diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 7b63f7549d..637f2ec3d1 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,3 +1,9 @@ +# all: Fix various spelling mistakes found by codespell 2.2.6. +cf490a70917a1b2d38ba9b58e763e0837d0f7ca7 + +# all: Fix spelling mistakes based on codespell check. +b1229efbd1509654dec6053865ab828d769e29db + # top: Update Python formatting to black "2023 stable style". 8b2748269244304854b3462cb8902952b4dcb892 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 2c17dcfe16..0000000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,135 +0,0 @@ - -# Adafruit Community Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and leaders pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level or type of -experience, education, socio-economic status, nationality, personal appearance, -race, religion, or sexual identity and orientation. - -## Our Standards - -We are committed to providing a friendly, safe and welcoming environment for -all. - -Examples of behavior that contributes to creating a positive environment -include: - -* Be kind and courteous to others -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Collaborating with other community members -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and sexual attention or advances -* The use of inappropriate images, including in a community member's avatar -* The use of inappropriate language, including in a community member's nickname -* Any spamming, flaming, baiting or other attention-stealing behavior -* Excessive or unwelcome helping; answering outside the scope of the question - asked -* Trolling, insulting/derogatory comments, and personal or political attacks -* Promoting or spreading disinformation, lies, or conspiracy theories against - a person, group, organisation, project, or community -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate - -The goal of the standards and moderation guidelines outlined here is to build -and maintain a respectful community. We ask that you don’t just aim to be -"technically unimpeachable", but rather try to be your best self. - -We value many things beyond technical expertise, including collaboration and -supporting others within our community. Providing a positive experience for -other community members can have a much more significant impact than simply -providing the correct answer. - -## Our Responsibilities - -Project leaders are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project leaders have the right and responsibility to remove, edit, or -reject messages, comments, commits, code, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any community member for other behaviors that they deem -inappropriate, threatening, offensive, or harmful. - -## Moderation - -Instances of behaviors that violate the Adafruit Community Code of Conduct -may be reported by any member of the community. Community members are -encouraged to report these situations, including situations they witness -involving other community members. - -You may report in the following ways: - -In any situation, you may send an email to . - -On the Adafruit Discord, you may send an open message from any channel -to all Community Moderators by tagging @community moderators. You may -also send an open message from any channel, or a direct message to -@kattni#1507, @tannewt#4653, @danh#1614, @cater#2442, -@sommersoft#0222, @Mr. Certainly#0472 or @Andon#8175. - -Email and direct message reports will be kept confidential. - -In situations on Discord where the issue is particularly egregious, possibly -illegal, requires immediate action, or violates the Discord terms of service, -you should also report the message directly to Discord. - -These are the steps for upholding our community’s standards of conduct. - -1. Any member of the community may report any situation that violates the -Adafruit Community Code of Conduct. All reports will be reviewed and -investigated. -2. If the behavior is an egregious violation, the community member who -committed the violation may be banned immediately, without warning. -3. Otherwise, moderators will first respond to such behavior with a warning. -4. Moderators follow a soft "three strikes" policy - the community member may -be given another chance, if they are receptive to the warning and change their -behavior. -5. If the community member is unreceptive or unreasonable when warned by a -moderator, or the warning goes unheeded, they may be banned for a first or -second offense. Repeated offenses will result in the community member being -banned. - -## Scope - -This Code of Conduct and the enforcement policies listed above apply to all -Adafruit Community venues. This includes but is not limited to any community -spaces (both public and private), the entire Adafruit Discord server, and -Adafruit GitHub repositories. Examples of Adafruit Community spaces include -but are not limited to meet-ups, audio chats on the Adafruit Discord, or -interaction at a conference. - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. As a community -member, you are representing our community, and are expected to behave -accordingly. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), -version 1.4, available at -, -and the [Rust Code of Conduct](https://www.rust-lang.org/en-US/conduct.html). - -For other projects adopting the Adafruit Community Code of -Conduct, please contact the maintainers of those projects for enforcement. -If you wish to use this code of conduct for your own project, consider -explicitly mentioning your moderation policy or making a copy with your -own moderation policy so as to avoid confusion. diff --git a/docs/library/gc.rst b/docs/library/gc.rst index 32afc7e982..d1625c0d8f 100644 --- a/docs/library/gc.rst +++ b/docs/library/gc.rst @@ -24,7 +24,7 @@ Functions .. function:: mem_alloc() - Return the number of bytes of heap RAM that are allocated. + Return the number of bytes of heap RAM that are allocated by Python code. .. admonition:: Difference to CPython :class: attention @@ -33,8 +33,8 @@ Functions .. function:: mem_free() - Return the number of bytes of available heap RAM, or -1 if this amount - is not known. + Return the number of bytes of heap RAM that is available for Python + code to allocate, or -1 if this amount is not known. .. admonition:: Difference to CPython :class: attention diff --git a/docs/library/gzip.rst b/docs/library/gzip.rst new file mode 100644 index 0000000000..f36f896db3 --- /dev/null +++ b/docs/library/gzip.rst @@ -0,0 +1,106 @@ +:mod:`gzip` -- gzip compression & decompression +=============================================== + +.. module:: gzip + :synopsis: gzip compression & decompression + +|see_cpython_module| :mod:`python:gzip`. + +This module allows compression and decompression of binary data with the +`DEFLATE algorithm `_ used by the gzip +file format. + +.. note:: Prefer to use :class:`deflate.DeflateIO` instead of the functions in this + module as it provides a streaming interface to compression and decompression + which is convenient and more memory efficient when working with reading or + writing compressed data to a file, socket, or stream. + +**Availability:** + +* This module is **not present by default** in official MicroPython firmware + releases as it duplicates functionality available in the :mod:`deflate + ` module. + +* A copy of this module can be installed (or frozen) + from :term:`micropython-lib` (`source `_). + See :ref:`packages` for more information. This documentation describes that module. + +* Compression support will only be available if compression support is enabled + in the built-in :mod:`deflate ` module. + +Functions +--------- + +.. function:: open(filename, mode, /) + + Wrapper around built-in :func:`open` returning a GzipFile instance. + +.. function:: decompress(data, /) + + Decompresses *data* into a bytes object. + +.. function:: compress(data, /) + + Compresses *data* into a bytes object. + +Classes +------- + +.. class:: GzipFile(*, fileobj, mode) + + This class can be used to wrap a *fileobj* which is any + :term:`stream-like ` object such as a file, socket, or stream + (including :class:`io.BytesIO`). It is itself a stream and implements the + standard read/readinto/write/close methods. + + When the *mode* argument is ``"rb"``, reads from the GzipFile instance will + decompress the data in the underlying stream and return decompressed data. + + If compression support is enabled then the *mode* argument can be set to + ``"wb"``, and writes to the GzipFile instance will be compressed and written + to the underlying stream. + + By default the GzipFile class will read and write data using the gzip file + format, including a header and footer with checksum and a window size of 512 + bytes. + + The **file**, **compresslevel**, and **mtime** arguments are not + supported. **fileobj** and **mode** must always be specified as keyword + arguments. + +Examples +-------- + +A typical use case for :class:`gzip.GzipFile` is to read or write a compressed +file from storage: + +.. code:: python + + import gzip + + # Reading: + with open("data.gz", "rb") as f: + with gzip.GzipFile(fileobj=f, mode="rb") as g: + # Use g.read(), g.readinto(), etc. + + # Same, but using gzip.open: + with gzip.open("data.gz", "rb") as f: + # Use f.read(), f.readinto(), etc. + + # Writing: + with open("data.gz", "wb") as f: + with gzip.GzipFile(fileobj=f, mode="wb") as g: + # Use g.write(...) etc + + # Same, but using gzip.open: + with gzip.open("data.gz", "wb") as f: + # Use f.write(...) etc + + # Write a dictionary as JSON in gzip format, with a + # small (64 byte) window size. + config = { ... } + with gzip.open("config.gz", "wb") as f: + json.dump(config, f) + +For guidance on working with gzip sources and choosing the window size see the +note at the :ref:`end of the deflate documentation `. diff --git a/docs/library/platform.rst b/docs/library/platform.rst new file mode 100644 index 0000000000..c091477d84 --- /dev/null +++ b/docs/library/platform.rst @@ -0,0 +1,38 @@ +:mod:`platform` -- access to underlying platform’s identifying data +=================================================================== + +.. module:: platform + :synopsis: access to underlying platform’s identifying data + +|see_cpython_module| :mod:`python:platform`. + +This module tries to retrieve as much platform-identifying data as possible. It +makes this information available via function APIs. + +Functions +--------- + +.. function:: platform() + + Returns a string identifying the underlying platform. This string is composed + of several substrings in the following order, delimited by dashes (``-``): + + - the name of the platform system (e.g. Unix, Windows or MicroPython) + - the MicroPython version + - the architecture of the platform + - the version of the underlying platform + - the concatenation of the name of the libc that MicroPython is linked to + and its corresponding version. + + For example, this could be + ``"MicroPython-1.20.0-xtensa-IDFv4.2.4-with-newlib3.0.0"``. + +.. function:: python_compiler() + + Returns a string identifying the compiler used for compiling MicroPython. + +.. function:: libc_ver() + + Returns a tuple of strings *(lib, version)*, where *lib* is the name of the + libc that MicroPython is linked to, and *version* the corresponding version + of this libc. diff --git a/docs/library/sys.rst b/docs/library/sys.rst index aa0a382748..c77f9a1c57 100644 --- a/docs/library/sys.rst +++ b/docs/library/sys.rst @@ -33,6 +33,7 @@ Constants * *name* - string "circuitpython" * *version* - tuple (major, minor, micro), e.g. (1, 7, 0) + * *_machine* - string describing the underlying machine * *_mpy* - supported mpy file-format version (optional attribute) This object is the recommended way to distinguish CircuitPython from other diff --git a/docs/reference/glossary.rst b/docs/reference/glossary.rst index 6ec613a315..efaa8f607f 100644 --- a/docs/reference/glossary.rst +++ b/docs/reference/glossary.rst @@ -32,7 +32,7 @@ Glossary callee-owned tuple This is a MicroPython-specific construct where, for efficiency - reasons, some built-in functions or methods may re-use the same + reasons, some built-in functions or methods may reuse the same underlying tuple object to return data. This avoids having to allocate a new tuple for every call, and reduces heap fragmentation. Programs should not hold references to callee-owned tuples and instead only @@ -120,7 +120,7 @@ Glossary `_ which provides implementations for many modules from CPython's standard library. - Some of the modules are are implemented in pure Python, and are able to + Some of the modules are implemented in pure Python, and are able to be used on all ports. However, the majority of these modules use :term:`FFI` to access operating system functionality, and as such can only be used on the :term:`MicroPython Unix port` (with limited support @@ -158,8 +158,10 @@ Glossary network-capable boards, and internally by tools such as :term:`mpremote`. + See :ref:`packages` for more information on using ``mip``. + mpremote - A tool for interacting with a MicroPython device. + A tool for interacting with a MicroPython device. See :ref:`mpremote`. .mpy file The output of the :term:`cross-compiler`. A compiled form of a diff --git a/examples/natmod/README.md b/examples/natmod/README.md new file mode 100644 index 0000000000..0cc4010ef4 --- /dev/null +++ b/examples/natmod/README.md @@ -0,0 +1,74 @@ +# Dynamic Native Modules + +Dynamic Native Modules are .mpy files that contain native machine code from a +language other than Python. For more info see [the documentation] +(https://docs.micropython.org/en/latest/develop/natmod.html). + +This should not be confused with [User C Modules] +(https://docs.micropython.org/en/latest/develop/cmodules.html) which are a +mechanism to add additional out-of-tree modules into the firmware build. + +## Examples + +This directory contains several examples of writing dynamic native modules, in +two main categories: + +1. Feature examples. + + * `features0` - A module containing a single "factorial" function which + demonstrates working with integers. + + * `features1` - A module that demonstrates some common tasks: + - defining simple functions exposed to Python + - defining local, helper C functions + - defining constant integers and strings exposed to Python + - getting and creating integer objects + - creating Python lists + - raising exceptions + - allocating memory + - BSS and constant data (rodata) + - relocated pointers in rodata + + * `features2` - This is a hybrid module containing both Python and C code, + and additionally the C code is spread over multiple files. It also + demonstrates using floating point (only when the target supports + hardware floating point). + + * `features3` - A module that shows how to use types, constant objects, + and creating dictionary instances. + + * `features4` - A module that demonstrates how to define a class. + +2. Dynamic version of existing built-ins. + + This provides a way to add missing functionality to firmware that doesn't + include certain built-in modules. See the `heapq`, `random`, `re`, + `deflate`, `btree`, and `framebuf` directories. + + So for example, if your firmware was compiled with `MICROPY_PY_FRAMEBUF` + disabled (e.g. to save flash space), then it would not include the + `framebuf` module. The `framebuf` native module provides a way to add the + `framebuf` module dynamically. + + The way these work is they define a dynamic native module which + `#include`'s the original module and then does the necessary + initialisation of the module's globals dict. + +## Build instructions + +To compile an example, you need to have the same toolchain available as +required for your target port. e.g. `arm-none-eabi-gcc` for any ARM Cortex M +target. See the port instructions for details. + +You also need to have the `pyelftools` Python package available, either via +your system package manager or installed from PyPI in a virtual environment +with `pip`. + +Each example provides a Makefile. You should specify the `ARCH` argument to +make (one of x86, x64, armv6m, armv7m, xtensa, xtensawin): + +``` +$ cd features0 +$ make ARCH=armv7m +$ mpremote cp features0.mpy : +``` diff --git a/examples/natmod/uzlib/Makefile b/examples/natmod/deflate/Makefile similarity index 87% rename from examples/natmod/uzlib/Makefile rename to examples/natmod/deflate/Makefile index 8761caf2dd..86ef29b632 100644 --- a/examples/natmod/uzlib/Makefile +++ b/examples/natmod/deflate/Makefile @@ -2,10 +2,10 @@ MPY_DIR = ../../.. # Name of module (different to built-in uzlib so it can coexist) -MOD = uzlib_$(ARCH) +MOD = deflate_$(ARCH) # Source files (.c or .py) -SRC = uzlib.c +SRC = deflate.c # Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) ARCH = x64 diff --git a/examples/natmod/deflate/deflate.c b/examples/natmod/deflate/deflate.c new file mode 100644 index 0000000000..fa475bd067 --- /dev/null +++ b/examples/natmod/deflate/deflate.c @@ -0,0 +1,70 @@ +#define MICROPY_PY_DEFLATE (1) +#define MICROPY_PY_DEFLATE_COMPRESS (1) + +#include "py/dynruntime.h" + +#if !defined(__linux__) +void *memcpy(void *dst, const void *src, size_t n) { + return mp_fun_table.memmove_(dst, src, n); +} +void *memset(void *s, int c, size_t n) { + return mp_fun_table.memset_(s, c, n); +} +#endif + +mp_obj_full_type_t deflateio_type; + +#include "extmod/moddeflate.c" + +// Re-implemented from py/stream.c, not yet available in dynruntime.h. +mp_obj_t mp_stream_close(mp_obj_t stream) { + const mp_stream_p_t *stream_p = mp_get_stream(stream); + int error; + mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_CLOSE, 0, &error); + if (res == MP_STREAM_ERROR) { + mp_raise_OSError(error); + } + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_close_obj, mp_stream_close); + +// Re-implemented from py/stream.c, not yet available in dynruntime.h. +STATIC mp_obj_t mp_stream___exit__(size_t n_args, const mp_obj_t *args) { + (void)n_args; + return mp_stream_close(args[0]); +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream___exit___obj, 4, 4, mp_stream___exit__); + +// Re-implemented from obj.c, not yet available in dynruntime.h. +mp_obj_t mp_identity(mp_obj_t self) { + return self; +} +MP_DEFINE_CONST_FUN_OBJ_1(mp_identity_obj, mp_identity); + +mp_map_elem_t deflateio_locals_dict_table[7]; +STATIC MP_DEFINE_CONST_DICT(deflateio_locals_dict, deflateio_locals_dict_table); + +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + MP_DYNRUNTIME_INIT_ENTRY + + deflateio_type.base.type = mp_fun_table.type_type; + deflateio_type.name = MP_QSTR_DeflateIO; + MP_OBJ_TYPE_SET_SLOT(&deflateio_type, make_new, &deflateio_make_new, 0); + MP_OBJ_TYPE_SET_SLOT(&deflateio_type, protocol, &deflateio_stream_p, 1); + deflateio_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) }; + deflateio_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) }; + deflateio_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) }; + deflateio_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_write), MP_OBJ_FROM_PTR(&mp_stream_write_obj) }; + deflateio_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_close), MP_OBJ_FROM_PTR(&mp_stream_close_obj) }; + deflateio_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR___enter__), MP_OBJ_FROM_PTR(&mp_identity_obj) }; + deflateio_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR___exit__), MP_OBJ_FROM_PTR(&mp_stream___exit___obj) }; + MP_OBJ_TYPE_SET_SLOT(&deflateio_type, locals_dict, (void*)&deflateio_locals_dict, 2); + + mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_deflate)); + mp_store_global(MP_QSTR_DeflateIO, MP_OBJ_FROM_PTR(&deflateio_type)); + mp_store_global(MP_QSTR_RAW, MP_OBJ_NEW_SMALL_INT(DEFLATEIO_FORMAT_RAW)); + mp_store_global(MP_QSTR_ZLIB, MP_OBJ_NEW_SMALL_INT(DEFLATEIO_FORMAT_ZLIB)); + mp_store_global(MP_QSTR_GZIP, MP_OBJ_NEW_SMALL_INT(DEFLATEIO_FORMAT_GZIP)); + + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/examples/natmod/features1/features1.c b/examples/natmod/features1/features1.c index a5e82252a1..d2494b2138 100644 --- a/examples/natmod/features1/features1.c +++ b/examples/natmod/features1/features1.c @@ -88,7 +88,7 @@ mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *a // This must be first, it sets up the globals dict and other things MP_DYNRUNTIME_INIT_ENTRY - // Messages can be printed as usually + // Messages can be printed as usual mp_printf(&mp_plat_print, "initialising module self=%p\n", self); // Make the functions available in the module's namespace diff --git a/examples/natmod/features2/test.py b/examples/natmod/features2/test.py index 5ac80120d7..af79b9692c 100644 --- a/examples/natmod/features2/test.py +++ b/examples/natmod/features2/test.py @@ -1,5 +1,7 @@ # This Python code will be merged with the C code in main.c +# ruff: noqa: F821 - this file is evaluated with C-defined names in scope + import array diff --git a/examples/natmod/features4/Makefile b/examples/natmod/features4/Makefile new file mode 100644 index 0000000000..f76a31a7cc --- /dev/null +++ b/examples/natmod/features4/Makefile @@ -0,0 +1,14 @@ +# Location of top-level MicroPython directory +MPY_DIR = ../../.. + +# Name of module +MOD = features4 + +# Source files (.c or .py) +SRC = features4.c + +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) +ARCH = x64 + +# Include to get the rules for compiling and linking the module +include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/features4/features4.c b/examples/natmod/features4/features4.c new file mode 100644 index 0000000000..336f4ecf64 --- /dev/null +++ b/examples/natmod/features4/features4.c @@ -0,0 +1,73 @@ +/* + This example extends on features0 but demonstrates how to define a class. + + The Factorial class constructor takes an integer, and then the calculate + method can be called to get the factorial. + + >>> import features4 + >>> f = features4.Factorial(4) + >>> f.calculate() + 24 +*/ + +// Include the header file to get access to the MicroPython API +#include "py/dynruntime.h" + +// This is type(Factorial) +mp_obj_full_type_t mp_type_factorial; + +// This is the internal state of a Factorial instance. +typedef struct { + mp_obj_base_t base; + mp_int_t n; +} mp_obj_factorial_t; + +// Essentially Factorial.__new__ (but also kind of __init__). +// Takes a single argument (the number to find the factorial of) +STATIC mp_obj_t factorial_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args_in) { + mp_arg_check_num(n_args, n_kw, 1, 1, false); + + mp_obj_factorial_t *o = mp_obj_malloc(mp_obj_factorial_t, type); + o->n = mp_obj_get_int(args_in[0]); + + return MP_OBJ_FROM_PTR(o); +} + +STATIC mp_int_t factorial_helper(mp_int_t x) { + if (x == 0) { + return 1; + } + return x * factorial_helper(x - 1); +} + +// Implements Factorial.calculate() +STATIC mp_obj_t factorial_calculate(mp_obj_t self_in) { + mp_obj_factorial_t *self = MP_OBJ_TO_PTR(self_in); + return mp_obj_new_int(factorial_helper(self->n)); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(factorial_calculate_obj, factorial_calculate); + +// Locals dict for the Factorial type (will have a single method, calculate, +// added in mpy_init). +mp_map_elem_t factorial_locals_dict_table[1]; +STATIC MP_DEFINE_CONST_DICT(factorial_locals_dict, factorial_locals_dict_table); + +// This is the entry point and is called when the module is imported +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + // This must be first, it sets up the globals dict and other things + MP_DYNRUNTIME_INIT_ENTRY + + // Initialise the type. + mp_type_factorial.base.type = (void*)&mp_type_type; + mp_type_factorial.flags = MP_TYPE_FLAG_NONE; + mp_type_factorial.name = MP_QSTR_Factorial; + MP_OBJ_TYPE_SET_SLOT(&mp_type_factorial, make_new, factorial_make_new, 0); + factorial_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_calculate), MP_OBJ_FROM_PTR(&factorial_calculate_obj) }; + MP_OBJ_TYPE_SET_SLOT(&mp_type_factorial, locals_dict, (void*)&factorial_locals_dict, 1); + + // Make the Factorial type available on the module. + mp_store_global(MP_QSTR_Factorial, MP_OBJ_FROM_PTR(&mp_type_factorial)); + + // This must be last, it restores the globals dict + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/examples/natmod/urandom/Makefile b/examples/natmod/heapq/Makefile similarity index 66% rename from examples/natmod/urandom/Makefile rename to examples/natmod/heapq/Makefile index 3f018baaf7..af45b472da 100644 --- a/examples/natmod/urandom/Makefile +++ b/examples/natmod/heapq/Makefile @@ -1,11 +1,11 @@ # Location of top-level MicroPython directory MPY_DIR = ../../.. -# Name of module (different to built-in urandom so it can coexist) -MOD = urandom_$(ARCH) +# Name of module (different to built-in heapq so it can coexist) +MOD = heapq_$(ARCH) # Source files (.c or .py) -SRC = urandom.c +SRC = heapq.c # Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) ARCH = x64 diff --git a/examples/natmod/heapq/heapq.c b/examples/natmod/heapq/heapq.c new file mode 100644 index 0000000000..ed19652a66 --- /dev/null +++ b/examples/natmod/heapq/heapq.c @@ -0,0 +1,16 @@ +#define MICROPY_PY_HEAPQ (1) + +#include "py/dynruntime.h" + +#include "extmod/modheapq.c" + +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + MP_DYNRUNTIME_INIT_ENTRY + + mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_heapq)); + mp_store_global(MP_QSTR_heappush, MP_OBJ_FROM_PTR(&mod_heapq_heappush_obj)); + mp_store_global(MP_QSTR_heappop, MP_OBJ_FROM_PTR(&mod_heapq_heappop_obj)); + mp_store_global(MP_QSTR_heapify, MP_OBJ_FROM_PTR(&mod_heapq_heapify_obj)); + + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/examples/natmod/uheapq/Makefile b/examples/natmod/random/Makefile similarity index 67% rename from examples/natmod/uheapq/Makefile rename to examples/natmod/random/Makefile index 55de3cc081..5c50227b15 100644 --- a/examples/natmod/uheapq/Makefile +++ b/examples/natmod/random/Makefile @@ -1,11 +1,11 @@ # Location of top-level MicroPython directory MPY_DIR = ../../.. -# Name of module (different to built-in uheapq so it can coexist) -MOD = uheapq_$(ARCH) +# Name of module (different to built-in random so it can coexist) +MOD = random_$(ARCH) # Source files (.c or .py) -SRC = uheapq.c +SRC = random.c # Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) ARCH = x64 diff --git a/examples/natmod/random/random.c b/examples/natmod/random/random.c new file mode 100644 index 0000000000..92257b8bc6 --- /dev/null +++ b/examples/natmod/random/random.c @@ -0,0 +1,33 @@ +#define MICROPY_PY_RANDOM (1) +#define MICROPY_PY_RANDOM_EXTRA_FUNCS (1) + +#include "py/dynruntime.h" + +// Dynamic native modules don't support a data section so these must go in the BSS +uint32_t yasmarang_pad, yasmarang_n, yasmarang_d; +uint8_t yasmarang_dat; + +#include "extmod/modrandom.c" + +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + MP_DYNRUNTIME_INIT_ENTRY + + yasmarang_pad = 0xeda4baba; + yasmarang_n = 69; + yasmarang_d = 233; + + mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_random)); + mp_store_global(MP_QSTR_getrandbits, MP_OBJ_FROM_PTR(&mod_random_getrandbits_obj)); + mp_store_global(MP_QSTR_seed, MP_OBJ_FROM_PTR(&mod_random_seed_obj)); + #if MICROPY_PY_RANDOM_EXTRA_FUNCS + mp_store_global(MP_QSTR_randrange, MP_OBJ_FROM_PTR(&mod_random_randrange_obj)); + mp_store_global(MP_QSTR_randint, MP_OBJ_FROM_PTR(&mod_random_randint_obj)); + mp_store_global(MP_QSTR_choice, MP_OBJ_FROM_PTR(&mod_random_choice_obj)); + #if MICROPY_PY_BUILTINS_FLOAT + mp_store_global(MP_QSTR_random, MP_OBJ_FROM_PTR(&mod_random_random_obj)); + mp_store_global(MP_QSTR_uniform, MP_OBJ_FROM_PTR(&mod_random_uniform_obj)); + #endif + #endif + + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/examples/natmod/ure/Makefile b/examples/natmod/re/Makefile similarity index 69% rename from examples/natmod/ure/Makefile rename to examples/natmod/re/Makefile index f5254298fd..1ba5401106 100644 --- a/examples/natmod/ure/Makefile +++ b/examples/natmod/re/Makefile @@ -1,11 +1,11 @@ # Location of top-level MicroPython directory MPY_DIR = ../../.. -# Name of module (different to built-in ure so it can coexist) -MOD = ure_$(ARCH) +# Name of module (different to built-in re so it can coexist) +MOD = re_$(ARCH) # Source files (.c or .py) -SRC = ure.c +SRC = re.c # Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) ARCH = x64 diff --git a/examples/natmod/ure/ure.c b/examples/natmod/re/re.c similarity index 92% rename from examples/natmod/ure/ure.c rename to examples/natmod/re/re.c index d4bd680abd..df89ea8353 100644 --- a/examples/natmod/ure/ure.c +++ b/examples/natmod/re/re.c @@ -1,8 +1,8 @@ #define MICROPY_STACK_CHECK (1) -#define MICROPY_PY_URE (1) -#define MICROPY_PY_URE_MATCH_GROUPS (1) -#define MICROPY_PY_URE_MATCH_SPAN_START_END (1) -#define MICROPY_PY_URE_SUB (0) // requires vstr interface +#define MICROPY_PY_RE (1) +#define MICROPY_PY_RE_MATCH_GROUPS (1) +#define MICROPY_PY_RE_MATCH_SPAN_START_END (1) +#define MICROPY_PY_RE_SUB (0) // requires vstr interface #include #include "py/dynruntime.h" @@ -35,7 +35,7 @@ void *memmove(void *dest, const void *src, size_t n) { mp_obj_full_type_t match_type; mp_obj_full_type_t re_type; -#include "extmod/modure.c" +#include "extmod/modre.c" mp_map_elem_t match_locals_dict_table[5]; STATIC MP_DEFINE_CONST_DICT(match_locals_dict, match_locals_dict_table); @@ -63,7 +63,7 @@ mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *a MP_OBJ_TYPE_SET_SLOT(&match_type, locals_dict, (void*)&match_locals_dict, 1); re_type.base.type = (void*)&mp_fun_table.type_type; - re_type.name = MP_QSTR_ure; + re_type.name = MP_QSTR_re; MP_OBJ_TYPE_SET_SLOT(&re_type, print, re_print, 0); re_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_match), MP_OBJ_FROM_PTR(&re_match_obj) }; re_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_search), MP_OBJ_FROM_PTR(&re_search_obj) }; diff --git a/examples/natmod/uheapq/uheapq.c b/examples/natmod/uheapq/uheapq.c deleted file mode 100644 index 9da6bb4ada..0000000000 --- a/examples/natmod/uheapq/uheapq.c +++ /dev/null @@ -1,16 +0,0 @@ -#define MICROPY_PY_UHEAPQ (1) - -#include "py/dynruntime.h" - -#include "extmod/moduheapq.c" - -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - MP_DYNRUNTIME_INIT_ENTRY - - mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_uheapq)); - mp_store_global(MP_QSTR_heappush, MP_OBJ_FROM_PTR(&mod_uheapq_heappush_obj)); - mp_store_global(MP_QSTR_heappop, MP_OBJ_FROM_PTR(&mod_uheapq_heappop_obj)); - mp_store_global(MP_QSTR_heapify, MP_OBJ_FROM_PTR(&mod_uheapq_heapify_obj)); - - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/natmod/urandom/urandom.c b/examples/natmod/urandom/urandom.c deleted file mode 100644 index e1fed6a554..0000000000 --- a/examples/natmod/urandom/urandom.c +++ /dev/null @@ -1,33 +0,0 @@ -#define MICROPY_PY_URANDOM (1) -#define MICROPY_PY_URANDOM_EXTRA_FUNCS (1) - -#include "py/dynruntime.h" - -// Dynamic native modules don't support a data section so these must go in the BSS -uint32_t yasmarang_pad, yasmarang_n, yasmarang_d; -uint8_t yasmarang_dat; - -#include "extmod/modurandom.c" - -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - MP_DYNRUNTIME_INIT_ENTRY - - yasmarang_pad = 0xeda4baba; - yasmarang_n = 69; - yasmarang_d = 233; - - mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_urandom)); - mp_store_global(MP_QSTR_getrandbits, MP_OBJ_FROM_PTR(&mod_urandom_getrandbits_obj)); - mp_store_global(MP_QSTR_seed, MP_OBJ_FROM_PTR(&mod_urandom_seed_obj)); - #if MICROPY_PY_URANDOM_EXTRA_FUNCS - mp_store_global(MP_QSTR_randrange, MP_OBJ_FROM_PTR(&mod_urandom_randrange_obj)); - mp_store_global(MP_QSTR_randint, MP_OBJ_FROM_PTR(&mod_urandom_randint_obj)); - mp_store_global(MP_QSTR_choice, MP_OBJ_FROM_PTR(&mod_urandom_choice_obj)); - #if MICROPY_PY_BUILTINS_FLOAT - mp_store_global(MP_QSTR_random, MP_OBJ_FROM_PTR(&mod_urandom_random_obj)); - mp_store_global(MP_QSTR_uniform, MP_OBJ_FROM_PTR(&mod_urandom_uniform_obj)); - #endif - #endif - - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/natmod/uzlib/uzlib.c b/examples/natmod/uzlib/uzlib.c deleted file mode 100644 index 9cf58b10e7..0000000000 --- a/examples/natmod/uzlib/uzlib.c +++ /dev/null @@ -1,35 +0,0 @@ -#define MICROPY_PY_UZLIB (1) - -#include "py/dynruntime.h" - -#if !defined(__linux__) -void *memset(void *s, int c, size_t n) { - return mp_fun_table.memset_(s, c, n); -} -#endif - -mp_obj_full_type_t decompio_type; - -#include "extmod/moduzlib.c" - -mp_map_elem_t decompio_locals_dict_table[3]; -STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table); - -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - MP_DYNRUNTIME_INIT_ENTRY - - decompio_type.base.type = mp_fun_table.type_type; - decompio_type.name = MP_QSTR_DecompIO; - MP_OBJ_TYPE_SET_SLOT(&decompio_type, make_new, &decompio_make_new, 0); - MP_OBJ_TYPE_SET_SLOT(&decompio_type, protocol, &decompio_stream_p, 1); - decompio_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) }; - decompio_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) }; - decompio_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) }; - MP_OBJ_TYPE_SET_SLOT(&decompio_type, locals_dict, (void*)&decompio_locals_dict, 2); - - mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_uzlib)); - mp_store_global(MP_QSTR_decompress, MP_OBJ_FROM_PTR(&mod_uzlib_decompress_obj)); - mp_store_global(MP_QSTR_DecompIO, MP_OBJ_FROM_PTR(&decompio_type)); - - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/usercmodule/cexample/examplemodule.c b/examples/usercmodule/cexample/examplemodule.c index ccce03bcbd..9416613ba9 100644 --- a/examples/usercmodule/cexample/examplemodule.c +++ b/examples/usercmodule/cexample/examplemodule.c @@ -68,7 +68,7 @@ MP_DEFINE_CONST_OBJ_TYPE( locals_dict, &example_Timer_locals_dict ); -// Define all properties of the module. +// Define all attributes of the module. // Table entries are key/value pairs of the attribute name (a string) // and the MicroPython object reference. // All identifiers and strings are written as MP_QSTR_xxx and will be diff --git a/examples/usercmodule/cexample/micropython.mk b/examples/usercmodule/cexample/micropython.mk index dbfe3c5cbd..d6801dac0b 100644 --- a/examples/usercmodule/cexample/micropython.mk +++ b/examples/usercmodule/cexample/micropython.mk @@ -1,9 +1,8 @@ -EXAMPLE_MOD_DIR := $(USERMOD_DIR) +CEXAMPLE_MOD_DIR := $(USERMOD_DIR) # Add all C files to SRC_USERMOD. -SRC_USERMOD += $(EXAMPLE_MOD_DIR)/examplemodule.c +SRC_USERMOD += $(CEXAMPLE_MOD_DIR)/examplemodule.c # We can add our module folder to include paths if needed # This is not actually needed in this example. -CFLAGS_USERMOD += -I$(EXAMPLE_MOD_DIR) -CEXAMPLE_MOD_DIR := $(USERMOD_DIR) +CFLAGS_USERMOD += -I$(CEXAMPLE_MOD_DIR) diff --git a/examples/usercmodule/cppexample/examplemodule.c b/examples/usercmodule/cppexample/examplemodule.c index 5c84eccd79..96a1a74438 100644 --- a/examples/usercmodule/cppexample/examplemodule.c +++ b/examples/usercmodule/cppexample/examplemodule.c @@ -4,7 +4,7 @@ // See example.cpp for the definition. STATIC MP_DEFINE_CONST_FUN_OBJ_2(cppfunc_obj, cppfunc); -// Define all properties of the module. +// Define all attributes of the module. // Table entries are key/value pairs of the attribute name (a string) // and the MicroPython object reference. // All identifiers and strings are written as MP_QSTR_xxx and will be diff --git a/examples/usercmodule/subpackage/README.md b/examples/usercmodule/subpackage/README.md new file mode 100644 index 0000000000..c7f2ee53a2 --- /dev/null +++ b/examples/usercmodule/subpackage/README.md @@ -0,0 +1 @@ +This is an example of a user C module that includes subpackages. diff --git a/examples/usercmodule/subpackage/micropython.cmake b/examples/usercmodule/subpackage/micropython.cmake new file mode 100644 index 0000000000..a51e7a8061 --- /dev/null +++ b/examples/usercmodule/subpackage/micropython.cmake @@ -0,0 +1,19 @@ +# Create an INTERFACE library for our C module. +add_library(usermod_subpackage_example INTERFACE) + +# Add our source files to the lib +target_sources(usermod_subpackage_example INTERFACE + ${CMAKE_CURRENT_LIST_DIR}/examplemodule.c +) + +# Add the current directory as an include directory. +target_include_directories(usermod_subpackage_example INTERFACE + ${CMAKE_CURRENT_LIST_DIR} +) + +target_compile_definitions(usermod_subpackage_example INTERFACE + MICROPY_MODULE_BUILTIN_SUBPACKAGES=1 +) + +# Link our INTERFACE library to the usermod target. +target_link_libraries(usermod INTERFACE usermod_subpackage_example) diff --git a/examples/usercmodule/subpackage/micropython.mk b/examples/usercmodule/subpackage/micropython.mk new file mode 100644 index 0000000000..99ebf13ec1 --- /dev/null +++ b/examples/usercmodule/subpackage/micropython.mk @@ -0,0 +1,10 @@ +SUBPACKAGE_EXAMPLE_MOD_DIR := $(USERMOD_DIR) + +# Add all C files to SRC_USERMOD. +SRC_USERMOD += $(SUBPACKAGE_EXAMPLE_MOD_DIR)/modexamplepackage.c + +# We can add our module folder to include paths if needed +# This is not actually needed in this example. +CFLAGS_USERMOD += -I$(SUBPACKAGE_EXAMPLE_MOD_DIR) -DMICROPY_MODULE_BUILTIN_SUBPACKAGES=1 + +QSTR_DEFS += $(SUBPACKAGE_EXAMPLE_MOD_DIR)/qstrdefsexamplepackage.h diff --git a/examples/usercmodule/subpackage/modexamplepackage.c b/examples/usercmodule/subpackage/modexamplepackage.c new file mode 100644 index 0000000000..70e1e4005b --- /dev/null +++ b/examples/usercmodule/subpackage/modexamplepackage.c @@ -0,0 +1,84 @@ +// Include MicroPython API. +#include "py/runtime.h" + +// Define example_package.foo.bar.f() +STATIC mp_obj_t example_package_foo_bar_f(void) { + mp_printf(&mp_plat_print, "example_package.foo.bar.f\n"); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(example_package_foo_bar_f_obj, example_package_foo_bar_f); + +// Define all attributes of the second-level sub-package (example_package.foo.bar). +STATIC const mp_rom_map_elem_t example_package_foo_bar_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_example_package_dot_foo_dot_bar) }, + { MP_ROM_QSTR(MP_QSTR_f), MP_ROM_PTR(&example_package_foo_bar_f_obj) }, +}; +STATIC MP_DEFINE_CONST_DICT(example_package_foo_bar_globals, example_package_foo_bar_globals_table); + +// Define example_package.foo.bar module object. +const mp_obj_module_t example_package_foo_bar_user_cmodule = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&example_package_foo_bar_globals, +}; + +// Define example_package.foo.f() +STATIC mp_obj_t example_package_foo_f(void) { + mp_printf(&mp_plat_print, "example_package.foo.f\n"); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(example_package_foo_f_obj, example_package_foo_f); + +// Define all attributes of the first-level sub-package (example_package.foo). +STATIC const mp_rom_map_elem_t example_package_foo_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_example_package_dot_foo) }, + { MP_ROM_QSTR(MP_QSTR_bar), MP_ROM_PTR(&example_package_foo_bar_user_cmodule) }, + { MP_ROM_QSTR(MP_QSTR_f), MP_ROM_PTR(&example_package_foo_f_obj) }, +}; +STATIC MP_DEFINE_CONST_DICT(example_package_foo_globals, example_package_foo_globals_table); + +// Define example_package.foo module object. +const mp_obj_module_t example_package_foo_user_cmodule = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&example_package_foo_globals, +}; + +// Define example_package.f() +STATIC mp_obj_t example_package_f(void) { + mp_printf(&mp_plat_print, "example_package.f\n"); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(example_package_f_obj, example_package_f); + +STATIC mp_obj_t example_package___init__(void) { + if (!MP_STATE_VM(example_package_initialised)) { + // __init__ for builtins is called each time the module is imported, + // so ensure that initialisation only happens once. + MP_STATE_VM(example_package_initialised) = true; + mp_printf(&mp_plat_print, "example_package.__init__\n"); + } + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(example_package___init___obj, example_package___init__); + +// The "initialised" state is stored on mp_state so that it is cleared on soft +// reset. +MP_REGISTER_ROOT_POINTER(int example_package_initialised); + +// Define all attributes of the top-level package (example_package). +STATIC const mp_rom_map_elem_t example_package_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_example_package) }, + { MP_ROM_QSTR(MP_QSTR___init__), MP_ROM_PTR(&example_package___init___obj) }, + { MP_ROM_QSTR(MP_QSTR_foo), MP_ROM_PTR(&example_package_foo_user_cmodule) }, + { MP_ROM_QSTR(MP_QSTR_f), MP_ROM_PTR(&example_package_f_obj) }, +}; +STATIC MP_DEFINE_CONST_DICT(example_package_globals, example_package_globals_table); + +// Define module object. +const mp_obj_module_t example_package_user_cmodule = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&example_package_globals, +}; + +// Register the module to make it available in Python. +// Note: subpackages should not be registered with MP_REGISTER_MODULE. +MP_REGISTER_MODULE(MP_QSTR_example_package, example_package_user_cmodule); diff --git a/examples/usercmodule/subpackage/qstrdefsexamplepackage.h b/examples/usercmodule/subpackage/qstrdefsexamplepackage.h new file mode 100644 index 0000000000..057ec5279d --- /dev/null +++ b/examples/usercmodule/subpackage/qstrdefsexamplepackage.h @@ -0,0 +1,2 @@ +Q(example_package.foo) +Q(example_package.foo.bar) diff --git a/extmod/extmod.mk b/extmod/extmod.mk index a1de6c6fe4..9a65a7a91a 100644 --- a/extmod/extmod.mk +++ b/extmod/extmod.mk @@ -1,20 +1,20 @@ # This makefile fragment adds the source code files for the core extmod modules # and provides rules to build 3rd-party components for extmod modules. +# CIRCUITPY has removed many extmod modules. SRC_EXTMOD_C += \ - extmod/moduasyncio.c \ - extmod/modubinascii.c \ + extmod/modasyncio.c \ + extmod/modbinascii.c \ + extmod/modhashlib.c \ + extmod/modheapq.c \ + extmod/modjson.c \ + extmod/modos.c \ + extmod/modplatform.c\ + extmod/modrandom.c \ + extmod/modre.c \ + extmod/modselect.c \ extmod/moductypes.c \ - extmod/moduhashlib.c \ - extmod/moduheapq.c \ - extmod/modujson.c \ - extmod/moduos.c \ - extmod/moduplatform.c\ - extmod/modurandom.c \ - extmod/modure.c \ - extmod/moduselect.c \ - extmod/moduzlib.c \ - extmod/utime_mphal.c \ + extmod/modzlib.c \ extmod/vfs.c \ extmod/vfs_blockdev.c \ extmod/vfs_fat.c \ @@ -81,10 +81,9 @@ endif ################################################################################ # ussl -ifeq ($(MICROPY_PY_USSL),1) -CFLAGS_EXTMOD += -DMICROPY_PY_USSL=1 +ifeq ($(MICROPY_PY_SSL),1) +CFLAGS_EXTMOD += -DMICROPY_PY_SSL=1 ifeq ($(MICROPY_SSL_AXTLS),1) -CFLAGS_MOD += -DMICROPY_SSL_AXTLS=1 -I$(TOP)/lib/axtls/ssl -I$(TOP)/lib/axtls/crypto -I$(TOP)/extmod/axtls-include AXTLS_DIR = lib/axtls GIT_SUBMODULES += $(AXTLS_DIR) CFLAGS_EXTMOD += -DMICROPY_SSL_AXTLS=1 -I$(TOP)/lib/axtls/ssl -I$(TOP)/lib/axtls/crypto -I$(TOP)/extmod/axtls-include @@ -146,7 +145,6 @@ SRC_THIRDPARTY_C += $(addprefix $(MBEDTLS_DIR)/library/,\ md4.c \ md5.c \ md.c \ - md_wrap.c \ oid.c \ padlock.c \ pem.c \ @@ -171,9 +169,11 @@ SRC_THIRDPARTY_C += $(addprefix $(MBEDTLS_DIR)/library/,\ ssl_cli.c \ ssl_cookie.c \ ssl_srv.c \ + ssl_msg.c \ ssl_ticket.c \ ssl_tls.c \ timing.c \ + constant_time.c \ x509.c \ x509_create.c \ x509_crl.c \ @@ -267,7 +267,7 @@ SRC_THIRDPARTY_C += $(addprefix $(BTREE_DIR)/,\ CFLAGS_EXTMOD += -DMICROPY_PY_BTREE=1 # we need to suppress certain warnings to get berkeley-db to compile cleanly # and we have separate BTREE_DEFS so the definitions don't interfere with other source code -$(BUILD)/$(BTREE_DIR)/%.o: CFLAGS += -Wno-old-style-definition -Wno-sign-compare -Wno-unused-parameter $(BTREE_DEFS) +$(BUILD)/$(BTREE_DIR)/%.o: CFLAGS += -Wno-old-style-definition -Wno-sign-compare -Wno-unused-parameter -Wno-deprecated-non-prototype -Wno-unknown-warning-option $(BTREE_DEFS) $(BUILD)/extmod/modbtree.o: CFLAGS += $(BTREE_DEFS) endif @@ -311,8 +311,51 @@ SRC_THIRDPARTY_C += $(addprefix $(WIZNET5K_DIR)/,\ Internet/DHCP/dhcp.c \ ) endif +endif # MICROPY_PY_NETWORK_WIZNET5K + +ifeq ($(MICROPY_PY_NETWORK_ESP_HOSTED),1) +ESP_HOSTED_DIR = drivers/esp-hosted +PROTOBUF_C_DIR = lib/protobuf-c +PROTOC ?= protoc-c +GIT_SUBMODULES += $(PROTOBUF_C_DIR) + +CFLAGS += -DMICROPY_PY_NETWORK_ESP_HOSTED=1 +CFLAGS_EXTMOD += -DMICROPY_PY_NETWORK_ESP_HOSTED=1 +INC += -I$(TOP)/$(ESP_HOSTED_DIR) + +ESP_HOSTED_SRC_C = $(addprefix $(ESP_HOSTED_DIR)/,\ + esp_hosted_wifi.c \ + esp_hosted_netif.c \ + esp_hosted_hal.c \ + ) + +ifeq ($(MICROPY_PY_BLUETOOTH),1) +ESP_HOSTED_SRC_C += $(ESP_HOSTED_DIR)/esp_hosted_bthci.c endif +# Include the protobuf-c support functions +ESP_HOSTED_SRC_C += $(addprefix $(PROTOBUF_C_DIR)/,\ + protobuf-c/protobuf-c.c \ + ) + +$(BUILD)/$(PROTOBUF_C_DIR)/%.o: CFLAGS += -Wno-unused-but-set-variable + +# Generate esp_hosted-pb-c.c|h from esp_hosted.proto +PROTO_GEN_SRC = $(BUILD)/extmod/esp_hosted.pb-c.c +ESP_HOSTED_SRC_C += $(PROTO_GEN_SRC) + +$(PROTO_GEN_SRC): $(TOP)/$(ESP_HOSTED_DIR)/esp_hosted.proto + $(PROTOC) --proto_path=$(dir $<) --c_out=$(dir $@) $< + +# Scope the protobuf include paths to the esp_hosted source files, only +ESP_HOSTED_OBJS = $(addprefix $(BUILD)/, $(ESP_HOSTED_SRC_C:.c=.o)) +$(ESP_HOSTED_OBJS): $(PROTO_GEN_SRC) +$(ESP_HOSTED_OBJS): CFLAGS += -I$(dir $(PROTO_GEN_SRC)) -I$(TOP)/$(PROTOBUF_C_DIR) + +DRIVERS_SRC_C += $(ESP_HOSTED_SRC_C) + +endif # MICROPY_PY_NETWORK_ESP_HOSTED + ################################################################################ # bluetooth diff --git a/extmod/misc.h b/extmod/misc.h index a9392aa10b..80e5b59a08 100644 --- a/extmod/misc.h +++ b/extmod/misc.h @@ -32,17 +32,17 @@ #include #include "py/runtime.h" -MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_uos_dupterm_obj); +MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_os_dupterm_obj); #if MICROPY_PY_OS_DUPTERM -bool mp_uos_dupterm_is_builtin_stream(mp_const_obj_t stream); -void mp_uos_dupterm_stream_detached_attached(mp_obj_t stream_detached, mp_obj_t stream_attached); -uintptr_t mp_uos_dupterm_poll(uintptr_t poll_flags); -int mp_uos_dupterm_rx_chr(void); -void mp_uos_dupterm_tx_strn(const char *str, size_t len); -void mp_uos_deactivate(size_t dupterm_idx, const char *msg, mp_obj_t exc); +bool mp_os_dupterm_is_builtin_stream(mp_const_obj_t stream); +void mp_os_dupterm_stream_detached_attached(mp_obj_t stream_detached, mp_obj_t stream_attached); +uintptr_t mp_os_dupterm_poll(uintptr_t poll_flags); +int mp_os_dupterm_rx_chr(void); +void mp_os_dupterm_tx_strn(const char *str, size_t len); +void mp_os_deactivate(size_t dupterm_idx, const char *msg, mp_obj_t exc); #else -#define mp_uos_dupterm_tx_strn(s, l) +#define mp_os_dupterm_tx_strn(s, l) #endif #endif // MICROPY_INCLUDED_EXTMOD_MISC_H diff --git a/extmod/moduasyncio.c b/extmod/modasyncio.c similarity index 88% rename from extmod/moduasyncio.c rename to extmod/modasyncio.c index 80e673c94a..96b7c5b0a0 100644 --- a/extmod/moduasyncio.c +++ b/extmod/modasyncio.c @@ -29,7 +29,7 @@ #include "py/pairheap.h" #include "py/mphal.h" -#if MICROPY_PY_UASYNCIO +#if MICROPY_PY_ASYNCIO #if CIRCUITPY && !(defined(__unix__) || defined(__APPLE__)) #include "shared-bindings/supervisor/__init__.h" @@ -68,14 +68,14 @@ STATIC mp_obj_t task_queue_make_new(const mp_obj_type_t *type, size_t n_args, si // Ticks for task ordering in pairing heap STATIC mp_obj_t ticks(void) { - return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_ms() & (MICROPY_PY_UTIME_TICKS_PERIOD - 1)); + return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_ms() & (MICROPY_PY_TIME_TICKS_PERIOD - 1)); } STATIC mp_int_t ticks_diff(mp_obj_t t1_in, mp_obj_t t0_in) { mp_uint_t t0 = MP_OBJ_SMALL_INT_VALUE(t0_in); mp_uint_t t1 = MP_OBJ_SMALL_INT_VALUE(t1_in); - mp_int_t diff = ((t1 - t0 + MICROPY_PY_UTIME_TICKS_PERIOD / 2) & (MICROPY_PY_UTIME_TICKS_PERIOD - 1)) - - MICROPY_PY_UTIME_TICKS_PERIOD / 2; + mp_int_t diff = ((t1 - t0 + MICROPY_PY_TIME_TICKS_PERIOD / 2) & (MICROPY_PY_TIME_TICKS_PERIOD - 1)) + - MICROPY_PY_TIME_TICKS_PERIOD / 2; return diff; } @@ -145,11 +145,6 @@ STATIC const mp_rom_map_elem_t task_queue_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_push), MP_ROM_PTR(&task_queue_push_obj) }, { MP_ROM_QSTR(MP_QSTR_pop), MP_ROM_PTR(&task_queue_pop_obj) }, { MP_ROM_QSTR(MP_QSTR_remove), MP_ROM_PTR(&task_queue_remove_obj) }, - - // CIRCUITPYTHON: Remove these in CircuitPython 10.0.0 - { MP_ROM_QSTR(MP_QSTR_push_head), MP_ROM_PTR(&task_queue_push_obj) }, - { MP_ROM_QSTR(MP_QSTR_push_sorted), MP_ROM_PTR(&task_queue_push_obj) }, - { MP_ROM_QSTR(MP_QSTR_pop_head), MP_ROM_PTR(&task_queue_pop_obj) }, }; STATIC MP_DEFINE_CONST_DICT(task_queue_locals_dict, task_queue_locals_dict_table); @@ -164,8 +159,8 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE( /******************************************************************************/ // Task class -// This is the core uasyncio context with cur_task, _task_queue and CancelledError. -STATIC mp_obj_t uasyncio_context = MP_OBJ_NULL; +// This is the core asyncio context with cur_task, _task_queue and CancelledError. +STATIC mp_obj_t asyncio_context = MP_OBJ_NULL; STATIC mp_obj_t task_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 1, 2, false); @@ -177,7 +172,7 @@ STATIC mp_obj_t task_make_new(const mp_obj_type_t *type, size_t n_args, size_t n self->state = TASK_STATE_RUNNING_NOT_WAITED_ON; self->ph_key = MP_OBJ_NEW_SMALL_INT(0); if (n_args == 2) { - uasyncio_context = args[1]; + asyncio_context = args[1]; } return MP_OBJ_FROM_PTR(self); } @@ -195,7 +190,7 @@ STATIC mp_obj_t task_cancel(mp_obj_t self_in) { return mp_const_false; } // Can't cancel self (not supported yet). - mp_obj_t cur_task = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); + mp_obj_t cur_task = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); if (self_in == cur_task) { mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("can't cancel self")); } @@ -204,7 +199,7 @@ STATIC mp_obj_t task_cancel(mp_obj_t self_in) { self = MP_OBJ_TO_PTR(self->data); } - mp_obj_t _task_queue = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR__task_queue)); + mp_obj_t _task_queue = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR__task_queue)); // Reschedule Task as a cancelled task. mp_obj_t dest[3]; @@ -227,7 +222,7 @@ STATIC mp_obj_t task_cancel(mp_obj_t self_in) { task_queue_push(2, dest); } - self->data = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_CancelledError)); + self->data = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_CancelledError)); return mp_const_true; } @@ -303,7 +298,7 @@ STATIC mp_obj_t task_iternext(mp_obj_t self_in) { } } else { // Put calling task on waiting queue. - mp_obj_t cur_task = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); + mp_obj_t cur_task = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); mp_obj_t args[2] = { self->state, cur_task }; task_queue_push(2, args); // Set calling task's data to this task that it waits on, to double-link it. @@ -327,20 +322,20 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE( ); /******************************************************************************/ -// C-level uasyncio module +// C-level asyncio module -STATIC const mp_rom_map_elem_t mp_module_uasyncio_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR__uasyncio) }, +STATIC const mp_rom_map_elem_t mp_module_asyncio_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR__asyncio) }, { MP_ROM_QSTR(MP_QSTR_TaskQueue), MP_ROM_PTR(&task_queue_type) }, { MP_ROM_QSTR(MP_QSTR_Task), MP_ROM_PTR(&task_type) }, }; -STATIC MP_DEFINE_CONST_DICT(mp_module_uasyncio_globals, mp_module_uasyncio_globals_table); +STATIC MP_DEFINE_CONST_DICT(mp_module_asyncio_globals, mp_module_asyncio_globals_table); -const mp_obj_module_t mp_module_uasyncio = { +const mp_obj_module_t mp_module_asyncio = { .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&mp_module_uasyncio_globals, + .globals = (mp_obj_dict_t *)&mp_module_asyncio_globals, }; -MP_REGISTER_MODULE(MP_QSTR__asyncio, mp_module_uasyncio); +MP_REGISTER_MODULE(MP_QSTR__asyncio, mp_module_asyncio); -#endif // MICROPY_PY_UASYNCIO +#endif // MICROPY_PY_ASYNCIO diff --git a/extmod/modubinascii.c b/extmod/modbinascii.c similarity index 96% rename from extmod/modubinascii.c rename to extmod/modbinascii.c index 59af21ace1..6d08ba1ad3 100644 --- a/extmod/modubinascii.c +++ b/extmod/modbinascii.c @@ -34,7 +34,7 @@ #include "lib/uzlib/tinf.h" -#if MICROPY_PY_UBINASCII +#if MICROPY_PY_BINASCII static void check_not_unicode(const mp_obj_t arg) { #if MICROPY_CPYTHON_COMPAT @@ -48,7 +48,6 @@ static void check_not_unicode(const mp_obj_t arg) { STATIC mp_obj_t bytes_hex_as_bytes(size_t n_args, const mp_obj_t *args) { return mp_obj_bytes_hex(n_args, args, &mp_type_bytes); } - STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(bytes_hex_as_bytes_obj, 1, 2, bytes_hex_as_bytes); STATIC mp_obj_t bytes_fromhex_bytes(mp_obj_t data) { @@ -82,7 +81,7 @@ STATIC mp_obj_t mod_binascii_a2b_base64(mp_obj_t data) { byte *in = bufinfo.buf; vstr_t vstr; - vstr_init(&vstr, (bufinfo.len / 4) * 3 + 1); // Potentially over-allocate + vstr_init(&vstr, (bufinfo.len * 3) / 4 + 1); // Potentially over-allocate byte *out = (byte *)vstr.buf; uint shift = 0; @@ -220,6 +219,8 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_binascii_b2a_base64_obj, 1, mod_binascii_b */ #if MICROPY_PY_UBINASCII_CRC32 +#include "lib/uzlib/tinf.h" + STATIC mp_obj_t mod_binascii_crc32(size_t n_args, const mp_obj_t *args) { mp_buffer_info_t bufinfo; // CIRCUITPY @@ -233,14 +234,14 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_binascii_crc32_obj, 1, 2, mod_bin #endif STATIC const mp_rom_map_elem_t mp_module_binascii_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ubinascii) }, + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_binascii) }, #if MICROPY_PY_BUILTINS_BYTES_HEX { MP_ROM_QSTR(MP_QSTR_hexlify), MP_ROM_PTR(&bytes_hex_as_bytes_obj) }, { MP_ROM_QSTR(MP_QSTR_unhexlify), MP_ROM_PTR(&bytes_fromhex_obj) }, #endif { MP_ROM_QSTR(MP_QSTR_a2b_base64), MP_ROM_PTR(&mod_binascii_a2b_base64_obj) }, { MP_ROM_QSTR(MP_QSTR_b2a_base64), MP_ROM_PTR(&mod_binascii_b2a_base64_obj) }, - #if MICROPY_PY_UBINASCII_CRC32 + #if MICROPY_PY_BINASCII_CRC32 { MP_ROM_QSTR(MP_QSTR_crc32), MP_ROM_PTR(&mod_binascii_crc32_obj) }, #endif }; @@ -252,6 +253,6 @@ const mp_obj_module_t mp_module_ubinascii = { .globals = (mp_obj_dict_t *)&mp_module_binascii_globals, }; -MP_REGISTER_MODULE(MP_QSTR_binascii, mp_module_ubinascii); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_binascii, mp_module_binascii); -#endif // MICROPY_PY_UBINASCII +#endif // MICROPY_PY_BINASCII diff --git a/extmod/moddeflate.c b/extmod/moddeflate.c new file mode 100644 index 0000000000..560ee3f0ab --- /dev/null +++ b/extmod/moddeflate.c @@ -0,0 +1,415 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023 Jim Mussared + * + * Based on extmod/modzlib.c + * Copyright (c) 2014-2016 Paul Sokolovsky + * Copyright (c) 2021-2023 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include + +#include "py/runtime.h" +#include "py/stream.h" +#include "py/mperrno.h" + +#if MICROPY_PY_DEFLATE + +#include "lib/uzlib/uzlib.h" + +#if 0 // print debugging info +#define DEBUG_printf DEBUG_printf +#else // don't print debugging info +#define DEBUG_printf(...) (void)0 +#endif + +typedef enum { + DEFLATEIO_FORMAT_MIN = 0, + DEFLATEIO_FORMAT_AUTO = DEFLATEIO_FORMAT_MIN, // Read mode this means auto-detect zlib/gzip, write mode this means RAW. + DEFLATEIO_FORMAT_RAW = 1, + DEFLATEIO_FORMAT_ZLIB = 2, + DEFLATEIO_FORMAT_GZIP = 3, + DEFLATEIO_FORMAT_MAX = DEFLATEIO_FORMAT_GZIP, +} deflateio_format_t; + +// This is used when the wbits is unset in the DeflateIO constructor. Default +// to the smallest window size (faster compression, less RAM usage, etc). +const int DEFLATEIO_DEFAULT_WBITS = 8; + +typedef struct { + void *window; + uzlib_uncomp_t decomp; + bool eof; +} mp_obj_deflateio_read_t; + +#if MICROPY_PY_DEFLATE_COMPRESS +typedef struct { + void *window; + size_t input_len; + uint32_t input_checksum; + uzlib_lz77_state_t lz77; +} mp_obj_deflateio_write_t; +#endif + +typedef struct { + mp_obj_base_t base; + mp_obj_t stream; + uint8_t format : 2; + uint8_t window_bits : 4; + bool close : 1; + mp_obj_deflateio_read_t *read; + #if MICROPY_PY_DEFLATE_COMPRESS + mp_obj_deflateio_write_t *write; + #endif +} mp_obj_deflateio_t; + +STATIC int deflateio_read_stream(void *data) { + mp_obj_deflateio_t *self = data; + const mp_stream_p_t *stream = mp_get_stream(self->stream); + int err; + byte c; + mp_uint_t out_sz = stream->read(self->stream, &c, 1, &err); + if (out_sz == MP_STREAM_ERROR) { + mp_raise_OSError(err); + } + if (out_sz == 0) { + mp_raise_type(&mp_type_EOFError); + } + return c; +} + +STATIC bool deflateio_init_read(mp_obj_deflateio_t *self) { + if (self->read) { + return true; + } + + mp_get_stream_raise(self->stream, MP_STREAM_OP_READ); + + self->read = m_new_obj(mp_obj_deflateio_read_t); + memset(&self->read->decomp, 0, sizeof(self->read->decomp)); + self->read->decomp.source_read_data = self; + self->read->decomp.source_read_cb = deflateio_read_stream; + self->read->eof = false; + + // Don't modify self->window_bits as it may also be used for write. + int wbits = self->window_bits; + + if (self->format == DEFLATEIO_FORMAT_RAW) { + if (wbits == 0) { + // The docs recommends always setting wbits explicitly when using + // RAW, but we still allow a default. + wbits = DEFLATEIO_DEFAULT_WBITS; + } + } else { + // Parse the header if we're in NONE/ZLIB/GZIP modes. + int header_wbits; + int header_type = uzlib_parse_zlib_gzip_header(&self->read->decomp, &header_wbits); + if (header_type < 0) { + // Stream header was invalid. + return false; + } + if ((self->format == DEFLATEIO_FORMAT_ZLIB && header_type != UZLIB_HEADER_ZLIB) || (self->format == DEFLATEIO_FORMAT_GZIP && header_type != UZLIB_HEADER_GZIP)) { + // Not what we expected. + return false; + } + // header_wbits will either be 15 (gzip) or 8-15 (zlib). + if (wbits == 0 || header_wbits < wbits) { + // If the header specified something lower, then use that instead. + // No point doing a bigger allocation than we need to. + wbits = header_wbits; + } + } + + size_t window_len = 1 << wbits; + self->read->window = m_new(uint8_t, window_len); + + uzlib_uncompress_init(&self->read->decomp, self->read->window, window_len); + + return true; +} + +#if MICROPY_PY_DEFLATE_COMPRESS +STATIC void deflateio_out_byte(void *data, uint8_t b) { + mp_obj_deflateio_t *self = data; + const mp_stream_p_t *stream = mp_get_stream(self->stream); + int err; + mp_uint_t ret = stream->write(self->stream, &b, 1, &err); + if (ret == MP_STREAM_ERROR) { + mp_raise_OSError(err); + } +} + +STATIC bool deflateio_init_write(mp_obj_deflateio_t *self) { + if (self->write) { + return true; + } + + const mp_stream_p_t *stream = mp_get_stream_raise(self->stream, MP_STREAM_OP_WRITE); + + self->write = m_new_obj(mp_obj_deflateio_write_t); + self->write->input_len = 0; + + int wbits = self->window_bits; + if (wbits == 0) { + // Same default wbits for all formats. + wbits = DEFLATEIO_DEFAULT_WBITS; + } + size_t window_len = 1 << wbits; + self->write->window = m_new(uint8_t, window_len); + + uzlib_lz77_init(&self->write->lz77, self->write->window, window_len); + self->write->lz77.dest_write_data = self; + self->write->lz77.dest_write_cb = deflateio_out_byte; + + // Write header if needed. + mp_uint_t ret = 0; + int err; + if (self->format == DEFLATEIO_FORMAT_ZLIB) { + // -----CMF------ ----------FLG--------------- + // CINFO(5) CM(3) FLEVEL(2) FDICT(1) FCHECK(5) + uint8_t buf[] = { 0x08, 0x80 }; // CM=2 (deflate), FLEVEL=2 (default), FDICT=0 (no dictionary) + buf[0] |= MAX(wbits - 8, 1) << 4; // base-2 logarithm of the LZ77 window size, minus eight. + buf[1] |= 31 - ((buf[0] * 256 + buf[1]) % 31); // (CMF*256 + FLG) % 31 == 0. + ret = stream->write(self->stream, buf, sizeof(buf), &err); + + self->write->input_checksum = 1; // ADLER32 + } else if (self->format == DEFLATEIO_FORMAT_GZIP) { + // ID1(8) ID2(8) CM(8) ---FLG--- MTIME(32) XFL(8) OS(8) + // FLG: x x x FCOMMENT FNAME FEXTRA FHCRC FTEXT + uint8_t buf[] = { 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03 }; // MTIME=0, XFL=4 (fastest), OS=3 (unix) + ret = stream->write(self->stream, buf, sizeof(buf), &err); + + self->write->input_checksum = ~0; // CRC32 + } + if (ret == MP_STREAM_ERROR) { + return false; + } + + // Write starting block. + uzlib_start_block(&self->write->lz77); + + return true; +} +#endif + +STATIC mp_obj_t deflateio_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args_in) { + // args: stream, format=NONE, wbits=0, close=False + mp_arg_check_num(n_args, n_kw, 1, 4, false); + + mp_int_t format = n_args > 1 ? mp_obj_get_int(args_in[1]) : DEFLATEIO_FORMAT_AUTO; + mp_int_t wbits = n_args > 2 ? mp_obj_get_int(args_in[2]) : 0; + + if (format < DEFLATEIO_FORMAT_MIN || format > DEFLATEIO_FORMAT_MAX) { + mp_raise_ValueError(MP_ERROR_TEXT("format")); + } + if (wbits != 0 && (wbits < 5 || wbits > 15)) { + mp_raise_ValueError(MP_ERROR_TEXT("wbits")); + } + + mp_obj_deflateio_t *self = mp_obj_malloc(mp_obj_deflateio_t, type); + self->stream = args_in[0]; + self->format = format; + self->window_bits = wbits; + self->read = NULL; + #if MICROPY_PY_DEFLATE_COMPRESS + self->write = NULL; + #endif + self->close = n_args > 3 ? mp_obj_is_true(args_in[3]) : false; + + return MP_OBJ_FROM_PTR(self); +} + +STATIC mp_uint_t deflateio_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { + mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(o_in); + + if (self->stream == MP_OBJ_NULL || !deflateio_init_read(self)) { + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } + + if (self->read->eof) { + return 0; + } + + self->read->decomp.dest = buf; + self->read->decomp.dest_limit = (uint8_t *)buf + size; + int st = uzlib_uncompress_chksum(&self->read->decomp); + if (st == UZLIB_DONE) { + self->read->eof = true; + } + if (st < 0) { + DEBUG_printf("uncompress error=" INT_FMT "\n", st); + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } + return self->read->decomp.dest - (uint8_t *)buf; +} + +#if MICROPY_PY_DEFLATE_COMPRESS +STATIC mp_uint_t deflateio_write(mp_obj_t self_in, const void *buf, mp_uint_t size, int *errcode) { + mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(self_in); + + if (self->stream == MP_OBJ_NULL || !deflateio_init_write(self)) { + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } + + self->write->input_len += size; + if (self->format == DEFLATEIO_FORMAT_ZLIB) { + self->write->input_checksum = uzlib_adler32(buf, size, self->write->input_checksum); + } else if (self->format == DEFLATEIO_FORMAT_GZIP) { + self->write->input_checksum = uzlib_crc32(buf, size, self->write->input_checksum); + } + + uzlib_lz77_compress(&self->write->lz77, buf, size); + return size; +} + +static inline void put_le32(char *buf, uint32_t value) { + buf[0] = value & 0xff; + buf[1] = value >> 8 & 0xff; + buf[2] = value >> 16 & 0xff; + buf[3] = value >> 24 & 0xff; +} + +static inline void put_be32(char *buf, uint32_t value) { + buf[3] = value & 0xff; + buf[2] = value >> 8 & 0xff; + buf[1] = value >> 16 & 0xff; + buf[0] = value >> 24 & 0xff; +} +#endif + +STATIC mp_uint_t deflateio_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, int *errcode) { + if (request == MP_STREAM_CLOSE) { + mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(self_in); + + mp_uint_t ret = 0; + + if (self->stream != MP_OBJ_NULL) { + #if MICROPY_PY_DEFLATE_COMPRESS + if (self->write) { + uzlib_finish_block(&self->write->lz77); + + const mp_stream_p_t *stream = mp_get_stream(self->stream); + + // Write footer if needed. + if (self->format == DEFLATEIO_FORMAT_ZLIB || self->format == DEFLATEIO_FORMAT_GZIP) { + char footer[8]; + size_t footer_len; + if (self->format == DEFLATEIO_FORMAT_ZLIB) { + put_be32(&footer[0], self->write->input_checksum); + footer_len = 4; + } else { // DEFLATEIO_FORMAT_GZIP + put_le32(&footer[0], ~self->write->input_checksum); + put_le32(&footer[4], self->write->input_len); + footer_len = 8; + } + if (stream->write(self->stream, footer, footer_len, errcode) == MP_STREAM_ERROR) { + ret = MP_STREAM_ERROR; + } + } + } + #endif + + // Only close the stream if required. e.g. when using io.BytesIO + // it needs to stay open so that getvalue() can be called. + if (self->close) { + mp_stream_close(self->stream); + } + + // Either way, free the reference to the stream. + self->stream = MP_OBJ_NULL; + } + + return ret; + } else { + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } +} + +STATIC const mp_stream_p_t deflateio_stream_p = { + .read = deflateio_read, + #if MICROPY_PY_DEFLATE_COMPRESS + .write = deflateio_write, + #endif + .ioctl = deflateio_ioctl, +}; + +#if !MICROPY_ENABLE_DYNRUNTIME +STATIC const mp_rom_map_elem_t deflateio_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, + { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, + { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, + #if MICROPY_PY_DEFLATE_COMPRESS + { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, + #endif + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, +}; +STATIC MP_DEFINE_CONST_DICT(deflateio_locals_dict, deflateio_locals_dict_table); + +STATIC MP_DEFINE_CONST_OBJ_TYPE( + deflateio_type, + MP_QSTR_DeflateIO, + MP_TYPE_FLAG_NONE, + make_new, deflateio_make_new, + protocol, &deflateio_stream_p, + locals_dict, &deflateio_locals_dict + ); + +STATIC const mp_rom_map_elem_t mp_module_deflate_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_deflate) }, + { MP_ROM_QSTR(MP_QSTR_DeflateIO), MP_ROM_PTR(&deflateio_type) }, + { MP_ROM_QSTR(MP_QSTR_AUTO), MP_ROM_INT(DEFLATEIO_FORMAT_AUTO) }, + { MP_ROM_QSTR(MP_QSTR_RAW), MP_ROM_INT(DEFLATEIO_FORMAT_RAW) }, + { MP_ROM_QSTR(MP_QSTR_ZLIB), MP_ROM_INT(DEFLATEIO_FORMAT_ZLIB) }, + { MP_ROM_QSTR(MP_QSTR_GZIP), MP_ROM_INT(DEFLATEIO_FORMAT_GZIP) }, +}; +STATIC MP_DEFINE_CONST_DICT(mp_module_deflate_globals, mp_module_deflate_globals_table); + +const mp_obj_module_t mp_module_deflate = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mp_module_deflate_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_deflate, mp_module_deflate); +#endif // !MICROPY_ENABLE_DYNRUNTIME + +// Source files #include'd here to make sure they're compiled in +// only if the module is enabled. + +#include "lib/uzlib/tinflate.c" +#include "lib/uzlib/header.c" +#include "lib/uzlib/adler32.c" +#include "lib/uzlib/crc32.c" + +#if MICROPY_PY_DEFLATE_COMPRESS +#include "lib/uzlib/lz77.c" +#endif + +#endif // MICROPY_PY_DEFLATE diff --git a/extmod/moduhashlib.c b/extmod/modhashlib.c similarity index 62% rename from extmod/moduhashlib.c rename to extmod/modhashlib.c index 6850434129..ef98a80166 100644 --- a/extmod/moduhashlib.c +++ b/extmod/modhashlib.c @@ -29,13 +29,13 @@ #include "py/runtime.h" -#if MICROPY_PY_UHASHLIB +#if MICROPY_PY_HASHLIB #if MICROPY_SSL_MBEDTLS #include "mbedtls/version.h" #endif -#if MICROPY_PY_UHASHLIB_SHA256 +#if MICROPY_PY_HASHLIB_SHA256 #if MICROPY_SSL_MBEDTLS #include "mbedtls/sha256.h" @@ -45,7 +45,7 @@ #endif -#if MICROPY_PY_UHASHLIB_SHA1 || MICROPY_PY_UHASHLIB_MD5 +#if MICROPY_PY_HASHLIB_SHA1 || MICROPY_PY_HASHLIB_MD5 #if MICROPY_SSL_AXTLS #include "lib/axtls/crypto/crypto.h" @@ -64,47 +64,47 @@ typedef struct _mp_obj_hash_t { uintptr_t state[0]; // must be aligned to a machine word } mp_obj_hash_t; -static void uhashlib_ensure_not_final(mp_obj_hash_t *self) { +static void hashlib_ensure_not_final(mp_obj_hash_t *self) { if (self->final) { mp_raise_ValueError(MP_ERROR_TEXT("hash is final")); } } -#if MICROPY_PY_UHASHLIB_SHA256 -STATIC mp_obj_t uhashlib_sha256_update(mp_obj_t self_in, mp_obj_t arg); +#if MICROPY_PY_HASHLIB_SHA256 +STATIC mp_obj_t hashlib_sha256_update(mp_obj_t self_in, mp_obj_t arg); #if MICROPY_SSL_MBEDTLS -#if MBEDTLS_VERSION_NUMBER < 0x02070000 +#if MBEDTLS_VERSION_NUMBER < 0x02070000 || MBEDTLS_VERSION_NUMBER >= 0x03000000 #define mbedtls_sha256_starts_ret mbedtls_sha256_starts #define mbedtls_sha256_update_ret mbedtls_sha256_update #define mbedtls_sha256_finish_ret mbedtls_sha256_finish #endif -STATIC mp_obj_t uhashlib_sha256_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { +STATIC mp_obj_t hashlib_sha256_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(mbedtls_sha256_context), type); o->final = false; mbedtls_sha256_init((mbedtls_sha256_context *)&o->state); mbedtls_sha256_starts_ret((mbedtls_sha256_context *)&o->state, 0); if (n_args == 1) { - uhashlib_sha256_update(MP_OBJ_FROM_PTR(o), args[0]); + hashlib_sha256_update(MP_OBJ_FROM_PTR(o), args[0]); } return MP_OBJ_FROM_PTR(o); } -STATIC mp_obj_t uhashlib_sha256_update(mp_obj_t self_in, mp_obj_t arg) { +STATIC mp_obj_t hashlib_sha256_update(mp_obj_t self_in, mp_obj_t arg) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); mp_buffer_info_t bufinfo; mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ); mbedtls_sha256_update_ret((mbedtls_sha256_context *)&self->state, bufinfo.buf, bufinfo.len); return mp_const_none; } -STATIC mp_obj_t uhashlib_sha256_digest(mp_obj_t self_in) { +STATIC mp_obj_t hashlib_sha256_digest(mp_obj_t self_in) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); self->final = true; vstr_t vstr; vstr_init_len(&vstr, 32); @@ -124,30 +124,29 @@ static void check_not_unicode(const mp_obj_t arg) { #include "lib/crypto-algorithms/sha256.c" -STATIC mp_obj_t uhashlib_sha256_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { +STATIC mp_obj_t hashlib_sha256_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(CRYAL_SHA256_CTX), type); o->final = false; sha256_init((CRYAL_SHA256_CTX *)o->state); if (n_args == 1) { - uhashlib_sha256_update(MP_OBJ_FROM_PTR(o), args[0]); + hashlib_sha256_update(MP_OBJ_FROM_PTR(o), args[0]); } return MP_OBJ_FROM_PTR(o); } -STATIC mp_obj_t uhashlib_sha256_update(mp_obj_t self_in, mp_obj_t arg) { - check_not_unicode(arg); +STATIC mp_obj_t hashlib_sha256_update(mp_obj_t self_in, mp_obj_t arg) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); mp_buffer_info_t bufinfo; mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ); sha256_update((CRYAL_SHA256_CTX *)self->state, bufinfo.buf, bufinfo.len); return mp_const_none; } -STATIC mp_obj_t uhashlib_sha256_digest(mp_obj_t self_in) { +STATIC mp_obj_t hashlib_sha256_digest(mp_obj_t self_in) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); self->final = true; vstr_t vstr; vstr_init_len(&vstr, SHA256_BLOCK_SIZE); @@ -156,52 +155,52 @@ STATIC mp_obj_t uhashlib_sha256_digest(mp_obj_t self_in) { } #endif -STATIC MP_DEFINE_CONST_FUN_OBJ_2(uhashlib_sha256_update_obj, uhashlib_sha256_update); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(uhashlib_sha256_digest_obj, uhashlib_sha256_digest); +STATIC MP_DEFINE_CONST_FUN_OBJ_2(hashlib_sha256_update_obj, hashlib_sha256_update); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(hashlib_sha256_digest_obj, hashlib_sha256_digest); -STATIC const mp_rom_map_elem_t uhashlib_sha256_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR_update), MP_ROM_PTR(&uhashlib_sha256_update_obj) }, - { MP_ROM_QSTR(MP_QSTR_digest), MP_ROM_PTR(&uhashlib_sha256_digest_obj) }, +STATIC const mp_rom_map_elem_t hashlib_sha256_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_update), MP_ROM_PTR(&hashlib_sha256_update_obj) }, + { MP_ROM_QSTR(MP_QSTR_digest), MP_ROM_PTR(&hashlib_sha256_digest_obj) }, }; -STATIC MP_DEFINE_CONST_DICT(uhashlib_sha256_locals_dict, uhashlib_sha256_locals_dict_table); +STATIC MP_DEFINE_CONST_DICT(hashlib_sha256_locals_dict, hashlib_sha256_locals_dict_table); STATIC MP_DEFINE_CONST_OBJ_TYPE( - uhashlib_sha256_type, + hashlib_sha256_type, MP_QSTR_sha256, MP_TYPE_FLAG_NONE, - make_new, uhashlib_sha256_make_new, - locals_dict, &uhashlib_sha256_locals_dict + make_new, hashlib_sha256_make_new, + locals_dict, &hashlib_sha256_locals_dict ); #endif -#if MICROPY_PY_UHASHLIB_SHA1 -STATIC mp_obj_t uhashlib_sha1_update(mp_obj_t self_in, mp_obj_t arg); +#if MICROPY_PY_HASHLIB_SHA1 +STATIC mp_obj_t hashlib_sha1_update(mp_obj_t self_in, mp_obj_t arg); #if MICROPY_SSL_AXTLS -STATIC mp_obj_t uhashlib_sha1_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { +STATIC mp_obj_t hashlib_sha1_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(SHA1_CTX), type); o->final = false; SHA1_Init((SHA1_CTX *)o->state); if (n_args == 1) { - uhashlib_sha1_update(MP_OBJ_FROM_PTR(o), args[0]); + hashlib_sha1_update(MP_OBJ_FROM_PTR(o), args[0]); } return MP_OBJ_FROM_PTR(o); } -STATIC mp_obj_t uhashlib_sha1_update(mp_obj_t self_in, mp_obj_t arg) { +STATIC mp_obj_t hashlib_sha1_update(mp_obj_t self_in, mp_obj_t arg) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); mp_buffer_info_t bufinfo; mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ); SHA1_Update((SHA1_CTX *)self->state, bufinfo.buf, bufinfo.len); return mp_const_none; } -STATIC mp_obj_t uhashlib_sha1_digest(mp_obj_t self_in) { +STATIC mp_obj_t hashlib_sha1_digest(mp_obj_t self_in) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); self->final = true; vstr_t vstr; vstr_init_len(&vstr, SHA1_SIZE); @@ -212,36 +211,36 @@ STATIC mp_obj_t uhashlib_sha1_digest(mp_obj_t self_in) { #if MICROPY_SSL_MBEDTLS -#if MBEDTLS_VERSION_NUMBER < 0x02070000 +#if MBEDTLS_VERSION_NUMBER < 0x02070000 || MBEDTLS_VERSION_NUMBER >= 0x03000000 #define mbedtls_sha1_starts_ret mbedtls_sha1_starts #define mbedtls_sha1_update_ret mbedtls_sha1_update #define mbedtls_sha1_finish_ret mbedtls_sha1_finish #endif -STATIC mp_obj_t uhashlib_sha1_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { +STATIC mp_obj_t hashlib_sha1_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(mbedtls_sha1_context), type); o->final = false; mbedtls_sha1_init((mbedtls_sha1_context *)o->state); mbedtls_sha1_starts_ret((mbedtls_sha1_context *)o->state); if (n_args == 1) { - uhashlib_sha1_update(MP_OBJ_FROM_PTR(o), args[0]); + hashlib_sha1_update(MP_OBJ_FROM_PTR(o), args[0]); } return MP_OBJ_FROM_PTR(o); } -STATIC mp_obj_t uhashlib_sha1_update(mp_obj_t self_in, mp_obj_t arg) { +STATIC mp_obj_t hashlib_sha1_update(mp_obj_t self_in, mp_obj_t arg) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); mp_buffer_info_t bufinfo; mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ); mbedtls_sha1_update_ret((mbedtls_sha1_context *)self->state, bufinfo.buf, bufinfo.len); return mp_const_none; } -STATIC mp_obj_t uhashlib_sha1_digest(mp_obj_t self_in) { +STATIC mp_obj_t hashlib_sha1_digest(mp_obj_t self_in) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); self->final = true; vstr_t vstr; vstr_init_len(&vstr, 20); @@ -251,51 +250,51 @@ STATIC mp_obj_t uhashlib_sha1_digest(mp_obj_t self_in) { } #endif -STATIC MP_DEFINE_CONST_FUN_OBJ_2(uhashlib_sha1_update_obj, uhashlib_sha1_update); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(uhashlib_sha1_digest_obj, uhashlib_sha1_digest); +STATIC MP_DEFINE_CONST_FUN_OBJ_2(hashlib_sha1_update_obj, hashlib_sha1_update); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(hashlib_sha1_digest_obj, hashlib_sha1_digest); -STATIC const mp_rom_map_elem_t uhashlib_sha1_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR_update), MP_ROM_PTR(&uhashlib_sha1_update_obj) }, - { MP_ROM_QSTR(MP_QSTR_digest), MP_ROM_PTR(&uhashlib_sha1_digest_obj) }, +STATIC const mp_rom_map_elem_t hashlib_sha1_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_update), MP_ROM_PTR(&hashlib_sha1_update_obj) }, + { MP_ROM_QSTR(MP_QSTR_digest), MP_ROM_PTR(&hashlib_sha1_digest_obj) }, }; -STATIC MP_DEFINE_CONST_DICT(uhashlib_sha1_locals_dict, uhashlib_sha1_locals_dict_table); +STATIC MP_DEFINE_CONST_DICT(hashlib_sha1_locals_dict, hashlib_sha1_locals_dict_table); STATIC MP_DEFINE_CONST_OBJ_TYPE( - uhashlib_sha1_type, + hashlib_sha1_type, MP_QSTR_sha1, MP_TYPE_FLAG_NONE, - make_new, uhashlib_sha1_make_new, - locals_dict, &uhashlib_sha1_locals_dict + make_new, hashlib_sha1_make_new, + locals_dict, &hashlib_sha1_locals_dict ); #endif -#if MICROPY_PY_UHASHLIB_MD5 -STATIC mp_obj_t uhashlib_md5_update(mp_obj_t self_in, mp_obj_t arg); +#if MICROPY_PY_HASHLIB_MD5 +STATIC mp_obj_t hashlib_md5_update(mp_obj_t self_in, mp_obj_t arg); #if MICROPY_SSL_AXTLS -STATIC mp_obj_t uhashlib_md5_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { +STATIC mp_obj_t hashlib_md5_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(MD5_CTX), type); o->final = false; MD5_Init((MD5_CTX *)o->state); if (n_args == 1) { - uhashlib_md5_update(MP_OBJ_FROM_PTR(o), args[0]); + hashlib_md5_update(MP_OBJ_FROM_PTR(o), args[0]); } return MP_OBJ_FROM_PTR(o); } -STATIC mp_obj_t uhashlib_md5_update(mp_obj_t self_in, mp_obj_t arg) { +STATIC mp_obj_t hashlib_md5_update(mp_obj_t self_in, mp_obj_t arg) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); mp_buffer_info_t bufinfo; mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ); MD5_Update((MD5_CTX *)self->state, bufinfo.buf, bufinfo.len); return mp_const_none; } -STATIC mp_obj_t uhashlib_md5_digest(mp_obj_t self_in) { +STATIC mp_obj_t hashlib_md5_digest(mp_obj_t self_in) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); self->final = true; vstr_t vstr; vstr_init_len(&vstr, MD5_SIZE); @@ -312,30 +311,30 @@ STATIC mp_obj_t uhashlib_md5_digest(mp_obj_t self_in) { #define mbedtls_md5_finish_ret mbedtls_md5_finish #endif -STATIC mp_obj_t uhashlib_md5_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { +STATIC mp_obj_t hashlib_md5_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(mbedtls_md5_context), type); o->final = false; mbedtls_md5_init((mbedtls_md5_context *)o->state); mbedtls_md5_starts_ret((mbedtls_md5_context *)o->state); if (n_args == 1) { - uhashlib_md5_update(MP_OBJ_FROM_PTR(o), args[0]); + hashlib_md5_update(MP_OBJ_FROM_PTR(o), args[0]); } return MP_OBJ_FROM_PTR(o); } -STATIC mp_obj_t uhashlib_md5_update(mp_obj_t self_in, mp_obj_t arg) { +STATIC mp_obj_t hashlib_md5_update(mp_obj_t self_in, mp_obj_t arg) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); mp_buffer_info_t bufinfo; mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ); mbedtls_md5_update_ret((mbedtls_md5_context *)self->state, bufinfo.buf, bufinfo.len); return mp_const_none; } -STATIC mp_obj_t uhashlib_md5_digest(mp_obj_t self_in) { +STATIC mp_obj_t hashlib_md5_digest(mp_obj_t self_in) { mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); - uhashlib_ensure_not_final(self); + hashlib_ensure_not_final(self); self->final = true; vstr_t vstr; vstr_init_len(&vstr, 16); @@ -345,44 +344,44 @@ STATIC mp_obj_t uhashlib_md5_digest(mp_obj_t self_in) { } #endif // MICROPY_SSL_MBEDTLS -STATIC MP_DEFINE_CONST_FUN_OBJ_2(uhashlib_md5_update_obj, uhashlib_md5_update); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(uhashlib_md5_digest_obj, uhashlib_md5_digest); +STATIC MP_DEFINE_CONST_FUN_OBJ_2(hashlib_md5_update_obj, hashlib_md5_update); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(hashlib_md5_digest_obj, hashlib_md5_digest); -STATIC const mp_rom_map_elem_t uhashlib_md5_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR_update), MP_ROM_PTR(&uhashlib_md5_update_obj) }, - { MP_ROM_QSTR(MP_QSTR_digest), MP_ROM_PTR(&uhashlib_md5_digest_obj) }, +STATIC const mp_rom_map_elem_t hashlib_md5_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_update), MP_ROM_PTR(&hashlib_md5_update_obj) }, + { MP_ROM_QSTR(MP_QSTR_digest), MP_ROM_PTR(&hashlib_md5_digest_obj) }, }; -STATIC MP_DEFINE_CONST_DICT(uhashlib_md5_locals_dict, uhashlib_md5_locals_dict_table); +STATIC MP_DEFINE_CONST_DICT(hashlib_md5_locals_dict, hashlib_md5_locals_dict_table); STATIC MP_DEFINE_CONST_OBJ_TYPE( - uhashlib_md5_type, + hashlib_md5_type, MP_QSTR_md5, MP_TYPE_FLAG_NONE, - make_new, uhashlib_md5_make_new, - locals_dict, &uhashlib_md5_locals_dict + make_new, hashlib_md5_make_new, + locals_dict, &hashlib_md5_locals_dict ); -#endif // MICROPY_PY_UHASHLIB_MD5 +#endif // MICROPY_PY_HASHLIB_MD5 -STATIC const mp_rom_map_elem_t mp_module_uhashlib_globals_table[] = { +STATIC const mp_rom_map_elem_t mp_module_hashlib_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_hashlib) }, - #if MICROPY_PY_UHASHLIB_SHA256 - { MP_ROM_QSTR(MP_QSTR_sha256), MP_ROM_PTR(&uhashlib_sha256_type) }, + #if MICROPY_PY_HASHLIB_SHA256 + { MP_ROM_QSTR(MP_QSTR_sha256), MP_ROM_PTR(&hashlib_sha256_type) }, #endif - #if MICROPY_PY_UHASHLIB_SHA1 - { MP_ROM_QSTR(MP_QSTR_sha1), MP_ROM_PTR(&uhashlib_sha1_type) }, + #if MICROPY_PY_HASHLIB_SHA1 + { MP_ROM_QSTR(MP_QSTR_sha1), MP_ROM_PTR(&hashlib_sha1_type) }, #endif - #if MICROPY_PY_UHASHLIB_MD5 - { MP_ROM_QSTR(MP_QSTR_md5), MP_ROM_PTR(&uhashlib_md5_type) }, + #if MICROPY_PY_HASHLIB_MD5 + { MP_ROM_QSTR(MP_QSTR_md5), MP_ROM_PTR(&hashlib_md5_type) }, #endif }; -STATIC MP_DEFINE_CONST_DICT(mp_module_uhashlib_globals, mp_module_uhashlib_globals_table); +STATIC MP_DEFINE_CONST_DICT(mp_module_hashlib_globals, mp_module_hashlib_globals_table); -const mp_obj_module_t mp_module_uhashlib = { +const mp_obj_module_t mp_module_hashlib = { .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&mp_module_uhashlib_globals, + .globals = (mp_obj_dict_t *)&mp_module_hashlib_globals, }; -MP_REGISTER_MODULE(MP_QSTR_hashlib, mp_module_uhashlib); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_hashlib, mp_module_hashlib); -#endif // MICROPY_PY_UHASHLIB +#endif // MICROPY_PY_HASHLIB diff --git a/extmod/moduheapq.c b/extmod/modheapq.c similarity index 65% rename from extmod/moduheapq.c rename to extmod/modheapq.c index a2317fb120..db1e35bac2 100644 --- a/extmod/moduheapq.c +++ b/extmod/modheapq.c @@ -27,18 +27,18 @@ #include "py/objlist.h" #include "py/runtime.h" -#if MICROPY_PY_UHEAPQ +#if MICROPY_PY_HEAPQ // the algorithm here is modelled on CPython's heapq.py -STATIC mp_obj_list_t *uheapq_get_heap(mp_obj_t heap_in) { +STATIC mp_obj_list_t *heapq_get_heap(mp_obj_t heap_in) { if (!mp_obj_is_type(heap_in, &mp_type_list)) { mp_raise_TypeError(MP_ERROR_TEXT("heap must be a list")); } return MP_OBJ_TO_PTR(heap_in); } -STATIC void uheapq_heap_siftdown(mp_obj_list_t *heap, mp_uint_t start_pos, mp_uint_t pos) { +STATIC void heapq_heap_siftdown(mp_obj_list_t *heap, mp_uint_t start_pos, mp_uint_t pos) { mp_obj_t item = heap->items[pos]; while (pos > start_pos) { mp_uint_t parent_pos = (pos - 1) >> 1; @@ -53,7 +53,7 @@ STATIC void uheapq_heap_siftdown(mp_obj_list_t *heap, mp_uint_t start_pos, mp_ui heap->items[pos] = item; } -STATIC void uheapq_heap_siftup(mp_obj_list_t *heap, mp_uint_t pos) { +STATIC void heapq_heap_siftup(mp_obj_list_t *heap, mp_uint_t pos) { mp_uint_t start_pos = pos; mp_uint_t end_pos = heap->len; mp_obj_t item = heap->items[pos]; @@ -67,19 +67,19 @@ STATIC void uheapq_heap_siftup(mp_obj_list_t *heap, mp_uint_t pos) { pos = child_pos; } heap->items[pos] = item; - uheapq_heap_siftdown(heap, start_pos, pos); + heapq_heap_siftdown(heap, start_pos, pos); } -STATIC mp_obj_t mod_uheapq_heappush(mp_obj_t heap_in, mp_obj_t item) { - mp_obj_list_t *heap = uheapq_get_heap(heap_in); +STATIC mp_obj_t mod_heapq_heappush(mp_obj_t heap_in, mp_obj_t item) { + mp_obj_list_t *heap = heapq_get_heap(heap_in); mp_obj_list_append(heap_in, item); - uheapq_heap_siftdown(heap, 0, heap->len - 1); + heapq_heap_siftdown(heap, 0, heap->len - 1); return mp_const_none; } -STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_uheapq_heappush_obj, mod_uheapq_heappush); +STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_heapq_heappush_obj, mod_heapq_heappush); -STATIC mp_obj_t mod_uheapq_heappop(mp_obj_t heap_in) { - mp_obj_list_t *heap = uheapq_get_heap(heap_in); +STATIC mp_obj_t mod_heapq_heappop(mp_obj_t heap_in) { + mp_obj_list_t *heap = heapq_get_heap(heap_in); if (heap->len == 0) { mp_raise_msg(&mp_type_IndexError, MP_ERROR_TEXT("empty heap")); } @@ -88,37 +88,37 @@ STATIC mp_obj_t mod_uheapq_heappop(mp_obj_t heap_in) { heap->items[0] = heap->items[heap->len]; heap->items[heap->len] = MP_OBJ_NULL; // so we don't retain a pointer if (heap->len) { - uheapq_heap_siftup(heap, 0); + heapq_heap_siftup(heap, 0); } return item; } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_uheapq_heappop_obj, mod_uheapq_heappop); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_heapq_heappop_obj, mod_heapq_heappop); -STATIC mp_obj_t mod_uheapq_heapify(mp_obj_t heap_in) { - mp_obj_list_t *heap = uheapq_get_heap(heap_in); +STATIC mp_obj_t mod_heapq_heapify(mp_obj_t heap_in) { + mp_obj_list_t *heap = heapq_get_heap(heap_in); for (mp_uint_t i = heap->len / 2; i > 0;) { - uheapq_heap_siftup(heap, --i); + heapq_heap_siftup(heap, --i); } return mp_const_none; } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_uheapq_heapify_obj, mod_uheapq_heapify); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_heapq_heapify_obj, mod_heapq_heapify); #if !MICROPY_ENABLE_DYNRUNTIME -STATIC const mp_rom_map_elem_t mp_module_uheapq_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uheapq) }, - { MP_ROM_QSTR(MP_QSTR_heappush), MP_ROM_PTR(&mod_uheapq_heappush_obj) }, - { MP_ROM_QSTR(MP_QSTR_heappop), MP_ROM_PTR(&mod_uheapq_heappop_obj) }, - { MP_ROM_QSTR(MP_QSTR_heapify), MP_ROM_PTR(&mod_uheapq_heapify_obj) }, +STATIC const mp_rom_map_elem_t mp_module_heapq_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_heapq) }, + { MP_ROM_QSTR(MP_QSTR_heappush), MP_ROM_PTR(&mod_heapq_heappush_obj) }, + { MP_ROM_QSTR(MP_QSTR_heappop), MP_ROM_PTR(&mod_heapq_heappop_obj) }, + { MP_ROM_QSTR(MP_QSTR_heapify), MP_ROM_PTR(&mod_heapq_heapify_obj) }, }; -STATIC MP_DEFINE_CONST_DICT(mp_module_uheapq_globals, mp_module_uheapq_globals_table); +STATIC MP_DEFINE_CONST_DICT(mp_module_heapq_globals, mp_module_heapq_globals_table); -const mp_obj_module_t mp_module_uheapq = { +const mp_obj_module_t mp_module_heapq = { .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&mp_module_uheapq_globals, + .globals = (mp_obj_dict_t *)&mp_module_heapq_globals, }; -MP_REGISTER_MODULE(MP_QSTR_heapq, mp_module_uheapq); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_heapq, mp_module_heapq); #endif -#endif // MICROPY_PY_UHEAPQ +#endif // MICROPY_PY_HEAPQ diff --git a/extmod/modujson.c b/extmod/modjson.c similarity index 85% rename from extmod/modujson.c rename to extmod/modjson.c index 5234ecfd0e..c39ffaf5fb 100644 --- a/extmod/modujson.c +++ b/extmod/modjson.c @@ -26,24 +26,22 @@ #include -#include "py/binary.h" -#include "py/objarray.h" #include "py/objlist.h" #include "py/objstringio.h" #include "py/parsenum.h" #include "py/runtime.h" #include "py/stream.h" -#if MICROPY_PY_UJSON +#if MICROPY_PY_JSON -#if MICROPY_PY_UJSON_SEPARATORS +#if MICROPY_PY_JSON_SEPARATORS enum { DUMP_MODE_TO_STRING = 1, DUMP_MODE_TO_STREAM = 2, }; -STATIC mp_obj_t mod_ujson_dump_helper(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args, unsigned int mode) { +STATIC mp_obj_t mod_json_dump_helper(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args, unsigned int mode) { enum { ARG_separators }; static const mp_arg_t allowed_args[] = { { MP_QSTR_separators, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_NONE} }, @@ -80,34 +78,35 @@ STATIC mp_obj_t mod_ujson_dump_helper(size_t n_args, const mp_obj_t *pos_args, m } } -STATIC mp_obj_t mod_ujson_dump(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { - return mod_ujson_dump_helper(n_args, pos_args, kw_args, DUMP_MODE_TO_STREAM); +STATIC mp_obj_t mod_json_dump(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { + return mod_json_dump_helper(n_args, pos_args, kw_args, DUMP_MODE_TO_STREAM); } -STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_ujson_dump_obj, 2, mod_ujson_dump); +STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_json_dump_obj, 2, mod_json_dump); -STATIC mp_obj_t mod_ujson_dumps(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { - return mod_ujson_dump_helper(n_args, pos_args, kw_args, DUMP_MODE_TO_STRING); +STATIC mp_obj_t mod_json_dumps(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { + return mod_json_dump_helper(n_args, pos_args, kw_args, DUMP_MODE_TO_STRING); } -STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_ujson_dumps_obj, 1, mod_ujson_dumps); +STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_json_dumps_obj, 1, mod_json_dumps); #else -STATIC mp_obj_t mod_ujson_dump(mp_obj_t obj, mp_obj_t stream) { +STATIC mp_obj_t mod_json_dump(mp_obj_t obj, mp_obj_t stream) { mp_get_stream_raise(stream, MP_STREAM_OP_WRITE); mp_print_t print = {MP_OBJ_TO_PTR(stream), mp_stream_write_adaptor}; mp_obj_print_helper(&print, obj, PRINT_JSON); return mp_const_none; } -STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_ujson_dump_obj, mod_ujson_dump); +STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_json_dump_obj, mod_json_dump); -STATIC mp_obj_t mod_ujson_dumps(mp_obj_t obj) { +STATIC mp_obj_t mod_json_dumps(mp_obj_t obj) { vstr_t vstr; mp_print_t print; vstr_init_print(&vstr, 8, &print); mp_obj_print_helper(&print, obj, PRINT_JSON); return mp_obj_new_str_from_utf8_vstr(&vstr); } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_ujson_dumps_obj, mod_ujson_dumps); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_json_dumps_obj, mod_json_dumps); + #endif #define JSON_DEBUG(...) (void)0 @@ -127,7 +126,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_ujson_dumps_obj, mod_ujson_dumps); // strings). It does 1 pass over the input stream. It tries to be fast and // small in code size, while not using more RAM than necessary. -typedef struct _ujson_stream_t { +typedef struct _json_stream_t { mp_obj_t stream_obj; mp_uint_t (*read)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode); int errcode; @@ -137,16 +136,15 @@ typedef struct _ujson_stream_t { size_t start; size_t end; byte cur; -} ujson_stream_t; +} json_stream_t; #define S_EOF (0) // null is not allowed in json stream so is ok as EOF marker #define S_END(s) ((s).cur == S_EOF) #define S_CUR(s) ((s).cur) -#define S_NEXT(s) (ujson_stream_next(&(s))) +#define S_NEXT(s) (json_stream_next(&(s))) -STATIC byte ujson_stream_next(ujson_stream_t *s) { +STATIC byte json_stream_next(json_stream_t *s) { mp_uint_t ret = s->read(s->stream_obj, &s->cur, 1, &s->errcode); - JSON_DEBUG(" usjon_stream_next err:%2d cur: %c \n", s->errcode, s->cur); if (s->errcode != 0) { mp_raise_OSError(s->errcode); } @@ -163,7 +161,7 @@ STATIC byte ujson_stream_next(ujson_stream_t *s) { #define CIRCUITPY_JSON_READ_CHUNK_SIZE 64 -STATIC mp_uint_t ujson_python_readinto(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode) { +STATIC mp_uint_t json_python_readinto(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode) { (void)size; // Ignore size because we know it's always 1. ujson_stream_t *s = obj; @@ -186,7 +184,7 @@ STATIC mp_uint_t ujson_python_readinto(mp_obj_t obj, void *buf, mp_uint_t size, return 1; } -STATIC mp_obj_t _mod_ujson_load(mp_obj_t stream_obj, bool return_first_json) { +STATIC mp_obj_t _mod_json_load(mp_obj_t stream_obj, bool return_first_json) { const mp_stream_p_t *stream_p = mp_proto_get(MP_QSTR_protocol_stream, stream_obj); ujson_stream_t s; uint8_t character_buffer[CIRCUITPY_JSON_READ_CHUNK_SIZE]; @@ -426,39 +424,35 @@ fail: mp_raise_ValueError(MP_ERROR_TEXT("syntax error in JSON")); } -STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) { - return _mod_ujson_load(stream_obj, true); +STATIC mp_obj_t mod_json_load(mp_obj_t stream_obj) { + return _mod_json_load(stream_obj, true); } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_ujson_load_obj, mod_ujson_load); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_json_load_obj, mod_json_load); -STATIC mp_obj_t mod_ujson_loads(mp_obj_t obj) { +STATIC mp_obj_t mod_json_loads(mp_obj_t obj) { mp_buffer_info_t bufinfo; mp_get_buffer_raise(obj, &bufinfo, MP_BUFFER_READ); vstr_t vstr = {bufinfo.len, bufinfo.len, (char *)bufinfo.buf, true}; mp_obj_stringio_t sio = {{&mp_type_stringio}, &vstr, 0, MP_OBJ_NULL}; - return _mod_ujson_load(MP_OBJ_FROM_PTR(&sio), false); + return mod_json_load(MP_OBJ_FROM_PTR(&sio)); } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_ujson_loads_obj, mod_ujson_loads); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_json_loads_obj, mod_json_loads); -STATIC const mp_rom_map_elem_t mp_module_ujson_globals_table[] = { - #if CIRCUITPY +STATIC const mp_rom_map_elem_t mp_module_json_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_json) }, - #else - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ujson) }, - #endif - { MP_ROM_QSTR(MP_QSTR_dump), MP_ROM_PTR(&mod_ujson_dump_obj) }, - { MP_ROM_QSTR(MP_QSTR_dumps), MP_ROM_PTR(&mod_ujson_dumps_obj) }, - { MP_ROM_QSTR(MP_QSTR_load), MP_ROM_PTR(&mod_ujson_load_obj) }, - { MP_ROM_QSTR(MP_QSTR_loads), MP_ROM_PTR(&mod_ujson_loads_obj) }, + { MP_ROM_QSTR(MP_QSTR_dump), MP_ROM_PTR(&mod_json_dump_obj) }, + { MP_ROM_QSTR(MP_QSTR_dumps), MP_ROM_PTR(&mod_json_dumps_obj) }, + { MP_ROM_QSTR(MP_QSTR_load), MP_ROM_PTR(&mod_json_load_obj) }, + { MP_ROM_QSTR(MP_QSTR_loads), MP_ROM_PTR(&mod_json_loads_obj) }, }; -STATIC MP_DEFINE_CONST_DICT(mp_module_ujson_globals, mp_module_ujson_globals_table); +STATIC MP_DEFINE_CONST_DICT(mp_module_json_globals, mp_module_json_globals_table); -const mp_obj_module_t mp_module_ujson = { +const mp_obj_module_t mp_module_json = { .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&mp_module_ujson_globals, + .globals = (mp_obj_dict_t *)&mp_module_json_globals, }; -MP_REGISTER_MODULE(MP_QSTR_json, mp_module_ujson); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_json, mp_module_json); -#endif // MICROPY_PY_UJSON +#endif // MICROPY_PY_JSON diff --git a/extmod/moduos.c b/extmod/modos.c similarity index 55% rename from extmod/moduos.c rename to extmod/modos.c index 4cbdf72627..6df49d7f84 100644 --- a/extmod/moduos.c +++ b/extmod/modos.c @@ -27,13 +27,17 @@ #include "py/objstr.h" #include "py/runtime.h" -#if MICROPY_PY_UOS +#if MICROPY_PY_OS #include "extmod/misc.h" #include "extmod/vfs.h" #if MICROPY_VFS_FAT #include "extmod/vfs_fat.h" +#if MICROPY_PY_OS_SYNC +#include "lib/oofatfs/ff.h" +#include "lib/oofatfs/diskio.h" +#endif #endif #if MICROPY_VFS_LFS1 || MICROPY_VFS_LFS2 @@ -44,12 +48,12 @@ #include "extmod/vfs_posix.h" #endif -#if MICROPY_PY_UOS_UNAME +#if MICROPY_PY_OS_UNAME #include "genhdr/mpversion.h" #endif -#ifdef MICROPY_PY_UOS_INCLUDEFILE -#include MICROPY_PY_UOS_INCLUDEFILE +#ifdef MICROPY_PY_OS_INCLUDEFILE +#include MICROPY_PY_OS_INCLUDEFILE #endif #ifdef MICROPY_BUILD_TYPE @@ -58,77 +62,87 @@ #define MICROPY_BUILD_TYPE_PAREN #endif -#if MICROPY_PY_UOS_UNAME +#if MICROPY_PY_OS_SYNC +// sync() +// Sync all filesystems. +STATIC mp_obj_t mp_os_sync(void) { + #if MICROPY_VFS_FAT + for (mp_vfs_mount_t *vfs = MP_STATE_VM(vfs_mount_table); vfs != NULL; vfs = vfs->next) { + // this assumes that vfs->obj is fs_user_mount_t with block device functions + disk_ioctl(MP_OBJ_TO_PTR(vfs->obj), CTRL_SYNC, NULL); + } + #endif + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_0(mp_os_sync_obj, mp_os_sync); +#endif -#if MICROPY_PY_UOS_UNAME_RELEASE_DYNAMIC +#if MICROPY_PY_OS_UNAME + +#if MICROPY_PY_OS_UNAME_RELEASE_DYNAMIC #define CONST_RELEASE #else #define CONST_RELEASE const #endif -STATIC const qstr mp_uos_uname_info_fields[] = { +STATIC const qstr mp_os_uname_info_fields[] = { MP_QSTR_sysname, MP_QSTR_nodename, MP_QSTR_release, MP_QSTR_version, MP_QSTR_machine }; -STATIC const MP_DEFINE_STR_OBJ(mp_uos_uname_info_sysname_obj, MICROPY_PY_SYS_PLATFORM); -STATIC const MP_DEFINE_STR_OBJ(mp_uos_uname_info_nodename_obj, MICROPY_PY_SYS_PLATFORM); -STATIC CONST_RELEASE MP_DEFINE_STR_OBJ(mp_uos_uname_info_release_obj, MICROPY_VERSION_STRING); -STATIC const MP_DEFINE_STR_OBJ(mp_uos_uname_info_version_obj, MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE MICROPY_BUILD_TYPE_PAREN); -STATIC const MP_DEFINE_STR_OBJ(mp_uos_uname_info_machine_obj, MICROPY_HW_BOARD_NAME " with " MICROPY_HW_MCU_NAME); +STATIC const MP_DEFINE_STR_OBJ(mp_os_uname_info_sysname_obj, MICROPY_PY_SYS_PLATFORM); +STATIC const MP_DEFINE_STR_OBJ(mp_os_uname_info_nodename_obj, MICROPY_PY_SYS_PLATFORM); +STATIC CONST_RELEASE MP_DEFINE_STR_OBJ(mp_os_uname_info_release_obj, MICROPY_VERSION_STRING); +STATIC const MP_DEFINE_STR_OBJ(mp_os_uname_info_version_obj, MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE MICROPY_BUILD_TYPE_PAREN); +STATIC const MP_DEFINE_STR_OBJ(mp_os_uname_info_machine_obj, MICROPY_HW_BOARD_NAME " with " MICROPY_HW_MCU_NAME); STATIC MP_DEFINE_ATTRTUPLE( - mp_uos_uname_info_obj, - mp_uos_uname_info_fields, + mp_os_uname_info_obj, + mp_os_uname_info_fields, 5, - MP_ROM_PTR(&mp_uos_uname_info_sysname_obj), - MP_ROM_PTR(&mp_uos_uname_info_nodename_obj), - MP_ROM_PTR(&mp_uos_uname_info_release_obj), - MP_ROM_PTR(&mp_uos_uname_info_version_obj), - MP_ROM_PTR(&mp_uos_uname_info_machine_obj) + MP_ROM_PTR(&mp_os_uname_info_sysname_obj), + MP_ROM_PTR(&mp_os_uname_info_nodename_obj), + MP_ROM_PTR(&mp_os_uname_info_release_obj), + MP_ROM_PTR(&mp_os_uname_info_version_obj), + MP_ROM_PTR(&mp_os_uname_info_machine_obj) ); -STATIC mp_obj_t mp_uos_uname(void) { - #if MICROPY_PY_UOS_UNAME_RELEASE_DYNAMIC - const char *release = mp_uos_uname_release(); - mp_uos_uname_info_release_obj.len = strlen(release); - mp_uos_uname_info_release_obj.data = (const byte *)release; +STATIC mp_obj_t mp_os_uname(void) { + #if MICROPY_PY_OS_UNAME_RELEASE_DYNAMIC + const char *release = mp_os_uname_release(); + mp_os_uname_info_release_obj.len = strlen(release); + mp_os_uname_info_release_obj.data = (const byte *)release; #endif - return MP_OBJ_FROM_PTR(&mp_uos_uname_info_obj); + return MP_OBJ_FROM_PTR(&mp_os_uname_info_obj); } -STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_uos_uname_obj, mp_uos_uname); +STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_os_uname_obj, mp_os_uname); #endif STATIC const mp_rom_map_elem_t os_module_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uos) }, + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_os) }, - #if MICROPY_PY_UOS_GETENV_PUTENV_UNSETENV - { MP_ROM_QSTR(MP_QSTR_getenv), MP_ROM_PTR(&mp_uos_getenv_obj) }, - #if defined(MICROPY_UNIX_COVERAGE) - { MP_ROM_QSTR(MP_QSTR_getenv_int), MP_ROM_PTR(&mp_uos_getenv_int_obj) }, - { MP_ROM_QSTR(MP_QSTR_getenv_str), MP_ROM_PTR(&mp_uos_getenv_str_obj) }, + #if MICROPY_PY_OS_GETENV_PUTENV_UNSETENV + { MP_ROM_QSTR(MP_QSTR_getenv), MP_ROM_PTR(&mp_os_getenv_obj) }, + { MP_ROM_QSTR(MP_QSTR_putenv), MP_ROM_PTR(&mp_os_putenv_obj) }, + { MP_ROM_QSTR(MP_QSTR_unsetenv), MP_ROM_PTR(&mp_os_unsetenv_obj) }, #endif - - { MP_ROM_QSTR(MP_QSTR_putenv), MP_ROM_PTR(&mp_uos_putenv_obj) }, - { MP_ROM_QSTR(MP_QSTR_unsetenv), MP_ROM_PTR(&mp_uos_unsetenv_obj) }, - #endif - #if MICROPY_PY_UOS_SEP + #if MICROPY_PY_OS_SEP { MP_ROM_QSTR(MP_QSTR_sep), MP_ROM_QSTR(MP_QSTR__slash_) }, #endif - #if MICROPY_PY_UOS_SYNC - { MP_ROM_QSTR(MP_QSTR_sync), MP_ROM_PTR(&mp_uos_sync_obj) }, + #if MICROPY_PY_OS_SYNC + { MP_ROM_QSTR(MP_QSTR_sync), MP_ROM_PTR(&mp_os_sync_obj) }, #endif - #if MICROPY_PY_UOS_SYSTEM - { MP_ROM_QSTR(MP_QSTR_system), MP_ROM_PTR(&mp_uos_system_obj) }, + #if MICROPY_PY_OS_SYSTEM + { MP_ROM_QSTR(MP_QSTR_system), MP_ROM_PTR(&mp_os_system_obj) }, #endif - #if MICROPY_PY_UOS_UNAME - { MP_ROM_QSTR(MP_QSTR_uname), MP_ROM_PTR(&mp_uos_uname_obj) }, + #if MICROPY_PY_OS_UNAME + { MP_ROM_QSTR(MP_QSTR_uname), MP_ROM_PTR(&mp_os_uname_obj) }, #endif - #if MICROPY_PY_UOS_URANDOM - { MP_ROM_QSTR(MP_QSTR_urandom), MP_ROM_PTR(&mp_uos_urandom_obj) }, + #if MICROPY_PY_OS_URANDOM + { MP_ROM_QSTR(MP_QSTR_urandom), MP_ROM_PTR(&mp_os_urandom_obj) }, #endif #if MICROPY_VFS @@ -147,13 +161,13 @@ STATIC const mp_rom_map_elem_t os_module_globals_table[] = { // The following are MicroPython extensions. #if MICROPY_PY_OS_DUPTERM - { MP_ROM_QSTR(MP_QSTR_dupterm), MP_ROM_PTR(&mp_uos_dupterm_obj) }, + { MP_ROM_QSTR(MP_QSTR_dupterm), MP_ROM_PTR(&mp_os_dupterm_obj) }, #endif - #if MICROPY_PY_UOS_DUPTERM_NOTIFY - { MP_ROM_QSTR(MP_QSTR_dupterm_notify), MP_ROM_PTR(&mp_uos_dupterm_notify_obj) }, + #if MICROPY_PY_OS_DUPTERM_NOTIFY + { MP_ROM_QSTR(MP_QSTR_dupterm_notify), MP_ROM_PTR(&mp_os_dupterm_notify_obj) }, #endif - #if MICROPY_PY_UOS_ERRNO - { MP_ROM_QSTR(MP_QSTR_errno), MP_ROM_PTR(&mp_uos_errno_obj) }, + #if MICROPY_PY_OS_ERRNO + { MP_ROM_QSTR(MP_QSTR_errno), MP_ROM_PTR(&mp_os_errno_obj) }, #endif #if MICROPY_VFS @@ -176,11 +190,11 @@ STATIC const mp_rom_map_elem_t os_module_globals_table[] = { }; STATIC MP_DEFINE_CONST_DICT(os_module_globals, os_module_globals_table); -const mp_obj_module_t mp_module_uos = { +const mp_obj_module_t mp_module_os = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&os_module_globals, }; -MP_REGISTER_MODULE(MP_QSTR_os, mp_module_uos); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_os, mp_module_os); -#endif // MICROPY_PY_UOS +#endif // MICROPY_PY_OS diff --git a/extmod/moduplatform.c b/extmod/modplatform.c similarity index 92% rename from extmod/moduplatform.c rename to extmod/modplatform.c index 3590943bb1..73846f946d 100644 --- a/extmod/moduplatform.c +++ b/extmod/modplatform.c @@ -29,10 +29,10 @@ #include "py/objtuple.h" #include "py/objstr.h" #include "py/mphal.h" -#include "extmod/moduplatform.h" +#include "extmod/modplatform.h" #include "genhdr/mpversion.h" -#if MICROPY_PY_UPLATFORM +#if MICROPY_PY_PLATFORM // platform - Access to underlying platform's identifying data @@ -62,7 +62,7 @@ STATIC mp_obj_t platform_libc_ver(size_t n_args, const mp_obj_t *pos_args, mp_ma STATIC MP_DEFINE_CONST_FUN_OBJ_KW(platform_libc_ver_obj, 0, platform_libc_ver); STATIC const mp_rom_map_elem_t modplatform_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uplatform) }, + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_platform) }, { MP_ROM_QSTR(MP_QSTR_platform), MP_ROM_PTR(&platform_platform_obj) }, { MP_ROM_QSTR(MP_QSTR_python_compiler), MP_ROM_PTR(&platform_python_compiler_obj) }, { MP_ROM_QSTR(MP_QSTR_libc_ver), MP_ROM_PTR(&platform_libc_ver_obj) }, @@ -70,11 +70,11 @@ STATIC const mp_rom_map_elem_t modplatform_globals_table[] = { STATIC MP_DEFINE_CONST_DICT(modplatform_globals, modplatform_globals_table); -const mp_obj_module_t mp_module_uplatform = { +const mp_obj_module_t mp_module_platform = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&modplatform_globals, }; -MP_REGISTER_MODULE(MP_QSTR_platform, mp_module_uplatform); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_platform, mp_module_platform); -#endif // MICROPY_PY_UPLATFORM +#endif // MICROPY_PY_PLATFORM diff --git a/extmod/moduplatform.h b/extmod/modplatform.h similarity index 93% rename from extmod/moduplatform.h rename to extmod/modplatform.h index 3597f7559f..56a50e53c5 100644 --- a/extmod/moduplatform.h +++ b/extmod/modplatform.h @@ -23,14 +23,14 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ -#ifndef MICROPY_INCLUDED_MODUPLATFORM_H -#define MICROPY_INCLUDED_MODUPLATFORM_H +#ifndef MICROPY_INCLUDED_MODPLATFORM_H +#define MICROPY_INCLUDED_MODPLATFORM_H #include "py/misc.h" // For MP_STRINGIFY. #include "py/mpconfig.h" // Preprocessor directives identifying the platform. -// The (u)platform module itself is guarded by MICROPY_PY_UPLATFORM, see the +// The platform module itself is guarded by MICROPY_PY_PLATFORM, see the // .c file, but these are made available because they're generally usable. // TODO: Add more architectures, compilers and libraries. // See: https://sourceforge.net/p/predef/wiki/Home/ @@ -43,6 +43,8 @@ #define MICROPY_PLATFORM_ARCH "x86" #elif defined(__xtensa__) #define MICROPY_PLATFORM_ARCH "xtensa" +#elif defined(__riscv) +#define MICROPY_PLATFORM_ARCH "riscv" #else #define MICROPY_PLATFORM_ARCH "" #endif @@ -102,4 +104,4 @@ #define MICROPY_PLATFORM_VERSION "" #endif -#endif // MICROPY_INCLUDED_MODUPLATFORM_H +#endif // MICROPY_INCLUDED_MODPLATFORM_H diff --git a/extmod/modurandom.c b/extmod/modrandom.c similarity index 98% rename from extmod/modurandom.c rename to extmod/modrandom.c index 68ebed32d3..33c77e8b93 100644 --- a/extmod/modurandom.c +++ b/extmod/modrandom.c @@ -29,10 +29,10 @@ #include "py/runtime.h" -#if MICROPY_PY_URANDOM +#if MICROPY_PY_RANDOM // Work out if the seed will be set on import or not. -#if MICROPY_MODULE_BUILTIN_INIT && defined(MICROPY_PY_URANDOM_SEED_INIT_FUNC) +#if MICROPY_MODULE_BUILTIN_INIT && defined(MICROPY_PY_RANDOM_SEED_INIT_FUNC) #define SEED_ON_IMPORT (1) #else #define SEED_ON_IMPORT (0) @@ -255,7 +255,7 @@ const mp_obj_module_t mp_module_urandom = { .globals = (mp_obj_dict_t *)&mp_module_urandom_globals, }; -MP_REGISTER_MODULE(MP_QSTR_random, mp_module_urandom); +MP_REGISTER_MODULE(MP_QSTR_urandom, mp_module_urandom); #endif #endif // MICROPY_PY_URANDOM diff --git a/extmod/modure.c b/extmod/modre.c similarity index 95% rename from extmod/modure.c rename to extmod/modre.c index 9e6ac76ecb..0c4e001acc 100644 --- a/extmod/modure.c +++ b/extmod/modre.c @@ -37,7 +37,7 @@ #include "py/unicode.h" #endif -#if MICROPY_PY_URE +#if MICROPY_PY_RE #define re1_5_stack_chk() MP_STACK_CHECK() @@ -167,10 +167,10 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(match_end_obj, 1, 2, match_end); #if !MICROPY_ENABLE_DYNRUNTIME STATIC const mp_rom_map_elem_t match_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_group), MP_ROM_PTR(&match_group_obj) }, - #if MICROPY_PY_URE_MATCH_GROUPS + #if MICROPY_PY_RE_MATCH_GROUPS { MP_ROM_QSTR(MP_QSTR_groups), MP_ROM_PTR(&match_groups_obj) }, #endif - #if MICROPY_PY_URE_MATCH_SPAN_START_END + #if MICROPY_PY_RE_MATCH_SPAN_START_END { MP_ROM_QSTR(MP_QSTR_span), MP_ROM_PTR(&match_span_obj) }, { MP_ROM_QSTR(MP_QSTR_start), MP_ROM_PTR(&match_start_obj) }, { MP_ROM_QSTR(MP_QSTR_end), MP_ROM_PTR(&match_end_obj) }, @@ -194,7 +194,7 @@ STATIC void re_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t mp_printf(print, "", self); } -STATIC mp_obj_t ure_exec(bool is_anchored, uint n_args, const mp_obj_t *args) { +STATIC mp_obj_t re_exec(bool is_anchored, uint n_args, const mp_obj_t *args) { (void)n_args; mp_obj_re_t *self; if (mp_obj_is_type(args[0], (mp_obj_type_t *)&re_type)) { @@ -206,7 +206,8 @@ STATIC mp_obj_t ure_exec(bool is_anchored, uint n_args, const mp_obj_t *args) { size_t len; subj.begin_line = subj.begin = mp_obj_str_get_data(args[1], &len); subj.end = subj.begin + len; - #if MICROPY_PY_URE_MATCH_SPAN_START_END && !(defined(MICROPY_ENABLE_DYNRUNTIME) && MICROPY_ENABLE_DYNRUNTIME) + // CIRCUITPY + #if MICROPY_PY_RE_MATCH_SPAN_START_END && !(defined(MICROPY_ENABLE_DYNRUNTIME) && MICROPY_ENABLE_DYNRUNTIME) if (n_args > 2) { const mp_obj_type_t *self_type = mp_obj_get_type(args[1]); @@ -253,12 +254,12 @@ STATIC mp_obj_t ure_exec(bool is_anchored, uint n_args, const mp_obj_t *args) { } STATIC mp_obj_t re_match(size_t n_args, const mp_obj_t *args) { - return ure_exec(true, n_args, args); + return re_exec(true, n_args, args); } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(re_match_obj, 2, 4, re_match); STATIC mp_obj_t re_search(size_t n_args, const mp_obj_t *args) { - return ure_exec(false, n_args, args); + return re_exec(false, n_args, args); } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(re_search_obj, 2, 4, re_search); @@ -307,7 +308,7 @@ STATIC mp_obj_t re_split(size_t n_args, const mp_obj_t *args) { } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(re_split_obj, 2, 3, re_split); -#if MICROPY_PY_URE_SUB +#if MICROPY_PY_RE_SUB STATIC mp_obj_t re_sub_helper(size_t n_args, const mp_obj_t *args) { mp_obj_re_t *self; @@ -435,7 +436,7 @@ STATIC const mp_rom_map_elem_t re_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_match), MP_ROM_PTR(&re_match_obj) }, { MP_ROM_QSTR(MP_QSTR_search), MP_ROM_PTR(&re_search_obj) }, { MP_ROM_QSTR(MP_QSTR_split), MP_ROM_PTR(&re_split_obj) }, - #if MICROPY_PY_URE_SUB + #if MICROPY_PY_RE_SUB { MP_ROM_QSTR(MP_QSTR_sub), MP_ROM_PTR(&re_sub_obj) }, #endif }; @@ -444,11 +445,7 @@ STATIC MP_DEFINE_CONST_DICT(re_locals_dict, re_locals_dict_table); STATIC MP_DEFINE_CONST_OBJ_TYPE( re_type, - #if CIRCUITPY MP_QSTR_re, - #else - MP_QSTR_ure, - #endif MP_TYPE_FLAG_NONE, print, re_print, locals_dict, &re_locals_dict @@ -463,7 +460,7 @@ STATIC mp_obj_t mod_re_compile(size_t n_args, const mp_obj_t *args) { goto error; } mp_obj_re_t *o = mp_obj_malloc_var(mp_obj_re_t, char, size, (mp_obj_type_t *)&re_type); - #if MICROPY_PY_URE_DEBUG + #if MICROPY_PY_RE_DEBUG int flags = 0; if (n_args > 1) { flags = mp_obj_get_int(args[1]); @@ -474,7 +471,7 @@ STATIC mp_obj_t mod_re_compile(size_t n_args, const mp_obj_t *args) { error: mp_raise_ValueError(MP_ERROR_TEXT("Error in regex")); } - #if MICROPY_PY_URE_DEBUG + #if MICROPY_PY_RE_DEBUG if (flags & FLAG_DEBUG) { re1_5_dumpcode(&o->re); } @@ -485,30 +482,26 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_re_compile_obj, 1, 2, mod_re_compile); #if !MICROPY_ENABLE_DYNRUNTIME STATIC const mp_rom_map_elem_t mp_module_re_globals_table[] = { - #if CIRCUITPY { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_re) }, - #else - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ure) }, - #endif { MP_ROM_QSTR(MP_QSTR_compile), MP_ROM_PTR(&mod_re_compile_obj) }, { MP_ROM_QSTR(MP_QSTR_match), MP_ROM_PTR(&re_match_obj) }, { MP_ROM_QSTR(MP_QSTR_search), MP_ROM_PTR(&re_search_obj) }, - #if MICROPY_PY_URE_SUB + #if MICROPY_PY_RE_SUB { MP_ROM_QSTR(MP_QSTR_sub), MP_ROM_PTR(&re_sub_obj) }, #endif - #if MICROPY_PY_URE_DEBUG + #if MICROPY_PY_RE_DEBUG { MP_ROM_QSTR(MP_QSTR_DEBUG), MP_ROM_INT(FLAG_DEBUG) }, #endif }; STATIC MP_DEFINE_CONST_DICT(mp_module_re_globals, mp_module_re_globals_table); -const mp_obj_module_t mp_module_ure = { +const mp_obj_module_t mp_module_re = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&mp_module_re_globals, }; -MP_REGISTER_MODULE(MP_QSTR_re, mp_module_ure); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_re, mp_module_re); #endif // Source files #include'd here to make sure they're compiled in @@ -520,11 +513,11 @@ MP_REGISTER_MODULE(MP_QSTR_re, mp_module_ure); #include "lib/re1.5/recursiveloop.c" #include "lib/re1.5/charclass.c" -#if MICROPY_PY_URE_DEBUG +#if MICROPY_PY_RE_DEBUG // Make sure the output print statements go to the same output as other Python output. #define printf(...) mp_printf(&mp_plat_print, __VA_ARGS__) #include "lib/re1.5/dumpcode.c" #undef printf #endif -#endif // MICROPY_PY_URE +#endif // MICROPY_PY_RE diff --git a/extmod/modselect.c b/extmod/modselect.c new file mode 100644 index 0000000000..f1169c5cd6 --- /dev/null +++ b/extmod/modselect.c @@ -0,0 +1,646 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014-2023 Damien P. George + * Copyright (c) 2015-2017 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/mpconfig.h" +#include "py/runtime.h" +#include "py/obj.h" +#include "py/objlist.h" +#include "py/stream.h" +#include "py/mperrno.h" +#include "py/mphal.h" +#include "shared/runtime/interrupt_char.h" + +#if MICROPY_PY_SELECT + +#if MICROPY_PY_SELECT_SELECT && MICROPY_PY_SELECT_POSIX_OPTIMISATIONS +#error "select.select is not supported with MICROPY_PY_SELECT_POSIX_OPTIMISATIONS" +#endif + +#if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS + +#include + +#if !((MP_STREAM_POLL_RD) == (POLLIN) && \ + (MP_STREAM_POLL_WR) == (POLLOUT) && \ + (MP_STREAM_POLL_ERR) == (POLLERR) && \ + (MP_STREAM_POLL_HUP) == (POLLHUP) && \ + (MP_STREAM_POLL_NVAL) == (POLLNVAL)) +#error "With MICROPY_PY_SELECT_POSIX_OPTIMISATIONS enabled, POLL constants must match" +#endif + +// When non-file-descriptor objects are on the list to be polled (the polling of +// which involves repeatedly calling ioctl(MP_STREAM_POLL)), this variable sets +// the period between polling these objects. +#define MICROPY_PY_SELECT_IOCTL_CALL_PERIOD_MS (1) + +#endif + +// Flags for ipoll() +#define FLAG_ONESHOT (1) + +// A single pollable object. +typedef struct _poll_obj_t { + mp_obj_t obj; + mp_uint_t (*ioctl)(mp_obj_t obj, mp_uint_t request, uintptr_t arg, int *errcode); + #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS + // If the pollable object has an associated file descriptor, then pollfd points to an entry + // in poll_set_t::pollfds, and the events/revents fields for this object are stored in the + // pollfd entry (and the nonfd_* members are unused). + // Otherwise the object is a non-file-descriptor object and pollfd==NULL, and the events/ + // revents fields are stored in the nonfd_* members (which are named as such so that code + // doesn't accidentally mix the use of these members when this optimisation is used). + struct pollfd *pollfd; + uint16_t nonfd_events; + uint16_t nonfd_revents; + #else + mp_uint_t events; + mp_uint_t revents; + #endif +} poll_obj_t; + +// A set of pollable objects. +typedef struct _poll_set_t { + // Map containing a dict with key=object to poll, value=its corresponding poll_obj_t. + mp_map_t map; + + #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS + // Array of pollfd entries for objects that have a file descriptor. + unsigned short alloc; // memory allocated for pollfds + unsigned short max_used; // maximum number of used entries in pollfds + unsigned short used; // actual number of used entries in pollfds + struct pollfd *pollfds; + #endif +} poll_set_t; + +STATIC void poll_set_init(poll_set_t *poll_set, size_t n) { + mp_map_init(&poll_set->map, n); + #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS + poll_set->alloc = 0; + poll_set->max_used = 0; + poll_set->used = 0; + poll_set->pollfds = NULL; + #endif +} + +#if MICROPY_PY_SELECT_SELECT +STATIC void poll_set_deinit(poll_set_t *poll_set) { + mp_map_deinit(&poll_set->map); +} +#endif + +#if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS + +STATIC mp_uint_t poll_obj_get_events(poll_obj_t *poll_obj) { + assert(poll_obj->pollfd == NULL); + return poll_obj->nonfd_events; +} + +STATIC void poll_obj_set_events(poll_obj_t *poll_obj, mp_uint_t events) { + if (poll_obj->pollfd != NULL) { + poll_obj->pollfd->events = events; + } else { + poll_obj->nonfd_events = events; + } +} + +STATIC mp_uint_t poll_obj_get_revents(poll_obj_t *poll_obj) { + if (poll_obj->pollfd != NULL) { + return poll_obj->pollfd->revents; + } else { + return poll_obj->nonfd_revents; + } +} + +STATIC void poll_obj_set_revents(poll_obj_t *poll_obj, mp_uint_t revents) { + if (poll_obj->pollfd != NULL) { + poll_obj->pollfd->revents = revents; + } else { + poll_obj->nonfd_revents = revents; + } +} + +STATIC struct pollfd *poll_set_add_fd(poll_set_t *poll_set, int fd) { + struct pollfd *free_slot = NULL; + + if (poll_set->used == poll_set->max_used) { + // No free slots below max_used, so expand max_used (and possibly allocate). + if (poll_set->max_used >= poll_set->alloc) { + poll_set->pollfds = m_renew(struct pollfd, poll_set->pollfds, poll_set->alloc, poll_set->alloc + 4); + poll_set->alloc += 4; + } + free_slot = &poll_set->pollfds[poll_set->max_used++]; + } else { + // There should be a free slot below max_used. + for (unsigned int i = 0; i < poll_set->max_used; ++i) { + struct pollfd *slot = &poll_set->pollfds[i]; + if (slot->fd == -1) { + free_slot = slot; + break; + } + } + assert(free_slot != NULL); + } + + free_slot->fd = fd; + ++poll_set->used; + + return free_slot; +} + +static inline bool poll_set_all_are_fds(poll_set_t *poll_set) { + return poll_set->map.used == poll_set->used; +} + +#else + +static inline mp_uint_t poll_obj_get_events(poll_obj_t *poll_obj) { + return poll_obj->events; +} + +static inline void poll_obj_set_events(poll_obj_t *poll_obj, mp_uint_t events) { + poll_obj->events = events; +} + +static inline mp_uint_t poll_obj_get_revents(poll_obj_t *poll_obj) { + return poll_obj->revents; +} + +static inline void poll_obj_set_revents(poll_obj_t *poll_obj, mp_uint_t revents) { + poll_obj->revents = revents; +} + +#endif + +STATIC void poll_set_add_obj(poll_set_t *poll_set, const mp_obj_t *obj, mp_uint_t obj_len, mp_uint_t events, bool or_events) { + for (mp_uint_t i = 0; i < obj_len; i++) { + mp_map_elem_t *elem = mp_map_lookup(&poll_set->map, mp_obj_id(obj[i]), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + if (elem->value == MP_OBJ_NULL) { + // object not found; get its ioctl and add it to the poll list + + // If an exception is raised below when adding the new object then the map entry for that + // object remains unpopulated, and methods like poll() may crash. This case is not handled. + + poll_obj_t *poll_obj = m_new_obj(poll_obj_t); + poll_obj->obj = obj[i]; + + #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS + int fd = -1; + if (mp_obj_is_int(obj[i])) { + // A file descriptor integer passed in as the object, so use it directly. + fd = mp_obj_get_int(obj[i]); + if (fd < 0) { + mp_raise_ValueError(NULL); + } + poll_obj->ioctl = NULL; + } else { + // An object passed in. Check if it has a file descriptor. + const mp_stream_p_t *stream_p = mp_get_stream_raise(obj[i], MP_STREAM_OP_IOCTL); + poll_obj->ioctl = stream_p->ioctl; + int err; + mp_uint_t res = stream_p->ioctl(obj[i], MP_STREAM_GET_FILENO, 0, &err); + if (res != MP_STREAM_ERROR) { + fd = res; + } + } + if (fd >= 0) { + // Object has a file descriptor so add it to pollfds. + poll_obj->pollfd = poll_set_add_fd(poll_set, fd); + } else { + // Object doesn't have a file descriptor. + poll_obj->pollfd = NULL; + } + #else + const mp_stream_p_t *stream_p = mp_get_stream_raise(obj[i], MP_STREAM_OP_IOCTL); + poll_obj->ioctl = stream_p->ioctl; + #endif + + poll_obj_set_events(poll_obj, events); + poll_obj_set_revents(poll_obj, 0); + elem->value = MP_OBJ_FROM_PTR(poll_obj); + } else { + // object exists; update its events + poll_obj_t *poll_obj = (poll_obj_t *)MP_OBJ_TO_PTR(elem->value); + #if MICROPY_PY_SELECT_SELECT + if (or_events) { + events |= poll_obj_get_events(poll_obj); + } + #else + (void)or_events; + #endif + poll_obj_set_events(poll_obj, events); + } + } +} + +// For each object in the poll set, poll it once. +STATIC mp_uint_t poll_set_poll_once(poll_set_t *poll_set, size_t *rwx_num) { + mp_uint_t n_ready = 0; + for (mp_uint_t i = 0; i < poll_set->map.alloc; ++i) { + if (!mp_map_slot_is_filled(&poll_set->map, i)) { + continue; + } + + poll_obj_t *poll_obj = MP_OBJ_TO_PTR(poll_set->map.table[i].value); + + #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS + if (poll_obj->pollfd != NULL) { + // Object has file descriptor so will be polled separately by poll(). + continue; + } + #endif + + int errcode; + mp_int_t ret = poll_obj->ioctl(poll_obj->obj, MP_STREAM_POLL, poll_obj_get_events(poll_obj), &errcode); + poll_obj_set_revents(poll_obj, ret); + + if (ret == -1) { + // error doing ioctl + mp_raise_OSError(errcode); + } + + if (ret != 0) { + // object is ready + n_ready += 1; + #if MICROPY_PY_SELECT_SELECT + if (rwx_num != NULL) { + if (ret & MP_STREAM_POLL_RD) { + rwx_num[0] += 1; + } + if (ret & MP_STREAM_POLL_WR) { + rwx_num[1] += 1; + } + if ((ret & ~(MP_STREAM_POLL_RD | MP_STREAM_POLL_WR)) != 0) { + rwx_num[2] += 1; + } + } + #else + (void)rwx_num; + #endif + } + } + return n_ready; +} + +STATIC mp_uint_t poll_set_poll_until_ready_or_timeout(poll_set_t *poll_set, size_t *rwx_num, mp_uint_t timeout) { + mp_uint_t start_ticks = mp_hal_ticks_ms(); + + #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS + + for (;;) { + MP_THREAD_GIL_EXIT(); + + // Compute the timeout. + int t = MICROPY_PY_SELECT_IOCTL_CALL_PERIOD_MS; + if (poll_set_all_are_fds(poll_set)) { + // All our pollables are file descriptors, so we can use a blocking + // poll and let it (the underlying system) handle the timeout. + if (timeout == (mp_uint_t)-1) { + t = -1; + } else { + mp_uint_t delta = mp_hal_ticks_ms() - start_ticks; + if (delta >= timeout) { + t = 0; + } else { + t = timeout - delta; + } + } + } + + // Call system poll for those objects that have a file descriptor. + int n_ready = poll(poll_set->pollfds, poll_set->max_used, t); + + MP_THREAD_GIL_ENTER(); + + // The call to poll() may have been interrupted, but per PEP 475 we must retry if the + // signal is EINTR (this implements a special case of calling MP_HAL_RETRY_SYSCALL()). + if (n_ready == -1) { + int err = errno; + if (err != EINTR) { + mp_raise_OSError(err); + } + n_ready = 0; + } + + // Explicitly poll any objects that do not have a file descriptor. + if (!poll_set_all_are_fds(poll_set)) { + n_ready += poll_set_poll_once(poll_set, rwx_num); + } + + // Return if an object is ready, or if the timeout expired. + if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_ticks >= timeout)) { + return n_ready; + } + + // This would be MICROPY_EVENT_POLL_HOOK but the call to poll() above already includes a delay. + mp_handle_pending(true); + } + + #else + + for (;;) { + // poll the objects + mp_uint_t n_ready = poll_set_poll_once(poll_set, rwx_num); + if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_ticks >= timeout)) { + return n_ready; + } + // CIRCUITPY + RUN_BACKGROUND_TASKS; + if (mp_hal_is_interrupted()) { + return 0; + } + // end CIRCUITPY + MICROPY_EVENT_POLL_HOOK + } + + #endif +} + +#if MICROPY_PY_SELECT_SELECT +// select(rlist, wlist, xlist[, timeout]) +STATIC mp_obj_t select_select(size_t n_args, const mp_obj_t *args) { + // get array data from tuple/list arguments + size_t rwx_len[3]; + mp_obj_t *r_array, *w_array, *x_array; + mp_obj_get_array(args[0], &rwx_len[0], &r_array); + mp_obj_get_array(args[1], &rwx_len[1], &w_array); + mp_obj_get_array(args[2], &rwx_len[2], &x_array); + + // get timeout + mp_uint_t timeout = -1; + if (n_args == 4) { + if (args[3] != mp_const_none) { + #if MICROPY_PY_BUILTINS_FLOAT + float timeout_f = mp_obj_get_float_to_f(args[3]); + if (timeout_f >= 0) { + timeout = (mp_uint_t)(timeout_f * 1000); + } + #else + timeout = mp_obj_get_int(args[3]) * 1000; + #endif + } + } + + // merge separate lists and get the ioctl function for each object + poll_set_t poll_set; + poll_set_init(&poll_set, rwx_len[0] + rwx_len[1] + rwx_len[2]); + poll_set_add_obj(&poll_set, r_array, rwx_len[0], MP_STREAM_POLL_RD, true); + poll_set_add_obj(&poll_set, w_array, rwx_len[1], MP_STREAM_POLL_WR, true); + poll_set_add_obj(&poll_set, x_array, rwx_len[2], MP_STREAM_POLL_ERR | MP_STREAM_POLL_HUP, true); + + // poll all objects + rwx_len[0] = rwx_len[1] = rwx_len[2] = 0; + poll_set_poll_until_ready_or_timeout(&poll_set, rwx_len, timeout); + + // one or more objects are ready, or we had a timeout + mp_obj_t list_array[3]; + list_array[0] = mp_obj_new_list(rwx_len[0], NULL); + list_array[1] = mp_obj_new_list(rwx_len[1], NULL); + list_array[2] = mp_obj_new_list(rwx_len[2], NULL); + rwx_len[0] = rwx_len[1] = rwx_len[2] = 0; + for (mp_uint_t i = 0; i < poll_set.map.alloc; ++i) { + if (!mp_map_slot_is_filled(&poll_set.map, i)) { + continue; + } + // CIRCUITPY + RUN_BACKGROUND_TASKS; + + poll_obj_t *poll_obj = MP_OBJ_TO_PTR(poll_set.map.table[i].value); + if (poll_obj->revents & MP_STREAM_POLL_RD) { + ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[0]))->items[rwx_len[0]++] = poll_obj->obj; + } + if (poll_obj->revents & MP_STREAM_POLL_WR) { + ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[1]))->items[rwx_len[1]++] = poll_obj->obj; + } + if ((poll_obj->revents & ~(MP_STREAM_POLL_RD | MP_STREAM_POLL_WR)) != 0) { + ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[2]))->items[rwx_len[2]++] = poll_obj->obj; + } + } + poll_set_deinit(&poll_set); + return mp_obj_new_tuple(3, list_array); +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_select_select_obj, 3, 4, select_select); +#endif // MICROPY_PY_SELECT_SELECT + +typedef struct _mp_obj_poll_t { + mp_obj_base_t base; + poll_set_t poll_set; + short iter_cnt; + short iter_idx; + int flags; + // callee-owned tuple + mp_obj_t ret_tuple; +} mp_obj_poll_t; + +// register(obj[, eventmask]) +STATIC mp_obj_t poll_register(size_t n_args, const mp_obj_t *args) { + mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); + mp_uint_t events; + if (n_args == 3) { + events = mp_obj_get_int(args[2]); + } else { + events = MP_STREAM_POLL_RD | MP_STREAM_POLL_WR; + } + poll_set_add_obj(&self->poll_set, &args[1], 1, events, false); + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_register_obj, 2, 3, poll_register); + +// unregister(obj) +STATIC mp_obj_t poll_unregister(mp_obj_t self_in, mp_obj_t obj_in) { + mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in); + mp_map_elem_t *elem = mp_map_lookup(&self->poll_set.map, mp_obj_id(obj_in), MP_MAP_LOOKUP_REMOVE_IF_FOUND); + + #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS + if (elem != NULL) { + poll_obj_t *poll_obj = (poll_obj_t *)MP_OBJ_TO_PTR(elem->value); + if (poll_obj->pollfd != NULL) { + poll_obj->pollfd->fd = -1; + --self->poll_set.used; + } + elem->value = MP_OBJ_NULL; + } + #else + (void)elem; + #endif + + // TODO raise KeyError if obj didn't exist in map + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_2(poll_unregister_obj, poll_unregister); + +// modify(obj, eventmask) +STATIC mp_obj_t poll_modify(mp_obj_t self_in, mp_obj_t obj_in, mp_obj_t eventmask_in) { + mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in); + mp_map_elem_t *elem = mp_map_lookup(&self->poll_set.map, mp_obj_id(obj_in), MP_MAP_LOOKUP); + if (elem == NULL) { + mp_raise_OSError(MP_ENOENT); + } + poll_obj_set_events((poll_obj_t *)MP_OBJ_TO_PTR(elem->value), mp_obj_get_int(eventmask_in)); + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_3(poll_modify_obj, poll_modify); + +STATIC mp_uint_t poll_poll_internal(uint n_args, const mp_obj_t *args) { + mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); + + // work out timeout (its given already in ms) + mp_uint_t timeout = -1; + int flags = 0; + if (n_args >= 2) { + if (args[1] != mp_const_none) { + mp_int_t timeout_i = mp_obj_get_int(args[1]); + if (timeout_i >= 0) { + timeout = timeout_i; + } + } + if (n_args >= 3) { + flags = mp_obj_get_int(args[2]); + } + } + + self->flags = flags; + + return poll_set_poll_until_ready_or_timeout(&self->poll_set, NULL, timeout); +} + +STATIC mp_obj_t poll_poll(size_t n_args, const mp_obj_t *args) { + mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); + mp_uint_t n_ready = poll_poll_internal(n_args, args); + + // one or more objects are ready, or we had a timeout + mp_obj_list_t *ret_list = MP_OBJ_TO_PTR(mp_obj_new_list(n_ready, NULL)); + n_ready = 0; + for (mp_uint_t i = 0; i < self->poll_set.map.alloc; ++i) { + if (!mp_map_slot_is_filled(&self->poll_set.map, i)) { + continue; + } + poll_obj_t *poll_obj = MP_OBJ_TO_PTR(self->poll_set.map.table[i].value); + if (poll_obj_get_revents(poll_obj) != 0) { + mp_obj_t tuple[2] = {poll_obj->obj, MP_OBJ_NEW_SMALL_INT(poll_obj_get_revents(poll_obj))}; + ret_list->items[n_ready++] = mp_obj_new_tuple(2, tuple); + } + } + return MP_OBJ_FROM_PTR(ret_list); +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_poll_obj, 1, 2, poll_poll); + +STATIC mp_obj_t poll_ipoll(size_t n_args, const mp_obj_t *args) { + mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); + + if (self->ret_tuple == MP_OBJ_NULL) { + self->ret_tuple = mp_obj_new_tuple(2, NULL); + } + + int n_ready = poll_poll_internal(n_args, args); + self->iter_cnt = n_ready; + self->iter_idx = 0; + + return args[0]; +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_ipoll_obj, 1, 3, poll_ipoll); + +STATIC mp_obj_t poll_iternext(mp_obj_t self_in) { + mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in); + + if (self->iter_cnt == 0) { + return MP_OBJ_STOP_ITERATION; + } + + self->iter_cnt--; + + for (mp_uint_t i = self->iter_idx; i < self->poll_set.map.alloc; ++i) { + self->iter_idx++; + if (!mp_map_slot_is_filled(&self->poll_set.map, i)) { + continue; + } + poll_obj_t *poll_obj = MP_OBJ_TO_PTR(self->poll_set.map.table[i].value); + if (poll_obj_get_revents(poll_obj) != 0) { + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(self->ret_tuple); + t->items[0] = poll_obj->obj; + t->items[1] = MP_OBJ_NEW_SMALL_INT(poll_obj_get_revents(poll_obj)); + if (self->flags & FLAG_ONESHOT) { + // Don't poll next time, until new event mask will be set explicitly + poll_obj_set_events(poll_obj, 0); + } + return MP_OBJ_FROM_PTR(t); + } + } + + assert(!"inconsistent number of poll active entries"); + self->iter_cnt = 0; + return MP_OBJ_STOP_ITERATION; +} + +STATIC const mp_rom_map_elem_t poll_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_register), MP_ROM_PTR(&poll_register_obj) }, + { MP_ROM_QSTR(MP_QSTR_unregister), MP_ROM_PTR(&poll_unregister_obj) }, + { MP_ROM_QSTR(MP_QSTR_modify), MP_ROM_PTR(&poll_modify_obj) }, + { MP_ROM_QSTR(MP_QSTR_poll), MP_ROM_PTR(&poll_poll_obj) }, + { MP_ROM_QSTR(MP_QSTR_ipoll), MP_ROM_PTR(&poll_ipoll_obj) }, +}; +STATIC MP_DEFINE_CONST_DICT(poll_locals_dict, poll_locals_dict_table); + +STATIC MP_DEFINE_CONST_OBJ_TYPE( + mp_type_poll, + MP_QSTR_poll, + MP_TYPE_FLAG_ITER_IS_ITERNEXT, + iter, poll_iternext, + locals_dict, &poll_locals_dict + ); + +// poll() +STATIC mp_obj_t select_poll(void) { + mp_obj_poll_t *poll = mp_obj_malloc(mp_obj_poll_t, &mp_type_poll); + poll_set_init(&poll->poll_set, 0); + poll->iter_cnt = 0; + poll->ret_tuple = MP_OBJ_NULL; + return MP_OBJ_FROM_PTR(poll); +} +MP_DEFINE_CONST_FUN_OBJ_0(mp_select_poll_obj, select_poll); + +STATIC const mp_rom_map_elem_t mp_module_select_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_select) }, + #if MICROPY_PY_SELECT_SELECT + { MP_ROM_QSTR(MP_QSTR_select), MP_ROM_PTR(&mp_select_select_obj) }, + #endif + { MP_ROM_QSTR(MP_QSTR_poll), MP_ROM_PTR(&mp_select_poll_obj) }, + { MP_ROM_QSTR(MP_QSTR_POLLIN), MP_ROM_INT(MP_STREAM_POLL_RD) }, + { MP_ROM_QSTR(MP_QSTR_POLLOUT), MP_ROM_INT(MP_STREAM_POLL_WR) }, + { MP_ROM_QSTR(MP_QSTR_POLLERR), MP_ROM_INT(MP_STREAM_POLL_ERR) }, + { MP_ROM_QSTR(MP_QSTR_POLLHUP), MP_ROM_INT(MP_STREAM_POLL_HUP) }, +}; + +STATIC MP_DEFINE_CONST_DICT(mp_module_select_globals, mp_module_select_globals_table); + +const mp_obj_module_t mp_module_select = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mp_module_select_globals, +}; + +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_select, mp_module_select); + +#endif // MICROPY_PY_SELECT diff --git a/extmod/modtime.c b/extmod/modtime.c new file mode 100644 index 0000000000..805c2621c0 --- /dev/null +++ b/extmod/modtime.c @@ -0,0 +1,236 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2023 Damien P. George + * Copyright (c) 2016 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/mphal.h" +#include "py/runtime.h" +#include "py/smallint.h" +#include "extmod/modtime.h" + +#if MICROPY_PY_TIME + +#ifdef MICROPY_PY_TIME_INCLUDEFILE +#include MICROPY_PY_TIME_INCLUDEFILE +#endif + +#if MICROPY_PY_TIME_GMTIME_LOCALTIME_MKTIME + +#include "shared/timeutils/timeutils.h" + +// localtime([secs]) +// Convert a time expressed in seconds since the Epoch into an 8-tuple which +// contains: (year, month, mday, hour, minute, second, weekday, yearday) +// If secs is not provided or None, then the current time is used. +// - year is the full year, eg 2000 +// - month is 1-12 +// - mday is 1-31 +// - hour is 0-23 +// - minute is 0-59 +// - second is 0-59 +// - weekday is 0-6 for Mon-Sun +// - yearday is 1-366 +STATIC mp_obj_t time_localtime(size_t n_args, const mp_obj_t *args) { + if (n_args == 0 || args[0] == mp_const_none) { + // Get current date and time. + return mp_time_localtime_get(); + } else { + // Convert given seconds to tuple. + mp_int_t seconds = mp_obj_get_int(args[0]); + timeutils_struct_time_t tm; + timeutils_seconds_since_epoch_to_struct_time(seconds, &tm); + mp_obj_t tuple[8] = { + tuple[0] = mp_obj_new_int(tm.tm_year), + tuple[1] = mp_obj_new_int(tm.tm_mon), + tuple[2] = mp_obj_new_int(tm.tm_mday), + tuple[3] = mp_obj_new_int(tm.tm_hour), + tuple[4] = mp_obj_new_int(tm.tm_min), + tuple[5] = mp_obj_new_int(tm.tm_sec), + tuple[6] = mp_obj_new_int(tm.tm_wday), + tuple[7] = mp_obj_new_int(tm.tm_yday), + }; + return mp_obj_new_tuple(8, tuple); + } +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_time_localtime_obj, 0, 1, time_localtime); + +// mktime() +// This is the inverse function of localtime. Its argument is a full 8-tuple +// which expresses a time as per localtime. It returns an integer which is +// the number of seconds since the Epoch (eg 1st Jan 1970, or 1st Jan 2000). +STATIC mp_obj_t time_mktime(mp_obj_t tuple) { + size_t len; + mp_obj_t *elem; + mp_obj_get_array(tuple, &len, &elem); + + // localtime generates a tuple of len 8. CPython uses 9, so we accept both. + if (len < 8 || len > 9) { + mp_raise_TypeError(MP_ERROR_TEXT("mktime needs a tuple of length 8 or 9")); + } + + return mp_obj_new_int_from_uint(timeutils_mktime(mp_obj_get_int(elem[0]), + mp_obj_get_int(elem[1]), mp_obj_get_int(elem[2]), mp_obj_get_int(elem[3]), + mp_obj_get_int(elem[4]), mp_obj_get_int(elem[5]))); +} +MP_DEFINE_CONST_FUN_OBJ_1(mp_time_mktime_obj, time_mktime); + +#endif // MICROPY_PY_TIME_GMTIME_LOCALTIME_MKTIME + +#if MICROPY_PY_TIME_TIME_TIME_NS + +// time() +// Return the number of seconds since the Epoch. +STATIC mp_obj_t time_time(void) { + return mp_time_time_get(); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_time_time_obj, time_time); + +// time_ns() +// Returns the number of nanoseconds since the Epoch, as an integer. +STATIC mp_obj_t time_time_ns(void) { + return mp_obj_new_int_from_ull(mp_hal_time_ns()); +} +MP_DEFINE_CONST_FUN_OBJ_0(mp_time_time_ns_obj, time_time_ns); + +#endif // MICROPY_PY_TIME_TIME_TIME_NS + +STATIC mp_obj_t time_sleep(mp_obj_t seconds_o) { + #ifdef MICROPY_PY_TIME_CUSTOM_SLEEP + mp_time_sleep(seconds_o); + #else + #if MICROPY_PY_BUILTINS_FLOAT + mp_hal_delay_ms((mp_uint_t)(1000 * mp_obj_get_float(seconds_o))); + #else + mp_hal_delay_ms(1000 * mp_obj_get_int(seconds_o)); + #endif + #endif + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_1(mp_time_sleep_obj, time_sleep); + +STATIC mp_obj_t time_sleep_ms(mp_obj_t arg) { + mp_int_t ms = mp_obj_get_int(arg); + if (ms >= 0) { + mp_hal_delay_ms(ms); + } + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_1(mp_time_sleep_ms_obj, time_sleep_ms); + +STATIC mp_obj_t time_sleep_us(mp_obj_t arg) { + mp_int_t us = mp_obj_get_int(arg); + if (us > 0) { + mp_hal_delay_us(us); + } + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_1(mp_time_sleep_us_obj, time_sleep_us); + +STATIC mp_obj_t time_ticks_ms(void) { + return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_ms() & (MICROPY_PY_TIME_TICKS_PERIOD - 1)); +} +MP_DEFINE_CONST_FUN_OBJ_0(mp_time_ticks_ms_obj, time_ticks_ms); + +STATIC mp_obj_t time_ticks_us(void) { + return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_us() & (MICROPY_PY_TIME_TICKS_PERIOD - 1)); +} +MP_DEFINE_CONST_FUN_OBJ_0(mp_time_ticks_us_obj, time_ticks_us); + +STATIC mp_obj_t time_ticks_cpu(void) { + return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_cpu() & (MICROPY_PY_TIME_TICKS_PERIOD - 1)); +} +MP_DEFINE_CONST_FUN_OBJ_0(mp_time_ticks_cpu_obj, time_ticks_cpu); + +STATIC mp_obj_t time_ticks_diff(mp_obj_t end_in, mp_obj_t start_in) { + // we assume that the arguments come from ticks_xx so are small ints + mp_uint_t start = MP_OBJ_SMALL_INT_VALUE(start_in); + mp_uint_t end = MP_OBJ_SMALL_INT_VALUE(end_in); + // Optimized formula avoiding if conditions. We adjust difference "forward", + // wrap it around and adjust back. + mp_int_t diff = ((end - start + MICROPY_PY_TIME_TICKS_PERIOD / 2) & (MICROPY_PY_TIME_TICKS_PERIOD - 1)) + - MICROPY_PY_TIME_TICKS_PERIOD / 2; + return MP_OBJ_NEW_SMALL_INT(diff); +} +MP_DEFINE_CONST_FUN_OBJ_2(mp_time_ticks_diff_obj, time_ticks_diff); + +STATIC mp_obj_t time_ticks_add(mp_obj_t ticks_in, mp_obj_t delta_in) { + // we assume that first argument come from ticks_xx so is small int + mp_uint_t ticks = MP_OBJ_SMALL_INT_VALUE(ticks_in); + mp_uint_t delta = mp_obj_get_int(delta_in); + + // Check that delta does not overflow the range that ticks_diff can handle. + // This ensures the following: + // - ticks_diff(ticks_add(T, delta), T) == delta + // - ticks_diff(T, ticks_add(T, delta)) == -delta + // The latter requires excluding delta=-TICKS_PERIOD/2. + // + // This unsigned comparison is equivalent to a signed comparison of: + // delta <= -TICKS_PERIOD/2 || delta >= TICKS_PERIOD/2 + if (delta + MICROPY_PY_TIME_TICKS_PERIOD / 2 - 1 >= MICROPY_PY_TIME_TICKS_PERIOD - 1) { + mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("ticks interval overflow")); + } + + return MP_OBJ_NEW_SMALL_INT((ticks + delta) & (MICROPY_PY_TIME_TICKS_PERIOD - 1)); +} +MP_DEFINE_CONST_FUN_OBJ_2(mp_time_ticks_add_obj, time_ticks_add); + +STATIC const mp_rom_map_elem_t mp_module_time_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_time) }, + + #if MICROPY_PY_TIME_GMTIME_LOCALTIME_MKTIME + { MP_ROM_QSTR(MP_QSTR_gmtime), MP_ROM_PTR(&mp_time_localtime_obj) }, + { MP_ROM_QSTR(MP_QSTR_localtime), MP_ROM_PTR(&mp_time_localtime_obj) }, + { MP_ROM_QSTR(MP_QSTR_mktime), MP_ROM_PTR(&mp_time_mktime_obj) }, + #endif + + #if MICROPY_PY_TIME_TIME_TIME_NS + { MP_ROM_QSTR(MP_QSTR_time), MP_ROM_PTR(&mp_time_time_obj) }, + { MP_ROM_QSTR(MP_QSTR_time_ns), MP_ROM_PTR(&mp_time_time_ns_obj) }, + #endif + + { MP_ROM_QSTR(MP_QSTR_sleep), MP_ROM_PTR(&mp_time_sleep_obj) }, + { MP_ROM_QSTR(MP_QSTR_sleep_ms), MP_ROM_PTR(&mp_time_sleep_ms_obj) }, + { MP_ROM_QSTR(MP_QSTR_sleep_us), MP_ROM_PTR(&mp_time_sleep_us_obj) }, + + { MP_ROM_QSTR(MP_QSTR_ticks_ms), MP_ROM_PTR(&mp_time_ticks_ms_obj) }, + { MP_ROM_QSTR(MP_QSTR_ticks_us), MP_ROM_PTR(&mp_time_ticks_us_obj) }, + { MP_ROM_QSTR(MP_QSTR_ticks_cpu), MP_ROM_PTR(&mp_time_ticks_cpu_obj) }, + { MP_ROM_QSTR(MP_QSTR_ticks_add), MP_ROM_PTR(&mp_time_ticks_add_obj) }, + { MP_ROM_QSTR(MP_QSTR_ticks_diff), MP_ROM_PTR(&mp_time_ticks_diff_obj) }, + + #ifdef MICROPY_PY_TIME_EXTRA_GLOBALS + MICROPY_PY_TIME_EXTRA_GLOBALS + #endif +}; +STATIC MP_DEFINE_CONST_DICT(mp_module_time_globals, mp_module_time_globals_table); + +const mp_obj_module_t mp_module_time = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mp_module_time_globals, +}; + +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_time, mp_module_time); + +#endif // MICROPY_PY_TIME diff --git a/extmod/utime_mphal.h b/extmod/modtime.h similarity index 67% rename from extmod/utime_mphal.h rename to extmod/modtime.h index 57fc348832..f5ea6b55bf 100644 --- a/extmod/utime_mphal.h +++ b/extmod/modtime.h @@ -24,19 +24,20 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ -#ifndef MICROPY_INCLUDED_EXTMOD_UTIME_MPHAL_H -#define MICROPY_INCLUDED_EXTMOD_UTIME_MPHAL_H +#ifndef MICROPY_INCLUDED_EXTMOD_MODUTIME_H +#define MICROPY_INCLUDED_EXTMOD_MODUTIME_H #include "py/obj.h" -MP_DECLARE_CONST_FUN_OBJ_1(mp_utime_sleep_obj); -MP_DECLARE_CONST_FUN_OBJ_1(mp_utime_sleep_ms_obj); -MP_DECLARE_CONST_FUN_OBJ_1(mp_utime_sleep_us_obj); -MP_DECLARE_CONST_FUN_OBJ_0(mp_utime_ticks_ms_obj); -MP_DECLARE_CONST_FUN_OBJ_0(mp_utime_ticks_us_obj); -MP_DECLARE_CONST_FUN_OBJ_0(mp_utime_ticks_cpu_obj); -MP_DECLARE_CONST_FUN_OBJ_2(mp_utime_ticks_diff_obj); -MP_DECLARE_CONST_FUN_OBJ_2(mp_utime_ticks_add_obj); -MP_DECLARE_CONST_FUN_OBJ_0(mp_utime_time_ns_obj); +MP_DECLARE_CONST_FUN_OBJ_1(mp_time_mktime_obj); +MP_DECLARE_CONST_FUN_OBJ_1(mp_time_sleep_obj); +MP_DECLARE_CONST_FUN_OBJ_1(mp_time_sleep_ms_obj); +MP_DECLARE_CONST_FUN_OBJ_1(mp_time_sleep_us_obj); +MP_DECLARE_CONST_FUN_OBJ_0(mp_time_ticks_ms_obj); +MP_DECLARE_CONST_FUN_OBJ_0(mp_time_ticks_us_obj); +MP_DECLARE_CONST_FUN_OBJ_0(mp_time_ticks_cpu_obj); +MP_DECLARE_CONST_FUN_OBJ_2(mp_time_ticks_diff_obj); +MP_DECLARE_CONST_FUN_OBJ_2(mp_time_ticks_add_obj); +MP_DECLARE_CONST_FUN_OBJ_0(mp_time_time_ns_obj); -#endif // MICROPY_INCLUDED_EXTMOD_UTIME_MPHAL_H +#endif // MICROPY_INCLUDED_EXTMOD_MODUTIME_H diff --git a/extmod/moductypes.c b/extmod/moductypes.c index 15c36290a9..63be621fbc 100644 --- a/extmod/moductypes.c +++ b/extmod/moductypes.c @@ -582,7 +582,7 @@ STATIC mp_obj_t uctypes_struct_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_ob STATIC mp_obj_t uctypes_struct_unary_op(mp_unary_op_t op, mp_obj_t self_in) { mp_obj_uctypes_struct_t *self = MP_OBJ_TO_PTR(self_in); switch (op) { - case MP_UNARY_OP_INT: + case MP_UNARY_OP_INT_MAYBE: if (mp_obj_is_type(self->desc, &mp_type_tuple)) { mp_obj_tuple_t *t = MP_OBJ_TO_PTR(self->desc); mp_int_t offset = MP_OBJ_SMALL_INT_VALUE(t->items[0]); @@ -718,6 +718,8 @@ const mp_obj_module_t mp_module_uctypes = { .globals = (mp_obj_dict_t *)&mp_module_uctypes_globals, }; +// uctypes is not a Python standard library module (hence "uctypes" +// not "ctypes") and therefore shouldn't be extensible. MP_REGISTER_MODULE(MP_QSTR_uctypes, mp_module_uctypes); #endif diff --git a/extmod/moduselect.c b/extmod/moduselect.c deleted file mode 100644 index a1513835b2..0000000000 --- a/extmod/moduselect.c +++ /dev/null @@ -1,384 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2014 Damien P. George - * Copyright (c) 2015-2017 Paul Sokolovsky - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "py/mpconfig.h" -#if MICROPY_PY_USELECT - -#include - -#include "py/runtime.h" -#include "py/obj.h" -#include "py/objlist.h" -#include "py/stream.h" -#include "py/mperrno.h" -#include "py/mphal.h" -#include "shared/runtime/interrupt_char.h" - -// Flags for poll() -#define FLAG_ONESHOT (1) - -typedef struct _poll_obj_t { - mp_obj_t obj; - mp_uint_t (*ioctl)(mp_obj_t obj, mp_uint_t request, uintptr_t arg, int *errcode); - mp_uint_t flags; - mp_uint_t flags_ret; -} poll_obj_t; - -STATIC void poll_map_add(mp_map_t *poll_map, const mp_obj_t *obj, mp_uint_t obj_len, mp_uint_t flags, bool or_flags) { - for (mp_uint_t i = 0; i < obj_len; i++) { - mp_map_elem_t *elem = mp_map_lookup(poll_map, mp_obj_id(obj[i]), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); - if (elem->value == MP_OBJ_NULL) { - // object not found; get its ioctl and add it to the poll list - const mp_stream_p_t *stream_p = mp_get_stream_raise(obj[i], MP_STREAM_OP_IOCTL); - poll_obj_t *poll_obj = m_new_obj(poll_obj_t); - poll_obj->obj = obj[i]; - poll_obj->ioctl = stream_p->ioctl; - poll_obj->flags = flags; - poll_obj->flags_ret = 0; - elem->value = MP_OBJ_FROM_PTR(poll_obj); - } else { - // object exists; update its flags - if (or_flags) { - ((poll_obj_t *)MP_OBJ_TO_PTR(elem->value))->flags |= flags; - } else { - ((poll_obj_t *)MP_OBJ_TO_PTR(elem->value))->flags = flags; - } - } - } -} - -// poll each object in the map -STATIC mp_uint_t poll_map_poll(mp_map_t *poll_map, size_t *rwx_num) { - mp_uint_t n_ready = 0; - for (mp_uint_t i = 0; i < poll_map->alloc; ++i) { - if (!mp_map_slot_is_filled(poll_map, i)) { - continue; - } - - poll_obj_t *poll_obj = MP_OBJ_TO_PTR(poll_map->table[i].value); - int errcode; - mp_int_t ret = poll_obj->ioctl(poll_obj->obj, MP_STREAM_POLL, poll_obj->flags, &errcode); - poll_obj->flags_ret = ret; - - if (ret == -1) { - // error doing ioctl - mp_raise_OSError(errcode); - } - - if (ret != 0) { - // object is ready - n_ready += 1; - if (rwx_num != NULL) { - if (ret & MP_STREAM_POLL_RD) { - rwx_num[0] += 1; - } - if (ret & MP_STREAM_POLL_WR) { - rwx_num[1] += 1; - } - if ((ret & ~(MP_STREAM_POLL_RD | MP_STREAM_POLL_WR)) != 0) { - rwx_num[2] += 1; - } - } - } - } - return n_ready; -} - -#if MICROPY_PY_USELECT_SELECT -// select(rlist, wlist, xlist[, timeout]) -STATIC mp_obj_t select_select(size_t n_args, const mp_obj_t *args) { - // get array data from tuple/list arguments - size_t rwx_len[3]; - mp_obj_t *r_array, *w_array, *x_array; - mp_obj_get_array(args[0], &rwx_len[0], &r_array); - mp_obj_get_array(args[1], &rwx_len[1], &w_array); - mp_obj_get_array(args[2], &rwx_len[2], &x_array); - - // get timeout - mp_uint_t timeout = -1; - if (n_args == 4) { - if (args[3] != mp_const_none) { - #if MICROPY_PY_BUILTINS_FLOAT - float timeout_f = mp_obj_get_float_to_f(args[3]); - if (timeout_f >= 0) { - timeout = (mp_uint_t)(timeout_f * 1000); - } - #else - timeout = mp_obj_get_int(args[3]) * 1000; - #endif - } - } - - // merge separate lists and get the ioctl function for each object - mp_map_t poll_map; - mp_map_init(&poll_map, rwx_len[0] + rwx_len[1] + rwx_len[2]); - poll_map_add(&poll_map, r_array, rwx_len[0], MP_STREAM_POLL_RD, true); - poll_map_add(&poll_map, w_array, rwx_len[1], MP_STREAM_POLL_WR, true); - poll_map_add(&poll_map, x_array, rwx_len[2], MP_STREAM_POLL_ERR | MP_STREAM_POLL_HUP, true); - - mp_uint_t start_tick = mp_hal_ticks_ms(); - rwx_len[0] = rwx_len[1] = rwx_len[2] = 0; - for (;;) { - // poll the objects - mp_uint_t n_ready = poll_map_poll(&poll_map, rwx_len); - - if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_tick >= timeout)) { - // one or more objects are ready, or we had a timeout - mp_obj_t list_array[3]; - list_array[0] = mp_obj_new_list(rwx_len[0], NULL); - list_array[1] = mp_obj_new_list(rwx_len[1], NULL); - list_array[2] = mp_obj_new_list(rwx_len[2], NULL); - rwx_len[0] = rwx_len[1] = rwx_len[2] = 0; - for (mp_uint_t i = 0; i < poll_map.alloc; ++i) { - if (!mp_map_slot_is_filled(&poll_map, i)) { - continue; - } - poll_obj_t *poll_obj = MP_OBJ_TO_PTR(poll_map.table[i].value); - if (poll_obj->flags_ret & MP_STREAM_POLL_RD) { - ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[0]))->items[rwx_len[0]++] = poll_obj->obj; - } - if (poll_obj->flags_ret & MP_STREAM_POLL_WR) { - ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[1]))->items[rwx_len[1]++] = poll_obj->obj; - } - if ((poll_obj->flags_ret & ~(MP_STREAM_POLL_RD | MP_STREAM_POLL_WR)) != 0) { - ((mp_obj_list_t *)MP_OBJ_TO_PTR(list_array[2]))->items[rwx_len[2]++] = poll_obj->obj; - } - } - mp_map_deinit(&poll_map); - return mp_obj_new_tuple(3, list_array); - } - // CIRCUITPY - RUN_BACKGROUND_TASKS; - } -} -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_select_select_obj, 3, 4, select_select); -#endif // MICROPY_PY_USELECT_SELECT - -typedef struct _mp_obj_poll_t { - mp_obj_base_t base; - mp_map_t poll_map; - short iter_cnt; - short iter_idx; - int flags; - // callee-owned tuple - mp_obj_t ret_tuple; -} mp_obj_poll_t; - -// register(obj[, eventmask]) -STATIC mp_obj_t poll_register(size_t n_args, const mp_obj_t *args) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); - mp_uint_t flags; - if (n_args == 3) { - flags = mp_obj_get_int(args[2]); - } else { - flags = MP_STREAM_POLL_RD | MP_STREAM_POLL_WR; - } - poll_map_add(&self->poll_map, &args[1], 1, flags, false); - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_register_obj, 2, 3, poll_register); - -// unregister(obj) -STATIC mp_obj_t poll_unregister(mp_obj_t self_in, mp_obj_t obj_in) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in); - mp_map_lookup(&self->poll_map, mp_obj_id(obj_in), MP_MAP_LOOKUP_REMOVE_IF_FOUND); - // TODO raise KeyError if obj didn't exist in map - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_2(poll_unregister_obj, poll_unregister); - -// modify(obj, eventmask) -STATIC mp_obj_t poll_modify(mp_obj_t self_in, mp_obj_t obj_in, mp_obj_t eventmask_in) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in); - mp_map_elem_t *elem = mp_map_lookup(&self->poll_map, mp_obj_id(obj_in), MP_MAP_LOOKUP); - if (elem == NULL) { - mp_raise_OSError(MP_ENOENT); - } - ((poll_obj_t *)MP_OBJ_TO_PTR(elem->value))->flags = mp_obj_get_int(eventmask_in); - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_3(poll_modify_obj, poll_modify); - -STATIC mp_uint_t poll_poll_internal(uint n_args, const mp_obj_t *args) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); - - // work out timeout (its given already in ms) - mp_uint_t timeout = -1; - int flags = 0; - if (n_args >= 2) { - if (args[1] != mp_const_none) { - mp_int_t timeout_i = mp_obj_get_int(args[1]); - if (timeout_i >= 0) { - timeout = timeout_i; - } - } - if (n_args >= 3) { - flags = mp_obj_get_int(args[2]); - } - } - - self->flags = flags; - - mp_uint_t start_tick = mp_hal_ticks_ms(); - mp_uint_t n_ready; - for (;;) { - // poll the objects - n_ready = poll_map_poll(&self->poll_map, NULL); - if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_tick >= timeout)) { - break; - } - // CIRCUITPY - RUN_BACKGROUND_TASKS; - if (mp_hal_is_interrupted()) { - return 0; - } - } - - return n_ready; -} - -STATIC mp_obj_t poll_poll(size_t n_args, const mp_obj_t *args) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); - mp_uint_t n_ready = poll_poll_internal(n_args, args); - - // one or more objects are ready, or we had a timeout - mp_obj_list_t *ret_list = MP_OBJ_TO_PTR(mp_obj_new_list(n_ready, NULL)); - n_ready = 0; - for (mp_uint_t i = 0; i < self->poll_map.alloc; ++i) { - if (!mp_map_slot_is_filled(&self->poll_map, i)) { - continue; - } - poll_obj_t *poll_obj = MP_OBJ_TO_PTR(self->poll_map.table[i].value); - if (poll_obj->flags_ret != 0) { - mp_obj_t tuple[2] = {poll_obj->obj, MP_OBJ_NEW_SMALL_INT(poll_obj->flags_ret)}; - ret_list->items[n_ready++] = mp_obj_new_tuple(2, tuple); - if (self->flags & FLAG_ONESHOT) { - // Don't poll next time, until new event flags will be set explicitly - poll_obj->flags = 0; - } - } - } - return MP_OBJ_FROM_PTR(ret_list); -} -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_poll_obj, 1, 3, poll_poll); - -STATIC mp_obj_t poll_ipoll(size_t n_args, const mp_obj_t *args) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); - - if (self->ret_tuple == MP_OBJ_NULL) { - self->ret_tuple = mp_obj_new_tuple(2, NULL); - } - - int n_ready = poll_poll_internal(n_args, args); - self->iter_cnt = n_ready; - self->iter_idx = 0; - - return args[0]; -} -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_ipoll_obj, 1, 3, poll_ipoll); - -STATIC mp_obj_t poll_iternext(mp_obj_t self_in) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in); - - if (self->iter_cnt == 0) { - return MP_OBJ_STOP_ITERATION; - } - - self->iter_cnt--; - - for (mp_uint_t i = self->iter_idx; i < self->poll_map.alloc; ++i) { - self->iter_idx++; - if (!mp_map_slot_is_filled(&self->poll_map, i)) { - continue; - } - poll_obj_t *poll_obj = MP_OBJ_TO_PTR(self->poll_map.table[i].value); - if (poll_obj->flags_ret != 0) { - mp_obj_tuple_t *t = MP_OBJ_TO_PTR(self->ret_tuple); - t->items[0] = poll_obj->obj; - t->items[1] = MP_OBJ_NEW_SMALL_INT(poll_obj->flags_ret); - if (self->flags & FLAG_ONESHOT) { - // Don't poll next time, until new event flags will be set explicitly - poll_obj->flags = 0; - } - return MP_OBJ_FROM_PTR(t); - } - } - - assert(!"inconsistent number of poll active entries"); - self->iter_cnt = 0; - return MP_OBJ_STOP_ITERATION; -} - -STATIC const mp_rom_map_elem_t poll_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR_register), MP_ROM_PTR(&poll_register_obj) }, - { MP_ROM_QSTR(MP_QSTR_unregister), MP_ROM_PTR(&poll_unregister_obj) }, - { MP_ROM_QSTR(MP_QSTR_modify), MP_ROM_PTR(&poll_modify_obj) }, - { MP_ROM_QSTR(MP_QSTR_poll), MP_ROM_PTR(&poll_poll_obj) }, - { MP_ROM_QSTR(MP_QSTR_ipoll), MP_ROM_PTR(&poll_ipoll_obj) }, -}; -STATIC MP_DEFINE_CONST_DICT(poll_locals_dict, poll_locals_dict_table); - -STATIC MP_DEFINE_CONST_OBJ_TYPE( - mp_type_poll, - MP_QSTR_poll, - MP_TYPE_FLAG_ITER_IS_ITERNEXT, - iter, poll_iternext, - locals_dict, &poll_locals_dict - ); - -// poll() -STATIC mp_obj_t select_poll(void) { - mp_obj_poll_t *poll = mp_obj_malloc(mp_obj_poll_t, &mp_type_poll); - mp_map_init(&poll->poll_map, 0); - poll->iter_cnt = 0; - poll->ret_tuple = MP_OBJ_NULL; - return MP_OBJ_FROM_PTR(poll); -} -MP_DEFINE_CONST_FUN_OBJ_0(mp_select_poll_obj, select_poll); - -STATIC const mp_rom_map_elem_t mp_module_select_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uselect) }, - #if MICROPY_PY_USELECT_SELECT - { MP_ROM_QSTR(MP_QSTR_select), MP_ROM_PTR(&mp_select_select_obj) }, - #endif - { MP_ROM_QSTR(MP_QSTR_poll), MP_ROM_PTR(&mp_select_poll_obj) }, - { MP_ROM_QSTR(MP_QSTR_POLLIN), MP_ROM_INT(MP_STREAM_POLL_RD) }, - { MP_ROM_QSTR(MP_QSTR_POLLOUT), MP_ROM_INT(MP_STREAM_POLL_WR) }, - { MP_ROM_QSTR(MP_QSTR_POLLERR), MP_ROM_INT(MP_STREAM_POLL_ERR) }, - { MP_ROM_QSTR(MP_QSTR_POLLHUP), MP_ROM_INT(MP_STREAM_POLL_HUP) }, -}; - -STATIC MP_DEFINE_CONST_DICT(mp_module_select_globals, mp_module_select_globals_table); - -const mp_obj_module_t mp_module_uselect = { - .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&mp_module_select_globals, -}; - -MP_REGISTER_MODULE(MP_QSTR_select, mp_module_uselect); - -#endif // MICROPY_PY_USELECT diff --git a/extmod/moduzlib.c b/extmod/modzlib.c similarity index 100% rename from extmod/moduzlib.c rename to extmod/modzlib.c diff --git a/extmod/utime_mphal.c b/extmod/utime_mphal.c deleted file mode 100644 index 9cd5d938d0..0000000000 --- a/extmod/utime_mphal.c +++ /dev/null @@ -1,121 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2013-2016 Damien P. George - * Copyright (c) 2016 Paul Sokolovsky - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "py/mpconfig.h" -#if MICROPY_PY_UTIME_MP_HAL - -#include - -#include "py/obj.h" -#include "py/mphal.h" -#include "py/smallint.h" -#include "py/runtime.h" -#include "extmod/utime_mphal.h" - -STATIC mp_obj_t time_sleep(mp_obj_t seconds_o) { - #if MICROPY_PY_BUILTINS_FLOAT - mp_hal_delay_ms((mp_uint_t)(1000 * mp_obj_get_float(seconds_o))); - #else - mp_hal_delay_ms(1000 * mp_obj_get_int(seconds_o)); - #endif - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_1(mp_utime_sleep_obj, time_sleep); - -STATIC mp_obj_t time_sleep_ms(mp_obj_t arg) { - mp_int_t ms = mp_obj_get_int(arg); - if (ms >= 0) { - mp_hal_delay_ms(ms); - } - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_1(mp_utime_sleep_ms_obj, time_sleep_ms); - -STATIC mp_obj_t time_sleep_us(mp_obj_t arg) { - mp_int_t us = mp_obj_get_int(arg); - if (us > 0) { - mp_hal_delay_us(us); - } - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_1(mp_utime_sleep_us_obj, time_sleep_us); - -STATIC mp_obj_t time_ticks_ms(void) { - return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_ms() & (MICROPY_PY_UTIME_TICKS_PERIOD - 1)); -} -MP_DEFINE_CONST_FUN_OBJ_0(mp_utime_ticks_ms_obj, time_ticks_ms); - -STATIC mp_obj_t time_ticks_us(void) { - return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_us() & (MICROPY_PY_UTIME_TICKS_PERIOD - 1)); -} -MP_DEFINE_CONST_FUN_OBJ_0(mp_utime_ticks_us_obj, time_ticks_us); - -STATIC mp_obj_t time_ticks_cpu(void) { - return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_cpu() & (MICROPY_PY_UTIME_TICKS_PERIOD - 1)); -} -MP_DEFINE_CONST_FUN_OBJ_0(mp_utime_ticks_cpu_obj, time_ticks_cpu); - -STATIC mp_obj_t time_ticks_diff(mp_obj_t end_in, mp_obj_t start_in) { - // we assume that the arguments come from ticks_xx so are small ints - mp_uint_t start = MP_OBJ_SMALL_INT_VALUE(start_in); - mp_uint_t end = MP_OBJ_SMALL_INT_VALUE(end_in); - // Optimized formula avoiding if conditions. We adjust difference "forward", - // wrap it around and adjust back. - mp_int_t diff = ((end - start + MICROPY_PY_UTIME_TICKS_PERIOD / 2) & (MICROPY_PY_UTIME_TICKS_PERIOD - 1)) - - MICROPY_PY_UTIME_TICKS_PERIOD / 2; - return MP_OBJ_NEW_SMALL_INT(diff); -} -MP_DEFINE_CONST_FUN_OBJ_2(mp_utime_ticks_diff_obj, time_ticks_diff); - -STATIC mp_obj_t time_ticks_add(mp_obj_t ticks_in, mp_obj_t delta_in) { - // we assume that first argument come from ticks_xx so is small int - mp_uint_t ticks = MP_OBJ_SMALL_INT_VALUE(ticks_in); - mp_uint_t delta = mp_obj_get_int(delta_in); - - // Check that delta does not overflow the range that ticks_diff can handle. - // This ensures the following: - // - ticks_diff(ticks_add(T, delta), T) == delta - // - ticks_diff(T, ticks_add(T, delta)) == -delta - // The latter requires excluding delta=-TICKS_PERIOD/2. - // - // This unsigned comparison is equivalent to a signed comparison of: - // delta <= -TICKS_PERIOD/2 || delta >= TICKS_PERIOD/2 - if (delta + MICROPY_PY_UTIME_TICKS_PERIOD / 2 - 1 >= MICROPY_PY_UTIME_TICKS_PERIOD - 1) { - mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("ticks interval overflow")); - } - - return MP_OBJ_NEW_SMALL_INT((ticks + delta) & (MICROPY_PY_UTIME_TICKS_PERIOD - 1)); -} -MP_DEFINE_CONST_FUN_OBJ_2(mp_utime_ticks_add_obj, time_ticks_add); - -// Returns the number of nanoseconds since the Epoch, as an integer. -STATIC mp_obj_t time_time_ns(void) { - return mp_obj_new_int_from_ull(mp_hal_time_ns()); -} -MP_DEFINE_CONST_FUN_OBJ_0(mp_utime_time_ns_obj, time_time_ns); - -#endif // MICROPY_PY_UTIME_MP_HAL diff --git a/extmod/vfs_fat_file.c b/extmod/vfs_fat_file.c index 72b712ab71..91c6bb8501 100644 --- a/extmod/vfs_fat_file.c +++ b/extmod/vfs_fat_file.c @@ -93,13 +93,6 @@ STATIC mp_uint_t file_obj_write(mp_obj_t self_in, const void *buf, mp_uint_t siz return sz_out; } - -STATIC mp_obj_t file_obj___exit__(size_t n_args, const mp_obj_t *args) { - (void)n_args; - return mp_stream_close(args[0]); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(file_obj___exit___obj, 4, 4, file_obj___exit__); - STATIC mp_uint_t file_obj_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { pyb_file_obj_t *self = MP_OBJ_TO_PTR(o_in); @@ -162,9 +155,8 @@ STATIC const mp_rom_map_elem_t vfs_fat_rawfile_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) }, { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&file_obj___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, }; -#define FILE_OPEN_NUM_ARGS MP_ARRAY_SIZE(file_open_args) STATIC MP_DEFINE_CONST_DICT(vfs_fat_rawfile_locals_dict, vfs_fat_rawfile_locals_dict_table); diff --git a/extmod/vfs_lfsx.c b/extmod/vfs_lfsx.c index de1f421977..fe0731eced 100644 --- a/extmod/vfs_lfsx.c +++ b/extmod/vfs_lfsx.c @@ -323,7 +323,7 @@ STATIC mp_obj_t MP_VFS_LFSx(chdir)(mp_obj_t self_in, mp_obj_t path_in) { size_t from = 1; char *cwd = vstr_str(&self->cur_dir); while (from < CWD_LEN) { - for (; cwd[from] == '/' && from < CWD_LEN; ++from) { + for (; from < CWD_LEN && cwd[from] == '/'; ++from) { // Scan for the start } if (from > to) { @@ -331,7 +331,7 @@ STATIC mp_obj_t MP_VFS_LFSx(chdir)(mp_obj_t self_in, mp_obj_t path_in) { vstr_cut_out_bytes(&self->cur_dir, to, from - to); from = to; } - for (; cwd[from] != '/' && from < CWD_LEN; ++from) { + for (; from < CWD_LEN && cwd[from] != '/'; ++from) { // Scan for the next / } if ((from - to) == 1 && cwd[to] == '.') { diff --git a/extmod/vfs_lfsx_file.c b/extmod/vfs_lfsx_file.c index 5f0155d3c5..879ba78973 100644 --- a/extmod/vfs_lfsx_file.c +++ b/extmod/vfs_lfsx_file.c @@ -123,12 +123,6 @@ mp_obj_t MP_VFS_LFSx(file_open)(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mod return MP_OBJ_FROM_PTR(o); } -STATIC mp_obj_t MP_VFS_LFSx(file___exit__)(size_t n_args, const mp_obj_t *args) { - (void)n_args; - return mp_stream_close(args[0]); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(MP_VFS_LFSx(file___exit___obj), 4, 4, MP_VFS_LFSx(file___exit__)); - STATIC mp_uint_t MP_VFS_LFSx(file_read)(mp_obj_t self_in, void *buf, mp_uint_t size, int *errcode) { MP_OBJ_VFS_LFSx_FILE *self = MP_OBJ_TO_PTR(self_in); MP_VFS_LFSx(check_open)(self); @@ -213,7 +207,7 @@ STATIC const mp_rom_map_elem_t MP_VFS_LFSx(file_locals_dict_table)[] = { { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) }, { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&MP_VFS_LFSx(file___exit___obj)) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, }; STATIC MP_DEFINE_CONST_DICT(MP_VFS_LFSx(file_locals_dict), MP_VFS_LFSx(file_locals_dict_table)); diff --git a/extmod/vfs_posix.c b/extmod/vfs_posix.c index 9b856e1f01..d63bb5be7b 100644 --- a/extmod/vfs_posix.c +++ b/extmod/vfs_posix.c @@ -332,7 +332,7 @@ STATIC mp_obj_t vfs_posix_stat(mp_obj_t self_in, mp_obj_t path_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_stat_obj, vfs_posix_stat); -#if MICROPY_PY_UOS_STATVFS +#if MICROPY_PY_OS_STATVFS #ifdef __ANDROID__ #define USE_STATFS 1 @@ -390,7 +390,7 @@ STATIC const mp_rom_map_elem_t vfs_posix_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_rename), MP_ROM_PTR(&vfs_posix_rename_obj) }, { MP_ROM_QSTR(MP_QSTR_rmdir), MP_ROM_PTR(&vfs_posix_rmdir_obj) }, { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&vfs_posix_stat_obj) }, - #if MICROPY_PY_UOS_STATVFS + #if MICROPY_PY_OS_STATVFS { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&vfs_posix_statvfs_obj) }, #endif }; diff --git a/extmod/vfs_posix_file.c b/extmod/vfs_posix_file.c index ea19de7fd0..81a608d2b6 100644 --- a/extmod/vfs_posix_file.c +++ b/extmod/vfs_posix_file.c @@ -115,12 +115,6 @@ STATIC mp_obj_t vfs_posix_file_fileno(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(vfs_posix_file_fileno_obj, vfs_posix_file_fileno); -STATIC mp_obj_t vfs_posix_file___exit__(size_t n_args, const mp_obj_t *args) { - (void)n_args; - return mp_stream_close(args[0]); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(vfs_posix_file___exit___obj, 4, 4, vfs_posix_file___exit__); - STATIC mp_uint_t vfs_posix_file_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { mp_obj_vfs_posix_file_t *o = MP_OBJ_TO_PTR(o_in); check_fd_is_open(o); @@ -159,12 +153,20 @@ STATIC mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_ switch (request) { case MP_STREAM_FLUSH: { int ret; + // fsync(stdin/stdout/stderr) may fail with EINVAL (or ENOTSUP on macos or EBADF + // on windows), because the OS doesn't buffer these except for instance when they + // are redirected from/to file, but don't propagate that error out. Because data + // is not buffered by us, and stdin/out/err.flush() should just be a no-op. + #if defined(__APPLE__) + #define VFS_POSIX_STREAM_STDIO_ERR_CATCH (err == EINVAL || err == ENOTSUP) + #elif defined(_MSC_VER) + #define VFS_POSIX_STREAM_STDIO_ERR_CATCH (err == EINVAL || err == EBADF) + #else + #define VFS_POSIX_STREAM_STDIO_ERR_CATCH (err == EINVAL) + #endif MP_HAL_RETRY_SYSCALL(ret, fsync(o->fd), { - if (err == EINVAL + if (VFS_POSIX_STREAM_STDIO_ERR_CATCH && (o->fd == STDIN_FILENO || o->fd == STDOUT_FILENO || o->fd == STDERR_FILENO)) { - // fsync(stdin/stdout/stderr) may fail with EINVAL, but don't propagate that - // error out. Because data is not buffered by us, and stdin/out/err.flush() - // should just be a no-op. return 0; } *errcode = err; @@ -194,7 +196,7 @@ STATIC mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_ return 0; case MP_STREAM_GET_FILENO: return o->fd; - #if MICROPY_PY_USELECT + #if MICROPY_PY_SELECT && !MICROPY_PY_SELECT_POSIX_OPTIMISATIONS case MP_STREAM_POLL: { #ifdef _WIN32 mp_raise_NotImplementedError(MP_ERROR_TEXT("poll on file not available on win32")); @@ -215,6 +217,15 @@ STATIC mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_ if (pfd.revents & POLLOUT) { ret |= MP_STREAM_POLL_WR; } + if (pfd.revents & POLLERR) { + ret |= MP_STREAM_POLL_ERR; + } + if (pfd.revents & POLLHUP) { + ret |= MP_STREAM_POLL_HUP; + } + if (pfd.revents & POLLNVAL) { + ret |= MP_STREAM_POLL_NVAL; + } } return ret; #endif @@ -239,7 +250,7 @@ STATIC const mp_rom_map_elem_t vfs_posix_rawfile_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&vfs_posix_file___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, }; STATIC MP_DEFINE_CONST_DICT(vfs_posix_rawfile_locals_dict, vfs_posix_rawfile_locals_dict_table); @@ -266,12 +277,58 @@ STATIC const mp_stream_p_t vfs_posix_textio_stream_p = { .is_text = true, }; +#if MICROPY_PY_SYS_STDIO_BUFFER + +const mp_obj_vfs_posix_file_t mp_sys_stdin_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDIN_FILENO}; +const mp_obj_vfs_posix_file_t mp_sys_stdout_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDOUT_FILENO}; +const mp_obj_vfs_posix_file_t mp_sys_stderr_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDERR_FILENO}; + +// Forward declarations. +const mp_obj_vfs_posix_file_t mp_sys_stdin_obj; +const mp_obj_vfs_posix_file_t mp_sys_stdout_obj; +const mp_obj_vfs_posix_file_t mp_sys_stderr_obj; + +STATIC void vfs_posix_textio_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + if (dest[0] != MP_OBJ_NULL) { + // These objects are read-only. + return; + } + + if (attr == MP_QSTR_buffer) { + // Implement the `buffer` attribute only on std{in,out,err} instances. + if (MP_OBJ_TO_PTR(self_in) == &mp_sys_stdin_obj) { + dest[0] = MP_OBJ_FROM_PTR(&mp_sys_stdin_buffer_obj); + return; + } + if (MP_OBJ_TO_PTR(self_in) == &mp_sys_stdout_obj) { + dest[0] = MP_OBJ_FROM_PTR(&mp_sys_stdout_buffer_obj); + return; + } + if (MP_OBJ_TO_PTR(self_in) == &mp_sys_stderr_obj) { + dest[0] = MP_OBJ_FROM_PTR(&mp_sys_stderr_buffer_obj); + return; + } + } + + // Any other attribute - forward to locals dict. + dest[1] = MP_OBJ_SENTINEL; +}; + +#define VFS_POSIX_TEXTIO_TYPE_ATTR attr, vfs_posix_textio_attr, + +#else + +#define VFS_POSIX_TEXTIO_TYPE_ATTR + +#endif // MICROPY_PY_SYS_STDIO_BUFFER + MP_DEFINE_CONST_OBJ_TYPE( mp_type_vfs_posix_textio, MP_QSTR_TextIOWrapper, MP_TYPE_FLAG_ITER_IS_STREAM, print, vfs_posix_file_print, protocol, &vfs_posix_textio_stream_p, + VFS_POSIX_TEXTIO_TYPE_ATTR locals_dict, &vfs_posix_rawfile_locals_dict ); diff --git a/lib/mbedtls b/lib/mbedtls deleted file mode 160000 index 1bc2c9cb8b..0000000000 --- a/lib/mbedtls +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1bc2c9cb8b8fe4659bd94b8ebba5a4c02029b7fa diff --git a/lib/mbedtls_errors/README.md b/lib/mbedtls_errors/README.md deleted file mode 100644 index 0e13021eb1..0000000000 --- a/lib/mbedtls_errors/README.md +++ /dev/null @@ -1,42 +0,0 @@ -MBEDTLS Error Strings for MicroPython -===================================== - -This directory contains source code and tools to rework the Mbedtls error strings for -micropython to use less space. In short, instead of storing and printing something like -"SSL - Our own certificate(s) is/are too large to send in an SSL message" it prints -the name of the error #define, which would be "MBEDTLS_ERR_SSL_CERTIFICATE_TOO_LARGE" in -this case, and only stores `SSL_CERTIFICATE_TOO_LARGE` in flash. The exact Mbedtls error -defines are used because they're easy to search for to find more detailed information. - -Mbedtls defines a specific format for error value #defines and -includes a Perl script to gather all `MBEDTLS_ERR` defines from includes files together with -english error text. From that the Perl script generates `mbedtls_strerror()`. The files in this -directory modify this process to produce a more space efficient error lookup table with -shorter error strings. - -The files are as follows: -- `generate_errors.diff` - diff for original mbedtls perl script -- `error.fmt` - modified code template for MicroPython -- `mp_mbedtls_errors.c` - source file with `mbedtls_strerror` this is built using the include - files in `../mbedtls` -- `do-mp.sh` - shell script to produce `mp_mbedtls_errors.c` -- `tester.c` - simple C main to test `mp_mbedtls_errors.c` locally on a dev box -- `do-test.sh` - shell script to produce `mp_mbedtls_errors.c` and compile the `tester` app -- `do-esp32.sh` - shell script to produce `esp32_mbedtls_errors.c` -- see below - -In order not to store multiple copies of `mbedtls_errors.c` -([https://github.com/micropython/micropython/pull/5819#discussion_r445528006](see)) -it is assumed that all ports use the same version of mbedtls with the same error #defines. -This is true as of MP v1.13, and ESP-IDF versions 3.3.2 and 4.0.1. If anything changes in the -future the `do-esp32.sh` script can be used to generate an esp32-specific version. - -### How-to - -- To build MicroPython all that is needed is to include the `mp_mbedtls_errors.c` into the build - (the Makefiles do this automatically). Note that Perl is not needed for routine MicroPython - builds. -- When a new version of Mbedtls is pulled-in the `do-mp.sh` script should be run to - re-generate `mp_mbedtls_errors.c`. -- The `tester` app should be run if changes to the string handling in `error.fmt` are made: - it tests that there is not an off-by-one error in the string copying/appending, etc. -- To include `mbedtls_strerror` error strings define `MBEDTLS_ERROR_C` in the build. diff --git a/lib/mbedtls_errors/do-esp32.sh b/lib/mbedtls_errors/do-esp32.sh deleted file mode 100755 index 6fd4682415..0000000000 --- a/lib/mbedtls_errors/do-esp32.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /bin/bash -e -# Generate esp32_mbedtls_errors.c for use in the Esp32 port, with the ESP-IDF version of mbedtls -# The IDF_PATH env var must be set to the top-level dir of ESPIDF -echo "IDF_PATH=$IDF_PATH" -MBEDTLS=$IDF_PATH/components/mbedtls/mbedtls -patch -o esp32_generate_errors.pl $MBEDTLS/scripts/generate_errors.pl -#endif - -#if defined(MBEDTLS_PLATFORM_C) -#include "mbedtls/platform.h" -#else -#define mbedtls_snprintf snprintf -#define mbedtls_time_t time_t -#endif - -#if defined(MBEDTLS_ERROR_C) - -#include - -HEADER_INCLUDED - -// Error code table type -struct ssl_errs { - int16_t errnum; - const char *errstr; -}; - -// Table of high level error codes -static const struct ssl_errs mbedtls_high_level_error_tab[] = { -// BEGIN generated code -HIGH_LEVEL_CODE_CHECKS -// END generated code -}; - -static const struct ssl_errs mbedtls_low_level_error_tab[] = { -// Low level error codes -// -// BEGIN generated code -LOW_LEVEL_CODE_CHECKS -// END generated code -}; - -static const char *mbedtls_err_prefix = "MBEDTLS_ERR_"; -#define MBEDTLS_ERR_PREFIX_LEN ( sizeof("MBEDTLS_ERR_")-1 ) - -// copy error text into buffer, ensure null termination, return strlen of result -static size_t mbedtls_err_to_str(int err, const struct ssl_errs tab[], int tab_len, char *buf, size_t buflen) { - if (buflen == 0) return 0; - - // prefix for all error names - strncpy(buf, mbedtls_err_prefix, buflen); - if (buflen <= MBEDTLS_ERR_PREFIX_LEN+1) { - buf[buflen-1] = 0; - return buflen-1; - } - - // append error name from table - for (int i = 0; i < tab_len; i++) { - if (tab[i].errnum == err) { - strncpy(buf+MBEDTLS_ERR_PREFIX_LEN, tab[i].errstr, buflen-MBEDTLS_ERR_PREFIX_LEN); - buf[buflen-1] = 0; - return strlen(buf); - } - } - - mbedtls_snprintf(buf+MBEDTLS_ERR_PREFIX_LEN, buflen-MBEDTLS_ERR_PREFIX_LEN, "UNKNOWN (0x%04X)", - err); - return strlen(buf); -} - -#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) - -void mbedtls_strerror(int ret, char *buf, size_t buflen) { - int use_ret; - - if (buflen == 0) return; - - buf[buflen-1] = 0; - - if (ret < 0) ret = -ret; - - // - // High-level error codes - // - uint8_t got_hl = (ret & 0xFF80) != 0; - if (got_hl) { - use_ret = ret & 0xFF80; - - // special case -#if defined(MBEDTLS_SSL_TLS_C) - if (use_ret == -(MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE)) { - strncpy(buf, "MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE", buflen); - buf[buflen-1] = 0; - return; - } -#endif - - size_t len = mbedtls_err_to_str(use_ret, mbedtls_high_level_error_tab, - ARRAY_SIZE(mbedtls_high_level_error_tab), buf, buflen); - - buf += len; - buflen -= len; - if (buflen == 0) return; - } - - // - // Low-level error codes - // - use_ret = ret & ~0xFF80; - - if (use_ret == 0) return; - - // If high level code is present, make a concatenation between both error strings. - if (got_hl) { - if (buflen < 2) return; - *buf++ = '+'; - buflen--; - } - - mbedtls_err_to_str(use_ret, mbedtls_low_level_error_tab, - ARRAY_SIZE(mbedtls_low_level_error_tab), buf, buflen); -} - -#else /* MBEDTLS_ERROR_C */ - -#if defined(MBEDTLS_ERROR_STRERROR_DUMMY) - -/* - * Provide an non-function in case MBEDTLS_ERROR_C is not defined - */ -void mbedtls_strerror( int ret, char *buf, size_t buflen ) -{ - ((void) ret); - - if( buflen > 0 ) - buf[0] = '\0'; -} - -#endif /* MBEDTLS_ERROR_STRERROR_DUMMY */ - -#endif /* MBEDTLS_ERROR_C */ diff --git a/lib/mbedtls_errors/generate_errors.diff b/lib/mbedtls_errors/generate_errors.diff deleted file mode 100644 index ad24c372fa..0000000000 --- a/lib/mbedtls_errors/generate_errors.diff +++ /dev/null @@ -1,22 +0,0 @@ ---- generate_errors_orig.pl 2020-06-20 08:40:38.819060379 -0700 -+++ generate_errors.pl 2020-06-20 08:47:26.511163591 -0700 -@@ -162,16 +162,12 @@ - - if ($error_name eq "MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE") - { -- ${$code_check} .= "${white_space}if( use_ret == -($error_name) )\n". -- "${white_space}\{\n". -- "${white_space} mbedtls_snprintf( buf, buflen, \"$module_name - $description\" );\n". -- "${white_space} return;\n". -- "${white_space}}\n" -+ # no-op, this case is hard-coded in error.fmt - } - else - { -- ${$code_check} .= "${white_space}if( use_ret == -($error_name) )\n". -- "${white_space} mbedtls_snprintf( buf, buflen, \"$module_name - $description\" );\n" -+ my $error_text = $error_name =~ s/^MBEDTLS_ERR_//r; -+ ${$code_check} .= "${white_space}{ -($error_name), \"$error_text\" },\n" - } - }; - diff --git a/lib/mbedtls_errors/mp_mbedtls_errors.c b/lib/mbedtls_errors/mp_mbedtls_errors.c deleted file mode 100644 index 03a91f0dc9..0000000000 --- a/lib/mbedtls_errors/mp_mbedtls_errors.c +++ /dev/null @@ -1,705 +0,0 @@ -/* - * Error message information - * - * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * This file is part of mbed TLS (https://tls.mbed.org) - */ - -#if !defined(MBEDTLS_CONFIG_FILE) -#include "mbedtls/config.h" -#else -#include MBEDTLS_CONFIG_FILE -#endif - -#if defined(MBEDTLS_ERROR_C) || defined(MBEDTLS_ERROR_STRERROR_DUMMY) -#include "mbedtls/error.h" -#include -#endif - -#if defined(MBEDTLS_PLATFORM_C) -#include "mbedtls/platform.h" -#else -#define mbedtls_snprintf snprintf -#define mbedtls_time_t time_t -#endif - -#if defined(MBEDTLS_ERROR_C) - -#include - -#if defined(MBEDTLS_AES_C) -#include "mbedtls/aes.h" -#endif - -#if defined(MBEDTLS_ARC4_C) -#include "mbedtls/arc4.h" -#endif - -#if defined(MBEDTLS_ARIA_C) -#include "mbedtls/aria.h" -#endif - -#if defined(MBEDTLS_BASE64_C) -#include "mbedtls/base64.h" -#endif - -#if defined(MBEDTLS_BIGNUM_C) -#include "mbedtls/bignum.h" -#endif - -#if defined(MBEDTLS_BLOWFISH_C) -#include "mbedtls/blowfish.h" -#endif - -#if defined(MBEDTLS_CAMELLIA_C) -#include "mbedtls/camellia.h" -#endif - -#if defined(MBEDTLS_CCM_C) -#include "mbedtls/ccm.h" -#endif - -#if defined(MBEDTLS_CHACHA20_C) -#include "mbedtls/chacha20.h" -#endif - -#if defined(MBEDTLS_CHACHAPOLY_C) -#include "mbedtls/chachapoly.h" -#endif - -#if defined(MBEDTLS_CIPHER_C) -#include "mbedtls/cipher.h" -#endif - -#if defined(MBEDTLS_CMAC_C) -#include "mbedtls/cmac.h" -#endif - -#if defined(MBEDTLS_CTR_DRBG_C) -#include "mbedtls/ctr_drbg.h" -#endif - -#if defined(MBEDTLS_DES_C) -#include "mbedtls/des.h" -#endif - -#if defined(MBEDTLS_DHM_C) -#include "mbedtls/dhm.h" -#endif - -#if defined(MBEDTLS_ECP_C) -#include "mbedtls/ecp.h" -#endif - -#if defined(MBEDTLS_ENTROPY_C) -#include "mbedtls/entropy.h" -#endif - -#if defined(MBEDTLS_GCM_C) -#include "mbedtls/gcm.h" -#endif - -#if defined(MBEDTLS_HKDF_C) -#include "mbedtls/hkdf.h" -#endif - -#if defined(MBEDTLS_HMAC_DRBG_C) -#include "mbedtls/hmac_drbg.h" -#endif - -#if defined(MBEDTLS_MD_C) -#include "mbedtls/md.h" -#endif - -#if defined(MBEDTLS_MD2_C) -#include "mbedtls/md2.h" -#endif - -#if defined(MBEDTLS_MD4_C) -#include "mbedtls/md4.h" -#endif - -#if defined(MBEDTLS_MD5_C) -#include "mbedtls/md5.h" -#endif - -#if defined(MBEDTLS_NET_C) -#include "mbedtls/net_sockets.h" -#endif - -#if defined(MBEDTLS_OID_C) -#include "mbedtls/oid.h" -#endif - -#if defined(MBEDTLS_PADLOCK_C) -#include "mbedtls/padlock.h" -#endif - -#if defined(MBEDTLS_PEM_PARSE_C) || defined(MBEDTLS_PEM_WRITE_C) -#include "mbedtls/pem.h" -#endif - -#if defined(MBEDTLS_PK_C) -#include "mbedtls/pk.h" -#endif - -#if defined(MBEDTLS_PKCS12_C) -#include "mbedtls/pkcs12.h" -#endif - -#if defined(MBEDTLS_PKCS5_C) -#include "mbedtls/pkcs5.h" -#endif - -#if defined(MBEDTLS_PLATFORM_C) -#include "mbedtls/platform.h" -#endif - -#if defined(MBEDTLS_POLY1305_C) -#include "mbedtls/poly1305.h" -#endif - -#if defined(MBEDTLS_RIPEMD160_C) -#include "mbedtls/ripemd160.h" -#endif - -#if defined(MBEDTLS_RSA_C) -#include "mbedtls/rsa.h" -#endif - -#if defined(MBEDTLS_SHA1_C) -#include "mbedtls/sha1.h" -#endif - -#if defined(MBEDTLS_SHA256_C) -#include "mbedtls/sha256.h" -#endif - -#if defined(MBEDTLS_SHA512_C) -#include "mbedtls/sha512.h" -#endif - -#if defined(MBEDTLS_SSL_TLS_C) -#include "mbedtls/ssl.h" -#endif - -#if defined(MBEDTLS_THREADING_C) -#include "mbedtls/threading.h" -#endif - -#if defined(MBEDTLS_X509_USE_C) || defined(MBEDTLS_X509_CREATE_C) -#include "mbedtls/x509.h" -#endif - -#if defined(MBEDTLS_XTEA_C) -#include "mbedtls/xtea.h" -#endif - - -// Error code table type -struct ssl_errs { - int16_t errnum; - const char *errstr; -}; - -// Table of high level error codes -static const struct ssl_errs mbedtls_high_level_error_tab[] = { -// BEGIN generated code -#if defined(MBEDTLS_CIPHER_C) - { -(MBEDTLS_ERR_CIPHER_FEATURE_UNAVAILABLE), "CIPHER_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA), "CIPHER_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_CIPHER_ALLOC_FAILED), "CIPHER_ALLOC_FAILED" }, - { -(MBEDTLS_ERR_CIPHER_INVALID_PADDING), "CIPHER_INVALID_PADDING" }, - { -(MBEDTLS_ERR_CIPHER_FULL_BLOCK_EXPECTED), "CIPHER_FULL_BLOCK_EXPECTED" }, - { -(MBEDTLS_ERR_CIPHER_AUTH_FAILED), "CIPHER_AUTH_FAILED" }, - { -(MBEDTLS_ERR_CIPHER_INVALID_CONTEXT), "CIPHER_INVALID_CONTEXT" }, - { -(MBEDTLS_ERR_CIPHER_HW_ACCEL_FAILED), "CIPHER_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_CIPHER_C */ - -#if defined(MBEDTLS_DHM_C) - { -(MBEDTLS_ERR_DHM_BAD_INPUT_DATA), "DHM_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_DHM_READ_PARAMS_FAILED), "DHM_READ_PARAMS_FAILED" }, - { -(MBEDTLS_ERR_DHM_MAKE_PARAMS_FAILED), "DHM_MAKE_PARAMS_FAILED" }, - { -(MBEDTLS_ERR_DHM_READ_PUBLIC_FAILED), "DHM_READ_PUBLIC_FAILED" }, - { -(MBEDTLS_ERR_DHM_MAKE_PUBLIC_FAILED), "DHM_MAKE_PUBLIC_FAILED" }, - { -(MBEDTLS_ERR_DHM_CALC_SECRET_FAILED), "DHM_CALC_SECRET_FAILED" }, - { -(MBEDTLS_ERR_DHM_INVALID_FORMAT), "DHM_INVALID_FORMAT" }, - { -(MBEDTLS_ERR_DHM_ALLOC_FAILED), "DHM_ALLOC_FAILED" }, - { -(MBEDTLS_ERR_DHM_FILE_IO_ERROR), "DHM_FILE_IO_ERROR" }, - { -(MBEDTLS_ERR_DHM_HW_ACCEL_FAILED), "DHM_HW_ACCEL_FAILED" }, - { -(MBEDTLS_ERR_DHM_SET_GROUP_FAILED), "DHM_SET_GROUP_FAILED" }, -#endif /* MBEDTLS_DHM_C */ - -#if defined(MBEDTLS_ECP_C) - { -(MBEDTLS_ERR_ECP_BAD_INPUT_DATA), "ECP_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_ECP_BUFFER_TOO_SMALL), "ECP_BUFFER_TOO_SMALL" }, - { -(MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE), "ECP_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_ECP_VERIFY_FAILED), "ECP_VERIFY_FAILED" }, - { -(MBEDTLS_ERR_ECP_ALLOC_FAILED), "ECP_ALLOC_FAILED" }, - { -(MBEDTLS_ERR_ECP_RANDOM_FAILED), "ECP_RANDOM_FAILED" }, - { -(MBEDTLS_ERR_ECP_INVALID_KEY), "ECP_INVALID_KEY" }, - { -(MBEDTLS_ERR_ECP_SIG_LEN_MISMATCH), "ECP_SIG_LEN_MISMATCH" }, - { -(MBEDTLS_ERR_ECP_HW_ACCEL_FAILED), "ECP_HW_ACCEL_FAILED" }, - { -(MBEDTLS_ERR_ECP_IN_PROGRESS), "ECP_IN_PROGRESS" }, -#endif /* MBEDTLS_ECP_C */ - -#if defined(MBEDTLS_MD_C) - { -(MBEDTLS_ERR_MD_FEATURE_UNAVAILABLE), "MD_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_MD_BAD_INPUT_DATA), "MD_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_MD_ALLOC_FAILED), "MD_ALLOC_FAILED" }, - { -(MBEDTLS_ERR_MD_FILE_IO_ERROR), "MD_FILE_IO_ERROR" }, - { -(MBEDTLS_ERR_MD_HW_ACCEL_FAILED), "MD_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_MD_C */ - -#if defined(MBEDTLS_PEM_PARSE_C) || defined(MBEDTLS_PEM_WRITE_C) - { -(MBEDTLS_ERR_PEM_NO_HEADER_FOOTER_PRESENT), "PEM_NO_HEADER_FOOTER_PRESENT" }, - { -(MBEDTLS_ERR_PEM_INVALID_DATA), "PEM_INVALID_DATA" }, - { -(MBEDTLS_ERR_PEM_ALLOC_FAILED), "PEM_ALLOC_FAILED" }, - { -(MBEDTLS_ERR_PEM_INVALID_ENC_IV), "PEM_INVALID_ENC_IV" }, - { -(MBEDTLS_ERR_PEM_UNKNOWN_ENC_ALG), "PEM_UNKNOWN_ENC_ALG" }, - { -(MBEDTLS_ERR_PEM_PASSWORD_REQUIRED), "PEM_PASSWORD_REQUIRED" }, - { -(MBEDTLS_ERR_PEM_PASSWORD_MISMATCH), "PEM_PASSWORD_MISMATCH" }, - { -(MBEDTLS_ERR_PEM_FEATURE_UNAVAILABLE), "PEM_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_PEM_BAD_INPUT_DATA), "PEM_BAD_INPUT_DATA" }, -#endif /* MBEDTLS_PEM_PARSE_C || MBEDTLS_PEM_WRITE_C */ - -#if defined(MBEDTLS_PK_C) - { -(MBEDTLS_ERR_PK_ALLOC_FAILED), "PK_ALLOC_FAILED" }, - { -(MBEDTLS_ERR_PK_TYPE_MISMATCH), "PK_TYPE_MISMATCH" }, - { -(MBEDTLS_ERR_PK_BAD_INPUT_DATA), "PK_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_PK_FILE_IO_ERROR), "PK_FILE_IO_ERROR" }, - { -(MBEDTLS_ERR_PK_KEY_INVALID_VERSION), "PK_KEY_INVALID_VERSION" }, - { -(MBEDTLS_ERR_PK_KEY_INVALID_FORMAT), "PK_KEY_INVALID_FORMAT" }, - { -(MBEDTLS_ERR_PK_UNKNOWN_PK_ALG), "PK_UNKNOWN_PK_ALG" }, - { -(MBEDTLS_ERR_PK_PASSWORD_REQUIRED), "PK_PASSWORD_REQUIRED" }, - { -(MBEDTLS_ERR_PK_PASSWORD_MISMATCH), "PK_PASSWORD_MISMATCH" }, - { -(MBEDTLS_ERR_PK_INVALID_PUBKEY), "PK_INVALID_PUBKEY" }, - { -(MBEDTLS_ERR_PK_INVALID_ALG), "PK_INVALID_ALG" }, - { -(MBEDTLS_ERR_PK_UNKNOWN_NAMED_CURVE), "PK_UNKNOWN_NAMED_CURVE" }, - { -(MBEDTLS_ERR_PK_FEATURE_UNAVAILABLE), "PK_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_PK_SIG_LEN_MISMATCH), "PK_SIG_LEN_MISMATCH" }, - { -(MBEDTLS_ERR_PK_HW_ACCEL_FAILED), "PK_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_PK_C */ - -#if defined(MBEDTLS_PKCS12_C) - { -(MBEDTLS_ERR_PKCS12_BAD_INPUT_DATA), "PKCS12_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_PKCS12_FEATURE_UNAVAILABLE), "PKCS12_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_PKCS12_PBE_INVALID_FORMAT), "PKCS12_PBE_INVALID_FORMAT" }, - { -(MBEDTLS_ERR_PKCS12_PASSWORD_MISMATCH), "PKCS12_PASSWORD_MISMATCH" }, -#endif /* MBEDTLS_PKCS12_C */ - -#if defined(MBEDTLS_PKCS5_C) - { -(MBEDTLS_ERR_PKCS5_BAD_INPUT_DATA), "PKCS5_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_PKCS5_INVALID_FORMAT), "PKCS5_INVALID_FORMAT" }, - { -(MBEDTLS_ERR_PKCS5_FEATURE_UNAVAILABLE), "PKCS5_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_PKCS5_PASSWORD_MISMATCH), "PKCS5_PASSWORD_MISMATCH" }, -#endif /* MBEDTLS_PKCS5_C */ - -#if defined(MBEDTLS_RSA_C) - { -(MBEDTLS_ERR_RSA_BAD_INPUT_DATA), "RSA_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_RSA_INVALID_PADDING), "RSA_INVALID_PADDING" }, - { -(MBEDTLS_ERR_RSA_KEY_GEN_FAILED), "RSA_KEY_GEN_FAILED" }, - { -(MBEDTLS_ERR_RSA_KEY_CHECK_FAILED), "RSA_KEY_CHECK_FAILED" }, - { -(MBEDTLS_ERR_RSA_PUBLIC_FAILED), "RSA_PUBLIC_FAILED" }, - { -(MBEDTLS_ERR_RSA_PRIVATE_FAILED), "RSA_PRIVATE_FAILED" }, - { -(MBEDTLS_ERR_RSA_VERIFY_FAILED), "RSA_VERIFY_FAILED" }, - { -(MBEDTLS_ERR_RSA_OUTPUT_TOO_LARGE), "RSA_OUTPUT_TOO_LARGE" }, - { -(MBEDTLS_ERR_RSA_RNG_FAILED), "RSA_RNG_FAILED" }, - { -(MBEDTLS_ERR_RSA_UNSUPPORTED_OPERATION), "RSA_UNSUPPORTED_OPERATION" }, - { -(MBEDTLS_ERR_RSA_HW_ACCEL_FAILED), "RSA_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_RSA_C */ - -#if defined(MBEDTLS_SSL_TLS_C) - { -(MBEDTLS_ERR_SSL_FEATURE_UNAVAILABLE), "SSL_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_SSL_BAD_INPUT_DATA), "SSL_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_SSL_INVALID_MAC), "SSL_INVALID_MAC" }, - { -(MBEDTLS_ERR_SSL_INVALID_RECORD), "SSL_INVALID_RECORD" }, - { -(MBEDTLS_ERR_SSL_CONN_EOF), "SSL_CONN_EOF" }, - { -(MBEDTLS_ERR_SSL_UNKNOWN_CIPHER), "SSL_UNKNOWN_CIPHER" }, - { -(MBEDTLS_ERR_SSL_NO_CIPHER_CHOSEN), "SSL_NO_CIPHER_CHOSEN" }, - { -(MBEDTLS_ERR_SSL_NO_RNG), "SSL_NO_RNG" }, - { -(MBEDTLS_ERR_SSL_NO_CLIENT_CERTIFICATE), "SSL_NO_CLIENT_CERTIFICATE" }, - { -(MBEDTLS_ERR_SSL_CERTIFICATE_TOO_LARGE), "SSL_CERTIFICATE_TOO_LARGE" }, - { -(MBEDTLS_ERR_SSL_CERTIFICATE_REQUIRED), "SSL_CERTIFICATE_REQUIRED" }, - { -(MBEDTLS_ERR_SSL_PRIVATE_KEY_REQUIRED), "SSL_PRIVATE_KEY_REQUIRED" }, - { -(MBEDTLS_ERR_SSL_CA_CHAIN_REQUIRED), "SSL_CA_CHAIN_REQUIRED" }, - { -(MBEDTLS_ERR_SSL_UNEXPECTED_MESSAGE), "SSL_UNEXPECTED_MESSAGE" }, - { -(MBEDTLS_ERR_SSL_PEER_VERIFY_FAILED), "SSL_PEER_VERIFY_FAILED" }, - { -(MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY), "SSL_PEER_CLOSE_NOTIFY" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_CLIENT_HELLO), "SSL_BAD_HS_CLIENT_HELLO" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_SERVER_HELLO), "SSL_BAD_HS_SERVER_HELLO" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_CERTIFICATE), "SSL_BAD_HS_CERTIFICATE" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_CERTIFICATE_REQUEST), "SSL_BAD_HS_CERTIFICATE_REQUEST" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_SERVER_KEY_EXCHANGE), "SSL_BAD_HS_SERVER_KEY_EXCHANGE" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_SERVER_HELLO_DONE), "SSL_BAD_HS_SERVER_HELLO_DONE" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_CLIENT_KEY_EXCHANGE), "SSL_BAD_HS_CLIENT_KEY_EXCHANGE" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_CLIENT_KEY_EXCHANGE_RP), "SSL_BAD_HS_CLIENT_KEY_EXCHANGE_RP" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_CLIENT_KEY_EXCHANGE_CS), "SSL_BAD_HS_CLIENT_KEY_EXCHANGE_CS" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_CERTIFICATE_VERIFY), "SSL_BAD_HS_CERTIFICATE_VERIFY" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_CHANGE_CIPHER_SPEC), "SSL_BAD_HS_CHANGE_CIPHER_SPEC" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_FINISHED), "SSL_BAD_HS_FINISHED" }, - { -(MBEDTLS_ERR_SSL_ALLOC_FAILED), "SSL_ALLOC_FAILED" }, - { -(MBEDTLS_ERR_SSL_HW_ACCEL_FAILED), "SSL_HW_ACCEL_FAILED" }, - { -(MBEDTLS_ERR_SSL_HW_ACCEL_FALLTHROUGH), "SSL_HW_ACCEL_FALLTHROUGH" }, - { -(MBEDTLS_ERR_SSL_COMPRESSION_FAILED), "SSL_COMPRESSION_FAILED" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_PROTOCOL_VERSION), "SSL_BAD_HS_PROTOCOL_VERSION" }, - { -(MBEDTLS_ERR_SSL_BAD_HS_NEW_SESSION_TICKET), "SSL_BAD_HS_NEW_SESSION_TICKET" }, - { -(MBEDTLS_ERR_SSL_SESSION_TICKET_EXPIRED), "SSL_SESSION_TICKET_EXPIRED" }, - { -(MBEDTLS_ERR_SSL_PK_TYPE_MISMATCH), "SSL_PK_TYPE_MISMATCH" }, - { -(MBEDTLS_ERR_SSL_UNKNOWN_IDENTITY), "SSL_UNKNOWN_IDENTITY" }, - { -(MBEDTLS_ERR_SSL_INTERNAL_ERROR), "SSL_INTERNAL_ERROR" }, - { -(MBEDTLS_ERR_SSL_COUNTER_WRAPPING), "SSL_COUNTER_WRAPPING" }, - { -(MBEDTLS_ERR_SSL_WAITING_SERVER_HELLO_RENEGO), "SSL_WAITING_SERVER_HELLO_RENEGO" }, - { -(MBEDTLS_ERR_SSL_HELLO_VERIFY_REQUIRED), "SSL_HELLO_VERIFY_REQUIRED" }, - { -(MBEDTLS_ERR_SSL_BUFFER_TOO_SMALL), "SSL_BUFFER_TOO_SMALL" }, - { -(MBEDTLS_ERR_SSL_NO_USABLE_CIPHERSUITE), "SSL_NO_USABLE_CIPHERSUITE" }, - { -(MBEDTLS_ERR_SSL_WANT_READ), "SSL_WANT_READ" }, - { -(MBEDTLS_ERR_SSL_WANT_WRITE), "SSL_WANT_WRITE" }, - { -(MBEDTLS_ERR_SSL_TIMEOUT), "SSL_TIMEOUT" }, - { -(MBEDTLS_ERR_SSL_CLIENT_RECONNECT), "SSL_CLIENT_RECONNECT" }, - { -(MBEDTLS_ERR_SSL_UNEXPECTED_RECORD), "SSL_UNEXPECTED_RECORD" }, - { -(MBEDTLS_ERR_SSL_NON_FATAL), "SSL_NON_FATAL" }, - { -(MBEDTLS_ERR_SSL_INVALID_VERIFY_HASH), "SSL_INVALID_VERIFY_HASH" }, - { -(MBEDTLS_ERR_SSL_CONTINUE_PROCESSING), "SSL_CONTINUE_PROCESSING" }, - { -(MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS), "SSL_ASYNC_IN_PROGRESS" }, - { -(MBEDTLS_ERR_SSL_EARLY_MESSAGE), "SSL_EARLY_MESSAGE" }, - { -(MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS), "SSL_CRYPTO_IN_PROGRESS" }, -#endif /* MBEDTLS_SSL_TLS_C */ - -#if defined(MBEDTLS_X509_USE_C) || defined(MBEDTLS_X509_CREATE_C) - { -(MBEDTLS_ERR_X509_FEATURE_UNAVAILABLE), "X509_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_X509_UNKNOWN_OID), "X509_UNKNOWN_OID" }, - { -(MBEDTLS_ERR_X509_INVALID_FORMAT), "X509_INVALID_FORMAT" }, - { -(MBEDTLS_ERR_X509_INVALID_VERSION), "X509_INVALID_VERSION" }, - { -(MBEDTLS_ERR_X509_INVALID_SERIAL), "X509_INVALID_SERIAL" }, - { -(MBEDTLS_ERR_X509_INVALID_ALG), "X509_INVALID_ALG" }, - { -(MBEDTLS_ERR_X509_INVALID_NAME), "X509_INVALID_NAME" }, - { -(MBEDTLS_ERR_X509_INVALID_DATE), "X509_INVALID_DATE" }, - { -(MBEDTLS_ERR_X509_INVALID_SIGNATURE), "X509_INVALID_SIGNATURE" }, - { -(MBEDTLS_ERR_X509_INVALID_EXTENSIONS), "X509_INVALID_EXTENSIONS" }, - { -(MBEDTLS_ERR_X509_UNKNOWN_VERSION), "X509_UNKNOWN_VERSION" }, - { -(MBEDTLS_ERR_X509_UNKNOWN_SIG_ALG), "X509_UNKNOWN_SIG_ALG" }, - { -(MBEDTLS_ERR_X509_SIG_MISMATCH), "X509_SIG_MISMATCH" }, - { -(MBEDTLS_ERR_X509_CERT_VERIFY_FAILED), "X509_CERT_VERIFY_FAILED" }, - { -(MBEDTLS_ERR_X509_CERT_UNKNOWN_FORMAT), "X509_CERT_UNKNOWN_FORMAT" }, - { -(MBEDTLS_ERR_X509_BAD_INPUT_DATA), "X509_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_X509_ALLOC_FAILED), "X509_ALLOC_FAILED" }, - { -(MBEDTLS_ERR_X509_FILE_IO_ERROR), "X509_FILE_IO_ERROR" }, - { -(MBEDTLS_ERR_X509_BUFFER_TOO_SMALL), "X509_BUFFER_TOO_SMALL" }, - { -(MBEDTLS_ERR_X509_FATAL_ERROR), "X509_FATAL_ERROR" }, -#endif /* MBEDTLS_X509_USE_C || MBEDTLS_X509_CREATE_C */ -// END generated code -}; - -static const struct ssl_errs mbedtls_low_level_error_tab[] = { -// Low level error codes -// -// BEGIN generated code -#if defined(MBEDTLS_AES_C) - { -(MBEDTLS_ERR_AES_INVALID_KEY_LENGTH), "AES_INVALID_KEY_LENGTH" }, - { -(MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH), "AES_INVALID_INPUT_LENGTH" }, - { -(MBEDTLS_ERR_AES_BAD_INPUT_DATA), "AES_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_AES_FEATURE_UNAVAILABLE), "AES_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_AES_HW_ACCEL_FAILED), "AES_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_AES_C */ - -#if defined(MBEDTLS_ARC4_C) - { -(MBEDTLS_ERR_ARC4_HW_ACCEL_FAILED), "ARC4_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_ARC4_C */ - -#if defined(MBEDTLS_ARIA_C) - { -(MBEDTLS_ERR_ARIA_BAD_INPUT_DATA), "ARIA_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_ARIA_INVALID_INPUT_LENGTH), "ARIA_INVALID_INPUT_LENGTH" }, - { -(MBEDTLS_ERR_ARIA_FEATURE_UNAVAILABLE), "ARIA_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_ARIA_HW_ACCEL_FAILED), "ARIA_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_ARIA_C */ - -#if defined(MBEDTLS_ASN1_PARSE_C) - { -(MBEDTLS_ERR_ASN1_OUT_OF_DATA), "ASN1_OUT_OF_DATA" }, - { -(MBEDTLS_ERR_ASN1_UNEXPECTED_TAG), "ASN1_UNEXPECTED_TAG" }, - { -(MBEDTLS_ERR_ASN1_INVALID_LENGTH), "ASN1_INVALID_LENGTH" }, - { -(MBEDTLS_ERR_ASN1_LENGTH_MISMATCH), "ASN1_LENGTH_MISMATCH" }, - { -(MBEDTLS_ERR_ASN1_INVALID_DATA), "ASN1_INVALID_DATA" }, - { -(MBEDTLS_ERR_ASN1_ALLOC_FAILED), "ASN1_ALLOC_FAILED" }, - { -(MBEDTLS_ERR_ASN1_BUF_TOO_SMALL), "ASN1_BUF_TOO_SMALL" }, -#endif /* MBEDTLS_ASN1_PARSE_C */ - -#if defined(MBEDTLS_BASE64_C) - { -(MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL), "BASE64_BUFFER_TOO_SMALL" }, - { -(MBEDTLS_ERR_BASE64_INVALID_CHARACTER), "BASE64_INVALID_CHARACTER" }, -#endif /* MBEDTLS_BASE64_C */ - -#if defined(MBEDTLS_BIGNUM_C) - { -(MBEDTLS_ERR_MPI_FILE_IO_ERROR), "MPI_FILE_IO_ERROR" }, - { -(MBEDTLS_ERR_MPI_BAD_INPUT_DATA), "MPI_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_MPI_INVALID_CHARACTER), "MPI_INVALID_CHARACTER" }, - { -(MBEDTLS_ERR_MPI_BUFFER_TOO_SMALL), "MPI_BUFFER_TOO_SMALL" }, - { -(MBEDTLS_ERR_MPI_NEGATIVE_VALUE), "MPI_NEGATIVE_VALUE" }, - { -(MBEDTLS_ERR_MPI_DIVISION_BY_ZERO), "MPI_DIVISION_BY_ZERO" }, - { -(MBEDTLS_ERR_MPI_NOT_ACCEPTABLE), "MPI_NOT_ACCEPTABLE" }, - { -(MBEDTLS_ERR_MPI_ALLOC_FAILED), "MPI_ALLOC_FAILED" }, -#endif /* MBEDTLS_BIGNUM_C */ - -#if defined(MBEDTLS_BLOWFISH_C) - { -(MBEDTLS_ERR_BLOWFISH_BAD_INPUT_DATA), "BLOWFISH_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_BLOWFISH_INVALID_INPUT_LENGTH), "BLOWFISH_INVALID_INPUT_LENGTH" }, - { -(MBEDTLS_ERR_BLOWFISH_HW_ACCEL_FAILED), "BLOWFISH_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_BLOWFISH_C */ - -#if defined(MBEDTLS_CAMELLIA_C) - { -(MBEDTLS_ERR_CAMELLIA_BAD_INPUT_DATA), "CAMELLIA_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_CAMELLIA_INVALID_INPUT_LENGTH), "CAMELLIA_INVALID_INPUT_LENGTH" }, - { -(MBEDTLS_ERR_CAMELLIA_HW_ACCEL_FAILED), "CAMELLIA_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_CAMELLIA_C */ - -#if defined(MBEDTLS_CCM_C) - { -(MBEDTLS_ERR_CCM_BAD_INPUT), "CCM_BAD_INPUT" }, - { -(MBEDTLS_ERR_CCM_AUTH_FAILED), "CCM_AUTH_FAILED" }, - { -(MBEDTLS_ERR_CCM_HW_ACCEL_FAILED), "CCM_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_CCM_C */ - -#if defined(MBEDTLS_CHACHA20_C) - { -(MBEDTLS_ERR_CHACHA20_BAD_INPUT_DATA), "CHACHA20_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_CHACHA20_FEATURE_UNAVAILABLE), "CHACHA20_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_CHACHA20_HW_ACCEL_FAILED), "CHACHA20_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_CHACHA20_C */ - -#if defined(MBEDTLS_CHACHAPOLY_C) - { -(MBEDTLS_ERR_CHACHAPOLY_BAD_STATE), "CHACHAPOLY_BAD_STATE" }, - { -(MBEDTLS_ERR_CHACHAPOLY_AUTH_FAILED), "CHACHAPOLY_AUTH_FAILED" }, -#endif /* MBEDTLS_CHACHAPOLY_C */ - -#if defined(MBEDTLS_CMAC_C) - { -(MBEDTLS_ERR_CMAC_HW_ACCEL_FAILED), "CMAC_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_CMAC_C */ - -#if defined(MBEDTLS_CTR_DRBG_C) - { -(MBEDTLS_ERR_CTR_DRBG_ENTROPY_SOURCE_FAILED), "CTR_DRBG_ENTROPY_SOURCE_FAILED" }, - { -(MBEDTLS_ERR_CTR_DRBG_REQUEST_TOO_BIG), "CTR_DRBG_REQUEST_TOO_BIG" }, - { -(MBEDTLS_ERR_CTR_DRBG_INPUT_TOO_BIG), "CTR_DRBG_INPUT_TOO_BIG" }, - { -(MBEDTLS_ERR_CTR_DRBG_FILE_IO_ERROR), "CTR_DRBG_FILE_IO_ERROR" }, -#endif /* MBEDTLS_CTR_DRBG_C */ - -#if defined(MBEDTLS_DES_C) - { -(MBEDTLS_ERR_DES_INVALID_INPUT_LENGTH), "DES_INVALID_INPUT_LENGTH" }, - { -(MBEDTLS_ERR_DES_HW_ACCEL_FAILED), "DES_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_DES_C */ - -#if defined(MBEDTLS_ENTROPY_C) - { -(MBEDTLS_ERR_ENTROPY_SOURCE_FAILED), "ENTROPY_SOURCE_FAILED" }, - { -(MBEDTLS_ERR_ENTROPY_MAX_SOURCES), "ENTROPY_MAX_SOURCES" }, - { -(MBEDTLS_ERR_ENTROPY_NO_SOURCES_DEFINED), "ENTROPY_NO_SOURCES_DEFINED" }, - { -(MBEDTLS_ERR_ENTROPY_NO_STRONG_SOURCE), "ENTROPY_NO_STRONG_SOURCE" }, - { -(MBEDTLS_ERR_ENTROPY_FILE_IO_ERROR), "ENTROPY_FILE_IO_ERROR" }, -#endif /* MBEDTLS_ENTROPY_C */ - -#if defined(MBEDTLS_GCM_C) - { -(MBEDTLS_ERR_GCM_AUTH_FAILED), "GCM_AUTH_FAILED" }, - { -(MBEDTLS_ERR_GCM_HW_ACCEL_FAILED), "GCM_HW_ACCEL_FAILED" }, - { -(MBEDTLS_ERR_GCM_BAD_INPUT), "GCM_BAD_INPUT" }, -#endif /* MBEDTLS_GCM_C */ - -#if defined(MBEDTLS_HKDF_C) - { -(MBEDTLS_ERR_HKDF_BAD_INPUT_DATA), "HKDF_BAD_INPUT_DATA" }, -#endif /* MBEDTLS_HKDF_C */ - -#if defined(MBEDTLS_HMAC_DRBG_C) - { -(MBEDTLS_ERR_HMAC_DRBG_REQUEST_TOO_BIG), "HMAC_DRBG_REQUEST_TOO_BIG" }, - { -(MBEDTLS_ERR_HMAC_DRBG_INPUT_TOO_BIG), "HMAC_DRBG_INPUT_TOO_BIG" }, - { -(MBEDTLS_ERR_HMAC_DRBG_FILE_IO_ERROR), "HMAC_DRBG_FILE_IO_ERROR" }, - { -(MBEDTLS_ERR_HMAC_DRBG_ENTROPY_SOURCE_FAILED), "HMAC_DRBG_ENTROPY_SOURCE_FAILED" }, -#endif /* MBEDTLS_HMAC_DRBG_C */ - -#if defined(MBEDTLS_MD2_C) - { -(MBEDTLS_ERR_MD2_HW_ACCEL_FAILED), "MD2_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_MD2_C */ - -#if defined(MBEDTLS_MD4_C) - { -(MBEDTLS_ERR_MD4_HW_ACCEL_FAILED), "MD4_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_MD4_C */ - -#if defined(MBEDTLS_MD5_C) - { -(MBEDTLS_ERR_MD5_HW_ACCEL_FAILED), "MD5_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_MD5_C */ - -#if defined(MBEDTLS_NET_C) - { -(MBEDTLS_ERR_NET_SOCKET_FAILED), "NET_SOCKET_FAILED" }, - { -(MBEDTLS_ERR_NET_CONNECT_FAILED), "NET_CONNECT_FAILED" }, - { -(MBEDTLS_ERR_NET_BIND_FAILED), "NET_BIND_FAILED" }, - { -(MBEDTLS_ERR_NET_LISTEN_FAILED), "NET_LISTEN_FAILED" }, - { -(MBEDTLS_ERR_NET_ACCEPT_FAILED), "NET_ACCEPT_FAILED" }, - { -(MBEDTLS_ERR_NET_RECV_FAILED), "NET_RECV_FAILED" }, - { -(MBEDTLS_ERR_NET_SEND_FAILED), "NET_SEND_FAILED" }, - { -(MBEDTLS_ERR_NET_CONN_RESET), "NET_CONN_RESET" }, - { -(MBEDTLS_ERR_NET_UNKNOWN_HOST), "NET_UNKNOWN_HOST" }, - { -(MBEDTLS_ERR_NET_BUFFER_TOO_SMALL), "NET_BUFFER_TOO_SMALL" }, - { -(MBEDTLS_ERR_NET_INVALID_CONTEXT), "NET_INVALID_CONTEXT" }, - { -(MBEDTLS_ERR_NET_POLL_FAILED), "NET_POLL_FAILED" }, - { -(MBEDTLS_ERR_NET_BAD_INPUT_DATA), "NET_BAD_INPUT_DATA" }, -#endif /* MBEDTLS_NET_C */ - -#if defined(MBEDTLS_OID_C) - { -(MBEDTLS_ERR_OID_NOT_FOUND), "OID_NOT_FOUND" }, - { -(MBEDTLS_ERR_OID_BUF_TOO_SMALL), "OID_BUF_TOO_SMALL" }, -#endif /* MBEDTLS_OID_C */ - -#if defined(MBEDTLS_PADLOCK_C) - { -(MBEDTLS_ERR_PADLOCK_DATA_MISALIGNED), "PADLOCK_DATA_MISALIGNED" }, -#endif /* MBEDTLS_PADLOCK_C */ - -#if defined(MBEDTLS_PLATFORM_C) - { -(MBEDTLS_ERR_PLATFORM_HW_ACCEL_FAILED), "PLATFORM_HW_ACCEL_FAILED" }, - { -(MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED), "PLATFORM_FEATURE_UNSUPPORTED" }, -#endif /* MBEDTLS_PLATFORM_C */ - -#if defined(MBEDTLS_POLY1305_C) - { -(MBEDTLS_ERR_POLY1305_BAD_INPUT_DATA), "POLY1305_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_POLY1305_FEATURE_UNAVAILABLE), "POLY1305_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_POLY1305_HW_ACCEL_FAILED), "POLY1305_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_POLY1305_C */ - -#if defined(MBEDTLS_RIPEMD160_C) - { -(MBEDTLS_ERR_RIPEMD160_HW_ACCEL_FAILED), "RIPEMD160_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_RIPEMD160_C */ - -#if defined(MBEDTLS_SHA1_C) - { -(MBEDTLS_ERR_SHA1_HW_ACCEL_FAILED), "SHA1_HW_ACCEL_FAILED" }, - { -(MBEDTLS_ERR_SHA1_BAD_INPUT_DATA), "SHA1_BAD_INPUT_DATA" }, -#endif /* MBEDTLS_SHA1_C */ - -#if defined(MBEDTLS_SHA256_C) - { -(MBEDTLS_ERR_SHA256_HW_ACCEL_FAILED), "SHA256_HW_ACCEL_FAILED" }, - { -(MBEDTLS_ERR_SHA256_BAD_INPUT_DATA), "SHA256_BAD_INPUT_DATA" }, -#endif /* MBEDTLS_SHA256_C */ - -#if defined(MBEDTLS_SHA512_C) - { -(MBEDTLS_ERR_SHA512_HW_ACCEL_FAILED), "SHA512_HW_ACCEL_FAILED" }, - { -(MBEDTLS_ERR_SHA512_BAD_INPUT_DATA), "SHA512_BAD_INPUT_DATA" }, -#endif /* MBEDTLS_SHA512_C */ - -#if defined(MBEDTLS_THREADING_C) - { -(MBEDTLS_ERR_THREADING_FEATURE_UNAVAILABLE), "THREADING_FEATURE_UNAVAILABLE" }, - { -(MBEDTLS_ERR_THREADING_BAD_INPUT_DATA), "THREADING_BAD_INPUT_DATA" }, - { -(MBEDTLS_ERR_THREADING_MUTEX_ERROR), "THREADING_MUTEX_ERROR" }, -#endif /* MBEDTLS_THREADING_C */ - -#if defined(MBEDTLS_XTEA_C) - { -(MBEDTLS_ERR_XTEA_INVALID_INPUT_LENGTH), "XTEA_INVALID_INPUT_LENGTH" }, - { -(MBEDTLS_ERR_XTEA_HW_ACCEL_FAILED), "XTEA_HW_ACCEL_FAILED" }, -#endif /* MBEDTLS_XTEA_C */ -// END generated code -}; - -static const char *mbedtls_err_prefix = "MBEDTLS_ERR_"; -#define MBEDTLS_ERR_PREFIX_LEN ( sizeof("MBEDTLS_ERR_")-1 ) - -// copy error text into buffer, ensure null termination, return strlen of result -static size_t mbedtls_err_to_str(int err, const struct ssl_errs tab[], int tab_len, char *buf, size_t buflen) { - if (buflen == 0) return 0; - - // prefix for all error names - strncpy(buf, mbedtls_err_prefix, buflen); - if (buflen <= MBEDTLS_ERR_PREFIX_LEN+1) { - buf[buflen-1] = 0; - return buflen-1; - } - - // append error name from table - for (int i = 0; i < tab_len; i++) { - if (tab[i].errnum == err) { - strncpy(buf+MBEDTLS_ERR_PREFIX_LEN, tab[i].errstr, buflen-MBEDTLS_ERR_PREFIX_LEN); - buf[buflen-1] = 0; - return strlen(buf); - } - } - - mbedtls_snprintf(buf+MBEDTLS_ERR_PREFIX_LEN, buflen-MBEDTLS_ERR_PREFIX_LEN, "UNKNOWN (0x%04X)", - err); - return strlen(buf); -} - -#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) - -void mbedtls_strerror(int ret, char *buf, size_t buflen) { - int use_ret; - - if (buflen == 0) return; - - buf[buflen-1] = 0; - - if (ret < 0) ret = -ret; - - // - // High-level error codes - // - uint8_t got_hl = (ret & 0xFF80) != 0; - if (got_hl) { - use_ret = ret & 0xFF80; - - // special case -#if defined(MBEDTLS_SSL_TLS_C) - if (use_ret == -(MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE)) { - strncpy(buf, "MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE", buflen); - buf[buflen-1] = 0; - return; - } -#endif - - size_t len = mbedtls_err_to_str(use_ret, mbedtls_high_level_error_tab, - ARRAY_SIZE(mbedtls_high_level_error_tab), buf, buflen); - - buf += len; - buflen -= len; - if (buflen == 0) return; - } - - // - // Low-level error codes - // - use_ret = ret & ~0xFF80; - - if (use_ret == 0) return; - - // If high level code is present, make a concatenation between both error strings. - if (got_hl) { - if (buflen < 2) return; - *buf++ = '+'; - buflen--; - } - - mbedtls_err_to_str(use_ret, mbedtls_low_level_error_tab, - ARRAY_SIZE(mbedtls_low_level_error_tab), buf, buflen); -} - -#else /* MBEDTLS_ERROR_C */ - -#if defined(MBEDTLS_ERROR_STRERROR_DUMMY) - -/* - * Provide an non-function in case MBEDTLS_ERROR_C is not defined - */ -void mbedtls_strerror( int ret, char *buf, size_t buflen ) -{ - ((void) ret); - - if( buflen > 0 ) - buf[0] = '\0'; -} - -#endif /* MBEDTLS_ERROR_STRERROR_DUMMY */ - -#endif /* MBEDTLS_ERROR_C */ diff --git a/lib/mbedtls_errors/tester.c b/lib/mbedtls_errors/tester.c deleted file mode 100644 index 6f1c788f50..0000000000 --- a/lib/mbedtls_errors/tester.c +++ /dev/null @@ -1,58 +0,0 @@ -#include "mbedtls/error.h" -#include -#include - -// test_code checks that the provided code results in the provided error string for any size -// buffer. It calls mbedtls_strerror() to fill a buffer that is from 1 to 100 bytes in length -// and then checks that the buffer contents is OK and that a few guard bytes before and after -// the buffer were not overwritten. -int test_code(int code, char *str) { - char buf[100]; - int ok = 1; - int res; - - // test zero-length buffer - memset(buf, -3, 100); - mbedtls_strerror(code, buf + 4, 0); - for (int i = 0; i < 10; i++) { - if (buf[i] != -3) { - printf("Error: guard overwritten buflen=0 i=%d buf[i]=%d\n", i, buf[i]); - ok = 0; - } - } - - // test - for (size_t buflen = 1; buflen < 90; buflen++) { - memset(buf, -3, 100); - mbedtls_strerror(code, buf + 4, buflen); - for (int i = 0; i < 4; i++) { - if (buf[i] != -3) { - printf("Error: pre-guard overwritten buflen=%d i=%d buf[i]=%d\n", buflen, i, buf[i]); - ok = 0; - } - } - for (int i = 4 + buflen; i < 100; i++) { - if (buf[i] != -3) { - printf("Error: post-guard overwritten buflen=%d i=%d buf[i]=%d\n", buflen, i, buf[i]); - ok = 0; - } - } - char exp[100]; - strncpy(exp, str, buflen); - exp[buflen - 1] = 0; - if (strcmp(buf + 4, exp) != 0) { - printf("Error: expected %s, got %s\n", exp, buf); - ok = 0; - } - } - - printf("Test %x -> %s is %s\n", code, str, ok?"OK":"*** BAD ***"); -} - -int main() { - test_code(0x7200, "MBEDTLS_ERR_SSL_INVALID_RECORD"); - test_code(0x7780, "MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE"); - test_code(0x0074, "MBEDTLS_ERR_SHA256_BAD_INPUT_DATA"); - test_code(0x6600 | 0x0074, "MBEDTLS_ERR_SSL_INVALID_VERIFY_HASH+MBEDTLS_ERR_SHA256_BAD_INPUT_DATA"); - test_code(103, "MBEDTLS_ERR_UNKNOWN (0x0067)"); -} diff --git a/lib/micropython-lib b/lib/micropython-lib index e6b89eafa3..e025c843b6 160000 --- a/lib/micropython-lib +++ b/lib/micropython-lib @@ -1 +1 @@ -Subproject commit e6b89eafa3b86d2e8e405450377d459600a30cd6 +Subproject commit e025c843b60e93689f0f991d753010bb5bd6a722 diff --git a/lib/oofatfs/ff.c b/lib/oofatfs/ff.c index dbcfa3efc3..12790c04a3 100644 --- a/lib/oofatfs/ff.c +++ b/lib/oofatfs/ff.c @@ -2834,7 +2834,7 @@ static FRESULT create_name ( /* FR_OK: successful, FR_INVALID_NAME: could not if (di >= FF_MAX_LFN) return FR_INVALID_NAME; /* Reject too long name */ lfn[di++] = wc; /* Store the Unicode character */ } - while (*p == '/' || *p == '\\') p++; /* Skip duplicated separators if exist */ + if (wc == '/' || wc == '\\') while (*p == '/' || *p == '\\') p++; /* Skip duplicated separators if exist */ *path = p; /* Return pointer to the next segment */ cf = (wc < ' ') ? NS_LAST : 0; /* Set last segment flag if end of the path */ diff --git a/lib/uzlib/adler32.c b/lib/uzlib/adler32.c index 1f1759493b..65fe1181b8 100644 --- a/lib/uzlib/adler32.c +++ b/lib/uzlib/adler32.c @@ -36,7 +36,7 @@ * Copyright (C) 1995-1998 Jean-loup Gailly and Mark Adler */ -#include "tinf.h" +#include "uzlib.h" #define A32_BASE 65521 #define A32_NMAX 5552 diff --git a/lib/uzlib/crc32.c b/lib/uzlib/crc32.c index e24c643b6a..1e3b1756b3 100644 --- a/lib/uzlib/crc32.c +++ b/lib/uzlib/crc32.c @@ -36,7 +36,7 @@ * Copyright (C) 1995-1998 Jean-loup Gailly and Mark Adler */ -#include "tinf.h" +#include "uzlib.h" static const unsigned int tinf_crc32tab[16] = { 0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, diff --git a/lib/uzlib/defl_static.c b/lib/uzlib/defl_static.c new file mode 100644 index 0000000000..208e9ea5d2 --- /dev/null +++ b/lib/uzlib/defl_static.c @@ -0,0 +1,181 @@ +/* + +Routines in this file are based on: +Zlib (RFC1950 / RFC1951) compression for PuTTY. + +PuTTY is copyright 1997-2014 Simon Tatham. + +Portions copyright Robert de Bath, Joris van Rantwijk, Delian +Delchev, Andreas Schultz, Jeroen Massar, Wez Furlong, Nicolas Barry, +Justin Bradford, Ben Harris, Malcolm Smith, Ahmad Khalifa, Markus +Kuhn, Colin Watson, and CORE SDI S.A. + +Optimised for MicroPython: +Copyright (c) 2023 by Jim Mussared + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation files +(the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#include +#include +#include +#include + +/* ---------------------------------------------------------------------- + * Zlib compression. We always use the static Huffman tree option. + * Mostly this is because it's hard to scan a block in advance to + * work out better trees; dynamic trees are great when you're + * compressing a large file under no significant time constraint, + * but when you're compressing little bits in real time, things get + * hairier. + * + * I suppose it's possible that I could compute Huffman trees based + * on the frequencies in the _previous_ block, as a sort of + * heuristic, but I'm not confident that the gain would balance out + * having to transmit the trees. + */ + +static void outbits(uzlib_lz77_state_t *state, unsigned long bits, int nbits) +{ + assert(state->noutbits + nbits <= 32); + state->outbits |= bits << state->noutbits; + state->noutbits += nbits; + while (state->noutbits >= 8) { + state->dest_write_cb(state->dest_write_data, state->outbits & 0xFF); + state->outbits >>= 8; + state->noutbits -= 8; + } +} + +static const unsigned char mirrornibbles[16] = { + 0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe, + 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf, +}; + +static unsigned int mirrorbyte(unsigned int b) { + return mirrornibbles[b & 0xf] << 4 | mirrornibbles[b >> 4]; +} + +static int int_log2(int x) { + int r = 0; + while (x >>= 1) { + ++r; + } + return r; +} + +static void uzlib_literal(uzlib_lz77_state_t *state, unsigned char c) +{ + if (c <= 143) { + /* 0 through 143 are 8 bits long starting at 00110000. */ + outbits(state, mirrorbyte(0x30 + c), 8); + } else { + /* 144 through 255 are 9 bits long starting at 110010000. */ + outbits(state, 1 + 2 * mirrorbyte(0x90 - 144 + c), 9); + } +} + +static void uzlib_match(uzlib_lz77_state_t *state, int distance, int len) +{ + assert(distance >= 1 && distance <= 32768); + distance -= 1; + + while (len > 0) { + int thislen; + + /* + * We can transmit matches of lengths 3 through 258 + * inclusive. So if len exceeds 258, we must transmit in + * several steps, with 258 or less in each step. + * + * Specifically: if len >= 261, we can transmit 258 and be + * sure of having at least 3 left for the next step. And if + * len <= 258, we can just transmit len. But if len == 259 + * or 260, we must transmit len-3. + */ + thislen = (len > 260 ? 258 : len <= 258 ? len : len - 3); + len -= thislen; + + assert(thislen >= 3 && thislen <= 258); + + thislen -= 3; + int lcode = 28; + int x = int_log2(thislen); + int y; + if (thislen < 255) { + if (x) { + --x; + } + y = (thislen >> (x ? x - 1 : 0)) & 3; + lcode = x * 4 + y; + } + + /* + * Transmit the length code. 256-279 are seven bits + * starting at 0000000; 280-287 are eight bits starting at + * 11000000. + */ + if (lcode <= 22) { + outbits(state, mirrorbyte((lcode + 1) * 2), 7); + } else { + outbits(state, mirrorbyte(lcode + 169), 8); + } + + /* + * Transmit the extra bits. + */ + if (thislen < 255 && x > 1) { + int extrabits = x - 1; + int lmin = (thislen >> extrabits) << extrabits; + outbits(state, thislen - lmin, extrabits); + } + + x = int_log2(distance); + y = (distance >> (x ? x - 1 : 0)) & 1; + + /* + * Transmit the distance code. Five bits starting at 00000. + */ + outbits(state, mirrorbyte((x * 2 + y) * 8), 5); + + /* + * Transmit the extra bits. + */ + if (x > 1) { + int dextrabits = x - 1; + int dmin = (distance >> dextrabits) << dextrabits; + outbits(state, distance - dmin, dextrabits); + } + } +} + +void uzlib_start_block(uzlib_lz77_state_t *state) +{ + // Final block (0b1) + // Static huffman block (0b01) + outbits(state, 3, 3); +} + +void uzlib_finish_block(uzlib_lz77_state_t *state) +{ + // Close block (0b0000000) + // Make sure all bits are flushed (0b0000000) + outbits(state, 0, 14); +} diff --git a/lib/uzlib/defl_static.h b/lib/uzlib/defl_static.h deleted file mode 100644 index 292734d773..0000000000 --- a/lib/uzlib/defl_static.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) uzlib authors - * - * This software is provided 'as-is', without any express - * or implied warranty. In no event will the authors be - * held liable for any damages arising from the use of - * this software. - * - * Permission is granted to anyone to use this software - * for any purpose, including commercial applications, - * and to alter it and redistribute it freely, subject to - * the following restrictions: - * - * 1. The origin of this software must not be - * misrepresented; you must not claim that you - * wrote the original software. If you use this - * software in a product, an acknowledgment in - * the product documentation would be appreciated - * but is not required. - * - * 2. Altered source versions must be plainly marked - * as such, and must not be misrepresented as - * being the original software. - * - * 3. This notice may not be removed or altered from - * any source distribution. - */ - -/* This files contains type declaration and prototypes for defl_static.c. - They may be altered/distinct from the originals used in PuTTY source - code. */ - -struct Outbuf { - unsigned char *outbuf; - int outlen, outsize; - unsigned long outbits; - int noutbits; - int comp_disabled; -}; - -void outbits(struct Outbuf *out, unsigned long bits, int nbits); -void zlib_start_block(struct Outbuf *ctx); -void zlib_finish_block(struct Outbuf *ctx); -void zlib_literal(struct Outbuf *ectx, unsigned char c); -void zlib_match(struct Outbuf *ectx, int distance, int len); diff --git a/lib/uzlib/header.c b/lib/uzlib/header.c new file mode 100644 index 0000000000..9c48d91391 --- /dev/null +++ b/lib/uzlib/header.c @@ -0,0 +1,137 @@ +/* + * uzlib - tiny deflate/inflate library (deflate, gzip, zlib) + * + * Copyright (c) 2003 by Joergen Ibsen / Jibz + * All Rights Reserved + * + * http://www.ibsensoftware.com/ + * + * Copyright (c) 2014-2018 by Paul Sokolovsky + * + * Optimised for MicroPython: + * Copyright (c) 2023 by Jim Mussared + * + * This software is provided 'as-is', without any express + * or implied warranty. In no event will the authors be + * held liable for any damages arising from the use of + * this software. + * + * Permission is granted to anyone to use this software + * for any purpose, including commercial applications, + * and to alter it and redistribute it freely, subject to + * the following restrictions: + * + * 1. The origin of this software must not be + * misrepresented; you must not claim that you + * wrote the original software. If you use this + * software in a product, an acknowledgment in + * the product documentation would be appreciated + * but is not required. + * + * 2. Altered source versions must be plainly marked + * as such, and must not be misrepresented as + * being the original software. + * + * 3. This notice may not be removed or altered from + * any source distribution. + */ + +#include "uzlib.h" + +#define FTEXT 1 +#define FHCRC 2 +#define FEXTRA 4 +#define FNAME 8 +#define FCOMMENT 16 + +void tinf_skip_bytes(uzlib_uncomp_t *d, int num); +uint16_t tinf_get_uint16(uzlib_uncomp_t *d); + +void tinf_skip_bytes(uzlib_uncomp_t *d, int num) +{ + while (num--) uzlib_get_byte(d); +} + +uint16_t tinf_get_uint16(uzlib_uncomp_t *d) +{ + unsigned int v = uzlib_get_byte(d); + v = (uzlib_get_byte(d) << 8) | v; + return v; +} + +int uzlib_parse_zlib_gzip_header(uzlib_uncomp_t *d, int *wbits) +{ + /* -- check format -- */ + unsigned char cmf = uzlib_get_byte(d); + unsigned char flg = uzlib_get_byte(d); + + /* check for gzip id bytes */ + if (cmf == 0x1f && flg == 0x8b) { + /* check method is deflate */ + if (uzlib_get_byte(d) != 8) return UZLIB_DATA_ERROR; + + /* get flag byte */ + flg = uzlib_get_byte(d); + + /* check that reserved bits are zero */ + if (flg & 0xe0) return UZLIB_DATA_ERROR; + + /* -- find start of compressed data -- */ + + /* skip rest of base header of 10 bytes */ + tinf_skip_bytes(d, 6); + + /* skip extra data if present */ + if (flg & FEXTRA) + { + unsigned int xlen = tinf_get_uint16(d); + tinf_skip_bytes(d, xlen); + } + + /* skip file name if present */ + if (flg & FNAME) { while (uzlib_get_byte(d)); } + + /* skip file comment if present */ + if (flg & FCOMMENT) { while (uzlib_get_byte(d)); } + + /* check header crc if present */ + if (flg & FHCRC) + { + /*unsigned int hcrc =*/ tinf_get_uint16(d); + + // TODO: Check! + // if (hcrc != (tinf_crc32(src, start - src) & 0x0000ffff)) + // return UZLIB_DATA_ERROR; + } + + /* initialize for crc32 checksum */ + d->checksum_type = UZLIB_CHKSUM_CRC; + d->checksum = ~0; + + /* gzip does not include the window size in the header, as it is expected that a + compressor will use wbits=15 (32kiB).*/ + *wbits = 15; + + return UZLIB_HEADER_GZIP; + } else { + /* check checksum */ + if ((256*cmf + flg) % 31) return UZLIB_DATA_ERROR; + + /* check method is deflate */ + if ((cmf & 0x0f) != 8) return UZLIB_DATA_ERROR; + + /* check window size is valid */ + if ((cmf >> 4) > 7) return UZLIB_DATA_ERROR; + + /* check there is no preset dictionary */ + if (flg & 0x20) return UZLIB_DATA_ERROR; + + /* initialize for adler32 checksum */ + d->checksum_type = UZLIB_CHKSUM_ADLER; + d->checksum = 1; + + *wbits = (cmf >> 4) + 8; + + return UZLIB_HEADER_ZLIB; + } +} diff --git a/lib/uzlib/lz77.c b/lib/uzlib/lz77.c new file mode 100644 index 0000000000..117ce50f70 --- /dev/null +++ b/lib/uzlib/lz77.c @@ -0,0 +1,87 @@ +/* + * Simple LZ77 streaming compressor. + * + * The scheme implemented here doesn't use a hash table and instead does a brute + * force search in the history for a previous string. It is relatively slow + * (but still O(N)) but gives good compression and minimal memory usage. For a + * small history window (eg 256 bytes) it's not too slow and compresses well. + * + * MIT license; Copyright (c) 2021 Damien P. George + */ + +#include "uzlib.h" + +#include "defl_static.c" + +#define MATCH_LEN_MIN (3) +#define MATCH_LEN_MAX (258) + +// hist should be a preallocated buffer of hist_max size bytes. +// hist_max should be greater than 0 a power of 2 (ie 1, 2, 4, 8, ...). +// It's possible to pass in hist=NULL, and then the history window will be taken from the +// src passed in to uzlib_lz77_compress (this is useful when not doing streaming compression). +void uzlib_lz77_init(uzlib_lz77_state_t *state, uint8_t *hist, size_t hist_max) { + memset(state, 0, sizeof(uzlib_lz77_state_t)); + state->hist_buf = hist; + state->hist_max = hist_max; + state->hist_start = 0; + state->hist_len = 0; +} + +// Push the given byte to the history. +// Search back in the history for the maximum match of the given src data, +// with support for searching beyond the end of the history and into the src buffer +// (effectively the history and src buffer are concatenated). +static size_t uzlib_lz77_search_max_match(uzlib_lz77_state_t *state, const uint8_t *src, size_t len, size_t *longest_offset) { + size_t longest_len = 0; + for (size_t hist_search = 0; hist_search < state->hist_len; ++hist_search) { + size_t match_len; + for (match_len = 0; match_len <= MATCH_LEN_MAX && match_len < len; ++match_len) { + uint8_t hist; + if (hist_search + match_len < state->hist_len) { + hist = state->hist_buf[(state->hist_start + hist_search + match_len) & (state->hist_max - 1)]; + } else { + hist = src[hist_search + match_len - state->hist_len]; + } + if (src[match_len] != hist) { + break; + } + } + if (match_len >= MATCH_LEN_MIN && match_len > longest_len) { + longest_len = match_len; + *longest_offset = state->hist_len - hist_search; + } + } + + return longest_len; +} + +// Compress the given chunk of data. +void uzlib_lz77_compress(uzlib_lz77_state_t *state, const uint8_t *src, unsigned len) { + const uint8_t *top = src + len; + while (src < top) { + // Look for a match in the history window. + size_t match_offset = 0; + size_t match_len = uzlib_lz77_search_max_match(state, src, top - src, &match_offset); + + // Encode the literal byte or the match. + if (match_len == 0) { + uzlib_literal(state, *src); + match_len = 1; + } else { + uzlib_match(state, match_offset, match_len); + } + + // Push the bytes into the history buffer. + size_t mask = state->hist_max - 1; + while (match_len--) { + uint8_t b = *src++; + state->hist_buf[(state->hist_start + state->hist_len) & mask] = b; + if (state->hist_len == state->hist_max) { + state->hist_start = (state->hist_start + 1) & mask; + } else { + ++state->hist_len; + } + } + } +} diff --git a/lib/uzlib/tinf.h b/lib/uzlib/tinf.h deleted file mode 100644 index ae6e1c4073..0000000000 --- a/lib/uzlib/tinf.h +++ /dev/null @@ -1,3 +0,0 @@ -/* Compatibility header for the original tinf lib/older versions of uzlib. - Note: may be removed in the future, please migrate to uzlib.h. */ -#include "uzlib.h" diff --git a/lib/uzlib/tinf_compat.h b/lib/uzlib/tinf_compat.h deleted file mode 100644 index f763804bd9..0000000000 --- a/lib/uzlib/tinf_compat.h +++ /dev/null @@ -1,9 +0,0 @@ -/* This header contains compatibility defines for the original tinf API - and uzlib 2.x and below API. These defines are deprecated and going - to be removed in the future, so applications should migrate to new - uzlib API. */ -#define TINF_DATA struct uzlib_uncomp - -#define destSize dest_size -#define destStart dest_start -#define readSource source_read_cb diff --git a/lib/uzlib/tinfgzip.c b/lib/uzlib/tinfgzip.c deleted file mode 100644 index 22b000df9a..0000000000 --- a/lib/uzlib/tinfgzip.c +++ /dev/null @@ -1,110 +0,0 @@ -/* - * uzlib - tiny deflate/inflate library (deflate, gzip, zlib) - * - * Copyright (c) 2003 by Joergen Ibsen / Jibz - * All Rights Reserved - * - * http://www.ibsensoftware.com/ - * - * Copyright (c) 2014-2018 by Paul Sokolovsky - * - * This software is provided 'as-is', without any express - * or implied warranty. In no event will the authors be - * held liable for any damages arising from the use of - * this software. - * - * Permission is granted to anyone to use this software - * for any purpose, including commercial applications, - * and to alter it and redistribute it freely, subject to - * the following restrictions: - * - * 1. The origin of this software must not be - * misrepresented; you must not claim that you - * wrote the original software. If you use this - * software in a product, an acknowledgment in - * the product documentation would be appreciated - * but is not required. - * - * 2. Altered source versions must be plainly marked - * as such, and must not be misrepresented as - * being the original software. - * - * 3. This notice may not be removed or altered from - * any source distribution. - */ - -#include "tinf.h" - -#define FTEXT 1 -#define FHCRC 2 -#define FEXTRA 4 -#define FNAME 8 -#define FCOMMENT 16 - -void tinf_skip_bytes(TINF_DATA *d, int num); -uint16_t tinf_get_uint16(TINF_DATA *d); - -void tinf_skip_bytes(TINF_DATA *d, int num) -{ - while (num--) uzlib_get_byte(d); -} - -uint16_t tinf_get_uint16(TINF_DATA *d) -{ - unsigned int v = uzlib_get_byte(d); - v = (uzlib_get_byte(d) << 8) | v; - return v; -} - -int uzlib_gzip_parse_header(TINF_DATA *d) -{ - unsigned char flg; - - /* -- check format -- */ - - /* check id bytes */ - if (uzlib_get_byte(d) != 0x1f || uzlib_get_byte(d) != 0x8b) return TINF_DATA_ERROR; - - /* check method is deflate */ - if (uzlib_get_byte(d) != 8) return TINF_DATA_ERROR; - - /* get flag byte */ - flg = uzlib_get_byte(d); - - /* check that reserved bits are zero */ - if (flg & 0xe0) return TINF_DATA_ERROR; - - /* -- find start of compressed data -- */ - - /* skip rest of base header of 10 bytes */ - tinf_skip_bytes(d, 6); - - /* skip extra data if present */ - if (flg & FEXTRA) - { - unsigned int xlen = tinf_get_uint16(d); - tinf_skip_bytes(d, xlen); - } - - /* skip file name if present */ - if (flg & FNAME) { while (uzlib_get_byte(d)); } - - /* skip file comment if present */ - if (flg & FCOMMENT) { while (uzlib_get_byte(d)); } - - /* check header crc if present */ - if (flg & FHCRC) - { - /*unsigned int hcrc =*/ tinf_get_uint16(d); - - // TODO: Check! -// if (hcrc != (tinf_crc32(src, start - src) & 0x0000ffff)) -// return TINF_DATA_ERROR; - } - - /* initialize for crc32 checksum */ - d->checksum_type = TINF_CHKSUM_CRC; - d->checksum = ~0; - - return TINF_OK; -} diff --git a/lib/uzlib/tinflate.c b/lib/uzlib/tinflate.c index 045952c755..53536272f3 100644 --- a/lib/uzlib/tinflate.c +++ b/lib/uzlib/tinflate.c @@ -7,6 +7,9 @@ * * Copyright (c) 2014-2018 by Paul Sokolovsky * + * Optimised for MicroPython: + * Copyright (c) 2023 by Jim Mussared + * * This software is provided 'as-is', without any express * or implied warranty. In no event will the authors be * held liable for any damages arising from the use of @@ -33,7 +36,7 @@ */ #include -#include "tinf.h" +#include "uzlib.h" #define UZLIB_DUMP_ARRAY(heading, arr, size) \ { \ @@ -44,52 +47,52 @@ printf("\n"); \ } -uint32_t tinf_get_le_uint32(TINF_DATA *d); -uint32_t tinf_get_be_uint32(TINF_DATA *d); +uint32_t tinf_get_le_uint32(uzlib_uncomp_t *d); +uint32_t tinf_get_be_uint32(uzlib_uncomp_t *d); /* --------------------------------------------------- * * -- uninitialized global data (static structures) -- * * --------------------------------------------------- */ -#ifdef RUNTIME_BITS_TABLES +typedef struct { + unsigned char length_bits : 3; + unsigned short length_base : 9; + unsigned char dist_bits : 4; + unsigned short dist_base : 15; +} lookup_table_entry_t; -/* extra bits and base tables for length codes */ -unsigned char length_bits[30]; -unsigned short length_base[30]; - -/* extra bits and base tables for distance codes */ -unsigned char dist_bits[30]; -unsigned short dist_base[30]; - -#else - -const unsigned char length_bits[30] = { - 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 2, 2, - 3, 3, 3, 3, 4, 4, 4, 4, - 5, 5, 5, 5 +const lookup_table_entry_t lookup_table[30] = { + {0, 3, 0, 1}, + {0, 4, 0, 2}, + {0, 5, 0, 3}, + {0, 6, 0, 4}, + {0, 7, 1, 5}, + {0, 8, 1, 7}, + {0, 9, 2, 9}, + {0, 10, 2, 13}, + {1, 11, 3, 17}, + {1, 13, 3, 25}, + {1, 15, 4, 33}, + {1, 17, 4, 49}, + {2, 19, 5, 65}, + {2, 23, 5, 97}, + {2, 27, 6, 129}, + {2, 31, 6, 193}, + {3, 35, 7, 257}, + {3, 43, 7, 385}, + {3, 51, 8, 513}, + {3, 59, 8, 769}, + {4, 67, 9, 1025}, + {4, 83, 9, 1537}, + {4, 99, 10, 2049}, + {4, 115, 10, 3073}, + {5, 131, 11, 4097}, + {5, 163, 11, 6145}, + {5, 195, 12, 8193}, + {5, 227, 12, 12289}, + {0, 258, 13, 16385}, + {0, 0, 13, 24577}, }; -const unsigned short length_base[30] = { - 3, 4, 5, 6, 7, 8, 9, 10, - 11, 13, 15, 17, 19, 23, 27, 31, - 35, 43, 51, 59, 67, 83, 99, 115, - 131, 163, 195, 227, 258 -}; - -const unsigned char dist_bits[30] = { - 0, 0, 0, 0, 1, 1, 2, 2, - 3, 3, 4, 4, 5, 5, 6, 6, - 7, 7, 8, 8, 9, 9, 10, 10, - 11, 11, 12, 12, 13, 13 -}; -const unsigned short dist_base[30] = { - 1, 2, 3, 4, 5, 7, 9, 13, - 17, 25, 33, 49, 65, 97, 129, 193, - 257, 385, 513, 769, 1025, 1537, 2049, 3073, - 4097, 6145, 8193, 12289, 16385, 24577 -}; - -#endif /* special ordering of code length codes */ const unsigned char clcidx[] = { @@ -102,25 +105,6 @@ const unsigned char clcidx[] = { * -- utility functions -- * * ----------------------- */ -#ifdef RUNTIME_BITS_TABLES -/* build extra bits and base tables */ -static void tinf_build_bits_base(unsigned char *bits, unsigned short *base, int delta, int first) -{ - int i, sum; - - /* build bits table */ - for (i = 0; i < delta; ++i) bits[i] = 0; - for (i = 0; i < 30 - delta; ++i) bits[i + delta] = i / delta; - - /* build base table */ - for (sum = first, i = 0; i < 30; ++i) - { - base[i] = sum; - sum += 1 << bits[i]; - } -} -#endif - /* build the fixed huffman trees */ static void tinf_build_fixed_trees(TINF_TREE *lt, TINF_TREE *dt) { @@ -189,7 +173,7 @@ static void tinf_build_tree(TINF_TREE *t, const unsigned char *lengths, unsigned * -- decode functions -- * * ---------------------- */ -unsigned char uzlib_get_byte(TINF_DATA *d) +unsigned char uzlib_get_byte(uzlib_uncomp_t *d) { /* If end of source buffer is not reached, return next byte from source buffer. */ @@ -200,14 +184,14 @@ unsigned char uzlib_get_byte(TINF_DATA *d) /* Otherwise if there's callback and we haven't seen EOF yet, try to read next byte using it. (Note: the callback can also update ->source and ->source_limit). */ - if (d->readSource && !d->eof) { - int val = d->readSource(d); + if (d->source_read_cb && !d->eof) { + int val = d->source_read_cb(d->source_read_data); if (val >= 0) { return (unsigned char)val; } } - /* Otherwise, we hit EOF (either from ->readSource() or from exhaustion + /* Otherwise, we hit EOF (either from ->source_read_cb() or from exhaustion of the buffer), and it will be "sticky", i.e. further calls to this function will end up here too. */ d->eof = true; @@ -215,7 +199,7 @@ unsigned char uzlib_get_byte(TINF_DATA *d) return 0; } -uint32_t tinf_get_le_uint32(TINF_DATA *d) +uint32_t tinf_get_le_uint32(uzlib_uncomp_t *d) { uint32_t val = 0; int i; @@ -225,7 +209,7 @@ uint32_t tinf_get_le_uint32(TINF_DATA *d) return val; } -uint32_t tinf_get_be_uint32(TINF_DATA *d) +uint32_t tinf_get_be_uint32(uzlib_uncomp_t *d) { uint32_t val = 0; int i; @@ -236,7 +220,7 @@ uint32_t tinf_get_be_uint32(TINF_DATA *d) } /* get one bit from source stream */ -static int tinf_getbit(TINF_DATA *d) +static int tinf_getbit(uzlib_uncomp_t *d) { unsigned int bit; @@ -256,7 +240,7 @@ static int tinf_getbit(TINF_DATA *d) } /* read a num bit value from a stream and add base */ -static unsigned int tinf_read_bits(TINF_DATA *d, int num, int base) +static unsigned int tinf_read_bits(uzlib_uncomp_t *d, int num, int base) { unsigned int val = 0; @@ -274,7 +258,7 @@ static unsigned int tinf_read_bits(TINF_DATA *d, int num, int base) } /* given a data stream and a tree, decode a symbol */ -static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t) +static int tinf_decode_symbol(uzlib_uncomp_t *d, TINF_TREE *t) { int sum = 0, cur = 0, len = 0; @@ -284,7 +268,7 @@ static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t) cur = 2*cur + tinf_getbit(d); if (++len == TINF_ARRAY_SIZE(t->table)) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } sum += t->table[len]; @@ -295,7 +279,7 @@ static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t) sum += cur; #if UZLIB_CONF_PARANOID_CHECKS if (sum < 0 || sum >= TINF_ARRAY_SIZE(t->trans)) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } #endif @@ -303,7 +287,7 @@ static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t) } /* given a data stream, decode dynamic trees from it */ -static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) +static int tinf_decode_trees(uzlib_uncomp_t *d, TINF_TREE *lt, TINF_TREE *dt) { /* code lengths for 288 literal/len symbols and 32 dist symbols */ unsigned char lengths[288+32]; @@ -348,7 +332,7 @@ static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) { case 16: /* copy previous code length 3-6 times (read 2 bits) */ - if (num == 0) return TINF_DATA_ERROR; + if (num == 0) return UZLIB_DATA_ERROR; fill_value = lengths[num - 1]; lbits = 2; break; @@ -370,7 +354,7 @@ static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) /* special code length 16-18 are handled here */ length = tinf_read_bits(d, lbits, lbase); - if (num + length > hlimit) return TINF_DATA_ERROR; + if (num + length > hlimit) return UZLIB_DATA_ERROR; for (; length; --length) { lengths[num++] = fill_value; @@ -387,7 +371,7 @@ static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) #if UZLIB_CONF_PARANOID_CHECKS /* Check that there's "end of block" symbol */ if (lengths[256] == 0) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } #endif @@ -395,7 +379,7 @@ static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) tinf_build_tree(lt, lengths, hlit); tinf_build_tree(dt, lengths + hlit, hdist); - return TINF_OK; + return UZLIB_OK; } /* ----------------------------- * @@ -403,7 +387,7 @@ static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) * ----------------------------- */ /* given a stream and two trees, inflate next byte of output */ -static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) +static int tinf_inflate_block_data(uzlib_uncomp_t *d, TINF_TREE *lt, TINF_TREE *dt) { if (d->curlen == 0) { unsigned int offs; @@ -412,41 +396,41 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) //printf("huff sym: %02x\n", sym); if (d->eof) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } /* literal byte */ if (sym < 256) { TINF_PUT(d, sym); - return TINF_OK; + return UZLIB_OK; } /* end of block */ if (sym == 256) { - return TINF_DONE; + return UZLIB_DONE; } /* substring from sliding dictionary */ sym -= 257; if (sym >= 29) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } /* possibly get more bits from length code */ - d->curlen = tinf_read_bits(d, length_bits[sym], length_base[sym]); + d->curlen = tinf_read_bits(d, lookup_table[sym].length_bits, lookup_table[sym].length_base); dist = tinf_decode_symbol(d, dt); if (dist >= 30) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } /* possibly get more bits from distance code */ - offs = tinf_read_bits(d, dist_bits[dist], dist_base[dist]); + offs = tinf_read_bits(d, lookup_table[dist].dist_bits, lookup_table[dist].dist_base); /* calculate and validate actual LZ offset to use */ if (d->dict_ring) { if (offs > d->dict_size) { - return TINF_DICT_ERROR; + return UZLIB_DICT_ERROR; } /* Note: unlike full-dest-in-memory case below, we don't try to catch offset which points to not yet filled @@ -464,8 +448,8 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) } } else { /* catch trying to point before the start of dest buffer */ - if (offs > (unsigned int)(d->dest - d->destStart)) { - return TINF_DATA_ERROR; + if (offs > (unsigned int)(d->dest - d->dest_start)) { + return UZLIB_DATA_ERROR; } d->lzOff = -offs; } @@ -482,11 +466,11 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) d->dest++; } d->curlen--; - return TINF_OK; + return UZLIB_OK; } /* inflate next byte from uncompressed block of data */ -static int tinf_inflate_uncompressed_block(TINF_DATA *d) +static int tinf_inflate_uncompressed_block(uzlib_uncomp_t *d) { if (d->curlen == 0) { unsigned int length, invlength; @@ -498,9 +482,9 @@ static int tinf_inflate_uncompressed_block(TINF_DATA *d) invlength = uzlib_get_byte(d); invlength += 256 * uzlib_get_byte(d); /* check length */ - if (length != (~invlength & 0x0000ffff)) return TINF_DATA_ERROR; + if (length != (~invlength & 0x0000ffff)) return UZLIB_DATA_ERROR; - /* increment length to properly return TINF_DONE below, without + /* increment length to properly return UZLIB_DONE below, without producing data at the same time */ d->curlen = length + 1; @@ -509,34 +493,20 @@ static int tinf_inflate_uncompressed_block(TINF_DATA *d) } if (--d->curlen == 0) { - return TINF_DONE; + return UZLIB_DONE; } unsigned char c = uzlib_get_byte(d); TINF_PUT(d, c); - return TINF_OK; + return UZLIB_OK; } /* ---------------------- * * -- public functions -- * * ---------------------- */ -/* initialize global (static) data */ -void uzlib_init(void) -{ -#ifdef RUNTIME_BITS_TABLES - /* build extra bits and base tables */ - tinf_build_bits_base(length_bits, length_base, 4, 3); - tinf_build_bits_base(dist_bits, dist_base, 2, 1); - - /* fix a special case */ - length_bits[28] = 0; - length_base[28] = 258; -#endif -} - /* initialize decompression structure */ -void uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen) +void uzlib_uncompress_init(uzlib_uncomp_t *d, void *dict, unsigned int dictLen) { d->eof = 0; d->bitcount = 0; @@ -549,7 +519,7 @@ void uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen) } /* inflate next output bytes from compressed stream */ -int uzlib_uncompress(TINF_DATA *d) +int uzlib_uncompress(uzlib_uncomp_t *d) { do { int res; @@ -572,7 +542,7 @@ next_blk: } else if (d->btype == 2) { /* decode trees from stream */ res = tinf_decode_trees(d, &d->ltree, &d->dtree); - if (res != TINF_OK) { + if (res != UZLIB_OK) { return res; } } @@ -592,27 +562,27 @@ next_blk: res = tinf_inflate_block_data(d, &d->ltree, &d->dtree); break; default: - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } - if (res == TINF_DONE && !d->bfinal) { + if (res == UZLIB_DONE && !d->bfinal) { /* the block has ended (without producing more data), but we can't return without data, so start procesing next block */ goto next_blk; } - if (res != TINF_OK) { + if (res != UZLIB_OK) { return res; } } while (d->dest < d->dest_limit); - return TINF_OK; + return UZLIB_OK; } /* inflate next output bytes from compressed stream, updating checksum, and at the end of stream, verify it */ -int uzlib_uncompress_chksum(TINF_DATA *d) +int uzlib_uncompress_chksum(uzlib_uncomp_t *d) { int res; unsigned char *data = d->dest; @@ -623,31 +593,31 @@ int uzlib_uncompress_chksum(TINF_DATA *d) switch (d->checksum_type) { - case TINF_CHKSUM_ADLER: + case UZLIB_CHKSUM_ADLER: d->checksum = uzlib_adler32(data, d->dest - data, d->checksum); break; - case TINF_CHKSUM_CRC: + case UZLIB_CHKSUM_CRC: d->checksum = uzlib_crc32(data, d->dest - data, d->checksum); break; } - if (res == TINF_DONE) { + if (res == UZLIB_DONE) { unsigned int val; switch (d->checksum_type) { - case TINF_CHKSUM_ADLER: + case UZLIB_CHKSUM_ADLER: val = tinf_get_be_uint32(d); if (d->checksum != val) { - return TINF_CHKSUM_ERROR; + return UZLIB_CHKSUM_ERROR; } break; - case TINF_CHKSUM_CRC: + case UZLIB_CHKSUM_CRC: val = tinf_get_le_uint32(d); if (~d->checksum != val) { - return TINF_CHKSUM_ERROR; + return UZLIB_CHKSUM_ERROR; } // Uncompressed size. TODO: Check val = tinf_get_le_uint32(d); diff --git a/lib/uzlib/tinfzlib.c b/lib/uzlib/tinfzlib.c deleted file mode 100644 index 5cb8852fcc..0000000000 --- a/lib/uzlib/tinfzlib.c +++ /dev/null @@ -1,66 +0,0 @@ -/* - * uzlib - tiny deflate/inflate library (deflate, gzip, zlib) - * - * Copyright (c) 2003 by Joergen Ibsen / Jibz - * All Rights Reserved - * - * http://www.ibsensoftware.com/ - * - * Copyright (c) 2014-2018 by Paul Sokolovsky - * - * This software is provided 'as-is', without any express - * or implied warranty. In no event will the authors be - * held liable for any damages arising from the use of - * this software. - * - * Permission is granted to anyone to use this software - * for any purpose, including commercial applications, - * and to alter it and redistribute it freely, subject to - * the following restrictions: - * - * 1. The origin of this software must not be - * misrepresented; you must not claim that you - * wrote the original software. If you use this - * software in a product, an acknowledgment in - * the product documentation would be appreciated - * but is not required. - * - * 2. Altered source versions must be plainly marked - * as such, and must not be misrepresented as - * being the original software. - * - * 3. This notice may not be removed or altered from - * any source distribution. - */ - -#include "tinf.h" - -int uzlib_zlib_parse_header(TINF_DATA *d) -{ - unsigned char cmf, flg; - - /* -- get header bytes -- */ - - cmf = uzlib_get_byte(d); - flg = uzlib_get_byte(d); - - /* -- check format -- */ - - /* check checksum */ - if ((256*cmf + flg) % 31) return TINF_DATA_ERROR; - - /* check method is deflate */ - if ((cmf & 0x0f) != 8) return TINF_DATA_ERROR; - - /* check window size is valid */ - if ((cmf >> 4) > 7) return TINF_DATA_ERROR; - - /* check there is no preset dictionary */ - if (flg & 0x20) return TINF_DATA_ERROR; - - /* initialize for adler32 checksum */ - d->checksum_type = TINF_CHKSUM_ADLER; - d->checksum = 1; - - return cmf >> 4; -} diff --git a/lib/uzlib/uzlib.h b/lib/uzlib/uzlib.h index 7d8949543e..16984a77d3 100644 --- a/lib/uzlib/uzlib.h +++ b/lib/uzlib/uzlib.h @@ -7,6 +7,9 @@ * * Copyright (c) 2014-2018 by Paul Sokolovsky * + * Optimised for MicroPython: + * Copyright (c) 2023 by Jim Mussared + * * This software is provided 'as-is', without any express * or implied warranty. In no event will the authors be * held liable for any damages arising from the use of @@ -39,38 +42,27 @@ #include #include -#include "defl_static.h" - #include "uzlib_conf.h" #if UZLIB_CONF_DEBUG_LOG #include #endif -/* calling convention */ -#ifndef TINFCC - #ifdef __WATCOMC__ - #define TINFCC __cdecl - #else - #define TINFCC - #endif -#endif - #ifdef __cplusplus extern "C" { #endif /* ok status, more data produced */ -#define TINF_OK 0 +#define UZLIB_OK 0 /* end of compressed stream reached */ -#define TINF_DONE 1 -#define TINF_DATA_ERROR (-3) -#define TINF_CHKSUM_ERROR (-4) -#define TINF_DICT_ERROR (-5) +#define UZLIB_DONE 1 +#define UZLIB_DATA_ERROR (-3) +#define UZLIB_CHKSUM_ERROR (-4) +#define UZLIB_DICT_ERROR (-5) /* checksum types */ -#define TINF_CHKSUM_NONE 0 -#define TINF_CHKSUM_ADLER 1 -#define TINF_CHKSUM_CRC 2 +#define UZLIB_CHKSUM_NONE 0 +#define UZLIB_CHKSUM_ADLER 1 +#define UZLIB_CHKSUM_CRC 2 /* helper macros */ #define TINF_ARRAY_SIZE(arr) (sizeof(arr) / sizeof(*(arr))) @@ -82,9 +74,7 @@ typedef struct { unsigned short trans[288]; /* code -> symbol translation table */ } TINF_TREE; -struct uzlib_uncomp { - /* Point to the CircuitPython object that owns this decompression stream */ - void *self; +typedef struct _uzlib_uncomp_t { /* Pointer to the next byte in the input buffer */ const unsigned char *source; /* Pointer to the next byte past the input buffer (source_limit = source + len) */ @@ -94,7 +84,8 @@ struct uzlib_uncomp { also return -1 in case of EOF (or irrecoverable error). Note that besides returning the next byte, it may also update source and source_limit fields, thus allowing for buffered operation. */ - int (*source_read_cb)(struct uzlib_uncomp *uncomp); + void *source_read_data; + int (*source_read_cb)(void *); unsigned int tag; unsigned int bitcount; @@ -121,9 +112,7 @@ struct uzlib_uncomp { TINF_TREE ltree; /* dynamic length/symbol tree */ TINF_TREE dtree; /* dynamic distance tree */ -}; - -#include "tinf_compat.h" +} uzlib_uncomp_t; #define TINF_PUT(d, c) \ { \ @@ -131,38 +120,43 @@ struct uzlib_uncomp { if (d->dict_ring) { d->dict_ring[d->dict_idx++] = c; if (d->dict_idx == d->dict_size) d->dict_idx = 0; } \ } -unsigned char TINFCC uzlib_get_byte(TINF_DATA *d); +unsigned char uzlib_get_byte(uzlib_uncomp_t *d); /* Decompression API */ -void TINFCC uzlib_init(void); -void TINFCC uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen); -int TINFCC uzlib_uncompress(TINF_DATA *d); -int TINFCC uzlib_uncompress_chksum(TINF_DATA *d); +void uzlib_uncompress_init(uzlib_uncomp_t *d, void *dict, unsigned int dictLen); +int uzlib_uncompress(uzlib_uncomp_t *d); +int uzlib_uncompress_chksum(uzlib_uncomp_t *d); -int TINFCC uzlib_zlib_parse_header(TINF_DATA *d); -int TINFCC uzlib_gzip_parse_header(TINF_DATA *d); +#define UZLIB_HEADER_ZLIB 0 +#define UZLIB_HEADER_GZIP 1 +int uzlib_parse_zlib_gzip_header(uzlib_uncomp_t *d, int *wbits); /* Compression API */ -typedef const uint8_t *uzlib_hash_entry_t; +typedef struct { + void *dest_write_data; + void (*dest_write_cb)(void *data, uint8_t byte); + unsigned long outbits; + int noutbits; + uint8_t *hist_buf; + size_t hist_max; + size_t hist_start; + size_t hist_len; +} uzlib_lz77_state_t; -struct uzlib_comp { - struct Outbuf out; +void uzlib_lz77_init(uzlib_lz77_state_t *state, uint8_t *hist, size_t hist_max); +void uzlib_lz77_compress(uzlib_lz77_state_t *state, const uint8_t *src, unsigned len); - uzlib_hash_entry_t *hash_table; - unsigned int hash_bits; - unsigned int dict_size; -}; - -void TINFCC uzlib_compress(struct uzlib_comp *c, const uint8_t *src, unsigned slen); +void uzlib_start_block(uzlib_lz77_state_t *state); +void uzlib_finish_block(uzlib_lz77_state_t *state); /* Checksum API */ /* prev_sum is previous value for incremental computation, 1 initially */ -uint32_t TINFCC uzlib_adler32(const void *data, unsigned int length, uint32_t prev_sum); +uint32_t uzlib_adler32(const void *data, unsigned int length, uint32_t prev_sum); /* crc is previous value for incremental computation, 0xffffffff initially */ -uint32_t TINFCC uzlib_crc32(const void *data, unsigned int length, uint32_t crc); +uint32_t uzlib_crc32(const void *data, unsigned int length, uint32_t crc); #ifdef __cplusplus } /* extern "C" */ diff --git a/mpy-cross/main.c b/mpy-cross/main.c index 67e73e0eba..78135574c4 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -48,6 +48,14 @@ mp_uint_t mp_verbose_flag = 0; // Make it larger on a 64 bit machine, because pointers are larger. long heap_size = 1024 * 1024 * (sizeof(mp_uint_t) / 4); +STATIC void stdout_print_strn(void *env, const char *str, size_t len) { + (void)env; + ssize_t dummy = write(STDOUT_FILENO, str, len); + (void)dummy; +} + +STATIC const mp_print_t mp_stdout_print = {NULL, stdout_print_strn}; + STATIC void stderr_print_strn(void *env, const char *str, size_t len) { (void)env; ssize_t dummy = write(STDERR_FILENO, str, len); @@ -59,7 +67,12 @@ STATIC const mp_print_t mp_stderr_print = {NULL, stderr_print_strn}; STATIC int compile_and_save(const char *file, const char *output_file, const char *source_file) { nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_lexer_t *lex = mp_lexer_new_from_file(file); + mp_lexer_t *lex; + if (strcmp(file, "-") == 0) { + lex = mp_lexer_new_from_fd(MP_QSTR__lt_stdin_gt_, STDIN_FILENO, false); + } else { + lex = mp_lexer_new_from_file(file); + } qstr source_name; if (source_file == NULL) { @@ -77,17 +90,23 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha cm.context = m_new_obj(mp_module_context_t); mp_compile_to_raw_code(&parse_tree, source_name, false, &cm); - vstr_t vstr; - vstr_init(&vstr, 16); - if (output_file == NULL) { - vstr_add_str(&vstr, file); - vstr_cut_tail_bytes(&vstr, 2); - vstr_add_str(&vstr, "mpy"); + if ((output_file != NULL && strcmp(output_file, "-") == 0) || + (output_file == NULL && strcmp(file, "-") == 0)) { + mp_raw_code_save(&cm, (mp_print_t *)&mp_stdout_print); } else { - vstr_add_str(&vstr, output_file); + vstr_t vstr; + vstr_init(&vstr, 16); + if (output_file == NULL) { + vstr_add_str(&vstr, file); + vstr_cut_tail_bytes(&vstr, 2); + vstr_add_str(&vstr, "mpy"); + } else { + vstr_add_str(&vstr, output_file); + } + + mp_raw_code_save_file(&cm, vstr_null_terminated_str(&vstr)); + vstr_clear(&vstr); } - mp_raw_code_save_file(&cm, vstr_null_terminated_str(&vstr)); - vstr_clear(&vstr); nlr_pop(); return 0; @@ -100,10 +119,10 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha STATIC int usage(char **argv) { printf( - "usage: %s [] [-X ] \n" + "usage: %s [] [-X ] [--] \n" "Options:\n" "--version : show version information\n" - "-o : output file for compiled bytecode (defaults to input with .mpy extension)\n" + "-o : output file for compiled bytecode (defaults to input filename with .mpy extension, or stdout if input is stdin)\n" "-s : source filename to embed in the compiled bytecode (defaults to input file)\n" "-v : verbose (trace various operations); can be multiple\n" "-O[N] : apply bytecode optimizations of level N\n" @@ -220,10 +239,11 @@ MP_NOINLINE int main_(int argc, char **argv) { const char *input_file = NULL; const char *output_file = NULL; const char *source_file = NULL; + bool option_parsing_active = true; // parse main options for (int a = 1; a < argc; a++) { - if (argv[a][0] == '-') { + if (option_parsing_active && argv[a][0] == '-' && argv[a][1] != '\0') { if (strcmp(argv[a], "-X") == 0) { a += 1; } else if (strcmp(argv[a], "--version") == 0) { @@ -309,6 +329,8 @@ MP_NOINLINE int main_(int argc, char **argv) { } else { return usage(argv); } + } else if (strcmp(argv[a], "--") == 0) { + option_parsing_active = false; } else { return usage(argv); } diff --git a/mpy-cross/mpy_cross/__init__.py b/mpy-cross/mpy_cross/__init__.py index 8eadbc8352..5020033e8c 100644 --- a/mpy-cross/mpy_cross/__init__.py +++ b/mpy-cross/mpy_cross/__init__.py @@ -100,7 +100,7 @@ def compile(src, dest=None, src_path=None, opt=None, march=None, mpy_cross=None, if not src: raise ValueError("src is required") if not os.path.exists(src): - raise CrossCompileError("Input .py file not found: {}.".format(src_py)) + raise CrossCompileError("Input .py file not found: {}.".format(src)) args = [] diff --git a/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1011/clocks.c b/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1011/clocks.c index 69ce673a0e..9d54839e28 100644 --- a/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1011/clocks.c +++ b/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1011/clocks.c @@ -178,7 +178,7 @@ void clocks_init(void) { CLOCK_DisableClock(kCLOCK_Lpuart3); CLOCK_DisableClock(kCLOCK_Lpuart4); /* Set UART_CLK_PODF. */ - CLOCK_SetDiv(kCLOCK_UartDiv, 0); + CLOCK_SetDiv(kCLOCK_UartDiv, 1); /* Set Uart clock source. */ CLOCK_SetMux(kCLOCK_UartMux, 0); /* Disable SPDIF clock gate. */ diff --git a/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1015/clocks.c b/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1015/clocks.c index 9906454658..e5aef56552 100644 --- a/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1015/clocks.c +++ b/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1015/clocks.c @@ -213,7 +213,7 @@ void clocks_init(void) { CLOCK_DisableClock(kCLOCK_Lpuart3); CLOCK_DisableClock(kCLOCK_Lpuart4); /* Set UART_CLK_PODF. */ - CLOCK_SetDiv(kCLOCK_UartDiv, 0); + CLOCK_SetDiv(kCLOCK_UartDiv, 1); /* Set Uart clock source. */ CLOCK_SetMux(kCLOCK_UartMux, 0); /* Disable SPDIF clock gate. */ diff --git a/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1021/clocks.c b/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1021/clocks.c index eab7f4eacc..6ee8d0c157 100644 --- a/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1021/clocks.c +++ b/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1021/clocks.c @@ -246,7 +246,7 @@ void clocks_init(void) { CLOCK_DisableClock(kCLOCK_Lpuart7); CLOCK_DisableClock(kCLOCK_Lpuart8); /* Set UART_CLK_PODF. */ - CLOCK_SetDiv(kCLOCK_UartDiv, 0); + CLOCK_SetDiv(kCLOCK_UartDiv, 1); /* Set Uart clock source. */ CLOCK_SetMux(kCLOCK_UartMux, 0); /* Disable SPDIF clock gate. */ diff --git a/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1042/clocks.c b/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1042/clocks.c index 97d3d04f8f..d4a73248ac 100644 --- a/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1042/clocks.c +++ b/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1042/clocks.c @@ -224,7 +224,7 @@ void clocks_init(void) { CLOCK_DisableClock(kCLOCK_Lpuart7); CLOCK_DisableClock(kCLOCK_Lpuart8); /* Set UART_CLK_PODF. */ - CLOCK_SetDiv(kCLOCK_UartDiv, 0); + CLOCK_SetDiv(kCLOCK_UartDiv, 1); /* Set Uart clock source. */ CLOCK_SetMux(kCLOCK_UartMux, 0); /* Disable LCDIF clock gate. */ diff --git a/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1052/clocks.c b/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1052/clocks.c index aafd0fde33..31639a5703 100644 --- a/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1052/clocks.c +++ b/ports/mimxrt10xx/peripherals/mimxrt10xx/MIMXRT1052/clocks.c @@ -231,7 +231,7 @@ void clocks_init(void) { CLOCK_DisableClock(kCLOCK_Lpuart7); CLOCK_DisableClock(kCLOCK_Lpuart8); /* Set UART_CLK_PODF. */ - CLOCK_SetDiv(kCLOCK_UartDiv, 0); + CLOCK_SetDiv(kCLOCK_UartDiv, 1); /* Set Uart clock source. */ CLOCK_SetMux(kCLOCK_UartMux, 0); /* Disable LCDIF clock gate. */ diff --git a/ports/unix/Makefile b/ports/unix/Makefile index f5b111a5fe..a44df0c731 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -27,8 +27,8 @@ FROZEN_MANIFEST ?= variants/manifest.py PROG ?= micropython # qstr definitions (must come before including py.mk) -QSTR_DEFS = qstrdefsport.h -QSTR_GLOBAL_DEPENDENCIES = $(VARIANT_DIR)/mpconfigvariant.h +QSTR_DEFS += qstrdefsport.h +QSTR_GLOBAL_DEPENDENCIES += $(VARIANT_DIR)/mpconfigvariant.h # OS name, for simple autoconfig UNAME_S := $(shell uname -s) @@ -59,8 +59,8 @@ endif # Remove unused sections. COPT += -fdata-sections -ffunction-sections -# Always enable symbols -- They're occasionally useful, and don't make it into the -# final .bin/.hex/.dfu so the extra size doesn't matter. +# Note: Symbols and debug information will still be stripped from the final binary +# unless "DEBUG=1" or "STRIP=" is passed to make, see README.md for details. CFLAGS += -g ifndef DEBUG @@ -138,7 +138,7 @@ CFLAGS += -DMICROPY_PY_THREAD=1 -DMICROPY_PY_THREAD_GIL=0 LDFLAGS += $(LIBPTHREAD) endif -ifeq ($(MICROPY_PY_USSL),1) +ifeq ($(MICROPY_PY_SSL),1) ifeq ($(MICROPY_SSL_AXTLS),1) endif diff --git a/ports/unix/README.md b/ports/unix/README.md index a3a0dba75a..e15fd93b23 100644 --- a/ports/unix/README.md +++ b/ports/unix/README.md @@ -40,8 +40,8 @@ or >>> import mip >>> mip.install("hmac") -Browse available modules at [micropython-lib] -(https://github.com/micropython/micropython-lib). See +Browse available modules at +[micropython-lib](https://github.com/micropython/micropython-lib). See [Package management](https://docs.micropython.org/en/latest/reference/packages.html) for more information about `mip`. @@ -70,5 +70,19 @@ or not). If you intend to build MicroPython with additional options (like cross-compiling), the same set of options should be passed to `make deplibs`. To actually enable/disable use of dependencies, edit the `ports/unix/mpconfigport.mk` file, which has inline descriptions of the -options. For example, to build the SSL module, `MICROPY_PY_USSL` should be +options. For example, to build the SSL module, `MICROPY_PY_SSL` should be set to 1. + +Debug Symbols +------------- + +By default, builds are stripped of symbols and debug information to save size. + +To build a debuggable version of the Unix port, there are two options + +1. Run `make [other arguments] DEBUG=1`. Note setting `DEBUG` also reduces the + optimisation level, so it's not a good option for builds that also want the + best performance. +2. Run `make [other arguments] STRIP=`. Note that the value of `STRIP` is + empty. This will skip the build step that strips symbols and debug + information, but changes nothing else in the build configuration. diff --git a/ports/unix/main.c b/ports/unix/main.c index 87f58014ec..d9a7c0ed35 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -48,7 +48,7 @@ #include "py/mphal.h" #include "py/mpthread.h" #include "extmod/misc.h" -#include "extmod/moduplatform.h" +#include "extmod/modplatform.h" #include "extmod/vfs.h" #include "extmod/vfs_posix.h" #include "genhdr/mpversion.h" @@ -69,12 +69,20 @@ long heap_size = 1024 * 1024 * (sizeof(mp_uint_t) / 4); #define MICROPY_GC_SPLIT_HEAP_N_HEAPS (1) #endif +#if !MICROPY_PY_SYS_PATH +#error "The unix port requires MICROPY_PY_SYS_PATH=1" +#endif + +#if !MICROPY_PY_SYS_ARGV +#error "The unix port requires MICROPY_PY_SYS_ARGV=1" +#endif + STATIC void stderr_print_strn(void *env, const char *str, size_t len) { (void)env; ssize_t ret; MP_HAL_RETRY_SYSCALL(ret, write(STDERR_FILENO, str, len), {}); #if MICROPY_PY_OS_DUPTERM - mp_uos_dupterm_tx_strn(str, len); + mp_os_dupterm_tx_strn(str, len); #endif } @@ -540,44 +548,40 @@ MP_NOINLINE int main_(int argc, char **argv) { } #endif - char *home = getenv("HOME"); - char *path = getenv("MICROPYPATH"); - if (path == NULL) { - path = MICROPY_PY_SYS_PATH_DEFAULT; - } - size_t path_num = 1; // [0] is for current dir (or base dir of the script) - if (*path == PATHLIST_SEP_CHAR) { - path_num++; - } - for (char *p = path; p != NULL; p = strchr(p, PATHLIST_SEP_CHAR)) { - path_num++; - if (p != NULL) { - p++; - } - } - mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), path_num); - mp_obj_t *path_items; - mp_obj_list_get(mp_sys_path, &path_num, &path_items); - path_items[0] = MP_OBJ_NEW_QSTR(MP_QSTR_); { - char *p = path; - for (mp_uint_t i = 1; i < path_num; i++) { - char *p1 = strchr(p, PATHLIST_SEP_CHAR); - if (p1 == NULL) { - p1 = p + strlen(p); + // sys.path starts as [""] + mp_sys_path = mp_obj_new_list(0, NULL); + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); + + // Add colon-separated entries from MICROPYPATH. + char *home = getenv("HOME"); + char *path = getenv("MICROPYPATH"); + if (path == NULL) { + path = MICROPY_PY_SYS_PATH_DEFAULT; + } + if (*path == PATHLIST_SEP_CHAR) { + // First entry is empty. We've already added an empty entry to sys.path, so skip it. + ++path; + } + bool path_remaining = *path; + while (path_remaining) { + char *path_entry_end = strchr(path, PATHLIST_SEP_CHAR); + if (path_entry_end == NULL) { + path_entry_end = path + strlen(path); + path_remaining = false; } - if (p[0] == '~' && p[1] == '/' && home != NULL) { + if (path[0] == '~' && path[1] == '/' && home != NULL) { // Expand standalone ~ to $HOME int home_l = strlen(home); vstr_t vstr; - vstr_init(&vstr, home_l + (p1 - p - 1) + 1); + vstr_init(&vstr, home_l + (path_entry_end - path - 1) + 1); vstr_add_strn(&vstr, home, home_l); - vstr_add_strn(&vstr, p + 1, p1 - p - 1); - path_items[i] = mp_obj_new_str_from_vstr(&vstr); + vstr_add_strn(&vstr, path + 1, path_entry_end - path - 1); + mp_obj_list_append(mp_sys_path, mp_obj_new_str_from_vstr(&vstr)); } else { - path_items[i] = mp_obj_new_str_via_qstr(p, p1 - p); + mp_obj_list_append(mp_sys_path, mp_obj_new_str_via_qstr(path, path_entry_end - path)); } - p = p1 + 1; + path = path_entry_end + 1; } } @@ -666,7 +670,10 @@ MP_NOINLINE int main_(int argc, char **argv) { return handle_uncaught_exception(nlr.ret_val) & 0xff; } - if (mp_obj_is_package(mod) && !subpkg_tried) { + // If this module is a package, see if it has a `__main__.py`. + mp_obj_t dest[2]; + mp_load_method_protected(mod, MP_QSTR___path__, dest, true); + if (dest[0] != MP_OBJ_NULL && !subpkg_tried) { subpkg_tried = true; vstr_t vstr; int len = strlen(argv[a + 1]); @@ -709,7 +716,7 @@ MP_NOINLINE int main_(int argc, char **argv) { // Set base dir of the script as first entry in sys.path. char *p = strrchr(basedir, '/'); - path_items[0] = mp_obj_new_str_via_qstr(basedir, p - basedir); + mp_obj_list_store(mp_sys_path, MP_OBJ_NEW_SMALL_INT(0), mp_obj_new_str_via_qstr(basedir, p - basedir)); free(pathbuf); set_sys_argv(argv, argc, a); diff --git a/ports/unix/mbedtls/mbedtls_config.h b/ports/unix/mbedtls/mbedtls_config.h index c8ffab0832..629064abcf 100644 --- a/ports/unix/mbedtls/mbedtls_config.h +++ b/ports/unix/mbedtls/mbedtls_config.h @@ -27,7 +27,7 @@ #define MICROPY_INCLUDED_MBEDTLS_CONFIG_H // Set mbedtls configuration -#define MBEDTLS_CIPHER_MODE_CTR // needed for MICROPY_PY_UCRYPTOLIB_CTR +#define MBEDTLS_CIPHER_MODE_CTR // needed for MICROPY_PY_CRYPTOLIB_CTR // Enable mbedtls modules #define MBEDTLS_HAVEGE_C diff --git a/ports/unix/modjni.c b/ports/unix/modjni.c index 10622f588f..a1cfca8149 100644 --- a/ports/unix/modjni.c +++ b/ports/unix/modjni.c @@ -329,8 +329,7 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE( unary_op, jobject_unary_op, attr, jobject_attr, subscr, jobject_subscr, - iter, subscr_getiter, - // .locals_dict = &jobject_locals_dict, + iter, subscr_getiter ); STATIC mp_obj_t new_jobject(jobject jo) { @@ -574,9 +573,7 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE( MP_QSTR_jmethod, MP_TYPE_FLAG_NONE, print, jmethod_print, - call, jmethod_call, - // .attr = jobject_attr, - // .locals_dict = &jobject_locals_dict, + call, jmethod_call ); #ifdef __ANDROID__ @@ -615,26 +612,26 @@ STATIC void create_jvm(void) { jclass Object_class = JJ(FindClass, "java/lang/Object"); Object_toString_mid = JJ(GetMethodID, Object_class, "toString", - MP_ERROR_TEXT("()Ljava/lang/String;")); + MP_COMPRESSED_ROM_TEXT("()Ljava/lang/String;")); Class_getName_mid = (*env)->GetMethodID(env, Class_class, "getName", - MP_ERROR_TEXT("()Ljava/lang/String;")); + MP_COMPRESSED_ROM_TEXT("()Ljava/lang/String;")); Class_getField_mid = (*env)->GetMethodID(env, Class_class, "getField", - MP_ERROR_TEXT("(Ljava/lang/String;)Ljava/lang/reflect/Field;")); + MP_COMPRESSED_ROM_TEXT("(Ljava/lang/String;)Ljava/lang/reflect/Field;")); Class_getMethods_mid = (*env)->GetMethodID(env, Class_class, "getMethods", - MP_ERROR_TEXT("()[Ljava/lang/reflect/Method;")); + MP_COMPRESSED_ROM_TEXT("()[Ljava/lang/reflect/Method;")); Class_getConstructors_mid = (*env)->GetMethodID(env, Class_class, "getConstructors", - MP_ERROR_TEXT("()[Ljava/lang/reflect/Constructor;")); + MP_COMPRESSED_ROM_TEXT("()[Ljava/lang/reflect/Constructor;")); Method_getName_mid = (*env)->GetMethodID(env, method_class, "getName", - MP_ERROR_TEXT("()Ljava/lang/String;")); + MP_COMPRESSED_ROM_TEXT("()Ljava/lang/String;")); List_class = JJ(FindClass, "java/util/List"); List_get_mid = JJ(GetMethodID, List_class, "get", - MP_ERROR_TEXT("(I)Ljava/lang/Object;")); + MP_COMPRESSED_ROM_TEXT("(I)Ljava/lang/Object;")); List_set_mid = JJ(GetMethodID, List_class, "set", - MP_ERROR_TEXT("(ILjava/lang/Object;)Ljava/lang/Object;")); + MP_COMPRESSED_ROM_TEXT("(ILjava/lang/Object;)Ljava/lang/Object;")); List_size_mid = JJ(GetMethodID, List_class, "size", - MP_ERROR_TEXT("()I")); + MP_COMPRESSED_ROM_TEXT("()I")); IndexException_class = JJ(FindClass, "java/lang/IndexOutOfBoundsException"); } diff --git a/ports/unix/modmachine.c b/ports/unix/modmachine.c index 542ff50025..dd3cbf96c0 100644 --- a/ports/unix/modmachine.c +++ b/ports/unix/modmachine.c @@ -86,7 +86,7 @@ MP_DEFINE_CONST_FUN_OBJ_0(machine_idle_obj, machine_idle); #endif STATIC const mp_rom_map_elem_t machine_module_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_umachine) }, + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_machine) }, { MP_ROM_QSTR(MP_QSTR_mem8), MP_ROM_PTR(&machine_mem8_obj) }, { MP_ROM_QSTR(MP_QSTR_mem16), MP_ROM_PTR(&machine_mem16_obj) }, @@ -110,6 +110,6 @@ const mp_obj_module_t mp_module_machine = { .globals = (mp_obj_dict_t *)&machine_module_globals, }; -MP_REGISTER_MODULE(MP_QSTR_umachine, mp_module_machine); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_machine, mp_module_machine); #endif // MICROPY_PY_MACHINE diff --git a/ports/unix/moduos.c b/ports/unix/modos.c similarity index 78% rename from ports/unix/moduos.c rename to ports/unix/modos.c index e99c53ff62..00c8c79c5e 100644 --- a/ports/unix/moduos.c +++ b/ports/unix/modos.c @@ -40,7 +40,7 @@ os_getenv_err_t common_hal_os_getenv_str(const char *key, char *value, size_t va os_getenv_err_t common_hal_os_getenv_int(const char *key, mp_int_t *value); #endif -STATIC mp_obj_t mp_uos_getenv(size_t n_args, const mp_obj_t *args) { +STATIC mp_obj_t mp_os_getenv(size_t n_args, const mp_obj_t *args) { mp_obj_t var_in = args[0]; #if defined(MICROPY_UNIX_COVERAGE) mp_obj_t result = common_hal_os_getenv(mp_obj_str_get_str(var_in), mp_const_none); @@ -50,14 +50,18 @@ STATIC mp_obj_t mp_uos_getenv(size_t n_args, const mp_obj_t *args) { #endif const char *s = getenv(mp_obj_str_get_str(var_in)); if (s == NULL) { - return n_args > 1 ? args[1] : mp_const_none; + if (n_args == 2) { + return args[1]; + } + return mp_const_none; } return mp_obj_new_str(s, strlen(s)); } -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_uos_getenv_obj, 1, 2, mp_uos_getenv); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_os_getenv_obj, 1, 2, mp_os_getenv); +// CIRCUITPY getenv differences #if defined(MICROPY_UNIX_COVERAGE) -STATIC mp_obj_t mp_uos_getenv_int(mp_obj_t var_in) { +STATIC mp_obj_t mp_os_getenv_int(mp_obj_t var_in) { mp_int_t value; os_getenv_err_t result = common_hal_os_getenv_int(mp_obj_str_get_str(var_in), &value); if (result == 0) { @@ -65,9 +69,9 @@ STATIC mp_obj_t mp_uos_getenv_int(mp_obj_t var_in) { } return mp_const_none; } -MP_DEFINE_CONST_FUN_OBJ_1(mp_uos_getenv_int_obj, mp_uos_getenv_int); +MP_DEFINE_CONST_FUN_OBJ_1(mp_os_getenv_int_obj, mp_os_getenv_int); -STATIC mp_obj_t mp_uos_getenv_str(mp_obj_t var_in) { +STATIC mp_obj_t mp_os_getenv_str(mp_obj_t var_in) { char buf[4096]; os_getenv_err_t result = common_hal_os_getenv_str(mp_obj_str_get_str(var_in), buf, sizeof(buf)); if (result == 0) { @@ -75,10 +79,10 @@ STATIC mp_obj_t mp_uos_getenv_str(mp_obj_t var_in) { } return mp_const_none; } -MP_DEFINE_CONST_FUN_OBJ_1(mp_uos_getenv_str_obj, mp_uos_getenv_str); +MP_DEFINE_CONST_FUN_OBJ_1(mp_os_getenv_str_obj, mp_os_getenv_str); #endif -STATIC mp_obj_t mp_uos_putenv(mp_obj_t key_in, mp_obj_t value_in) { +STATIC mp_obj_t mp_os_putenv(mp_obj_t key_in, mp_obj_t value_in) { const char *key = mp_obj_str_get_str(key_in); const char *value = mp_obj_str_get_str(value_in); int ret; @@ -94,9 +98,9 @@ STATIC mp_obj_t mp_uos_putenv(mp_obj_t key_in, mp_obj_t value_in) { } return mp_const_none; } -STATIC MP_DEFINE_CONST_FUN_OBJ_2(mp_uos_putenv_obj, mp_uos_putenv); +STATIC MP_DEFINE_CONST_FUN_OBJ_2(mp_os_putenv_obj, mp_os_putenv); -STATIC mp_obj_t mp_uos_unsetenv(mp_obj_t key_in) { +STATIC mp_obj_t mp_os_unsetenv(mp_obj_t key_in) { const char *key = mp_obj_str_get_str(key_in); int ret; @@ -111,9 +115,9 @@ STATIC mp_obj_t mp_uos_unsetenv(mp_obj_t key_in) { } return mp_const_none; } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_uos_unsetenv_obj, mp_uos_unsetenv); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_os_unsetenv_obj, mp_os_unsetenv); -STATIC mp_obj_t mp_uos_system(mp_obj_t cmd_in) { +STATIC mp_obj_t mp_os_system(mp_obj_t cmd_in) { const char *cmd = mp_obj_str_get_str(cmd_in); MP_THREAD_GIL_EXIT(); @@ -124,18 +128,18 @@ STATIC mp_obj_t mp_uos_system(mp_obj_t cmd_in) { return MP_OBJ_NEW_SMALL_INT(r); } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_uos_system_obj, mp_uos_system); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_os_system_obj, mp_os_system); -STATIC mp_obj_t mp_uos_urandom(mp_obj_t num) { +STATIC mp_obj_t mp_os_urandom(mp_obj_t num) { mp_int_t n = mp_obj_get_int(num); vstr_t vstr; vstr_init_len(&vstr, n); mp_hal_get_random(n, vstr.buf); return mp_obj_new_bytes_from_vstr(&vstr); } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_uos_urandom_obj, mp_uos_urandom); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_os_urandom_obj, mp_os_urandom); -STATIC mp_obj_t mp_uos_errno(size_t n_args, const mp_obj_t *args) { +STATIC mp_obj_t mp_os_errno(size_t n_args, const mp_obj_t *args) { if (n_args == 0) { return MP_OBJ_NEW_SMALL_INT(errno); } diff --git a/ports/unix/modtime.c b/ports/unix/modtime.c index cec8baa8cd..c9c98243e4 100644 --- a/ports/unix/modtime.c +++ b/ports/unix/modtime.c @@ -4,7 +4,7 @@ * The MIT License (MIT) * * Copyright (c) 2014-2017 Paul Sokolovsky - * Copyright (c) 2014-2017 Damien P. George + * Copyright (c) 2014-2023 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -25,9 +25,6 @@ * THE SOFTWARE. */ -#include "py/mpconfig.h" -#if MICROPY_PY_UTIME - #include #include #include @@ -35,10 +32,8 @@ #include #include -#include "py/runtime.h" -#include "py/smallint.h" #include "py/mphal.h" -#include "extmod/utime_mphal.h" +#include "py/runtime.h" #ifdef _WIN32 static inline int msec_sleep_tv(struct timeval *tv) { @@ -66,7 +61,7 @@ static inline int msec_sleep_tv(struct timeval *tv) { #error Unsupported clock() implementation #endif -STATIC mp_obj_t mod_time_time(void) { +STATIC mp_obj_t mp_time_time_get(void) { #if MICROPY_PY_BUILTINS_FLOAT && MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE struct timeval tv; gettimeofday(&tv, NULL); @@ -91,7 +86,7 @@ STATIC mp_obj_t mod_time_clock(void) { } STATIC MP_DEFINE_CONST_FUN_OBJ_0(mod_time_clock_obj, mod_time_clock); -STATIC mp_obj_t mod_time_sleep(mp_obj_t arg) { +STATIC mp_obj_t mp_time_sleep(mp_obj_t arg) { #if MICROPY_PY_BUILTINS_FLOAT struct timeval tv; mp_float_t val = mp_obj_get_float(arg); diff --git a/ports/unix/moduselect.c b/ports/unix/moduselect.c deleted file mode 100644 index dbbc86a053..0000000000 --- a/ports/unix/moduselect.c +++ /dev/null @@ -1,356 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2014 Damien P. George - * Copyright (c) 2015-2017 Paul Sokolovsky - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "py/mpconfig.h" - -#if MICROPY_PY_USELECT_POSIX - -#if MICROPY_PY_USELECT -#error "Can't have both MICROPY_PY_USELECT and MICROPY_PY_USELECT_POSIX." -#endif - -#include -#include -#include - -#include "py/runtime.h" -#include "py/stream.h" -#include "py/obj.h" -#include "py/objlist.h" -#include "py/objtuple.h" -#include "py/mphal.h" -#include "py/mpthread.h" - -#define DEBUG 0 - -#if MICROPY_PY_SOCKET -extern const mp_obj_type_t mp_type_socket; -#endif - -// Flags for poll() -#define FLAG_ONESHOT (1) - -/// \class Poll - poll class - -typedef struct _mp_obj_poll_t { - mp_obj_base_t base; - unsigned short alloc; - unsigned short len; - struct pollfd *entries; - mp_obj_t *obj_map; - short iter_cnt; - short iter_idx; - int flags; - // callee-owned tuple - mp_obj_t ret_tuple; -} mp_obj_poll_t; - -STATIC int get_fd(mp_obj_t fdlike) { - if (mp_obj_is_obj(fdlike)) { - const mp_stream_p_t *stream_p = mp_get_stream_raise(fdlike, MP_STREAM_OP_IOCTL); - int err; - mp_uint_t res = stream_p->ioctl(fdlike, MP_STREAM_GET_FILENO, 0, &err); - if (res != MP_STREAM_ERROR) { - return res; - } - } - return mp_obj_get_int(fdlike); -} - -/// \method register(obj[, eventmask]) -STATIC mp_obj_t poll_register(size_t n_args, const mp_obj_t *args) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); - bool is_fd = mp_obj_is_int(args[1]); - int fd = get_fd(args[1]); - - mp_uint_t flags; - if (n_args == 3) { - flags = mp_obj_get_int(args[2]); - } else { - flags = POLLIN | POLLOUT; - } - - struct pollfd *free_slot = NULL; - - struct pollfd *entry = self->entries; - for (int i = 0; i < self->len; i++, entry++) { - int entry_fd = entry->fd; - if (entry_fd == fd) { - entry->events = flags; - return mp_const_false; - } - if (entry_fd == -1) { - free_slot = entry; - } - } - - if (free_slot == NULL) { - if (self->len >= self->alloc) { - self->entries = m_renew(struct pollfd, self->entries, self->alloc, self->alloc + 4); - if (self->obj_map) { - self->obj_map = m_renew(mp_obj_t, self->obj_map, self->alloc, self->alloc + 4); - } - self->alloc += 4; - } - free_slot = &self->entries[self->len++]; - } - - if (!is_fd) { - if (self->obj_map == NULL) { - self->obj_map = m_new0(mp_obj_t, self->alloc); - } - self->obj_map[free_slot - self->entries] = args[1]; - } - - free_slot->fd = fd; - free_slot->events = flags; - free_slot->revents = 0; - return mp_const_true; -} -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_register_obj, 2, 3, poll_register); - -/// \method unregister(obj) -STATIC mp_obj_t poll_unregister(mp_obj_t self_in, mp_obj_t obj_in) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in); - struct pollfd *entries = self->entries; - int fd = get_fd(obj_in); - for (int i = self->len - 1; i >= 0; i--) { - if (entries->fd == fd) { - entries->fd = -1; - if (self->obj_map) { - self->obj_map[entries - self->entries] = MP_OBJ_NULL; - } - break; - } - entries++; - } - - // TODO raise KeyError if obj didn't exist in map - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_2(poll_unregister_obj, poll_unregister); - -/// \method modify(obj, eventmask) -STATIC mp_obj_t poll_modify(mp_obj_t self_in, mp_obj_t obj_in, mp_obj_t eventmask_in) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in); - struct pollfd *entries = self->entries; - int fd = get_fd(obj_in); - for (int i = self->len - 1; i >= 0; i--) { - if (entries->fd == fd) { - entries->events = mp_obj_get_int(eventmask_in); - return mp_const_none; - } - entries++; - } - - // obj doesn't exist in poller - mp_raise_OSError(MP_ENOENT); -} -MP_DEFINE_CONST_FUN_OBJ_3(poll_modify_obj, poll_modify); - -STATIC int poll_poll_internal(size_t n_args, const mp_obj_t *args) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); - - // work out timeout (it's given already in ms) - int timeout = -1; - int flags = 0; - if (n_args >= 2) { - if (args[1] != mp_const_none) { - mp_int_t timeout_i = mp_obj_get_int(args[1]); - if (timeout_i >= 0) { - timeout = timeout_i; - } - } - if (n_args >= 3) { - flags = mp_obj_get_int(args[2]); - } - } - - self->flags = flags; - - int n_ready; - MP_HAL_RETRY_SYSCALL(n_ready, poll(self->entries, self->len, timeout), mp_raise_OSError(err)); - return n_ready; -} - -/// \method poll([timeout]) -/// Timeout is in milliseconds. -STATIC mp_obj_t poll_poll(size_t n_args, const mp_obj_t *args) { - int n_ready = poll_poll_internal(n_args, args); - - if (n_ready == 0) { - return mp_const_empty_tuple; - } - - mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); - - mp_obj_list_t *ret_list = MP_OBJ_TO_PTR(mp_obj_new_list(n_ready, NULL)); - int ret_i = 0; - struct pollfd *entries = self->entries; - for (int i = 0; i < self->len; i++, entries++) { - if (entries->revents != 0) { - mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(2, NULL)); - // If there's an object stored, return it, otherwise raw fd - if (self->obj_map && self->obj_map[i] != MP_OBJ_NULL) { - t->items[0] = self->obj_map[i]; - } else { - t->items[0] = MP_OBJ_NEW_SMALL_INT(entries->fd); - } - t->items[1] = MP_OBJ_NEW_SMALL_INT(entries->revents); - ret_list->items[ret_i++] = MP_OBJ_FROM_PTR(t); - if (self->flags & FLAG_ONESHOT) { - entries->events = 0; - } - } - } - - return MP_OBJ_FROM_PTR(ret_list); -} -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_poll_obj, 1, 3, poll_poll); - -STATIC mp_obj_t poll_ipoll(size_t n_args, const mp_obj_t *args) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); - - if (self->ret_tuple == MP_OBJ_NULL) { - self->ret_tuple = mp_obj_new_tuple(2, NULL); - } - - int n_ready = poll_poll_internal(n_args, args); - self->iter_cnt = n_ready; - self->iter_idx = 0; - - return args[0]; -} -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_ipoll_obj, 1, 3, poll_ipoll); - -STATIC mp_obj_t poll_iternext(mp_obj_t self_in) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in); - - if (self->iter_cnt == 0) { - return MP_OBJ_STOP_ITERATION; - } - - self->iter_cnt--; - - struct pollfd *entries = self->entries + self->iter_idx; - for (int i = self->iter_idx; i < self->len; i++, entries++) { - self->iter_idx++; - if (entries->revents != 0) { - mp_obj_tuple_t *t = MP_OBJ_TO_PTR(self->ret_tuple); - // If there's an object stored, return it, otherwise raw fd - if (self->obj_map && self->obj_map[i] != MP_OBJ_NULL) { - t->items[0] = self->obj_map[i]; - } else { - t->items[0] = MP_OBJ_NEW_SMALL_INT(entries->fd); - } - t->items[1] = MP_OBJ_NEW_SMALL_INT(entries->revents); - if (self->flags & FLAG_ONESHOT) { - entries->events = 0; - } - return MP_OBJ_FROM_PTR(t); - } - } - - assert(!"inconsistent number of poll active entries"); - self->iter_cnt = 0; - return MP_OBJ_STOP_ITERATION; -} - -#if DEBUG -STATIC mp_obj_t poll_dump(mp_obj_t self_in) { - mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in); - - struct pollfd *entries = self->entries; - for (int i = self->len - 1; i >= 0; i--) { - printf("fd: %d ev: %x rev: %x", entries->fd, entries->events, entries->revents); - if (self->obj_map) { - printf(" obj: %p", self->obj_map[entries - self->entries]); - } - printf("\n"); - entries++; - } - - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_1(poll_dump_obj, poll_dump); -#endif - -STATIC const mp_rom_map_elem_t poll_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR_register), MP_ROM_PTR(&poll_register_obj) }, - { MP_ROM_QSTR(MP_QSTR_unregister), MP_ROM_PTR(&poll_unregister_obj) }, - { MP_ROM_QSTR(MP_QSTR_modify), MP_ROM_PTR(&poll_modify_obj) }, - { MP_ROM_QSTR(MP_QSTR_poll), MP_ROM_PTR(&poll_poll_obj) }, - { MP_ROM_QSTR(MP_QSTR_ipoll), MP_ROM_PTR(&poll_ipoll_obj) }, - #if DEBUG - { MP_ROM_QSTR(MP_QSTR_dump), MP_ROM_PTR(&poll_dump_obj) }, - #endif -}; -STATIC MP_DEFINE_CONST_DICT(poll_locals_dict, poll_locals_dict_table); - -STATIC MP_DEFINE_CONST_OBJ_TYPE( - mp_type_poll, - MP_QSTR_poll, - MP_TYPE_FLAG_ITER_IS_ITERNEXT, - iter, poll_iternext, - locals_dict, &poll_locals_dict - ); - -STATIC mp_obj_t select_poll(size_t n_args, const mp_obj_t *args) { - int alloc = 4; - if (n_args > 0) { - alloc = mp_obj_get_int(args[0]); - } - mp_obj_poll_t *poll = mp_obj_malloc(mp_obj_poll_t, &mp_type_poll); - poll->entries = m_new(struct pollfd, alloc); - poll->alloc = alloc; - poll->len = 0; - poll->obj_map = NULL; - poll->iter_cnt = 0; - poll->ret_tuple = MP_OBJ_NULL; - return MP_OBJ_FROM_PTR(poll); -} -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_select_poll_obj, 0, 1, select_poll); - -STATIC const mp_rom_map_elem_t mp_module_select_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uselect) }, - { MP_ROM_QSTR(MP_QSTR_poll), MP_ROM_PTR(&mp_select_poll_obj) }, - { MP_ROM_QSTR(MP_QSTR_POLLIN), MP_ROM_INT(POLLIN) }, - { MP_ROM_QSTR(MP_QSTR_POLLOUT), MP_ROM_INT(POLLOUT) }, - { MP_ROM_QSTR(MP_QSTR_POLLERR), MP_ROM_INT(POLLERR) }, - { MP_ROM_QSTR(MP_QSTR_POLLHUP), MP_ROM_INT(POLLHUP) }, -}; - -STATIC MP_DEFINE_CONST_DICT(mp_module_select_globals, mp_module_select_globals_table); - -const mp_obj_module_t mp_module_uselect = { - .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&mp_module_select_globals, -}; - -MP_REGISTER_MODULE(MP_QSTR_select, mp_module_uselect); - -#endif // MICROPY_PY_USELECT_POSIX diff --git a/ports/unix/mpconfigport.h b/ports/unix/mpconfigport.h index 86ec15b822..e24d869e03 100644 --- a/ports/unix/mpconfigport.h +++ b/ports/unix/mpconfigport.h @@ -140,12 +140,6 @@ typedef long mp_off_t; #define MICROPY_STACKLESS_STRICT (0) #endif -// If settrace is enabled then we need code saving. -#if MICROPY_PY_SYS_SETTRACE -#define MICROPY_PERSISTENT_CODE_SAVE (1) -#define MICROPY_COMP_CONST (0) -#endif - // Unix-specific configuration of machine.mem*. #define MICROPY_MACHINE_MEM_GET_READ_ADDR mod_machine_mem_get_addr #define MICROPY_MACHINE_MEM_GET_WRITE_ADDR mod_machine_mem_get_addr @@ -162,13 +156,13 @@ typedef long mp_off_t; // Ensure builtinimport.c works with -m. #define MICROPY_MODULE_OVERRIDE_MAIN_IMPORT (1) -// Don't default sys.argv because we do that in main. +// Don't default sys.argv and sys.path because we do that in main. #define MICROPY_PY_SYS_PATH_ARGV_DEFAULTS (0) // Enable sys.executable. #define MICROPY_PY_SYS_EXECUTABLE (1) -#define MICROPY_PY_USOCKET_LISTEN_BACKLOG_DEFAULT (SOMAXCONN < 128 ? SOMAXCONN : 128) +#define MICROPY_PY_SOCKET_LISTEN_BACKLOG_DEFAULT (SOMAXCONN < 128 ? SOMAXCONN : 128) // Bare-metal ports don't have stderr. Printing debug to stderr may give tests // which check stdout a chance to pass, etc. @@ -189,10 +183,10 @@ void mp_unix_mark_exec(void); #endif // If enabled, configure how to seed random on init. -#ifdef MICROPY_PY_URANDOM_SEED_INIT_FUNC +#ifdef MICROPY_PY_RANDOM_SEED_INIT_FUNC #include void mp_hal_get_random(size_t n, void *buf); -static inline unsigned long mp_urandom_seed_init(void) { +static inline unsigned long mp_random_seed_init(void) { unsigned long r; mp_hal_get_random(sizeof(r), &r); return r; diff --git a/ports/unix/mpconfigport.mk b/ports/unix/mpconfigport.mk index 2bdd87534e..d278f14123 100644 --- a/ports/unix/mpconfigport.mk +++ b/ports/unix/mpconfigport.mk @@ -8,8 +8,8 @@ MICROPY_FORCE_32BIT = 0 # 1 - use MicroPython version of readline MICROPY_USE_READLINE = 1 -# btree module using Berkeley DB 1.xx # CIRCUITPY: not present +# btree module using Berkeley DB 1.xx MICROPY_PY_BTREE = 0 # _thread module using pthreads @@ -18,19 +18,22 @@ MICROPY_PY_THREAD = 1 # Subset of CPython termios module MICROPY_PY_TERMIOS = 1 +# CIRCUITPY: not present # Subset of CPython socket module MICROPY_PY_SOCKET = 0 # ffi module requires libffi (libffi-dev Debian package) MICROPY_PY_FFI = 1 -# ussl module requires one of the TLS libraries below -MICROPY_PY_USSL = 0 +# CIRCUITPY: not present +# ssl module requires one of the TLS libraries below +MICROPY_PY_SSL = 0 # axTLS has minimal size but implements only a subset of modern TLS # functionality, so may have problems with some servers. MICROPY_SSL_AXTLS = 0 # mbedTLS is more up to date and complete implementation, but also # more bloated. +# CIRCUITPY: not present MICROPY_SSL_MBEDTLS = 0 # jni module requires JVM/JNI diff --git a/ports/unix/mpthreadport.c b/ports/unix/mpthreadport.c index 6a267e7236..2190bf4ad1 100644 --- a/ports/unix/mpthreadport.c +++ b/ports/unix/mpthreadport.c @@ -191,6 +191,10 @@ void mp_thread_set_state(mp_state_thread_t *state) { pthread_setspecific(tls_key, state); } +mp_uint_t mp_thread_get_id(void) { + return (mp_uint_t)pthread_self(); +} + void mp_thread_start(void) { // enable realtime priority if `-X realtime` command line parameter was set #if defined(__APPLE__) @@ -210,7 +214,7 @@ void mp_thread_start(void) { mp_thread_unix_end_atomic_section(); } -void mp_thread_create(void *(*entry)(void *), void *arg, size_t *stack_size) { +mp_uint_t mp_thread_create(void *(*entry)(void *), void *arg, size_t *stack_size) { // default stack size is 8k machine-words if (*stack_size == 0) { *stack_size = 8192 * sizeof(void *); @@ -265,7 +269,8 @@ void mp_thread_create(void *(*entry)(void *), void *arg, size_t *stack_size) { mp_thread_unix_end_atomic_section(); - return; + MP_STATIC_ASSERT(sizeof(mp_uint_t) >= sizeof(pthread_t)); + return (mp_uint_t)id; er: mp_raise_OSError(ret); diff --git a/ports/unix/unix_mphal.c b/ports/unix/unix_mphal.c index a385121775..4626c2f0c0 100644 --- a/ports/unix/unix_mphal.c +++ b/ports/unix/unix_mphal.c @@ -192,8 +192,9 @@ main_term:; void mp_hal_stdout_tx_strn(const char *str, size_t len) { ssize_t ret; MP_HAL_RETRY_SYSCALL(ret, write(STDOUT_FILENO, str, len), {}); + // CIRCUITPY: need to conditionalize MICROPY_PY_OS_DUPTERM #if MICROPY_PY_OS_DUPTERM - mp_uos_dupterm_tx_strn(str, len); + mp_os_dupterm_tx_strn(str, len); #endif } @@ -262,7 +263,7 @@ void mp_hal_get_random(size_t n, void *buf) { #ifdef _HAVE_GETRANDOM RAISE_ERRNO(getrandom(buf, n, 0), errno); #else - int fd = open("/dev/urandom", O_RDONLY); + int fd = open("/dev/random", O_RDONLY); RAISE_ERRNO(fd, errno); RAISE_ERRNO(read(fd, buf, n), errno); close(fd); diff --git a/ports/unix/variants/coverage/mpconfigvariant.h b/ports/unix/variants/coverage/mpconfigvariant.h index 72c25263b2..5d0bfaf058 100644 --- a/ports/unix/variants/coverage/mpconfigvariant.h +++ b/ports/unix/variants/coverage/mpconfigvariant.h @@ -33,13 +33,17 @@ // Enable extra Unix features. #include "../mpconfigvariant_common.h" +// Enable testing of split heap. +#define MICROPY_GC_SPLIT_HEAP (1) +#define MICROPY_GC_SPLIT_HEAP_N_HEAPS (4) + // Enable additional features. #define MICROPY_DEBUG_PARSE_RULE_NAME (1) #define MICROPY_TRACKED_ALLOC (1) #define MICROPY_WARNINGS_CATEGORY (1) -// Disable things never used in circuitpython -#define MICROPY_PY_UCRYPTOLIB (0) -#define MICROPY_PY_UCRYPTOLIB_CTR (0) +// CIRCUITPY Disable things never used in circuitpython +#define MICROPY_PY_CRYPTOLIB (0) +#define MICROPY_PY_CRYPTOLIB_CTR (0) #define MICROPY_PY_STRUCT (0) // uses shared-bindings struct #define MICROPY_GC_SPLIT_HEAP (0) diff --git a/ports/unix/variants/minimal/mpconfigvariant.h b/ports/unix/variants/minimal/mpconfigvariant.h index 6b107e7790..0dbfbb3d1c 100644 --- a/ports/unix/variants/minimal/mpconfigvariant.h +++ b/ports/unix/variants/minimal/mpconfigvariant.h @@ -63,7 +63,7 @@ // Enable just the sys and os built-in modules. #define MICROPY_PY_SYS (1) -#define MICROPY_PY_UOS (1) +#define MICROPY_PY_OS (1) // The minimum sets this to 1 to save flash. #define MICROPY_QSTR_BYTES_IN_HASH (2) diff --git a/ports/unix/variants/minimal/mpconfigvariant.mk b/ports/unix/variants/minimal/mpconfigvariant.mk index d5c2a52e9a..19b3460ed5 100644 --- a/ports/unix/variants/minimal/mpconfigvariant.mk +++ b/ports/unix/variants/minimal/mpconfigvariant.mk @@ -7,7 +7,7 @@ MICROPY_PY_FFI = 0 MICROPY_PY_SOCKET = 0 MICROPY_PY_THREAD = 0 MICROPY_PY_TERMIOS = 0 -MICROPY_PY_USSL = 0 +MICROPY_PY_SSL = 0 MICROPY_USE_READLINE = 0 MICROPY_VFS_FAT = 0 diff --git a/ports/unix/variants/mpconfigvariant_common.h b/ports/unix/variants/mpconfigvariant_common.h index de3411087c..082938ed5f 100644 --- a/ports/unix/variants/mpconfigvariant_common.h +++ b/ports/unix/variants/mpconfigvariant_common.h @@ -58,7 +58,7 @@ #endif // Seed random on import. -#define MICROPY_PY_URANDOM_SEED_INIT_FUNC (mp_urandom_seed_init()) +#define MICROPY_PY_RANDOM_SEED_INIT_FUNC (mp_random_seed_init()) // Allow exception details in low-memory conditions. #define MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF (1) @@ -87,37 +87,31 @@ #define MICROPY_PY_SYS_EXC_INFO (1) // Configure the "os" module with extra unix features. -#define MICROPY_PY_UOS_INCLUDEFILE "ports/unix/moduos.c" -#define MICROPY_PY_UOS_ERRNO (1) -#define MICROPY_PY_UOS_GETENV_PUTENV_UNSETENV (1) -#define MICROPY_PY_UOS_SEP (1) -#define MICROPY_PY_UOS_SYSTEM (1) -#define MICROPY_PY_UOS_URANDOM (1) +#define MICROPY_PY_OS_INCLUDEFILE "ports/unix/modos.c" +#define MICROPY_PY_OS_ERRNO (1) +#define MICROPY_PY_OS_GETENV_PUTENV_UNSETENV (1) +#define MICROPY_PY_OS_SEP (1) +#define MICROPY_PY_OS_SYSTEM (1) +#define MICROPY_PY_OS_URANDOM (1) // Enable the unix-specific "time" module. -#define MICROPY_PY_UTIME (1) -#define MICROPY_PY_UTIME_MP_HAL (1) +#define MICROPY_PY_TIME (1) +#define MICROPY_PY_TIME_TIME_TIME_NS (1) +#define MICROPY_PY_TIME_CUSTOM_SLEEP (1) +#define MICROPY_PY_TIME_INCLUDEFILE "ports/unix/modtime.c" -// Enable the utimeq module used by the previous (v2) version of uasyncio. -#define MICROPY_PY_UTIMEQ (1) - -#if MICROPY_PY_USSL -#define MICROPY_PY_UHASHLIB_MD5 (1) -#define MICROPY_PY_UHASHLIB_SHA1 (1) -#define MICROPY_PY_UCRYPTOLIB (1) +#if MICROPY_PY_SSL +#define MICROPY_PY_HASHLIB_MD5 (1) +#define MICROPY_PY_HASHLIB_SHA1 (1) +#define MICROPY_PY_CRYPTOLIB (1) #endif -// Use the posix implementation of the "select" module (unless the variant -// specifically asks for the MicroPython version). -#ifndef MICROPY_PY_USELECT -#define MICROPY_PY_USELECT (0) -#endif -#ifndef MICROPY_PY_USELECT_POSIX -#define MICROPY_PY_USELECT_POSIX (!MICROPY_PY_USELECT) -#endif +// The "select" module is enabled by default, but disable select.select(). +#define MICROPY_PY_SELECT_POSIX_OPTIMISATIONS (1) +#define MICROPY_PY_SELECT_SELECT (0) // Enable the "websocket" module. -#define MICROPY_PY_UWEBSOCKET (1) +#define MICROPY_PY_WEBSOCKET (1) // Enable the "machine" module, mostly for machine.mem*. #define MICROPY_PY_MACHINE (1) diff --git a/py/asmthumb.c b/py/asmthumb.c index 49574c43a7..395134028a 100644 --- a/py/asmthumb.c +++ b/py/asmthumb.c @@ -290,7 +290,7 @@ bool asm_thumb_b_n_label(asm_thumb_t *as, uint label) { #define OP_BCC_N(cond, byte_offset) (0xd000 | ((cond) << 8) | (((byte_offset) >> 1) & 0x00ff)) -// all these bit arithmetics need coverage testing! +// all these bit-arithmetic operations need coverage testing! #define OP_BCC_W_HI(cond, byte_offset) (0xf000 | ((cond) << 6) | (((byte_offset) >> 10) & 0x0400) | (((byte_offset) >> 14) & 0x003f)) #define OP_BCC_W_LO(byte_offset) (0x8000 | ((byte_offset) & 0x2000) | (((byte_offset) >> 1) & 0x0fff)) diff --git a/py/asmx86.h b/py/asmx86.h index af7bb84988..f344c78f09 100644 --- a/py/asmx86.h +++ b/py/asmx86.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George + * Copyright (c) 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/builtin.h b/py/builtin.h index 7232142b77..81d0789802 100644 --- a/py/builtin.h +++ b/py/builtin.h @@ -133,7 +133,7 @@ extern const mp_obj_module_t mp_module_builtins; extern const mp_obj_module_t mp_module_sys; // Modules needed by the parser when MICROPY_COMP_MODULE_CONST is enabled. -extern const mp_obj_module_t mp_module_uerrno; +extern const mp_obj_module_t mp_module_errno; extern const mp_obj_module_t mp_module_uctypes; extern const mp_obj_module_t mp_module_machine; diff --git a/py/builtinevex.c b/py/builtinevex.c index 173978ef52..97eab7fad9 100644 --- a/py/builtinevex.c +++ b/py/builtinevex.c @@ -45,12 +45,18 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE( ); STATIC mp_obj_t code_execute(mp_obj_code_t *self, mp_obj_dict_t *globals, mp_obj_dict_t *locals) { - // save context and set new context - mp_obj_dict_t *old_globals = mp_globals_get(); - mp_obj_dict_t *old_locals = mp_locals_get(); + // save context + nlr_jump_callback_node_globals_locals_t ctx; + ctx.globals = mp_globals_get(); + ctx.locals = mp_locals_get(); + + // set new context mp_globals_set(globals); mp_locals_set(locals); + // set exception handler to restore context if an exception is raised + nlr_push_jump_callback(&ctx.callback, mp_globals_locals_set_from_nlr_jump_callback); + // a bit of a hack: fun_bc will re-set globals, so need to make sure it's // the correct one if (mp_obj_is_type(self->module_fun, &mp_type_fun_bc)) { @@ -59,19 +65,13 @@ STATIC mp_obj_t code_execute(mp_obj_code_t *self, mp_obj_dict_t *globals, mp_obj } // execute code - nlr_buf_t nlr; - if (nlr_push(&nlr) == 0) { - mp_obj_t ret = mp_call_function_0(self->module_fun); - nlr_pop(); - mp_globals_set(old_globals); - mp_locals_set(old_locals); - return ret; - } else { - // exception; restore context and re-raise same exception - mp_globals_set(old_globals); - mp_locals_set(old_locals); - nlr_jump(nlr.ret_val); - } + mp_obj_t ret = mp_call_function_0(self->module_fun); + + // deregister exception handler and restore context + nlr_pop_jump_callback(true); + + // return value + return ret; } STATIC mp_obj_t mp_builtin_compile(size_t n_args, const mp_obj_t *args) { diff --git a/py/builtinhelp.c b/py/builtinhelp.c index fc922935ed..579b8887bb 100644 --- a/py/builtinhelp.c +++ b/py/builtinhelp.c @@ -77,18 +77,14 @@ STATIC void mp_help_add_from_names(mp_obj_t list, const char *name) { } #endif -// These externs were originally declared inside mp_help_print_modules(), -// but they triggered -Wnested-externs, so they were moved outside. -#if MICROPY_MODULE_FROZEN -extern const char mp_frozen_names[]; -#endif - STATIC void mp_help_print_modules(void) { mp_obj_t list = mp_obj_new_list(0, NULL); mp_help_add_from_map(list, &mp_builtin_module_map); + mp_help_add_from_map(list, &mp_builtin_extensible_module_map); #if MICROPY_MODULE_FROZEN + extern const char mp_frozen_names[]; mp_help_add_from_names(list, mp_frozen_names); #endif @@ -158,12 +154,7 @@ STATIC void mp_help_print_obj(const mp_obj_t obj) { if (map != NULL) { for (uint i = 0; i < map->alloc; i++) { mp_obj_t key = map->table[i].key; - if (key != MP_OBJ_NULL - #if MICROPY_MODULE_ATTR_DELEGATION - // MP_MODULE_ATTR_DELEGATION_ENTRY entries have MP_QSTRnull as qstr key. - && key != MP_OBJ_NEW_QSTR(MP_QSTRnull) - #endif - ) { + if (key != MP_OBJ_NULL) { mp_help_print_info_about_object(key, map->table[i].value); } } diff --git a/py/builtinimport.c b/py/builtinimport.c index a57841cd8c..a8847b7376 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -31,7 +31,6 @@ #include #include "py/compile.h" -#include "py/gc.h" #include "py/objmodule.h" #include "py/persistentcode.h" #include "py/runtime.h" @@ -46,15 +45,6 @@ #define DEBUG_printf(...) (void)0 #endif -#if MICROPY_MODULE_WEAK_LINKS -STATIC qstr make_weak_link_name(vstr_t *buffer, qstr name) { - vstr_reset(buffer); - vstr_add_char(buffer, 'u'); - vstr_add_str(buffer, qstr_str(name)); - return qstr_from_strn(buffer->buf, buffer->len); -} -#endif - #if MICROPY_ENABLE_EXTERNAL_IMPORT // Must be a string of one byte. @@ -63,39 +53,39 @@ STATIC qstr make_weak_link_name(vstr_t *buffer, qstr name) { // Virtual sys.path entry that maps to the frozen modules. #define MP_FROZEN_PATH_PREFIX ".frozen/" -bool mp_obj_is_package(mp_obj_t module) { - mp_obj_t dest[2]; - mp_load_method_maybe(module, MP_QSTR___path__, dest); - return dest[0] != MP_OBJ_NULL; -} - // Wrapper for mp_import_stat (which is provided by the port, and typically // uses mp_vfs_import_stat) to also search frozen modules. Given an exact // path to a file or directory (e.g. "foo/bar", foo/bar.py" or "foo/bar.mpy"), // will return whether the path is a file, directory, or doesn't exist. -STATIC mp_import_stat_t stat_path_or_frozen(const char *path) { +STATIC mp_import_stat_t stat_path(const char *path) { #if MICROPY_MODULE_FROZEN // Only try and load as a frozen module if it starts with .frozen/. const int frozen_path_prefix_len = strlen(MP_FROZEN_PATH_PREFIX); if (strncmp(path, MP_FROZEN_PATH_PREFIX, frozen_path_prefix_len) == 0) { + // Just stat (which is the return value), don't get the data. return mp_find_frozen_module(path + frozen_path_prefix_len, NULL, NULL); } #endif return mp_import_stat(path); } -// Given a path to a .py file, try and find this path as either a .py or .mpy -// in either the filesystem or frozen modules. +// Stat a given filesystem path to a .py file. If the file does not exist, +// then attempt to stat the corresponding .mpy file, and update the path +// argument. This is the logic that makes .py files take precedent over .mpy +// files. This uses stat_path above, rather than mp_import_stat directly, so +// that the .frozen path prefix is handled. STATIC mp_import_stat_t stat_file_py_or_mpy(vstr_t *path) { - mp_import_stat_t stat = stat_path_or_frozen(vstr_null_terminated_str(path)); + mp_import_stat_t stat = stat_path(vstr_null_terminated_str(path)); if (stat == MP_IMPORT_STAT_FILE) { return stat; } #if MICROPY_PERSISTENT_CODE_LOAD // Didn't find .py -- try the .mpy instead by inserting an 'm' into the '.py'. + // Note: There's no point doing this if it's a frozen path, but adding the check + // would be extra code, and no harm letting mp_find_frozen_module fail instead. vstr_ins_byte(path, path->len - 2, 'm'); - stat = stat_path_or_frozen(vstr_null_terminated_str(path)); + stat = stat_path(vstr_null_terminated_str(path)); if (stat == MP_IMPORT_STAT_FILE) { return stat; } @@ -105,54 +95,60 @@ STATIC mp_import_stat_t stat_file_py_or_mpy(vstr_t *path) { } // Given an import path (e.g. "foo/bar"), try and find "foo/bar" (a directory) -// or "foo/bar.(m)py" in either the filesystem or frozen modules. -STATIC mp_import_stat_t stat_dir_or_file(vstr_t *path) { - mp_import_stat_t stat = stat_path_or_frozen(vstr_null_terminated_str(path)); +// or "foo/bar.(m)py" in either the filesystem or frozen modules. If the +// result is a file, the path argument will be updated to include the file +// extension. +STATIC mp_import_stat_t stat_module(vstr_t *path) { + mp_import_stat_t stat = stat_path(vstr_null_terminated_str(path)); DEBUG_printf("stat %s: %d\n", vstr_str(path), stat); if (stat == MP_IMPORT_STAT_DIR) { return stat; } - // not a directory, add .py and try as a file + // Not a directory, add .py and try as a file. vstr_add_str(path, ".py"); return stat_file_py_or_mpy(path); } -// Given a top-level module, try and find it in each of the sys.path entries -// via stat_dir_or_file. -STATIC mp_import_stat_t stat_top_level_dir_or_file(qstr mod_name, vstr_t *dest) { - DEBUG_printf("stat_top_level_dir_or_file: '%s'\n", qstr_str(mod_name)); +// Given a top-level module name, try and find it in each of the sys.path +// entries. Note: On success, the dest argument will be updated to the matching +// path (i.e. "/mod_name(.py)"). +STATIC mp_import_stat_t stat_top_level(qstr mod_name, vstr_t *dest) { + DEBUG_printf("stat_top_level: '%s'\n", qstr_str(mod_name)); #if MICROPY_PY_SYS size_t path_num; mp_obj_t *path_items; - mp_obj_list_get(mp_sys_path, &path_num, &path_items); + mp_obj_get_array(mp_sys_path, &path_num, &path_items); - if (path_num > 0) { - // go through each path looking for a directory or file - for (size_t i = 0; i < path_num; i++) { - vstr_reset(dest); - size_t p_len; - const char *p = mp_obj_str_get_data(path_items[i], &p_len); - if (p_len > 0) { - vstr_add_strn(dest, p, p_len); - vstr_add_char(dest, PATH_SEP_CHAR[0]); - } - vstr_add_str(dest, qstr_str(mod_name)); - mp_import_stat_t stat = stat_dir_or_file(dest); - if (stat != MP_IMPORT_STAT_NO_EXIST) { - return stat; - } + // go through each sys.path entry, trying to import "/". + for (size_t i = 0; i < path_num; i++) { + vstr_reset(dest); + size_t p_len; + const char *p = mp_obj_str_get_data(path_items[i], &p_len); + if (p_len > 0) { + // Add the path separator (unless the entry is "", i.e. cwd). + vstr_add_strn(dest, p, p_len); + vstr_add_char(dest, PATH_SEP_CHAR[0]); + } + vstr_add_str(dest, qstr_str(mod_name)); + mp_import_stat_t stat = stat_module(dest); + if (stat != MP_IMPORT_STAT_NO_EXIST) { + return stat; } - - // could not find a directory or file - return MP_IMPORT_STAT_NO_EXIST; } #endif - // mp_sys_path is empty (or not enabled), so just stat the given path - // directly. + // sys.path was empty or no matches, do not search the filesystem or + // frozen code. + return MP_IMPORT_STAT_NO_EXIST; + + #else + + // mp_sys_path is not enabled, so just stat the given path directly. vstr_add_str(dest, qstr_str(mod_name)); - return stat_dir_or_file(dest); + return stat_module(dest); + + #endif } #if MICROPY_MODULE_FROZEN_STR || MICROPY_ENABLE_COMPILER @@ -180,28 +176,23 @@ STATIC void do_execute_raw_code(const mp_module_context_t *context, const mp_raw mp_obj_dict_t *mod_globals = context->module.globals; // save context - mp_obj_dict_t *volatile old_globals = mp_globals_get(); - mp_obj_dict_t *volatile old_locals = mp_locals_get(); + nlr_jump_callback_node_globals_locals_t ctx; + ctx.globals = mp_globals_get(); + ctx.locals = mp_locals_get(); // set new context mp_globals_set(mod_globals); mp_locals_set(mod_globals); - nlr_buf_t nlr; - if (nlr_push(&nlr) == 0) { - mp_obj_t module_fun = mp_make_function_from_raw_code(rc, context, NULL); - mp_call_function_0(module_fun); + // set exception handler to restore context if an exception is raised + nlr_push_jump_callback(&ctx.callback, mp_globals_locals_set_from_nlr_jump_callback); - // finish nlr block, restore context - nlr_pop(); - mp_globals_set(old_globals); - mp_locals_set(old_locals); - } else { - // exception; restore context and re-raise same exception - mp_globals_set(old_globals); - mp_locals_set(old_locals); - nlr_jump(nlr.ret_val); - } + // make and execute the function + mp_obj_t module_fun = mp_make_function_from_raw_code(rc, context, NULL); + mp_call_function_0(module_fun); + + // deregister exception handler and restore context + nlr_pop_jump_callback(true); } #endif @@ -291,7 +282,7 @@ STATIC void evaluate_relative_import(mp_int_t level, const char **module_name, s #endif // If we have a __path__ in the globals dict, then we're a package. - bool is_pkg = mp_map_lookup(&mp_globals_get()->map, MP_OBJ_NEW_QSTR(MP_QSTR___path__), MP_MAP_LOOKUP) != NULL; + bool is_pkg = mp_map_lookup(&mp_globals_get()->map, MP_OBJ_NEW_QSTR(MP_QSTR___path__), MP_MAP_LOOKUP); #if DEBUG_PRINT DEBUG_printf("Current module/package: "); @@ -347,137 +338,190 @@ STATIC void evaluate_relative_import(mp_int_t level, const char **module_name, s *module_name_len = new_module_name_len; } +typedef struct _nlr_jump_callback_node_unregister_module_t { + nlr_jump_callback_node_t callback; + qstr name; +} nlr_jump_callback_node_unregister_module_t; + +STATIC void unregister_module_from_nlr_jump_callback(void *ctx_in) { + nlr_jump_callback_node_unregister_module_t *ctx = ctx_in; + mp_map_t *mp_loaded_modules_map = &MP_STATE_VM(mp_loaded_modules_dict).map; + mp_map_lookup(mp_loaded_modules_map, MP_OBJ_NEW_QSTR(ctx->name), MP_MAP_LOOKUP_REMOVE_IF_FOUND); +} + // Load a module at the specified absolute path, possibly as a submodule of the given outer module. -// full_mod_name: The full absolute path to this module (e.g. "foo.bar.baz"). +// full_mod_name: The full absolute path up to this level (e.g. "foo.bar.baz"). // level_mod_name: The final component of the path (e.g. "baz"). // outer_module_obj: The parent module (we need to store this module as an // attribute on it) (or MP_OBJ_NULL for top-level). -// path: The filesystem path where we found the parent module -// (or empty for a top level module). // override_main: Whether to set the __name__ to "__main__" (and use __main__ // for the actual path). -STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, mp_obj_t outer_module_obj, vstr_t *path, bool override_main) { +STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, mp_obj_t outer_module_obj, bool override_main) { + // Immediately return if the module at this level is already loaded. + mp_map_elem_t *elem; + + #if MICROPY_PY_SYS + // If sys.path is empty, the intention is to force using a built-in. This + // means we should also ignore any loaded modules with the same name + // which may have come from the filesystem. + size_t path_num; + mp_obj_t *path_items; + mp_obj_get_array(mp_sys_path, &path_num, &path_items); + if (path_num) + #endif + { + elem = mp_map_lookup(&MP_STATE_VM(mp_loaded_modules_dict).map, MP_OBJ_NEW_QSTR(full_mod_name), MP_MAP_LOOKUP); + if (elem) { + return elem->value; + } + } + + VSTR_FIXED(path, MICROPY_ALLOC_PATH_MAX); mp_import_stat_t stat = MP_IMPORT_STAT_NO_EXIST; - - // Exact-match of built-in (or already-loaded) takes priority. - mp_obj_t module_obj = mp_module_get_loaded_or_builtin(full_mod_name); - - // Even if we find the module, go through the motions of searching for it - // because we may actually be in the process of importing a sub-module. - // So we need to (re-)find the correct path to be finding the sub-module - // on the next iteration of process_import_at_level. + mp_obj_t module_obj; if (outer_module_obj == MP_OBJ_NULL) { + // First module in the dotted-name path. DEBUG_printf("Searching for top-level module\n"); - // First module in the dotted-name; search for a directory or file - // relative to all the locations in sys.path. - stat = stat_top_level_dir_or_file(full_mod_name, path); + // An import of a non-extensible built-in will always bypass the + // filesystem. e.g. `import micropython` or `import pyb`. So try and + // match a non-extensible built-ins first. + module_obj = mp_module_get_builtin(level_mod_name, false); + if (module_obj != MP_OBJ_NULL) { + return module_obj; + } - // If the module "foo" doesn't exist on the filesystem, and it's not a - // builtin, try and find "ufoo" as a built-in. (This feature was - // formerly known as "weak links"). - #if MICROPY_MODULE_WEAK_LINKS - if (stat == MP_IMPORT_STAT_NO_EXIST && module_obj == MP_OBJ_NULL) { - qstr umodule_name = make_weak_link_name(path, level_mod_name); - module_obj = mp_module_get_builtin(umodule_name); + // Next try the filesystem. Search for a directory or file relative to + // all the locations in sys.path. + stat = stat_top_level(level_mod_name, &path); + + // If filesystem failed, now try and see if it matches an extensible + // built-in module. + if (stat == MP_IMPORT_STAT_NO_EXIST) { + module_obj = mp_module_get_builtin(level_mod_name, true); + if (module_obj != MP_OBJ_NULL) { + return module_obj; + } } - #elif MICROPY_PY_SYS - if (stat == MP_IMPORT_STAT_NO_EXIST && module_obj == MP_OBJ_NULL && level_mod_name == MP_QSTR_sys) { - module_obj = MP_OBJ_FROM_PTR(&mp_module_sys); - } - #endif } else { DEBUG_printf("Searching for sub-module\n"); - // Add the current part of the module name to the path. - vstr_add_char(path, PATH_SEP_CHAR[0]); - vstr_add_str(path, qstr_str(level_mod_name)); - - // Because it's not top level, we already know which path the parent was found in. - stat = stat_dir_or_file(path); - } - DEBUG_printf("Current path: %.*s\n", (int)vstr_len(path), vstr_str(path)); - - if (module_obj == MP_OBJ_NULL) { - // Not a built-in and not already-loaded. - - if (stat == MP_IMPORT_STAT_NO_EXIST) { - // And the file wasn't found -- fail. - #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE - mp_raise_msg(&mp_type_ImportError, MP_ERROR_TEXT("module not found")); - #else - mp_raise_msg_varg(&mp_type_ImportError, MP_ERROR_TEXT("no module named '%q'"), full_mod_name); - #endif - } - - // Not a built-in but found on the filesystem, try and load it. - - DEBUG_printf("Found path: %.*s\n", (int)vstr_len(path), vstr_str(path)); - - // Prepare for loading from the filesystem. Create a new shell module. - module_obj = mp_obj_new_module(full_mod_name); - - #if MICROPY_MODULE_OVERRIDE_MAIN_IMPORT - // If this module is being loaded via -m on unix, then - // override __name__ to "__main__". Do this only for *modules* - // however - packages never have their names replaced, instead - // they're -m'ed using a special __main__ submodule in them. (This all - // apparently is done to not touch the package name itself, which is - // important for future imports). - if (override_main && stat != MP_IMPORT_STAT_DIR) { - mp_obj_module_t *o = MP_OBJ_TO_PTR(module_obj); - mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR___main__)); - #if MICROPY_CPYTHON_COMPAT - // Store module as "__main__" in the dictionary of loaded modules (returned by sys.modules). - mp_obj_dict_store(MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_loaded_modules_dict)), MP_OBJ_NEW_QSTR(MP_QSTR___main__), module_obj); - // Store real name in "__main__" attribute. Need this for - // resolving relative imports later. "__main__ was chosen - // semi-randonly, to reuse existing qstr's. - mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___main__), MP_OBJ_NEW_QSTR(full_mod_name)); - #endif - } - #endif // MICROPY_MODULE_OVERRIDE_MAIN_IMPORT - - if (stat == MP_IMPORT_STAT_DIR) { - // Directory -- execute "path/__init__.py". - DEBUG_printf("%.*s is dir\n", (int)vstr_len(path), vstr_str(path)); - // Store the __path__ attribute onto this module. - // https://docs.python.org/3/reference/import.html - // "Specifically, any module that contains a __path__ attribute is considered a package." - mp_store_attr(module_obj, MP_QSTR___path__, mp_obj_new_str(vstr_str(path), vstr_len(path))); - size_t orig_path_len = path->len; - vstr_add_str(path, PATH_SEP_CHAR "__init__.py"); - if (stat_file_py_or_mpy(path) == MP_IMPORT_STAT_FILE) { - do_load(MP_OBJ_TO_PTR(module_obj), path); - } else { - // No-op. Nothing to load. - // mp_warning("%s is imported as namespace package", vstr_str(&path)); + #if MICROPY_MODULE_BUILTIN_SUBPACKAGES + // If the outer module is a built-in (because its map is in ROM), then + // treat it like a package if it contains this submodule in its + // globals dict. + mp_obj_module_t *mod = MP_OBJ_TO_PTR(outer_module_obj); + if (mod->globals->map.is_fixed) { + elem = mp_map_lookup(&mod->globals->map, MP_OBJ_NEW_QSTR(level_mod_name), MP_MAP_LOOKUP); + // Also verify that the entry in the globals dict is in fact a module. + if (elem && mp_obj_is_type(elem->value, &mp_type_module)) { + return elem->value; } - // Remove /__init__.py suffix. - path->len = orig_path_len; - } else { // MP_IMPORT_STAT_FILE - // File -- execute "path.(m)py". - do_load(MP_OBJ_TO_PTR(module_obj), path); - // Note: This should be the last component in the import path. If - // there are remaining components then it's an ImportError - // because the current path(the module that was just loaded) is - // not a package. This will be caught on the next iteration - // because the file will not exist. } + #endif - // CIRCUITPY - // Loading a module thrashes the heap significantly so we explicitly clean up - // afterwards. - gc_collect(); + // If the outer module is a package, it will have __path__ set. + // We can use that as the path to search inside. + mp_obj_t dest[2]; + mp_load_method_maybe(outer_module_obj, MP_QSTR___path__, dest); + if (dest[0] != MP_OBJ_NULL) { + // e.g. __path__ will be "/foo/bar" + vstr_add_str(&path, mp_obj_str_get_str(dest[0])); + + // Add the level module name to the path to get "/foo/bar/baz". + vstr_add_char(&path, PATH_SEP_CHAR[0]); + vstr_add_str(&path, qstr_str(level_mod_name)); + + stat = stat_module(&path); + } } - if (outer_module_obj != MP_OBJ_NULL && VERIFY_PTR(MP_OBJ_TO_PTR(outer_module_obj))) { - // If it's a sub-module (not a built-in one), then make it available on - // the parent module. + // Not already loaded, and not a built-in, so look at the stat result from the filesystem/frozen. + + if (stat == MP_IMPORT_STAT_NO_EXIST) { + // Not found -- fail. + #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE + mp_raise_msg(&mp_type_ImportError, MP_ERROR_TEXT("module not found")); + #else + mp_raise_msg_varg(&mp_type_ImportError, MP_ERROR_TEXT("no module named '%q'"), full_mod_name); + #endif + } + + // Module was found on the filesystem/frozen, try and load it. + DEBUG_printf("Found path to load: %.*s\n", (int)vstr_len(&path), vstr_str(&path)); + + // Prepare for loading from the filesystem. Create a new shell module + // and register it in sys.modules. Also make sure we remove it if + // there is any problem below. + module_obj = mp_obj_new_module(full_mod_name); + nlr_jump_callback_node_unregister_module_t ctx; + ctx.name = full_mod_name; + nlr_push_jump_callback(&ctx.callback, unregister_module_from_nlr_jump_callback); + + #if MICROPY_MODULE_OVERRIDE_MAIN_IMPORT + // If this module is being loaded via -m on unix, then + // override __name__ to "__main__". Do this only for *modules* + // however - packages never have their names replaced, instead + // they're -m'ed using a special __main__ submodule in them. (This all + // apparently is done to not touch the package name itself, which is + // important for future imports). + if (override_main && stat != MP_IMPORT_STAT_DIR) { + mp_obj_module_t *o = MP_OBJ_TO_PTR(module_obj); + mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR___main__)); + #if MICROPY_CPYTHON_COMPAT + // Store module as "__main__" in the dictionary of loaded modules (returned by sys.modules). + mp_obj_dict_store(MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_loaded_modules_dict)), MP_OBJ_NEW_QSTR(MP_QSTR___main__), module_obj); + // Store real name in "__main__" attribute. Need this for + // resolving relative imports later. "__main__ was chosen + // semi-randonly, to reuse existing qstr's. + mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___main__), MP_OBJ_NEW_QSTR(full_mod_name)); + #endif + } + #endif // MICROPY_MODULE_OVERRIDE_MAIN_IMPORT + + if (stat == MP_IMPORT_STAT_DIR) { + // Directory (i.e. a package). + DEBUG_printf("%.*s is dir\n", (int)vstr_len(&path), vstr_str(&path)); + + // Store the __path__ attribute onto this module. + // https://docs.python.org/3/reference/import.html + // "Specifically, any module that contains a __path__ attribute is considered a package." + // This gets used later to locate any subpackages of this module. + mp_store_attr(module_obj, MP_QSTR___path__, mp_obj_new_str(vstr_str(&path), vstr_len(&path))); + size_t orig_path_len = path.len; + vstr_add_str(&path, PATH_SEP_CHAR "__init__.py"); + + // execute "path/__init__.py" (if available). + if (stat_file_py_or_mpy(&path) == MP_IMPORT_STAT_FILE) { + do_load(MP_OBJ_TO_PTR(module_obj), &path); + } else { + // No-op. Nothing to load. + // mp_warning("%s is imported as namespace package", vstr_str(&path)); + } + // Remove /__init__.py suffix from path. + path.len = orig_path_len; + } else { // MP_IMPORT_STAT_FILE + // File -- execute "path.(m)py". + do_load(MP_OBJ_TO_PTR(module_obj), &path); + // Note: This should be the last component in the import path. If + // there are remaining components then in the next call to + // process_import_at_level will detect that it doesn't have + // a __path__ attribute, and not attempt to stat it. + } + + // CIRCUITPY + // Loading a module thrashes the heap significantly so we explicitly clean up + // afterwards. + gc_collect(); + + if (outer_module_obj != MP_OBJ_NULL) { + // If it's a sub-module then make it available on the parent module. mp_store_attr(outer_module_obj, level_mod_name, module_obj); } + nlr_pop_jump_callback(false); + return module_obj; } @@ -507,7 +551,6 @@ mp_obj_t mp_builtin___import___default(size_t n_args, const mp_obj_t *args) { // i.e. "from . import foo" --> level=1 // i.e. "from ...foo.bar import baz" --> level=3 mp_int_t level = 0; - if (n_args >= 4) { fromtuple = args[3]; if (n_args >= 5) { @@ -522,8 +565,10 @@ mp_obj_t mp_builtin___import___default(size_t n_args, const mp_obj_t *args) { const char *module_name = mp_obj_str_get_data(module_name_obj, &module_name_len); if (level != 0) { - // Turn "foo.bar" into ".foo.bar". + // Turn "foo.bar" with level=3 into ".foo.bar". + // Current module name is extracted from globals().__name__. evaluate_relative_import(level, &module_name, &module_name_len); + // module_name is now an absolute module path. } if (module_name_len == 0) { @@ -532,11 +577,12 @@ mp_obj_t mp_builtin___import___default(size_t n_args, const mp_obj_t *args) { DEBUG_printf("Starting module search for '%s'\n", module_name); - VSTR_FIXED(path, MICROPY_ALLOC_PATH_MAX) mp_obj_t top_module_obj = MP_OBJ_NULL; mp_obj_t outer_module_obj = MP_OBJ_NULL; - // Search for the end of each component. + // Iterate the absolute path, finding the end of each component of the path. + // foo.bar.baz + // ^ ^ ^ size_t current_component_start = 0; for (size_t i = 1; i <= module_name_len; i++) { if (i == module_name_len || module_name[i] == '.') { @@ -546,18 +592,18 @@ mp_obj_t mp_builtin___import___default(size_t n_args, const mp_obj_t *args) { qstr level_mod_name = qstr_from_strn(module_name + current_component_start, i - current_component_start); DEBUG_printf("Processing module: '%s' at level '%s'\n", qstr_str(full_mod_name), qstr_str(level_mod_name)); - DEBUG_printf("Previous path: =%.*s=\n", (int)vstr_len(&path), vstr_str(&path)); #if MICROPY_MODULE_OVERRIDE_MAIN_IMPORT - // On unix, if this is being loaded via -m (magic mp_const_false), - // then handle that if it's the final component. + // On unix, if this is being loaded via -m (indicated by sentinel + // fromtuple=mp_const_false), then handle that if it's the final + // component. bool override_main = (i == module_name_len && fromtuple == mp_const_false); #else bool override_main = false; #endif // Import this module. - mp_obj_t module_obj = process_import_at_level(full_mod_name, level_mod_name, outer_module_obj, &path, override_main); + mp_obj_t module_obj = process_import_at_level(full_mod_name, level_mod_name, outer_module_obj, override_main); // Set this as the parent module, and remember the top-level module if it's the first. outer_module_obj = module_obj; @@ -580,32 +626,29 @@ mp_obj_t mp_builtin___import___default(size_t n_args, const mp_obj_t *args) { #else // MICROPY_ENABLE_EXTERNAL_IMPORT -bool mp_obj_is_package(mp_obj_t module) { - return false; -} - mp_obj_t mp_builtin___import___default(size_t n_args, const mp_obj_t *args) { - // Check that it's not a relative import + // Check that it's not a relative import. if (n_args >= 5 && MP_OBJ_SMALL_INT_VALUE(args[4]) != 0) { mp_raise_NotImplementedError(MP_ERROR_TEXT("relative import")); } - // Check if module already exists, and return it if it does - qstr module_name_qstr = mp_obj_str_get_qstr(args[0]); - mp_obj_t module_obj = mp_module_get_loaded_or_builtin(module_name_qstr); - if (module_obj != MP_OBJ_NULL) { - return module_obj; + // Check if the module is already loaded. + mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_loaded_modules_dict).map, args[0], MP_MAP_LOOKUP); + if (elem) { + return elem->value; } - #if MICROPY_MODULE_WEAK_LINKS - // Check if there is a weak link to this module - VSTR_FIXED(umodule_path, MICROPY_ALLOC_PATH_MAX); - qstr umodule_name_qstr = make_weak_link_name(&umodule_path, module_name_qstr); - module_obj = mp_module_get_loaded_or_builtin(umodule_name_qstr); + // Try the name directly as a non-extensible built-in (e.g. `micropython`). + qstr module_name_qstr = mp_obj_str_get_qstr(args[0]); + mp_obj_t module_obj = mp_module_get_builtin(module_name_qstr, false); + if (module_obj != MP_OBJ_NULL) { + return module_obj; + } + // Now try as an extensible built-in (e.g. `time`). + module_obj = mp_module_get_builtin(module_name_qstr, true); if (module_obj != MP_OBJ_NULL) { return module_obj; } - #endif // Couldn't find the module, so fail #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE diff --git a/py/compile.c b/py/compile.c index b1addc76fc..4f91ca49b9 100644 --- a/py/compile.c +++ b/py/compile.c @@ -255,7 +255,7 @@ STATIC void compile_error_set_line(compiler_t *comp, mp_parse_node_t pn) { } } -STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const compressed_string_t *msg) { +STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, mp_rom_error_text_t msg) { // only register the error if there has been no other error if (comp->compile_error == MP_OBJ_NULL) { comp->compile_error = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg); @@ -423,21 +423,6 @@ STATIC void c_if_cond(compiler_t *comp, mp_parse_node_t pn, bool jump_if, int la } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_not_test_2) { c_if_cond(comp, pns->nodes[0], !jump_if, label); return; - } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_atom_paren) { - // cond is something in parenthesis - if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) { - // empty tuple, acts as false for the condition - if (jump_if == false) { - EMIT_ARG(jump, label); - } - } else { - assert(MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_comp)); - // non-empty tuple, acts as true for the condition - if (jump_if == true) { - EMIT_ARG(jump, label); - } - } - return; } } @@ -850,21 +835,12 @@ STATIC bool compile_built_in_decorator(compiler_t *comp, size_t name_len, mp_par qstr attr = MP_PARSE_NODE_LEAF_ARG(name_nodes[1]); if (attr == MP_QSTR_bytecode) { *emit_options = MP_EMIT_OPT_BYTECODE; - // @micropython.native decorator. + #if MICROPY_EMIT_NATIVE } else if (attr == MP_QSTR_native) { - // CIRCUITPY - // Different from MicroPython: native doesn't raise SyntaxError if native support isn't - // compiled, it just passes through the function unmodified. - #if MICROPY_EMIT_NATIVE *emit_options = MP_EMIT_OPT_NATIVE_PYTHON; - #else - return true; - #endif - #if MICROPY_EMIT_NATIVE - // @micropython.viper decorator. } else if (attr == MP_QSTR_viper) { *emit_options = MP_EMIT_OPT_VIPER; - #endif + #endif #if MICROPY_EMIT_INLINE_ASM #if MICROPY_DYNAMIC_COMPILER } else if (attr == MP_QSTR_asm_thumb) { @@ -883,11 +859,11 @@ STATIC bool compile_built_in_decorator(compiler_t *comp, size_t name_len, mp_par #if MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER if (*emit_options == MP_EMIT_OPT_NATIVE_PYTHON || *emit_options == MP_EMIT_OPT_VIPER) { if (emit_native_table[mp_dynamic_compiler.native_arch] == NULL) { - compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid architecture")); + compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid arch")); } } else if (*emit_options == MP_EMIT_OPT_ASM) { if (emit_asm_table[mp_dynamic_compiler.native_arch] == NULL) { - compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid architecture")); + compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid arch")); } } #endif @@ -947,7 +923,7 @@ STATIC void compile_decorated(compiler_t *comp, mp_parse_node_struct_t *pns) { mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t *)pns_body->nodes[0]; body_name = compile_funcdef_helper(comp, pns0, emit_options); scope_t *fscope = (scope_t *)pns0->nodes[4]; - fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR | MP_SCOPE_FLAG_ASYNC; + fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR; #endif } else { assert(MP_PARSE_NODE_STRUCT_KIND(pns_body) == PN_classdef); // should be @@ -1031,16 +1007,13 @@ STATIC void compile_del_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { STATIC void compile_break_cont_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { uint16_t label; - const compressed_string_t *error_msg; if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_break_stmt) { label = comp->break_label; - error_msg = MP_ERROR_TEXT("'break' outside loop"); } else { label = comp->continue_label; - error_msg = MP_ERROR_TEXT("'continue' outside loop"); } if (label == INVALID_LABEL) { - compile_syntax_error(comp, (mp_parse_node_t)pns, error_msg); + compile_syntax_error(comp, (mp_parse_node_t)pns, MP_ERROR_TEXT("'break'/'continue' outside loop")); } assert(comp->cur_except_level >= comp->break_continue_except_level); EMIT_ARG(unwind_jump, label, comp->cur_except_level - comp->break_continue_except_level); @@ -1795,17 +1768,21 @@ STATIC void compile_await_object_method(compiler_t *comp, qstr method) { } STATIC void compile_async_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { - // comp->break_label |= MP_EMIT_BREAK_FROM_FOR; - qstr context = MP_PARSE_NODE_LEAF_ARG(pns->nodes[1]); + // Allocate labels. uint while_else_label = comp_next_label(comp); uint try_exception_label = comp_next_label(comp); uint try_else_label = comp_next_label(comp); uint try_finally_label = comp_next_label(comp); + // Stack: (...) + + // Compile the iterator expression and load and call its __aiter__ method. compile_node(comp, pns->nodes[1]); // iterator + // Stack: (..., iterator) EMIT_ARG(load_method, MP_QSTR___aiter__, false); + // Stack: (..., iterator, __aiter__) EMIT_ARG(call_method, 0, 0, 0); - compile_store_id(comp, context); + // Stack: (..., iterable) START_BREAK_CONTINUE_BLOCK @@ -1813,9 +1790,15 @@ STATIC void compile_async_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns compile_increase_except_level(comp, try_exception_label, MP_EMIT_SETUP_BLOCK_EXCEPT); - compile_load_id(comp, context); + EMIT(dup_top); + // Stack: (..., iterable, iterable) + + // Compile: yield from iterable.__anext__() compile_await_object_method(comp, MP_QSTR___anext__); + // Stack: (..., iterable, yielded_value) + c_assign(comp, pns->nodes[0], ASSIGN_STORE); // variable + // Stack: (..., iterable) EMIT_ARG(pop_except_jump, try_else_label, false); EMIT_ARG(label_assign, try_exception_label); @@ -1832,6 +1815,8 @@ STATIC void compile_async_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns compile_decrease_except_level(comp); EMIT(end_except_handler); + // Stack: (..., iterable) + EMIT_ARG(label_assign, try_else_label); compile_node(comp, pns->nodes[2]); // body @@ -1843,6 +1828,10 @@ STATIC void compile_async_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns compile_node(comp, pns->nodes[3]); // else EMIT_ARG(label_assign, break_label); + // Stack: (..., iterable) + + EMIT(pop_top); + // Stack: (...) } STATIC void compile_async_with_stmt_helper(compiler_t *comp, size_t n, mp_parse_node_t *nodes, mp_parse_node_t body) { @@ -1974,13 +1963,13 @@ STATIC void compile_async_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { // async def compile_funcdef(comp, pns0); scope_t *fscope = (scope_t *)pns0->nodes[4]; - fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR | MP_SCOPE_FLAG_ASYNC; + fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR; } else { // async for/with; first verify the scope is a generator int scope_flags = comp->scope_cur->scope_flags; if (!(scope_flags & MP_SCOPE_FLAG_GENERATOR)) { compile_syntax_error(comp, (mp_parse_node_t)pns0, - MP_ERROR_TEXT("'await', 'async for' or 'async with' outside async function")); + MP_ERROR_TEXT("async for/with outside async function")); return; } @@ -2749,12 +2738,6 @@ STATIC void compile_yield_expr(compiler_t *comp, mp_parse_node_struct_t *pns) { reserve_labels_for_native(comp, 1); } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_yield_arg_from)) { pns = (mp_parse_node_struct_t *)pns->nodes[0]; - #if MICROPY_PY_ASYNC_AWAIT - if ((comp->scope_cur->scope_flags & MP_SCOPE_FLAG_ASYNC) != 0) { - compile_syntax_error(comp, (mp_parse_node_t)pns, MP_ERROR_TEXT("'yield from' inside async function")); - return; - } - #endif compile_node(comp, pns->nodes[0]); compile_yield_from(comp); } else { @@ -2771,15 +2754,7 @@ STATIC void compile_atom_expr_await(compiler_t *comp, mp_parse_node_struct_t *pn return; } compile_atom_expr_normal(comp, pns); - - // If it's an awaitable thing, need to reach for the __await__ method for the coroutine. - // async def functions' __await__ return themselves, which are able to receive a send(), - // while other types with custom __await__ implementations return async generators. - EMIT_ARG(load_method, MP_QSTR___await__, false); - EMIT_ARG(call_method, 0, 0, 0); - EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); - EMIT_ARG(yield, MP_EMIT_YIELD_FROM); - reserve_labels_for_native(comp, 3); + compile_yield_from(comp); } #endif diff --git a/py/dynruntime.h b/py/dynruntime.h index 9303638055..9be2d5b643 100644 --- a/py/dynruntime.h +++ b/py/dynruntime.h @@ -79,7 +79,7 @@ static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) { /******************************************************************************/ // Types and objects -#define MP_OBJ_NEW_QSTR(x) MP_OBJ_NEW_QSTR_##x +#define MP_OBJ_NEW_QSTR(x) (mp_fun_table.native_to_obj(x, MP_NATIVE_TYPE_QSTR)) #define mp_type_type (*mp_fun_table.type_type) #define mp_type_NoneType (*mp_obj_get_type(mp_const_none)) diff --git a/py/emitglue.c b/py/emitglue.c index d4d8439881..00940d663f 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -87,7 +87,7 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, mp_prof_extract_prelude(code, prelude); #endif - #ifdef DEBUG_PRINT + #if defined(DEBUG_PRINT) && DEBUG_PRINT #if !(MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS) const size_t len = 0; #endif @@ -149,7 +149,7 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void rc->n_pos_args = n_pos_args; rc->type_sig = type_sig; - #ifdef DEBUG_PRINT + #if DEBUG_PRINT DEBUG_printf("assign native: kind=%d fun=%p len=" UINT_FMT " n_pos_args=" UINT_FMT " flags=%x\n", kind, fun_data, fun_len, n_pos_args, (uint)scope_flags); for (mp_uint_t i = 0; i < fun_len; i++) { if (i > 0 && i % 16 == 0) { diff --git a/py/emitinlinethumb.c b/py/emitinlinethumb.c index 4bde921b99..3a5dd1a2a4 100644 --- a/py/emitinlinethumb.c +++ b/py/emitinlinethumb.c @@ -74,6 +74,7 @@ static inline bool emit_inline_thumb_allow_float(emit_inline_asm_t *emit) { #endif +// CIRCUITPY STATIC void emit_inline_thumb_error_msg(emit_inline_asm_t *emit, const compressed_string_t *msg) { *emit->error_slot = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg); } diff --git a/py/emitinlinextensa.c b/py/emitinlinextensa.c index f044cfd094..45ffceb450 100644 --- a/py/emitinlinextensa.c +++ b/py/emitinlinextensa.c @@ -43,6 +43,7 @@ struct _emit_inline_asm_t { qstr *label_lookup; }; +// CIRCUITPY STATIC void emit_inline_xtensa_error_msg(emit_inline_asm_t *emit, const compressed_string_t *msg) { *emit->error_slot = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg); } diff --git a/py/emitnative.c b/py/emitnative.c index 619f75bdbb..5ec675a939 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -774,7 +774,7 @@ STATIC void adjust_stack(emit_t *emit, mp_int_t stack_size_delta) { if (emit->pass > MP_PASS_SCOPE && emit->stack_size > emit->scope->stack_size) { emit->scope->stack_size = emit->stack_size; } - #ifdef DEBUG_PRINT + #if DEBUG_PRINT DEBUG_printf(" adjust_stack; stack_size=%d+%d; stack now:", emit->stack_size - stack_size_delta, stack_size_delta); for (int i = 0; i < emit->stack_size; i++) { stack_info_t *si = &emit->stack_info[i]; diff --git a/py/formatfloat.c b/py/formatfloat.c index 050d3a9dfb..7cd471018d 100644 --- a/py/formatfloat.c +++ b/py/formatfloat.c @@ -230,7 +230,7 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch } } else { // Build positive exponent. - // We don't modify f at this point to avoid innaccuracies from + // We don't modify f at this point to avoid inaccuracies from // scaling it. Instead, we find the product of powers of 10 // that is not greater than it, and use that to start the // mantissa. diff --git a/py/gc.c b/py/gc.c index cccc096e57..ddc236b2e4 100644 --- a/py/gc.c +++ b/py/gc.c @@ -90,7 +90,7 @@ #define ATB_3_IS_FREE(a) (((a) & ATB_MASK_3) == 0) #if MICROPY_GC_SPLIT_HEAP -#define NEXT_AREA(area) (area->next) +#define NEXT_AREA(area) ((area)->next) #else #define NEXT_AREA(area) (NULL) #endif @@ -131,6 +131,7 @@ #define GC_EXIT() #endif +// CIRCUITPY #ifdef LOG_HEAP_ACTIVITY volatile uint32_t change_me; #pragma GCC push_options @@ -151,7 +152,13 @@ STATIC void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { // => T = A * (1 + BLOCKS_PER_ATB / BLOCKS_PER_FTB + BLOCKS_PER_ATB * BYTES_PER_BLOCK) size_t total_byte_len = (byte *)end - (byte *)start; #if MICROPY_ENABLE_FINALISER - area->gc_alloc_table_byte_len = (total_byte_len - ALLOC_TABLE_GAP_BYTE) * MP_BITS_PER_BYTE / (MP_BITS_PER_BYTE + MP_BITS_PER_BYTE * BLOCKS_PER_ATB / BLOCKS_PER_FTB + MP_BITS_PER_BYTE * BLOCKS_PER_ATB * BYTES_PER_BLOCK); + area->gc_alloc_table_byte_len = (total_byte_len - ALLOC_TABLE_GAP_BYTE) + * MP_BITS_PER_BYTE + / ( + MP_BITS_PER_BYTE + + MP_BITS_PER_BYTE * BLOCKS_PER_ATB / BLOCKS_PER_FTB + + MP_BITS_PER_BYTE * BLOCKS_PER_ATB * BYTES_PER_BLOCK + ); #else area->gc_alloc_table_byte_len = (total_byte_len - ALLOC_TABLE_GAP_BYTE) / (1 + MP_BITS_PER_BYTE / 2 * BYTES_PER_BLOCK); #endif @@ -172,25 +179,34 @@ STATIC void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { #endif #if MICROPY_ENABLE_FINALISER - // clear ATBs and FTBs + // clear ATB's and FTB's memset(area->gc_alloc_table_start, 0, gc_finaliser_table_byte_len + area->gc_alloc_table_byte_len + ALLOC_TABLE_GAP_BYTE); #else - // clear ATBs + // clear ATB's memset(area->gc_alloc_table_start, 0, area->gc_alloc_table_byte_len + ALLOC_TABLE_GAP_BYTE); #endif area->gc_last_free_atb_index = 0; + area->gc_last_used_block = 0; #if MICROPY_GC_SPLIT_HEAP area->next = NULL; #endif DEBUG_printf("GC layout:\n"); - DEBUG_printf(" alloc table at %p, length " UINT_FMT " bytes, " UINT_FMT " blocks\n", MP_STATE_MEM(area).gc_alloc_table_start, MP_STATE_MEM(area).gc_alloc_table_byte_len, MP_STATE_MEM(area).gc_alloc_table_byte_len * BLOCKS_PER_ATB); + DEBUG_printf(" alloc table at %p, length " UINT_FMT " bytes, " + UINT_FMT " blocks\n", + area->gc_alloc_table_start, area->gc_alloc_table_byte_len, + area->gc_alloc_table_byte_len * BLOCKS_PER_ATB); #if MICROPY_ENABLE_FINALISER - DEBUG_printf(" finaliser table at %p, length " UINT_FMT " bytes, " UINT_FMT " blocks\n", MP_STATE_MEM(area).gc_finaliser_table_start, gc_finaliser_table_byte_len, gc_finaliser_table_byte_len * BLOCKS_PER_FTB); + DEBUG_printf(" finaliser table at %p, length " UINT_FMT " bytes, " + UINT_FMT " blocks\n", area->gc_finaliser_table_start, + gc_finaliser_table_byte_len, + gc_finaliser_table_byte_len * BLOCKS_PER_FTB); #endif - DEBUG_printf(" pool at %p, length " UINT_FMT " bytes, " UINT_FMT " blocks\n", MP_STATE_MEM(area).gc_pool_start, gc_pool_block_len * BYTES_PER_BLOCK, gc_pool_block_len); + DEBUG_printf(" pool at %p, length " UINT_FMT " bytes, " + UINT_FMT " blocks\n", area->gc_pool_start, + gc_pool_block_len * BYTES_PER_BLOCK, gc_pool_block_len); } void gc_init(void *start, void *end) { @@ -243,7 +259,6 @@ void gc_add(void *start, void *end) { // Add this area to the linked list prev_area->next = area; } -#endif // CIRCUITPY // TODO FOR MERGE: fix this for multiple areas?? @@ -252,7 +267,84 @@ void gc_deinit(void) { gc_sweep_all(); MP_STATIC_ASSERT(!MICROPY_GC_SPLIT_HEAP); memset(&MP_STATE_MEM(area), 0, sizeof(MP_STATE_MEM(area))); + +#if MICROPY_GC_SPLIT_HEAP_AUTO +// Try to automatically add a heap area large enough to fulfill 'failed_alloc'. +STATIC bool gc_try_add_heap(size_t failed_alloc) { + // 'needed' is the size of a heap large enough to hold failed_alloc, with + // the additional metadata overheads as calculated in gc_setup_area(). + // + // Rather than reproduce all of that logic here, we approximate that adding + // (13/512) is enough overhead for sufficiently large heap areas (the + // overhead converges to 3/128, but there's some fixed overhead and some + // rounding up of partial block sizes). + size_t needed = failed_alloc + MAX(2048, failed_alloc * 13 / 512); + + size_t avail = gc_get_max_new_split(); + + DEBUG_printf("gc_try_add_heap failed_alloc " UINT_FMT ", " + "needed " UINT_FMT ", avail " UINT_FMT " bytes \n", + failed_alloc, + needed, + avail); + + if (avail < needed) { + // Can't fit this allocation, or system heap has nearly run out anyway + return false; + } + + // Deciding how much to grow the total heap by each time is tricky: + // + // - Grow by too small amounts, leads to heap fragmentation issues. + // + // - Grow by too large amounts, may lead to system heap running out of + // space. + // + // Currently, this implementation is: + // + // - At minimum, aim to double the total heap size each time we add a new + // heap. i.e. without any large single allocations, total size will be + // 64KB -> 128KB -> 256KB -> 512KB -> 1MB, etc + // + // - If the failed allocation is too large to fit in that size, the new + // heap is made exactly large enough for that allocation. Future growth + // will double the total heap size again. + // + // - If the new heap won't fit in the available free space, add the largest + // new heap that will fit (this may lead to failed system heap allocations + // elsewhere, but some allocation will likely fail in this circumstance!) + size_t total_heap = 0; + for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); + area != NULL; + area = NEXT_AREA(area)) { + total_heap += area->gc_pool_end - area->gc_alloc_table_start; + total_heap += ALLOC_TABLE_GAP_BYTE + sizeof(mp_state_mem_area_t); + } + + DEBUG_printf("total_heap " UINT_FMT " bytes\n", total_heap); + + size_t to_alloc = MIN(avail, MAX(total_heap, needed)); + + mp_state_mem_area_t *new_heap = MP_PLAT_ALLOC_HEAP(to_alloc); + + DEBUG_printf("MP_PLAT_ALLOC_HEAP " UINT_FMT " = %p\n", + to_alloc, new_heap); + + if (new_heap == NULL) { + // This should only fail: + // - In a threaded environment if another thread has + // allocated while this function ran. + // - If there is a bug in gc_get_max_new_split(). + return false; + } + + gc_add(new_heap, (void *)new_heap + to_alloc); + + return true; } +#endif + +#endif void gc_lock(void) { // This does not need to be atomic or have the GC mutex because: @@ -312,10 +404,8 @@ STATIC void MP_NO_INSTRUMENT PLACE_IN_ITCM(gc_mark_subtree)(size_t block) // Start with the block passed in the argument. size_t sp = 0; for (;;) { - MICROPY_GC_HOOK_LOOP - #if !MICROPY_GC_SPLIT_HEAP - mp_state_mem_area_t * area = &MP_STATE_MEM(area); + mp_state_mem_area_t *area = &MP_STATE_MEM(area); #endif // work out number of consecutive blocks in the chain starting with this one @@ -330,7 +420,7 @@ STATIC void MP_NO_INSTRUMENT PLACE_IN_ITCM(gc_mark_subtree)(size_t block) // check this block's children void **ptrs = (void **)PTR_FROM_BLOCK(area, block); for (size_t i = n_blocks * BYTES_PER_BLOCK / sizeof(void *); i > 0; i--, ptrs++) { - MICROPY_GC_HOOK_LOOP + MICROPY_GC_HOOK_LOOP(i); void *ptr = *ptrs; // If this is a heap pointer that hasn't been marked, mark it and push // it's children to the stack. @@ -386,7 +476,7 @@ STATIC void gc_deal_with_stack_overflow(void) { // scan entire memory looking for blocks which have been marked but not their children for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { for (size_t block = 0; block < area->gc_alloc_table_byte_len * BLOCKS_PER_ATB; block++) { - MICROPY_GC_HOOK_LOOP + MICROPY_GC_HOOK_LOOP(block); // trace (again) if mark bit set if (ATB_GET_KIND(area, block) == AT_MARK) { #if MICROPY_GC_SPLIT_HEAP @@ -406,9 +496,19 @@ STATIC void gc_sweep(void) { #endif // free unmarked heads and their tails int free_tail = 0; + #if MICROPY_GC_SPLIT_HEAP_AUTO + mp_state_mem_area_t *prev_area = NULL; + #endif for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { - for (size_t block = 0; block < area->gc_alloc_table_byte_len * BLOCKS_PER_ATB; block++) { - MICROPY_GC_HOOK_LOOP + size_t end_block = area->gc_alloc_table_byte_len * BLOCKS_PER_ATB; + if (area->gc_last_used_block < end_block) { + end_block = area->gc_last_used_block + 1; + } + + size_t last_used_block = 0; + + for (size_t block = 0; block < end_block; block++) { + MICROPY_GC_HOOK_LOOP(block); switch (ATB_GET_KIND(area, block)) { case AT_HEAD: #if MICROPY_ENABLE_FINALISER @@ -447,15 +547,31 @@ STATIC void gc_sweep(void) { #if CLEAR_ON_SWEEP memset((void *)PTR_FROM_BLOCK(area, block), 0, BYTES_PER_BLOCK); #endif + } else { + last_used_block = block; } break; case AT_MARK: ATB_MARK_TO_HEAD(area, block); free_tail = 0; + last_used_block = block; break; } } + + area->gc_last_used_block = last_used_block; + + #if MICROPY_GC_SPLIT_HEAP_AUTO + // Free any empty area, aside from the first one + if (last_used_block == 0 && prev_area != NULL) { + DEBUG_printf("gc_sweep free empty area %p\n", area); + NEXT_AREA(prev_area) = NEXT_AREA(area); + MP_PLAT_FREE_HEAP(area); + area = prev_area; + } + prev_area = area; + #endif } } @@ -494,6 +610,7 @@ void gc_collect_ptr(void *ptr) { #if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) __attribute__((no_sanitize_address)) #endif +// CIRCUITPY static void *MP_NO_INSTRUMENT PLACE_IN_ITCM(gc_get_ptr)(void **ptrs, int i) { #if MICROPY_DEBUG_VALGRIND if (!VALGRIND_CHECK_MEM_IS_ADDRESSABLE(&ptrs[i], sizeof(*ptrs))) { @@ -508,7 +625,7 @@ void gc_collect_root(void **ptrs, size_t len) { mp_state_mem_area_t *area = &MP_STATE_MEM(area); #endif for (size_t i = 0; i < len; i++) { - MICROPY_GC_HOOK_LOOP + MICROPY_GC_HOOK_LOOP(i); void *ptr = gc_get_ptr(ptrs, i); #if MICROPY_GC_SPLIT_HEAP mp_state_mem_area_t *area = gc_get_ptr_area(ptr); @@ -566,6 +683,7 @@ void gc_info(gc_info_t *info) { bool finish = false; info->total += area->gc_pool_end - area->gc_pool_start; for (size_t block = 0, len = 0, len_free = 0; !finish;) { + MICROPY_GC_HOOK_LOOP(block); size_t kind = ATB_GET_KIND(area, block); switch (kind) { case AT_FREE: @@ -617,9 +735,15 @@ void gc_info(gc_info_t *info) { info->used *= BYTES_PER_BLOCK; info->free *= BYTES_PER_BLOCK; + + #if MICROPY_GC_SPLIT_HEAP_AUTO + info->max_new_split = gc_get_max_new_split(); + #endif + GC_EXIT(); } +// CIRCUITPY bool gc_alloc_possible(void) { #if MICROPY_GC_SPLIT_HEAP return MP_STATE_MEM(gc_last_free_area) != 0; @@ -651,6 +775,9 @@ void *gc_alloc(size_t n_bytes, unsigned int alloc_flags) { size_t start_block; size_t n_free; int collected = !MP_STATE_MEM(gc_auto_collect_enabled); + #if MICROPY_GC_SPLIT_HEAP_AUTO + bool added = false; + #endif #if MICROPY_GC_ALLOC_THRESHOLD if (!collected && MP_STATE_MEM(gc_alloc_amount) >= MP_STATE_MEM(gc_alloc_threshold)) { @@ -669,6 +796,7 @@ void *gc_alloc(size_t n_bytes, unsigned int alloc_flags) { area = &MP_STATE_MEM(area); #endif + // CIRCUITPY if (area == 0) { reset_into_safe_mode(SAFE_MODE_GC_ALLOC_OUTSIDE_VM); } @@ -677,6 +805,7 @@ void *gc_alloc(size_t n_bytes, unsigned int alloc_flags) { for (; area != NULL; area = NEXT_AREA(area), i = 0) { n_free = 0; for (i = area->gc_last_free_atb_index; i < area->gc_alloc_table_byte_len; i++) { + MICROPY_GC_HOOK_LOOP(i); byte a = area->gc_alloc_table_start[i]; // *FORMAT-OFF* if (ATB_0_IS_FREE(a)) { if (++n_free >= n_blocks) { i = i * BLOCKS_PER_ATB + 0; goto found; } } else { n_free = 0; } @@ -699,6 +828,12 @@ void *gc_alloc(size_t n_bytes, unsigned int alloc_flags) { GC_EXIT(); // nothing found! if (collected) { + #if MICROPY_GC_SPLIT_HEAP_AUTO + if (!added && gc_try_add_heap(n_bytes)) { + added = true; + continue; + } + #endif return NULL; } DEBUG_printf("gc_alloc(" UINT_FMT "): no free mem, triggering GC\n", n_bytes); @@ -730,6 +865,8 @@ found: gc_log_change(start_block, end_block - start_block + 1); #endif + area->gc_last_used_block = MAX(area->gc_last_used_block, end_block); + // mark first block as used head ATB_FREE_TO_HEAD(area, start_block); @@ -857,6 +994,7 @@ void gc_free(void *ptr) { area->gc_last_free_atb_index = block / BLOCKS_PER_ATB; } + // CIRCUITPY #ifdef LOG_HEAP_ACTIVITY gc_log_change(start_block, 0); #endif @@ -1027,6 +1165,7 @@ void *gc_realloc(void *ptr_in, size_t n_bytes, bool allow_move) { gc_dump_alloc_table(&mp_plat_print); #endif + // CIRCUITPY #ifdef LOG_HEAP_ACTIVITY gc_log_change(block, new_blocks); #endif @@ -1041,11 +1180,14 @@ void *gc_realloc(void *ptr_in, size_t n_bytes, bool allow_move) { // check if we can expand in place if (new_blocks <= n_blocks + n_free) { // mark few more blocks as used tail - for (size_t bl = block + n_blocks; bl < block + new_blocks; bl++) { + size_t end_block = block + new_blocks; + for (size_t bl = block + n_blocks; bl < end_block; bl++) { assert(ATB_GET_KIND(area, bl) == AT_FREE); ATB_FREE_TO_TAIL(area, bl); } + area->gc_last_used_block = MAX(area->gc_last_used_block, end_block); + GC_EXIT(); #if MICROPY_GC_CONSERVATIVE_CLEAR @@ -1060,6 +1202,7 @@ void *gc_realloc(void *ptr_in, size_t n_bytes, bool allow_move) { gc_dump_alloc_table(&mp_plat_print); #endif + // CIRCUITPY #ifdef LOG_HEAP_ACTIVITY gc_log_change(block, new_blocks); #endif @@ -1099,6 +1242,7 @@ void *gc_realloc(void *ptr_in, size_t n_bytes, bool allow_move) { } #endif // Alternative gc_realloc impl +// CIRCUITPY bool gc_never_free(void *ptr) { // Check to make sure the pointer is on the heap in the first place. if (gc_nbytes(ptr) == 0) { @@ -1134,9 +1278,12 @@ bool gc_never_free(void *ptr) { void gc_dump_info(const mp_print_t *print) { gc_info_t info; gc_info(&info); - mp_printf(print, "GC: total: %u, used: %u, free: %u\n", + mp_printf(print, "GC: total: %u, used: %u, free: %u", (uint)info.total, (uint)info.used, (uint)info.free); - mp_printf(print, " No. of 1-blocks: %u, 2-blocks: %u, max blk sz: %u, max free sz: %u\n", + #if MICROPY_GC_SPLIT_HEAP_AUTO + mp_printf(print, ", max new split: %u", (uint)info.max_new_split); + #endif + mp_printf(print, "\n No. of 1-blocks: %u, 2-blocks: %u, max blk sz: %u, max free sz: %u\n", (uint)info.num_1block, (uint)info.num_2block, (uint)info.max_block, (uint)info.max_free); } @@ -1205,6 +1352,7 @@ void gc_dump_alloc_table(const mp_print_t *print) { */ /* this prints the uPy object type of the head block */ case AT_HEAD: { + // CIRCUITPY compiler warning avoidance #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-align" void **ptr = (void **)(area->gc_pool_start + bl * BYTES_PER_BLOCK); @@ -1270,6 +1418,7 @@ void gc_dump_alloc_table(const mp_print_t *print) { } mp_print_str(print, "\n"); } + // CIRCUITPY mp_print_str(&mp_plat_print, "\n"); GC_EXIT(); } diff --git a/py/gc.h b/py/gc.h index 8d0a4a8607..b700f75b31 100644 --- a/py/gc.h +++ b/py/gc.h @@ -34,6 +34,7 @@ #include "py/mpstate.h" #include "py/misc.h" +// CIRCUITPY - this may change when use split heap #if !MICROPY_GC_SPLIT_HEAP #define HEAP_PTR(ptr) ( \ MP_STATE_MEM(area).gc_pool_start != 0 /* Not on the heap if it isn't inited */ \ @@ -41,6 +42,7 @@ && ptr < (void *)MP_STATE_MEM(area).gc_pool_end /* must be below end of pool */ \ ) +// CIRCUITPY: defined here so available outside of gc.c // ptr should be of type void* #define VERIFY_PTR(ptr) ( \ ((uintptr_t)(ptr) & (MICROPY_BYTES_PER_GC_BLOCK - 1)) == 0 /* must be aligned on a block */ \ @@ -49,12 +51,19 @@ #endif void gc_init(void *start, void *end); +// CIRCUITPY void gc_deinit(void); #if MICROPY_GC_SPLIT_HEAP // Used to add additional memory areas to the heap. void gc_add(void *start, void *end); -#endif + +#if MICROPY_GC_SPLIT_HEAP_AUTO +// Port must implement this function to return the maximum available block of +// RAM to allocate a new heap area into using MP_PLAT_ALLOC_HEAP. +size_t gc_get_max_new_split(void); +#endif // MICROPY_GC_SPLIT_HEAP_AUTO +#endif // MICROPY_GC_SPLIT_HEAP // These lock/unlock functions can be nested. // They can be used to prevent the GC from allocating/freeing. @@ -65,10 +74,12 @@ bool gc_is_locked(void); // A given port must implement gc_collect by using the other collect functions. void gc_collect(void); void gc_collect_start(void); +// CIRCUITPY void gc_collect_ptr(void *ptr); void gc_collect_root(void **ptrs, size_t len); void gc_collect_end(void); +// CIRCUITPY // Is the gc heap available? bool gc_alloc_possible(void); @@ -85,6 +96,7 @@ size_t gc_nbytes(const void *ptr); bool gc_has_finaliser(const void *ptr); void *gc_realloc(void *ptr, size_t n_bytes, bool allow_move); +// CIRCUITPY // Prevents a pointer from ever being freed because it establishes a permanent reference to it. Use // very sparingly because it can leak memory. bool gc_never_free(void *ptr); @@ -97,6 +109,9 @@ typedef struct _gc_info_t { size_t num_1block; size_t num_2block; size_t max_block; + #if MICROPY_GC_SPLIT_HEAP_AUTO + size_t max_new_split; + #endif } gc_info_t; void gc_info(gc_info_t *info); diff --git a/py/lexer.c b/py/lexer.c index e7d0e81440..6b28f22152 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -361,13 +361,25 @@ STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) vstr_add_byte(&lex->fstring_args, '('); // remember the start of this argument (if we need it for f'{a=}'). size_t i = lex->fstring_args.len; - // extract characters inside the { until we reach the - // format specifier or closing }. - // (MicroPython limitation) note: this is completely unaware of - // Python syntax and will not handle any expression containing '}' or ':'. - // e.g. f'{"}"}' or f'{foo({})}'. + // Extract characters inside the { until the bracket level + // is zero and we reach the conversion specifier '!', + // format specifier ':', or closing '}'. The conversion + // and format specifiers are left unchanged in the format + // string to be handled by str.format. + // (MicroPython limitation) note: this is completely + // unaware of Python syntax and will not handle any + // expression containing '}' or ':'. e.g. f'{"}"}' or f' + // {foo({})}'. However, detection of the '!' will + // specifically ensure that it's followed by [rs] and + // then either the format specifier or the closing + // brace. This allows the use of e.g. != in expressions. unsigned int nested_bracket_level = 0; - while (!is_end(lex) && (nested_bracket_level != 0 || !is_char_or(lex, ':', '}'))) { + while (!is_end(lex) && (nested_bracket_level != 0 + || !(is_char_or(lex, ':', '}') + || (is_char(lex, '!') + && is_char_following_or(lex, 'r', 's') + && is_char_following_following_or(lex, ':', '}')))) + ) { unichar c = CUR_CHAR(lex); if (c == '[' || c == '{') { nested_bracket_level += 1; @@ -834,6 +846,7 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) { vstr_init(&lex->vstr, 32); #if MICROPY_PY_FSTRINGS vstr_init(&lex->fstring_args, 0); + lex->fstring_args_idx = 0; #endif // store sentinel for first indentation level diff --git a/py/makemoduledefs.py b/py/makemoduledefs.py index 9061cd890b..29162ab387 100644 --- a/py/makemoduledefs.py +++ b/py/makemoduledefs.py @@ -1,8 +1,17 @@ """ This pre-processor parses a single file containing a list of -MP_REGISTER_MODULE(module_name, obj_module) -These are used to generate a header with the required entries for -"mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c +`MP_REGISTER_MODULE(MP_QSTR_module_name, obj_module)` or +`MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_module_name, obj_module)` +(i.e. the output of `py/makeqstrdefs.py cat module`). + +The output is a header (typically moduledefs.h) which is included by +py/objmodule.c that contains entries to be included in the definition of + - mp_rom_map_elem_t mp_builtin_module_table[] + - mp_rom_map_elem_t mp_builtin_extensible_module_table[] + +Extensible modules are modules that can be overridden from the filesystem, see +py/builtinimnport.c:process_import_at_level. Regular modules will always use +the built-in version. """ from __future__ import print_function @@ -13,7 +22,15 @@ import io import argparse -pattern = re.compile(r"\s*MP_REGISTER_MODULE\((.*?),\s*(.*?)\);", flags=re.DOTALL) +register_pattern = re.compile( + r"\s*(MP_REGISTER_MODULE|MP_REGISTER_EXTENSIBLE_MODULE)\(MP_QSTR_(.*?),\s*(.*?)\);", + flags=re.DOTALL, +) + +delegation_pattern = re.compile( + r"\s*(?:MP_REGISTER_MODULE_DELEGATION)\((.*?),\s*(.*?)\);", + flags=re.DOTALL, +) def find_module_registrations(filename): @@ -25,7 +42,8 @@ def find_module_registrations(filename): global pattern with io.open(filename, encoding="utf-8") as c_file_obj: - return set(re.findall(pattern, c_file_obj.read())) + c = c_file_obj.read() + return set(re.findall(register_pattern, c)), set(re.findall(delegation_pattern, c)) def generate_module_table_header(modules): @@ -37,14 +55,22 @@ def generate_module_table_header(modules): # Print header file for all external modules. mod_defs = set() - print("// Automatically generated by makemoduledefs.py.\n") - for module_name, obj_module in modules: + extensible_mod_defs = set() + for macro_name, module_name, obj_module in modules: mod_def = "MODULE_DEF_{}".format(module_name.upper()) - mod_defs.add(mod_def) + if macro_name == "MP_REGISTER_MODULE": + mod_defs.add(mod_def) + elif macro_name == "MP_REGISTER_EXTENSIBLE_MODULE": + extensible_mod_defs.add(mod_def) if "," in obj_module: print( - "ERROR: Call to MP_REGISTER_MODULE({}, {}) should be MP_REGISTER_MODULE({}, {})\n".format( - module_name, obj_module, module_name, obj_module.split(",")[0] + "ERROR: Call to {}({}, {}) should be {}({}, {})\n".format( + macro_name, + module_name, + obj_module, + macro_name, + module_name, + obj_module.split(",")[0], ), file=sys.stderr, ) @@ -53,7 +79,7 @@ def generate_module_table_header(modules): ( "extern const struct _mp_obj_module_t {obj_module};\n" "#undef {mod_def}\n" - "#define {mod_def} {{ MP_ROM_QSTR({module_name}), MP_ROM_PTR(&{obj_module}) }},\n" + "#define {mod_def} {{ MP_ROM_QSTR(MP_QSTR_{module_name}), MP_ROM_PTR(&{obj_module}) }},\n" ).format( module_name=module_name, obj_module=obj_module, @@ -68,14 +94,41 @@ def generate_module_table_header(modules): print("// MICROPY_REGISTERED_MODULES") + print("\n#define MICROPY_REGISTERED_EXTENSIBLE_MODULES \\") + + for mod_def in sorted(extensible_mod_defs): + print(" {mod_def} \\".format(mod_def=mod_def)) + + print("// MICROPY_REGISTERED_EXTENSIBLE_MODULES") + + +def generate_module_delegations(delegations): + if not delegations: + return + + print() + for obj_module, fun_name in delegations: + print("extern void {}(mp_obj_t self_in, qstr attr, mp_obj_t *dest);".format(fun_name)) + print("#define MICROPY_MODULE_DELEGATIONS \\") + for obj_module, fun_name in delegations: + print( + " {{ MP_ROM_PTR(&{obj_module}), {fun_name} }}, \\".format( + obj_module=obj_module, fun_name=fun_name + ) + ) + print("// MICROPY_MODULE_DELEGATIONS") + def main(): parser = argparse.ArgumentParser() parser.add_argument("file", nargs=1, help="file with MP_REGISTER_MODULE definitions") args = parser.parse_args() - modules = find_module_registrations(args.file[0]) + print("// Automatically generated by makemoduledefs.py.\n") + + modules, delegations = find_module_registrations(args.file[0]) generate_module_table_header(sorted(modules)) + generate_module_delegations(sorted(delegations)) if __name__ == "__main__": diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index d476f5988a..cf51b62d46 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -3,6 +3,7 @@ Process raw qstr file and output qstr data with length, hash and data bytes. This script works with Python 2.7, 3.3 and 3.4. +CIRCUITPY: For documentation about the format of compressed translated strings, see supervisor/shared/translate/translate.h """ @@ -271,6 +272,7 @@ def qstr_escape(qst): return re.sub(r"[^A-Za-z0-9_]", esc_char, qst) +# CIRCUITPY: add translations handling def parse_input_headers_with_translations(infiles): qcfgs = {} qstrs = {} @@ -304,6 +306,7 @@ def parse_input_headers_with_translations(infiles): qcfgs[match.group(1)] = value continue + # CIRCUITPY match = re.match(r'^TRANSLATE\("(.*)"\)$', line) if match: translations.add(match.group(1)) @@ -342,6 +345,7 @@ def parse_input_headers_with_translations(infiles): order = -190000 elif ident.startswith("__"): order -= 100000 + # CIRCUITPY elif ident.startswith("_lt"): order -= 100000 elif ident.startswith("_gt"): @@ -355,7 +359,7 @@ def parse_input_headers_with_translations(infiles): return qcfgs, qstrs, translations -# Used externally by mpy-tool.py. Don't pass back translations. +# CIRCUITPY: Used externally by mpy-tool.py. Don't pass back translations. def parse_input_headers(infiles): qcfgs, qstrs, translations = parse_input_headers_with_translations(infiles) return (qcfgs, qstrs) @@ -381,6 +385,7 @@ def make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr): return '%d, %d, "%s"' % (qhash, qlen, qdata) +# CIRCUITPY: add translations def print_qstr_data(qcfgs, qstrs, translations): # get config variables cfg_bytes_len = int(qcfgs["BYTES_IN_LEN"]) diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py index 8990f2a034..aed10b1e4d 100644 --- a/py/makeqstrdefs.py +++ b/py/makeqstrdefs.py @@ -57,7 +57,7 @@ _MODE_QSTR = "qstr" # Extract MP_COMPRESSED_ROM_TEXT("") macros. (Which come from MP_ERROR_TEXT) _MODE_COMPRESS = "compress" -# Extract MP_REGISTER_MODULE(...) macros. +# Extract MP_REGISTER_(EXTENSIBLE_)MODULE(...) macros. _MODE_MODULE = "module" # Extract MP_REGISTER_ROOT_POINTER(...) macros. @@ -141,7 +141,9 @@ def process_file(f): elif args.mode == _MODE_COMPRESS: re_match = re.compile(r'MP_COMPRESSED_ROM_TEXT\("([^"]*)"\)') elif args.mode == _MODE_MODULE: - re_match = re.compile(r"MP_REGISTER_MODULE\(.*?,\s*.*?\);") + re_match = re.compile( + r"(?:MP_REGISTER_MODULE|MP_REGISTER_EXTENSIBLE_MODULE|MP_REGISTER_MODULE_DELEGATION)\(.*?,\s*.*?\);" + ) elif args.mode == _MODE_ROOT_POINTER: re_match = re.compile(r"MP_REGISTER_ROOT_POINTER\(.*?\);") re_translate = re.compile(r"translate\(\"((?:(?=(\\?))\2.)*?)\"\)") diff --git a/py/malloc.c b/py/malloc.c index efdff75396..ddf139e386 100644 --- a/py/malloc.c +++ b/py/malloc.c @@ -185,7 +185,7 @@ void *m_realloc_maybe(void *ptr, size_t new_num_bytes, bool allow_move) #if MICROPY_MALLOC_USES_ALLOCATED_SIZE DEBUG_printf("realloc %p, %d, %d : %p\n", ptr, old_num_bytes, new_num_bytes, new_ptr); #else - DEBUG_printf("realloc %p, %d, %d : %p\n", ptr, new_num_bytes, new_ptr); + DEBUG_printf("realloc %p, %d : %p\n", ptr, new_num_bytes, new_ptr); #endif return new_ptr; } diff --git a/py/misc.h b/py/misc.h index 78cddc394a..9ea3148ea3 100644 --- a/py/misc.h +++ b/py/misc.h @@ -134,6 +134,7 @@ size_t m_get_peak_bytes_allocated(void); // align ptr to the nearest multiple of "alignment" #define MP_ALIGN(ptr, alignment) (void *)(((uintptr_t)(ptr) + ((alignment) - 1)) & ~((alignment) - 1)) +// CIRCUITPY #define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) /** unichar / UTF-8 *********************************************/ diff --git a/py/mkenv.mk b/py/mkenv.mk index ee2a96e258..7d321da82c 100644 --- a/py/mkenv.mk +++ b/py/mkenv.mk @@ -12,6 +12,7 @@ endif THIS_MAKEFILE := $(lastword $(MAKEFILE_LIST)) TOP := $(patsubst %/py/mkenv.mk,%,$(THIS_MAKEFILE)) +# CIRCUITPY: verbosity differences, STEPECHO # Turn on increased build verbosity by defining BUILD_VERBOSE in your main # Makefile or in your environment. You can also use V="steps commands rules" or any combination thereof # on the make command line. @@ -19,7 +20,6 @@ TOP := $(patsubst %/py/mkenv.mk,%,$(THIS_MAKEFILE)) ifeq ("$(origin V)", "command line") BUILD_VERBOSE=$(V) endif - ifndef BUILD_VERBOSE $(info - Verbosity options: any combination of "steps commands rules", as `make V=...` or env var BUILD_VERBOSE) BUILD_VERBOSE = "" @@ -50,20 +50,15 @@ endif PY_SRC ?= $(TOP)/py BUILD ?= build -ECHO = @echo - -CAT = cat -CD = cd -CP = cp -FIND = find -MKDIR = mkdir -PYTHON = python3 RM = rm -RSYNC = rsync +ECHO = @echo +CP = cp +MKDIR = mkdir SED = sed +CAT = cat TOUCH = touch -# Linux has 'nproc', macOS has 'sysctl -n hw.logicalcpu', this is cross-platform -NPROC = $(PYTHON) -c 'import multiprocessing as mp; print(mp.cpu_count())' +PYTHON = python3 +ZIP = zip AS = $(CROSS_COMPILE)as CC = $(CROSS_COMPILE)gcc @@ -79,6 +74,7 @@ AR = $(CROSS_COMPILE)ar MAKE_MANIFEST = $(PYTHON) $(TOP)/tools/makemanifest.py MAKE_FROZEN = $(PYTHON) $(TOP)/tools/make-frozen.py MPY_TOOL = $(PYTHON) $(TOP)/tools/mpy-tool.py +# CIRCUITPY PREPROCESS_FROZEN_MODULES = PYTHONPATH=$(TOP)/tools/python-semver $(TOP)/tools/preprocess_frozen_modules.py MPY_LIB_SUBMODULE_DIR = $(TOP)/lib/micropython-lib diff --git a/py/mkrules.cmake b/py/mkrules.cmake index 7eb5fcf018..02e3148f2b 100644 --- a/py/mkrules.cmake +++ b/py/mkrules.cmake @@ -178,7 +178,12 @@ if(MICROPY_FROZEN_MANIFEST) set(MICROPY_LIB_DIR ${MICROPY_DIR}/lib/micropython-lib) endif() - if(NOT (${ECHO_SUBMODULES}) AND NOT EXISTS ${MICROPY_LIB_DIR}/README.md) + if(ECHO_SUBMODULES) + # No-op, we're just doing submodule/variant discovery. + # Note: All the following rules are safe to run in discovery mode even + # though the submodule might not be available as they do not directly depend + # on anything from the submodule. + elseif(NOT EXISTS ${MICROPY_LIB_DIR}/README.md) message(FATAL_ERROR " micropython-lib not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'") endif() @@ -218,9 +223,3 @@ if(ECHO_SUBMODULES) execute_process(COMMAND ${CMAKE_COMMAND} -E echo "GIT_SUBMODULES=${GIT_SUBMODULES}") message(FATAL_ERROR "Done") endif() - -# Display BOARD_VARIANTS -if(ECHO_BOARD_VARIANTS) - execute_process(COMMAND ${CMAKE_COMMAND} -E echo "BOARD_VARIANTS=${BOARD_VARIANTS}") - message(FATAL_ERROR "Done") -endif() diff --git a/py/mkrules.mk b/py/mkrules.mk index 87db5f24e9..2ff6bac3f8 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -22,7 +22,7 @@ endif # QSTR generation uses the same CFLAGS, with these modifications. QSTR_GEN_FLAGS = -DNO_QSTR -# Note: := to force evalulation immediately. +# Note: := to force evaluation immediately. QSTR_GEN_CFLAGS := $(CFLAGS) QSTR_GEN_CFLAGS += $(QSTR_GEN_FLAGS) QSTR_GEN_CXXFLAGS := $(CXXFLAGS) @@ -44,6 +44,7 @@ QSTR_GEN_CXXFLAGS += $(QSTR_GEN_FLAGS) # can be located. By following this scheme, it allows a single build rule # to be used to compile all .c files. +# CIRCUITPY adds STEPECHO vpath %.S . $(TOP) $(USER_C_MODULES) $(BUILD)/%.o: %.S $(STEPECHO) "CC $<" @@ -166,7 +167,7 @@ $(HEADER_BUILD): ifneq ($(MICROPY_MPYCROSS_DEPENDENCY),) # to automatically build mpy-cross, if needed $(MICROPY_MPYCROSS_DEPENDENCY): - $(MAKE) -C $(TOP)/mpy-cross + $(MAKE) -C $(abspath $(dir $@)..) endif ifneq ($(FROZEN_DIR),) @@ -184,6 +185,11 @@ ifeq ($(MPY_LIB_DIR),$(MPY_LIB_SUBMODULE_DIR)) GIT_SUBMODULES += lib/micropython-lib endif +# Set compile options needed to enable frozen code. +CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool +CFLAGS += -DMICROPY_MODULE_FROZEN_MPY +CFLAGS += -DMICROPY_MODULE_FROZEN_STR + # to build frozen_content.c from a manifest # CIRCUITPY: FROZEN_MANIFEST is constructed at build time $(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h $(BUILD)/genhdr/root_pointers.h $(FROZEN_MANIFEST) | $(MICROPY_MPYCROSS_DEPENDENCY) @@ -209,7 +215,9 @@ $(BUILD)/$(PROG): $(OBJ) # we may want to compile using Thumb, but link with non-Thumb libc. $(Q)$(CC) -o $@ $^ $(LIB) $(LDFLAGS) ifndef DEBUG +ifdef STRIP $(Q)$(STRIP) $(STRIPFLAGS_EXTRA) $@ +endif endif $(Q)$(SIZE) $$(find $(BUILD) -path "$(BUILD)/build/frozen*.o") $@ @@ -257,9 +265,3 @@ print-def: @$(RM) -f __empty__.c -include $(OBJ:.o=.P) - -# CIRCUITPY addition -# Print out the value of a make variable. -# https://stackoverflow.com/questions/16467718/how-to-print-out-a-variable-in-makefile -print-%: - @echo $* = $($*) diff --git a/py/modarray.c b/py/modarray.c index cfed0fbb59..ac2e56ed38 100644 --- a/py/modarray.c +++ b/py/modarray.c @@ -40,6 +40,6 @@ const mp_obj_module_t mp_module_array = { .globals = (mp_obj_dict_t *)&mp_module_array_globals, }; -MP_REGISTER_MODULE(MP_QSTR_array, mp_module_array); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_array, mp_module_array); #endif diff --git a/py/modbuiltins.c b/py/modbuiltins.c index e5695bfb53..46dced9c01 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -188,7 +188,7 @@ STATIC mp_obj_t mp_builtin_dir(size_t n_args, const mp_obj_t *args) { // Implemented by probing all possible qstrs with mp_load_method_maybe size_t nqstr = QSTR_TOTAL(); for (size_t i = MP_QSTR_ + 1; i < nqstr; ++i) { - // CIRCUITPY changes #6539 + // CIRCUITPY changes PR #6539 mp_obj_t dest[2] = {}; mp_load_method_protected(args[0], i, dest, true); if (dest[0] != MP_OBJ_NULL) { @@ -398,6 +398,7 @@ STATIC mp_obj_t mp_builtin_pow(size_t n_args, const mp_obj_t *args) { } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_pow_obj, 2, 3, mp_builtin_pow); +// CIRCUITPY adds flush() STATIC mp_obj_t mp_builtin_print(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { enum { ARG_sep, ARG_end, ARG_flush, ARG_file }; static const mp_arg_t allowed_args[] = { diff --git a/py/modcollections.c b/py/modcollections.c index a56fe069ea..30a5881bc2 100644 --- a/py/modcollections.c +++ b/py/modcollections.c @@ -46,6 +46,6 @@ const mp_obj_module_t mp_module_collections = { .globals = (mp_obj_dict_t *)&mp_module_collections_globals, }; -MP_REGISTER_MODULE(MP_QSTR_collections, mp_module_collections); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_collections, mp_module_collections); #endif // MICROPY_PY_COLLECTIONS diff --git a/py/moduerrno.c b/py/moderrno.c similarity index 56% rename from py/moduerrno.c rename to py/moderrno.c index 2332b5df96..9df30d68c0 100644 --- a/py/moduerrno.c +++ b/py/moderrno.c @@ -30,10 +30,12 @@ #include "py/obj.h" #include "py/mperrno.h" +#if MICROPY_PY_ERRNO + // This list can be defined per port in mpconfigport.h to tailor it to a // specific port's needs. If it's not defined then we provide a default. -#ifndef MICROPY_PY_UERRNO_LIST -#define MICROPY_PY_UERRNO_LIST \ +#ifndef MICROPY_PY_ERRNO_LIST +#define MICROPY_PY_ERRNO_LIST \ X(EPERM) \ X(ENOENT) \ X(EIO) \ @@ -59,9 +61,7 @@ #endif -#if MICROPY_PY_UERRNO - -#if MICROPY_PY_UERRNO_ERRORCODE +#if MICROPY_PY_ERRNO_ERRORCODE STATIC const mp_rom_map_elem_t errorcode_table[] = { #define X(e) { MP_ROM_INT(MP_##e), MP_ROM_QSTR(MP_QSTR_##e) }, MICROPY_PY_UERRNO_LIST @@ -81,33 +81,28 @@ STATIC const mp_obj_dict_t errorcode_dict = { }; #endif -STATIC const mp_rom_map_elem_t mp_module_uerrno_globals_table[] = { - #if CIRCUITPY +STATIC const mp_rom_map_elem_t mp_module_errno_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_errno) }, - #else - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uerrno) }, - #endif - #if MICROPY_PY_UERRNO_ERRORCODE + #if MICROPY_PY_ERRNO_ERRORCODE { MP_ROM_QSTR(MP_QSTR_errorcode), MP_ROM_PTR(&errorcode_dict) }, #endif #define X(e) { MP_ROM_QSTR(MP_QSTR_##e), MP_ROM_INT(MP_##e) }, - MICROPY_PY_UERRNO_LIST + MICROPY_PY_ERRNO_LIST #undef X }; -STATIC MP_DEFINE_CONST_DICT(mp_module_uerrno_globals, mp_module_uerrno_globals_table); +STATIC MP_DEFINE_CONST_DICT(mp_module_errno_globals, mp_module_errno_globals_table); -const mp_obj_module_t mp_module_uerrno = { +const mp_obj_module_t mp_module_errno = { .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&mp_module_uerrno_globals, + .globals = (mp_obj_dict_t *)&mp_module_errno_globals, }; -MP_REGISTER_MODULE(MP_QSTR_errno, mp_module_uerrno); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_errno, mp_module_errno); qstr mp_errno_to_str(mp_obj_t errno_val) { - // Otherwise, return the Exxxx string for that error code - #if MICROPY_PY_UERRNO_ERRORCODE + #if MICROPY_PY_ERRNO_ERRORCODE // We have the errorcode dict so can do a lookup using the hash map mp_map_elem_t *elem = mp_map_lookup((mp_map_t *)&errorcode_dict.map, errno_val, MP_MAP_LOOKUP); if (elem == NULL) { @@ -117,62 +112,13 @@ qstr mp_errno_to_str(mp_obj_t errno_val) { } #else // We don't have the errorcode dict so do a simple search in the modules dict - for (size_t i = 0; i < MP_ARRAY_SIZE(mp_module_uerrno_globals_table); ++i) { - if (errno_val == mp_module_uerrno_globals_table[i].value) { - return MP_OBJ_QSTR_VALUE(mp_module_uerrno_globals_table[i].key); + for (size_t i = 0; i < MP_ARRAY_SIZE(mp_module_errno_globals_table); ++i) { + if (errno_val == mp_module_errno_globals_table[i].value) { + return MP_OBJ_QSTR_VALUE(mp_module_errno_globals_table[i].key); } } return MP_QSTRnull; #endif } -#endif // MICROPY_PY_UERRNO - - -// For commonly encountered errors, return human readable strings, otherwise try errno name -const char *mp_common_errno_to_str(mp_obj_t errno_val, char *buf, size_t len) { - if (!mp_obj_is_small_int(errno_val)) { - return NULL; - } - - const compressed_string_t *desc = NULL; - switch (MP_OBJ_SMALL_INT_VALUE(errno_val)) { - case EPERM: - desc = MP_ERROR_TEXT("Operation not permitted"); - break; - case ENOENT: - desc = MP_ERROR_TEXT("No such file/directory"); - break; - case EIO: - desc = MP_ERROR_TEXT("Input/output error"); - break; - case EACCES: - desc = MP_ERROR_TEXT("Permission denied"); - break; - case EEXIST: - desc = MP_ERROR_TEXT("File exists"); - break; - case ENODEV: - desc = MP_ERROR_TEXT("No such device"); - break; - case EINVAL: - desc = MP_ERROR_TEXT("Invalid argument"); - break; - case ENOSPC: - desc = MP_ERROR_TEXT("No space left on device"); - break; - case EROFS: - desc = MP_ERROR_TEXT("Read-only filesystem"); - break; - } - if (desc != NULL && decompress_length(desc) <= len) { - decompress(desc, buf); - return buf; - } - - const char *msg = ""; - #if MICROPY_PY_UERRNO - msg = qstr_str(mp_errno_to_str(errno_val)); - #endif - return msg[0] != '\0' ? msg : NULL; -} +#endif // MICROPY_PY_ERRNO diff --git a/py/modgc.c b/py/modgc.c index c11bcaecd7..7b18045b08 100644 --- a/py/modgc.c +++ b/py/modgc.c @@ -64,7 +64,12 @@ MP_DEFINE_CONST_FUN_OBJ_0(gc_isenabled_obj, gc_isenabled); STATIC mp_obj_t gc_mem_free(void) { gc_info_t info; gc_info(&info); + #if MICROPY_GC_SPLIT_HEAP_AUTO + // Include max_new_split value here as a more useful heuristic + return MP_OBJ_NEW_SMALL_INT(info.free + info.max_new_split); + #else return MP_OBJ_NEW_SMALL_INT(info.free); + #endif } MP_DEFINE_CONST_FUN_OBJ_0(gc_mem_free_obj, gc_mem_free); diff --git a/py/modio.c b/py/modio.c index 0ccdd5c337..39317c52d5 100644 --- a/py/modio.c +++ b/py/modio.c @@ -203,7 +203,7 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE( #endif // MICROPY_PY_IO_BUFFEREDWRITER STATIC const mp_rom_map_elem_t mp_module_io_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uio) }, + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_io) }, // Note: mp_builtin_open_obj should be defined by port, it's not // part of the core. { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&mp_builtin_open_obj) }, @@ -226,6 +226,6 @@ const mp_obj_module_t mp_module_io = { .globals = (mp_obj_dict_t *)&mp_module_io_globals, }; -MP_REGISTER_MODULE(MP_QSTR_io, mp_module_io); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_io, mp_module_io); #endif diff --git a/py/modmath.c b/py/modmath.c index 81cf0cb8fa..879930515a 100644 --- a/py/modmath.c +++ b/py/modmath.c @@ -252,6 +252,7 @@ STATIC mp_obj_t mp_math_log(size_t n_args, const mp_obj_t *args) { if (base <= (mp_float_t)0.0) { math_error(); } else if (base == (mp_float_t)1.0) { + // CIRCUITPY: remove redundant text error message mp_raise_ZeroDivisionError(); } return mp_obj_new_float(l / MICROPY_FLOAT_C_FUN(log)(base)); diff --git a/py/modstruct.c b/py/modstruct.c index c772977013..8dbcfa79f1 100644 --- a/py/modstruct.c +++ b/py/modstruct.c @@ -92,14 +92,13 @@ STATIC size_t calc_size_items(const char *fmt, size_t *total_sz) { cnt = get_fmt_num(&fmt); } - if (*fmt == 's') { + if (*fmt == 'x') { + size += cnt; + } else if (*fmt == 's') { total_cnt += 1; size += cnt; } else { - // Pad bytes are skipped and don't get included in the item count. - if (*fmt != 'x') { - total_cnt += cnt; - } + total_cnt += cnt; size_t align; size_t sz = mp_binary_get_size(fmt_type, *fmt, &align); while (cnt--) { @@ -162,17 +161,16 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) { cnt = get_fmt_num(&fmt); } mp_obj_t item; - if (*fmt == 's') { + if (*fmt == 'x') { + p += cnt; + } else if (*fmt == 's') { item = mp_obj_new_bytes(p, cnt); p += cnt; res->items[i++] = item; } else { while (cnt--) { item = mp_binary_get_val(fmt_type, *fmt, p_base, &p); - // Pad bytes ('x') are just skipped. - if (*fmt != 'x') { - res->items[i++] = item; - } + res->items[i++] = item; } } fmt++; @@ -183,6 +181,7 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_unpack_from_obj, 2, 3, struct_unpack_ // This function assumes there is enough room in p to store all the values STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, const mp_obj_t *args) { + // CIRCUITPY additional error checking size_t size; size_t count = calc_size_items(mp_obj_str_get_str(fmt_in), &size); if (count != n_args) { @@ -199,11 +198,18 @@ STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, c size_t i; for (i = 0; i < n_args;) { mp_uint_t cnt = 1; + if (*fmt == '\0') { + // more arguments given than used by format string; CPython raises struct.error here + break; + } if (unichar_isdigit(*fmt)) { cnt = get_fmt_num(&fmt); } - if (*fmt == 's') { + if (*fmt == 'x') { + memset(p, 0, cnt); + p += cnt; + } else if (*fmt == 's') { mp_buffer_info_t bufinfo; mp_get_buffer_raise(args[i++], &bufinfo, MP_BUFFER_READ); mp_uint_t to_copy = cnt; @@ -214,14 +220,9 @@ STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, c memset(p + to_copy, 0, cnt - to_copy); p += cnt; } else { - while (cnt--) { - // Pad bytes don't have a corresponding argument. - if (*fmt == 'x') { - mp_binary_set_val(fmt_type, *fmt, MP_OBJ_NEW_SMALL_INT(0), p_base, &p); - } else { - mp_binary_set_val(fmt_type, *fmt, args[i], p_base, &p); - i++; - } + // If we run out of args then we just finish; CPython would raise struct.error + while (cnt-- && i < n_args) { + mp_binary_set_val(fmt_type, *fmt, args[i++], p_base, &p); } } fmt++; @@ -229,6 +230,7 @@ STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, c } STATIC mp_obj_t struct_pack(size_t n_args, const mp_obj_t *args) { + // TODO: "The arguments must match the values required by the format exactly." mp_int_t size = MP_OBJ_SMALL_INT_VALUE(struct_calcsize(args[0])); vstr_t vstr; vstr_init_len(&vstr, size); @@ -266,7 +268,7 @@ STATIC mp_obj_t struct_pack_into(size_t n_args, const mp_obj_t *args) { MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_pack_into_obj, 3, MP_OBJ_FUN_ARGS_MAX, struct_pack_into); STATIC const mp_rom_map_elem_t mp_module_struct_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ustruct) }, + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_struct) }, { MP_ROM_QSTR(MP_QSTR_calcsize), MP_ROM_PTR(&struct_calcsize_obj) }, { MP_ROM_QSTR(MP_QSTR_pack), MP_ROM_PTR(&struct_pack_obj) }, { MP_ROM_QSTR(MP_QSTR_pack_into), MP_ROM_PTR(&struct_pack_into_obj) }, @@ -276,11 +278,11 @@ STATIC const mp_rom_map_elem_t mp_module_struct_globals_table[] = { STATIC MP_DEFINE_CONST_DICT(mp_module_struct_globals, mp_module_struct_globals_table); -const mp_obj_module_t mp_module_ustruct = { +const mp_obj_module_t mp_module_struct = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&mp_module_struct_globals, }; -MP_REGISTER_MODULE(MP_QSTR_struct, mp_module_ustruct); +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_struct, mp_module_struct); #endif diff --git a/py/modsys.c b/py/modsys.c index 226a422833..f3e74d9ec5 100644 --- a/py/modsys.c +++ b/py/modsys.c @@ -36,7 +36,7 @@ #include "py/smallint.h" #include "py/runtime.h" #include "py/persistentcode.h" -#include "extmod/moduplatform.h" +#include "extmod/modplatform.h" #include "genhdr/mpversion.h" #if MICROPY_PY_SYS_SETTRACE @@ -58,7 +58,7 @@ const mp_print_t mp_sys_stdout_print = {&mp_sys_stdout_obj, mp_stream_write_adap #endif // version - Python language version that this implementation conforms to, as a string -STATIC const MP_DEFINE_STR_OBJ(mp_sys_version_obj, "3.4.0"); +STATIC const MP_DEFINE_STR_OBJ(mp_sys_version_obj, "3.4.0; " MICROPY_BANNER_NAME_AND_VERSION); // version_info - Python language version that this implementation conforms to, as a tuple of ints #define I(n) MP_OBJ_NEW_SMALL_INT(n) @@ -72,20 +72,24 @@ STATIC const mp_obj_tuple_t mp_sys_implementation_version_info_obj = { 3, { I(MICROPY_VERSION_MAJOR), I(MICROPY_VERSION_MINOR), I(MICROPY_VERSION_MICRO) } }; +STATIC const MP_DEFINE_STR_OBJ(mp_sys_implementation_machine_obj, MICROPY_BANNER_MACHINE); #if MICROPY_PERSISTENT_CODE_LOAD #define SYS_IMPLEMENTATION_ELEMS \ MP_ROM_QSTR(MP_QSTR_circuitpython), \ MP_ROM_PTR(&mp_sys_implementation_version_info_obj), \ + MP_ROM_PTR(&mp_sys_implementation_machine_obj), \ MP_ROM_INT(MPY_FILE_HEADER_INT) #else #define SYS_IMPLEMENTATION_ELEMS \ MP_ROM_QSTR(MP_QSTR_circuitpython), \ - MP_ROM_PTR(&mp_sys_implementation_version_info_obj) + MP_ROM_PTR(&mp_sys_implementation_version_info_obj), \ + MP_ROM_PTR(&mp_sys_implementation_machine_obj) #endif #if MICROPY_PY_ATTRTUPLE STATIC const qstr impl_fields[] = { MP_QSTR_name, MP_QSTR_version, + MP_QSTR__machine, #if MICROPY_PERSISTENT_CODE_LOAD MP_QSTR__mpy, #endif @@ -93,13 +97,13 @@ STATIC const qstr impl_fields[] = { STATIC MP_DEFINE_ATTRTUPLE( mp_sys_implementation_obj, impl_fields, - 2 + MICROPY_PERSISTENT_CODE_LOAD, + 3 + MICROPY_PERSISTENT_CODE_LOAD, SYS_IMPLEMENTATION_ELEMS ); #else STATIC const mp_rom_obj_tuple_t mp_sys_implementation_obj = { {&mp_type_tuple}, - 2 + MICROPY_PERSISTENT_CODE_LOAD, + 3 + MICROPY_PERSISTENT_CODE_LOAD, { SYS_IMPLEMENTATION_ELEMS } @@ -162,6 +166,7 @@ STATIC mp_obj_t mp_sys_exc_info(void) { t->items[0] = MP_OBJ_FROM_PTR(mp_obj_get_type(cur_exc)); t->items[1] = cur_exc; + // CIRCUITPY has traceback obj t->items[2] = mp_obj_exception_get_traceback_obj(cur_exc); return MP_OBJ_FROM_PTR(t); } @@ -193,8 +198,30 @@ STATIC mp_obj_t mp_sys_settrace(mp_obj_t obj) { MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_settrace_obj, mp_sys_settrace); #endif // MICROPY_PY_SYS_SETTRACE +#if MICROPY_PY_SYS_PATH && !MICROPY_PY_SYS_ATTR_DELEGATION +#error "MICROPY_PY_SYS_PATH requires MICROPY_PY_SYS_ATTR_DELEGATION" +#endif + +#if MICROPY_PY_SYS_PS1_PS2 && !MICROPY_PY_SYS_ATTR_DELEGATION +#error "MICROPY_PY_SYS_PS1_PS2 requires MICROPY_PY_SYS_ATTR_DELEGATION" +#endif + +#if MICROPY_PY_SYS_TRACEBACKLIMIT && !MICROPY_PY_SYS_ATTR_DELEGATION +#error "MICROPY_PY_SYS_TRACEBACKLIMIT requires MICROPY_PY_SYS_ATTR_DELEGATION" +#endif + +#if MICROPY_PY_SYS_ATTR_DELEGATION && !MICROPY_MODULE_ATTR_DELEGATION +#error "MICROPY_PY_SYS_ATTR_DELEGATION requires MICROPY_MODULE_ATTR_DELEGATION" +#endif + #if MICROPY_PY_SYS_ATTR_DELEGATION +// Must be kept in sync with the enum at the top of mpstate.h. STATIC const uint16_t sys_mutable_keys[] = { + #if MICROPY_PY_SYS_PATH + // Code should access this (as an mp_obj_t) for use with e.g. + // mp_obj_list_append by using the `mp_sys_path` macro defined in runtime.h. + MP_QSTR_path, + #endif #if MICROPY_PY_SYS_PS1_PS2 MP_QSTR_ps1, MP_QSTR_ps2, @@ -205,7 +232,7 @@ STATIC const uint16_t sys_mutable_keys[] = { MP_QSTRnull, }; -STATIC void mp_module_sys_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { +void mp_module_sys_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { MP_STATIC_ASSERT(MP_ARRAY_SIZE(sys_mutable_keys) == MP_SYS_MUTABLE_NUM + 1); MP_STATIC_ASSERT(MP_ARRAY_SIZE(MP_STATE_VM(sys_mutable)) == MP_SYS_MUTABLE_NUM); mp_module_generic_attr(attr, dest, sys_mutable_keys, MP_STATE_VM(sys_mutable)); @@ -215,8 +242,9 @@ STATIC void mp_module_sys_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_sys) }, - { MP_ROM_QSTR(MP_QSTR_path), MP_ROM_PTR(&MP_STATE_VM(mp_sys_path_obj)) }, + #if MICROPY_PY_SYS_ARGV { MP_ROM_QSTR(MP_QSTR_argv), MP_ROM_PTR(&MP_STATE_VM(mp_sys_argv_obj)) }, + #endif { MP_ROM_QSTR(MP_QSTR_version), MP_ROM_PTR(&mp_sys_version_obj) }, { MP_ROM_QSTR(MP_QSTR_version_info), MP_ROM_PTR(&mp_sys_version_info_obj) }, { MP_ROM_QSTR(MP_QSTR_implementation), MP_ROM_PTR(&mp_sys_implementation_obj) }, @@ -278,11 +306,6 @@ STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = { #if MICROPY_PY_SYS_ATEXIT { MP_ROM_QSTR(MP_QSTR_atexit), MP_ROM_PTR(&mp_sys_atexit_obj) }, #endif - - #if MICROPY_PY_SYS_ATTR_DELEGATION - // Delegation of attr lookup. - MP_MODULE_ATTR_DELEGATION_ENTRY(&mp_module_sys_attr), - #endif }; STATIC MP_DEFINE_CONST_DICT(mp_module_sys_globals, mp_module_sys_globals_table); @@ -292,12 +315,16 @@ const mp_obj_module_t mp_module_sys = { .globals = (mp_obj_dict_t *)&mp_module_sys_globals, }; +// Unlike the other CPython-compatible modules, sys is not extensible from the +// filesystem. We rely on it to work so that things like sys.path are always +// available. MP_REGISTER_MODULE(MP_QSTR_sys, mp_module_sys); -// If MICROPY_PY_SYS_PATH_ARGV_DEFAULTS is not enabled then these two lists -// must be initialised after the call to mp_init. -MP_REGISTER_ROOT_POINTER(mp_obj_list_t mp_sys_path_obj); +#if MICROPY_PY_SYS_ARGV +// Code should access this (as an mp_obj_t) for use with e.g. +// mp_obj_list_append by using the `mp_sys_argv` macro defined in runtime.h. MP_REGISTER_ROOT_POINTER(mp_obj_list_t mp_sys_argv_obj); +#endif #if MICROPY_PY_SYS_EXC_INFO // current exception being handled, for sys.exc_info() @@ -312,6 +339,7 @@ MP_REGISTER_ROOT_POINTER(mp_obj_t sys_exitfunc); #if MICROPY_PY_SYS_ATTR_DELEGATION // Contains mutable sys attributes. MP_REGISTER_ROOT_POINTER(mp_obj_t sys_mutable[MP_SYS_MUTABLE_NUM]); +MP_REGISTER_MODULE_DELEGATION(mp_module_sys, mp_module_sys_attr); #endif #endif // MICROPY_PY_SYS diff --git a/py/modthread.c b/py/modthread.c index 51d63e4703..53b4cc6d1c 100644 --- a/py/modthread.c +++ b/py/modthread.c @@ -129,6 +129,7 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE( STATIC size_t thread_stack_size = 0; STATIC mp_obj_t mod_thread_get_ident(void) { + // CIRCUITPY: uintptr_t cast to avoid warning return mp_obj_new_int_from_uint((uintptr_t)mp_thread_get_state()); } STATIC MP_DEFINE_CONST_FUN_OBJ_0(mod_thread_get_ident_obj, mod_thread_get_ident); @@ -268,9 +269,7 @@ STATIC mp_obj_t mod_thread_start_new_thread(size_t n_args, const mp_obj_t *args) th_args->fun = args[0]; // spawn the thread! - mp_thread_create(thread_entry, th_args, &th_args->stack_size); - - return mp_const_none; + return mp_obj_new_int_from_uint(mp_thread_create(thread_entry, th_args, &th_args->stack_size)); } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_thread_start_new_thread_obj, 2, 3, mod_thread_start_new_thread); diff --git a/py/mpconfig.h b/py/mpconfig.h index d7fd785fdc..222878558d 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -32,7 +32,7 @@ #else // Current version of MicroPython #define MICROPY_VERSION_MAJOR 1 -#define MICROPY_VERSION_MINOR 20 +#define MICROPY_VERSION_MINOR 21 #define MICROPY_VERSION_MICRO 0 // Combined version as a 32-bit number for convenience @@ -58,6 +58,15 @@ #define CIRCUITPY 0 #endif +// mpconfigport.h is a file containing configuration settings for a +// particular port. mpconfigport.h is actually a default name for +// such config, and it can be overridden using MP_CONFIGFILE preprocessor +// define (you can do that by passing CFLAGS_EXTRA='-DMP_CONFIGFILE=""' +// argument to make when using standard MicroPython makefiles). +// This is useful to have more than one config per port, for example, +// release vs debug configs, etc. Note that if you switch from one config +// to another, you must rebuild from scratch using "-B" switch to make. + // Disable all optional features (i.e. minimal port). #define MICROPY_CONFIG_ROM_LEVEL_MINIMUM (0) // Only enable core features (constrained flash, e.g. STM32L072) @@ -71,15 +80,6 @@ // Enable everything (e.g. coverage) #define MICROPY_CONFIG_ROM_LEVEL_EVERYTHING (50) -// mpconfigport.h is a file containing configuration settings for a -// particular port. mpconfigport.h is actually a default name for -// such config, and it can be overridden using MP_CONFIGFILE preprocessor -// define (you can do that by passing CFLAGS_EXTRA='-DMP_CONFIGFILE=""' -// argument to make when using standard MicroPython makefiles). -// This is useful to have more than one config per port, for example, -// release vs debug configs, etc. Note that if you switch from one config -// to another, you must rebuild from scratch using "-B" switch to make. - #ifdef MP_CONFIGFILE #include MP_CONFIGFILE #else @@ -323,9 +323,11 @@ #define MICROPY_PERSISTENT_CODE_LOAD (0) #endif -// Whether to support saving of persistent code +// Whether to support saving of persistent code, i.e. for mpy-cross to +// generate .mpy files. Enabling this enables additional metadata on raw code +// objects which is also required for sys.settrace. #ifndef MICROPY_PERSISTENT_CODE_SAVE -#define MICROPY_PERSISTENT_CODE_SAVE (0) +#define MICROPY_PERSISTENT_CODE_SAVE (MICROPY_PY_SYS_SETTRACE) #endif // Whether to support saving persistent code to a file via mp_raw_code_save_file @@ -538,15 +540,6 @@ #define MICROPY_OPT_COMPUTED_GOTO (0) #endif -// CIRCUITPY -// Whether to save trade flash space for speed in MICROPY_OPT_COMPUTED_GOTO. -// Costs about 3% speed, saves about 1500 bytes space. In addition to the assumptions -// of MICROPY_OPT_COMPUTED_GOTO, also assumes that mp_execute_bytecode is less than -// 32kB in size. -#ifndef MICROPY_OPT_COMPUTED_GOTO_SAVE_SPACE -#define MICROPY_OPT_COMPUTED_GOTO_SAVE_SPACE (0) -#endif - // Optimise the fast path for loading attributes from instance types. Increases // Thumb2 code size by about 48 bytes. #ifndef MICROPY_OPT_LOAD_ATTR_FAST_PATH @@ -604,12 +597,6 @@ #define MICROPY_HAS_FILE_READER (MICROPY_READER_POSIX || MICROPY_READER_VFS) #endif -// CIRCUITPY -// Number of VFS mounts to persist across soft-reset. -#ifndef MICROPY_FATFS_NUM_PERSISTENT -#define MICROPY_FATFS_NUM_PERSISTENT (0) -#endif - // Hook for the VM at the start of the opcode loop (can contain variable // definitions usable by the other hook functions) #ifndef MICROPY_VM_HOOK_INIT @@ -642,9 +629,15 @@ #define MICROPY_GC_SPLIT_HEAP (0) #endif +// Whether regions should be added/removed from the split heap as needed. +#ifndef MICROPY_GC_SPLIT_HEAP_AUTO +#define MICROPY_GC_SPLIT_HEAP_AUTO (0) +#endif + // Hook to run code during time consuming garbage collector operations +// *i* is the loop index variable (e.g. can be used to run every x loops) #ifndef MICROPY_GC_HOOK_LOOP -#define MICROPY_GC_HOOK_LOOP +#define MICROPY_GC_HOOK_LOOP(i) #endif // Whether to provide m_tracked_calloc, m_tracked_free functions @@ -877,27 +870,38 @@ typedef double mp_float_t; #define MICROPY_STREAMS_POSIX_API (0) #endif -// Whether modules can use MP_MODULE_ATTR_DELEGATION_ENTRY() to delegate failed -// attribute lookups. +// Whether modules can use MP_REGISTER_MODULE_DELEGATION() to delegate failed +// attribute lookups to a custom handler function. #ifndef MICROPY_MODULE_ATTR_DELEGATION -#define MICROPY_MODULE_ATTR_DELEGATION (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#define MICROPY_MODULE_ATTR_DELEGATION (MICROPY_PY_SYS_ATTR_DELEGATION || MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -// Whether to call __init__ when importing builtin modules for the first time +// Whether to call __init__ when importing builtin modules for the first time. +// Modules using this need to handle the possibility that __init__ might be +// called multiple times. #ifndef MICROPY_MODULE_BUILTIN_INIT #define MICROPY_MODULE_BUILTIN_INIT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif +// Whether to allow built-in modules to have sub-packages (by making the +// sub-package a member of their locals dict). Sub-packages should not be +// registered with MP_REGISTER_MODULE, instead they should be added as +// members of the parent's globals dict. To match CPython behavior, +// their __name__ should be "foo.bar"(i.e. QSTR_foo_dot_bar) which will +// require an entry in qstrdefs, although it does also work to just call +// it "bar". Also, because subpackages can be accessed without being +// imported (e.g. as foo.bar after `import foo`), they should not +// have __init__ methods. Instead, the top-level package's __init__ should +// initialise all sub-packages. +#ifndef MICROPY_MODULE_BUILTIN_SUBPACKAGES +#define MICROPY_MODULE_BUILTIN_SUBPACKAGES (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#endif + // Whether to support module-level __getattr__ (see PEP 562) #ifndef MICROPY_MODULE_GETATTR #define MICROPY_MODULE_GETATTR (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif -// Whether module weak links are supported -#ifndef MICROPY_MODULE_WEAK_LINKS -#define MICROPY_MODULE_WEAK_LINKS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) -#endif - // Whether to enable importing foo.py with __name__ set to '__main__' // Used by the unix port for the -m flag. #ifndef MICROPY_MODULE_OVERRIDE_MAIN_IMPORT @@ -1156,7 +1160,7 @@ typedef double mp_float_t; #define MICROPY_PY_BUILTINS_RANGE_BINOP (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) #endif -// Support for callling next() with second argument +// Support for calling next() with second argument #ifndef MICROPY_PY_BUILTINS_NEXT2 #define MICROPY_PY_BUILTINS_NEXT2 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) #endif @@ -1175,13 +1179,13 @@ typedef double mp_float_t; #define MICROPY_PY_ALL_SPECIAL_METHODS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -// Whether to support all inplace arithmetic operarion methods +// Whether to support all inplace arithmetic operation methods // (__imul__, etc.) #ifndef MICROPY_PY_ALL_INPLACE_SPECIAL_METHODS #define MICROPY_PY_ALL_INPLACE_SPECIAL_METHODS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) #endif -// Whether to support reverse arithmetic operarion methods +// Whether to support reverse arithmetic operation methods // (__radd__, etc.). Additionally gated by // MICROPY_PY_ALL_SPECIAL_METHODS. #ifndef MICROPY_PY_REVERSE_SPECIAL_METHODS @@ -1300,7 +1304,7 @@ typedef double mp_float_t; #define MICROPY_PY_COLLECTIONS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif -// Whether to provide "ucollections.deque" type +// Whether to provide "collections.deque" type #ifndef MICROPY_PY_COLLECTIONS_DEQUE #define MICROPY_PY_COLLECTIONS_DEQUE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif @@ -1405,11 +1409,6 @@ typedef double mp_float_t; #define MICROPY_PY_STRUCT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif -// Whether to provide non-standard typecodes in "struct" module -#ifndef MICROPY_NONSTANDARD_TYPECODES -#define MICROPY_NONSTANDARD_TYPECODES (1) -#endif - // Whether to provide "sys" module #ifndef MICROPY_PY_SYS #define MICROPY_PY_SYS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) @@ -1453,6 +1452,23 @@ typedef double mp_float_t; #define MICROPY_PY_SYS_ATEXIT (0) #endif +// Whether to provide the "sys.path" attribute (which forces module delegation +// and mutable sys attributes to be enabled). +// If MICROPY_PY_SYS_PATH_ARGV_DEFAULTS is enabled, this is initialised in +// mp_init to an empty list. Otherwise the port must initialise it using +// `mp_sys_path = mp_obj_new_list(...)`. +#ifndef MICROPY_PY_SYS_PATH +#define MICROPY_PY_SYS_PATH (1) +#endif + +// Whether to provide the "sys.argv" attribute. +// If MICROPY_PY_SYS_PATH_ARGV_DEFAULTS is enabled, this is initialised in +// mp_init to an empty list. Otherwise the port must initialise it using +// `mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), ...);` +#ifndef MICROPY_PY_SYS_ARGV +#define MICROPY_PY_SYS_ARGV (1) +#endif + // Whether to provide sys.{ps1,ps2} mutable attributes, to control REPL prompts #ifndef MICROPY_PY_SYS_PS1_PS2 #define MICROPY_PY_SYS_PS1_PS2 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) @@ -1487,45 +1503,59 @@ typedef double mp_float_t; // Whether the sys module supports attribute delegation // This is enabled automatically when needed by other features #ifndef MICROPY_PY_SYS_ATTR_DELEGATION -#define MICROPY_PY_SYS_ATTR_DELEGATION (MICROPY_PY_SYS_PS1_PS2 || MICROPY_PY_SYS_TRACEBACKLIMIT) +#define MICROPY_PY_SYS_ATTR_DELEGATION (MICROPY_PY_SYS_PATH || MICROPY_PY_SYS_PS1_PS2 || MICROPY_PY_SYS_TRACEBACKLIMIT) #endif -// Whether to provide "uerrno" module -#ifndef MICROPY_PY_UERRNO -#define MICROPY_PY_UERRNO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +// Whether to provide "errno" module +#ifndef MICROPY_PY_ERRNO +#define MICROPY_PY_ERRNO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -// Whether to provide the uerrno.errorcode dict -#ifndef MICROPY_PY_UERRNO_ERRORCODE -#define MICROPY_PY_UERRNO_ERRORCODE (1) +// Whether to provide the errno.errorcode dict +#ifndef MICROPY_PY_ERRNO_ERRORCODE +#define MICROPY_PY_ERRNO_ERRORCODE (1) #endif -// Whether to provide "uselect" module (baremetal implementation) -#ifndef MICROPY_PY_USELECT -#define MICROPY_PY_USELECT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +// Whether to provide "select" module +#ifndef MICROPY_PY_SELECT +#define MICROPY_PY_SELECT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -// Whether to enable the select() function in the "uselect" module (baremetal +// Whether to enable POSIX optimisations in the "select" module (requires system poll) +#ifndef MICROPY_PY_SELECT_POSIX_OPTIMISATIONS +#define MICROPY_PY_SELECT_POSIX_OPTIMISATIONS (0) +#endif + +// Whether to enable the select() function in the "select" module (baremetal // implementation). This is present for compatibility but can be disabled to // save space. -#ifndef MICROPY_PY_USELECT_SELECT -#define MICROPY_PY_USELECT_SELECT (1) +#ifndef MICROPY_PY_SELECT_SELECT +#define MICROPY_PY_SELECT_SELECT (1) #endif -// Whether to provide "utime" module functions implementation -// in terms of mp_hal_* functions. -#ifndef MICROPY_PY_UTIME_MP_HAL -#define MICROPY_PY_UTIME_MP_HAL (0) +// Whether to provide the "time" module +#ifndef MICROPY_PY_TIME +#define MICROPY_PY_TIME (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_BASIC_FEATURES) #endif -// Period of values returned by utime.ticks_ms(), ticks_us(), ticks_cpu() +// Whether to provide time.gmtime/localtime/mktime functions +#ifndef MICROPY_PY_TIME_GMTIME_LOCALTIME_MKTIME +#define MICROPY_PY_TIME_GMTIME_LOCALTIME_MKTIME (0) +#endif + +// Whether to provide time.time/time_ns functions +#ifndef MICROPY_PY_TIME_TIME_TIME_NS +#define MICROPY_PY_TIME_TIME_TIME_NS (0) +#endif + +// Period of values returned by time.ticks_ms(), ticks_us(), ticks_cpu() // functions. Should be power of two. All functions above use the same // period, so if underlying hardware/API has different periods, the // minimum of them should be used. The value below is the maximum value // this parameter can take (corresponding to 30 bit tick values on 32-bit // system). -#ifndef MICROPY_PY_UTIME_TICKS_PERIOD -#define MICROPY_PY_UTIME_TICKS_PERIOD (MP_SMALL_INT_POSITIVE_MASK + 1) +#ifndef MICROPY_PY_TIME_TICKS_PERIOD +#define MICROPY_PY_TIME_TICKS_PERIOD (MP_SMALL_INT_POSITIVE_MASK + 1) #endif // Whether to provide "_thread" module @@ -1547,8 +1577,8 @@ typedef double mp_float_t; // Extended modules -#ifndef MICROPY_PY_UASYNCIO -#define MICROPY_PY_UASYNCIO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#ifndef MICROPY_PY_ASYNCIO +#define MICROPY_PY_ASYNCIO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif #ifndef MICROPY_PY_UCTYPES @@ -1561,105 +1591,107 @@ typedef double mp_float_t; #define MICROPY_PY_UCTYPES_NATIVE_C_TYPES (1) #endif -#ifndef MICROPY_PY_UZLIB -#define MICROPY_PY_UZLIB (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +// CIRCUITPY +// TODO? CIRCUITPY_ZLIB instead +#ifndef MICROPY_PY_ZLIB +#define MICROPY_PY_ZLIB (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) + +// Whether to provide "deflate" module (decompression-only by default) +#ifndef MICROPY_PY_DEFLATE +#define MICROPY_PY_DEFLATE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -#ifndef MICROPY_PY_UJSON -#define MICROPY_PY_UJSON (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +// Whether to provide compression support in "deflate" module +#ifndef MICROPY_PY_DEFLATE_COMPRESS +#define MICROPY_PY_DEFLATE_COMPRESS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES) +#endif + +#ifndef MICROPY_PY_JSON +#define MICROPY_PY_JSON (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support the "separators" argument to dump, dumps -#ifndef MICROPY_PY_UJSON_SEPARATORS -#define MICROPY_PY_UJSON_SEPARATORS (1) +#ifndef MICROPY_PY_JSON_SEPARATORS +#define MICROPY_PY_JSON_SEPARATORS (1) #endif -#ifndef MICROPY_PY_UOS -#define MICROPY_PY_UOS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#ifndef MICROPY_PY_OS +#define MICROPY_PY_OS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -#ifndef MICROPY_PY_UOS_STATVFS -#define MICROPY_PY_UOS_STATVFS (MICROPY_PY_UOS) +#ifndef MICROPY_PY_OS_STATVFS +#define MICROPY_PY_OS_STATVFS (MICROPY_PY_OS) #endif -#ifndef CIRCUITPY_ULAB -#define CIRCUITPY_ULAB (0) +#ifndef MICROPY_PY_RE +#define MICROPY_PY_RE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -#ifndef MICROPY_PY_URE -#define MICROPY_PY_URE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#ifndef MICROPY_PY_RE_DEBUG +#define MICROPY_PY_RE_DEBUG (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) #endif -#ifndef MICROPY_PY_URE_DEBUG -#define MICROPY_PY_URE_DEBUG (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#ifndef MICROPY_PY_RE_MATCH_GROUPS +#define MICROPY_PY_RE_MATCH_GROUPS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) #endif -#ifndef MICROPY_PY_URE_MATCH_GROUPS -#define MICROPY_PY_URE_MATCH_GROUPS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#ifndef MICROPY_PY_RE_MATCH_SPAN_START_END +#define MICROPY_PY_RE_MATCH_SPAN_START_END (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) #endif -#ifndef MICROPY_PY_URE_MATCH_SPAN_START_END -#define MICROPY_PY_URE_MATCH_SPAN_START_END (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#ifndef MICROPY_PY_RE_SUB +#define MICROPY_PY_RE_SUB (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -#ifndef MICROPY_PY_URE_SUB -#define MICROPY_PY_URE_SUB (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#ifndef MICROPY_PY_HEAPQ +#define MICROPY_PY_HEAPQ (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -#ifndef MICROPY_PY_UHEAPQ -#define MICROPY_PY_UHEAPQ (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#ifndef MICROPY_PY_HASHLIB +#define MICROPY_PY_HASHLIB (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -// Optimized heap queue for relative timestamps (only used by uasyncio v2) -#ifndef MICROPY_PY_UTIMEQ -#define MICROPY_PY_UTIMEQ (0) +#ifndef MICROPY_PY_HASHLIB_MD5 +#define MICROPY_PY_HASHLIB_MD5 (0) #endif -#ifndef MICROPY_PY_UHASHLIB -#define MICROPY_PY_UHASHLIB (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#ifndef MICROPY_PY_HASHLIB_SHA1 +#define MICROPY_PY_HASHLIB_SHA1 (0) #endif -#ifndef MICROPY_PY_UHASHLIB_MD5 -#define MICROPY_PY_UHASHLIB_MD5 (0) +#ifndef MICROPY_PY_HASHLIB_SHA256 +#define MICROPY_PY_HASHLIB_SHA256 (1) #endif -#ifndef MICROPY_PY_UHASHLIB_SHA1 -#define MICROPY_PY_UHASHLIB_SHA1 (0) +#ifndef MICROPY_PY_CRYPTOLIB +#define MICROPY_PY_CRYPTOLIB (0) #endif -#ifndef MICROPY_PY_UHASHLIB_SHA256 -#define MICROPY_PY_UHASHLIB_SHA256 (1) +// Depends on MICROPY_PY_CRYPTOLIB +#ifndef MICROPY_PY_CRYPTOLIB_CTR +#define MICROPY_PY_CRYPTOLIB_CTR (0) #endif -#ifndef MICROPY_PY_UCRYPTOLIB -#define MICROPY_PY_UCRYPTOLIB (0) +#ifndef MICROPY_PY_CRYPTOLIB_CONSTS +#define MICROPY_PY_CRYPTOLIB_CONSTS (0) #endif -// Depends on MICROPY_PY_UCRYPTOLIB -#ifndef MICROPY_PY_UCRYPTOLIB_CTR -#define MICROPY_PY_UCRYPTOLIB_CTR (0) +#ifndef MICROPY_PY_BINASCII +#define MICROPY_PY_BINASCII (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -#ifndef MICROPY_PY_UCRYPTOLIB_CONSTS -#define MICROPY_PY_UCRYPTOLIB_CONSTS (0) +// CIRCUITPY: does not depend on MICROPY_PY_DEFLATE +#ifndef MICROPY_PY_BINASCII_CRC32 +#define MICROPY_PY_BINASCII_CRC32 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -#ifndef MICROPY_PY_UBINASCII -#define MICROPY_PY_UBINASCII (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) -#endif - -// Depends on MICROPY_PY_UZLIB -#ifndef MICROPY_PY_UBINASCII_CRC32 -#define MICROPY_PY_UBINASCII_CRC32 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) -#endif - -#ifndef MICROPY_PY_URANDOM -#define MICROPY_PY_URANDOM (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#ifndef MICROPY_PY_RANDOM +#define MICROPY_PY_RANDOM (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to include: randrange, randint, choice, random, uniform -#ifndef MICROPY_PY_URANDOM_EXTRA_FUNCS -#define MICROPY_PY_URANDOM_EXTRA_FUNCS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#ifndef MICROPY_PY_RANDOM_EXTRA_FUNCS +#define MICROPY_PY_RANDOM_EXTRA_FUNCS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif #ifndef MICROPY_PY_MACHINE @@ -1705,21 +1737,21 @@ typedef double mp_float_t; #endif // The default backlog value for socket.listen(backlog) -#ifndef MICROPY_PY_USOCKET_LISTEN_BACKLOG_DEFAULT -#define MICROPY_PY_USOCKET_LISTEN_BACKLOG_DEFAULT (2) +#ifndef MICROPY_PY_SOCKET_LISTEN_BACKLOG_DEFAULT +#define MICROPY_PY_SOCKET_LISTEN_BACKLOG_DEFAULT (2) #endif -#ifndef MICROPY_PY_USSL -#define MICROPY_PY_USSL (0) +#ifndef MICROPY_PY_SSL +#define MICROPY_PY_SSL (0) #endif -// Whether to add finaliser code to ussl objects -#ifndef MICROPY_PY_USSL_FINALISER -#define MICROPY_PY_USSL_FINALISER (0) +// Whether to add finaliser code to ssl objects +#ifndef MICROPY_PY_SSL_FINALISER +#define MICROPY_PY_SSL_FINALISER (MICROPY_ENABLE_FINALISER) #endif -#ifndef MICROPY_PY_UWEBSOCKET -#define MICROPY_PY_UWEBSOCKET (0) +#ifndef MICROPY_PY_WEBSOCKET +#define MICROPY_PY_WEBSOCKET (0) #endif #ifndef MICROPY_PY_FRAMEBUF @@ -1730,18 +1762,14 @@ typedef double mp_float_t; #define MICROPY_PY_BTREE (0) #endif -#ifndef MICROPY_HW_ENABLE_USB -#define MICROPY_HW_ENABLE_USB (0) -#endif - // Whether to provide the low-level "_onewire" module #ifndef MICROPY_PY_ONEWIRE #define MICROPY_PY_ONEWIRE (0) #endif // Whether to provide the "platform" module -#ifndef MICROPY_PY_UPLATFORM -#define MICROPY_PY_UPLATFORM (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#ifndef MICROPY_PY_PLATFORM +#define MICROPY_PY_PLATFORM (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif /*****************************************************************************/ @@ -1904,6 +1932,16 @@ typedef double mp_float_t; #define MP_PLAT_FREE_EXEC(ptr, size) m_del(byte, ptr, size) #endif +// Allocating new heap area at runtime requires port to be able to allocate from system heap +#if MICROPY_GC_SPLIT_HEAP_AUTO +#ifndef MP_PLAT_ALLOC_HEAP +#define MP_PLAT_ALLOC_HEAP(size) malloc(size) +#endif +#ifndef MP_PLAT_FREE_HEAP +#define MP_PLAT_FREE_HEAP(ptr) free(ptr) +#endif +#endif + // This macro is used to do all output (except when MICROPY_PY_IO is defined) #ifndef MP_PLAT_PRINT_STRN #define MP_PLAT_PRINT_STRN(str, len) mp_hal_stdout_tx_strn_cooked(str, len) @@ -1939,17 +1977,6 @@ typedef double mp_float_t; #define MP_WEAK __attribute__((weak)) #endif -// Modifier for functions which should not be instrumented when tracing with -// -finstrument-functions -#ifndef MP_NO_INSTRUMENT -#define MP_NO_INSTRUMENT __attribute__((no_instrument_function)) -#endif - -// Modifier for functions which should ideally inlined -#ifndef MP_INLINE -#define MP_INLINE inline MP_NO_INSTRUMENT -#endif - // Modifier for functions which should be never inlined #ifndef MP_NOINLINE #define MP_NOINLINE __attribute__((noinline)) @@ -1970,12 +1997,6 @@ typedef double mp_float_t; #define MP_UNLIKELY(x) __builtin_expect((x), 0) #endif -// Modifier for functions which aren't often used. Calls will also be considered -// unlikely. Section names are `.text.unlikely` for use in linker scripts. -#ifndef MP_COLD -#define MP_COLD __attribute__((cold)) -#endif - // To annotate that code is unreachable #ifndef MP_UNREACHABLE #if defined(__GNUC__) @@ -2023,14 +2044,4 @@ typedef double mp_float_t; #define MP_WARN_CAT(x) (NULL) #endif -// Feature dependency check. -#if MICROPY_PY_SYS_SETTRACE -#if !MICROPY_PERSISTENT_CODE_SAVE -#error "MICROPY_PY_SYS_SETTRACE requires MICROPY_PERSISTENT_CODE_SAVE to be enabled" -#endif -#if MICROPY_COMP_CONST -#error "MICROPY_PY_SYS_SETTRACE requires MICROPY_COMP_CONST to be disabled" -#endif -#endif - #endif // MICROPY_INCLUDED_PY_MPCONFIG_H diff --git a/py/mperrno.h b/py/mperrno.h index 4c9af57ee8..94116a0773 100644 --- a/py/mperrno.h +++ b/py/mperrno.h @@ -142,7 +142,13 @@ #endif +#if MICROPY_PY_ERRNO + +#include "py/obj.h" + qstr mp_errno_to_str(mp_obj_t errno_val); +// CIRCUITPY const char *mp_common_errno_to_str(mp_obj_t errno_val, char *buf, size_t len); +#endif #endif // MICROPY_INCLUDED_PY_MPERRNO_H diff --git a/py/mpprint.c b/py/mpprint.c index 527acd471d..0d214a01f6 100644 --- a/py/mpprint.c +++ b/py/mpprint.c @@ -376,6 +376,7 @@ int mp_print_float(const mp_print_t *print, mp_float_t f, char fmt, int flags, c } #endif +// CIRCUITPY static int print_str_common(const mp_print_t *print, const char *str, int prec, size_t len, int flags, int fill, int width) { if (prec >= 0 && (size_t)prec < len) { len = prec; diff --git a/py/mpprint.h b/py/mpprint.h index 562afdad44..5f7a07af98 100644 --- a/py/mpprint.h +++ b/py/mpprint.h @@ -79,6 +79,7 @@ int mp_printf(const mp_print_t *print, const char *fmt, ...); int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args); #endif +// CIRCUITPY struct compressed_string; int mp_cprintf(const mp_print_t *print, const struct compressed_string *compressed_fmt, ...); #ifdef va_start diff --git a/py/mpstate.c b/py/mpstate.c index 32f1d60a59..d7eee3cd00 100644 --- a/py/mpstate.c +++ b/py/mpstate.c @@ -25,6 +25,7 @@ */ #include "py/mpstate.h" +// CIRCUITPY #include "supervisor/linker.h" #if MICROPY_DYNAMIC_COMPILER diff --git a/py/mpstate.h b/py/mpstate.h index 1519efb71d..0b17d22646 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -46,7 +46,12 @@ // memory system, runtime and virtual machine. The state is a global // variable, but in the future it is hoped that the state can become local. +#if MICROPY_PY_SYS_ATTR_DELEGATION +// Must be kept in sync with sys_mutable_keys in modsys.c. enum { + #if MICROPY_PY_SYS_PATH + MP_SYS_MUTABLE_PATH, + #endif #if MICROPY_PY_SYS_PS1_PS2 MP_SYS_MUTABLE_PS1, MP_SYS_MUTABLE_PS2, @@ -56,6 +61,7 @@ enum { #endif MP_SYS_MUTABLE_NUM, }; +#endif // MICROPY_PY_SYS_ATTR_DELEGATION // This structure contains dynamic configuration for the compiler. #if MICROPY_DYNAMIC_COMPILER @@ -93,6 +99,7 @@ typedef struct _mp_state_mem_area_t { byte *gc_pool_end; size_t gc_last_free_atb_index; + size_t gc_last_used_block; // The block ID of the highest block allocated in the area } mp_state_mem_area_t; // This structure hold information about the memory allocation system. @@ -291,6 +298,7 @@ typedef struct _mp_state_thread_t { mp_obj_dict_t *dict_globals; nlr_buf_t *nlr_top; + nlr_jump_callback_node_t *nlr_jump_callback_top; // pending exception object (MP_OBJ_NULL if not pending) volatile mp_obj_t mp_pending_exception; diff --git a/py/mpthread.h b/py/mpthread.h index e611ef4c11..f335cc0291 100644 --- a/py/mpthread.h +++ b/py/mpthread.h @@ -40,7 +40,8 @@ struct _mp_state_thread_t; struct _mp_state_thread_t *mp_thread_get_state(void); void mp_thread_set_state(struct _mp_state_thread_t *state); -void mp_thread_create(void *(*entry)(void *), void *arg, size_t *stack_size); +mp_uint_t mp_thread_create(void *(*entry)(void *), void *arg, size_t *stack_size); +mp_uint_t mp_thread_get_id(void); void mp_thread_start(void); void mp_thread_finish(void); void mp_thread_mutex_init(mp_thread_mutex_t *mutex); diff --git a/py/mpz.c b/py/mpz.c index 3218d5f392..9b62f87ab0 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -452,7 +452,7 @@ STATIC size_t mpn_mul(mpz_dig_t *idig, mpz_dig_t *jdig, size_t jlen, mpz_dig_t * } ilen = id - oidig; - // check to prevent usb starvation + // CIRCUITPY: check to prevent usb starvation #ifdef RUN_BACKGROUND_TASKS RUN_BACKGROUND_TASKS; #endif diff --git a/py/mpz.h b/py/mpz.h index 0de0b31f73..9e4c3d6469 100644 --- a/py/mpz.h +++ b/py/mpz.h @@ -142,6 +142,7 @@ void mpz_divmod_inpl(mpz_t *dest_quo, mpz_t *dest_rem, const mpz_t *lhs, const m static inline size_t mpz_max_num_bits(const mpz_t *z) { return z->len * MPZ_DIG_SIZE; } +// CIRCUITPY static inline size_t mpz_num_bits(const mpz_t *z) { if (mpz_is_zero(z)) { return 0; diff --git a/py/nativeglue.c b/py/nativeglue.c index 2afe466428..f15f1033be 100644 --- a/py/nativeglue.c +++ b/py/nativeglue.c @@ -105,6 +105,8 @@ mp_obj_t mp_native_to_obj(mp_uint_t val, mp_uint_t type) { return mp_obj_new_int(val); case MP_NATIVE_TYPE_UINT: return mp_obj_new_int_from_uint(val); + case MP_NATIVE_TYPE_QSTR: + return MP_OBJ_NEW_QSTR(val); default: // a pointer // we return just the value of the pointer as an integer return mp_obj_new_int_from_uint(val); diff --git a/py/nlr.c b/py/nlr.c index 6435df66fb..69d2526178 100644 --- a/py/nlr.c +++ b/py/nlr.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * Copyright (c) 2013-2017 Damien P. George + * Copyright (c) 2013-2023 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -50,6 +50,36 @@ void nlr_pop(void) { *top = (*top)->prev; } +void nlr_push_jump_callback(nlr_jump_callback_node_t *node, nlr_jump_callback_fun_t fun) { + nlr_jump_callback_node_t **top = &MP_STATE_THREAD(nlr_jump_callback_top); + node->prev = *top; + node->fun = fun; + *top = node; +} + +void nlr_pop_jump_callback(bool run_callback) { + nlr_jump_callback_node_t **top = &MP_STATE_THREAD(nlr_jump_callback_top); + nlr_jump_callback_node_t *cur = *top; + *top = (*top)->prev; + if (run_callback) { + cur->fun(cur); + } +} + +// This function pops and runs all callbacks that were registered after `nlr` +// was pushed (via nlr_push). It assumes: +// - a descending C stack, +// - that all nlr_jump_callback_node_t's in the linked-list pointed to by +// nlr_jump_callback_top are on the C stack +// It works by popping each node in turn until the next node is NULL or above +// the `nlr` pointer on the C stack (and so pushed before `nlr` was pushed). +void nlr_call_jump_callbacks(nlr_buf_t *nlr) { + nlr_jump_callback_node_t **top = &MP_STATE_THREAD(nlr_jump_callback_top); + while (*top != NULL && (void *)*top < (void *)nlr) { + nlr_pop_jump_callback(true); + } +} + #if MICROPY_ENABLE_VM_ABORT NORETURN void nlr_jump_abort(void) { MP_STATE_THREAD(nlr_top) = MP_STATE_VM(nlr_abort); diff --git a/py/nlr.h b/py/nlr.h index 2b8ec4b64e..827979f34f 100644 --- a/py/nlr.h +++ b/py/nlr.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013-2023 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,6 +31,7 @@ #include #include +#include #include "py/mpconfig.h" @@ -133,6 +134,15 @@ struct _nlr_buf_t { #endif }; +typedef void (*nlr_jump_callback_fun_t)(void *ctx); + +typedef struct _nlr_jump_callback_node_t nlr_jump_callback_node_t; + +struct _nlr_jump_callback_node_t { + nlr_jump_callback_node_t *prev; + nlr_jump_callback_fun_t fun; +}; + // Helper macros to save/restore the pystack state #if MICROPY_ENABLE_PYSTACK #define MP_NLR_SAVE_PYSTACK(nlr_buf) (nlr_buf)->pystack = MP_STATE_THREAD(pystack_cur) @@ -150,6 +160,7 @@ struct _nlr_buf_t { nlr_jump_fail(val); \ } \ top->ret_val = val; \ + nlr_call_jump_callbacks(top); \ MP_NLR_RESTORE_PYSTACK(top); \ *_top_ptr = top->prev; \ @@ -181,11 +192,9 @@ NORETURN void nlr_jump_fail(void *val); #ifndef MICROPY_DEBUG_NLR #define nlr_raise(val) nlr_jump(MP_OBJ_TO_PTR(val)) #else -#include "mpstate.h" + #define nlr_raise(val) \ do { \ - /*printf("nlr_raise: nlr_top=%p\n", MP_STATE_THREAD(nlr_top)); \ - fflush(stdout);*/ \ void *_val = MP_OBJ_TO_PTR(val); \ assert(_val != NULL); \ assert(mp_obj_is_exception_instance(val)); \ @@ -195,13 +204,20 @@ NORETURN void nlr_jump_fail(void *val); #if !MICROPY_NLR_SETJMP #define nlr_push(val) \ assert(MP_STATE_THREAD(nlr_top) != val), nlr_push(val) - -/* -#define nlr_push(val) \ - printf("nlr_push: before: nlr_top=%p, val=%p\n", MP_STATE_THREAD(nlr_top), val),assert(MP_STATE_THREAD(nlr_top) != val),nlr_push(val) -*/ #endif #endif +// Push a callback on to the linked-list of NLR jump callbacks. The `node` pointer must +// be on the C stack. The `fun` callback will be executed if an NLR jump is taken which +// unwinds the C stack through this `node`. +void nlr_push_jump_callback(nlr_jump_callback_node_t *node, nlr_jump_callback_fun_t fun); + +// Pop a callback from the linked-list of NLR jump callbacks. The corresponding function +// will be called if `run_callback` is true. +void nlr_pop_jump_callback(bool run_callback); + +// Pop and call all NLR jump callbacks that were registered after `nlr` buffer was pushed. +void nlr_call_jump_callbacks(nlr_buf_t *nlr); + #endif // MICROPY_INCLUDED_PY_NLR_H diff --git a/py/nlraarch64.c b/py/nlraarch64.c index 9b00a2856a..5410aa4526 100644 --- a/py/nlraarch64.c +++ b/py/nlraarch64.c @@ -62,13 +62,14 @@ NORETURN void nlr_jump(void *val) { MP_STATIC_ASSERT(offsetof(nlr_buf_t, regs) == 16); // asm assumes it __asm volatile ( - "ldr x29, [%0, #112]\n" - "ldp x27, x28, [%0, #96]\n" - "ldp x25, x26, [%0, #80]\n" - "ldp x23, x24, [%0, #64]\n" - "ldp x21, x22, [%0, #48]\n" - "ldp x19, x20, [%0, #32]\n" - "ldp lr, x9, [%0, #16]\n" // 16 == offsetof(nlr_buf_t, regs) + "mov x0, %0 \n" + "ldr x29, [x0, #112]\n" + "ldp x27, x28, [x0, #96]\n" + "ldp x25, x26, [x0, #80]\n" + "ldp x23, x24, [x0, #64]\n" + "ldp x21, x22, [x0, #48]\n" + "ldp x19, x20, [x0, #32]\n" + "ldp lr, x9, [x0, #16]\n" // 16 == offsetof(nlr_buf_t, regs) "mov sp, x9 \n" "mov x0, #1 \n" // non-local return "ret \n" diff --git a/py/nlrsetjmp.c b/py/nlrsetjmp.c index 960dd86f52..73fbe81261 100644 --- a/py/nlrsetjmp.c +++ b/py/nlrsetjmp.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * Copyright (c) 2013-2017 Damien P. George + * Copyright (c) 2013-2023 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,14 +29,7 @@ #if MICROPY_NLR_SETJMP void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - top->ret_val = val; - MP_NLR_RESTORE_PYSTACK(top); - *top_ptr = top->prev; + MP_NLR_JUMP_HEAD(val, top); longjmp(top->jmpbuf, 1); } diff --git a/py/nlrx64.c b/py/nlrx64.c index df620ed9f7..7c271fef2e 100644 --- a/py/nlrx64.c +++ b/py/nlrx64.c @@ -35,8 +35,27 @@ __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); +#if !MICROPY_NLR_OS_WINDOWS +#if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 8) +#define USE_NAKED 1 +#else +// On older gcc the equivalent here is to force omit-frame-pointer +__attribute__((optimize("omit-frame-pointer"))) +#endif +#endif + +#if !defined(USE_NAKED) +#define USE_NAKED 0 +#endif + +#if USE_NAKED +// nlr_push prelude should never push frame pointer register ebp onto the stack +__attribute__((naked)) +#endif unsigned int nlr_push(nlr_buf_t *nlr) { + #if !USE_NAKED (void)nlr; + #endif #if MICROPY_NLR_OS_WINDOWS @@ -58,9 +77,6 @@ unsigned int nlr_push(nlr_buf_t *nlr) { #else __asm volatile ( - #if defined(__APPLE__) && defined(__MACH__) - "pop %rbp \n" // undo function's prelude - #endif "movq (%rsp), %rax \n" // load return %rip "movq %rax, 16(%rdi) \n" // store %rip into nlr_buf "movq %rbp, 24(%rdi) \n" // store %rbp into nlr_buf @@ -79,7 +95,9 @@ unsigned int nlr_push(nlr_buf_t *nlr) { #endif + #if !USE_NAKED return 0; // needed to silence compiler warning + #endif } NORETURN void nlr_jump(void *val) { diff --git a/py/obj.c b/py/obj.c index f8316ce5b2..8929a04dd3 100644 --- a/py/obj.c +++ b/py/obj.c @@ -107,12 +107,14 @@ const mp_obj_type_t *MICROPY_WRAP_MP_OBJ_GET_TYPE(mp_obj_get_type)(mp_const_obj_ } const char *mp_obj_get_type_str(mp_const_obj_t o_in) { + // CIRCUITPY return qstr_str(mp_obj_get_type_qstr(o_in)); } void mp_obj_print_helper(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { // There can be data structures nested too deep, or just recursive MP_STACK_CHECK(); + // CIRCUITPY #ifdef RUN_BACKGROUND_TASKS RUN_BACKGROUND_TASKS; #endif @@ -141,6 +143,7 @@ void mp_obj_print(mp_obj_t o_in, mp_print_kind_t kind) { mp_obj_print_helper(MP_PYTHON_PRINTER, o_in, kind); } +// CIRCUITPY static void mp_obj_print_inner_exception(const mp_print_t *print, mp_obj_t self_in, mp_int_t limit) { #if MICROPY_CPYTHON_EXCEPTION_CHAIN mp_obj_exception_t *self = mp_obj_exception_get_native(self_in); @@ -163,6 +166,7 @@ static void mp_obj_print_inner_exception(const mp_print_t *print, mp_obj_t self_ #endif } +// CIRCUITPY // helper function to print an exception with traceback void mp_obj_print_exception_with_limit(const mp_print_t *print, mp_obj_t exc, mp_int_t limit) { if (mp_obj_is_exception_instance(exc) && stack_ok()) { @@ -223,6 +227,7 @@ void mp_obj_print_exception_with_limit(const mp_print_t *print, mp_obj_t exc, mp mp_print_str(print, "\n"); } +// CIRCUITPY void mp_obj_print_exception(const mp_print_t *print, mp_obj_t exc) { mp_obj_print_exception_with_limit(print, exc, 0); } @@ -358,7 +363,7 @@ mp_obj_t mp_obj_equal_not_equal(mp_binary_op_t op, mp_obj_t o1, mp_obj_t o2) { o2 = temp; } - // equality not implemented, so fall back to pointer conparison + // equality not implemented, so fall back to pointer comparison return (o1 == o2) ? local_true : local_false; } @@ -370,18 +375,11 @@ mp_int_t mp_obj_get_int(mp_const_obj_t arg) { // This function essentially performs implicit type conversion to int // Note that Python does NOT provide implicit type conversion from // float to int in the core expression language, try some_list[1.0]. - if (arg == mp_const_false) { - return 0; - } else if (arg == mp_const_true) { - return 1; - } else if (mp_obj_is_small_int(arg)) { - return MP_OBJ_SMALL_INT_VALUE(arg); - } else if (mp_obj_is_exact_type(arg, &mp_type_int)) { - return mp_obj_int_get_checked(arg); - } else { - mp_obj_t res = mp_unary_op(MP_UNARY_OP_INT, (mp_obj_t)arg); - return mp_obj_int_get_checked(res); + mp_int_t val; + if (!mp_obj_get_int_maybe(arg, &val)) { + mp_raise_TypeError_int_conversion(arg); } + return val; } mp_int_t mp_obj_get_int_truncated(mp_const_obj_t arg) { @@ -405,7 +403,12 @@ bool mp_obj_get_int_maybe(mp_const_obj_t arg, mp_int_t *value) { } else if (mp_obj_is_exact_type(arg, &mp_type_int)) { *value = mp_obj_int_get_checked(arg); } else { - return false; + arg = mp_unary_op(MP_UNARY_OP_INT_MAYBE, (mp_obj_t)arg); + if (arg != MP_OBJ_NULL) { + *value = mp_obj_int_get_checked(arg); + } else { + return false; + } } return true; } @@ -485,6 +488,7 @@ void mp_obj_get_complex(mp_obj_t arg, mp_float_t *real, mp_float_t *imag) { // note: returned value in *items may point to the interior of a GC block void mp_obj_get_array(mp_obj_t o, size_t *len, mp_obj_t **items) { + // CIRCUITPY if (mp_obj_is_tuple_compatible(o)) { mp_obj_tuple_get(o, len, items); } else if (mp_obj_is_type(o, &mp_type_list)) { @@ -538,6 +542,7 @@ size_t mp_get_index(const mp_obj_type_t *type, size_t len, mp_obj_t index, bool i = len; } } else { + // CIRCUITPY mp_arg_validate_index_range(i, 0, len - 1, MP_QSTR_index); } @@ -695,12 +700,3 @@ void mp_get_buffer_raise(mp_obj_t obj, mp_buffer_info_t *bufinfo, mp_uint_t flag mp_raise_TypeError(MP_ERROR_TEXT("object with buffer protocol required")); } } - -mp_obj_t mp_generic_unary_op(mp_unary_op_t op, mp_obj_t o_in) { - switch (op) { - case MP_UNARY_OP_HASH: - return MP_OBJ_NEW_SMALL_INT((mp_uint_t)o_in); - default: - return MP_OBJ_NULL; // op not supported - } -} diff --git a/py/obj.h b/py/obj.h index 789e05b884..cf65c6a777 100644 --- a/py/obj.h +++ b/py/obj.h @@ -34,6 +34,7 @@ #include "py/mpprint.h" #include "py/runtime0.h" +// CIRCUITPY #include "supervisor/shared/translate/compressed_string.h" // This is the definition of the opaque MicroPython object type. @@ -465,6 +466,13 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t; // param obj_module: mp_obj_module_t instance #define MP_REGISTER_MODULE(module_name, obj_module) +// As above, but allow this module to be extended from the filesystem. +#define MP_REGISTER_EXTENSIBLE_MODULE(module_name, obj_module) + +// Add a custom handler for a builtin module that will be called to delegate +// failed attribute lookups. +#define MP_REGISTER_MODULE_DELEGATION(obj_module, fun_name) + // Declare a root pointer (to avoid garbage collection of a global static variable). // param variable_declaration: a valid C variable declaration #define MP_REGISTER_ROOT_POINTER(variable_declaration) @@ -825,6 +833,7 @@ extern const mp_obj_type_t mp_type_bytearray; extern const mp_obj_type_t mp_type_memoryview; extern const mp_obj_type_t mp_type_float; extern const mp_obj_type_t mp_type_complex; +// CIRCUITPY extern const mp_obj_type_t mp_type_traceback; extern const mp_obj_type_t mp_type_tuple; extern const mp_obj_type_t mp_type_list; @@ -842,10 +851,13 @@ extern const mp_obj_type_t mp_type_zip; extern const mp_obj_type_t mp_type_array; extern const mp_obj_type_t mp_type_super; extern const mp_obj_type_t mp_type_gen_wrap; +// CIRCUITPY distinguishes generators and coroutines extern const mp_obj_type_t mp_type_coro_wrap; extern const mp_obj_type_t mp_type_native_gen_wrap; +// CIRCUITPY distinguishes generators and coroutines extern const mp_obj_type_t mp_type_native_coro_wrap; extern const mp_obj_type_t mp_type_gen_instance; +// CIRCUITPY distinguishes generators and coroutines extern const mp_obj_type_t mp_type_coro_instance; extern const mp_obj_type_t mp_type_fun_builtin_0; extern const mp_obj_type_t mp_type_fun_builtin_1; @@ -877,6 +889,7 @@ extern const mp_obj_type_t mp_type_ImportError; extern const mp_obj_type_t mp_type_IndentationError; extern const mp_obj_type_t mp_type_IndexError; extern const mp_obj_type_t mp_type_KeyboardInterrupt; +// CIRCUITPY extern const mp_obj_type_t mp_type_ReloadException; extern const mp_obj_type_t mp_type_KeyError; extern const mp_obj_type_t mp_type_LookupError; @@ -884,7 +897,9 @@ extern const mp_obj_type_t mp_type_MemoryError; extern const mp_obj_type_t mp_type_NameError; extern const mp_obj_type_t mp_type_NotImplementedError; extern const mp_obj_type_t mp_type_OSError; +// CIRCUITPY extern const mp_obj_type_t mp_type_ConnectionError; +// CIRCUITPY extern const mp_obj_type_t mp_type_BrokenPipeError; extern const mp_obj_type_t mp_type_OverflowError; extern const mp_obj_type_t mp_type_RuntimeError; @@ -928,6 +943,7 @@ extern const struct _mp_obj_str_t mp_const_empty_bytes_obj; extern const struct _mp_obj_tuple_t mp_const_empty_tuple_obj; extern const struct _mp_obj_dict_t mp_const_empty_dict_obj; extern const struct _mp_obj_singleton_t mp_const_ellipsis_obj; +// CIRCUITPY next several lines extern const struct _mp_obj_traceback_t mp_const_empty_traceback_obj; extern const struct _mp_obj_singleton_t mp_const_notimplemented_obj; #if MICROPY_CONST_GENERATOREXIT_OBJ @@ -976,6 +992,7 @@ void *mp_obj_malloc_helper(size_t num_bytes, const mp_obj_type_t *type); #define mp_obj_is_str_or_bytes(o) (mp_obj_is_qstr(o) || (mp_obj_is_obj(o) && MP_OBJ_TYPE_GET_SLOT_OR_NULL(((mp_obj_base_t *)MP_OBJ_TO_PTR(o))->type, binary_op) == mp_obj_str_binary_op)) bool mp_obj_is_dict_or_ordereddict(mp_obj_t o); #define mp_obj_is_fun(o) (mp_obj_is_obj(o) && (((mp_obj_base_t *)MP_OBJ_TO_PTR(o))->type->name == MP_QSTR_function)) +// CIRCUITPY // type check is done on iter method to allow tuple, namedtuple, attrtuple #define mp_obj_is_tuple_compatible(o) (MP_OBJ_TYPE_GET_SLOT_OR_NULL(mp_obj_get_type(o), iter) == mp_obj_tuple_getiter) @@ -1018,10 +1035,12 @@ mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args, #define mp_obj_new_exception_msg(exc_type, msg) mp_obj_new_exception(exc_type) #define mp_obj_new_exception_msg_varg(exc_type, ...) mp_obj_new_exception(exc_type) #else +// CIRCUITPY mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const compressed_string_t *msg); mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const compressed_string_t *fmt, ...); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!) #endif #ifdef va_start +// CIRCUITPY mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const compressed_string_t *fmt, va_list arg); // same fmt restrictions as above #endif mp_obj_t mp_obj_new_gen_wrap(mp_obj_t fun); @@ -1038,6 +1057,7 @@ mp_obj_t mp_obj_new_memoryview(byte typecode, size_t nitems, void *items); const mp_obj_type_t *mp_obj_get_type(mp_const_obj_t o_in); const char *mp_obj_get_type_str(mp_const_obj_t o_in); +CIRCUITPY #define mp_obj_get_type_qstr(o_in) (mp_obj_get_type((o_in))->name) bool mp_obj_is_subclass_fast(mp_const_obj_t object, mp_const_obj_t classinfo); // arguments should be type objects mp_obj_t mp_obj_cast_to_native_base(mp_obj_t self_in, mp_const_obj_t native_type); @@ -1045,6 +1065,7 @@ mp_obj_t mp_obj_cast_to_native_base(mp_obj_t self_in, mp_const_obj_t native_type void mp_obj_print_helper(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind); void mp_obj_print(mp_obj_t o, mp_print_kind_t kind); void mp_obj_print_exception(const mp_print_t *print, mp_obj_t exc); +// CIRCUITPY void mp_obj_print_exception_with_limit(const mp_print_t *print, mp_obj_t exc, mp_int_t limit); bool mp_obj_is_true(mp_obj_t arg); @@ -1073,7 +1094,6 @@ mp_obj_t mp_obj_id(mp_obj_t o_in); mp_obj_t mp_obj_len(mp_obj_t o_in); mp_obj_t mp_obj_len_maybe(mp_obj_t o_in); // may return MP_OBJ_NULL mp_obj_t mp_obj_subscr(mp_obj_t base, mp_obj_t index, mp_obj_t val); -mp_obj_t mp_generic_unary_op(mp_unary_op_t op, mp_obj_t o_in); // cell @@ -1108,6 +1128,7 @@ bool mp_obj_exception_match(mp_obj_t exc, mp_const_obj_t exc_type); void mp_obj_exception_clear_traceback(mp_obj_t self_in); void mp_obj_exception_add_traceback(mp_obj_t self_in, qstr file, size_t line, qstr block); void mp_obj_exception_get_traceback(mp_obj_t self_in, size_t *n, size_t **values); +// CIRCUITPY mp_obj_t mp_obj_exception_get_traceback_obj(mp_obj_t self_in); mp_obj_t mp_obj_exception_get_value(mp_obj_t self_in); mp_obj_t mp_obj_exception_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *args); @@ -1184,6 +1205,7 @@ void mp_obj_tuple_del(mp_obj_t self_in); mp_int_t mp_obj_tuple_hash(mp_obj_t self_in); // list +// CIRCUITPY mp_obj_t mp_obj_list_clear(mp_obj_t self_in); mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg); mp_obj_t mp_obj_list_remove(mp_obj_t self_in, mp_obj_t value); @@ -1253,6 +1275,7 @@ qstr mp_obj_fun_get_name(mp_const_obj_t fun); mp_obj_t mp_identity(mp_obj_t self); MP_DECLARE_CONST_FUN_OBJ_1(mp_identity_obj); +// CIRCUITPY // Generic iterator that uses unary op and subscr to iterate over a native type. It will be slower // than a custom iterator but applies broadly. mp_obj_t mp_obj_generic_subscript_getiter(mp_obj_t self, mp_obj_iter_buf_t *iter_buf); @@ -1265,8 +1288,6 @@ typedef struct _mp_obj_module_t { static inline mp_obj_dict_t *mp_obj_module_get_globals(mp_obj_t module) { return ((mp_obj_module_t *)MP_OBJ_TO_PTR(module))->globals; } -// check if given module object is a package -bool mp_obj_is_package(mp_obj_t module); // staticmethod and classmethod types; defined here so we can make const versions // this structure is used for instances of both staticmethod and classmethod @@ -1280,10 +1301,12 @@ typedef struct _mp_rom_obj_static_class_method_t { } mp_rom_obj_static_class_method_t; // property +// CIRCUITPY const mp_obj_t *mp_obj_property_get(mp_obj_t self_in, size_t *n_proxy); // sequence helpers +// CIRCUITPY // Compute the new length of a sequence and ensure an exception is thrown on overflow. size_t mp_seq_multiply_len(size_t item_sz, size_t len); void mp_seq_multiply(const void *items, size_t item_sz, size_t len, size_t times, void *dest); diff --git a/py/objarray.c b/py/objarray.c index d54f586e1e..76de8eaf76 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -65,6 +65,7 @@ STATIC mp_obj_t array_iterator_new(mp_obj_t array_in, mp_obj_iter_buf_t *iter_bu STATIC mp_obj_t array_append(mp_obj_t self_in, mp_obj_t arg); STATIC mp_obj_t array_extend(mp_obj_t self_in, mp_obj_t arg_in); STATIC mp_int_t array_get_buffer(mp_obj_t o_in, mp_buffer_info_t *bufinfo, mp_uint_t flags); +// CIRCUITPY #if MICROPY_CPYTHON_COMPAT STATIC mp_obj_t array_decode(size_t n_args, const mp_obj_t *args); #endif @@ -99,6 +100,7 @@ STATIC void array_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t #if MICROPY_PY_BUILTINS_BYTEARRAY || MICROPY_PY_ARRAY STATIC mp_obj_array_t *array_new(char typecode, size_t n) { + // CIRCUITPY if (typecode == 'x') { mp_raise_ValueError(MP_ERROR_TEXT("bad typecode")); } @@ -132,6 +134,7 @@ STATIC mp_obj_t array_construct(char typecode, mp_obj_t initializer) { && mp_get_buffer(initializer, &bufinfo, MP_BUFFER_READ)) { // construct array from raw bytes size_t sz = mp_binary_get_size('@', typecode, NULL); + // CIRCUITPY if (bufinfo.len % sz) { mp_raise_ValueError(MP_ERROR_TEXT("bytes length not a multiple of item size")); } @@ -363,6 +366,12 @@ STATIC mp_obj_t array_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs return MP_OBJ_FROM_PTR(res); } case MP_BINARY_OP_ADD: { + #if MICROPY_PY_BUILTINS_MEMORYVIEW + if (lhs->base.type == &mp_type_memoryview) { + return MP_OBJ_NULL; // op not supported + } + #endif + // allow to add anything that has the buffer protocol (extension to CPython) mp_buffer_info_t lhs_bufinfo; mp_buffer_info_t rhs_bufinfo; @@ -724,6 +733,7 @@ STATIC mp_int_t array_get_buffer(mp_obj_t o_in, mp_buffer_info_t *bufinfo, mp_ui } +// CIRCUITPY #if MICROPY_CPYTHON_COMPAT && MICROPY_PY_BUILTINS_BYTEARRAY // Directly lifted from objstr.c STATIC mp_obj_t array_decode(size_t n_args, const mp_obj_t *args) { @@ -798,7 +808,7 @@ MP_DEFINE_CONST_OBJ_TYPE( #define MEMORYVIEW_TYPE_ATTR #endif -#if MICROPY_CPYTHON_COMPAT // CIRCUITPY provides csat +#if MICROPY_CPYTHON_COMPAT // CIRCUITPY provides cast STATIC const mp_rom_map_elem_t memoryview_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_cast), MP_ROM_PTR(&memoryview_cast_obj) }, #if MICROPY_PY_BUILTINS_BYTES_HEX @@ -841,6 +851,7 @@ mp_obj_t mp_obj_new_bytearray(size_t n, const void *items) { return MP_OBJ_FROM_PTR(o); } +// CIRCUITPY mp_obj_t mp_obj_new_bytearray_of_zeros(size_t n) { mp_obj_array_t *o = array_new(BYTEARRAY_TYPECODE, n); memset(o->items, 0, n); diff --git a/py/objbool.c b/py/objbool.c index 3267ff98bb..96f0e60dd1 100644 --- a/py/objbool.c +++ b/py/objbool.c @@ -45,7 +45,7 @@ typedef struct _mp_obj_bool_t { STATIC void bool_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { bool value = BOOL_VALUE(self_in); - if (MICROPY_PY_UJSON && kind == PRINT_JSON) { + if (MICROPY_PY_JSON && kind == PRINT_JSON) { if (value) { mp_print_str(print, "true"); } else { diff --git a/py/objcomplex.c b/py/objcomplex.c index 9c71455a97..43ab4b889b 100644 --- a/py/objcomplex.c +++ b/py/objcomplex.c @@ -36,6 +36,7 @@ #include #include "py/formatfloat.h" +// CIRCUITPY compilation warning removal #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wfloat-equal" diff --git a/py/objdeque.c b/py/objdeque.c index 67a5baa257..8b52b8d387 100644 --- a/py/objdeque.c +++ b/py/objdeque.c @@ -28,7 +28,6 @@ #include #include "py/mpconfig.h" - #if MICROPY_PY_COLLECTIONS_DEQUE #include "py/runtime.h" diff --git a/py/objdict.c b/py/objdict.c index f378fd3e87..eb8eabc1a1 100644 --- a/py/objdict.c +++ b/py/objdict.c @@ -75,10 +75,10 @@ STATIC void dict_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_ bool first = true; const char *item_separator = ", "; const char *key_separator = ": "; - if (!(MICROPY_PY_UJSON && kind == PRINT_JSON)) { + if (!(MICROPY_PY_JSON && kind == PRINT_JSON)) { kind = PRINT_REPR; } else { - #if MICROPY_PY_UJSON_SEPARATORS + #if MICROPY_PY_JSON_SEPARATORS item_separator = MP_PRINT_GET_EXT(print)->item_separator; key_separator = MP_PRINT_GET_EXT(print)->key_separator; #endif @@ -94,7 +94,7 @@ STATIC void dict_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_ mp_print_str(print, item_separator); } first = false; - bool add_quote = MICROPY_PY_UJSON && kind == PRINT_JSON && !mp_obj_is_str_or_bytes(next->key); + bool add_quote = MICROPY_PY_JSON && kind == PRINT_JSON && !mp_obj_is_str_or_bytes(next->key); if (add_quote) { mp_print_str(print, "\""); } @@ -305,6 +305,7 @@ STATIC mp_obj_t dict_fromkeys(size_t n_args, const mp_obj_t *args) { mp_obj_t len = mp_obj_len_maybe(args[1]); if (len == MP_OBJ_NULL) { /* object's type doesn't have a __len__ slot */ + // CIRCUITPY uses dict_new_typed() here. self_out = dict_new_typed(type, 0); } else { self_out = dict_new_typed(type, MP_OBJ_SMALL_INT_VALUE(len)); @@ -441,6 +442,7 @@ STATIC mp_obj_t dict_update(size_t n_args, const mp_obj_t *args, mp_map_t *kwarg } STATIC MP_DEFINE_CONST_FUN_OBJ_KW(dict_update_obj, 1, dict_update); +// CIRCUITPY #if MICROPY_PY_COLLECTIONS_ORDEREDDICT STATIC mp_obj_t dict_move_to_end(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { mp_obj_dict_t *self = MP_OBJ_TO_PTR(pos_args[0]); @@ -576,6 +578,16 @@ STATIC void dict_view_print(const mp_print_t *print, mp_obj_t self_in, mp_print_ mp_print_str(print, "])"); } +// CIRCUITPY +STATIC mp_obj_t dict_view_unary_op(mp_unary_op_t op, mp_obj_t o_in) { + mp_obj_dict_view_t *o = MP_OBJ_TO_PTR(o_in); + // only dict.values() supports __hash__. + if (op == MP_UNARY_OP_HASH && o->kind == MP_DICT_VIEW_VALUES) { + return MP_OBJ_NEW_SMALL_INT((mp_uint_t)o_in); + } + return MP_OBJ_NULL; +} + STATIC mp_obj_t dict_view_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { // only supported for the 'keys' kind until sets and dicts are refactored mp_obj_dict_view_t *o = MP_OBJ_TO_PTR(lhs_in); @@ -593,6 +605,7 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE( MP_QSTR_dict_view, MP_TYPE_FLAG_ITER_IS_GETITER, print, dict_view_print, + unary_op, dict_view_unary_op, binary_op, dict_view_binary_op, iter, dict_view_getiter ); @@ -650,6 +663,7 @@ STATIC const mp_rom_map_elem_t dict_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_get), MP_ROM_PTR(&dict_get_obj) }, { MP_ROM_QSTR(MP_QSTR_items), MP_ROM_PTR(&dict_items_obj) }, { MP_ROM_QSTR(MP_QSTR_keys), MP_ROM_PTR(&dict_keys_obj) }, + // CIRCUITPY #if MICROPY_PY_COLLECTIONS_ORDEREDDICT { MP_ROM_QSTR(MP_QSTR_move_to_end), MP_ROM_PTR(&dict_move_to_end_obj) }, #endif diff --git a/py/objexcept.c b/py/objexcept.c index 0890afa3c6..506d50c36d 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -116,6 +116,7 @@ bool mp_obj_is_native_exception_instance(mp_obj_t self_in) { return MP_OBJ_TYPE_GET_SLOT_OR_NULL(mp_obj_get_type(self_in), make_new) == mp_obj_exception_make_new; } +// CIRCUITPY mp_obj_exception_t *mp_obj_exception_get_native(mp_obj_t self_in) { assert(mp_obj_is_exception_instance(self_in)); if (mp_obj_is_native_exception_instance(self_in)) { @@ -179,7 +180,7 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin return; } - #if MICROPY_PY_UERRNO + #if MICROPY_PY_ERRNO // try to provide a nice OSError error message if (o->base.type == &mp_type_OSError && o->args->len > 0 && o->args->len < 3 && mp_obj_is_small_int(o->args->items[0])) { // CIRCUITPY can print a whole string, not just the errno qstr @@ -223,6 +224,7 @@ mp_obj_t mp_obj_exception_make_new(const mp_obj_type_t *type, size_t n_args, siz // Populate the exception object o_exc->base.type = type; + // CIRCUITPY o_exc->traceback = (mp_obj_traceback_t *)&mp_const_empty_traceback_obj; mp_obj_tuple_t *o_tuple; @@ -263,6 +265,7 @@ mp_obj_t mp_obj_exception_make_new(const mp_obj_type_t *type, size_t n_args, siz // Get exception "value" - that is, first argument, or None mp_obj_t mp_obj_exception_get_value(mp_obj_t self_in) { + // CIRCUITPY mp_obj_exception_t *self = mp_obj_exception_get_native(self_in); if (self->args->len == 0) { return mp_const_none; @@ -276,6 +279,7 @@ void mp_obj_exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { mp_obj_exception_t *self = MP_OBJ_TO_PTR(self_in); if (dest[0] != MP_OBJ_NULL) { // store/delete attribute + // CIRCUITPY changes till end of function #if MICROPY_CONST_GENERATOREXIT_OBJ if (self == &mp_const_GeneratorExit_obj) { mp_raise_AttributeError(MP_ERROR_TEXT("can't set attribute")); @@ -406,6 +410,7 @@ MP_DEFINE_EXCEPTION(Exception, BaseException) MP_DEFINE_EXCEPTION(BlockingIOError, OSError) MP_DEFINE_EXCEPTION(ChildProcessError, OSError) */ + // CIRCUITPY MP_DEFINE_EXCEPTION(ConnectionError, OSError) MP_DEFINE_EXCEPTION(BrokenPipeError, ConnectionError) /* @@ -418,6 +423,7 @@ MP_DEFINE_EXCEPTION(Exception, BaseException) MP_DEFINE_EXCEPTION(PermissionError, OSError) MP_DEFINE_EXCEPTION(ProcessLookupError, OSError) */ + // CIRCUITPY MP_DEFINE_EXCEPTION(TimeoutError, OSError) /* MP_DEFINE_EXCEPTION(FileExistsError, OSError) @@ -472,6 +478,7 @@ mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args, #if MICROPY_ERROR_REPORTING != MICROPY_ERROR_REPORTING_NONE mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const compressed_string_t *msg) { + // CIRCUITPY is different here and for many lines below. return mp_obj_new_exception_msg_varg(exc_type, msg); } @@ -705,6 +712,7 @@ void mp_obj_exception_get_traceback(mp_obj_t self_in, size_t *n, size_t **values } } +// CIRCUITPY #if MICROPY_PY_SYS_EXC_INFO STATIC const mp_obj_namedtuple_type_t code_type_obj = { NAMEDTUPLE_TYPE_BASE_AND_SLOTS(MP_QSTR_code), diff --git a/py/objexcept.h b/py/objexcept.h index c7e1d89f2d..f32572b1e8 100644 --- a/py/objexcept.h +++ b/py/objexcept.h @@ -28,6 +28,7 @@ #include "py/obj.h" #include "py/objtuple.h" +// CIRCUITPY changes here and below for traceback. #include "py/objtraceback.h" typedef struct _mp_obj_exception_t { diff --git a/py/objfloat.c b/py/objfloat.c index 90aeef648b..b66450874e 100644 --- a/py/objfloat.c +++ b/py/objfloat.c @@ -37,6 +37,7 @@ #include #include "py/formatfloat.h" +// CIRCUITPY avoid compiler warning #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wfloat-equal" diff --git a/py/objfun.c b/py/objfun.c index e10b79d089..df12f0b401 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -58,8 +58,7 @@ STATIC mp_obj_t PLACE_IN_ITCM(fun_builtin_0_call)(mp_obj_t self_in, size_t n_arg MP_DEFINE_CONST_OBJ_TYPE( mp_type_fun_builtin_0, MP_QSTR_function, MP_TYPE_FLAG_BINDS_SELF | MP_TYPE_FLAG_BUILTIN_FUN, - call, fun_builtin_0_call, - unary_op, mp_generic_unary_op + call, fun_builtin_0_call ); STATIC mp_obj_t PLACE_IN_ITCM(fun_builtin_1_call)(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { @@ -71,8 +70,7 @@ STATIC mp_obj_t PLACE_IN_ITCM(fun_builtin_1_call)(mp_obj_t self_in, size_t n_arg MP_DEFINE_CONST_OBJ_TYPE( mp_type_fun_builtin_1, MP_QSTR_function, MP_TYPE_FLAG_BINDS_SELF | MP_TYPE_FLAG_BUILTIN_FUN, - call, fun_builtin_1_call, - unary_op, mp_generic_unary_op + call, fun_builtin_1_call ); STATIC mp_obj_t fun_builtin_2_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { @@ -84,8 +82,7 @@ STATIC mp_obj_t fun_builtin_2_call(mp_obj_t self_in, size_t n_args, size_t n_kw, MP_DEFINE_CONST_OBJ_TYPE( mp_type_fun_builtin_2, MP_QSTR_function, MP_TYPE_FLAG_BINDS_SELF | MP_TYPE_FLAG_BUILTIN_FUN, - call, fun_builtin_2_call, - unary_op, mp_generic_unary_op + call, fun_builtin_2_call ); STATIC mp_obj_t PLACE_IN_ITCM(fun_builtin_3_call)(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { @@ -97,8 +94,7 @@ STATIC mp_obj_t PLACE_IN_ITCM(fun_builtin_3_call)(mp_obj_t self_in, size_t n_arg MP_DEFINE_CONST_OBJ_TYPE( mp_type_fun_builtin_3, MP_QSTR_function, MP_TYPE_FLAG_BINDS_SELF | MP_TYPE_FLAG_BUILTIN_FUN, - call, fun_builtin_3_call, - unary_op, mp_generic_unary_op + call, fun_builtin_3_call ); STATIC mp_obj_t fun_builtin_var_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { @@ -126,8 +122,7 @@ STATIC mp_obj_t fun_builtin_var_call(mp_obj_t self_in, size_t n_args, size_t n_k MP_DEFINE_CONST_OBJ_TYPE( mp_type_fun_builtin_var, MP_QSTR_function, MP_TYPE_FLAG_BINDS_SELF | MP_TYPE_FLAG_BUILTIN_FUN, - call, fun_builtin_var_call, - unary_op, mp_generic_unary_op + call, fun_builtin_var_call ); /******************************************************************************/ @@ -370,8 +365,7 @@ MP_DEFINE_CONST_OBJ_TYPE( MP_TYPE_FLAG_BINDS_SELF, FUN_BC_TYPE_PRINT FUN_BC_TYPE_ATTR - call, fun_bc_call, - unary_op, mp_generic_unary_op + call, fun_bc_call ); mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_module_context_t *context, struct _mp_raw_code_t *const *child_table) { @@ -432,8 +426,7 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE( MP_TYPE_FLAG_BINDS_SELF, FUN_BC_TYPE_PRINT FUN_BC_TYPE_ATTR - call, fun_native_call, - unary_op, mp_generic_unary_op + call, fun_native_call ); mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) { @@ -540,8 +533,7 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE( mp_type_fun_asm, MP_QSTR_function, MP_TYPE_FLAG_BINDS_SELF, - call, fun_asm_call, - unary_op, mp_generic_unary_op + call, fun_asm_call ); mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig) { diff --git a/py/objgenerator.c b/py/objgenerator.c index e1873040f9..ebb4a1a8ac 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -94,8 +94,7 @@ MP_DEFINE_CONST_OBJ_TYPE( MP_QSTR_generator, MP_TYPE_FLAG_BINDS_SELF, GEN_WRAP_TYPE_ATTR - call, gen_wrap_call, - unary_op, mp_generic_unary_op + call, gen_wrap_call ); #if MICROPY_PY_ASYNC_AWAIT @@ -172,7 +171,7 @@ STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_k } #if MICROPY_PY_FUNCTION_ATTRS -#define NATIVE_GEN_WRAP_TYPE_ATTR attr, mp_obj_fun_bc_attr, +#define NATIVE_GEN_WRAP_TYPE_ATTR , attr, mp_obj_fun_bc_attr #else #define NATIVE_GEN_WRAP_TYPE_ATTR #endif @@ -181,9 +180,8 @@ MP_DEFINE_CONST_OBJ_TYPE( mp_type_native_gen_wrap, MP_QSTR_generator, MP_TYPE_FLAG_BINDS_SELF, - call, native_gen_wrap_call, + call, native_gen_wrap_call NATIVE_GEN_WRAP_TYPE_ATTR - unary_op, mp_generic_unary_op ); @@ -452,7 +450,6 @@ MP_DEFINE_CONST_OBJ_TYPE( MP_QSTR_generator, MP_TYPE_FLAG_ITER_IS_ITERNEXT, print, gen_instance_print, - unary_op, mp_generic_unary_op, iter, gen_instance_iternext, locals_dict, &gen_instance_locals_dict ); diff --git a/py/objint.c b/py/objint.c index 68541353d8..7cded790ba 100644 --- a/py/objint.c +++ b/py/objint.c @@ -48,22 +48,22 @@ STATIC mp_obj_t mp_obj_int_make_new(const mp_obj_type_t *type_in, size_t n_args, case 0: return MP_OBJ_NEW_SMALL_INT(0); - case 1: - if (mp_obj_is_int(args[0])) { - // already an int (small or long), just return it - return args[0]; - } else if (mp_obj_is_str_or_bytes(args[0])) { - // a string, parse it - size_t l; - const char *s = mp_obj_str_get_data(args[0], &l); - return mp_parse_num_integer(s, l, 0, NULL); + case 1: { + mp_buffer_info_t bufinfo; + mp_obj_t o = mp_unary_op(MP_UNARY_OP_INT_MAYBE, args[0]); + if (o != MP_OBJ_NULL) { + return o; + } else if (mp_get_buffer(args[0], &bufinfo, MP_BUFFER_READ)) { + // a textual representation, parse it + return mp_parse_num_integer(bufinfo.buf, bufinfo.len, 0, NULL); #if MICROPY_PY_BUILTINS_FLOAT } else if (mp_obj_is_float(args[0])) { return mp_obj_new_int_from_float(mp_obj_float_get(args[0])); #endif } else { - return mp_unary_op(MP_UNARY_OP_INT, args[0]); + mp_raise_TypeError_int_conversion(args[0]); } + } case 2: default: { @@ -300,6 +300,7 @@ char *mp_obj_int_formatted(char **buf, size_t *buf_size, size_t *fmt_size, mp_co return b; } +// CIRCUITPY #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE void mp_obj_int_buffer_overflow_check(mp_obj_t self_in, size_t nbytes, bool is_signed) { @@ -394,6 +395,7 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i // This is called only with strings whose value doesn't fit in SMALL_INT mp_obj_t mp_obj_new_int_from_str_len(const char **str, size_t len, bool neg, unsigned int base) { + // CIRCUITPY different error message mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("No long integer support")); return mp_const_none; } @@ -456,6 +458,7 @@ mp_obj_t mp_obj_int_binary_op_extra_cases(mp_binary_op_t op, mp_obj_t lhs_in, mp return MP_OBJ_NULL; // op not supported } +// CIRCUITPY #if MICROPY_CPYTHON_COMPAT STATIC mp_obj_t int_bit_length(mp_obj_t self_in) { #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE @@ -477,6 +480,7 @@ STATIC mp_obj_t int_bit_length(mp_obj_t self_in) { MP_DEFINE_CONST_FUN_OBJ_1(int_bit_length_obj, int_bit_length); #endif +// CIRCUITPY more functionality // this is a classmethod STATIC mp_obj_t int_from_bytes(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { // TODO: Support signed param (assumes signed=False at the moment) @@ -571,6 +575,7 @@ STATIC mp_obj_t int_to_bytes(size_t n_args, const mp_obj_t *pos_args, mp_map_t * STATIC MP_DEFINE_CONST_FUN_OBJ_KW(int_to_bytes_obj, 3, int_to_bytes); STATIC const mp_rom_map_elem_t int_locals_dict_table[] = { + // CIRCUITPY #if MICROPY_CPYTHON_COMPAT { MP_ROM_QSTR(MP_QSTR_bit_length), MP_ROM_PTR(&int_bit_length_obj) }, #endif diff --git a/py/objint.h b/py/objint.h index 83f12bac6c..1cd457bc00 100644 --- a/py/objint.h +++ b/py/objint.h @@ -53,6 +53,7 @@ char *mp_obj_int_formatted(char **buf, size_t *buf_size, size_t *fmt_size, mp_co int base, const char *prefix, char base_char, char comma); char *mp_obj_int_formatted_impl(char **buf, size_t *buf_size, size_t *fmt_size, mp_const_obj_t self_in, int base, const char *prefix, char base_char, char comma); +// CIRCUITPY #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE void mp_obj_int_buffer_overflow_check(mp_obj_t self_in, size_t nbytes, bool is_signed); #endif @@ -60,6 +61,7 @@ void mp_obj_int_buffer_overflow_check(mp_obj_t self_in, size_t nbytes, bool is_s void mp_small_int_buffer_overflow_check(mp_int_t val, size_t nbytes, bool is_signed); mp_int_t mp_obj_int_hash(mp_obj_t self_in); +// CIRCUITPY mp_obj_t mp_obj_int_bit_length_impl(mp_obj_t self_in); mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf); void mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, size_t len, byte *buf); diff --git a/py/objint_longlong.c b/py/objint_longlong.c index a2d6e8382b..5e1e26f690 100644 --- a/py/objint_longlong.c +++ b/py/objint_longlong.c @@ -43,6 +43,7 @@ const mp_obj_int_t mp_sys_maxsize_obj = {{&mp_type_int}, MP_SSIZE_MAX}; #endif +// CIRCUITPY mp_obj_t mp_obj_int_bit_length_impl(mp_obj_t self_in) { assert(mp_obj_is_type(self_in, &mp_type_int)); mp_obj_int_t *self = self_in; @@ -130,6 +131,8 @@ mp_obj_t mp_obj_int_unary_op(mp_unary_op_t op, mp_obj_t o_in) { self->val = -self->val; return MP_OBJ_FROM_PTR(self); } + case MP_UNARY_OP_INT_MAYBE: + return o_in; default: return MP_OBJ_NULL; // op not supported } diff --git a/py/objint_mpz.c b/py/objint_mpz.c index 81293b0593..e04c8ca86b 100644 --- a/py/objint_mpz.c +++ b/py/objint_mpz.c @@ -106,6 +106,7 @@ char *mp_obj_int_formatted_impl(char **buf, size_t *buf_size, size_t *fmt_size, return str; } +// CIRCUITPY mp_obj_t mp_obj_int_bit_length_impl(mp_obj_t self_in) { assert(mp_obj_is_exact_type(self_in, &mp_type_int)); mp_obj_int_t *self = MP_OBJ_TO_PTR(self_in); @@ -171,6 +172,8 @@ mp_obj_t mp_obj_int_unary_op(mp_unary_op_t op, mp_obj_t o_in) { mpz_abs_inpl(&self2->mpz, &self->mpz); return MP_OBJ_FROM_PTR(self2); } + case MP_UNARY_OP_INT_MAYBE: + return o_in; default: return MP_OBJ_NULL; // op not supported } diff --git a/py/objlist.c b/py/objlist.c index aa857fbfc0..b183eda5a8 100644 --- a/py/objlist.c +++ b/py/objlist.c @@ -39,16 +39,19 @@ STATIC mp_obj_t list_pop(size_t n_args, const mp_obj_t *args); // TODO: Move to mpconfig.h #define LIST_MIN_ALLOC 4 +// CIRCUITPY: native_list() and other changes here for broadcom port +// https://github.com/adafruit/circuitpython/pull/5610 + /******************************************************************************/ /* list */ STATIC void list_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { mp_obj_list_t *o = MP_OBJ_TO_PTR(o_in); const char *item_separator = ", "; - if (!(MICROPY_PY_UJSON && kind == PRINT_JSON)) { + if (!(MICROPY_PY_JSON && kind == PRINT_JSON)) { kind = PRINT_REPR; } else { - #if MICROPY_PY_UJSON_SEPARATORS + #if MICROPY_PY_JSON_SEPARATORS item_separator = MP_PRINT_GET_EXT(print)->item_separator; #endif } @@ -290,6 +293,7 @@ inline mp_obj_t mp_obj_list_pop(mp_obj_list_t *self, size_t index) { return ret; } +// CIRCUITPY STATIC mp_obj_t list_pop(size_t n_args, const mp_obj_t *args) { mp_check_self(mp_obj_is_type(args[0], &mp_type_list)); mp_obj_list_t *self = native_list(args[0]); diff --git a/py/objlist.h b/py/objlist.h index b015261f61..aa2a666020 100644 --- a/py/objlist.h +++ b/py/objlist.h @@ -37,6 +37,7 @@ typedef struct _mp_obj_list_t { void mp_obj_list_init(mp_obj_list_t *o, size_t n); mp_obj_t mp_obj_list_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *args); +// CIRCUITPY mp_obj_t mp_obj_list_pop(mp_obj_list_t *self, size_t index); void mp_obj_list_insert(mp_obj_list_t *self, size_t index, mp_obj_t obj); diff --git a/py/objmodule.c b/py/objmodule.c index 63e9fa4757..5266421b79 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -34,16 +34,6 @@ #include "py/runtime.h" #include "py/builtin.h" -#ifndef NO_QSTR -// Only include module definitions when not doing qstr extraction, because the -// qstr extraction stage also generates this module definition header file. -#include "genhdr/moduledefs.h" -#endif - -#if MICROPY_MODULE_BUILTIN_INIT -STATIC void mp_module_call_init(mp_obj_t module_name, mp_obj_t module_obj); -#endif - STATIC void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { (void)kind; mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in); @@ -67,20 +57,7 @@ STATIC void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kin mp_printf(print, "", module_name); } -STATIC void module_attr_try_delegation(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { - #if MICROPY_MODULE_ATTR_DELEGATION - // Delegate lookup to a module's custom attr method (found in last lot of globals dict). - mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in); - mp_map_t *map = &self->globals->map; - if (map->table[map->alloc - 1].key == MP_OBJ_NEW_QSTR(MP_QSTRnull)) { - ((mp_attr_fun_t)MP_OBJ_TO_PTR(map->table[map->alloc - 1].value))(self_in, attr, dest); - } - #else - (void)self_in; - (void)attr; - (void)dest; - #endif -} +STATIC void module_attr_try_delegation(mp_obj_t self_in, qstr attr, mp_obj_t *dest); STATIC void module_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in); @@ -170,75 +147,98 @@ mp_obj_t mp_obj_new_module(qstr module_name) { // Global module table and related functions STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = { - // builtin modules declared with MP_REGISTER_MODULE() + // built-in modules declared with MP_REGISTER_MODULE() MICROPY_REGISTERED_MODULES }; - MP_DEFINE_CONST_MAP(mp_builtin_module_map, mp_builtin_module_table); -// Tries to find a loaded module, otherwise attempts to load a builtin, otherwise MP_OBJ_NULL. -mp_obj_t mp_module_get_loaded_or_builtin(qstr module_name) { - // First try loaded modules. - mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_loaded_modules_dict).map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP); +STATIC const mp_rom_map_elem_t mp_builtin_extensible_module_table[] = { + // built-in modules declared with MP_REGISTER_EXTENSIBLE_MODULE() + MICROPY_REGISTERED_EXTENSIBLE_MODULES +}; +MP_DEFINE_CONST_MAP(mp_builtin_extensible_module_map, mp_builtin_extensible_module_table); +#if MICROPY_MODULE_ATTR_DELEGATION && defined(MICROPY_MODULE_DELEGATIONS) +typedef struct _mp_module_delegation_entry_t { + mp_rom_obj_t mod; + mp_attr_fun_t fun; +} mp_module_delegation_entry_t; + +STATIC const mp_module_delegation_entry_t mp_builtin_module_delegation_table[] = { + // delegation entries declared with MP_REGISTER_MODULE_DELEGATION() + MICROPY_MODULE_DELEGATIONS +}; +#endif + +// Attempts to find (and initialise) a built-in, otherwise returns +// MP_OBJ_NULL. +mp_obj_t mp_module_get_builtin(qstr module_name, bool extensible) { + mp_map_elem_t *elem = mp_map_lookup((mp_map_t *)(extensible ? &mp_builtin_extensible_module_map : &mp_builtin_module_map), MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP); if (!elem) { - #if MICROPY_MODULE_WEAK_LINKS - return mp_module_get_builtin(module_name); - #else - // Otherwise try builtin. - elem = mp_map_lookup((mp_map_t *)&mp_builtin_module_map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP); - if (!elem) { + #if MICROPY_PY_SYS + // Special case for sys, which isn't extensible but can always be + // imported with the alias `usys`. + if (module_name == MP_QSTR_usys) { + return MP_OBJ_FROM_PTR(&mp_module_sys); + } + #endif + + if (extensible) { + // At this point we've already tried non-extensible built-ins, the + // filesystem, and now extensible built-ins. No match, so fail + // the import. return MP_OBJ_NULL; } - #if MICROPY_MODULE_BUILTIN_INIT - // If found, it's a newly loaded built-in, so init it. - mp_module_call_init(MP_OBJ_NEW_QSTR(module_name), elem->value); - #endif - #endif - } - - return elem->value; -} - -#if MICROPY_MODULE_WEAK_LINKS -// Tries to find a loaded module, otherwise attempts to load a builtin, otherwise MP_OBJ_NULL. -mp_obj_t mp_module_get_builtin(qstr module_name) { - // Try builtin. - mp_map_elem_t *elem = mp_map_lookup((mp_map_t *)&mp_builtin_module_map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP); - if (!elem) { - return MP_OBJ_NULL; + // We're trying to match a non-extensible built-in (i.e. before trying + // the filesystem), but if the user is importing `ufoo`, _and_ `foo` + // is an extensible module, then allow it as a way of forcing the + // built-in. Essentially, this makes it as if all the extensible + // built-ins also had non-extensible aliases named `ufoo`. Newer code + // should be using sys.path to force the built-in, but this retains + // the old behaviour of the u-prefix being used to force a built-in + // import. + size_t module_name_len; + const char *module_name_str = (const char *)qstr_data(module_name, &module_name_len); + if (module_name_str[0] != 'u') { + return MP_OBJ_NULL; + } + elem = mp_map_lookup((mp_map_t *)&mp_builtin_extensible_module_map, MP_OBJ_NEW_QSTR(qstr_from_strn(module_name_str + 1, module_name_len - 1)), MP_MAP_LOOKUP); + if (!elem) { + return MP_OBJ_NULL; + } } #if MICROPY_MODULE_BUILTIN_INIT - // If found, it's a newly loaded built-in, so init it. - mp_module_call_init(MP_OBJ_NEW_QSTR(module_name), elem->value); + // If found, it's a newly loaded built-in, so init it. This can run + // multiple times, so the module must ensure that it handles being + // initialised multiple times. + mp_obj_t dest[2]; + mp_load_method_maybe(elem->value, MP_QSTR___init__, dest); + if (dest[0] != MP_OBJ_NULL) { + mp_call_method_n_kw(0, 0, dest); + } #endif return elem->value; } -#endif -#if MICROPY_MODULE_BUILTIN_INIT -STATIC void mp_module_register(mp_obj_t module_name, mp_obj_t module) { - mp_map_t *mp_loaded_modules_map = &MP_STATE_VM(mp_loaded_modules_dict).map; - mp_map_lookup(mp_loaded_modules_map, module_name, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = module; -} - -STATIC void mp_module_call_init(mp_obj_t module_name, mp_obj_t module_obj) { - // Look for __init__ and call it if it exists - mp_obj_t dest[2]; - mp_load_method_maybe(module_obj, MP_QSTR___init__, dest); - if (dest[0] != MP_OBJ_NULL) { - mp_call_method_n_kw(0, 0, dest); - // Register module so __init__ is not called again. - // If a module can be referenced by more than one name (eg due to weak links) - // then __init__ will still be called for each distinct import, and it's then - // up to the particular module to make sure it's __init__ code only runs once. - mp_module_register(module_name, module_obj); +STATIC void module_attr_try_delegation(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + #if MICROPY_MODULE_ATTR_DELEGATION && defined(MICROPY_MODULE_DELEGATIONS) + // Delegate lookup to a module's custom attr method. + size_t n = MP_ARRAY_SIZE(mp_builtin_module_delegation_table); + for (size_t i = 0; i < n; ++i) { + if (*(mp_obj_t *)(&mp_builtin_module_delegation_table[i].mod) == self_in) { + mp_builtin_module_delegation_table[i].fun(self_in, attr, dest); + break; + } } + #else + (void)self_in; + (void)attr; + (void)dest; + #endif } -#endif void mp_module_generic_attr(qstr attr, mp_obj_t *dest, const uint16_t *keys, mp_obj_t *values) { for (size_t i = 0; keys[i] != MP_QSTRnull; ++i) { diff --git a/py/objmodule.h b/py/objmodule.h index d11d5bcd74..8b14cd9fc3 100644 --- a/py/objmodule.h +++ b/py/objmodule.h @@ -28,15 +28,16 @@ #include "py/obj.h" -// Place at the very end of a module's globals_table. -#define MP_MODULE_ATTR_DELEGATION_ENTRY(ptr) { MP_ROM_QSTR(MP_QSTRnull), MP_ROM_PTR(ptr) } +#ifndef NO_QSTR +// Only include module definitions when not doing qstr extraction, because the +// qstr extraction stage also generates this module definition header file. +#include "genhdr/moduledefs.h" +#endif extern const mp_map_t mp_builtin_module_map; +extern const mp_map_t mp_builtin_extensible_module_map; -mp_obj_t mp_module_get_loaded_or_builtin(qstr module_name); -#if MICROPY_MODULE_WEAK_LINKS -mp_obj_t mp_module_get_builtin(qstr module_name); -#endif +mp_obj_t mp_module_get_builtin(qstr module_name, bool extensible); void mp_module_generic_attr(qstr attr, mp_obj_t *dest, const uint16_t *keys, mp_obj_t *values); diff --git a/py/objnamedtuple.c b/py/objnamedtuple.c index ed2b1de42e..fa07c8b00b 100644 --- a/py/objnamedtuple.c +++ b/py/objnamedtuple.c @@ -44,6 +44,7 @@ size_t mp_obj_namedtuple_find_field(const mp_obj_namedtuple_type_t *type, qstr n return (size_t)-1; } +// CIRCUITPY differences #if MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT STATIC mp_obj_t namedtuple_asdict(mp_obj_t self_in) { mp_obj_namedtuple_t *self = MP_OBJ_TO_PTR(self_in); diff --git a/py/objnone.c b/py/objnone.c index 4f8996e897..5b25cd38c4 100644 --- a/py/objnone.c +++ b/py/objnone.c @@ -36,7 +36,7 @@ typedef struct _mp_obj_none_t { STATIC void none_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { (void)self_in; - if (MICROPY_PY_UJSON && kind == PRINT_JSON) { + if (MICROPY_PY_JSON && kind == PRINT_JSON) { mp_print_str(print, "null"); } else { mp_print_str(print, "None"); @@ -47,8 +47,7 @@ MP_DEFINE_CONST_OBJ_TYPE( mp_type_NoneType, MP_QSTR_NoneType, MP_TYPE_FLAG_NONE, - print, none_print, - unary_op, mp_generic_unary_op + print, none_print ); #if !MICROPY_OBJ_IMMEDIATE_OBJS diff --git a/py/objobject.c b/py/objobject.c index 4f296106ce..11dc25e2c3 100644 --- a/py/objobject.c +++ b/py/objobject.c @@ -49,6 +49,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(object___init___obj, object___init__); STATIC mp_obj_t object___new__(mp_obj_t cls) { if (!mp_obj_is_type(cls, &mp_type_type) || !mp_obj_is_instance_type((mp_obj_type_t *)MP_OBJ_TO_PTR(cls))) { + // CIRCUITPY better error mp_raise_TypeError(MP_ERROR_TEXT("__new__ arg must be a user-type")); } // This executes only "__new__" part of instance creation. diff --git a/py/objproperty.c b/py/objproperty.c index 98011f2782..aa3a54aa0f 100644 --- a/py/objproperty.c +++ b/py/objproperty.c @@ -31,6 +31,9 @@ #include "py/objproperty.h" #include "py/runtime.h" +// CIRCUITPY changes to reduce property proxy table size +// when possible + #if MICROPY_PY_BUILTINS_PROPERTY STATIC mp_obj_t property_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { diff --git a/py/objsingleton.c b/py/objsingleton.c index dc73d28c27..6537676c52 100644 --- a/py/objsingleton.c +++ b/py/objsingleton.c @@ -45,8 +45,7 @@ STATIC void singleton_print(const mp_print_t *print, mp_obj_t self_in, mp_print_ MP_DEFINE_CONST_OBJ_TYPE( mp_type_singleton, MP_QSTR_, MP_TYPE_FLAG_NONE, - print, singleton_print, - unary_op, mp_generic_unary_op + print, singleton_print ); const mp_obj_singleton_t mp_const_ellipsis_obj = {{&mp_type_singleton}, MP_QSTR_Ellipsis}; diff --git a/py/objslice.c b/py/objslice.c index d165699ecf..19c69db34a 100644 --- a/py/objslice.c +++ b/py/objslice.c @@ -47,6 +47,12 @@ STATIC void slice_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t mp_print_str(print, ")"); } +STATIC mp_obj_t slice_unary_op(mp_unary_op_t op, mp_obj_t o_in) { + // Needed to explicitly opt out of default __hash__. + // REVISIT: CPython implements comparison operators for slice. + return MP_OBJ_NULL; +} + #if MICROPY_PY_BUILTINS_SLICE_INDICES STATIC mp_obj_t slice_indices(mp_obj_t self_in, mp_obj_t length_obj) { mp_int_t length = mp_obj_int_get_checked(length_obj); @@ -138,6 +144,7 @@ MP_DEFINE_CONST_OBJ_TYPE( mp_type_slice, MP_QSTR_slice, MP_TYPE_FLAG_NONE, + unary_op, slice_unary_op, SLICE_TYPE_ATTR_OR_LOCALS_DICT SLICE_MAKE_NEW print, slice_print diff --git a/py/objstr.c b/py/objstr.c index 994194540f..fae7ed0a6f 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -116,7 +116,7 @@ void mp_str_print_quoted(const mp_print_t *print, const byte *str_data, size_t s mp_printf(print, "%c", quote_char); } -#if MICROPY_PY_UJSON +#if MICROPY_PY_JSON void mp_str_print_json(const mp_print_t *print, const byte *str_data, size_t str_len) { // for JSON spec, see http://www.ietf.org/rfc/rfc4627.txt // if we are given a valid utf8-encoded string, we will print it in a JSON-conforming way @@ -144,7 +144,7 @@ void mp_str_print_json(const mp_print_t *print, const byte *str_data, size_t str STATIC void str_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { GET_STR_DATA_LEN(self_in, str_data, str_len); - #if MICROPY_PY_UJSON + #if MICROPY_PY_JSON if (kind == PRINT_JSON) { mp_str_print_json(print, str_data, str_len); return; @@ -420,7 +420,16 @@ mp_obj_t mp_obj_str_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i } else { // LHS is str and RHS has an incompatible type // (except if operation is EQUAL, but that's handled by mp_obj_equal) - bad_implicit_conversion(rhs_in); + + // CONTAINS must fail with a bad-implicit-conversion exception, because + // otherwise mp_binary_op() will fallback to `list(lhs).__contains__(rhs)`. + if (op == MP_BINARY_OP_CONTAINS) { + bad_implicit_conversion(rhs_in); + } + + // All other operations are not supported, and may be handled by another + // type, eg for reverse operations. + return MP_OBJ_NULL; } switch (op) { @@ -1015,6 +1024,7 @@ STATIC vstr_t mp_obj_str_format_helper(const char *str, const char *top, int *ar #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE terse_str_format_value_error(); #else + // CIRCUITPY better error message mp_raise_ValueError_varg(MP_ERROR_TEXT("unmatched '%c' in format"), '}'); #endif } @@ -1653,7 +1663,9 @@ STATIC mp_obj_t str_modulo_format(mp_obj_t pattern, size_t n_args, const mp_obj_ } } - if (arg_i != n_args) { + if (dict == MP_OBJ_NULL && arg_i != n_args) { + // NOTE: if `dict` exists, then `n_args` is 1 and the dict is always consumed; either + // positionally, or as a map of named args, even if none were actually referenced. mp_raise_TypeError(MP_ERROR_TEXT("not all arguments converted during string formatting")); } diff --git a/py/objstringio.c b/py/objstringio.c index 1a083449ba..a3c66ed010 100644 --- a/py/objstringio.c +++ b/py/objstringio.c @@ -170,12 +170,6 @@ STATIC mp_obj_t stringio_getvalue(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(stringio_getvalue_obj, stringio_getvalue); -STATIC mp_obj_t stringio___exit__(size_t n_args, const mp_obj_t *args) { - (void)n_args; - return mp_stream_close(args[0]); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(stringio___exit___obj, 4, 4, stringio___exit__); - STATIC mp_obj_stringio_t *stringio_new(const mp_obj_type_t *type) { mp_obj_stringio_t *o = mp_obj_malloc(mp_obj_stringio_t, type); o->pos = 0; @@ -232,7 +226,7 @@ STATIC const mp_rom_map_elem_t stringio_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR_getvalue), MP_ROM_PTR(&stringio_getvalue_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&stringio___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, }; STATIC MP_DEFINE_CONST_DICT(stringio_locals_dict, stringio_locals_dict_table); diff --git a/py/objstrunicode.c b/py/objstrunicode.c index 505380c4b5..d878e1e060 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -73,6 +73,7 @@ STATIC void uni_print_quoted(const mp_print_t *print, const byte *str_data, uint mp_print_str(print, "\\r"); } else if (ch == '\t') { mp_print_str(print, "\\t"); + // CIRCUITPY difference: print printable Unicode chars } else if (ch <= 0x1f || (0x7f <= ch && ch <= 0xa0) || ch == 0xad) { mp_printf(print, "\\x%02x", ch); } else if ((0x2000 <= ch && ch <= 0x200f) || ch == 0x2028 || ch == 0x2029) { @@ -88,7 +89,7 @@ STATIC void uni_print_quoted(const mp_print_t *print, const byte *str_data, uint STATIC void uni_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { GET_STR_DATA_LEN(self_in, str_data, str_len); - #if MICROPY_PY_UJSON + #if MICROPY_PY_JSON if (kind == PRINT_JSON) { mp_str_print_json(print, str_data, str_len); return; diff --git a/py/objtuple.c b/py/objtuple.c index d1705982b8..37a02a358f 100644 --- a/py/objtuple.c +++ b/py/objtuple.c @@ -31,15 +31,18 @@ #include "py/objtuple.h" #include "py/runtime.h" +// type check is done on getiter method to allow tuple, namedtuple, attrtuple +#define mp_obj_is_tuple_compatible(o) (MP_OBJ_TYPE_GET_SLOT_OR_NULL(mp_obj_get_type(o), iter) == mp_obj_tuple_getiter) + /******************************************************************************/ /* tuple */ void mp_obj_tuple_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { mp_obj_tuple_t *o = MP_OBJ_TO_PTR(o_in); const char *item_separator = ", "; - if (MICROPY_PY_UJSON && kind == PRINT_JSON) { + if (MICROPY_PY_JSON && kind == PRINT_JSON) { mp_print_str(print, "["); - #if MICROPY_PY_UJSON_SEPARATORS + #if MICROPY_PY_JSON_SEPARATORS item_separator = MP_PRINT_GET_EXT(print)->item_separator; #endif } else { @@ -52,7 +55,7 @@ void mp_obj_tuple_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t } mp_obj_print_helper(print, o->items[i], kind); } - if (MICROPY_PY_UJSON && kind == PRINT_JSON) { + if (MICROPY_PY_JSON && kind == PRINT_JSON) { mp_print_str(print, "]"); } else { if (o->len == 1) { diff --git a/py/objtuple.h b/py/objtuple.h index 2d3026fca5..6ba276459d 100644 --- a/py/objtuple.h +++ b/py/objtuple.h @@ -50,6 +50,7 @@ mp_obj_t mp_obj_tuple_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf); extern const mp_obj_type_t mp_type_attrtuple; +// CIRCUITPY // Relies on gcc Variadic Macros and Statement Expressions #define MP_OBJ_NEW_TUPLE(...) ({mp_obj_t _z[] = {__VA_ARGS__}; mp_obj_new_tuple(MP_ARRAY_SIZE(_z), _z);}) diff --git a/py/objtype.c b/py/objtype.c index 89730bb9a0..e39b1e32a4 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -82,6 +82,7 @@ STATIC int instance_count_native_bases(const mp_obj_type_t *type, const mp_obj_t } } +// CIRCUITPY differences // This wrapper function is allows a subclass of a native type to call the // __init__() method (corresponding to type->make_new) of the native type. STATIC mp_obj_t native_base_init_wrapper(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { @@ -127,6 +128,7 @@ mp_obj_instance_t *mp_obj_new_instance(const mp_obj_type_t *class, const mp_obj_ return o; } +// CIRCUITPY // When instances are first created they have the base_init wrapper as their native parent's // instance because make_new combines __new__ and __init__. This object is invalid for the native // code so it must call this method to ensure that the given object has been __init__'d and is @@ -301,6 +303,7 @@ STATIC void instance_print(const mp_print_t *print, mp_obj_t self_in, mp_print_k } // TODO: CPython prints fully-qualified type name + // CIRCUITPY: use qstr mp_printf(print, "<%q object at %p>", mp_obj_get_type_qstr(self_in), self); } @@ -401,7 +404,7 @@ const byte mp_unary_op_method_name[MP_UNARY_OP_NUM_RUNTIME] = { [MP_UNARY_OP_BOOL] = MP_QSTR___bool__, [MP_UNARY_OP_LEN] = MP_QSTR___len__, [MP_UNARY_OP_HASH] = MP_QSTR___hash__, - [MP_UNARY_OP_INT] = MP_QSTR___int__, + [MP_UNARY_OP_INT_MAYBE] = MP_QSTR___int__, #if MICROPY_PY_ALL_SPECIAL_METHODS [MP_UNARY_OP_POSITIVE] = MP_QSTR___pos__, [MP_UNARY_OP_NEGATIVE] = MP_QSTR___neg__, @@ -459,7 +462,7 @@ STATIC mp_obj_t instance_unary_op(mp_unary_op_t op, mp_obj_t self_in) { // __hash__ must return a small int val = MP_OBJ_NEW_SMALL_INT(mp_obj_get_int_truncated(val)); break; - case MP_UNARY_OP_INT: + case MP_UNARY_OP_INT_MAYBE: // Must return int if (!mp_obj_is_int(val)) { mp_raise_TypeError(NULL); @@ -561,7 +564,6 @@ STATIC mp_obj_t instance_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t // Note: For ducktyping, CPython does not look in the instance members or use // __getattr__ or __getattribute__. It only looks in the class dictionary. mp_obj_instance_t *lhs = MP_OBJ_TO_PTR(lhs_in); -retry:; qstr op_name = mp_binary_op_method_name[op]; /* Still try to lookup native slot if (op_name == 0) { @@ -586,22 +588,12 @@ retry:; res = mp_call_method_n_kw(1, 0, dest); res = op == MP_BINARY_OP_CONTAINS ? mp_obj_new_bool(mp_obj_is_true(res)) : res; } else { - // If this was an inplace method, fallback to normal method - // https://docs.python.org/3/reference/datamodel.html#object.__iadd__ : - // "If a specific method is not defined, the augmented assignment - // falls back to the normal methods." - if (op >= MP_BINARY_OP_INPLACE_OR && op <= MP_BINARY_OP_INPLACE_POWER) { - op -= MP_BINARY_OP_INPLACE_OR - MP_BINARY_OP_OR; - goto retry; - } return MP_OBJ_NULL; // op not supported } #if MICROPY_PY_BUILTINS_NOTIMPLEMENTED // NotImplemented means "try other fallbacks (like calling __rop__ - // instead of __op__) and if nothing works, raise TypeError". As - // MicroPython doesn't implement any fallbacks, signal to raise - // TypeError right away. + // instead of __op__) and if nothing works, raise TypeError". if (res == mp_const_notimplemented) { return MP_OBJ_NULL; // op not supported } @@ -740,6 +732,7 @@ STATIC bool mp_obj_instance_store_attr(mp_obj_t self_in, qstr attr, mp_obj_t val // would be called by the descriptor code down below. But that way // requires overhead for the nested mp_call's and overhead for // the code. + // CIRCUITPY: variable number of proxies size_t n_proxy; const mp_obj_t *proxy = mp_obj_property_get(member[0], &n_proxy); mp_obj_t dest[2] = {self_in, value}; @@ -1141,7 +1134,6 @@ MP_DEFINE_CONST_OBJ_TYPE( make_new, type_make_new, print, type_print, call, type_call, - unary_op, mp_generic_unary_op, attr, type_attr ); diff --git a/py/parse.c b/py/parse.c index 50f9586835..369ab9957b 100644 --- a/py/parse.c +++ b/py/parse.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2017 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -61,7 +61,7 @@ enum { // define rules with a compile function #define DEF_RULE(rule, comp, kind, ...) RULE_##rule, #define DEF_RULE_NC(rule, kind, ...) - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC RULE_const_object, // special node for a constant, generic Python object @@ -69,7 +69,7 @@ enum { // define rules without a compile function #define DEF_RULE(rule, comp, kind, ...) #define DEF_RULE_NC(rule, kind, ...) RULE_##rule, - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC }; @@ -86,7 +86,7 @@ STATIC const uint8_t rule_act_table[] = { #define DEF_RULE(rule, comp, kind, ...) kind, #define DEF_RULE_NC(rule, kind, ...) - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC @@ -94,7 +94,7 @@ STATIC const uint8_t rule_act_table[] = { #define DEF_RULE(rule, comp, kind, ...) #define DEF_RULE_NC(rule, kind, ...) kind, - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC @@ -115,13 +115,13 @@ STATIC const uint16_t rule_arg_combined_table[] = { #define DEF_RULE(rule, comp, kind, ...) __VA_ARGS__, #define DEF_RULE_NC(rule, kind, ...) - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC #define DEF_RULE(rule, comp, kind, ...) #define DEF_RULE_NC(rule, kind, ...) __VA_ARGS__, - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC @@ -141,12 +141,12 @@ STATIC const uint16_t rule_arg_combined_table[] = { enum { #define DEF_RULE(rule, comp, kind, ...) RULE_PADDING(rule, __VA_ARGS__) #define DEF_RULE_NC(rule, kind, ...) - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC #define DEF_RULE(rule, comp, kind, ...) #define DEF_RULE_NC(rule, kind, ...) RULE_PADDING(rule, __VA_ARGS__) - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC }; @@ -164,13 +164,13 @@ enum { STATIC const uint8_t rule_arg_offset_table[] = { #define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) & 0xff, #define DEF_RULE_NC(rule, kind, ...) - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC 0, // RULE_const_object #define DEF_RULE(rule, comp, kind, ...) #define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) & 0xff, - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC }; @@ -187,20 +187,20 @@ static const size_t FIRST_RULE_WITH_OFFSET_ABOVE_255 = #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC - 0; +0; #if MICROPY_DEBUG_PARSE_RULE_NAME // Define an array of rule names corresponding to each rule STATIC const char *const rule_name_table[] = { #define DEF_RULE(rule, comp, kind, ...) #rule, #define DEF_RULE_NC(rule, kind, ...) - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC "", // RULE_const_object #define DEF_RULE(rule, comp, kind, ...) #define DEF_RULE_NC(rule, kind, ...) #rule, - #include "py/grammar.h" +#include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC }; @@ -242,6 +242,8 @@ typedef struct _parser_t { #endif } parser_t; +STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t num_args); + STATIC const uint16_t *get_rule_arg(uint8_t r_id) { size_t off = rule_arg_offset_table[r_id]; if (r_id >= FIRST_RULE_WITH_OFFSET_ABOVE_255) { @@ -250,6 +252,7 @@ STATIC const uint16_t *get_rule_arg(uint8_t r_id) { return &rule_arg_combined_table[off]; } +// CIRCUITPY ignore compiler warning #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-align" @@ -634,10 +637,12 @@ STATIC void push_result_token(parser_t *parser, uint8_t rule_id) { push_result_node(parser, pn); } +#if MICROPY_COMP_CONST_FOLDING + #if MICROPY_COMP_MODULE_CONST STATIC const mp_rom_map_elem_t mp_constants_table[] = { - #if MICROPY_PY_UERRNO - { MP_ROM_QSTR(MP_QSTR_errno), MP_ROM_PTR(&mp_module_uerrno) }, + #if MICROPY_PY_ERRNO + { MP_ROM_QSTR(MP_QSTR_errno), MP_ROM_PTR(&mp_module_errno) }, #endif #if MICROPY_PY_UCTYPES { MP_ROM_QSTR(MP_QSTR_uctypes), MP_ROM_PTR(&mp_module_uctypes) }, @@ -899,7 +904,8 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { return true; } -#endif + +#endif // MICROPY_COMP_CONST_FOLDING #if MICROPY_COMP_CONST_TUPLE STATIC bool build_tuple_from_stack(parser_t *parser, size_t src_line, size_t num_args) { @@ -1012,7 +1018,7 @@ STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, #if MICROPY_COMP_CONST_TUPLE if (build_tuple(parser, src_line, rule_id, num_args)) { - // we built a tuple from this rule so return straight away + // we built a tuple from this rule so return straightaway return; } #endif @@ -1031,6 +1037,9 @@ STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, } mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { + // Set exception handler to free the lexer if an exception is raised. + MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, mp_lexer_free, lex); + nlr_push_jump_callback(&ctx.callback, mp_call_function_1_from_nlr_jump_callback); // initialise parser and allocate memory for its stacks @@ -1038,6 +1047,8 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { parser.rule_stack_alloc = MICROPY_ALLOC_PARSE_RULE_INIT; parser.rule_stack_top = 0; + // CIRCUITPY make parsing more memory flexible + // https://github.com/adafruit/circuitpython/pull/552 parser.rule_stack = NULL; while (parser.rule_stack_alloc > 1) { parser.rule_stack = m_new_maybe(rule_stack_t, parser.rule_stack_alloc); @@ -1121,7 +1132,6 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { backtrack = false; } for (; i < n; ++i) { - // printf("--> inside for @L924\n"); uint16_t kind = rule_arg[i] & RULE_ARG_KIND_MASK; if (kind == RULE_ARG_TOK) { if (lex->tok_kind == (rule_arg[i] & RULE_ARG_ARG_MASK)) { @@ -1398,8 +1408,8 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { m_del(rule_stack_t, parser.rule_stack, parser.rule_stack_alloc); m_del(mp_parse_node_t, parser.result_stack, parser.result_stack_alloc); - // we also free the lexer on behalf of the caller - mp_lexer_free(lex); + // Deregister exception handler and free the lexer. + nlr_pop_jump_callback(true); return parser.tree; } diff --git a/py/parse.h b/py/parse.h index d9d0e15314..5531e35cbb 100644 --- a/py/parse.h +++ b/py/parse.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/parsenum.c b/py/parsenum.c index b2658eac77..796eaf8113 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -53,6 +53,7 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m mp_obj_t ret_val; // check radix base + // CIRCUITPY use validator if (base != 0) { // this won't be reached if lex!=NULL mp_arg_validate_int_range(base, 2, 36, MP_QSTR_base); @@ -214,7 +215,7 @@ static void accept_digit(mp_float_t *p_dec_val, int dig, int *p_exp_extra, int i } } } -#endif // MICROPY_BUILTINS_FLOAT +#endif // MICROPY_PY_BUILTINS_FLOAT #if MICROPY_PY_BUILTINS_COMPLEX mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool force_complex, mp_lexer_t *lex) diff --git a/py/parsenum.h b/py/parsenum.h index a0c738659c..f444632d23 100644 --- a/py/parsenum.h +++ b/py/parsenum.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/parsenumbase.c b/py/parsenumbase.c index 0802b435f0..94523a666d 100644 --- a/py/parsenumbase.c +++ b/py/parsenumbase.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/parsenumbase.h b/py/parsenumbase.h index 43dcc2353a..3a525f993c 100644 --- a/py/parsenumbase.h +++ b/py/parsenumbase.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/persistentcode.c b/py/persistentcode.c index abbf8033a4..65e08d1421 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -87,6 +87,7 @@ void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) { size_t addr = read_uint(ri->reader); if ((addr & 1) == 0) { // Point to somewhere in text + // CIRCUITPY avoid compiler warnings #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-align" addr_to_adjust = &((uintptr_t *)text)[addr >> 1]; @@ -397,9 +398,14 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co } void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) { + // Set exception handler to close the reader if an exception is raised. + MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, reader->close, reader->data); + nlr_push_jump_callback(&ctx.callback, mp_call_function_1_from_nlr_jump_callback); + byte header[4]; read_bytes(reader, header, sizeof(header)); byte arch = MPY_FEATURE_DECODE_ARCH(header[2]); + // CIRCUITPY: 'C', not 'M' if (header[0] != 'C' || header[1] != MPY_VERSION || (arch != MP_NATIVE_ARCH_NONE && MPY_FEATURE_DECODE_SUB_VERSION(header[2]) != MPY_SUB_VERSION) @@ -441,7 +447,8 @@ void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) { cm->n_obj = n_obj; #endif - reader->close(reader->data); + // Deregister exception handler and close the reader. + nlr_pop_jump_callback(true); } void mp_raw_code_load_mem(const byte *buf, size_t len, mp_compiled_module_t *context) { diff --git a/py/persistentcode.h b/py/persistentcode.h index f68ce1a7e1..d363f544ad 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2016 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/profile.c b/py/profile.c index fd2d61caa9..274089d702 100644 --- a/py/profile.c +++ b/py/profile.c @@ -31,6 +31,12 @@ #if MICROPY_PY_SYS_SETTRACE +#if !MICROPY_PERSISTENT_CODE_SAVE +// The settrace feature requires that we maintain additional metadata on the raw +// code object which is normally only done when writing .mpy files. +#error "MICROPY_PY_SYS_SETTRACE requires MICROPY_PERSISTENT_CODE_SAVE to be enabled" +#endif + #define prof_trace_cb MP_STATE_THREAD(prof_trace_callback) #define QSTR_MAP(context, idx) (context->constants.qstr_table[idx]) @@ -177,7 +183,6 @@ MP_DEFINE_CONST_OBJ_TYPE( MP_QSTR_code, MP_TYPE_FLAG_NONE, print, code_print, - unary_op, mp_generic_unary_op, attr, code_attr ); @@ -247,7 +252,6 @@ MP_DEFINE_CONST_OBJ_TYPE( MP_QSTR_frame, MP_TYPE_FLAG_NONE, print, frame_print, - unary_op, mp_generic_unary_op, attr, frame_attr ); diff --git a/py/py.cmake b/py/py.cmake index d33e8a801f..8904b6406e 100644 --- a/py/py.cmake +++ b/py/py.cmake @@ -47,7 +47,7 @@ set(MICROPY_SOURCE_PY ${MICROPY_PY_DIR}/modstruct.c ${MICROPY_PY_DIR}/modsys.c ${MICROPY_PY_DIR}/modthread.c - ${MICROPY_PY_DIR}/moduerrno.c + ${MICROPY_PY_DIR}/moderrno.c ${MICROPY_PY_DIR}/mpprint.c ${MICROPY_PY_DIR}/mpstate.c ${MICROPY_PY_DIR}/mpz.c diff --git a/py/py.mk b/py/py.mk index bf258b678f..4fb54fd2fe 100644 --- a/py/py.mk +++ b/py/py.mk @@ -7,6 +7,7 @@ HEADER_BUILD = $(BUILD)/genhdr # file containing qstr defs for the core Python bit PY_QSTR_DEFS = $(PY_SRC)/qstrdefs.h +# CIRCUITPY TRANSLATION ?= en_US # If qstr autogeneration is not disabled we specify the output header @@ -203,7 +204,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ modmicropython.o \ modstruct.o \ modsys.o \ - moduerrno.o \ + moderrno.o \ modthread.o \ vm.o \ bc.o \ diff --git a/py/pystack.h b/py/pystack.h index 4bb717c482..ea8fddcf2f 100644 --- a/py/pystack.h +++ b/py/pystack.h @@ -26,7 +26,6 @@ #ifndef MICROPY_INCLUDED_PY_PYSTACK_H #define MICROPY_INCLUDED_PY_PYSTACK_H -#include "py/mpconfig.h" #include "py/mpstate.h" // Enable this debugging option to check that the amount of memory freed is @@ -41,7 +40,7 @@ void *mp_pystack_alloc(size_t n_bytes); // This function can free multiple continuous blocks at once: just pass the // pointer to the block that was allocated first and it and all subsequently // allocated blocks will be freed. -static MP_INLINE void mp_pystack_free(void *ptr) { +static inline void mp_pystack_free(void *ptr) { assert((uint8_t *)ptr >= MP_STATE_THREAD(pystack_start)); assert((uint8_t *)ptr <= MP_STATE_THREAD(pystack_cur)); #if MP_PYSTACK_DEBUG @@ -59,16 +58,16 @@ static MP_INLINE void mp_pystack_free(void *ptr) { MP_STATE_THREAD(pystack_cur) = (uint8_t *)ptr; } -static MP_INLINE void mp_pystack_realloc(void *ptr, size_t n_bytes) { +static inline void mp_pystack_realloc(void *ptr, size_t n_bytes) { mp_pystack_free(ptr); mp_pystack_alloc(n_bytes); } -static MP_INLINE size_t mp_pystack_usage(void) { +static inline size_t mp_pystack_usage(void) { return MP_STATE_THREAD(pystack_cur) - MP_STATE_THREAD(pystack_start); } -static MP_INLINE size_t mp_pystack_limit(void) { +static inline size_t mp_pystack_limit(void) { return MP_STATE_THREAD(pystack_end) - MP_STATE_THREAD(pystack_start); } @@ -78,43 +77,43 @@ static MP_INLINE size_t mp_pystack_limit(void) { #define mp_local_alloc(n_bytes) alloca(n_bytes) -static MP_INLINE void mp_local_free(void *ptr) { +static inline void mp_local_free(void *ptr) { (void)ptr; } -static MP_INLINE void *mp_nonlocal_alloc(size_t n_bytes) { +static inline void *mp_nonlocal_alloc(size_t n_bytes) { return m_new(uint8_t, n_bytes); } -static MP_INLINE void *mp_nonlocal_realloc(void *ptr, size_t old_n_bytes, size_t new_n_bytes) { +static inline void *mp_nonlocal_realloc(void *ptr, size_t old_n_bytes, size_t new_n_bytes) { return m_renew(uint8_t, ptr, old_n_bytes, new_n_bytes); } -static MP_INLINE void mp_nonlocal_free(void *ptr, size_t n_bytes) { +static inline void mp_nonlocal_free(void *ptr, size_t n_bytes) { m_del(uint8_t, ptr, n_bytes); } #else -static MP_INLINE void *mp_local_alloc(size_t n_bytes) { +static inline void *mp_local_alloc(size_t n_bytes) { return mp_pystack_alloc(n_bytes); } -static MP_INLINE void mp_local_free(void *ptr) { +static inline void mp_local_free(void *ptr) { mp_pystack_free(ptr); } -static MP_INLINE void *mp_nonlocal_alloc(size_t n_bytes) { +static inline void *mp_nonlocal_alloc(size_t n_bytes) { return mp_pystack_alloc(n_bytes); } -static MP_INLINE void *mp_nonlocal_realloc(void *ptr, size_t old_n_bytes, size_t new_n_bytes) { +static inline void *mp_nonlocal_realloc(void *ptr, size_t old_n_bytes, size_t new_n_bytes) { (void)old_n_bytes; mp_pystack_realloc(ptr, new_n_bytes); return ptr; } -static MP_INLINE void mp_nonlocal_free(void *ptr, size_t n_bytes) { +static inline void mp_nonlocal_free(void *ptr, size_t n_bytes) { (void)n_bytes; mp_pystack_free(ptr); } diff --git a/py/qstr.c b/py/qstr.c index f7a5e33bfb..47566af1e5 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -33,6 +33,8 @@ #include "py/gc.h" #include "py/runtime.h" +// CIRCUITPY changes for TRANSLATION + // NOTE: we are using linear arrays to store and search for qstr's (unique strings, interned strings) // ultimately we will replace this with a static hash table of some kind @@ -119,6 +121,7 @@ extern const qstr_pool_t MICROPY_QSTR_EXTRA_POOL; #define CONST_POOL mp_qstr_const_pool #endif +// CIRCUITPY void qstr_reset(void) { MP_STATE_VM(last_pool) = (qstr_pool_t *)&CONST_POOL; // we won't modify the const_pool since it has no allocated room left MP_STATE_VM(qstr_last_chunk) = NULL; diff --git a/py/qstr.h b/py/qstr.h index 20a6d2f28e..64c6a5f78b 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -39,6 +39,7 @@ enum { #ifndef NO_QSTR #define QDEF(id, hash, len, str) id, +// CIRCUITPY #define TRANSLATION(english_id, number) #include "genhdr/qstrdefs.generated.h" #undef QDEF diff --git a/py/qstrdefs.h b/py/qstrdefs.h index e1cebb1db3..4971a9acf2 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -35,6 +35,8 @@ QCFG(BYTES_IN_LEN, MICROPY_QSTR_BYTES_IN_LEN) QCFG(BYTES_IN_HASH, MICROPY_QSTR_BYTES_IN_HASH) +// CIRCUITPY translatable messages removed + Q() Q(*) Q(_) diff --git a/py/repl.c b/py/repl.c index a07dded50c..cf69d3d899 100644 --- a/py/repl.c +++ b/py/repl.c @@ -162,8 +162,8 @@ STATIC bool test_qstr(mp_obj_t obj, qstr name) { return dest[0] != MP_OBJ_NULL; } else { // try builtin module - return mp_map_lookup((mp_map_t *)&mp_builtin_module_map, - MP_OBJ_NEW_QSTR(name), MP_MAP_LOOKUP) != NULL; + return mp_map_lookup((mp_map_t *)&mp_builtin_module_map, MP_OBJ_NEW_QSTR(name), MP_MAP_LOOKUP) || + mp_map_lookup((mp_map_t *)&mp_builtin_extensible_module_map, MP_OBJ_NEW_QSTR(name), MP_MAP_LOOKUP); } } diff --git a/py/ringbuf.c b/py/ringbuf.c index 19295afa44..2906ebd0a2 100644 --- a/py/ringbuf.c +++ b/py/ringbuf.c @@ -5,6 +5,7 @@ * * Copyright (c) 2016 Paul Sokolovsky * Copyright (c) 2019 Jim Mussared + * Copyright (c) 2020 Dan Halbert for Adafruit Industries LLC * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -25,6 +26,8 @@ * THE SOFTWARE. */ +// CIRCUITPY thoroughly reworked + #include "ringbuf.h" bool ringbuf_init(ringbuf_t *r, uint8_t *buf, size_t size) { diff --git a/py/ringbuf.h b/py/ringbuf.h index b6863599cc..a6048bb1d2 100644 --- a/py/ringbuf.h +++ b/py/ringbuf.h @@ -31,6 +31,8 @@ #include #include +// CIRCUITPY thoroughly reworked + typedef struct _ringbuf_t { uint8_t *buf; uint32_t size; @@ -62,4 +64,7 @@ size_t ringbuf_get_n(ringbuf_t *r, uint8_t *buf, size_t bufsize); int ringbuf_get16(ringbuf_t *r); int ringbuf_put16(ringbuf_t *r, uint16_t v); +int ringbuf_get_bytes(ringbuf_t *r, uint8_t *data, size_t data_len); +int ringbuf_put_bytes(ringbuf_t *r, const uint8_t *data, size_t data_len); + #endif // MICROPY_INCLUDED_PY_RINGBUF_H diff --git a/py/runtime.c b/py/runtime.c index d83e624be1..677920122e 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -71,11 +71,10 @@ void mp_init(void) { // no pending exceptions to start with MP_STATE_THREAD(mp_pending_exception) = MP_OBJ_NULL; #if MICROPY_ENABLE_SCHEDULER + // no pending callbacks to start with + MP_STATE_VM(sched_state) = MP_SCHED_IDLE; #if MICROPY_SCHEDULER_STATIC_NODES - if (MP_STATE_VM(sched_head) == NULL) { - // no pending callbacks to start with - MP_STATE_VM(sched_state) = MP_SCHED_IDLE; - } else { + if (MP_STATE_VM(sched_head) != NULL) { // pending callbacks are on the list, eg from before a soft reset MP_STATE_VM(sched_state) = MP_SCHED_PENDING; } @@ -129,12 +128,6 @@ void mp_init(void) { MP_STATE_VM(track_reloc_code_list) = MP_OBJ_NULL; #endif - #ifdef MICROPY_FSUSERMOUNT - // zero out the pointers to the user-mounted devices - memset(MP_STATE_VM(fs_user_mount) + MICROPY_FATFS_NUM_PERSISTENT, 0, - sizeof(MP_STATE_VM(fs_user_mount)) - MICROPY_FATFS_NUM_PERSISTENT); - #endif - #if MICROPY_PY_OS_DUPTERM for (size_t i = 0; i < MICROPY_PY_OS_DUPTERM; ++i) { MP_STATE_VM(dupterm_objs[i]) = MP_OBJ_NULL; @@ -149,13 +142,17 @@ void mp_init(void) { #endif #if MICROPY_PY_SYS_PATH_ARGV_DEFAULTS - mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), 0); + #if MICROPY_PY_SYS_PATH + mp_sys_path = mp_obj_new_list(0, NULL); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script) #if MICROPY_MODULE_FROZEN mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); #endif + #endif + #if MICROPY_PY_SYS_ARGV mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), 0); #endif + #endif // MICROPY_PY_SYS_PATH_ARGV_DEFAULTS #if MICROPY_PY_SYS_ATEXIT MP_STATE_VM(sys_exitfunc) = mp_const_none; @@ -201,6 +198,17 @@ void mp_deinit(void) { #endif } +void mp_globals_locals_set_from_nlr_jump_callback(void *ctx_in) { + nlr_jump_callback_node_globals_locals_t *ctx = ctx_in; + mp_globals_set(ctx->globals); + mp_locals_set(ctx->locals); +} + +void mp_call_function_1_from_nlr_jump_callback(void *ctx_in) { + nlr_jump_callback_node_call_function_1_t *ctx = ctx_in; + ctx->func(ctx->arg); +} + mp_obj_t MICROPY_WRAP_MP_LOAD_NAME(mp_load_name)(qstr qst) { // logic: search locals, globals, builtins DEBUG_OP_printf("load name %s\n", qstr_str(qst)); @@ -240,6 +248,8 @@ mp_obj_t MICROPY_WRAP_MP_LOAD_GLOBAL(mp_load_global)(qstr qst) { return elem->value; } +// CIRCUITPY noinline +// https://github.com/adafruit/circuitpython/pull/8071 mp_obj_t __attribute__((noinline)) mp_load_build_class(void) { DEBUG_OP_printf("load_build_class\n"); #if MICROPY_CAN_OVERRIDE_BUILTINS @@ -290,7 +300,7 @@ mp_obj_t mp_unary_op(mp_unary_op_t op, mp_obj_t arg) { case MP_UNARY_OP_HASH: return arg; case MP_UNARY_OP_POSITIVE: - case MP_UNARY_OP_INT: + case MP_UNARY_OP_INT_MAYBE: return arg; case MP_UNARY_OP_NEGATIVE: // check for overflow @@ -327,6 +337,9 @@ mp_obj_t mp_unary_op(mp_unary_op_t op, mp_obj_t arg) { if (result != MP_OBJ_NULL) { return result; } + } else if (op == MP_UNARY_OP_HASH) { + // Type doesn't have unary_op so use hash of object instance. + return MP_OBJ_NEW_SMALL_INT((mp_uint_t)arg); } if (op == MP_UNARY_OP_BOOL) { // Type doesn't have unary_op (or didn't handle MP_UNARY_OP_BOOL), @@ -334,30 +347,23 @@ mp_obj_t mp_unary_op(mp_unary_op_t op, mp_obj_t arg) { // if arg==mp_const_none. return mp_const_true; } - #if MICROPY_PY_BUILTINS_FLOAT - if (op == MP_UNARY_OP_FLOAT_MAYBE + if (op == MP_UNARY_OP_INT_MAYBE + #if MICROPY_PY_BUILTINS_FLOAT + || op == MP_UNARY_OP_FLOAT_MAYBE #if MICROPY_PY_BUILTINS_COMPLEX || op == MP_UNARY_OP_COMPLEX_MAYBE #endif + #endif ) { + // These operators may return MP_OBJ_NULL if they are not supported by the type. return MP_OBJ_NULL; } - #endif - // With MP_UNARY_OP_INT, mp_unary_op() becomes a fallback for mp_obj_get_int(). - // In this case provide a more focused error message to not confuse, e.g. chr(1.0) #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE - if (op == MP_UNARY_OP_INT) { - mp_raise_TypeError(MP_ERROR_TEXT("can't convert to int")); - } else { - mp_raise_TypeError(MP_ERROR_TEXT("unsupported type for operator")); - } + mp_raise_TypeError(MP_ERROR_TEXT("unsupported type for operator")); #else - if (op == MP_UNARY_OP_INT) { - mp_raise_TypeError_varg(MP_ERROR_TEXT("can't convert %q to %q"), mp_obj_get_type_qstr(arg), MP_QSTR_int); - } else { - mp_raise_TypeError_varg(MP_ERROR_TEXT("unsupported type for %q: '%q'"), - mp_unary_op_method_name[op], mp_obj_get_type_qstr(arg)); - } + mp_raise_msg_varg(&mp_type_TypeError, + MP_ERROR_TEXT("unsupported type for %q: '%s'"), + mp_unary_op_method_name[op], mp_obj_get_type_str(arg)); #endif } } @@ -633,6 +639,15 @@ generic_binary_op: } } + // If this was an inplace method, fallback to the corresponding normal method. + // https://docs.python.org/3/reference/datamodel.html#object.__iadd__ : + // "If a specific method is not defined, the augmented assignment falls back + // to the normal methods." + if (op >= MP_BINARY_OP_INPLACE_OR && op <= MP_BINARY_OP_INPLACE_POWER) { + op += MP_BINARY_OP_OR - MP_BINARY_OP_INPLACE_OR; + goto generic_binary_op; + } + #if MICROPY_PY_REVERSE_SPECIAL_METHODS if (op >= MP_BINARY_OP_OR && op <= MP_BINARY_OP_POWER) { mp_obj_t t = rhs; @@ -935,6 +950,7 @@ mp_obj_t mp_call_method_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ob } // unpacked items are stored in reverse order into the array pointed to by items +// CIRCUITPY noline void __attribute__((noinline, )) mp_unpack_sequence(mp_obj_t seq_in, size_t num, mp_obj_t *items) { size_t seq_len; if (mp_obj_is_type(seq_in, &mp_type_tuple) || mp_obj_is_type(seq_in, &mp_type_list)) { @@ -982,6 +998,7 @@ too_long: } // unpacked items are stored in reverse order into the array pointed to by items +// CIRCUITPY noinline void __attribute__((noinline)) mp_unpack_ex(mp_obj_t seq_in, size_t num_in, mp_obj_t *items) { size_t num_left = num_in & 0xff; size_t num_right = (num_in >> 8) & 0xff; @@ -1074,12 +1091,12 @@ STATIC mp_obj_t checked_fun_call(mp_obj_t self_in, size_t n_args, size_t n_kw, c if (n_args > 0) { const mp_obj_type_t *arg0_type = mp_obj_get_type(args[0]); if (arg0_type != self->type) { - if (MICROPY_ERROR_REPORTING != MICROPY_ERROR_REPORTING_DETAILED) { - mp_raise_TypeError(MP_ERROR_TEXT("argument has wrong type")); - } else { - mp_raise_TypeError_varg(MP_ERROR_TEXT("argument should be a '%q' not a '%q'"), - self->type->name, arg0_type->name); - } + #if MICROPY_ERROR_REPORTING != MICROPY_ERROR_REPORTING_DETAILED + mp_raise_TypeError(MP_ERROR_TEXT("argument has wrong type")); + #else + mp_raise_msg_varg(&mp_type_TypeError, + MP_ERROR_TEXT("argument should be a '%q' not a '%q'"), self->type->name, arg0_type->name); + #endif } } return mp_call_function_n_kw(self->fun, n_args, n_kw, args); @@ -1147,6 +1164,8 @@ void mp_convert_member_lookup(mp_obj_t self, const mp_obj_type_t *type, mp_obj_t } dest[0] = ((mp_obj_static_class_method_t *)MP_OBJ_TO_PTR(member))->fun; dest[1] = MP_OBJ_FROM_PTR(type); + // CIRCUITPY + // https://github.com/adafruit/circuitpython/commit/8fae7d2e3024de6336affc4b2a8fa992c946e017 #if MICROPY_PY_BUILTINS_PROPERTY // If self is MP_OBJ_NULL, we looking at the class itself, not an instance. } else if (mp_obj_is_type(member, &mp_type_property) && mp_obj_is_native_type(type) && self != MP_OBJ_NULL) { @@ -1278,6 +1297,7 @@ void mp_store_attr(mp_obj_t base, qstr attr, mp_obj_t value) { // success return; } + // CIRCUITPY https://github.com/adafruit/circuitpython/pull/50 #if MICROPY_PY_BUILTINS_PROPERTY } else if (MP_OBJ_TYPE_HAS_SLOT(type, locals_dict)) { // generic method lookup @@ -1314,6 +1334,7 @@ void mp_store_attr(mp_obj_t base, qstr attr, mp_obj_t value) { mp_raise_AttributeError(MP_ERROR_TEXT("no such attribute")); #else mp_raise_msg_varg(&mp_type_AttributeError, + // CIRCUITPY better error message MP_ERROR_TEXT("can't set attribute '%q'"), attr); #endif @@ -1372,6 +1393,7 @@ mp_obj_t mp_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) { STATIC mp_fun_1_t type_get_iternext(const mp_obj_type_t *type) { if ((type->flags & MP_TYPE_FLAG_ITER_IS_STREAM) == MP_TYPE_FLAG_ITER_IS_STREAM) { + mp_obj_t mp_stream_unbuffered_iter(mp_obj_t self); return mp_stream_unbuffered_iter; } else if (type->flags & MP_TYPE_FLAG_ITER_IS_ITERNEXT) { return (mp_fun_1_t)MP_OBJ_TYPE_GET_SLOT(type, iter); @@ -1574,6 +1596,7 @@ mp_obj_t mp_import_name(qstr name, mp_obj_t fromlist, mp_obj_t level) { return mp_builtin___import__(5, args); } +// CIRCUITPY noinline mp_obj_t __attribute__((noinline, )) mp_import_from(mp_obj_t module, qstr name) { DEBUG_printf("import from %p %s\n", module, qstr_str(name)); @@ -1594,7 +1617,8 @@ mp_obj_t __attribute__((noinline, )) mp_import_from(mp_obj_t module, qstr name) #if MICROPY_ENABLE_EXTERNAL_IMPORT // See if it's a package, then can try FS import - if (!mp_obj_is_package(module)) { + mp_load_method_maybe(module, MP_QSTR___path__, dest); + if (dest[0] == MP_OBJ_NULL) { goto import_error; } @@ -1644,43 +1668,40 @@ void mp_import_all(mp_obj_t module) { mp_obj_t mp_parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t parse_input_kind, mp_obj_dict_t *globals, mp_obj_dict_t *locals) { // save context - mp_obj_dict_t *volatile old_globals = mp_globals_get(); - mp_obj_dict_t *volatile old_locals = mp_locals_get(); + nlr_jump_callback_node_globals_locals_t ctx; + ctx.globals = mp_globals_get(); + ctx.locals = mp_locals_get(); // set new context mp_globals_set(globals); mp_locals_set(locals); - nlr_buf_t nlr; - if (nlr_push(&nlr) == 0) { - qstr source_name = lex->source_name; - mp_parse_tree_t parse_tree = mp_parse(lex, parse_input_kind); - mp_obj_t module_fun = mp_compile(&parse_tree, source_name, parse_input_kind == MP_PARSE_SINGLE_INPUT); + // set exception handler to restore context if an exception is raised + nlr_push_jump_callback(&ctx.callback, mp_globals_locals_set_from_nlr_jump_callback); - mp_obj_t ret; - if (MICROPY_PY_BUILTINS_COMPILE && globals == NULL) { - // for compile only, return value is the module function - ret = module_fun; - } else { - // execute module function and get return value - ret = mp_call_function_0(module_fun); - } + qstr source_name = lex->source_name; + mp_parse_tree_t parse_tree = mp_parse(lex, parse_input_kind); + mp_obj_t module_fun = mp_compile(&parse_tree, source_name, parse_input_kind == MP_PARSE_SINGLE_INPUT); - // finish nlr block, restore context and return value - nlr_pop(); - mp_globals_set(old_globals); - mp_locals_set(old_locals); - return ret; + mp_obj_t ret; + if (MICROPY_PY_BUILTINS_COMPILE && globals == NULL) { + // for compile only, return value is the module function + ret = module_fun; } else { - // exception; restore context and re-raise same exception - mp_globals_set(old_globals); - mp_locals_set(old_locals); - nlr_jump(nlr.ret_val); + // execute module function and get return value + ret = mp_call_function_0(module_fun); } + + // deregister exception handler and restore context + nlr_pop_jump_callback(true); + + // return value + return ret; } #endif // MICROPY_ENABLE_COMPILER +// CIRCUITPY MP_COLD are CIRCUITPY NORETURN MP_COLD void m_malloc_fail(size_t num_bytes) { DEBUG_printf("memory allocation failed, allocating %u bytes\n", (uint)num_bytes); #if MICROPY_ENABLE_GC @@ -1851,6 +1872,16 @@ NORETURN void mp_raise_StopIteration(mp_obj_t arg) { } } +NORETURN void mp_raise_TypeError_int_conversion(mp_const_obj_t arg) { + #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE + (void)arg; + mp_raise_TypeError(MP_ERROR_TEXT("can't convert to int")); + #else + mp_raise_msg_varg(&mp_type_TypeError, + MP_ERROR_TEXT("can't convert %s to int"), mp_obj_get_type_str(arg)); + #endif +} + NORETURN MP_COLD void mp_raise_OSError(int errno_) { mp_raise_type_arg(&mp_type_OSError, MP_OBJ_NEW_SMALL_INT(errno_)); } diff --git a/py/runtime.h b/py/runtime.h index 770e892de7..332acf7d94 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -31,9 +31,17 @@ #include "py/mpstate.h" #include "py/pystack.h" +// CIRCUITPY #include "supervisor/linker.h" #include "supervisor/shared/translate/translate.h" +// For use with mp_call_function_1_from_nlr_jump_callback. +#define MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, f, a) \ + nlr_jump_callback_node_call_function_1_t ctx = { \ + .func = (void (*)(void *))(f), \ + .arg = (a), \ + } + typedef enum { MP_VM_RETURN_NORMAL, MP_VM_RETURN_YIELD, @@ -71,6 +79,20 @@ typedef struct _mp_sched_node_t { struct _mp_sched_node_t *next; } mp_sched_node_t; +// For use with mp_globals_locals_set_from_nlr_jump_callback. +typedef struct _nlr_jump_callback_node_globals_locals_t { + nlr_jump_callback_node_t callback; + mp_obj_dict_t *globals; + mp_obj_dict_t *locals; +} nlr_jump_callback_node_globals_locals_t; + +// For use with mp_call_function_1_from_nlr_jump_callback. +typedef struct _nlr_jump_callback_node_call_function_1_t { + nlr_jump_callback_node_t callback; + void (*func)(void *); + void *arg; +} nlr_jump_callback_node_call_function_1_t; + // Tables mapping operator enums to qstrs, defined in objtype.c extern const byte mp_unary_op_method_name[]; extern const byte mp_binary_op_method_name[]; @@ -80,8 +102,10 @@ void mp_deinit(void); void mp_sched_exception(mp_obj_t exc); void mp_sched_keyboard_interrupt(void); +#if MICROPY_ENABLE_VM_ABORT +void mp_sched_vm_abort(void); +#endif void mp_handle_pending(bool raise_exc); -void mp_handle_pending_tail(mp_uint_t atomic_state); #if MICROPY_ENABLE_SCHEDULER void mp_sched_lock(void); @@ -103,6 +127,7 @@ void mp_arg_parse_all_kw_array(size_t n_pos, size_t n_kw, const mp_obj_t *args, NORETURN void mp_arg_error_terse_mismatch(void); NORETURN void mp_arg_error_unimpl_kw(void); +// CIRCUITPY arg validation routines NORETURN void mp_arg_error_invalid(qstr arg_name); mp_int_t mp_arg_validate_int(mp_int_t i, mp_int_t required_i, qstr arg_name); mp_int_t mp_arg_validate_int_min(mp_int_t i, mp_int_t min, qstr arg_name); @@ -249,6 +274,7 @@ NORETURN void mp_raise_RuntimeError_varg(const compressed_string_t *fmt, ...); NORETURN void mp_raise_StopIteration(mp_obj_t arg); NORETURN void mp_raise_TypeError(const compressed_string_t *msg); NORETURN void mp_raise_TypeError_varg(const compressed_string_t *fmt, ...); +NORETURN void mp_raise_TypeError_int_conversion(mp_const_obj_t arg); NORETURN void mp_raise_ValueError(const compressed_string_t *msg); NORETURN void mp_raise_ValueError_varg(const compressed_string_t *fmt, ...); NORETURN void mp_raise_ZeroDivisionError(void); @@ -268,8 +294,13 @@ int mp_native_type_from_qstr(qstr qst); mp_uint_t mp_native_from_obj(mp_obj_t obj, mp_uint_t type); mp_obj_t mp_native_to_obj(mp_uint_t val, mp_uint_t type); -#define mp_sys_path (MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_sys_path_obj))) +#if MICROPY_PY_SYS_PATH +#define mp_sys_path (MP_STATE_VM(sys_mutable[MP_SYS_MUTABLE_PATH])) +#endif + +#if MICROPY_PY_SYS_ARGV #define mp_sys_argv (MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_sys_argv_obj))) +#endif #if MICROPY_WARNINGS #ifndef mp_warning diff --git a/py/runtime0.h b/py/runtime0.h index 994c7d9324..d5ea4ddaab 100644 --- a/py/runtime0.h +++ b/py/runtime0.h @@ -35,6 +35,7 @@ #define MP_SCOPE_FLAG_VARKEYWORDS (0x02) #define MP_SCOPE_FLAG_VARARGS (0x04) #define MP_SCOPE_FLAG_DEFKWARGS (0x08) +// CIRCUITPY #define MP_SCOPE_FLAG_ASYNC (0x10) #define MP_SCOPE_FLAG_REFGLOBALS (0x20) // used only if native emitter enabled #define MP_SCOPE_FLAG_HASCONSTS (0x40) // used only if native emitter enabled @@ -53,6 +54,9 @@ #define MP_NATIVE_TYPE_PTR16 (0x06) #define MP_NATIVE_TYPE_PTR32 (0x07) +// Not use for viper, but for dynamic native modules +#define MP_NATIVE_TYPE_QSTR (0x08) + // Bytecode and runtime boundaries for unary ops #define MP_UNARY_OP_NUM_BYTECODE (MP_UNARY_OP_NOT + 1) #define MP_UNARY_OP_NUM_RUNTIME (MP_UNARY_OP_SIZEOF + 1) @@ -78,7 +82,7 @@ typedef enum { MP_UNARY_OP_LEN, // __len__ MP_UNARY_OP_HASH, // __hash__; must return a small int MP_UNARY_OP_ABS, // __abs__ - MP_UNARY_OP_INT, // __int__ + MP_UNARY_OP_INT_MAYBE, // __int__; must return MP_OBJ_NULL, or an object satisfying mp_obj_is_int() MP_UNARY_OP_FLOAT_MAYBE, // __float__ MP_UNARY_OP_COMPLEX_MAYBE, // __complex__ MP_UNARY_OP_SIZEOF, // for sys.getsizeof() diff --git a/py/scheduler.c b/py/scheduler.c index 41001d6af9..33cef16777 100644 --- a/py/scheduler.c +++ b/py/scheduler.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2017 Damien P. George + * Copyright (c) 2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -46,6 +46,7 @@ void MICROPY_WRAP_MP_SCHED_EXCEPTION(mp_sched_exception)(mp_obj_t exc) { #if MICROPY_KBD_EXCEPTION // This function may be called asynchronously at any time so only do the bare minimum. void MICROPY_WRAP_MP_SCHED_KEYBOARD_INTERRUPT(mp_sched_keyboard_interrupt)(void) { + // CIRCUITPY MP_STATE_VM(mp_kbd_exception).traceback = (mp_obj_traceback_t *)&mp_const_empty_traceback_obj; mp_sched_exception(MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_kbd_exception))); } diff --git a/py/stackctrl.c b/py/stackctrl.c index fa5f16d662..fe0f43e3c1 100644 --- a/py/stackctrl.c +++ b/py/stackctrl.c @@ -28,16 +28,17 @@ #include "py/stackctrl.h" void mp_stack_ctrl_init(void) { - // Force routine to not be inlined. Better guarantee than MP_NOINLINE for -flto. + // CIRCUITPY: Force routine to not be inlined. Better guarantee than MP_NOINLINE for -flto. __asm volatile (""); + #if __GNUC__ >= 13 #pragma GCC diagnostic push - #if __GNUC__ > 12 - // Introduced in GCC 13 #pragma GCC diagnostic ignored "-Wdangling-pointer" #endif volatile int stack_dummy; MP_STATE_THREAD(stack_top) = (char *)&stack_dummy; + #if __GNUC__ >= 13 #pragma GCC diagnostic pop + #endif } void mp_stack_set_top(void *top) { @@ -46,7 +47,7 @@ void mp_stack_set_top(void *top) { mp_uint_t PLACE_IN_ITCM(mp_stack_usage)(void) { // Assumes descending stack - // Force routine to not be inlined. Better guarantee than MP_NOINLINE for -flto. + // CIRCUITPY: Force routine to not be inlined. Better guarantee than MP_NOINLINE for -flto. __asm volatile (""); volatile int stack_dummy; return MP_STATE_THREAD(stack_top) - (char *)&stack_dummy; @@ -65,41 +66,3 @@ void PLACE_IN_ITCM(mp_stack_check)(void) { } #endif // MICROPY_STACK_CHECK - -#if MICROPY_MAX_STACK_USAGE - -// Fill stack space with this unusual value. -const char MP_MAX_STACK_USAGE_SENTINEL_BYTE = 0xEE; - -// Record absolute bottom (logical limit) of stack. -void mp_stack_set_bottom(void *stack_bottom) { - MP_STATE_THREAD(stack_bottom) = stack_bottom; -} - -// Return the current frame pointer. This can be used as an -// approximation for the stack pointer of the _calling_ function. -// This routine must not be inlined. This method is -// architecture-independent, as opposed to using asm("sp") or similar. -// -// The stack_dummy approach used elsewhere in this file is not safe in -// all cases. That value may be below the actual top of the stack. -static void *approx_stack_pointer(void) { - __asm volatile (""); - return __builtin_frame_address(0); -} - -// Fill stack space down toward the stack limit with a known unusual value. -void mp_stack_fill_with_sentinel(void) { - // Force routine to not be inlined. Better guarantee than MP_NOINLINE for -flto. - __asm volatile (""); - // Start filling stack just below the current stack frame. - // Continue until we've hit the bottom of the stack (lowest address, - // logical "ceiling" of stack). - char *p = (char *)approx_stack_pointer() - 1; - - while (p >= MP_STATE_THREAD(stack_bottom)) { - *p-- = MP_MAX_STACK_USAGE_SENTINEL_BYTE; - } -} - -#endif // MICROPY_MAX_STACK_USAGE diff --git a/py/stream.c b/py/stream.c index 6dd8f90970..b8c803d2e4 100644 --- a/py/stream.c +++ b/py/stream.c @@ -438,6 +438,12 @@ mp_obj_t mp_stream_close(mp_obj_t stream) { } MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_close_obj, mp_stream_close); +STATIC mp_obj_t mp_stream___exit__(size_t n_args, const mp_obj_t *args) { + (void)n_args; + return mp_stream_close(args[0]); +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream___exit___obj, 4, 4, mp_stream___exit__); + STATIC mp_obj_t stream_seek(size_t n_args, const mp_obj_t *args) { struct mp_stream_seek_t seek_s; // TODO: Could be uint64 diff --git a/py/stream.h b/py/stream.h index e776d8d5d1..9bdb5e8390 100644 --- a/py/stream.h +++ b/py/stream.h @@ -28,6 +28,7 @@ #define MICROPY_INCLUDED_PY_STREAM_H #include "py/obj.h" +// CIRCUITPY #include "py/proto.h" #include "py/mperrno.h" @@ -68,6 +69,7 @@ struct mp_stream_seek_t { // Stream protocol typedef struct _mp_stream_p_t { + // CIRCUITPY MP_PROTOCOL_HEAD // On error, functions should return MP_STREAM_ERROR and fill in *errcode (values // are implementation-dependent, but will be exposed to user, e.g. via exception). @@ -88,6 +90,7 @@ MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_unbuffered_readlines_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write_obj); MP_DECLARE_CONST_FUN_OBJ_2(mp_stream_write1_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_close_obj); +MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream___exit___obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_seek_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_tell_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_flush_obj); @@ -120,7 +123,8 @@ mp_uint_t mp_stream_rw(mp_obj_t stream, void *buf, mp_uint_t size, int *errcode, #define mp_stream_read_exactly(stream, buf, size, err) mp_stream_rw(stream, buf, size, err, MP_STREAM_RW_READ) void mp_stream_write_adaptor(void *self, const char *buf, size_t len); -mp_obj_t mp_stream_flush(mp_obj_t self); // CIRCUITPY + // CIRCUITPY +mp_obj_t mp_stream_flush(mp_obj_t self); #if MICROPY_STREAMS_POSIX_API #include diff --git a/py/vm.c b/py/vm.c index 0e6c1c249d..950425c877 100644 --- a/py/vm.c +++ b/py/vm.c @@ -195,6 +195,7 @@ #define TRACE_TICK(current_ip, current_sp, is_exception) #endif // MICROPY_PY_SYS_SETTRACE +// CIRCUITPY STATIC mp_obj_t get_active_exception(mp_exc_stack_t *exc_sp, mp_exc_stack_t *exc_stack) { for (mp_exc_stack_t *e = exc_sp; e >= exc_stack; --e) { if (e->prev_exc != NULL) { @@ -236,6 +237,7 @@ mp_vm_return_kind_t MICROPY_WRAP_MP_EXECUTE_BYTECODE(mp_execute_bytecode)(mp_cod #endif #if MICROPY_OPT_COMPUTED_GOTO #include "py/vmentrytable.h" + // CIRCUITPY #if MICROPY_OPT_COMPUTED_GOTO_SAVE_SPACE #define ONE_TRUE_DISPATCH() one_true_dispatch : do { \ TRACE(ip); \ @@ -326,6 +328,7 @@ outer_dispatch_loop: for (;;) { dispatch_loop: #if MICROPY_OPT_COMPUTED_GOTO + // CIRCUITPY ONE_TRUE_DISPATCH(); #else TRACE(ip); @@ -1169,7 +1172,8 @@ unwind_return: ENTRY(MP_BC_RAISE_LAST): { MARK_EXC_IP_SELECTIVE(); // search for the inner-most previous exception, to reraise it - mp_obj_t obj = get_active_exception(exc_sp, exc_stack); + // CIRCUITPY + mp_obj_t obj = get_active_exception(exc_sp, exc_stack); if (obj == MP_OBJ_NULL) { obj = mp_obj_new_exception_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("no active exception to reraise")); } @@ -1179,6 +1183,7 @@ unwind_return: ENTRY(MP_BC_RAISE_OBJ): { MARK_EXC_IP_SELECTIVE(); mp_obj_t obj = mp_make_raise_obj(TOP()); + // CIRCUITPY #if MICROPY_CPYTHON_EXCEPTION_CHAIN mp_obj_t active_exception = get_active_exception(exc_sp, exc_stack); if (active_exception != MP_OBJ_NULL && active_exception != obj) { @@ -1190,6 +1195,7 @@ unwind_return: ENTRY(MP_BC_RAISE_FROM): { MARK_EXC_IP_SELECTIVE(); + // CIRCUITPY mp_obj_t cause = POP(); mp_obj_t obj = mp_make_raise_obj(TOP()); #if MICROPY_CPYTHON_EXCEPTION_CHAIN diff --git a/py/vstr.c b/py/vstr.c index ef5fe06eb2..3b62bce22f 100644 --- a/py/vstr.c +++ b/py/vstr.c @@ -51,6 +51,7 @@ void vstr_init(vstr_t *vstr, size_t alloc) { // Init the vstr so it allocs exactly enough ram to hold a null-terminated // string of the given length, and set the length. void vstr_init_len(vstr_t *vstr, size_t len) { + // CIRCUITPY if (len == SIZE_MAX) { m_malloc_fail(len); } diff --git a/shared/libc/printf.c b/shared/libc/printf.c index e52ed523ef..715181229e 100644 --- a/shared/libc/printf.c +++ b/shared/libc/printf.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -72,16 +72,14 @@ int vprintf(const char *fmt, va_list ap) { // need this because gcc optimises printf("%c", c) -> putchar(c), and printf("a") -> putchar('a') int putchar(int c) { char chr = c; - mp_hal_stdout_tx_strn_cooked(&chr, 1); + MICROPY_INTERNAL_PRINTF_PRINTER->print_strn(MICROPY_INTERNAL_PRINTF_PRINTER->data, &chr, 1); return chr; } // need this because gcc optimises printf("string\n") -> puts("string") int puts(const char *s) { - mp_hal_stdout_tx_strn_cooked(s, strlen(s)); - char chr = '\n'; - mp_hal_stdout_tx_strn_cooked(&chr, 1); - return 1; + MICROPY_INTERNAL_PRINTF_PRINTER->print_strn(MICROPY_INTERNAL_PRINTF_PRINTER->data, s, strlen(s)); + return putchar('\n'); // will return 10, which is >0 per specs of puts } typedef struct _strn_print_env_t { diff --git a/shared/libc/string0.c b/shared/libc/string0.c index 92b063c552..c45d3e4893 100644 --- a/shared/libc/string0.c +++ b/shared/libc/string0.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -34,6 +34,7 @@ #define likely(x) __builtin_expect((x), 1) #endif +// CIRCUITPY avoid compiler warnings #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-align" void *memcpy(void *dst, const void *src, size_t n) { @@ -74,6 +75,7 @@ void *memcpy(void *dst, const void *src, size_t n) { return dst; } +// CIRCUITPY extern extern void *__memcpy_chk(void *dest, const void *src, size_t len, size_t slen); void *__memcpy_chk(void *dest, const void *src, size_t len, size_t slen) { if (len > slen) { diff --git a/shared/memzip/README.md b/shared/memzip/README.md index 9aa12a94c8..3938e3198c 100644 --- a/shared/memzip/README.md +++ b/shared/memzip/README.md @@ -25,3 +25,4 @@ $(BUILD)/memzip-files.c: $(shell find ${MEMZIP_DIR} -type f) @$(ECHO) "Creating $@" $(Q)$(PYTHON) $(MAKE_MEMZIP) --zip-file $(BUILD)/memzip-files.zip --c-file $@ $(MEMZIP_DIR) ``` + diff --git a/shared/memzip/lexermemzip.c b/shared/memzip/lexermemzip.c index c1f21c1caa..6b26961bdc 100644 --- a/shared/memzip/lexermemzip.c +++ b/shared/memzip/lexermemzip.c @@ -16,3 +16,4 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) return mp_lexer_new_from_str_len(qstr_from_str(filename), (const char *)data, (mp_uint_t)len, 0); } + diff --git a/shared/memzip/make-memzip.py b/shared/memzip/make-memzip.py index dd1ac93da4..9fc92f2c4d 100755 --- a/shared/memzip/make-memzip.py +++ b/shared/memzip/make-memzip.py @@ -36,7 +36,7 @@ def create_c_from_file(c_filename, zip_filename): break print(' ', end='', file=c_file) for byte in buf: - if type(byte) is types.StringType: + if isinstance(byte, types.StringType): print(' 0x{:02x},'.format(ord(byte)), end='', file=c_file) else: print(' 0x{:02x},'.format(byte), end='', file=c_file) diff --git a/shared/netutils/dhcpserver.c b/shared/netutils/dhcpserver.c index 0a84da60d6..1609ea9646 100644 --- a/shared/netutils/dhcpserver.c +++ b/shared/netutils/dhcpserver.c @@ -34,6 +34,8 @@ #include "py/mphal.h" #include "lwip/opt.h" +// Used in CIRCUITPY without MICROPY_PY_LWIP + #if LWIP_UDP #include "shared/netutils/dhcpserver.h" @@ -119,7 +121,7 @@ static int dhcp_socket_bind(struct udp_pcb **udp, uint32_t ip, uint16_t port) { return udp_bind(*udp, &addr, port); } -static int dhcp_socket_sendto(struct udp_pcb **udp, const void *buf, size_t len, uint32_t ip, uint16_t port) { +static int dhcp_socket_sendto(struct udp_pcb **udp, struct netif *netif, const void *buf, size_t len, uint32_t ip, uint16_t port) { if (len > 0xffff) { len = 0xffff; } @@ -133,7 +135,12 @@ static int dhcp_socket_sendto(struct udp_pcb **udp, const void *buf, size_t len, ip_addr_t dest; IP4_ADDR(&dest, ip >> 24 & 0xff, ip >> 16 & 0xff, ip >> 8 & 0xff, ip & 0xff); - err_t err = udp_sendto(*udp, p, &dest, port); + err_t err; + if (netif != NULL) { + err = udp_sendto_if(*udp, p, &dest, port, netif); + } else { + err = udp_sendto(*udp, p, &dest, port); + } pbuf_free(p); @@ -154,7 +161,7 @@ static uint8_t *opt_find(uint8_t *opt, uint8_t cmd) { return NULL; } -static void opt_write_n(uint8_t **opt, uint8_t cmd, size_t n, void *data) { +static void opt_write_n(uint8_t **opt, uint8_t cmd, size_t n, const void *data) { uint8_t *o = *opt; *o++ = cmd; *o++ = n; @@ -282,7 +289,8 @@ static void dhcp_server_process(void *arg, struct udp_pcb *upcb, struct pbuf *p, opt_write_u32(&opt, DHCP_OPT_DNS, DEFAULT_DNS); // can have multiple addresses opt_write_u32(&opt, DHCP_OPT_IP_LEASE_TIME, DEFAULT_LEASE_TIME_S); *opt++ = DHCP_OPT_END; - dhcp_socket_sendto(&d->udp, &dhcp_msg, opt - (uint8_t *)&dhcp_msg, 0xffffffff, PORT_DHCP_CLIENT); + struct netif *netif = ip_current_input_netif(); + dhcp_socket_sendto(&d->udp, netif, &dhcp_msg, opt - (uint8_t *)&dhcp_msg, 0xffffffff, PORT_DHCP_CLIENT); ignore_request: pbuf_free(p); diff --git a/shared/netutils/netutils.c b/shared/netutils/netutils.c index ab2b592427..84b4405c41 100644 --- a/shared/netutils/netutils.c +++ b/shared/netutils/netutils.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * Copyright (c) 2015 Daniel Campora * * Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/shared/netutils/netutils.h b/shared/netutils/netutils.h index dfc554bf8a..58113d17b9 100644 --- a/shared/netutils/netutils.h +++ b/shared/netutils/netutils.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * Copyright (c) 2015 Daniel Campora * * Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/shared/readline/readline.c b/shared/readline/readline.c index db4653b82b..4060c6412f 100644 --- a/shared/readline/readline.c +++ b/shared/readline/readline.c @@ -40,6 +40,8 @@ #define DEBUG_printf(...) (void)0 #endif +// CIRCUITPY a number of changes + #define READLINE_HIST_SIZE (MP_ARRAY_SIZE(MP_STATE_PORT(readline_hist))) // flags for readline_t.auto_indent_state diff --git a/shared/readline/readline.h b/shared/readline/readline.h index 9af19b181e..cc77a89dac 100644 --- a/shared/readline/readline.h +++ b/shared/readline/readline.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,8 @@ #ifndef MICROPY_INCLUDED_LIB_MP_READLINE_READLINE_H #define MICROPY_INCLUDED_LIB_MP_READLINE_READLINE_H +// CIRCUITPY: a number of changes + #include "py/misc.h" #define CHAR_CTRL_A (1) diff --git a/shared/runtime/interrupt_char.c b/shared/runtime/interrupt_char.c index de4218420d..eb5470f96a 100644 --- a/shared/runtime/interrupt_char.c +++ b/shared/runtime/interrupt_char.c @@ -36,6 +36,7 @@ void mp_hal_set_interrupt_char(int c) { mp_interrupt_char = c; } +// CIRCUITPY // Check to see if we've been CTRL-C'ed by autoreload or the user. bool mp_hal_is_interrupted(void) { return MP_STATE_THREAD(mp_pending_exception) != MP_OBJ_FROM_PTR(NULL); diff --git a/shared/runtime/interrupt_char.h b/shared/runtime/interrupt_char.h index 41b0bd8faa..bca8b63417 100644 --- a/shared/runtime/interrupt_char.h +++ b/shared/runtime/interrupt_char.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2016 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ #ifndef MICROPY_INCLUDED_LIB_UTILS_INTERRUPT_CHAR_H #define MICROPY_INCLUDED_LIB_UTILS_INTERRUPT_CHAR_H +// CIRCUITPY changes #include extern int mp_interrupt_char; diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 7b71fe4dea..1e74ccf1d8 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -43,6 +43,8 @@ #include "shared/runtime/pyexec.h" #include "genhdr/mpversion.h" +// CIRCUITPY multiple changes for atexit(), interrupts + #if CIRCUITPY_ATEXIT #include "shared-module/atexit/__init__.h" #endif @@ -542,6 +544,8 @@ int pyexec_event_repl_process_char(int c) { return res; } +MP_REGISTER_ROOT_POINTER(vstr_t * repl_line); + #else // MICROPY_REPL_EVENT_DRIVEN int pyexec_raw_repl(void) { @@ -639,7 +643,7 @@ friendly_repl_reset: // If the user gets to here and interrupts are disabled then // they'll never see the prompt, traceback etc. The USB REPL needs // interrupts to be enabled or no transfers occur. So we try to - // do the user a favor and reenable interrupts. + // do the user a favor and re-enable interrupts. if (query_irq() == IRQ_STATE_DISABLED) { enable_irq(IRQ_STATE_ENABLED); mp_hal_stdout_tx_str("MPY: enabling IRQs\r\n"); @@ -796,5 +800,3 @@ mp_obj_t pyb_set_repl_info(mp_obj_t o_value) { } MP_DEFINE_CONST_FUN_OBJ_1(pyb_set_repl_info_obj, pyb_set_repl_info); #endif - -MP_REGISTER_ROOT_POINTER(vstr_t * repl_line); diff --git a/shared/runtime/pyexec.h b/shared/runtime/pyexec.h index 0e1a354036..7f855169e4 100644 --- a/shared/runtime/pyexec.h +++ b/shared/runtime/pyexec.h @@ -28,6 +28,8 @@ #include "py/obj.h" +// CIRCUITPY multiple changes + typedef enum { PYEXEC_MODE_FRIENDLY_REPL, PYEXEC_MODE_RAW_REPL, diff --git a/shared/runtime/stdout_helpers.c b/shared/runtime/stdout_helpers.c index 500b9748be..c88b496517 100644 --- a/shared/runtime/stdout_helpers.c +++ b/shared/runtime/stdout_helpers.c @@ -1,3 +1,29 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2021 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + #include #include #include "py/mpconfig.h" @@ -9,6 +35,7 @@ * implementation below can be used. */ +// CIRCUITPY changes // Send "cooked" string of given length, where every occurrence of // LF character is replaced with CR LF. void mp_hal_stdout_tx_strn_cooked(const char *str, size_t len) { diff --git a/shared/runtime/sys_stdio_mphal.c b/shared/runtime/sys_stdio_mphal.c index 236c6f3eb3..84ce5828ef 100644 --- a/shared/runtime/sys_stdio_mphal.c +++ b/shared/runtime/sys_stdio_mphal.c @@ -89,19 +89,14 @@ STATIC mp_uint_t stdio_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, (void)self_in; if (request == MP_STREAM_POLL) { return mp_hal_stdio_poll(arg); + } else if (request == MP_STREAM_CLOSE) { + return 0; } else { *errcode = MP_EINVAL; return MP_STREAM_ERROR; } } -STATIC mp_obj_t stdio_obj___exit__(size_t n_args, const mp_obj_t *args) { - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(stdio_obj___exit___obj, 4, 4, stdio_obj___exit__); - -// TODO gc hook to close the file if not already closed - STATIC const mp_rom_map_elem_t stdio_locals_dict_table[] = { #if MICROPY_PY_SYS_STDIO_BUFFER { MP_ROM_QSTR(MP_QSTR_buffer), MP_ROM_PTR(&stdio_buffer_obj) }, @@ -112,9 +107,8 @@ STATIC const mp_rom_map_elem_t stdio_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_readlines), MP_ROM_PTR(&mp_stream_unbuffered_readlines_obj)}, { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_identity_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&stdio_obj___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, }; STATIC MP_DEFINE_CONST_DICT(stdio_locals_dict, stdio_locals_dict_table); diff --git a/shared/timeutils/timeutils.c b/shared/timeutils/timeutils.c index d1a10a4c93..6bf3eca84a 100644 --- a/shared/timeutils/timeutils.c +++ b/shared/timeutils/timeutils.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2015 Daniel Campora + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2015 Daniel Campora * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/shared/timeutils/timeutils.h b/shared/timeutils/timeutils.h index f82e57fe96..9f4b500caa 100644 --- a/shared/timeutils/timeutils.h +++ b/shared/timeutils/timeutils.h @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2015 Daniel Campora + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2015 Daniel Campora * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/shared/tinyusb/mp_cdc_common.c b/shared/tinyusb/mp_cdc_common.c index cd4f5d1013..4ccae4bac1 100644 --- a/shared/tinyusb/mp_cdc_common.c +++ b/shared/tinyusb/mp_cdc_common.c @@ -39,7 +39,13 @@ STATIC void usbd_cdc_run_bootloader_task(mp_sched_node_t *node) { machine_bootloader(0, NULL); } -void tud_cdc_line_state_cb(uint8_t itf, bool dtr, bool rts) { +void +#if MICROPY_HW_USB_EXTERNAL_TINYUSB +mp_usbd_line_state_cb +#else +tud_cdc_line_state_cb +#endif + (uint8_t itf, bool dtr, bool rts) { if (dtr == false && rts == false) { // Device is disconnected. cdc_line_coding_t line_coding; diff --git a/shared/tinyusb/mp_usbd_descriptor.c b/shared/tinyusb/mp_usbd_descriptor.c index 8fab599b67..72a2652179 100644 --- a/shared/tinyusb/mp_usbd_descriptor.c +++ b/shared/tinyusb/mp_usbd_descriptor.c @@ -34,7 +34,7 @@ #include "mp_usbd_internal.h" #define USBD_CDC_CMD_MAX_SIZE (8) -#define USBD_CDC_IN_OUT_MAX_SIZE (64) +#define USBD_CDC_IN_OUT_MAX_SIZE ((CFG_TUD_MAX_SPEED == OPT_MODE_HIGH_SPEED) ? 512 : 64) const tusb_desc_device_t mp_usbd_desc_device_static = { .bLength = sizeof(tusb_desc_device_t), diff --git a/shared/tinyusb/tusb_config.h b/shared/tinyusb/tusb_config.h index 28bee09a59..266cb88cc2 100644 --- a/shared/tinyusb/tusb_config.h +++ b/shared/tinyusb/tusb_config.h @@ -43,7 +43,21 @@ #define MICROPY_HW_USB_CDC_INTERFACE_STRING "Board CDC" #endif +#ifndef MICROPY_HW_USB_MSC_INQUIRY_VENDOR_STRING +#define MICROPY_HW_USB_MSC_INQUIRY_VENDOR_STRING "MicroPy" +#endif + +#ifndef MICROPY_HW_USB_MSC_INQUIRY_PRODUCT_STRING +#define MICROPY_HW_USB_MSC_INQUIRY_PRODUCT_STRING "Mass Storage" +#endif + +#ifndef MICROPY_HW_USB_MSC_INQUIRY_REVISION_STRING +#define MICROPY_HW_USB_MSC_INQUIRY_REVISION_STRING "1.00" +#endif + +#ifndef CFG_TUSB_RHPORT0_MODE #define CFG_TUSB_RHPORT0_MODE (OPT_MODE_DEVICE) +#endif #if MICROPY_HW_USB_CDC #define CFG_TUD_CDC (1) @@ -59,8 +73,8 @@ // CDC Configuration #if CFG_TUD_CDC -#define CFG_TUD_CDC_RX_BUFSIZE (256) -#define CFG_TUD_CDC_TX_BUFSIZE (256) +#define CFG_TUD_CDC_RX_BUFSIZE ((CFG_TUD_MAX_SPEED == OPT_MODE_HIGH_SPEED) ? 512 : 256) +#define CFG_TUD_CDC_TX_BUFSIZE ((CFG_TUD_MAX_SPEED == OPT_MODE_HIGH_SPEED) ? 512 : 256) #endif // MSC Configuration diff --git a/shared/upytesthelper/upytesthelper.c b/shared/upytesthelper/upytesthelper.c index ce60732424..ba20037f7a 100644 --- a/shared/upytesthelper/upytesthelper.c +++ b/shared/upytesthelper/upytesthelper.c @@ -31,6 +31,14 @@ #include "py/compile.h" #include "upytesthelper.h" +#if !MICROPY_PY_SYS_PATH +#error "upytesthelper requires MICROPY_PY_SYS_PATH=1" +#endif + +#if !MICROPY_PY_SYS_ARGV +#error "upytesthelper requires MICROPY_PY_SYS_ARGV=1" +#endif + static const char *test_exp_output; static int test_exp_output_len, test_rem_output_len; static int test_failed; @@ -60,8 +68,8 @@ bool upytest_is_failed(void) { } // MP_PLAT_PRINT_STRN() should be redirected to this function. -// It will pass-thru any content to mp_hal_stdout_tx_strn_cooked() -// (the dfault value of MP_PLAT_PRINT_STRN), but will also match +// It will pass-through any content to mp_hal_stdout_tx_strn_cooked() +// (the default value of MP_PLAT_PRINT_STRN), but will also match // it to the expected output as set by upytest_set_expected_output(). // If mismatch happens, upytest_is_failed() returns true. void upytest_output(const char *str, mp_uint_t len) { @@ -93,7 +101,7 @@ void upytest_execute_test(const char *src) { // reinitialized before running each. gc_init(heap_start, heap_end); mp_init(); - mp_obj_list_init(mp_sys_path, 0); + mp_sys_path = mp_obj_new_list(0, NULL); #if MICROPY_MODULE_FROZEN mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); #endif diff --git a/tests/README.md b/tests/README.md index b0bff4872e..c89930d0c0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -147,3 +147,30 @@ the test runs, and the absolute difference value is unreliable. High error percentages are particularly common on PC builds, where the host OS may influence test run times. Increasing the `N` value may help average this out by running each test longer. + +## internal_bench + +The `internal_bench` directory contains a set of tests for benchmarking +different internal Python features. By default, tests are run on the (unix or +Windows) host, but the `--pyboard` option allows them to be run on an attached +board instead. + +Tests are grouped by the first part of the file name, and the test runner compares +output between each group of tests. + +The benchmarks measure the elapsed (wall time) for each test, according +to MicroPython's own time module. + +If run without any arguments, all test groups are run. Otherwise, it's possible +to manually specify which test cases to run. + +Example: + +``` +$ ./run-internalbench.py internal_bench/bytebuf-*.py +internal_bench/bytebuf: + 0.094s (+00.00%) internal_bench/bytebuf-1-inplace.py + 0.471s (+399.24%) internal_bench/bytebuf-2-join_map_bytes.py + 0.177s (+87.78%) internal_bench/bytebuf-3-bytarray_map.py +1 tests performed (3 individual testcases) +``` diff --git a/tests/basics/array_construct.py b/tests/basics/array_construct.py index 4985244d13..2221de9906 100644 --- a/tests/basics/array_construct.py +++ b/tests/basics/array_construct.py @@ -1,13 +1,10 @@ # test construction of array.array from different objects try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit # tuple, list print(array('b', (1, 2))) diff --git a/tests/basics/array_construct2.py b/tests/basics/array_construct2.py index d1b1e9d158..c305b7f011 100644 --- a/tests/basics/array_construct2.py +++ b/tests/basics/array_construct2.py @@ -1,11 +1,8 @@ try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit # construct from something with unknown length (requires generators) print(array('i', (i for i in range(10)))) diff --git a/tests/basics/array_construct_endian.py b/tests/basics/array_construct_endian.py index 82a962fbe0..990d7b1ea0 100644 --- a/tests/basics/array_construct_endian.py +++ b/tests/basics/array_construct_endian.py @@ -1,13 +1,10 @@ # test construction of array.array from different objects try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit # raw copy from bytes, bytearray print(array('h', b'12')) diff --git a/tests/basics/array_intbig.py b/tests/basics/array_intbig.py index a3b24c2cdb..ef3b567935 100644 --- a/tests/basics/array_intbig.py +++ b/tests/basics/array_intbig.py @@ -3,13 +3,10 @@ import skip_if skip_if.no_bigint() try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit print(array('L', [0, 2**32-1])) print(array('l', [-2**31, 0, 2**31-1])) diff --git a/tests/basics/async_for.py b/tests/basics/async_for.py index 5fd0540828..f54f70238c 100644 --- a/tests/basics/async_for.py +++ b/tests/basics/async_for.py @@ -1,29 +1,75 @@ # test basic async for execution # example taken from PEP0492 + class AsyncIteratorWrapper: def __init__(self, obj): - print('init') - self._it = iter(obj) + print("init") + self._obj = obj + + def __repr__(self): + return "AsyncIteratorWrapper-" + self._obj def __aiter__(self): - print('aiter') - return self + print("aiter") + return AsyncIteratorWrapperIterator(self._obj) + + +class AsyncIteratorWrapperIterator: + def __init__(self, obj): + print("init") + self._it = iter(obj) async def __anext__(self): - print('anext') + print("anext") try: value = next(self._it) except StopIteration: raise StopAsyncIteration return value -async def coro(): - async for letter in AsyncIteratorWrapper('abc'): + +def run_coro(c): + print("== start ==") + try: + c.send(None) + except StopIteration: + print("== finish ==") + + +async def coro0(): + async for letter in AsyncIteratorWrapper("abc"): print(letter) -o = coro() -try: - o.send(None) -except StopIteration: - print('finished') + +run_coro(coro0()) + + +async def coro1(): + a = AsyncIteratorWrapper("def") + async for letter in a: + print(letter) + print(a) + + +run_coro(coro1()) + +a_global = AsyncIteratorWrapper("ghi") + + +async def coro2(): + async for letter in a_global: + print(letter) + print(a_global) + + +run_coro(coro2()) + + +async def coro3(a): + async for letter in a: + print(letter) + print(a) + + +run_coro(coro3(AsyncIteratorWrapper("jkl"))) diff --git a/tests/basics/async_for.py.exp b/tests/basics/async_for.py.exp index 1f728a66c8..6f59979c06 100644 --- a/tests/basics/async_for.py.exp +++ b/tests/basics/async_for.py.exp @@ -1,5 +1,7 @@ +== start == init aiter +init anext a anext @@ -7,4 +9,43 @@ b anext c anext -finished +== finish == +== start == +init +aiter +init +anext +d +anext +e +anext +f +anext +AsyncIteratorWrapper-def +== finish == +init +== start == +aiter +init +anext +g +anext +h +anext +i +anext +AsyncIteratorWrapper-ghi +== finish == +init +== start == +aiter +init +anext +j +anext +k +anext +l +anext +AsyncIteratorWrapper-jkl +== finish == diff --git a/tests/basics/builtin_compile.py b/tests/basics/builtin_compile.py index a2f2cbe550..41dc746ad1 100644 --- a/tests/basics/builtin_compile.py +++ b/tests/basics/builtin_compile.py @@ -42,4 +42,7 @@ def test(): print("NameError") print(x) # check 'x' still exists as a global + # hashing a compiled function object + print(type(hash(compile("", "", "exec")))) + test() diff --git a/tests/basics/builtin_hash.py b/tests/basics/builtin_hash.py index 704895fbb6..89e1cfb47a 100644 --- a/tests/basics/builtin_hash.py +++ b/tests/basics/builtin_hash.py @@ -5,6 +5,24 @@ print(hash(True)) print({():1}) # hash tuple print({(1,):1}) # hash non-empty tuple print(hash in {hash:1}) # hash function +print(type(hash(list.pop))) # hash checked function (mp_type_checked_fun) +print(type(hash([].pop))) # hash bound method +print(type(hash(object()))) # hash object instance +print(type(hash(super(object, object)))) # hash super +print(type(hash(classmethod(hash)))) # hash classmethod +print(type(hash(staticmethod(hash)))) # hash staticmethod +print(type(hash(iter("")))) # hash string iterator +print(type(hash(iter(b"")))) # hash bytes iterator +print(type(hash(iter(range(0))))) # hash range iterator +print(type(hash(map(None, [])))) # hash map object +print(type(hash(zip([])))) # hash zip object + +def f(x): + def g(): + return x + return g + +print(type(hash(f(1)))) # hash closure try: hash([]) diff --git a/tests/basics/bytearray_construct_array.py b/tests/basics/bytearray_construct_array.py index 52eaa7c6ef..bde5fa08bd 100644 --- a/tests/basics/bytearray_construct_array.py +++ b/tests/basics/bytearray_construct_array.py @@ -1,12 +1,9 @@ # test construction of bytearray from different objects try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit # arrays print(bytearray(array('b', [1, 2]))) diff --git a/tests/basics/bytearray_construct_endian.py b/tests/basics/bytearray_construct_endian.py index 332b43e686..0002f19c5f 100644 --- a/tests/basics/bytearray_construct_endian.py +++ b/tests/basics/bytearray_construct_endian.py @@ -1,12 +1,9 @@ # test construction of bytearray from different objects try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit # arrays print(bytearray(array('h', [1, 2]))) diff --git a/tests/basics/bytes_construct_array.py b/tests/basics/bytes_construct_array.py index 7bdd8f10df..453eb59010 100644 --- a/tests/basics/bytes_construct_array.py +++ b/tests/basics/bytes_construct_array.py @@ -1,12 +1,9 @@ # test construction of bytes from different objects try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit # arrays print(bytes(array('b', [1, 2]))) diff --git a/tests/basics/bytes_construct_endian.py b/tests/basics/bytes_construct_endian.py index 294c5f23f5..cf1a9f408f 100644 --- a/tests/basics/bytes_construct_endian.py +++ b/tests/basics/bytes_construct_endian.py @@ -1,13 +1,10 @@ # test construction of bytes from different objects try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit # arrays print(bytes(array('h', [1, 2]))) diff --git a/tests/basics/class_ordereddict.py b/tests/basics/class_ordereddict.py index 4dd25eb6e1..03a211ddad 100644 --- a/tests/basics/class_ordereddict.py +++ b/tests/basics/class_ordereddict.py @@ -1,13 +1,10 @@ # test using an OrderedDict as the locals to construct a class try: - from ucollections import OrderedDict + from collections import OrderedDict except ImportError: - try: - from collections import OrderedDict - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit if not hasattr(int, "__dict__"): print("SKIP") diff --git a/tests/basics/class_reverse_op.py b/tests/basics/class_reverse_op.py index e157661c9a..a00928818b 100644 --- a/tests/basics/class_reverse_op.py +++ b/tests/basics/class_reverse_op.py @@ -17,5 +17,38 @@ class A: def __repr__(self): return "A({})".format(self.v) + print(A(3) + 1) print(2 + A(5)) + + +# Test user type with strings. +class B: + def __init__(self, v): + self.v = v + + def __repr__(self): + return "B({})".format(self.v) + + def __ror__(self, o): + return B(o + "|" + self.v) + + def __radd__(self, o): + return B(o + "+" + self.v) + + def __rmul__(self, o): + return B(o + "*" + self.v) + + def __rtruediv__(self, o): + return B(o + "/" + self.v) + + +print("a" | B("b")) +print("a" + B("b")) +print("a" * B("b")) +print("a" / B("b")) + +x = "a"; x |= B("b"); print(x) +x = "a"; x += B("b"); print(x) +x = "a"; x *= B("b"); print(x) +x = "a"; x /= B("b"); print(x) diff --git a/tests/basics/deque1.py b/tests/basics/deque1.py index 19966fcb07..8b7874e2b1 100644 --- a/tests/basics/deque1.py +++ b/tests/basics/deque1.py @@ -1,8 +1,5 @@ try: - try: - from ucollections import deque - except ImportError: - from collections import deque + from collections import deque except ImportError: print("SKIP") raise SystemExit diff --git a/tests/basics/deque2.py b/tests/basics/deque2.py index 22d370e943..80fcd66785 100644 --- a/tests/basics/deque2.py +++ b/tests/basics/deque2.py @@ -1,10 +1,7 @@ # Tests for deques with "check overflow" flag and other extensions # wrt to CPython. try: - try: - from ucollections import deque - except ImportError: - from collections import deque + from collections import deque except ImportError: print("SKIP") raise SystemExit diff --git a/tests/basics/dict_views.py b/tests/basics/dict_views.py index 7ebcc1f56d..a82f47b6be 100644 --- a/tests/basics/dict_views.py +++ b/tests/basics/dict_views.py @@ -18,4 +18,22 @@ try: except TypeError: print('TypeError') +# keys dict_view is not hashable + +try: + hash({}.keys()) +except TypeError: + print('TypeError') + +# values dict_view is hashable + +print(type(hash({}.values()))) + +# items dict_view is not hashable + +try: + hash({}.items()) +except TypeError: + print('TypeError') + # set operations still to come diff --git a/tests/basics/ifcond.py b/tests/basics/ifcond.py index 9264aa74dd..5eba143420 100644 --- a/tests/basics/ifcond.py +++ b/tests/basics/ifcond.py @@ -57,6 +57,20 @@ if not (1,): else: print('b') +# test evaluation of the if-condition with tuples as arguments +# non-constant tuples should be evaluated even though they will evaluate to true + +def f(x): + print("f", x) + +if (f(1),): + print(18) + +if (f(2), f(3)): + print(19) + +# test if-conditions within a function + f2 = 0 def f(t1, t2, f1): diff --git a/tests/basics/int_big1.py b/tests/basics/int_big1.py index f6e4b61d1d..de2ba9fc4f 100644 --- a/tests/basics/int_big1.py +++ b/tests/basics/int_big1.py @@ -10,6 +10,9 @@ print(y) print('%#X' % (x - x)) # print prefix print('{:#,}'.format(x)) # print with commas +# construction +print(int(x)) + # addition print(x + 1) print(x + y) diff --git a/tests/basics/int_parse.py b/tests/basics/int_parse.py new file mode 100644 index 0000000000..cc41158c59 --- /dev/null +++ b/tests/basics/int_parse.py @@ -0,0 +1,12 @@ +# Test parsing ints. + +try: + bytearray + memoryview +except NameError: + print("SKIP") + raise SystemExit + +print(int(b"123")) +print(int(bytearray(b"123"))) +print(int(memoryview(b"123"))) diff --git a/tests/basics/io_buffered_writer.py b/tests/basics/io_buffered_writer.py index b3bab54900..11df0324e2 100644 --- a/tests/basics/io_buffered_writer.py +++ b/tests/basics/io_buffered_writer.py @@ -25,3 +25,6 @@ bts = io.BytesIO() buf = io.BufferedWriter(bts, 1) buf.write(b"foo") print(bts.getvalue()) + +# hashing a BufferedWriter +print(type(hash(buf))) diff --git a/tests/basics/io_buffered_writer.py.exp b/tests/basics/io_buffered_writer.py.exp index d086935a92..2209348f5a 100644 --- a/tests/basics/io_buffered_writer.py.exp +++ b/tests/basics/io_buffered_writer.py.exp @@ -3,3 +3,4 @@ b'foobarfo' b'foobarfoobar' b'foobarfoobar' b'foo' + diff --git a/tests/basics/io_bytesio_cow.py b/tests/basics/io_bytesio_cow.py index 543dd484ab..2edb7136a9 100644 --- a/tests/basics/io_bytesio_cow.py +++ b/tests/basics/io_bytesio_cow.py @@ -1,7 +1,6 @@ # Make sure that write operations on io.BytesIO don't # change original object it was constructed from. import io - b = b"foobar" a = io.BytesIO(b) diff --git a/tests/basics/io_bytesio_ext.py b/tests/basics/io_bytesio_ext.py index eee20baf3b..4d4c60c136 100644 --- a/tests/basics/io_bytesio_ext.py +++ b/tests/basics/io_bytesio_ext.py @@ -1,6 +1,5 @@ # Extended stream operations on io.BytesIO import io - a = io.BytesIO(b"foobar") a.seek(10) print(a.read(10)) diff --git a/tests/basics/io_bytesio_ext2.py b/tests/basics/io_bytesio_ext2.py index 5bede33f58..414ac90a3b 100644 --- a/tests/basics/io_bytesio_ext2.py +++ b/tests/basics/io_bytesio_ext2.py @@ -1,5 +1,4 @@ import io - a = io.BytesIO(b"foobar") try: a.seek(-10) diff --git a/tests/basics/io_iobase.py b/tests/basics/io_iobase.py index 6d27fe1e74..b2ee5cd63a 100644 --- a/tests/basics/io_iobase.py +++ b/tests/basics/io_iobase.py @@ -1,5 +1,4 @@ import io - try: io.IOBase except AttributeError: @@ -13,5 +12,4 @@ class MyIO(io.IOBase): print("write", len(buf)) return len(buf) - print("test", file=MyIO()) diff --git a/tests/basics/io_stringio1.py b/tests/basics/io_stringio1.py index 2174db00cd..7d355930f5 100644 --- a/tests/basics/io_stringio1.py +++ b/tests/basics/io_stringio1.py @@ -1,7 +1,6 @@ import io - a = io.StringIO() -print("io.StringIO" in repr(a)) +print('io.StringIO' in repr(a)) print(a.getvalue()) print(a.read()) @@ -37,7 +36,7 @@ print(a.tell()) a = io.StringIO() a.close() -for f in [a.read, a.getvalue, lambda: a.write("")]: +for f in [a.read, a.getvalue, lambda:a.write("")]: # CPython throws for operations on closed I/O, MicroPython makes # the underlying string empty unless MICROPY_CPYTHON_COMPAT defined try: diff --git a/tests/basics/io_stringio_base.py b/tests/basics/io_stringio_base.py index dffc879074..0f65fb3fab 100644 --- a/tests/basics/io_stringio_base.py +++ b/tests/basics/io_stringio_base.py @@ -1,10 +1,7 @@ # Checks that an instance type inheriting from a native base that uses # MP_TYPE_FLAG_ITER_IS_STREAM will still have a getiter. -try: - import uio as io -except ImportError: - import io +import io a = io.StringIO() a.write("hello\nworld\nmicro\npython\n") diff --git a/tests/basics/io_stringio_with.py b/tests/basics/io_stringio_with.py index b30395672c..a3aa6ec84e 100644 --- a/tests/basics/io_stringio_with.py +++ b/tests/basics/io_stringio_with.py @@ -1,5 +1,4 @@ import io - # test __enter__/__exit__ with io.StringIO() as b: b.write("foo") diff --git a/tests/basics/list_mult.py b/tests/basics/list_mult.py index 548f88534e..125c548eec 100644 --- a/tests/basics/list_mult.py +++ b/tests/basics/list_mult.py @@ -11,6 +11,16 @@ a = [1, 2, 3] c = a * 3 print(a, c) +# check inplace multiplication +a = [4, 5, 6] +a *= 3 +print(a) + +# check reverse inplace multiplication +a = 3 +a *= [7, 8, 9] +print(a) + # unsupported type on RHS try: [] * None diff --git a/tests/basics/memoryview2.py b/tests/basics/memoryview2.py index eacc227c28..fa5514e072 100644 --- a/tests/basics/memoryview2.py +++ b/tests/basics/memoryview2.py @@ -5,13 +5,10 @@ except: print("SKIP") raise SystemExit try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit print(list(memoryview(b'\x7f\x80\x81\xff'))) print(list(memoryview(array('b', [0x7f, -0x80])))) diff --git a/tests/basics/memoryview_intbig.py b/tests/basics/memoryview_intbig.py index 4800a70cc2..72951d6eaa 100644 --- a/tests/basics/memoryview_intbig.py +++ b/tests/basics/memoryview_intbig.py @@ -5,13 +5,10 @@ except: print("SKIP") raise SystemExit try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit print(list(memoryview(array('i', [0x7f000000, -0x80000000])))) print(list(memoryview(array('I', [0x7f000000, 0x80000000, 0x81000000, 0xffffffff])))) diff --git a/tests/basics/memoryview_itemsize.py b/tests/basics/memoryview_itemsize.py index 64a8822b8b..5428a41785 100644 --- a/tests/basics/memoryview_itemsize.py +++ b/tests/basics/memoryview_itemsize.py @@ -4,13 +4,10 @@ except: print("SKIP") raise SystemExit try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit for code in ['b', 'h', 'i', 'q', 'f', 'd']: print(memoryview(array(code)).itemsize) diff --git a/tests/basics/namedtuple1.py b/tests/basics/namedtuple1.py index eee8a3909c..362c60583e 100644 --- a/tests/basics/namedtuple1.py +++ b/tests/basics/namedtuple1.py @@ -68,6 +68,11 @@ try: except TypeError: print("TypeError") +# Try single string +T3 = namedtuple("TupComma", "foo bar") +t = T3(1, 2) +print(t.foo, t.bar) + # Try tuple T4 = namedtuple("TupTuple", ("foo", "bar")) t = T4(1, 2) diff --git a/tests/basics/namedtuple_asdict.py b/tests/basics/namedtuple_asdict.py index 34c4e6f713..e85281bb73 100644 --- a/tests/basics/namedtuple_asdict.py +++ b/tests/basics/namedtuple_asdict.py @@ -1,8 +1,5 @@ try: - try: - from ucollections import namedtuple - except ImportError: - from collections import namedtuple + from collections import namedtuple except ImportError: print("SKIP") raise SystemExit diff --git a/tests/basics/op_error_memoryview.py b/tests/basics/op_error_memoryview.py index 233f7f9ab7..4853704289 100644 --- a/tests/basics/op_error_memoryview.py +++ b/tests/basics/op_error_memoryview.py @@ -6,8 +6,18 @@ except: raise SystemExit # unsupported binary operators +try: + memoryview(b"") + b"" +except TypeError: + print("TypeError") + +try: + memoryview(b"") + memoryview(b"") +except TypeError: + print("TypeError") + try: m = memoryview(bytearray()) m += bytearray() except TypeError: - print('TypeError') + print("TypeError") diff --git a/tests/basics/slice_op.py b/tests/basics/slice_op.py new file mode 100644 index 0000000000..f1e83c5e27 --- /dev/null +++ b/tests/basics/slice_op.py @@ -0,0 +1,15 @@ + +try: + t = [][:] +except: + print("SKIP") + raise SystemExit + + +# REVISIT: slice comparison operators are not implemented in MicroPython + +# test that slice is not hashable, i.e. it can't be used to copy a dict +try: + {}[:] = {} +except TypeError: + print('TypeError') diff --git a/tests/basics/special_methods.py b/tests/basics/special_methods.py index d8af8e0797..4afc60f3f3 100644 --- a/tests/basics/special_methods.py +++ b/tests/basics/special_methods.py @@ -37,9 +37,11 @@ class Cud(): def __floordiv__(self, other): print("__floordiv__ called") + # CIRCUITPY def __index__(self, other): print("__index__ called") + # CIRCUITPY def __inv__(self): print("__inv__ called") diff --git a/tests/basics/special_methods2.py b/tests/basics/special_methods2.py index 31f330ab42..8a47690b2e 100644 --- a/tests/basics/special_methods2.py +++ b/tests/basics/special_methods2.py @@ -38,12 +38,6 @@ class Cud(): def __floordiv__(self, other): print("__floordiv__ called") - def __index__(self, other): - print("__index__ called") - - def __inv__(self): - print("__inv__ called") - def __invert__(self): print("__invert__ called") diff --git a/tests/basics/special_methods_intbig.py b/tests/basics/special_methods_intbig.py new file mode 100644 index 0000000000..653422f213 --- /dev/null +++ b/tests/basics/special_methods_intbig.py @@ -0,0 +1,8 @@ +# Test class special methods, that use a bigint. + +class A: + def __int__(self): + return 1 << 100 + + +print(int(A())) diff --git a/tests/basics/string_format_modulo.py b/tests/basics/string_format_modulo.py index 01f8e7ed24..7bddd96750 100644 --- a/tests/basics/string_format_modulo.py +++ b/tests/basics/string_format_modulo.py @@ -13,6 +13,12 @@ print("=%s=" % [1, 2]) print("=%s=" % "str") print("=%r=" % "str") +# test calling __int__ +class A: + def __int__(self): + return 123 +print("%d" % A()) + try: print("=%s=%s=" % 1) except TypeError: @@ -45,8 +51,9 @@ print('%c' % True) # Should be able to print dicts; in this case they aren't used # to lookup keywords in formats like %(foo)s -print('%s' % {}) -print('%s' % ({},)) +print('%s' % {}) # dict treated as the single (positional) arg to % +print('%s' % ({},)) # dict is the first (and only) arg in the positional arg tuple +print('foo' % {}) # no error, dict treated as an empty map of named args # Cases when "*" used and there's not enough values total try: @@ -59,7 +66,11 @@ except TypeError: print("TypeError") print("%(foo)s" % {"foo": "bar", "baz": False}) -print("%s %(foo)s %(foo)s" % {"foo": 1}) +print("%s %(foo)s %(foo)s" % {"foo": 1}) # dict consumed positionally, then used as map - ok +try: + print("%(foo)s %s %(foo)s" % {"foo": 1}) # used as map, then positionally - not enough args +except TypeError: + print("TypeError") try: print("%(foo)s" % {}) except KeyError: diff --git a/tests/basics/string_fstring.py b/tests/basics/string_fstring.py index 8907a5c478..1a3960680e 100644 --- a/tests/basics/string_fstring.py +++ b/tests/basics/string_fstring.py @@ -61,3 +61,22 @@ except (ValueError, SyntaxError): print(f"a {1,} b") print(f"a {x,y,} b") print(f"a {x,1} b") + +# f-strings with conversion specifiers (only support !r and !s). +a = "123" +print(f"{a!r}") +print(f"{a!s}") +try: + eval('print(f"{a!x}")') +except (ValueError, SyntaxError): + # CPython detects this at compile time, MicroPython fails with ValueError + # when the str.format is executed. + print("ValueError") + +# Mixing conversion specifiers with formatting. +print(f"{a!r:8s}") +print(f"{a!s:8s}") + +# Still allow ! in expressions. +print(f"{'1' if a != '456' else '0'!r:8s}") +print(f"{'1' if a != '456' else '0'!s:8s}") diff --git a/tests/basics/string_mult.py b/tests/basics/string_mult.py index c0713c1d3a..5a7d822947 100644 --- a/tests/basics/string_mult.py +++ b/tests/basics/string_mult.py @@ -10,3 +10,13 @@ for i in (-4, -2, 0, 2, 4): a = '123' c = a * 3 print(a, c) + +# check inplace multiplication +a = '456' +a *= 3 +print(a) + +# check reverse inplace multiplication +a = 3 +a *= '789' +print(a) diff --git a/tests/basics/struct1.py b/tests/basics/struct1.py index f6c521d6ce..4342e49640 100644 --- a/tests/basics/struct1.py +++ b/tests/basics/struct1.py @@ -20,6 +20,8 @@ print(struct.pack("h", 1)) print(struct.pack("b", 1)) +print(struct.pack("x")) print(struct.pack("bI", -128, 256)) @@ -29,6 +31,13 @@ print(struct.calcsize("97sI")) print(struct.unpack("<6sH", b"foo\0\0\0\x12\x34")) print(struct.pack("<6sH", b"foo", 10000)) +print(struct.calcsize("7xx")) +print(struct.pack("7xx")) + +print(struct.calcsize(">bxI3xH")) +print(struct.pack(">bxI3xH", 1, 2, 3)) +print(struct.unpack(">bxI3xH", b"\x01\0\0\0\0\x02\0\0\0\0\x03")) + s = struct.pack("BHBI", 10, 100, 200, 300) v = struct.unpack("BHBI", s) print(v == (10, 100, 200, 300)) @@ -116,6 +125,7 @@ try: except: print('struct.error') +# CIRCUITPY # check padding bytes print(struct.pack("xb", 3)) # Make sure pack doesn't reuse a larger value and error diff --git a/tests/basics/struct2.py b/tests/basics/struct2.py index 5c535fc432..9cbf5ad867 100644 --- a/tests/basics/struct2.py +++ b/tests/basics/struct2.py @@ -1,4 +1,4 @@ -# test ustruct with a count specified before the type +# test struct with a count specified before the type try: import struct diff --git a/tests/basics/struct_endian.py b/tests/basics/struct_endian.py index 5c3181b092..6eabda0188 100644 --- a/tests/basics/struct_endian.py +++ b/tests/basics/struct_endian.py @@ -1,4 +1,4 @@ -# test ustruct and endian specific things +# test struct and endian specific things try: import struct diff --git a/tests/basics/subclass_native_exc_new.py b/tests/basics/subclass_native_exc_new.py index c1bd89a6fa..a431392eaa 100644 --- a/tests/basics/subclass_native_exc_new.py +++ b/tests/basics/subclass_native_exc_new.py @@ -29,7 +29,6 @@ except Exception as bad: print(type(bad), bad.args[0]) try: - def gen(): yield diff --git a/tests/basics/sys1.py b/tests/basics/sys1.py index c8dbfc3541..6c427dffb6 100644 --- a/tests/basics/sys1.py +++ b/tests/basics/sys1.py @@ -1,7 +1,6 @@ # test sys module import sys - print(sys.__name__) print(type(sys.path)) print(type(sys.argv)) @@ -14,6 +13,7 @@ except AttributeError: print(True) try: + # CIRCUITPY print(sys.implementation.name in ('cpython', 'micropython', 'circuitpython')) except AttributeError: # Effectively skip subtests diff --git a/tests/basics/sys_exit.py b/tests/basics/sys_exit.py index b1f71549db..4994af4161 100644 --- a/tests/basics/sys_exit.py +++ b/tests/basics/sys_exit.py @@ -1,7 +1,6 @@ # test sys module's exit function import sys - try: sys.exit except AttributeError: diff --git a/tests/basics/sys_getsizeof.py b/tests/basics/sys_getsizeof.py index fe1b403e04..779e992d1f 100644 --- a/tests/basics/sys_getsizeof.py +++ b/tests/basics/sys_getsizeof.py @@ -16,7 +16,7 @@ print(sys.getsizeof(A()) > 0) # Only test deque if we have it try: - from ucollections import deque + from collections import deque assert sys.getsizeof(deque((), 1)) > 0 except ImportError: pass diff --git a/tests/basics/sys_path.py b/tests/basics/sys_path.py index ede1fd3a8f..576bd66c94 100644 --- a/tests/basics/sys_path.py +++ b/tests/basics/sys_path.py @@ -1,7 +1,6 @@ # test sys.path import sys - # check that this script was executed from a file of the same name if "__file__" not in globals() or "sys_path.py" not in __file__: print("SKIP") diff --git a/tests/cpydiff/core_fstring_repr.py b/tests/cpydiff/core_fstring_repr.py index df80abf795..d37fb48db7 100644 --- a/tests/cpydiff/core_fstring_repr.py +++ b/tests/cpydiff/core_fstring_repr.py @@ -1,18 +1,8 @@ """ categories: Core -description: f-strings don't support the !r, !s, and !a conversions -cause: MicroPython is optimised for code space. -workaround: Use repr(), str(), and ascii() explicitly. +description: f-strings don't support !a conversions +cause: MicropPython does not implement ascii() +workaround: None """ - -class X: - def __repr__(self): - return "repr" - - def __str__(self): - return "str" - - -print(f"{X()!r}") -print(f"{X()!s}") +f"{'unicode text'!a}" diff --git a/tests/cpydiff/core_import_prereg.py b/tests/cpydiff/core_import_prereg.py deleted file mode 100644 index 3ce2340c68..0000000000 --- a/tests/cpydiff/core_import_prereg.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -categories: Core,import -description: Failed to load modules are still registered as loaded -cause: To make module handling more efficient, it's not wrapped with exception handling. -workaround: Test modules before production use; during development, use ``del sys.modules["name"]``, or just soft or hard reset the board. -""" -import sys - -try: - from modules import foo -except NameError as e: - print(e) -try: - from modules import foo - - print("Should not get here") -except NameError as e: - print(e) diff --git a/tests/extmod/asyncio_as_uasyncio.py b/tests/extmod/asyncio_as_uasyncio.py new file mode 100644 index 0000000000..612292299c --- /dev/null +++ b/tests/extmod/asyncio_as_uasyncio.py @@ -0,0 +1,12 @@ +try: + import uasyncio + import asyncio +except ImportError: + print("SKIP") + raise SystemExit + +x = set(dir(uasyncio)) +y = set(dir(asyncio)) - set(["event", "lock", "stream", "funcs"]) + +print(x - y) +print(y - x) diff --git a/tests/extmod/asyncio_as_uasyncio.py.exp b/tests/extmod/asyncio_as_uasyncio.py.exp new file mode 100644 index 0000000000..9405b80109 --- /dev/null +++ b/tests/extmod/asyncio_as_uasyncio.py.exp @@ -0,0 +1,2 @@ +set() +set() diff --git a/tests/extmod/uasyncio_await_return.py b/tests/extmod/asyncio_await_return.py similarity index 100% rename from tests/extmod/uasyncio_await_return.py rename to tests/extmod/asyncio_await_return.py diff --git a/tests/extmod/uasyncio_await_return.py.exp b/tests/extmod/asyncio_await_return.py.exp similarity index 100% rename from tests/extmod/uasyncio_await_return.py.exp rename to tests/extmod/asyncio_await_return.py.exp diff --git a/tests/extmod/uasyncio_basic.py b/tests/extmod/asyncio_basic.py similarity index 91% rename from tests/extmod/uasyncio_basic.py rename to tests/extmod/asyncio_basic.py index 148a361b4b..584e10c131 100644 --- a/tests/extmod/uasyncio_basic.py +++ b/tests/extmod/asyncio_basic.py @@ -4,7 +4,7 @@ except ImportError: print("SKIP") raise SystemExit - +# CIRCUITPY provides __await()__ async def foo(): return 42 @@ -15,14 +15,12 @@ except AttributeError: print("SKIP") raise SystemExit -try: - import time +import time +if hasattr(time, "ticks_ms"): ticks = time.ticks_ms ticks_diff = time.ticks_diff -except: - import time - +else: ticks = lambda: int(time.time() * 1000) ticks_diff = lambda t1, t0: t1 - t0 diff --git a/tests/extmod/uasyncio_basic.py.exp b/tests/extmod/asyncio_basic.py.exp similarity index 100% rename from tests/extmod/uasyncio_basic.py.exp rename to tests/extmod/asyncio_basic.py.exp diff --git a/tests/extmod/uasyncio_basic2.py b/tests/extmod/asyncio_basic2.py similarity index 100% rename from tests/extmod/uasyncio_basic2.py rename to tests/extmod/asyncio_basic2.py diff --git a/tests/extmod/uasyncio_basic2.py.exp b/tests/extmod/asyncio_basic2.py.exp similarity index 100% rename from tests/extmod/uasyncio_basic2.py.exp rename to tests/extmod/asyncio_basic2.py.exp diff --git a/tests/extmod/uasyncio_cancel_fair.py b/tests/extmod/asyncio_cancel_fair.py similarity index 100% rename from tests/extmod/uasyncio_cancel_fair.py rename to tests/extmod/asyncio_cancel_fair.py diff --git a/tests/extmod/uasyncio_cancel_fair.py.exp b/tests/extmod/asyncio_cancel_fair.py.exp similarity index 100% rename from tests/extmod/uasyncio_cancel_fair.py.exp rename to tests/extmod/asyncio_cancel_fair.py.exp diff --git a/tests/extmod/uasyncio_cancel_fair2.py b/tests/extmod/asyncio_cancel_fair2.py similarity index 100% rename from tests/extmod/uasyncio_cancel_fair2.py rename to tests/extmod/asyncio_cancel_fair2.py diff --git a/tests/extmod/uasyncio_cancel_fair2.py.exp b/tests/extmod/asyncio_cancel_fair2.py.exp similarity index 100% rename from tests/extmod/uasyncio_cancel_fair2.py.exp rename to tests/extmod/asyncio_cancel_fair2.py.exp diff --git a/tests/extmod/uasyncio_cancel_self.py b/tests/extmod/asyncio_cancel_self.py similarity index 100% rename from tests/extmod/uasyncio_cancel_self.py rename to tests/extmod/asyncio_cancel_self.py diff --git a/tests/extmod/uasyncio_cancel_self.py.exp b/tests/extmod/asyncio_cancel_self.py.exp similarity index 100% rename from tests/extmod/uasyncio_cancel_self.py.exp rename to tests/extmod/asyncio_cancel_self.py.exp diff --git a/tests/extmod/uasyncio_cancel_task.py b/tests/extmod/asyncio_cancel_task.py similarity index 100% rename from tests/extmod/uasyncio_cancel_task.py rename to tests/extmod/asyncio_cancel_task.py diff --git a/tests/extmod/uasyncio_cancel_task.py.exp b/tests/extmod/asyncio_cancel_task.py.exp similarity index 100% rename from tests/extmod/uasyncio_cancel_task.py.exp rename to tests/extmod/asyncio_cancel_task.py.exp diff --git a/tests/extmod/uasyncio_cancel_wait_on_finished.py b/tests/extmod/asyncio_cancel_wait_on_finished.py similarity index 100% rename from tests/extmod/uasyncio_cancel_wait_on_finished.py rename to tests/extmod/asyncio_cancel_wait_on_finished.py diff --git a/tests/extmod/uasyncio_cancel_wait_on_finished.py.exp b/tests/extmod/asyncio_cancel_wait_on_finished.py.exp similarity index 100% rename from tests/extmod/uasyncio_cancel_wait_on_finished.py.exp rename to tests/extmod/asyncio_cancel_wait_on_finished.py.exp diff --git a/tests/extmod/uasyncio_current_task.py b/tests/extmod/asyncio_current_task.py similarity index 100% rename from tests/extmod/uasyncio_current_task.py rename to tests/extmod/asyncio_current_task.py diff --git a/tests/extmod/uasyncio_current_task.py.exp b/tests/extmod/asyncio_current_task.py.exp similarity index 100% rename from tests/extmod/uasyncio_current_task.py.exp rename to tests/extmod/asyncio_current_task.py.exp diff --git a/tests/extmod/uasyncio_event.py b/tests/extmod/asyncio_event.py similarity index 100% rename from tests/extmod/uasyncio_event.py rename to tests/extmod/asyncio_event.py diff --git a/tests/extmod/uasyncio_event.py.exp b/tests/extmod/asyncio_event.py.exp similarity index 100% rename from tests/extmod/uasyncio_event.py.exp rename to tests/extmod/asyncio_event.py.exp diff --git a/tests/extmod/uasyncio_event_fair.py b/tests/extmod/asyncio_event_fair.py similarity index 96% rename from tests/extmod/uasyncio_event_fair.py rename to tests/extmod/asyncio_event_fair.py index 1029b54c5f..9ef63c9f16 100644 --- a/tests/extmod/uasyncio_event_fair.py +++ b/tests/extmod/asyncio_event_fair.py @@ -8,6 +8,7 @@ except ImportError: raise SystemExit +# CIRCUITPY provides __await__() async def foo(): return 42 diff --git a/tests/extmod/uasyncio_event_fair.py.exp b/tests/extmod/asyncio_event_fair.py.exp similarity index 100% rename from tests/extmod/uasyncio_event_fair.py.exp rename to tests/extmod/asyncio_event_fair.py.exp diff --git a/tests/extmod/uasyncio_exception.py b/tests/extmod/asyncio_exception.py similarity index 100% rename from tests/extmod/uasyncio_exception.py rename to tests/extmod/asyncio_exception.py diff --git a/tests/extmod/uasyncio_exception.py.exp b/tests/extmod/asyncio_exception.py.exp similarity index 100% rename from tests/extmod/uasyncio_exception.py.exp rename to tests/extmod/asyncio_exception.py.exp diff --git a/tests/extmod/uasyncio_fair.py b/tests/extmod/asyncio_fair.py similarity index 100% rename from tests/extmod/uasyncio_fair.py rename to tests/extmod/asyncio_fair.py diff --git a/tests/extmod/uasyncio_fair.py.exp b/tests/extmod/asyncio_fair.py.exp similarity index 100% rename from tests/extmod/uasyncio_fair.py.exp rename to tests/extmod/asyncio_fair.py.exp diff --git a/tests/extmod/uasyncio_gather.py b/tests/extmod/asyncio_gather.py similarity index 98% rename from tests/extmod/uasyncio_gather.py rename to tests/extmod/asyncio_gather.py index 5e5bb62d7c..5f3a84c1f3 100644 --- a/tests/extmod/uasyncio_gather.py +++ b/tests/extmod/asyncio_gather.py @@ -7,6 +7,7 @@ except ImportError: raise SystemExit +# CIRCUITPY provides __await__() async def foo(): return 42 diff --git a/tests/extmod/uasyncio_gather.py.exp b/tests/extmod/asyncio_gather.py.exp similarity index 100% rename from tests/extmod/uasyncio_gather.py.exp rename to tests/extmod/asyncio_gather.py.exp diff --git a/tests/extmod/uasyncio_gather_notimpl.py b/tests/extmod/asyncio_gather_notimpl.py similarity index 92% rename from tests/extmod/uasyncio_gather_notimpl.py rename to tests/extmod/asyncio_gather_notimpl.py index 0b12bb804b..91b26362d0 100644 --- a/tests/extmod/uasyncio_gather_notimpl.py +++ b/tests/extmod/asyncio_gather_notimpl.py @@ -1,4 +1,4 @@ -# Test asyncio.gather() function, featres that are not implemented. +# Test asyncio.gather() function, features that are not implemented. try: import asyncio @@ -7,6 +7,7 @@ except ImportError: raise SystemExit +# CIRCUITPY provides __await__() async def foo(): return 42 diff --git a/tests/extmod/uasyncio_gather_notimpl.py.exp b/tests/extmod/asyncio_gather_notimpl.py.exp similarity index 100% rename from tests/extmod/uasyncio_gather_notimpl.py.exp rename to tests/extmod/asyncio_gather_notimpl.py.exp diff --git a/tests/extmod/uasyncio_get_event_loop.py b/tests/extmod/asyncio_get_event_loop.py similarity index 100% rename from tests/extmod/uasyncio_get_event_loop.py rename to tests/extmod/asyncio_get_event_loop.py diff --git a/tests/extmod/uasyncio_heaplock.py b/tests/extmod/asyncio_heaplock.py similarity index 98% rename from tests/extmod/uasyncio_heaplock.py rename to tests/extmod/asyncio_heaplock.py index 6ea9270c7a..8326443f0e 100644 --- a/tests/extmod/uasyncio_heaplock.py +++ b/tests/extmod/asyncio_heaplock.py @@ -1,6 +1,6 @@ # test that the following do not use the heap: # - basic scheduling of tasks -# - uasyncio.sleep_ms +# - asyncio.sleep_ms # - StreamWriter.write, stream is blocked and data to write is a bytes object # - StreamWriter.write, when stream is not blocked diff --git a/tests/extmod/uasyncio_heaplock.py.exp b/tests/extmod/asyncio_heaplock.py.exp similarity index 100% rename from tests/extmod/uasyncio_heaplock.py.exp rename to tests/extmod/asyncio_heaplock.py.exp diff --git a/tests/extmod/uasyncio_lock.py b/tests/extmod/asyncio_lock.py similarity index 100% rename from tests/extmod/uasyncio_lock.py rename to tests/extmod/asyncio_lock.py diff --git a/tests/extmod/uasyncio_lock.py.exp b/tests/extmod/asyncio_lock.py.exp similarity index 100% rename from tests/extmod/uasyncio_lock.py.exp rename to tests/extmod/asyncio_lock.py.exp diff --git a/tests/extmod/uasyncio_lock_cancel.py b/tests/extmod/asyncio_lock_cancel.py similarity index 95% rename from tests/extmod/uasyncio_lock_cancel.py rename to tests/extmod/asyncio_lock_cancel.py index a501185e90..87e82219b7 100644 --- a/tests/extmod/uasyncio_lock_cancel.py +++ b/tests/extmod/asyncio_lock_cancel.py @@ -1,12 +1,13 @@ # Test that locks work when cancelling multiple waiters on the lock try: - import asyncio + import uasyncio as asyncio except ImportError: print("SKIP") raise SystemExit +# CIRCUITPY provides __await__() async def foo(): return 42 diff --git a/tests/extmod/uasyncio_lock_cancel.py.exp b/tests/extmod/asyncio_lock_cancel.py.exp similarity index 100% rename from tests/extmod/uasyncio_lock_cancel.py.exp rename to tests/extmod/asyncio_lock_cancel.py.exp diff --git a/tests/extmod/uasyncio_loop_stop.py b/tests/extmod/asyncio_loop_stop.py similarity index 100% rename from tests/extmod/uasyncio_loop_stop.py rename to tests/extmod/asyncio_loop_stop.py diff --git a/tests/extmod/uasyncio_loop_stop.py.exp b/tests/extmod/asyncio_loop_stop.py.exp similarity index 100% rename from tests/extmod/uasyncio_loop_stop.py.exp rename to tests/extmod/asyncio_loop_stop.py.exp diff --git a/tests/extmod/uasyncio_micropython.py b/tests/extmod/asyncio_micropython.py similarity index 78% rename from tests/extmod/uasyncio_micropython.py rename to tests/extmod/asyncio_micropython.py index c27a134cd9..3985307043 100644 --- a/tests/extmod/uasyncio_micropython.py +++ b/tests/extmod/asyncio_micropython.py @@ -22,6 +22,12 @@ async def main(): await asyncio.sleep_ms(1) print(time.ticks_diff(time.ticks_ms(), t0) < 100) + try: + # Sleep 1ms beyond maximum allowed sleep value + await asyncio.sleep_ms(time.ticks_add(0, -1) // 2 + 1) + except OverflowError: + print("OverflowError") + # When task finished before the timeout print(await asyncio.wait_for_ms(task(1, 5), 50)) @@ -34,4 +40,4 @@ async def main(): print("finish") -asyncio.run(main()) +uasyncio.run(main()) diff --git a/tests/extmod/uasyncio_micropython.py.exp b/tests/extmod/asyncio_micropython.py.exp similarity index 100% rename from tests/extmod/uasyncio_micropython.py.exp rename to tests/extmod/asyncio_micropython.py.exp diff --git a/tests/extmod/uasyncio_new_event_loop.py b/tests/extmod/asyncio_new_event_loop.py similarity index 100% rename from tests/extmod/uasyncio_new_event_loop.py rename to tests/extmod/asyncio_new_event_loop.py diff --git a/tests/extmod/uasyncio_new_event_loop.py.exp b/tests/extmod/asyncio_new_event_loop.py.exp similarity index 100% rename from tests/extmod/uasyncio_new_event_loop.py.exp rename to tests/extmod/asyncio_new_event_loop.py.exp diff --git a/tests/extmod/uasyncio_set_exception_handler.py b/tests/extmod/asyncio_set_exception_handler.py similarity index 100% rename from tests/extmod/uasyncio_set_exception_handler.py rename to tests/extmod/asyncio_set_exception_handler.py diff --git a/tests/extmod/uasyncio_set_exception_handler.py.exp b/tests/extmod/asyncio_set_exception_handler.py.exp similarity index 100% rename from tests/extmod/uasyncio_set_exception_handler.py.exp rename to tests/extmod/asyncio_set_exception_handler.py.exp diff --git a/tests/extmod/uasyncio_task_done.py b/tests/extmod/asyncio_task_done.py similarity index 100% rename from tests/extmod/uasyncio_task_done.py rename to tests/extmod/asyncio_task_done.py diff --git a/tests/extmod/uasyncio_task_done.py.exp b/tests/extmod/asyncio_task_done.py.exp similarity index 100% rename from tests/extmod/uasyncio_task_done.py.exp rename to tests/extmod/asyncio_task_done.py.exp diff --git a/tests/extmod/uasyncio_threadsafeflag.py b/tests/extmod/asyncio_threadsafeflag.py similarity index 86% rename from tests/extmod/uasyncio_threadsafeflag.py rename to tests/extmod/asyncio_threadsafeflag.py index a70015f8b6..abb55930ce 100644 --- a/tests/extmod/uasyncio_threadsafeflag.py +++ b/tests/extmod/asyncio_threadsafeflag.py @@ -11,23 +11,13 @@ import micropython try: micropython.schedule + # CIRCUITPY: no ThreadSafeFlag asyncio.ThreadSafeFlag except AttributeError: print("SKIP") raise SystemExit -try: - # Unix port can't select/poll on user-defined types. - import select - - poller = select.poll() - poller.register(asyncio.ThreadSafeFlag()) -except TypeError: - print("SKIP") - raise SystemExit - - async def task(id, flag): print("task", id) await flag.wait() @@ -35,9 +25,7 @@ async def task(id, flag): def set_from_schedule(flag): - print("schedule") flag.set() - print("schedule done") async def main(): diff --git a/tests/extmod/uasyncio_threadsafeflag.py.exp b/tests/extmod/asyncio_threadsafeflag.py.exp similarity index 91% rename from tests/extmod/uasyncio_threadsafeflag.py.exp rename to tests/extmod/asyncio_threadsafeflag.py.exp index 757115ac4b..0c62eae9d0 100644 --- a/tests/extmod/uasyncio_threadsafeflag.py.exp +++ b/tests/extmod/asyncio_threadsafeflag.py.exp @@ -9,8 +9,6 @@ yield task 2 set event yield -schedule -schedule done wait task task 2 done ---- diff --git a/tests/extmod/uasyncio_wait_for.py b/tests/extmod/asyncio_wait_for.py similarity index 100% rename from tests/extmod/uasyncio_wait_for.py rename to tests/extmod/asyncio_wait_for.py diff --git a/tests/extmod/uasyncio_wait_for.py.exp b/tests/extmod/asyncio_wait_for.py.exp similarity index 100% rename from tests/extmod/uasyncio_wait_for.py.exp rename to tests/extmod/asyncio_wait_for.py.exp diff --git a/tests/extmod/uasyncio_wait_for_fwd.py b/tests/extmod/asyncio_wait_for_fwd.py similarity index 98% rename from tests/extmod/uasyncio_wait_for_fwd.py rename to tests/extmod/asyncio_wait_for_fwd.py index fd74ed01d5..6e1bff2b51 100644 --- a/tests/extmod/uasyncio_wait_for_fwd.py +++ b/tests/extmod/asyncio_wait_for_fwd.py @@ -7,6 +7,7 @@ except ImportError: raise SystemExit +# CIRCUITPY provides __await__() async def foo(): return 42 diff --git a/tests/extmod/uasyncio_wait_for_fwd.py.exp b/tests/extmod/asyncio_wait_for_fwd.py.exp similarity index 100% rename from tests/extmod/uasyncio_wait_for_fwd.py.exp rename to tests/extmod/asyncio_wait_for_fwd.py.exp diff --git a/tests/extmod/uasyncio_wait_task.py b/tests/extmod/asyncio_wait_task.py similarity index 96% rename from tests/extmod/uasyncio_wait_task.py rename to tests/extmod/asyncio_wait_task.py index 1b1762befe..bce426d971 100644 --- a/tests/extmod/uasyncio_wait_task.py +++ b/tests/extmod/asyncio_wait_task.py @@ -7,14 +7,12 @@ except ImportError: raise SystemExit -try: - import time +import time +if hasattr(time, "ticks_ms"): ticks = time.ticks_ms ticks_diff = time.ticks_diff -except: - import time - +else: ticks = lambda: int(time.time() * 1000) ticks_diff = lambda t1, t0: t1 - t0 diff --git a/tests/extmod/uasyncio_wait_task.py.exp b/tests/extmod/asyncio_wait_task.py.exp similarity index 100% rename from tests/extmod/uasyncio_wait_task.py.exp rename to tests/extmod/asyncio_wait_task.py.exp diff --git a/tests/extmod/ubinascii_a2b_base64.py b/tests/extmod/binascii_a2b_base64.py similarity index 100% rename from tests/extmod/ubinascii_a2b_base64.py rename to tests/extmod/binascii_a2b_base64.py diff --git a/tests/extmod/ubinascii_b2a_base64.py b/tests/extmod/binascii_b2a_base64.py similarity index 100% rename from tests/extmod/ubinascii_b2a_base64.py rename to tests/extmod/binascii_b2a_base64.py diff --git a/tests/extmod/ubinascii_crc32.py b/tests/extmod/binascii_crc32.py similarity index 100% rename from tests/extmod/ubinascii_crc32.py rename to tests/extmod/binascii_crc32.py diff --git a/tests/extmod/ubinascii_hexlify.py b/tests/extmod/binascii_hexlify.py similarity index 100% rename from tests/extmod/ubinascii_hexlify.py rename to tests/extmod/binascii_hexlify.py diff --git a/tests/extmod/ubinascii_unhexlify.py b/tests/extmod/binascii_unhexlify.py similarity index 100% rename from tests/extmod/ubinascii_unhexlify.py rename to tests/extmod/binascii_unhexlify.py diff --git a/tests/extmod/deflate_compress.py b/tests/extmod/deflate_compress.py new file mode 100644 index 0000000000..612af663e4 --- /dev/null +++ b/tests/extmod/deflate_compress.py @@ -0,0 +1,148 @@ +try: + # Check if deflate is available. + import deflate + import io +except ImportError: + print("SKIP") + raise SystemExit + +# Check if compression is enabled. +if not hasattr(deflate.DeflateIO, "write"): + print("SKIP") + raise SystemExit + +# Simple compression & decompression. +b = io.BytesIO() +g = deflate.DeflateIO(b, deflate.RAW) +data = b"micropython" +N = 10 +for i in range(N): + g.write(data) +g.close() +result_raw = b.getvalue() +print(len(result_raw) < len(data) * N) +b = io.BytesIO(result_raw) +g = deflate.DeflateIO(b, deflate.RAW) +print(g.read()) + +# Same, but using a context manager. +b = io.BytesIO() +with deflate.DeflateIO(b, deflate.RAW) as g: + for i in range(N): + g.write(data) +result_raw = b.getvalue() +print(len(result_raw) < len(data) * N) +b = io.BytesIO(result_raw) +with deflate.DeflateIO(b, deflate.RAW) as g: + print(g.read()) + +# Writing to a closed underlying stream. +b = io.BytesIO() +g = deflate.DeflateIO(b, deflate.RAW) +g.write(b"micropython") +b.close() +try: + g.write(b"micropython") +except ValueError: + print("ValueError") + +# Writing to a closed DeflateIO. +b = io.BytesIO() +g = deflate.DeflateIO(b, deflate.RAW) +g.write(b"micropython") +g.close() +try: + g.write(b"micropython") +except OSError: + print("OSError") + + +def decompress(data, *args): + buf = io.BytesIO(data) + with deflate.DeflateIO(buf, *args) as g: + return g.read() + + +def compress(data, *args): + b = io.BytesIO() + with deflate.DeflateIO(b, *args) as g: + g.write(data) + return b.getvalue() + + +def compress_error(data, *args): + try: + compress(data, *args) + except OSError: + print("OSError") + except ValueError: + print("ValueError") + + +# More test patterns. +PATTERNS_RAW = ( + (b"0", b"3\x00\x00"), + (b"a", b"K\x04\x00"), + (b"0" * 100, b"3\xa0\x03\x00\x00"), + ( + bytes(range(64)), + b"c`dbfaec\xe7\xe0\xe4\xe2\xe6\xe1\xe5\xe3\x17\x10\x14\x12\x16\x11\x15\x13\x97\x90\x94\x92\x96\x91\x95\x93WPTRVQUS\xd7\xd0\xd4\xd2\xd6\xd1\xd5\xd370426153\xb7\xb0\xb4\xb2\xb6\xb1\xb5\xb3\x07\x00", + ), +) +for unpacked, packed in PATTERNS_RAW: + print(compress(unpacked) == packed) + print(compress(unpacked, deflate.RAW) == packed) + +# Verify header and checksum format. +unpacked = b"hello" +packed = b"\xcbH\xcd\xc9\xc9\x07\x00" + + +def check_header(n, a, b): + if a == b: + print(n) + else: + print(n, a, b) + + +check_header("RAW", compress(unpacked, deflate.RAW), packed) +check_header( + "ZLIB(9)", compress(unpacked, deflate.ZLIB, 9), b"\x18\x95" + packed + b"\x06,\x02\x15" +) +check_header( + "ZLIB(15)", compress(unpacked, deflate.ZLIB, 15), b"\x78\x9c" + packed + b"\x06,\x02\x15" +) +check_header( + "GZIP", + compress(unpacked, deflate.GZIP, 9), + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x04\x03" + packed + b"\x86\xa6\x106\x05\x00\x00\x00", +) + +# Valid wbits values. +compress_error(unpacked, deflate.RAW, -1) +print(len(compress(unpacked, deflate.RAW, 0))) +compress_error(unpacked, deflate.RAW, 1) +compress_error(unpacked, deflate.RAW, 4) +for i in range(5, 16): + print(len(compress(unpacked, deflate.RAW, i))) +compress_error(unpacked, deflate.RAW, 16) + +# Invalid values for format. +compress_error(unpacked, -1) +compress_error(unpacked, 5) + +# Fill buf with a predictable pseudorandom sequence. +buf = bytearray(1024) +lfsr = 1 << 15 | 1 +for i in range(len(buf)): + bit = (lfsr ^ (lfsr >> 1) ^ (lfsr >> 3) ^ (lfsr >> 12)) & 1 + lfsr = (lfsr >> 1) | (bit << 15) + buf[i] = lfsr & 0xFF + +# Verify that compression improves as the window size increases. +prev_len = len(buf) +for wbits in range(5, 10): + result = compress(buf, deflate.RAW, wbits) + next_len = len(result) + print(next_len < prev_len and decompress(result, deflate.RAW, wbits) == buf) + prev_len = next_len diff --git a/tests/extmod/deflate_compress.py.exp b/tests/extmod/deflate_compress.py.exp new file mode 100644 index 0000000000..5da70f491b --- /dev/null +++ b/tests/extmod/deflate_compress.py.exp @@ -0,0 +1,41 @@ +True +b'micropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropython' +True +b'micropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropython' +ValueError +OSError +True +True +True +True +True +True +True +True +RAW +ZLIB(9) +ZLIB(15) +GZIP +ValueError +7 +ValueError +ValueError +7 +7 +7 +7 +7 +7 +7 +7 +7 +7 +7 +ValueError +ValueError +ValueError +False +True +True +True +True diff --git a/tests/extmod/deflate_decompress.py b/tests/extmod/deflate_decompress.py new file mode 100644 index 0000000000..3ac8880af2 --- /dev/null +++ b/tests/extmod/deflate_decompress.py @@ -0,0 +1,185 @@ +try: + # Check if deflate is available. + import deflate + import io +except ImportError: + print("SKIP") + raise SystemExit + +try: + # Check there's enough memory to deflate gzip streams. + # zlib.compress(b'', wbits=25) + empty_gzip = ( + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00" + ) + deflate.DeflateIO(io.BytesIO(empty_gzip)).read() +except MemoryError: + print("SKIP") + raise SystemExit + +# zlib.compress(b'micropython hello world hello world micropython', wbits=-9) +data_raw = b'\xcb\xcdL.\xca/\xa8,\xc9\xc8\xcfS\xc8H\xcd\xc9\xc9W(\xcf/\xcaIAa\xe7"\xd4\x00\x00' +# zlib.compress(b'micropython hello world hello world micropython', wbits=9) +data_zlib = b'\x18\x95\xcb\xcdL.\xca/\xa8,\xc9\xc8\xcfS\xc8H\xcd\xc9\xc9W(\xcf/\xcaIAa\xe7"\xd4\x00\x00\xbc\xfa\x12\x91' +# zlib.compress(b'micropython hello world hello world micropython', wbits=25) +data_gzip = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\xcb\xcdL.\xca/\xa8,\xc9\xc8\xcfS\xc8H\xcd\xc9\xc9W(\xcf/\xcaIAa\xe7"\xd4\x00\x00"\xeb\xc4\x98/\x00\x00\x00' + +# compress(b'hello' + bytearray(300) + b'hello', format=deflate.RAW, 5) +data_wbits_5 = b"\xcbH\xcd\xc9\xc9g\x18\xe9\x00\x08\x88\x95\xcfH\xcd\xc9\xc9\x07\x00" +# compress(b'hello' + bytearray(300) + b'hello', format=deflate.RAW, 6) +data_wbits_6 = b"\xcbH\xcd\xc9\xc9g\x18\xe9\x00\x08\x88\xd5\x9f\x91\x9a\x93\x93\x0f\x00" +# compress(b'hello' + bytearray(300) + b'hello', format=deflate.RAW, 8) +data_wbits_8 = b"\xcbH\xcd\xc9\xc9g\x18\xe9\x00\x08\x88\xf5\x7fFjNN>\x00" +# compress(b'hello' + bytearray(2000) + b'hello', format=deflate.RAW, 10) +data_wbits_10 = b"\xcbH\xcd\xc9\xc9g\x18\xe9\x00\x08Fz\x18\x00\xc3`\xa4'\x03`2\x18\xe99\x01\x98\x13Fz\xfe\x07\xe6\xff\x91\x9e\xff\x81\xf9\x7f\xa4\xe7\x7f`\xfe\x1f\xba\xf9?#5''\x1f\x00" + + +def decompress(data, *args): + buf = io.BytesIO(data) + with deflate.DeflateIO(buf, *args) as g: + return g.read() + + +def decompress_error(data, *args): + try: + decompress(data, *args) + except OSError: + print("OSError") + except EOFError: + print("EOFError") + except ValueError: + print("ValueError") + + +# Basic handling of format and detection. +print(decompress(data_raw, deflate.RAW)) +print(decompress(data_zlib, deflate.ZLIB)) +print(decompress(data_gzip, deflate.GZIP)) +print(decompress(data_zlib)) # detect zlib/gzip. +print(decompress(data_gzip)) # detect zlib/gzip. + +decompress_error(data_raw) # cannot detect zlib/gzip from raw stream +decompress_error(data_raw, deflate.ZLIB) +decompress_error(data_raw, deflate.GZIP) +decompress_error(data_zlib, deflate.RAW) +decompress_error(data_zlib, deflate.GZIP) +decompress_error(data_gzip, deflate.RAW) +decompress_error(data_gzip, deflate.ZLIB) + +# Invalid data stream. +decompress_error(b"abcef", deflate.RAW) + +# Invalid block type. final-block, block-type=3. +decompress_error(b"\x07", deflate.RAW) + +# Truncated stream. +decompress_error(data_raw[:10], deflate.RAW) + +# Partial reads. +buf = io.BytesIO(data_zlib) +with deflate.DeflateIO(buf) as g: + print(buf.seek(0, 1)) # verify stream is not read until first read of the DeflateIO stream. + print(g.read(1)) + print(buf.seek(0, 1)) # verify that only the minimal amount is read from the source + print(g.read(1)) + print(buf.seek(0, 1)) + print(g.read(2)) + print(buf.seek(0, 1)) + print(g.read()) + print(buf.seek(0, 1)) + print(g.read(1)) + print(buf.seek(0, 1)) + print(g.read()) + +# Invalid zlib checksum (+ length for gzip). Note: only checksum errors are +# currently detected, see the end of uzlib_uncompress_chksum(). +decompress_error(data_zlib[:-4] + b"\x00\x00\x00\x00") +decompress_error(data_gzip[:-8] + b"\x00\x00\x00\x00\x00\x00\x00\x00") +decompress_error(data_zlib[:-4] + b"\x00\x00\x00\x00", deflate.ZLIB) +decompress_error(data_gzip[:-8] + b"\x00\x00\x00\x00\x00\x00\x00\x00", deflate.GZIP) + +# Reading from a closed underlying stream. +b = io.BytesIO(data_raw) +g = deflate.DeflateIO(b, deflate.RAW) +g.read(4) +b.close() +try: + g.read(4) +except ValueError: + print("ValueError") + +# Reading from a closed DeflateIO. +b = io.BytesIO(data_raw) +g = deflate.DeflateIO(b, deflate.RAW) +g.read(4) +g.close() +try: + g.read(4) +except OSError: + print("OSError") + +# Gzip header with extra flags (FCOMMENT FNAME FEXTRA FHCRC) enabled. +data_gzip_header_extra = b"\x1f\x8b\x08\x1e}\x9a\x9bd\x02\x00\x00\x00\x00\x00\x00\xff\xcb\xcdL.\xca/\xa8,\xc9\xc8\xcf\x03\x00\xf2KF>\x0b\x00\x00\x00" +print(decompress(data_gzip_header_extra)) + +# Test patterns. +PATTERNS_ZLIB = [ + # Packed results produced by CPy's zlib.compress() + (b"0", b"x\x9c3\x00\x00\x001\x001"), + (b"a", b"x\x9cK\x04\x00\x00b\x00b"), + (b"0" * 100, b"x\x9c30\xa0=\x00\x00\xb3q\x12\xc1"), + ( + bytes(range(64)), + b"x\x9cc`dbfaec\xe7\xe0\xe4\xe2\xe6\xe1\xe5\xe3\x17\x10\x14\x12\x16\x11\x15\x13\x97\x90\x94\x92\x96\x91\x95\x93WPTRVQUS\xd7\xd0\xd4\xd2\xd6\xd1\xd5\xd370426153\xb7\xb0\xb4\xb2\xb6\xb1\xb5\xb3\x07\x00\xaa\xe0\x07\xe1", + ), + (b"hello", b"x\x01\x01\x05\x00\xfa\xffhello\x06,\x02\x15"), # compression level 0 + # adaptive/dynamic huffman tree + ( + b"13371813150|13764518736|12345678901", + b"x\x9c\x05\xc1\x81\x01\x000\x04\x04\xb1\x95\\\x1f\xcfn\x86o\x82d\x06Qq\xc8\x9d\xc5X}I}\x00\x951D>I}\x00\x951D>I}\x00\x951D>I}\x00\x951D", + b"x\x9c\x05\xc11\x01\x00\x00\x00\x010\x95\x14py\x84\x12C_\x9bR\x8cV\x8a\xd1J1Z)F\x1fw`\x089", + ), +] +for unpacked, packed in PATTERNS_ZLIB: + print(decompress(packed) == unpacked) + print(decompress(packed, deflate.ZLIB) == unpacked) + +# Older version's of CPython's zlib module still included the checksum and length (as if it were a zlib/gzip stream). +# Make sure there're no problem decompressing this. +data_raw_with_footer = data_raw + b"\x00\x00\x00\x00\x00\x00\x00\x00" +print(decompress(data_raw_with_footer, deflate.RAW)) + +# Valid wbits values. +decompress_error(data_wbits_5, deflate.RAW, -1) +print(len(decompress(data_wbits_5, deflate.RAW, 0))) +decompress_error(data_wbits_5, deflate.RAW, 1) +decompress_error(data_wbits_5, deflate.RAW, 4) +for i in range(5, 16): + print(len(decompress(data_wbits_5, deflate.RAW, i))) +decompress_error(data_wbits_5, deflate.RAW, 16) + +# Invalid values for format. +decompress_error(data_raw, -1) +decompress_error(data_raw, 5) + +# Data that requires a higher wbits value. +decompress_error(data_wbits_6, deflate.RAW, 5) +print(len(decompress(data_wbits_6, deflate.RAW, 6))) +print(len(decompress(data_wbits_6, deflate.RAW, 7))) +decompress_error(data_wbits_8, deflate.RAW, 7) +print(len(decompress(data_wbits_8, deflate.RAW, 8))) +print(len(decompress(data_wbits_8, deflate.RAW, 9))) +decompress_error(data_wbits_10, deflate.RAW) +decompress_error(data_wbits_10, deflate.RAW, 9) +print(len(decompress(data_wbits_10, deflate.RAW, 10))) + +# zlib header sets the size, so works with wbits unset or wbits >= 10. +data_wbits_10_zlib = b"(\x91\xcbH\xcd\xc9\xc9g\x18\xe9\x00\x08Fz\x18\x00\xc3`\xa4'\x03`2\x18\xe99\x01\x98\x13Fz\xfe\x07\xe6\xff\x91\x9e\xff\x81\xf9\x7f\xa4\xe7\x7f`\xfe\x1f\xba\xf9?#5''\x1f\x00[\xbc\x04)" +print(len(decompress(data_wbits_10_zlib, deflate.ZLIB))) +decompress_error(data_wbits_10_zlib, deflate.ZLIB, 9) +print(len(decompress(data_wbits_10_zlib, deflate.ZLIB, 10))) +print(len(decompress(data_wbits_10_zlib))) diff --git a/tests/extmod/deflate_decompress.py.exp b/tests/extmod/deflate_decompress.py.exp new file mode 100644 index 0000000000..381f2b0685 --- /dev/null +++ b/tests/extmod/deflate_decompress.py.exp @@ -0,0 +1,80 @@ +b'micropython hello world hello world micropython' +b'micropython hello world hello world micropython' +b'micropython hello world hello world micropython' +b'micropython hello world hello world micropython' +b'micropython hello world hello world micropython' +OSError +OSError +OSError +OSError +OSError +OSError +OSError +OSError +OSError +EOFError +0 +b'm' +4 +b'i' +5 +b'cr' +7 +b'opython hello world hello world micropython' +36 +b'' +36 +b'' +OSError +OSError +OSError +OSError +ValueError +OSError +b'micropython' +True +True +True +True +True +True +True +True +True +True +True +True +True +True +b'micropython hello world hello world micropython' +ValueError +310 +ValueError +ValueError +310 +310 +310 +310 +310 +310 +310 +310 +310 +310 +310 +ValueError +ValueError +ValueError +OSError +310 +310 +OSError +310 +310 +OSError +OSError +2010 +2010 +OSError +2010 +2010 diff --git a/tests/extmod/deflate_stream_error.py b/tests/extmod/deflate_stream_error.py new file mode 100644 index 0000000000..aee6b28033 --- /dev/null +++ b/tests/extmod/deflate_stream_error.py @@ -0,0 +1,89 @@ +# Test deflate module with stream errors. + +try: + # Check if deflate & IOBase are available. + import deflate, io + + io.IOBase +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + +# Check if compression is enabled. +if not hasattr(deflate.DeflateIO, "write"): + print("SKIP") + raise SystemExit + +formats = (deflate.RAW, deflate.ZLIB, deflate.GZIP) + +# Test error on read when decompressing. + + +class Stream(io.IOBase): + def readinto(self, buf): + print("Stream.readinto", len(buf)) + return -1 + + +try: + deflate.DeflateIO(Stream()).read() +except OSError as er: + print(repr(er)) + +# Test error on write when compressing. + + +class Stream(io.IOBase): + def write(self, buf): + print("Stream.write", buf) + return -1 + + +for format in formats: + try: + deflate.DeflateIO(Stream(), format).write("a") + except OSError as er: + print(repr(er)) + +# Test write after close. + + +class Stream(io.IOBase): + def write(self, buf): + print("Stream.write", buf) + return -1 + + def ioctl(self, cmd, arg): + print("Stream.ioctl", cmd, arg) + return 0 + + +try: + d = deflate.DeflateIO(Stream(), deflate.RAW, 0, True) + d.close() + d.write("a") +except OSError as er: + print(repr(er)) + +# Test error on write when closing. + + +class Stream(io.IOBase): + def __init__(self): + self.num_writes = 0 + + def write(self, buf): + print("Stream.write", buf) + if self.num_writes >= 4: + return -1 + self.num_writes += 1 + return len(buf) + + +for format in formats: + d = deflate.DeflateIO(Stream(), format) + d.write("a") + try: + d.close() + except OSError as er: + print(repr(er)) diff --git a/tests/extmod/deflate_stream_error.py.exp b/tests/extmod/deflate_stream_error.py.exp new file mode 100644 index 0000000000..4ec90d7997 --- /dev/null +++ b/tests/extmod/deflate_stream_error.py.exp @@ -0,0 +1,25 @@ +Stream.readinto 1 +OSError(1,) +Stream.write bytearray(b'K') +OSError(1,) +Stream.write bytearray(b'\x18\x95') +OSError(22,) +Stream.write bytearray(b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x04\x03') +OSError(22,) +Stream.ioctl 4 0 +OSError(22,) +Stream.write bytearray(b'K') +Stream.write bytearray(b'\x04') +Stream.write bytearray(b'\x00') +Stream.write bytearray(b'\x18\x95') +Stream.write bytearray(b'K') +Stream.write bytearray(b'\x04') +Stream.write bytearray(b'\x00') +Stream.write bytearray(b'\x00b\x00b') +OSError(1,) +Stream.write bytearray(b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x04\x03') +Stream.write bytearray(b'K') +Stream.write bytearray(b'\x04') +Stream.write bytearray(b'\x00') +Stream.write bytearray(b'C\xbe\xb7\xe8\x01\x00\x00\x00') +OSError(1,) diff --git a/tests/extmod/uhashlib_final.py b/tests/extmod/hashlib_final.py similarity index 100% rename from tests/extmod/uhashlib_final.py rename to tests/extmod/hashlib_final.py diff --git a/tests/extmod/uhashlib_final.py.exp b/tests/extmod/hashlib_final.py.exp similarity index 100% rename from tests/extmod/uhashlib_final.py.exp rename to tests/extmod/hashlib_final.py.exp diff --git a/tests/extmod/uhashlib_md5.py b/tests/extmod/hashlib_md5.py similarity index 100% rename from tests/extmod/uhashlib_md5.py rename to tests/extmod/hashlib_md5.py diff --git a/tests/extmod/uhashlib_sha1.py b/tests/extmod/hashlib_sha1.py similarity index 74% rename from tests/extmod/uhashlib_sha1.py rename to tests/extmod/hashlib_sha1.py index 882f570843..af23033a59 100644 --- a/tests/extmod/uhashlib_sha1.py +++ b/tests/extmod/hashlib_sha1.py @@ -16,10 +16,3 @@ except AttributeError: sha1 = hashlib.sha1(b"hello") sha1.update(b"world") print(sha1.digest()) - -sha1 = hashlib.sha1(b"hello") -try: - sha1.update("world") -except TypeError as e: - print("TypeError") -print(sha1.digest()) diff --git a/tests/extmod/uhashlib_sha256.py b/tests/extmod/hashlib_sha256.py similarity index 100% rename from tests/extmod/uhashlib_sha256.py rename to tests/extmod/hashlib_sha256.py diff --git a/tests/extmod/uheapq1.py b/tests/extmod/heapq1.py similarity index 100% rename from tests/extmod/uheapq1.py rename to tests/extmod/heapq1.py diff --git a/tests/extmod/ujson_dump.py b/tests/extmod/json_dump.py similarity index 100% rename from tests/extmod/ujson_dump.py rename to tests/extmod/json_dump.py diff --git a/tests/extmod/ujson_dump_iobase.py b/tests/extmod/json_dump_iobase.py similarity index 100% rename from tests/extmod/ujson_dump_iobase.py rename to tests/extmod/json_dump_iobase.py diff --git a/tests/extmod/ujson_dump_separators.py b/tests/extmod/json_dump_separators.py similarity index 100% rename from tests/extmod/ujson_dump_separators.py rename to tests/extmod/json_dump_separators.py diff --git a/tests/extmod/ujson_dumps.py b/tests/extmod/json_dumps.py similarity index 100% rename from tests/extmod/ujson_dumps.py rename to tests/extmod/json_dumps.py diff --git a/tests/extmod/ujson_dumps_extra.py b/tests/extmod/json_dumps_extra.py similarity index 100% rename from tests/extmod/ujson_dumps_extra.py rename to tests/extmod/json_dumps_extra.py diff --git a/tests/extmod/ujson_dumps_extra.py.exp b/tests/extmod/json_dumps_extra.py.exp similarity index 100% rename from tests/extmod/ujson_dumps_extra.py.exp rename to tests/extmod/json_dumps_extra.py.exp diff --git a/tests/extmod/ujson_dumps_float.py b/tests/extmod/json_dumps_float.py similarity index 100% rename from tests/extmod/ujson_dumps_float.py rename to tests/extmod/json_dumps_float.py diff --git a/tests/extmod/ujson_dumps_ordereddict.py b/tests/extmod/json_dumps_ordereddict.py similarity index 100% rename from tests/extmod/ujson_dumps_ordereddict.py rename to tests/extmod/json_dumps_ordereddict.py diff --git a/tests/extmod/ujson_dumps_separators.py b/tests/extmod/json_dumps_separators.py similarity index 100% rename from tests/extmod/ujson_dumps_separators.py rename to tests/extmod/json_dumps_separators.py diff --git a/tests/extmod/ujson_load.py b/tests/extmod/json_load.py similarity index 100% rename from tests/extmod/ujson_load.py rename to tests/extmod/json_load.py diff --git a/tests/extmod/ujson_load_readinto.py b/tests/extmod/json_load_readinto.py similarity index 100% rename from tests/extmod/ujson_load_readinto.py rename to tests/extmod/json_load_readinto.py diff --git a/tests/extmod/ujson_load_readinto.py.exp b/tests/extmod/json_load_readinto.py.exp similarity index 100% rename from tests/extmod/ujson_load_readinto.py.exp rename to tests/extmod/json_load_readinto.py.exp diff --git a/tests/extmod/ujson_loads.py b/tests/extmod/json_loads.py similarity index 100% rename from tests/extmod/ujson_loads.py rename to tests/extmod/json_loads.py diff --git a/tests/extmod/ujson_loads_bytes.py b/tests/extmod/json_loads_bytes.py similarity index 100% rename from tests/extmod/ujson_loads_bytes.py rename to tests/extmod/json_loads_bytes.py diff --git a/tests/extmod/ujson_loads_bytes.py.exp b/tests/extmod/json_loads_bytes.py.exp similarity index 100% rename from tests/extmod/ujson_loads_bytes.py.exp rename to tests/extmod/json_loads_bytes.py.exp diff --git a/tests/extmod/ujson_loads_float.py b/tests/extmod/json_loads_float.py similarity index 100% rename from tests/extmod/ujson_loads_float.py rename to tests/extmod/json_loads_float.py diff --git a/tests/extmod/umsgpack_pack.py b/tests/extmod/msgpack_pack.py similarity index 100% rename from tests/extmod/umsgpack_pack.py rename to tests/extmod/msgpack_pack.py diff --git a/tests/extmod/umsgpack_pack.py.ext b/tests/extmod/msgpack_pack.py.ext similarity index 100% rename from tests/extmod/umsgpack_pack.py.ext rename to tests/extmod/msgpack_pack.py.ext diff --git a/tests/extmod/urandom_basic.py b/tests/extmod/random_basic.py similarity index 85% rename from tests/extmod/urandom_basic.py rename to tests/extmod/random_basic.py index e42373e30d..7cb992fede 100644 --- a/tests/extmod/urandom_basic.py +++ b/tests/extmod/random_basic.py @@ -1,11 +1,8 @@ try: import random except ImportError: - try: - import random - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit # check getrandbits returns a value within the bit range for b in (1, 2, 3, 4, 16, 32): diff --git a/tests/extmod/urandom_basic.py.exp b/tests/extmod/random_basic.py.exp similarity index 100% rename from tests/extmod/urandom_basic.py.exp rename to tests/extmod/random_basic.py.exp diff --git a/tests/extmod/urandom_extra.py b/tests/extmod/random_extra.py similarity index 100% rename from tests/extmod/urandom_extra.py rename to tests/extmod/random_extra.py diff --git a/tests/extmod/urandom_extra_float.py b/tests/extmod/random_extra_float.py similarity index 100% rename from tests/extmod/urandom_extra_float.py rename to tests/extmod/random_extra_float.py diff --git a/tests/extmod/urandom_seed_default.py b/tests/extmod/random_seed_default.py similarity index 91% rename from tests/extmod/urandom_seed_default.py rename to tests/extmod/random_seed_default.py index 241649c14b..2fb16282ca 100644 --- a/tests/extmod/urandom_seed_default.py +++ b/tests/extmod/random_seed_default.py @@ -1,4 +1,4 @@ -# test urandom.seed() without any arguments +# test random.seed() without any arguments try: import random diff --git a/tests/extmod/ure1.py b/tests/extmod/re1.py similarity index 100% rename from tests/extmod/ure1.py rename to tests/extmod/re1.py diff --git a/tests/extmod/ure_debug.py b/tests/extmod/re_debug.py similarity index 100% rename from tests/extmod/ure_debug.py rename to tests/extmod/re_debug.py diff --git a/tests/extmod/ure_debug.py.exp b/tests/extmod/re_debug.py.exp similarity index 100% rename from tests/extmod/ure_debug.py.exp rename to tests/extmod/re_debug.py.exp diff --git a/tests/extmod/ure_error.py b/tests/extmod/re_error.py similarity index 100% rename from tests/extmod/ure_error.py rename to tests/extmod/re_error.py diff --git a/tests/extmod/ure_group.py b/tests/extmod/re_group.py similarity index 100% rename from tests/extmod/ure_group.py rename to tests/extmod/re_group.py diff --git a/tests/extmod/ure_groups.py b/tests/extmod/re_groups.py similarity index 100% rename from tests/extmod/ure_groups.py rename to tests/extmod/re_groups.py diff --git a/tests/extmod/ure_limit.py b/tests/extmod/re_limit.py similarity index 100% rename from tests/extmod/ure_limit.py rename to tests/extmod/re_limit.py diff --git a/tests/extmod/ure_limit.py.exp b/tests/extmod/re_limit.py.exp similarity index 100% rename from tests/extmod/ure_limit.py.exp rename to tests/extmod/re_limit.py.exp diff --git a/tests/extmod/ure_namedclass.py b/tests/extmod/re_namedclass.py similarity index 100% rename from tests/extmod/ure_namedclass.py rename to tests/extmod/re_namedclass.py diff --git a/tests/extmod/ure_span.py b/tests/extmod/re_span.py similarity index 100% rename from tests/extmod/ure_span.py rename to tests/extmod/re_span.py diff --git a/tests/extmod/ure_split.py b/tests/extmod/re_split.py similarity index 100% rename from tests/extmod/ure_split.py rename to tests/extmod/re_split.py diff --git a/tests/extmod/ure_split_empty.py b/tests/extmod/re_split_empty.py similarity index 100% rename from tests/extmod/ure_split_empty.py rename to tests/extmod/re_split_empty.py diff --git a/tests/extmod/ure_split_empty.py.exp b/tests/extmod/re_split_empty.py.exp similarity index 100% rename from tests/extmod/ure_split_empty.py.exp rename to tests/extmod/re_split_empty.py.exp diff --git a/tests/extmod/ure_split_notimpl.py b/tests/extmod/re_split_notimpl.py similarity index 100% rename from tests/extmod/ure_split_notimpl.py rename to tests/extmod/re_split_notimpl.py diff --git a/tests/extmod/ure_split_notimpl.py.exp b/tests/extmod/re_split_notimpl.py.exp similarity index 100% rename from tests/extmod/ure_split_notimpl.py.exp rename to tests/extmod/re_split_notimpl.py.exp diff --git a/tests/extmod/ure_stack_overflow.py b/tests/extmod/re_stack_overflow.py similarity index 100% rename from tests/extmod/ure_stack_overflow.py rename to tests/extmod/re_stack_overflow.py diff --git a/tests/extmod/ure_stack_overflow.py.exp b/tests/extmod/re_stack_overflow.py.exp similarity index 100% rename from tests/extmod/ure_stack_overflow.py.exp rename to tests/extmod/re_stack_overflow.py.exp diff --git a/tests/extmod/ure_sub.py b/tests/extmod/re_sub.py similarity index 100% rename from tests/extmod/ure_sub.py rename to tests/extmod/re_sub.py diff --git a/tests/extmod/ure_sub_unmatched.py b/tests/extmod/re_sub_unmatched.py similarity index 100% rename from tests/extmod/ure_sub_unmatched.py rename to tests/extmod/re_sub_unmatched.py diff --git a/tests/extmod/ure_sub_unmatched.py.exp b/tests/extmod/re_sub_unmatched.py.exp similarity index 100% rename from tests/extmod/ure_sub_unmatched.py.exp rename to tests/extmod/re_sub_unmatched.py.exp diff --git a/tests/extmod/select_ipoll.py b/tests/extmod/select_ipoll.py new file mode 100644 index 0000000000..0b661c11c8 --- /dev/null +++ b/tests/extmod/select_ipoll.py @@ -0,0 +1,55 @@ +# Test select.ipoll(). + +try: + import socket, select +except ImportError: + print("SKIP") + raise SystemExit + + +def print_poll_output(lst): + print([(type(obj), flags) for obj, flags in lst]) + + +poller = select.poll() + +# Use a new UDP socket for tests, which should be writable but not readable. +try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.bind(socket.getaddrinfo("127.0.0.1", 8000)[0][-1]) +except OSError: + print("SKIP") + raise SystemExit + +poller.register(s) + +# Basic polling. +print_poll_output(poller.ipoll(0)) + +# Pass in flags=1 for one-shot behaviour. +print_poll_output(poller.ipoll(0, 1)) + +# Socket should be deregistered and poll should return nothing. +print_poll_output(poller.ipoll(0)) + +# Create a second socket. +s2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s2.bind(socket.getaddrinfo("127.0.0.1", 8001)[0][-1]) + +# Register both sockets (to reset the first one). +poller.register(s) +poller.register(s2) + +# Basic polling with two sockets. +print_poll_output(poller.ipoll(0)) + +# Unregister the first socket, to test polling the remaining one. +poller.unregister(s) +print_poll_output(poller.ipoll(0)) + +# Unregister the second socket, to test polling none. +poller.unregister(s2) +print_poll_output(poller.ipoll(0)) + +s2.close() +s.close() diff --git a/tests/extmod/select_ipoll.py.exp b/tests/extmod/select_ipoll.py.exp new file mode 100644 index 0000000000..cbeabdce90 --- /dev/null +++ b/tests/extmod/select_ipoll.py.exp @@ -0,0 +1,6 @@ +[(, 4)] +[(, 4)] +[] +[(, 4), (, 4)] +[(, 4)] +[] diff --git a/tests/extmod/select_poll_custom.py b/tests/extmod/select_poll_custom.py new file mode 100644 index 0000000000..b854a8a14d --- /dev/null +++ b/tests/extmod/select_poll_custom.py @@ -0,0 +1,102 @@ +# Test custom pollable objects implemented in Python. + +from micropython import const + +try: + import socket, select, io +except ImportError: + print("SKIP") + raise SystemExit + +_MP_STREAM_POLL = const(3) +_MP_STREAM_GET_FILENO = const(10) + +_MP_STREAM_POLL_RD = const(0x0001) +_MP_STREAM_POLL_WR = const(0x0004) + + +def print_poll_output(lst): + print([(type(obj), flags) for obj, flags in lst]) + + +class CustomPollable(io.IOBase): + def __init__(self): + self.poll_state = 0 + + def ioctl(self, cmd, arg): + if cmd == _MP_STREAM_GET_FILENO: + # Bare-metal ports don't call this ioctl, so don't print it. + return -1 + + print("CustomPollable.ioctl", cmd, arg) + if cmd == _MP_STREAM_POLL: + if self.poll_state == "delay_rd": + self.poll_state = _MP_STREAM_POLL_RD + return 0 + elif self.poll_state < 0: + return self.poll_state + else: + return self.poll_state & arg + + +poller = select.poll() + +# Use a new UDP socket for tests, which should be writable but not readable. +try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.bind(socket.getaddrinfo("127.0.0.1", 8000)[0][-1]) +except OSError: + print("SKIP") + raise SystemExit + +x = CustomPollable() + +# Register both a file-descriptor-based object and a custom pure-Python object. +poller.register(s) +poller.register(x) + +# Modify the flags for the custom object. +poller.modify(x, select.POLLIN) + +# Test polling. +print_poll_output(poller.poll(0)) +x.poll_state = _MP_STREAM_POLL_WR +print_poll_output(poller.poll(0)) +x.poll_state = _MP_STREAM_POLL_RD +print_poll_output(poller.poll(0)) + +# The custom object becomes readable only after being polled. +poller.modify(s, select.POLLIN) +x.poll_state = "delay_rd" +print_poll_output(poller.poll()) + +# The custom object returns an error. +x.poll_state = -1000 +try: + poller.poll(0) +except OSError as er: + print("OSError", er.errno) + +# Register then unregister a socket (a native stream), then test +# that the Python object is still pollable. +s2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +x.poll_state = _MP_STREAM_POLL_RD +poller.register(s2) +poller.unregister(s2) +print_poll_output(poller.poll()) + +# Test registering and unregistering multiple times. +for _ in range(2): + poller.unregister(s) + poller.unregister(x) + poller.register(s2) + poller.register(s, select.POLLIN) + poller.register(x, select.POLLIN) + poller.unregister(s2) + print_poll_output(poller.poll()) + +# Clean up. +poller.unregister(x) +poller.unregister(s) +s2.close() +s.close() diff --git a/tests/extmod/select_poll_custom.py.exp b/tests/extmod/select_poll_custom.py.exp new file mode 100644 index 0000000000..d85508bab2 --- /dev/null +++ b/tests/extmod/select_poll_custom.py.exp @@ -0,0 +1,17 @@ +CustomPollable.ioctl 3 1 +[(, 4)] +CustomPollable.ioctl 3 1 +[(, 4)] +CustomPollable.ioctl 3 1 +[(, 4), (, 1)] +CustomPollable.ioctl 3 1 +CustomPollable.ioctl 3 1 +[(, 1)] +CustomPollable.ioctl 3 1 +OSError 1000 +CustomPollable.ioctl 3 1 +[(, 1)] +CustomPollable.ioctl 3 1 +[(, 1)] +CustomPollable.ioctl 3 1 +[(, 1)] diff --git a/tests/extmod/select_poll_eintr.py b/tests/extmod/select_poll_eintr.py new file mode 100644 index 0000000000..e1cbc2aaf5 --- /dev/null +++ b/tests/extmod/select_poll_eintr.py @@ -0,0 +1,50 @@ +# Test interruption of select.poll by EINTR signal, when +# MICROPY_PY_SELECT_POSIX_OPTIMISATIONS is enabled. + +try: + import time, gc, select, socket, _thread + + time.time_ns # Check for time_ns on MicroPython + select.poll # Raises AttributeError for CPython implementations without poll() +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + + +def thread_main(): + lock.acquire() + time.sleep(0.2) + print("thread gc start") + # The unix gc.collect() implementation will raise EINTR on other threads. + # Could possibly use _thread._interrupt_main() instead if MicroPython had it. + gc.collect() + print("thread gc end") + + +# Start a thread to interrupt the main thread during its call to poll. +lock = _thread.allocate_lock() +lock.acquire() +_thread.start_new_thread(thread_main, ()) + +# Use a new UDP socket for tests, which should be writable but not readable. +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.bind(socket.getaddrinfo("127.0.0.1", 8000)[0][-1]) + +# Create the poller object. +poller = select.poll() +poller.register(s, select.POLLIN) + +# Poll on the UDP socket for a set timeout, which should be reached. +print("poll") +lock.release() +t0 = time.time_ns() +result = poller.poll(400) +dt_ms = (time.time_ns() - t0) / 1e6 +print("result:", result) +if 380 <= dt_ms <= 600: + print("dt in range") +else: + print("dt not in range:", dt_ms) + +# Clean up. +s.close() diff --git a/tests/extmod/select_poll_fd.py b/tests/extmod/select_poll_fd.py new file mode 100644 index 0000000000..fab9c0e0e9 --- /dev/null +++ b/tests/extmod/select_poll_fd.py @@ -0,0 +1,44 @@ +# Test select.poll in combination with file descriptors. + +try: + import select, errno + + select.poll # Raises AttributeError for CPython implementations without poll() +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + +# Check that poll supports registering file descriptors (integers). +try: + select.poll().register(0) +except OSError: + print("SKIP") + raise SystemExit + +# Register invalid file descriptor. +try: + select.poll().register(-1) +except ValueError: + print("ValueError") + +# Test polling stdout, it should be writable. +poller = select.poll() +poller.register(1) +poller.modify(1, select.POLLOUT) +print(poller.poll()) + +# Unregister then re-register. +poller.unregister(1) +poller.register(1, select.POLLIN) + +# Poll for input, should return an empty list. +print(poller.poll(0)) + +# Test registering a very large number of file descriptors. +poller = select.poll() +for fd in range(6000): + poller.register(fd) +try: + poller.poll() +except OSError as er: + print(er.errno == errno.EINVAL) diff --git a/tests/extmod/socket_udp_nonblock.py b/tests/extmod/socket_udp_nonblock.py new file mode 100644 index 0000000000..1e74e2917d --- /dev/null +++ b/tests/extmod/socket_udp_nonblock.py @@ -0,0 +1,21 @@ +# test non-blocking UDP sockets + +try: + import socket, errno +except ImportError: + print("SKIP") + raise SystemExit + +try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.bind(socket.getaddrinfo("127.0.0.1", 8000)[0][-1]) +except OSError: + print("SKIP") + raise SystemExit + +s.settimeout(0) + +try: + s.recv(1) +except OSError as er: + print("EAGAIN:", er.errno == errno.EAGAIN) diff --git a/tests/extmod/ssl_cadata.py b/tests/extmod/ssl_cadata.py new file mode 100644 index 0000000000..e66f6ca825 --- /dev/null +++ b/tests/extmod/ssl_cadata.py @@ -0,0 +1,18 @@ +# Test ssl.wrap_socket() with cadata passed in. + +try: + import io + import ssl +except ImportError: + print("SKIP") + raise SystemExit + +# Invalid cadata. +try: + ssl.wrap_socket(io.BytesIO(), cadata=b"!") +except TypeError: + # "cadata" keyword argument is not supported by axtls. + print("SKIP") + raise SystemExit +except ValueError as er: + print(repr(er)) diff --git a/tests/extmod/ssl_cadata.py.exp b/tests/extmod/ssl_cadata.py.exp new file mode 100644 index 0000000000..9f1cf732e3 --- /dev/null +++ b/tests/extmod/ssl_cadata.py.exp @@ -0,0 +1 @@ +ValueError('invalid cert',) diff --git a/tests/extmod/ssl_ioctl.py b/tests/extmod/ssl_ioctl.py new file mode 100644 index 0000000000..4db7c2df82 --- /dev/null +++ b/tests/extmod/ssl_ioctl.py @@ -0,0 +1,31 @@ +# Test SSL ioctl method. +# Direct access to this method is only available if MICROPY_UNIX_COVERAGE is enabled. + +try: + import io, ssl + + io.BytesIO +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + +_MP_STREAM_POLL = 3 +_MP_STREAM_CLOSE = 4 +_MP_STREAM_GET_FILENO = 10 + +s = ssl.wrap_socket(io.BytesIO(), server_side=1, do_handshake=0) + +if not hasattr(s, "ioctl"): + print("SKIP") + raise SystemExit + +# These ioctl's should be unsupported. +for request in (-1, 0, _MP_STREAM_GET_FILENO): + try: + s.ioctl(request, 0) + except OSError: + print(request, "OSError") + +# These ioctl's should be supported. +for request in (_MP_STREAM_CLOSE, _MP_STREAM_POLL, _MP_STREAM_CLOSE): + print(request, s.ioctl(request, 0)) diff --git a/tests/extmod/ssl_ioctl.py.exp b/tests/extmod/ssl_ioctl.py.exp new file mode 100644 index 0000000000..22208b00cc --- /dev/null +++ b/tests/extmod/ssl_ioctl.py.exp @@ -0,0 +1,6 @@ +-1 OSError +0 OSError +10 OSError +4 0 +3 32 +4 0 diff --git a/tests/extmod/ussl_poll.py b/tests/extmod/ssl_poll.py similarity index 94% rename from tests/extmod/ussl_poll.py rename to tests/extmod/ssl_poll.py index 8080ec5112..347e5f7d37 100644 --- a/tests/extmod/ussl_poll.py +++ b/tests/extmod/ssl_poll.py @@ -1,8 +1,8 @@ try: - import uselect - import ussl + import select + import ssl import io - import ubinascii as binascii + import binascii except ImportError: print("SKIP") raise SystemExit @@ -121,9 +121,9 @@ client_io, server_io = _Pipe.new_pair() client_io.block_reads = True client_io.block_writes = True -client_sock = ussl.wrap_socket(client_io, do_handshake=False) +client_sock = ssl.wrap_socket(client_io, do_handshake=False) -server_sock = ussl.wrap_socket(server_io, key=key, cert=cert, server_side=True, do_handshake=False) +server_sock = ssl.wrap_socket(server_io, key=key, cert=cert, server_side=True, do_handshake=False) # Do a test read, at this point the TLS handshake wants to write, # so it returns None: @@ -175,7 +175,7 @@ assert server_sock.read(3) == b"bar" # Polling on a closed socket errors out: client_io, _ = _Pipe.new_pair() -client_sock = ussl.wrap_socket(client_io, do_handshake=False) +client_sock = ssl.wrap_socket(client_io, do_handshake=False) client_sock.close() assert_poll( client_sock, client_io, _MP_STREAM_POLL_RD, None, _MP_STREAM_POLL_NVAL @@ -184,7 +184,7 @@ assert_poll( # Errors propagates to poll: client_io, server_io = _Pipe.new_pair() -client_sock = ussl.wrap_socket(client_io, do_handshake=False) +client_sock = ssl.wrap_socket(client_io, do_handshake=False) # The server returns garbage: server_io.write(b"fooba") # Needs to be exactly 5 bytes diff --git a/tests/extmod/ubinascii_micropython.py b/tests/extmod/ssl_poll.py.exp similarity index 100% rename from tests/extmod/ubinascii_micropython.py rename to tests/extmod/ssl_poll.py.exp diff --git a/tests/extmod/ssl_sslcontext.py b/tests/extmod/ssl_sslcontext.py new file mode 100644 index 0000000000..23ff9c2964 --- /dev/null +++ b/tests/extmod/ssl_sslcontext.py @@ -0,0 +1,25 @@ +# Very basic test of ssl.SSLContext class. + +try: + import socket, ssl +except ImportError: + print("SKIP") + raise SystemExit + +# Test constructing with arguments. +ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) +ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + +# Test printing object. +ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) +print("SSLContext" in str(ctx)) + +# Coverage test for destructor, and calling it twice. +if hasattr(ctx, "__del__"): + ctx.__del__() + ctx.__del__() + +# Test calling .wrap_socket() method, multiple times. +ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +ctx.wrap_socket(socket.socket(), do_handshake_on_connect=False) +ctx.wrap_socket(socket.socket(), do_handshake_on_connect=False) diff --git a/tests/extmod/ssl_sslcontext_micropython.py b/tests/extmod/ssl_sslcontext_micropython.py new file mode 100644 index 0000000000..136fb8a054 --- /dev/null +++ b/tests/extmod/ssl_sslcontext_micropython.py @@ -0,0 +1,29 @@ +# Test MicroPython-specific behaviour of ssl.SSLContext. + +try: + import ssl +except ImportError: + print("SKIP") + raise SystemExit + +# Test constructing without any arguments (in CPython it's a DeprecationWarning). +try: + ssl.SSLContext() +except TypeError: + print("TypeError") + +# Test attributes that don't exist (in CPython new attributes can be added). +# This test is needed for coverage because SSLContext implements a custom attr handler. +ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) +try: + ctx.does_not_exist +except AttributeError: + print("AttributeError on load") +try: + ctx.does_not_exist = None +except AttributeError: + print("AttributeError on store") +try: + del ctx.does_not_exist +except AttributeError: + print("AttributeError on delete") diff --git a/tests/extmod/ssl_sslcontext_micropython.py.exp b/tests/extmod/ssl_sslcontext_micropython.py.exp new file mode 100644 index 0000000000..21e65258ff --- /dev/null +++ b/tests/extmod/ssl_sslcontext_micropython.py.exp @@ -0,0 +1,4 @@ +TypeError +AttributeError on load +AttributeError on store +AttributeError on delete diff --git a/tests/extmod/ssl_sslcontext_verify_mode.py b/tests/extmod/ssl_sslcontext_verify_mode.py new file mode 100644 index 0000000000..daccc2f4a9 --- /dev/null +++ b/tests/extmod/ssl_sslcontext_verify_mode.py @@ -0,0 +1,24 @@ +# Test ssl.SSLContext.verify_mode attribute. +# It's not available in the axtls implementation, so has an independent test. + +try: + import ssl +except ImportError: + print("SKIP") + raise SystemExit + +if not hasattr(ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT), "verify_mode"): + print("SKIP") + raise SystemExit + +# Test default verify_mode for server (client default is different in MicroPython). +ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +print(ctx.verify_mode == ssl.CERT_NONE) + +# Test setting and getting verify_mode. +ctx.verify_mode = ssl.CERT_NONE +print(ctx.verify_mode == ssl.CERT_NONE) +ctx.verify_mode = ssl.CERT_OPTIONAL +print(ctx.verify_mode == ssl.CERT_OPTIONAL) +ctx.verify_mode = ssl.CERT_REQUIRED +print(ctx.verify_mode == ssl.CERT_REQUIRED) diff --git a/tests/extmod/ticks_add.py b/tests/extmod/ticks_add.py index 2f1ba6c810..4f465f3cfb 100644 --- a/tests/extmod/ticks_add.py +++ b/tests/extmod/ticks_add.py @@ -1,5 +1,5 @@ try: - from utime import ticks_diff, ticks_add + from time import ticks_diff, ticks_add except ImportError: print("SKIP") raise SystemExit diff --git a/tests/extmod/time_ms_us.py b/tests/extmod/time_ms_us.py index 67c469d0a3..e26c6e677a 100644 --- a/tests/extmod/time_ms_us.py +++ b/tests/extmod/time_ms_us.py @@ -17,7 +17,7 @@ t0 = time.ticks_us() t1 = time.ticks_us() print(0 <= time.ticks_diff(t1, t0) <= 500) -# ticks_cpu may not be implemented, at least make sre it doesn't decrease +# ticks_cpu may not be implemented, at least make sure it doesn't decrease t0 = time.ticks_cpu() t1 = time.ticks_cpu() print(time.ticks_diff(t1, t0) >= 0) diff --git a/tests/extmod/utime_res.py b/tests/extmod/time_res.py similarity index 100% rename from tests/extmod/utime_res.py rename to tests/extmod/time_res.py diff --git a/tests/extmod/utime_res.py.exp b/tests/extmod/time_res.py.exp similarity index 100% rename from tests/extmod/utime_res.py.exp rename to tests/extmod/time_res.py.exp diff --git a/tests/extmod/utime_time_ns.py b/tests/extmod/time_time_ns.py similarity index 100% rename from tests/extmod/utime_time_ns.py rename to tests/extmod/time_time_ns.py diff --git a/tests/extmod/utime_time_ns.py.exp b/tests/extmod/time_time_ns.py.exp similarity index 100% rename from tests/extmod/utime_time_ns.py.exp rename to tests/extmod/time_time_ns.py.exp diff --git a/tests/extmod/uasyncio_task_exception.py b/tests/extmod/uasyncio_task_exception.py deleted file mode 100644 index 9fb26857b5..0000000000 --- a/tests/extmod/uasyncio_task_exception.py +++ /dev/null @@ -1,39 +0,0 @@ -# In MicroPython, a non-awaited task with a pending exception will raise to -# the loop's exception handler the second time it is scheduled. This is -# because without reference counting we have no way to know when the task is -# truly "non awaited" -- i.e. we only know that it wasn't awaited in the time -# it took to be re-scheduled. - -# If the task _is_ subsequently awaited, then the await should succeed without -# raising. - -try: - import asyncio -except ImportError: - print("SKIP") - raise SystemExit - - -def custom_handler(loop, context): - print("exception handler", type(context["exception"]).__name__) - - -async def main(): - loop = asyncio.get_event_loop() - loop.set_exception_handler(custom_handler) - - async def task(): - print("raise") - raise OSError - - print("create") - t = asyncio.create_task(task()) - print("sleep 1") - await asyncio.sleep(0) - print("sleep 2") - await asyncio.sleep(0) - print("await") - await t # should not raise. - - -asyncio.run(main()) diff --git a/tests/extmod/uasyncio_task_exception.py.exp b/tests/extmod/uasyncio_task_exception.py.exp deleted file mode 100644 index 44dae61e1c..0000000000 --- a/tests/extmod/uasyncio_task_exception.py.exp +++ /dev/null @@ -1,6 +0,0 @@ -create -sleep 1 -raise -sleep 2 -exception handler OSError -await diff --git a/tests/extmod/uctypes_array_assign_le.py b/tests/extmod/uctypes_array_assign_le.py index d822faf7e8..b7fcdfb49c 100644 --- a/tests/extmod/uctypes_array_assign_le.py +++ b/tests/extmod/uctypes_array_assign_le.py @@ -18,7 +18,7 @@ desc = { "arr8": (uctypes.ARRAY | 1, 1, {"l": uctypes.UINT32 | 0}), } -data = bytearray(5) +data = bytearray(6) S = uctypes.struct(uctypes.addressof(data), desc, uctypes.LITTLE_ENDIAN) diff --git a/tests/extmod/ussl_poll.py.exp b/tests/extmod/ussl_poll.py.exp deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/extmod/uzlib_decompio.py.exp b/tests/extmod/uzlib_decompio.py.exp deleted file mode 100644 index 3f5f360fa3..0000000000 --- a/tests/extmod/uzlib_decompio.py.exp +++ /dev/null @@ -1,12 +0,0 @@ -0 -b'h' -2 -b'el' -b'lo' -7 -b'' -b'' -7 -b'0000000000' -b'000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000' -OSError(22,) diff --git a/tests/extmod/uzlib_decompio_gz.py.exp b/tests/extmod/uzlib_decompio_gz.py.exp deleted file mode 100644 index 20a30c82a3..0000000000 --- a/tests/extmod/uzlib_decompio_gz.py.exp +++ /dev/null @@ -1,13 +0,0 @@ -16 -b'h' -18 -b'el' -b'lo' -31 -b'' -b'' -31 -b'hello' -b'hello' -ValueError -OSError(22,) diff --git a/tests/extmod/vfs_blockdev.py b/tests/extmod/vfs_blockdev.py index 333ac7f183..f4c84ea921 100644 --- a/tests/extmod/vfs_blockdev.py +++ b/tests/extmod/vfs_blockdev.py @@ -65,6 +65,8 @@ def test(bdev, vfs_class): try: + import os + bdev = RAMBlockDevice(50) except MemoryError: print("SKIP") diff --git a/tests/extmod/vfs_fat_fileio2.py b/tests/extmod/vfs_fat_fileio2.py index 7aed7c4ce4..0e3808c314 100644 --- a/tests/extmod/vfs_fat_fileio2.py +++ b/tests/extmod/vfs_fat_fileio2.py @@ -22,13 +22,11 @@ class RAMFS: # print("readblocks(%s, %x(%d))" % (n, id(buf), len(buf))) for i in range(len(buf)): buf[i] = self.data[n * self.SEC_SIZE + i] - return 0 def writeblocks(self, n, buf): # print("writeblocks(%s, %x)" % (n, id(buf))) for i in range(len(buf)): self.data[n * self.SEC_SIZE + i] = buf[i] - return 0 def ioctl(self, op, arg): # print("ioctl(%d, %r)" % (op, arg)) @@ -69,6 +67,7 @@ try: except OSError as e: print(e.errno == errno.ENOENT) +# CIRCUITPY test try: vfs.rename("foo_dir", "foo_dir/inside_itself") except OSError as e: diff --git a/tests/extmod/vfs_fat_finaliser.py b/tests/extmod/vfs_fat_finaliser.py index 37929fd1e3..a2cce7846c 100644 --- a/tests/extmod/vfs_fat_finaliser.py +++ b/tests/extmod/vfs_fat_finaliser.py @@ -31,6 +31,8 @@ class RAMBlockDevice: # Create block device, and skip test if not enough RAM try: + import errno, os + bdev = RAMBlockDevice(50) except MemoryError: print("SKIP") @@ -48,6 +50,8 @@ import micropython micropython.heap_lock() try: + import errno, os + vfs.open("x", "r") except MemoryError: print("MemoryError") @@ -60,7 +64,7 @@ import gc # in turn allocate new qstrs and/or a new qstr pool). f = None n = None -names = ["x%d" % i for i in range(4)] +names = ["x%d" % i for i in range(5)] # Do a large number of single-block allocations to move the GC head forwards, # ensuring that the files are allocated from never-before-used blocks and @@ -70,12 +74,13 @@ for i in range(1024): [] # Run the test: create files without closing them, run GC, then read back files. +# Only read back N-1 files because the last one may not be finalised due to +# references to it being left on the C stack. for n in names: f = vfs.open(n, "w") f.write(n) f = None # release f without closing - sorted([0, 1, 2, 3, 4, 5], key=lambda x: x) # use up Python and C stack so f is really gone -gc.collect() # should finalise all N files by closing them -for n in names: +gc.collect() # should finalise at least the first N-1 files by closing them +for n in names[:-1]: with vfs.open(n, "r") as f: print(f.read()) diff --git a/tests/extmod/vfs_fat_ilistdir_del.py b/tests/extmod/vfs_fat_ilistdir_del.py index ccdacc57c2..055836ad71 100644 --- a/tests/extmod/vfs_fat_ilistdir_del.py +++ b/tests/extmod/vfs_fat_ilistdir_del.py @@ -2,9 +2,9 @@ import gc try: - import uos + import os - uos.VfsFat + os.VfsFat except (ImportError, AttributeError): print("SKIP") raise SystemExit @@ -77,4 +77,4 @@ except MemoryError: print("SKIP") raise SystemExit -test(bdev, uos.VfsFat) +test(bdev, os.VfsFat) diff --git a/tests/extmod/vfs_lfs_ilistdir_del.py b/tests/extmod/vfs_lfs_ilistdir_del.py index 073576986d..ff66717147 100644 --- a/tests/extmod/vfs_lfs_ilistdir_del.py +++ b/tests/extmod/vfs_lfs_ilistdir_del.py @@ -2,9 +2,9 @@ import gc try: - import uos + import os - uos.VfsLfs2 + os.VfsLfs2 except (ImportError, AttributeError): print("SKIP") raise SystemExit @@ -72,4 +72,4 @@ def test(bdev, vfs_class): bdev = RAMBlockDevice(30) -test(bdev, uos.VfsLfs2) +test(bdev, os.VfsLfs2) diff --git a/tests/extmod/vfs_lfs_mtime.py b/tests/extmod/vfs_lfs_mtime.py index 0a330b3e94..2f8bfae555 100644 --- a/tests/extmod/vfs_lfs_mtime.py +++ b/tests/extmod/vfs_lfs_mtime.py @@ -107,4 +107,4 @@ except MemoryError: print("SKIP") raise SystemExit -test(bdev, uos.VfsLfs2) +test(bdev, os.VfsLfs2) diff --git a/tests/extmod/vfs_posix.py b/tests/extmod/vfs_posix.py index 3723339c1e..05dc0f537e 100644 --- a/tests/extmod/vfs_posix.py +++ b/tests/extmod/vfs_posix.py @@ -13,6 +13,8 @@ except (ImportError, AttributeError): # Skip the test if it does exist. temp_dir = "micropy_test_dir" try: + import os + os.stat(temp_dir) print("SKIP") raise SystemExit @@ -103,6 +105,8 @@ print(os.listdir(temp_dir)) # remove with error try: + import os + os.remove(temp_dir + "/test2") except OSError: print("remove OSError") @@ -113,6 +117,8 @@ print(temp_dir in os.listdir()) # rmdir with error try: + import os + os.rmdir(temp_dir) except OSError: print("rmdir OSError") diff --git a/tests/extmod/vfs_userfs.py b/tests/extmod/vfs_userfs.py index 518373c70a..36f1088870 100644 --- a/tests/extmod/vfs_userfs.py +++ b/tests/extmod/vfs_userfs.py @@ -68,6 +68,8 @@ user_files = { "/data.txt": b"some data in a text file", "/usermod1.py": b"print('in usermod1')\nimport usermod2", "/usermod2.py": b"print('in usermod2')", + "/usermod3.py": b"syntax error", + "/usermod4.mpy": b"syntax error", } os.mount(UserFS(user_files), "/userfs") @@ -79,6 +81,18 @@ print(f.read()) sys.path.append("/userfs") import usermod1 +# import a .py file with a syntax error (file should be closed on error) +try: + import usermod3 +except SyntaxError: + print("SyntaxError in usermod3") + +# import a .mpy file with a syntax error (file should be closed on error) +try: + import usermod4 +except ValueError: + print("ValueError in usermod4") + # unmount and undo path addition os.umount("/userfs") sys.path.pop() diff --git a/tests/extmod/vfs_userfs.py.exp b/tests/extmod/vfs_userfs.py.exp index 00ddd95fca..05cff0452c 100644 --- a/tests/extmod/vfs_userfs.py.exp +++ b/tests/extmod/vfs_userfs.py.exp @@ -10,3 +10,14 @@ stat /usermod2.py open /usermod2.py rb ioctl 4 0 in usermod2 +stat /usermod3 +stat /usermod3.py +open /usermod3.py rb +ioctl 4 0 +SyntaxError in usermod3 +stat /usermod4 +stat /usermod4.py +stat /usermod4.mpy +open /usermod4.mpy rb +ioctl 4 0 +ValueError in usermod4 diff --git a/tests/extmod/uzlib_decompio.py b/tests/extmod/zlib_decompio.py similarity index 100% rename from tests/extmod/uzlib_decompio.py rename to tests/extmod/zlib_decompio.py diff --git a/tests/extmod/uzlib_decompio_gz.py b/tests/extmod/zlib_decompio_gz.py similarity index 100% rename from tests/extmod/uzlib_decompio_gz.py rename to tests/extmod/zlib_decompio_gz.py diff --git a/tests/extmod/uzlib_decompress.py b/tests/extmod/zlib_decompress.py similarity index 100% rename from tests/extmod/uzlib_decompress.py rename to tests/extmod/zlib_decompress.py diff --git a/tests/feature_check/README b/tests/feature_check/README index 0f55e7ece4..3f4804cfc9 100644 --- a/tests/feature_check/README +++ b/tests/feature_check/README @@ -1,4 +1,4 @@ This directory doesn't contain real tests, but code snippets to detect -various interpreter features, which can't be/inconvenient to detected by +various interpreter features, which can't be/inconvenient to detect by other means. Scripts here are executed by run-tests.py at the beginning of testsuite to decide what other test groups to run/exclude. diff --git a/tests/float/array_construct.py b/tests/float/array_construct.py index f6a3a9dc9d..6b5c996312 100644 --- a/tests/float/array_construct.py +++ b/tests/float/array_construct.py @@ -1,13 +1,10 @@ # test construction of array from array with float type try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit print(array("f", array("h", [1, 2]))) print(array("d", array("f", [1, 2]))) diff --git a/tests/float/bytearray_construct_endian.py b/tests/float/bytearray_construct_endian.py index 47f2b793c0..a971d706e2 100644 --- a/tests/float/bytearray_construct_endian.py +++ b/tests/float/bytearray_construct_endian.py @@ -1,12 +1,9 @@ # test construction of bytearray from array with float type try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit print(bytearray(array("f", [1, 2.5]))) diff --git a/tests/float/bytes_construct_endian.py b/tests/float/bytes_construct_endian.py index 4e15acc8bc..4dbcf390ea 100644 --- a/tests/float/bytes_construct_endian.py +++ b/tests/float/bytes_construct_endian.py @@ -1,12 +1,9 @@ # test construction of bytes from array with float type try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit print(bytes(array("f", [1, 2.5]))) diff --git a/tests/float/float2int_doubleprec_intbig.py b/tests/float/float2int_doubleprec_intbig.py index e2876f5d7c..402966cace 100644 --- a/tests/float/float2int_doubleprec_intbig.py +++ b/tests/float/float2int_doubleprec_intbig.py @@ -1,7 +1,6 @@ # check cases converting float to int, requiring double precision float import struct - import sys maxsize_bits = 0 diff --git a/tests/float/float2int_fp30_intbig.py b/tests/float/float2int_fp30_intbig.py index 3d3795f94d..1b22fe9646 100644 --- a/tests/float/float2int_fp30_intbig.py +++ b/tests/float/float2int_fp30_intbig.py @@ -1,7 +1,6 @@ # check cases converting float to int, relying only on single precision float import struct - import sys maxsize_bits = 0 diff --git a/tests/float/float2int_intbig.py b/tests/float/float2int_intbig.py index a9d7b57211..d047f247f2 100644 --- a/tests/float/float2int_intbig.py +++ b/tests/float/float2int_intbig.py @@ -1,7 +1,6 @@ # check cases converting float to int, relying only on single precision float import struct - import sys maxsize_bits = 0 diff --git a/tests/float/float_array.py b/tests/float/float_array.py index 219b6b86ae..3d128da838 100644 --- a/tests/float/float_array.py +++ b/tests/float/float_array.py @@ -1,11 +1,8 @@ try: - from uarray import array + from array import array except ImportError: - try: - from array import array - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit def test(a): diff --git a/tests/float/float_format_ints.py b/tests/float/float_format_ints.py index 0bf4baf12d..df4444166c 100644 --- a/tests/float/float_format_ints.py +++ b/tests/float/float_format_ints.py @@ -12,12 +12,6 @@ for b in [13, 123, 457, 23456]: print(title, "with format", f_fmt, "gives", f_fmt.format(f)) print(title, "with format", g_fmt, "gives", g_fmt.format(f)) -# Check that powers of 10 (that fit in float32) format correctly. -for i in range(31): - # It works to 12 digits on all platforms *except* qemu-arm, where - # 10^11 comes out as 10000000820 or something. - print("{:.7g}".format(float("1e" + str(i)))) - # 16777215 is 2^24 - 1, the largest integer that can be completely held # in a float32. print("{:f}".format(16777215)) diff --git a/tests/float/float_format_ints_power10.py b/tests/float/float_format_ints_power10.py new file mode 100644 index 0000000000..98900c135b --- /dev/null +++ b/tests/float/float_format_ints_power10.py @@ -0,0 +1,8 @@ +# Test that integers format to exact values. +# This test requires at least 32-bit floats (won't work with 30-bit). + +# Check that powers of 10 (that fit in float32) format correctly. +for i in range(31): + # It works to 12 digits on all platforms *except* qemu-arm, where + # 10^11 comes out as 10000000820 or something. + print(i, "{:.7g}".format(float("1e" + str(i)))) diff --git a/tests/float/math_domain.py b/tests/float/math_domain.py index 606a655284..b1273c6c11 100644 --- a/tests/float/math_domain.py +++ b/tests/float/math_domain.py @@ -10,25 +10,25 @@ inf = float("inf") nan = float("nan") # single argument functions -for name, f, args in ( - ("fabs", math.fabs, ()), - ("ceil", math.ceil, ()), - ("floor", math.floor, ()), - ("trunc", math.trunc, ()), - ("sqrt", math.sqrt, (-1, 0)), - ("exp", math.exp, ()), - ("log", math.log, ()), - ("sin", math.sin, ()), - ("cos", math.cos, ()), - ("tan", math.tan, ()), - ("asin", math.asin, (-1.1, 1, 1.1)), - ("acos", math.acos, (-1.1, 1, 1.1)), - ("atan", math.atan, ()), - ("ldexp", lambda x: math.ldexp(x, 0), ()), - ("radians", math.radians, ()), - ("degrees", math.degrees, ()), +for name, f in ( + ("fabs", math.fabs), + ("ceil", math.ceil), + ("floor", math.floor), + ("trunc", math.trunc), + ("sqrt", math.sqrt), + ("exp", math.exp), + ("log", math.log), + ("sin", math.sin), + ("cos", math.cos), + ("tan", math.tan), + ("asin", math.asin), + ("acos", math.acos), + ("atan", math.atan), + ("ldexp", lambda x: math.ldexp(x, 0)), + ("radians", math.radians), + ("degrees", math.degrees), ): - for x in args + (inf, -inf, nan): + for x in (0, 1, 1.12, -1, -1.12, inf, -inf, nan): try: ans = "%.4f" % f(x) except ValueError: @@ -38,35 +38,54 @@ for name, f, args in ( print("%s(%.4f) = %s" % (name, x, ans)) # double argument functions -for name, f, args in ( - ( - "pow", - math.pow, - ( - (0, 2), - (-1, 2), - (0, -1), - (-1, 2.3), - (0.5, inf), - (-0.5, inf), - (0.5, -inf), - (-0.5, -inf), - (1.5, inf), - (-1.5, inf), - (1.5, -inf), - (-1.5, -inf), - (nan, 0), - (1, nan), - ), - ), - ("log", math.log, ()), - ("fmod", math.fmod, ((1.2, inf), (1.2, -inf), (1.2, 0), (inf, 1.2))), - ("atan2", math.atan2, ((0, 0), (-inf, inf), (-inf, -inf), (inf, -inf))), - ("copysign", math.copysign, ()), +for name, f in ( + ("pow", math.pow), + ("log", math.log), + ("fmod", math.fmod), + ("atan2", math.atan2), + ("copysign", math.copysign), ): - for x in args + ((0, inf), (inf, 0), (inf, inf), (inf, nan), (nan, inf), (nan, nan)): + for x in ( + (0, 0), + (0, 2), + (0, -1), + (1, 0), + (1.2, 0), + (-1, 0), + (-1, 2), + (-1, 2.3), + (0, inf), + (0.5, inf), + (0.5, -inf), + (0.9, inf), + (0.9, -inf), + (1.2, inf), + (1.2, -inf), + (-0.5, inf), + (-0.5, -inf), + (-0.9, inf), + (-0.9, -inf), + (-1.2, inf), + (-1.2, -inf), + (inf, 0), + (inf, 1.2), + (inf, -1.2), + (inf, inf), + (inf, -inf), + (-inf, inf), + (-inf, -inf), + (0, nan), + (nan, 0), + (1, nan), + (nan, 1), + (inf, nan), + (nan, inf), + (nan, nan), + ): try: ans = "%.4f" % f(*x) except ValueError: ans = "ValueError" + except ZeroDivisionError: + ans = "ZeroDivisionError" print("%s(%.4f, %.4f) = %s" % (name, x[0], x[1], ans)) diff --git a/tests/float/math_domain_python311.py b/tests/float/math_domain_python311.py new file mode 100644 index 0000000000..f56fd55852 --- /dev/null +++ b/tests/float/math_domain_python311.py @@ -0,0 +1,26 @@ +# Tests domain errors in math functions. +# This is split out from math_domain.py because math.pow(0, -inf) was changed +# in Python 3.11, and so this test requires a .py.exp file. +# (See https://github.com/python/cpython/issues/88505) + +try: + import math +except ImportError: + print("SKIP") + raise SystemExit + +inf = float("inf") + +for name, f in ( + ("pow", math.pow), + ("log", math.log), + ("fmod", math.fmod), + ("atan2", math.atan2), + ("copysign", math.copysign), +): + for x in ((0, -inf),): + try: + ans = "%.4f" % f(*x) + except ValueError: + ans = "ValueError" + print("%s(%.4f, %.4f) = %s" % (name, x[0], x[1], ans)) diff --git a/tests/float/math_domain_python311.py.exp b/tests/float/math_domain_python311.py.exp new file mode 100644 index 0000000000..dfc93bb510 --- /dev/null +++ b/tests/float/math_domain_python311.py.exp @@ -0,0 +1,5 @@ +pow(0.0000, -inf) = inf +log(0.0000, -inf) = ValueError +fmod(0.0000, -inf) = 0.0000 +atan2(0.0000, -inf) = 3.1416 +copysign(0.0000, -inf) = -0.0000 diff --git a/tests/import/broken/pkg2_and_zerodiv.py b/tests/import/broken/pkg2_and_zerodiv.py new file mode 100644 index 0000000000..3580628ff5 --- /dev/null +++ b/tests/import/broken/pkg2_and_zerodiv.py @@ -0,0 +1,2 @@ +import pkg2 +import broken.zerodiv diff --git a/tests/import/broken/zerodiv.py b/tests/import/broken/zerodiv.py new file mode 100644 index 0000000000..72dca4d5e4 --- /dev/null +++ b/tests/import/broken/zerodiv.py @@ -0,0 +1 @@ +1 / 0 diff --git a/tests/import/builtin_ext.py b/tests/import/builtin_ext.py new file mode 100644 index 0000000000..87465f1d59 --- /dev/null +++ b/tests/import/builtin_ext.py @@ -0,0 +1,40 @@ +# Verify that sys is a builtin. +import sys + +print(sys, hasattr(sys, "__file__")) + +sys.path.clear() +sys.path.append("ext") + +# All three should only get builtins, despite sys.py, usys.py, and +# micropython.py being in the path. +# usys isn't extensible, but has a special-cased alias for backwards +# compatibility. +import micropython + +print(micropython, hasattr(micropython, "__file__")) +import sys + +print(sys, hasattr(sys, "__file__")) +import usys + +print(usys, hasattr(usys, "__file__")) + +# This should get os.py, which uses uos to get the builtin. +import os + +print(os, hasattr(os, "__file__"), os.sep, os.extra) + +# This should get time.py, which uses empty sys.path to get the builtin. +import time + +print(time, hasattr(time, "__file__"), time.sleep, time.extra) + +# These should get the builtins. +import uos + +print(uos, hasattr(uos, "__file__"), hasattr(uos, "extra")) + +import utime + +print(utime, hasattr(utime, "__file__"), hasattr(utime, "extra")) diff --git a/tests/import/builtin_ext.py.exp b/tests/import/builtin_ext.py.exp new file mode 100644 index 0000000000..e09af2843a --- /dev/null +++ b/tests/import/builtin_ext.py.exp @@ -0,0 +1,10 @@ + False + False + False + False +os from filesystem + True / 1 +time from filesystem + True 1 + False False + False False diff --git a/tests/import/circular/main.py b/tests/import/circular/main.py new file mode 100644 index 0000000000..5d63d507c3 --- /dev/null +++ b/tests/import/circular/main.py @@ -0,0 +1,4 @@ +x = 1 +import circular.sub + +print(circular.sub.y) diff --git a/tests/import/circular/sub.py b/tests/import/circular/sub.py new file mode 100644 index 0000000000..50d7afe07b --- /dev/null +++ b/tests/import/circular/sub.py @@ -0,0 +1,3 @@ +from circular.main import x + +y = x + 20 diff --git a/tests/import/ext/micropython.py b/tests/import/ext/micropython.py new file mode 100644 index 0000000000..88c8b770e1 --- /dev/null +++ b/tests/import/ext/micropython.py @@ -0,0 +1,2 @@ +# micropython is always builtin and cannot be overriden by the filesystem. +print("ERROR: micropython from filesystem") diff --git a/tests/import/ext/os.py b/tests/import/ext/os.py new file mode 100644 index 0000000000..e1448805d0 --- /dev/null +++ b/tests/import/ext/os.py @@ -0,0 +1,5 @@ +print("os from filesystem") + +from uos import * + +extra = 1 diff --git a/tests/import/ext/sys.py b/tests/import/ext/sys.py new file mode 100644 index 0000000000..71ee633e11 --- /dev/null +++ b/tests/import/ext/sys.py @@ -0,0 +1,2 @@ +# sys is always builtin and cannot be overriden by the filesystem. +print("ERROR: sys from filesystem") diff --git a/tests/import/ext/time.py b/tests/import/ext/time.py new file mode 100644 index 0000000000..6b460b4235 --- /dev/null +++ b/tests/import/ext/time.py @@ -0,0 +1,14 @@ +print("time from filesystem") + +# Tests the CPython-compatible / non-u-prefix way of forcing a builtin +# import. +import sys + +_path = sys.path +sys.path = () +from time import * + +sys.path = _path +del _path + +extra = 1 diff --git a/tests/import/ext/usys.py b/tests/import/ext/usys.py new file mode 100644 index 0000000000..d7629a4499 --- /dev/null +++ b/tests/import/ext/usys.py @@ -0,0 +1,3 @@ +# usys (and any u-prefix) is always builtin and cannot be overriden by the +# filesystem. +print("ERROR: usys from filesystem") diff --git a/tests/import/import_broken.py b/tests/import/import_broken.py new file mode 100644 index 0000000000..3c7cf4a498 --- /dev/null +++ b/tests/import/import_broken.py @@ -0,0 +1,32 @@ +import sys, pkg + +# Modules we import are usually added to sys.modules. +print("pkg" in sys.modules) + +try: + from broken.zerodiv import x +except Exception as e: + print(e.__class__.__name__) + +# The broken module we tried to import should not be in sys.modules. +print("broken.zerodiv" in sys.modules) + +# If we try to import the module again, the code should +# run again and we should get the same error. +try: + from broken.zerodiv import x +except Exception as e: + print(e.__class__.__name__) + +# Import a module that successfully imports some other modules +# before importing the problematic module. +try: + import broken.pkg2_and_zerodiv +except ZeroDivisionError: + pass + +print("pkg2" in sys.modules) +print("pkg2.mod1" in sys.modules) +print("pkg2.mod2" in sys.modules) +print("broken.zerodiv" in sys.modules) +print("broken.pkg2_and_zerodiv" in sys.modules) diff --git a/tests/import/import_circular.py b/tests/import/import_circular.py new file mode 100644 index 0000000000..388efdd130 --- /dev/null +++ b/tests/import/import_circular.py @@ -0,0 +1 @@ +import circular.main diff --git a/tests/import/import_pkg9.py b/tests/import/import_pkg9.py new file mode 100644 index 0000000000..4de028494f --- /dev/null +++ b/tests/import/import_pkg9.py @@ -0,0 +1,16 @@ +# tests that import only sets subpackage attribute on first import + +import pkg9 + +pkg9.mod1() +pkg9.mod2() + +import pkg9.mod1 + +pkg9.mod1() +pkg9.mod2() + +import pkg9.mod2 + +pkg9.mod1() +print(pkg9.mod2.__name__, type(pkg9.mod2).__name__) diff --git a/tests/import/pkg9/__init__.py b/tests/import/pkg9/__init__.py new file mode 100644 index 0000000000..5d08d4b4a9 --- /dev/null +++ b/tests/import/pkg9/__init__.py @@ -0,0 +1,5 @@ +from .mod1 import mod1 + + +def mod2(): + print("mod2") diff --git a/tests/import/pkg9/mod1.py b/tests/import/pkg9/mod1.py new file mode 100644 index 0000000000..7e7066bada --- /dev/null +++ b/tests/import/pkg9/mod1.py @@ -0,0 +1,2 @@ +def mod1(): + print("mod1") diff --git a/tests/import/pkg9/mod2.py b/tests/import/pkg9/mod2.py new file mode 100644 index 0000000000..f4b3e265fb --- /dev/null +++ b/tests/import/pkg9/mod2.py @@ -0,0 +1 @@ +from . import mod2 diff --git a/tests/io/file1.py b/tests/io/file1.py index de30045d31..1274a653d1 100644 --- a/tests/io/file1.py +++ b/tests/io/file1.py @@ -1,20 +1,20 @@ -f = open("io/data/file1") +f = open("data/file1") print(f.read(5)) print(f.readline()) print(f.read()) -f = open("io/data/file1") +f = open("data/file1") print(f.readlines()) -f = open("io/data/file1", "r") +f = open("data/file1", "r") print(f.readlines()) -f = open("io/data/file1", "rb") +f = open("data/file1", "rb") print(f.readlines()) -f = open("io/data/file1", mode="r") +f = open("data/file1", mode="r") print(f.readlines()) -f = open("io/data/file1", mode="rb") +f = open("data/file1", mode="rb") print(f.readlines()) # write() error -f = open("io/data/file1", "r") +f = open("data/file1", "r") try: f.write("x") except OSError: @@ -22,7 +22,7 @@ except OSError: f.close() # read(n) error on binary file -f = open("io/data/file1", "ab") +f = open("data/file1", "ab") try: f.read(1) except OSError: @@ -30,7 +30,7 @@ except OSError: f.close() # read(n) error on text file -f = open("io/data/file1", "at") +f = open("data/file1", "at") try: f.read(1) except OSError: @@ -38,7 +38,7 @@ except OSError: f.close() # read() w/o args error -f = open("io/data/file1", "ab") +f = open("data/file1", "ab") try: f.read() except OSError: diff --git a/tests/io/file_iter.py b/tests/io/file_iter.py index 48e8739966..26e82b9b1a 100644 --- a/tests/io/file_iter.py +++ b/tests/io/file_iter.py @@ -1,3 +1,3 @@ -f = open("io/data/file1") +f = open("data/file1") for l in f: print(l) diff --git a/tests/io/file_long_read.py b/tests/io/file_long_read.py index 8bdd484504..3f57d5594d 100644 --- a/tests/io/file_long_read.py +++ b/tests/io/file_long_read.py @@ -1,3 +1,3 @@ -f = open("io/data/file1") +f = open("data/file1") b = f.read(100) print(len(b)) diff --git a/tests/io/file_long_read2.py b/tests/io/file_long_read2.py index 337a5fba96..ea87f91f10 100644 --- a/tests/io/file_long_read2.py +++ b/tests/io/file_long_read2.py @@ -1,4 +1,4 @@ -f = open("io/data/bigfile1") +f = open("data/bigfile1") b = f.read() print(len(b)) print(b) diff --git a/tests/io/file_long_read3.py b/tests/io/file_long_read3.py index d8b0cce550..1ea47e1853 100644 --- a/tests/io/file_long_read3.py +++ b/tests/io/file_long_read3.py @@ -1,4 +1,4 @@ -f = open("io/data/bigfile1", "rb") +f = open("data/bigfile1", "rb") b = f.read(512) print(len(b)) print(b) diff --git a/tests/io/file_readinto.py b/tests/io/file_readinto.py index 1f3702a217..f9004013d9 100644 --- a/tests/io/file_readinto.py +++ b/tests/io/file_readinto.py @@ -1,13 +1,13 @@ b = bytearray(30) -f = open("io/data/file1", "rb") +f = open("data/file1", "rb") print(f.readinto(b)) print(b) -f = open("io/data/file2", "rb") +f = open("data/file2", "rb") print(f.readinto(b)) print(b) # readinto() on writable file -f = open("io/data/file1", "ab") +f = open("data/file1", "ab") try: f.readinto(bytearray(4)) except OSError: diff --git a/tests/io/file_readinto_len.py b/tests/io/file_readinto_len.py index 84cc8cf5e1..d6eb1dfc41 100644 --- a/tests/io/file_readinto_len.py +++ b/tests/io/file_readinto_len.py @@ -1,10 +1,10 @@ b = bytearray(30) -f = open("io/data/file1", "rb") +f = open("data/file1", "rb") # 2nd arg (length to read) is extension to CPython print(f.readinto(b, 8)) print(b) b = bytearray(4) -f = open("io/data/file1", "rb") +f = open("data/file1", "rb") print(f.readinto(b, 8)) print(b) diff --git a/tests/io/file_readline.py b/tests/io/file_readline.py index 86d010eaf6..3d270db514 100644 --- a/tests/io/file_readline.py +++ b/tests/io/file_readline.py @@ -1,4 +1,4 @@ -f = open("io/data/file1") +f = open("data/file1") print(f.readline()) print(f.readline(3)) print(f.readline(4)) @@ -6,7 +6,7 @@ print(f.readline(5)) print(f.readline()) # readline() on writable file -f = open("io/data/file1", "ab") +f = open("data/file1", "ab") try: f.readline() except OSError: diff --git a/tests/io/file_seek.py b/tests/io/file_seek.py index 2fe57692c6..3990df8409 100644 --- a/tests/io/file_seek.py +++ b/tests/io/file_seek.py @@ -1,4 +1,4 @@ -f = open("io/data/file1", "rb") +f = open("data/file1", "rb") print(f.seek(6)) print(f.read(5)) print(f.tell()) @@ -18,14 +18,14 @@ print(f.tell()) f.close() # test text mode -f = open("io/data/file1", "rt") +f = open("data/file1", "rt") print(f.seek(6)) print(f.read(5)) print(f.tell()) f.close() # seek closed file -f = open("io/data/file1", "r") +f = open("data/file1", "r") f.close() try: f.seek(1) diff --git a/tests/io/file_stdio.py b/tests/io/file_stdio.py index cbdb070163..d714bffd4d 100644 --- a/tests/io/file_stdio.py +++ b/tests/io/file_stdio.py @@ -2,3 +2,4 @@ import sys print(sys.stdin.fileno()) print(sys.stdout.fileno()) +print(sys.stderr.fileno()) diff --git a/tests/io/file_stdio2.py b/tests/io/file_stdio2.py new file mode 100644 index 0000000000..5b8a5a7692 --- /dev/null +++ b/tests/io/file_stdio2.py @@ -0,0 +1,65 @@ +# Test sys.std*.buffer objects. + +import sys + +try: + sys.stdout.buffer + sys.stdin.buffer + sys.stderr.buffer +except AttributeError: + print("SKIP") + raise SystemExit + + +# force cpython to flush after every print +# this is to sequence stdout and stderr +def print_flush(*args, **kwargs): + try: + print(*args, **kwargs, flush=True) + except TypeError: + print(*args, **kwargs) + + +print_flush("==stdin==") +print_flush(sys.stdin.buffer.fileno()) + + +print_flush("==stdout==") +print_flush(sys.stdout.buffer.fileno()) +n_text = sys.stdout.write("The quick brown fox jumps over the lazy dog\n") +sys.stdout.flush() +n_binary = sys.stdout.buffer.write("The quick brown fox jumps over the lazy dog\n".encode("utf-8")) +sys.stdout.buffer.flush() +print_flush("n_text:{} n_binary:{}".format(n_text, n_binary)) + +# temporarily disabling unicode tests until future PR which fixes unicode write character count +# n_text = sys.stdout.write("🚀") +# sys.stdout.flush() +# n_binary = sys.stdout.buffer.write("🚀".encode("utf-8")) +# sys.stdout.buffer.flush() +# print_flush("") +# print_flush("n_text:{} n_binary:{}".format(n_text, n_binary)) +# n_text = sys.stdout.write("1🚀2a3α4b5β6c7γ8d9δ0ぁ1🙐") +# sys.stdout.flush() +# n_binary = sys.stdout.buffer.write("1🚀2a3α4b5β6c7γ8d9δ0ぁ1🙐".encode("utf-8")) +# sys.stdout.buffer.flush() +# print_flush("") +# print_flush("n_text:{} n_binary:{}".format(n_text, n_binary)) + + +print_flush("==stderr==") +print_flush(sys.stderr.buffer.fileno()) +n_text = sys.stderr.write("The quick brown fox jumps over the lazy dog\n") +sys.stderr.flush() +n_binary = sys.stderr.buffer.write("The quick brown fox jumps over the lazy dog\n".encode("utf-8")) +sys.stderr.buffer.flush() +print_flush("n_text:{} n_binary:{}".format(n_text, n_binary)) + +# temporarily disabling unicode tests until future PR which fixes unicode write character count +# n_text = sys.stderr.write("🚀") +# sys.stderr.flush() +# n_binary = sys.stderr.buffer.write("🚀".encode("utf-8")) +# sys.stderr.buffer.flush() +# print_flush("") +# print_flush("n_text:{} n_binary:{}".format(n_text, n_binary)) +# print_flush("") diff --git a/tests/io/file_with.py b/tests/io/file_with.py index 899c0f9287..d5217dfe96 100644 --- a/tests/io/file_with.py +++ b/tests/io/file_with.py @@ -1,4 +1,4 @@ -f = open("io/data/file1") +f = open("data/file1") with f as f2: print(f2.read()) diff --git a/tests/micropython/builtin_execfile.py b/tests/micropython/builtin_execfile.py index 8a8ce79f78..5a26ccf0ad 100644 --- a/tests/micropython/builtin_execfile.py +++ b/tests/micropython/builtin_execfile.py @@ -1,17 +1,17 @@ # Test builtin execfile function using VFS. try: - import uio, uos + import io, os execfile - uio.IOBase - uos.mount + io.IOBase + os.mount except (ImportError, NameError, AttributeError): print("SKIP") raise SystemExit -class File(uio.IOBase): +class File(io.IOBase): def __init__(self, data): self.data = data self.off = 0 @@ -44,21 +44,25 @@ class Filesystem: # First umount any existing mount points the target may have. try: - uos.umount("/") + import io, os + + os.umount("/") except OSError: pass -for path in uos.listdir("/"): - uos.umount("/" + path) +for path in os.listdir("/"): + os.umount("/" + path) # Create and mount the VFS object. files = { "/test.py": "print(123)", } fs = Filesystem(files) -uos.mount(fs, "/test_mnt") +os.mount(fs, "/test_mnt") # Test execfile with a file that doesn't exist. try: + import io, os + execfile("/test_mnt/noexist.py") except OSError: print("OSError") @@ -67,4 +71,4 @@ except OSError: execfile("/test_mnt/test.py") # Unmount the VFS object. -uos.umount(fs) +os.umount(fs) diff --git a/tests/micropython/emg_exc.py b/tests/micropython/emg_exc.py index e26fd62e49..43b9044d19 100644 --- a/tests/micropython/emg_exc.py +++ b/tests/micropython/emg_exc.py @@ -24,6 +24,7 @@ def f(): exc = er micropython.heap_unlock() + # CIRCUITPY print(repr(exc)) diff --git a/tests/micropython/import_mpy_native_gc.py b/tests/micropython/import_mpy_native_gc.py index 0e489c74e2..a945910f5c 100644 --- a/tests/micropython/import_mpy_native_gc.py +++ b/tests/micropython/import_mpy_native_gc.py @@ -47,6 +47,10 @@ class UserFS: # Pre-compiled examples/natmod/features0 example for various architectures, keyed # by the required value of sys.implementation._mpy (without sub-version). +# cd examples/natmod/features0 +# make clean +# make ARCH=x64 # or ARCH=armv6m +# cat features0.mpy | python -c 'import sys; print(sys.stdin.buffer.read())' features0_file_contents = { # -march=x64 0x806: b'C\x06\t\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \x11$\r&\xa5 \x01"\xff', diff --git a/tests/misc/non_compliant.py b/tests/misc/non_compliant.py index 4c06977d11..da90f90ac3 100644 --- a/tests/misc/non_compliant.py +++ b/tests/misc/non_compliant.py @@ -55,7 +55,7 @@ except NotImplementedError: try: str(b"abc", encoding="utf8") except NotImplementedError: - print("TypeError") + print("NotImplementedError") # str.rsplit(None, n) not implemented try: @@ -105,6 +105,12 @@ try: except NotImplementedError: print("NotImplementedError") +# struct pack with too many args, not checked by uPy +print(struct.pack("bb", 1, 2, 3)) + +# struct pack with too few args, not checked by uPy +print(struct.pack("bb", 1)) + # array slice assignment with unsupported RHS try: bytearray(4)[0:1] = [1, 2] diff --git a/tests/misc/sys_settrace_features.py b/tests/misc/sys_settrace_features.py index 8a38b7534f..8ca6b382e3 100644 --- a/tests/misc/sys_settrace_features.py +++ b/tests/misc/sys_settrace_features.py @@ -22,7 +22,9 @@ def print_stacktrace(frame, level=0): frame.f_code.co_name, # Keep just the filename. "sys_settrace_" + frame.f_code.co_filename.split("sys_settrace_")[-1], - frame.f_lineno, + max( + 1, frame.f_lineno + ), # CPython 3.11 emits `0` where CPython 3.6 emits `1` for the "call" event corresponding to import. ) ) @@ -65,6 +67,11 @@ def trace_tick_handler(frame, event, arg): if any(name in frame_name for name in to_ignore): return + # Lines 4,5,7 create the `const` lambda, and line `15` is a `_X = const()` which + # MicroPython will not see as it's optimised out. + if "sys_settrace_importme" in frame.f_code.co_filename and frame.f_lineno in (4, 5, 7, 15): + return trace_tick_handler + print("### trace_handler::main event:", event) __prof__.trace_tick(frame, event, arg) diff --git a/tests/misc/sys_settrace_subdir/sys_settrace_importme.py b/tests/misc/sys_settrace_subdir/sys_settrace_importme.py index de561ef217..fdfa06134e 100644 --- a/tests/misc/sys_settrace_subdir/sys_settrace_importme.py +++ b/tests/misc/sys_settrace_subdir/sys_settrace_importme.py @@ -3,12 +3,18 @@ print("Yep, I got imported.") try: x = const(1) except NameError: - print("const not defined") + # Either running on CPython or MICROPY_COMP_CONST disabled. + const = lambda x: x -const = lambda x: x +# No const optimisation. _CNT01 = "CONST01" + +# Const assigned to an underscore name. Invisible to MicroPython with +# MICROPY_COMP_CONST enabled. _CNT02 = const(123) + +# Consts assigned to regular name, executed normally. A123 = const(123) a123 = const(123) diff --git a/tests/perf_bench/benchrun.py b/tests/perf_bench/benchrun.py index 6e6135fd98..9c55f338cc 100644 --- a/tests/perf_bench/benchrun.py +++ b/tests/perf_bench/benchrun.py @@ -1,6 +1,6 @@ def bm_run(N, M): try: - from utime import ticks_us, ticks_diff + from time import ticks_us, ticks_diff except ImportError: import time diff --git a/tests/perf_bench/core_import_mpy_multi.py b/tests/perf_bench/core_import_mpy_multi.py index 0f37aedf03..364c325042 100644 --- a/tests/perf_bench/core_import_mpy_multi.py +++ b/tests/perf_bench/core_import_mpy_multi.py @@ -1,6 +1,6 @@ # Test performance of importing an .mpy file many times. -import usys, io, os +import sys, io, os if not (hasattr(io, "IOBase") and hasattr(os, "mount")): print("SKIP") @@ -47,7 +47,7 @@ class FS: pass def stat(self, path): - if path == "__injected.mpy": + if path == "/__injected.mpy": return tuple(0 for _ in range(10)) else: raise OSError(-2) # ENOENT @@ -58,13 +58,13 @@ class FS: def mount(): os.mount(FS(), "/__remote") - os.chdir("/__remote") + sys.path.insert(0, "/__remote") def test(r): global result for _ in r: - usys.modules.clear() + sys.modules.clear() module = __import__("__injected") result = module.result diff --git a/tests/perf_bench/core_import_mpy_single.py b/tests/perf_bench/core_import_mpy_single.py index 1b411fc3fb..5757c3eaf1 100644 --- a/tests/perf_bench/core_import_mpy_single.py +++ b/tests/perf_bench/core_import_mpy_single.py @@ -2,7 +2,7 @@ # The first import of a module will intern strings that don't already exist, and # this test should be representative of what happens in a real application. -import io, os +import io, os, sys if not (hasattr(io, "IOBase") and hasattr(os, "mount")): print("SKIP") @@ -102,7 +102,7 @@ class FS: pass def stat(self, path): - if path == "__injected.mpy": + if path == "/__injected.mpy": return tuple(0 for _ in range(10)) else: raise OSError(-2) # ENOENT @@ -113,7 +113,7 @@ class FS: def mount(): os.mount(FS(), "/__remote") - os.chdir("/__remote") + sys.path.insert(0, "/__remote") def test(): diff --git a/tests/run-internalbench.py b/tests/run-internalbench.py index 2cf7f3443b..c9f783e474 100755 --- a/tests/run-internalbench.py +++ b/tests/run-internalbench.py @@ -8,16 +8,11 @@ import re from glob import glob from collections import defaultdict -# Tests require at least CPython 3.3. If your default python3 executable -# is of lower version, you can point MICROPY_CPYTHON3 environment var -# to the correct executable. if os.name == "nt": - CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3.exe") MICROPYTHON = os.getenv( "MICROPY_MICROPYTHON", "../ports/windows/build-standard/micropython.exe" ) else: - CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3") MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/unix/build-standard/micropython") diff --git a/tests/run-multitests.py b/tests/run-multitests.py index 81db31ea94..5ae8b8ea48 100755 --- a/tests/run-multitests.py +++ b/tests/run-multitests.py @@ -162,6 +162,14 @@ class PyInstanceSubProcess(PyInstance): def __str__(self): return self.argv[0].rsplit("/")[-1] + def prepare_script_from_file(self, filename, prepend, append): + # Make tests run in an isolated environment (i.e. `import io` would + # otherwise get the `tests/io` directory). + remove_cwd_from_sys_path = b"import sys\nsys.path.remove('')\n\n" + return remove_cwd_from_sys_path + super().prepare_script_from_file( + filename, prepend, append + ) + def run_script(self, script): output = b"" err = None @@ -582,7 +590,7 @@ def main(): cmd_args = cmd_parser.parse_args() # clear search path to make sure tests use only builtin modules and those in extmod - os.environ["MICROPYPATH"] = os.pathsep.join(("", ".frozen", "../extmod")) + os.environ["MICROPYPATH"] = os.pathsep.join((".frozen", "../extmod")) test_files = prepare_test_file_list(cmd_args.files) max_instances = max(t[1] for t in test_files) diff --git a/tests/run-natmodtests.py b/tests/run-natmodtests.py index db093bc5c5..d6ec5fe525 100755 --- a/tests/run-natmodtests.py +++ b/tests/run-natmodtests.py @@ -17,11 +17,10 @@ NATMOD_EXAMPLE_DIR = "../examples/natmod/" # Supported tests and their corresponding mpy module TEST_MAPPINGS = { - "uheapq": "uheapq/uheapq_$(ARCH).mpy", - "urandom": "urandom/urandom_$(ARCH).mpy", - "ure": "ure/ure_$(ARCH).mpy", - "zlib": "zlib/zlib_$(ARCH).mpy", -} + "heapq": "heapq/heapq_$(ARCH).mpy", + "random": "random/random_$(ARCH).mpy", + "re": "re/re_$(ARCH).mpy", + "zlib": "zlib/zlib_$(ARCH).mpy",} # Code to allow a target MicroPython to import an .mpy from RAM injected_import_hook_code = """\ @@ -41,14 +40,14 @@ class __FS: def chdir(self, path): pass def stat(self, path): - if path == '__injected.mpy': + if path == '/__injected.mpy': return tuple(0 for _ in range(10)) else: raise OSError(-2) # ENOENT def open(self, path, mode): return __File() os.mount(__FS(), '/__remote') -os.chdir('/__remote') +sys.path.insert(0, '/__remote') sys.modules['{}'] = __import__('__injected') """ @@ -106,9 +105,10 @@ def run_tests(target_truth, target, args, stats): test_file_data = f.read() # Create full test with embedded .mpy + test_script = b"import sys\nsys.path.remove('')\n\n" try: with open(NATMOD_EXAMPLE_DIR + test_mpy, "rb") as f: - test_script = b"__buf=" + bytes(repr(f.read()), "ascii") + b"\n" + test_script += b"__buf=" + bytes(repr(f.read()), "ascii") + b"\n" except OSError: print("---- {} - mpy file not compiled".format(test_file)) continue diff --git a/tests/run-perfbench.py b/tests/run-perfbench.py index 578f975bb8..81d873c459 100755 --- a/tests/run-perfbench.py +++ b/tests/run-perfbench.py @@ -109,8 +109,9 @@ def run_benchmarks(args, target, param_n, param_m, n_average, test_list): continue # Create test script + test_script = b"import sys\nsys.path.remove('')\n\n" with open(test_file, "rb") as f: - test_script = f.read() + test_script += f.read() with open(BENCH_SCRIPT_DIR + "benchrun.py", "rb") as f: test_script += f.read() test_script += b"bm_run(%u, %u)\n" % (param_n, param_m) diff --git a/tests/run-tests.py b/tests/run-tests.py index e2968d2d54..6b3d130ae5 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -56,8 +56,8 @@ os.environ["PYTHONIOENCODING"] = "utf-8" # Code to allow a target MicroPython to import an .mpy from RAM injected_import_hook_code = """\ -import usys, uos, uio -class __File(uio.IOBase): +import sys, os, io +class __File(io.IOBase): def __init__(self): self.off = 0 def ioctl(self, request, arg): @@ -80,8 +80,8 @@ class __FS: raise OSError(-2) # ENOENT def open(self, path, mode): return __File() -uos.mount(__FS(), '/__vfstest') -uos.chdir('/__vfstest') +os.mount(__FS(), '/__vfstest') +os.chdir('/__vfstest') __import__('__injected_test') """ @@ -261,29 +261,31 @@ def run_micropython(pyb, args, test_file, is_special=False): # a standard test run on PC # create system command - cmdlist = [MICROPYTHON, "-X", "emit=" + args.emit] + cmdlist = [os.path.abspath(MICROPYTHON), "-X", "emit=" + args.emit] if args.heapsize is not None: cmdlist.extend(["-X", "heapsize=" + args.heapsize]) if sys.platform == "darwin": cmdlist.extend(["-X", "realtime"]) + cwd = os.path.dirname(test_file) + # if running via .mpy, first compile the .py file if args.via_mpy: - mpy_modname = tempfile.mktemp(dir="") - mpy_filename = mpy_modname + ".mpy" + mpy_filename = tempfile.mktemp(dir=cwd, suffix=".mpy") subprocess.check_output( [MPYCROSS] + args.mpy_cross_flags.split() + ["-o", mpy_filename, "-X", "emit=" + args.emit, test_file] ) + mpy_modname = os.path.splitext(os.path.basename(mpy_filename))[0] cmdlist.extend(["-m", mpy_modname]) else: - cmdlist.append(test_file) + cmdlist.append(os.path.abspath(test_file)) # run the actual test try: output_mupy = subprocess.check_output( - cmdlist, stderr=subprocess.STDOUT, timeout=TEST_TIMEOUT + cmdlist, stderr=subprocess.STDOUT, timeout=TEST_TIMEOUT, cwd=cwd ) except subprocess.CalledProcessError as er: had_crash = True @@ -565,10 +567,9 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): for t in "bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le".split() } ) # requires uctypes - skip_tests.add("extmod/zlibd_decompress.py") # requires zlib - skip_tests.add("extmod/uheapq1.py") # uheapq not supported by WiPy - skip_tests.add("extmod/urandom_basic.py") # requires urandom - skip_tests.add("extmod/urandom_extra.py") # requires urandom + skip_tests.add("extmod/heapq1.py") # heapq not supported by WiPy + skip_tests.add("extmod/random_basic.py") # requires random + skip_tests.add("extmod/random_extra.py") # requires random elif args.target == "esp8266": skip_tests.add("misc/rge_sm.py") # too large elif args.target == "minimal": @@ -580,7 +581,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_tests.add("micropython/opt_level.py") # don't assume line numbers are stored elif args.target == "nrf": skip_tests.add("basics/memoryview1.py") # no item assignment for memoryview - skip_tests.add("extmod/urandom_basic.py") # unimplemented: urandom.seed + skip_tests.add("extmod/random_basic.py") # unimplemented: random.seed skip_tests.add("micropython/opt_level.py") # no support for line numbers skip_tests.add("misc/non_compliant.py") # no item assignment for bytearray for t in tests: @@ -588,7 +589,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_tests.add(t) elif args.target == "renesas-ra": skip_tests.add( - "extmod/utime_time_ns.py" + "extmod/time_time_ns.py" ) # RA fsp rtc function doesn't support nano sec info elif args.target == "qemu-arm": skip_tests.add("misc/print_exception.py") # requires sys stdfiles @@ -622,6 +623,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_tests.add("basics/sys_tracebacklimit.py") # requires traceback info skip_tests.add("basics/try_finally_return2.py") # requires raise_varargs skip_tests.add("basics/unboundlocal.py") # requires checking for unbound local + # CIRCUITPY skip_tests.update( ( "basics/chained_exception.py", @@ -629,10 +631,10 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): "circuitpython/traceback_test_chained.py", ) ) # because native doesn't have proper traceback info - skip_tests.add("extmod/uasyncio_event.py") # unknown issue - skip_tests.add("extmod/uasyncio_lock.py") # requires async with - skip_tests.add("extmod/uasyncio_micropython.py") # unknown issue - skip_tests.add("extmod/uasyncio_wait_for.py") # unknown issue + skip_tests.add("extmod/asyncio_event.py") # unknown issue + skip_tests.add("extmod/asyncio_lock.py") # requires async with + skip_tests.add("extmod/asyncio_micropython.py") # unknown issue + skip_tests.add("extmod/asyncio_wait_for.py") # unknown issue skip_tests.add("misc/features.py") # requires raise_varargs skip_tests.add( "misc/print_exception.py" @@ -677,7 +679,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray") is_set_type = test_name.startswith(("set_", "frozenset")) or test_name.endswith("_set") is_slice = test_name.find("slice") != -1 or test_name in misc_slice_tests - is_async = test_name.startswith(("async_", "uasyncio_")) + is_async = test_name.startswith(("async_", "asyncio_")) is_const = test_name.startswith("const") is_io_module = test_name.startswith("io_") is_fstring = test_name.startswith("string_fstring") @@ -716,7 +718,11 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): e = {"PYTHONPATH": "testlib", "PATH": os.environ["PATH"], "LANG": "en_US.UTF-8"} # run CPython to work out expected output try: - output_expected = subprocess.check_output(CPYTHON3_CMD + [test_file], env=e) + output_expected = subprocess.check_output( + CPYTHON3_CMD + [os.path.abspath(test_file)], + cwd=os.path.dirname(test_file), + stderr=subprocess.STDOUT, + ) if args.write_exp: with open(test_file_expected, "wb") as f: f.write(output_expected) @@ -1016,16 +1022,9 @@ the last matching regex is used: tests = args.files if not args.keep_path: - # clear search path to make sure tests use only builtin modules and those that can be frozen - os.environ["MICROPYPATH"] = os.pathsep.join( - [ - "", - "testlib", - ".frozen", - base_path("../frozen/Adafruit_CircuitPython_asyncio"), - base_path("../frozen/Adafruit_CircuitPython_Ticks"), - ] - ) + # clear search path to make sure tests use only builtin modules and those in extmod + os.environ["MICROPYPATH"] = ".frozen" + os.pathsep + base_path("../extmod") + try: os.makedirs(args.result_dir, exist_ok=True) res = run_tests(pyb, tests, args, args.result_dir, args.jobs) diff --git a/tests/stress/bytecode_limit.py b/tests/stress/bytecode_limit.py index 8cca413cf2..ad090637f6 100644 --- a/tests/stress/bytecode_limit.py +++ b/tests/stress/bytecode_limit.py @@ -3,7 +3,7 @@ body = " with f()()() as a:\n try:\n f()()()\n except Exception:\n pass\n" # Test overflow of jump offset. -for n in (430, 431, 432, 433): +for n in (433, 432, 431, 430): try: exec("cond = 0\nif cond:\n" + body * n + "else:\n print('cond false')\n") except MemoryError: diff --git a/tests/stress/bytecode_limit.py.exp b/tests/stress/bytecode_limit.py.exp index 74ab06c09b..1d892250b0 100644 --- a/tests/stress/bytecode_limit.py.exp +++ b/tests/stress/bytecode_limit.py.exp @@ -1,5 +1,5 @@ -cond false -cond false RuntimeError RuntimeError +cond false +cond false [123] diff --git a/tests/thread/stress_create.py b/tests/thread/stress_create.py index 877424cdf5..98f272f5cd 100644 --- a/tests/thread/stress_create.py +++ b/tests/thread/stress_create.py @@ -1,13 +1,12 @@ # stress test for creating many threads -try: - import utime - - sleep_ms = utime.sleep_ms -except ImportError: - import time +import time +if hasattr(time, "sleep_ms"): + sleep_ms = time.sleep_ms +else: sleep_ms = lambda t: time.sleep(t / 1000) + import _thread diff --git a/tests/thread/stress_schedule.py b/tests/thread/stress_schedule.py index 8be7f2d737..40191e6620 100644 --- a/tests/thread/stress_schedule.py +++ b/tests/thread/stress_schedule.py @@ -2,7 +2,7 @@ # while dealing with concurrent access from multiple threads. import _thread -import utime +import time import micropython import gc @@ -32,15 +32,15 @@ def thread(): micropython.schedule(task, None) except RuntimeError: # Queue full, back off. - utime.sleep_ms(10) + time.sleep_ms(10) for i in range(8): _thread.start_new_thread(thread, ()) # Wait up to 10 seconds for 10000 tasks to be scheduled. -t = utime.ticks_ms() -while n < _NUM_TASKS and utime.ticks_diff(utime.ticks_ms(), t) < _TIMEOUT_MS: +t = time.ticks_ms() +while n < _NUM_TASKS and time.ticks_diff(time.ticks_ms(), t) < _TIMEOUT_MS: pass if n < _NUM_TASKS: diff --git a/tests/thread/thread_exc2.py b/tests/thread/thread_exc2.py index 2863e1dec1..6f77bdbffa 100644 --- a/tests/thread/thread_exc2.py +++ b/tests/thread/thread_exc2.py @@ -1,5 +1,5 @@ # test raising exception within thread which is not caught -import utime +import time import _thread @@ -8,5 +8,5 @@ def thread_entry(): _thread.start_new_thread(thread_entry, ()) -utime.sleep(1) +time.sleep(1) print("done") diff --git a/tests/thread/thread_ident1.py b/tests/thread/thread_ident1.py index 1ba342e2b6..5ecf3774f1 100644 --- a/tests/thread/thread_ident1.py +++ b/tests/thread/thread_ident1.py @@ -7,7 +7,11 @@ import _thread +tid = None + + def thread_entry(): + global tid tid = _thread.get_ident() print("thread", type(tid) == int, tid != 0, tid != tid_main) global finished @@ -18,8 +22,9 @@ tid_main = _thread.get_ident() print("main", type(tid_main) == int, tid_main != 0) finished = False -_thread.start_new_thread(thread_entry, ()) +new_tid = _thread.start_new_thread(thread_entry, ()) while not finished: pass -print("done") + +print("done", type(new_tid) == int, new_tid == tid) diff --git a/tests/thread/thread_sleep1.py b/tests/thread/thread_sleep1.py index d81d537399..711e356220 100644 --- a/tests/thread/thread_sleep1.py +++ b/tests/thread/thread_sleep1.py @@ -4,13 +4,11 @@ # # SPDX-License-Identifier: MIT -try: - import utime - - sleep_ms = utime.sleep_ms -except ImportError: - import time +import time +if hasattr(time, "sleep_ms"): + sleep_ms = time.sleep_ms +else: sleep_ms = lambda t: time.sleep(t / 1000) import _thread diff --git a/tests/thread/thread_stacksize1.py b/tests/thread/thread_stacksize1.py index 2c70b90b5b..140d165cb3 100644 --- a/tests/thread/thread_stacksize1.py +++ b/tests/thread/thread_stacksize1.py @@ -1,8 +1,6 @@ # test setting the thread stack size # -# SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd -# -# SPDX-License-Identifier: MIT +# MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd import sys import _thread diff --git a/tests/unicode/file1.py b/tests/unicode/file1.py index 08b7d25041..6508b5f716 100644 --- a/tests/unicode/file1.py +++ b/tests/unicode/file1.py @@ -1,4 +1,4 @@ -f = open("unicode/data/utf-8_1.txt", encoding="utf-8") +f = open("data/utf-8_1.txt", encoding="utf-8") l = f.readline() print(l) print(len(l)) diff --git a/tests/unicode/file2.py b/tests/unicode/file2.py index 1a5b933c89..c35c18789c 100644 --- a/tests/unicode/file2.py +++ b/tests/unicode/file2.py @@ -6,7 +6,7 @@ def do(mode): enc = None else: enc = "utf-8" - f = open("unicode/data/utf-8_2.txt", mode=mode, encoding=enc) + f = open("data/utf-8_2.txt", mode=mode, encoding=enc) print(f.read(1)) print(f.read(1)) print(f.read(2)) diff --git a/tests/unicode/file_invalid.py b/tests/unicode/file_invalid.py index 3f7f184062..5c42302519 100644 --- a/tests/unicode/file_invalid.py +++ b/tests/unicode/file_invalid.py @@ -1,5 +1,5 @@ try: - f = open("unicode/data/utf-8_invalid.txt", encoding="utf-8") + f = open("data/utf-8_invalid.txt", encoding="utf-8") f.read() except UnicodeError: print("UnicodeError") diff --git a/tests/unicode/unicode.py b/tests/unicode/unicode.py index 8ac34ca282..072e049fde 100644 --- a/tests/unicode/unicode.py +++ b/tests/unicode/unicode.py @@ -19,7 +19,8 @@ print(enc, enc.decode() == s) # printing of unicode chars using repr # NOTE: for some characters (eg \u10ff) we differ to CPython -print(repr("a\u2000")) +print(repr("a\uffff")) +print(repr("a\U0001ffff")) # test invalid escape code try: diff --git a/tests/unicode/unicode_ure.py b/tests/unicode/unicode_ure.py index 5a5dc60054..c268b9e6fe 100644 --- a/tests/unicode/unicode_ure.py +++ b/tests/unicode/unicode_ure.py @@ -1,13 +1,10 @@ # test match.span() for unicode strings try: - import ure as re + import re except ImportError: - try: - import re - except ImportError: - print("SKIP") - raise SystemExit + print("SKIP") + raise SystemExit try: m = re.match(".", "a") diff --git a/tests/unix/extra_coverage.py b/tests/unix/extra_coverage.py index 9d9ccb75c1..0ea8f7886b 100644 --- a/tests/unix/extra_coverage.py +++ b/tests/unix/extra_coverage.py @@ -96,3 +96,20 @@ print(returns_NULL()) import frozentest print(frozentest.__file__) + +# test for builtin sub-packages +from example_package.foo import bar + +print(bar) +bar.f() +import example_package + +print(example_package, example_package.foo, example_package.foo.bar) +example_package.f() +example_package.foo.f() +example_package.foo.bar.f() +print(bar == example_package.foo.bar) +from example_package.foo import f as foo_f + +foo_f() +print(foo_f == example_package.foo.f) diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index bbea65fa1f..7c52eb19c8 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -15,12 +15,12 @@ false true abc % # GC -0x0 -0x0 +0 +0 # GC part 2 pass # tracked allocation -m_tracked_head = 0x0 +m_tracked_head = 0 0 1 1 1 2 1 @@ -37,7 +37,7 @@ m_tracked_head = 0x0 5 1 6 1 7 1 -m_tracked_head = 0x0 +m_tracked_head = 0 # vstr tests sts @@ -48,7 +48,7 @@ RuntimeError: RuntimeError: # repl ame__ -port +mport builtins micropython __future__ _asyncio _thread aesio array audiocore @@ -212,3 +212,13 @@ b'bytes 1234\x01' 2 3 frozentest.py +example_package.__init__ + +example_package.foo.bar.f + +example_package.f +example_package.foo.f +example_package.foo.bar.f +True +example_package.foo.f +True diff --git a/tests/unix/mod_os.py b/tests/unix/mod_os.py index 17554d9375..f69fa45b2b 100644 --- a/tests/unix/mod_os.py +++ b/tests/unix/mod_os.py @@ -12,7 +12,7 @@ os.unsetenv("TEST_VARIABLE") print(os.getenv("TEST_VARIABLE")) print(os.getenv("TEST_VARIABLE", "TEST_DEFAULT_VALUE")) -print(os.system("true")) +print(os.system("exit 0")) rand = os.urandom(4) print(type(rand) is bytes, len(rand)) diff --git a/tests/unix/time.py b/tests/unix/time_mktime_localtime.py similarity index 100% rename from tests/unix/time.py rename to tests/unix/time_mktime_localtime.py diff --git a/tools/autobuild/build-boards.sh b/tools/autobuild/build-boards.sh index 0a0e127f2e..4b8cf315a2 100755 --- a/tools/autobuild/build-boards.sh +++ b/tools/autobuild/build-boards.sh @@ -3,8 +3,36 @@ # The functions in this file can be run independently to build boards. # For example: # -# $ source build-boards.sh -# $ MICROPY_AUTOBUILD_MAKE=make build_rp2_boards -latest /tmp +# $ source tools/autobuild/build-boards.sh +# $ cd ports/rp2 +# $ MICROPY_AUTOBUILD_MAKE="make -j8" build_rp2_boards -latest /tmp +# +# Or to build a single board: +# +# $ source tools/autobuild/build-boards.sh +# $ cd ports/rp2 +# $ MICROPY_AUTOBUILD_MAKE="make -j8" build_board boards/PICO/board.json -latest /tmp uf2 + +function copy_artefacts { + local dest_dir=$1 + local descr=$2 + local fw_tag=$3 + local build_dir=$4 + shift 4 + + for ext in $@; do + dest=$dest_dir/$descr$fw_tag.$ext + if [ -r $build_dir/firmware.$ext ]; then + mv $build_dir/firmware.$ext $dest + elif [ -r $build_dir/micropython.$ext ]; then + # esp32 has micropython.elf, etc + mv $build_dir/micropython.$ext $dest + elif [ $ext = app-bin -a -r $build_dir/micropython.bin ]; then + # esp32 has micropython.bin which is just the application + mv $build_dir/micropython.bin $dest + fi + done +} function build_board { # check/get parameters @@ -13,30 +41,29 @@ function build_board { return 1 fi - board_json=$1 - fw_tag=$2 - dest_dir=$3 - shift - shift - shift + local board_json=$1 + local fw_tag=$2 + local dest_dir=$3 + shift 3 - board=$(echo $board_json | awk -F '/' '{ print $2 }') - descr=$(cat $board_json | python3 -c "import json,sys; print(json.load(sys.stdin).get('id', '$board'))") - build_dir=/tmp/micropython-build-$board + local board=$(echo $board_json | awk -F '/' '{ print $2 }') + local descr=$(cat $board_json | python3 -c "import json,sys; print(json.load(sys.stdin).get('id', '$board'))") + # Build the "default" variant. For most boards this is the only thing we build. echo "building $descr $board" - $MICROPY_AUTOBUILD_MAKE BOARD=$board BUILD=$build_dir && ( - for ext in $@; do - dest=$dest_dir/$descr$fw_tag.$ext - if [ -r $build_dir/firmware.$ext ]; then - mv $build_dir/firmware.$ext $dest - elif [ -r $build_dir/micropython.$ext ]; then - # esp32 has micropython.elf, etc - mv $build_dir/micropython.$ext $dest - fi - done - ) + local build_dir=/tmp/micropython-build-$board + $MICROPY_AUTOBUILD_MAKE BOARD=$board BUILD=$build_dir && copy_artefacts $dest_dir $descr $fw_tag $build_dir $@ rm -rf $build_dir + + # Query variants from board.json and build them. Ignore the special "IDF3" + # variant for ESP32 boards (this allows the downloads page to still have + # the idf3 files for older releases that used to be explicitly built). + for variant in `cat $board_json | python3 -c "import json,sys; print(' '.join(v for v in json.load(sys.stdin).get('variants', {}).keys() if v != 'IDF3'))"`; do + local variant_build_dir=$build_dir-$variant + echo "building variant $descr $board $variant" + $MICROPY_AUTOBUILD_MAKE BOARD=$board BOARD_VARIANT=$variant BUILD=$variant_build_dir && copy_artefacts $dest_dir $descr-$variant $fw_tag $variant_build_dir $@ + rm -rf $variant_build_dir + done } function build_boards { @@ -46,7 +73,7 @@ function build_boards { return 1 fi - check_file=$1 + local check_file=$1 shift # check we are in the correct directory @@ -61,37 +88,16 @@ function build_boards { done } +function build_cc3200_boards { + build_boards hal/cc3200_hal.c $1 $2 zip +} + function build_esp32_boards { - # check/get parameters - if [ $# != 2 ]; then - echo "usage: $0 " - return 1 - fi + build_boards modesp32.c $1 $2 bin elf map uf2 app-bin +} - fw_tag=$1 - dest_dir=$2 - - # check we are in the correct directory - if [ ! -r modesp32.c ]; then - echo "must be in esp32 directory" - return 1 - fi - - # build the boards, based on the IDF version - for board_json in $(find boards/ -name board.json | sort); do - mcu=$(cat $board_json | python3 -c "import json,sys; print(json.load(sys.stdin).get('mcu', 'unknown'))") - if idf.py --version | grep -q v4.2; then - if [ $mcu = esp32 ]; then - # build standard esp32-based boards with IDF v4.2 - build_board $board_json $fw_tag $dest_dir bin elf map - fi - else - if [ $mcu != esp32 ]; then - # build esp32-s2/s3/c3 based boards with IDF v4.4+ - build_board $board_json $fw_tag $dest_dir bin elf map uf2 - fi - fi - done +function build_esp8266_boards { + build_boards modesp.c $1 $2 bin elf map } function build_mimxrt_boards { @@ -103,7 +109,7 @@ function build_nrf_boards { } function build_renesas_ra_boards { - build_boards ra_it.c $1 $2 hex + build_boards ra_it.c $1 $2 bin hex } function build_rp2_boards { diff --git a/tools/autobuild/build-downloads.py b/tools/autobuild/build-downloads.py index 0f532e8bf0..c03d98aa5d 100755 --- a/tools/autobuild/build-downloads.py +++ b/tools/autobuild/build-downloads.py @@ -5,6 +5,42 @@ import json import os import sys +VALID_FEATURES = { + # Connectivity + "BLE", + "CAN", + "Ethernet", + "LoRa", + "USB", + "USB-C", + "WiFi", + # MCU features + "Dual-core", + "External Flash", + "External RAM", + # Form factor + "Feather", + # Connectors / sockets + "JST-PH", + "JST-SH", + "mikroBUS", + "microSD", + "SDCard", + # Sensors + "Environment Sensor", + "IMU", + # Other + "Audio Codec", + "Battery Charging", + "Camera", + "DAC", + "Display", + "Microphone", + "PoE", + "RGB LED", + "Secure Element", +} + def main(repo_path, output_path): boards_index = [] @@ -19,6 +55,16 @@ def main(repo_path, output_path): with open(board_json, "r") as f: blob = json.load(f) + features = set(blob.get("features", [])) + if not features.issubset(VALID_FEATURES): + print( + board_json, + "unknown features:", + features.difference(VALID_FEATURES), + file=sys.stderr, + ) + sys.exit(1) + # Use "id" if specified, otherwise default to board dir (e.g. "PYBV11"). # We allow boards to override ID for the historical build names. blob["id"] = blob.get("id", os.path.basename(board_dir)) diff --git a/tools/ci.sh b/tools/ci.sh index 8127a5134c..efa492bd19 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -28,6 +28,17 @@ function ci_code_formatting_run { tools/codeformat.py -v } +######################################################################################## +# code spelling + +function ci_code_spell_setup { + pip3 install codespell tomli +} + +function ci_code_spell_run { + codespell +} + ######################################################################################## # commit formatting @@ -50,8 +61,8 @@ function ci_code_size_setup { function ci_code_size_build { # check the following ports for the change in their code size - PORTS_TO_CHECK=bmusp - SUBMODULES="lib/berkeley-db-1.xx lib/mbedtls lib/micropython-lib lib/pico-sdk lib/stm32lib lib/tinyusb" + PORTS_TO_CHECK=bmusxpd + SUBMODULES="lib/asf4 lib/berkeley-db-1.xx lib/mbedtls lib/micropython-lib lib/nxp_driver lib/pico-sdk lib/stm32lib lib/tinyusb" # starts off at either the ref/pull/N/merge FETCH_HEAD, or the current branch HEAD git checkout -b pull_request # save the current location @@ -104,35 +115,13 @@ function ci_cc3200_build { ######################################################################################## # ports/esp32 -function ci_esp32_setup_helper { +function ci_esp32_idf50_setup { pip3 install pyelftools git clone https://github.com/espressif/esp-idf.git - git -C esp-idf checkout $1 - git -C esp-idf submodule update --init \ - components/bt/host/nimble/nimble \ - components/esp_wifi \ - components/esptool_py/esptool \ - components/lwip/lwip \ - components/mbedtls/mbedtls - if [ -d esp-idf/components/bt/controller/esp32 ]; then - git -C esp-idf submodule update --init \ - components/bt/controller/lib_esp32 \ - components/bt/controller/lib_esp32c3_family - else - git -C esp-idf submodule update --init \ - components/bt/controller/lib - fi + git -C esp-idf checkout v5.0.2 ./esp-idf/install.sh } -function ci_esp32_idf402_setup { - ci_esp32_setup_helper v4.0.2 -} - -function ci_esp32_idf44_setup { - ci_esp32_setup_helper v4.4 -} - function ci_esp32_build { source esp-idf/export.sh make ${MAKEOPTS} -C mpy-cross @@ -140,15 +129,9 @@ function ci_esp32_build { make ${MAKEOPTS} -C ports/esp32 \ USER_C_MODULES=../../../examples/usercmodule/micropython.cmake \ FROZEN_MANIFEST=$(pwd)/ports/esp32/boards/manifest_test.py - if [ -d $IDF_PATH/components/esp32c3 ]; then - make ${MAKEOPTS} -C ports/esp32 BOARD=GENERIC_C3 - fi - if [ -d $IDF_PATH/components/esp32s2 ]; then - make ${MAKEOPTS} -C ports/esp32 BOARD=GENERIC_S2 - fi - if [ -d $IDF_PATH/components/esp32s3 ]; then - make ${MAKEOPTS} -C ports/esp32 BOARD=GENERIC_S3 - fi + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_C3 + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_S2 + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_S3 # Test building native .mpy with xtensawin architecture. ci_native_mpy_modules_build xtensawin @@ -172,9 +155,9 @@ function ci_esp8266_path { function ci_esp8266_build { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/esp8266 submodules - make ${MAKEOPTS} -C ports/esp8266 - make ${MAKEOPTS} -C ports/esp8266 BOARD=GENERIC_512K - make ${MAKEOPTS} -C ports/esp8266 BOARD=GENERIC_1M + make ${MAKEOPTS} -C ports/esp8266 BOARD=ESP8266_GENERIC + make ${MAKEOPTS} -C ports/esp8266 BOARD=ESP8266_GENERIC BOARD_VARIANT=FLASH_512K + make ${MAKEOPTS} -C ports/esp8266 BOARD=ESP8266_GENERIC BOARD_VARIANT=FLASH_1M } ######################################################################################## @@ -221,10 +204,10 @@ function ci_nrf_build { ports/nrf/drivers/bluetooth/download_ble_stack.sh s140_nrf52_6_1_1 make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/nrf submodules - make ${MAKEOPTS} -C ports/nrf BOARD=pca10040 - make ${MAKEOPTS} -C ports/nrf BOARD=microbit - make ${MAKEOPTS} -C ports/nrf BOARD=pca10056 SD=s140 - make ${MAKEOPTS} -C ports/nrf BOARD=pca10090 + make ${MAKEOPTS} -C ports/nrf BOARD=PCA10040 + make ${MAKEOPTS} -C ports/nrf BOARD=MICROBIT + make ${MAKEOPTS} -C ports/nrf BOARD=PCA10056 SD=s140 + make ${MAKEOPTS} -C ports/nrf BOARD=PCA10090 } ######################################################################################## @@ -266,13 +249,19 @@ function ci_qemu_arm_build { function ci_renesas_ra_setup { ci_gcc_arm_setup + sudo apt-get install protobuf-c-compiler } function ci_renesas_ra_board_build { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/renesas-ra submodules make ${MAKEOPTS} -C ports/renesas-ra BOARD=RA4M1_CLICKER - make ${MAKEOPTS} -C ports/renesas-ra BOARD=RA6M2_EK + make ${MAKEOPTS} -C ports/renesas-ra BOARD=EK_RA6M2 + make ${MAKEOPTS} -C ports/renesas-ra BOARD=EK_RA6M1 + make ${MAKEOPTS} -C ports/renesas-ra BOARD=EK_RA4M1 + make ${MAKEOPTS} -C ports/renesas-ra BOARD=EK_RA4W1 + make ${MAKEOPTS} -C ports/renesas-ra BOARD=ARDUINO_PORTENTA_C33 submodules + make ${MAKEOPTS} -C ports/renesas-ra BOARD=ARDUINO_PORTENTA_C33 } ######################################################################################## @@ -286,8 +275,8 @@ function ci_rp2_build { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/rp2 submodules make ${MAKEOPTS} -C ports/rp2 - make ${MAKEOPTS} -C ports/rp2 clean - make ${MAKEOPTS} -C ports/rp2 USER_C_MODULES=../../examples/usercmodule/micropython.cmake + make ${MAKEOPTS} -C ports/rp2 BOARD=RPI_PICO_W submodules + make ${MAKEOPTS} -C ports/rp2 BOARD=RPI_PICO_W USER_C_MODULES=../../examples/usercmodule/micropython.cmake make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO submodules make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO @@ -306,7 +295,8 @@ function ci_samd_setup { function ci_samd_build { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/samd submodules - make ${MAKEOPTS} -C ports/samd + make ${MAKEOPTS} -C ports/samd BOARD=ADAFRUIT_ITSYBITSY_M0_EXPRESS + make ${MAKEOPTS} -C ports/samd BOARD=ADAFRUIT_ITSYBITSY_M4_EXPRESS } ######################################################################################## @@ -343,6 +333,7 @@ function ci_stm32_nucleo_build { # Test building various MCU families, some with additional options. make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_F091RC + make ${MAKEOPTS} -C ports/stm32 BOARD=STM32H573I_DK make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_H743ZI COPT=-O2 CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1' make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_L073RZ make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_L476RG DEBUG=1 @@ -380,14 +371,14 @@ function ci_teensy_build { CI_UNIX_OPTS_SYS_SETTRACE=( MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 - MICROPY_PY_USSL=0 + MICROPY_PY_SSL=0 CFLAGS_EXTRA="-DMICROPY_PY_SYS_SETTRACE=1" ) CI_UNIX_OPTS_SYS_SETTRACE_STACKLESS=( MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 - MICROPY_PY_USSL=0 + MICROPY_PY_SSL=0 CFLAGS_EXTRA="-DMICROPY_STACKLESS=1 -DMICROPY_STACKLESS_STRICT=1 -DMICROPY_PY_SYS_SETTRACE=1" ) @@ -437,12 +428,13 @@ function ci_native_mpy_modules_build { make -C examples/natmod/features1 ARCH=$arch make -C examples/natmod/features2 ARCH=$arch make -C examples/natmod/features3 ARCH=$arch + make -C examples/natmod/features4 ARCH=$arch make -C examples/natmod/btree ARCH=$arch + make -C examples/natmod/deflate ARCH=$arch make -C examples/natmod/framebuf ARCH=$arch - make -C examples/natmod/uheapq ARCH=$arch - make -C examples/natmod/urandom ARCH=$arch - make -C examples/natmod/ure ARCH=$arch - make -C examples/natmod/uzlib ARCH=$arch + make -C examples/natmod/heapq ARCH=$arch + make -C examples/natmod/random ARCH=$arch + make -C examples/natmod/re ARCH=$arch } function ci_native_mpy_modules_32bit_build { @@ -508,7 +500,7 @@ function ci_unix_coverage_run_mpy_merge_tests { function ci_unix_coverage_run_native_mpy_tests { MICROPYPATH=examples/natmod/features2 ./ports/unix/build-coverage/micropython -m features2 - (cd tests && ./run-natmodtests.py "$@" extmod/{btree*,framebuf*,uheapq*,urandom*,ure*,uzlib*}.py) + (cd tests && ./run-natmodtests.py "$@" extmod/{btree*,deflate*,framebuf*,heapq*,random*,re*}.py) } function ci_unix_32bit_setup { @@ -615,8 +607,8 @@ function ci_unix_macos_build { function ci_unix_macos_run_tests { # Issues with macOS tests: # - import_pkg7 has a problem with relative imports - # - urandom_basic has a problem with getrandbits(0) - (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-standard/micropython ./run-tests.py --exclude 'import_pkg7.py' --exclude 'urandom_basic.py') + # - random_basic has a problem with getrandbits(0) + (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-standard/micropython ./run-tests.py --exclude 'import_pkg7.py' --exclude 'random_basic.py') } function ci_unix_qemu_mips_setup { diff --git a/tools/insert-usb-ids.py b/tools/insert-usb-ids.py index b0843ab3ee..4691d5a710 100644 --- a/tools/insert-usb-ids.py +++ b/tools/insert-usb-ids.py @@ -20,7 +20,7 @@ def parse_usb_ids(filename): rv = dict() for line in open(filename).readlines(): line = line.rstrip("\r\n") - match = re.match("^#define\s+(\w+)\s+\(0x([0-9A-Fa-f]+)\)$", line) + match = re.match(r"^#define\s+(\w+)\s+\(0x([0-9A-Fa-f]+)\)$", line) if match and match.group(1).startswith(config_prefix): key = match.group(1).replace(config_prefix, "USB_") val = match.group(2) diff --git a/tools/manifestfile.py b/tools/manifestfile.py index f9b8a70357..295170c34c 100644 --- a/tools/manifestfile.py +++ b/tools/manifestfile.py @@ -106,7 +106,7 @@ class ManifestPackageMetadata: self.stdlib = False # Allows a python-ecosys package to be annotated with the - # corresponding name in PyPI. e.g. micropython-lib/urequests is based + # corresponding name in PyPI. e.g. micropython-lib/requests is based # on pypi/requests. self.pypi = None # For a micropython package, this is the name that we will publish it @@ -143,7 +143,7 @@ class ManifestPackageMetadata: self.description = description self.version = version - self.license = version + self.license = license self.author = author self.pypi = pypi self.pypi_publish = pypi_publish diff --git a/tools/merge_micropython.py b/tools/merge_micropython.py index b596cd59c5..74c32bea6b 100644 --- a/tools/merge_micropython.py +++ b/tools/merge_micropython.py @@ -3,7 +3,13 @@ This is a helper script for merging in new versions of MicroPython. You *must* evaluate its correctness and adapt it for each MP version. This is committed in the repo more for reference than "fire and forget" use. -Updated for v1.20.0 merge - dhalbert +I have found I have to run each piece separately, because there are some errors. +For instance, there are file renames in the porcelain output that are not handled. +I add a sys.exit(0) after a section, and once a section runs, I delete it temporarily +and move on to the next section. -- dhalbert + +Updated for v1.21.0 merge - dhalbert + """ from io import StringIO @@ -36,7 +42,7 @@ ports_to_delete = [ ] for p in ports_to_delete: try: - git.rm("-rf", "ports/" + p) + git.rm("-rf", "ports/" + p)XC except sh.ErrorReturnCode_128: pass @@ -145,6 +151,7 @@ for d in docs_to_delete: tests_to_delete = [ "esp32", "multi_bluetooth", + "multi_espnow", "multi_net", "net_hosted", "net_inet", @@ -170,6 +177,7 @@ libs_to_delete = [ "nrfx", "nxp_driver", "pico-sdk", + "protobuf-c", "stm32lib", "wiznet5k", ] @@ -191,10 +199,10 @@ extmod_to_delete = [ "modnetwork.*", "modonewire.*", "moducryptolib.*", - "modusocket.*", - "modussl_*.*", - "modutimeq.*", - "moduwebsocket.*", + "modsocket.*", + "modssl_*.*", + "modtimeq.*", + "modwebsocket.*", "modwebrepl.*", "mpbthci.*", "network_*.*", diff --git a/tools/metrics.py b/tools/metrics.py index da4d188568..9c5ed1d7d5 100755 --- a/tools/metrics.py +++ b/tools/metrics.py @@ -64,12 +64,12 @@ port_data = { "n": PortData("unix nanbox", "unix", "build-nanbox/micropython", "VARIANT=nanbox"), "s": PortData("stm32", "stm32", "build-PYBV10/firmware.elf", "BOARD=PYBV10"), "c": PortData("cc3200", "cc3200", "build/WIPY/release/application.axf", "BTARGET=application"), - "8": PortData("esp8266", "esp8266", "build-GENERIC/firmware.elf"), - "3": PortData("esp32", "esp32", "build-GENERIC/micropython.elf"), + "8": PortData("esp8266", "esp8266", "build-ESP8266_GENERIC/firmware.elf"), + "3": PortData("esp32", "esp32", "build-ESP32_GENERIC/micropython.elf"), "x": PortData("mimxrt", "mimxrt", "build-TEENSY40/firmware.elf"), - "e": PortData("renesas-ra", "renesas-ra", "build-RA6M2_EK/firmware.elf"), - "r": PortData("nrf", "nrf", "build-pca10040/firmware.elf"), - "p": PortData("rp2", "rp2", "build-PICO/firmware.elf"), + "e": PortData("renesas-ra", "renesas-ra", "build-EK_RA6M2/firmware.elf"), + "r": PortData("nrf", "nrf", "build-PCA10040/firmware.elf"), + "p": PortData("rp2", "rp2", "build-RPI_PICO/firmware.elf"), "d": PortData("samd", "samd", "build-ADAFRUIT_ITSYBITSY_M4_EXPRESS/firmware.elf"), } diff --git a/tools/mpremote/mpremote/transport.py b/tools/mpremote/mpremote/transport.py new file mode 100644 index 0000000000..6e9a77b2bb --- /dev/null +++ b/tools/mpremote/mpremote/transport.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2023 Jim Mussared +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + +class TransportError(Exception): + pass + + +class Transport: + pass diff --git a/tools/mpremote/mpremote/transport_serial.py b/tools/mpremote/mpremote/transport_serial.py new file mode 100644 index 0000000000..e04f5b4ac9 --- /dev/null +++ b/tools/mpremote/mpremote/transport_serial.py @@ -0,0 +1,1211 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2014-2021 Damien P. George +# Copyright (c) 2017 Paul Sokolovsky +# Copyright (c) 2023 Jim Mussared +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# This is based on the serial-only parts of tools/pyboard.py, with Python 2 +# support removed, and is currently in the process of being refactored to +# support multiple transports (webrepl, socket, BLE, etc). At the moment, +# SerialTransport is just the old Pyboard+PyboardExtended class without any +# of this refactoring. The API is going to change significantly. + +# Once the API is stabilised, the idea is that mpremote can be used both +# as a command line tool and a library for interacting with devices. + +import ast, io, errno, os, re, struct, sys, time +from collections import namedtuple +from errno import EPERM +from .console import VT_ENABLED +from .transport import TransportError, Transport + + +def stdout_write_bytes(b): + b = b.replace(b"\x04", b"") + sys.stdout.buffer.write(b) + sys.stdout.buffer.flush() + + +listdir_result = namedtuple("dir_result", ["name", "st_mode", "st_ino", "st_size"]) + + +def reraise_filesystem_error(e, info): + if len(e.args) >= 3: + if b"OSError" in e.args[2] and b"ENOENT" in e.args[2]: + raise FileNotFoundError(info) + raise + + +class SerialTransport(Transport): + def __init__(self, device, baudrate=115200, wait=0, exclusive=True): + self.in_raw_repl = False + self.use_raw_paste = True + self.device_name = device + self.mounted = False + + import serial + import serial.tools.list_ports + + # Set options, and exclusive if pyserial supports it + serial_kwargs = {"baudrate": baudrate, "interCharTimeout": 1} + if serial.__version__ >= "3.3": + serial_kwargs["exclusive"] = exclusive + + delayed = False + for attempt in range(wait + 1): + try: + if device.startswith("rfc2217://"): + self.serial = serial.serial_for_url(device, **serial_kwargs) + elif os.name == "nt": + self.serial = serial.Serial(**serial_kwargs) + self.serial.port = device + portinfo = list(serial.tools.list_ports.grep(device)) # type: ignore + if portinfo and portinfo[0].manufacturer != "Microsoft": + # ESP8266/ESP32 boards use RTS/CTS for flashing and boot mode selection. + # DTR False: to avoid using the reset button will hang the MCU in bootloader mode + # RTS False: to prevent pulses on rts on serial.close() that would POWERON_RESET an ESPxx + self.serial.dtr = False # DTR False = gpio0 High = Normal boot + self.serial.rts = False # RTS False = EN High = MCU enabled + self.serial.open() + else: + self.serial = serial.Serial(device, **serial_kwargs) + break + except OSError: + if wait == 0: + continue + if attempt == 0: + sys.stdout.write("Waiting {} seconds for pyboard ".format(wait)) + delayed = True + time.sleep(1) + sys.stdout.write(".") + sys.stdout.flush() + else: + if delayed: + print("") + raise TransportError("failed to access " + device) + if delayed: + print("") + + def close(self): + self.serial.close() + + def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None): + # if data_consumer is used then data is not accumulated and the ending must be 1 byte long + assert data_consumer is None or len(ending) == 1 + + data = self.serial.read(min_num_bytes) + if data_consumer: + data_consumer(data) + timeout_count = 0 + while True: + if data.endswith(ending): + break + elif self.serial.inWaiting() > 0: + new_data = self.serial.read(1) + if data_consumer: + data_consumer(new_data) + data = new_data + else: + data = data + new_data + timeout_count = 0 + else: + timeout_count += 1 + if timeout is not None and timeout_count >= 100 * timeout: + break + time.sleep(0.01) + return data + + def enter_raw_repl(self, soft_reset=True): + self.serial.write(b"\r\x03\x03") # ctrl-C twice: interrupt any running program + + # flush input (without relying on serial.flushInput()) + n = self.serial.inWaiting() + while n > 0: + self.serial.read(n) + n = self.serial.inWaiting() + + self.serial.write(b"\r\x01") # ctrl-A: enter raw REPL + + if soft_reset: + data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n>") + if not data.endswith(b"raw REPL; CTRL-B to exit\r\n>"): + print(data) + raise TransportError("could not enter raw repl") + + self.serial.write(b"\x04") # ctrl-D: soft reset + + # Waiting for "soft reboot" independently to "raw REPL" (done below) + # allows boot.py to print, which will show up after "soft reboot" + # and before "raw REPL". + data = self.read_until(1, b"soft reboot\r\n") + if not data.endswith(b"soft reboot\r\n"): + print(data) + raise TransportError("could not enter raw repl") + + data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n") + if not data.endswith(b"raw REPL; CTRL-B to exit\r\n"): + print(data) + raise TransportError("could not enter raw repl") + + self.in_raw_repl = True + + def exit_raw_repl(self): + self.serial.write(b"\r\x02") # ctrl-B: enter friendly REPL + self.in_raw_repl = False + + def follow(self, timeout, data_consumer=None): + # wait for normal output + data = self.read_until(1, b"\x04", timeout=timeout, data_consumer=data_consumer) + if not data.endswith(b"\x04"): + raise TransportError("timeout waiting for first EOF reception") + data = data[:-1] + + # wait for error output + data_err = self.read_until(1, b"\x04", timeout=timeout) + if not data_err.endswith(b"\x04"): + raise TransportError("timeout waiting for second EOF reception") + data_err = data_err[:-1] + + # return normal and error output + return data, data_err + + def raw_paste_write(self, command_bytes): + # Read initial header, with window size. + data = self.serial.read(2) + window_size = struct.unpack("") + if not data.endswith(b">"): + raise TransportError("could not enter raw repl") + + if self.use_raw_paste: + # Try to enter raw-paste mode. + self.serial.write(b"\x05A\x01") + data = self.serial.read(2) + if data == b"R\x00": + # Device understood raw-paste command but doesn't support it. + pass + elif data == b"R\x01": + # Device supports raw-paste mode, write out the command using this mode. + return self.raw_paste_write(command_bytes) + else: + # Device doesn't support raw-paste, fall back to normal raw REPL. + data = self.read_until(1, b"w REPL; CTRL-B to exit\r\n>") + if not data.endswith(b"w REPL; CTRL-B to exit\r\n>"): + print(data) + raise TransportError("could not enter raw repl") + # Don't try to use raw-paste mode again for this connection. + self.use_raw_paste = False + + # Write command using standard raw REPL, 256 bytes every 10ms. + for i in range(0, len(command_bytes), 256): + self.serial.write(command_bytes[i : min(i + 256, len(command_bytes))]) + time.sleep(0.01) + self.serial.write(b"\x04") + + # check if we could exec command + data = self.serial.read(2) + if data != b"OK": + raise TransportError("could not exec command (response: %r)" % data) + + def exec_raw(self, command, timeout=10, data_consumer=None): + self.exec_raw_no_follow(command) + return self.follow(timeout, data_consumer) + + def eval(self, expression, parse=False): + if parse: + ret = self.exec("print(repr({}))".format(expression)) + ret = ret.strip() + return ast.literal_eval(ret.decode()) + else: + ret = self.exec("print({})".format(expression)) + ret = ret.strip() + return ret + + def exec(self, command, data_consumer=None): + ret, ret_err = self.exec_raw(command, data_consumer=data_consumer) + if ret_err: + raise TransportError("exception", ret, ret_err) + return ret + + def execfile(self, filename): + with open(filename, "rb") as f: + pyfile = f.read() + return self.exec(pyfile) + + def fs_exists(self, src): + try: + self.exec("import os\nos.stat(%s)" % (("'%s'" % src) if src else "")) + return True + except TransportError: + return False + + def fs_ls(self, src): + cmd = ( + "import os\nfor f in os.ilistdir(%s):\n" + " print('{:12} {}{}'.format(f[3]if len(f)>3 else 0,f[0],'/'if f[1]&0x4000 else ''))" + % (("'%s'" % src) if src else "") + ) + self.exec(cmd, data_consumer=stdout_write_bytes) + + def fs_listdir(self, src=""): + buf = bytearray() + + def repr_consumer(b): + buf.extend(b.replace(b"\x04", b"")) + + cmd = "import os\nfor f in os.ilistdir(%s):\n" " print(repr(f), end=',')" % ( + ("'%s'" % src) if src else "" + ) + try: + buf.extend(b"[") + self.exec(cmd, data_consumer=repr_consumer) + buf.extend(b"]") + except TransportError as e: + reraise_filesystem_error(e, src) + + return [ + listdir_result(*f) if len(f) == 4 else listdir_result(*(f + (0,))) + for f in ast.literal_eval(buf.decode()) + ] + + def fs_stat(self, src): + try: + self.exec("import os") + return os.stat_result(self.eval("os.stat(%s)" % (("'%s'" % src)), parse=True)) + except TransportError as e: + reraise_filesystem_error(e, src) + + def fs_cat(self, src, chunk_size=256): + cmd = ( + "with open('%s') as f:\n while 1:\n" + " b=f.read(%u)\n if not b:break\n print(b,end='')" % (src, chunk_size) + ) + self.exec(cmd, data_consumer=stdout_write_bytes) + + def fs_readfile(self, src, chunk_size=256): + buf = bytearray() + + def repr_consumer(b): + buf.extend(b.replace(b"\x04", b"")) + + cmd = ( + "with open('%s', 'rb') as f:\n while 1:\n" + " b=f.read(%u)\n if not b:break\n print(b,end='')" % (src, chunk_size) + ) + try: + self.exec(cmd, data_consumer=repr_consumer) + except TransportError as e: + reraise_filesystem_error(e, src) + return ast.literal_eval(buf.decode()) + + def fs_writefile(self, dest, data, chunk_size=256): + self.exec("f=open('%s','wb')\nw=f.write" % dest) + while data: + chunk = data[:chunk_size] + self.exec("w(" + repr(chunk) + ")") + data = data[len(chunk) :] + self.exec("f.close()") + + def fs_cp(self, src, dest, chunk_size=256, progress_callback=None): + if progress_callback: + src_size = self.fs_stat(src).st_size + written = 0 + self.exec("fr=open('%s','rb')\nr=fr.read\nfw=open('%s','wb')\nw=fw.write" % (src, dest)) + while True: + data_len = int(self.exec("d=r(%u)\nw(d)\nprint(len(d))" % chunk_size)) + if not data_len: + break + if progress_callback: + written += data_len + progress_callback(written, src_size) + self.exec("fr.close()\nfw.close()") + + def fs_get(self, src, dest, chunk_size=256, progress_callback=None): + if progress_callback: + src_size = self.fs_stat(src).st_size + written = 0 + self.exec("f=open('%s','rb')\nr=f.read" % src) + with open(dest, "wb") as f: + while True: + data = bytearray() + self.exec("print(r(%u))" % chunk_size, data_consumer=lambda d: data.extend(d)) + assert data.endswith(b"\r\n\x04") + try: + data = ast.literal_eval(str(data[:-3], "ascii")) + if not isinstance(data, bytes): + raise ValueError("Not bytes") + except (UnicodeError, ValueError) as e: + raise TransportError("fs_get: Could not interpret received data: %s" % str(e)) + if not data: + break + f.write(data) + if progress_callback: + written += len(data) + progress_callback(written, src_size) + self.exec("f.close()") + + def fs_put(self, src, dest, chunk_size=256, progress_callback=None): + if progress_callback: + src_size = os.path.getsize(src) + written = 0 + self.exec("f=open('%s','wb')\nw=f.write" % dest) + with open(src, "rb") as f: + while True: + data = f.read(chunk_size) + if not data: + break + if sys.version_info < (3,): + self.exec("w(b" + repr(data) + ")") + else: + self.exec("w(" + repr(data) + ")") + if progress_callback: + written += len(data) + progress_callback(written, src_size) + self.exec("f.close()") + + def fs_mkdir(self, dir): + self.exec("import os\nos.mkdir('%s')" % dir) + + def fs_rmdir(self, dir): + self.exec("import os\nos.rmdir('%s')" % dir) + + def fs_rm(self, src): + self.exec("import os\nos.remove('%s')" % src) + + def fs_touch(self, src): + self.exec("f=open('%s','a')\nf.close()" % src) + + def filesystem_command(self, args, progress_callback=None, verbose=False): + def fname_remote(src): + if src.startswith(":"): + src = src[1:] + # Convert all path separators to "/", because that's what a remote device uses. + return src.replace(os.path.sep, "/") + + def fname_cp_dest(src, dest): + _, src = os.path.split(src) + if dest is None or dest == "": + dest = src + elif dest == ".": + dest = "./" + src + elif dest.endswith("/"): + dest += src + return dest + + cmd = args[0] + args = args[1:] + try: + if cmd == "cp": + srcs = args[:-1] + dest = args[-1] + if dest.startswith(":"): + op_remote_src = self.fs_cp + op_local_src = self.fs_put + else: + op_remote_src = self.fs_get + op_local_src = lambda src, dest, **_: __import__("shutil").copy(src, dest) + for src in srcs: + if verbose: + print("cp %s %s" % (src, dest)) + if src.startswith(":"): + op = op_remote_src + else: + op = op_local_src + src2 = fname_remote(src) + dest2 = fname_cp_dest(src2, fname_remote(dest)) + op(src2, dest2, progress_callback=progress_callback) + else: + ops = { + "cat": self.fs_cat, + "ls": self.fs_ls, + "mkdir": self.fs_mkdir, + "rm": self.fs_rm, + "rmdir": self.fs_rmdir, + "touch": self.fs_touch, + } + if cmd not in ops: + raise TransportError("'{}' is not a filesystem command".format(cmd)) + if cmd == "ls" and not args: + args = [""] + for src in args: + src = fname_remote(src) + if verbose: + print("%s :%s" % (cmd, src)) + ops[cmd](src) + except TransportError as er: + if len(er.args) > 1: + print(str(er.args[2], "ascii")) + else: + print(er) + self.exit_raw_repl() + self.close() + sys.exit(1) + + def mount_local(self, path, unsafe_links=False): + fout = self.serial + if self.eval('"RemoteFS" in globals()') == b"False": + self.exec(fs_hook_code) + self.exec("__mount()") + self.mounted = True + self.cmd = PyboardCommand(self.serial, fout, path, unsafe_links=unsafe_links) + self.serial = SerialIntercept(self.serial, self.cmd) + + def write_ctrl_d(self, out_callback): + self.serial.write(b"\x04") + if not self.mounted: + return + + # Read response from the device until it is quiet (with a timeout). + INITIAL_TIMEOUT = 0.5 + BANNER_TIMEOUT = 2 + QUIET_TIMEOUT = 0.1 + FULL_TIMEOUT = 5 + t_start = t_last_activity = time.monotonic() + data_all = b"" + soft_reboot_started = False + soft_reboot_banner = False + while True: + t = time.monotonic() + n = self.serial.inWaiting() + if n > 0: + data = self.serial.read(n) + out_callback(data) + data_all += data + t_last_activity = t + else: + if len(data_all) == 0: + if t - t_start > INITIAL_TIMEOUT: + return + else: + if t - t_start > FULL_TIMEOUT: + if soft_reboot_started: + break + return + + next_data_timeout = QUIET_TIMEOUT + + if not soft_reboot_started and data_all.find(b"MPY: soft reboot") != -1: + soft_reboot_started = True + + if soft_reboot_started and not soft_reboot_banner: + # Once soft reboot has been initiated, give some more time for the startup + # banner to be shown + if data_all.find(b"\nMicroPython ") != -1: + soft_reboot_banner = True + elif data_all.find(b"\nraw REPL; CTRL-B to exit\r\n") != -1: + soft_reboot_banner = True + else: + next_data_timeout = BANNER_TIMEOUT + + if t - t_last_activity > next_data_timeout: + break + + if not soft_reboot_started: + return + + if not soft_reboot_banner: + out_callback(b"Warning: Could not remount local filesystem\r\n") + return + + # Determine type of prompt + if data_all.endswith(b">"): + in_friendly_repl = False + prompt = b">" + else: + in_friendly_repl = True + prompt = data_all.rsplit(b"\r\n", 1)[-1] + + # Clear state while board remounts, it will be re-set once mounted. + self.mounted = False + self.serial = self.serial.orig_serial + + # Provide a message about the remount. + out_callback(bytes(f"\r\nRemount local directory {self.cmd.root} at /remote\r\n", "utf8")) + + # Enter raw REPL and re-mount the remote filesystem. + self.serial.write(b"\x01") + self.exec(fs_hook_code) + self.exec("__mount()") + self.mounted = True + + # Exit raw REPL if needed, and wait for the friendly REPL prompt. + if in_friendly_repl: + self.exit_raw_repl() + self.read_until(len(prompt), prompt) + out_callback(prompt) + self.serial = SerialIntercept(self.serial, self.cmd) + + def umount_local(self): + if self.mounted: + self.exec('os.umount("/remote")') + self.mounted = False + self.serial = self.serial.orig_serial + + +fs_hook_cmds = { + "CMD_STAT": 1, + "CMD_ILISTDIR_START": 2, + "CMD_ILISTDIR_NEXT": 3, + "CMD_OPEN": 4, + "CMD_CLOSE": 5, + "CMD_READ": 6, + "CMD_WRITE": 7, + "CMD_SEEK": 8, + "CMD_REMOVE": 9, + "CMD_RENAME": 10, + "CMD_MKDIR": 11, + "CMD_RMDIR": 12, +} + +fs_hook_code = """\ +import os, io, struct, micropython + +SEEK_SET = 0 + +class RemoteCommand: + def __init__(self): + import select, sys + self.buf4 = bytearray(4) + self.fout = sys.stdout.buffer + self.fin = sys.stdin.buffer + self.poller = select.poll() + self.poller.register(self.fin, select.POLLIN) + + def poll_in(self): + for _ in self.poller.ipoll(1000): + return + self.end() + raise Exception('timeout waiting for remote') + + def rd(self, n): + buf = bytearray(n) + self.rd_into(buf, n) + return buf + + def rd_into(self, buf, n): + # implement reading with a timeout in case other side disappears + if n == 0: + return + self.poll_in() + r = self.fin.readinto(buf, n) + if r < n: + mv = memoryview(buf) + while r < n: + self.poll_in() + r += self.fin.readinto(mv[r:], n - r) + + def begin(self, type): + micropython.kbd_intr(-1) + buf4 = self.buf4 + buf4[0] = 0x18 + buf4[1] = type + self.fout.write(buf4, 2) + # Wait for sync byte 0x18, but don't get stuck forever + for i in range(30): + self.poller.poll(1000) + self.fin.readinto(buf4, 1) + if buf4[0] == 0x18: + break + + def end(self): + micropython.kbd_intr(3) + + def rd_s8(self): + self.rd_into(self.buf4, 1) + n = self.buf4[0] + if n & 0x80: + n -= 0x100 + return n + + def rd_s32(self): + buf4 = self.buf4 + self.rd_into(buf4, 4) + n = buf4[0] | buf4[1] << 8 | buf4[2] << 16 | buf4[3] << 24 + if buf4[3] & 0x80: + n -= 0x100000000 + return n + + def rd_u32(self): + buf4 = self.buf4 + self.rd_into(buf4, 4) + return buf4[0] | buf4[1] << 8 | buf4[2] << 16 | buf4[3] << 24 + + def rd_bytes(self, buf): + # TODO if n is large (eg >256) then we may miss bytes on stdin + n = self.rd_s32() + if buf is None: + ret = buf = bytearray(n) + else: + ret = n + self.rd_into(buf, n) + return ret + + def rd_str(self): + n = self.rd_s32() + if n == 0: + return '' + else: + return str(self.rd(n), 'utf8') + + def wr_s8(self, i): + self.buf4[0] = i + self.fout.write(self.buf4, 1) + + def wr_s32(self, i): + struct.pack_into(' {len(buf)}") + + def do_seek(self): + fd = self.rd_s8() + n = self.rd_s32() + whence = self.rd_s8() + # self.log_cmd(f"seek {fd} {n}") + try: + n = self.data_files[fd][0].seek(n, whence) + except io.UnsupportedOperation: + n = -1 + self.wr_s32(n) + + def do_write(self): + fd = self.rd_s8() + buf = self.rd_bytes() + if self.data_files[fd][1]: + buf = str(buf, "utf8") + n = self.data_files[fd][0].write(buf) + self.wr_s32(n) + # self.log_cmd(f"write {fd} {len(buf)} -> {n}") + + def do_remove(self): + path = self.root + self.rd_str() + # self.log_cmd(f"remove {path}") + try: + self.path_check(path) + os.remove(path) + ret = 0 + except OSError as er: + ret = -abs(er.errno) + self.wr_s32(ret) + + def do_rename(self): + old = self.root + self.rd_str() + new = self.root + self.rd_str() + # self.log_cmd(f"rename {old} {new}") + try: + self.path_check(old) + self.path_check(new) + os.rename(old, new) + ret = 0 + except OSError as er: + ret = -abs(er.errno) + self.wr_s32(ret) + + def do_mkdir(self): + path = self.root + self.rd_str() + # self.log_cmd(f"mkdir {path}") + try: + self.path_check(path) + os.mkdir(path) + ret = 0 + except OSError as er: + ret = -abs(er.errno) + self.wr_s32(ret) + + def do_rmdir(self): + path = self.root + self.rd_str() + # self.log_cmd(f"rmdir {path}") + try: + self.path_check(path) + os.rmdir(path) + ret = 0 + except OSError as er: + ret = -abs(er.errno) + self.wr_s32(ret) + + cmd_table = { + fs_hook_cmds["CMD_STAT"]: do_stat, + fs_hook_cmds["CMD_ILISTDIR_START"]: do_ilistdir_start, + fs_hook_cmds["CMD_ILISTDIR_NEXT"]: do_ilistdir_next, + fs_hook_cmds["CMD_OPEN"]: do_open, + fs_hook_cmds["CMD_CLOSE"]: do_close, + fs_hook_cmds["CMD_READ"]: do_read, + fs_hook_cmds["CMD_WRITE"]: do_write, + fs_hook_cmds["CMD_SEEK"]: do_seek, + fs_hook_cmds["CMD_REMOVE"]: do_remove, + fs_hook_cmds["CMD_RENAME"]: do_rename, + fs_hook_cmds["CMD_MKDIR"]: do_mkdir, + fs_hook_cmds["CMD_RMDIR"]: do_rmdir, + } + + +class SerialIntercept: + def __init__(self, serial, cmd): + self.orig_serial = serial + self.cmd = cmd + self.buf = b"" + self.orig_serial.timeout = 5.0 + + def _check_input(self, blocking): + if blocking or self.orig_serial.inWaiting() > 0: + c = self.orig_serial.read(1) + if c == b"\x18": + # a special command + c = self.orig_serial.read(1)[0] + self.orig_serial.write(b"\x18") # Acknowledge command + PyboardCommand.cmd_table[c](self.cmd) + elif not VT_ENABLED and c == b"\x1b": + # ESC code, ignore these on windows + esctype = self.orig_serial.read(1) + if esctype == b"[": # CSI + while not (0x40 < self.orig_serial.read(1)[0] < 0x7E): + # Looking for "final byte" of escape sequence + pass + else: + self.buf += c + + @property + def fd(self): + return self.orig_serial.fd + + def close(self): + self.orig_serial.close() + + def inWaiting(self): + self._check_input(False) + return len(self.buf) + + def read(self, n): + while len(self.buf) < n: + self._check_input(True) + out = self.buf[:n] + self.buf = self.buf[n:] + return out + + def write(self, buf): + self.orig_serial.write(buf) diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index a02104322c..e2ebc18ae2 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -33,9 +33,9 @@ if platform.python_version_tuple()[0] == "2": str_cons = lambda val, enc=None: str(val) bytes_cons = lambda val, enc=None: bytearray(val) - is_str_type = lambda o: type(o) is str + is_str_type = lambda o: isinstance(o, str) is_bytes_type = lambda o: type(o) is bytearray - is_int_type = lambda o: type(o) is int or type(o) is long + is_int_type = lambda o: isinstance(o, int) or isinstance(o, long) # noqa: F821 def hexlify_to_str(b): x = hexlify_py2(b) @@ -46,9 +46,9 @@ else: str_cons = str bytes_cons = bytes - is_str_type = lambda o: type(o) is str - is_bytes_type = lambda o: type(o) is bytes - is_int_type = lambda o: type(o) is int + is_str_type = lambda o: isinstance(o, str) + is_bytes_type = lambda o: isinstance(o, bytes) + is_int_type = lambda o: isinstance(o, int) def hexlify_to_str(b): return str(hexlify(b, ":"), "ascii") @@ -127,9 +127,9 @@ MP_PERSISTENT_OBJ_COMPLEX = 9 MP_PERSISTENT_OBJ_TUPLE = 10 # Circuitpython: this does not match upstream because we added MP_SCOPE_FLAG_ASYNC -MP_SCOPE_FLAG_VIPERRELOC = 0x10 -MP_SCOPE_FLAG_VIPERRODATA = 0x20 -MP_SCOPE_FLAG_VIPERBSS = 0x40 +MP_SCOPE_FLAG_VIPERRELOC = 0x20 +MP_SCOPE_FLAG_VIPERRODATA = 0x40 +MP_SCOPE_FLAG_VIPERBSS = 0x80 MP_BC_MASK_EXTRA_BYTE = 0x9E @@ -755,7 +755,7 @@ class CompiledModule: const_int_content += (digs.count(",") + 1) * bits_per_dig // 8 const_obj_content += 4 * 4 return "MP_ROM_PTR(&%s)" % obj_name - elif type(obj) is float: + elif isinstance(obj, float): macro_name = "%s_macro" % obj_name print( "#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B" @@ -776,7 +776,7 @@ class CompiledModule: print("#endif") const_obj_content += 3 * 4 return macro_name - elif type(obj) is complex: + elif isinstance(obj, complex): print( "static const mp_obj_complex_t %s = {{&mp_type_complex}, (mp_float_t)%.16g, (mp_float_t)%.16g};" % (obj_name, obj.real, obj.imag) @@ -1119,7 +1119,6 @@ class RawCodeNative(RawCode): i_top = len(self.fun_data) i = 0 - qi = 0 while i < i_top: # copy machine code (max 16 bytes) i16 = min(i + 16, i_top) @@ -1275,7 +1274,7 @@ def read_raw_code(reader, parent_name, qstr_table, obj_table, segments): if native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA: rodata_size = reader.read_uint() if native_scope_flags & MP_SCOPE_FLAG_VIPERBSS: - bss_size = reader.read_uint() + reader.read_uint() # bss_size if native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA: reader.read_bytes(rodata_size) if native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC: @@ -1284,10 +1283,10 @@ def read_raw_code(reader, parent_name, qstr_table, obj_table, segments): if op == 0xFF: break if op & 1: - addr = reader.read_uint() + reader.read_uint() # addr op >>= 1 if op <= 5 and op & 1: - n = reader.read_uint() + reader.read_uint() # n else: assert kind == MP_CODE_NATIVE_ASM native_n_pos_args = reader.read_uint() diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index 69a3ee48cb..0fd9e03847 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -233,34 +233,20 @@ def extract_qstrs(source_files): def read_qstrs(f): with open(f) as f: vals = set() - objs = set() for line in f: - while line: - m = re.search(r"MP_OBJ_NEW_QSTR\((MP_QSTR_[A-Za-z0-9_]*)\)", line) - if m: - objs.add(m.group(1)) - else: - m = re.search(r"MP_QSTR_[A-Za-z0-9_]*", line) - if m: - vals.add(m.group()) - if m: - s = m.span() - line = line[: s[0]] + line[s[1] :] - else: - line = "" - return vals, objs + for m in re.finditer(r"MP_QSTR_[A-Za-z0-9_]*", line): + vals.add(m.group()) + return vals static_qstrs = ["MP_QSTR_" + qstrutil.qstr_escape(q) for q in qstrutil.static_qstr_list] qstr_vals = set() - qstr_objs = set() for f in source_files: - vals, objs = read_qstrs(f) + vals = read_qstrs(f) qstr_vals.update(vals) - qstr_objs.update(objs) qstr_vals.difference_update(static_qstrs) - return static_qstrs, qstr_vals, qstr_objs + return static_qstrs, qstr_vals ################################################################################ @@ -472,7 +458,6 @@ def do_relocation_text(env, text_addr, r): # Extract relevant info about symbol that's being relocated s = r.sym s_bind = s.entry["st_info"]["bind"] - s_shndx = s.entry["st_shndx"] s_type = s.entry["st_info"]["type"] r_offset = r["r_offset"] + text_addr r_info_type = r["r_info_type"] @@ -732,7 +717,7 @@ def load_object_file(env, felf): env.unresolved_syms.append(sym) -def link_objects(env, native_qstr_vals_len, native_qstr_objs_len): +def link_objects(env, native_qstr_vals_len): # Build GOT information if env.arch.name == "EM_XTENSA": build_got_xtensa(env) @@ -763,7 +748,7 @@ def link_objects(env, native_qstr_vals_len, native_qstr_objs_len): # Create section to contain mp_native_obj_table env.obj_table_section = Section( ".external.obj_table", - bytearray(native_qstr_objs_len * env.arch.word_size), + bytearray(0 * env.arch.word_size), # currently empty env.arch.word_size, ) @@ -902,7 +887,7 @@ class MPYOutput: self.write_uint(n) -def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): +def build_mpy(env, entry_offset, fmpy, native_qstr_vals): # Write jump instruction to start of text jump = env.arch.asm_jump(entry_offset) env.full_text[: len(jump)] = jump @@ -930,7 +915,7 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): out.write_uint(1 + len(native_qstr_vals)) # MPY: n_obj - out.write_uint(len(native_qstr_objs)) + out.write_uint(0) # MPY: qstr table out.write_qstr(fmpy) # filename @@ -938,10 +923,7 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): out.write_qstr(q) # MPY: object table - for q in native_qstr_objs: - out.write_bytes(bytearray([MP_PERSISTENT_OBJ_STR])) - out.write_uint(len(q)) - out.write_bytes(bytes(q, "utf8") + b"\x00") + # # MPY: kind/len out.write_uint(len(env.full_text) << 3 | (MP_CODE_NATIVE_VIPER - MP_CODE_BYTECODE)) @@ -968,11 +950,15 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): out.write_bytes(env.full_rodata) # MPY: relocation information + # See py/persistentcode.c:mp_native_relocate for meaning of the `kind` integer values. prev_kind = None + prev_base = None + prev_offset = None + prev_n = None for base, addr, kind in env.mpy_relocs: if isinstance(kind, str) and kind.startswith(".text"): kind = 0 - elif kind in (".rodata", ".data.rel.ro"): + elif isinstance(kind, str) and kind.startswith((".rodata", ".data.rel.ro")): if env.arch.separate_rodata: kind = rodata_const_table_idx else: @@ -1016,7 +1002,7 @@ def do_preprocess(args): if args.output is None: assert args.files[0].endswith(".c") args.output = args.files[0][:-1] + "config.h" - static_qstrs, qstr_vals, qstr_objs = extract_qstrs(args.files) + static_qstrs, qstr_vals = extract_qstrs(args.files) with open(args.output, "w") as f: print( "#include \n" @@ -1029,11 +1015,6 @@ def do_preprocess(args): print("#define %s (%u)" % (q, i + 1), file=f) for i, q in enumerate(sorted(qstr_vals)): print("#define %s (mp_native_qstr_table[%d])" % (q, i + 1), file=f) - for i, q in enumerate(sorted(qstr_objs)): - print( - "#define MP_OBJ_NEW_QSTR_%s ((mp_obj_t)mp_native_obj_table[%d])" % (q, i), - file=f, - ) print("extern const uint16_t mp_native_qstr_table[];", file=f) print("extern const mp_uint_t mp_native_obj_table[];", file=f) @@ -1043,25 +1024,19 @@ def do_link(args): assert args.files[0].endswith(".o") args.output = args.files[0][:-1] + "mpy" native_qstr_vals = [] - native_qstr_objs = [] if args.qstrs is not None: with open(args.qstrs) as f: for l in f: m = re.match(r"#define MP_QSTR_([A-Za-z0-9_]*) \(mp_native_", l) if m: native_qstr_vals.append(m.group(1)) - else: - m = re.match(r"#define MP_OBJ_NEW_QSTR_MP_QSTR_([A-Za-z0-9_]*)", l) - if m: - native_qstr_objs.append(m.group(1)) log(LOG_LEVEL_2, "qstr vals: " + ", ".join(native_qstr_vals)) - log(LOG_LEVEL_2, "qstr objs: " + ", ".join(native_qstr_objs)) env = LinkEnv(args.arch) try: for file in args.files: load_object_file(env, file) - link_objects(env, len(native_qstr_vals), len(native_qstr_objs)) - build_mpy(env, env.find_addr("mpy_init"), args.output, native_qstr_vals, native_qstr_objs) + link_objects(env, len(native_qstr_vals)) + build_mpy(env, env.find_addr("mpy_init"), args.output, native_qstr_vals) except LinkError as er: print("LinkError:", er.args[0]) sys.exit(1) diff --git a/tools/pyboard.py b/tools/pyboard.py index bfec3e1972..a9977cec58 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -49,6 +49,7 @@ Or: """ import ast +import errno import os import struct import sys @@ -226,7 +227,7 @@ class ProcessPtyToTerminal: pty = m.group() # rtscts, dsrdtr params are to workaround pyserial bug: # http://stackoverflow.com/questions/34831131/pyserial-does-not-play-well-with-virtual-port - self.ser = serial.Serial(pty, interCharTimeout=1, rtscts=True, dsrdtr=True) + self.serial = serial.Serial(pty, interCharTimeout=1, rtscts=True, dsrdtr=True) def close(self): import signal @@ -234,13 +235,13 @@ class ProcessPtyToTerminal: os.killpg(os.getpgid(self.subp.pid), signal.SIGTERM) def read(self, size=1): - return self.ser.read(size) + return self.serial.read(size) def write(self, data): - return self.ser.write(data) + return self.serial.write(data) def inWaiting(self): - return self.ser.inWaiting() + return self.serial.inWaiting() class Pyboard: @@ -258,6 +259,7 @@ class Pyboard: self.serial = TelnetToSerial(device, user, password, read_timeout=10) else: import serial + import serial.tools.list_ports # Set options, and exclusive if pyserial supports it serial_kwargs = {"baudrate": baudrate, "interCharTimeout": 1} @@ -488,14 +490,14 @@ class Pyboard: def fs_exists(self, src): try: - self.exec_("import uos\nuos.stat(%s)" % (("'%s'" % src) if src else "")) + self.exec_("import os\nos.stat(%s)" % (("'%s'" % src) if src else "")) return True except PyboardError: return False def fs_ls(self, src): cmd = ( - "import uos\nfor f in uos.ilistdir(%s):\n" + "import os\nfor f in os.ilistdir(%s):\n" " print('{:12} {}{}'.format(f[3]if len(f)>3 else 0,f[0],'/'if f[1]&0x4000 else ''))" % (("'%s'" % src) if src else "") ) @@ -507,7 +509,7 @@ class Pyboard: def repr_consumer(b): buf.extend(b.replace(b"\x04", b"")) - cmd = "import uos\nfor f in uos.ilistdir(%s):\n" " print(repr(f), end=',')" % ( + cmd = "import os\nfor f in os.ilistdir(%s):\n" " print(repr(f), end=',')" % ( ("'%s'" % src) if src else "" ) try: @@ -524,8 +526,8 @@ class Pyboard: def fs_stat(self, src): try: - self.exec_("import uos") - return os.stat_result(self.eval("uos.stat(%s)" % (("'%s'" % src)), parse=True)) + self.exec_("import os") + return os.stat_result(self.eval("os.stat(%s)" % (("'%s'" % src)), parse=True)) except PyboardError as e: raise e.convert(src) @@ -618,13 +620,13 @@ class Pyboard: self.exec_("f.close()") def fs_mkdir(self, dir): - self.exec_("import uos\nuos.mkdir('%s')" % dir) + self.exec_("import os\nos.mkdir('%s')" % dir) def fs_rmdir(self, dir): - self.exec_("import uos\nuos.rmdir('%s')" % dir) + self.exec_("import os\nos.rmdir('%s')" % dir) def fs_rm(self, src): - self.exec_("import uos\nuos.remove('%s')" % src) + self.exec_("import os\nos.remove('%s')" % src) def fs_touch(self, src): self.exec_("f=open('%s','a')\nf.close()" % src) @@ -665,6 +667,10 @@ def filesystem_command(pyb, args, progress_callback=None, verbose=False): args = args[1:] try: if cmd == "cp": + if len(args) == 1: + raise PyboardError( + "cp: missing destination file operand after '{}'".format(args[0]) + ) srcs = args[:-1] dest = args[-1] if dest.startswith(":"): @@ -712,9 +718,9 @@ def filesystem_command(pyb, args, progress_callback=None, verbose=False): _injected_import_hook_code = """\ -import uos, uio +import os, io class _FS: - class File(uio.IOBase): + class File(io.IOBase): def __init__(self): self.off = 0 def ioctl(self, request, arg): @@ -731,10 +737,10 @@ class _FS: raise OSError(-2) # ENOENT def open(self, path, mode): return self.File() -uos.mount(_FS(), '/_') -uos.chdir('/_') +os.mount(_FS(), '/_') +os.chdir('/_') from _injected import * -uos.umount('/_') +os.umount('/_') del _injected_buf, _FS """ diff --git a/tools/pydfu.py b/tools/pydfu.py index 57a8708e5d..e1e4074a6b 100755 --- a/tools/pydfu.py +++ b/tools/pydfu.py @@ -1,15 +1,14 @@ #!/usr/bin/env python - -# SPDX-FileCopyrightText: Copyright (c) 2013/2014 Ibrahim Abdelkader -# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -# -# SPDX-License-Identifier: MIT +# This file is part of the OpenMV project. +# Copyright (c) 2013/2014 Ibrahim Abdelkader +# This work is licensed under the MIT license, see the file LICENSE for +# details. """This module implements enough functionality to program the STM32F4xx over DFU, without requiring dfu-util. See app note AN3156 for a description of the DFU protocol. -See document UM0391 for a dscription of the DFuse file. +See document UM0391 for a description of the DFuse file. """ from __future__ import print_function @@ -76,9 +75,11 @@ __verbose = None # USB DFU interface __DFU_INTERFACE = 0 -import inspect +# Python 3 deprecated getargspec in favour of getfullargspec, but +# Python 2 doesn't have the latter, so detect which one to use +getargspec = getattr(inspect, "getfullargspec", getattr(inspect, "getargspec", None)) -if "length" in inspect.getfullargspec(usb.util.get_string).args: +if "length" in getargspec(usb.util.get_string).args: # PyUSB 1.0.0.b1 has the length argument def get_string(dev, index): return usb.util.get_string(dev, 255, index) diff --git a/tools/tinytest-codegen.py b/tools/tinytest-codegen.py index 79b03f1383..3d2f4d1f73 100755 --- a/tools/tinytest-codegen.py +++ b/tools/tinytest-codegen.py @@ -86,11 +86,10 @@ exclude_tests = ( "basics/bytes_compare3.py", "extmod/ticks_diff.py", "extmod/time_ms_us.py", - "extmod/uheapq_timeq.py", # unicode char issue - "extmod/ujson_loads.py", + "extmod/json_loads.py", # doesn't output to python stdout - "extmod/ure_debug.py", + "extmod/re_debug.py", "extmod/vfs_basic.py", "extmod/vfs_fat_ramdisk.py", "extmod/vfs_fat_fileio.py", diff --git a/tools/verifygitlog.py b/tools/verifygitlog.py index ee3e119f37..ad9385e7ac 100755 --- a/tools/verifygitlog.py +++ b/tools/verifygitlog.py @@ -49,17 +49,17 @@ def git_log(pretty_format, *args): def diagnose_subject_line(subject_line, subject_line_format, err): err.error("Subject line: " + subject_line) if not subject_line.endswith("."): - err.error('* should end with "."') + err.error('* must end with "."') if not re.match(r"^[^!]+: ", subject_line): - err.error('* should start with "path: "') + err.error('* must start with "path: "') if re.match(r"^[^!]+: *$", subject_line): - err.error("* should contain a subject after the path.") + err.error("* must contain a subject after the path.") m = re.match(r"^[^!]+: ([a-z][^ ]*)", subject_line) if m: - err.error('* first word of subject ("{}") should be capitalised.'.format(m.group(1))) + err.error('* first word of subject ("{}") must be capitalised.'.format(m.group(1))) if re.match(r"^[^!]+: [^ ]+$", subject_line): - err.error("* subject should contain more than one word.") - err.error("* should match: " + repr(subject_line_format)) + err.error("* subject must contain more than one word.") + err.error("* must match: " + repr(subject_line_format)) err.error('* Example: "py/runtime: Add support for foo to bar."') @@ -94,11 +94,11 @@ def verify_message_body(raw_body, err): if not re.match(subject_line_format, subject_line): diagnose_subject_line(subject_line, subject_line_format, err) if len(subject_line) >= 73: - err.error("Subject line should be 72 or fewer characters: " + subject_line) + err.error("Subject line must be 72 or fewer characters: " + subject_line) # Second one divides subject and body. if len(raw_body) > 1 and raw_body[1]: - err.error("Second message line should be empty: " + raw_body[1]) + err.error("Second message line must be empty: " + raw_body[1]) # Message body lines. for line in raw_body[2:]: @@ -107,7 +107,7 @@ def verify_message_body(raw_body, err): err.error("Message lines should be 75 or less characters: " + line) if not raw_body[-1].startswith("Signed-off-by: ") or "@" not in raw_body[-1]: - err.warning('Message should be signed-off. Use "git commit -s".') + err.error('Message must be signed-off. Use "git commit -s".') def run(args):