diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index c5039f58b5..0ffa4fcfd9 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,3 +1,15 @@ +#all: Reformat remaining C code that doesn't have a space after a comma. +5b700b0af90591d6b1a2c087bb8de6b7f1bfdd2d + +# ports: Reformat more C and Python source code. +5c32111fa0e31e451b0f1666bdf926be2fdfd82c + +# all: Update Python formatting to latest Black version 22.1.0. +ab2923dfa1174dc177f0a90cb00a7e4ff87958d2 + +# all: Update Python formatting to latest Black version 21.12b0. +3770fab33449a5dadf8eb06edfae0767e75320a6 + # tools/gen-cpydiff.py: Fix formatting of doc strings for new Black. 0f78c36c5aa458a954eed39a46942209107a553e diff --git a/docs/library/array.rst b/docs/library/array.rst index 8801ca6ab9..a78ba82404 100644 --- a/docs/library/array.rst +++ b/docs/library/array.rst @@ -1,10 +1,10 @@ :mod:`array` -- arrays of numeric data -======================================= +====================================== .. module:: array :synopsis: efficient arrays of numeric data -|see_cpython_module| :mod:`cpython:array`. +|see_cpython_module| :mod:`python:array`. Supported format codes: ``b``, ``B``, ``h``, ``H``, ``i``, ``I``, ``l``, ``L``, ``q``, ``Q``, ``f``, ``d`` (the latter 2 depending on the diff --git a/docs/library/binascii.rst b/docs/library/binascii.rst index f43eeb83c4..296a4e9d85 100644 --- a/docs/library/binascii.rst +++ b/docs/library/binascii.rst @@ -1,10 +1,10 @@ :mod:`binascii` -- binary/ASCII conversions -============================================ +=========================================== .. module:: binascii :synopsis: binary/ASCII conversions -|see_cpython_module| :mod:`cpython:binascii`. +|see_cpython_module| :mod:`python:binascii`. This module implements conversions between binary data and various encodings of it in ASCII form (in both directions). @@ -31,11 +31,11 @@ Functions Conforms to `RFC 2045 s.6.8 `_. Returns a bytes object. -.. function:: b2a_base64(data) +.. function:: b2a_base64(data, *, newline=True) Encode binary data in base64 format, as in `RFC 3548 `_. Returns the encoded data - followed by a newline character, as a bytes object. + followed by a newline character if ``newline``is true, as a bytes object. .. function:: crc32(data, value=0, /) diff --git a/docs/library/btree.rst b/docs/library/btree.rst index e160ebe13d..3d58abae6c 100644 --- a/docs/library/btree.rst +++ b/docs/library/btree.rst @@ -7,7 +7,7 @@ :synopsis: simple BTree database The ``btree`` module implements a simple key-value database using external -storage (disk files, or in general case, a random-access ``stream``). Keys are +storage (disk files, or in general case, a random-access `stream`). Keys are stored sorted in the database, and besides efficient retrieval by a key value, a database also supports efficient ordered range scans (retrieval of values with the keys in a given range). On the application interface @@ -80,7 +80,7 @@ Functions .. function:: open(stream, *, flags=0, pagesize=0, cachesize=0, minkeypage=0) - Open a database from a random-access ``stream`` (like an open file). All + Open a database from a random-access `stream` (like an open file). All other parameters are optional and keyword-only, and allow to tweak advanced parameters of the database operation (most users will not need them): diff --git a/docs/library/builtins.rst b/docs/library/builtins.rst index 3a675e7ec1..6003c23fb1 100644 --- a/docs/library/builtins.rst +++ b/docs/library/builtins.rst @@ -33,7 +33,7 @@ Functions and types .. class:: bytes() - |see_cpython| `bytes`. + |see_cpython| `python:bytes`. .. function:: callable() @@ -68,7 +68,7 @@ Functions and types .. class:: frozenset() -`frozenset()` is not enabled on non-Express CircuitPython boards. +`frozenset()` is not enabled on the smallest CircuitPython boards for space reasons. .. function:: getattr() @@ -88,12 +88,12 @@ Functions and types .. classmethod:: from_bytes(bytes, byteorder) - In CircuitPython, ``byteorder`` parameter must be positional (this is + In CircuitPython, the ``byteorder`` parameter must be positional (this is compatible with CPython). .. method:: to_bytes(size, byteorder) - In CircuitPython, ``byteorder`` parameter must be positional (this is + In CircuitPython, the ``byteorder`` parameter must be positional (this is compatible with CPython). .. function:: isinstance() @@ -138,7 +138,7 @@ Functions and types .. function:: reversed() -`reversed()` is not enabled on non-Express CircuitPython boards. +`reversed()` is not enabled on the smallest CircuitPython boards for space reasons. .. function:: round() @@ -224,10 +224,14 @@ Exceptions .. exception:: SystemExit + |see_cpython| `python:SystemExit`. + .. exception:: TimeoutError .. exception:: TypeError + |see_cpython| `python:TypeError`. + .. exception:: UnicodeError .. exception:: ValueError diff --git a/docs/library/collections.rst b/docs/library/collections.rst index 2cc1a215d1..7cc6d9989b 100644 --- a/docs/library/collections.rst +++ b/docs/library/collections.rst @@ -1,12 +1,12 @@ :mod:`collections` -- collection and container types -===================================================== +==================================================== -.. include:: ../templates/unsupported_in_circuitpython.inc +**Limitations:** Not implemented on the smallest CircuitPython boards for space reasons. .. module:: collections :synopsis: collection and container types -|see_cpython_module| :mod:`cpython:collections`. +|see_cpython_module| :mod:`python:collections`. This module implements advanced collection and container types to hold/accumulate various objects. @@ -14,7 +14,7 @@ hold/accumulate various objects. Classes ------- -.. function:: deque(iterable, maxlen[, flags]) +.. class:: deque(iterable, maxlen[, flags]) Deques (double-ended queues) are a list-like container that support O(1) appends and pops from either side of the deque. New deques are created @@ -59,7 +59,7 @@ Classes print(t1.name) assert t2.name == t2[1] -.. function:: OrderedDict(...) +.. class:: OrderedDict(...) ``dict`` type subclass which remembers and preserves the order of keys added. When ordered dict is iterated over, keys/items are returned in diff --git a/docs/library/errno.rst b/docs/library/errno.rst index 1f2c7e14ea..61970291df 100644 --- a/docs/library/errno.rst +++ b/docs/library/errno.rst @@ -4,10 +4,10 @@ .. module:: errno :synopsis: system error codes -|see_cpython_module| :mod:`cpython:errno`. +|see_cpython_module| :mod:`python:errno`. This module provides access to symbolic error codes for `OSError` exception. -A particular inventory of codes depends on :term:`MicroPython port`. +The codes available may vary per CircuitPython build. Constants --------- @@ -15,14 +15,13 @@ Constants .. data:: EEXIST, EAGAIN, etc. Error codes, based on ANSI C/POSIX standard. All error codes start with - "E". As mentioned above, inventory of the codes depends on - :term:`MicroPython port`. Errors are usually accessible as ``exc.args[0]`` + "E". Errors are usually accessible as ``exc.errno`` where ``exc`` is an instance of `OSError`. Usage example:: try: os.mkdir("my_dir") except OSError as exc: - if exc.args[0] == errno.EEXIST: + if exc.errno == errno.EEXIST: print("Directory already exists") .. data:: errorcode diff --git a/docs/library/gc.rst b/docs/library/gc.rst index 1a6c3d68c0..32afc7e982 100644 --- a/docs/library/gc.rst +++ b/docs/library/gc.rst @@ -1,12 +1,10 @@ :mod:`gc` -- control the garbage collector ========================================== -.. include:: ../templates/unsupported_in_circuitpython.inc - .. module:: gc :synopsis: control the garbage collector -|see_cpython_module| :mod:`cpython:gc`. +|see_cpython_module| :mod:`python:gc`. Functions --------- @@ -63,6 +61,6 @@ Functions .. admonition:: Difference to CPython :class: attention - This function is a a MicroPython extension. CPython has a similar + This function is a MicroPython extension. CPython has a similar function - ``set_threshold()``, but due to different GC implementations, its signature and semantics are different. diff --git a/docs/library/heapq.rst b/docs/library/heapq.rst index 7269422847..71a998479f 100644 --- a/docs/library/heapq.rst +++ b/docs/library/heapq.rst @@ -6,7 +6,7 @@ .. module:: heapq :synopsis: heap queue algorithm -|see_cpython_module| :mod:`cpython:heapq`. +|see_cpython_module| :mod:`python:heapq`. This module implements the `min heap queue algorithm `_. diff --git a/docs/library/index.rst b/docs/library/index.rst index 2a8e37a8f4..48b87a7721 100644 --- a/docs/library/index.rst +++ b/docs/library/index.rst @@ -15,7 +15,7 @@ CircuitPython's long-term goal is that code written in CircuitPython using Python standard libraries will be runnable on CPython without changes. These libraries are not enabled on CircuitPython builds with -limited flash memory, usually on non-Express builds: +limited flash memory: ``binascii``, ``errno``, ``json``, ``re``. These libraries are not currently enabled in any CircuitPython build, but may be in the future: @@ -42,21 +42,6 @@ Omitted functions in the ``string`` library ------------------------------------------- A few string operations are not enabled on small builds -(usually non-Express), due to limited flash memory: +due to limited flash memory: ``string.center()``, ``string.partition()``, ``string.splitlines()``, ``string.reversed()``. - - -CircuitPython/MicroPython-specific libraries --------------------------------------------- - -Functionality specific to the CircuitPython/MicroPython implementation is available in -the following libraries. These libraries may change significantly or be removed in future -versions of CircuitPython. - -.. toctree:: - :maxdepth: 1 - - btree.rst - framebuf.rst - micropython.rst diff --git a/docs/library/io.rst b/docs/library/io.rst index 37e3eb7c94..ad1fb03c77 100644 --- a/docs/library/io.rst +++ b/docs/library/io.rst @@ -1,10 +1,10 @@ :mod:`io` -- input/output streams -================================== +================================= .. module:: io :synopsis: input/output streams -|see_cpython_module| :mod:`cpython:io`. +|see_cpython_module| :mod:`python:io`. This module contains additional types of ``stream`` (file-like) objects and helper functions. @@ -112,3 +112,20 @@ Classes .. method:: getvalue() Get the current contents of the underlying buffer which holds data. + +.. class:: StringIO(alloc_size) + :noindex: +.. class:: BytesIO(alloc_size) + :noindex: + + Create an empty `StringIO`/`BytesIO` object, preallocated to hold up + to *alloc_size* number of bytes. That means that writing that amount + of bytes won't lead to reallocation of the buffer, and thus won't hit + out-of-memory situation or lead to memory fragmentation. These constructors + are a MicroPython extension and are recommended for usage only in special + cases and in system-level libraries, not for end-user applications. + + .. admonition:: Difference to CPython + :class: attention + + These constructors are a MicroPython extension. diff --git a/docs/library/json.rst b/docs/library/json.rst index 21574e556b..d65bc0e5a2 100644 --- a/docs/library/json.rst +++ b/docs/library/json.rst @@ -1,10 +1,10 @@ :mod:`json` -- JSON encoding and decoding -========================================== +========================================= .. module:: json :synopsis: JSON encoding and decoding -|see_cpython_module| :mod:`cpython:json`. +|see_cpython_module| :mod:`python:json`. This modules allows to convert between Python objects and the JSON data format. @@ -12,14 +12,20 @@ data format. Functions --------- -.. function:: dump(obj, stream) +.. function:: dump(obj, stream, separators=None) Serialise ``obj`` to a JSON string, writing it to the given *stream*. -.. function:: dumps(obj) + If specified, separators should be an ``(item_separator, key_separator)`` + tuple. The default is ``(', ', ': ')``. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate whitespace. + +.. function:: dumps(obj, separators=None) Return ``obj`` represented as a JSON string. + The arguments have the same meaning as in `dump`. + .. function:: load(stream) Parse the given ``stream``, interpreting it as a JSON string and diff --git a/docs/library/micropython.rst b/docs/library/micropython.rst index dddb813d34..166192de3a 100644 --- a/docs/library/micropython.rst +++ b/docs/library/micropython.rst @@ -1,7 +1,5 @@ -:mod:`micropython` -- access and control MicroPython internals -============================================================== - -.. include:: ../templates/unsupported_in_circuitpython.inc +:mod:`micropython` -- MicroPython extensions and internals +========================================================== .. module:: micropython :synopsis: access and control MicroPython internals @@ -11,7 +9,7 @@ Functions .. function:: const(expr) - Used to declare that the expression is a constant so that the compile can + Used to declare that the expression is a constant so that the compiler can optimise it. The use of this function should be as follows:: from micropython import const @@ -28,111 +26,3 @@ Functions provided as part of the :mod:`micropython` module mainly so that scripts can be written which run under both CPython and MicroPython, by following the above pattern. - -.. function:: opt_level([level]) - - If *level* is given then this function sets the optimisation level for subsequent - compilation of scripts, and returns ``None``. Otherwise it returns the current - optimisation level. - - The optimisation level controls the following compilation features: - - - Assertions: at level 0 assertion statements are enabled and compiled into the - bytecode; at levels 1 and higher assertions are not compiled. - - Built-in ``__debug__`` variable: at level 0 this variable expands to ``True``; - at levels 1 and higher it expands to ``False``. - - Source-code line numbers: at levels 0, 1 and 2 source-code line number are - stored along with the bytecode so that exceptions can report the line number - they occurred at; at levels 3 and higher line numbers are not stored. - - The default optimisation level is usually level 0. - -.. function:: mem_info([verbose]) - - Print information about currently used memory. If the *verbose* argument - is given then extra information is printed. - - The information that is printed is implementation dependent, but currently - includes the amount of stack and heap used. In verbose mode it prints out - the entire heap indicating which blocks are used and which are free. - -.. function:: qstr_info([verbose]) - - Print information about currently interned strings. If the *verbose* - argument is given then extra information is printed. - - The information that is printed is implementation dependent, but currently - includes the number of interned strings and the amount of RAM they use. In - verbose mode it prints out the names of all RAM-interned strings. - -.. function:: stack_use() - - Return an integer representing the current amount of stack that is being - used. The absolute value of this is not particularly useful, rather it - should be used to compute differences in stack usage at different points. - -.. function:: heap_lock() -.. function:: heap_unlock() -.. function:: heap_locked() - - Lock or unlock the heap. When locked no memory allocation can occur and a - ``MemoryError`` will be raised if any heap allocation is attempted. - `heap_locked()` returns a true value if the heap is currently locked. - - These functions can be nested, ie `heap_lock()` can be called multiple times - in a row and the lock-depth will increase, and then `heap_unlock()` must be - called the same number of times to make the heap available again. - - Both `heap_unlock()` and `heap_locked()` return the current lock depth - (after unlocking for the former) as a non-negative integer, with 0 meaning - the heap is not locked. - - If the REPL becomes active with the heap locked then it will be forcefully - unlocked. - - Note: `heap_locked()` is not enabled on most ports by default, - requires ``MICROPY_PY_MICROPYTHON_HEAP_LOCKED``. - -.. function:: kbd_intr(chr) - - Set the character that will raise a `KeyboardInterrupt` exception. By - default this is set to 3 during script execution, corresponding to Ctrl-C. - Passing -1 to this function will disable capture of Ctrl-C, and passing 3 - will restore it. - - This function can be used to prevent the capturing of Ctrl-C on the - incoming stream of characters that is usually used for the REPL, in case - that stream is used for other purposes. - -.. function:: schedule(func, arg) - - Schedule the function *func* to be executed "very soon". The function - is passed the value *arg* as its single argument. "Very soon" means that - the MicroPython runtime will do its best to execute the function at the - earliest possible time, given that it is also trying to be efficient, and - that the following conditions hold: - - - A scheduled function will never preempt another scheduled function. - - Scheduled functions are always executed "between opcodes" which means - that all fundamental Python operations (such as appending to a list) - are guaranteed to be atomic. - - A given port may define "critical regions" within which scheduled - functions will never be executed. Functions may be scheduled within - a critical region but they will not be executed until that region - is exited. An example of a critical region is a preempting interrupt - handler (an IRQ). - - A use for this function is to schedule a callback from a preempting IRQ. - Such an IRQ puts restrictions on the code that runs in the IRQ (for example - the heap may be locked) and scheduling a function to call later will lift - those restrictions. - - Note: If `schedule()` is called from a preempting IRQ, when memory - allocation is not allowed and the callback to be passed to `schedule()` is - a bound method, passing this directly will fail. This is because creating a - reference to a bound method causes memory allocation. A solution is to - create a reference to the method in the class constructor and to pass that - reference to `schedule()`. - - There is a finite queue to hold the scheduled functions and `schedule()` - will raise a `RuntimeError` if the queue is full. diff --git a/docs/library/re.rst b/docs/library/re.rst index d09ef59ba4..19b15d2d2c 100644 --- a/docs/library/re.rst +++ b/docs/library/re.rst @@ -1,10 +1,10 @@ :mod:`re` -- simple regular expressions -======================================== +======================================= .. module:: re :synopsis: regular expressions -|see_cpython_module| :mod:`cpython:re`. +|see_cpython_module| :mod:`python:re`. This module implements regular expression operations. Regular expression syntax supported is a subset of CPython ``re`` module (and actually is diff --git a/docs/library/select.rst b/docs/library/select.rst index 84792bc53a..08bbe08df2 100644 --- a/docs/library/select.rst +++ b/docs/library/select.rst @@ -1,8 +1,6 @@ :mod:`select` -- wait for events on a set of streams ==================================================== -.. include:: ../templates/unsupported_in_circuitpython.inc - .. module:: select :synopsis: wait for events on a set of streams @@ -86,7 +84,7 @@ Methods .. method:: poll.ipoll(timeout=-1, flags=0, /) Like :meth:`poll.poll`, but instead returns an iterator which yields a - ``callee-owned tuples``. This function provides efficient, allocation-free + "callee-owned tuple". This function provides an efficient, allocation-free way to poll on streams. If *flags* is 1, one-shot behaviour for events is employed: streams for diff --git a/docs/library/sys.rst b/docs/library/sys.rst index a0675f0f50..aa0a382748 100644 --- a/docs/library/sys.rst +++ b/docs/library/sys.rst @@ -1,12 +1,10 @@ :mod:`sys` -- system specific functions -======================================== - -.. include:: ../templates/unsupported_in_circuitpython.inc +======================================= .. module:: sys :synopsis: system specific functions -|see_cpython_module| :mod:`cpython:sys`. +|see_cpython_module| :mod:`python:sys`. Functions --------- @@ -35,6 +33,7 @@ Constants * *name* - string "circuitpython" * *version* - tuple (major, minor, micro), e.g. (1, 7, 0) + * *_mpy* - supported mpy file-format version (optional attribute) This object is the recommended way to distinguish CircuitPython from other Python implementations (note that it still may not exist in the very @@ -97,6 +96,12 @@ Constants If you need to check whether your program runs on CircuitPython (vs other Python implementation), use `sys.implementation` instead. +.. data:: ps1 + ps2 + + Mutable attributes holding strings, which are used for the REPL prompt. The defaults + give the standard Python prompt of ``>>>`` and ``...``. + .. data:: stderr Standard error ``stream``. @@ -109,6 +114,14 @@ Constants Standard output ``stream``. +.. data:: tracebacklimit + + A mutable attribute holding an integer value which is the maximum number of traceback + entries to store in an exception. Set to 0 to disable adding tracebacks. Defaults + to 1000. + + Note: this is not available on all ports. + .. data:: version Python language version that this implementation conforms to, as a string. diff --git a/docs/reference/glossary.rst b/docs/reference/glossary.rst index 5aae70af0a..fe65ddaa14 100644 --- a/docs/reference/glossary.rst +++ b/docs/reference/glossary.rst @@ -159,7 +159,7 @@ Glossary typically accessible on a host PC via USB. stream - Also known as a "file-like object". An Python object which provides + Also known as a "file-like object". A Python object which provides sequential read-write access to the underlying data. A stream object implements a corresponding interface, which consists of methods like ``read()``, ``write()``, ``readinto()``, ``seek()``, ``flush()``, diff --git a/docs/reference/mpyfiles.rst b/docs/reference/mpyfiles.rst new file mode 100644 index 0000000000..4995672b52 --- /dev/null +++ b/docs/reference/mpyfiles.rst @@ -0,0 +1,199 @@ +.. _mpy_files: + +MicroPython .mpy files +====================== + +MicroPython defines the concept of an .mpy file which is a binary container +file format that holds precompiled code, and which can be imported like a +normal .py module. The file ``foo.mpy`` can be imported via ``import foo``, +as long as ``foo.mpy`` can be found in the usual way by the import machinery. +Usually, each directory listed in ``sys.path`` is searched in order. When +searching a particular directory ``foo.py`` is looked for first and if that +is not found then ``foo.mpy`` is looked for, then the search continues in the +next directory if neither is found. As such, ``foo.py`` will take precedence +over ``foo.mpy``. + +These .mpy files can contain bytecode which is usually generated from Python +source files (.py files) via the ``mpy-cross`` program. For some architectures +an .mpy file can also contain native machine code, which can be generated in +a variety of ways, most notably from C source code. + +Versioning and compatibility of .mpy files +------------------------------------------ + +A given .mpy file may or may not be compatible with a given MicroPython system. +Compatibility is based on the following: + +* Version of the .mpy file: the version of the file must match the version + supported by the system loading it. + +* Sub-version of the .mpy file: if the .mpy file contains native machine code + then the sub-version of the file must match the version support by the + system loading it. Otherwise, if there is no native machine code in the .mpy + file, then the sub-version is ignored when loading. + +* Small integer bits: the .mpy file will require a minimum number of bits in + a small integer and the system loading it must support at least this many + bits. + +* Native architecture: if the .mpy file contains native machine code then + it will specify the architecture of that machine code and the system + loading it must support execution of that architecture's code. + +If a MicroPython system supports importing .mpy files then the +``sys.implementation._mpy`` field will exist and return an integer which +encodes the version (lower 8 bits), features and native architecture. + +Trying to import an .mpy file that fails one of the first four tests will +raise ``ValueError('incompatible .mpy file')``. Trying to import an .mpy +file that fails the native architecture test (if it contains native machine +code) will raise ``ValueError('incompatible .mpy arch')``. + +If importing an .mpy file fails then try the following: + +* Determine the .mpy version and flags supported by your MicroPython system + by executing:: + + import sys + sys_mpy = sys.implementation._mpy + arch = [None, 'x86', 'x64', + 'armv6', 'armv6m', 'armv7m', 'armv7em', 'armv7emsp', 'armv7emdp', + 'xtensa', 'xtensawin'][sys_mpy >> 10] + print('mpy version:', sys_mpy & 0xff) + print('mpy flags:', end='') + if arch: + print(' -march=' + arch, end='') + print() + +* Check the validity of the .mpy file by inspecting the first two bytes of + the file. The first byte should be an uppercase 'M' and the second byte + will be the version number, which should match the system version from above. + If it doesn't match then rebuild the .mpy file. + +* Check if the system .mpy version matches the version emitted by ``mpy-cross`` + that was used to build the .mpy file, found by ``mpy-cross --version``. + If it doesn't match then recompile ``mpy-cross`` from the Git repository + checked out at the tag (or hash) reported by ``mpy-cross --version``. + +* Make sure you are using the correct ``mpy-cross`` flags, found by the code + above, or by inspecting the ``MPY_CROSS_FLAGS`` Makefile variable for the + port that you are using. + +The following table shows the correspondence between MicroPython release +and .mpy version. + +=================== ============ +MicroPython release .mpy version +=================== ============ +v1.19 and up 6 +v1.12 - v1.18 5 +v1.11 4 +v1.9.3 - v1.10 3 +v1.9 - v1.9.2 2 +v1.5.1 - v1.8.7 0 +=================== ============ + +For completeness, the next table shows the Git commit of the main +MicroPython repository at which the .mpy version was changed. + +=================== ======================================== +.mpy version change Git commit +=================== ======================================== +5 to 6 f2040bfc7ee033e48acef9f289790f3b4e6b74e5 +4 to 5 5716c5cf65e9b2cb46c2906f40302401bdd27517 +3 to 4 9a5f92ea72754c01cc03e5efcdfe94021120531e +2 to 3 ff93fd4f50321c6190e1659b19e64fef3045a484 +1 to 2 dd11af209d226b7d18d5148b239662e30ed60bad +0 to 1 6a11048af1d01c78bdacddadd1b72dc7ba7c6478 +initial version 0 d8c834c95d506db979ec871417de90b7951edc30 +=================== ======================================== + +Binary encoding of .mpy files +----------------------------- + +MicroPython .mpy files are a binary container format with code objects (bytecode +and native machine code) stored internally in a nested hierarchy. The code for +the outer module is stored first, and then its children follow. Each child may +have further children, for example in the case of a class having methods, or a +function defining a lambda or comprehension. To keep files small while still +providing a large range of possible values it uses the concept of a +variably-encoded-unsigned-integer (vuint) in many places. Similar to utf-8 +encoding, this encoding stores 7 bits per byte with the 8th bit (MSB) set +if one or more bytes follow. The bits of the unsigned integer are stored +in the vuint in LSB form. + +The top-level of an .mpy file consists of three parts: + +* The header. + +* The global qstr and constant tables. + +* The raw-code for the outer scope of the module. + This outer scope is executed when the .mpy file is imported. + +You can inspect the contents of a .mpy file by using ``mpy-tool.py``, for +example (run from the root of the main MicroPython repository):: + + $ ./tools/mpy-tool.py -xd myfile.mpy + +The header +~~~~~~~~~~ + +The .mpy header is: + +====== ================================ +size field +====== ================================ +byte value 0x4d (ASCII 'M') +byte .mpy version number +byte feature flags +byte number of bits in a small int +====== ================================ + +The global qstr and constant tables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An .mpy file contains a single qstr table, and a single constant object table. +These are global to the .mpy file, they are referenced by all nested raw-code +objects. The qstr table maps internal qstr number (internal to the .mpy file) +to the resolved qstr number of the runtime that the .mpy file is imported into. +This links the .mpy file with the rest of the system that it executes within. +The constant object table is populated with references to all constant objects +that the .mpy file needs. + +====== ================================ +size field +====== ================================ +vuint number of qstrs +vuint number of constant objects +... qstr data +... encoded constant objects +====== ================================ + +Raw code elements +~~~~~~~~~~~~~~~~~ + +A raw-code element contains code, either bytecode or native machine code. Its +contents are: + +====== ================================ +size field +====== ================================ +vuint type, size and whether there are sub-raw-code elements +... code (bytecode or machine code) +vuint number of sub-raw-code elements (only if non-zero) +... sub-raw-code elements +====== ================================ + +The first vuint in a raw-code element encodes the type of code stored in this +element (the two least-significant bits), whether this raw-code has any +children (the third least-significant bit), and the length of the code that +follows (the amount of RAM to allocate for it). + +Following the vuint comes the code itself. Unless the code type is viper code +with relocations, this code is constant data and does not need to be modified. + +If this raw-code has any children (as indicated by a bit in the first vuint), +following the code comes a vuint counting the number of sub-raw-code elements. + +Finally any sub-raw-code elements are stored, recursively. diff --git a/docs/workflows.md b/docs/workflows.md index 453b425491..10d2897870 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -94,6 +94,10 @@ MDNS is used to resolve [`circuitpython.local`](http://circuitpython.local) to a hostname of the form `cpy-XXXXXX.local`. The `XXXXXX` is based on network MAC address. The device also provides the MDNS service with service type `_circuitpython` and protocol `_tcp`. +Since port 80 (or the port assigned to `CIRCUITPY_WEB_API_PORT`) is used for web workflow, the `mdns` +[module](https://docs.circuitpython.org/en/latest/shared-bindings/mdns/index.html#mdns.Server.advertise_service) +can't advertise an additional service on that port. + ### HTTP The web server is HTTP 1.1 and may use chunked responses so that it doesn't need to precompute content length. diff --git a/examples/natmod/.gitignore b/examples/natmod/.gitignore deleted file mode 100644 index 4815d20f06..0000000000 --- a/examples/natmod/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.mpy diff --git a/examples/natmod/btree/Makefile b/examples/natmod/btree/Makefile deleted file mode 100644 index d795102b4a..0000000000 --- a/examples/natmod/btree/Makefile +++ /dev/null @@ -1,37 +0,0 @@ -# Location of top-level MicroPython directory -MPY_DIR = ../../.. - -# Name of module (different to built-in btree so it can coexist) -MOD = btree_$(ARCH) - -# Source files (.c or .py) -SRC = btree_c.c btree_py.py - -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) -ARCH = x64 - -BTREE_DIR = $(MPY_DIR)/lib/berkeley-db-1.xx -BTREE_DEFS = -D__DBINTERFACE_PRIVATE=1 -Dmpool_error="(void)" -Dabort=abort_ "-Dvirt_fd_t=void*" $(BTREE_DEFS_EXTRA) -CFLAGS += -I$(BTREE_DIR)/PORT/include -CFLAGS += -Wno-old-style-definition -Wno-sign-compare -Wno-unused-parameter $(BTREE_DEFS) - -SRC += $(addprefix $(realpath $(BTREE_DIR))/,\ - btree/bt_close.c \ - btree/bt_conv.c \ - btree/bt_delete.c \ - btree/bt_get.c \ - btree/bt_open.c \ - btree/bt_overflow.c \ - btree/bt_page.c \ - btree/bt_put.c \ - btree/bt_search.c \ - btree/bt_seq.c \ - btree/bt_split.c \ - btree/bt_utils.c \ - mpool/mpool.c \ - ) - -include $(MPY_DIR)/py/dynruntime.mk - -# btree needs gnu99 defined -CFLAGS += -std=gnu99 diff --git a/examples/natmod/btree/btree_c.c b/examples/natmod/btree/btree_c.c deleted file mode 100644 index b44b7bb300..0000000000 --- a/examples/natmod/btree/btree_c.c +++ /dev/null @@ -1,148 +0,0 @@ -#define MICROPY_PY_BTREE (1) - -#include "py/dynruntime.h" - -#include - -#if !defined(__linux__) -void *memcpy(void *dst, const void *src, size_t n) { - return mp_fun_table.memmove_(dst, src, n); -} -void *memset(void *s, int c, size_t n) { - return mp_fun_table.memset_(s, c, n); -} -#endif - -void *memmove(void *dest, const void *src, size_t n) { - return mp_fun_table.memmove_(dest, src, n); -} - -void *malloc(size_t n) { - void *ptr = m_malloc(n, false); - return ptr; -} -void *realloc(void *ptr, size_t n) { - mp_printf(&mp_plat_print, "UNDEF %d\n", __LINE__); - return NULL; -} -void *calloc(size_t n, size_t m) { - void *ptr = m_malloc(n * m, false); - // memory already cleared by conservative GC - return ptr; -} - -void free(void *ptr) { - m_free(ptr); -} - -void abort_(void) { - nlr_raise(mp_obj_new_exception(mp_load_global(MP_QSTR_RuntimeError))); -} - -int native_errno; -#if defined(__linux__) -int *__errno_location (void) -#else -int *__errno (void) -#endif -{ - return &native_errno; -} - -ssize_t mp_stream_posix_write(void *stream, const void *buf, size_t len) { - mp_obj_base_t* o = stream; - const mp_stream_p_t *stream_p = o->type->ext[0].protocol; - mp_uint_t out_sz = stream_p->write(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno); - if (out_sz == MP_STREAM_ERROR) { - return -1; - } else { - return out_sz; - } -} - -ssize_t mp_stream_posix_read(void *stream, void *buf, size_t len) { - mp_obj_base_t* o = stream; - const mp_stream_p_t *stream_p = o->type->ext[0].protocol; - mp_uint_t out_sz = stream_p->read(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno); - if (out_sz == MP_STREAM_ERROR) { - return -1; - } else { - return out_sz; - } -} - -off_t mp_stream_posix_lseek(void *stream, off_t offset, int whence) { - const mp_obj_base_t* o = stream; - const mp_stream_p_t *stream_p = o->type->ext[0].protocol; - struct mp_stream_seek_t seek_s; - seek_s.offset = offset; - seek_s.whence = whence; - mp_uint_t res = stream_p->ioctl(MP_OBJ_FROM_PTR(stream), MP_STREAM_SEEK, (mp_uint_t)(uintptr_t)&seek_s, &native_errno); - if (res == MP_STREAM_ERROR) { - return -1; - } - return seek_s.offset; -} - -int mp_stream_posix_fsync(void *stream) { - mp_obj_base_t* o = stream; - const mp_stream_p_t *stream_p = o->type->ext[0].protocol; - mp_uint_t res = stream_p->ioctl(MP_OBJ_FROM_PTR(stream), MP_STREAM_FLUSH, 0, &native_errno); - if (res == MP_STREAM_ERROR) { - return -1; - } - return res; -} - -mp_obj_full_type_t btree_type; - -#include "extmod/modbtree.c" - -mp_map_elem_t btree_locals_dict_table[8]; -STATIC MP_DEFINE_CONST_DICT(btree_locals_dict, btree_locals_dict_table); - -STATIC mp_obj_t btree_open(size_t n_args, const mp_obj_t *args) { - // Make sure we got a stream object - mp_get_stream_raise(args[0], MP_STREAM_OP_READ | MP_STREAM_OP_WRITE | MP_STREAM_OP_IOCTL); - - BTREEINFO openinfo = {0}; - openinfo.flags = mp_obj_get_int(args[1]); - openinfo.cachesize = mp_obj_get_int(args[2]); - openinfo.psize = mp_obj_get_int(args[3]); - openinfo.minkeypage = mp_obj_get_int(args[4]); - DB *db = __bt_open(MP_OBJ_TO_PTR(args[0]), &btree_stream_fvtable, &openinfo, 0); - if (db == NULL) { - mp_raise_OSError(native_errno); - } - - return MP_OBJ_FROM_PTR(btree_new(db, args[0])); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(btree_open_obj, 5, 5, btree_open); - -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - MP_DYNRUNTIME_INIT_ENTRY - - btree_type.base.type = (void*)&mp_fun_table.type_type; - btree_type.flags = MP_TYPE_FLAG_EXTENDED; - btree_type.name = MP_QSTR_btree; - btree_type.print = btree_print; - btree_type.ext[0].getiter = btree_getiter; - btree_type.ext[0].iternext = btree_iternext; - btree_type.ext[0].binary_op = btree_binary_op; - btree_type.ext[0].subscr = btree_subscr; - btree_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_close), MP_OBJ_FROM_PTR(&btree_close_obj) }; - btree_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_flush), MP_OBJ_FROM_PTR(&btree_flush_obj) }; - btree_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_get), MP_OBJ_FROM_PTR(&btree_get_obj) }; - btree_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_put), MP_OBJ_FROM_PTR(&btree_put_obj) }; - btree_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_seq), MP_OBJ_FROM_PTR(&btree_seq_obj) }; - btree_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_keys), MP_OBJ_FROM_PTR(&btree_keys_obj) }; - btree_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_values), MP_OBJ_FROM_PTR(&btree_values_obj) }; - btree_locals_dict_table[7] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_items), MP_OBJ_FROM_PTR(&btree_items_obj) }; - btree_type.locals_dict = (void*)&btree_locals_dict; - - mp_store_global(MP_QSTR__open, MP_OBJ_FROM_PTR(&btree_open_obj)); - mp_store_global(MP_QSTR_INCL, MP_OBJ_NEW_SMALL_INT(FLAG_END_KEY_INCL)); - mp_store_global(MP_QSTR_DESC, MP_OBJ_NEW_SMALL_INT(FLAG_DESC)); - - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/natmod/btree/btree_py.py b/examples/natmod/btree/btree_py.py deleted file mode 100644 index bd53c084a1..0000000000 --- a/examples/natmod/btree/btree_py.py +++ /dev/null @@ -1,3 +0,0 @@ -# Implemented in Python to support keyword arguments -def open(stream, *, flags=0, cachesize=0, pagesize=0, minkeypage=0): - return _open(stream, flags, cachesize, pagesize, minkeypage) diff --git a/examples/natmod/features0/Makefile b/examples/natmod/features0/Makefile deleted file mode 100644 index 57490df90a..0000000000 --- a/examples/natmod/features0/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -# Location of top-level MicroPython directory -MPY_DIR = ../../.. - -# Name of module -MOD = features0 - -# Source files (.c or .py) -SRC = features0.c - -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) -ARCH = x64 - -# Include to get the rules for compiling and linking the module -include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/features0/features0.c b/examples/natmod/features0/features0.c deleted file mode 100644 index 1b1867a3bc..0000000000 --- a/examples/natmod/features0/features0.c +++ /dev/null @@ -1,40 +0,0 @@ -/* This example demonstrates the following features in a native module: - - defining a simple function exposed to Python - - defining a local, helper C function - - getting and creating integer objects -*/ - -// Include the header file to get access to the MicroPython API -#include "py/dynruntime.h" - -// Helper function to compute factorial -STATIC mp_int_t factorial_helper(mp_int_t x) { - if (x == 0) { - return 1; - } - return x * factorial_helper(x - 1); -} - -// This is the function which will be called from Python, as factorial(x) -STATIC mp_obj_t factorial(mp_obj_t x_obj) { - // Extract the integer from the MicroPython input object - mp_int_t x = mp_obj_get_int(x_obj); - // Calculate the factorial - mp_int_t result = factorial_helper(x); - // Convert the result to a MicroPython integer object and return it - return mp_obj_new_int(result); -} -// Define a Python reference to the function above -STATIC MP_DEFINE_CONST_FUN_OBJ_1(factorial_obj, factorial); - -// This is the entry point and is called when the module is imported -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - // This must be first, it sets up the globals dict and other things - MP_DYNRUNTIME_INIT_ENTRY - - // Make the function available in the module's namespace - mp_store_global(MP_QSTR_factorial, MP_OBJ_FROM_PTR(&factorial_obj)); - - // This must be last, it restores the globals dict - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/natmod/features1/Makefile b/examples/natmod/features1/Makefile deleted file mode 100644 index 010640daf9..0000000000 --- a/examples/natmod/features1/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -# Location of top-level MicroPython directory -MPY_DIR = ../../.. - -# Name of module -MOD = features1 - -# Source files (.c or .py) -SRC = features1.c - -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) -ARCH = x64 - -# Include to get the rules for compiling and linking the module -include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/features1/features1.c b/examples/natmod/features1/features1.c deleted file mode 100644 index a5e82252a1..0000000000 --- a/examples/natmod/features1/features1.c +++ /dev/null @@ -1,106 +0,0 @@ -/* This example demonstrates the following features in a native module: - - defining simple functions exposed to Python - - defining local, helper C functions - - defining constant integers and strings exposed to Python - - getting and creating integer objects - - creating Python lists - - raising exceptions - - allocating memory - - BSS and constant data (rodata) - - relocated pointers in rodata -*/ - -// Include the header file to get access to the MicroPython API -#include "py/dynruntime.h" - -// BSS (zero) data -uint16_t data16[4]; - -// Constant data (rodata) -const uint8_t table8[] = { 0, 1, 1, 2, 3, 5, 8, 13 }; -const uint16_t table16[] = { 0x1000, 0x2000 }; - -// Constant data pointing to BSS/constant data -uint16_t *const table_ptr16a[] = { &data16[0], &data16[1], &data16[2], &data16[3] }; -const uint16_t *const table_ptr16b[] = { &table16[0], &table16[1] }; - -// A simple function that adds its 2 arguments (must be integers) -STATIC mp_obj_t add(mp_obj_t x_in, mp_obj_t y_in) { - mp_int_t x = mp_obj_get_int(x_in); - mp_int_t y = mp_obj_get_int(y_in); - return mp_obj_new_int(x + y); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_obj, add); - -// A local helper function (not exposed to Python) -STATIC mp_int_t fibonacci_helper(mp_int_t x) { - if (x < MP_ARRAY_SIZE(table8)) { - return table8[x]; - } else { - return fibonacci_helper(x - 1) + fibonacci_helper(x - 2); - } -} - -// A function which computes Fibonacci numbers -STATIC mp_obj_t fibonacci(mp_obj_t x_in) { - mp_int_t x = mp_obj_get_int(x_in); - if (x < 0) { - mp_raise_ValueError(MP_ERROR_TEXT("can't compute negative Fibonacci number")); - } - return mp_obj_new_int(fibonacci_helper(x)); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(fibonacci_obj, fibonacci); - -// A function that accesses the BSS data -STATIC mp_obj_t access(size_t n_args, const mp_obj_t *args) { - if (n_args == 0) { - // Create a list holding all items from data16 - mp_obj_list_t *lst = MP_OBJ_TO_PTR(mp_obj_new_list(MP_ARRAY_SIZE(data16), NULL)); - for (int i = 0; i < MP_ARRAY_SIZE(data16); ++i) { - lst->items[i] = mp_obj_new_int(data16[i]); - } - return MP_OBJ_FROM_PTR(lst); - } else if (n_args == 1) { - // Get one item from data16 - mp_int_t idx = mp_obj_get_int(args[0]) & 3; - return mp_obj_new_int(data16[idx]); - } else { - // Set one item in data16 (via table_ptr16a) - mp_int_t idx = mp_obj_get_int(args[0]) & 3; - *table_ptr16a[idx] = mp_obj_get_int(args[1]); - return mp_const_none; - } -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(access_obj, 0, 2, access); - -// A function that allocates memory and creates a bytearray -STATIC mp_obj_t make_array(void) { - uint16_t *ptr = m_new(uint16_t, MP_ARRAY_SIZE(table_ptr16b)); - for (int i = 0; i < MP_ARRAY_SIZE(table_ptr16b); ++i) { - ptr[i] = *table_ptr16b[i]; - } - return mp_obj_new_bytearray_by_ref(sizeof(uint16_t) * MP_ARRAY_SIZE(table_ptr16b), ptr); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_0(make_array_obj, make_array); - -// This is the entry point and is called when the module is imported -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - // This must be first, it sets up the globals dict and other things - MP_DYNRUNTIME_INIT_ENTRY - - // Messages can be printed as usually - mp_printf(&mp_plat_print, "initialising module self=%p\n", self); - - // Make the functions available in the module's namespace - mp_store_global(MP_QSTR_add, MP_OBJ_FROM_PTR(&add_obj)); - mp_store_global(MP_QSTR_fibonacci, MP_OBJ_FROM_PTR(&fibonacci_obj)); - mp_store_global(MP_QSTR_access, MP_OBJ_FROM_PTR(&access_obj)); - mp_store_global(MP_QSTR_make_array, MP_OBJ_FROM_PTR(&make_array_obj)); - - // Add some constants to the module's namespace - mp_store_global(MP_QSTR_VAL, MP_OBJ_NEW_SMALL_INT(42)); - mp_store_global(MP_QSTR_MSG, MP_OBJ_NEW_QSTR(MP_QSTR_HELLO_MICROPYTHON)); - - // This must be last, it restores the globals dict - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/natmod/features2/Makefile b/examples/natmod/features2/Makefile deleted file mode 100644 index 4fd23c6879..0000000000 --- a/examples/natmod/features2/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -# Location of top-level MicroPython directory -MPY_DIR = ../../.. - -# Name of module -MOD = features2 - -# Source files (.c or .py) -SRC = main.c prod.c test.py - -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) -ARCH = x64 - -# Include to get the rules for compiling and linking the module -include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/features2/main.c b/examples/natmod/features2/main.c deleted file mode 100644 index 1a39700dc4..0000000000 --- a/examples/natmod/features2/main.c +++ /dev/null @@ -1,83 +0,0 @@ -/* This example demonstrates the following features in a native module: - - using floats - - defining additional code in Python (see test.py) - - have extra C code in a separate file (see prod.c) -*/ - -// Include the header file to get access to the MicroPython API -#include "py/dynruntime.h" - -// Include the header for auxiliary C code for this module -#include "prod.h" - -// Automatically detect if this module should include double-precision code. -// If double precision is supported by the target architecture then it can -// be used in native module regardless of what float setting the target -// MicroPython runtime uses (being none, float or double). -#if defined(__i386__) || defined(__x86_64__) || (defined(__ARM_FP) && (__ARM_FP & 8)) -#define USE_DOUBLE 1 -#else -#define USE_DOUBLE 0 -#endif - -// A function that uses the default float type configured for the current target -// This default can be overridden by specifying MICROPY_FLOAT_IMPL at the make level -STATIC mp_obj_t add(mp_obj_t x, mp_obj_t y) { - return mp_obj_new_float(mp_obj_get_float(x) + mp_obj_get_float(y)); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_obj, add); - -// A function that explicitly uses single precision floats -STATIC mp_obj_t add_f(mp_obj_t x, mp_obj_t y) { - return mp_obj_new_float_from_f(mp_obj_get_float_to_f(x) + mp_obj_get_float_to_f(y)); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_f_obj, add_f); - -#if USE_DOUBLE -// A function that explicitly uses double precision floats -STATIC mp_obj_t add_d(mp_obj_t x, mp_obj_t y) { - return mp_obj_new_float_from_d(mp_obj_get_float_to_d(x) + mp_obj_get_float_to_d(y)); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_d_obj, add_d); -#endif - -// A function that computes the product of floats in an array. -// This function uses the most general C argument interface, which is more difficult -// to use but has access to the globals dict of the module via self->globals. -STATIC mp_obj_t productf(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - // Check number of arguments is valid - mp_arg_check_num(n_args, n_kw, 1, 1, false); - - // Extract buffer pointer and verify typecode - mp_buffer_info_t bufinfo; - mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_RW); - if (bufinfo.typecode != 'f') { - mp_raise_ValueError(MP_ERROR_TEXT("expecting float array")); - } - - // Compute product, store result back in first element of array - float *ptr = bufinfo.buf; - float prod = prod_array(bufinfo.len / sizeof(*ptr), ptr); - ptr[0] = prod; - - return mp_const_none; -} - -// This is the entry point and is called when the module is imported -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - // This must be first, it sets up the globals dict and other things - MP_DYNRUNTIME_INIT_ENTRY - - // Make the functions available in the module's namespace - mp_store_global(MP_QSTR_add, MP_OBJ_FROM_PTR(&add_obj)); - mp_store_global(MP_QSTR_add_f, MP_OBJ_FROM_PTR(&add_f_obj)); - #if USE_DOUBLE - mp_store_global(MP_QSTR_add_d, MP_OBJ_FROM_PTR(&add_d_obj)); - #endif - - // The productf function uses the most general C argument interface - mp_store_global(MP_QSTR_productf, MP_DYNRUNTIME_MAKE_FUNCTION(productf)); - - // This must be last, it restores the globals dict - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/natmod/features2/prod.c b/examples/natmod/features2/prod.c deleted file mode 100644 index 7791dcad1d..0000000000 --- a/examples/natmod/features2/prod.c +++ /dev/null @@ -1,9 +0,0 @@ -#include "prod.h" - -float prod_array(int n, float *ar) { - float ans = 1; - for (int i = 0; i < n; ++i) { - ans *= ar[i]; - } - return ans; -} diff --git a/examples/natmod/features2/prod.h b/examples/natmod/features2/prod.h deleted file mode 100644 index f27dd8d033..0000000000 --- a/examples/natmod/features2/prod.h +++ /dev/null @@ -1 +0,0 @@ -float prod_array(int n, float *ar); diff --git a/examples/natmod/features2/test.py b/examples/natmod/features2/test.py deleted file mode 100644 index 5ac80120d7..0000000000 --- a/examples/natmod/features2/test.py +++ /dev/null @@ -1,29 +0,0 @@ -# This Python code will be merged with the C code in main.c - -import array - - -def isclose(a, b): - return abs(a - b) < 1e-3 - - -def test(): - tests = [ - isclose(add(0.1, 0.2), 0.3), - isclose(add_f(0.1, 0.2), 0.3), - ] - - ar = array.array("f", [1, 2, 3.5]) - productf(ar) - tests.append(isclose(ar[0], 7)) - - if "add_d" in globals(): - tests.append(isclose(add_d(0.1, 0.2), 0.3)) - - print(tests) - - if not all(tests): - raise SystemExit(1) - - -test() diff --git a/examples/natmod/framebuf/Makefile b/examples/natmod/framebuf/Makefile deleted file mode 100644 index 2e2b815975..0000000000 --- a/examples/natmod/framebuf/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# Location of top-level MicroPython directory -MPY_DIR = ../../.. - -# Name of module (different to built-in framebuf so it can coexist) -MOD = framebuf_$(ARCH) - -# Source files (.c or .py) -SRC = framebuf.c - -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) -ARCH = x64 - -include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/framebuf/framebuf.c b/examples/natmod/framebuf/framebuf.c deleted file mode 100644 index 4497ab3318..0000000000 --- a/examples/natmod/framebuf/framebuf.c +++ /dev/null @@ -1,50 +0,0 @@ -#define MICROPY_PY_FRAMEBUF (1) - -#include "py/dynruntime.h" - -#if !defined(__linux__) -void *memset(void *s, int c, size_t n) { - return mp_fun_table.memset_(s, c, n); -} -#endif - -mp_obj_full_type_t mp_type_framebuf; - -#include "extmod/modframebuf.c" - -mp_map_elem_t framebuf_locals_dict_table[10]; -STATIC MP_DEFINE_CONST_DICT(framebuf_locals_dict, framebuf_locals_dict_table); - -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - MP_DYNRUNTIME_INIT_ENTRY - - mp_type_framebuf.base.type = (void*)&mp_type_type; - mp_type_framebuf.flags = MP_TYPE_FLAG_EXTENDED; - mp_type_framebuf.name = MP_QSTR_FrameBuffer; - mp_type_framebuf.make_new = framebuf_make_new; - mp_type_framebuf.ext[0].buffer_p.get_buffer = framebuf_get_buffer; - framebuf_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_fill), MP_OBJ_FROM_PTR(&framebuf_fill_obj) }; - framebuf_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_fill_rect), MP_OBJ_FROM_PTR(&framebuf_fill_rect_obj) }; - framebuf_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_pixel), MP_OBJ_FROM_PTR(&framebuf_pixel_obj) }; - framebuf_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_hline), MP_OBJ_FROM_PTR(&framebuf_hline_obj) }; - framebuf_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_vline), MP_OBJ_FROM_PTR(&framebuf_vline_obj) }; - framebuf_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_rect), MP_OBJ_FROM_PTR(&framebuf_rect_obj) }; - framebuf_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_line), MP_OBJ_FROM_PTR(&framebuf_line_obj) }; - framebuf_locals_dict_table[7] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_blit), MP_OBJ_FROM_PTR(&framebuf_blit_obj) }; - framebuf_locals_dict_table[8] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_scroll), MP_OBJ_FROM_PTR(&framebuf_scroll_obj) }; - framebuf_locals_dict_table[9] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_text), MP_OBJ_FROM_PTR(&framebuf_text_obj) }; - mp_type_framebuf.locals_dict = (void*)&framebuf_locals_dict; - - mp_store_global(MP_QSTR_FrameBuffer, MP_OBJ_FROM_PTR(&mp_type_framebuf)); - mp_store_global(MP_QSTR_FrameBuffer1, MP_OBJ_FROM_PTR(&legacy_framebuffer1_obj)); - mp_store_global(MP_QSTR_MVLSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MVLSB)); - mp_store_global(MP_QSTR_MONO_VLSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MVLSB)); - mp_store_global(MP_QSTR_RGB565, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_RGB565)); - mp_store_global(MP_QSTR_GS2_HMSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_GS2_HMSB)); - mp_store_global(MP_QSTR_GS4_HMSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_GS4_HMSB)); - mp_store_global(MP_QSTR_GS8, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_GS8)); - mp_store_global(MP_QSTR_MONO_HLSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MHLSB)); - mp_store_global(MP_QSTR_MONO_HMSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MHMSB)); - - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/natmod/uheapq/Makefile b/examples/natmod/uheapq/Makefile deleted file mode 100644 index 55de3cc081..0000000000 --- a/examples/natmod/uheapq/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# Location of top-level MicroPython directory -MPY_DIR = ../../.. - -# Name of module (different to built-in uheapq so it can coexist) -MOD = uheapq_$(ARCH) - -# Source files (.c or .py) -SRC = uheapq.c - -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) -ARCH = x64 - -include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/uheapq/uheapq.c b/examples/natmod/uheapq/uheapq.c deleted file mode 100644 index 9da6bb4ada..0000000000 --- a/examples/natmod/uheapq/uheapq.c +++ /dev/null @@ -1,16 +0,0 @@ -#define MICROPY_PY_UHEAPQ (1) - -#include "py/dynruntime.h" - -#include "extmod/moduheapq.c" - -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - MP_DYNRUNTIME_INIT_ENTRY - - mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_uheapq)); - mp_store_global(MP_QSTR_heappush, MP_OBJ_FROM_PTR(&mod_uheapq_heappush_obj)); - mp_store_global(MP_QSTR_heappop, MP_OBJ_FROM_PTR(&mod_uheapq_heappop_obj)); - mp_store_global(MP_QSTR_heapify, MP_OBJ_FROM_PTR(&mod_uheapq_heapify_obj)); - - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/natmod/urandom/Makefile b/examples/natmod/urandom/Makefile deleted file mode 100644 index 3f018baaf7..0000000000 --- a/examples/natmod/urandom/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# Location of top-level MicroPython directory -MPY_DIR = ../../.. - -# Name of module (different to built-in urandom so it can coexist) -MOD = urandom_$(ARCH) - -# Source files (.c or .py) -SRC = urandom.c - -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) -ARCH = x64 - -include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/urandom/urandom.c b/examples/natmod/urandom/urandom.c deleted file mode 100644 index e1fed6a554..0000000000 --- a/examples/natmod/urandom/urandom.c +++ /dev/null @@ -1,33 +0,0 @@ -#define MICROPY_PY_URANDOM (1) -#define MICROPY_PY_URANDOM_EXTRA_FUNCS (1) - -#include "py/dynruntime.h" - -// Dynamic native modules don't support a data section so these must go in the BSS -uint32_t yasmarang_pad, yasmarang_n, yasmarang_d; -uint8_t yasmarang_dat; - -#include "extmod/modurandom.c" - -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - MP_DYNRUNTIME_INIT_ENTRY - - yasmarang_pad = 0xeda4baba; - yasmarang_n = 69; - yasmarang_d = 233; - - mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_urandom)); - mp_store_global(MP_QSTR_getrandbits, MP_OBJ_FROM_PTR(&mod_urandom_getrandbits_obj)); - mp_store_global(MP_QSTR_seed, MP_OBJ_FROM_PTR(&mod_urandom_seed_obj)); - #if MICROPY_PY_URANDOM_EXTRA_FUNCS - mp_store_global(MP_QSTR_randrange, MP_OBJ_FROM_PTR(&mod_urandom_randrange_obj)); - mp_store_global(MP_QSTR_randint, MP_OBJ_FROM_PTR(&mod_urandom_randint_obj)); - mp_store_global(MP_QSTR_choice, MP_OBJ_FROM_PTR(&mod_urandom_choice_obj)); - #if MICROPY_PY_BUILTINS_FLOAT - mp_store_global(MP_QSTR_random, MP_OBJ_FROM_PTR(&mod_urandom_random_obj)); - mp_store_global(MP_QSTR_uniform, MP_OBJ_FROM_PTR(&mod_urandom_uniform_obj)); - #endif - #endif - - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/natmod/ure/Makefile b/examples/natmod/ure/Makefile deleted file mode 100644 index f5254298fd..0000000000 --- a/examples/natmod/ure/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# Location of top-level MicroPython directory -MPY_DIR = ../../.. - -# Name of module (different to built-in ure so it can coexist) -MOD = ure_$(ARCH) - -# Source files (.c or .py) -SRC = ure.c - -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) -ARCH = x64 - -include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/ure/ure.c b/examples/natmod/ure/ure.c deleted file mode 100644 index 175b93e395..0000000000 --- a/examples/natmod/ure/ure.c +++ /dev/null @@ -1,78 +0,0 @@ -#define MICROPY_STACK_CHECK (1) -#define MICROPY_PY_URE (1) -#define MICROPY_PY_URE_MATCH_GROUPS (1) -#define MICROPY_PY_URE_MATCH_SPAN_START_END (1) -#define MICROPY_PY_URE_SUB (0) // requires vstr interface - -#include -#include "py/dynruntime.h" - -#define STACK_LIMIT (2048) - -const char *stack_top; - -void mp_stack_check(void) { - // Assumes descending stack on target - volatile char dummy; - if (stack_top - &dummy >= STACK_LIMIT) { - mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("maximum recursion depth exceeded")); - } -} - -#if !defined(__linux__) -void *memcpy(void *dst, const void *src, size_t n) { - return mp_fun_table.memmove_(dst, src, n); -} -void *memset(void *s, int c, size_t n) { - return mp_fun_table.memset_(s, c, n); -} -#endif - -void *memmove(void *dest, const void *src, size_t n) { - return mp_fun_table.memmove_(dest, src, n); -} - -mp_obj_type_t match_type; -mp_obj_type_t re_type; - -#include "extmod/modure.c" - -mp_map_elem_t match_locals_dict_table[5]; -STATIC MP_DEFINE_CONST_DICT(match_locals_dict, match_locals_dict_table); - -mp_map_elem_t re_locals_dict_table[3]; -STATIC MP_DEFINE_CONST_DICT(re_locals_dict, re_locals_dict_table); - -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - MP_DYNRUNTIME_INIT_ENTRY - - char dummy; - stack_top = &dummy; - - // Because MP_QSTR_start/end/split are static, xtensa and xtensawin will make a small data section - // to copy in this key/value pair if they are specified as a struct, so assign them separately. - - match_type.base.type = (void*)&mp_fun_table.type_type; - match_type.name = MP_QSTR_match; - match_type.print = match_print; - match_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_group), MP_OBJ_FROM_PTR(&match_group_obj) }; - match_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_groups), MP_OBJ_FROM_PTR(&match_groups_obj) }; - match_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_span), MP_OBJ_FROM_PTR(&match_span_obj) }; - match_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_start), MP_OBJ_FROM_PTR(&match_start_obj) }; - match_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_end), MP_OBJ_FROM_PTR(&match_end_obj) }; - match_type.locals_dict = (void*)&match_locals_dict; - - re_type.base.type = (void*)&mp_fun_table.type_type; - re_type.name = MP_QSTR_ure; - re_type.print = re_print; - re_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_match), MP_OBJ_FROM_PTR(&re_match_obj) }; - re_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_search), MP_OBJ_FROM_PTR(&re_search_obj) }; - re_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_split), MP_OBJ_FROM_PTR(&re_split_obj) }; - re_type.locals_dict = (void*)&re_locals_dict; - - mp_store_global(MP_QSTR_compile, MP_OBJ_FROM_PTR(&mod_re_compile_obj)); - mp_store_global(MP_QSTR_match, MP_OBJ_FROM_PTR(&re_match_obj)); - mp_store_global(MP_QSTR_search, MP_OBJ_FROM_PTR(&re_search_obj)); - - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/natmod/uzlib/Makefile b/examples/natmod/uzlib/Makefile deleted file mode 100644 index 8761caf2dd..0000000000 --- a/examples/natmod/uzlib/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# Location of top-level MicroPython directory -MPY_DIR = ../../.. - -# Name of module (different to built-in uzlib so it can coexist) -MOD = uzlib_$(ARCH) - -# Source files (.c or .py) -SRC = uzlib.c - -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) -ARCH = x64 - -include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/uzlib/uzlib.c b/examples/natmod/uzlib/uzlib.c deleted file mode 100644 index 6df9944d6a..0000000000 --- a/examples/natmod/uzlib/uzlib.c +++ /dev/null @@ -1,40 +0,0 @@ -#define MICROPY_PY_UZLIB (1) - -#include "py/dynruntime.h" - -#if !defined(__linux__) -void *memset(void *s, int c, size_t n) { - return mp_fun_table.memset_(s, c, n); -} -#endif - -mp_obj_full_type_t decompio_type; -mp_stream_p_t decompio_stream_p; - -#include "extmod/moduzlib.c" - -mp_map_elem_t decompio_locals_dict_table[3]; -STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table); - -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - MP_DYNRUNTIME_INIT_ENTRY - - decompio_stream_p.name = MP_QSTR_protocol_stream; - decompio_stream_p.read = decompio_read; - - decompio_type.base.type = mp_fun_table.type_type; - decompio_type.flags = MP_TYPE_FLAG_EXTENDED; - decompio_type.name = MP_QSTR_DecompIO; - decompio_type.make_new = decompio_make_new; - decompio_type.ext[0].protocol = &decompio_stream_p; - decompio_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) }; - decompio_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) }; - decompio_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) }; - decompio_type.locals_dict = (void*)&decompio_locals_dict; - - mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_uzlib)); - mp_store_global(MP_QSTR_decompress, MP_OBJ_FROM_PTR(&mod_uzlib_decompress_obj)); - mp_store_global(MP_QSTR_DecompIO, MP_OBJ_FROM_PTR(&decompio_type)); - - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/examples/usercmodule/cexample/examplemodule.c b/examples/usercmodule/cexample/examplemodule.c deleted file mode 100644 index 49ebc7aaaf..0000000000 --- a/examples/usercmodule/cexample/examplemodule.c +++ /dev/null @@ -1,37 +0,0 @@ -// Include MicroPython API. -#include "py/runtime.h" - -// This is the function which will be called from Python as cexample.add_ints(a, b). -STATIC mp_obj_t example_add_ints(mp_obj_t a_obj, mp_obj_t b_obj) { - // Extract the ints from the micropython input objects. - int a = mp_obj_get_int(a_obj); - int b = mp_obj_get_int(b_obj); - - // Calculate the addition and convert to MicroPython object. - return mp_obj_new_int(a + b); -} -// Define a Python reference to the function above. -STATIC MP_DEFINE_CONST_FUN_OBJ_2(example_add_ints_obj, example_add_ints); - -// Define all properties of the module. -// Table entries are key/value pairs of the attribute name (a string) -// and the MicroPython object reference. -// All identifiers and strings are written as MP_QSTR_xxx and will be -// optimized to word-sized integers by the build system (interned strings). -STATIC const mp_rom_map_elem_t example_module_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_cexample) }, - { MP_ROM_QSTR(MP_QSTR_add_ints), MP_ROM_PTR(&example_add_ints_obj) }, -}; -STATIC MP_DEFINE_CONST_DICT(example_module_globals, example_module_globals_table); - -// Define module object. -const mp_obj_module_t example_user_cmodule = { - .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&example_module_globals, -}; - -// Register the module to make it available in Python. -// Note: the "1" in the third argument means this module is always enabled. -// This "1" can be optionally replaced with a macro like MODULE_CEXAMPLE_ENABLED -// which can then be used to conditionally enable this module. -MP_REGISTER_MODULE(MP_QSTR_cexample, example_user_cmodule, 1); diff --git a/examples/usercmodule/cexample/micropython.cmake b/examples/usercmodule/cexample/micropython.cmake deleted file mode 100644 index ba076a16b2..0000000000 --- a/examples/usercmodule/cexample/micropython.cmake +++ /dev/null @@ -1,15 +0,0 @@ -# Create an INTERFACE library for our C module. -add_library(usermod_cexample INTERFACE) - -# Add our source files to the lib -target_sources(usermod_cexample INTERFACE - ${CMAKE_CURRENT_LIST_DIR}/examplemodule.c -) - -# Add the current directory as an include directory. -target_include_directories(usermod_cexample INTERFACE - ${CMAKE_CURRENT_LIST_DIR} -) - -# Link our INTERFACE library to the usermod target. -target_link_libraries(usermod INTERFACE usermod_cexample) diff --git a/examples/usercmodule/cexample/micropython.mk b/examples/usercmodule/cexample/micropython.mk deleted file mode 100644 index dbfe3c5cbd..0000000000 --- a/examples/usercmodule/cexample/micropython.mk +++ /dev/null @@ -1,9 +0,0 @@ -EXAMPLE_MOD_DIR := $(USERMOD_DIR) - -# Add all C files to SRC_USERMOD. -SRC_USERMOD += $(EXAMPLE_MOD_DIR)/examplemodule.c - -# We can add our module folder to include paths if needed -# This is not actually needed in this example. -CFLAGS_USERMOD += -I$(EXAMPLE_MOD_DIR) -CEXAMPLE_MOD_DIR := $(USERMOD_DIR) diff --git a/examples/usercmodule/cppexample/example.cpp b/examples/usercmodule/cppexample/example.cpp deleted file mode 100644 index 06809732a4..0000000000 --- a/examples/usercmodule/cppexample/example.cpp +++ /dev/null @@ -1,17 +0,0 @@ -extern "C" { -#include - -// Here we implement the function using C++ code, but since it's -// declaration has to be compatible with C everything goes in extern "C" scope. -mp_obj_t cppfunc(mp_obj_t a_obj, mp_obj_t b_obj) { - // Prove we have (at least) C++11 features. - const auto a = mp_obj_get_int(a_obj); - const auto b = mp_obj_get_int(b_obj); - const auto sum = [&]() { - return mp_obj_new_int(a + b); - } (); - // Prove we're being scanned for QSTRs. - mp_obj_t tup[] = {sum, MP_ROM_QSTR(MP_QSTR_hellocpp)}; - return mp_obj_new_tuple(2, tup); -} -} diff --git a/examples/usercmodule/cppexample/examplemodule.c b/examples/usercmodule/cppexample/examplemodule.c deleted file mode 100644 index dfb7856837..0000000000 --- a/examples/usercmodule/cppexample/examplemodule.c +++ /dev/null @@ -1,28 +0,0 @@ -#include - -// Define a Python reference to the function we'll make available. -// See example.cpp for the definition. -STATIC MP_DEFINE_CONST_FUN_OBJ_2(cppfunc_obj, cppfunc); - -// Define all properties of the module. -// Table entries are key/value pairs of the attribute name (a string) -// and the MicroPython object reference. -// All identifiers and strings are written as MP_QSTR_xxx and will be -// optimized to word-sized integers by the build system (interned strings). -STATIC const mp_rom_map_elem_t cppexample_module_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_cppexample) }, - { MP_ROM_QSTR(MP_QSTR_cppfunc), MP_ROM_PTR(&cppfunc_obj) }, -}; -STATIC MP_DEFINE_CONST_DICT(cppexample_module_globals, cppexample_module_globals_table); - -// Define module object. -const mp_obj_module_t cppexample_user_cmodule = { - .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&cppexample_module_globals, -}; - -// Register the module to make it available in Python. -// Note: the "1" in the third argument means this module is always enabled. -// This "1" can be optionally replaced with a macro like MODULE_CPPEXAMPLE_ENABLED -// which can then be used to conditionally enable this module. -MP_REGISTER_MODULE(MP_QSTR_cppexample, cppexample_user_cmodule, 1); diff --git a/examples/usercmodule/cppexample/examplemodule.h b/examples/usercmodule/cppexample/examplemodule.h deleted file mode 100644 index d89384a630..0000000000 --- a/examples/usercmodule/cppexample/examplemodule.h +++ /dev/null @@ -1,5 +0,0 @@ -// Include MicroPython API. -#include "py/runtime.h" - -// Declare the function we'll make available in Python as cppexample.cppfunc(). -extern mp_obj_t cppfunc(mp_obj_t a_obj, mp_obj_t b_obj); diff --git a/examples/usercmodule/cppexample/micropython.cmake b/examples/usercmodule/cppexample/micropython.cmake deleted file mode 100644 index 6da972c94e..0000000000 --- a/examples/usercmodule/cppexample/micropython.cmake +++ /dev/null @@ -1,16 +0,0 @@ -# Create an INTERFACE library for our CPP module. -add_library(usermod_cppexample INTERFACE) - -# Add our source files to the library. -target_sources(usermod_cppexample INTERFACE - ${CMAKE_CURRENT_LIST_DIR}/example.cpp - ${CMAKE_CURRENT_LIST_DIR}/examplemodule.c -) - -# Add the current directory as an include directory. -target_include_directories(usermod_cppexample INTERFACE - ${CMAKE_CURRENT_LIST_DIR} -) - -# Link our INTERFACE library to the usermod target. -target_link_libraries(usermod INTERFACE usermod_cppexample) diff --git a/examples/usercmodule/cppexample/micropython.mk b/examples/usercmodule/cppexample/micropython.mk deleted file mode 100644 index e10d965a00..0000000000 --- a/examples/usercmodule/cppexample/micropython.mk +++ /dev/null @@ -1,12 +0,0 @@ -CPPEXAMPLE_MOD_DIR := $(USERMOD_DIR) - -# Add our source files to the respective variables. -SRC_USERMOD += $(CPPEXAMPLE_MOD_DIR)/examplemodule.c -SRC_USERMOD_CXX += $(CPPEXAMPLE_MOD_DIR)/example.cpp - -# Add our module directory to the include path. -CFLAGS_USERMOD += -I$(CPPEXAMPLE_MOD_DIR) -CXXFLAGS_USERMOD += -I$(CPPEXAMPLE_MOD_DIR) - -# We use C++ features so have to link against the standard library. -LDFLAGS_USERMOD += -lstdc++ diff --git a/examples/usercmodule/micropython.cmake b/examples/usercmodule/micropython.cmake deleted file mode 100644 index b9802401fa..0000000000 --- a/examples/usercmodule/micropython.cmake +++ /dev/null @@ -1,10 +0,0 @@ -# This top-level micropython.cmake is responsible for listing -# the individual modules we want to include. -# Paths are absolute, and ${CMAKE_CURRENT_LIST_DIR} can be -# used to prefix subdirectories. - -# Add the C example. -include(${CMAKE_CURRENT_LIST_DIR}/cexample/micropython.cmake) - -# Add the CPP example. -include(${CMAKE_CURRENT_LIST_DIR}/cppexample/micropython.cmake) diff --git a/extmod/axtls-include/axtls_os_port.h b/extmod/axtls-include/axtls_os_port.h index ef2683acfc..057642f974 100644 --- a/extmod/axtls-include/axtls_os_port.h +++ b/extmod/axtls-include/axtls_os_port.h @@ -26,7 +26,11 @@ #ifndef AXTLS_OS_PORT_H #define AXTLS_OS_PORT_H +#ifndef __ets__ +#include +#endif #include +#include #include "py/stream.h" #include "lib/crypto-algorithms/sha256.h" diff --git a/extmod/extmod.cmake b/extmod/extmod.cmake deleted file mode 100644 index 67f7d8fd39..0000000000 --- a/extmod/extmod.cmake +++ /dev/null @@ -1,98 +0,0 @@ -# CMake fragment for MicroPython extmod component - -set(MICROPY_EXTMOD_DIR "${MICROPY_DIR}/extmod") -set(MICROPY_OOFATFS_DIR "${MICROPY_DIR}/lib/oofatfs") - -set(MICROPY_SOURCE_EXTMOD - ${MICROPY_DIR}/shared/libc/abort_.c - ${MICROPY_DIR}/shared/libc/printf.c - ${MICROPY_EXTMOD_DIR}/machine_bitstream.c - ${MICROPY_EXTMOD_DIR}/machine_i2c.c - ${MICROPY_EXTMOD_DIR}/machine_mem.c - ${MICROPY_EXTMOD_DIR}/machine_pulse.c - ${MICROPY_EXTMOD_DIR}/machine_pwm.c - ${MICROPY_EXTMOD_DIR}/machine_signal.c - ${MICROPY_EXTMOD_DIR}/machine_spi.c - ${MICROPY_EXTMOD_DIR}/modbluetooth.c - ${MICROPY_EXTMOD_DIR}/modbtree.c - ${MICROPY_EXTMOD_DIR}/modframebuf.c - ${MICROPY_EXTMOD_DIR}/modnetwork.c - ${MICROPY_EXTMOD_DIR}/modonewire.c - ${MICROPY_EXTMOD_DIR}/moduasyncio.c - ${MICROPY_EXTMOD_DIR}/modubinascii.c - ${MICROPY_EXTMOD_DIR}/moducryptolib.c - ${MICROPY_EXTMOD_DIR}/moductypes.c - ${MICROPY_EXTMOD_DIR}/moduhashlib.c - ${MICROPY_EXTMOD_DIR}/moduheapq.c - ${MICROPY_EXTMOD_DIR}/modujson.c - ${MICROPY_EXTMOD_DIR}/moduplatform.c - ${MICROPY_EXTMOD_DIR}/modurandom.c - ${MICROPY_EXTMOD_DIR}/modure.c - ${MICROPY_EXTMOD_DIR}/moduselect.c - ${MICROPY_EXTMOD_DIR}/modusocket.c - ${MICROPY_EXTMOD_DIR}/modussl_axtls.c - ${MICROPY_EXTMOD_DIR}/modussl_mbedtls.c - ${MICROPY_EXTMOD_DIR}/modutimeq.c - ${MICROPY_EXTMOD_DIR}/moduwebsocket.c - ${MICROPY_EXTMOD_DIR}/moduzlib.c - ${MICROPY_EXTMOD_DIR}/modwebrepl.c - ${MICROPY_EXTMOD_DIR}/uos_dupterm.c - ${MICROPY_EXTMOD_DIR}/utime_mphal.c - ${MICROPY_EXTMOD_DIR}/vfs.c - ${MICROPY_EXTMOD_DIR}/vfs_blockdev.c - ${MICROPY_EXTMOD_DIR}/vfs_fat.c - ${MICROPY_EXTMOD_DIR}/vfs_fat_diskio.c - ${MICROPY_EXTMOD_DIR}/vfs_fat_file.c - ${MICROPY_EXTMOD_DIR}/vfs_lfs.c - ${MICROPY_EXTMOD_DIR}/vfs_posix.c - ${MICROPY_EXTMOD_DIR}/vfs_posix_file.c - ${MICROPY_EXTMOD_DIR}/vfs_reader.c - ${MICROPY_EXTMOD_DIR}/virtpin.c - ${MICROPY_EXTMOD_DIR}/nimble/modbluetooth_nimble.c -) - -# Library for btree module and associated code - -set(MICROPY_LIB_BERKELEY_DIR "${MICROPY_DIR}/lib/berkeley-db-1.xx") - -if(EXISTS "${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c") - add_library(micropy_extmod_btree OBJECT - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c - ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c - ${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c - ) - - target_include_directories(micropy_extmod_btree PRIVATE - ${MICROPY_LIB_BERKELEY_DIR}/PORT/include - ) - - target_compile_definitions(micropy_extmod_btree PRIVATE - __DBINTERFACE_PRIVATE=1 - mpool_error=printf - abort=abort_ - "virt_fd_t=void*" - ) - - # The include directories and compile definitions below are needed to build - # modbtree.c and should be added to the main MicroPython target. - - list(APPEND MICROPY_INC_CORE - "${MICROPY_LIB_BERKELEY_DIR}/PORT/include" - ) - - list(APPEND MICROPY_DEF_CORE - __DBINTERFACE_PRIVATE=1 - "virt_fd_t=void*" - ) -endif() diff --git a/extmod/extmod.mk b/extmod/extmod.mk index 4cdeaedaf9..ff24a549a9 100644 --- a/extmod/extmod.mk +++ b/extmod/extmod.mk @@ -28,8 +28,6 @@ SRC_MOD += $(addprefix $(LITTLEFS_DIR)/,\ lfs1.c \ lfs1_util.c \ ) -else -CFLAGS_MOD += -DMICROPY_VFS_LFS1=0 endif ifeq ($(MICROPY_VFS_LFS2),1) @@ -39,8 +37,6 @@ SRC_MOD += $(addprefix $(LITTLEFS_DIR)/,\ lfs2.c \ lfs2_util.c \ ) -else -CFLAGS_MOD += -DMICROPY_VFS_LFS2=0 $(BUILD)/$(LITTLEFS_DIR)/lfs2.o: CFLAGS += -Wno-missing-field-initializers endif @@ -232,3 +228,4 @@ CFLAGS_MOD += -DMICROPY_PY_BTREE=1 $(BUILD)/$(BTREE_DIR)/%.o: CFLAGS += -Wno-old-style-definition -Wno-sign-compare -Wno-unused-parameter $(BTREE_DEFS) $(BUILD)/extmod/modbtree.o: CFLAGS += $(BTREE_DEFS) endif + diff --git a/extmod/font_petme128_8x8.h b/extmod/font_petme128_8x8.h index 632397dfe3..0ee897d968 100644 --- a/extmod/font_petme128_8x8.h +++ b/extmod/font_petme128_8x8.h @@ -1,108 +1,128 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George -// -// SPDX-License-Identifier: MIT - +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013, 2014 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #ifndef MICROPY_INCLUDED_STM32_FONT_PETME128_8X8_H #define MICROPY_INCLUDED_STM32_FONT_PETME128_8X8_H static const uint8_t font_petme128_8x8[] = { - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, // 32= - 0x00,0x00,0x00,0x4f,0x4f,0x00,0x00,0x00, // 33=! - 0x00,0x07,0x07,0x00,0x00,0x07,0x07,0x00, // 34=" - 0x14,0x7f,0x7f,0x14,0x14,0x7f,0x7f,0x14, // 35=# - 0x00,0x24,0x2e,0x6b,0x6b,0x3a,0x12,0x00, // 36=$ - 0x00,0x63,0x33,0x18,0x0c,0x66,0x63,0x00, // 37=% - 0x00,0x32,0x7f,0x4d,0x4d,0x77,0x72,0x50, // 38=& - 0x00,0x00,0x00,0x04,0x06,0x03,0x01,0x00, // 39=' - 0x00,0x00,0x1c,0x3e,0x63,0x41,0x00,0x00, // 40=( - 0x00,0x00,0x41,0x63,0x3e,0x1c,0x00,0x00, // 41=) - 0x08,0x2a,0x3e,0x1c,0x1c,0x3e,0x2a,0x08, // 42=* - 0x00,0x08,0x08,0x3e,0x3e,0x08,0x08,0x00, // 43=+ - 0x00,0x00,0x80,0xe0,0x60,0x00,0x00,0x00, // 44=, - 0x00,0x08,0x08,0x08,0x08,0x08,0x08,0x00, // 45=- - 0x00,0x00,0x00,0x60,0x60,0x00,0x00,0x00, // 46=. - 0x00,0x40,0x60,0x30,0x18,0x0c,0x06,0x02, // 47=/ - 0x00,0x3e,0x7f,0x49,0x45,0x7f,0x3e,0x00, // 48=0 - 0x00,0x40,0x44,0x7f,0x7f,0x40,0x40,0x00, // 49=1 - 0x00,0x62,0x73,0x51,0x49,0x4f,0x46,0x00, // 50=2 - 0x00,0x22,0x63,0x49,0x49,0x7f,0x36,0x00, // 51=3 - 0x00,0x18,0x18,0x14,0x16,0x7f,0x7f,0x10, // 52=4 - 0x00,0x27,0x67,0x45,0x45,0x7d,0x39,0x00, // 53=5 - 0x00,0x3e,0x7f,0x49,0x49,0x7b,0x32,0x00, // 54=6 - 0x00,0x03,0x03,0x79,0x7d,0x07,0x03,0x00, // 55=7 - 0x00,0x36,0x7f,0x49,0x49,0x7f,0x36,0x00, // 56=8 - 0x00,0x26,0x6f,0x49,0x49,0x7f,0x3e,0x00, // 57=9 - 0x00,0x00,0x00,0x24,0x24,0x00,0x00,0x00, // 58=: - 0x00,0x00,0x80,0xe4,0x64,0x00,0x00,0x00, // 59=; - 0x00,0x08,0x1c,0x36,0x63,0x41,0x41,0x00, // 60=< - 0x00,0x14,0x14,0x14,0x14,0x14,0x14,0x00, // 61== - 0x00,0x41,0x41,0x63,0x36,0x1c,0x08,0x00, // 62=> - 0x00,0x02,0x03,0x51,0x59,0x0f,0x06,0x00, // 63=? - 0x00,0x3e,0x7f,0x41,0x4d,0x4f,0x2e,0x00, // 64=@ - 0x00,0x7c,0x7e,0x0b,0x0b,0x7e,0x7c,0x00, // 65=A - 0x00,0x7f,0x7f,0x49,0x49,0x7f,0x36,0x00, // 66=B - 0x00,0x3e,0x7f,0x41,0x41,0x63,0x22,0x00, // 67=C - 0x00,0x7f,0x7f,0x41,0x63,0x3e,0x1c,0x00, // 68=D - 0x00,0x7f,0x7f,0x49,0x49,0x41,0x41,0x00, // 69=E - 0x00,0x7f,0x7f,0x09,0x09,0x01,0x01,0x00, // 70=F - 0x00,0x3e,0x7f,0x41,0x49,0x7b,0x3a,0x00, // 71=G - 0x00,0x7f,0x7f,0x08,0x08,0x7f,0x7f,0x00, // 72=H - 0x00,0x00,0x41,0x7f,0x7f,0x41,0x00,0x00, // 73=I - 0x00,0x20,0x60,0x41,0x7f,0x3f,0x01,0x00, // 74=J - 0x00,0x7f,0x7f,0x1c,0x36,0x63,0x41,0x00, // 75=K - 0x00,0x7f,0x7f,0x40,0x40,0x40,0x40,0x00, // 76=L - 0x00,0x7f,0x7f,0x06,0x0c,0x06,0x7f,0x7f, // 77=M - 0x00,0x7f,0x7f,0x0e,0x1c,0x7f,0x7f,0x00, // 78=N - 0x00,0x3e,0x7f,0x41,0x41,0x7f,0x3e,0x00, // 79=O - 0x00,0x7f,0x7f,0x09,0x09,0x0f,0x06,0x00, // 80=P - 0x00,0x1e,0x3f,0x21,0x61,0x7f,0x5e,0x00, // 81=Q - 0x00,0x7f,0x7f,0x19,0x39,0x6f,0x46,0x00, // 82=R - 0x00,0x26,0x6f,0x49,0x49,0x7b,0x32,0x00, // 83=S - 0x00,0x01,0x01,0x7f,0x7f,0x01,0x01,0x00, // 84=T - 0x00,0x3f,0x7f,0x40,0x40,0x7f,0x3f,0x00, // 85=U - 0x00,0x1f,0x3f,0x60,0x60,0x3f,0x1f,0x00, // 86=V - 0x00,0x7f,0x7f,0x30,0x18,0x30,0x7f,0x7f, // 87=W - 0x00,0x63,0x77,0x1c,0x1c,0x77,0x63,0x00, // 88=X - 0x00,0x07,0x0f,0x78,0x78,0x0f,0x07,0x00, // 89=Y - 0x00,0x61,0x71,0x59,0x4d,0x47,0x43,0x00, // 90=Z - 0x00,0x00,0x7f,0x7f,0x41,0x41,0x00,0x00, // 91=[ - 0x00,0x02,0x06,0x0c,0x18,0x30,0x60,0x40, // 92='\' - 0x00,0x00,0x41,0x41,0x7f,0x7f,0x00,0x00, // 93=] - 0x00,0x08,0x0c,0x06,0x06,0x0c,0x08,0x00, // 94=^ - 0xc0,0xc0,0xc0,0xc0,0xc0,0xc0,0xc0,0xc0, // 95=_ - 0x00,0x00,0x01,0x03,0x06,0x04,0x00,0x00, // 96=` - 0x00,0x20,0x74,0x54,0x54,0x7c,0x78,0x00, // 97=a - 0x00,0x7f,0x7f,0x44,0x44,0x7c,0x38,0x00, // 98=b - 0x00,0x38,0x7c,0x44,0x44,0x6c,0x28,0x00, // 99=c - 0x00,0x38,0x7c,0x44,0x44,0x7f,0x7f,0x00, // 100=d - 0x00,0x38,0x7c,0x54,0x54,0x5c,0x58,0x00, // 101=e - 0x00,0x08,0x7e,0x7f,0x09,0x03,0x02,0x00, // 102=f - 0x00,0x98,0xbc,0xa4,0xa4,0xfc,0x7c,0x00, // 103=g - 0x00,0x7f,0x7f,0x04,0x04,0x7c,0x78,0x00, // 104=h - 0x00,0x00,0x00,0x7d,0x7d,0x00,0x00,0x00, // 105=i - 0x00,0x40,0xc0,0x80,0x80,0xfd,0x7d,0x00, // 106=j - 0x00,0x7f,0x7f,0x30,0x38,0x6c,0x44,0x00, // 107=k - 0x00,0x00,0x41,0x7f,0x7f,0x40,0x00,0x00, // 108=l - 0x00,0x7c,0x7c,0x18,0x30,0x18,0x7c,0x7c, // 109=m - 0x00,0x7c,0x7c,0x04,0x04,0x7c,0x78,0x00, // 110=n - 0x00,0x38,0x7c,0x44,0x44,0x7c,0x38,0x00, // 111=o - 0x00,0xfc,0xfc,0x24,0x24,0x3c,0x18,0x00, // 112=p - 0x00,0x18,0x3c,0x24,0x24,0xfc,0xfc,0x00, // 113=q - 0x00,0x7c,0x7c,0x04,0x04,0x0c,0x08,0x00, // 114=r - 0x00,0x48,0x5c,0x54,0x54,0x74,0x20,0x00, // 115=s - 0x04,0x04,0x3f,0x7f,0x44,0x64,0x20,0x00, // 116=t - 0x00,0x3c,0x7c,0x40,0x40,0x7c,0x3c,0x00, // 117=u - 0x00,0x1c,0x3c,0x60,0x60,0x3c,0x1c,0x00, // 118=v - 0x00,0x1c,0x7c,0x30,0x18,0x30,0x7c,0x1c, // 119=w - 0x00,0x44,0x6c,0x38,0x38,0x6c,0x44,0x00, // 120=x - 0x00,0x9c,0xbc,0xa0,0xa0,0xfc,0x7c,0x00, // 121=y - 0x00,0x44,0x64,0x74,0x5c,0x4c,0x44,0x00, // 122=z - 0x00,0x08,0x08,0x3e,0x77,0x41,0x41,0x00, // 123={ - 0x00,0x00,0x00,0xff,0xff,0x00,0x00,0x00, // 124=| - 0x00,0x41,0x41,0x77,0x3e,0x08,0x08,0x00, // 125=} - 0x00,0x02,0x03,0x01,0x03,0x02,0x03,0x01, // 126=~ - 0xaa,0x55,0xaa,0x55,0xaa,0x55,0xaa,0x55, // 127 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 32= + 0x00, 0x00, 0x00, 0x4f, 0x4f, 0x00, 0x00, 0x00, // 33=! + 0x00, 0x07, 0x07, 0x00, 0x00, 0x07, 0x07, 0x00, // 34=" + 0x14, 0x7f, 0x7f, 0x14, 0x14, 0x7f, 0x7f, 0x14, // 35=# + 0x00, 0x24, 0x2e, 0x6b, 0x6b, 0x3a, 0x12, 0x00, // 36=$ + 0x00, 0x63, 0x33, 0x18, 0x0c, 0x66, 0x63, 0x00, // 37=% + 0x00, 0x32, 0x7f, 0x4d, 0x4d, 0x77, 0x72, 0x50, // 38=& + 0x00, 0x00, 0x00, 0x04, 0x06, 0x03, 0x01, 0x00, // 39=' + 0x00, 0x00, 0x1c, 0x3e, 0x63, 0x41, 0x00, 0x00, // 40=( + 0x00, 0x00, 0x41, 0x63, 0x3e, 0x1c, 0x00, 0x00, // 41=) + 0x08, 0x2a, 0x3e, 0x1c, 0x1c, 0x3e, 0x2a, 0x08, // 42=* + 0x00, 0x08, 0x08, 0x3e, 0x3e, 0x08, 0x08, 0x00, // 43=+ + 0x00, 0x00, 0x80, 0xe0, 0x60, 0x00, 0x00, 0x00, // 44=, + 0x00, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x00, // 45=- + 0x00, 0x00, 0x00, 0x60, 0x60, 0x00, 0x00, 0x00, // 46=. + 0x00, 0x40, 0x60, 0x30, 0x18, 0x0c, 0x06, 0x02, // 47=/ + 0x00, 0x3e, 0x7f, 0x49, 0x45, 0x7f, 0x3e, 0x00, // 48=0 + 0x00, 0x40, 0x44, 0x7f, 0x7f, 0x40, 0x40, 0x00, // 49=1 + 0x00, 0x62, 0x73, 0x51, 0x49, 0x4f, 0x46, 0x00, // 50=2 + 0x00, 0x22, 0x63, 0x49, 0x49, 0x7f, 0x36, 0x00, // 51=3 + 0x00, 0x18, 0x18, 0x14, 0x16, 0x7f, 0x7f, 0x10, // 52=4 + 0x00, 0x27, 0x67, 0x45, 0x45, 0x7d, 0x39, 0x00, // 53=5 + 0x00, 0x3e, 0x7f, 0x49, 0x49, 0x7b, 0x32, 0x00, // 54=6 + 0x00, 0x03, 0x03, 0x79, 0x7d, 0x07, 0x03, 0x00, // 55=7 + 0x00, 0x36, 0x7f, 0x49, 0x49, 0x7f, 0x36, 0x00, // 56=8 + 0x00, 0x26, 0x6f, 0x49, 0x49, 0x7f, 0x3e, 0x00, // 57=9 + 0x00, 0x00, 0x00, 0x24, 0x24, 0x00, 0x00, 0x00, // 58=: + 0x00, 0x00, 0x80, 0xe4, 0x64, 0x00, 0x00, 0x00, // 59=; + 0x00, 0x08, 0x1c, 0x36, 0x63, 0x41, 0x41, 0x00, // 60=< + 0x00, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x00, // 61== + 0x00, 0x41, 0x41, 0x63, 0x36, 0x1c, 0x08, 0x00, // 62=> + 0x00, 0x02, 0x03, 0x51, 0x59, 0x0f, 0x06, 0x00, // 63=? + 0x00, 0x3e, 0x7f, 0x41, 0x4d, 0x4f, 0x2e, 0x00, // 64=@ + 0x00, 0x7c, 0x7e, 0x0b, 0x0b, 0x7e, 0x7c, 0x00, // 65=A + 0x00, 0x7f, 0x7f, 0x49, 0x49, 0x7f, 0x36, 0x00, // 66=B + 0x00, 0x3e, 0x7f, 0x41, 0x41, 0x63, 0x22, 0x00, // 67=C + 0x00, 0x7f, 0x7f, 0x41, 0x63, 0x3e, 0x1c, 0x00, // 68=D + 0x00, 0x7f, 0x7f, 0x49, 0x49, 0x41, 0x41, 0x00, // 69=E + 0x00, 0x7f, 0x7f, 0x09, 0x09, 0x01, 0x01, 0x00, // 70=F + 0x00, 0x3e, 0x7f, 0x41, 0x49, 0x7b, 0x3a, 0x00, // 71=G + 0x00, 0x7f, 0x7f, 0x08, 0x08, 0x7f, 0x7f, 0x00, // 72=H + 0x00, 0x00, 0x41, 0x7f, 0x7f, 0x41, 0x00, 0x00, // 73=I + 0x00, 0x20, 0x60, 0x41, 0x7f, 0x3f, 0x01, 0x00, // 74=J + 0x00, 0x7f, 0x7f, 0x1c, 0x36, 0x63, 0x41, 0x00, // 75=K + 0x00, 0x7f, 0x7f, 0x40, 0x40, 0x40, 0x40, 0x00, // 76=L + 0x00, 0x7f, 0x7f, 0x06, 0x0c, 0x06, 0x7f, 0x7f, // 77=M + 0x00, 0x7f, 0x7f, 0x0e, 0x1c, 0x7f, 0x7f, 0x00, // 78=N + 0x00, 0x3e, 0x7f, 0x41, 0x41, 0x7f, 0x3e, 0x00, // 79=O + 0x00, 0x7f, 0x7f, 0x09, 0x09, 0x0f, 0x06, 0x00, // 80=P + 0x00, 0x1e, 0x3f, 0x21, 0x61, 0x7f, 0x5e, 0x00, // 81=Q + 0x00, 0x7f, 0x7f, 0x19, 0x39, 0x6f, 0x46, 0x00, // 82=R + 0x00, 0x26, 0x6f, 0x49, 0x49, 0x7b, 0x32, 0x00, // 83=S + 0x00, 0x01, 0x01, 0x7f, 0x7f, 0x01, 0x01, 0x00, // 84=T + 0x00, 0x3f, 0x7f, 0x40, 0x40, 0x7f, 0x3f, 0x00, // 85=U + 0x00, 0x1f, 0x3f, 0x60, 0x60, 0x3f, 0x1f, 0x00, // 86=V + 0x00, 0x7f, 0x7f, 0x30, 0x18, 0x30, 0x7f, 0x7f, // 87=W + 0x00, 0x63, 0x77, 0x1c, 0x1c, 0x77, 0x63, 0x00, // 88=X + 0x00, 0x07, 0x0f, 0x78, 0x78, 0x0f, 0x07, 0x00, // 89=Y + 0x00, 0x61, 0x71, 0x59, 0x4d, 0x47, 0x43, 0x00, // 90=Z + 0x00, 0x00, 0x7f, 0x7f, 0x41, 0x41, 0x00, 0x00, // 91=[ + 0x00, 0x02, 0x06, 0x0c, 0x18, 0x30, 0x60, 0x40, // 92='\' + 0x00, 0x00, 0x41, 0x41, 0x7f, 0x7f, 0x00, 0x00, // 93=] + 0x00, 0x08, 0x0c, 0x06, 0x06, 0x0c, 0x08, 0x00, // 94=^ + 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, // 95=_ + 0x00, 0x00, 0x01, 0x03, 0x06, 0x04, 0x00, 0x00, // 96=` + 0x00, 0x20, 0x74, 0x54, 0x54, 0x7c, 0x78, 0x00, // 97=a + 0x00, 0x7f, 0x7f, 0x44, 0x44, 0x7c, 0x38, 0x00, // 98=b + 0x00, 0x38, 0x7c, 0x44, 0x44, 0x6c, 0x28, 0x00, // 99=c + 0x00, 0x38, 0x7c, 0x44, 0x44, 0x7f, 0x7f, 0x00, // 100=d + 0x00, 0x38, 0x7c, 0x54, 0x54, 0x5c, 0x58, 0x00, // 101=e + 0x00, 0x08, 0x7e, 0x7f, 0x09, 0x03, 0x02, 0x00, // 102=f + 0x00, 0x98, 0xbc, 0xa4, 0xa4, 0xfc, 0x7c, 0x00, // 103=g + 0x00, 0x7f, 0x7f, 0x04, 0x04, 0x7c, 0x78, 0x00, // 104=h + 0x00, 0x00, 0x00, 0x7d, 0x7d, 0x00, 0x00, 0x00, // 105=i + 0x00, 0x40, 0xc0, 0x80, 0x80, 0xfd, 0x7d, 0x00, // 106=j + 0x00, 0x7f, 0x7f, 0x30, 0x38, 0x6c, 0x44, 0x00, // 107=k + 0x00, 0x00, 0x41, 0x7f, 0x7f, 0x40, 0x00, 0x00, // 108=l + 0x00, 0x7c, 0x7c, 0x18, 0x30, 0x18, 0x7c, 0x7c, // 109=m + 0x00, 0x7c, 0x7c, 0x04, 0x04, 0x7c, 0x78, 0x00, // 110=n + 0x00, 0x38, 0x7c, 0x44, 0x44, 0x7c, 0x38, 0x00, // 111=o + 0x00, 0xfc, 0xfc, 0x24, 0x24, 0x3c, 0x18, 0x00, // 112=p + 0x00, 0x18, 0x3c, 0x24, 0x24, 0xfc, 0xfc, 0x00, // 113=q + 0x00, 0x7c, 0x7c, 0x04, 0x04, 0x0c, 0x08, 0x00, // 114=r + 0x00, 0x48, 0x5c, 0x54, 0x54, 0x74, 0x20, 0x00, // 115=s + 0x04, 0x04, 0x3f, 0x7f, 0x44, 0x64, 0x20, 0x00, // 116=t + 0x00, 0x3c, 0x7c, 0x40, 0x40, 0x7c, 0x3c, 0x00, // 117=u + 0x00, 0x1c, 0x3c, 0x60, 0x60, 0x3c, 0x1c, 0x00, // 118=v + 0x00, 0x1c, 0x7c, 0x30, 0x18, 0x30, 0x7c, 0x1c, // 119=w + 0x00, 0x44, 0x6c, 0x38, 0x38, 0x6c, 0x44, 0x00, // 120=x + 0x00, 0x9c, 0xbc, 0xa0, 0xa0, 0xfc, 0x7c, 0x00, // 121=y + 0x00, 0x44, 0x64, 0x74, 0x5c, 0x4c, 0x44, 0x00, // 122=z + 0x00, 0x08, 0x08, 0x3e, 0x77, 0x41, 0x41, 0x00, // 123={ + 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, // 124=| + 0x00, 0x41, 0x41, 0x77, 0x3e, 0x08, 0x08, 0x00, // 125=} + 0x00, 0x02, 0x03, 0x01, 0x03, 0x02, 0x03, 0x01, // 126=~ + 0xaa, 0x55, 0xaa, 0x55, 0xaa, 0x55, 0xaa, 0x55, // 127 }; #endif // MICROPY_INCLUDED_STM32_FONT_PETME128_8X8_H diff --git a/extmod/modbtree.c b/extmod/modbtree.c deleted file mode 100644 index 5f4b4696b4..0000000000 --- a/extmod/modbtree.c +++ /dev/null @@ -1,367 +0,0 @@ -// Copyright (c) 2016 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// -// SPDX-License-Identifier: MIT - -#include -#include -#include // for declaration of global errno variable -#include - -#include "py/runtime.h" -#include "py/stream.h" - -#if MICROPY_PY_BTREE - -#include -#include <../../btree/btree.h> - -typedef struct _mp_obj_btree_t { - mp_obj_base_t base; - mp_obj_t stream; // retain a reference to prevent GC from reclaiming it - DB *db; - mp_obj_t start_key; - mp_obj_t end_key; - #define FLAG_END_KEY_INCL 1 - #define FLAG_DESC 2 - #define FLAG_ITER_TYPE_MASK 0xc0 - #define FLAG_ITER_KEYS 0x40 - #define FLAG_ITER_VALUES 0x80 - #define FLAG_ITER_ITEMS 0xc0 - byte flags; - byte next_flags; -} mp_obj_btree_t; - -#if !MICROPY_ENABLE_DYNRUNTIME -STATIC const mp_obj_type_t btree_type; -#endif - -#define CHECK_ERROR(res) \ - if (res == RET_ERROR) { \ - mp_raise_OSError(errno); \ - } - -void __dbpanic(DB *db) { - mp_printf(&mp_plat_print, "__dbpanic(%p)\n", db); -} - -STATIC mp_obj_btree_t *btree_new(DB *db, mp_obj_t stream) { - mp_obj_btree_t *o = m_new_obj(mp_obj_btree_t); - o->base.type = (mp_obj_type_t *)&btree_type; - o->stream = stream; - o->db = db; - o->start_key = mp_const_none; - o->end_key = mp_const_none; - o->next_flags = 0; - return o; -} - -STATIC void btree_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { - (void)kind; - mp_obj_btree_t *self = MP_OBJ_TO_PTR(self_in); - mp_printf(print, "", self->db); -} - -STATIC mp_obj_t btree_flush(mp_obj_t self_in) { - mp_obj_btree_t *self = MP_OBJ_TO_PTR(self_in); - return MP_OBJ_NEW_SMALL_INT(__bt_sync(self->db, 0)); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(btree_flush_obj, btree_flush); - -STATIC mp_obj_t btree_close(mp_obj_t self_in) { - mp_obj_btree_t *self = MP_OBJ_TO_PTR(self_in); - return MP_OBJ_NEW_SMALL_INT(__bt_close(self->db)); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(btree_close_obj, btree_close); - -STATIC mp_obj_t btree_put(size_t n_args, const mp_obj_t *args) { - (void)n_args; - mp_obj_btree_t *self = MP_OBJ_TO_PTR(args[0]); - DBT key, val; - key.data = (void *)mp_obj_str_get_data(args[1], &key.size); - val.data = (void *)mp_obj_str_get_data(args[2], &val.size); - return MP_OBJ_NEW_SMALL_INT(__bt_put(self->db, &key, &val, 0)); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(btree_put_obj, 3, 4, btree_put); - -STATIC mp_obj_t btree_get(size_t n_args, const mp_obj_t *args) { - mp_obj_btree_t *self = MP_OBJ_TO_PTR(args[0]); - DBT key, val; - key.data = (void *)mp_obj_str_get_data(args[1], &key.size); - int res = __bt_get(self->db, &key, &val, 0); - if (res == RET_SPECIAL) { - if (n_args > 2) { - return args[2]; - } else { - return mp_const_none; - } - } - CHECK_ERROR(res); - return mp_obj_new_bytes(val.data, val.size); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(btree_get_obj, 2, 3, btree_get); - -STATIC mp_obj_t btree_seq(size_t n_args, const mp_obj_t *args) { - mp_obj_btree_t *self = MP_OBJ_TO_PTR(args[0]); - int flags = MP_OBJ_SMALL_INT_VALUE(args[1]); - DBT key, val; - if (n_args > 2) { - key.data = (void *)mp_obj_str_get_data(args[2], &key.size); - } - - int res = __bt_seq(self->db, &key, &val, flags); - CHECK_ERROR(res); - if (res == RET_SPECIAL) { - return mp_const_none; - } - - mp_obj_t pair_o = mp_obj_new_tuple(2, NULL); - mp_obj_tuple_t *pair = MP_OBJ_TO_PTR(pair_o); - pair->items[0] = mp_obj_new_bytes(key.data, key.size); - pair->items[1] = mp_obj_new_bytes(val.data, val.size); - return pair_o; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(btree_seq_obj, 2, 4, btree_seq); - -STATIC mp_obj_t btree_init_iter(size_t n_args, const mp_obj_t *args, byte type) { - mp_obj_btree_t *self = MP_OBJ_TO_PTR(args[0]); - self->next_flags = type; - self->start_key = mp_const_none; - self->end_key = mp_const_none; - if (n_args > 1) { - self->start_key = args[1]; - if (n_args > 2) { - self->end_key = args[2]; - if (n_args > 3) { - self->next_flags = type | MP_OBJ_SMALL_INT_VALUE(args[3]); - } - } - } - return args[0]; -} - -STATIC mp_obj_t btree_keys(size_t n_args, const mp_obj_t *args) { - return btree_init_iter(n_args, args, FLAG_ITER_KEYS); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(btree_keys_obj, 1, 4, btree_keys); - -STATIC mp_obj_t btree_values(size_t n_args, const mp_obj_t *args) { - return btree_init_iter(n_args, args, FLAG_ITER_VALUES); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(btree_values_obj, 1, 4, btree_values); - -STATIC mp_obj_t btree_items(size_t n_args, const mp_obj_t *args) { - return btree_init_iter(n_args, args, FLAG_ITER_ITEMS); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(btree_items_obj, 1, 4, btree_items); - -STATIC mp_obj_t btree_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { - (void)iter_buf; - mp_obj_btree_t *self = MP_OBJ_TO_PTR(self_in); - if (self->next_flags != 0) { - // If we're called immediately after keys(), values(), or items(), - // use their setup for iteration. - self->flags = self->next_flags; - self->next_flags = 0; - } else { - // Otherwise, iterate over all keys. - self->flags = FLAG_ITER_KEYS; - self->start_key = mp_const_none; - self->end_key = mp_const_none; - } - - return self_in; -} - -STATIC mp_obj_t btree_iternext(mp_obj_t self_in) { - mp_obj_btree_t *self = MP_OBJ_TO_PTR(self_in); - DBT key, val; - int res; - bool desc = self->flags & FLAG_DESC; - if (self->start_key != MP_OBJ_NULL) { - int flags = R_FIRST; - if (self->start_key != mp_const_none) { - key.data = (void *)mp_obj_str_get_data(self->start_key, &key.size); - flags = R_CURSOR; - } else if (desc) { - flags = R_LAST; - } - res = __bt_seq(self->db, &key, &val, flags); - self->start_key = MP_OBJ_NULL; - } else { - res = __bt_seq(self->db, &key, &val, desc ? R_PREV : R_NEXT); - } - - if (res == RET_SPECIAL) { - return MP_OBJ_STOP_ITERATION; - } - CHECK_ERROR(res); - - if (self->end_key != mp_const_none) { - DBT end_key; - end_key.data = (void *)mp_obj_str_get_data(self->end_key, &end_key.size); - BTREE *t = self->db->internal; - int cmp = t->bt_cmp(&key, &end_key); - if (desc) { - cmp = -cmp; - } - if (self->flags & FLAG_END_KEY_INCL) { - cmp--; - } - if (cmp >= 0) { - self->end_key = MP_OBJ_NULL; - return MP_OBJ_STOP_ITERATION; - } - } - - switch (self->flags & FLAG_ITER_TYPE_MASK) { - case FLAG_ITER_KEYS: - return mp_obj_new_bytes(key.data, key.size); - case FLAG_ITER_VALUES: - return mp_obj_new_bytes(val.data, val.size); - default: { - mp_obj_t pair_o = mp_obj_new_tuple(2, NULL); - mp_obj_tuple_t *pair = MP_OBJ_TO_PTR(pair_o); - pair->items[0] = mp_obj_new_bytes(key.data, key.size); - pair->items[1] = mp_obj_new_bytes(val.data, val.size); - return pair_o; - } - } -} - -STATIC mp_obj_t btree_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { - mp_obj_btree_t *self = mp_obj_cast_to_native_base(self_in, &btree_type); - if (value == MP_OBJ_NULL) { - // delete - DBT key; - key.data = (void *)mp_obj_str_get_data(index, &key.size); - int res = __bt_delete(self->db, &key, 0); - if (res == RET_SPECIAL) { - mp_raise_type(&mp_type_KeyError); - } - CHECK_ERROR(res); - return mp_const_none; - } else if (value == MP_OBJ_SENTINEL) { - // load - DBT key, val; - key.data = (void *)mp_obj_str_get_data(index, &key.size); - int res = __bt_get(self->db, &key, &val, 0); - if (res == RET_SPECIAL) { - mp_raise_type(&mp_type_KeyError); - } - CHECK_ERROR(res); - return mp_obj_new_bytes(val.data, val.size); - } else { - // store - DBT key, val; - key.data = (void *)mp_obj_str_get_data(index, &key.size); - val.data = (void *)mp_obj_str_get_data(value, &val.size); - int res = __bt_put(self->db, &key, &val, 0); - CHECK_ERROR(res); - return mp_const_none; - } -} - -STATIC mp_obj_t btree_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { - mp_obj_btree_t *self = MP_OBJ_TO_PTR(lhs_in); - switch (op) { - case MP_BINARY_OP_CONTAINS: { - DBT key, val; - key.data = (void *)mp_obj_str_get_data(rhs_in, &key.size); - int res = __bt_get(self->db, &key, &val, 0); - CHECK_ERROR(res); - return mp_obj_new_bool(res != RET_SPECIAL); - } - default: - // op not supported - return MP_OBJ_NULL; - } -} - -#if !MICROPY_ENABLE_DYNRUNTIME -STATIC const mp_rom_map_elem_t btree_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&btree_close_obj) }, - { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&btree_flush_obj) }, - { MP_ROM_QSTR(MP_QSTR_get), MP_ROM_PTR(&btree_get_obj) }, - { MP_ROM_QSTR(MP_QSTR_put), MP_ROM_PTR(&btree_put_obj) }, - { MP_ROM_QSTR(MP_QSTR_seq), MP_ROM_PTR(&btree_seq_obj) }, - { MP_ROM_QSTR(MP_QSTR_keys), MP_ROM_PTR(&btree_keys_obj) }, - { MP_ROM_QSTR(MP_QSTR_values), MP_ROM_PTR(&btree_values_obj) }, - { MP_ROM_QSTR(MP_QSTR_items), MP_ROM_PTR(&btree_items_obj) }, -}; - -STATIC MP_DEFINE_CONST_DICT(btree_locals_dict, btree_locals_dict_table); - -STATIC const mp_obj_type_t btree_type = { - { &mp_type_type }, - // Save on qstr's, reuse same as for module - .flags = MP_TYPE_FLAG_EXTENDED, - .name = MP_QSTR_btree, - .print = btree_print, - .locals_dict = (void *)&btree_locals_dict, - MP_TYPE_EXTENDED_FIELDS( - .getiter = btree_getiter, - .iternext = btree_iternext, - .binary_op = btree_binary_op, - .subscr = btree_subscr, - ), -}; -#endif - -STATIC const FILEVTABLE btree_stream_fvtable = { - mp_stream_posix_read, - mp_stream_posix_write, - mp_stream_posix_lseek, - mp_stream_posix_fsync -}; - -#if !MICROPY_ENABLE_DYNRUNTIME -STATIC mp_obj_t mod_btree_open(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { - static const mp_arg_t allowed_args[] = { - { MP_QSTR_flags, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} }, - { MP_QSTR_cachesize, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} }, - { MP_QSTR_pagesize, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} }, - { MP_QSTR_minkeypage, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} }, - }; - - // Make sure we got a stream object - mp_get_stream_raise(pos_args[0], MP_STREAM_OP_READ | MP_STREAM_OP_WRITE | MP_STREAM_OP_IOCTL); - - struct { - mp_arg_val_t flags; - mp_arg_val_t cachesize; - mp_arg_val_t pagesize; - mp_arg_val_t minkeypage; - } args; - mp_arg_parse_all(n_args - 1, pos_args + 1, kw_args, - MP_ARRAY_SIZE(allowed_args), allowed_args, (mp_arg_val_t *)&args); - BTREEINFO openinfo = {0}; - openinfo.flags = args.flags.u_int; - openinfo.cachesize = args.cachesize.u_int; - openinfo.psize = args.pagesize.u_int; - openinfo.minkeypage = args.minkeypage.u_int; - - DB *db = __bt_open(MP_OBJ_TO_PTR(pos_args[0]), &btree_stream_fvtable, &openinfo, /*dflags*/ 0); - if (db == NULL) { - mp_raise_OSError(errno); - } - return MP_OBJ_FROM_PTR(btree_new(db, pos_args[0])); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_btree_open_obj, 1, mod_btree_open); - -STATIC const mp_rom_map_elem_t mp_module_btree_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_btree) }, - { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&mod_btree_open_obj) }, - { MP_ROM_QSTR(MP_QSTR_INCL), MP_ROM_INT(FLAG_END_KEY_INCL) }, - { MP_ROM_QSTR(MP_QSTR_DESC), MP_ROM_INT(FLAG_DESC) }, -}; - -STATIC MP_DEFINE_CONST_DICT(mp_module_btree_globals, mp_module_btree_globals_table); - -const mp_obj_module_t mp_module_btree = { - .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&mp_module_btree_globals, -}; -#endif - -#endif // MICROPY_PY_BTREE diff --git a/extmod/modframebuf.c b/extmod/modframebuf.c deleted file mode 100644 index a90f00c347..0000000000 --- a/extmod/modframebuf.c +++ /dev/null @@ -1,667 +0,0 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George -// -// SPDX-License-Identifier: MIT - -#include -#include - -#include "py/runtime.h" -#include "py/objtype.h" -#include "py/proto.h" - -#if MICROPY_PY_FRAMEBUF - -#include "font_petme128_8x8.h" - -typedef struct _mp_obj_framebuf_t { - mp_obj_base_t base; - mp_obj_t buf_obj; // need to store this to prevent GC from reclaiming buf - void *buf; - uint16_t width, height, stride; - uint8_t format; -} mp_obj_framebuf_t; - -#if !MICROPY_ENABLE_DYNRUNTIME -STATIC const mp_obj_type_t mp_type_framebuf; -#endif - -typedef void (*setpixel_t)(const mp_obj_framebuf_t *, unsigned int, unsigned int, uint32_t); -typedef uint32_t (*getpixel_t)(const mp_obj_framebuf_t *, unsigned int, unsigned int); -typedef void (*fill_rect_t)(const mp_obj_framebuf_t *, unsigned int, unsigned int, unsigned int, unsigned int, uint32_t); - -typedef struct _mp_framebuf_p_t { - setpixel_t setpixel; - getpixel_t getpixel; - fill_rect_t fill_rect; -} mp_framebuf_p_t; - -// constants for formats -#define FRAMEBUF_MVLSB (0) -#define FRAMEBUF_RGB565 (1) -#define FRAMEBUF_GS2_HMSB (5) -#define FRAMEBUF_GS4_HMSB (2) -#define FRAMEBUF_GS8 (6) -#define FRAMEBUF_MHLSB (3) -#define FRAMEBUF_MHMSB (4) - -// Functions for MHLSB and MHMSB - -STATIC void mono_horiz_setpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, uint32_t col) { - size_t index = (x + y * fb->stride) >> 3; - unsigned int offset = fb->format == FRAMEBUF_MHMSB ? x & 0x07 : 7 - (x & 0x07); - ((uint8_t *)fb->buf)[index] = (((uint8_t *)fb->buf)[index] & ~(0x01 << offset)) | ((col != 0) << offset); -} - -STATIC uint32_t mono_horiz_getpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y) { - size_t index = (x + y * fb->stride) >> 3; - unsigned int offset = fb->format == FRAMEBUF_MHMSB ? x & 0x07 : 7 - (x & 0x07); - return (((uint8_t *)fb->buf)[index] >> (offset)) & 0x01; -} - -STATIC void mono_horiz_fill_rect(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, unsigned int w, unsigned int h, uint32_t col) { - unsigned int reverse = fb->format == FRAMEBUF_MHMSB; - unsigned int advance = fb->stride >> 3; - while (w--) { - uint8_t *b = &((uint8_t *)fb->buf)[(x >> 3) + y * advance]; - unsigned int offset = reverse ? x & 7 : 7 - (x & 7); - for (unsigned int hh = h; hh; --hh) { - *b = (*b & ~(0x01 << offset)) | ((col != 0) << offset); - b += advance; - } - ++x; - } -} - -// Functions for MVLSB format - -STATIC void mvlsb_setpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, uint32_t col) { - size_t index = (y >> 3) * fb->stride + x; - uint8_t offset = y & 0x07; - ((uint8_t *)fb->buf)[index] = (((uint8_t *)fb->buf)[index] & ~(0x01 << offset)) | ((col != 0) << offset); -} - -STATIC uint32_t mvlsb_getpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y) { - return (((uint8_t *)fb->buf)[(y >> 3) * fb->stride + x] >> (y & 0x07)) & 0x01; -} - -STATIC void mvlsb_fill_rect(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, unsigned int w, unsigned int h, uint32_t col) { - while (h--) { - uint8_t *b = &((uint8_t *)fb->buf)[(y >> 3) * fb->stride + x]; - uint8_t offset = y & 0x07; - for (unsigned int ww = w; ww; --ww) { - *b = (*b & ~(0x01 << offset)) | ((col != 0) << offset); - ++b; - } - ++y; - } -} - -// Functions for RGB565 format - -STATIC void rgb565_setpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, uint32_t col) { - ((uint16_t *)fb->buf)[x + y * fb->stride] = col; -} - -STATIC uint32_t rgb565_getpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y) { - return ((uint16_t *)fb->buf)[x + y * fb->stride]; -} - -STATIC void rgb565_fill_rect(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, unsigned int w, unsigned int h, uint32_t col) { - uint16_t *b = &((uint16_t *)fb->buf)[x + y * fb->stride]; - while (h--) { - for (unsigned int ww = w; ww; --ww) { - *b++ = col; - } - b += fb->stride - w; - } -} - -// Functions for GS2_HMSB format - -STATIC void gs2_hmsb_setpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, uint32_t col) { - uint8_t *pixel = &((uint8_t *)fb->buf)[(x + y * fb->stride) >> 2]; - uint8_t shift = (x & 0x3) << 1; - uint8_t mask = 0x3 << shift; - uint8_t color = (col & 0x3) << shift; - *pixel = color | (*pixel & (~mask)); -} - -STATIC uint32_t gs2_hmsb_getpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y) { - uint8_t pixel = ((uint8_t *)fb->buf)[(x + y * fb->stride) >> 2]; - uint8_t shift = (x & 0x3) << 1; - return (pixel >> shift) & 0x3; -} - -STATIC void gs2_hmsb_fill_rect(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, unsigned int w, unsigned int h, uint32_t col) { - for (unsigned int xx = x; xx < x + w; xx++) { - for (unsigned int yy = y; yy < y + h; yy++) { - gs2_hmsb_setpixel(fb, xx, yy, col); - } - } -} - -// Functions for GS4_HMSB format - -STATIC void gs4_hmsb_setpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, uint32_t col) { - uint8_t *pixel = &((uint8_t *)fb->buf)[(x + y * fb->stride) >> 1]; - - if (x % 2) { - *pixel = ((uint8_t)col & 0x0f) | (*pixel & 0xf0); - } else { - *pixel = ((uint8_t)col << 4) | (*pixel & 0x0f); - } -} - -STATIC uint32_t gs4_hmsb_getpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y) { - if (x % 2) { - return ((uint8_t *)fb->buf)[(x + y * fb->stride) >> 1] & 0x0f; - } - - return ((uint8_t *)fb->buf)[(x + y * fb->stride) >> 1] >> 4; -} - -STATIC void gs4_hmsb_fill_rect(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, unsigned int w, unsigned int h, uint32_t col) { - col &= 0x0f; - uint8_t *pixel_pair = &((uint8_t *)fb->buf)[(x + y * fb->stride) >> 1]; - uint8_t col_shifted_left = col << 4; - uint8_t col_pixel_pair = col_shifted_left | col; - unsigned int pixel_count_till_next_line = (fb->stride - w) >> 1; - bool odd_x = (x % 2 == 1); - - while (h--) { - unsigned int ww = w; - - if (odd_x && ww > 0) { - *pixel_pair = (*pixel_pair & 0xf0) | col; - pixel_pair++; - ww--; - } - - memset(pixel_pair, col_pixel_pair, ww >> 1); - pixel_pair += ww >> 1; - - if (ww % 2) { - *pixel_pair = col_shifted_left | (*pixel_pair & 0x0f); - if (!odd_x) { - pixel_pair++; - } - } - - pixel_pair += pixel_count_till_next_line; - } -} - -// Functions for GS8 format - -STATIC void gs8_setpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, uint32_t col) { - uint8_t *pixel = &((uint8_t *)fb->buf)[(x + y * fb->stride)]; - *pixel = col & 0xff; -} - -STATIC uint32_t gs8_getpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y) { - return ((uint8_t *)fb->buf)[(x + y * fb->stride)]; -} - -STATIC void gs8_fill_rect(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, unsigned int w, unsigned int h, uint32_t col) { - uint8_t *pixel = &((uint8_t *)fb->buf)[(x + y * fb->stride)]; - while (h--) { - memset(pixel, col, w); - pixel += fb->stride; - } -} - -STATIC const mp_framebuf_p_t formats[] = { - [FRAMEBUF_MVLSB] = {mvlsb_setpixel, mvlsb_getpixel, mvlsb_fill_rect}, - [FRAMEBUF_RGB565] = {rgb565_setpixel, rgb565_getpixel, rgb565_fill_rect}, - [FRAMEBUF_GS2_HMSB] = {gs2_hmsb_setpixel, gs2_hmsb_getpixel, gs2_hmsb_fill_rect}, - [FRAMEBUF_GS4_HMSB] = {gs4_hmsb_setpixel, gs4_hmsb_getpixel, gs4_hmsb_fill_rect}, - [FRAMEBUF_GS8] = {gs8_setpixel, gs8_getpixel, gs8_fill_rect}, - [FRAMEBUF_MHLSB] = {mono_horiz_setpixel, mono_horiz_getpixel, mono_horiz_fill_rect}, - [FRAMEBUF_MHMSB] = {mono_horiz_setpixel, mono_horiz_getpixel, mono_horiz_fill_rect}, -}; - -static inline void setpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y, uint32_t col) { - formats[fb->format].setpixel(fb, x, y, col); -} - -static inline uint32_t getpixel(const mp_obj_framebuf_t *fb, unsigned int x, unsigned int y) { - return formats[fb->format].getpixel(fb, x, y); -} - -STATIC void fill_rect(const mp_obj_framebuf_t *fb, int x, int y, int w, int h, uint32_t col) { - if (h < 1 || w < 1 || x + w <= 0 || y + h <= 0 || y >= fb->height || x >= fb->width) { - // No operation needed. - return; - } - - // clip to the framebuffer - int xend = MIN(fb->width, x + w); - int yend = MIN(fb->height, y + h); - x = MAX(x, 0); - y = MAX(y, 0); - - formats[fb->format].fill_rect(fb, x, y, xend - x, yend - y, col); -} - -STATIC mp_obj_t framebuf_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { - mp_arg_check_num(n_args, n_kw, 4, 5, false); - - mp_obj_framebuf_t *o = m_new_obj(mp_obj_framebuf_t); - o->base.type = type; - o->buf_obj = args[0]; - - mp_buffer_info_t bufinfo; - mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_WRITE); - o->buf = bufinfo.buf; - - o->width = mp_obj_get_int(args[1]); - o->height = mp_obj_get_int(args[2]); - o->format = mp_obj_get_int(args[3]); - if (n_args >= 5) { - o->stride = mp_obj_get_int(args[4]); - } else { - o->stride = o->width; - } - - switch (o->format) { - case FRAMEBUF_MVLSB: - case FRAMEBUF_RGB565: - break; - case FRAMEBUF_MHLSB: - case FRAMEBUF_MHMSB: - o->stride = (o->stride + 7) & ~7; - break; - case FRAMEBUF_GS2_HMSB: - o->stride = (o->stride + 3) & ~3; - break; - case FRAMEBUF_GS4_HMSB: - o->stride = (o->stride + 1) & ~1; - break; - case FRAMEBUF_GS8: - break; - default: - mp_raise_ValueError(MP_ERROR_TEXT("invalid format")); - } - - return MP_OBJ_FROM_PTR(o); -} - -#if !(defined(MICROPY_ENABLE_DYNRUNTIME) && MICROPY_ENABLE_DYNRUNTIME) -STATIC const mp_obj_type_t mp_type_framebuf; -#endif - -// Helper to ensure we have the native super class instead of a subclass. -static mp_obj_framebuf_t *native_framebuf(mp_obj_t framebuf_obj) { - mp_obj_t native_framebuf = mp_obj_cast_to_native_base(framebuf_obj, &mp_type_framebuf); - mp_obj_assert_native_inited(native_framebuf); - if (native_framebuf == MP_OBJ_NULL) { - mp_raise_TypeError(NULL); - } - return MP_OBJ_TO_PTR(native_framebuf); -} - -STATIC mp_int_t framebuf_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, mp_uint_t flags) { - (void)flags; - mp_obj_framebuf_t *self = native_framebuf(self_in); - bufinfo->buf = self->buf; - bufinfo->len = self->stride * self->height * (self->format == FRAMEBUF_RGB565 ? 2 : 1); - bufinfo->typecode = 'B'; // view framebuf as bytes - return 0; -} - -STATIC mp_obj_t framebuf_fill(mp_obj_t self_in, mp_obj_t col_in) { - mp_obj_framebuf_t *self = native_framebuf(self_in); - mp_int_t col = mp_obj_get_int(col_in); - formats[self->format].fill_rect(self, 0, 0, self->width, self->height, col); - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_2(framebuf_fill_obj, framebuf_fill); - -STATIC mp_obj_t framebuf_fill_rect(size_t n_args, const mp_obj_t *args) { - (void)n_args; - - mp_obj_framebuf_t *self = native_framebuf(args[0]); - mp_int_t x = mp_obj_get_int(args[1]); - mp_int_t y = mp_obj_get_int(args[2]); - mp_int_t width = mp_obj_get_int(args[3]); - mp_int_t height = mp_obj_get_int(args[4]); - mp_int_t col = mp_obj_get_int(args[5]); - - fill_rect(self, x, y, width, height, col); - - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(framebuf_fill_rect_obj, 6, 6, framebuf_fill_rect); - -STATIC mp_obj_t framebuf_pixel(size_t n_args, const mp_obj_t *args) { - mp_obj_framebuf_t *self = native_framebuf(args[0]); - mp_int_t x = mp_obj_get_int(args[1]); - mp_int_t y = mp_obj_get_int(args[2]); - if (0 <= x && x < self->width && 0 <= y && y < self->height) { - if (n_args == 3) { - // get - return MP_OBJ_NEW_SMALL_INT(getpixel(self, x, y)); - } else { - // set - setpixel(self, x, y, mp_obj_get_int(args[3])); - } - } - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(framebuf_pixel_obj, 3, 4, framebuf_pixel); - -STATIC mp_obj_t framebuf_hline(size_t n_args, const mp_obj_t *args) { - (void)n_args; - - mp_obj_framebuf_t *self = native_framebuf(args[0]); - mp_int_t x = mp_obj_get_int(args[1]); - mp_int_t y = mp_obj_get_int(args[2]); - mp_int_t w = mp_obj_get_int(args[3]); - mp_int_t col = mp_obj_get_int(args[4]); - - fill_rect(self, x, y, w, 1, col); - - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(framebuf_hline_obj, 5, 5, framebuf_hline); - -STATIC mp_obj_t framebuf_vline(size_t n_args, const mp_obj_t *args) { - (void)n_args; - - mp_obj_framebuf_t *self = native_framebuf(args[0]); - mp_int_t x = mp_obj_get_int(args[1]); - mp_int_t y = mp_obj_get_int(args[2]); - mp_int_t h = mp_obj_get_int(args[3]); - mp_int_t col = mp_obj_get_int(args[4]); - - fill_rect(self, x, y, 1, h, col); - - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(framebuf_vline_obj, 5, 5, framebuf_vline); - -STATIC mp_obj_t framebuf_rect(size_t n_args, const mp_obj_t *args) { - (void)n_args; - - mp_obj_framebuf_t *self = native_framebuf(args[0]); - mp_int_t x = mp_obj_get_int(args[1]); - mp_int_t y = mp_obj_get_int(args[2]); - mp_int_t w = mp_obj_get_int(args[3]); - mp_int_t h = mp_obj_get_int(args[4]); - mp_int_t col = mp_obj_get_int(args[5]); - - fill_rect(self, x, y, w, 1, col); - fill_rect(self, x, y + h - 1, w, 1, col); - fill_rect(self, x, y, 1, h, col); - fill_rect(self, x + w - 1, y, 1, h, col); - - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(framebuf_rect_obj, 6, 6, framebuf_rect); - -STATIC mp_obj_t framebuf_line(size_t n_args, const mp_obj_t *args) { - (void)n_args; - - mp_obj_framebuf_t *self = native_framebuf(args[0]); - mp_int_t x1 = mp_obj_get_int(args[1]); - mp_int_t y1 = mp_obj_get_int(args[2]); - mp_int_t x2 = mp_obj_get_int(args[3]); - mp_int_t y2 = mp_obj_get_int(args[4]); - mp_int_t col = mp_obj_get_int(args[5]); - - mp_int_t dx = x2 - x1; - mp_int_t sx; - if (dx > 0) { - sx = 1; - } else { - dx = -dx; - sx = -1; - } - - mp_int_t dy = y2 - y1; - mp_int_t sy; - if (dy > 0) { - sy = 1; - } else { - dy = -dy; - sy = -1; - } - - bool steep; - if (dy > dx) { - mp_int_t temp; - temp = x1; - x1 = y1; - y1 = temp; - temp = dx; - dx = dy; - dy = temp; - temp = sx; - sx = sy; - sy = temp; - steep = true; - } else { - steep = false; - } - - mp_int_t e = 2 * dy - dx; - for (mp_int_t i = 0; i < dx; ++i) { - if (steep) { - if (0 <= y1 && y1 < self->width && 0 <= x1 && x1 < self->height) { - setpixel(self, y1, x1, col); - } - } else { - if (0 <= x1 && x1 < self->width && 0 <= y1 && y1 < self->height) { - setpixel(self, x1, y1, col); - } - } - while (e >= 0) { - y1 += sy; - e -= 2 * dx; - } - x1 += sx; - e += 2 * dy; - } - - if (0 <= x2 && x2 < self->width && 0 <= y2 && y2 < self->height) { - setpixel(self, x2, y2, col); - } - - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(framebuf_line_obj, 6, 6, framebuf_line); - -STATIC mp_obj_t framebuf_blit(size_t n_args, const mp_obj_t *args) { - mp_obj_framebuf_t *self = native_framebuf(args[0]); - mp_obj_framebuf_t *source = native_framebuf(args[1]); - mp_int_t x = mp_obj_get_int(args[2]); - mp_int_t y = mp_obj_get_int(args[3]); - mp_int_t key = -1; - if (n_args > 4) { - key = mp_obj_get_int(args[4]); - } - mp_obj_framebuf_t *palette = NULL; - if (n_args > 5 && args[5] != mp_const_none) { - palette = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(args[5], MP_OBJ_FROM_PTR(&mp_type_framebuf))); - } - - if ( - (x >= self->width) || - (y >= self->height) || - (-x >= source->width) || - (-y >= source->height) - ) { - // Out of bounds, no-op. - return mp_const_none; - } - - // Clip. - int x0 = MAX(0, x); - int y0 = MAX(0, y); - int x1 = MAX(0, -x); - int y1 = MAX(0, -y); - int x0end = MIN(self->width, x + source->width); - int y0end = MIN(self->height, y + source->height); - - for (; y0 < y0end; ++y0) { - int cx1 = x1; - for (int cx0 = x0; cx0 < x0end; ++cx0) { - uint32_t col = getpixel(source, cx1, y1); - if (palette) { - col = getpixel(palette, col, 0); - } - if (col != (uint32_t)key) { - setpixel(self, cx0, y0, col); - } - ++cx1; - } - ++y1; - } - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(framebuf_blit_obj, 4, 6, framebuf_blit); - -STATIC mp_obj_t framebuf_scroll(mp_obj_t self_in, mp_obj_t xstep_in, mp_obj_t ystep_in) { - mp_obj_framebuf_t *self = native_framebuf(self_in); - mp_int_t xstep = mp_obj_get_int(xstep_in); - mp_int_t ystep = mp_obj_get_int(ystep_in); - int sx, y, xend, yend, dx, dy; - if (xstep < 0) { - sx = 0; - xend = self->width + xstep; - dx = 1; - } else { - sx = self->width - 1; - xend = xstep - 1; - dx = -1; - } - if (ystep < 0) { - y = 0; - yend = self->height + ystep; - dy = 1; - } else { - y = self->height - 1; - yend = ystep - 1; - dy = -1; - } - for (; y != yend; y += dy) { - for (int x = sx; x != xend; x += dx) { - setpixel(self, x, y, getpixel(self, x - xstep, y - ystep)); - } - } - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_3(framebuf_scroll_obj, framebuf_scroll); - -STATIC mp_obj_t framebuf_text(size_t n_args, const mp_obj_t *args) { - // extract arguments - mp_obj_framebuf_t *self = native_framebuf(args[0]); - const char *str = mp_obj_str_get_str(args[1]); - mp_int_t x0 = mp_obj_get_int(args[2]); - mp_int_t y0 = mp_obj_get_int(args[3]); - mp_int_t col = 1; - if (n_args >= 5) { - col = mp_obj_get_int(args[4]); - } - - // loop over chars - for (; *str; ++str) { - // get char and make sure its in range of font - int chr = *(uint8_t *)str; - if (chr < 32 || chr > 127) { - chr = 127; - } - // get char data - const uint8_t *chr_data = &font_petme128_8x8[(chr - 32) * 8]; - // loop over char data - for (int j = 0; j < 8; j++, x0++) { - if (0 <= x0 && x0 < self->width) { // clip x - uint vline_data = chr_data[j]; // each byte is a column of 8 pixels, LSB at top - for (int y = y0; vline_data; vline_data >>= 1, y++) { // scan over vertical column - if (vline_data & 1) { // only draw if pixel set - if (0 <= y && y < self->height) { // clip y - setpixel(self, x0, y, col); - } - } - } - } - } - } - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(framebuf_text_obj, 4, 5, framebuf_text); - -#if !MICROPY_ENABLE_DYNRUNTIME -STATIC const mp_rom_map_elem_t framebuf_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR_fill), MP_ROM_PTR(&framebuf_fill_obj) }, - { MP_ROM_QSTR(MP_QSTR_fill_rect), MP_ROM_PTR(&framebuf_fill_rect_obj) }, - { MP_ROM_QSTR(MP_QSTR_pixel), MP_ROM_PTR(&framebuf_pixel_obj) }, - { MP_ROM_QSTR(MP_QSTR_hline), MP_ROM_PTR(&framebuf_hline_obj) }, - { MP_ROM_QSTR(MP_QSTR_vline), MP_ROM_PTR(&framebuf_vline_obj) }, - { MP_ROM_QSTR(MP_QSTR_rect), MP_ROM_PTR(&framebuf_rect_obj) }, - { MP_ROM_QSTR(MP_QSTR_line), MP_ROM_PTR(&framebuf_line_obj) }, - { MP_ROM_QSTR(MP_QSTR_blit), MP_ROM_PTR(&framebuf_blit_obj) }, - { MP_ROM_QSTR(MP_QSTR_scroll), MP_ROM_PTR(&framebuf_scroll_obj) }, - { MP_ROM_QSTR(MP_QSTR_text), MP_ROM_PTR(&framebuf_text_obj) }, -}; -STATIC MP_DEFINE_CONST_DICT(framebuf_locals_dict, framebuf_locals_dict_table); - -STATIC const mp_obj_type_t mp_type_framebuf = { - { &mp_type_type }, - .flags = MP_TYPE_FLAG_EXTENDED, - .name = MP_QSTR_FrameBuffer, - .make_new = framebuf_make_new, - .locals_dict = (mp_obj_dict_t *)&framebuf_locals_dict, - MP_TYPE_EXTENDED_FIELDS( - .buffer_p = { .get_buffer = framebuf_get_buffer }, - ), -}; -#endif - -// this factory function is provided for backwards compatibility with old FrameBuffer1 class -STATIC mp_obj_t legacy_framebuffer1(size_t n_args, const mp_obj_t *args) { - mp_obj_framebuf_t *o = m_new_obj(mp_obj_framebuf_t); - o->base.type = (mp_obj_type_t *)&mp_type_framebuf; - - mp_buffer_info_t bufinfo; - mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_WRITE); - o->buf = bufinfo.buf; - - o->width = mp_obj_get_int(args[1]); - o->height = mp_obj_get_int(args[2]); - o->format = FRAMEBUF_MVLSB; - if (n_args >= 4) { - o->stride = mp_obj_get_int(args[3]); - } else { - o->stride = o->width; - } - - return MP_OBJ_FROM_PTR(o); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(legacy_framebuffer1_obj, 3, 4, legacy_framebuffer1); - -#if !MICROPY_ENABLE_DYNRUNTIME -STATIC const mp_rom_map_elem_t framebuf_module_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_framebuf) }, - { MP_ROM_QSTR(MP_QSTR_FrameBuffer), MP_ROM_PTR(&mp_type_framebuf) }, - { MP_ROM_QSTR(MP_QSTR_FrameBuffer1), MP_ROM_PTR(&legacy_framebuffer1_obj) }, - { MP_ROM_QSTR(MP_QSTR_MVLSB), MP_ROM_INT(FRAMEBUF_MVLSB) }, - { MP_ROM_QSTR(MP_QSTR_MONO_VLSB), MP_ROM_INT(FRAMEBUF_MVLSB) }, - { MP_ROM_QSTR(MP_QSTR_RGB565), MP_ROM_INT(FRAMEBUF_RGB565) }, - { MP_ROM_QSTR(MP_QSTR_GS2_HMSB), MP_ROM_INT(FRAMEBUF_GS2_HMSB) }, - { MP_ROM_QSTR(MP_QSTR_GS4_HMSB), MP_ROM_INT(FRAMEBUF_GS4_HMSB) }, - { MP_ROM_QSTR(MP_QSTR_GS8), MP_ROM_INT(FRAMEBUF_GS8) }, - { MP_ROM_QSTR(MP_QSTR_MONO_HLSB), MP_ROM_INT(FRAMEBUF_MHLSB) }, - { MP_ROM_QSTR(MP_QSTR_MONO_HMSB), MP_ROM_INT(FRAMEBUF_MHMSB) }, -}; - -STATIC MP_DEFINE_CONST_DICT(framebuf_module_globals, framebuf_module_globals_table); - -const mp_obj_module_t mp_module_framebuf = { - .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&framebuf_module_globals, -}; -#endif - -#endif // MICROPY_PY_FRAMEBUF diff --git a/extmod/modonewire.c b/extmod/modonewire.c deleted file mode 100644 index ba7405fbae..0000000000 --- a/extmod/modonewire.c +++ /dev/null @@ -1,145 +0,0 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2015-2017 Damien P. George -// -// SPDX-License-Identifier: MIT - -#include -#include - -#include "py/obj.h" -#include "py/mphal.h" - -#if MICROPY_PY_ONEWIRE - -/******************************************************************************/ -// Low-level 1-Wire routines - -#define TIMING_RESET1 (480) -#define TIMING_RESET2 (70) -#define TIMING_RESET3 (410) -#define TIMING_READ1 (5) -#define TIMING_READ2 (5) -#define TIMING_READ3 (40) -#define TIMING_WRITE1 (10) -#define TIMING_WRITE2 (50) -#define TIMING_WRITE3 (10) - -STATIC int onewire_bus_reset(mp_hal_pin_obj_t pin) { - mp_hal_pin_od_low(pin); - mp_hal_delay_us(TIMING_RESET1); - uint32_t i = mp_hal_quiet_timing_enter(); - mp_hal_pin_od_high(pin); - mp_hal_delay_us_fast(TIMING_RESET2); - int status = !mp_hal_pin_read(pin); - mp_hal_quiet_timing_exit(i); - mp_hal_delay_us(TIMING_RESET3); - return status; -} - -STATIC int onewire_bus_readbit(mp_hal_pin_obj_t pin) { - mp_hal_pin_od_high(pin); - uint32_t i = mp_hal_quiet_timing_enter(); - mp_hal_pin_od_low(pin); - mp_hal_delay_us_fast(TIMING_READ1); - mp_hal_pin_od_high(pin); - mp_hal_delay_us_fast(TIMING_READ2); - int value = mp_hal_pin_read(pin); - mp_hal_quiet_timing_exit(i); - mp_hal_delay_us_fast(TIMING_READ3); - return value; -} - -STATIC void onewire_bus_writebit(mp_hal_pin_obj_t pin, int value) { - uint32_t i = mp_hal_quiet_timing_enter(); - mp_hal_pin_od_low(pin); - mp_hal_delay_us_fast(TIMING_WRITE1); - if (value) { - mp_hal_pin_od_high(pin); - } - mp_hal_delay_us_fast(TIMING_WRITE2); - mp_hal_pin_od_high(pin); - mp_hal_delay_us_fast(TIMING_WRITE3); - mp_hal_quiet_timing_exit(i); -} - -/******************************************************************************/ -// MicroPython bindings - -STATIC mp_obj_t onewire_reset(mp_obj_t pin_in) { - return mp_obj_new_bool(onewire_bus_reset(mp_hal_get_pin_obj(pin_in))); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(onewire_reset_obj, onewire_reset); - -STATIC mp_obj_t onewire_readbit(mp_obj_t pin_in) { - return MP_OBJ_NEW_SMALL_INT(onewire_bus_readbit(mp_hal_get_pin_obj(pin_in))); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(onewire_readbit_obj, onewire_readbit); - -STATIC mp_obj_t onewire_readbyte(mp_obj_t pin_in) { - mp_hal_pin_obj_t pin = mp_hal_get_pin_obj(pin_in); - uint8_t value = 0; - for (int i = 0; i < 8; ++i) { - value |= onewire_bus_readbit(pin) << i; - } - return MP_OBJ_NEW_SMALL_INT(value); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(onewire_readbyte_obj, onewire_readbyte); - -STATIC mp_obj_t onewire_writebit(mp_obj_t pin_in, mp_obj_t value_in) { - onewire_bus_writebit(mp_hal_get_pin_obj(pin_in), mp_obj_get_int(value_in)); - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_2(onewire_writebit_obj, onewire_writebit); - -STATIC mp_obj_t onewire_writebyte(mp_obj_t pin_in, mp_obj_t value_in) { - mp_hal_pin_obj_t pin = mp_hal_get_pin_obj(pin_in); - int value = mp_obj_get_int(value_in); - for (int i = 0; i < 8; ++i) { - onewire_bus_writebit(pin, value & 1); - value >>= 1; - } - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_2(onewire_writebyte_obj, onewire_writebyte); - -STATIC mp_obj_t onewire_crc8(mp_obj_t data) { - mp_buffer_info_t bufinfo; - mp_get_buffer_raise(data, &bufinfo, MP_BUFFER_READ); - uint8_t crc = 0; - for (size_t i = 0; i < bufinfo.len; ++i) { - uint8_t byte = ((uint8_t *)bufinfo.buf)[i]; - for (int b = 0; b < 8; ++b) { - uint8_t fb_bit = (crc ^ byte) & 0x01; - if (fb_bit == 0x01) { - crc = crc ^ 0x18; - } - crc = (crc >> 1) & 0x7f; - if (fb_bit == 0x01) { - crc = crc | 0x80; - } - byte = byte >> 1; - } - } - return MP_OBJ_NEW_SMALL_INT(crc); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(onewire_crc8_obj, onewire_crc8); - -STATIC const mp_rom_map_elem_t onewire_module_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_onewire) }, - - { MP_ROM_QSTR(MP_QSTR_reset), MP_ROM_PTR(&onewire_reset_obj) }, - { MP_ROM_QSTR(MP_QSTR_readbit), MP_ROM_PTR(&onewire_readbit_obj) }, - { MP_ROM_QSTR(MP_QSTR_readbyte), MP_ROM_PTR(&onewire_readbyte_obj) }, - { MP_ROM_QSTR(MP_QSTR_writebit), MP_ROM_PTR(&onewire_writebit_obj) }, - { MP_ROM_QSTR(MP_QSTR_writebyte), MP_ROM_PTR(&onewire_writebyte_obj) }, - { MP_ROM_QSTR(MP_QSTR_crc8), MP_ROM_PTR(&onewire_crc8_obj) }, -}; - -STATIC MP_DEFINE_CONST_DICT(onewire_module_globals, onewire_module_globals_table); - -const mp_obj_module_t mp_module_onewire = { - .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&onewire_module_globals, -}; - -#endif // MICROPY_PY_ONEWIRE diff --git a/extmod/moduasyncio.c b/extmod/moduasyncio.c index 1e1c5de7e7..4c0436b2db 100644 --- a/extmod/moduasyncio.c +++ b/extmod/moduasyncio.c @@ -70,6 +70,7 @@ STATIC mp_obj_t task_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf); /******************************************************************************/ // Ticks for task ordering in pairing heap +// CIRCUITPY-style ticks #define _TICKS_PERIOD (1lu << 29) #define _TICKS_MAX (_TICKS_PERIOD - 1) #define _TICKS_HALFPERIOD (_TICKS_PERIOD >> 1) @@ -105,8 +106,7 @@ STATIC int task_lt(mp_pairheap_t *n1, mp_pairheap_t *n2) { STATIC mp_obj_t task_queue_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { (void)args; mp_arg_check_num(n_args, n_kw, 0, 0, false); - mp_obj_task_queue_t *self = m_new_obj(mp_obj_task_queue_t); - self->base.type = type; + mp_obj_task_queue_t *self = mp_obj_malloc(mp_obj_task_queue_t, type); self->heap = (mp_obj_task_t *)mp_pairheap_new(task_lt); return MP_OBJ_FROM_PTR(self); } @@ -121,7 +121,7 @@ STATIC mp_obj_t task_queue_peek(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(task_queue_peek_obj, task_queue_peek); -STATIC mp_obj_t task_queue_push_sorted(size_t n_args, const mp_obj_t *args) { +STATIC mp_obj_t task_queue_push(size_t n_args, const mp_obj_t *args) { mp_obj_task_queue_t *self = MP_OBJ_TO_PTR(args[0]); mp_obj_task_t *task = MP_OBJ_TO_PTR(args[1]); task->data = mp_const_none; @@ -134,9 +134,9 @@ STATIC mp_obj_t task_queue_push_sorted(size_t n_args, const mp_obj_t *args) { self->heap = (mp_obj_task_t *)mp_pairheap_push(task_lt, TASK_PAIRHEAP(self->heap), TASK_PAIRHEAP(task)); return mp_const_none; } -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(task_queue_push_sorted_obj, 2, 3, task_queue_push_sorted); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(task_queue_push_obj, 2, 3, task_queue_push); -STATIC mp_obj_t task_queue_pop_head(mp_obj_t self_in) { +STATIC mp_obj_t task_queue_pop(mp_obj_t self_in) { mp_obj_task_queue_t *self = MP_OBJ_TO_PTR(self_in); mp_obj_task_t *head = (mp_obj_task_t *)mp_pairheap_peek(task_lt, &self->heap->pairheap); if (head == NULL) { @@ -145,7 +145,7 @@ STATIC mp_obj_t task_queue_pop_head(mp_obj_t self_in) { self->heap = (mp_obj_task_t *)mp_pairheap_pop(task_lt, &self->heap->pairheap); return MP_OBJ_FROM_PTR(head); } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(task_queue_pop_head_obj, task_queue_pop_head); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(task_queue_pop_obj, task_queue_pop); STATIC mp_obj_t task_queue_remove(mp_obj_t self_in, mp_obj_t task_in) { mp_obj_task_queue_t *self = MP_OBJ_TO_PTR(self_in); @@ -157,9 +157,8 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(task_queue_remove_obj, task_queue_remove); STATIC const mp_rom_map_elem_t task_queue_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_peek), MP_ROM_PTR(&task_queue_peek_obj) }, - { MP_ROM_QSTR(MP_QSTR_push_sorted), MP_ROM_PTR(&task_queue_push_sorted_obj) }, - { MP_ROM_QSTR(MP_QSTR_push_head), MP_ROM_PTR(&task_queue_push_sorted_obj) }, - { MP_ROM_QSTR(MP_QSTR_pop_head), MP_ROM_PTR(&task_queue_pop_head_obj) }, + { MP_ROM_QSTR(MP_QSTR_push), MP_ROM_PTR(&task_queue_push_obj) }, + { MP_ROM_QSTR(MP_QSTR_pop), MP_ROM_PTR(&task_queue_pop_obj) }, { MP_ROM_QSTR(MP_QSTR_remove), MP_ROM_PTR(&task_queue_remove_obj) }, }; STATIC MP_DEFINE_CONST_DICT(task_queue_locals_dict, task_queue_locals_dict_table); @@ -223,18 +222,18 @@ STATIC mp_obj_t task_cancel(mp_obj_t self_in) { // Not on the main running queue, remove the task from the queue it's on. dest[2] = MP_OBJ_FROM_PTR(self); mp_call_method_n_kw(1, 0, dest); - // _task_queue.push_head(self) + // _task_queue.push(self) dest[0] = _task_queue; dest[1] = MP_OBJ_FROM_PTR(self); - task_queue_push_sorted(2, dest); + task_queue_push(2, dest); } else if (ticks_diff(self->ph_key, ticks()) > 0) { // On the main running queue but scheduled in the future, so bring it forward to now. // _task_queue.remove(self) task_queue_remove(_task_queue, MP_OBJ_FROM_PTR(self)); - // _task_queue.push_head(self) + // _task_queue.push(self) dest[0] = _task_queue; dest[1] = MP_OBJ_FROM_PTR(self); - task_queue_push_sorted(2, dest); + task_queue_push(2, dest); } self->data = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_CancelledError)); @@ -243,6 +242,7 @@ STATIC mp_obj_t task_cancel(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(task_cancel_obj, task_cancel); +// CIRCUITPY provides __await__(). STATIC mp_obj_t task_await(mp_obj_t self_in) { return task_getiter(self_in, NULL); } @@ -266,6 +266,7 @@ STATIC void task_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { dest[1] = self_in; } else if (attr == MP_QSTR_ph_key) { dest[0] = self->ph_key; + // CIRCUITPY provides __await__(). } else if (attr == MP_QSTR___await__) { dest[0] = MP_OBJ_FROM_PTR(&task_await_obj); dest[1] = self_in; @@ -291,6 +292,9 @@ STATIC mp_obj_t task_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { } else if (self->state == TASK_STATE_RUNNING_NOT_WAITED_ON) { // Allocate the waiting queue. self->state = task_queue_make_new(&task_queue_type, 0, 0, NULL); + } else if (mp_obj_get_type(self->state) != &task_queue_type) { + // Task has state used for another purpose, so can't also wait on it. + mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("can't wait")); } return self_in; } @@ -298,6 +302,7 @@ STATIC mp_obj_t task_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { STATIC mp_obj_t task_iternext(mp_obj_t self_in) { mp_obj_task_t *self = MP_OBJ_TO_PTR(self_in); if (TASK_IS_DONE(self)) { + // CIRCUITPY if (self->data == mp_const_none) { // Task finished but has already been sent to the loop's exception handler. mp_raise_StopIteration(MP_OBJ_NULL); @@ -309,7 +314,7 @@ STATIC mp_obj_t task_iternext(mp_obj_t self_in) { // Put calling task on waiting queue. mp_obj_t cur_task = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); mp_obj_t args[2] = { self->state, cur_task }; - task_queue_push_sorted(2, args); + task_queue_push(2, args); // Set calling task's data to this task that it waits on, to double-link it. ((mp_obj_task_t *)MP_OBJ_TO_PTR(cur_task))->data = self_in; } @@ -347,6 +352,6 @@ const mp_obj_module_t mp_module_uasyncio = { .globals = (mp_obj_dict_t *)&mp_module_uasyncio_globals, }; -MP_REGISTER_MODULE(MP_QSTR__asyncio, mp_module_uasyncio, MICROPY_PY_UASYNCIO); +MP_REGISTER_MODULE(MP_QSTR__uasyncio, mp_module_uasyncio); #endif // MICROPY_PY_UASYNCIO diff --git a/extmod/modubinascii.c b/extmod/modubinascii.c index 5aa9eacefd..dc5a6e29ec 100644 --- a/extmod/modubinascii.c +++ b/extmod/modubinascii.c @@ -1,8 +1,28 @@ -// Copyright (c) 2014 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: 2022 Beat Ludin for Adafruit Industries -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include #include @@ -20,6 +40,7 @@ static void check_not_unicode(const mp_obj_t arg) { } #endif } +#if MICROPY_PY_UBINASCII STATIC mp_obj_t mod_binascii_hexlify(size_t n_args, const mp_obj_t *args) { // First argument is the data to convert. @@ -154,13 +175,20 @@ STATIC mp_obj_t mod_binascii_a2b_base64(mp_obj_t data) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_binascii_a2b_base64_obj, mod_binascii_a2b_base64); -STATIC mp_obj_t mod_binascii_b2a_base64(mp_obj_t data) { - check_not_unicode(data); +STATIC mp_obj_t mod_binascii_b2a_base64(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { + enum { ARG_newline }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_newline, MP_ARG_BOOL, {.u_bool = true} }, + }; + + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + mp_arg_parse_all(n_args - 1, pos_args + 1, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + uint8_t newline = args[ARG_newline].u_bool; mp_buffer_info_t bufinfo; - mp_get_buffer_raise(data, &bufinfo, MP_BUFFER_READ); + mp_get_buffer_raise(pos_args[0], &bufinfo, MP_BUFFER_READ); vstr_t vstr; - vstr_init_len(&vstr, ((bufinfo.len != 0) ? (((bufinfo.len - 1) / 3) + 1) * 4 : 0) + 1); + vstr_init_len(&vstr, ((bufinfo.len != 0) ? (((bufinfo.len - 1) / 3) + 1) * 4 : 0) + newline); // First pass, we convert input buffer to numeric base 64 values byte *in = bufinfo.buf, *out = (byte *)vstr.buf; @@ -186,7 +214,7 @@ STATIC mp_obj_t mod_binascii_b2a_base64(mp_obj_t data) { // Second pass, we convert number base 64 values to actual base64 ascii encoding out = (byte *)vstr.buf; - for (mp_uint_t j = vstr.len - 1; j--;) { + for (mp_uint_t j = vstr.len - newline; j--;) { if (*out < 26) { *out += 'A'; } else if (*out < 52) { @@ -202,10 +230,15 @@ STATIC mp_obj_t mod_binascii_b2a_base64(mp_obj_t data) { } out++; } - *out = '\n'; + if (newline) { + *out = '\n'; + } return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr); } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_binascii_b2a_base64_obj, mod_binascii_b2a_base64); +STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_binascii_b2a_base64_obj, 1, mod_binascii_b2a_base64); + +// CIRCUITPY uses a self-contained implementation of CRC32, +// instead of depending on uzlib, like MicroPython. /* * CRC32 checksum @@ -296,4 +329,6 @@ const mp_obj_module_t mp_module_ubinascii = { .globals = (mp_obj_dict_t *)&mp_module_binascii_globals, }; -MP_REGISTER_MODULE(MP_QSTR_binascii, mp_module_ubinascii, MICROPY_PY_UBINASCII); +MP_REGISTER_MODULE(MP_QSTR_ubinascii, mp_module_ubinascii); + +#endif // MICROPY_PY_UBINASCII diff --git a/extmod/moductypes.c b/extmod/moductypes.c index 38f700c055..78cf44a17a 100644 --- a/extmod/moductypes.c +++ b/extmod/moductypes.c @@ -1,7 +1,28 @@ -// Copyright (c) 2014-2018 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014-2018 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include #include @@ -76,8 +97,7 @@ STATIC NORETURN void syntax_error(void) { STATIC mp_obj_t uctypes_struct_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 2, 3, false); - mp_obj_uctypes_struct_t *o = m_new_obj(mp_obj_uctypes_struct_t); - o->base.type = type; + mp_obj_uctypes_struct_t *o = mp_obj_malloc(mp_obj_uctypes_struct_t, type); o->addr = (void *)(uintptr_t)mp_obj_int_get_truncated(args[0]); o->desc = args[1]; o->flags = LAYOUT_NATIVE; @@ -444,8 +464,7 @@ STATIC mp_obj_t uctypes_struct_attr_op(mp_obj_t self_in, qstr attr, mp_obj_t set switch (agg_type) { case STRUCT: { - mp_obj_uctypes_struct_t *o = m_new_obj(mp_obj_uctypes_struct_t); - o->base.type = &uctypes_struct_type; + mp_obj_uctypes_struct_t *o = mp_obj_malloc(mp_obj_uctypes_struct_t, &uctypes_struct_type); o->desc = sub->items[1]; o->addr = self->addr + offset; o->flags = self->flags; @@ -460,8 +479,7 @@ STATIC mp_obj_t uctypes_struct_attr_op(mp_obj_t self_in, qstr attr, mp_obj_t set MP_FALLTHROUGH } case PTR: { - mp_obj_uctypes_struct_t *o = m_new_obj(mp_obj_uctypes_struct_t); - o->base.type = &uctypes_struct_type; + mp_obj_uctypes_struct_t *o = mp_obj_malloc(mp_obj_uctypes_struct_t, &uctypes_struct_type); o->desc = MP_OBJ_FROM_PTR(sub); o->addr = self->addr + offset; o->flags = self->flags; @@ -533,8 +551,7 @@ STATIC mp_obj_t uctypes_struct_subscr(mp_obj_t base_in, mp_obj_t index_in, mp_ob } else if (value == MP_OBJ_SENTINEL) { mp_uint_t dummy = 0; mp_uint_t size = uctypes_struct_size(t->items[2], self->flags, &dummy); - mp_obj_uctypes_struct_t *o = m_new_obj(mp_obj_uctypes_struct_t); - o->base.type = &uctypes_struct_type; + mp_obj_uctypes_struct_t *o = mp_obj_malloc(mp_obj_uctypes_struct_t, &uctypes_struct_type); o->desc = t->items[2]; o->addr = self->addr + size * index; o->flags = self->flags; @@ -551,8 +568,7 @@ STATIC mp_obj_t uctypes_struct_subscr(mp_obj_t base_in, mp_obj_t index_in, mp_ob } else { mp_uint_t dummy = 0; mp_uint_t size = uctypes_struct_size(t->items[1], self->flags, &dummy); - mp_obj_uctypes_struct_t *o = m_new_obj(mp_obj_uctypes_struct_t); - o->base.type = &uctypes_struct_type; + mp_obj_uctypes_struct_t *o = mp_obj_malloc(mp_obj_uctypes_struct_t, &uctypes_struct_type); o->desc = t->items[1]; o->addr = p + size * index; o->flags = self->flags; @@ -706,4 +722,6 @@ const mp_obj_module_t mp_module_uctypes = { .globals = (mp_obj_dict_t *)&mp_module_uctypes_globals, }; +MP_REGISTER_MODULE(MP_QSTR_uctypes, mp_module_uctypes); + #endif diff --git a/extmod/moduhashlib.c b/extmod/moduhashlib.c index a84254c566..42bfb8573e 100644 --- a/extmod/moduhashlib.c +++ b/extmod/moduhashlib.c @@ -1,7 +1,28 @@ -// Copyright (c) 2014 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include #include @@ -39,7 +60,6 @@ #endif - typedef struct _mp_obj_hash_t { mp_obj_base_t base; bool final; // if set, update and digest raise an exception @@ -65,8 +85,7 @@ STATIC mp_obj_t uhashlib_sha256_update(mp_obj_t self_in, mp_obj_t arg); STATIC mp_obj_t uhashlib_sha256_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); - mp_obj_hash_t *o = m_new_obj_var(mp_obj_hash_t, char, sizeof(mbedtls_sha256_context)); - o->base.type = type; + mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(mbedtls_sha256_context), type); o->final = false; mbedtls_sha256_init((mbedtls_sha256_context *)&o->state); mbedtls_sha256_starts_ret((mbedtls_sha256_context *)&o->state, 0); @@ -110,8 +129,7 @@ static void check_not_unicode(const mp_obj_t arg) { STATIC mp_obj_t uhashlib_sha256_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); - mp_obj_hash_t *o = m_new_obj_var(mp_obj_hash_t, char, sizeof(CRYAL_SHA256_CTX)); - o->base.type = type; + mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(CRYAL_SHA256_CTX), type); o->final = false; sha256_init((CRYAL_SHA256_CTX *)o->state); if (n_args == 1) { @@ -159,16 +177,13 @@ STATIC const mp_obj_type_t uhashlib_sha256_type = { }; #endif -#endif - #if MICROPY_PY_UHASHLIB_SHA1 STATIC mp_obj_t uhashlib_sha1_update(mp_obj_t self_in, mp_obj_t arg); #if MICROPY_SSL_AXTLS -STATIC mp_obj_t uhashlib_sha1_make_new(const mp_obj_type_t *type, size_t n_args, const mp_obj_t *args, mp_map_t *kw_args) { - mp_arg_check_num(n_args, kw_args, 0, 1, false); - mp_obj_hash_t *o = m_new_obj_var(mp_obj_hash_t, char, sizeof(SHA1_CTX)); - o->base.type = type; +STATIC mp_obj_t uhashlib_sha1_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_arg_check_num(n_args, n_kw, 0, 1, false); + mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(SHA1_CTX), type); o->final = false; SHA1_Init((SHA1_CTX *)o->state); if (n_args == 1) { @@ -178,7 +193,6 @@ STATIC mp_obj_t uhashlib_sha1_make_new(const mp_obj_type_t *type, size_t n_args, } STATIC mp_obj_t uhashlib_sha1_update(mp_obj_t self_in, mp_obj_t arg) { - check_not_unicode(arg); mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in); uhashlib_ensure_not_final(self); mp_buffer_info_t bufinfo; @@ -208,8 +222,7 @@ STATIC mp_obj_t uhashlib_sha1_digest(mp_obj_t self_in) { STATIC mp_obj_t uhashlib_sha1_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); - mp_obj_hash_t *o = m_new_obj_var(mp_obj_hash_t, char, sizeof(mbedtls_sha1_context)); - o->base.type = type; + mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(mbedtls_sha1_context), type); o->final = false; mbedtls_sha1_init((mbedtls_sha1_context *)o->state); mbedtls_sha1_starts_ret((mbedtls_sha1_context *)o->state); @@ -263,8 +276,7 @@ STATIC mp_obj_t uhashlib_md5_update(mp_obj_t self_in, mp_obj_t arg); #if MICROPY_SSL_AXTLS STATIC mp_obj_t uhashlib_md5_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); - mp_obj_hash_t *o = m_new_obj_var(mp_obj_hash_t, char, sizeof(MD5_CTX)); - o->base.type = type; + mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(MD5_CTX), type); o->final = false; MD5_Init((MD5_CTX *)o->state); if (n_args == 1) { @@ -303,8 +315,7 @@ STATIC mp_obj_t uhashlib_md5_digest(mp_obj_t self_in) { STATIC mp_obj_t uhashlib_md5_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); - mp_obj_hash_t *o = m_new_obj_var(mp_obj_hash_t, char, sizeof(mbedtls_md5_context)); - o->base.type = type; + mp_obj_hash_t *o = mp_obj_malloc_var(mp_obj_hash_t, char, sizeof(mbedtls_md5_context), type); o->final = false; mbedtls_md5_init((mbedtls_md5_context *)o->state); mbedtls_md5_starts_ret((mbedtls_md5_context *)o->state); @@ -372,4 +383,6 @@ const mp_obj_module_t mp_module_uhashlib = { .globals = (mp_obj_dict_t *)&mp_module_uhashlib_globals, }; +MP_REGISTER_MODULE(MP_QSTR_uhashlib, mp_module_uhashlib, MICROPY_PY_UHASHLIB); + #endif // MICROPY_PY_UHASHLIB diff --git a/extmod/moduheapq.c b/extmod/moduheapq.c index 0200245f86..f48d9bceb1 100644 --- a/extmod/moduheapq.c +++ b/extmod/moduheapq.c @@ -1,7 +1,28 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include "py/objlist.h" #include "py/runtime.h" @@ -98,6 +119,8 @@ const mp_obj_module_t mp_module_uheapq = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&mp_module_uheapq_globals, }; + +MP_REGISTER_MODULE(MP_QSTR_uheapq, mp_module_uheapq); #endif #endif // MICROPY_PY_UHEAPQ diff --git a/extmod/modujson.c b/extmod/modujson.c index cff02ae95c..14e45e2cc1 100644 --- a/extmod/modujson.c +++ b/extmod/modujson.c @@ -1,7 +1,28 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2014-2019 Damien P. George -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014-2019 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include @@ -112,6 +133,7 @@ typedef struct _ujson_stream_t { mp_obj_t stream_obj; mp_uint_t (*read)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode); int errcode; + // CIRCUITPY mp_obj_t python_readinto[2 + 1]; mp_obj_array_t bytearray_obj; size_t start; @@ -136,6 +158,8 @@ STATIC byte ujson_stream_next(ujson_stream_t *s) { return s->cur; } +// CIRCUITPY + // We read from an object's `readinto` method in chunks larger than the json // parser needs to reduce the number of function calls done. @@ -375,6 +399,8 @@ STATIC mp_obj_t _mod_ujson_load(mp_obj_t stream_obj, bool return_first_json) { } } success: + // CIRCUITPY + // It is legal for a stream to have contents after JSON. // E.g., A UART is not closed after receiving an object; in load() we will // return the first complete JSON object, while in loads() we will retain @@ -432,6 +458,6 @@ const mp_obj_module_t mp_module_ujson = { .globals = (mp_obj_dict_t *)&mp_module_ujson_globals, }; -MP_REGISTER_MODULE(MP_QSTR_json, mp_module_ujson, MICROPY_PY_UJSON); +MP_REGISTER_MODULE(MP_QSTR_ujson, mp_module_ujson); #endif // MICROPY_PY_UJSON diff --git a/extmod/moduos.c b/extmod/moduos.c new file mode 100644 index 0000000000..87a611148d --- /dev/null +++ b/extmod/moduos.c @@ -0,0 +1,181 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2016-2022 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/objstr.h" +#include "py/runtime.h" + +#if MICROPY_PY_UOS + +#include "extmod/misc.h" +#include "extmod/vfs.h" + +#if MICROPY_VFS_FAT +#include "extmod/vfs_fat.h" +#endif + +#if MICROPY_VFS_LFS1 || MICROPY_VFS_LFS2 +#include "extmod/vfs_lfs.h" +#endif + +#if MICROPY_VFS_POSIX +#include "extmod/vfs_posix.h" +#endif + +#if MICROPY_PY_UOS_UNAME +#include "genhdr/mpversion.h" +#endif + +#ifdef MICROPY_PY_UOS_INCLUDEFILE +#include MICROPY_PY_UOS_INCLUDEFILE +#endif + +#ifdef MICROPY_BUILD_TYPE +#define MICROPY_BUILD_TYPE_PAREN " (" MICROPY_BUILD_TYPE ")" +#else +#define MICROPY_BUILD_TYPE_PAREN +#endif + +#if MICROPY_PY_UOS_UNAME + +#if MICROPY_PY_UOS_UNAME_RELEASE_DYNAMIC +#define CONST_RELEASE +#else +#define CONST_RELEASE const +#endif + +STATIC const qstr mp_uos_uname_info_fields[] = { + MP_QSTR_sysname, + MP_QSTR_nodename, + MP_QSTR_release, + MP_QSTR_version, + MP_QSTR_machine +}; +STATIC const MP_DEFINE_STR_OBJ(mp_uos_uname_info_sysname_obj, MICROPY_PY_SYS_PLATFORM); +STATIC const MP_DEFINE_STR_OBJ(mp_uos_uname_info_nodename_obj, MICROPY_PY_SYS_PLATFORM); +STATIC CONST_RELEASE MP_DEFINE_STR_OBJ(mp_uos_uname_info_release_obj, MICROPY_VERSION_STRING); +STATIC const MP_DEFINE_STR_OBJ(mp_uos_uname_info_version_obj, MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE MICROPY_BUILD_TYPE_PAREN); +STATIC const MP_DEFINE_STR_OBJ(mp_uos_uname_info_machine_obj, MICROPY_HW_BOARD_NAME " with " MICROPY_HW_MCU_NAME); + +STATIC MP_DEFINE_ATTRTUPLE( + mp_uos_uname_info_obj, + mp_uos_uname_info_fields, + 5, + MP_ROM_PTR(&mp_uos_uname_info_sysname_obj), + MP_ROM_PTR(&mp_uos_uname_info_nodename_obj), + MP_ROM_PTR(&mp_uos_uname_info_release_obj), + MP_ROM_PTR(&mp_uos_uname_info_version_obj), + MP_ROM_PTR(&mp_uos_uname_info_machine_obj) + ); + +STATIC mp_obj_t mp_uos_uname(void) { + #if MICROPY_PY_UOS_UNAME_RELEASE_DYNAMIC + const char *release = mp_uos_uname_release(); + mp_uos_uname_info_release_obj.len = strlen(release); + mp_uos_uname_info_release_obj.data = (const byte *)release; + #endif + return MP_OBJ_FROM_PTR(&mp_uos_uname_info_obj); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_uos_uname_obj, mp_uos_uname); + +#endif + +STATIC const mp_rom_map_elem_t os_module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uos) }, + + #if MICROPY_PY_UOS_GETENV_PUTENV_UNSETENV + { MP_ROM_QSTR(MP_QSTR_getenv), MP_ROM_PTR(&mp_uos_getenv_obj) }, + { MP_ROM_QSTR(MP_QSTR_putenv), MP_ROM_PTR(&mp_uos_putenv_obj) }, + { MP_ROM_QSTR(MP_QSTR_unsetenv), MP_ROM_PTR(&mp_uos_unsetenv_obj) }, + #endif + #if MICROPY_PY_UOS_SEP + { MP_ROM_QSTR(MP_QSTR_sep), MP_ROM_QSTR(MP_QSTR__slash_) }, + #endif + #if MICROPY_PY_UOS_SYNC + { MP_ROM_QSTR(MP_QSTR_sync), MP_ROM_PTR(&mp_uos_sync_obj) }, + #endif + #if MICROPY_PY_UOS_SYSTEM + { MP_ROM_QSTR(MP_QSTR_system), MP_ROM_PTR(&mp_uos_system_obj) }, + #endif + #if MICROPY_PY_UOS_UNAME + { MP_ROM_QSTR(MP_QSTR_uname), MP_ROM_PTR(&mp_uos_uname_obj) }, + #endif + #if MICROPY_PY_UOS_URANDOM + { MP_ROM_QSTR(MP_QSTR_urandom), MP_ROM_PTR(&mp_uos_urandom_obj) }, + #endif + + #if MICROPY_VFS + { MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&mp_vfs_chdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_getcwd), MP_ROM_PTR(&mp_vfs_getcwd_obj) }, + { MP_ROM_QSTR(MP_QSTR_listdir), MP_ROM_PTR(&mp_vfs_listdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_mkdir), MP_ROM_PTR(&mp_vfs_mkdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_remove), MP_ROM_PTR(&mp_vfs_remove_obj) }, + { MP_ROM_QSTR(MP_QSTR_rename), MP_ROM_PTR(&mp_vfs_rename_obj) }, + { MP_ROM_QSTR(MP_QSTR_rmdir), MP_ROM_PTR(&mp_vfs_rmdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&mp_vfs_stat_obj) }, + { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&mp_vfs_statvfs_obj) }, + { MP_ROM_QSTR(MP_QSTR_unlink), MP_ROM_PTR(&mp_vfs_remove_obj) }, // unlink aliases to remove + #endif + + // The following are MicroPython extensions. + + #if MICROPY_PY_OS_DUPTERM + { MP_ROM_QSTR(MP_QSTR_dupterm), MP_ROM_PTR(&mp_uos_dupterm_obj) }, + #endif + #if MICROPY_PY_UOS_DUPTERM_NOTIFY + { MP_ROM_QSTR(MP_QSTR_dupterm_notify), MP_ROM_PTR(&mp_uos_dupterm_notify_obj) }, + #endif + #if MICROPY_PY_UOS_ERRNO + { MP_ROM_QSTR(MP_QSTR_errno), MP_ROM_PTR(&mp_uos_errno_obj) }, + #endif + + #if MICROPY_VFS + { MP_ROM_QSTR(MP_QSTR_ilistdir), MP_ROM_PTR(&mp_vfs_ilistdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&mp_vfs_mount_obj) }, + { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&mp_vfs_umount_obj) }, + #if MICROPY_VFS_FAT + { MP_ROM_QSTR(MP_QSTR_VfsFat), MP_ROM_PTR(&mp_fat_vfs_type) }, + #endif + #if MICROPY_VFS_LFS1 + { MP_ROM_QSTR(MP_QSTR_VfsLfs1), MP_ROM_PTR(&mp_type_vfs_lfs1) }, + #endif + #if MICROPY_VFS_LFS2 + { MP_ROM_QSTR(MP_QSTR_VfsLfs2), MP_ROM_PTR(&mp_type_vfs_lfs2) }, + #endif + #if MICROPY_VFS_POSIX + { MP_ROM_QSTR(MP_QSTR_VfsPosix), MP_ROM_PTR(&mp_type_vfs_posix) }, + #endif + #endif +}; +STATIC MP_DEFINE_CONST_DICT(os_module_globals, os_module_globals_table); + +const mp_obj_module_t mp_module_uos = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&os_module_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_uos, mp_module_uos); + +#endif // MICROPY_PY_UOS diff --git a/extmod/moduplatform.c b/extmod/moduplatform.c index 820feb312a..1b35b08aa7 100644 --- a/extmod/moduplatform.c +++ b/extmod/moduplatform.c @@ -29,87 +29,19 @@ #include "py/objtuple.h" #include "py/objstr.h" #include "py/mphal.h" +#include "extmod/moduplatform.h" #include "genhdr/mpversion.h" #if MICROPY_PY_UPLATFORM // platform - Access to underlying platform's identifying data -// TODO: Add more architectures, compilers and libraries. -// See: https://sourceforge.net/p/predef/wiki/Home/ - -#if defined(__ARM_ARCH) -#define PLATFORM_ARCH "arm" -#elif defined(__x86_64__) || defined(_WIN64) -#define PLATFORM_ARCH "x86_64" -#elif defined(__i386__) || defined(_M_IX86) -#define PLATFORM_ARCH "x86" -#elif defined(__xtensa__) || defined(_M_IX86) -#define PLATFORM_ARCH "xtensa" -#else -#define PLATFORM_ARCH "" -#endif - -#if defined(__GNUC__) -#define PLATFORM_COMPILER \ - "GCC " \ - MP_STRINGIFY(__GNUC__) "." \ - MP_STRINGIFY(__GNUC_MINOR__) "." \ - MP_STRINGIFY(__GNUC_PATCHLEVEL__) -#elif defined(__ARMCC_VERSION) -#define PLATFORM_COMPILER \ - "ARMCC " \ - MP_STRINGIFY((__ARMCC_VERSION / 1000000)) "." \ - MP_STRINGIFY((__ARMCC_VERSION / 10000 % 100)) "." \ - MP_STRINGIFY((__ARMCC_VERSION % 10000)) -#elif defined(_MSC_VER) -#if defined(_WIN64) -#define COMPILER_BITS "64 bit" -#elif defined(_M_IX86) -#define COMPILER_BITS "32 bit" -#else -#define COMPILER_BITS "" -#endif -#define PLATFORM_COMPILER \ - "MSC v." MP_STRINGIFY(_MSC_VER) " " COMPILER_BITS -#else -#define PLATFORM_COMPILER "" -#endif - -#if defined(__GLIBC__) -#define PLATFORM_LIBC_LIB "glibc" -#define PLATFORM_LIBC_VER \ - MP_STRINGIFY(__GLIBC__) "." \ - MP_STRINGIFY(__GLIBC_MINOR__) -#elif defined(__NEWLIB__) -#define PLATFORM_LIBC_LIB "newlib" -#define PLATFORM_LIBC_VER _NEWLIB_VERSION -#else -#define PLATFORM_LIBC_LIB "" -#define PLATFORM_LIBC_VER "" -#endif - -#if defined(__linux) -#define PLATFORM_SYSTEM "Linux" -#elif defined(__unix__) -#define PLATFORM_SYSTEM "Unix" -#elif defined(__CYGWIN__) -#define PLATFORM_SYSTEM "Cygwin" -#elif defined(_WIN32) -#define PLATFORM_SYSTEM "Windows" -#else -#define PLATFORM_SYSTEM "MicroPython" -#endif - -#ifndef MICROPY_PLATFORM_VERSION -#define MICROPY_PLATFORM_VERSION "" -#endif - -STATIC const MP_DEFINE_STR_OBJ(info_platform_obj, PLATFORM_SYSTEM "-" MICROPY_VERSION_STRING "-" \ - PLATFORM_ARCH "-" MICROPY_PLATFORM_VERSION "-with-" PLATFORM_LIBC_LIB "" PLATFORM_LIBC_VER); -STATIC const MP_DEFINE_STR_OBJ(info_python_compiler_obj, PLATFORM_COMPILER); -STATIC const MP_DEFINE_STR_OBJ(info_libc_lib_obj, PLATFORM_LIBC_LIB); -STATIC const MP_DEFINE_STR_OBJ(info_libc_ver_obj, PLATFORM_LIBC_VER); +STATIC const MP_DEFINE_STR_OBJ(info_platform_obj, MICROPY_PLATFORM_SYSTEM "-" \ + MICROPY_VERSION_STRING "-" MICROPY_PLATFORM_ARCH "-" MICROPY_PLATFORM_VERSION "-with-" \ + MICROPY_PLATFORM_LIBC_LIB "" MICROPY_PLATFORM_LIBC_VER); +STATIC const MP_DEFINE_STR_OBJ(info_python_compiler_obj, MICROPY_PLATFORM_COMPILER); +STATIC const MP_DEFINE_STR_OBJ(info_libc_lib_obj, MICROPY_PLATFORM_LIBC_LIB); +STATIC const MP_DEFINE_STR_OBJ(info_libc_ver_obj, MICROPY_PLATFORM_LIBC_VER); STATIC const mp_rom_obj_tuple_t info_libc_tuple_obj = { {&mp_type_tuple}, 2, {MP_ROM_PTR(&info_libc_lib_obj), MP_ROM_PTR(&info_libc_ver_obj)} }; @@ -143,4 +75,6 @@ const mp_obj_module_t mp_module_uplatform = { .globals = (mp_obj_dict_t *)&modplatform_globals, }; +MP_REGISTER_MODULE(MP_QSTR_uplatform, mp_module_uplatform); + #endif // MICROPY_PY_UPLATFORM diff --git a/extmod/moduplatform.h b/extmod/moduplatform.h new file mode 100644 index 0000000000..2b9ad3ae49 --- /dev/null +++ b/extmod/moduplatform.h @@ -0,0 +1,105 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2021 Ibrahim Abdelkader + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_MODUPLATFORM_H +#define MICROPY_INCLUDED_MODUPLATFORM_H + +#include "py/misc.h" // For MP_STRINGIFY. +#include "py/mpconfig.h" + +// Preprocessor directives indentifying the platform. +// The (u)platform module itself is guarded by MICROPY_PY_UPLATFORM, see the +// .c file, but these are made available because they're generally usable. +// TODO: Add more architectures, compilers and libraries. +// See: https://sourceforge.net/p/predef/wiki/Home/ + +#if defined(__ARM_ARCH) +#define MICROPY_PLATFORM_ARCH "arm" +#elif defined(__x86_64__) || defined(_WIN64) +#define MICROPY_PLATFORM_ARCH "x86_64" +#elif defined(__i386__) || defined(_M_IX86) +#define MICROPY_PLATFORM_ARCH "x86" +#elif defined(__xtensa__) || defined(_M_IX86) +#define MICROPY_PLATFORM_ARCH "xtensa" +#else +#define MICROPY_PLATFORM_ARCH "" +#endif + +#if defined(__GNUC__) +#define MICROPY_PLATFORM_COMPILER \ + "GCC " \ + MP_STRINGIFY(__GNUC__) "." \ + MP_STRINGIFY(__GNUC_MINOR__) "." \ + MP_STRINGIFY(__GNUC_PATCHLEVEL__) +#elif defined(__ARMCC_VERSION) +#define MICROPY_PLATFORM_COMPILER \ + "ARMCC " \ + MP_STRINGIFY((__ARMCC_VERSION / 1000000)) "." \ + MP_STRINGIFY((__ARMCC_VERSION / 10000 % 100)) "." \ + MP_STRINGIFY((__ARMCC_VERSION % 10000)) +#elif defined(_MSC_VER) +#if defined(_WIN64) +#define MICROPY_PLATFORM_COMPILER_BITS "64 bit" +#elif defined(_M_IX86) +#define MICROPY_PLATFORM_COMPILER_BITS "32 bit" +#else +#define MICROPY_PLATFORM_COMPILER_BITS "" +#endif +#define MICROPY_PLATFORM_COMPILER \ + "MSC v." MP_STRINGIFY(_MSC_VER) " " MICROPY_PLATFORM_COMPILER_BITS +#else +#define MICROPY_PLATFORM_COMPILER "" +#endif + +#if defined(__GLIBC__) +#define MICROPY_PLATFORM_LIBC_LIB "glibc" +#define MICROPY_PLATFORM_LIBC_VER \ + MP_STRINGIFY(__GLIBC__) "." \ + MP_STRINGIFY(__GLIBC_MINOR__) +#elif defined(__NEWLIB__) +#define MICROPY_PLATFORM_LIBC_LIB "newlib" +#define MICROPY_PLATFORM_LIBC_VER _NEWLIB_VERSION +#else +#define MICROPY_PLATFORM_LIBC_LIB "" +#define MICROPY_PLATFORM_LIBC_VER "" +#endif + +#if defined(__linux) +#define MICROPY_PLATFORM_SYSTEM "Linux" +#elif defined(__unix__) +#define MICROPY_PLATFORM_SYSTEM "Unix" +#elif defined(__CYGWIN__) +#define MICROPY_PLATFORM_SYSTEM "Cygwin" +#elif defined(_WIN32) +#define MICROPY_PLATFORM_SYSTEM "Windows" +#else +#define MICROPY_PLATFORM_SYSTEM "MicroPython" +#endif + +#ifndef MICROPY_PLATFORM_VERSION +#define MICROPY_PLATFORM_VERSION "" +#endif + +#endif // MICROPY_INCLUDED_MODUPLATFORM_H diff --git a/extmod/modurandom.c b/extmod/modurandom.c index e304d0b728..b661627190 100644 --- a/extmod/modurandom.c +++ b/extmod/modurandom.c @@ -1,7 +1,28 @@ -// Copyright (c) 2016 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2016 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include #include @@ -195,7 +216,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_urandom_uniform_obj, mod_urandom_uniform); #endif // MICROPY_PY_URANDOM_EXTRA_FUNCS #if SEED_ON_IMPORT -STATIC mp_obj_t mod_urandom___init__() { +STATIC mp_obj_t mod_urandom___init__(void) { // This module may be imported by more than one name so need to ensure // that it's only ever seeded once. static bool seeded = false; @@ -233,6 +254,8 @@ const mp_obj_module_t mp_module_urandom = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&mp_module_urandom_globals, }; + +MP_REGISTER_MODULE(MP_QSTR_urandom, mp_module_urandom); #endif #endif // MICROPY_PY_URANDOM diff --git a/extmod/modure.c b/extmod/modure.c index c65eddb988..bac8cd4550 100644 --- a/extmod/modure.c +++ b/extmod/modure.c @@ -1,7 +1,28 @@ -// Copyright (c) 2014 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include #include @@ -18,6 +39,7 @@ #include "lib/re1.5/re1.5.h" +//CIRCUITPY #if MICROPY_PY_URE_DEBUG #define FLAG_DEBUG 0x1000 #endif @@ -168,7 +190,7 @@ STATIC mp_obj_t ure_exec(bool is_anchored, uint n_args, const mp_obj_t *args) { } Subject subj; size_t len; - subj.begin = mp_obj_str_get_data(args[1], &len); + subj.begin_line = subj.begin = mp_obj_str_get_data(args[1], &len); subj.end = subj.begin + len; #if MICROPY_PY_URE_MATCH_SPAN_START_END && !(defined(MICROPY_ENABLE_DYNRUNTIME) && MICROPY_ENABLE_DYNRUNTIME) @@ -231,7 +253,7 @@ STATIC mp_obj_t re_split(size_t n_args, const mp_obj_t *args) { Subject subj; size_t len; const mp_obj_type_t *str_type = mp_obj_get_type(args[1]); - subj.begin = mp_obj_str_get_data(args[1], &len); + subj.begin_line = subj.begin = mp_obj_str_get_data(args[1], &len); subj.end = subj.begin + len; int caps_num = (self->re.sub + 1) * 2; @@ -291,7 +313,7 @@ STATIC mp_obj_t re_sub_helper(size_t n_args, const mp_obj_t *args) { size_t where_len; const char *where_str = mp_obj_str_get_data(where, &where_len); Subject subj; - subj.begin = where_str; + subj.begin_line = subj.begin = where_str; subj.end = subj.begin + where_len; int caps_num = (self->re.sub + 1) * 2; @@ -421,8 +443,7 @@ STATIC mp_obj_t mod_re_compile(size_t n_args, const mp_obj_t *args) { if (size == -1) { goto error; } - mp_obj_re_t *o = m_new_obj_var(mp_obj_re_t, char, size); - o->base.type = &re_type; + mp_obj_re_t *o = mp_obj_malloc_var(mp_obj_re_t, char, size, &re_type); #if MICROPY_PY_URE_DEBUG int flags = 0; if (n_args > 1) { @@ -470,7 +491,7 @@ const mp_obj_module_t mp_module_ure = { .globals = (mp_obj_dict_t *)&mp_module_re_globals, }; -MP_REGISTER_MODULE(MP_QSTR_re, mp_module_ure, MICROPY_PY_URE); +MP_REGISTER_MODULE(MP_QSTR_ure, mp_module_ure); #endif // Source files #include'd here to make sure they're compiled in diff --git a/extmod/moduselect.c b/extmod/moduselect.c index d388438313..98d9042ec8 100644 --- a/extmod/moduselect.c +++ b/extmod/moduselect.c @@ -1,8 +1,29 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George -// SPDX-FileCopyrightText: Copyright (c) 2015-2017 Paul Sokolovsky -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014 Damien P. George + * Copyright (c) 2015-2017 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include "py/mpconfig.h" #if MICROPY_PY_USELECT @@ -151,6 +172,7 @@ STATIC mp_obj_t select_select(size_t n_args, const mp_obj_t *args) { mp_map_deinit(&poll_map); return mp_obj_new_tuple(3, list_array); } + // CIRCUITPY RUN_BACKGROUND_TASKS; } } @@ -230,6 +252,7 @@ STATIC mp_uint_t poll_poll_internal(uint n_args, const mp_obj_t *args) { if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_tick >= timeout)) { break; } + // CIRCUITPY RUN_BACKGROUND_TASKS; if (mp_hal_is_interrupted()) { return 0; @@ -324,6 +347,8 @@ STATIC const mp_obj_type_t mp_type_poll = { { &mp_type_type }, .flags = MP_TYPE_FLAG_EXTENDED, .name = MP_QSTR_poll, + .getiter = mp_identity_getiter, + .iternext = poll_iternext, .locals_dict = (void *)&poll_locals_dict, MP_TYPE_EXTENDED_FIELDS( .getiter = mp_identity_getiter, @@ -333,8 +358,7 @@ STATIC const mp_obj_type_t mp_type_poll = { // poll() STATIC mp_obj_t select_poll(void) { - mp_obj_poll_t *poll = m_new_obj(mp_obj_poll_t); - poll->base.type = &mp_type_poll; + mp_obj_poll_t *poll = mp_obj_malloc(mp_obj_poll_t, &mp_type_poll); mp_map_init(&poll->poll_map, 0); poll->iter_cnt = 0; poll->ret_tuple = MP_OBJ_NULL; @@ -361,6 +385,6 @@ const mp_obj_module_t mp_module_uselect = { .globals = (mp_obj_dict_t *)&mp_module_select_globals, }; -MP_REGISTER_MODULE(MP_QSTR_select, mp_module_uselect, MICROPY_PY_USELECT); +MP_REGISTER_MODULE(MP_QSTR_uselect, mp_module_uselect); #endif // MICROPY_PY_USELECT diff --git a/extmod/modutimeq.c b/extmod/modutimeq.c index fcef14ce98..7cabfee4be 100644 --- a/extmod/modutimeq.c +++ b/extmod/modutimeq.c @@ -1,8 +1,29 @@ -// Copyright (c) 2016-2017 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014 Damien P. George + * Copyright (c) 2016-2017 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include @@ -58,8 +79,7 @@ STATIC bool time_less_than(struct qentry *item, struct qentry *parent) { STATIC mp_obj_t utimeq_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 1, 1, false); mp_uint_t alloc = mp_obj_get_int(args[0]); - mp_obj_utimeq_t *o = m_new_obj_var(mp_obj_utimeq_t, struct qentry, alloc); - o->base.type = type; + mp_obj_utimeq_t *o = mp_obj_malloc_var(mp_obj_utimeq_t, struct qentry, alloc, type); memset(o->items, 0, sizeof(*o->items) * alloc); o->alloc = alloc; o->len = 0; @@ -214,4 +234,6 @@ const mp_obj_module_t mp_module_utimeq = { .globals = (mp_obj_dict_t *)&mp_module_utimeq_globals, }; +MP_REGISTER_MODULE(MP_QSTR_utimeq, mp_module_utimeq); + #endif // MICROPY_PY_UTIMEQ diff --git a/extmod/moduzlib.c b/extmod/moduzlib.c index 6a1a2d6111..5c16ea732c 100644 --- a/extmod/moduzlib.c +++ b/extmod/moduzlib.c @@ -1,7 +1,28 @@ -// Copyright (c) 2014-2016 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014-2016 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include #include @@ -51,15 +72,14 @@ STATIC int read_src_stream(TINF_DATA *data) { STATIC mp_obj_t decompio_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 1, 2, false); mp_get_stream_raise(args[0], MP_STREAM_OP_READ); - mp_obj_decompio_t *o = m_new_obj(mp_obj_decompio_t); - o->base.type = type; + mp_obj_decompio_t *o = mp_obj_malloc(mp_obj_decompio_t, type); memset(&o->decomp, 0, sizeof(o->decomp)); o->decomp.readSource = read_src_stream; o->src_stream = args[0]; o->eof = false; mp_int_t dict_opt = 0; - int dict_sz; + uint dict_sz; if (n_args > 1) { dict_opt = mp_obj_get_int(args[1]); } @@ -76,7 +96,10 @@ STATIC mp_obj_t decompio_make_new(const mp_obj_type_t *type, size_t n_args, size header_error: mp_raise_ValueError(MP_ERROR_TEXT("compression header")); } - dict_sz = 1 << dict_opt; + // RFC 1950 section 2.2: + // CINFO is the base-2 logarithm of the LZ77 window size, + // minus eight (CINFO=7 indicates a 32K window size) + dict_sz = 1 << (dict_opt + 8); } else { dict_sz = 1 << -dict_opt; } @@ -92,12 +115,13 @@ STATIC mp_uint_t decompio_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *er } o->decomp.dest = buf; - o->decomp.dest_limit = (unsigned char *)buf + size; + o->decomp.dest_limit = (byte *)buf + size; int st = uzlib_uncompress_chksum(&o->decomp); if (st == TINF_DONE) { o->eof = true; } if (st < 0) { + DEBUG_printf("uncompress error=" INT_FMT "\n", st); *errcode = MP_EINVAL; return MP_STREAM_ERROR; } @@ -146,9 +170,10 @@ STATIC mp_obj_t mod_uzlib_decompress(size_t n_args, const mp_obj_t *args) { decomp->dest = dest_buf; decomp->dest_limit = dest_buf + dest_buf_size; - DEBUG_printf("uzlib: Initial out buffer: " UINT_FMT " bytes\n", decomp->destSize); + DEBUG_printf("uzlib: Initial out buffer: " UINT_FMT " bytes\n", dest_buf_size); decomp->source = bufinfo.buf; - decomp->source_limit = (unsigned char *)bufinfo.buf + bufinfo.len; + decomp->source_limit = (byte *)bufinfo.buf + bufinfo.len; + int st; bool is_zlib = true; @@ -175,7 +200,7 @@ STATIC mp_obj_t mod_uzlib_decompress(size_t n_args, const mp_obj_t *args) { dest_buf = m_renew(byte, dest_buf, dest_buf_size, dest_buf_size + 256); dest_buf_size += 256; decomp->dest = dest_buf + offset; - decomp->dest_limit = dest_buf + offset + 256; + decomp->dest_limit = decomp->dest + 256; } mp_uint_t final_sz = decomp->dest - dest_buf; @@ -203,6 +228,9 @@ const mp_obj_module_t mp_module_uzlib = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&mp_module_uzlib_globals, }; + + +MP_REGISTER_MODULE(MP_QSTR_uzlib, mp_module_uzlib); #endif // Source files #include'd here to make sure they're compiled in diff --git a/extmod/uasyncio/__init__.py b/extmod/uasyncio/__init__.py deleted file mode 100644 index fa64438f6b..0000000000 --- a/extmod/uasyncio/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# MicroPython uasyncio module -# MIT license; Copyright (c) 2019 Damien P. George - -from .core import * - -__version__ = (3, 0, 0) - -_attrs = { - "wait_for": "funcs", - "wait_for_ms": "funcs", - "gather": "funcs", - "Event": "event", - "ThreadSafeFlag": "event", - "Lock": "lock", - "open_connection": "stream", - "start_server": "stream", - "StreamReader": "stream", - "StreamWriter": "stream", -} - -# Lazy loader, effectively does: -# global attr -# from .mod import attr -def __getattr__(attr): - mod = _attrs.get(attr, None) - if mod is None: - raise AttributeError(attr) - value = getattr(__import__(mod, None, None, True, 1), attr) - globals()[attr] = value - return value diff --git a/extmod/uasyncio/core.py b/extmod/uasyncio/core.py deleted file mode 100644 index c3ce3ccafa..0000000000 --- a/extmod/uasyncio/core.py +++ /dev/null @@ -1,295 +0,0 @@ -# MicroPython uasyncio module -# MIT license; Copyright (c) 2019 Damien P. George - -from time import ticks_ms as ticks, ticks_diff, ticks_add -import sys, select - -# Import TaskQueue and Task, preferring built-in C code over Python code -try: - from _uasyncio import TaskQueue, Task -except: - from .task import TaskQueue, Task - - -################################################################################ -# Exceptions - - -class CancelledError(BaseException): - pass - - -class TimeoutError(Exception): - pass - - -# Used when calling Loop.call_exception_handler -_exc_context = {"message": "Task exception wasn't retrieved", "exception": None, "future": None} - - -################################################################################ -# Sleep functions - -# "Yield" once, then raise StopIteration -class SingletonGenerator: - def __init__(self): - self.state = None - self.exc = StopIteration() - - def __await__(self): - return self - - def __next__(self): - if self.state is not None: - _task_queue.push_sorted(cur_task, self.state) - self.state = None - return None - else: - self.exc.__traceback__ = None - raise self.exc - - -# Pause task execution for the given time (integer in milliseconds, uPy extension) -# Use a SingletonGenerator to do it without allocating on the heap -def sleep_ms(t, sgen=SingletonGenerator()): - assert sgen.state is None - sgen.state = ticks_add(ticks(), max(0, t)) - return sgen - - -# Pause task execution for the given time (in seconds) -def sleep(t): - return sleep_ms(int(t * 1000)) - - -################################################################################ -# Queue and poller for stream IO - - -class IOQueue: - def __init__(self): - self.poller = select.poll() - self.map = {} # maps id(stream) to [task_waiting_read, task_waiting_write, stream] - - def _enqueue(self, s, idx): - if id(s) not in self.map: - entry = [None, None, s] - entry[idx] = cur_task - self.map[id(s)] = entry - self.poller.register(s, select.POLLIN if idx == 0 else select.POLLOUT) - else: - sm = self.map[id(s)] - assert sm[idx] is None - assert sm[1 - idx] is not None - sm[idx] = cur_task - self.poller.modify(s, select.POLLIN | select.POLLOUT) - # Link task to this IOQueue so it can be removed if needed - cur_task.data = self - - def _dequeue(self, s): - del self.map[id(s)] - self.poller.unregister(s) - - def queue_read(self, s): - self._enqueue(s, 0) - - def queue_write(self, s): - self._enqueue(s, 1) - - def remove(self, task): - while True: - del_s = None - for k in self.map: # Iterate without allocating on the heap - q0, q1, s = self.map[k] - if q0 is task or q1 is task: - del_s = s - break - if del_s is not None: - self._dequeue(s) - else: - break - - def wait_io_event(self, dt): - for s, ev in self.poller.ipoll(dt): - sm = self.map[id(s)] - # print('poll', s, sm, ev) - if ev & ~select.POLLOUT and sm[0] is not None: - # POLLIN or error - _task_queue.push_head(sm[0]) - sm[0] = None - if ev & ~select.POLLIN and sm[1] is not None: - # POLLOUT or error - _task_queue.push_head(sm[1]) - sm[1] = None - if sm[0] is None and sm[1] is None: - self._dequeue(s) - elif sm[0] is None: - self.poller.modify(s, select.POLLOUT) - else: - self.poller.modify(s, select.POLLIN) - - -################################################################################ -# Main run loop - -# Ensure the awaitable is a task -def _promote_to_task(aw): - return aw if isinstance(aw, Task) else create_task(aw) - - -# Create and schedule a new task from a coroutine -def create_task(coro): - if not hasattr(coro, "send"): - raise TypeError("coroutine expected") - t = Task(coro, globals()) - _task_queue.push_head(t) - return t - - -# Keep scheduling tasks until there are none left to schedule -def run_until_complete(main_task=None): - global cur_task - excs_all = (CancelledError, Exception) # To prevent heap allocation in loop - excs_stop = (CancelledError, StopIteration) # To prevent heap allocation in loop - while True: - # Wait until the head of _task_queue is ready to run - dt = 1 - while dt > 0: - dt = -1 - t = _task_queue.peek() - if t: - # A task waiting on _task_queue; "ph_key" is time to schedule task at - dt = max(0, ticks_diff(t.ph_key, ticks())) - elif not _io_queue.map: - # No tasks can be woken so finished running - return - # print('(poll {})'.format(dt), len(_io_queue.map)) - _io_queue.wait_io_event(dt) - - # Get next task to run and continue it - t = _task_queue.pop_head() - cur_task = t - try: - # Continue running the coroutine, it's responsible for rescheduling itself - exc = t.data - if not exc: - t.coro.send(None) - else: - # If the task is finished and on the run queue and gets here, then it - # had an exception and was not await'ed on. Throwing into it now will - # raise StopIteration and the code below will catch this and run the - # call_exception_handler function. - t.data = None - t.coro.throw(exc) - except excs_all as er: - # Check the task is not on any event queue - assert t.data is None - # This task is done, check if it's the main task and then loop should stop - if t is main_task: - if isinstance(er, StopIteration): - return er.value - raise er - if t.state: - # Task was running but is now finished. - waiting = False - if t.state is True: - # "None" indicates that the task is complete and not await'ed on (yet). - t.state = None - else: - # Schedule any other tasks waiting on the completion of this task. - while t.state.peek(): - _task_queue.push_head(t.state.pop_head()) - waiting = True - # "False" indicates that the task is complete and has been await'ed on. - t.state = False - if not waiting and not isinstance(er, excs_stop): - # An exception ended this detached task, so queue it for later - # execution to handle the uncaught exception if no other task retrieves - # the exception in the meantime (this is handled by Task.throw). - _task_queue.push_head(t) - # Save return value of coro to pass up to caller. - t.data = er - elif t.state is None: - # Task is already finished and nothing await'ed on the task, - # so call the exception handler. - _exc_context["exception"] = exc - _exc_context["future"] = t - Loop.call_exception_handler(_exc_context) - - -# Create a new task from a coroutine and run it until it finishes -def run(coro): - return run_until_complete(create_task(coro)) - - -################################################################################ -# Event loop wrapper - - -async def _stopper(): - pass - - -_stop_task = None - - -class Loop: - _exc_handler = None - - def create_task(coro): - return create_task(coro) - - def run_forever(): - global _stop_task - _stop_task = Task(_stopper(), globals()) - run_until_complete(_stop_task) - # TODO should keep running until .stop() is called, even if there're no tasks left - - def run_until_complete(aw): - return run_until_complete(_promote_to_task(aw)) - - def stop(): - global _stop_task - if _stop_task is not None: - _task_queue.push_head(_stop_task) - # If stop() is called again, do nothing - _stop_task = None - - def close(): - pass - - def set_exception_handler(handler): - Loop._exc_handler = handler - - def get_exception_handler(): - return Loop._exc_handler - - def default_exception_handler(loop, context): - print(context["message"]) - print("future:", context["future"], "coro=", context["future"].coro) - sys.print_exception(context["exception"]) - - def call_exception_handler(context): - (Loop._exc_handler or Loop.default_exception_handler)(Loop, context) - - -# The runq_len and waitq_len arguments are for legacy uasyncio compatibility -def get_event_loop(runq_len=0, waitq_len=0): - return Loop - - -def current_task(): - return cur_task - - -def new_event_loop(): - global _task_queue, _io_queue - # TaskQueue of Task instances - _task_queue = TaskQueue() - # Task queue and poller for stream IO - _io_queue = IOQueue() - return Loop - - -# Initialise default event loop -new_event_loop() diff --git a/extmod/uasyncio/event.py b/extmod/uasyncio/event.py deleted file mode 100644 index a5b3bf9f47..0000000000 --- a/extmod/uasyncio/event.py +++ /dev/null @@ -1,33 +0,0 @@ -# MicroPython uasyncio module -# MIT license; Copyright (c) 2019-2020 Damien P. George - -from . import core - -# Event class for primitive events that can be waited on, set, and cleared -class Event: - def __init__(self): - self.state = False # False=unset; True=set - self.waiting = core.TaskQueue() # Queue of Tasks waiting on completion of this event - - def is_set(self): - return self.state - - def set(self): - # Event becomes set, schedule any tasks waiting on it - # Note: This must not be called from anything except the thread running - # the asyncio loop (i.e. neither hard or soft IRQ, or a different thread). - while self.waiting.peek(): - core._task_queue.push_head(self.waiting.pop_head()) - self.state = True - - def clear(self): - self.state = False - - async def wait(self): - if not self.state: - # Event not set, put the calling task on the event's waiting queue - self.waiting.push_head(core.cur_task) - # Set calling task's data to the event's queue so it can be removed if needed - core.cur_task.data = self.waiting - yield - return True diff --git a/extmod/uasyncio/funcs.py b/extmod/uasyncio/funcs.py deleted file mode 100644 index 0ce48b015c..0000000000 --- a/extmod/uasyncio/funcs.py +++ /dev/null @@ -1,74 +0,0 @@ -# MicroPython uasyncio module -# MIT license; Copyright (c) 2019-2020 Damien P. George - -from . import core - - -async def wait_for(aw, timeout, sleep=core.sleep): - aw = core._promote_to_task(aw) - if timeout is None: - return await aw - - def runner(waiter, aw): - nonlocal status, result - try: - result = await aw - s = True - except BaseException as er: - s = er - if status is None: - # The waiter is still waiting, set status for it and cancel it. - status = s - waiter.cancel() - - # Run aw in a separate runner task that manages its exceptions. - status = None - result = None - runner_task = core.create_task(runner(core.cur_task, aw)) - - try: - # Wait for the timeout to elapse. - await sleep(timeout) - except core.CancelledError as er: - if status is True: - # aw completed successfully and cancelled the sleep, so return aw's result. - return result - elif status is None: - # This wait_for was cancelled externally, so cancel aw and re-raise. - status = True - runner_task.cancel() - raise er - else: - # aw raised an exception, propagate it out to the caller. - raise status - - # The sleep finished before aw, so cancel aw and raise TimeoutError. - status = True - runner_task.cancel() - await runner_task - raise core.TimeoutError - - -def wait_for_ms(aw, timeout): - return wait_for(aw, timeout, core.sleep_ms) - - -async def gather(*aws, return_exceptions=False): - ts = [core._promote_to_task(aw) for aw in aws] - for i in range(len(ts)): - try: - # TODO handle cancel of gather itself - # if ts[i].coro: - # iter(ts[i]).waiting.push_head(cur_task) - # try: - # yield - # except CancelledError as er: - # # cancel all waiting tasks - # raise er - ts[i] = await ts[i] - except (core.CancelledError, Exception) as er: - if return_exceptions: - ts[i] = er - else: - raise er - return ts diff --git a/extmod/uasyncio/lock.py b/extmod/uasyncio/lock.py deleted file mode 100644 index bddca295b6..0000000000 --- a/extmod/uasyncio/lock.py +++ /dev/null @@ -1,53 +0,0 @@ -# MicroPython uasyncio module -# MIT license; Copyright (c) 2019-2020 Damien P. George - -from . import core - -# Lock class for primitive mutex capability -class Lock: - def __init__(self): - # The state can take the following values: - # - 0: unlocked - # - 1: locked - # - : unlocked but this task has been scheduled to acquire the lock next - self.state = 0 - # Queue of Tasks waiting to acquire this Lock - self.waiting = core.TaskQueue() - - def locked(self): - return self.state == 1 - - def release(self): - if self.state != 1: - raise RuntimeError("Lock not acquired") - if self.waiting.peek(): - # Task(s) waiting on lock, schedule next Task - self.state = self.waiting.pop_head() - core._task_queue.push_head(self.state) - else: - # No Task waiting so unlock - self.state = 0 - - async def acquire(self): - if self.state != 0: - # Lock unavailable, put the calling Task on the waiting queue - self.waiting.push_head(core.cur_task) - # Set calling task's data to the lock's queue so it can be removed if needed - core.cur_task.data = self.waiting - try: - yield - except core.CancelledError as er: - if self.state == core.cur_task: - # Cancelled while pending on resume, schedule next waiting Task - self.state = 1 - self.release() - raise er - # Lock available, set it as locked - self.state = 1 - return True - - async def __aenter__(self): - return await self.acquire() - - async def __aexit__(self, exc_type, exc, tb): - return self.release() diff --git a/extmod/uasyncio/manifest.py b/extmod/uasyncio/manifest.py deleted file mode 100644 index f5fa27bfca..0000000000 --- a/extmod/uasyncio/manifest.py +++ /dev/null @@ -1,13 +0,0 @@ -# This list of frozen files doesn't include task.py because that's provided by the C module. -freeze( - "..", - ( - "uasyncio/__init__.py", - "uasyncio/core.py", - "uasyncio/event.py", - "uasyncio/funcs.py", - "uasyncio/lock.py", - "uasyncio/stream.py", - ), - opt=3, -) diff --git a/extmod/uasyncio/stream.py b/extmod/uasyncio/stream.py deleted file mode 100644 index af3b8feab3..0000000000 --- a/extmod/uasyncio/stream.py +++ /dev/null @@ -1,164 +0,0 @@ -# MicroPython uasyncio module -# MIT license; Copyright (c) 2019-2020 Damien P. George - -from . import core - - -class Stream: - def __init__(self, s, e={}): - self.s = s - self.e = e - self.out_buf = b"" - - def get_extra_info(self, v): - return self.e[v] - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc, tb): - await self.close() - - def close(self): - pass - - async def wait_closed(self): - # TODO yield? - self.s.close() - - async def read(self, n): - yield core._io_queue.queue_read(self.s) - return self.s.read(n) - - async def readinto(self, buf): - yield core._io_queue.queue_read(self.s) - return self.s.readinto(buf) - - async def readexactly(self, n): - r = b"" - while n: - yield core._io_queue.queue_read(self.s) - r2 = self.s.read(n) - if r2 is not None: - if not len(r2): - raise EOFError - r += r2 - n -= len(r2) - return r - - async def readline(self): - l = b"" - while True: - yield core._io_queue.queue_read(self.s) - l2 = self.s.readline() # may do multiple reads but won't block - l += l2 - if not l2 or l[-1] == 10: # \n (check l in case l2 is str) - return l - - def write(self, buf): - self.out_buf += buf - - async def drain(self): - mv = memoryview(self.out_buf) - off = 0 - while off < len(mv): - yield core._io_queue.queue_write(self.s) - ret = self.s.write(mv[off:]) - if ret is not None: - off += ret - self.out_buf = b"" - - -# Stream can be used for both reading and writing to save code size -StreamReader = Stream -StreamWriter = Stream - - -# Create a TCP stream connection to a remote host -async def open_connection(host, port): - from uerrno import EINPROGRESS - import usocket as socket - - ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)[0] # TODO this is blocking! - s = socket.socket(ai[0], ai[1], ai[2]) - s.setblocking(False) - ss = Stream(s) - try: - s.connect(ai[-1]) - except OSError as er: - if er.errno != EINPROGRESS: - raise er - yield core._io_queue.queue_write(s) - return ss, ss - - -# Class representing a TCP stream server, can be closed and used in "async with" -class Server: - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc, tb): - self.close() - await self.wait_closed() - - def close(self): - self.task.cancel() - - async def wait_closed(self): - await self.task - - async def _serve(self, s, cb): - # Accept incoming connections - while True: - try: - yield core._io_queue.queue_read(s) - except core.CancelledError: - # Shutdown server - s.close() - return - try: - s2, addr = s.accept() - except: - # Ignore a failed accept - continue - s2.setblocking(False) - s2s = Stream(s2, {"peername": addr}) - core.create_task(cb(s2s, s2s)) - - -# Helper function to start a TCP stream server, running as a new task -# TODO could use an accept-callback on socket read activity instead of creating a task -async def start_server(cb, host, port, backlog=5): - import usocket as socket - - # Create and bind server socket. - host = socket.getaddrinfo(host, port)[0] # TODO this is blocking! - s = socket.socket() - s.setblocking(False) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - s.bind(host[-1]) - s.listen(backlog) - - # Create and return server object and task. - srv = Server() - srv.task = core.create_task(srv._serve(s, cb)) - return srv - - -################################################################################ -# Legacy uasyncio compatibility - - -async def stream_awrite(self, buf, off=0, sz=-1): - if off != 0 or sz != -1: - buf = memoryview(buf) - if sz == -1: - sz = len(buf) - buf = buf[off : off + sz] - self.write(buf) - await self.drain() - - -Stream.aclose = Stream.wait_closed -Stream.awrite = stream_awrite -Stream.awritestr = stream_awrite # TODO explicitly convert to bytes? diff --git a/extmod/uasyncio/task.py b/extmod/uasyncio/task.py deleted file mode 100644 index 7a80b25208..0000000000 --- a/extmod/uasyncio/task.py +++ /dev/null @@ -1,179 +0,0 @@ -# MicroPython uasyncio module -# MIT license; Copyright (c) 2019-2020 Damien P. George - -# This file contains the core TaskQueue based on a pairing heap, and the core Task class. -# They can optionally be replaced by C implementations. - -from . import core - - -# pairing-heap meld of 2 heaps; O(1) -def ph_meld(h1, h2): - if h1 is None: - return h2 - if h2 is None: - return h1 - lt = core.ticks_diff(h1.ph_key, h2.ph_key) < 0 - if lt: - if h1.ph_child is None: - h1.ph_child = h2 - else: - h1.ph_child_last.ph_next = h2 - h1.ph_child_last = h2 - h2.ph_next = None - h2.ph_rightmost_parent = h1 - return h1 - else: - h1.ph_next = h2.ph_child - h2.ph_child = h1 - if h1.ph_next is None: - h2.ph_child_last = h1 - h1.ph_rightmost_parent = h2 - return h2 - - -# pairing-heap pairing operation; amortised O(log N) -def ph_pairing(child): - heap = None - while child is not None: - n1 = child - child = child.ph_next - n1.ph_next = None - if child is not None: - n2 = child - child = child.ph_next - n2.ph_next = None - n1 = ph_meld(n1, n2) - heap = ph_meld(heap, n1) - return heap - - -# pairing-heap delete of a node; stable, amortised O(log N) -def ph_delete(heap, node): - if node is heap: - child = heap.ph_child - node.ph_child = None - return ph_pairing(child) - # Find parent of node - parent = node - while parent.ph_next is not None: - parent = parent.ph_next - parent = parent.ph_rightmost_parent - # Replace node with pairing of its children - if node is parent.ph_child and node.ph_child is None: - parent.ph_child = node.ph_next - node.ph_next = None - return heap - elif node is parent.ph_child: - child = node.ph_child - next = node.ph_next - node.ph_child = None - node.ph_next = None - node = ph_pairing(child) - parent.ph_child = node - else: - n = parent.ph_child - while node is not n.ph_next: - n = n.ph_next - child = node.ph_child - next = node.ph_next - node.ph_child = None - node.ph_next = None - node = ph_pairing(child) - if node is None: - node = n - else: - n.ph_next = node - node.ph_next = next - if next is None: - node.ph_rightmost_parent = parent - parent.ph_child_last = node - return heap - - -# TaskQueue class based on the above pairing-heap functions. -class TaskQueue: - def __init__(self): - self.heap = None - - def peek(self): - return self.heap - - def push_sorted(self, v, key): - v.data = None - v.ph_key = key - v.ph_child = None - v.ph_next = None - self.heap = ph_meld(v, self.heap) - - def push_head(self, v): - self.push_sorted(v, core.ticks()) - - def pop_head(self): - v = self.heap - self.heap = ph_pairing(self.heap.ph_child) - return v - - def remove(self, v): - self.heap = ph_delete(self.heap, v) - - -# Task class representing a coroutine, can be waited on and cancelled. -class Task: - def __init__(self, coro, globals=None): - self.coro = coro # Coroutine of this Task - self.data = None # General data for queue it is waiting on - self.state = True # None, False, True or a TaskQueue instance - self.ph_key = 0 # Pairing heap - self.ph_child = None # Paring heap - self.ph_child_last = None # Paring heap - self.ph_next = None # Paring heap - self.ph_rightmost_parent = None # Paring heap - - def __await__(self): - if not self.state: - # Task finished, signal that is has been await'ed on. - self.state = False - elif self.state is True: - # Allocated head of linked list of Tasks waiting on completion of this task. - self.state = TaskQueue() - return self - - def __next__(self): - if not self.state: - if self.data is None: - # Task finished but has already been sent to the loop's exception handler. - raise StopIteration - else: - # Task finished, raise return value to caller so it can continue. - raise self.data - else: - # Put calling task on waiting queue. - self.state.push_head(core.cur_task) - # Set calling task's data to this task that it waits on, to double-link it. - core.cur_task.data = self - - def done(self): - return not self.state - - def cancel(self): - # Check if task is already finished. - if not self.state: - return False - # Can't cancel self (not supported yet). - if self is core.cur_task: - raise RuntimeError("can't cancel self") - # If Task waits on another task then forward the cancel to the one it's waiting on. - while isinstance(self.data, Task): - self = self.data - # Reschedule Task as a cancelled task. - if hasattr(self.data, "remove"): - # Not on the main running queue, remove the task from the queue it's on. - self.data.remove(self) - core._task_queue.push_head(self) - elif core.ticks_diff(self.ph_key, core.ticks()) > 0: - # On the main running queue but scheduled in the future, so bring it forward to now. - core._task_queue.remove(self) - core._task_queue.push_head(self) - self.data = core.CancelledError - return True diff --git a/extmod/utime_mphal.c b/extmod/utime_mphal.c index 0d6437a920..3d1cdfd820 100644 --- a/extmod/utime_mphal.c +++ b/extmod/utime_mphal.c @@ -1,8 +1,29 @@ -// Copyright (c) 2016 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2013-2016 Damien P. George -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2016 Damien P. George + * Copyright (c) 2016 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include "py/mpconfig.h" #if MICROPY_PY_UTIME_MP_HAL diff --git a/extmod/utime_mphal.h b/extmod/utime_mphal.h index 61a851cec0..57fc348832 100644 --- a/extmod/utime_mphal.h +++ b/extmod/utime_mphal.h @@ -1,9 +1,29 @@ -// Copyright (c) 2016 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2013-2016 Damien P. George -// -// SPDX-License-Identifier: MIT - +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2016 Damien P. George + * Copyright (c) 2016 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #ifndef MICROPY_INCLUDED_EXTMOD_UTIME_MPHAL_H #define MICROPY_INCLUDED_EXTMOD_UTIME_MPHAL_H diff --git a/extmod/vfs.c b/extmod/vfs.c index 7aa82a6082..61d85b293b 100644 --- a/extmod/vfs.c +++ b/extmod/vfs.c @@ -1,7 +1,28 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2017 Damien P. George -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include #include @@ -386,8 +407,7 @@ mp_obj_t mp_vfs_ilistdir(size_t n_args, const mp_obj_t *args) { if (vfs == MP_VFS_ROOT) { // list the root directory - mp_vfs_ilistdir_it_t *iter = m_new_obj(mp_vfs_ilistdir_it_t); - iter->base.type = &mp_type_polymorph_iter; + mp_vfs_ilistdir_it_t *iter = mp_obj_malloc(mp_vfs_ilistdir_it_t, &mp_type_polymorph_iter); iter->iternext = mp_vfs_ilistdir_it_iternext; iter->cur.vfs = MP_STATE_VM(vfs_mount_table); iter->is_str = mp_obj_get_type(path_in) == &mp_type_str; diff --git a/extmod/vfs.h b/extmod/vfs.h index abb563e9a9..49c6d87e58 100644 --- a/extmod/vfs.h +++ b/extmod/vfs.h @@ -1,12 +1,32 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2017 Damien P. George -// -// SPDX-License-Identifier: MIT - +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #ifndef MICROPY_INCLUDED_EXTMOD_VFS_H #define MICROPY_INCLUDED_EXTMOD_VFS_H -#include "py/lexer.h" +#include "py/builtin.h" #include "py/obj.h" #include "py/proto.h" diff --git a/extmod/vfs_fat.c b/extmod/vfs_fat.c index 0a9737e53a..4d4e7e881a 100644 --- a/extmod/vfs_fat.c +++ b/extmod/vfs_fat.c @@ -1,8 +1,29 @@ -// Copyright (c) 2016 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014 Damien P. George + * Copyright (c) 2016 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include "py/mpconfig.h" #if MICROPY_VFS_FAT @@ -54,8 +75,7 @@ STATIC mp_obj_t fat_vfs_make_new(const mp_obj_type_t *type, size_t n_args, size_ mp_arg_check_num(n_args, n_kw, 1, 1, false); // create new object - fs_user_mount_t *vfs = m_new_obj(fs_user_mount_t); - vfs->base.type = type; + fs_user_mount_t *vfs = mp_obj_malloc(fs_user_mount_t, type); vfs->fatfs.drv = vfs; // Initialise underlying block device @@ -171,8 +191,7 @@ STATIC mp_obj_t fat_vfs_ilistdir_func(size_t n_args, const mp_obj_t *args) { } // Create a new iterator object to list the dir - mp_vfs_fat_ilistdir_it_t *iter = m_new_obj(mp_vfs_fat_ilistdir_it_t); - iter->base.type = &mp_type_polymorph_iter; + mp_vfs_fat_ilistdir_it_t *iter = mp_obj_malloc(mp_vfs_fat_ilistdir_it_t, &mp_type_polymorph_iter); iter->iternext = mp_vfs_fat_ilistdir_it_iternext; iter->is_str = is_str_type; FRESULT res = f_opendir(&self->fatfs, &iter->dir, path); diff --git a/extmod/vfs_fat.h b/extmod/vfs_fat.h index d64aaf30a2..cd003e2fe1 100644 --- a/extmod/vfs_fat.h +++ b/extmod/vfs_fat.h @@ -1,8 +1,28 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George -// -// SPDX-License-Identifier: MIT - +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013, 2014 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #ifndef MICROPY_INCLUDED_EXTMOD_VFS_FAT_H #define MICROPY_INCLUDED_EXTMOD_VFS_FAT_H diff --git a/extmod/vfs_fat_diskio.c b/extmod/vfs_fat_diskio.c index 9561a6740b..1bcd471f21 100644 --- a/extmod/vfs_fat_diskio.c +++ b/extmod/vfs_fat_diskio.c @@ -1,7 +1,31 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * Original template for this file comes from: + * Low level disk I/O module skeleton for FatFs, (C)ChaN, 2013 + * + * The MIT License (MIT) + * + * Copyright (c) 2013, 2014 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include "py/mpconfig.h" #if MICROPY_VFS && MICROPY_VFS_FAT diff --git a/extmod/vfs_fat_file.c b/extmod/vfs_fat_file.c index 2415f7d3c1..6abe638780 100644 --- a/extmod/vfs_fat_file.c +++ b/extmod/vfs_fat_file.c @@ -1,7 +1,28 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013, 2014 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include "py/mpconfig.h" #if MICROPY_VFS && MICROPY_VFS_FAT @@ -136,6 +157,7 @@ STATIC const mp_arg_t file_open_args[] = { }; #define FILE_OPEN_NUM_ARGS MP_ARRAY_SIZE(file_open_args) +// CIRCUITPY is more careful about validating the open mode. STATIC mp_obj_t file_open(fs_user_mount_t *vfs, const mp_obj_type_t *type, mp_arg_val_t *args) { int mode = 0; const char *mode_s = mp_obj_str_get_str(args[1].u_obj); diff --git a/extmod/vfs_lfsx.c b/extmod/vfs_lfsx.c index 872d7f8428..0cd3295faf 100644 --- a/extmod/vfs_lfsx.c +++ b/extmod/vfs_lfsx.c @@ -203,8 +203,7 @@ STATIC mp_obj_t MP_VFS_LFSx(ilistdir_func)(size_t n_args, const mp_obj_t *args) path = vstr_null_terminated_str(&self->cur_dir); } - MP_VFS_LFSx(ilistdir_it_t) * iter = m_new_obj(MP_VFS_LFSx(ilistdir_it_t)); - iter->base.type = &mp_type_polymorph_iter; + MP_VFS_LFSx(ilistdir_it_t) * iter = mp_obj_malloc(MP_VFS_LFSx(ilistdir_it_t), &mp_type_polymorph_iter); iter->iternext = MP_VFS_LFSx(ilistdir_it_iternext); iter->is_str = is_str_type; iter->vfs = self; diff --git a/extmod/vfs_posix.c b/extmod/vfs_posix.c index 1d0aa09664..4e73acea6f 100644 --- a/extmod/vfs_posix.c +++ b/extmod/vfs_posix.c @@ -1,7 +1,28 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2017-2018 Damien P. George -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017-2018 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include "py/runtime.h" #include "py/mperrno.h" @@ -16,6 +37,9 @@ #include #include #include +#ifdef _MSC_VER +#include // For mkdir etc. +#endif typedef struct _mp_obj_vfs_posix_t { mp_obj_base_t base; @@ -74,8 +98,7 @@ STATIC mp_import_stat_t mp_vfs_posix_import_stat(void *self_in, const char *path STATIC mp_obj_t vfs_posix_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, 1, false); - mp_obj_vfs_posix_t *vfs = m_new_obj(mp_obj_vfs_posix_t); - vfs->base.type = type; + mp_obj_vfs_posix_t *vfs = mp_obj_malloc(mp_obj_vfs_posix_t, type); vstr_init(&vfs->root, 0); if (n_args == 1) { vstr_add_str(&vfs->root, mp_obj_str_get_str(args[0])); @@ -205,8 +228,7 @@ STATIC mp_obj_t vfs_posix_ilistdir_it_iternext(mp_obj_t self_in) { STATIC mp_obj_t vfs_posix_ilistdir(mp_obj_t self_in, mp_obj_t path_in) { mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); - vfs_posix_ilistdir_it_t *iter = m_new_obj(vfs_posix_ilistdir_it_t); - iter->base.type = &mp_type_polymorph_iter; + vfs_posix_ilistdir_it_t *iter = mp_obj_malloc(vfs_posix_ilistdir_it_t, &mp_type_polymorph_iter); iter->iternext = vfs_posix_ilistdir_it_iternext; iter->is_str = mp_obj_get_type(path_in) == &mp_type_str; const char *path = vfs_posix_get_path_str(self, path_in); @@ -233,7 +255,11 @@ STATIC mp_obj_t vfs_posix_mkdir(mp_obj_t self_in, mp_obj_t path_in) { mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); const char *path = vfs_posix_get_path_str(self, path_in); MP_THREAD_GIL_EXIT(); + #ifdef _WIN32 + int ret = mkdir(path); + #else int ret = mkdir(path, 0777); + #endif MP_THREAD_GIL_ENTER(); if (ret != 0) { mp_raise_OSError(errno); @@ -287,6 +313,8 @@ STATIC mp_obj_t vfs_posix_stat(mp_obj_t self_in, mp_obj_t path_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_stat_obj, vfs_posix_stat); +#if MICROPY_PY_UOS_STATVFS + #ifdef __ANDROID__ #define USE_STATFS 1 #endif @@ -328,6 +356,8 @@ STATIC mp_obj_t vfs_posix_statvfs(mp_obj_t self_in, mp_obj_t path_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_statvfs_obj, vfs_posix_statvfs); +#endif + STATIC const mp_rom_map_elem_t vfs_posix_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&vfs_posix_mount_obj) }, { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&vfs_posix_umount_obj) }, @@ -341,7 +371,9 @@ STATIC const mp_rom_map_elem_t vfs_posix_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_rename), MP_ROM_PTR(&vfs_posix_rename_obj) }, { MP_ROM_QSTR(MP_QSTR_rmdir), MP_ROM_PTR(&vfs_posix_rmdir_obj) }, { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&vfs_posix_stat_obj) }, + #if MICROPY_PY_UOS_STATVFS { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&vfs_posix_statvfs_obj) }, + #endif }; STATIC MP_DEFINE_CONST_DICT(vfs_posix_locals_dict, vfs_posix_locals_dict_table); diff --git a/extmod/vfs_posix.h b/extmod/vfs_posix.h index 4540baedf0..32299b8269 100644 --- a/extmod/vfs_posix.h +++ b/extmod/vfs_posix.h @@ -1,8 +1,28 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2018 Damien P. George -// -// SPDX-License-Identifier: MIT - +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2018 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #ifndef MICROPY_INCLUDED_EXTMOD_VFS_POSIX_H #define MICROPY_INCLUDED_EXTMOD_VFS_POSIX_H diff --git a/extmod/vfs_posix_file.c b/extmod/vfs_posix_file.c index 94ddbe5f7b..534b11c08c 100644 --- a/extmod/vfs_posix_file.c +++ b/extmod/vfs_posix_file.c @@ -1,7 +1,28 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2013-2018 Damien P. George -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include "py/mphal.h" #include "py/mpthread.h" diff --git a/extmod/vfs_reader.c b/extmod/vfs_reader.c index c87fcd76fd..d3904c5c50 100644 --- a/extmod/vfs_reader.c +++ b/extmod/vfs_reader.c @@ -1,7 +1,28 @@ -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// SPDX-FileCopyrightText: Copyright (c) 2013-2017 Damien P. George -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2017 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include #include diff --git a/extmod/virtpin.c b/extmod/virtpin.c index e19c05c812..510ba8dbaf 100644 --- a/extmod/virtpin.c +++ b/extmod/virtpin.c @@ -1,7 +1,28 @@ -// SPDX-FileCopyrightText: Copyright (c) 2016 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2016 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include "extmod/virtpin.h" #include "py/proto.h" diff --git a/extmod/virtpin.h b/extmod/virtpin.h index 9b43406bd1..591249e48d 100644 --- a/extmod/virtpin.h +++ b/extmod/virtpin.h @@ -1,8 +1,28 @@ -// Copyright (c) 2016 Paul Sokolovsky -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// -// SPDX-License-Identifier: MIT - +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2016 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #ifndef MICROPY_INCLUDED_EXTMOD_VIRTPIN_H #define MICROPY_INCLUDED_EXTMOD_VIRTPIN_H diff --git a/lib/littlefs/lfs2.c b/lib/littlefs/lfs2.c index 39bd2f0d1c..534e4a1a65 100644 --- a/lib/littlefs/lfs2.c +++ b/lib/littlefs/lfs2.c @@ -1,16 +1,33 @@ /* * The little filesystem * + * Copyright (c) 2022, The littlefs authors. * Copyright (c) 2017, Arm Limited. All rights reserved. * SPDX-License-Identifier: BSD-3-Clause */ #include "lfs2.h" #include "lfs2_util.h" + +// some constants used throughout the code #define LFS2_BLOCK_NULL ((lfs2_block_t)-1) #define LFS2_BLOCK_INLINE ((lfs2_block_t)-2) +enum { + LFS2_OK_RELOCATED = 1, + LFS2_OK_DROPPED = 2, + LFS2_OK_ORPHANED = 3, +}; + +enum { + LFS2_CMP_EQ = 0, + LFS2_CMP_LT = 1, + LFS2_CMP_GT = 2, +}; + + /// Caching block device operations /// + static inline void lfs2_cache_drop(lfs2_t *lfs2, lfs2_cache_t *rcache) { // do not zero, cheaper if cache is readonly or only going to be // written with identical data (during relocates) @@ -107,12 +124,6 @@ static int lfs2_bd_read(lfs2_t *lfs2, return 0; } -enum { - LFS2_CMP_EQ = 0, - LFS2_CMP_LT = 1, - LFS2_CMP_GT = 2, -}; - static int lfs2_bd_cmp(lfs2_t *lfs2, const lfs2_cache_t *pcache, lfs2_cache_t *rcache, lfs2_size_t hint, lfs2_block_t block, lfs2_off_t off, @@ -268,22 +279,26 @@ static inline int lfs2_pair_cmp( paira[0] == pairb[1] || paira[1] == pairb[0]); } +#ifndef LFS2_READONLY static inline bool lfs2_pair_sync( const lfs2_block_t paira[2], const lfs2_block_t pairb[2]) { return (paira[0] == pairb[0] && paira[1] == pairb[1]) || (paira[0] == pairb[1] && paira[1] == pairb[0]); } +#endif static inline void lfs2_pair_fromle32(lfs2_block_t pair[2]) { pair[0] = lfs2_fromle32(pair[0]); pair[1] = lfs2_fromle32(pair[1]); } +#ifndef LFS2_READONLY static inline void lfs2_pair_tole32(lfs2_block_t pair[2]) { pair[0] = lfs2_tole32(pair[0]); pair[1] = lfs2_tole32(pair[1]); } +#endif // operations on 32-bit entry tags typedef uint32_t lfs2_tag_t; @@ -365,6 +380,7 @@ static inline bool lfs2_gstate_iszero(const lfs2_gstate_t *a) { return true; } +#ifndef LFS2_READONLY static inline bool lfs2_gstate_hasorphans(const lfs2_gstate_t *a) { return lfs2_tag_size(a->tag); } @@ -376,6 +392,7 @@ static inline uint8_t lfs2_gstate_getorphans(const lfs2_gstate_t *a) { static inline bool lfs2_gstate_hasmove(const lfs2_gstate_t *a) { return lfs2_tag_type1(a->tag); } +#endif static inline bool lfs2_gstate_hasmovehere(const lfs2_gstate_t *a, const lfs2_block_t *pair) { @@ -388,11 +405,13 @@ static inline void lfs2_gstate_fromle32(lfs2_gstate_t *a) { a->pair[1] = lfs2_fromle32(a->pair[1]); } +#ifndef LFS2_READONLY static inline void lfs2_gstate_tole32(lfs2_gstate_t *a) { a->tag = lfs2_tole32(a->tag); a->pair[0] = lfs2_tole32(a->pair[0]); a->pair[1] = lfs2_tole32(a->pair[1]); } +#endif // other endianness operations static void lfs2_ctz_fromle32(struct lfs2_ctz *ctz) { @@ -416,6 +435,7 @@ static inline void lfs2_superblock_fromle32(lfs2_superblock_t *superblock) { superblock->attr_max = lfs2_fromle32(superblock->attr_max); } +#ifndef LFS2_READONLY static inline void lfs2_superblock_tole32(lfs2_superblock_t *superblock) { superblock->version = lfs2_tole32(superblock->version); superblock->block_size = lfs2_tole32(superblock->block_size); @@ -424,9 +444,10 @@ static inline void lfs2_superblock_tole32(lfs2_superblock_t *superblock) { superblock->file_max = lfs2_tole32(superblock->file_max); superblock->attr_max = lfs2_tole32(superblock->attr_max); } +#endif #ifndef LFS2_NO_ASSERT -static inline bool lfs2_mlist_isopen(struct lfs2_mlist *head, +static bool lfs2_mlist_isopen(struct lfs2_mlist *head, struct lfs2_mlist *node) { for (struct lfs2_mlist **p = &head; *p; p = &(*p)->next) { if (*p == (struct lfs2_mlist*)node) { @@ -438,7 +459,7 @@ static inline bool lfs2_mlist_isopen(struct lfs2_mlist *head, } #endif -static inline void lfs2_mlist_remove(lfs2_t *lfs2, struct lfs2_mlist *mlist) { +static void lfs2_mlist_remove(lfs2_t *lfs2, struct lfs2_mlist *mlist) { for (struct lfs2_mlist **p = &lfs2->mlist; *p; p = &(*p)->next) { if (*p == mlist) { *p = (*p)->next; @@ -447,7 +468,7 @@ static inline void lfs2_mlist_remove(lfs2_t *lfs2, struct lfs2_mlist *mlist) { } } -static inline void lfs2_mlist_append(lfs2_t *lfs2, struct lfs2_mlist *mlist) { +static void lfs2_mlist_append(lfs2_t *lfs2, struct lfs2_mlist *mlist) { mlist->next = lfs2->mlist; lfs2->mlist = mlist; } @@ -460,22 +481,22 @@ static int lfs2_dir_commit(lfs2_t *lfs2, lfs2_mdir_t *dir, static int lfs2_dir_compact(lfs2_t *lfs2, lfs2_mdir_t *dir, const struct lfs2_mattr *attrs, int attrcount, lfs2_mdir_t *source, uint16_t begin, uint16_t end); - +static lfs2_ssize_t lfs2_file_flushedwrite(lfs2_t *lfs2, lfs2_file_t *file, + const void *buffer, lfs2_size_t size); static lfs2_ssize_t lfs2_file_rawwrite(lfs2_t *lfs2, lfs2_file_t *file, const void *buffer, lfs2_size_t size); static int lfs2_file_rawsync(lfs2_t *lfs2, lfs2_file_t *file); static int lfs2_file_outline(lfs2_t *lfs2, lfs2_file_t *file); static int lfs2_file_flush(lfs2_t *lfs2, lfs2_file_t *file); -static void lfs2_fs_preporphans(lfs2_t *lfs2, int8_t orphans); +static int lfs2_fs_deorphan(lfs2_t *lfs2, bool powerloss); +static int lfs2_fs_preporphans(lfs2_t *lfs2, int8_t orphans); static void lfs2_fs_prepmove(lfs2_t *lfs2, uint16_t id, const lfs2_block_t pair[2]); static int lfs2_fs_pred(lfs2_t *lfs2, const lfs2_block_t dir[2], lfs2_mdir_t *pdir); static lfs2_stag_t lfs2_fs_parent(lfs2_t *lfs2, const lfs2_block_t dir[2], lfs2_mdir_t *parent); -static int lfs2_fs_relocate(lfs2_t *lfs2, - const lfs2_block_t oldpair[2], lfs2_block_t newpair[2]); static int lfs2_fs_forceconsistency(lfs2_t *lfs2); #endif @@ -486,6 +507,8 @@ static int lfs21_traverse(lfs2_t *lfs2, static int lfs2_dir_rawrewind(lfs2_t *lfs2, lfs2_dir_t *dir); +static lfs2_ssize_t lfs2_file_flushedread(lfs2_t *lfs2, lfs2_file_t *file, + void *buffer, lfs2_size_t size); static lfs2_ssize_t lfs2_file_rawread(lfs2_t *lfs2, lfs2_file_t *file, void *buffer, lfs2_size_t size); static int lfs2_file_rawclose(lfs2_t *lfs2, lfs2_file_t *file); @@ -726,6 +749,7 @@ static int lfs2_dir_traverse_filter(void *p, (LFS2_MKTAG(0x7ff, 0x3ff, 0) & tag) == ( LFS2_MKTAG(LFS2_TYPE_DELETE, 0, 0) | (LFS2_MKTAG(0, 0x3ff, 0) & *filtertag))) { + *filtertag = LFS2_MKTAG(LFS2_FROM_NOOP, 0, 0); return true; } @@ -740,98 +764,227 @@ static int lfs2_dir_traverse_filter(void *p, #endif #ifndef LFS2_READONLY +// maximum recursive depth of lfs2_dir_traverse, the deepest call: +// +// traverse with commit +// '-> traverse with move +// '-> traverse with filter +// +#define LFS2_DIR_TRAVERSE_DEPTH 3 + +struct lfs2_dir_traverse { + const lfs2_mdir_t *dir; + lfs2_off_t off; + lfs2_tag_t ptag; + const struct lfs2_mattr *attrs; + int attrcount; + + lfs2_tag_t tmask; + lfs2_tag_t ttag; + uint16_t begin; + uint16_t end; + int16_t diff; + + int (*cb)(void *data, lfs2_tag_t tag, const void *buffer); + void *data; + + lfs2_tag_t tag; + const void *buffer; + struct lfs2_diskoff disk; +}; + static int lfs2_dir_traverse(lfs2_t *lfs2, const lfs2_mdir_t *dir, lfs2_off_t off, lfs2_tag_t ptag, const struct lfs2_mattr *attrs, int attrcount, lfs2_tag_t tmask, lfs2_tag_t ttag, uint16_t begin, uint16_t end, int16_t diff, int (*cb)(void *data, lfs2_tag_t tag, const void *buffer), void *data) { + // This function in inherently recursive, but bounded. To allow tool-based + // analysis without unnecessary code-cost we use an explicit stack + struct lfs2_dir_traverse stack[LFS2_DIR_TRAVERSE_DEPTH-1]; + unsigned sp = 0; + int res; + // iterate over directory and attrs + lfs2_tag_t tag; + const void *buffer; + struct lfs2_diskoff disk; while (true) { - lfs2_tag_t tag; - const void *buffer; - struct lfs2_diskoff disk; - if (off+lfs2_tag_dsize(ptag) < dir->off) { - off += lfs2_tag_dsize(ptag); - int err = lfs2_bd_read(lfs2, - NULL, &lfs2->rcache, sizeof(tag), - dir->pair[0], off, &tag, sizeof(tag)); - if (err) { - return err; + { + if (off+lfs2_tag_dsize(ptag) < dir->off) { + off += lfs2_tag_dsize(ptag); + int err = lfs2_bd_read(lfs2, + NULL, &lfs2->rcache, sizeof(tag), + dir->pair[0], off, &tag, sizeof(tag)); + if (err) { + return err; + } + + tag = (lfs2_frombe32(tag) ^ ptag) | 0x80000000; + disk.block = dir->pair[0]; + disk.off = off+sizeof(lfs2_tag_t); + buffer = &disk; + ptag = tag; + } else if (attrcount > 0) { + tag = attrs[0].tag; + buffer = attrs[0].buffer; + attrs += 1; + attrcount -= 1; + } else { + // finished traversal, pop from stack? + res = 0; + break; } - tag = (lfs2_frombe32(tag) ^ ptag) | 0x80000000; - disk.block = dir->pair[0]; - disk.off = off+sizeof(lfs2_tag_t); - buffer = &disk; - ptag = tag; - } else if (attrcount > 0) { - tag = attrs[0].tag; - buffer = attrs[0].buffer; - attrs += 1; - attrcount -= 1; - } else { - return 0; + // do we need to filter? + lfs2_tag_t mask = LFS2_MKTAG(0x7ff, 0, 0); + if ((mask & tmask & tag) != (mask & tmask & ttag)) { + continue; + } + + if (lfs2_tag_id(tmask) != 0) { + LFS2_ASSERT(sp < LFS2_DIR_TRAVERSE_DEPTH); + // recurse, scan for duplicates, and update tag based on + // creates/deletes + stack[sp] = (struct lfs2_dir_traverse){ + .dir = dir, + .off = off, + .ptag = ptag, + .attrs = attrs, + .attrcount = attrcount, + .tmask = tmask, + .ttag = ttag, + .begin = begin, + .end = end, + .diff = diff, + .cb = cb, + .data = data, + .tag = tag, + .buffer = buffer, + .disk = disk, + }; + sp += 1; + + tmask = 0; + ttag = 0; + begin = 0; + end = 0; + diff = 0; + cb = lfs2_dir_traverse_filter; + data = &stack[sp-1].tag; + continue; + } } - lfs2_tag_t mask = LFS2_MKTAG(0x7ff, 0, 0); - if ((mask & tmask & tag) != (mask & tmask & ttag)) { +popped: + // in filter range? + if (lfs2_tag_id(tmask) != 0 && + !(lfs2_tag_id(tag) >= begin && lfs2_tag_id(tag) < end)) { continue; } - // do we need to filter? inlining the filtering logic here allows - // for some minor optimizations - if (lfs2_tag_id(tmask) != 0) { - // scan for duplicates and update tag based on creates/deletes - int filter = lfs2_dir_traverse(lfs2, - dir, off, ptag, attrs, attrcount, - 0, 0, 0, 0, 0, - lfs2_dir_traverse_filter, &tag); - if (filter < 0) { - return filter; - } - - if (filter) { - continue; - } - - // in filter range? - if (!(lfs2_tag_id(tag) >= begin && lfs2_tag_id(tag) < end)) { - continue; - } - } - // handle special cases for mcu-side operations if (lfs2_tag_type3(tag) == LFS2_FROM_NOOP) { // do nothing } else if (lfs2_tag_type3(tag) == LFS2_FROM_MOVE) { + // Without this condition, lfs2_dir_traverse can exhibit an + // extremely expensive O(n^3) of nested loops when renaming. + // This happens because lfs2_dir_traverse tries to filter tags by + // the tags in the source directory, triggering a second + // lfs2_dir_traverse with its own filter operation. + // + // traverse with commit + // '-> traverse with filter + // '-> traverse with move + // '-> traverse with filter + // + // However we don't actually care about filtering the second set of + // tags, since duplicate tags have no effect when filtering. + // + // This check skips this unnecessary recursive filtering explicitly, + // reducing this runtime from O(n^3) to O(n^2). + if (cb == lfs2_dir_traverse_filter) { + continue; + } + + // recurse into move + stack[sp] = (struct lfs2_dir_traverse){ + .dir = dir, + .off = off, + .ptag = ptag, + .attrs = attrs, + .attrcount = attrcount, + .tmask = tmask, + .ttag = ttag, + .begin = begin, + .end = end, + .diff = diff, + .cb = cb, + .data = data, + .tag = LFS2_MKTAG(LFS2_FROM_NOOP, 0, 0), + }; + sp += 1; + uint16_t fromid = lfs2_tag_size(tag); uint16_t toid = lfs2_tag_id(tag); - int err = lfs2_dir_traverse(lfs2, - buffer, 0, 0xffffffff, NULL, 0, - LFS2_MKTAG(0x600, 0x3ff, 0), - LFS2_MKTAG(LFS2_TYPE_STRUCT, 0, 0), - fromid, fromid+1, toid-fromid+diff, - cb, data); - if (err) { - return err; - } + dir = buffer; + off = 0; + ptag = 0xffffffff; + attrs = NULL; + attrcount = 0; + tmask = LFS2_MKTAG(0x600, 0x3ff, 0); + ttag = LFS2_MKTAG(LFS2_TYPE_STRUCT, 0, 0); + begin = fromid; + end = fromid+1; + diff = toid-fromid+diff; } else if (lfs2_tag_type3(tag) == LFS2_FROM_USERATTRS) { for (unsigned i = 0; i < lfs2_tag_size(tag); i++) { const struct lfs2_attr *a = buffer; - int err = cb(data, LFS2_MKTAG(LFS2_TYPE_USERATTR + a[i].type, + res = cb(data, LFS2_MKTAG(LFS2_TYPE_USERATTR + a[i].type, lfs2_tag_id(tag) + diff, a[i].size), a[i].buffer); - if (err) { - return err; + if (res < 0) { + return res; + } + + if (res) { + break; } } } else { - int err = cb(data, tag + LFS2_MKTAG(0, diff, 0), buffer); - if (err) { - return err; + res = cb(data, tag + LFS2_MKTAG(0, diff, 0), buffer); + if (res < 0) { + return res; + } + + if (res) { + break; } } } + + if (sp > 0) { + // pop from the stack and return, fortunately all pops share + // a destination + dir = stack[sp-1].dir; + off = stack[sp-1].off; + ptag = stack[sp-1].ptag; + attrs = stack[sp-1].attrs; + attrcount = stack[sp-1].attrcount; + tmask = stack[sp-1].tmask; + ttag = stack[sp-1].ttag; + begin = stack[sp-1].begin; + end = stack[sp-1].end; + diff = stack[sp-1].diff; + cb = stack[sp-1].cb; + data = stack[sp-1].data; + tag = stack[sp-1].tag; + buffer = stack[sp-1].buffer; + disk = stack[sp-1].disk; + sp -= 1; + goto popped; + } else { + return res; + } } #endif @@ -1449,7 +1602,7 @@ static int lfs2_dir_alloc(lfs2_t *lfs2, lfs2_mdir_t *dir) { } } - // zero for reproducability in case initial block is unreadable + // zero for reproducibility in case initial block is unreadable dir->rev = 0; // rather than clobbering one of the blocks we just pretend @@ -1508,8 +1661,7 @@ static int lfs2_dir_drop(lfs2_t *lfs2, lfs2_mdir_t *dir, lfs2_mdir_t *tail) { static int lfs2_dir_split(lfs2_t *lfs2, lfs2_mdir_t *dir, const struct lfs2_mattr *attrs, int attrcount, lfs2_mdir_t *source, uint16_t split, uint16_t end) { - // create tail directory - lfs2_alloc_ack(lfs2); + // create tail metadata pair lfs2_mdir_t tail; int err = lfs2_dir_alloc(lfs2, &tail); if (err) { @@ -1520,9 +1672,10 @@ static int lfs2_dir_split(lfs2_t *lfs2, tail.tail[0] = dir->tail[0]; tail.tail[1] = dir->tail[1]; - err = lfs2_dir_compact(lfs2, &tail, attrs, attrcount, source, split, end); - if (err) { - return err; + // note we don't care about LFS2_OK_RELOCATED + int res = lfs2_dir_compact(lfs2, &tail, attrs, attrcount, source, split, end); + if (res < 0) { + return res; } dir->tail[0] = tail.pair[0]; @@ -1564,105 +1717,44 @@ static int lfs2_dir_commit_commit(void *p, lfs2_tag_t tag, const void *buffer) { #endif #ifndef LFS2_READONLY -static int lfs2_dir_compact(lfs2_t *lfs2, - lfs2_mdir_t *dir, const struct lfs2_mattr *attrs, int attrcount, - lfs2_mdir_t *source, uint16_t begin, uint16_t end) { - // save some state in case block is bad - const lfs2_block_t oldpair[2] = {dir->pair[0], dir->pair[1]}; - bool relocated = false; - bool tired = false; - - // should we split? - while (end - begin > 1) { - // find size - lfs2_size_t size = 0; - int err = lfs2_dir_traverse(lfs2, - source, 0, 0xffffffff, attrs, attrcount, - LFS2_MKTAG(0x400, 0x3ff, 0), - LFS2_MKTAG(LFS2_TYPE_NAME, 0, 0), - begin, end, -begin, - lfs2_dir_commit_size, &size); - if (err) { - return err; - } - - // space is complicated, we need room for tail, crc, gstate, - // cleanup delete, and we cap at half a block to give room - // for metadata updates. - if (end - begin < 0xff && - size <= lfs2_min(lfs2->cfg->block_size - 36, - lfs2_alignup(lfs2->cfg->block_size/2, - lfs2->cfg->prog_size))) { - break; - } - - // can't fit, need to split, we should really be finding the - // largest size that fits with a small binary search, but right now - // it's not worth the code size - uint16_t split = (end - begin) / 2; - err = lfs2_dir_split(lfs2, dir, attrs, attrcount, - source, begin+split, end); - if (err) { - // if we fail to split, we may be able to overcompact, unless - // we're too big for even the full block, in which case our - // only option is to error - if (err == LFS2_ERR_NOSPC && size <= lfs2->cfg->block_size - 36) { - break; - } - return err; - } - - end = begin + split; - } - - // increment revision count - dir->rev += 1; +static bool lfs2_dir_needsrelocation(lfs2_t *lfs2, lfs2_mdir_t *dir) { // If our revision count == n * block_cycles, we should force a relocation, // this is how littlefs wear-levels at the metadata-pair level. Note that we // actually use (block_cycles+1)|1, this is to avoid two corner cases: // 1. block_cycles = 1, which would prevent relocations from terminating // 2. block_cycles = 2n, which, due to aliasing, would only ever relocate // one metadata block in the pair, effectively making this useless - if (lfs2->cfg->block_cycles > 0 && - (dir->rev % ((lfs2->cfg->block_cycles+1)|1) == 0)) { - if (lfs2_pair_cmp(dir->pair, (const lfs2_block_t[2]){0, 1}) == 0) { - // oh no! we're writing too much to the superblock, - // should we expand? - lfs2_ssize_t res = lfs2_fs_rawsize(lfs2); - if (res < 0) { - return res; - } - - // do we have extra space? littlefs can't reclaim this space - // by itself, so expand cautiously - if ((lfs2_size_t)res < lfs2->cfg->block_count/2) { - LFS2_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); - int err = lfs2_dir_split(lfs2, dir, attrs, attrcount, - source, begin, end); - if (err && err != LFS2_ERR_NOSPC) { - return err; - } - - // welp, we tried, if we ran out of space there's not much - // we can do, we'll error later if we've become frozen - if (!err) { - end = begin; - } - } -#ifdef LFS2_MIGRATE - } else if (lfs2->lfs21) { - // do not proactively relocate blocks during migrations, this - // can cause a number of failure states such: clobbering the - // v1 superblock if we relocate root, and invalidating directory - // pointers if we relocate the head of a directory. On top of - // this, relocations increase the overall complexity of - // lfs2_migration, which is already a delicate operation. + return (lfs2->cfg->block_cycles > 0 + && ((dir->rev + 1) % ((lfs2->cfg->block_cycles+1)|1) == 0)); +} #endif - } else { - // we're writing too much, time to relocate - tired = true; - goto relocate; - } + +#ifndef LFS2_READONLY +static int lfs2_dir_compact(lfs2_t *lfs2, + lfs2_mdir_t *dir, const struct lfs2_mattr *attrs, int attrcount, + lfs2_mdir_t *source, uint16_t begin, uint16_t end) { + // save some state in case block is bad + bool relocated = false; + bool tired = lfs2_dir_needsrelocation(lfs2, dir); + + // increment revision count + dir->rev += 1; + + // do not proactively relocate blocks during migrations, this + // can cause a number of failure states such: clobbering the + // v1 superblock if we relocate root, and invalidating directory + // pointers if we relocate the head of a directory. On top of + // this, relocations increase the overall complexity of + // lfs2_migration, which is already a delicate operation. +#ifdef LFS2_MIGRATE + if (lfs2->lfs21) { + tired = false; + } +#endif + + if (tired && lfs2_pair_cmp(dir->pair, (const lfs2_block_t[2]){0, 1}) != 0) { + // we're writing too much, time to relocate + goto relocate; } // begin loop to commit compaction to blocks until a compact sticks @@ -1676,7 +1768,8 @@ static int lfs2_dir_compact(lfs2_t *lfs2, .crc = 0xffffffff, .begin = 0, - .end = lfs2->cfg->block_size - 8, + .end = (lfs2->cfg->metadata_max ? + lfs2->cfg->metadata_max : lfs2->cfg->block_size) - 8, }; // erase block to write to @@ -1805,23 +1898,279 @@ relocate: continue; } - if (relocated) { - // update references if we relocated - LFS2_DEBUG("Relocating {0x%"PRIx32", 0x%"PRIx32"} " - "-> {0x%"PRIx32", 0x%"PRIx32"}", - oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); - int err = lfs2_fs_relocate(lfs2, oldpair, dir->pair); - if (err) { - return err; - } - } - - return 0; + return relocated ? LFS2_OK_RELOCATED : 0; } #endif #ifndef LFS2_READONLY -static int lfs2_dir_commit(lfs2_t *lfs2, lfs2_mdir_t *dir, +static int lfs2_dir_splittingcompact(lfs2_t *lfs2, lfs2_mdir_t *dir, + const struct lfs2_mattr *attrs, int attrcount, + lfs2_mdir_t *source, uint16_t begin, uint16_t end) { + while (true) { + // find size of first split, we do this by halving the split until + // the metadata is guaranteed to fit + // + // Note that this isn't a true binary search, we never increase the + // split size. This may result in poorly distributed metadata but isn't + // worth the extra code size or performance hit to fix. + lfs2_size_t split = begin; + while (end - split > 1) { + lfs2_size_t size = 0; + int err = lfs2_dir_traverse(lfs2, + source, 0, 0xffffffff, attrs, attrcount, + LFS2_MKTAG(0x400, 0x3ff, 0), + LFS2_MKTAG(LFS2_TYPE_NAME, 0, 0), + split, end, -split, + lfs2_dir_commit_size, &size); + if (err) { + return err; + } + + // space is complicated, we need room for tail, crc, gstate, + // cleanup delete, and we cap at half a block to give room + // for metadata updates. + if (end - split < 0xff + && size <= lfs2_min(lfs2->cfg->block_size - 36, + lfs2_alignup( + (lfs2->cfg->metadata_max + ? lfs2->cfg->metadata_max + : lfs2->cfg->block_size)/2, + lfs2->cfg->prog_size))) { + break; + } + + split = split + ((end - split) / 2); + } + + if (split == begin) { + // no split needed + break; + } + + // split into two metadata pairs and continue + int err = lfs2_dir_split(lfs2, dir, attrs, attrcount, + source, split, end); + if (err && err != LFS2_ERR_NOSPC) { + return err; + } + + if (err) { + // we can't allocate a new block, try to compact with degraded + // performance + LFS2_WARN("Unable to split {0x%"PRIx32", 0x%"PRIx32"}", + dir->pair[0], dir->pair[1]); + break; + } else { + end = split; + } + } + + if (lfs2_dir_needsrelocation(lfs2, dir) + && lfs2_pair_cmp(dir->pair, (const lfs2_block_t[2]){0, 1}) == 0) { + // oh no! we're writing too much to the superblock, + // should we expand? + lfs2_ssize_t size = lfs2_fs_rawsize(lfs2); + if (size < 0) { + return size; + } + + // do we have extra space? littlefs can't reclaim this space + // by itself, so expand cautiously + if ((lfs2_size_t)size < lfs2->cfg->block_count/2) { + LFS2_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); + int err = lfs2_dir_split(lfs2, dir, attrs, attrcount, + source, begin, end); + if (err && err != LFS2_ERR_NOSPC) { + return err; + } + + if (err) { + // welp, we tried, if we ran out of space there's not much + // we can do, we'll error later if we've become frozen + LFS2_WARN("Unable to expand superblock"); + } else { + end = begin; + } + } + } + + return lfs2_dir_compact(lfs2, dir, attrs, attrcount, source, begin, end); +} +#endif + +#ifndef LFS2_READONLY +static int lfs2_dir_relocatingcommit(lfs2_t *lfs2, lfs2_mdir_t *dir, + const lfs2_block_t pair[2], + const struct lfs2_mattr *attrs, int attrcount, + lfs2_mdir_t *pdir) { + int state = 0; + + // calculate changes to the directory + bool hasdelete = false; + for (int i = 0; i < attrcount; i++) { + if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_CREATE) { + dir->count += 1; + } else if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_DELETE) { + LFS2_ASSERT(dir->count > 0); + dir->count -= 1; + hasdelete = true; + } else if (lfs2_tag_type1(attrs[i].tag) == LFS2_TYPE_TAIL) { + dir->tail[0] = ((lfs2_block_t*)attrs[i].buffer)[0]; + dir->tail[1] = ((lfs2_block_t*)attrs[i].buffer)[1]; + dir->split = (lfs2_tag_chunk(attrs[i].tag) & 1); + lfs2_pair_fromle32(dir->tail); + } + } + + // should we actually drop the directory block? + if (hasdelete && dir->count == 0) { + LFS2_ASSERT(pdir); + int err = lfs2_fs_pred(lfs2, dir->pair, pdir); + if (err && err != LFS2_ERR_NOENT) { + return err; + } + + if (err != LFS2_ERR_NOENT && pdir->split) { + state = LFS2_OK_DROPPED; + goto fixmlist; + } + } + + if (dir->erased) { + // try to commit + struct lfs2_commit commit = { + .block = dir->pair[0], + .off = dir->off, + .ptag = dir->etag, + .crc = 0xffffffff, + + .begin = dir->off, + .end = (lfs2->cfg->metadata_max ? + lfs2->cfg->metadata_max : lfs2->cfg->block_size) - 8, + }; + + // traverse attrs that need to be written out + lfs2_pair_tole32(dir->tail); + int err = lfs2_dir_traverse(lfs2, + dir, dir->off, dir->etag, attrs, attrcount, + 0, 0, 0, 0, 0, + lfs2_dir_commit_commit, &(struct lfs2_dir_commit_commit){ + lfs2, &commit}); + lfs2_pair_fromle32(dir->tail); + if (err) { + if (err == LFS2_ERR_NOSPC || err == LFS2_ERR_CORRUPT) { + goto compact; + } + return err; + } + + // commit any global diffs if we have any + lfs2_gstate_t delta = {0}; + lfs2_gstate_xor(&delta, &lfs2->gstate); + lfs2_gstate_xor(&delta, &lfs2->gdisk); + lfs2_gstate_xor(&delta, &lfs2->gdelta); + delta.tag &= ~LFS2_MKTAG(0, 0, 0x3ff); + if (!lfs2_gstate_iszero(&delta)) { + err = lfs2_dir_getgstate(lfs2, dir, &delta); + if (err) { + return err; + } + + lfs2_gstate_tole32(&delta); + err = lfs2_dir_commitattr(lfs2, &commit, + LFS2_MKTAG(LFS2_TYPE_MOVESTATE, 0x3ff, + sizeof(delta)), &delta); + if (err) { + if (err == LFS2_ERR_NOSPC || err == LFS2_ERR_CORRUPT) { + goto compact; + } + return err; + } + } + + // finalize commit with the crc + err = lfs2_dir_commitcrc(lfs2, &commit); + if (err) { + if (err == LFS2_ERR_NOSPC || err == LFS2_ERR_CORRUPT) { + goto compact; + } + return err; + } + + // successful commit, update dir + LFS2_ASSERT(commit.off % lfs2->cfg->prog_size == 0); + dir->off = commit.off; + dir->etag = commit.ptag; + // and update gstate + lfs2->gdisk = lfs2->gstate; + lfs2->gdelta = (lfs2_gstate_t){0}; + + goto fixmlist; + } + +compact: + // fall back to compaction + lfs2_cache_drop(lfs2, &lfs2->pcache); + + state = lfs2_dir_splittingcompact(lfs2, dir, attrs, attrcount, + dir, 0, dir->count); + if (state < 0) { + return state; + } + + goto fixmlist; + +fixmlist:; + // this complicated bit of logic is for fixing up any active + // metadata-pairs that we may have affected + // + // note we have to make two passes since the mdir passed to + // lfs2_dir_commit could also be in this list, and even then + // we need to copy the pair so they don't get clobbered if we refetch + // our mdir. + lfs2_block_t oldpair[2] = {pair[0], pair[1]}; + for (struct lfs2_mlist *d = lfs2->mlist; d; d = d->next) { + if (lfs2_pair_cmp(d->m.pair, oldpair) == 0) { + d->m = *dir; + if (d->m.pair != pair) { + for (int i = 0; i < attrcount; i++) { + if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_DELETE && + d->id == lfs2_tag_id(attrs[i].tag)) { + d->m.pair[0] = LFS2_BLOCK_NULL; + d->m.pair[1] = LFS2_BLOCK_NULL; + } else if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_DELETE && + d->id > lfs2_tag_id(attrs[i].tag)) { + d->id -= 1; + if (d->type == LFS2_TYPE_DIR) { + ((lfs2_dir_t*)d)->pos -= 1; + } + } else if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_CREATE && + d->id >= lfs2_tag_id(attrs[i].tag)) { + d->id += 1; + if (d->type == LFS2_TYPE_DIR) { + ((lfs2_dir_t*)d)->pos += 1; + } + } + } + } + + while (d->id >= d->m.count && d->m.split) { + // we split and id is on tail now + d->id -= d->m.count; + int err = lfs2_dir_fetch(lfs2, &d->m, d->m.tail); + if (err) { + return err; + } + } + } + } + + return state; +} +#endif + +#ifndef LFS2_READONLY +static int lfs2_dir_orphaningcommit(lfs2_t *lfs2, lfs2_mdir_t *dir, const struct lfs2_mattr *attrs, int attrcount) { // check for any inline files that aren't RAM backed and // forcefully evict them, needed for filesystem consistency @@ -1841,168 +2190,193 @@ static int lfs2_dir_commit(lfs2_t *lfs2, lfs2_mdir_t *dir, } } - // calculate changes to the directory - lfs2_mdir_t olddir = *dir; - bool hasdelete = false; - for (int i = 0; i < attrcount; i++) { - if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_CREATE) { - dir->count += 1; - } else if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_DELETE) { - LFS2_ASSERT(dir->count > 0); - dir->count -= 1; - hasdelete = true; - } else if (lfs2_tag_type1(attrs[i].tag) == LFS2_TYPE_TAIL) { - dir->tail[0] = ((lfs2_block_t*)attrs[i].buffer)[0]; - dir->tail[1] = ((lfs2_block_t*)attrs[i].buffer)[1]; - dir->split = (lfs2_tag_chunk(attrs[i].tag) & 1); - lfs2_pair_fromle32(dir->tail); - } + lfs2_block_t lpair[2] = {dir->pair[0], dir->pair[1]}; + lfs2_mdir_t ldir = *dir; + lfs2_mdir_t pdir; + int state = lfs2_dir_relocatingcommit(lfs2, &ldir, dir->pair, + attrs, attrcount, &pdir); + if (state < 0) { + return state; } - // should we actually drop the directory block? - if (hasdelete && dir->count == 0) { - lfs2_mdir_t pdir; - int err = lfs2_fs_pred(lfs2, dir->pair, &pdir); - if (err && err != LFS2_ERR_NOENT) { - *dir = olddir; + // update if we're not in mlist, note we may have already been + // updated if we are in mlist + if (lfs2_pair_cmp(dir->pair, lpair) == 0) { + *dir = ldir; + } + + // commit was successful, but may require other changes in the + // filesystem, these would normally be tail recursive, but we have + // flattened them here avoid unbounded stack usage + + // need to drop? + if (state == LFS2_OK_DROPPED) { + // steal state + int err = lfs2_dir_getgstate(lfs2, dir, &lfs2->gdelta); + if (err) { return err; } - if (err != LFS2_ERR_NOENT && pdir.split) { - err = lfs2_dir_drop(lfs2, &pdir, dir); - if (err) { - *dir = olddir; - return err; - } - } - } - - if (dir->erased || dir->count >= 0xff) { - // try to commit - struct lfs2_commit commit = { - .block = dir->pair[0], - .off = dir->off, - .ptag = dir->etag, - .crc = 0xffffffff, - - .begin = dir->off, - .end = lfs2->cfg->block_size - 8, - }; - - // traverse attrs that need to be written out + // steal tail, note that this can't create a recursive drop + lpair[0] = pdir.pair[0]; + lpair[1] = pdir.pair[1]; lfs2_pair_tole32(dir->tail); - int err = lfs2_dir_traverse(lfs2, - dir, dir->off, dir->etag, attrs, attrcount, - 0, 0, 0, 0, 0, - lfs2_dir_commit_commit, &(struct lfs2_dir_commit_commit){ - lfs2, &commit}); + state = lfs2_dir_relocatingcommit(lfs2, &pdir, lpair, LFS2_MKATTRS( + {LFS2_MKTAG(LFS2_TYPE_TAIL + dir->split, 0x3ff, 8), + dir->tail}), + NULL); lfs2_pair_fromle32(dir->tail); - if (err) { - if (err == LFS2_ERR_NOSPC || err == LFS2_ERR_CORRUPT) { - goto compact; - } - *dir = olddir; - return err; + if (state < 0) { + return state; } - // commit any global diffs if we have any - lfs2_gstate_t delta = {0}; - lfs2_gstate_xor(&delta, &lfs2->gstate); - lfs2_gstate_xor(&delta, &lfs2->gdisk); - lfs2_gstate_xor(&delta, &lfs2->gdelta); - delta.tag &= ~LFS2_MKTAG(0, 0, 0x3ff); - if (!lfs2_gstate_iszero(&delta)) { - err = lfs2_dir_getgstate(lfs2, dir, &delta); + ldir = pdir; + } + + // need to relocate? + bool orphans = false; + while (state == LFS2_OK_RELOCATED) { + LFS2_DEBUG("Relocating {0x%"PRIx32", 0x%"PRIx32"} " + "-> {0x%"PRIx32", 0x%"PRIx32"}", + lpair[0], lpair[1], ldir.pair[0], ldir.pair[1]); + state = 0; + + // update internal root + if (lfs2_pair_cmp(lpair, lfs2->root) == 0) { + lfs2->root[0] = ldir.pair[0]; + lfs2->root[1] = ldir.pair[1]; + } + + // update internally tracked dirs + for (struct lfs2_mlist *d = lfs2->mlist; d; d = d->next) { + if (lfs2_pair_cmp(lpair, d->m.pair) == 0) { + d->m.pair[0] = ldir.pair[0]; + d->m.pair[1] = ldir.pair[1]; + } + + if (d->type == LFS2_TYPE_DIR && + lfs2_pair_cmp(lpair, ((lfs2_dir_t*)d)->head) == 0) { + ((lfs2_dir_t*)d)->head[0] = ldir.pair[0]; + ((lfs2_dir_t*)d)->head[1] = ldir.pair[1]; + } + } + + // find parent + lfs2_stag_t tag = lfs2_fs_parent(lfs2, lpair, &pdir); + if (tag < 0 && tag != LFS2_ERR_NOENT) { + return tag; + } + + bool hasparent = (tag != LFS2_ERR_NOENT); + if (tag != LFS2_ERR_NOENT) { + // note that if we have a parent, we must have a pred, so this will + // always create an orphan + int err = lfs2_fs_preporphans(lfs2, +1); if (err) { - *dir = olddir; return err; } - lfs2_gstate_tole32(&delta); - err = lfs2_dir_commitattr(lfs2, &commit, - LFS2_MKTAG(LFS2_TYPE_MOVESTATE, 0x3ff, - sizeof(delta)), &delta); - if (err) { - if (err == LFS2_ERR_NOSPC || err == LFS2_ERR_CORRUPT) { - goto compact; - } - *dir = olddir; - return err; - } - } - - // finalize commit with the crc - err = lfs2_dir_commitcrc(lfs2, &commit); - if (err) { - if (err == LFS2_ERR_NOSPC || err == LFS2_ERR_CORRUPT) { - goto compact; - } - *dir = olddir; - return err; - } - - // successful commit, update dir - LFS2_ASSERT(commit.off % lfs2->cfg->prog_size == 0); - dir->off = commit.off; - dir->etag = commit.ptag; - // and update gstate - lfs2->gdisk = lfs2->gstate; - lfs2->gdelta = (lfs2_gstate_t){0}; - } else { -compact: - // fall back to compaction - lfs2_cache_drop(lfs2, &lfs2->pcache); - - int err = lfs2_dir_compact(lfs2, dir, attrs, attrcount, - dir, 0, dir->count); - if (err) { - *dir = olddir; - return err; - } - } - - // this complicated bit of logic is for fixing up any active - // metadata-pairs that we may have affected - // - // note we have to make two passes since the mdir passed to - // lfs2_dir_commit could also be in this list, and even then - // we need to copy the pair so they don't get clobbered if we refetch - // our mdir. - for (struct lfs2_mlist *d = lfs2->mlist; d; d = d->next) { - if (&d->m != dir && lfs2_pair_cmp(d->m.pair, olddir.pair) == 0) { - d->m = *dir; - for (int i = 0; i < attrcount; i++) { - if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_DELETE && - d->id == lfs2_tag_id(attrs[i].tag)) { - d->m.pair[0] = LFS2_BLOCK_NULL; - d->m.pair[1] = LFS2_BLOCK_NULL; - } else if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_DELETE && - d->id > lfs2_tag_id(attrs[i].tag)) { - d->id -= 1; - if (d->type == LFS2_TYPE_DIR) { - ((lfs2_dir_t*)d)->pos -= 1; - } - } else if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_CREATE && - d->id >= lfs2_tag_id(attrs[i].tag)) { - d->id += 1; - if (d->type == LFS2_TYPE_DIR) { - ((lfs2_dir_t*)d)->pos += 1; - } + // fix pending move in this pair? this looks like an optimization but + // is in fact _required_ since relocating may outdate the move. + uint16_t moveid = 0x3ff; + if (lfs2_gstate_hasmovehere(&lfs2->gstate, pdir.pair)) { + moveid = lfs2_tag_id(lfs2->gstate.tag); + LFS2_DEBUG("Fixing move while relocating " + "{0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16"\n", + pdir.pair[0], pdir.pair[1], moveid); + lfs2_fs_prepmove(lfs2, 0x3ff, NULL); + if (moveid < lfs2_tag_id(tag)) { + tag -= LFS2_MKTAG(0, 1, 0); } } - } - } - for (struct lfs2_mlist *d = lfs2->mlist; d; d = d->next) { - if (lfs2_pair_cmp(d->m.pair, olddir.pair) == 0) { - while (d->id >= d->m.count && d->m.split) { - // we split and id is on tail now - d->id -= d->m.count; - int err = lfs2_dir_fetch(lfs2, &d->m, d->m.tail); + lfs2_block_t ppair[2] = {pdir.pair[0], pdir.pair[1]}; + lfs2_pair_tole32(ldir.pair); + state = lfs2_dir_relocatingcommit(lfs2, &pdir, ppair, LFS2_MKATTRS( + {LFS2_MKTAG_IF(moveid != 0x3ff, + LFS2_TYPE_DELETE, moveid, 0), NULL}, + {tag, ldir.pair}), + NULL); + lfs2_pair_fromle32(ldir.pair); + if (state < 0) { + return state; + } + + if (state == LFS2_OK_RELOCATED) { + lpair[0] = ppair[0]; + lpair[1] = ppair[1]; + ldir = pdir; + orphans = true; + continue; + } + } + + // find pred + int err = lfs2_fs_pred(lfs2, lpair, &pdir); + if (err && err != LFS2_ERR_NOENT) { + return err; + } + LFS2_ASSERT(!(hasparent && err == LFS2_ERR_NOENT)); + + // if we can't find dir, it must be new + if (err != LFS2_ERR_NOENT) { + if (lfs2_gstate_hasorphans(&lfs2->gstate)) { + // next step, clean up orphans + err = lfs2_fs_preporphans(lfs2, -hasparent); if (err) { return err; } } + + // fix pending move in this pair? this looks like an optimization + // but is in fact _required_ since relocating may outdate the move. + uint16_t moveid = 0x3ff; + if (lfs2_gstate_hasmovehere(&lfs2->gstate, pdir.pair)) { + moveid = lfs2_tag_id(lfs2->gstate.tag); + LFS2_DEBUG("Fixing move while relocating " + "{0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16"\n", + pdir.pair[0], pdir.pair[1], moveid); + lfs2_fs_prepmove(lfs2, 0x3ff, NULL); + } + + // replace bad pair, either we clean up desync, or no desync occured + lpair[0] = pdir.pair[0]; + lpair[1] = pdir.pair[1]; + lfs2_pair_tole32(ldir.pair); + state = lfs2_dir_relocatingcommit(lfs2, &pdir, lpair, LFS2_MKATTRS( + {LFS2_MKTAG_IF(moveid != 0x3ff, + LFS2_TYPE_DELETE, moveid, 0), NULL}, + {LFS2_MKTAG(LFS2_TYPE_TAIL + pdir.split, 0x3ff, 8), + ldir.pair}), + NULL); + lfs2_pair_fromle32(ldir.pair); + if (state < 0) { + return state; + } + + ldir = pdir; + } + } + + return orphans ? LFS2_OK_ORPHANED : 0; +} +#endif + +#ifndef LFS2_READONLY +static int lfs2_dir_commit(lfs2_t *lfs2, lfs2_mdir_t *dir, + const struct lfs2_mattr *attrs, int attrcount) { + int orphans = lfs2_dir_orphaningcommit(lfs2, dir, attrs, attrcount); + if (orphans < 0) { + return orphans; + } + + if (orphans) { + // make sure we've removed all orphans, this is a noop if there + // are none, but if we had nested blocks failures we may have + // created some + int err = lfs2_fs_deorphan(lfs2, false); + if (err) { + return err; } } @@ -2060,10 +2434,13 @@ static int lfs2_rawmkdir(lfs2_t *lfs2, const char *path) { return err; } - // current block end of list? + // current block not end of list? if (cwd.m.split) { // update tails, this creates a desync - lfs2_fs_preporphans(lfs2, +1); + err = lfs2_fs_preporphans(lfs2, +1); + if (err) { + return err; + } // it's possible our predecessor has to be relocated, and if // our parent is our predecessor's predecessor, this could have @@ -2083,7 +2460,10 @@ static int lfs2_rawmkdir(lfs2_t *lfs2, const char *path) { } lfs2->mlist = cwd.next; - lfs2_fs_preporphans(lfs2, -1); + err = lfs2_fs_preporphans(lfs2, -1); + if (err) { + return err; + } } // now insert into our parent block @@ -2504,8 +2884,11 @@ static int lfs2_file_rawopencfg(lfs2_t *lfs2, lfs2_file_t *file, {LFS2_MKTAG(LFS2_TYPE_CREATE, file->id, 0), NULL}, {LFS2_MKTAG(LFS2_TYPE_REG, file->id, nlen), path}, {LFS2_MKTAG(LFS2_TYPE_INLINESTRUCT, file->id, 0), NULL})); + + // it may happen that the file name doesn't fit in the metadata blocks, e.g., a 256 byte file name will + // not fit in a 128 byte block. + err = (err == LFS2_ERR_NOSPC) ? LFS2_ERR_NAMETOOLONG : err; if (err) { - err = LFS2_ERR_NAMETOOLONG; goto cleanup; } @@ -2610,12 +2993,14 @@ cleanup: return err; } +#ifndef LFS2_NO_MALLOC static int lfs2_file_rawopen(lfs2_t *lfs2, lfs2_file_t *file, const char *path, int flags) { static const struct lfs2_file_config defaults = {0}; int err = lfs2_file_rawopencfg(lfs2, file, path, flags, &defaults); return err; } +#endif static int lfs2_file_rawclose(lfs2_t *lfs2, lfs2_file_t *file) { #ifndef LFS2_READONLY @@ -2721,7 +3106,6 @@ static int lfs2_file_outline(lfs2_t *lfs2, lfs2_file_t *file) { } #endif -#ifndef LFS2_READONLY static int lfs2_file_flush(lfs2_t *lfs2, lfs2_file_t *file) { if (file->flags & LFS2_F_READING) { if (!(file->flags & LFS2_F_INLINE)) { @@ -2730,6 +3114,7 @@ static int lfs2_file_flush(lfs2_t *lfs2, lfs2_file_t *file) { file->flags &= ~LFS2_F_READING; } +#ifndef LFS2_READONLY if (file->flags & LFS2_F_WRITING) { lfs2_off_t pos = file->pos; @@ -2748,12 +3133,12 @@ static int lfs2_file_flush(lfs2_t *lfs2, lfs2_file_t *file) { // copy over a byte at a time, leave it up to caching // to make this efficient uint8_t data; - lfs2_ssize_t res = lfs2_file_rawread(lfs2, &orig, &data, 1); + lfs2_ssize_t res = lfs2_file_flushedread(lfs2, &orig, &data, 1); if (res < 0) { return res; } - res = lfs2_file_rawwrite(lfs2, file, &data, 1); + res = lfs2_file_flushedwrite(lfs2, file, &data, 1); if (res < 0) { return res; } @@ -2796,10 +3181,10 @@ relocate: file->pos = pos; } +#endif return 0; } -#endif #ifndef LFS2_READONLY static int lfs2_file_rawsync(lfs2_t *lfs2, lfs2_file_t *file) { @@ -2854,23 +3239,11 @@ static int lfs2_file_rawsync(lfs2_t *lfs2, lfs2_file_t *file) { } #endif -static lfs2_ssize_t lfs2_file_rawread(lfs2_t *lfs2, lfs2_file_t *file, +static lfs2_ssize_t lfs2_file_flushedread(lfs2_t *lfs2, lfs2_file_t *file, void *buffer, lfs2_size_t size) { - LFS2_ASSERT((file->flags & LFS2_O_RDONLY) == LFS2_O_RDONLY); - uint8_t *data = buffer; lfs2_size_t nsize = size; -#ifndef LFS2_READONLY - if (file->flags & LFS2_F_WRITING) { - // flush out any writes - int err = lfs2_file_flush(lfs2, file); - if (err) { - return err; - } - } -#endif - if (file->pos >= file->ctz.size) { // eof if past end return 0; @@ -2927,48 +3300,36 @@ static lfs2_ssize_t lfs2_file_rawread(lfs2_t *lfs2, lfs2_file_t *file, return size; } +static lfs2_ssize_t lfs2_file_rawread(lfs2_t *lfs2, lfs2_file_t *file, + void *buffer, lfs2_size_t size) { + LFS2_ASSERT((file->flags & LFS2_O_RDONLY) == LFS2_O_RDONLY); + #ifndef LFS2_READONLY -static lfs2_ssize_t lfs2_file_rawwrite(lfs2_t *lfs2, lfs2_file_t *file, - const void *buffer, lfs2_size_t size) { - LFS2_ASSERT((file->flags & LFS2_O_WRONLY) == LFS2_O_WRONLY); - - const uint8_t *data = buffer; - lfs2_size_t nsize = size; - - if (file->flags & LFS2_F_READING) { - // drop any reads + if (file->flags & LFS2_F_WRITING) { + // flush out any writes int err = lfs2_file_flush(lfs2, file); if (err) { return err; } } +#endif - if ((file->flags & LFS2_O_APPEND) && file->pos < file->ctz.size) { - file->pos = file->ctz.size; - } + return lfs2_file_flushedread(lfs2, file, buffer, size); +} - if (file->pos + size > lfs2->file_max) { - // Larger than file limit? - return LFS2_ERR_FBIG; - } - if (!(file->flags & LFS2_F_WRITING) && file->pos > file->ctz.size) { - // fill with zeros - lfs2_off_t pos = file->pos; - file->pos = file->ctz.size; - - while (file->pos < pos) { - lfs2_ssize_t res = lfs2_file_rawwrite(lfs2, file, &(uint8_t){0}, 1); - if (res < 0) { - return res; - } - } - } +#ifndef LFS2_READONLY +static lfs2_ssize_t lfs2_file_flushedwrite(lfs2_t *lfs2, lfs2_file_t *file, + const void *buffer, lfs2_size_t size) { + const uint8_t *data = buffer; + lfs2_size_t nsize = size; if ((file->flags & LFS2_F_INLINE) && lfs2_max(file->pos+nsize, file->ctz.size) > lfs2_min(0x3fe, lfs2_min( - lfs2->cfg->cache_size, lfs2->cfg->block_size/8))) { + lfs2->cfg->cache_size, + (lfs2->cfg->metadata_max ? + lfs2->cfg->metadata_max : lfs2->cfg->block_size) / 8))) { // inline file doesn't fit anymore int err = lfs2_file_outline(lfs2, file); if (err) { @@ -3043,29 +3404,72 @@ relocate: lfs2_alloc_ack(lfs2); } - file->flags &= ~LFS2_F_ERRED; return size; } + +static lfs2_ssize_t lfs2_file_rawwrite(lfs2_t *lfs2, lfs2_file_t *file, + const void *buffer, lfs2_size_t size) { + LFS2_ASSERT((file->flags & LFS2_O_WRONLY) == LFS2_O_WRONLY); + + if (file->flags & LFS2_F_READING) { + // drop any reads + int err = lfs2_file_flush(lfs2, file); + if (err) { + return err; + } + } + + if ((file->flags & LFS2_O_APPEND) && file->pos < file->ctz.size) { + file->pos = file->ctz.size; + } + + if (file->pos + size > lfs2->file_max) { + // Larger than file limit? + return LFS2_ERR_FBIG; + } + + if (!(file->flags & LFS2_F_WRITING) && file->pos > file->ctz.size) { + // fill with zeros + lfs2_off_t pos = file->pos; + file->pos = file->ctz.size; + + while (file->pos < pos) { + lfs2_ssize_t res = lfs2_file_flushedwrite(lfs2, file, &(uint8_t){0}, 1); + if (res < 0) { + return res; + } + } + } + + lfs2_ssize_t nsize = lfs2_file_flushedwrite(lfs2, file, buffer, size); + if (nsize < 0) { + return nsize; + } + + file->flags &= ~LFS2_F_ERRED; + return nsize; +} #endif static lfs2_soff_t lfs2_file_rawseek(lfs2_t *lfs2, lfs2_file_t *file, lfs2_soff_t off, int whence) { -#ifndef LFS2_READONLY - // write out everything beforehand, may be noop if rdonly - int err = lfs2_file_flush(lfs2, file); - if (err) { - return err; - } -#endif - // find new pos lfs2_off_t npos = file->pos; if (whence == LFS2_SEEK_SET) { npos = off; } else if (whence == LFS2_SEEK_CUR) { - npos = file->pos + off; + if ((lfs2_soff_t)file->pos + off < 0) { + return LFS2_ERR_INVAL; + } else { + npos = file->pos + off; + } } else if (whence == LFS2_SEEK_END) { - npos = file->ctz.size + off; + lfs2_soff_t res = lfs2_file_rawsize(lfs2, file) + off; + if (res < 0) { + return LFS2_ERR_INVAL; + } else { + npos = res; + } } if (npos > lfs2->file_max) { @@ -3073,6 +3477,38 @@ static lfs2_soff_t lfs2_file_rawseek(lfs2_t *lfs2, lfs2_file_t *file, return LFS2_ERR_INVAL; } + if (file->pos == npos) { + // noop - position has not changed + return npos; + } + + // if we're only reading and our new offset is still in the file's cache + // we can avoid flushing and needing to reread the data + if ( +#ifndef LFS2_READONLY + !(file->flags & LFS2_F_WRITING) +#else + true +#endif + ) { + int oindex = lfs2_ctz_index(lfs2, &(lfs2_off_t){file->pos}); + lfs2_off_t noff = npos; + int nindex = lfs2_ctz_index(lfs2, &noff); + if (oindex == nindex + && noff >= file->cache.off + && noff < file->cache.off + file->cache.size) { + file->pos = npos; + file->off = noff; + return npos; + } + } + + // write out everything beforehand, may be noop if rdonly + int err = lfs2_file_flush(lfs2, file); + if (err) { + return err; + } + // update pos file->pos = npos; return npos; @@ -3103,21 +3539,22 @@ static int lfs2_file_rawtruncate(lfs2_t *lfs2, lfs2_file_t *file, lfs2_off_t siz return err; } + // need to set pos/block/off consistently so seeking back to + // the old position does not get confused + file->pos = size; file->ctz.head = file->block; file->ctz.size = size; file->flags |= LFS2_F_DIRTY | LFS2_F_READING; } else if (size > oldsize) { // flush+seek if not already at end - if (file->pos != oldsize) { - lfs2_soff_t res = lfs2_file_rawseek(lfs2, file, 0, LFS2_SEEK_END); - if (res < 0) { - return (int)res; - } + lfs2_soff_t res = lfs2_file_rawseek(lfs2, file, 0, LFS2_SEEK_END); + if (res < 0) { + return (int)res; } // fill with zeros while (file->pos < size) { - lfs2_ssize_t res = lfs2_file_rawwrite(lfs2, file, &(uint8_t){0}, 1); + res = lfs2_file_rawwrite(lfs2, file, &(uint8_t){0}, 1); if (res < 0) { return (int)res; } @@ -3208,7 +3645,10 @@ static int lfs2_rawremove(lfs2_t *lfs2, const char *path) { } // mark fs as orphaned - lfs2_fs_preporphans(lfs2, +1); + err = lfs2_fs_preporphans(lfs2, +1); + if (err) { + return err; + } // I know it's crazy but yes, dir can be changed by our parent's // commit (if predecessor is child) @@ -3228,7 +3668,10 @@ static int lfs2_rawremove(lfs2_t *lfs2, const char *path) { lfs2->mlist = dir.next; if (lfs2_tag_type3(tag) == LFS2_TYPE_DIR) { // fix orphan - lfs2_fs_preporphans(lfs2, -1); + err = lfs2_fs_preporphans(lfs2, -1); + if (err) { + return err; + } err = lfs2_fs_pred(lfs2, dir.m.pair, &cwd); if (err) { @@ -3314,7 +3757,10 @@ static int lfs2_rawrename(lfs2_t *lfs2, const char *oldpath, const char *newpath } // mark fs as orphaned - lfs2_fs_preporphans(lfs2, +1); + err = lfs2_fs_preporphans(lfs2, +1); + if (err) { + return err; + } // I know it's crazy but yes, dir can be changed by our parent's // commit (if predecessor is child) @@ -3355,9 +3801,13 @@ static int lfs2_rawrename(lfs2_t *lfs2, const char *oldpath, const char *newpath } lfs2->mlist = prevdir.next; - if (prevtag != LFS2_ERR_NOENT && lfs2_tag_type3(prevtag) == LFS2_TYPE_DIR) { + if (prevtag != LFS2_ERR_NOENT + && lfs2_tag_type3(prevtag) == LFS2_TYPE_DIR) { // fix orphan - lfs2_fs_preporphans(lfs2, -1); + err = lfs2_fs_preporphans(lfs2, -1); + if (err) { + return err; + } err = lfs2_fs_pred(lfs2, prevdir.m.pair, &newcwd); if (err) { @@ -3538,6 +3988,8 @@ static int lfs2_init(lfs2_t *lfs2, const struct lfs2_config *cfg) { lfs2->attr_max = LFS2_ATTR_MAX; } + LFS2_ASSERT(lfs2->cfg->metadata_max <= lfs2->cfg->block_size); + // setup default state lfs2->root[0] = LFS2_BLOCK_NULL; lfs2->root[1] = LFS2_BLOCK_NULL; @@ -3618,12 +4070,6 @@ static int lfs2_rawformat(lfs2_t *lfs2, const struct lfs2_config *cfg) { goto cleanup; } - // sanity check that fetch works - err = lfs2_dir_fetch(lfs2, &root, (const lfs2_block_t[2]){0, 1}); - if (err) { - goto cleanup; - } - // force compaction to prevent accidentally mounting any // older version of littlefs that may live on disk root.erased = false; @@ -3631,6 +4077,12 @@ static int lfs2_rawformat(lfs2_t *lfs2, const struct lfs2_config *cfg) { if (err) { goto cleanup; } + + // sanity check that fetch works + err = lfs2_dir_fetch(lfs2, &root, (const lfs2_block_t[2]){0, 1}); + if (err) { + goto cleanup; + } } cleanup: @@ -3730,6 +4182,20 @@ static int lfs2_rawmount(lfs2_t *lfs2, const struct lfs2_config *cfg) { lfs2->attr_max = superblock.attr_max; } + + if (superblock.block_count != lfs2->cfg->block_count) { + LFS2_ERROR("Invalid block count (%"PRIu32" != %"PRIu32")", + superblock.block_count, lfs2->cfg->block_count); + err = LFS2_ERR_INVAL; + goto cleanup; + } + + if (superblock.block_size != lfs2->cfg->block_size) { + LFS2_ERROR("Invalid block size (%"PRIu32" != %"PRIu32")", + superblock.block_count, lfs2->cfg->block_count); + err = LFS2_ERR_INVAL; + goto cleanup; + } } // has gstate? @@ -3957,108 +4423,13 @@ static lfs2_stag_t lfs2_fs_parent(lfs2_t *lfs2, const lfs2_block_t pair[2], #endif #ifndef LFS2_READONLY -static int lfs2_fs_relocate(lfs2_t *lfs2, - const lfs2_block_t oldpair[2], lfs2_block_t newpair[2]) { - // update internal root - if (lfs2_pair_cmp(oldpair, lfs2->root) == 0) { - lfs2->root[0] = newpair[0]; - lfs2->root[1] = newpair[1]; - } - - // update internally tracked dirs - for (struct lfs2_mlist *d = lfs2->mlist; d; d = d->next) { - if (lfs2_pair_cmp(oldpair, d->m.pair) == 0) { - d->m.pair[0] = newpair[0]; - d->m.pair[1] = newpair[1]; - } - - if (d->type == LFS2_TYPE_DIR && - lfs2_pair_cmp(oldpair, ((lfs2_dir_t*)d)->head) == 0) { - ((lfs2_dir_t*)d)->head[0] = newpair[0]; - ((lfs2_dir_t*)d)->head[1] = newpair[1]; - } - } - - // find parent - lfs2_mdir_t parent; - lfs2_stag_t tag = lfs2_fs_parent(lfs2, oldpair, &parent); - if (tag < 0 && tag != LFS2_ERR_NOENT) { - return tag; - } - - if (tag != LFS2_ERR_NOENT) { - // update disk, this creates a desync - lfs2_fs_preporphans(lfs2, +1); - - // fix pending move in this pair? this looks like an optimization but - // is in fact _required_ since relocating may outdate the move. - uint16_t moveid = 0x3ff; - if (lfs2_gstate_hasmovehere(&lfs2->gstate, parent.pair)) { - moveid = lfs2_tag_id(lfs2->gstate.tag); - LFS2_DEBUG("Fixing move while relocating " - "{0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16"\n", - parent.pair[0], parent.pair[1], moveid); - lfs2_fs_prepmove(lfs2, 0x3ff, NULL); - if (moveid < lfs2_tag_id(tag)) { - tag -= LFS2_MKTAG(0, 1, 0); - } - } - - lfs2_pair_tole32(newpair); - int err = lfs2_dir_commit(lfs2, &parent, LFS2_MKATTRS( - {LFS2_MKTAG_IF(moveid != 0x3ff, - LFS2_TYPE_DELETE, moveid, 0), NULL}, - {tag, newpair})); - lfs2_pair_fromle32(newpair); - if (err) { - return err; - } - - // next step, clean up orphans - lfs2_fs_preporphans(lfs2, -1); - } - - // find pred - int err = lfs2_fs_pred(lfs2, oldpair, &parent); - if (err && err != LFS2_ERR_NOENT) { - return err; - } - - // if we can't find dir, it must be new - if (err != LFS2_ERR_NOENT) { - // fix pending move in this pair? this looks like an optimization but - // is in fact _required_ since relocating may outdate the move. - uint16_t moveid = 0x3ff; - if (lfs2_gstate_hasmovehere(&lfs2->gstate, parent.pair)) { - moveid = lfs2_tag_id(lfs2->gstate.tag); - LFS2_DEBUG("Fixing move while relocating " - "{0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16"\n", - parent.pair[0], parent.pair[1], moveid); - lfs2_fs_prepmove(lfs2, 0x3ff, NULL); - } - - // replace bad pair, either we clean up desync, or no desync occured - lfs2_pair_tole32(newpair); - err = lfs2_dir_commit(lfs2, &parent, LFS2_MKATTRS( - {LFS2_MKTAG_IF(moveid != 0x3ff, - LFS2_TYPE_DELETE, moveid, 0), NULL}, - {LFS2_MKTAG(LFS2_TYPE_TAIL + parent.split, 0x3ff, 8), newpair})); - lfs2_pair_fromle32(newpair); - if (err) { - return err; - } - } - - return 0; -} -#endif - -#ifndef LFS2_READONLY -static void lfs2_fs_preporphans(lfs2_t *lfs2, int8_t orphans) { +static int lfs2_fs_preporphans(lfs2_t *lfs2, int8_t orphans) { LFS2_ASSERT(lfs2_tag_size(lfs2->gstate.tag) > 0 || orphans >= 0); lfs2->gstate.tag += orphans; lfs2->gstate.tag = ((lfs2->gstate.tag & ~LFS2_MKTAG(0x800, 0, 0)) | ((uint32_t)lfs2_gstate_hasorphans(&lfs2->gstate) << 31)); + + return 0; } #endif @@ -4105,78 +4476,129 @@ static int lfs2_fs_demove(lfs2_t *lfs2) { #endif #ifndef LFS2_READONLY -static int lfs2_fs_deorphan(lfs2_t *lfs2) { +static int lfs2_fs_deorphan(lfs2_t *lfs2, bool powerloss) { if (!lfs2_gstate_hasorphans(&lfs2->gstate)) { return 0; } - // Fix any orphans - lfs2_mdir_t pdir = {.split = true, .tail = {0, 1}}; - lfs2_mdir_t dir; + int8_t found = 0; +restart: + { + // Fix any orphans + lfs2_mdir_t pdir = {.split = true, .tail = {0, 1}}; + lfs2_mdir_t dir; - // iterate over all directory directory entries - while (!lfs2_pair_isnull(pdir.tail)) { - int err = lfs2_dir_fetch(lfs2, &dir, pdir.tail); - if (err) { - return err; - } - - // check head blocks for orphans - if (!pdir.split) { - // check if we have a parent - lfs2_mdir_t parent; - lfs2_stag_t tag = lfs2_fs_parent(lfs2, pdir.tail, &parent); - if (tag < 0 && tag != LFS2_ERR_NOENT) { - return tag; + // iterate over all directory directory entries + while (!lfs2_pair_isnull(pdir.tail)) { + int err = lfs2_dir_fetch(lfs2, &dir, pdir.tail); + if (err) { + return err; } - if (tag == LFS2_ERR_NOENT) { - // we are an orphan - LFS2_DEBUG("Fixing orphan {0x%"PRIx32", 0x%"PRIx32"}", - pdir.tail[0], pdir.tail[1]); - - err = lfs2_dir_drop(lfs2, &pdir, &dir); - if (err) { - return err; + // check head blocks for orphans + if (!pdir.split) { + // check if we have a parent + lfs2_mdir_t parent; + lfs2_stag_t tag = lfs2_fs_parent(lfs2, pdir.tail, &parent); + if (tag < 0 && tag != LFS2_ERR_NOENT) { + return tag; } - // refetch tail - continue; - } + // note we only check for full orphans if we may have had a + // power-loss, otherwise orphans are created intentionally + // during operations such as lfs2_mkdir + if (tag == LFS2_ERR_NOENT && powerloss) { + // we are an orphan + LFS2_DEBUG("Fixing orphan {0x%"PRIx32", 0x%"PRIx32"}", + pdir.tail[0], pdir.tail[1]); - lfs2_block_t pair[2]; - lfs2_stag_t res = lfs2_dir_get(lfs2, &parent, - LFS2_MKTAG(0x7ff, 0x3ff, 0), tag, pair); - if (res < 0) { - return res; - } - lfs2_pair_fromle32(pair); + // steal state + err = lfs2_dir_getgstate(lfs2, &dir, &lfs2->gdelta); + if (err) { + return err; + } - if (!lfs2_pair_sync(pair, pdir.tail)) { - // we have desynced - LFS2_DEBUG("Fixing half-orphan {0x%"PRIx32", 0x%"PRIx32"} " - "-> {0x%"PRIx32", 0x%"PRIx32"}", - pdir.tail[0], pdir.tail[1], pair[0], pair[1]); + // steal tail + lfs2_pair_tole32(dir.tail); + int state = lfs2_dir_orphaningcommit(lfs2, &pdir, LFS2_MKATTRS( + {LFS2_MKTAG(LFS2_TYPE_TAIL + dir.split, 0x3ff, 8), + dir.tail})); + lfs2_pair_fromle32(dir.tail); + if (state < 0) { + return state; + } - lfs2_pair_tole32(pair); - err = lfs2_dir_commit(lfs2, &pdir, LFS2_MKATTRS( - {LFS2_MKTAG(LFS2_TYPE_SOFTTAIL, 0x3ff, 8), pair})); - lfs2_pair_fromle32(pair); - if (err) { - return err; + found += 1; + + // did our commit create more orphans? + if (state == LFS2_OK_ORPHANED) { + goto restart; + } + + // refetch tail + continue; } - // refetch tail - continue; - } - } + if (tag != LFS2_ERR_NOENT) { + lfs2_block_t pair[2]; + lfs2_stag_t state = lfs2_dir_get(lfs2, &parent, + LFS2_MKTAG(0x7ff, 0x3ff, 0), tag, pair); + if (state < 0) { + return state; + } + lfs2_pair_fromle32(pair); - pdir = dir; + if (!lfs2_pair_sync(pair, pdir.tail)) { + // we have desynced + LFS2_DEBUG("Fixing half-orphan " + "{0x%"PRIx32", 0x%"PRIx32"} " + "-> {0x%"PRIx32", 0x%"PRIx32"}", + pdir.tail[0], pdir.tail[1], pair[0], pair[1]); + + // fix pending move in this pair? this looks like an + // optimization but is in fact _required_ since + // relocating may outdate the move. + uint16_t moveid = 0x3ff; + if (lfs2_gstate_hasmovehere(&lfs2->gstate, pdir.pair)) { + moveid = lfs2_tag_id(lfs2->gstate.tag); + LFS2_DEBUG("Fixing move while fixing orphans " + "{0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16"\n", + pdir.pair[0], pdir.pair[1], moveid); + lfs2_fs_prepmove(lfs2, 0x3ff, NULL); + } + + lfs2_pair_tole32(pair); + state = lfs2_dir_orphaningcommit(lfs2, &pdir, LFS2_MKATTRS( + {LFS2_MKTAG_IF(moveid != 0x3ff, + LFS2_TYPE_DELETE, moveid, 0), NULL}, + {LFS2_MKTAG(LFS2_TYPE_SOFTTAIL, 0x3ff, 8), + pair})); + lfs2_pair_fromle32(pair); + if (state < 0) { + return state; + } + + found += 1; + + // did our commit create more orphans? + if (state == LFS2_OK_ORPHANED) { + goto restart; + } + + // refetch tail + continue; + } + } + } + + pdir = dir; + } } // mark orphans as fixed - lfs2_fs_preporphans(lfs2, -lfs2_gstate_getorphans(&lfs2->gstate)); - return 0; + return lfs2_fs_preporphans(lfs2, -lfs2_min( + lfs2_gstate_getorphans(&lfs2->gstate), + found)); } #endif @@ -4187,7 +4609,7 @@ static int lfs2_fs_forceconsistency(lfs2_t *lfs2) { return err; } - err = lfs2_fs_deorphan(lfs2); + err = lfs2_fs_deorphan(lfs2, true); if (err) { return err; } @@ -4725,7 +5147,7 @@ static int lfs2_rawmigrate(lfs2_t *lfs2, const struct lfs2_config *cfg) { lfs21_entry_tole32(&entry1.d); err = lfs2_dir_commit(lfs2, &dir2, LFS2_MKATTRS( - {LFS2_MKTAG(LFS2_TYPE_CREATE, id, 0)}, + {LFS2_MKTAG(LFS2_TYPE_CREATE, id, 0), NULL}, {LFS2_MKTAG_IF_ELSE(isdir, LFS2_TYPE_DIR, id, entry1.d.nlen, LFS2_TYPE_REG, id, entry1.d.nlen), @@ -4830,7 +5252,7 @@ static int lfs2_rawmigrate(lfs2_t *lfs2, const struct lfs2_config *cfg) { lfs2_superblock_tole32(&superblock); err = lfs2_dir_commit(lfs2, &dir2, LFS2_MKATTRS( - {LFS2_MKTAG(LFS2_TYPE_CREATE, 0, 0)}, + {LFS2_MKTAG(LFS2_TYPE_CREATE, 0, 0), NULL}, {LFS2_MKTAG(LFS2_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, {LFS2_MKTAG(LFS2_TYPE_INLINESTRUCT, 0, sizeof(superblock)), &superblock})); @@ -5044,6 +5466,7 @@ int lfs2_removeattr(lfs2_t *lfs2, const char *path, uint8_t type) { } #endif +#ifndef LFS2_NO_MALLOC int lfs2_file_open(lfs2_t *lfs2, lfs2_file_t *file, const char *path, int flags) { int err = LFS2_LOCK(lfs2->cfg); if (err) { @@ -5059,6 +5482,7 @@ int lfs2_file_open(lfs2_t *lfs2, lfs2_file_t *file, const char *path, int flags) LFS2_UNLOCK(lfs2->cfg); return err; } +#endif int lfs2_file_opencfg(lfs2_t *lfs2, lfs2_file_t *file, const char *path, int flags, diff --git a/lib/littlefs/lfs2.h b/lib/littlefs/lfs2.h index f3b66d76ff..715764f7ce 100644 --- a/lib/littlefs/lfs2.h +++ b/lib/littlefs/lfs2.h @@ -1,6 +1,7 @@ /* * The little filesystem * + * Copyright (c) 2022, The littlefs authors. * Copyright (c) 2017, Arm Limited. All rights reserved. * SPDX-License-Identifier: BSD-3-Clause */ @@ -22,7 +23,7 @@ extern "C" // Software library version // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS2_VERSION 0x00020003 +#define LFS2_VERSION 0x00020005 #define LFS2_VERSION_MAJOR (0xffff & (LFS2_VERSION >> 16)) #define LFS2_VERSION_MINOR (0xffff & (LFS2_VERSION >> 0)) @@ -159,49 +160,49 @@ struct lfs2_config { // information to the block device operations void *context; - // Read a region in a block. Negative error codes are propogated + // Read a region in a block. Negative error codes are propagated // to the user. int (*read)(const struct lfs2_config *c, lfs2_block_t block, lfs2_off_t off, void *buffer, lfs2_size_t size); // Program a region in a block. The block must have previously - // been erased. Negative error codes are propogated to the user. + // been erased. Negative error codes are propagated to the user. // May return LFS2_ERR_CORRUPT if the block should be considered bad. int (*prog)(const struct lfs2_config *c, lfs2_block_t block, lfs2_off_t off, const void *buffer, lfs2_size_t size); // Erase a block. A block must be erased before being programmed. // The state of an erased block is undefined. Negative error codes - // are propogated to the user. + // are propagated to the user. // May return LFS2_ERR_CORRUPT if the block should be considered bad. int (*erase)(const struct lfs2_config *c, lfs2_block_t block); // Sync the state of the underlying block device. Negative error codes - // are propogated to the user. + // are propagated to the user. int (*sync)(const struct lfs2_config *c); #ifdef LFS2_THREADSAFE // Lock the underlying block device. Negative error codes - // are propogated to the user. + // are propagated to the user. int (*lock)(const struct lfs2_config *c); // Unlock the underlying block device. Negative error codes - // are propogated to the user. + // are propagated to the user. int (*unlock)(const struct lfs2_config *c); #endif - // Minimum size of a block read. All read operations will be a + // Minimum size of a block read in bytes. All read operations will be a // multiple of this value. lfs2_size_t read_size; - // Minimum size of a block program. All program operations will be a - // multiple of this value. + // Minimum size of a block program in bytes. All program operations will be + // a multiple of this value. lfs2_size_t prog_size; - // Size of an erasable block. This does not impact ram consumption and - // may be larger than the physical erase size. However, non-inlined files - // take up at minimum one block. Must be a multiple of the read - // and program sizes. + // Size of an erasable block in bytes. This does not impact ram consumption + // and may be larger than the physical erase size. However, non-inlined + // files take up at minimum one block. Must be a multiple of the read and + // program sizes. lfs2_size_t block_size; // Number of erasable blocks on the device. @@ -215,11 +216,11 @@ struct lfs2_config { // Set to -1 to disable block-level wear-leveling. int32_t block_cycles; - // Size of block caches. Each cache buffers a portion of a block in RAM. - // The littlefs needs a read cache, a program cache, and one additional + // Size of block caches in bytes. Each cache buffers a portion of a block in + // RAM. The littlefs needs a read cache, a program cache, and one additional // cache per file. Larger caches can improve performance by storing more - // data and reducing the number of disk accesses. Must be a multiple of - // the read and program sizes, and a factor of the block size. + // data and reducing the number of disk accesses. Must be a multiple of the + // read and program sizes, and a factor of the block size. lfs2_size_t cache_size; // Size of the lookahead buffer in bytes. A larger lookahead buffer @@ -256,6 +257,12 @@ struct lfs2_config { // larger attributes size but must be <= LFS2_ATTR_MAX. Defaults to // LFS2_ATTR_MAX when zero. lfs2_size_t attr_max; + + // Optional upper limit on total space given to metadata pairs in bytes. On + // devices with large blocks (e.g. 128kB) setting this to a low size (2-8kB) + // can help bound the metadata compaction time. Must be <= block_size. + // Defaults to block_size when zero. + lfs2_size_t metadata_max; }; // File info structure @@ -479,7 +486,7 @@ int lfs2_stat(lfs2_t *lfs2, const char *path, struct lfs2_info *info); // Returns the size of the attribute, or a negative error code on failure. // Note, the returned size is the size of the attribute on disk, irrespective // of the size of the buffer. This can be used to dynamically allocate a buffer -// or check for existance. +// or check for existence. lfs2_ssize_t lfs2_getattr(lfs2_t *lfs2, const char *path, uint8_t type, void *buffer, lfs2_size_t size); @@ -507,6 +514,7 @@ int lfs2_removeattr(lfs2_t *lfs2, const char *path, uint8_t type); /// File operations /// +#ifndef LFS2_NO_MALLOC // Open a file // // The mode that the file is opened in is determined by the flags, which @@ -516,6 +524,10 @@ int lfs2_removeattr(lfs2_t *lfs2, const char *path, uint8_t type); int lfs2_file_open(lfs2_t *lfs2, lfs2_file_t *file, const char *path, int flags); +// if LFS2_NO_MALLOC is defined, lfs2_file_open() will fail with LFS2_ERR_NOMEM +// thus use lfs2_file_opencfg() with config.buffer set. +#endif + // Open a file with extra configuration // // The mode that the file is opened in is determined by the flags, which diff --git a/lib/littlefs/lfs2_util.c b/lib/littlefs/lfs2_util.c index 083a99c36c..c9850e7886 100644 --- a/lib/littlefs/lfs2_util.c +++ b/lib/littlefs/lfs2_util.c @@ -1,6 +1,7 @@ /* * lfs2 util functions * + * Copyright (c) 2022, The littlefs authors. * Copyright (c) 2017, Arm Limited. All rights reserved. * SPDX-License-Identifier: BSD-3-Clause */ diff --git a/lib/littlefs/lfs2_util.h b/lib/littlefs/lfs2_util.h index 70bca717c9..6a4c8ffb50 100644 --- a/lib/littlefs/lfs2_util.h +++ b/lib/littlefs/lfs2_util.h @@ -1,6 +1,7 @@ /* * lfs2 utility functions * + * Copyright (c) 2022, The littlefs authors. * Copyright (c) 2017, Arm Limited. All rights reserved. * SPDX-License-Identifier: BSD-3-Clause */ @@ -49,6 +50,7 @@ extern "C" // code footprint // Logging functions +#ifndef LFS2_TRACE #ifdef LFS2_YES_TRACE #define LFS2_TRACE_(fmt, ...) \ printf("%s:%d:trace: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__) @@ -56,7 +58,9 @@ extern "C" #else #define LFS2_TRACE(...) #endif +#endif +#ifndef LFS2_DEBUG #ifndef LFS2_NO_DEBUG #define LFS2_DEBUG_(fmt, ...) \ printf("%s:%d:debug: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__) @@ -64,7 +68,9 @@ extern "C" #else #define LFS2_DEBUG(...) #endif +#endif +#ifndef LFS2_WARN #ifndef LFS2_NO_WARN #define LFS2_WARN_(fmt, ...) \ printf("%s:%d:warn: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__) @@ -72,7 +78,9 @@ extern "C" #else #define LFS2_WARN(...) #endif +#endif +#ifndef LFS2_ERROR #ifndef LFS2_NO_ERROR #define LFS2_ERROR_(fmt, ...) \ printf("%s:%d:error: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__) @@ -80,13 +88,16 @@ extern "C" #else #define LFS2_ERROR(...) #endif +#endif // Runtime assertions +#ifndef LFS2_ASSERT #ifndef LFS2_NO_ASSERT #define LFS2_ASSERT(test) assert(test) #else #define LFS2_ASSERT(test) #endif +#endif // Builtin functions, these may be replaced by more efficient diff --git a/lib/oofatfs/ffconf.h b/lib/oofatfs/ffconf.h index 52ff6d0a7a..35c010084a 100644 --- a/lib/oofatfs/ffconf.h +++ b/lib/oofatfs/ffconf.h @@ -6,7 +6,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2019 Damien P. George + * Copyright (c) 2013-2019 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/lib/re1.5/re1.5.h b/lib/re1.5/re1.5.h index 995e2d4d21..43a0416160 100644 --- a/lib/re1.5/re1.5.h +++ b/lib/re1.5/re1.5.h @@ -132,6 +132,7 @@ Sub *update(Sub*, int, const char*); void decref(Sub*); struct Subject { + const char *begin_line; const char *begin; const char *end; }; diff --git a/lib/re1.5/recursiveloop.c b/lib/re1.5/recursiveloop.c index f8cb926292..17ecea3378 100644 --- a/lib/re1.5/recursiveloop.c +++ b/lib/re1.5/recursiveloop.c @@ -68,7 +68,7 @@ recursiveloop(char *pc, const char *sp, Subject *input, const char **subp, int n subp[off] = old; return 0; case Bol: - if(sp != input->begin) + if(sp != input->begin_line) return 0; continue; case Eol: diff --git a/mpy-cross/gccollect.c b/mpy-cross/gccollect.c index 79c47f3b68..72d4204c28 100644 --- a/mpy-cross/gccollect.c +++ b/mpy-cross/gccollect.c @@ -1,7 +1,28 @@ -// SPDX-FileCopyrightText: Copyright (c) 2013-2014 Damien P. George -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2014 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include diff --git a/mpy-cross/main.c b/mpy-cross/main.c index 077e84f473..88d84915f8 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -1,13 +1,35 @@ -// SPDX-FileCopyrightText: Copyright (c) 2013-2016 Damien P. George -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2016 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include #include #include #include +#include "py/builtin.h" #include "py/compile.h" #include "py/persistentcode.h" #include "py/runtime.h" @@ -51,7 +73,8 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha #endif mp_parse_tree_t parse_tree = mp_parse(lex, MP_PARSE_FILE_INPUT); - mp_raw_code_t *rc = mp_compile_to_raw_code(&parse_tree, source_name, false); + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + mp_compiled_module_t cm = mp_compile_to_raw_code(&parse_tree, source_name, false, ctx); vstr_t vstr; vstr_init(&vstr, 16); @@ -62,7 +85,7 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha } else { vstr_add_str(&vstr, output_file); } - mp_raw_code_save_file(rc, vstr_null_terminated_str(&vstr)); + mp_raw_code_save_file(&cm, vstr_null_terminated_str(&vstr)); vstr_clear(&vstr); nlr_pop(); @@ -86,8 +109,7 @@ STATIC int usage(char **argv) { "\n" "Target specific options:\n" "-msmall-int-bits=number : set the maximum bits used to encode a small-int\n" - "-mno-unicode : don't support unicode in compiled strings\n" - "-march= : set architecture for native emitter; x86, x64, armv6, armv7m, armv7em, armv7emsp, armv7emdp, xtensa, xtensawin\n" + "-march= : set architecture for native emitter; x86, x64, armv6, armv6m, armv7m, armv7em, armv7emsp, armv7emdp, xtensa, xtensawin\n" "\n" "Implementation specific options:\n", argv[0] ); @@ -181,7 +203,6 @@ MP_NOINLINE int main_(int argc, char **argv) { // set default compiler configuration mp_dynamic_compiler.small_int_bits = 31; - mp_dynamic_compiler.py_builtins_str_unicode = 1; #if defined(__i386__) mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_X86; mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_X86; @@ -239,10 +260,6 @@ MP_NOINLINE int main_(int argc, char **argv) { return usage(argv); } // TODO check that small_int_bits is within range of host's capabilities - } else if (strcmp(argv[a], "-mno-unicode") == 0) { - mp_dynamic_compiler.py_builtins_str_unicode = 0; - } else if (strcmp(argv[a], "-municode") == 0) { - mp_dynamic_compiler.py_builtins_str_unicode = 1; } else if (strncmp(argv[a], "-march=", sizeof("-march=") - 1) == 0) { const char *arch = argv[a] + sizeof("-march=") - 1; if (strcmp(arch, "x86") == 0) { @@ -254,6 +271,9 @@ MP_NOINLINE int main_(int argc, char **argv) { } else if (strcmp(arch, "armv6") == 0) { mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_ARMV6; mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_ARM_THUMB_FP; + } else if (strcmp(arch, "armv6m") == 0) { + mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_ARMV6M; + mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_ARM_THUMB_FP; // need to be conservative so this code can run on armv7emdp } else if (strcmp(arch, "armv7m") == 0) { mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_ARMV7M; mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_ARM_THUMB_FP; diff --git a/mpy-cross/mpconfigport.h b/mpy-cross/mpconfigport.h index d91e1d6d6e..c4c4c9a4b0 100644 --- a/mpy-cross/mpconfigport.h +++ b/mpy-cross/mpconfigport.h @@ -1,7 +1,28 @@ -// SPDX-FileCopyrightText: Copyright (c) 2013-2015 Damien P. George -// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) -// -// SPDX-License-Identifier: MIT +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2015 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ // options to control how MicroPython is built @@ -21,8 +42,6 @@ #define MICROPY_EMIT_X86 (1) #define MICROPY_EMIT_THUMB (1) #define MICROPY_EMIT_INLINE_THUMB (1) -#define MICROPY_EMIT_INLINE_THUMB_ARMV7M (1) -#define MICROPY_EMIT_INLINE_THUMB_FLOAT (1) #define MICROPY_EMIT_ARM (1) #define MICROPY_EMIT_XTENSA (1) #define MICROPY_EMIT_INLINE_XTENSA (1) @@ -163,7 +182,7 @@ typedef long mp_off_t; #define MP_NOINLINE __declspec(noinline) #define MP_LIKELY(x) (x) #define MP_UNLIKELY(x) (x) -#define MICROPY_PORT_CONSTANTS { "dummy", 0 } +#define MICROPY_PORT_CONSTANTS { MP_ROM_QSTR(MP_QSTR_dummy), MP_ROM_PTR(NULL) } #ifdef _WIN64 #define MP_SSIZE_MAX _I64_MAX #else diff --git a/mpy-cross/mpy-cross.vcxproj b/mpy-cross/mpy-cross.vcxproj index e70b29ae14..53cb0fa1fe 100644 --- a/mpy-cross/mpy-cross.vcxproj +++ b/mpy-cross/mpy-cross.vcxproj @@ -1,5 +1,5 @@  - + Debug diff --git a/ports/espressif/common-hal/hashlib/Hash.c b/ports/espressif/common-hal/hashlib/Hash.c index 8090128acb..295fd0bdf8 100644 --- a/ports/espressif/common-hal/hashlib/Hash.c +++ b/ports/espressif/common-hal/hashlib/Hash.c @@ -3,7 +3,12 @@ * * The MIT License (MIT) * +<<<<<<<< HEAD:ports/espressif/common-hal/hashlib/Hash.c * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries +======== + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2022 Renesas Electronics Corporation +>>>>>>>> v1.19.1:ports/renesas-ra/moduos.c * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,6 +29,7 @@ * THE SOFTWARE. */ +<<<<<<<< HEAD:ports/espressif/common-hal/hashlib/Hash.c #include "shared-bindings/hashlib/Hash.h" #include "components/mbedtls/mbedtls/include/mbedtls/ssl.h" @@ -54,4 +60,40 @@ size_t common_hal_hashlib_hash_get_digest_size(hashlib_hash_obj_t *self) { return 20; } return 0; +======== +#include "py/runtime.h" +#include "uart.h" + +#if MICROPY_VFS_FAT +#include "lib/oofatfs/ff.h" +#include "lib/oofatfs/diskio.h" +#endif + +// sync() +// Sync all filesystems. +STATIC mp_obj_t mp_uos_sync(void) { + #if MICROPY_VFS_FAT + for (mp_vfs_mount_t *vfs = MP_STATE_VM(vfs_mount_table); vfs != NULL; vfs = vfs->next) { + // this assumes that vfs->obj is fs_user_mount_t with block device functions + disk_ioctl(MP_OBJ_TO_PTR(vfs->obj), CTRL_SYNC, NULL); + } + #endif + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_0(mp_uos_sync_obj, mp_uos_sync); + +bool mp_uos_dupterm_is_builtin_stream(mp_const_obj_t stream) { + const mp_obj_type_t *type = mp_obj_get_type(stream); + return type == &machine_uart_type; +} + +void mp_uos_dupterm_stream_detached_attached(mp_obj_t stream_detached, mp_obj_t stream_attached) { + if (mp_obj_get_type(stream_detached) == &machine_uart_type) { + uart_attach_to_repl(MP_OBJ_TO_PTR(stream_detached), false); + } + + if (mp_obj_get_type(stream_attached) == &machine_uart_type) { + uart_attach_to_repl(MP_OBJ_TO_PTR(stream_attached), true); + } +>>>>>>>> v1.19.1:ports/renesas-ra/moduos.c } diff --git a/ports/mimxrt10xx/boards/teensy40/board.c b/ports/mimxrt10xx/boards/teensy40/board.c index 8ece1546d7..4f92b4f623 100644 --- a/ports/mimxrt10xx/boards/teensy40/board.c +++ b/ports/mimxrt10xx/boards/teensy40/board.c @@ -25,8 +25,15 @@ * THE SOFTWARE. */ +<<<<<<<< HEAD:ports/mimxrt10xx/boards/teensy40/board.c #include "supervisor/board.h" #include "shared-bindings/microcontroller/Pin.h" +======== +#include +#include "py/mpconfig.h" +#include "fsl_edma.h" +#include "dma_manager.h" +>>>>>>>> v1.19.1:ports/raspberrypi/dma_manager.c // These pins should never ever be reset; doing so could interfere with basic operation. // Used in common-hal/microcontroller/Pin.c @@ -54,4 +61,35 @@ const mcu_pin_obj_t *mimxrt10xx_reset_forbidden_pins[] = { NULL, // Must end in NULL. }; +<<<<<<<< HEAD:ports/mimxrt10xx/boards/teensy40/board.c // Use the MP_WEAK supervisor/shared/board.c versions of routines not defined here. +======== +STATIC bool dma_initialized = false; + +// allocate_channel(): retrieve an available channel. Return the number or -1 +int allocate_dma_channel(void) { + for (int i = 0; i < ARRAY_SIZE(channel_list); i++) { + if (channel_list[i] == false) { // Channel available + channel_list[i] = true; + return i; + } + } + return -1; +} + +// free_channel(n): Declare channel as free +void free_dma_channel(int n) { + if (n >= 0 && n <= ARRAY_SIZE(channel_list)) { + channel_list[n] = false; + } +} + +void dma_init(void) { + if (!dma_initialized) { + edma_config_t dmaConfig; + EDMA_GetDefaultConfig(&dmaConfig); + EDMA_Init(DMA0, &dmaConfig); + dma_initialized = true; + } +} +>>>>>>>> v1.19.1:ports/raspberrypi/dma_manager.c diff --git a/ports/nrf/boards/pca10100/mpconfigboard.h b/ports/nrf/boards/pca10100/mpconfigboard.h index 1456f29949..8b1ff19a89 100644 --- a/ports/nrf/boards/pca10100/mpconfigboard.h +++ b/ports/nrf/boards/pca10100/mpconfigboard.h @@ -31,8 +31,6 @@ #define MICROPY_HW_MCU_NAME "nRF52833" #define MICROPY_HW_LED_STATUS (&pin_P0_13) -#define MICROPY_HW_LED_TX (&pin_P0_14) -#define MICROPY_HW_LED_RX (&pin_P0_15) #define CIRCUITPY_INTERNAL_NVM_SIZE 0 #define CIRCUITPY_INTERNAL_FLASH_FILESYSTEM_SIZE (60 * 1024) diff --git a/ports/nrf/boards/pca10100/pins.c b/ports/nrf/boards/pca10100/pins.c index 68d87d0e8f..e6ca54eb2e 100644 --- a/ports/nrf/boards/pca10100/pins.c +++ b/ports/nrf/boards/pca10100/pins.c @@ -45,10 +45,14 @@ STATIC const mp_rom_map_elem_t board_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR_LED3), MP_ROM_PTR(&pin_P0_15) }, { MP_ROM_QSTR(MP_QSTR_LED4), MP_ROM_PTR(&pin_P0_16) }, + { MP_ROM_QSTR(MP_QSTR_BUTTON1), MP_ROM_PTR(&pin_P0_11) }, { MP_ROM_QSTR(MP_QSTR_BUTTON1_DEFAULT), MP_ROM_PTR(&pin_P0_11) }, { MP_ROM_QSTR(MP_QSTR_BUTTON1_OPTIONAL), MP_ROM_PTR(&pin_P1_07) }, + + { MP_ROM_QSTR(MP_QSTR_BUTTON2), MP_ROM_PTR(&pin_P0_12) }, { MP_ROM_QSTR(MP_QSTR_BUTTON2_DEFAULT), MP_ROM_PTR(&pin_P0_12) }, { MP_ROM_QSTR(MP_QSTR_BUTTON2_OPTIONAL), MP_ROM_PTR(&pin_P1_08) }, + { MP_ROM_QSTR(MP_QSTR_BUTTON3), MP_ROM_PTR(&pin_P0_24) }, { MP_ROM_QSTR(MP_QSTR_BUTTON4), MP_ROM_PTR(&pin_P0_25) }, }; diff --git a/ports/nrf/mpconfigport.mk b/ports/nrf/mpconfigport.mk index 7973453c2f..58700817d5 100644 --- a/ports/nrf/mpconfigport.mk +++ b/ports/nrf/mpconfigport.mk @@ -20,6 +20,7 @@ CIRCUITPY_AUDIOBUSIO ?= 1 CIRCUITPY_AUDIOCORE ?= 1 CIRCUITPY_AUDIOMIXER ?= 1 CIRCUITPY_AUDIOPWMIO ?= 1 +CIRCUITPY_SYNTHIO_MAX_CHANNELS = 12 # Native BLEIO is not compatible with HCI _bleio. CIRCUITPY_BLEIO_HCI = 0 diff --git a/ports/raspberrypi/common-hal/board/__init__.c b/ports/raspberrypi/common-hal/board/__init__.c index 3c7f30df22..13462afe4d 100644 --- a/ports/raspberrypi/common-hal/board/__init__.c +++ b/ports/raspberrypi/common-hal/board/__init__.c @@ -30,5 +30,13 @@ #include "py/mphal.h" #include "common-hal/microcontroller/Pin.h" +<<<<<<<< HEAD:ports/raspberrypi/common-hal/board/__init__.c // Pins aren't actually defined here. They are in the board specific directory // such as boards/arduino_zero/pins.c. +======== +int allocate_dma_channel(void); +void free_dma_channel(int n); +void dma_init(void); + +#endif // MICROPY_INCLUDED_MIMXRT_DMACHANNEL_H +>>>>>>>> v1.19.1:ports/raspberrypi/dma_manager.h diff --git a/ports/raspberrypi/mbedtls/mbedtls_config.h b/ports/raspberrypi/mbedtls/mbedtls_config.h index 3791924a13..c182bc2c6b 100644 --- a/ports/raspberrypi/mbedtls/mbedtls_config.h +++ b/ports/raspberrypi/mbedtls/mbedtls_config.h @@ -81,9 +81,13 @@ #define MBEDTLS_BIGNUM_C #define MBEDTLS_CIPHER_C #define MBEDTLS_CTR_DRBG_C +<<<<<<<< HEAD:ports/raspberrypi/mbedtls/mbedtls_config.h #define MBEDTLS_ECDH_C #define MBEDTLS_ECDSA_C #define MBEDTLS_ECP_C +======== +// #define MBEDTLS_ECP_C +>>>>>>>> v1.19.1:ports/stm32/mbedtls/mbedtls_config.h #define MBEDTLS_ENTROPY_C #define MBEDTLS_ERROR_C #define MBEDTLS_GCM_C diff --git a/ports/silabs/license.md b/ports/silabs/license.md index 5b8797814f..4833861a01 100644 --- a/ports/silabs/license.md +++ b/ports/silabs/license.md @@ -1,4 +1,4 @@ -MIT License +The MIT License (MIT) Copyright (c) 2013-2022 Damien P. George and others diff --git a/ports/stm/boards/stm32f746g_discovery/board.c b/ports/stm/boards/stm32f746g_discovery/board.c deleted file mode 100644 index 56c90bd8c6..0000000000 --- a/ports/stm/boards/stm32f746g_discovery/board.c +++ /dev/null @@ -1,51 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2017 Scott Shawcroft for Adafruit Industries - * Copyright (c) 2020 Mark Olsson - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "supervisor/board.h" -#include "stm32f7xx_hal.h" -#include "common-hal/microcontroller/Pin.h" - -void board_init(void) { - GPIO_InitTypeDef GPIO_InitStructure; - - __HAL_RCC_GPIOK_CLK_ENABLE(); - - /* - * Turn off the backlight as it is distracting during development. - * LCD_BL_CTRL = PK3 - */ - GPIO_InitStructure.Pin = GPIO_PIN_3; - GPIO_InitStructure.Mode = GPIO_MODE_OUTPUT_PP; - GPIO_InitStructure.Pull = GPIO_PULLUP; - GPIO_InitStructure.Speed = GPIO_SPEED_FREQ_VERY_HIGH; - HAL_GPIO_Init(GPIOK, &GPIO_InitStructure); - HAL_GPIO_WritePin(GPIOK, GPIO_PIN_3, GPIO_PIN_RESET); - - never_reset_pin_number(10, 3); -} - -// Use the MP_WEAK supervisor/shared/board.c versions of routines not defined here. diff --git a/ports/stm/peripherals/stm32f4/stm32f411xe/clocks.h b/ports/stm/peripherals/stm32f4/stm32f411xe/clocks.h index 0b38d610cb..ea8b33cfff 100644 --- a/ports/stm/peripherals/stm32f4/stm32f411xe/clocks.h +++ b/ports/stm/peripherals/stm32f4/stm32f411xe/clocks.h @@ -3,7 +3,11 @@ * * The MIT License (MIT) * +<<<<<<<< HEAD:ports/stm/peripherals/stm32f4/stm32f411xe/clocks.h * Copyright (c) 2020 Lucian Copeland for Adafruit Industries +======== + * Copyright (c) 2022 Arduino SA +>>>>>>>> v1.19.1:ports/stm/boards/ARDUINO_PORTENTA_H7/bdev.c * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,6 +28,7 @@ * THE SOFTWARE. */ +<<<<<<<< HEAD:ports/stm/peripherals/stm32f4/stm32f411xe/clocks.h #include "stm32f4xx_hal.h" // Chip: STM32F411 @@ -65,3 +70,24 @@ #ifndef BOARD_HSE_SOURCE #define BOARD_HSE_SOURCE (RCC_HSE_ON) #endif +======== +#include "storage.h" +#include "qspi.h" + +#if MICROPY_HW_SPIFLASH_ENABLE_CACHE +// Shared cache for first and second SPI block devices +STATIC mp_spiflash_cache_t spi_bdev_cache; +#endif + +// First external SPI flash uses hardware QSPI interface +const mp_spiflash_config_t spiflash_config = { + .bus_kind = MP_SPIFLASH_BUS_QSPI, + .bus.u_qspi.data = NULL, + .bus.u_qspi.proto = &qspi_proto, + #if MICROPY_HW_SPIFLASH_ENABLE_CACHE + .cache = &spi_bdev_cache, + #endif +}; + +spi_bdev_t spi_bdev; +>>>>>>>> v1.19.1:ports/stm/boards/ARDUINO_PORTENTA_H7/bdev.c diff --git a/ports/stm/peripherals/stm32f4/stm32f412cx/clocks.h b/ports/stm/peripherals/stm32f4/stm32f412cx/clocks.h index 20a7b41ba3..ff8b2b5ce1 100644 --- a/ports/stm/peripherals/stm32f4/stm32f412cx/clocks.h +++ b/ports/stm/peripherals/stm32f4/stm32f412cx/clocks.h @@ -3,7 +3,11 @@ * * The MIT License (MIT) * +<<<<<<<< HEAD:ports/stm/peripherals/stm32f4/stm32f412cx/clocks.h * Copyright (c) 2020 Lucian Copeland for Adafruit Industries +======== + * Copyright (c) 2013, 2014 Damien P. George +>>>>>>>> v1.19.1:ports/renesas-ra/gccollect.c * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,6 +28,7 @@ * THE SOFTWARE. */ +<<<<<<<< HEAD:ports/stm/peripherals/stm32f4/stm32f412cx/clocks.h #include "stm32f4xx_hal.h" // Chip: STM32F412Cx @@ -64,3 +69,50 @@ #ifndef BOARD_HSE_SOURCE #define BOARD_HSE_SOURCE (RCC_HSE_ON) #endif +======== +#include +#include + +#include "py/mpstate.h" +#include "py/gc.h" +#include "py/mpthread.h" +#include "shared/runtime/gchelper.h" +#include "gccollect.h" +#include "softtimer.h" +#include "systick.h" + +void gc_collect(void) { + // get current time, in case we want to time the GC + #if 0 + uint32_t start = mp_hal_ticks_us(); + #endif + + // start the GC + gc_collect_start(); + + // trace the stack and registers + gc_helper_collect_regs_and_stack(); + + // trace root pointers from any threads + #if MICROPY_PY_THREAD + mp_thread_gc_others(); + #endif + + // trace soft timer nodes + soft_timer_gc_mark_all(); + + // end the GC + gc_collect_end(); + + #if 0 + // print GC info + uint32_t ticks = mp_hal_ticks_us() - start; + gc_info_t info; + gc_info(&info); + printf("GC@%lu %lums\n", start, ticks); + printf(" " UINT_FMT " total\n", info.total); + printf(" " UINT_FMT " : " UINT_FMT "\n", info.used, info.free); + printf(" 1=" UINT_FMT " 2=" UINT_FMT " m=" UINT_FMT "\n", info.num_1block, info.num_2block, info.max_block); + #endif +} +>>>>>>>> v1.19.1:ports/renesas-ra/gccollect.c diff --git a/ports/unix/Makefile b/ports/unix/Makefile index 5141beff4c..ac3e2ef11d 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -29,7 +29,7 @@ UNAME_S := $(shell uname -s) # include py core make definitions include $(TOP)/py/py.mk -GIT_SUBMODULES = lib/axtls lib/berkeley-db-1.xx lib/libffi +GIT_SUBMODULES += lib/axtls lib/berkeley-db-1.xx lib/libffi INC += -I. INC += -I$(TOP) @@ -264,7 +264,6 @@ ifneq ($(FROZEN_MANIFEST),) # freeze, then invoke make with FROZEN_MANIFEST=manifest.py (be sure to build from scratch). CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool CFLAGS += -DMICROPY_MODULE_FROZEN_MPY -CFLAGS += -DMICROPY_MODULE_FROZEN_STR CFLAGS += -DMPZ_DIG_SIZE=16 # force 16 bits to work on both 32 and 64 bit archs CFLAGS += -DMICROPY_MODULE_FROZEN_STR endif @@ -367,5 +366,3 @@ install: $(PROG) uninstall: -rm $(BINDIR)/$(PROG) - -$(BUILD)/supervisor/shared/translate/translate.o: $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/compression.generated.h diff --git a/ports/unix/coverage.c b/ports/unix/coverage.c index b04bc2158c..188fa5e937 100644 --- a/ports/unix/coverage.c +++ b/ports/unix/coverage.c @@ -2,6 +2,7 @@ #include #include "py/obj.h" +#include "py/objfun.h" #include "py/objstr.h" #include "py/runtime.h" #include "py/gc.h" @@ -100,7 +101,6 @@ STATIC const mp_rom_map_elem_t rawfile_locals_dict_table[] = { STATIC MP_DEFINE_CONST_DICT(rawfile_locals_dict, rawfile_locals_dict_table); STATIC const mp_stream_p_t fileio_stream_p = { - MP_PROTO_IMPLEMENT(MP_QSTR_protocol_stream) .read = stest_read, .write = stest_write, .ioctl = stest_ioctl, @@ -108,11 +108,8 @@ STATIC const mp_stream_p_t fileio_stream_p = { STATIC const mp_obj_type_t mp_type_stest_fileio = { { &mp_type_type }, - .flags = MP_TYPE_FLAG_EXTENDED, + .protocol = &fileio_stream_p, .locals_dict = (mp_obj_dict_t *)&rawfile_locals_dict, - MP_TYPE_EXTENDED_FIELDS( - .protocol = &fileio_stream_p, - ), }; // stream read returns non-blocking error @@ -131,7 +128,6 @@ STATIC const mp_rom_map_elem_t rawfile_locals_dict_table2[] = { STATIC MP_DEFINE_CONST_DICT(rawfile_locals_dict2, rawfile_locals_dict_table2); STATIC const mp_stream_p_t textio_stream_p2 = { - MP_PROTO_IMPLEMENT(MP_QSTR_protocol_stream) .read = stest_read2, .write = NULL, .is_text = true, @@ -139,11 +135,8 @@ STATIC const mp_stream_p_t textio_stream_p2 = { STATIC const mp_obj_type_t mp_type_stest_textio2 = { { &mp_type_type }, - .flags = MP_TYPE_FLAG_EXTENDED, + .protocol = &textio_stream_p2, .locals_dict = (mp_obj_dict_t *)&rawfile_locals_dict2, - MP_TYPE_EXTENDED_FIELDS( - .protocol = &textio_stream_p2, - ), }; // str/bytes objects without a valid hash @@ -220,13 +213,62 @@ STATIC mp_obj_t extra_coverage(void) { gc_unlock(); // using gc_realloc to resize to 0, which means free the memory - void *p = gc_alloc(4, false, false); + void *p = gc_alloc(4, false); mp_printf(&mp_plat_print, "%p\n", gc_realloc(p, 0, false)); // calling gc_nbytes with a non-heap pointer mp_printf(&mp_plat_print, "%p\n", gc_nbytes(NULL)); } + // tracked allocation + { + #define NUM_PTRS (8) + #define NUM_BYTES (128) + #define FLIP_POINTER(p) ((uint8_t *)((uintptr_t)(p) ^ 0x0f)) + + mp_printf(&mp_plat_print, "# tracked allocation\n"); + mp_printf(&mp_plat_print, "m_tracked_head = %p\n", MP_STATE_VM(m_tracked_head)); + + uint8_t *ptrs[NUM_PTRS]; + + // allocate memory blocks + for (size_t i = 0; i < NUM_PTRS; ++i) { + ptrs[i] = m_tracked_calloc(1, NUM_BYTES); + bool all_zero = true; + for (size_t j = 0; j < NUM_BYTES; ++j) { + if (ptrs[i][j] != 0) { + all_zero = false; + break; + } + ptrs[i][j] = j; + } + mp_printf(&mp_plat_print, "%d %d\n", i, all_zero); + + // hide the pointer from the GC and collect + ptrs[i] = FLIP_POINTER(ptrs[i]); + gc_collect(); + } + + // check the memory blocks have the correct content + for (size_t i = 0; i < NUM_PTRS; ++i) { + bool correct_contents = true; + for (size_t j = 0; j < NUM_BYTES; ++j) { + if (FLIP_POINTER(ptrs[i])[j] != j) { + correct_contents = false; + break; + } + } + mp_printf(&mp_plat_print, "%d %d\n", i, correct_contents); + } + + // free the memory blocks + for (size_t i = 0; i < NUM_PTRS; ++i) { + m_tracked_free(FLIP_POINTER(ptrs[i])); + } + + mp_printf(&mp_plat_print, "m_tracked_head = %p\n", MP_STATE_VM(m_tracked_head)); + } + // vstr { mp_printf(&mp_plat_print, "# vstr\n"); @@ -457,7 +499,10 @@ STATIC mp_obj_t extra_coverage(void) { mp_printf(&mp_plat_print, "# VM\n"); // call mp_execute_bytecode with invalid bytecode (should raise NotImplementedError) + mp_module_context_t context; mp_obj_fun_bc_t fun_bc; + fun_bc.context = &context; + fun_bc.child_table = NULL; fun_bc.bytecode = (const byte *)"\x01"; // just needed for n_state mp_code_state_t *code_state = m_new_obj_var(mp_code_state_t, mp_obj_t, 1); code_state->fun_bc = &fun_bc; @@ -526,49 +571,47 @@ STATIC mp_obj_t extra_coverage(void) { // ringbuf { - #define RINGBUF_SIZE 99 - - byte buf[RINGBUF_SIZE]; - ringbuf_t ringbuf; - ringbuf_init(&ringbuf, &buf[0], sizeof(buf)); + byte buf[100]; + ringbuf_t ringbuf = {buf, sizeof(buf), 0, 0}; mp_printf(&mp_plat_print, "# ringbuf\n"); // Single-byte put/get with empty ringbuf. - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", ringbuf_free(&ringbuf), ringbuf_avail(&ringbuf)); ringbuf_put(&ringbuf, 22); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", ringbuf_free(&ringbuf), ringbuf_avail(&ringbuf)); mp_printf(&mp_plat_print, "%d\n", ringbuf_get(&ringbuf)); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", ringbuf_free(&ringbuf), ringbuf_avail(&ringbuf)); // Two-byte put/get with empty ringbuf. ringbuf_put16(&ringbuf, 0xaa55); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", ringbuf_free(&ringbuf), ringbuf_avail(&ringbuf)); mp_printf(&mp_plat_print, "%04x\n", ringbuf_get16(&ringbuf)); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", ringbuf_free(&ringbuf), ringbuf_avail(&ringbuf)); // Two-byte put with full ringbuf. - for (int i = 0; i < RINGBUF_SIZE; ++i) { + for (int i = 0; i < 99; ++i) { ringbuf_put(&ringbuf, i); } - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", ringbuf_free(&ringbuf), ringbuf_avail(&ringbuf)); mp_printf(&mp_plat_print, "%d\n", ringbuf_put16(&ringbuf, 0x11bb)); // Two-byte put with one byte free. ringbuf_get(&ringbuf); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", ringbuf_free(&ringbuf), ringbuf_avail(&ringbuf)); mp_printf(&mp_plat_print, "%d\n", ringbuf_put16(&ringbuf, 0x3377)); ringbuf_get(&ringbuf); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", ringbuf_free(&ringbuf), ringbuf_avail(&ringbuf)); mp_printf(&mp_plat_print, "%d\n", ringbuf_put16(&ringbuf, 0xcc99)); - for (int i = 0; i < RINGBUF_SIZE - 2; ++i) { + for (int i = 0; i < 97; ++i) { ringbuf_get(&ringbuf); } mp_printf(&mp_plat_print, "%04x\n", ringbuf_get16(&ringbuf)); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", ringbuf_free(&ringbuf), ringbuf_avail(&ringbuf)); // Two-byte put with wrap around on first byte: - ringbuf_clear(&ringbuf); - for (int i = 0; i < RINGBUF_SIZE; ++i) { + ringbuf.iput = 0; + ringbuf.iget = 0; + for (int i = 0; i < 99; ++i) { ringbuf_put(&ringbuf, i); ringbuf_get(&ringbuf); } @@ -576,8 +619,9 @@ STATIC mp_obj_t extra_coverage(void) { mp_printf(&mp_plat_print, "%04x\n", ringbuf_get16(&ringbuf)); // Two-byte put with wrap around on second byte: - ringbuf_clear(&ringbuf); - for (int i = 0; i < RINGBUF_SIZE - 1; ++i) { + ringbuf.iput = 0; + ringbuf.iget = 0; + for (int i = 0; i < 98; ++i) { ringbuf_put(&ringbuf, i); ringbuf_get(&ringbuf); } @@ -585,11 +629,13 @@ STATIC mp_obj_t extra_coverage(void) { mp_printf(&mp_plat_print, "%04x\n", ringbuf_get16(&ringbuf)); // Two-byte get from empty ringbuf. - ringbuf_clear(&ringbuf); + ringbuf.iput = 0; + ringbuf.iget = 0; mp_printf(&mp_plat_print, "%d\n", ringbuf_get16(&ringbuf)); // Two-byte get from ringbuf with one byte available. - ringbuf_clear(&ringbuf); + ringbuf.iput = 0; + ringbuf.iget = 0; ringbuf_put(&ringbuf, 0xaa); mp_printf(&mp_plat_print, "%d\n", ringbuf_get16(&ringbuf)); } @@ -642,14 +688,12 @@ STATIC mp_obj_t extra_coverage(void) { mp_printf(&mp_plat_print, "# end coverage.c\n"); - mp_obj_streamtest_t *s = m_new_obj(mp_obj_streamtest_t); - s->base.type = &mp_type_stest_fileio; + mp_obj_streamtest_t *s = mp_obj_malloc(mp_obj_streamtest_t, &mp_type_stest_fileio); s->buf = NULL; s->len = 0; s->pos = 0; s->error_code = 0; - mp_obj_streamtest_t *s2 = m_new_obj(mp_obj_streamtest_t); - s2->base.type = &mp_type_stest_textio2; + mp_obj_streamtest_t *s2 = mp_obj_malloc(mp_obj_streamtest_t, &mp_type_stest_textio2); // return a tuple of data for testing on the Python side mp_obj_t items[] = {(mp_obj_t)&str_no_hash_obj, (mp_obj_t)&bytes_no_hash_obj, MP_OBJ_FROM_PTR(s), MP_OBJ_FROM_PTR(s2)}; diff --git a/ports/unix/gccollect.c b/ports/unix/gccollect.c index d9ac53d7f8..79b17663c3 100644 --- a/ports/unix/gccollect.c +++ b/ports/unix/gccollect.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/ports/unix/input.c b/ports/unix/input.c index d66487a7af..c5bf719738 100644 --- a/ports/unix/input.c +++ b/ports/unix/input.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/ports/unix/main.c b/ports/unix/main.c index dcc1c68fe6..959640dc8c 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014-2017 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014-2017 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -39,7 +39,6 @@ #include #include "py/compile.h" -#include "py/frozenmod.h" #include "py/runtime.h" #include "py/builtin.h" #include "py/repl.h" @@ -47,6 +46,8 @@ #include "py/stackctrl.h" #include "py/mphal.h" #include "py/mpthread.h" +#include "extmod/misc.h" +#include "extmod/moduplatform.h" #include "extmod/vfs.h" #include "extmod/vfs_posix.h" #include "genhdr/mpversion.h" @@ -176,8 +177,9 @@ STATIC char *strjoin(const char *s1, int sep_char, const char *s2) { #endif STATIC int do_repl(void) { - mp_hal_stdout_tx_str("MicroPython " MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE "; " - MICROPY_PY_SYS_PLATFORM " version\nUse Ctrl-D to exit, Ctrl-E for paste mode\n"); + mp_hal_stdout_tx_str(MICROPY_BANNER_NAME_AND_VERSION); + mp_hal_stdout_tx_str("; " MICROPY_BANNER_MACHINE); + mp_hal_stdout_tx_str("\nUse Ctrl-D to exit, Ctrl-E for paste mode\n"); #if MICROPY_USE_READLINE == 1 @@ -190,7 +192,7 @@ STATIC int do_repl(void) { input_restart: vstr_reset(&line); - int ret = readline(&line, ">>> "); + int ret = readline(&line, mp_repl_get_ps1()); mp_parse_input_kind_t parse_input_kind = MP_PARSE_SINGLE_INPUT; if (ret == CHAR_CTRL_C) { @@ -237,7 +239,7 @@ STATIC int do_repl(void) { // got a line with non-zero length, see if it needs continuing while (mp_repl_continue_with_input(vstr_null_terminated_str(&line))) { vstr_add_byte(&line, '\n'); - ret = readline(&line, "... "); + ret = readline(&line, mp_repl_get_ps2()); if (ret == CHAR_CTRL_C) { // cancel everything printf("\n"); @@ -262,13 +264,13 @@ STATIC int do_repl(void) { // use simple readline for (;;) { - char *line = prompt(">>> "); + char *line = prompt((char *)mp_repl_get_ps1()); if (line == NULL) { // EOF return 0; } while (mp_repl_continue_with_input(line)) { - char *line2 = prompt("... "); + char *line2 = prompt((char *)mp_repl_get_ps2()); if (line2 == NULL) { break; } @@ -325,6 +327,10 @@ STATIC void print_help(char **argv) { , heap_size); impl_opts_cnt++; #endif + #if defined(__APPLE__) + printf(" realtime -- set thread priority to realtime\n"); + impl_opts_cnt++; + #endif if (impl_opts_cnt == 0) { printf(" (none)\n"); @@ -396,6 +402,15 @@ STATIC void pre_process_options(int argc, char **argv) { goto invalid_arg; } #endif + #if defined(__APPLE__) + } else if (strcmp(argv[a + 1], "realtime") == 0) { + #if MICROPY_PY_THREAD + mp_thread_is_realtime_enabled = true; + #endif + // main thread was already intialized before the option + // was parsed, so we have to enable realtime here. + mp_thread_set_realtime(); + #endif } else { invalid_arg: exit(invalid_args()); @@ -695,6 +710,11 @@ MP_NOINLINE int main_(int argc, char **argv) { } #endif + #if MICROPY_PY_BLUETOOTH + void mp_bluetooth_deinit(void); + mp_bluetooth_deinit(); + #endif + #if MICROPY_PY_THREAD mp_thread_deinit(); #endif @@ -715,38 +735,6 @@ MP_NOINLINE int main_(int argc, char **argv) { return ret & 0xff; } -#if !MICROPY_VFS -mp_import_stat_t mp_import_stat(const char *path) { - struct stat st; - if (stat(path, &st) == 0) { - if (S_ISDIR(st.st_mode)) { - return MP_IMPORT_STAT_DIR; - } else if (S_ISREG(st.st_mode)) { - return MP_IMPORT_STAT_FILE; - } - } - return MP_IMPORT_STAT_NO_EXIST; -} - -#if MICROPY_PY_IO -// Factory function for I/O stream classes, only needed if generic VFS subsystem isn't used. -// Note: buffering and encoding are currently ignored. -mp_obj_t mp_builtin_open(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kwargs) { - enum { ARG_file, ARG_mode }; - STATIC const mp_arg_t allowed_args[] = { - { MP_QSTR_file, MP_ARG_OBJ | MP_ARG_REQUIRED, {.u_rom_obj = MP_ROM_NONE} }, - { MP_QSTR_mode, MP_ARG_OBJ, {.u_obj = MP_OBJ_NEW_QSTR(MP_QSTR_r)} }, - { MP_QSTR_buffering, MP_ARG_INT, {.u_int = -1} }, - { MP_QSTR_encoding, MP_ARG_OBJ, {.u_rom_obj = MP_ROM_NONE} }, - }; - mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; - mp_arg_parse_all(n_args, pos_args, kwargs, MP_ARRAY_SIZE(allowed_args), allowed_args, args); - return mp_vfs_posix_file_open(&mp_type_textio, args[ARG_file].u_obj, args[ARG_mode].u_obj); -} -MP_DEFINE_CONST_FUN_OBJ_KW(mp_builtin_open_obj, 1, mp_builtin_open); -#endif -#endif - void nlr_jump_fail(void *val) { fprintf(stderr, "FATAL: uncaught NLR %p\n", val); exit(1); diff --git a/ports/unix/modffi.c b/ports/unix/modffi.c index d8b63b571e..6ea3d29fd2 100644 --- a/ports/unix/modffi.c +++ b/ports/unix/modffi.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014-2018 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014-2018 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -228,8 +228,7 @@ STATIC mp_obj_t make_func(mp_obj_t rettype_in, void *func, mp_obj_t argtypes_in) const char *argtypes = mp_obj_str_get_str(argtypes_in); mp_int_t nparams = MP_OBJ_SMALL_INT_VALUE(mp_obj_len_maybe(argtypes_in)); - mp_obj_ffifunc_t *o = m_new_obj_var(mp_obj_ffifunc_t, ffi_type *, nparams); - o->base.type = &ffifunc_type; + mp_obj_ffifunc_t *o = mp_obj_malloc_var(mp_obj_ffifunc_t, ffi_type *, nparams, &ffifunc_type); o->func = func; o->rettype = *rettype; @@ -245,7 +244,7 @@ STATIC mp_obj_t make_func(mp_obj_t rettype_in, void *func, mp_obj_t argtypes_in) int res = ffi_prep_cif(&o->cif, FFI_DEFAULT_ABI, nparams, char2ffi_type(*rettype), o->params); if (res != FFI_OK) { - mp_raise_ValueError(MP_ERROR_TEXT("Error in ffi_prep_cif")); + mp_raise_ValueError(MP_ERROR_TEXT("error in ffi_prep_cif")); } return MP_OBJ_FROM_PTR(o); @@ -336,8 +335,7 @@ STATIC mp_obj_t mod_ffi_callback(size_t n_args, const mp_obj_t *pos_args, mp_map const char *rettype = mp_obj_str_get_str(rettype_in); mp_int_t nparams = MP_OBJ_SMALL_INT_VALUE(mp_obj_len_maybe(paramtypes_in)); - mp_obj_fficallback_t *o = m_new_obj_var(mp_obj_fficallback_t, ffi_type *, nparams); - o->base.type = &fficallback_type; + mp_obj_fficallback_t *o = mp_obj_malloc_var(mp_obj_fficallback_t, ffi_type *, nparams, &fficallback_type); o->clo = ffi_closure_alloc(sizeof(ffi_closure), &o->func); @@ -376,8 +374,7 @@ STATIC mp_obj_t ffimod_var(mp_obj_t self_in, mp_obj_t vartype_in, mp_obj_t symna if (sym == NULL) { mp_raise_OSError(MP_ENOENT); } - mp_obj_ffivar_t *o = m_new_obj(mp_obj_ffivar_t); - o->base.type = &ffivar_type; + mp_obj_ffivar_t *o = mp_obj_malloc(mp_obj_ffivar_t, &ffivar_type); o->var = sym; o->type = *rettype; @@ -410,8 +407,7 @@ STATIC mp_obj_t ffimod_make_new(const mp_obj_type_t *type, size_t n_args, size_t if (mod == NULL) { mp_raise_OSError(errno); } - mp_obj_ffimod_t *o = m_new_obj(mp_obj_ffimod_t); - o->base.type = type; + mp_obj_ffimod_t *o = mp_obj_malloc(mp_obj_ffimod_t, type); o->handle = mod; return MP_OBJ_FROM_PTR(o); } @@ -509,22 +505,19 @@ STATIC mp_obj_t ffifunc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const } else if (mp_obj_is_str(a)) { const char *s = mp_obj_str_get_str(a); values[i].ffi = (ffi_arg)(intptr_t)s; - } else { - mp_getbuffer_fun_t get_buffer = mp_type_get_getbuffer_slot(((mp_obj_base_t *)MP_OBJ_TO_PTR(a))->type); - if (get_buffer != NULL) { - mp_obj_base_t *o = (mp_obj_base_t *)MP_OBJ_TO_PTR(a); - mp_buffer_info_t bufinfo; - int ret = get_buffer(MP_OBJ_FROM_PTR(o), &bufinfo, MP_BUFFER_READ); // TODO: MP_BUFFER_READ? - if (ret != 0) { - goto error; - } - values[i].ffi = (ffi_arg)(intptr_t)bufinfo.buf; - } else if (mp_obj_is_type(a, &fficallback_type)) { - mp_obj_fficallback_t *p = MP_OBJ_TO_PTR(a); - values[i].ffi = (ffi_arg)(intptr_t)p->func; - } else { + } else if (((mp_obj_base_t *)MP_OBJ_TO_PTR(a))->type->buffer_p.get_buffer != NULL) { + mp_obj_base_t *o = (mp_obj_base_t *)MP_OBJ_TO_PTR(a); + mp_buffer_info_t bufinfo; + int ret = o->type->buffer_p.get_buffer(MP_OBJ_FROM_PTR(o), &bufinfo, MP_BUFFER_READ); // TODO: MP_BUFFER_READ? + if (ret != 0) { goto error; } + values[i].ffi = (ffi_arg)(intptr_t)bufinfo.buf; + } else if (mp_obj_is_type(a, &fficallback_type)) { + mp_obj_fficallback_t *p = MP_OBJ_TO_PTR(a); + values[i].ffi = (ffi_arg)(intptr_t)p->func; + } else { + goto error; } valueptrs[i] = &values[i]; } diff --git a/ports/unix/modjni.c b/ports/unix/modjni.c index bd151050ea..c86f30653c 100644 --- a/ports/unix/modjni.c +++ b/ports/unix/modjni.c @@ -35,6 +35,8 @@ #include +#if MICROPY_PY_JNI + #define JJ(call, ...) (*env)->call(env, __VA_ARGS__) #define JJ1(call) (*env)->call(env) #define MATCH(s, static) (!strncmp(s, static, sizeof(static) - 1)) @@ -145,8 +147,7 @@ STATIC void jclass_attr(mp_obj_t self_in, qstr attr_in, mp_obj_t *dest) { // JJ1(ExceptionDescribe); JJ1(ExceptionClear); - mp_obj_jmethod_t *o = m_new_obj(mp_obj_jmethod_t); - o->base.type = &jmethod_type; + mp_obj_jmethod_t *o = mp_obj_malloc(mp_obj_jmethod_t, &jmethod_type); o->name = attr_in; o->meth = NULL; o->obj = self->cls; @@ -183,8 +184,7 @@ STATIC const mp_obj_type_t jclass_type = { }; STATIC mp_obj_t new_jclass(jclass jc) { - mp_obj_jclass_t *o = m_new_obj(mp_obj_jclass_t); - o->base.type = &jclass_type; + mp_obj_jclass_t *o = mp_obj_malloc(mp_obj_jclass_t, &jclass_type); o->cls = jc; return MP_OBJ_FROM_PTR(o); } @@ -223,8 +223,7 @@ STATIC void jobject_attr(mp_obj_t self_in, qstr attr_in, mp_obj_t *dest) { // JJ1(ExceptionDescribe); JJ1(ExceptionClear); - mp_obj_jmethod_t *o = m_new_obj(mp_obj_jmethod_t); - o->base.type = &jmethod_type; + mp_obj_jmethod_t *o = mp_obj_malloc(mp_obj_jmethod_t, &jmethod_type); o->name = attr_in; o->meth = NULL; o->obj = self->obj; @@ -343,8 +342,7 @@ STATIC mp_obj_t new_jobject(jobject jo) { } else if (JJ(IsInstanceOf, jo, Class_class)) { return new_jclass(jo); } else { - mp_obj_jobject_t *o = m_new_obj(mp_obj_jobject_t); - o->base.type = &jobject_type; + mp_obj_jobject_t *o = mp_obj_malloc(mp_obj_jobject_t, &jobject_type); o->obj = jo; return MP_OBJ_FROM_PTR(o); } @@ -644,8 +642,7 @@ STATIC mp_obj_t mod_jni_cls(mp_obj_t cls_name_in) { } jclass cls = JJ(FindClass, cls_name); - mp_obj_jclass_t *o = m_new_obj(mp_obj_jclass_t); - o->base.type = &jclass_type; + mp_obj_jclass_t *o = mp_obj_malloc(mp_obj_jclass_t, &jclass_type); o->cls = cls; return MP_OBJ_FROM_PTR(o); } @@ -717,3 +714,7 @@ const mp_obj_module_t mp_module_jni = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&mp_module_jni_globals, }; + +MP_REGISTER_MODULE(MP_QSTR_jni, mp_module_jni); + +#endif // MICROPY_PY_JNI diff --git a/ports/unix/modmachine.c b/ports/unix/modmachine.c index 987c70b26e..bcbf39406f 100644 --- a/ports/unix/modmachine.c +++ b/ports/unix/modmachine.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2015 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2015 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -112,4 +112,6 @@ const mp_obj_module_t mp_module_machine = { .globals = (mp_obj_dict_t *)&machine_module_globals, }; +MP_REGISTER_MODULE(MP_QSTR_umachine, mp_module_machine); + #endif // MICROPY_PY_MACHINE diff --git a/ports/unix/modos.c b/ports/unix/modos.c deleted file mode 100644 index 8373c07a5c..0000000000 --- a/ports/unix/modos.c +++ /dev/null @@ -1,395 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * SPDX-FileCopyrightText: Copyright (c) 2014-2018 Paul Sokolovsky - * SPDX-FileCopyrightText: Copyright (c) 2014-2018 Damien P. George - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef _MSC_VER -#include // For mkdir -#endif -#include "py/mpconfig.h" - -#include "py/runtime.h" -#include "py/objtuple.h" -#include "py/mphal.h" -#include "py/mpthread.h" -#include "extmod/vfs.h" - -#ifdef __ANDROID__ -#define USE_STATFS 1 -#endif - -#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) -#if __GLIBC_PREREQ(2, 25) -#include -#define _HAVE_GETRANDOM -#endif -#endif - -#if defined(MICROPY_UNIX_COVERAGE) -#include "py/objstr.h" -typedef int os_getenv_err_t; -mp_obj_t common_hal_os_getenv(const char *key, mp_obj_t default_); -os_getenv_err_t common_hal_os_getenv_str(const char *key, char *value, size_t value_len); -os_getenv_err_t common_hal_os_getenv_int(const char *key, mp_int_t *value); -#endif - -STATIC mp_obj_t mod_os_urandom(mp_obj_t num) { - mp_int_t n = mp_obj_get_int(num); - vstr_t vstr; - vstr_init_len(&vstr, n); - #ifdef _HAVE_GETRANDOM - RAISE_ERRNO(getrandom(vstr.buf, n, 0), errno); - #else - int fd = open("/dev/urandom", O_RDONLY); - RAISE_ERRNO(fd, errno); - RAISE_ERRNO(read(fd, vstr.buf, n), errno); - close(fd); - #endif - return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_os_urandom_obj, mod_os_urandom); - -STATIC mp_obj_t mod_os_stat(mp_obj_t path_in) { - struct stat sb; - const char *path = mp_obj_str_get_str(path_in); - - int res; - MP_HAL_RETRY_SYSCALL(res, stat(path, &sb), mp_raise_OSError(err)); - - mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL)); - t->items[0] = MP_OBJ_NEW_SMALL_INT(sb.st_mode); - t->items[1] = mp_obj_new_int_from_uint(sb.st_ino); - t->items[2] = mp_obj_new_int_from_uint(sb.st_dev); - t->items[3] = mp_obj_new_int_from_uint(sb.st_nlink); - t->items[4] = mp_obj_new_int_from_uint(sb.st_uid); - t->items[5] = mp_obj_new_int_from_uint(sb.st_gid); - t->items[6] = mp_obj_new_int_from_uint(sb.st_size); - t->items[7] = mp_obj_new_int_from_uint(sb.st_atime); - t->items[8] = mp_obj_new_int_from_uint(sb.st_mtime); - t->items[9] = mp_obj_new_int_from_uint(sb.st_ctime); - return MP_OBJ_FROM_PTR(t); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_os_stat_obj, mod_os_stat); - -#if MICROPY_PY_OS_STATVFS - -#if USE_STATFS -#include -#define STRUCT_STATVFS struct statfs -#define STATVFS statfs -#define F_FAVAIL sb.f_ffree -#define F_NAMEMAX sb.f_namelen -#define F_FLAG sb.f_flags -#else -#include -#define STRUCT_STATVFS struct statvfs -#define STATVFS statvfs -#define F_FAVAIL sb.f_favail -#define F_NAMEMAX sb.f_namemax -#define F_FLAG sb.f_flag -#endif - -STATIC mp_obj_t mod_os_statvfs(mp_obj_t path_in) { - STRUCT_STATVFS sb; - const char *path = mp_obj_str_get_str(path_in); - - int res; - MP_HAL_RETRY_SYSCALL(res, STATVFS(path, &sb), mp_raise_OSError(err)); - - mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL)); - t->items[0] = MP_OBJ_NEW_SMALL_INT(sb.f_bsize); - t->items[1] = MP_OBJ_NEW_SMALL_INT(sb.f_frsize); - t->items[2] = MP_OBJ_NEW_SMALL_INT(sb.f_blocks); - t->items[3] = MP_OBJ_NEW_SMALL_INT(sb.f_bfree); - t->items[4] = MP_OBJ_NEW_SMALL_INT(sb.f_bavail); - t->items[5] = MP_OBJ_NEW_SMALL_INT(sb.f_files); - t->items[6] = MP_OBJ_NEW_SMALL_INT(sb.f_ffree); - t->items[7] = MP_OBJ_NEW_SMALL_INT(F_FAVAIL); - t->items[8] = MP_OBJ_NEW_SMALL_INT(F_FLAG); - t->items[9] = MP_OBJ_NEW_SMALL_INT(F_NAMEMAX); - return MP_OBJ_FROM_PTR(t); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_os_statvfs_obj, mod_os_statvfs); -#endif - -STATIC mp_obj_t mod_os_remove(mp_obj_t path_in) { - const char *path = mp_obj_str_get_str(path_in); - - // Note that POSIX requires remove() to be able to delete a directory - // too (act as rmdir()). This is POSIX extension to ANSI C semantics - // of that function. But Python remove() follows ANSI C, and explicitly - // required to raise exception on attempt to remove a directory. Thus, - // call POSIX unlink() here. - MP_THREAD_GIL_EXIT(); - int r = unlink(path); - MP_THREAD_GIL_ENTER(); - - RAISE_ERRNO(r, errno); - - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_os_remove_obj, mod_os_remove); - -STATIC mp_obj_t mod_os_rename(mp_obj_t old_path_in, mp_obj_t new_path_in) { - const char *old_path = mp_obj_str_get_str(old_path_in); - const char *new_path = mp_obj_str_get_str(new_path_in); - - MP_THREAD_GIL_EXIT(); - int r = rename(old_path, new_path); - MP_THREAD_GIL_ENTER(); - - RAISE_ERRNO(r, errno); - - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_os_rename_obj, mod_os_rename); - -STATIC mp_obj_t mod_os_rmdir(mp_obj_t path_in) { - const char *path = mp_obj_str_get_str(path_in); - - MP_THREAD_GIL_EXIT(); - int r = rmdir(path); - MP_THREAD_GIL_ENTER(); - - RAISE_ERRNO(r, errno); - - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_os_rmdir_obj, mod_os_rmdir); - -STATIC mp_obj_t mod_os_system(mp_obj_t cmd_in) { - const char *cmd = mp_obj_str_get_str(cmd_in); - - MP_THREAD_GIL_EXIT(); - int r = system(cmd); - MP_THREAD_GIL_ENTER(); - - RAISE_ERRNO(r, errno); - - return MP_OBJ_NEW_SMALL_INT(r); -} -MP_DEFINE_CONST_FUN_OBJ_1(mod_os_system_obj, mod_os_system); - -STATIC mp_obj_t mod_os_getenv(mp_obj_t var_in) { - #if defined(MICROPY_UNIX_COVERAGE) - mp_obj_t result = common_hal_os_getenv(mp_obj_str_get_str(var_in), mp_const_none); - if (result != mp_const_none) { - return result; - } - #endif - const char *s = getenv(mp_obj_str_get_str(var_in)); - if (s == NULL) { - return mp_const_none; - } - return mp_obj_new_str(s, strlen(s)); -} -MP_DEFINE_CONST_FUN_OBJ_1(mod_os_getenv_obj, mod_os_getenv); - -#if defined(MICROPY_UNIX_COVERAGE) -STATIC mp_obj_t mod_os_getenv_int(mp_obj_t var_in) { - mp_int_t value; - os_getenv_err_t result = common_hal_os_getenv_int(mp_obj_str_get_str(var_in), &value); - if (result == 0) { - return mp_obj_new_int(value); - } - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_1(mod_os_getenv_int_obj, mod_os_getenv_int); - -STATIC mp_obj_t mod_os_getenv_str(mp_obj_t var_in) { - char buf[4096]; - os_getenv_err_t result = common_hal_os_getenv_str(mp_obj_str_get_str(var_in), buf, sizeof(buf)); - if (result == 0) { - return mp_obj_new_str_copy(&mp_type_str, (byte *)buf, strlen(buf)); - } - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_1(mod_os_getenv_str_obj, mod_os_getenv_str); -#endif - -STATIC mp_obj_t mod_os_putenv(mp_obj_t key_in, mp_obj_t value_in) { - const char *key = mp_obj_str_get_str(key_in); - const char *value = mp_obj_str_get_str(value_in); - int ret; - - #if _WIN32 - ret = _putenv_s(key, value); - #else - ret = setenv(key, value, 1); - #endif - - if (ret == -1) { - mp_raise_OSError(errno); - } - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_2(mod_os_putenv_obj, mod_os_putenv); - -STATIC mp_obj_t mod_os_unsetenv(mp_obj_t key_in) { - const char *key = mp_obj_str_get_str(key_in); - int ret; - - #if _WIN32 - ret = _putenv_s(key, ""); - #else - ret = unsetenv(key); - #endif - - if (ret == -1) { - mp_raise_OSError(errno); - } - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_1(mod_os_unsetenv_obj, mod_os_unsetenv); - -STATIC mp_obj_t mod_os_mkdir(mp_obj_t path_in) { - // TODO: Accept mode param - const char *path = mp_obj_str_get_str(path_in); - MP_THREAD_GIL_EXIT(); - #ifdef _WIN32 - int r = mkdir(path); - #else - int r = mkdir(path, 0777); - #endif - MP_THREAD_GIL_ENTER(); - RAISE_ERRNO(r, errno); - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_os_mkdir_obj, mod_os_mkdir); - -typedef struct _mp_obj_listdir_t { - mp_obj_base_t base; - mp_fun_1_t iternext; - DIR *dir; -} mp_obj_listdir_t; - -STATIC mp_obj_t listdir_next(mp_obj_t self_in) { - mp_obj_listdir_t *self = MP_OBJ_TO_PTR(self_in); - - if (self->dir == NULL) { - goto done; - } - MP_THREAD_GIL_EXIT(); - struct dirent *dirent = readdir(self->dir); - if (dirent == NULL) { - closedir(self->dir); - MP_THREAD_GIL_ENTER(); - self->dir = NULL; - done: - return MP_OBJ_STOP_ITERATION; - } - MP_THREAD_GIL_ENTER(); - - mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(3, NULL)); - t->items[0] = mp_obj_new_str(dirent->d_name, strlen(dirent->d_name)); - - #ifdef _DIRENT_HAVE_D_TYPE - #ifdef DTTOIF - t->items[1] = MP_OBJ_NEW_SMALL_INT(DTTOIF(dirent->d_type)); - #else - if (dirent->d_type == DT_DIR) { - t->items[1] = MP_OBJ_NEW_SMALL_INT(MP_S_IFDIR); - } else if (dirent->d_type == DT_REG) { - t->items[1] = MP_OBJ_NEW_SMALL_INT(MP_S_IFREG); - } else { - t->items[1] = MP_OBJ_NEW_SMALL_INT(dirent->d_type); - } - #endif - #else - // DT_UNKNOWN should have 0 value on any reasonable system - t->items[1] = MP_OBJ_NEW_SMALL_INT(0); - #endif - - #ifdef _DIRENT_HAVE_D_INO - t->items[2] = MP_OBJ_NEW_SMALL_INT(dirent->d_ino); - #else - t->items[2] = MP_OBJ_NEW_SMALL_INT(0); - #endif - return MP_OBJ_FROM_PTR(t); -} - -STATIC mp_obj_t mod_os_ilistdir(size_t n_args, const mp_obj_t *args) { - const char *path = "."; - if (n_args > 0) { - path = mp_obj_str_get_str(args[0]); - } - mp_obj_listdir_t *o = m_new_obj(mp_obj_listdir_t); - o->base.type = &mp_type_polymorph_iter; - MP_THREAD_GIL_EXIT(); - o->dir = opendir(path); - MP_THREAD_GIL_ENTER(); - o->iternext = listdir_next; - return MP_OBJ_FROM_PTR(o); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_os_ilistdir_obj, 0, 1, mod_os_ilistdir); - -STATIC mp_obj_t mod_os_errno(size_t n_args, const mp_obj_t *args) { - if (n_args == 0) { - return MP_OBJ_NEW_SMALL_INT(errno); - } - - errno = mp_obj_get_int(args[0]); - return mp_const_none; -} -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_os_errno_obj, 0, 1, mod_os_errno); - -STATIC const mp_rom_map_elem_t mp_module_os_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uos) }, - { MP_ROM_QSTR(MP_QSTR_errno), MP_ROM_PTR(&mod_os_errno_obj) }, - { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&mod_os_stat_obj) }, - { MP_ROM_QSTR(MP_QSTR_urandom), MP_ROM_PTR(&mod_os_urandom_obj) }, - #if MICROPY_PY_OS_STATVFS - { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&mod_os_statvfs_obj) }, - #endif - { MP_ROM_QSTR(MP_QSTR_system), MP_ROM_PTR(&mod_os_system_obj) }, - { MP_ROM_QSTR(MP_QSTR_remove), MP_ROM_PTR(&mod_os_remove_obj) }, - { MP_ROM_QSTR(MP_QSTR_rename), MP_ROM_PTR(&mod_os_rename_obj) }, - { MP_ROM_QSTR(MP_QSTR_rmdir), MP_ROM_PTR(&mod_os_rmdir_obj) }, - { MP_ROM_QSTR(MP_QSTR_getenv), MP_ROM_PTR(&mod_os_getenv_obj) }, - #if defined(MICROPY_UNIX_COVERAGE) - { MP_ROM_QSTR(MP_QSTR_getenv_int), MP_ROM_PTR(&mod_os_getenv_int_obj) }, - { MP_ROM_QSTR(MP_QSTR_getenv_str), MP_ROM_PTR(&mod_os_getenv_str_obj) }, - #endif - { MP_ROM_QSTR(MP_QSTR_putenv), MP_ROM_PTR(&mod_os_putenv_obj) }, - { MP_ROM_QSTR(MP_QSTR_unsetenv), MP_ROM_PTR(&mod_os_unsetenv_obj) }, - { MP_ROM_QSTR(MP_QSTR_mkdir), MP_ROM_PTR(&mod_os_mkdir_obj) }, - { MP_ROM_QSTR(MP_QSTR_ilistdir), MP_ROM_PTR(&mod_os_ilistdir_obj) }, -}; - -STATIC MP_DEFINE_CONST_DICT(mp_module_os_globals, mp_module_os_globals_table); - -const mp_obj_module_t mp_module_os = { - .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&mp_module_os_globals, -}; diff --git a/ports/unix/modtermios.c b/ports/unix/modtermios.c index 4adef8a380..4f9751e274 100644 --- a/ports/unix/modtermios.c +++ b/ports/unix/modtermios.c @@ -33,6 +33,8 @@ #include "py/runtime.h" #include "py/mphal.h" +#if MICROPY_PY_TERMIOS + STATIC mp_obj_t mod_termios_tcgetattr(mp_obj_t fd_in) { struct termios term; int fd = mp_obj_get_int(fd_in); @@ -148,3 +150,7 @@ const mp_obj_module_t mp_module_termios = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&mp_module_termios_globals, }; + +MP_REGISTER_MODULE(MP_QSTR_termios, mp_module_termios); + +#endif // MICROPY_PY_TERMIOS diff --git a/ports/unix/modtime.c b/ports/unix/modtime.c index e72c611d1b..c9eb09c15b 100644 --- a/ports/unix/modtime.c +++ b/ports/unix/modtime.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014-2017 Paul Sokolovsky - * SPDX-FileCopyrightText: Copyright (c) 2014-2017 Damien P. George + * Copyright (c) 2014-2017 Paul Sokolovsky + * Copyright (c) 2014-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -45,7 +45,7 @@ static inline int msec_sleep_tv(struct timeval *tv) { msec_sleep(tv->tv_sec * 1000.0 + tv->tv_usec / 1000.0); return 0; } -#define sleep_select(a,b,c,d,e) msec_sleep_tv((e)) +#define sleep_select(a, b, c, d, e) msec_sleep_tv((e)) #else #define sleep_select select #endif @@ -232,4 +232,6 @@ const mp_obj_module_t mp_module_time = { .globals = (mp_obj_dict_t *)&mp_module_time_globals, }; +MP_REGISTER_MODULE(MP_QSTR_utime, mp_module_time); + #endif // MICROPY_PY_UTIME diff --git a/ports/unix/moduos.c b/ports/unix/moduos.c new file mode 100644 index 0000000000..848a3612d1 --- /dev/null +++ b/ports/unix/moduos.c @@ -0,0 +1,108 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014-2018 Paul Sokolovsky + * Copyright (c) 2017-2022 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include + +#include "py/runtime.h" +#include "py/mphal.h" + +STATIC mp_obj_t mp_uos_getenv(mp_obj_t var_in) { + const char *s = getenv(mp_obj_str_get_str(var_in)); + if (s == NULL) { + return mp_const_none; + } + return mp_obj_new_str(s, strlen(s)); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_uos_getenv_obj, mp_uos_getenv); + +STATIC mp_obj_t mp_uos_putenv(mp_obj_t key_in, mp_obj_t value_in) { + const char *key = mp_obj_str_get_str(key_in); + const char *value = mp_obj_str_get_str(value_in); + int ret; + + #if _WIN32 + ret = _putenv_s(key, value); + #else + ret = setenv(key, value, 1); + #endif + + if (ret == -1) { + mp_raise_OSError(errno); + } + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(mp_uos_putenv_obj, mp_uos_putenv); + +STATIC mp_obj_t mp_uos_unsetenv(mp_obj_t key_in) { + const char *key = mp_obj_str_get_str(key_in); + int ret; + + #if _WIN32 + ret = _putenv_s(key, ""); + #else + ret = unsetenv(key); + #endif + + if (ret == -1) { + mp_raise_OSError(errno); + } + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_uos_unsetenv_obj, mp_uos_unsetenv); + +STATIC mp_obj_t mp_uos_system(mp_obj_t cmd_in) { + const char *cmd = mp_obj_str_get_str(cmd_in); + + MP_THREAD_GIL_EXIT(); + int r = system(cmd); + MP_THREAD_GIL_ENTER(); + + RAISE_ERRNO(r, errno); + + return MP_OBJ_NEW_SMALL_INT(r); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_uos_system_obj, mp_uos_system); + +STATIC mp_obj_t mp_uos_urandom(mp_obj_t num) { + mp_int_t n = mp_obj_get_int(num); + vstr_t vstr; + vstr_init_len(&vstr, n); + mp_hal_get_random(n, vstr.buf); + return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_uos_urandom_obj, mp_uos_urandom); + +STATIC mp_obj_t mp_uos_errno(size_t n_args, const mp_obj_t *args) { + if (n_args == 0) { + return MP_OBJ_NEW_SMALL_INT(errno); + } + + errno = mp_obj_get_int(args[0]); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_uos_errno_obj, 0, 1, mp_uos_errno); diff --git a/ports/unix/moduos_vfs.c b/ports/unix/moduos_vfs.c deleted file mode 100644 index 495679290a..0000000000 --- a/ports/unix/moduos_vfs.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * SPDX-FileCopyrightText: Copyright (c) 2017 Damien P. George - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include - -#include "extmod/vfs.h" -#include "extmod/vfs_posix.h" -#include "extmod/vfs_fat.h" -#include "extmod/vfs_lfs.h" - -#if MICROPY_VFS - -// These are defined in modos.c -MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mod_os_errno_obj); -MP_DECLARE_CONST_FUN_OBJ_1(mod_os_getenv_obj); -#if defined(MICROPY_UNIX_COVERAGE) -MP_DECLARE_CONST_FUN_OBJ_1(mod_os_getenv_int_obj); -MP_DECLARE_CONST_FUN_OBJ_1(mod_os_getenv_str_obj); -#endif -MP_DECLARE_CONST_FUN_OBJ_1(mod_os_putenv_obj); -MP_DECLARE_CONST_FUN_OBJ_1(mod_os_unsetenv_obj); -MP_DECLARE_CONST_FUN_OBJ_1(mod_os_system_obj); - -STATIC const mp_rom_map_elem_t uos_vfs_module_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uos_vfs) }, - { MP_ROM_QSTR(MP_QSTR_sep), MP_ROM_QSTR(MP_QSTR__slash_) }, - - { MP_ROM_QSTR(MP_QSTR_errno), MP_ROM_PTR(&mod_os_errno_obj) }, - { MP_ROM_QSTR(MP_QSTR_getenv), MP_ROM_PTR(&mod_os_getenv_obj) }, - #if defined(MICROPY_UNIX_COVERAGE) - { MP_ROM_QSTR(MP_QSTR_getenv_int), MP_ROM_PTR(&mod_os_getenv_int_obj) }, - { MP_ROM_QSTR(MP_QSTR_getenv_str), MP_ROM_PTR(&mod_os_getenv_str_obj) }, - #endif - { MP_ROM_QSTR(MP_QSTR_putenv), MP_ROM_PTR(&mod_os_putenv_obj) }, - { MP_ROM_QSTR(MP_QSTR_unsetenv), MP_ROM_PTR(&mod_os_unsetenv_obj) }, - { MP_ROM_QSTR(MP_QSTR_system), MP_ROM_PTR(&mod_os_system_obj) }, - - { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&mp_vfs_mount_obj) }, - { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&mp_vfs_umount_obj) }, - - { MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&mp_vfs_chdir_obj) }, - { MP_ROM_QSTR(MP_QSTR_getcwd), MP_ROM_PTR(&mp_vfs_getcwd_obj) }, - { MP_ROM_QSTR(MP_QSTR_ilistdir), MP_ROM_PTR(&mp_vfs_ilistdir_obj) }, - { MP_ROM_QSTR(MP_QSTR_listdir), MP_ROM_PTR(&mp_vfs_listdir_obj) }, - { MP_ROM_QSTR(MP_QSTR_mkdir), MP_ROM_PTR(&mp_vfs_mkdir_obj) }, - { MP_ROM_QSTR(MP_QSTR_remove), MP_ROM_PTR(&mp_vfs_remove_obj) }, - { MP_ROM_QSTR(MP_QSTR_rename),MP_ROM_PTR(&mp_vfs_rename_obj) }, - { MP_ROM_QSTR(MP_QSTR_rmdir), MP_ROM_PTR(&mp_vfs_rmdir_obj) }, - { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&mp_vfs_stat_obj) }, - { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&mp_vfs_statvfs_obj) }, - { MP_ROM_QSTR(MP_QSTR_unlink), MP_ROM_PTR(&mp_vfs_remove_obj) }, // unlink aliases to remove - - #if MICROPY_VFS_POSIX - { MP_ROM_QSTR(MP_QSTR_VfsPosix), MP_ROM_PTR(&mp_type_vfs_posix) }, - #endif - #if MICROPY_VFS_FAT - { MP_ROM_QSTR(MP_QSTR_VfsFat), MP_ROM_PTR(&mp_fat_vfs_type) }, - #endif - #if MICROPY_VFS_LFS1 - { MP_ROM_QSTR(MP_QSTR_VfsLfs1), MP_ROM_PTR(&mp_type_vfs_lfs1) }, - #endif - #if MICROPY_VFS_LFS2 - { MP_ROM_QSTR(MP_QSTR_VfsLfs2), MP_ROM_PTR(&mp_type_vfs_lfs2) }, - #endif -}; - -STATIC MP_DEFINE_CONST_DICT(uos_vfs_module_globals, uos_vfs_module_globals_table); - -const mp_obj_module_t mp_module_uos_vfs = { - .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&uos_vfs_module_globals, -}; - -#endif // MICROPY_VFS diff --git a/ports/unix/moduselect.c b/ports/unix/moduselect.c index fe09c33c85..7bfcf8fc56 100644 --- a/ports/unix/moduselect.c +++ b/ports/unix/moduselect.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George + * Copyright (c) 2014 Damien P. George * Copyright (c) 2015-2017 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -327,8 +327,7 @@ STATIC mp_obj_t select_poll(size_t n_args, const mp_obj_t *args) { if (n_args > 0) { alloc = mp_obj_get_int(args[0]); } - mp_obj_poll_t *poll = m_new_obj(mp_obj_poll_t); - poll->base.type = &mp_type_poll; + mp_obj_poll_t *poll = mp_obj_malloc(mp_obj_poll_t, &mp_type_poll); poll->entries = m_new(struct pollfd, alloc); poll->alloc = alloc; poll->len = 0; @@ -355,4 +354,6 @@ const mp_obj_module_t mp_module_uselect = { .globals = (mp_obj_dict_t *)&mp_module_select_globals, }; +MP_REGISTER_MODULE(MP_QSTR_uselect, mp_module_uselect); + #endif // MICROPY_PY_USELECT_POSIX diff --git a/ports/unix/mpconfigport.h b/ports/unix/mpconfigport.h index 237b99bfe8..b1733f9c1a 100644 --- a/ports/unix/mpconfigport.h +++ b/ports/unix/mpconfigport.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -34,6 +34,74 @@ // If we're building the minimal variant, ignore the rest of this file. #ifndef MICROPY_UNIX_MINIMAL +// If the variant did not set a feature level then configure a set of features. +#ifndef MICROPY_CONFIG_ROM_LEVEL +#define MICROPY_COMP_MODULE_CONST (1) +#define MICROPY_COMP_TRIPLE_TUPLE_ASSIGN (1) +#define MICROPY_COMP_RETURN_IF_EXPR (1) +#ifndef MICROPY_OPT_LOAD_ATTR_FAST_PATH +#define MICROPY_OPT_LOAD_ATTR_FAST_PATH (1) +#endif +#ifndef MICROPY_OPT_MAP_LOOKUP_CACHE +#define MICROPY_OPT_MAP_LOOKUP_CACHE (1) +#endif +#define MICROPY_ENABLE_FINALISER (1) +#define MICROPY_STACK_CHECK (1) +#define MICROPY_KBD_EXCEPTION (1) +#define MICROPY_HELPER_REPL (1) +#define MICROPY_REPL_EMACS_KEYS (1) +#define MICROPY_REPL_AUTO_INDENT (1) +#define MICROPY_ENABLE_SOURCE_LINE (1) +#ifndef MICROPY_STREAMS_NON_BLOCK +#define MICROPY_STREAMS_NON_BLOCK (1) +#endif +#define MICROPY_MODULE_WEAK_LINKS (1) +#define MICROPY_CAN_OVERRIDE_BUILTINS (1) +#define MICROPY_PY_FUNCTION_ATTRS (1) +#define MICROPY_PY_DESCRIPTORS (1) +#define MICROPY_PY_DELATTR_SETATTR (1) +#define MICROPY_PY_FSTRINGS (1) +#define MICROPY_PY_BUILTINS_STR_UNICODE (1) +#define MICROPY_PY_BUILTINS_STR_CENTER (1) +#define MICROPY_PY_BUILTINS_STR_PARTITION (1) +#define MICROPY_PY_BUILTINS_STR_SPLITLINES (1) +#define MICROPY_PY_BUILTINS_MEMORYVIEW (1) +#define MICROPY_PY_BUILTINS_SLICE_ATTRS (1) +#define MICROPY_PY_BUILTINS_SLICE_INDICES (1) +#define MICROPY_PY_BUILTINS_FROZENSET (1) +#define MICROPY_PY_BUILTINS_ROUND_INT (1) +#define MICROPY_PY_ALL_SPECIAL_METHODS (1) +#define MICROPY_PY_REVERSE_SPECIAL_METHODS (1) +#define MICROPY_PY_BUILTINS_COMPILE (1) +#define MICROPY_PY_BUILTINS_NOTIMPLEMENTED (1) +#define MICROPY_PY_BUILTINS_INPUT (1) +#define MICROPY_PY_BUILTINS_POW3 (1) +#define MICROPY_PY_MICROPYTHON_MEM_INFO (1) +#define MICROPY_PY_ARRAY_SLICE_ASSIGN (1) +#define MICROPY_PY_COLLECTIONS_DEQUE (1) +#define MICROPY_PY_COLLECTIONS_ORDEREDDICT (1) +#ifndef MICROPY_PY_MATH_SPECIAL_FUNCTIONS +#define MICROPY_PY_MATH_SPECIAL_FUNCTIONS (1) +#endif +#define MICROPY_PY_MATH_ISCLOSE (MICROPY_PY_MATH_SPECIAL_FUNCTIONS) +#define MICROPY_PY_CMATH (1) +#define MICROPY_PY_IO_IOBASE (1) +#define MICROPY_PY_IO_FILEIO (1) +#define MICROPY_PY_SYS_MAXSIZE (1) +#define MICROPY_PY_SYS_STDFILES (1) +#define MICROPY_PY_UERRNO (1) +#define MICROPY_PY_UCTYPES (1) +#define MICROPY_PY_UZLIB (1) +#define MICROPY_PY_UJSON (1) +#define MICROPY_PY_UOS (1) +#define MICROPY_PY_URE (1) +#define MICROPY_PY_UHEAPQ (1) +#define MICROPY_PY_UHASHLIB (1) +#define MICROPY_PY_UBINASCII (1) +#define MICROPY_PY_UBINASCII_CRC32 (1) +#define MICROPY_PY_URANDOM (1) +#endif + #define MICROPY_ALLOC_PATH_MAX (PATH_MAX) #define MICROPY_PERSISTENT_CODE_LOAD (1) #if !defined(MICROPY_EMIT_X64) && defined(__x86_64__) @@ -51,69 +119,29 @@ #if !defined(MICROPY_EMIT_ARM) && defined(__arm__) && !defined(__thumb2__) #define MICROPY_EMIT_ARM (1) #endif -#define MICROPY_COMP_MODULE_CONST (1) -#define MICROPY_COMP_TRIPLE_TUPLE_ASSIGN (1) -#define MICROPY_COMP_RETURN_IF_EXPR (1) #define MICROPY_ENABLE_GC (1) -#define MICROPY_ENABLE_FINALISER (1) -#define MICROPY_STACK_CHECK (1) #define MICROPY_MALLOC_USES_ALLOCATED_SIZE (1) #define MICROPY_MEM_STATS (1) #define MICROPY_DEBUG_PRINTERS (1) -#define CIRCUITPY_MICROPYTHON_ADVANCED (1) // Printing debug to stderr may give tests which // check stdout a chance to pass, etc. #define MICROPY_DEBUG_PRINTER (&mp_stderr_print) #define MICROPY_READER_POSIX (1) +#define MICROPY_READER_VFS (1) #define MICROPY_USE_READLINE_HISTORY (1) -#define MICROPY_HELPER_REPL (1) -#define MICROPY_REPL_EMACS_KEYS (1) -#define MICROPY_REPL_AUTO_INDENT (1) #define MICROPY_HELPER_LEXER_UNIX (1) -#define MICROPY_ENABLE_SOURCE_LINE (1) #ifndef MICROPY_FLOAT_IMPL #define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_DOUBLE) #endif #define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_MPZ) -#ifndef MICROPY_STREAMS_NON_BLOCK -#define MICROPY_STREAMS_NON_BLOCK (1) -#endif #define MICROPY_STREAMS_POSIX_API (1) #define MICROPY_OPT_COMPUTED_GOTO (1) -#ifndef MICROPY_OPT_LOAD_ATTR_FAST_PATH -#define MICROPY_OPT_LOAD_ATTR_FAST_PATH (1) -#endif -#ifndef MICROPY_OPT_MAP_LOOKUP_CACHE -#define MICROPY_OPT_MAP_LOOKUP_CACHE (1) -#endif -#define MICROPY_MODULE_WEAK_LINKS (1) #define MICROPY_MODULE_OVERRIDE_MAIN_IMPORT (1) -#define MICROPY_CAN_OVERRIDE_BUILTINS (1) -#define MICROPY_VFS_POSIX_FILE (1) -#define MICROPY_PY_FUNCTION_ATTRS (1) -#define MICROPY_PY_DESCRIPTORS (1) -#define MICROPY_PY_DELATTR_SETATTR (1) -#define MICROPY_PY_FSTRINGS (1) -#define MICROPY_PY_BUILTINS_STR_UNICODE (1) -#define MICROPY_PY_BUILTINS_STR_CENTER (1) -#define MICROPY_PY_BUILTINS_STR_PARTITION (1) -#define MICROPY_PY_BUILTINS_STR_SPLITLINES (1) -#define MICROPY_PY_BUILTINS_MEMORYVIEW (1) -#define MICROPY_PY_BUILTINS_FROZENSET (1) -#define MICROPY_PY_BUILTINS_COMPILE (1) -#define MICROPY_PY_BUILTINS_NOTIMPLEMENTED (1) -#define MICROPY_PY_BUILTINS_INPUT (1) -#define MICROPY_PY_BUILTINS_POW3 (1) -#define MICROPY_PY_BUILTINS_ROUND_INT (1) -#define MICROPY_PY_MICROPYTHON_MEM_INFO (1) -#define MICROPY_PY_ALL_SPECIAL_METHODS (1) -#define MICROPY_PY_REVERSE_SPECIAL_METHODS (1) -#define MICROPY_PY_ARRAY_SLICE_ASSIGN (1) -#define MICROPY_PY_BUILTINS_SLICE_ATTRS (1) -#define MICROPY_PY_BUILTINS_SLICE_INDICES (1) -#define MICROPY_PY_SYS_ATEXIT (1) +#define MICROPY_VFS (1) +#define MICROPY_VFS_POSIX (1) #define MICROPY_PY_SYS_PATH_ARGV_DEFAULTS (0) #define MICROPY_PY_SYS_EXIT (1) +#define MICROPY_PY_SYS_ATEXIT (1) #if MICROPY_PY_SYS_SETTRACE #define MICROPY_PERSISTENT_CODE_SAVE (1) #define MICROPY_COMP_CONST (0) @@ -128,17 +156,7 @@ #ifndef MICROPY_PY_SYS_PATH_DEFAULT #define MICROPY_PY_SYS_PATH_DEFAULT ".frozen:~/.micropython/lib:/usr/lib/micropython" #endif -#define MICROPY_PY_SYS_MAXSIZE (1) -#define MICROPY_PY_SYS_STDFILES (1) #define MICROPY_PY_SYS_EXC_INFO (1) -#define MICROPY_PY_COLLECTIONS_DEQUE (1) -#ifndef MICROPY_PY_MATH_SPECIAL_FUNCTIONS -#define MICROPY_PY_MATH_SPECIAL_FUNCTIONS (1) -#endif -#define MICROPY_PY_MATH_ISCLOSE (MICROPY_PY_MATH_SPECIAL_FUNCTIONS) -#define MICROPY_PY_CMATH (1) -#define MICROPY_PY_IO_IOBASE (1) -#define MICROPY_PY_IO_FILEIO (1) #define MICROPY_PY_GC_COLLECT_RETVAL (1) #ifndef MICROPY_STACKLESS @@ -146,25 +164,21 @@ #define MICROPY_STACKLESS_STRICT (0) #endif -#define MICROPY_PY_OS_STATVFS (1) +#define MICROPY_PY_UOS_INCLUDEFILE "ports/unix/moduos.c" +#define MICROPY_PY_UOS_ERRNO (1) +#define MICROPY_PY_UOS_GETENV_PUTENV_UNSETENV (1) +#define MICROPY_PY_UOS_SEP (1) +#define MICROPY_PY_UOS_SYSTEM (1) +#define MICROPY_PY_UOS_URANDOM (1) #define MICROPY_PY_UTIME (1) #define MICROPY_PY_UTIME_MP_HAL (1) -#define MICROPY_PY_UERRNO (1) -#define MICROPY_PY_UCTYPES (1) -#define MICROPY_PY_UZLIB (1) -#define MICROPY_PY_UJSON (1) -#define MICROPY_PY_URE (1) -#define MICROPY_PY_UHEAPQ (1) #define MICROPY_PY_UTIMEQ (1) -#define MICROPY_PY_UHASHLIB (1) +#define MICROPY_PY_USOCKET_LISTEN_BACKLOG_DEFAULT (SOMAXCONN < 128 ? SOMAXCONN : 128) #if MICROPY_PY_USSL #define MICROPY_PY_UHASHLIB_MD5 (1) #define MICROPY_PY_UHASHLIB_SHA1 (1) #define MICROPY_PY_UCRYPTOLIB (1) #endif -#define MICROPY_PY_UBINASCII (1) -#define MICROPY_PY_UBINASCII_CRC32 (1) -#define MICROPY_PY_URANDOM (1) #ifndef MICROPY_PY_USELECT_POSIX #define MICROPY_PY_USELECT_POSIX (1) #endif @@ -203,73 +217,11 @@ extern const struct _mp_print_t mp_stderr_print; #define MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF (1) #define MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE (256) -#define MICROPY_KBD_EXCEPTION (1) #define MICROPY_ASYNC_KBD_INTR (1) #define mp_type_fileio mp_type_vfs_posix_fileio #define mp_type_textio mp_type_vfs_posix_textio -extern const struct _mp_obj_module_t mp_module_machine; -extern const struct _mp_obj_module_t mp_module_os; -extern const struct _mp_obj_module_t mp_module_uos_vfs; -extern const struct _mp_obj_module_t mp_module_uselect; -extern const struct _mp_obj_module_t mp_module_time; -extern const struct _mp_obj_module_t mp_module_termios; -extern const struct _mp_obj_module_t mp_module_socket; -extern const struct _mp_obj_module_t mp_module_ffi; -extern const struct _mp_obj_module_t mp_module_jni; - -#if MICROPY_PY_UOS_VFS -#define MICROPY_PY_UOS_DEF { MP_ROM_QSTR(MP_QSTR_uos), MP_ROM_PTR(&mp_module_uos_vfs) }, -#else -#define MICROPY_PY_UOS_DEF { MP_ROM_QSTR(MP_QSTR_uos), MP_ROM_PTR(&mp_module_os) }, -#endif -#if MICROPY_PY_FFI -#define MICROPY_PY_FFI_DEF { MP_ROM_QSTR(MP_QSTR_ffi), MP_ROM_PTR(&mp_module_ffi) }, -#else -#define MICROPY_PY_FFI_DEF -#endif -#if MICROPY_PY_MACHINE -#define MICROPY_PY_MACHINE_DEF { MP_ROM_QSTR(MP_QSTR_umachine), MP_ROM_PTR(&mp_module_machine) }, -#else -#define MICROPY_PY_MACHINE_DEF -#endif -#if MICROPY_PY_JNI -#define MICROPY_PY_JNI_DEF { MP_ROM_QSTR(MP_QSTR_jni), MP_ROM_PTR(&mp_module_jni) }, -#else -#define MICROPY_PY_JNI_DEF -#endif -#if MICROPY_PY_UTIME -#define MICROPY_PY_UTIME_DEF { MP_ROM_QSTR(MP_QSTR_utime), MP_ROM_PTR(&mp_module_time) }, -#else -#define MICROPY_PY_UTIME_DEF -#endif -#if MICROPY_PY_TERMIOS -#define MICROPY_PY_TERMIOS_DEF { MP_ROM_QSTR(MP_QSTR_termios), MP_ROM_PTR(&mp_module_termios) }, -#else -#define MICROPY_PY_TERMIOS_DEF -#endif -#if MICROPY_PY_SOCKET -#define MICROPY_PY_SOCKET_DEF { MP_ROM_QSTR(MP_QSTR_usocket), MP_ROM_PTR(&mp_module_socket) }, -#else -#define MICROPY_PY_SOCKET_DEF -#endif -#if MICROPY_PY_USELECT_POSIX -#define MICROPY_PY_USELECT_DEF { MP_ROM_QSTR(MP_QSTR_uselect), MP_ROM_PTR(&mp_module_uselect) }, -#else -#define MICROPY_PY_USELECT_DEF -#endif - -#define MICROPY_PORT_BUILTIN_MODULES \ - MICROPY_PY_FFI_DEF \ - MICROPY_PY_JNI_DEF \ - MICROPY_PY_UTIME_DEF \ - MICROPY_PY_SOCKET_DEF \ - MICROPY_PY_MACHINE_DEF \ - MICROPY_PY_UOS_DEF \ - MICROPY_PY_USELECT_DEF \ - MICROPY_PY_TERMIOS_DEF \ - // type definitions for the specific machine // For size_t and ssize_t @@ -306,6 +258,17 @@ void mp_unix_mark_exec(void); #define MICROPY_FORCE_PLAT_ALLOC_EXEC (1) #endif +#ifdef MICROPY_PY_URANDOM_SEED_INIT_FUNC +// Support for seeding the random module on import. +#include +void mp_hal_get_random(size_t n, void *buf); +static inline unsigned long mp_urandom_seed_init(void) { + unsigned long r; + mp_hal_get_random(sizeof(r), &r); + return r; +} +#endif + #ifdef __linux__ // Can access physical memory using /dev/mem #define MICROPY_PLAT_DEV_MEM (1) @@ -324,14 +287,26 @@ void mp_unix_mark_exec(void); #endif #endif -#define MICROPY_PORT_BUILTINS \ - { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&mp_builtin_open_obj) }, - #define MP_STATE_PORT MP_STATE_VM +#if MICROPY_PY_BLUETOOTH +#if MICROPY_BLUETOOTH_BTSTACK +struct _mp_bluetooth_btstack_root_pointers_t; +#define MICROPY_BLUETOOTH_ROOT_POINTERS struct _mp_bluetooth_btstack_root_pointers_t *bluetooth_btstack_root_pointers; +#endif +#if MICROPY_BLUETOOTH_NIMBLE +struct _mp_bluetooth_nimble_root_pointers_t; +struct _mp_bluetooth_nimble_malloc_t; +#define MICROPY_BLUETOOTH_ROOT_POINTERS struct _mp_bluetooth_nimble_malloc_t *bluetooth_nimble_memory; struct _mp_bluetooth_nimble_root_pointers_t *bluetooth_nimble_root_pointers; +#endif +#else +#define MICROPY_BLUETOOTH_ROOT_POINTERS +#endif + #define MICROPY_PORT_ROOT_POINTERS \ const char *readline_hist[50]; \ void *mmap_region_head; \ + MICROPY_BLUETOOTH_ROOT_POINTERS \ // We need to provide a declaration/definition of alloca() // unless support for it is disabled. diff --git a/ports/unix/mphalport.h b/ports/unix/mphalport.h index 9242bef1ad..e3c2a8035b 100644 --- a/ports/unix/mphalport.h +++ b/ports/unix/mphalport.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2015 Damien P. George + * Copyright (c) 2015 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -92,3 +92,13 @@ static inline void mp_hal_delay_us(mp_uint_t us) { #define RAISE_ERRNO(err_flag, error_val) \ { if (err_flag == -1) \ { mp_raise_OSError(error_val); } } + +void mp_hal_get_random(size_t n, void *buf); + +#if MICROPY_PY_BLUETOOTH +enum { + MP_HAL_MAC_BDADDR, +}; + +void mp_hal_get_mac(int idx, uint8_t buf[6]); +#endif diff --git a/ports/unix/mpthreadport.c b/ports/unix/mpthreadport.c index 6a0ce59377..6a267e7236 100644 --- a/ports/unix/mpthreadport.c +++ b/ports/unix/mpthreadport.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd + * Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -53,12 +53,12 @@ #define THREAD_STACK_OVERFLOW_MARGIN (8192) // this structure forms a linked list, one node per active thread -typedef struct _thread_t { +typedef struct _mp_thread_t { pthread_t id; // system id of thread int ready; // whether the thread is ready and running void *arg; // thread Python args, a GC root pointer - struct _thread_t *next; -} thread_t; + struct _mp_thread_t *next; +} mp_thread_t; STATIC pthread_key_t tls_key; @@ -66,7 +66,7 @@ STATIC pthread_key_t tls_key; // Specifically for thread management, access to the linked list is one example. // But also, e.g. scheduler state. STATIC pthread_mutex_t thread_mutex; -STATIC thread_t *thread; +STATIC mp_thread_t *thread; // this is used to synchronise the signal handler of the thread // it's needed because we can't use any pthread calls in a signal handler @@ -119,7 +119,7 @@ void mp_thread_init(void) { pthread_mutex_init(&thread_mutex, &thread_mutex_attr); // create first entry in linked list of all threads - thread = malloc(sizeof(thread_t)); + thread = malloc(sizeof(mp_thread_t)); thread->id = pthread_self(); thread->ready = 1; thread->arg = NULL; @@ -143,7 +143,7 @@ void mp_thread_init(void) { void mp_thread_deinit(void) { mp_thread_unix_begin_atomic_section(); while (thread->next != NULL) { - thread_t *th = thread; + mp_thread_t *th = thread; thread = thread->next; pthread_cancel(th->id); free(th); @@ -165,7 +165,7 @@ void mp_thread_deinit(void) { // garbage collection and tracing these pointers. void mp_thread_gc_others(void) { mp_thread_unix_begin_atomic_section(); - for (thread_t *th = thread; th != NULL; th = th->next) { + for (mp_thread_t *th = thread; th != NULL; th = th->next) { gc_collect_root(&th->arg, 1); if (th->id == pthread_self()) { continue; @@ -192,9 +192,16 @@ void mp_thread_set_state(mp_state_thread_t *state) { } void mp_thread_start(void) { + // enable realtime priority if `-X realtime` command line parameter was set + #if defined(__APPLE__) + if (mp_thread_is_realtime_enabled) { + mp_thread_set_realtime(); + } + #endif + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); mp_thread_unix_begin_atomic_section(); - for (thread_t *th = thread; th != NULL; th = th->next) { + for (mp_thread_t *th = thread; th != NULL; th = th->next) { if (th->id == pthread_self()) { th->ready = 1; break; @@ -249,7 +256,7 @@ void mp_thread_create(void *(*entry)(void *), void *arg, size_t *stack_size) { *stack_size -= THREAD_STACK_OVERFLOW_MARGIN; // add thread to linked list of all threads - thread_t *th = malloc(sizeof(thread_t)); + mp_thread_t *th = malloc(sizeof(mp_thread_t)); th->id = id; th->ready = 0; th->arg = arg; @@ -266,8 +273,8 @@ er: void mp_thread_finish(void) { mp_thread_unix_begin_atomic_section(); - thread_t *prev = NULL; - for (thread_t *th = thread; th != NULL; th = th->next) { + mp_thread_t *prev = NULL; + for (mp_thread_t *th = thread; th != NULL; th = th->next) { if (th->id == pthread_self()) { if (prev == NULL) { thread = th->next; @@ -310,3 +317,39 @@ void mp_thread_mutex_unlock(mp_thread_mutex_t *mutex) { } #endif // MICROPY_PY_THREAD + +// this is used even when MICROPY_PY_THREAD is disabled + +#if defined(__APPLE__) +#include +#include +#include +#include + +bool mp_thread_is_realtime_enabled; + +// based on https://developer.apple.com/library/archive/technotes/tn2169/_index.html +void mp_thread_set_realtime(void) { + mach_timebase_info_data_t timebase_info; + + mach_timebase_info(&timebase_info); + + const uint64_t NANOS_PER_MSEC = 1000000ULL; + double clock2abs = ((double)timebase_info.denom / (double)timebase_info.numer) * NANOS_PER_MSEC; + + thread_time_constraint_policy_data_t policy; + policy.period = 0; + policy.computation = (uint32_t)(5 * clock2abs); // 5 ms of work + policy.constraint = (uint32_t)(10 * clock2abs); + policy.preemptible = FALSE; + + int kr = thread_policy_set(pthread_mach_thread_np(pthread_self()), + THREAD_TIME_CONSTRAINT_POLICY, + (thread_policy_t)&policy, + THREAD_TIME_CONSTRAINT_POLICY_COUNT); + + if (kr != KERN_SUCCESS) { + mach_error("thread_policy_set:", kr); + } +} +#endif diff --git a/ports/unix/mpthreadport.h b/ports/unix/mpthreadport.h index 94edbe2f04..b365f200ed 100644 --- a/ports/unix/mpthreadport.h +++ b/ports/unix/mpthreadport.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd + * Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -25,6 +25,7 @@ */ #include +#include typedef pthread_mutex_t mp_thread_mutex_t; @@ -36,3 +37,9 @@ void mp_thread_gc_others(void); // Functions as a port-global lock for any code that must be serialised. void mp_thread_unix_begin_atomic_section(void); void mp_thread_unix_end_atomic_section(void); + +// for `-X realtime` command line option +#if defined(__APPLE__) +extern bool mp_thread_is_realtime_enabled; +void mp_thread_set_realtime(void); +#endif diff --git a/ports/unix/qstrdefsport.h b/ports/unix/qstrdefsport.h index 0c99d91c6e..8b827a8d64 100644 --- a/ports/unix/qstrdefsport.h +++ b/ports/unix/qstrdefsport.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/ports/unix/unix_mphal.c b/ports/unix/unix_mphal.c index 9d50652e39..08d5d35723 100644 --- a/ports/unix/unix_mphal.c +++ b/ports/unix/unix_mphal.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2015 Damien P. George + * Copyright (c) 2015 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,10 +29,19 @@ #include #include #include +#include #include "py/mphal.h" #include "py/mpthread.h" #include "py/runtime.h" +#include "extmod/misc.h" + +#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) +#if __GLIBC_PREREQ(2, 25) +#include +#define _HAVE_GETRANDOM +#endif +#endif #ifndef _WIN32 #include @@ -86,6 +95,7 @@ void mp_hal_set_interrupt_char(char c) { } } +// CIRCUITPY bool mp_hal_is_interrupted(void) { return false; } @@ -185,3 +195,14 @@ void mp_hal_delay_ms(mp_uint_t ms) { usleep(ms * 1000); #endif } + +void mp_hal_get_random(size_t n, void *buf) { + #ifdef _HAVE_GETRANDOM + RAISE_ERRNO(getrandom(buf, n, 0), errno); + #else + int fd = open("/dev/urandom", O_RDONLY); + RAISE_ERRNO(fd, errno); + RAISE_ERRNO(read(fd, buf, n), errno); + close(fd); + #endif +} diff --git a/ports/unix/variants/coverage/frzmpy/frzmpy3.py b/ports/unix/variants/coverage/frzmpy/frzmpy3.py new file mode 100644 index 0000000000..617fac5523 --- /dev/null +++ b/ports/unix/variants/coverage/frzmpy/frzmpy3.py @@ -0,0 +1,6 @@ +# Test freezing objects that may not be handled well by the build process. + +print("\\") +print("\nX") +print(repr("\x1b")) +print(b"\x00\xff") diff --git a/ports/unix/variants/coverage/mpconfigvariant.h b/ports/unix/variants/coverage/mpconfigvariant.h index 0c6e10aeae..5828cb5616 100644 --- a/ports/unix/variants/coverage/mpconfigvariant.h +++ b/ports/unix/variants/coverage/mpconfigvariant.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2016 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -27,11 +27,20 @@ // This config enables almost all possible features such that it can be used // for coverage testing. +// Set base feature level. +#define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_EXTRA_FEATURES) + #define MICROPY_VFS (1) #define MICROPY_PY_UOS_VFS (1) +// Disable some features that come enabled by default with the feature level. +#define MICROPY_PY_BUILTINS_EXECFILE (0) +#define MICROPY_PY_SYS_STDIO_BUFFER (0) +#define MICROPY_PY_USELECT (0) + +// Enable additional features. #define MICROPY_DEBUG_PARSE_RULE_NAME (1) -#define MICROPY_OPT_MATH_FACTORIAL (1) +#define MICROPY_TRACKED_ALLOC (1) #define MICROPY_FLOAT_HIGH_QUALITY_HASH (1) #define MICROPY_ENABLE_SCHEDULER (1) #define MICROPY_READER_VFS (1) @@ -48,10 +57,12 @@ #define MICROPY_PY_BUILTINS_HELP (1) #define MICROPY_PY_BUILTINS_HELP_MODULES (1) #define MICROPY_PY_SYS_GETSIZEOF (1) +#define MICROPY_PY_SYS_TRACEBACKLIMIT (1) #define MICROPY_PY_MATH_FACTORIAL (1) #define MICROPY_PY_URANDOM_EXTRA_FUNCS (1) #define MICROPY_PY_IO_BUFFEREDWRITER (1) #define MICROPY_PY_UASYNCIO (1) +#define MICROPY_PY_URANDOM_SEED_INIT_FUNC (mp_urandom_seed_init()) #define MICROPY_PY_URE_DEBUG (1) #define MICROPY_PY_URE_MATCH_GROUPS (1) #define MICROPY_PY_URE_MATCH_SPAN_START_END (1) @@ -66,9 +77,3 @@ #define MICROPY_PY_UCRYPTOLIB (1) #define MICROPY_PY_UCRYPTOLIB_CTR (1) #define MICROPY_PY_MICROPYTHON_HEAP_LOCKED (1) -#define MICROPY_CPYTHON_EXCEPTION_CHAIN (1) - -// use vfs's functions for import stat and builtin open -#define mp_import_stat mp_vfs_import_stat -#define mp_builtin_open mp_vfs_open -#define mp_builtin_open_obj mp_vfs_open_obj diff --git a/ports/unix/variants/dev/mpconfigvariant.h b/ports/unix/variants/dev/mpconfigvariant.h index 54ad993236..61092394cd 100644 --- a/ports/unix/variants/dev/mpconfigvariant.h +++ b/ports/unix/variants/dev/mpconfigvariant.h @@ -24,24 +24,16 @@ * THE SOFTWARE. */ -#define MICROPY_READER_VFS (1) +// Set base feature level. +#define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_EXTRA_FEATURES) + +// Disable some features that come enabled by default with the feature level. +#define MICROPY_PY_BUILTINS_EXECFILE (0) +#define MICROPY_PY_SYS_STDIO_BUFFER (0) +#define MICROPY_PY_USELECT (0) + +// Enable some additional features. #define MICROPY_REPL_EMACS_WORDS_MOVE (1) #define MICROPY_REPL_EMACS_EXTRA_WORDS_MOVE (1) -#define MICROPY_ENABLE_SCHEDULER (1) -#define MICROPY_VFS (1) -#define MICROPY_VFS_POSIX (1) - -#define MICROPY_PY_BUILTINS_HELP (1) -#define MICROPY_PY_BUILTINS_HELP_MODULES (1) #define MICROPY_PY_SYS_SETTRACE (1) -#define MICROPY_PY_UOS_VFS (1) -#define MICROPY_PY_URANDOM_EXTRA_FUNCS (1) - -#ifndef MICROPY_PY_UASYNCIO -#define MICROPY_PY_UASYNCIO (1) -#endif - -// Use vfs's functions for import stat and builtin open. -#define mp_import_stat mp_vfs_import_stat -#define mp_builtin_open mp_vfs_open -#define mp_builtin_open_obj mp_vfs_open_obj +#define MICROPY_PY_URANDOM_SEED_INIT_FUNC (mp_urandom_seed_init()) diff --git a/ports/unix/variants/dev/mpconfigvariant.mk b/ports/unix/variants/dev/mpconfigvariant.mk index 16d033c6ce..91bd28da9b 100644 --- a/ports/unix/variants/dev/mpconfigvariant.mk +++ b/ports/unix/variants/dev/mpconfigvariant.mk @@ -2,6 +2,7 @@ PROG ?= micropython-dev FROZEN_MANIFEST ?= $(VARIANT_DIR)/manifest.py +MICROPY_ROM_TEXT_COMPRESSION = 1 MICROPY_VFS_FAT = 1 MICROPY_VFS_LFS1 = 1 MICROPY_VFS_LFS2 = 1 diff --git a/ports/unix/variants/fast/mpconfigvariant.h b/ports/unix/variants/fast/mpconfigvariant.h index ba9981db1b..8a531b056a 100644 --- a/ports/unix/variants/fast/mpconfigvariant.h +++ b/ports/unix/variants/fast/mpconfigvariant.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/ports/unix/variants/freedos/mpconfigvariant.h b/ports/unix/variants/freedos/mpconfigvariant.h index 9e93661ddf..562c783ca3 100644 --- a/ports/unix/variants/freedos/mpconfigvariant.h +++ b/ports/unix/variants/freedos/mpconfigvariant.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2015 Damien P. George + * Copyright (c) 2015 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/ports/unix/variants/minimal/mpconfigvariant.h b/ports/unix/variants/minimal/mpconfigvariant.h index 09dc37d1c3..60d194b6a1 100644 --- a/ports/unix/variants/minimal/mpconfigvariant.h +++ b/ports/unix/variants/minimal/mpconfigvariant.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2015 Damien P. George + * Copyright (c) 2015 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -44,6 +44,7 @@ #define MICROPY_MEM_STATS (0) #define MICROPY_DEBUG_PRINTERS (0) #define MICROPY_READER_POSIX (1) +#define MICROPY_READER_VFS (1) #define MICROPY_HELPER_REPL (1) #define MICROPY_HELPER_LEXER_UNIX (1) #define MICROPY_ENABLE_SOURCE_LINE (0) @@ -59,6 +60,8 @@ #define MICROPY_OPT_MAP_LOOKUP_CACHE (0) #define MICROPY_CAN_OVERRIDE_BUILTINS (0) #define MICROPY_BUILTIN_METHOD_CHECK_SELF_ARG (0) +#define MICROPY_VFS (1) +#define MICROPY_VFS_POSIX (1) #define MICROPY_CPYTHON_COMPAT (0) #define MICROPY_PY_BUILTINS_BYTEARRAY (0) #define MICROPY_PY_BUILTINS_MEMORYVIEW (0) @@ -98,18 +101,17 @@ #define MICROPY_PY_UTIME (0) #define MICROPY_PY_UZLIB (0) #define MICROPY_PY_UJSON (0) +#define MICROPY_PY_UOS (1) #define MICROPY_PY_URE (0) #define MICROPY_PY_UHEAPQ (0) #define MICROPY_PY_UHASHLIB (0) #define MICROPY_PY_UBINASCII (0) -extern const struct _mp_obj_module_t mp_module_os; - -#define MICROPY_PORT_BUILTIN_MODULES \ - { MP_ROM_QSTR(MP_QSTR_uos), MP_ROM_PTR(&mp_module_os) }, \ - #define MICROPY_PORT_ROOT_POINTERS \ +#define mp_type_fileio mp_type_vfs_posix_fileio +#define mp_type_textio mp_type_vfs_posix_textio + ////////////////////////////////////////// // Do not change anything beyond this line ////////////////////////////////////////// diff --git a/ports/unix/variants/minimal/mpconfigvariant.mk b/ports/unix/variants/minimal/mpconfigvariant.mk index 84343557aa..ec3b21c0b9 100644 --- a/ports/unix/variants/minimal/mpconfigvariant.mk +++ b/ports/unix/variants/minimal/mpconfigvariant.mk @@ -3,6 +3,7 @@ PROG = micropython-minimal FROZEN_MANIFEST = +MICROPY_ROM_TEXT_COMPRESSION = 1 MICROPY_PY_BTREE = 0 MICROPY_PY_FFI = 0 MICROPY_PY_SOCKET = 0 diff --git a/ports/unix/variants/standard/mpconfigvariant.h b/ports/unix/variants/standard/mpconfigvariant.h index 3cdcfa8e9b..1ec46ef92d 100644 --- a/ports/unix/variants/standard/mpconfigvariant.h +++ b/ports/unix/variants/standard/mpconfigvariant.h @@ -24,5 +24,22 @@ * THE SOFTWARE. */ -#define MICROPY_PY_BUILTINS_HELP (1) -#define MICROPY_PY_BUILTINS_HELP_MODULES (1) +// Set base feature level. +#define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_EXTRA_FEATURES) + +// Disable some features that come enabled by default with the feature level. +#define MICROPY_OPT_MPZ_BITWISE (0) +#define MICROPY_OPT_MATH_FACTORIAL (0) +#define MICROPY_MODULE_ATTR_DELEGATION (0) +#define MICROPY_MODULE_BUILTIN_INIT (0) +#define MICROPY_ENABLE_SCHEDULER (0) +#define MICROPY_PY_BUILTINS_EXECFILE (0) +#define MICROPY_PY_MATH_CONSTANTS (0) +#define MICROPY_PY_MATH_FACTORIAL (0) +#define MICROPY_PY_SYS_PS1_PS2 (0) +#define MICROPY_PY_SYS_STDIO_BUFFER (0) +#define MICROPY_PY_USELECT (0) +#define MICROPY_PY_UASYNCIO (0) +#define MICROPY_PY_URE_SUB (0) +#define MICROPY_PY_URANDOM_EXTRA_FUNCS (0) +#define MICROPY_PY_FRAMEBUF (0) diff --git a/py/argcheck.c b/py/argcheck.c index 8a4419f88f..c3bde804d4 100644 --- a/py/argcheck.c +++ b/py/argcheck.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/asmarm.c b/py/asmarm.c index 3b637d34ef..42724e4d4b 100644 --- a/py/asmarm.c +++ b/py/asmarm.c @@ -4,7 +4,7 @@ * The MIT License (MIT) * * Copyright (c) 2014 Fabian Vogt - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -304,6 +304,11 @@ void asm_arm_ldrh_reg_reg(asm_arm_t *as, uint rd, uint rn) { emit_al(as, 0x1d000b0 | (rn << 16) | (rd << 12)); } +void asm_arm_ldrh_reg_reg_offset(asm_arm_t *as, uint rd, uint rn, uint byte_offset) { + // ldrh rd, [rn, #off] + emit_al(as, 0x1f000b0 | (rn << 16) | (rd << 12) | ((byte_offset & 0xf0) << 4) | (byte_offset & 0xf)); +} + void asm_arm_ldrb_reg_reg(asm_arm_t *as, uint rd, uint rn) { // ldrb rd, [rn] emit_al(as, 0x5d00000 | (rn << 16) | (rd << 12)); diff --git a/py/asmarm.h b/py/asmarm.h index cab4de79c2..ed8dc5f039 100644 --- a/py/asmarm.h +++ b/py/asmarm.h @@ -4,7 +4,7 @@ * The MIT License (MIT) * * Copyright (c) 2014 Fabian Vogt - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -109,6 +109,7 @@ void asm_arm_asr_reg_reg(asm_arm_t *as, uint rd, uint rs); // memory void asm_arm_ldr_reg_reg(asm_arm_t *as, uint rd, uint rn, uint byte_offset); void asm_arm_ldrh_reg_reg(asm_arm_t *as, uint rd, uint rn); +void asm_arm_ldrh_reg_reg_offset(asm_arm_t *as, uint rd, uint rn, uint byte_offset); void asm_arm_ldrb_reg_reg(asm_arm_t *as, uint rd, uint rn); void asm_arm_str_reg_reg(asm_arm_t *as, uint rd, uint rm, uint byte_offset); void asm_arm_strh_reg_reg(asm_arm_t *as, uint rd, uint rm); @@ -203,6 +204,7 @@ void asm_arm_bx_reg(asm_arm_t *as, uint reg_src); #define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_arm_ldr_reg_reg((as), (reg_dest), (reg_base), 4 * (word_offset)) #define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) asm_arm_ldrb_reg_reg((as), (reg_dest), (reg_base)) #define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) asm_arm_ldrh_reg_reg((as), (reg_dest), (reg_base)) +#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, uint16_offset) asm_arm_ldrh_reg_reg_offset((as), (reg_dest), (reg_base), 2 * (uint16_offset)) #define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) asm_arm_ldr_reg_reg((as), (reg_dest), (reg_base), 0) #define ASM_STORE_REG_REG(as, reg_value, reg_base) asm_arm_str_reg_reg((as), (reg_value), (reg_base), 0) diff --git a/py/asmbase.c b/py/asmbase.c index ff1dd80ebe..4a3fd089cb 100644 --- a/py/asmbase.c +++ b/py/asmbase.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George + * Copyright (c) 2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -61,7 +61,8 @@ void mp_asm_base_start_pass(mp_asm_base_t *as, int pass) { // all functions must go through this one to emit bytes // if as->pass < MP_ASM_PASS_EMIT, then this function just counts the number // of bytes needed and returns NULL, and callers should not store any data -uint8_t *mp_asm_base_get_cur_to_write_bytes(mp_asm_base_t *as, size_t num_bytes_to_write) { +uint8_t *mp_asm_base_get_cur_to_write_bytes(void *as_in, size_t num_bytes_to_write) { + mp_asm_base_t *as = as_in; uint8_t *c = NULL; if (as->pass == MP_ASM_PASS_EMIT) { assert(as->code_offset + num_bytes_to_write <= as->code_size); diff --git a/py/asmbase.h b/py/asmbase.h index f2932d6e20..960be7685f 100644 --- a/py/asmbase.h +++ b/py/asmbase.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George + * Copyright (c) 2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -45,7 +45,7 @@ typedef struct _mp_asm_base_t { void mp_asm_base_init(mp_asm_base_t *as, size_t max_num_labels); void mp_asm_base_deinit(mp_asm_base_t *as, bool free_code); void mp_asm_base_start_pass(mp_asm_base_t *as, int pass); -uint8_t *mp_asm_base_get_cur_to_write_bytes(mp_asm_base_t *as, size_t num_bytes_to_write); +uint8_t *mp_asm_base_get_cur_to_write_bytes(void *as, size_t num_bytes_to_write); void mp_asm_base_label_assign(mp_asm_base_t *as, size_t label); void mp_asm_base_align(mp_asm_base_t *as, unsigned int align); void mp_asm_base_data(mp_asm_base_t *as, unsigned int bytesize, uintptr_t val); diff --git a/py/asmthumb.c b/py/asmthumb.c index 1b1923370b..49574c43a7 100644 --- a/py/asmthumb.c +++ b/py/asmthumb.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -34,9 +34,25 @@ #if MICROPY_EMIT_THUMB || MICROPY_EMIT_INLINE_THUMB #include "py/mpstate.h" -#include "py/persistentcode.h" #include "py/asmthumb.h" +#ifdef _MSC_VER +#include + +static uint32_t mp_clz(uint32_t x) { + unsigned long lz = 0; + return _BitScanReverse(&lz, x) ? (sizeof(x) * 8 - 1) - lz : 0; +} + +static uint32_t mp_ctz(uint32_t x) { + unsigned long tz = 0; + return _BitScanForward(&tz, x) ? tz : 0; +} +#else +#define mp_clz(x) __builtin_clz(x) +#define mp_ctz(x) __builtin_ctz(x) +#endif + #define UNSIGNED_FIT5(x) ((uint32_t)(x) < 32) #define UNSIGNED_FIT7(x) ((uint32_t)(x) < 128) #define UNSIGNED_FIT8(x) (((x) & 0xffffff00) == 0) @@ -46,7 +62,6 @@ #define SIGNED_FIT12(x) (((x) & 0xfffff800) == 0) || (((x) & 0xfffff800) == 0xfffff800) #define SIGNED_FIT23(x) (((x) & 0xffc00000) == 0) || (((x) & 0xffc00000) == 0xffc00000) -#if MICROPY_EMIT_THUMB_ARMV7M // Note: these actually take an imm12 but the high-bit is not encoded here #define OP_ADD_W_RRI_HI(reg_src) (0xf200 | (reg_src)) #define OP_ADD_W_RRI_LO(reg_dest, imm11) ((imm11 << 4 & 0x7000) | reg_dest << 8 | (imm11 & 0xff)) @@ -55,7 +70,9 @@ #define OP_LDR_W_HI(reg_base) (0xf8d0 | (reg_base)) #define OP_LDR_W_LO(reg_dest, imm12) ((reg_dest) << 12 | (imm12)) -#endif + +#define OP_LDRH_W_HI(reg_base) (0xf8b0 | (reg_base)) +#define OP_LDRH_W_LO(reg_dest, imm12) ((reg_dest) << 12 | (imm12)) static inline byte *asm_thumb_get_cur_to_write_bytes(asm_thumb_t *as, int n) { return mp_asm_base_get_cur_to_write_bytes(&as->base, n); @@ -158,21 +175,21 @@ void asm_thumb_entry(asm_thumb_t *as, int num_locals) { } asm_thumb_op16(as, OP_PUSH_RLIST_LR(reglist)); if (stack_adjust > 0) { - #if MICROPY_EMIT_THUMB_ARMV7M - if (UNSIGNED_FIT7(stack_adjust)) { - asm_thumb_op16(as, OP_SUB_SP(stack_adjust)); + if (asm_thumb_allow_armv7m(as)) { + if (UNSIGNED_FIT7(stack_adjust)) { + asm_thumb_op16(as, OP_SUB_SP(stack_adjust)); + } else { + asm_thumb_op32(as, OP_SUB_W_RRI_HI(ASM_THUMB_REG_SP), OP_SUB_W_RRI_LO(ASM_THUMB_REG_SP, stack_adjust * 4)); + } } else { - asm_thumb_op32(as, OP_SUB_W_RRI_HI(ASM_THUMB_REG_SP), OP_SUB_W_RRI_LO(ASM_THUMB_REG_SP, stack_adjust * 4)); + int adj = stack_adjust; + // we don't expect the stack_adjust to be massive + while (!UNSIGNED_FIT7(adj)) { + asm_thumb_op16(as, OP_SUB_SP(127)); + adj -= 127; + } + asm_thumb_op16(as, OP_SUB_SP(adj)); } - #else - int adj = stack_adjust; - // we don't expect the stack_adjust to be massive - while (!UNSIGNED_FIT7(adj)) { - asm_thumb_op16(as, OP_SUB_SP(127)); - adj -= 127; - } - asm_thumb_op16(as, OP_SUB_SP(adj)); - #endif } as->push_reglist = reglist; as->stack_adjust = stack_adjust; @@ -180,21 +197,21 @@ void asm_thumb_entry(asm_thumb_t *as, int num_locals) { void asm_thumb_exit(asm_thumb_t *as) { if (as->stack_adjust > 0) { - #if MICROPY_EMIT_THUMB_ARMV7M - if (UNSIGNED_FIT7(as->stack_adjust)) { - asm_thumb_op16(as, OP_ADD_SP(as->stack_adjust)); + if (asm_thumb_allow_armv7m(as)) { + if (UNSIGNED_FIT7(as->stack_adjust)) { + asm_thumb_op16(as, OP_ADD_SP(as->stack_adjust)); + } else { + asm_thumb_op32(as, OP_ADD_W_RRI_HI(ASM_THUMB_REG_SP), OP_ADD_W_RRI_LO(ASM_THUMB_REG_SP, as->stack_adjust * 4)); + } } else { - asm_thumb_op32(as, OP_ADD_W_RRI_HI(ASM_THUMB_REG_SP), OP_ADD_W_RRI_LO(ASM_THUMB_REG_SP, as->stack_adjust * 4)); + int adj = as->stack_adjust; + // we don't expect the stack_adjust to be massive + while (!UNSIGNED_FIT7(adj)) { + asm_thumb_op16(as, OP_ADD_SP(127)); + adj -= 127; + } + asm_thumb_op16(as, OP_ADD_SP(adj)); } - #else - int adj = as->stack_adjust; - // we don't expect the stack_adjust to be massive - while (!UNSIGNED_FIT7(adj)) { - asm_thumb_op16(as, OP_ADD_SP(127)); - adj -= 127; - } - asm_thumb_op16(as, OP_ADD_SP(adj)); - #endif } asm_thumb_op16(as, OP_POP_RLIST_PC(as->push_reglist)); } @@ -248,27 +265,19 @@ void asm_thumb_mov_reg_reg(asm_thumb_t *as, uint reg_dest, uint reg_src) { asm_thumb_op16(as, 0x4600 | op_lo); } -#if MICROPY_EMIT_THUMB_ARMV7M - // if loading lo half with movw, the i16 value will be zero extended into the r32 register! -size_t asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src) { +void asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src) { assert(reg_dest < ASM_THUMB_REG_R15); - size_t loc = mp_asm_base_get_code_pos(&as->base); // mov[wt] reg_dest, #i16_src asm_thumb_op32(as, mov_op | ((i16_src >> 1) & 0x0400) | ((i16_src >> 12) & 0xf), ((i16_src << 4) & 0x7000) | (reg_dest << 8) | (i16_src & 0xff)); - return loc; } -#else - -void asm_thumb_mov_rlo_i16(asm_thumb_t *as, uint rlo_dest, int i16_src) { +static void asm_thumb_mov_rlo_i16(asm_thumb_t *as, uint rlo_dest, int i16_src) { asm_thumb_mov_rlo_i8(as, rlo_dest, (i16_src >> 8) & 0xff); asm_thumb_lsl_rlo_rlo_i5(as, rlo_dest, rlo_dest, 8); asm_thumb_add_rlo_i8(as, rlo_dest, i16_src & 0xff); } -#endif - #define OP_B_N(byte_offset) (0xe000 | (((byte_offset) >> 1) & 0x07ff)) bool asm_thumb_b_n_label(asm_thumb_t *as, uint label) { @@ -281,7 +290,7 @@ bool asm_thumb_b_n_label(asm_thumb_t *as, uint label) { #define OP_BCC_N(cond, byte_offset) (0xd000 | ((cond) << 8) | (((byte_offset) >> 1) & 0x00ff)) -// all these bit arithmetic need coverage testing! +// all these bit arithmetics need coverage testing! #define OP_BCC_W_HI(cond, byte_offset) (0xf000 | ((cond) << 6) | (((byte_offset) >> 10) & 0x0400) | (((byte_offset) >> 14) & 0x003f)) #define OP_BCC_W_LO(byte_offset) (0x8000 | ((byte_offset) & 0x2000) | (((byte_offset) >> 1) & 0x0fff)) @@ -292,14 +301,12 @@ bool asm_thumb_bcc_nw_label(asm_thumb_t *as, int cond, uint label, bool wide) { if (!wide) { asm_thumb_op16(as, OP_BCC_N(cond, rel)); return as->base.pass != MP_ASM_PASS_EMIT || SIGNED_FIT9(rel); - } else { - #if MICROPY_EMIT_THUMB_ARMV7M + } else if (asm_thumb_allow_armv7m(as)) { asm_thumb_op32(as, OP_BCC_W_HI(cond, rel), OP_BCC_W_LO(rel)); return true; - #else + } else { // this method should not be called for ARMV6M return false; - #endif } } @@ -320,30 +327,30 @@ size_t asm_thumb_mov_reg_i32(asm_thumb_t *as, uint reg_dest, mp_uint_t i32) { size_t loc = mp_asm_base_get_code_pos(&as->base); - #if MICROPY_EMIT_THUMB_ARMV7M - asm_thumb_mov_reg_i16(as, ASM_THUMB_OP_MOVW, reg_dest, i32); - asm_thumb_mov_reg_i16(as, ASM_THUMB_OP_MOVT, reg_dest, i32 >> 16); - #else - // should only be called with lo reg for ARMV6M - assert(reg_dest < ASM_THUMB_REG_R8); + if (asm_thumb_allow_armv7m(as)) { + asm_thumb_mov_reg_i16(as, ASM_THUMB_OP_MOVW, reg_dest, i32); + asm_thumb_mov_reg_i16(as, ASM_THUMB_OP_MOVT, reg_dest, i32 >> 16); + } else { + // should only be called with lo reg for ARMV6M + assert(reg_dest < ASM_THUMB_REG_R8); - // sanity check that generated code is aligned - assert(!as->base.code_base || !(3u & (uintptr_t)as->base.code_base)); + // sanity check that generated code is aligned + assert(!as->base.code_base || !(3u & (uintptr_t)as->base.code_base)); - // basically: - // (nop) - // ldr reg_dest, _data - // b 1f - // _data: .word i32 - // 1: - if (as->base.code_offset & 2u) { - asm_thumb_op16(as, ASM_THUMB_OP_NOP); + // basically: + // (nop) + // ldr reg_dest, _data + // b 1f + // _data: .word i32 + // 1: + if (as->base.code_offset & 2u) { + asm_thumb_op16(as, ASM_THUMB_OP_NOP); + } + asm_thumb_ldr_rlo_pcrel_i8(as, reg_dest, 0); + asm_thumb_op16(as, OP_B_N(2)); + asm_thumb_op16(as, i32 & 0xffff); + asm_thumb_op16(as, i32 >> 16); } - asm_thumb_ldr_rlo_pcrel_i8(as, reg_dest, 0); - asm_thumb_op16(as, OP_B_N(2)); - asm_thumb_op16(as, i32 & 0xffff); - asm_thumb_op16(as, i32 >> 16); - #endif return loc; } @@ -351,14 +358,13 @@ size_t asm_thumb_mov_reg_i32(asm_thumb_t *as, uint reg_dest, mp_uint_t i32) { void asm_thumb_mov_reg_i32_optimised(asm_thumb_t *as, uint reg_dest, int i32) { if (reg_dest < 8 && UNSIGNED_FIT8(i32)) { asm_thumb_mov_rlo_i8(as, reg_dest, i32); - } else { - #if MICROPY_EMIT_THUMB_ARMV7M + } else if (asm_thumb_allow_armv7m(as)) { if (UNSIGNED_FIT16(i32)) { asm_thumb_mov_reg_i16(as, ASM_THUMB_OP_MOVW, reg_dest, i32); } else { asm_thumb_mov_reg_i32(as, reg_dest, i32); } - #else + } else { uint rlo_dest = reg_dest; assert(rlo_dest < ASM_THUMB_REG_R8); // should never be called for ARMV6M @@ -367,8 +373,8 @@ void asm_thumb_mov_reg_i32_optimised(asm_thumb_t *as, uint reg_dest, int i32) { i32 = -i32; } - uint clz = __builtin_clz(i32); - uint ctz = i32 ? __builtin_ctz(i32) : 0; + uint clz = mp_clz(i32); + uint ctz = i32 ? mp_ctz(i32) : 0; assert(clz + ctz <= 32); if (clz + ctz >= 24) { asm_thumb_mov_rlo_i8(as, rlo_dest, (i32 >> ctz) & 0xff); @@ -386,7 +392,6 @@ void asm_thumb_mov_reg_i32_optimised(asm_thumb_t *as, uint reg_dest, int i32) { if (negate) { asm_thumb_neg_rlo_rlo(as, rlo_dest, rlo_dest); } - #endif } } @@ -429,62 +434,76 @@ void asm_thumb_mov_reg_pcrel(asm_thumb_t *as, uint rlo_dest, uint label) { mp_uint_t dest = get_label_dest(as, label); mp_int_t rel = dest - as->base.code_offset; rel |= 1; // to stay in Thumb state when jumping to this address - #if MICROPY_EMIT_THUMB_ARMV7M - rel -= 4 + 4; // adjust for mov_reg_i16 and then PC+4 prefetch of add_reg_reg - asm_thumb_mov_reg_i16(as, ASM_THUMB_OP_MOVW, rlo_dest, rel); // 4 bytes - #else - rel -= 8 + 4; // adjust for four instructions and then PC+4 prefetch of add_reg_reg - // 6 bytes - asm_thumb_mov_rlo_i16(as, rlo_dest, rel); - // 2 bytes - not always needed, but we want to keep the size the same - asm_thumb_sxth_rlo_rlo(as, rlo_dest, rlo_dest); - #endif + if (asm_thumb_allow_armv7m(as)) { + rel -= 6 + 4; // adjust for mov_reg_i16, sxth_rlo_rlo and then PC+4 prefetch of add_reg_reg + asm_thumb_mov_reg_i16(as, ASM_THUMB_OP_MOVW, rlo_dest, rel); // 4 bytes + asm_thumb_sxth_rlo_rlo(as, rlo_dest, rlo_dest); // 2 bytes + } else { + rel -= 8 + 4; // adjust for four instructions and then PC+4 prefetch of add_reg_reg + // 6 bytes + asm_thumb_mov_rlo_i16(as, rlo_dest, rel); + // 2 bytes - not always needed, but we want to keep the size the same + asm_thumb_sxth_rlo_rlo(as, rlo_dest, rlo_dest); + } asm_thumb_add_reg_reg(as, rlo_dest, ASM_THUMB_REG_R15); // 2 bytes } -#if MICROPY_EMIT_THUMB_ARMV7M +// ARMv7-M only static inline void asm_thumb_ldr_reg_reg_i12(asm_thumb_t *as, uint reg_dest, uint reg_base, uint word_offset) { asm_thumb_op32(as, OP_LDR_W_HI(reg_base), OP_LDR_W_LO(reg_dest, word_offset * 4)); } -#endif + +// emits code for: reg_dest = reg_base + offset << offset_shift +static void asm_thumb_add_reg_reg_offset(asm_thumb_t *as, uint reg_dest, uint reg_base, uint offset, uint offset_shift) { + if (reg_dest < ASM_THUMB_REG_R8 && reg_base < ASM_THUMB_REG_R8) { + if (offset << offset_shift < 256) { + if (reg_dest != reg_base) { + asm_thumb_mov_reg_reg(as, reg_dest, reg_base); + } + asm_thumb_add_rlo_i8(as, reg_dest, offset << offset_shift); + } else if (UNSIGNED_FIT8(offset) && reg_dest != reg_base) { + asm_thumb_mov_rlo_i8(as, reg_dest, offset); + asm_thumb_lsl_rlo_rlo_i5(as, reg_dest, reg_dest, offset_shift); + asm_thumb_add_rlo_rlo_rlo(as, reg_dest, reg_dest, reg_base); + } else if (reg_dest != reg_base) { + asm_thumb_mov_rlo_i16(as, reg_dest, offset << offset_shift); + asm_thumb_add_rlo_rlo_rlo(as, reg_dest, reg_dest, reg_dest); + } else { + uint reg_other = reg_dest ^ 7; + asm_thumb_op16(as, OP_PUSH_RLIST((1 << reg_other))); + asm_thumb_mov_rlo_i16(as, reg_other, offset << offset_shift); + asm_thumb_add_rlo_rlo_rlo(as, reg_dest, reg_dest, reg_other); + asm_thumb_op16(as, OP_POP_RLIST((1 << reg_other))); + } + } else { + assert(0); // should never be called for ARMV6M + } +} void asm_thumb_ldr_reg_reg_i12_optimised(asm_thumb_t *as, uint reg_dest, uint reg_base, uint word_offset) { if (reg_dest < ASM_THUMB_REG_R8 && reg_base < ASM_THUMB_REG_R8 && UNSIGNED_FIT5(word_offset)) { asm_thumb_ldr_rlo_rlo_i5(as, reg_dest, reg_base, word_offset); - } else { - #if MICROPY_EMIT_THUMB_ARMV7M + } else if (asm_thumb_allow_armv7m(as)) { asm_thumb_ldr_reg_reg_i12(as, reg_dest, reg_base, word_offset); - #else - word_offset -= 31; - if (reg_dest < ASM_THUMB_REG_R8 && reg_base < ASM_THUMB_REG_R8) { - if (UNSIGNED_FIT8(word_offset) && (word_offset < 64 || reg_dest != reg_base)) { - if (word_offset < 64) { - if (reg_dest != reg_base) { - asm_thumb_mov_reg_reg(as, reg_dest, reg_base); - } - asm_thumb_add_rlo_i8(as, reg_dest, word_offset * 4); - } else { - asm_thumb_mov_rlo_i8(as, reg_dest, word_offset); - asm_thumb_lsl_rlo_rlo_i5(as, reg_dest, reg_dest, 2); - asm_thumb_add_rlo_rlo_rlo(as, reg_dest, reg_dest, reg_base); - } - } else { - if (reg_dest != reg_base) { - asm_thumb_mov_rlo_i16(as, reg_dest, word_offset * 4); - asm_thumb_add_rlo_rlo_rlo(as, reg_dest, reg_dest, reg_dest); - } else { - uint reg_other = reg_dest ^ 7; - asm_thumb_op16(as, OP_PUSH_RLIST((1 << reg_other))); - asm_thumb_mov_rlo_i16(as, reg_other, word_offset * 4); - asm_thumb_add_rlo_rlo_rlo(as, reg_dest, reg_dest, reg_other); - asm_thumb_op16(as, OP_POP_RLIST((1 << reg_other))); - } - } - } else { - assert(0); // should never be called for ARMV6M - } + } else { + asm_thumb_add_reg_reg_offset(as, reg_dest, reg_base, word_offset - 31, 2); asm_thumb_ldr_rlo_rlo_i5(as, reg_dest, reg_dest, 31); - #endif + } +} + +// ARMv7-M only +static inline void asm_thumb_ldrh_reg_reg_i12(asm_thumb_t *as, uint reg_dest, uint reg_base, uint uint16_offset) { + asm_thumb_op32(as, OP_LDRH_W_HI(reg_base), OP_LDRH_W_LO(reg_dest, uint16_offset * 2)); +} + +void asm_thumb_ldrh_reg_reg_i12_optimised(asm_thumb_t *as, uint reg_dest, uint reg_base, uint uint16_offset) { + if (reg_dest < ASM_THUMB_REG_R8 && reg_base < ASM_THUMB_REG_R8 && UNSIGNED_FIT5(uint16_offset)) { + asm_thumb_ldrh_rlo_rlo_i5(as, reg_dest, reg_base, uint16_offset); + } else if (asm_thumb_allow_armv7m(as)) { + asm_thumb_ldrh_reg_reg_i12(as, reg_dest, reg_base, uint16_offset); + } else { + asm_thumb_add_reg_reg_offset(as, reg_dest, reg_base, uint16_offset - 31, 1); + asm_thumb_ldrh_rlo_rlo_i5(as, reg_dest, reg_dest, 31); } } @@ -496,20 +515,21 @@ void asm_thumb_b_label(asm_thumb_t *as, uint label) { mp_uint_t dest = get_label_dest(as, label); mp_int_t rel = dest - as->base.code_offset; rel -= 4; // account for instruction prefetch, PC is 4 bytes ahead of this instruction + if (dest != (mp_uint_t)-1 && rel <= -4) { // is a backwards jump, so we know the size of the jump on the first pass // calculate rel assuming 12 bit relative jump if (SIGNED_FIT12(rel)) { asm_thumb_op16(as, OP_B_N(rel)); - } else { - goto large_jump; + return; } - } else { - // is a forwards jump, so need to assume it's large - large_jump: - #if MICROPY_EMIT_THUMB_ARMV7M + } + + // is a large backwards jump, or a forwards jump (that must be assumed large) + + if (asm_thumb_allow_armv7m(as)) { asm_thumb_op32(as, OP_BW_HI(rel), OP_BW_LO(rel)); - #else + } else { if (SIGNED_FIT12(rel)) { // this code path has to be the same number of instructions irrespective of rel asm_thumb_op16(as, OP_B_N(rel)); @@ -520,7 +540,6 @@ void asm_thumb_b_label(asm_thumb_t *as, uint label) { mp_raise_NotImplementedError(MP_ERROR_TEXT("native method too big")); } } - #endif } } @@ -528,24 +547,24 @@ void asm_thumb_bcc_label(asm_thumb_t *as, int cond, uint label) { mp_uint_t dest = get_label_dest(as, label); mp_int_t rel = dest - as->base.code_offset; rel -= 4; // account for instruction prefetch, PC is 4 bytes ahead of this instruction + if (dest != (mp_uint_t)-1 && rel <= -4) { // is a backwards jump, so we know the size of the jump on the first pass // calculate rel assuming 9 bit relative jump if (SIGNED_FIT9(rel)) { asm_thumb_op16(as, OP_BCC_N(cond, rel)); - } else { - goto large_jump; + return; } - } else { - // is a forwards jump, so need to assume it's large - large_jump: - #if MICROPY_EMIT_THUMB_ARMV7M + } + + // is a large backwards jump, or a forwards jump (that must be assumed large) + + if (asm_thumb_allow_armv7m(as)) { asm_thumb_op32(as, OP_BCC_W_HI(cond, rel), OP_BCC_W_LO(rel)); - #else + } else { // reverse the sense of the branch to jump over a longer branch asm_thumb_op16(as, OP_BCC_N(cond ^ 1, 0)); asm_thumb_b_label(as, label); - #endif } } diff --git a/py/asmthumb.h b/py/asmthumb.h index ea48f7ebf3..8961b0f796 100644 --- a/py/asmthumb.h +++ b/py/asmthumb.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,6 +29,7 @@ #include #include "py/misc.h" #include "py/asmbase.h" +#include "py/persistentcode.h" #define ASM_THUMB_REG_R0 (0) #define ASM_THUMB_REG_R1 (1) @@ -70,6 +71,21 @@ typedef struct _asm_thumb_t { uint32_t stack_adjust; } asm_thumb_t; +#if MICROPY_DYNAMIC_COMPILER + +static inline bool asm_thumb_allow_armv7m(asm_thumb_t *as) { + return MP_NATIVE_ARCH_ARMV7M <= mp_dynamic_compiler.native_arch + && mp_dynamic_compiler.native_arch <= MP_NATIVE_ARCH_ARMV7EMDP; +} + +#else + +static inline bool asm_thumb_allow_armv7m(asm_thumb_t *as) { + return MICROPY_EMIT_THUMB_ARMV7M; +} + +#endif + static inline void asm_thumb_end_pass(asm_thumb_t *as) { (void)as; } @@ -263,8 +279,8 @@ static inline void asm_thumb_str_rlo_rlo_i5(asm_thumb_t *as, uint rlo_src, uint static inline void asm_thumb_strb_rlo_rlo_i5(asm_thumb_t *as, uint rlo_src, uint rlo_base, uint byte_offset) { asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_9_STR | ASM_THUMB_FORMAT_9_BYTE_TRANSFER, rlo_src, rlo_base, byte_offset); } -static inline void asm_thumb_strh_rlo_rlo_i5(asm_thumb_t *as, uint rlo_src, uint rlo_base, uint byte_offset) { - asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_10_STRH, rlo_src, rlo_base, byte_offset); +static inline void asm_thumb_strh_rlo_rlo_i5(asm_thumb_t *as, uint rlo_src, uint rlo_base, uint uint16_offset) { + asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_10_STRH, rlo_src, rlo_base, uint16_offset); } static inline void asm_thumb_ldr_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uint rlo_base, uint word_offset) { asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_9_LDR | ASM_THUMB_FORMAT_9_WORD_TRANSFER, rlo_dest, rlo_base, word_offset); @@ -272,8 +288,8 @@ static inline void asm_thumb_ldr_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uint static inline void asm_thumb_ldrb_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uint rlo_base, uint byte_offset) { asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_9_LDR | ASM_THUMB_FORMAT_9_BYTE_TRANSFER, rlo_dest, rlo_base, byte_offset); } -static inline void asm_thumb_ldrh_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uint rlo_base, uint byte_offset) { - asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_10_LDRH, rlo_dest, rlo_base, byte_offset); +static inline void asm_thumb_ldrh_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uint rlo_base, uint uint16_offset) { + asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_10_LDRH, rlo_dest, rlo_base, uint16_offset); } static inline void asm_thumb_lsl_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uint rlo_src, uint shift) { asm_thumb_format_1(as, ASM_THUMB_FORMAT_1_LSL, rlo_dest, rlo_src, shift); @@ -308,12 +324,7 @@ static inline void asm_thumb_sxth_rlo_rlo(asm_thumb_t *as, uint rlo_dest, uint r #define ASM_THUMB_OP_MOVT (0xf2c0) void asm_thumb_mov_reg_reg(asm_thumb_t *as, uint reg_dest, uint reg_src); - -#if MICROPY_EMIT_THUMB_ARMV7M -size_t asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src); -#else -void asm_thumb_mov_rlo_i16(asm_thumb_t *as, uint rlo_dest, int i16_src); -#endif +void asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src); // these return true if the destination is in range, false otherwise bool asm_thumb_b_n_label(asm_thumb_t *as, uint label); @@ -327,7 +338,8 @@ void asm_thumb_mov_reg_local(asm_thumb_t *as, uint rlo_dest, int local_num); // void asm_thumb_mov_reg_local_addr(asm_thumb_t *as, uint rlo_dest, int local_num); // convenience void asm_thumb_mov_reg_pcrel(asm_thumb_t *as, uint rlo_dest, uint label); -void asm_thumb_ldr_reg_reg_i12_optimised(asm_thumb_t *as, uint reg_dest, uint reg_base, uint byte_offset); // convenience +void asm_thumb_ldr_reg_reg_i12_optimised(asm_thumb_t *as, uint reg_dest, uint reg_base, uint word_offset); // convenience +void asm_thumb_ldrh_reg_reg_i12_optimised(asm_thumb_t *as, uint reg_dest, uint reg_base, uint uint16_offset); // convenience void asm_thumb_b_label(asm_thumb_t *as, uint label); // convenience: picks narrow or wide branch void asm_thumb_bcc_label(asm_thumb_t *as, int cc, uint label); // convenience: picks narrow or wide branch @@ -389,11 +401,6 @@ void asm_thumb_b_rel12(asm_thumb_t *as, int rel); #define ASM_MOV_LOCAL_REG(as, local_num, reg) asm_thumb_mov_local_reg((as), (local_num), (reg)) #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_thumb_mov_reg_i32_optimised((as), (reg_dest), (imm)) -#if MICROPY_EMIT_THUMB_ARMV7M -#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_thumb_mov_reg_i16((as), ASM_THUMB_OP_MOVW, (reg_dest), (imm)) -#else -#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_thumb_mov_rlo_i16((as), (reg_dest), (imm)) -#endif #define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_thumb_mov_reg_i32((as), (reg_dest), (imm)) #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_thumb_mov_reg_local((as), (reg_dest), (local_num)) #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_thumb_mov_reg_reg((as), (reg_dest), (reg_src)) @@ -414,6 +421,7 @@ void asm_thumb_b_rel12(asm_thumb_t *as, int rel); #define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_thumb_ldr_reg_reg_i12_optimised((as), (reg_dest), (reg_base), (word_offset)) #define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) asm_thumb_ldrb_rlo_rlo_i5((as), (reg_dest), (reg_base), 0) #define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) asm_thumb_ldrh_rlo_rlo_i5((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, uint16_offset) asm_thumb_ldrh_reg_reg_i12_optimised((as), (reg_dest), (reg_base), (uint16_offset)) #define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) asm_thumb_ldr_rlo_rlo_i5((as), (reg_dest), (reg_base), 0) #define ASM_STORE_REG_REG(as, reg_src, reg_base) asm_thumb_str_rlo_rlo_i5((as), (reg_src), (reg_base), 0) diff --git a/py/asmx64.c b/py/asmx64.c index 62df5c6d4a..5c923a523c 100644 --- a/py/asmx64.c +++ b/py/asmx64.c @@ -319,9 +319,7 @@ void asm_x64_mov_mem64_to_r64(asm_x64_t *as, int src_r64, int src_disp, int dest STATIC void asm_x64_lea_disp_to_r64(asm_x64_t *as, int src_r64, int src_disp, int dest_r64) { // use REX prefix for 64 bit operation - assert(src_r64 < 8); - assert(dest_r64 < 8); - asm_x64_write_byte_2(as, REX_PREFIX | REX_W, OPCODE_LEA_MEM_TO_R64); + asm_x64_write_byte_2(as, REX_PREFIX | REX_W | REX_R_FROM_R64(dest_r64) | REX_B_FROM_R64(src_r64), OPCODE_LEA_MEM_TO_R64); asm_x64_write_r64_disp(as, dest_r64, src_r64, src_disp); } diff --git a/py/asmx64.h b/py/asmx64.h index ec9a088f88..a4eaa12984 100644 --- a/py/asmx64.h +++ b/py/asmx64.h @@ -207,6 +207,7 @@ void asm_x64_call_ind(asm_x64_t *as, size_t fun_id, int temp_r32); #define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_x64_mov_mem64_to_r64((as), (reg_base), 8 * (word_offset), (reg_dest)) #define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) asm_x64_mov_mem8_to_r64zx((as), (reg_base), 0, (reg_dest)) #define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) asm_x64_mov_mem16_to_r64zx((as), (reg_base), 0, (reg_dest)) +#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, uint16_offset) asm_x64_mov_mem16_to_r64zx((as), (reg_base), 2 * (uint16_offset), (reg_dest)) #define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) asm_x64_mov_mem32_to_r64zx((as), (reg_base), 0, (reg_dest)) #define ASM_STORE_REG_REG(as, reg_src, reg_base) asm_x64_mov_r64_to_mem64((as), (reg_src), (reg_base), 0) diff --git a/py/asmx86.c b/py/asmx86.c index b44da76dab..4b0f8047f6 100644 --- a/py/asmx86.c +++ b/py/asmx86.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George + * Copyright (c) 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/asmx86.h b/py/asmx86.h index a8fb4e3f9e..af7bb84988 100644 --- a/py/asmx86.h +++ b/py/asmx86.h @@ -202,6 +202,7 @@ void asm_x86_call_ind(asm_x86_t *as, size_t fun_id, mp_uint_t n_args, int temp_r #define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_x86_mov_mem32_to_r32((as), (reg_base), 4 * (word_offset), (reg_dest)) #define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) asm_x86_mov_mem8_to_r32zx((as), (reg_base), 0, (reg_dest)) #define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) asm_x86_mov_mem16_to_r32zx((as), (reg_base), 0, (reg_dest)) +#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, uint16_offset) asm_x86_mov_mem16_to_r32zx((as), (reg_base), 2 * (uint16_offset), (reg_dest)) #define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) asm_x86_mov_mem32_to_r32((as), (reg_base), 0, (reg_dest)) #define ASM_STORE_REG_REG(as, reg_src, reg_base) asm_x86_mov_r32_to_mem32((as), (reg_src), (reg_base), 0) diff --git a/py/asmxtensa.c b/py/asmxtensa.c index fa53e8351c..8ac914ec41 100644 --- a/py/asmxtensa.c +++ b/py/asmxtensa.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George + * Copyright (c) 2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -27,7 +27,7 @@ #include #include -#include "py/mpconfig.h" +#include "py/runtime.h" // wrapper around everything in this file #if MICROPY_EMIT_XTENSA || MICROPY_EMIT_INLINE_XTENSA || MICROPY_EMIT_XTENSAWIN @@ -232,21 +232,33 @@ void asm_xtensa_mov_reg_pcrel(asm_xtensa_t *as, uint reg_dest, uint label) { asm_xtensa_op_add_n(as, reg_dest, reg_dest, ASM_XTENSA_REG_A0); } -void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx) { - if (idx < 16) { - asm_xtensa_op_l32i_n(as, ASM_XTENSA_REG_A0, ASM_XTENSA_REG_FUN_TABLE, idx); +void asm_xtensa_l32i_optimised(asm_xtensa_t *as, uint reg_dest, uint reg_base, uint word_offset) { + if (word_offset < 16) { + asm_xtensa_op_l32i_n(as, reg_dest, reg_base, word_offset); + } else if (word_offset < 256) { + asm_xtensa_op_l32i(as, reg_dest, reg_base, word_offset); } else { - asm_xtensa_op_l32i(as, ASM_XTENSA_REG_A0, ASM_XTENSA_REG_FUN_TABLE, idx); + mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("asm overflow")); } +} + +void asm_xtensa_s32i_optimised(asm_xtensa_t *as, uint reg_src, uint reg_base, uint word_offset) { + if (word_offset < 16) { + asm_xtensa_op_s32i_n(as, reg_src, reg_base, word_offset); + } else if (word_offset < 256) { + asm_xtensa_op_s32i(as, reg_src, reg_base, word_offset); + } else { + mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("asm overflow")); + } +} + +void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx) { + asm_xtensa_l32i_optimised(as, ASM_XTENSA_REG_A0, ASM_XTENSA_REG_FUN_TABLE, idx); asm_xtensa_op_callx0(as, ASM_XTENSA_REG_A0); } void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx) { - if (idx < 16) { - asm_xtensa_op_l32i_n(as, ASM_XTENSA_REG_A8, ASM_XTENSA_REG_FUN_TABLE_WIN, idx); - } else { - asm_xtensa_op_l32i(as, ASM_XTENSA_REG_A8, ASM_XTENSA_REG_FUN_TABLE_WIN, idx); - } + asm_xtensa_l32i_optimised(as, ASM_XTENSA_REG_A8, ASM_XTENSA_REG_FUN_TABLE_WIN, idx); asm_xtensa_op_callx8(as, ASM_XTENSA_REG_A8); } diff --git a/py/asmxtensa.h b/py/asmxtensa.h index 529bfa5f0c..60205f8f9c 100644 --- a/py/asmxtensa.h +++ b/py/asmxtensa.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George + * Copyright (c) 2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -278,6 +278,8 @@ void asm_xtensa_mov_local_reg(asm_xtensa_t *as, int local_num, uint reg_src); void asm_xtensa_mov_reg_local(asm_xtensa_t *as, uint reg_dest, int local_num); void asm_xtensa_mov_reg_local_addr(asm_xtensa_t *as, uint reg_dest, int local_num); void asm_xtensa_mov_reg_pcrel(asm_xtensa_t *as, uint reg_dest, uint label); +void asm_xtensa_l32i_optimised(asm_xtensa_t *as, uint reg_dest, uint reg_base, uint word_offset); +void asm_xtensa_s32i_optimised(asm_xtensa_t *as, uint reg_src, uint reg_base, uint word_offset); void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx); void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx); @@ -393,12 +395,13 @@ void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx); #define ASM_SUB_REG_REG(as, reg_dest, reg_src) asm_xtensa_op_sub((as), (reg_dest), (reg_dest), (reg_src)) #define ASM_MUL_REG_REG(as, reg_dest, reg_src) asm_xtensa_op_mull((as), (reg_dest), (reg_dest), (reg_src)) -#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_xtensa_op_l32i_n((as), (reg_dest), (reg_base), (word_offset)) +#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_xtensa_l32i_optimised((as), (reg_dest), (reg_base), (word_offset)) #define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) asm_xtensa_op_l8ui((as), (reg_dest), (reg_base), 0) #define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) asm_xtensa_op_l16ui((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, uint16_offset) asm_xtensa_op_l16ui((as), (reg_dest), (reg_base), (uint16_offset)) #define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) asm_xtensa_op_l32i_n((as), (reg_dest), (reg_base), 0) -#define ASM_STORE_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_xtensa_op_s32i_n((as), (reg_dest), (reg_base), (word_offset)) +#define ASM_STORE_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_xtensa_s32i_optimised((as), (reg_dest), (reg_base), (word_offset)) #define ASM_STORE8_REG_REG(as, reg_src, reg_base) asm_xtensa_op_s8i((as), (reg_src), (reg_base), 0) #define ASM_STORE16_REG_REG(as, reg_src, reg_base) asm_xtensa_op_s16i((as), (reg_src), (reg_base), 0) #define ASM_STORE32_REG_REG(as, reg_src, reg_base) asm_xtensa_op_s32i_n((as), (reg_src), (reg_base), 0) diff --git a/py/bc.c b/py/bc.c index e7ad43389a..ecd2f60879 100644 --- a/py/bc.c +++ b/py/bc.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George + * Copyright (c) 2014 Damien P. George * Copyright (c) 2014 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -29,11 +29,9 @@ #include #include -#include "py/runtime.h" #include "py/bc0.h" #include "py/bc.h" - -#include "supervisor/shared/translate/translate.h" +#include "py/objfun.h" #if MICROPY_DEBUG_VERBOSE // print debugging info #define DEBUG_PRINT (1) @@ -42,7 +40,23 @@ #define DEBUG_printf(...) (void)0 #endif -#if !MICROPY_PERSISTENT_CODE +void mp_encode_uint(void *env, mp_encode_uint_allocator_t allocator, mp_uint_t val) { + // We store each 7 bits in a separate byte, and that's how many bytes needed + byte buf[MP_ENCODE_UINT_MAX_BYTES]; + byte *p = buf + sizeof(buf); + // We encode in little-ending order, but store in big-endian, to help decoding + do { + *--p = val & 0x7f; + val >>= 7; + } while (val != 0); + byte *c = allocator(env, buf + sizeof(buf) - p); + if (c != NULL) { + while (p != buf + sizeof(buf) - 1) { + *c++ = *p++ | 0x80; + } + *c = *p; + } +} mp_uint_t mp_decode_uint(const byte **ptr) { mp_uint_t unum = 0; @@ -74,8 +88,6 @@ const byte *mp_decode_uint_skip(const byte *ptr) { return ptr; } -#endif - STATIC NORETURN void fun_pos_args_mismatch(mp_obj_fun_bc_t *f, size_t expected, size_t given) { #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE // generic message, used also for other argument issues @@ -109,46 +121,36 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { // On entry code_state should be allocated somewhere (stack/heap) and // contain the following valid entries: // - code_state->fun_bc should contain a pointer to the function object -// - code_state->ip should contain the offset in bytes from the pointer -// code_state->fun_bc->bytecode to the entry n_state (0 for bytecode, non-zero for native) -void PLACE_IN_ITCM(mp_setup_code_state)(mp_code_state_t * code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) { +// - code_state->ip should contain a pointer to the beginning of the prelude +// - code_state->sp should be: &code_state->state[0] - 1 +// - code_state->n_state should be the number of objects in the local state +STATIC void mp_setup_code_state_helper(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) { // This function is pretty complicated. It's main aim is to be efficient in speed and RAM // usage for the common case of positional only args. // get the function object that we want to set up (could be bytecode or native code) mp_obj_fun_bc_t *self = code_state->fun_bc; - // ip comes in as an offset into bytecode, so turn it into a true pointer - code_state->ip = self->bytecode + (size_t)code_state->ip; - - #if MICROPY_STACKLESS - code_state->prev = NULL; - #endif - - #if MICROPY_PY_SYS_SETTRACE - code_state->prev_state = NULL; - code_state->frame = NULL; - #endif - // Get cached n_state (rather than decode it again) size_t n_state = code_state->n_state; // Decode prelude size_t n_state_unused, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args; MP_BC_PRELUDE_SIG_DECODE_INTO(code_state->ip, n_state_unused, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args); + MP_BC_PRELUDE_SIZE_DECODE(code_state->ip); (void)n_state_unused; (void)n_exc_stack_unused; - code_state->sp = &code_state->state[0] - 1; + mp_obj_t *code_state_state = code_state->sp + 1; code_state->exc_sp_idx = 0; // zero out the local stack to begin with - memset(code_state->state, 0, n_state * sizeof(*code_state->state)); + memset(code_state_state, 0, n_state * sizeof(*code_state->state)); const mp_obj_t *kwargs = args + n_args; // var_pos_kw_args points to the stack where the var-args tuple, and var-kw dict, should go (if they are needed) - mp_obj_t *var_pos_kw_args = &code_state->state[n_state - 1 - n_pos_args - n_kwonly_args]; + mp_obj_t *var_pos_kw_args = &code_state_state[n_state - 1 - n_pos_args - n_kwonly_args]; // check positional arguments @@ -171,7 +173,7 @@ void PLACE_IN_ITCM(mp_setup_code_state)(mp_code_state_t * code_state, size_t n_a if (n_args >= (size_t)(n_pos_args - n_def_pos_args)) { // given enough arguments, but may need to use some default arguments for (size_t i = n_args; i < n_pos_args; i++) { - code_state->state[n_state - 1 - i] = self->extra_args[i - (n_pos_args - n_def_pos_args)]; + code_state_state[n_state - 1 - i] = self->extra_args[i - (n_pos_args - n_def_pos_args)]; } } else { fun_pos_args_mismatch(self, n_pos_args - n_def_pos_args, n_args); @@ -181,14 +183,14 @@ void PLACE_IN_ITCM(mp_setup_code_state)(mp_code_state_t * code_state, size_t n_a // copy positional args into state for (size_t i = 0; i < n_args; i++) { - code_state->state[n_state - 1 - i] = args[i]; + code_state_state[n_state - 1 - i] = args[i]; } // check keyword arguments if (n_kw != 0 || (scope_flags & MP_SCOPE_FLAG_DEFKWARGS) != 0) { DEBUG_printf("Initial args: "); - dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); + dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); mp_obj_t dict = MP_OBJ_NULL; if ((scope_flags & MP_SCOPE_FLAG_VARKEYWORDS) != 0) { @@ -196,26 +198,25 @@ void PLACE_IN_ITCM(mp_setup_code_state)(mp_code_state_t * code_state, size_t n_a *var_pos_kw_args = dict; } - // get pointer to arg_names array - const mp_obj_t *arg_names = (const mp_obj_t *)self->const_table; - for (size_t i = 0; i < n_kw; i++) { // the keys in kwargs are expected to be qstr objects mp_obj_t wanted_arg_name = kwargs[2 * i]; - if (MP_UNLIKELY(!mp_obj_is_qstr(wanted_arg_name))) { - #if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE - mp_raise_TypeError(MP_ERROR_TEXT("unexpected keyword argument")); - #else - mp_raise_TypeError(MP_ERROR_TEXT("keywords must be strings")); - #endif - } + + // get pointer to arg_names array + const uint8_t *arg_names = code_state->ip; + arg_names = mp_decode_uint_skip(arg_names); + for (size_t j = 0; j < n_pos_args + n_kwonly_args; j++) { - if (wanted_arg_name == arg_names[j]) { - if (code_state->state[n_state - 1 - j] != MP_OBJ_NULL) { - mp_raise_TypeError_varg( + qstr arg_qstr = mp_decode_uint(&arg_names); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + arg_qstr = self->context->constants.qstr_table[arg_qstr]; + #endif + if (wanted_arg_name == MP_OBJ_NEW_QSTR(arg_qstr)) { + if (code_state_state[n_state - 1 - j] != MP_OBJ_NULL) { + mp_raise_msg_varg(&mp_type_TypeError, MP_ERROR_TEXT("function got multiple values for argument '%q'"), MP_OBJ_QSTR_VALUE(wanted_arg_name)); } - code_state->state[n_state - 1 - j] = kwargs[2 * i + 1]; + code_state_state[n_state - 1 - j] = kwargs[2 * i + 1]; goto continue2; } } @@ -224,7 +225,7 @@ void PLACE_IN_ITCM(mp_setup_code_state)(mp_code_state_t * code_state, size_t n_a #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE mp_raise_TypeError(MP_ERROR_TEXT("unexpected keyword argument")); #else - mp_raise_TypeError_varg( + mp_raise_msg_varg(&mp_type_TypeError, MP_ERROR_TEXT("unexpected keyword argument '%q'"), MP_OBJ_QSTR_VALUE(wanted_arg_name)); #endif } @@ -233,10 +234,10 @@ void PLACE_IN_ITCM(mp_setup_code_state)(mp_code_state_t * code_state, size_t n_a } DEBUG_printf("Args with kws flattened: "); - dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); + dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); // fill in defaults for positional args - mp_obj_t *d = &code_state->state[n_state - n_pos_args]; + mp_obj_t *d = &code_state_state[n_state - n_pos_args]; mp_obj_t *s = &self->extra_args[n_def_pos_args - 1]; for (size_t i = n_def_pos_args; i > 0; i--, d++, s--) { if (*d == MP_OBJ_NULL) { @@ -245,26 +246,34 @@ void PLACE_IN_ITCM(mp_setup_code_state)(mp_code_state_t * code_state, size_t n_a } DEBUG_printf("Args after filling default positional: "); - dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); + dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); // Check that all mandatory positional args are specified - while (d < &code_state->state[n_state]) { + while (d < &code_state_state[n_state]) { if (*d++ == MP_OBJ_NULL) { - mp_raise_TypeError_varg( - MP_ERROR_TEXT("function missing required positional argument #%d"), &code_state->state[n_state] - d); + mp_raise_msg_varg(&mp_type_TypeError, + MP_ERROR_TEXT("function missing required positional argument #%d"), &code_state_state[n_state] - d); } } // Check that all mandatory keyword args are specified // Fill in default kw args if we have them + const uint8_t *arg_names = mp_decode_uint_skip(code_state->ip); + for (size_t i = 0; i < n_pos_args; i++) { + arg_names = mp_decode_uint_skip(arg_names); + } for (size_t i = 0; i < n_kwonly_args; i++) { - if (code_state->state[n_state - 1 - n_pos_args - i] == MP_OBJ_NULL) { + qstr arg_qstr = mp_decode_uint(&arg_names); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + arg_qstr = self->context->constants.qstr_table[arg_qstr]; + #endif + if (code_state_state[n_state - 1 - n_pos_args - i] == MP_OBJ_NULL) { mp_map_elem_t *elem = NULL; if ((scope_flags & MP_SCOPE_FLAG_DEFKWARGS) != 0) { - elem = mp_map_lookup(&((mp_obj_dict_t *)MP_OBJ_TO_PTR(self->extra_args[n_def_pos_args]))->map, arg_names[n_pos_args + i], MP_MAP_LOOKUP); + elem = mp_map_lookup(&((mp_obj_dict_t *)MP_OBJ_TO_PTR(self->extra_args[n_def_pos_args]))->map, MP_OBJ_NEW_QSTR(arg_qstr), MP_MAP_LOOKUP); } if (elem != NULL) { - code_state->state[n_state - 1 - n_pos_args - i] = elem->value; + code_state_state[n_state - 1 - n_pos_args - i] = elem->value; } else { mp_raise_TypeError_varg( MP_ERROR_TEXT("function missing required keyword argument '%q'"), @@ -283,57 +292,49 @@ void PLACE_IN_ITCM(mp_setup_code_state)(mp_code_state_t * code_state, size_t n_a } } - // read the size part of the prelude - const byte *ip = code_state->ip; - MP_BC_PRELUDE_SIZE_DECODE(ip); - - // jump over code info (source file and line-number mapping) - ip += n_info; + // jump over code info (source file, argument names and line-number mapping) + const uint8_t *ip = code_state->ip + n_info; // bytecode prelude: initialise closed over variables for (; n_cell; --n_cell) { size_t local_num = *ip++; - code_state->state[n_state - 1 - local_num] = - mp_obj_new_cell(code_state->state[n_state - 1 - local_num]); + code_state_state[n_state - 1 - local_num] = + mp_obj_new_cell(code_state_state[n_state - 1 - local_num]); } - #if !MICROPY_PERSISTENT_CODE - // so bytecode is aligned - ip = MP_ALIGN(ip, sizeof(mp_uint_t)); - #endif - // now that we skipped over the prelude, set the ip for the VM code_state->ip = ip; DEBUG_printf("Calling: n_pos_args=%d, n_kwonly_args=%d\n", n_pos_args, n_kwonly_args); - dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); - dump_args(code_state->state, n_state); + dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); + dump_args(code_state_state, n_state); } -#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE - -// The following table encodes the number of bytes that a specific opcode -// takes up. Some opcodes have an extra byte, defined by MP_BC_MASK_EXTRA_BYTE. -uint mp_opcode_format(const byte *ip, size_t *opcode_size, bool count_var_uint) { - uint f = MP_BC_FORMAT(*ip); - const byte *ip_start = ip; - if (f == MP_BC_FORMAT_QSTR) { - ip += 3; - } else { - int extra_byte = (*ip & MP_BC_MASK_EXTRA_BYTE) == 0; - ip += 1; - if (f == MP_BC_FORMAT_VAR_UINT) { - if (count_var_uint) { - while ((*ip++ & 0x80) != 0) { - } - } - } else if (f == MP_BC_FORMAT_OFFSET) { - ip += 2; - } - ip += extra_byte; - } - *opcode_size = ip - ip_start; - return f; +// On entry code_state should be allocated somewhere (stack/heap) and +// contain the following valid entries: +// - code_state->fun_bc should contain a pointer to the function object +// - code_state->n_state should be the number of objects in the local state +void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) { + code_state->ip = code_state->fun_bc->bytecode; + code_state->sp = &code_state->state[0] - 1; + #if MICROPY_STACKLESS + code_state->prev = NULL; + #endif + #if MICROPY_PY_SYS_SETTRACE + code_state->prev_state = NULL; + code_state->frame = NULL; + #endif + mp_setup_code_state_helper(code_state, n_args, n_kw, args); } -#endif // MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE +#if MICROPY_EMIT_NATIVE +// On entry code_state should be allocated somewhere (stack/heap) and +// contain the following valid entries: +// - code_state->fun_bc should contain a pointer to the function object +// - code_state->ip should contain a pointer to the beginning of the prelude +// - code_state->n_state should be the number of objects in the local state +void mp_setup_code_state_native(mp_code_state_native_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) { + code_state->sp = &code_state->state[0] - 1; + mp_setup_code_state_helper((mp_code_state_t *)code_state, n_args, n_kw, args); +} +#endif diff --git a/py/bc.h b/py/bc.h index eeccc3412f..d620be8e82 100644 --- a/py/bc.h +++ b/py/bc.h @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,7 +28,6 @@ #define MICROPY_INCLUDED_PY_BC_H #include "py/runtime.h" -#include "py/objfun.h" // bytecode layout: // @@ -50,7 +49,9 @@ // // source info section: // simple_name : var qstr -// source_file : var qstr +// argname0 : var qstr +// ... : var qstr +// argnameN : var qstr N = num_pos_args + num_kwonly_args - 1 // // // closure section: @@ -58,19 +59,16 @@ // ... : byte // local_numN : byte N = n_cells-1 // -// only needed if bytecode contains pointers -// // // // // constant table layout: // -// argname0 : obj (qstr) -// ... : obj (qstr) -// argnameN : obj (qstr) N = num_pos_args + num_kwonly_args // const0 : obj // constN : obj +#define MP_ENCODE_UINT_MAX_BYTES ((MP_BYTES_PER_OBJ_WORD * 8 + 6) / 7) + #define MP_BC_PRELUDE_SIG_ENCODE(S, E, scope, out_byte, out_env) \ do { \ /*// Get values to store in prelude */ \ @@ -187,9 +185,9 @@ typedef struct _mp_bytecode_prelude_t { uint n_pos_args; uint n_kwonly_args; uint n_def_pos_args; - qstr qstr_block_name; - qstr qstr_source_file; + qstr qstr_block_name_idx; const byte *line_info; + const byte *line_info_top; const byte *opcodes; } mp_bytecode_prelude_t; @@ -203,12 +201,46 @@ typedef struct _mp_exc_stack_t { mp_obj_base_t *prev_exc; } mp_exc_stack_t; +// Constants associated with a module, to interface bytecode with runtime. +typedef struct _mp_module_constants_t { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + qstr_short_t *qstr_table; + #else + qstr source_file; + #endif + mp_obj_t *obj_table; +} mp_module_constants_t; + +// State associated with a module. +typedef struct _mp_module_context_t { + mp_obj_module_t module; + mp_module_constants_t constants; +} mp_module_context_t; + +// Outer level struct defining a compiled module. +typedef struct _mp_compiled_module_t { + const mp_module_context_t *context; + const struct _mp_raw_code_t *rc; + #if MICROPY_PERSISTENT_CODE_SAVE + bool has_native; + size_t n_qstr; + size_t n_obj; + #endif +} mp_compiled_module_t; + +// Outer level struct defining a frozen module. +typedef struct _mp_frozen_module_t { + const mp_module_constants_t constants; + const struct _mp_raw_code_t *rc; +} mp_frozen_module_t; + +// State for an executing function. typedef struct _mp_code_state_t { // The fun_bc entry points to the underlying function object that is being executed. // It is needed to access the start of bytecode and the const_table. // It is also needed to prevent the GC from reclaiming the bytecode during execution, // because the ip pointer below will always point to the interior of the bytecode. - mp_obj_fun_bc_t *fun_bc; + struct _mp_obj_fun_bc_t *fun_bc; const byte *ip; mp_obj_t *sp; uint16_t n_state; @@ -227,17 +259,37 @@ typedef struct _mp_code_state_t { // mp_exc_stack_t exc_state[0]; } mp_code_state_t; +// State for an executing native function (based on mp_code_state_t). +typedef struct _mp_code_state_native_t { + struct _mp_obj_fun_bc_t *fun_bc; + const byte *ip; + mp_obj_t *sp; + uint16_t n_state; + uint16_t exc_sp_idx; + mp_obj_dict_t *old_globals; + mp_obj_t state[0]; +} mp_code_state_native_t; + +// Allocator may return NULL, in which case data is not stored (can be used to compute size). +typedef uint8_t *(*mp_encode_uint_allocator_t)(void *env, size_t nbytes); + +void mp_encode_uint(void *env, mp_encode_uint_allocator_t allocator, mp_uint_t val); mp_uint_t mp_decode_uint(const byte **ptr); mp_uint_t mp_decode_uint_value(const byte *ptr); const byte *mp_decode_uint_skip(const byte *ptr); -mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state, volatile mp_obj_t inject_exc); +mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state, +#ifndef __cplusplus + volatile +#endif + mp_obj_t inject_exc); mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t func, size_t n_args, size_t n_kw, const mp_obj_t *args); void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); -void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *code, mp_uint_t len, const mp_uint_t *const_table); -void mp_bytecode_print2(const mp_print_t *print, const byte *code, size_t len, const mp_uint_t *const_table); -const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip); -#define mp_bytecode_print_inst(print, code, const_table) mp_bytecode_print2(print, code, 1, const_table) +void mp_setup_code_state_native(mp_code_state_native_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); +void mp_bytecode_print(const mp_print_t *print, const struct _mp_raw_code_t *rc, const mp_module_constants_t *cm); +void mp_bytecode_print2(const mp_print_t *print, const byte *ip, size_t len, struct _mp_raw_code_t *const *child_table, const mp_module_constants_t *cm); +const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip_start, const byte *ip, struct _mp_raw_code_t *const *child_table, const mp_module_constants_t *cm); +#define mp_bytecode_print_inst(print, code, x_table) mp_bytecode_print2(print, code, 1, x_table) // Helper macros to access pointer with least significant bits holding flags #define MP_TAGPTR_PTR(x) ((void *)((uintptr_t)(x) & ~((uintptr_t)3))) @@ -245,16 +297,26 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip); #define MP_TAGPTR_TAG1(x) ((uintptr_t)(x) & 2) #define MP_TAGPTR_MAKE(ptr, tag) ((void *)((uintptr_t)(ptr) | (tag))) -#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE +static inline void mp_module_context_alloc_tables(mp_module_context_t *context, size_t n_qstr, size_t n_obj) { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + size_t nq = (n_qstr * sizeof(qstr_short_t) + sizeof(mp_uint_t) - 1) / sizeof(mp_uint_t); + size_t no = n_obj; + mp_uint_t *mem = m_new(mp_uint_t, nq + no); + context->constants.qstr_table = (qstr_short_t *)mem; + context->constants.obj_table = (mp_obj_t *)(mem + nq); + #else + if (n_obj == 0) { + context->constants.obj_table = NULL; + } else { + context->constants.obj_table = m_new(mp_obj_t, n_obj); + } + #endif +} -uint mp_opcode_format(const byte *ip, size_t *opcode_size, bool count_var_uint); - -#endif - -static inline size_t mp_bytecode_get_source_line(const byte *line_info, size_t bc_offset) { +static inline size_t mp_bytecode_get_source_line(const byte *line_info, const byte *line_info_top, size_t bc_offset) { size_t source_line = 1; - size_t c; - while ((c = *line_info)) { + while (line_info < line_info_top) { + size_t c = *line_info; size_t b, l; if ((c & 0x80) == 0) { // 0b0LLBBBBB encoding diff --git a/py/bc0.h b/py/bc0.h index 500dee5e7f..a4a0acf937 100644 --- a/py/bc0.h +++ b/py/bc0.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,18 @@ // MicroPython bytecode opcodes, grouped based on the format of the opcode +// All opcodes are encoded as a byte with an optional argument. Arguments are +// variable-length encoded so they can be as small as possible. The possible +// encodings for arguments are (ip[0] is the opcode): +// +// - unsigned relative bytecode offset: +// - if ip[1] high bit is clear then: arg = ip[1] +// - if ip[1] high bit is set then: arg = ip[1] & 0x7f | ip[2] << 7 +// +// - signed relative bytecode offset: +// - if ip[1] high bit is clear then: arg = ip[1] - 0x40 +// - if ip[1] high bit is set then: arg = (ip[1] & 0x7f | ip[2] << 7) - 0x4000 + #define MP_BC_MASK_FORMAT (0xf0) #define MP_BC_MASK_EXTRA_BYTE (0x9e) @@ -101,17 +113,17 @@ #define MP_BC_ROT_TWO (MP_BC_BASE_BYTE_O + 0x0a) #define MP_BC_ROT_THREE (MP_BC_BASE_BYTE_O + 0x0b) -#define MP_BC_JUMP (MP_BC_BASE_JUMP_E + 0x02) // rel byte code offset, 16-bit signed, in excess -#define MP_BC_POP_JUMP_IF_TRUE (MP_BC_BASE_JUMP_E + 0x03) // rel byte code offset, 16-bit signed, in excess -#define MP_BC_POP_JUMP_IF_FALSE (MP_BC_BASE_JUMP_E + 0x04) // rel byte code offset, 16-bit signed, in excess -#define MP_BC_JUMP_IF_TRUE_OR_POP (MP_BC_BASE_JUMP_E + 0x05) // rel byte code offset, 16-bit signed, in excess -#define MP_BC_JUMP_IF_FALSE_OR_POP (MP_BC_BASE_JUMP_E + 0x06) // rel byte code offset, 16-bit signed, in excess -#define MP_BC_UNWIND_JUMP (MP_BC_BASE_JUMP_E + 0x00) // rel byte code offset, 16-bit signed, in excess; then a byte -#define MP_BC_SETUP_WITH (MP_BC_BASE_JUMP_E + 0x07) // rel byte code offset, 16-bit unsigned -#define MP_BC_SETUP_EXCEPT (MP_BC_BASE_JUMP_E + 0x08) // rel byte code offset, 16-bit unsigned -#define MP_BC_SETUP_FINALLY (MP_BC_BASE_JUMP_E + 0x09) // rel byte code offset, 16-bit unsigned -#define MP_BC_POP_EXCEPT_JUMP (MP_BC_BASE_JUMP_E + 0x0a) // rel byte code offset, 16-bit unsigned -#define MP_BC_FOR_ITER (MP_BC_BASE_JUMP_E + 0x0b) // rel byte code offset, 16-bit unsigned +#define MP_BC_UNWIND_JUMP (MP_BC_BASE_JUMP_E + 0x00) // signed relative bytecode offset; then a byte +#define MP_BC_JUMP (MP_BC_BASE_JUMP_E + 0x02) // signed relative bytecode offset +#define MP_BC_POP_JUMP_IF_TRUE (MP_BC_BASE_JUMP_E + 0x03) // signed relative bytecode offset +#define MP_BC_POP_JUMP_IF_FALSE (MP_BC_BASE_JUMP_E + 0x04) // signed relative bytecode offset +#define MP_BC_JUMP_IF_TRUE_OR_POP (MP_BC_BASE_JUMP_E + 0x05) // unsigned relative bytecode offset +#define MP_BC_JUMP_IF_FALSE_OR_POP (MP_BC_BASE_JUMP_E + 0x06) // unsigned relative bytecode offset +#define MP_BC_SETUP_WITH (MP_BC_BASE_JUMP_E + 0x07) // unsigned relative bytecode offset +#define MP_BC_SETUP_EXCEPT (MP_BC_BASE_JUMP_E + 0x08) // unsigned relative bytecode offset +#define MP_BC_SETUP_FINALLY (MP_BC_BASE_JUMP_E + 0x09) // unsigned relative bytecode offset +#define MP_BC_POP_EXCEPT_JUMP (MP_BC_BASE_JUMP_E + 0x0a) // unsigned relative bytecode offset +#define MP_BC_FOR_ITER (MP_BC_BASE_JUMP_E + 0x0b) // unsigned relative bytecode offset #define MP_BC_WITH_CLEANUP (MP_BC_BASE_BYTE_O + 0x0c) #define MP_BC_END_FINALLY (MP_BC_BASE_BYTE_O + 0x0d) #define MP_BC_GET_ITER (MP_BC_BASE_BYTE_O + 0x0e) diff --git a/py/binary.c b/py/binary.c index 439993c0f7..60a74d6162 100644 --- a/py/binary.c +++ b/py/binary.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014-2017 Paul Sokolovsky - * SPDX-FileCopyrightText: Copyright (c) 2014-2019 Damien P. George + * Copyright (c) 2014-2017 Paul Sokolovsky + * Copyright (c) 2014-2019 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -217,6 +217,7 @@ long long mp_binary_get_int(size_t size, bool is_signed, bool big_endian, const val = -1; } for (uint i = 0; i < size; i++) { + // CIRCUITPY fix val *= 256; val |= *src; src += delta; diff --git a/py/binary.h b/py/binary.h index 9fade810f8..5c645bcaaa 100644 --- a/py/binary.h +++ b/py/binary.h @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014 Paul Sokolovsky - * SPDX-FileCopyrightText: Copyright (c) 2014-2017 Damien P. George + * Copyright (c) 2014 Paul Sokolovsky + * Copyright (c) 2014-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/builtin.h b/py/builtin.h index 4fa4b08f9d..db45f9841d 100644 --- a/py/builtin.h +++ b/py/builtin.h @@ -28,8 +28,43 @@ #include "py/obj.h" -mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args); +typedef enum { + MP_IMPORT_STAT_NO_EXIST, + MP_IMPORT_STAT_DIR, + MP_IMPORT_STAT_FILE, +} mp_import_stat_t; + +#if MICROPY_VFS + +// Delegate to the VFS for import stat and builtin open. + +#define mp_builtin_open_obj mp_vfs_open_obj + +mp_import_stat_t mp_vfs_import_stat(const char *path); +mp_obj_t mp_vfs_open(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs); + +MP_DECLARE_CONST_FUN_OBJ_KW(mp_vfs_open_obj); + +static inline mp_import_stat_t mp_import_stat(const char *path) { + return mp_vfs_import_stat(path); +} + +static inline mp_obj_t mp_builtin_open(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs) { + return mp_vfs_open(n_args, args, kwargs); +} + +#else + +// A port can provide implementations of these functions. +mp_import_stat_t mp_import_stat(const char *path); mp_obj_t mp_builtin_open(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs); + +// A port can provide this object. +MP_DECLARE_CONST_FUN_OBJ_KW(mp_builtin_open_obj); + +#endif + +mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args); mp_obj_t mp_micropython_mem_info(size_t n_args, const mp_obj_t *args); MP_DECLARE_CONST_FUN_OBJ_VAR(mp_builtin___build_class___obj); @@ -76,9 +111,7 @@ MP_DECLARE_CONST_FUN_OBJ_1(mp_builtin_repr_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_round_obj); MP_DECLARE_CONST_FUN_OBJ_KW(mp_builtin_sorted_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_sum_obj); -// Defined by a port, but declared here for simplicity MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_input_obj); -MP_DECLARE_CONST_FUN_OBJ_KW(mp_builtin_open_obj); MP_DECLARE_CONST_FUN_OBJ_2(mp_namedtuple_obj); diff --git a/py/builtinevex.c b/py/builtinevex.c index 46d4a1277a..b6b27c2a67 100644 --- a/py/builtinevex.c +++ b/py/builtinevex.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -56,7 +56,7 @@ STATIC mp_obj_t code_execute(mp_obj_code_t *self, mp_obj_dict_t *globals, mp_obj // the correct one if (mp_obj_is_type(self->module_fun, &mp_type_fun_bc)) { mp_obj_fun_bc_t *fun_bc = MP_OBJ_TO_PTR(self->module_fun); - fun_bc->globals = globals; + ((mp_module_context_t *)fun_bc->context)->module.globals = globals; } // execute code @@ -105,8 +105,7 @@ STATIC mp_obj_t mp_builtin_compile(size_t n_args, const mp_obj_t *args) { mp_raise_ValueError(MP_ERROR_TEXT("bad compile mode")); } - mp_obj_code_t *code = m_new_obj(mp_obj_code_t); - code->base.type = &mp_type_code; + mp_obj_code_t *code = mp_obj_malloc(mp_obj_code_t, &mp_type_code); code->module_fun = mp_parse_compile_execute(lex, parse_input_kind, NULL, NULL); return MP_OBJ_FROM_PTR(code); } diff --git a/py/builtinhelp.c b/py/builtinhelp.c index 86632f6dde..c26a1a5490 100644 --- a/py/builtinhelp.c +++ b/py/builtinhelp.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2016 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/builtinimport.c b/py/builtinimport.c index 92f3150306..5ba4da0b43 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2019 Damien P. George + * Copyright (c) 2013-2019 Damien P. George * Copyright (c) 2014 Paul Sokolovsky * Copyright (c) 2021 Jim Mussared * @@ -150,28 +150,29 @@ STATIC mp_import_stat_t stat_top_level_dir_or_file(qstr mod_name, vstr_t *dest) } #if MICROPY_MODULE_FROZEN_STR || MICROPY_ENABLE_COMPILER -STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex) { +STATIC void do_load_from_lexer(mp_module_context_t *context, mp_lexer_t *lex) { #if MICROPY_PY___FILE__ qstr source_name = lex->source_name; - mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); + mp_store_attr(MP_OBJ_FROM_PTR(&context->module), MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); #endif // parse, compile and execute the module in its context - mp_obj_dict_t *mod_globals = mp_obj_module_get_globals(module_obj); + mp_obj_dict_t *mod_globals = context->module.globals; mp_parse_compile_execute(lex, MP_PARSE_FILE_INPUT, mod_globals, mod_globals); mp_obj_module_set_globals(module_obj, make_dict_long_lived(mod_globals, 10)); } #endif #if (MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD) || MICROPY_MODULE_FROZEN_MPY -STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, const char *source_name) { +STATIC void do_execute_raw_code(mp_module_context_t *context, const mp_raw_code_t *rc, const mp_module_context_t *mc, const char *source_name) { (void)source_name; + #if MICROPY_PY___FILE__ - mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(source_name))); + mp_store_attr(MP_OBJ_FROM_PTR(&context->module), MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(source_name))); #endif // execute the module in its context - mp_obj_dict_t *mod_globals = mp_obj_module_get_globals(module_obj); + mp_obj_dict_t *mod_globals = context->module.globals; // save context mp_obj_dict_t *volatile old_globals = mp_globals_get(); @@ -183,7 +184,7 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, co nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_obj_t module_fun = mp_make_function_from_raw_code(raw_code, MP_OBJ_NULL, MP_OBJ_NULL); + mp_obj_t module_fun = mp_make_function_from_raw_code(rc, mc, NULL); mp_call_function_0(module_fun); // finish nlr block, restore context @@ -201,7 +202,7 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, co } #endif -STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { +STATIC void do_load(mp_module_context_t *module_obj, vstr_t *file) { #if MICROPY_MODULE_FROZEN || MICROPY_ENABLE_COMPILER || (MICROPY_PERSISTENT_CODE_LOAD && MICROPY_HAS_FILE_READER) const char *file_str = vstr_null_terminated_str(file); #endif @@ -228,7 +229,9 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { // its data) in the list of frozen files, execute it. #if MICROPY_MODULE_FROZEN_MPY if (frozen_type == MP_FROZEN_MPY) { - do_execute_raw_code(module_obj, modref, file_str + frozen_path_prefix_len); + const mp_frozen_module_t *frozen = modref; + module_obj->constants = frozen->constants; + do_execute_raw_code(module_obj, frozen->rc, module_obj, file_str + frozen_path_prefix_len); return; } #endif @@ -239,8 +242,8 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { // the correct format and, if so, load and execute the file. #if MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD if (file_str[file->len - 3] == 'm') { - mp_raw_code_t *raw_code = mp_raw_code_load_file(file_str); - do_execute_raw_code(module_obj, raw_code, file_str); + mp_compiled_module_t cm = mp_raw_code_load_file(file_str, module_obj); + do_execute_raw_code(module_obj, cm.rc, cm.context, file_str); return; } #endif @@ -376,6 +379,10 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, qstr umodule_name = qstr_from_str(umodule_buf); module_obj = mp_module_get_builtin(umodule_name); } + #elif MICROPY_PY_SYS + if (stat == MP_IMPORT_STAT_NO_EXIST && module_obj == MP_OBJ_NULL && level_mod_name == MP_QSTR_sys) { + module_obj = MP_OBJ_FROM_PTR(&mp_module_sys); + } #endif } else { DEBUG_printf("Searching for sub-module\n"); @@ -439,7 +446,7 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, size_t orig_path_len = path->len; vstr_add_str(path, PATH_SEP_CHAR "__init__.py"); if (stat_file_py_or_mpy(path) == MP_IMPORT_STAT_FILE) { - do_load(module_obj, path); + do_load(MP_OBJ_TO_PTR(module_obj), path); } else { // No-op. Nothing to load. // mp_warning("%s is imported as namespace package", vstr_str(&path)); @@ -448,7 +455,7 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, path->len = orig_path_len; } else { // MP_IMPORT_STAT_FILE // File -- execute "path.(m)py". - do_load(module_obj, path); + do_load(MP_OBJ_TO_PTR(module_obj), path); // Note: This should be the last component in the import path. If // there are remaining components then it's an ImportError // because the current path(the module that was just loaded) is diff --git a/py/compile.c b/py/compile.c index 3668bee320..82ab354eb4 100644 --- a/py/compile.c +++ b/py/compile.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2020 Damien P. George + * Copyright (c) 2013-2020 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -35,7 +35,9 @@ #include "py/compile.h" #include "py/runtime.h" #include "py/asmbase.h" +#include "py/nativeglue.h" #include "py/persistentcode.h" +#include "py/smallint.h" #include "supervisor/shared/translate/translate.h" @@ -90,7 +92,7 @@ typedef enum { #if MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER #define NATIVE_EMITTER(f) emit_native_table[mp_dynamic_compiler.native_arch]->emit_##f -#define NATIVE_EMITTER_TABLE emit_native_table[mp_dynamic_compiler.native_arch] +#define NATIVE_EMITTER_TABLE (emit_native_table[mp_dynamic_compiler.native_arch]) STATIC const emit_method_table_t *emit_native_table[] = { NULL, @@ -123,7 +125,7 @@ STATIC const emit_method_table_t *emit_native_table[] = { #else #error "unknown native emitter" #endif -#define NATIVE_EMITTER_TABLE &NATIVE_EMITTER(method_table) +#define NATIVE_EMITTER_TABLE (&NATIVE_EMITTER(method_table)) #endif #if MICROPY_EMIT_INLINE_ASM && MICROPY_DYNAMIC_COMPILER @@ -164,8 +166,6 @@ STATIC const emit_inline_asm_method_table_t *emit_asm_table[] = { // elements in this struct are ordered to make it compact typedef struct _compiler_t { - qstr source_file; - uint8_t is_repl; uint8_t pass; // holds enum type pass_kind_t uint8_t have_star; @@ -196,8 +196,60 @@ typedef struct _compiler_t { emit_inline_asm_t *emit_inline_asm; // current emitter for inline asm const emit_inline_asm_method_table_t *emit_inline_asm_method_table; // current emit method table for inline asm #endif + + mp_emit_common_t emit_common; } compiler_t; +/******************************************************************************/ +// mp_emit_common_t helper functions +// These are defined here so they can be inlined, to reduce code size. + +STATIC void mp_emit_common_init(mp_emit_common_t *emit, qstr source_file) { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + mp_map_init(&emit->qstr_map, 1); + + // add the source file as the first entry in the qstr table + mp_map_elem_t *elem = mp_map_lookup(&emit->qstr_map, MP_OBJ_NEW_QSTR(source_file), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + elem->value = MP_OBJ_NEW_SMALL_INT(0); + #endif + mp_obj_list_init(&emit->const_obj_list, 0); +} + +STATIC void mp_emit_common_start_pass(mp_emit_common_t *emit, pass_kind_t pass) { + emit->pass = pass; + if (pass == MP_PASS_CODE_SIZE) { + if (emit->ct_cur_child == 0) { + emit->children = NULL; + } else { + emit->children = m_new0(mp_raw_code_t *, emit->ct_cur_child); + } + } + emit->ct_cur_child = 0; +} + +STATIC void mp_emit_common_populate_module_context(mp_emit_common_t *emit, qstr source_file, mp_module_context_t *context) { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + size_t qstr_map_used = emit->qstr_map.used; + mp_module_context_alloc_tables(context, qstr_map_used, emit->const_obj_list.len); + for (size_t i = 0; i < emit->qstr_map.alloc; ++i) { + if (mp_map_slot_is_filled(&emit->qstr_map, i)) { + size_t idx = MP_OBJ_SMALL_INT_VALUE(emit->qstr_map.table[i].value); + qstr qst = MP_OBJ_QSTR_VALUE(emit->qstr_map.table[i].key); + context->constants.qstr_table[idx] = qst; + } + } + #else + mp_module_context_alloc_tables(context, 0, emit->const_obj_list.len); + context->constants.source_file = source_file; + #endif + + for (size_t i = 0; i < emit->const_obj_list.len; ++i) { + context->constants.obj_table[i] = emit->const_obj_list.items[i]; + } +} + +/******************************************************************************/ + STATIC void compile_error_set_line(compiler_t *comp, mp_parse_node_t pn) { // if the line of the error is unknown then try to update it from the pn if (comp->compile_error_line == 0 && MP_PARSE_NODE_IS_STRUCT(pn)) { @@ -248,7 +300,7 @@ STATIC void compile_decrease_except_level(compiler_t *comp) { } STATIC scope_t *scope_new_and_link(compiler_t *comp, scope_kind_t kind, mp_parse_node_t pn, uint emit_options) { - scope_t *scope = scope_new(kind, pn, comp->source_file, emit_options); + scope_t *scope = scope_new(kind, pn, emit_options); scope->parent = comp->scope_cur; scope->next = NULL; if (comp->scope_head == NULL) { @@ -802,6 +854,7 @@ STATIC bool compile_built_in_decorator(compiler_t *comp, size_t name_len, mp_par *emit_options = MP_EMIT_OPT_BYTECODE; // @micropython.native decorator. } else if (attr == MP_QSTR_native) { + // CIRCUITPY // Different from MicroPython: native doesn't raise SyntaxError if native support isn't // compiled, it just passes through the function unmodified. #if MICROPY_EMIT_NATIVE @@ -829,7 +882,7 @@ STATIC bool compile_built_in_decorator(compiler_t *comp, size_t name_len, mp_par compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid micropython decorator")); } - #if MICROPY_DYNAMIC_COMPILER + #if MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER if (*emit_options == MP_EMIT_OPT_NATIVE_PYTHON || *emit_options == MP_EMIT_OPT_VIPER) { if (emit_native_table[mp_dynamic_compiler.native_arch] == NULL) { compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid architecture")); @@ -1076,14 +1129,19 @@ STATIC void do_import_name(compiler_t *comp, mp_parse_node_t pn, qstr *q_base) { if (!is_as) { *q_base = MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]); } - int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); - int len = n - 1; - for (int i = 0; i < n; i++) { + size_t n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); + if (n == 0) { + // There must be at least one node in this PN_dotted_name. + // Let the compiler know this so it doesn't warn, and can generate better code. + MP_UNREACHABLE; + } + size_t len = n - 1; + for (size_t i = 0; i < n; i++) { len += qstr_len(MP_PARSE_NODE_LEAF_ARG(pns->nodes[i])); } char *q_ptr = mp_local_alloc(len); char *str_dest = q_ptr; - for (int i = 0; i < n; i++) { + for (size_t i = 0; i < n; i++) { if (i > 0) { *str_dest++ = '.'; } @@ -1096,7 +1154,7 @@ STATIC void do_import_name(compiler_t *comp, mp_parse_node_t pn, qstr *q_base) { mp_local_free(q_ptr); EMIT_ARG(import, q_full, MP_EMIT_IMPORT_NAME); if (is_as) { - for (int i = 1; i < n; i++) { + for (size_t i = 1; i < n; i++) { EMIT_ARG(attr, MP_PARSE_NODE_LEAF_ARG(pns->nodes[i]), MP_EMIT_ATTR_LOAD); } } @@ -2337,24 +2395,36 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar int n_positional = n_positional_extra; uint n_keyword = 0; uint star_flags = 0; - mp_parse_node_struct_t *star_args_node = NULL, *dblstar_args_node = NULL; + mp_uint_t star_args = 0; for (size_t i = 0; i < n_args; i++) { if (MP_PARSE_NODE_IS_STRUCT(args[i])) { mp_parse_node_struct_t *pns_arg = (mp_parse_node_struct_t *)args[i]; if (MP_PARSE_NODE_STRUCT_KIND(pns_arg) == PN_arglist_star) { - if (star_flags & MP_EMIT_STAR_FLAG_SINGLE) { - compile_syntax_error(comp, (mp_parse_node_t)pns_arg, MP_ERROR_TEXT("can't have multiple *x")); + if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) { + compile_syntax_error(comp, (mp_parse_node_t)pns_arg, MP_ERROR_TEXT("* arg after **")); + return; + } + #if MICROPY_DYNAMIC_COMPILER + if (i >= (size_t)mp_dynamic_compiler.small_int_bits - 1) + #else + if (i >= MP_SMALL_INT_BITS - 1) + #endif + { + // If there are not enough bits in a small int to fit the flag, then we consider + // it a syntax error. It should be unlikely to have this many args in practice. + compile_syntax_error(comp, (mp_parse_node_t)pns_arg, MP_ERROR_TEXT("too many args")); return; } star_flags |= MP_EMIT_STAR_FLAG_SINGLE; - star_args_node = pns_arg; + star_args |= (mp_uint_t)1 << i; + compile_node(comp, pns_arg->nodes[0]); + n_positional++; } else if (MP_PARSE_NODE_STRUCT_KIND(pns_arg) == PN_arglist_dbl_star) { - if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) { - compile_syntax_error(comp, (mp_parse_node_t)pns_arg, MP_ERROR_TEXT("can't have multiple **x")); - return; - } star_flags |= MP_EMIT_STAR_FLAG_DOUBLE; - dblstar_args_node = pns_arg; + // double-star args are stored as kw arg with key of None + EMIT(load_null); + compile_node(comp, pns_arg->nodes[0]); + n_keyword++; } else if (MP_PARSE_NODE_STRUCT_KIND(pns_arg) == PN_argument) { #if MICROPY_PY_ASSIGN_EXPR if (MP_PARSE_NODE_IS_STRUCT_KIND(pns_arg->nodes[1], PN_argument_3)) { @@ -2369,7 +2439,7 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar } EMIT_ARG(load_const_str, MP_PARSE_NODE_LEAF_ARG(pns_arg->nodes[0])); compile_node(comp, pns_arg->nodes[1]); - n_keyword += 1; + n_keyword++; } else { compile_comprehension(comp, pns_arg, SCOPE_GEN_EXPR); n_positional++; @@ -2379,12 +2449,12 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar } } else { normal_argument: - if (star_flags) { - compile_syntax_error(comp, args[i], MP_ERROR_TEXT("non-keyword arg after */**")); + if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) { + compile_syntax_error(comp, args[i], MP_ERROR_TEXT("positional arg after **")); return; } if (n_keyword > 0) { - compile_syntax_error(comp, args[i], MP_ERROR_TEXT("non-keyword arg after keyword arg")); + compile_syntax_error(comp, args[i], MP_ERROR_TEXT("positional arg after keyword arg")); return; } compile_node(comp, args[i]); @@ -2392,19 +2462,9 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar } } - // compile the star/double-star arguments if we had them - // if we had one but not the other then we load "null" as a place holder if (star_flags != 0) { - if (star_args_node == NULL) { - EMIT(load_null); - } else { - compile_node(comp, star_args_node->nodes[0]); - } - if (dblstar_args_node == NULL) { - EMIT(load_null); - } else { - compile_node(comp, dblstar_args_node->nodes[0]); - } + // one extra object that contains the star_args map + EMIT_ARG(load_const_small_int, star_args); } // emit the function/method call @@ -2714,12 +2774,7 @@ STATIC void compile_atom_expr_await(compiler_t *comp, mp_parse_node_struct_t *pn #endif STATIC mp_obj_t get_const_object(mp_parse_node_struct_t *pns) { - #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D - // nodes are 32-bit pointers, but need to extract 64-bit object - return (uint64_t)pns->nodes[0] | ((uint64_t)pns->nodes[1] << 32); - #else - return (mp_obj_t)pns->nodes[0]; - #endif + return mp_parse_node_extract_const_object(pns); } STATIC void compile_const_object(compiler_t *comp, mp_parse_node_struct_t *pns) { @@ -2744,23 +2799,7 @@ STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn) { // pass } else if (MP_PARSE_NODE_IS_SMALL_INT(pn)) { mp_int_t arg = MP_PARSE_NODE_LEAF_SMALL_INT(pn); - #if MICROPY_DYNAMIC_COMPILER - mp_uint_t sign_mask = -((mp_uint_t)1 << (mp_dynamic_compiler.small_int_bits - 1)); - if ((arg & sign_mask) == 0 || (arg & sign_mask) == sign_mask) { - // integer fits in target runtime's small-int - EMIT_ARG(load_const_small_int, arg); - } else { - // integer doesn't fit, so create a multi-precision int object - // (but only create the actual object on the last pass) - if (comp->pass != MP_PASS_EMIT) { - EMIT_ARG(load_const_obj, mp_const_none); - } else { - EMIT_ARG(load_const_obj, mp_obj_new_int_from_ll(arg)); - } - } - #else EMIT_ARG(load_const_small_int, arg); - #endif } else if (MP_PARSE_NODE_IS_LEAF(pn)) { uintptr_t arg = MP_PARSE_NODE_LEAF_ARG(pn); switch (MP_PARSE_NODE_LEAF_KIND(pn)) { @@ -2770,16 +2809,6 @@ STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn) { case MP_PARSE_NODE_STRING: EMIT_ARG(load_const_str, arg); break; - case MP_PARSE_NODE_BYTES: - // only create and load the actual bytes object on the last pass - if (comp->pass != MP_PASS_EMIT) { - EMIT_ARG(load_const_obj, mp_const_none); - } else { - size_t len; - const byte *data = qstr_data(arg, &len); - EMIT_ARG(load_const_obj, mp_obj_new_bytes(data, len)); - } - break; case MP_PARSE_NODE_TOKEN: default: if (arg == MP_TOKEN_NEWLINE) { @@ -2998,10 +3027,11 @@ STATIC void check_for_doc_string(compiler_t *comp, mp_parse_node_t pn) { #endif } -STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { +STATIC bool compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { comp->pass = pass; comp->scope_cur = scope; comp->next_label = 0; + mp_emit_common_start_pass(&comp->emit_common, pass); EMIT_ARG(start_pass, pass, scope); reserve_labels_for_native(comp, 6); // used by native's start_pass @@ -3155,10 +3185,12 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { EMIT(return_value); } - EMIT(end_pass); + bool pass_complete = EMIT(end_pass); // make sure we match all the exception levels assert(comp->cur_except_level == 0); + + return pass_complete; } #if MICROPY_EMIT_INLINE_ASM @@ -3322,9 +3354,10 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind f, mp_asm_base_get_code_size((mp_asm_base_t *)comp->emit_inline_asm), NULL, #if MICROPY_PERSISTENT_CODE_SAVE - 0, 0, 0, 0, NULL, + 0, + 0, #endif - comp->scope_cur->num_pos_args, 0, type_sig); + 0, comp->scope_cur->num_pos_args, type_sig); } } @@ -3432,15 +3465,15 @@ STATIC void scope_compute_things(scope_t *scope) { #if !MICROPY_PERSISTENT_CODE_SAVE STATIC #endif -mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) { +mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_module_context_t *context) { // put compiler state on the stack, it's relatively small compiler_t comp_state = {0}; compiler_t *comp = &comp_state; - comp->source_file = source_file; comp->is_repl = is_repl; comp->break_label = INVALID_LABEL; comp->continue_label = INVALID_LABEL; + mp_emit_common_init(&comp->emit_common, source_file); // create the module scope #if MICROPY_EMIT_NATIVE @@ -3451,9 +3484,9 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f scope_t *module_scope = scope_new_and_link(comp, SCOPE_MODULE, parse_tree->root, emit_opt); // create standard emitter; it's used at least for MP_PASS_SCOPE - emit_t *emit_bc = emit_bc_new(); + emit_t *emit_bc = emit_bc_new(&comp->emit_common); - // compile pass 1 + // compile MP_PASS_SCOPE comp->emit = emit_bc; #if MICROPY_EMIT_NATIVE comp->emit_method_table = &emit_bc_method_table; @@ -3491,7 +3524,7 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f // set max number of labels now that it's calculated emit_bc_set_max_num_labels(emit_bc, max_num_labels); - // compile pass 2 and 3 + // compile MP_PASS_STACK_SIZE, MP_PASS_CODE_SIZE, MP_PASS_EMIT #if MICROPY_EMIT_NATIVE emit_t *emit_native = NULL; #endif @@ -3531,7 +3564,7 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f case MP_EMIT_OPT_NATIVE_PYTHON: case MP_EMIT_OPT_VIPER: if (emit_native == NULL) { - emit_native = NATIVE_EMITTER(new)(&comp->compile_error, &comp->next_label, max_num_labels); + emit_native = NATIVE_EMITTER(new)(&comp->emit_common, &comp->compile_error, &comp->next_label, max_num_labels); } comp->emit_method_table = NATIVE_EMITTER_TABLE; comp->emit = emit_native; @@ -3555,8 +3588,10 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f } // final pass: emit code + // the emitter can request multiple of these passes if (comp->compile_error == MP_OBJ_NULL) { - compile_scope(comp, s, MP_PASS_EMIT); + while (!compile_scope(comp, s, MP_PASS_EMIT)) { + } } } } @@ -3566,10 +3601,45 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f // number for the start of this scope compile_error_set_line(comp, comp->scope_cur->pn); // add a traceback to the exception using relevant source info - mp_obj_exception_add_traceback(comp->compile_error, comp->source_file, + mp_obj_exception_add_traceback(comp->compile_error, source_file, comp->compile_error_line, comp->scope_cur->simple_name); } + // construct the global qstr/const table for this module + mp_compiled_module_t cm; + cm.rc = module_scope->raw_code; + cm.context = context; + #if MICROPY_PERSISTENT_CODE_SAVE + cm.has_native = false; + #if MICROPY_EMIT_NATIVE + if (emit_native != NULL) { + cm.has_native = true; + } + #endif + #if MICROPY_EMIT_INLINE_ASM + if (comp->emit_inline_asm != NULL) { + cm.has_native = true; + } + #endif + cm.n_qstr = comp->emit_common.qstr_map.used; + cm.n_obj = comp->emit_common.const_obj_list.len; + #endif + if (comp->compile_error == MP_OBJ_NULL) { + mp_emit_common_populate_module_context(&comp->emit_common, source_file, context); + + #if MICROPY_DEBUG_PRINTERS + // now that the module context is valid, the raw codes can be printed + if (mp_verbose_flag >= 2) { + for (scope_t *s = comp->scope_head; s != NULL; s = s->next) { + mp_raw_code_t *rc = s->raw_code; + if (rc->kind == MP_CODE_BYTECODE) { + mp_bytecode_print(&mp_plat_print, rc, &cm.context->constants); + } + } + } + #endif + } + // free the emitters emit_bc_free(emit_bc); @@ -3588,7 +3658,6 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f mp_parse_tree_clear(parse_tree); // free the scopes - mp_raw_code_t *outer_raw_code = module_scope->raw_code; for (scope_t *s = module_scope; s;) { scope_t *next = s->next; scope_free(s); @@ -3597,15 +3666,17 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f if (comp->compile_error != MP_OBJ_NULL) { nlr_raise(comp->compile_error); - } else { - return outer_raw_code; } + + return cm; } mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) { - mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, is_repl); + mp_module_context_t *context = m_new_obj(mp_module_context_t); + context->module.globals = mp_globals_get(); + mp_compiled_module_t cm = mp_compile_to_raw_code(parse_tree, source_file, is_repl, context); // return function that executes the outer module - return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL); + return mp_make_function_from_raw_code(cm.rc, cm.context, NULL); } #endif // MICROPY_ENABLE_COMPILER diff --git a/py/compile.h b/py/compile.h index 348beff15f..ae87bf2a04 100644 --- a/py/compile.h +++ b/py/compile.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,11 +32,12 @@ // the compiler will raise an exception if an error occurred // the compiler will clear the parse tree before it returns +// mp_globals_get() will be used for the context mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl); #if MICROPY_PERSISTENT_CODE_SAVE // this has the same semantics as mp_compile -mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl); +mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_module_context_t *globals); #endif // this is implemented in runtime.c diff --git a/py/dynruntime.h b/py/dynruntime.h index 608cdec097..254d0fee78 100644 --- a/py/dynruntime.h +++ b/py/dynruntime.h @@ -30,6 +30,7 @@ // MicroPython runtime API defined in py/obj.h and py/runtime.h. #include "py/nativeglue.h" +#include "py/objfun.h" #include "py/objstr.h" #include "py/objtype.h" @@ -43,6 +44,7 @@ #undef mp_const_none #undef mp_const_false #undef mp_const_true +#undef mp_const_empty_bytes #undef mp_const_empty_tuple #undef nlr_raise @@ -80,7 +82,11 @@ static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) { #define MP_OBJ_NEW_QSTR(x) MP_OBJ_NEW_QSTR_##x #define mp_type_type (*mp_fun_table.type_type) +#define mp_type_NoneType (*mp_obj_get_type(mp_const_none)) +#define mp_type_bool (*mp_obj_get_type(mp_const_false)) +#define mp_type_int (*(mp_obj_type_t *)(mp_load_global(MP_QSTR_int))) #define mp_type_str (*mp_fun_table.type_str) +#define mp_type_bytes (*(mp_obj_type_t *)(mp_load_global(MP_QSTR_bytes))) #define mp_type_tuple (*((mp_obj_base_t *)mp_const_empty_tuple)->type) #define mp_type_list (*mp_fun_table.type_list) #define mp_type_EOFError (*(mp_obj_type_t *)(mp_load_global(MP_QSTR_EOFError))) @@ -99,6 +105,7 @@ static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) { #define mp_const_none ((mp_obj_t)mp_fun_table.const_none) #define mp_const_false ((mp_obj_t)mp_fun_table.const_false) #define mp_const_true ((mp_obj_t)mp_fun_table.const_true) +#define mp_const_empty_bytes (mp_type_bytes.make_new(NULL, 0, 0, NULL)) #define mp_const_empty_tuple (mp_fun_table.new_tuple(0, NULL)) #define mp_obj_new_bool(b) ((b) ? (mp_obj_t)mp_fun_table.const_true : (mp_obj_t)mp_fun_table.const_false) @@ -110,6 +117,7 @@ static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) { #define mp_obj_new_bytearray_by_ref(n, i) (mp_fun_table.obj_new_bytearray_by_ref((n), (i))) #define mp_obj_new_tuple(n, items) (mp_fun_table.new_tuple((n), (items))) #define mp_obj_new_list(n, items) (mp_fun_table.new_list((n), (items))) +#define mp_obj_new_dict(n) (mp_fun_table.new_dict((n))) #define mp_obj_get_type(o) (mp_fun_table.obj_get_type((o))) #define mp_obj_cast_to_native_base(o, t) (mp_obj_cast_to_native_base_dyn((o), (t))) @@ -124,6 +132,9 @@ static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) { #define mp_obj_subscr(base, index, val) (mp_fun_table.obj_subscr((base), (index), (val))) #define mp_obj_get_array(o, len, items) (mp_obj_get_array_dyn((o), (len), (items))) #define mp_obj_list_append(list, item) (mp_fun_table.list_append((list), (item))) +#define mp_obj_dict_store(dict, key, val) (mp_fun_table.dict_store((dict), (key), (val))) + +#define mp_obj_malloc_helper(n, t) (mp_obj_malloc_helper_dyn(n, t)) #define mp_obj_assert_native_inited(o) (mp_fun_table.assert_native_inited((o))) @@ -140,8 +151,7 @@ static inline mp_obj_t mp_obj_cast_to_native_base_dyn(mp_obj_t self_in, mp_const if (MP_OBJ_FROM_PTR(self_type) == native_type) { return self_in; - } - if (self_type->parent != native_type) { + } else if (self_type->parent != native_type) { // The self_in object is not a direct descendant of native_type, so fail the cast. // This is a very simple version of mp_obj_is_subclass_fast that could be improved. return MP_OBJ_NULL; @@ -165,7 +175,6 @@ static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) { return mp_fun_table.call_function_n_kw(mp_fun_table.load_name(MP_QSTR_len), 1, &o); } - /******************************************************************************/ // General runtime functions @@ -181,8 +190,8 @@ static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) { #define mp_unary_op(op, obj) (mp_fun_table.unary_op((op), (obj))) #define mp_binary_op(op, lhs, rhs) (mp_fun_table.binary_op((op), (lhs), (rhs))) -#define mp_make_function_from_raw_code(rc, def_args, def_kw_args) \ - (mp_fun_table.make_function_from_raw_code((rc), (def_args), (def_kw_args))) +#define mp_make_function_from_raw_code(rc, context, def_args) \ + (mp_fun_table.make_function_from_raw_code((rc), (context), (def_args))) #define mp_call_function_n_kw(fun, n_args, n_kw, args) \ (mp_fun_table.call_function_n_kw((fun), (n_args) | ((n_kw) << 8), args)) @@ -191,11 +200,10 @@ static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) { (mp_fun_table.arg_check_num_sig((n_args), (n_kw), MP_OBJ_FUN_MAKE_SIG((n_args_min), (n_args_max), (takes_kw)))) #define MP_DYNRUNTIME_INIT_ENTRY \ - mp_obj_t old_globals = mp_fun_table.swap_globals(self->globals); \ + mp_obj_t old_globals = mp_fun_table.swap_globals(self->context->module.globals); \ mp_raw_code_t rc; \ rc.kind = MP_CODE_NATIVE_VIPER; \ rc.scope_flags = 0; \ - rc.const_table = (void *)self->const_table; \ (void)rc; #define MP_DYNRUNTIME_INIT_EXIT \ diff --git a/py/dynruntime.mk b/py/dynruntime.mk index db06d41e73..09cbb2dd37 100644 --- a/py/dynruntime.mk +++ b/py/dynruntime.mk @@ -55,6 +55,13 @@ CROSS = CFLAGS += -fno-stack-protector MICROPY_FLOAT_IMPL ?= double +else ifeq ($(ARCH),armv6m) + +# thumb +CROSS = arm-none-eabi- +CFLAGS += -mthumb -mcpu=cortex-m0 +MICROPY_FLOAT_IMPL ?= none + else ifeq ($(ARCH),armv7m) # thumb diff --git a/py/emit.h b/py/emit.h index 2797fb84ea..608734552a 100644 --- a/py/emit.h +++ b/py/emit.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -43,7 +43,7 @@ typedef enum { MP_PASS_SCOPE = 1, // work out id's and their kind, and number of labels MP_PASS_STACK_SIZE = 2, // work out maximum stack size MP_PASS_CODE_SIZE = 3, // work out code size and label offsets - MP_PASS_EMIT = 4, // emit code + MP_PASS_EMIT = 4, // emit code (may be run multiple times if the emitter requests it) } pass_kind_t; #define MP_EMIT_STAR_FLAG_SINGLE (0x01) @@ -92,6 +92,16 @@ typedef enum { typedef struct _emit_t emit_t; +typedef struct _mp_emit_common_t { + pass_kind_t pass; + uint16_t ct_cur_child; + mp_raw_code_t **children; + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + mp_map_t qstr_map; + #endif + mp_obj_list_t const_obj_list; +} mp_emit_common_t; + typedef struct _mp_emit_method_table_id_ops_t { void (*local)(emit_t *emit, qstr qst, mp_uint_t local_num, int kind); void (*global)(emit_t *emit, qstr qst, int kind); @@ -99,12 +109,12 @@ typedef struct _mp_emit_method_table_id_ops_t { typedef struct _emit_method_table_t { #if MICROPY_DYNAMIC_COMPILER - emit_t *(*emit_new)(mp_obj_t * error_slot, uint *label_slot, mp_uint_t max_num_labels); + emit_t *(*emit_new)(mp_emit_common_t * emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); void (*emit_free)(emit_t *emit); #endif void (*start_pass)(emit_t *emit, pass_kind_t pass, scope_t *scope); - void (*end_pass)(emit_t *emit); + bool (*end_pass)(emit_t *emit); bool (*last_emit_was_return_value)(emit_t *emit); void (*adjust_stack_size)(emit_t *emit, mp_int_t delta); void (*set_source_line)(emit_t *emit, mp_uint_t line); @@ -161,6 +171,23 @@ typedef struct _emit_method_table_t { void (*end_except_handler)(emit_t *emit); } emit_method_table_t; +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE +qstr_short_t mp_emit_common_use_qstr(mp_emit_common_t *emit, qstr qst); +#else +static inline qstr_short_t mp_emit_common_use_qstr(mp_emit_common_t *emit, qstr qst) { + return qst; +} +#endif + +size_t mp_emit_common_use_const_obj(mp_emit_common_t *emit, mp_obj_t const_obj); + +static inline size_t mp_emit_common_alloc_const_child(mp_emit_common_t *emit, mp_raw_code_t *rc) { + if (emit->pass == MP_PASS_EMIT) { + emit->children[emit->ct_cur_child] = rc; + } + return emit->ct_cur_child++; +} + static inline void mp_emit_common_get_id_for_load(scope_t *scope, qstr qst) { scope_find_or_add_id(scope, qst, ID_INFO_KIND_GLOBAL_IMPLICIT); } @@ -180,13 +207,13 @@ extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_load_id_ops; extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_store_id_ops; extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_delete_id_ops; -emit_t *emit_bc_new(void); -emit_t *emit_native_x64_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_x86_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_thumb_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_arm_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_xtensa_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_xtensawin_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_bc_new(mp_emit_common_t *emit_common); +emit_t *emit_native_x64_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_x86_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_thumb_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_arm_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_xtensa_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_xtensawin_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); void emit_bc_set_max_num_labels(emit_t *emit, mp_uint_t max_num_labels); @@ -199,7 +226,7 @@ void emit_native_xtensa_free(emit_t *emit); void emit_native_xtensawin_free(emit_t *emit); void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope); -void mp_emit_bc_end_pass(emit_t *emit); +bool mp_emit_bc_end_pass(emit_t *emit); bool mp_emit_bc_last_emit_was_return_value(emit_t *emit); void mp_emit_bc_adjust_stack_size(emit_t *emit, mp_int_t delta); void mp_emit_bc_set_source_line(emit_t *emit, mp_uint_t line); diff --git a/py/emitbc.c b/py/emitbc.c index 80d0bf489b..2007975c5e 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2019 Damien P. George + * Copyright (c) 2013-2019 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,16 +28,17 @@ #include #include #include +#include #include #include "py/mpstate.h" +#include "py/smallint.h" #include "py/emit.h" #include "py/bc0.h" #if MICROPY_ENABLE_COMPILER -#define BYTES_FOR_INT ((MP_BYTES_PER_OBJ_WORD * 8 + 6) / 7) -#define DUMMY_DATA_SIZE (BYTES_FOR_INT) +#define DUMMY_DATA_SIZE (MP_ENCODE_UINT_MAX_BYTES) struct _emit_t { // Accessed as mp_obj_t, so must be aligned as such, and we rely on the @@ -50,66 +51,45 @@ struct _emit_t { int stack_size; + mp_emit_common_t *emit_common; scope_t *scope; mp_uint_t last_source_line_offset; mp_uint_t last_source_line; - mp_uint_t max_num_labels; - mp_uint_t *label_offsets; + size_t max_num_labels; + size_t *label_offsets; size_t code_info_offset; size_t code_info_size; size_t bytecode_offset; size_t bytecode_size; byte *code_base; // stores both byte code and code info + bool overflow; size_t n_info; size_t n_cell; - - #if MICROPY_PERSISTENT_CODE - uint16_t ct_cur_obj; - uint16_t ct_num_obj; - uint16_t ct_cur_raw_code; - #endif - mp_uint_t *const_table; }; -emit_t *emit_bc_new(void) { +emit_t *emit_bc_new(mp_emit_common_t *emit_common) { emit_t *emit = m_new0(emit_t, 1); + emit->emit_common = emit_common; return emit; } void emit_bc_set_max_num_labels(emit_t *emit, mp_uint_t max_num_labels) { emit->max_num_labels = max_num_labels; - emit->label_offsets = m_new(mp_uint_t, emit->max_num_labels); + emit->label_offsets = m_new(size_t, emit->max_num_labels); } void emit_bc_free(emit_t *emit) { - m_del(mp_uint_t, emit->label_offsets, emit->max_num_labels); + m_del(size_t, emit->label_offsets, emit->max_num_labels); m_del_obj(emit_t, emit); } -typedef byte *(*emit_allocator_t)(emit_t *emit, int nbytes); - -STATIC void emit_write_uint(emit_t *emit, emit_allocator_t allocator, mp_uint_t val) { - // We store each 7 bits in a separate byte, and that's how many bytes needed - byte buf[BYTES_FOR_INT]; - byte *p = buf + sizeof(buf); - // We encode in little-ending order, but store in big-endian, to help decoding - do { - *--p = val & 0x7f; - val >>= 7; - } while (val != 0); - byte *c = allocator(emit, buf + sizeof(buf) - p); - while (p != buf + sizeof(buf) - 1) { - *c++ = *p++ | 0x80; - } - *c = *p; -} - // all functions must go through this one to emit code info -STATIC byte *emit_get_cur_to_write_code_info(emit_t *emit, int num_bytes_to_write) { +STATIC uint8_t *emit_get_cur_to_write_code_info(void *emit_in, size_t num_bytes_to_write) { + emit_t *emit = emit_in; if (emit->pass < MP_PASS_EMIT) { emit->code_info_offset += num_bytes_to_write; return emit->dummy_data; @@ -126,14 +106,7 @@ STATIC void emit_write_code_info_byte(emit_t *emit, byte val) { } STATIC void emit_write_code_info_qstr(emit_t *emit, qstr qst) { - #if MICROPY_PERSISTENT_CODE - assert((qst >> 16) == 0); - byte *c = emit_get_cur_to_write_code_info(emit, 2); - c[0] = qst; - c[1] = qst >> 8; - #else - emit_write_uint(emit, emit_get_cur_to_write_code_info, qst); - #endif + mp_encode_uint(emit, emit_get_cur_to_write_code_info, mp_emit_common_use_qstr(emit->emit_common, qst)); } #if MICROPY_ENABLE_SOURCE_LINE @@ -166,7 +139,8 @@ STATIC void emit_write_code_info_bytes_lines(emit_t *emit, mp_uint_t bytes_to_sk #endif // all functions must go through this one to emit byte code -STATIC byte *emit_get_cur_to_write_bytecode(emit_t *emit, int num_bytes_to_write) { +STATIC uint8_t *emit_get_cur_to_write_bytecode(void *emit_in, size_t num_bytes_to_write) { + emit_t *emit = emit_in; if (emit->pass < MP_PASS_EMIT) { emit->bytecode_offset += num_bytes_to_write; return emit->dummy_data; @@ -189,12 +163,12 @@ STATIC void emit_write_bytecode_byte(emit_t *emit, int stack_adj, byte b1) { c[0] = b1; } -// Similar to emit_write_bytecode_uint(), just some extra handling to encode sign +// Similar to mp_encode_uint(), just some extra handling to encode sign STATIC void emit_write_bytecode_byte_int(emit_t *emit, int stack_adj, byte b1, mp_int_t num) { emit_write_bytecode_byte(emit, stack_adj, b1); // We store each 7 bits in a separate byte, and that's how many bytes needed - byte buf[BYTES_FOR_INT]; + byte buf[MP_ENCODE_UINT_MAX_BYTES]; byte *p = buf + sizeof(buf); // We encode in little-ending order, but store in big-endian, to help decoding do { @@ -218,94 +192,81 @@ STATIC void emit_write_bytecode_byte_int(emit_t *emit, int stack_adj, byte b1, m STATIC void emit_write_bytecode_byte_uint(emit_t *emit, int stack_adj, byte b, mp_uint_t val) { emit_write_bytecode_byte(emit, stack_adj, b); - emit_write_uint(emit, emit_get_cur_to_write_bytecode, val); + mp_encode_uint(emit, emit_get_cur_to_write_bytecode, val); } -#if MICROPY_PERSISTENT_CODE -STATIC void emit_write_bytecode_byte_const(emit_t *emit, int stack_adj, byte b, mp_uint_t n, mp_uint_t c) { - if (emit->pass == MP_PASS_EMIT) { - emit->const_table[n] = c; - } +STATIC void emit_write_bytecode_byte_const(emit_t *emit, int stack_adj, byte b, mp_uint_t n) { emit_write_bytecode_byte_uint(emit, stack_adj, b, n); } -#endif STATIC void emit_write_bytecode_byte_qstr(emit_t *emit, int stack_adj, byte b, qstr qst) { - #if MICROPY_PERSISTENT_CODE - assert((qst >> 16) == 0); - mp_emit_bc_adjust_stack_size(emit, stack_adj); - byte *c = emit_get_cur_to_write_bytecode(emit, 3); - c[0] = b; - c[1] = qst; - c[2] = qst >> 8; - #else - emit_write_bytecode_byte_uint(emit, stack_adj, b, qst); - #endif + emit_write_bytecode_byte_uint(emit, stack_adj, b, mp_emit_common_use_qstr(emit->emit_common, qst)); } STATIC void emit_write_bytecode_byte_obj(emit_t *emit, int stack_adj, byte b, mp_obj_t obj) { - #if MICROPY_PERSISTENT_CODE - emit_write_bytecode_byte_const(emit, stack_adj, b, - emit->scope->num_pos_args + emit->scope->num_kwonly_args - + emit->ct_cur_obj++, (mp_uint_t)obj); - #else - // aligns the pointer so it is friendly to GC - emit_write_bytecode_byte(emit, stack_adj, b); - emit->bytecode_offset = (size_t)MP_ALIGN(emit->bytecode_offset, sizeof(mp_obj_t)); - mp_obj_t *c = (mp_obj_t *)emit_get_cur_to_write_bytecode(emit, sizeof(mp_obj_t)); - // Verify that c is already uint-aligned - assert(c == MP_ALIGN(c, sizeof(mp_obj_t))); - *c = obj; - #endif + emit_write_bytecode_byte_const(emit, stack_adj, b, mp_emit_common_use_const_obj(emit->emit_common, obj)); } -STATIC void emit_write_bytecode_byte_raw_code(emit_t *emit, int stack_adj, byte b, mp_raw_code_t *rc) { - #if MICROPY_PERSISTENT_CODE +STATIC void emit_write_bytecode_byte_child(emit_t *emit, int stack_adj, byte b, mp_raw_code_t *rc) { emit_write_bytecode_byte_const(emit, stack_adj, b, - emit->scope->num_pos_args + emit->scope->num_kwonly_args - + emit->ct_num_obj + emit->ct_cur_raw_code++, (mp_uint_t)(uintptr_t)rc); - #else - // aligns the pointer so it is friendly to GC - emit_write_bytecode_byte(emit, stack_adj, b); - emit->bytecode_offset = (size_t)MP_ALIGN(emit->bytecode_offset, sizeof(void *)); - void **c = (void **)emit_get_cur_to_write_bytecode(emit, sizeof(void *)); - // Verify that c is already uint-aligned - assert(c == MP_ALIGN(c, sizeof(void *))); - *c = rc; - #endif + mp_emit_common_alloc_const_child(emit->emit_common, rc)); #if MICROPY_PY_SYS_SETTRACE rc->line_of_definition = emit->last_source_line; #endif } -// unsigned labels are relative to ip following this instruction, stored as 16 bits -STATIC void emit_write_bytecode_byte_unsigned_label(emit_t *emit, int stack_adj, byte b1, mp_uint_t label) { +// Emit a jump opcode to a destination label. +// The offset to the label is relative to the ip following this instruction. +// The offset is encoded as either 1 or 2 bytes, depending on how big it is. +// The encoding of this jump opcode can change size from one pass to the next, +// but it must only ever decrease in size on successive passes. +STATIC void emit_write_bytecode_byte_label(emit_t *emit, int stack_adj, byte b1, mp_uint_t label) { mp_emit_bc_adjust_stack_size(emit, stack_adj); - mp_uint_t bytecode_offset; - if (emit->pass < MP_PASS_EMIT) { - bytecode_offset = 0; - } else { - bytecode_offset = emit->label_offsets[label] - emit->bytecode_offset - 3; - } - byte *c = emit_get_cur_to_write_bytecode(emit, 3); - c[0] = b1; - c[1] = bytecode_offset; - c[2] = bytecode_offset >> 8; -} -// signed labels are relative to ip following this instruction, stored as 16 bits, in excess -STATIC void emit_write_bytecode_byte_signed_label(emit_t *emit, int stack_adj, byte b1, mp_uint_t label) { - mp_emit_bc_adjust_stack_size(emit, stack_adj); - int bytecode_offset; - if (emit->pass < MP_PASS_EMIT) { - bytecode_offset = 0; - } else { - bytecode_offset = emit->label_offsets[label] - emit->bytecode_offset - 3 + 0x8000; + // Determine if the jump offset is signed or unsigned, based on the opcode. + const bool is_signed = b1 <= MP_BC_POP_JUMP_IF_FALSE; + + // Default to a 2-byte encoding (the largest) with an unknown jump offset. + unsigned int jump_encoding_size = 1; + ssize_t bytecode_offset = 0; + + // Compute the jump size and offset only when code size is known. + if (emit->pass >= MP_PASS_CODE_SIZE) { + // The -2 accounts for this jump opcode taking 2 bytes (at least). + bytecode_offset = emit->label_offsets[label] - emit->bytecode_offset - 2; + + // Check if the bytecode_offset is small enough to use a 1-byte encoding. + if ((is_signed && -64 <= bytecode_offset && bytecode_offset <= 63) + || (!is_signed && (size_t)bytecode_offset <= 127)) { + // Use a 1-byte jump offset. + jump_encoding_size = 0; + } + + // Adjust the offset depending on the size of the encoding of the offset. + bytecode_offset -= jump_encoding_size; + + assert(is_signed || bytecode_offset >= 0); } - byte *c = emit_get_cur_to_write_bytecode(emit, 3); + + // Emit the opcode. + byte *c = emit_get_cur_to_write_bytecode(emit, 2 + jump_encoding_size); c[0] = b1; - c[1] = bytecode_offset; - c[2] = bytecode_offset >> 8; + if (jump_encoding_size == 0) { + if (is_signed) { + bytecode_offset += 0x40; + } + assert(0 <= bytecode_offset && bytecode_offset <= 0x7f); + c[1] = bytecode_offset; + } else { + if (is_signed) { + bytecode_offset += 0x4000; + } + if (emit->pass == MP_PASS_EMIT && !(0 <= bytecode_offset && bytecode_offset <= 0x7fff)) { + emit->overflow = true; + } + c[1] = 0x80 | (bytecode_offset & 0x7f); + c[2] = bytecode_offset >> 7; + } } void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { @@ -315,14 +276,9 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { emit->scope = scope; emit->last_source_line_offset = 0; emit->last_source_line = 1; - #ifndef NDEBUG - // With debugging enabled labels are checked for unique assignment - if (pass < MP_PASS_EMIT && emit->label_offsets != NULL) { - memset(emit->label_offsets, -1, emit->max_num_labels * sizeof(mp_uint_t)); - } - #endif emit->bytecode_offset = 0; emit->code_info_offset = 0; + emit->overflow = false; // Write local state size, exception stack size, scope flags and number of arguments { @@ -343,27 +299,19 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { } // Write number of cells and size of the source code info - if (pass >= MP_PASS_CODE_SIZE) { - MP_BC_PRELUDE_SIZE_ENCODE(emit->n_info, emit->n_cell, emit_write_code_info_byte, emit); + if (emit->pass >= MP_PASS_CODE_SIZE) { + size_t n_info = emit->n_info; + size_t n_cell = emit->n_cell; + MP_BC_PRELUDE_SIZE_ENCODE(n_info, n_cell, emit_write_code_info_byte, emit); } emit->n_info = emit->code_info_offset; - // Write the name and source file of this function. + // Write the name of this function. emit_write_code_info_qstr(emit, scope->simple_name); - emit_write_code_info_qstr(emit, scope->source_file); - - #if MICROPY_PERSISTENT_CODE - emit->ct_cur_obj = 0; - emit->ct_cur_raw_code = 0; - #endif - - if (pass == MP_PASS_EMIT) { - // Write argument names (needed to resolve positional args passed as - // keywords). We store them as full word-sized objects for efficient access - // in mp_setup_code_state this is the start of the prelude and is guaranteed - // to be aligned on a word boundary. + // Write argument names, needed to resolve positional args passed as keywords. + { // For a given argument position (indexed by i) we need to find the // corresponding id_info which is a parameter, as it has the correct // qstr name to use as the argument name. Note that it's not a simple @@ -383,21 +331,19 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { break; } } - emit->const_table[i] = (mp_uint_t)MP_OBJ_NEW_QSTR(qst); + emit_write_code_info_qstr(emit, qst); } } } -void mp_emit_bc_end_pass(emit_t *emit) { +bool mp_emit_bc_end_pass(emit_t *emit) { if (emit->pass == MP_PASS_SCOPE) { - return; + return true; } // check stack is back to zero size assert(emit->stack_size == 0); - emit_write_code_info_byte(emit, 0); // end of line number info - // Calculate size of source code info section emit->n_info = emit->code_info_offset - emit->n_info; @@ -412,42 +358,43 @@ void mp_emit_bc_end_pass(emit_t *emit) { } } - #if MICROPY_PERSISTENT_CODE - assert(emit->pass <= MP_PASS_STACK_SIZE || (emit->ct_num_obj == emit->ct_cur_obj)); - emit->ct_num_obj = emit->ct_cur_obj; - #endif - if (emit->pass == MP_PASS_CODE_SIZE) { - #if !MICROPY_PERSISTENT_CODE - // so bytecode is aligned - emit->code_info_offset = (size_t)MP_ALIGN(emit->code_info_offset, sizeof(mp_uint_t)); - #endif - // calculate size of total code-info + bytecode, in bytes emit->code_info_size = emit->code_info_offset; emit->bytecode_size = emit->bytecode_offset; emit->code_base = m_new0(byte, emit->code_info_size + emit->bytecode_size); - #if MICROPY_PERSISTENT_CODE - emit->const_table = m_new0(mp_uint_t, - emit->scope->num_pos_args + emit->scope->num_kwonly_args - + emit->ct_cur_obj + emit->ct_cur_raw_code); - #else - emit->const_table = m_new0(mp_uint_t, - emit->scope->num_pos_args + emit->scope->num_kwonly_args); - #endif - } else if (emit->pass == MP_PASS_EMIT) { + // Code info and/or bytecode can shrink during this pass. + assert(emit->code_info_offset <= emit->code_info_size); + assert(emit->bytecode_offset <= emit->bytecode_size); + + if (emit->code_info_offset != emit->code_info_size + || emit->bytecode_offset != emit->bytecode_size) { + // Code info and/or bytecode changed size in this pass, so request the + // compiler to do another pass with these updated sizes. + emit->code_info_size = emit->code_info_offset; + emit->bytecode_size = emit->bytecode_offset; + return false; + } + + if (emit->overflow) { + mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("bytecode overflow")); + } + + // Bytecode is finalised, assign it to the raw code object. mp_emit_glue_assign_bytecode(emit->scope->raw_code, emit->code_base, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS emit->code_info_size + emit->bytecode_size, #endif - emit->const_table, + emit->emit_common->children, #if MICROPY_PERSISTENT_CODE_SAVE - emit->ct_cur_obj, emit->ct_cur_raw_code, + emit->emit_common->ct_cur_child, #endif emit->scope->scope_flags); } + + return true; } bool mp_emit_bc_last_emit_was_return_value(emit_t *emit) { @@ -490,15 +437,16 @@ void mp_emit_bc_label_assign(emit_t *emit, mp_uint_t l) { if (emit->pass == MP_PASS_SCOPE) { return; } + + // Label offsets can change from one pass to the next, but they must only + // decrease (ie code can only shrink). There will be multiple MP_PASS_EMIT + // stages until the labels no longer change, which is when the code size + // stays constant after a MP_PASS_EMIT. assert(l < emit->max_num_labels); - if (emit->pass < MP_PASS_EMIT) { - // assign label offset - assert(emit->label_offsets[l] == (mp_uint_t)-1); - emit->label_offsets[l] = emit->bytecode_offset; - } else { - // ensure label offset has not changed from MP_PASS_CODE_SIZE to MP_PASS_EMIT - assert(emit->label_offsets[l] == emit->bytecode_offset); - } + assert(emit->pass == MP_PASS_STACK_SIZE || emit->bytecode_offset <= emit->label_offsets[l]); + + // Assign label offset. + emit->label_offsets[l] = emit->bytecode_offset; } void mp_emit_bc_import(emit_t *emit, qstr qst, int kind) { @@ -523,6 +471,7 @@ void mp_emit_bc_load_const_tok(emit_t *emit, mp_token_kind_t tok) { } void mp_emit_bc_load_const_small_int(emit_t *emit, mp_int_t arg) { + assert(MP_SMALL_INT_FITS(arg)); if (-MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS <= arg && arg < MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM - MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS) { emit_write_bytecode_byte(emit, 1, @@ -646,22 +595,22 @@ void mp_emit_bc_rot_three(emit_t *emit) { } void mp_emit_bc_jump(emit_t *emit, mp_uint_t label) { - emit_write_bytecode_byte_signed_label(emit, 0, MP_BC_JUMP, label); + emit_write_bytecode_byte_label(emit, 0, MP_BC_JUMP, label); } void mp_emit_bc_pop_jump_if(emit_t *emit, bool cond, mp_uint_t label) { if (cond) { - emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_POP_JUMP_IF_TRUE, label); + emit_write_bytecode_byte_label(emit, -1, MP_BC_POP_JUMP_IF_TRUE, label); } else { - emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_POP_JUMP_IF_FALSE, label); + emit_write_bytecode_byte_label(emit, -1, MP_BC_POP_JUMP_IF_FALSE, label); } } void mp_emit_bc_jump_if_or_pop(emit_t *emit, bool cond, mp_uint_t label) { if (cond) { - emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_JUMP_IF_TRUE_OR_POP, label); + emit_write_bytecode_byte_label(emit, -1, MP_BC_JUMP_IF_TRUE_OR_POP, label); } else { - emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_JUMP_IF_FALSE_OR_POP, label); + emit_write_bytecode_byte_label(emit, -1, MP_BC_JUMP_IF_FALSE_OR_POP, label); } } @@ -675,9 +624,9 @@ void mp_emit_bc_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t except_dept emit_write_bytecode_raw_byte(emit, MP_BC_POP_TOP); } } - emit_write_bytecode_byte_signed_label(emit, 0, MP_BC_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR); + emit_write_bytecode_byte_label(emit, 0, MP_BC_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR); } else { - emit_write_bytecode_byte_signed_label(emit, 0, MP_BC_UNWIND_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR); + emit_write_bytecode_byte_label(emit, 0, MP_BC_UNWIND_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR); emit_write_bytecode_raw_byte(emit, ((label & MP_EMIT_BREAK_FROM_FOR) ? 0x80 : 0) | except_depth); } } @@ -689,7 +638,7 @@ void mp_emit_bc_setup_block(emit_t *emit, mp_uint_t label, int kind) { // The SETUP_WITH opcode pops ctx_mgr from the top of the stack // and then pushes 3 entries: __exit__, ctx_mgr, as_value. int stack_adj = kind == MP_EMIT_SETUP_BLOCK_WITH ? 2 : 0; - emit_write_bytecode_byte_unsigned_label(emit, stack_adj, MP_BC_SETUP_WITH + kind, label); + emit_write_bytecode_byte_label(emit, stack_adj, MP_BC_SETUP_WITH + kind, label); } void mp_emit_bc_with_cleanup(emit_t *emit, mp_uint_t label) { @@ -711,7 +660,7 @@ void mp_emit_bc_get_iter(emit_t *emit, bool use_stack) { } void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label) { - emit_write_bytecode_byte_unsigned_label(emit, 1, MP_BC_FOR_ITER, label); + emit_write_bytecode_byte_label(emit, 1, MP_BC_FOR_ITER, label); } void mp_emit_bc_for_iter_end(emit_t *emit) { @@ -720,7 +669,7 @@ void mp_emit_bc_for_iter_end(emit_t *emit) { void mp_emit_bc_pop_except_jump(emit_t *emit, mp_uint_t label, bool within_exc_handler) { (void)within_exc_handler; - emit_write_bytecode_byte_unsigned_label(emit, 0, MP_BC_POP_EXCEPT_JUMP, label); + emit_write_bytecode_byte_label(emit, 0, MP_BC_POP_EXCEPT_JUMP, label); } void mp_emit_bc_unary_op(emit_t *emit, mp_unary_op_t op) { @@ -783,28 +732,30 @@ void mp_emit_bc_unpack_ex(emit_t *emit, mp_uint_t n_left, mp_uint_t n_right) { void mp_emit_bc_make_function(emit_t *emit, scope_t *scope, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { if (n_pos_defaults == 0 && n_kw_defaults == 0) { - emit_write_bytecode_byte_raw_code(emit, 1, MP_BC_MAKE_FUNCTION, scope->raw_code); + emit_write_bytecode_byte_child(emit, 1, MP_BC_MAKE_FUNCTION, scope->raw_code); } else { - emit_write_bytecode_byte_raw_code(emit, -1, MP_BC_MAKE_FUNCTION_DEFARGS, scope->raw_code); + emit_write_bytecode_byte_child(emit, -1, MP_BC_MAKE_FUNCTION_DEFARGS, scope->raw_code); } } void mp_emit_bc_make_closure(emit_t *emit, scope_t *scope, mp_uint_t n_closed_over, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { if (n_pos_defaults == 0 && n_kw_defaults == 0) { int stack_adj = -n_closed_over + 1; - emit_write_bytecode_byte_raw_code(emit, stack_adj, MP_BC_MAKE_CLOSURE, scope->raw_code); + emit_write_bytecode_byte_child(emit, stack_adj, MP_BC_MAKE_CLOSURE, scope->raw_code); emit_write_bytecode_raw_byte(emit, n_closed_over); } else { assert(n_closed_over <= 255); int stack_adj = -2 - (mp_int_t)n_closed_over + 1; - emit_write_bytecode_byte_raw_code(emit, stack_adj, MP_BC_MAKE_CLOSURE_DEFARGS, scope->raw_code); + emit_write_bytecode_byte_child(emit, stack_adj, MP_BC_MAKE_CLOSURE_DEFARGS, scope->raw_code); emit_write_bytecode_raw_byte(emit, n_closed_over); } } STATIC void emit_bc_call_function_method_helper(emit_t *emit, int stack_adj, mp_uint_t bytecode_base, mp_uint_t n_positional, mp_uint_t n_keyword, mp_uint_t star_flags) { if (star_flags) { - stack_adj -= (int)n_positional + 2 * (int)n_keyword + 2; + // each positional arg is one object, each kwarg is two objects, the key + // and the value and one extra object for the star args bitmap. + stack_adj -= (int)n_positional + 2 * (int)n_keyword + 1; emit_write_bytecode_byte_uint(emit, stack_adj, bytecode_base + 1, (n_keyword << 8) | n_positional); // TODO make it 2 separate uints? } else { stack_adj -= (int)n_positional + 2 * (int)n_keyword; diff --git a/py/emitcommon.c b/py/emitcommon.c index 177418c30a..679ef1d973 100644 --- a/py/emitcommon.c +++ b/py/emitcommon.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -27,15 +27,76 @@ #include #include "py/emit.h" +#include "py/nativeglue.h" #if MICROPY_ENABLE_COMPILER +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE +qstr_short_t mp_emit_common_use_qstr(mp_emit_common_t *emit, qstr qst) { + mp_map_elem_t *elem = mp_map_lookup(&emit->qstr_map, MP_OBJ_NEW_QSTR(qst), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + if (elem->value == MP_OBJ_NULL) { + elem->value = MP_OBJ_NEW_SMALL_INT(emit->qstr_map.used - 1); + } + return MP_OBJ_SMALL_INT_VALUE(elem->value); +} +#endif + +// Compare two objects for strict equality, including equality of type. This is +// different to the semantics of mp_obj_equal which, eg, has (True,) == (1.0,). +static bool strictly_equal(mp_obj_t a, mp_obj_t b) { + if (a == b) { + return true; + } + + #if MICROPY_EMIT_NATIVE + if (a == MP_OBJ_FROM_PTR(&mp_fun_table) || b == MP_OBJ_FROM_PTR(&mp_fun_table)) { + return false; + } + #endif + + const mp_obj_type_t *a_type = mp_obj_get_type(a); + const mp_obj_type_t *b_type = mp_obj_get_type(b); + if (a_type != b_type) { + return false; + } + if (a_type == &mp_type_tuple) { + mp_obj_tuple_t *a_tuple = MP_OBJ_TO_PTR(a); + mp_obj_tuple_t *b_tuple = MP_OBJ_TO_PTR(b); + if (a_tuple->len != b_tuple->len) { + return false; + } + for (size_t i = 0; i < a_tuple->len; ++i) { + if (!strictly_equal(a_tuple->items[i], b_tuple->items[i])) { + return false; + } + } + return true; + } else { + return mp_obj_equal(a, b); + } +} + +size_t mp_emit_common_use_const_obj(mp_emit_common_t *emit, mp_obj_t const_obj) { + for (size_t i = 0; i < emit->const_obj_list.len; ++i) { + if (strictly_equal(emit->const_obj_list.items[i], const_obj)) { + return i; + } + } + mp_obj_list_append(MP_OBJ_FROM_PTR(&emit->const_obj_list), const_obj); + return emit->const_obj_list.len - 1; +} + void mp_emit_common_get_id_for_modification(scope_t *scope, qstr qst) { // name adding/lookup id_info_t *id = scope_find_or_add_id(scope, qst, ID_INFO_KIND_GLOBAL_IMPLICIT); - if (SCOPE_IS_FUNC_LIKE(scope->kind) && id->kind == ID_INFO_KIND_GLOBAL_IMPLICIT) { - // rebind as a local variable - id->kind = ID_INFO_KIND_LOCAL; + if (id->kind == ID_INFO_KIND_GLOBAL_IMPLICIT) { + if (SCOPE_IS_FUNC_LIKE(scope->kind)) { + // rebind as a local variable + id->kind = ID_INFO_KIND_LOCAL; + } else { + // mark this as assigned, to prevent it from being closed over + id->kind = ID_INFO_KIND_GLOBAL_IMPLICIT_ASSIGNED; + } } } @@ -46,7 +107,7 @@ void mp_emit_common_id_op(emit_t *emit, const mp_emit_method_table_id_ops_t *emi assert(id != NULL); // call the emit backend with the correct code - if (id->kind == ID_INFO_KIND_GLOBAL_IMPLICIT) { + if (id->kind == ID_INFO_KIND_GLOBAL_IMPLICIT || id->kind == ID_INFO_KIND_GLOBAL_IMPLICIT_ASSIGNED) { emit_method_table->global(emit, qst, MP_EMIT_IDOP_GLOBAL_NAME); } else if (id->kind == ID_INFO_KIND_GLOBAL_EXPLICIT) { emit_method_table->global(emit, qst, MP_EMIT_IDOP_GLOBAL_GLOBAL); diff --git a/py/emitglue.c b/py/emitglue.c index e701d0808f..95be7f661a 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -34,6 +34,7 @@ #include "py/emitglue.h" #include "py/runtime0.h" #include "py/bc.h" +#include "py/objfun.h" #include "py/profile.h" #if MICROPY_DEBUG_VERBOSE // print debugging info @@ -63,20 +64,22 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS size_t len, #endif - const mp_uint_t *const_table, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE - uint16_t n_obj, uint16_t n_raw_code, + size_t n_children, #endif mp_uint_t scope_flags) { rc->kind = MP_CODE_BYTECODE; rc->scope_flags = scope_flags; rc->fun_data = code; - rc->const_table = const_table; - #if MICROPY_PERSISTENT_CODE_SAVE + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS rc->fun_data_len = len; - rc->n_obj = n_obj; - rc->n_raw_code = n_raw_code; + #endif + rc->children = children; + + #if MICROPY_PERSISTENT_CODE_SAVE + rc->n_children = n_children; #endif #if MICROPY_PY_SYS_SETTRACE @@ -85,26 +88,21 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, #endif #ifdef DEBUG_PRINT - #if !MICROPY_DEBUG_PRINTERS + #if !(MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS) const size_t len = 0; #endif DEBUG_printf("assign byte code: code=%p len=" UINT_FMT " flags=%x\n", code, len, (uint)scope_flags); #endif - #if MICROPY_DEBUG_PRINTERS - if (mp_verbose_flag >= 2) { - mp_bytecode_print(&mp_plat_print, rc, code, len, const_table); - } - #endif } #if MICROPY_EMIT_MACHINE_CODE -void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, const mp_uint_t *const_table, +void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE + size_t n_children, uint16_t prelude_offset, - uint16_t n_obj, uint16_t n_raw_code, - uint16_t n_qstr, mp_qstr_link_entry_t *qstr_link, #endif - mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig) { + mp_uint_t scope_flags, mp_uint_t n_pos_args, mp_uint_t type_sig) { assert(kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER || kind == MP_CODE_NATIVE_ASM); @@ -112,7 +110,7 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void // so that the generated native code which was created in data RAM will // be available for execution from instruction RAM. #if MICROPY_EMIT_THUMB || MICROPY_EMIT_INLINE_THUMB - #if defined(__ICACHE_PRESENT) && __ICACHE_PRESENT == 1 + #if __ICACHE_PRESENT == 1 // Flush D-cache, so the code emitted is stored in RAM. MP_HAL_CLEAN_DCACHE(fun_data, fun_len); // Invalidate I-cache, so the newly-created code is reloaded from RAM. @@ -135,20 +133,22 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void rc->kind = kind; rc->scope_flags = scope_flags; - rc->n_pos_args = n_pos_args; rc->fun_data = fun_data; - rc->const_table = const_table; - rc->type_sig = type_sig; + + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS + rc->fun_data_len = fun_len; + #endif + rc->children = children; #if MICROPY_PERSISTENT_CODE_SAVE - rc->fun_data_len = fun_len; + rc->n_children = n_children; rc->prelude_offset = prelude_offset; - rc->n_obj = n_obj; - rc->n_raw_code = n_raw_code; - rc->n_qstr = n_qstr; - rc->qstr_link = qstr_link; #endif + // These two entries are only needed for MP_CODE_NATIVE_ASM. + rc->n_pos_args = n_pos_args; + rc->type_sig = type_sig; + #ifdef DEBUG_PRINT DEBUG_printf("assign native: kind=%d fun=%p len=" UINT_FMT " n_pos_args=" UINT_FMT " flags=%x\n", kind, fun_data, fun_len, n_pos_args, (uint)scope_flags); for (mp_uint_t i = 0; i < fun_len; i++) { @@ -170,15 +170,15 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void } #endif -mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args) { +mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, const mp_obj_t *def_args) { DEBUG_OP_printf("make_function_from_raw_code %p\n", rc); assert(rc != NULL); // def_args must be MP_OBJ_NULL or a tuple - assert(def_args == MP_OBJ_NULL || mp_obj_is_type(def_args, &mp_type_tuple)); + assert(def_args == NULL || def_args[0] == MP_OBJ_NULL || mp_obj_is_type(def_args[0], &mp_type_tuple)); // def_kw_args must be MP_OBJ_NULL or a dict - assert(def_kw_args == MP_OBJ_NULL || mp_obj_is_type(def_kw_args, &mp_type_dict)); + assert(def_args == NULL || def_args[1] == MP_OBJ_NULL || mp_obj_is_type(def_args[1], &mp_type_dict)); // make the function, depending on the raw code kind mp_obj_t fun; @@ -186,7 +186,11 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar #if MICROPY_EMIT_NATIVE case MP_CODE_NATIVE_PY: case MP_CODE_NATIVE_VIPER: - fun = mp_obj_new_fun_native(def_args, def_kw_args, rc->fun_data, rc->const_table); + fun = mp_obj_new_fun_native(def_args, rc->fun_data, context, rc->children); + // Check for a generator function, and if so change the type of the object + if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) { + ((mp_obj_base_t *)MP_OBJ_TO_PTR(fun))->type = &mp_type_native_gen_wrap; + } break; #endif #if MICROPY_EMIT_INLINE_ASM @@ -197,35 +201,33 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar default: // rc->kind should always be set and BYTECODE is the only remaining case assert(rc->kind == MP_CODE_BYTECODE); - fun = mp_obj_new_fun_bc(def_args, def_kw_args, rc->fun_data, rc->const_table); + fun = mp_obj_new_fun_bc(def_args, rc->fun_data, context, rc->children); + // check for generator functions and if so change the type of the object + if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) { + ((mp_obj_base_t *)MP_OBJ_TO_PTR(fun))->type = &mp_type_gen_wrap; + } + + #if MICROPY_PY_SYS_SETTRACE + mp_obj_fun_bc_t *self_fun = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(fun); + self_fun->rc = rc; + #endif + break; } - // check for generator functions and if so wrap in generator object - if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) { - fun = mp_obj_new_gen_wrap(fun, (rc->scope_flags & MP_SCOPE_FLAG_ASYNC) != 0); - } - - #if MICROPY_PY_SYS_SETTRACE - if (rc->kind == MP_CODE_BYTECODE) { - mp_obj_fun_bc_t *self_fun = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(fun); - self_fun->rc = rc; - } - #endif - return fun; } -mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args) { +mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, mp_uint_t n_closed_over, const mp_obj_t *args) { DEBUG_OP_printf("make_closure_from_raw_code %p " UINT_FMT " %p\n", rc, n_closed_over, args); // make function object mp_obj_t ffun; if (n_closed_over & 0x100) { // default positional and keyword args given - ffun = mp_make_function_from_raw_code(rc, args[0], args[1]); + ffun = mp_make_function_from_raw_code(rc, context, args); } else { // default positional and keyword args not given - ffun = mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL); + ffun = mp_make_function_from_raw_code(rc, context, NULL); } // wrap function in closure object return mp_obj_new_closure(ffun, n_closed_over & 0xff, args + ((n_closed_over >> 7) & 2)); diff --git a/py/emitglue.h b/py/emitglue.h index cbe69d6e44..4ddf74011f 100644 --- a/py/emitglue.h +++ b/py/emitglue.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -49,21 +49,20 @@ typedef enum { MP_CODE_NATIVE_ASM, } mp_raw_code_kind_t; -typedef struct _mp_qstr_link_entry_t { - uint16_t off; - uint16_t qst; -} mp_qstr_link_entry_t; - +// compiled bytecode: instance in RAM, referenced by outer scope, usually freed after first (and only) use +// mpy file: instance in RAM, created when .mpy file is loaded (same comments as above) +// frozen: instance in ROM typedef struct _mp_raw_code_t { - uint8_t kind; // of type mp_raw_code_kind_t - uint8_t scope_flags; - uint16_t n_pos_args; + mp_uint_t kind : 3; // of type mp_raw_code_kind_t + mp_uint_t scope_flags : 7; + mp_uint_t n_pos_args : 11; const void *fun_data; - const mp_uint_t *const_table; + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS + size_t fun_data_len; // so mp_raw_code_save and mp_bytecode_print work + #endif + struct _mp_raw_code_t **children; #if MICROPY_PERSISTENT_CODE_SAVE - size_t fun_data_len; - uint16_t n_obj; - uint16_t n_raw_code; + size_t n_children; #if MICROPY_PY_SYS_SETTRACE mp_bytecode_prelude_t prelude; // line_of_definition is a Python source line where the raw_code was @@ -74,8 +73,6 @@ typedef struct _mp_raw_code_t { #endif #if MICROPY_EMIT_MACHINE_CODE uint16_t prelude_offset; - uint16_t n_qstr; - mp_qstr_link_entry_t *qstr_link; #endif #endif #if MICROPY_EMIT_MACHINE_CODE @@ -89,22 +86,21 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS size_t len, #endif - const mp_uint_t *const_table, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE - uint16_t n_obj, uint16_t n_raw_code, + size_t n_children, #endif mp_uint_t scope_flags); void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, - const mp_uint_t *const_table, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE + size_t n_children, uint16_t prelude_offset, - uint16_t n_obj, uint16_t n_raw_code, - uint16_t n_qstr, mp_qstr_link_entry_t *qstr_link, #endif - mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig); + mp_uint_t scope_flags, mp_uint_t n_pos_args, mp_uint_t type_sig); -mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args); -mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args); +mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, const mp_obj_t *def_args); +mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, mp_uint_t n_closed_over, const mp_obj_t *args); #endif // MICROPY_INCLUDED_PY_EMITGLUE_H diff --git a/py/emitinlinethumb.c b/py/emitinlinethumb.c index 65168a5211..29487f1048 100644 --- a/py/emitinlinethumb.c +++ b/py/emitinlinethumb.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -59,7 +59,22 @@ struct _emit_inline_asm_t { qstr *label_lookup; }; -STATIC void emit_inline_thumb_error_msg(emit_inline_asm_t *emit, const compressed_string_t *msg) { +#if MICROPY_DYNAMIC_COMPILER + +static inline bool emit_inline_thumb_allow_float(emit_inline_asm_t *emit) { + return MP_NATIVE_ARCH_ARMV7EMSP <= mp_dynamic_compiler.native_arch + && mp_dynamic_compiler.native_arch <= MP_NATIVE_ARCH_ARMV7EMDP; +} + +#else + +static inline bool emit_inline_thumb_allow_float(emit_inline_asm_t *emit) { + return MICROPY_EMIT_INLINE_THUMB_FLOAT; +} + +#endif + +STATIC void emit_inline_thumb_error_msg(emit_inline_asm_t *emit, mp_rom_error_text_t msg) { *emit->error_slot = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg); } @@ -216,7 +231,6 @@ STATIC mp_uint_t get_arg_special_reg(emit_inline_asm_t *emit, const char *op, mp return 0; } -#if MICROPY_EMIT_INLINE_THUMB_FLOAT STATIC mp_uint_t get_arg_vfpreg(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { const char *reg_str = get_arg_str(pn); if (reg_str[0] == 's' && reg_str[1] != '\0') { @@ -243,7 +257,6 @@ malformed: MP_ERROR_TEXT("'%s' expects an FPU register"), op)); return 0; } -#endif STATIC mp_uint_t get_arg_reglist(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { // a register list looks like {r0, r1, r2} and is parsed as a Python set @@ -409,10 +422,10 @@ STATIC const format_9_10_op_t format_9_10_op_table[] = { }; #undef X -#if MICROPY_EMIT_INLINE_THUMB_FLOAT // actual opcodes are: 0xee00 | op.hi_nibble, 0x0a00 | op.lo_nibble -typedef struct _format_vfp_op_t { byte op; - char name[3]; +typedef struct _format_vfp_op_t { + byte op; + char name[3]; } format_vfp_op_t; STATIC const format_vfp_op_t format_vfp_op_table[] = { { 0x30, "add" }, @@ -420,10 +433,9 @@ STATIC const format_vfp_op_t format_vfp_op_table[] = { { 0x20, "mul" }, { 0x80, "div" }, }; -#endif // shorthand alias for whether we allow ARMv7-M instructions -#define ARMV7M MICROPY_EMIT_INLINE_THUMB_ARMV7M +#define ARMV7M asm_thumb_allow_armv7m(&emit->as) STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_args, mp_parse_node_t *pn_args) { // TODO perhaps make two tables: @@ -439,8 +451,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a size_t op_len; const char *op_str = (const char *)qstr_data(op, &op_len); - #if MICROPY_EMIT_INLINE_THUMB_FLOAT - if (op_str[0] == 'v') { + if (emit_inline_thumb_allow_float(emit) && op_str[0] == 'v') { // floating point operations if (n_args == 2) { mp_uint_t op_code = 0x0ac0, op_code_hi; @@ -535,7 +546,6 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a } return; } - #endif if (n_args == 0) { if (op == MP_QSTR_nop) { @@ -621,8 +631,13 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a asm_thumb_op16(&emit->as, ASM_THUMB_OP_CPSIE_I); } else if (op == MP_QSTR_push) { mp_uint_t reglist = get_arg_reglist(emit, op_str, pn_args[0]); - if ((reglist & 0xff00) == 0) { - asm_thumb_op16(&emit->as, 0xb400 | reglist); + if ((reglist & 0xbf00) == 0) { + if ((reglist & (1 << 14)) == 0) { + asm_thumb_op16(&emit->as, 0xb400 | reglist); + } else { + // 16-bit encoding for pushing low registers and LR + asm_thumb_op16(&emit->as, 0xb500 | (reglist & 0xff)); + } } else { if (!ARMV7M) { goto unknown_op; @@ -631,8 +646,13 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a } } else if (op == MP_QSTR_pop) { mp_uint_t reglist = get_arg_reglist(emit, op_str, pn_args[0]); - if ((reglist & 0xff00) == 0) { - asm_thumb_op16(&emit->as, 0xbc00 | reglist); + if ((reglist & 0x7f00) == 0) { + if ((reglist & (1 << 15)) == 0) { + asm_thumb_op16(&emit->as, 0xbc00 | reglist); + } else { + // 16-bit encoding for popping low registers and PC, i.e., returning + asm_thumb_op16(&emit->as, 0xbd00 | (reglist & 0xff)); + } } else { if (!ARMV7M) { goto unknown_op; @@ -705,24 +725,23 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a } else if (op == MP_QSTR_sub) { op_code = ASM_THUMB_FORMAT_3_SUB; goto op_format_3; - #if ARMV7M - } else if (op == MP_QSTR_movw) { + } else if (ARMV7M && op == MP_QSTR_movw) { op_code = ASM_THUMB_OP_MOVW; mp_uint_t reg_dest; op_movw_movt: reg_dest = get_arg_reg(emit, op_str, pn_args[0], 15); int i_src = get_arg_i(emit, op_str, pn_args[1], 0xffff); asm_thumb_mov_reg_i16(&emit->as, op_code, reg_dest, i_src); - } else if (op == MP_QSTR_movt) { + } else if (ARMV7M && op == MP_QSTR_movt) { op_code = ASM_THUMB_OP_MOVT; goto op_movw_movt; - } else if (op == MP_QSTR_movwt) { + } else if (ARMV7M && op == MP_QSTR_movwt) { // this is a convenience instruction mp_uint_t reg_dest = get_arg_reg(emit, op_str, pn_args[0], 15); uint32_t i_src = get_arg_i(emit, op_str, pn_args[1], 0xffffffff); asm_thumb_mov_reg_i16(&emit->as, ASM_THUMB_OP_MOVW, reg_dest, i_src & 0xffff); asm_thumb_mov_reg_i16(&emit->as, ASM_THUMB_OP_MOVT, reg_dest, (i_src >> 16) & 0xffff); - } else if (op == MP_QSTR_ldrex) { + } else if (ARMV7M && op == MP_QSTR_ldrex) { mp_uint_t r_dest = get_arg_reg(emit, op_str, pn_args[0], 15); mp_parse_node_t pn_base, pn_offset; if (get_arg_addr(emit, op_str, pn_args[1], &pn_base, &pn_offset)) { @@ -730,7 +749,6 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a mp_uint_t i8 = get_arg_i(emit, op_str, pn_offset, 0xff) >> 2; asm_thumb_op32(&emit->as, 0xe850 | r_base, 0x0f00 | (r_dest << 12) | i8); } - #endif } else { // search table for ldr/str instructions for (mp_uint_t i = 0; i < MP_ARRAY_SIZE(format_9_10_op_table); i++) { diff --git a/py/emitinlinextensa.c b/py/emitinlinextensa.c index 17b91163f1..f044cfd094 100644 --- a/py/emitinlinextensa.c +++ b/py/emitinlinextensa.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2016 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/emitnarm.c b/py/emitnarm.c index 8297ad6192..59075b6074 100644 --- a/py/emitnarm.c +++ b/py/emitnarm.c @@ -10,8 +10,6 @@ // Word indices of REG_LOCAL_x in nlr_buf_t #define NLR_BUF_IDX_LOCAL_1 (3) // r4 -#define NLR_BUF_IDX_LOCAL_2 (4) // r5 -#define NLR_BUF_IDX_LOCAL_3 (5) // r6 #define N_ARM (1) #define EXPORT_FUN(name) emit_native_arm_##name diff --git a/py/emitnative.c b/py/emitnative.c index 7d615d5816..9fafa1104a 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -48,6 +48,7 @@ #include "py/emit.h" #include "py/nativeglue.h" +#include "py/objfun.h" #include "py/objstr.h" #if MICROPY_DEBUG_VERBOSE // print debugging info @@ -57,6 +58,7 @@ #define DEBUG_printf(...) (void)0 #endif +// CIRCUITPY: force definitions #ifndef N_X64 #define N_X64 (0) #endif @@ -90,20 +92,26 @@ // C stack layout for native functions: // 0: nlr_buf_t [optional] -// emit->code_state_start: mp_code_state_t +// return_value [optional word] +// exc_handler_unwind [optional word] +// emit->code_state_start: mp_code_state_native_t // emit->stack_start: Python object stack | emit->n_state // locals (reversed, L0 at end) | // // C stack layout for native generator functions: // 0=emit->stack_start: nlr_buf_t +// return_value +// exc_handler_unwind [optional word] // // Then REG_GENERATOR_STATE points to: -// 0=emit->code_state_start: mp_code_state_t +// 0=emit->code_state_start: mp_code_state_native_t // emit->stack_start: Python object stack | emit->n_state // locals (reversed, L0 at end) | // // C stack layout for viper functions: // 0: nlr_buf_t [optional] +// return_value [optional word] +// exc_handler_unwind [optional word] // emit->code_state_start: fun_obj, old_globals [optional] // emit->stack_start: Python object stack | emit->n_state // locals (reversed, L0 at end) | @@ -115,14 +123,18 @@ #else #define SIZEOF_NLR_BUF (sizeof(nlr_buf_t) / sizeof(uintptr_t)) #endif -#define SIZEOF_CODE_STATE (sizeof(mp_code_state_t) / sizeof(uintptr_t)) -#define OFFSETOF_CODE_STATE_STATE (offsetof(mp_code_state_t, state) / sizeof(uintptr_t)) -#define OFFSETOF_CODE_STATE_FUN_BC (offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t)) -#define OFFSETOF_CODE_STATE_IP (offsetof(mp_code_state_t, ip) / sizeof(uintptr_t)) -#define OFFSETOF_CODE_STATE_SP (offsetof(mp_code_state_t, sp) / sizeof(uintptr_t)) -#define OFFSETOF_OBJ_FUN_BC_GLOBALS (offsetof(mp_obj_fun_bc_t, globals) / sizeof(uintptr_t)) +#define SIZEOF_CODE_STATE (sizeof(mp_code_state_native_t) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_STATE (offsetof(mp_code_state_native_t, state) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_FUN_BC (offsetof(mp_code_state_native_t, fun_bc) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_IP (offsetof(mp_code_state_native_t, ip) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_SP (offsetof(mp_code_state_native_t, sp) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_N_STATE (offsetof(mp_code_state_native_t, n_state) / sizeof(uintptr_t)) +#define OFFSETOF_OBJ_FUN_BC_CONTEXT (offsetof(mp_obj_fun_bc_t, context) / sizeof(uintptr_t)) +#define OFFSETOF_OBJ_FUN_BC_CHILD_TABLE (offsetof(mp_obj_fun_bc_t, child_table) / sizeof(uintptr_t)) #define OFFSETOF_OBJ_FUN_BC_BYTECODE (offsetof(mp_obj_fun_bc_t, bytecode) / sizeof(uintptr_t)) -#define OFFSETOF_OBJ_FUN_BC_CONST_TABLE (offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t)) +#define OFFSETOF_MODULE_CONTEXT_QSTR_TABLE (offsetof(mp_module_context_t, constants.qstr_table) / sizeof(uintptr_t)) +#define OFFSETOF_MODULE_CONTEXT_OBJ_TABLE (offsetof(mp_module_context_t, constants.obj_table) / sizeof(uintptr_t)) +#define OFFSETOF_MODULE_CONTEXT_GLOBALS (offsetof(mp_module_context_t, module.globals) / sizeof(uintptr_t)) // If not already defined, set parent args to same as child call registers #ifndef REG_PARENT_RET @@ -144,6 +156,9 @@ #define NEED_GLOBAL_EXC_HANDLER(emit) ((emit)->scope->exc_stack_size > 0 \ || ((emit)->scope->scope_flags & (MP_SCOPE_FLAG_GENERATOR | MP_SCOPE_FLAG_REFGLOBALS))) +// Whether a slot is needed to store LOCAL_IDX_EXC_HANDLER_UNWIND +#define NEED_EXC_HANDLER_UNWIND(emit) ((emit)->scope->exc_stack_size > 0) + // Whether registers can be used to store locals (only true if there are no // exception handlers, because otherwise an nlr_jump will restore registers to // their state at the start of the function and updates to locals will be lost) @@ -152,14 +167,41 @@ // Indices within the local C stack for various variables #define LOCAL_IDX_EXC_VAL(emit) (NLR_BUF_IDX_RET_VAL) #define LOCAL_IDX_EXC_HANDLER_PC(emit) (NLR_BUF_IDX_LOCAL_1) -#define LOCAL_IDX_EXC_HANDLER_UNWIND(emit) (NLR_BUF_IDX_LOCAL_2) -#define LOCAL_IDX_RET_VAL(emit) (NLR_BUF_IDX_LOCAL_3) +#define LOCAL_IDX_EXC_HANDLER_UNWIND(emit) (SIZEOF_NLR_BUF + 1) // this needs a dedicated variable outside nlr_buf_t +#define LOCAL_IDX_RET_VAL(emit) (SIZEOF_NLR_BUF) // needed when NEED_GLOBAL_EXC_HANDLER is true #define LOCAL_IDX_FUN_OBJ(emit) ((emit)->code_state_start + OFFSETOF_CODE_STATE_FUN_BC) #define LOCAL_IDX_OLD_GLOBALS(emit) ((emit)->code_state_start + OFFSETOF_CODE_STATE_IP) #define LOCAL_IDX_GEN_PC(emit) ((emit)->code_state_start + OFFSETOF_CODE_STATE_IP) #define LOCAL_IDX_LOCAL_VAR(emit, local_num) ((emit)->stack_start + (emit)->n_state - 1 - (local_num)) +#if MICROPY_PERSISTENT_CODE_SAVE + +// When building with the ability to save native code to .mpy files: +// - Qstrs are indirect via qstr_table, and REG_LOCAL_3 always points to qstr_table. +// - In a generator no registers are used to store locals, and REG_LOCAL_2 points to the generator state. +// - At most 2 registers hold local variables (see CAN_USE_REGS_FOR_LOCALS for when this is possible). + +#define REG_GENERATOR_STATE (REG_LOCAL_2) +#define REG_QSTR_TABLE (REG_LOCAL_3) +#define MAX_REGS_FOR_LOCAL_VARS (2) + +STATIC const uint8_t reg_local_table[MAX_REGS_FOR_LOCAL_VARS] = {REG_LOCAL_1, REG_LOCAL_2}; + +#else + +// When building without the ability to save native code to .mpy files: +// - Qstrs values are written directly into the machine code. +// - In a generator no registers are used to store locals, and REG_LOCAL_3 points to the generator state. +// - At most 3 registers hold local variables (see CAN_USE_REGS_FOR_LOCALS for when this is possible). + #define REG_GENERATOR_STATE (REG_LOCAL_3) +#define MAX_REGS_FOR_LOCAL_VARS (3) + +STATIC const uint8_t reg_local_table[MAX_REGS_FOR_LOCAL_VARS] = {REG_LOCAL_1, REG_LOCAL_2, REG_LOCAL_3}; + +#endif + +#define REG_LOCAL_LAST (reg_local_table[MAX_REGS_FOR_LOCAL_VARS - 1]) #define EMIT_NATIVE_VIPER_TYPE_ERROR(emit, ...) do { \ *emit->error_slot = mp_obj_new_exception_msg_varg(&mp_type_ViperTypeError, __VA_ARGS__); \ @@ -233,6 +275,7 @@ typedef struct _exc_stack_entry_t { } exc_stack_entry_t; struct _emit_t { + mp_emit_common_t *emit_common; mp_obj_t *error_slot; uint *label_slot; uint exit_label; @@ -253,23 +296,15 @@ struct _emit_t { exc_stack_entry_t *exc_stack; int prelude_offset; + int prelude_ptr_index; int start_offset; int n_state; uint16_t code_state_start; uint16_t stack_start; int stack_size; + uint16_t n_info; uint16_t n_cell; - uint16_t const_table_cur_obj; - uint16_t const_table_num_obj; - uint16_t const_table_cur_raw_code; - mp_uint_t *const_table; - - #if MICROPY_PERSISTENT_CODE_SAVE - uint16_t qstr_link_cur; - mp_qstr_link_entry_t *qstr_link; - #endif - bool last_emit_was_return_value; scope_t *scope; @@ -277,14 +312,14 @@ struct _emit_t { ASM_T *as; }; -STATIC const uint8_t reg_local_table[REG_LOCAL_NUM] = {REG_LOCAL_1, REG_LOCAL_2, REG_LOCAL_3}; - +STATIC void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj); STATIC void emit_native_global_exc_entry(emit_t *emit); STATIC void emit_native_global_exc_exit(emit_t *emit); STATIC void emit_native_load_const_obj(emit_t *emit, mp_obj_t obj); -emit_t *EXPORT_FUN(new)(mp_obj_t * error_slot, uint *label_slot, mp_uint_t max_num_labels) { +emit_t *EXPORT_FUN(new)(mp_emit_common_t * emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels) { emit_t *emit = m_new0(emit_t, 1); + emit->emit_common = emit_common; emit->error_slot = error_slot; emit->label_slot = label_slot; emit->stack_info_alloc = 8; @@ -338,12 +373,7 @@ STATIC void emit_native_mov_reg_state_addr(emit_t *emit, int reg_dest, int local STATIC void emit_native_mov_reg_qstr(emit_t *emit, int arg_reg, qstr qst) { #if MICROPY_PERSISTENT_CODE_SAVE - size_t loc = ASM_MOV_REG_IMM_FIX_U16(emit->as, arg_reg, qst); - size_t link_idx = emit->qstr_link_cur++; - if (emit->pass == MP_PASS_EMIT) { - emit->qstr_link[link_idx].off = loc << 2 | 1; - emit->qstr_link[link_idx].qst = qst; - } + ASM_LOAD16_REG_REG_OFFSET(emit->as, arg_reg, REG_QSTR_TABLE, mp_emit_common_use_qstr(emit->emit_common, qst)); #else ASM_MOV_REG_IMM(emit->as, arg_reg, qst); #endif @@ -351,12 +381,7 @@ STATIC void emit_native_mov_reg_qstr(emit_t *emit, int arg_reg, qstr qst) { STATIC void emit_native_mov_reg_qstr_obj(emit_t *emit, int reg_dest, qstr qst) { #if MICROPY_PERSISTENT_CODE_SAVE - size_t loc = ASM_MOV_REG_IMM_FIX_WORD(emit->as, reg_dest, (mp_uint_t)MP_OBJ_NEW_QSTR(qst)); - size_t link_idx = emit->qstr_link_cur++; - if (emit->pass == MP_PASS_EMIT) { - emit->qstr_link[link_idx].off = loc << 2 | 2; - emit->qstr_link[link_idx].qst = qst; - } + emit_load_reg_with_object(emit, reg_dest, MP_OBJ_NEW_QSTR(qst)); #else ASM_MOV_REG_IMM(emit->as, reg_dest, (mp_uint_t)MP_OBJ_NEW_QSTR(qst)); #endif @@ -368,33 +393,12 @@ STATIC void emit_native_mov_reg_qstr_obj(emit_t *emit, int reg_dest, qstr qst) { emit_native_mov_state_reg((emit), (local_num), (reg_temp)); \ } while (false) -#define emit_native_mov_state_imm_fix_u16_via(emit, local_num, imm, reg_temp) \ - do { \ - ASM_MOV_REG_IMM_FIX_U16((emit)->as, (reg_temp), (imm)); \ - emit_native_mov_state_reg((emit), (local_num), (reg_temp)); \ - } while (false) - -#define emit_native_mov_state_imm_fix_word_via(emit, local_num, imm, reg_temp) \ - do { \ - ASM_MOV_REG_IMM_FIX_WORD((emit)->as, (reg_temp), (imm)); \ - emit_native_mov_state_reg((emit), (local_num), (reg_temp)); \ - } while (false) - STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { DEBUG_printf("start_pass(pass=%u, scope=%p)\n", pass, scope); emit->pass = pass; emit->do_viper_types = scope->emit_options == MP_EMIT_OPT_VIPER; emit->stack_size = 0; - #if N_PRELUDE_AS_BYTES_OBJ - emit->const_table_cur_obj = emit->do_viper_types ? 0 : 1; // reserve first obj for prelude bytes obj - #else - emit->const_table_cur_obj = 0; - #endif - emit->const_table_cur_raw_code = 0; - #if MICROPY_PERSISTENT_CODE_SAVE - emit->qstr_link_cur = 0; - #endif emit->last_emit_was_return_value = false; emit->scope = scope; @@ -442,12 +446,18 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop // generate code for entry to function - // Work out start of code state (mp_code_state_t or reduced version for viper) + // Work out start of code state (mp_code_state_native_t or reduced version for viper) emit->code_state_start = 0; if (NEED_GLOBAL_EXC_HANDLER(emit)) { - emit->code_state_start = SIZEOF_NLR_BUF; + emit->code_state_start = SIZEOF_NLR_BUF; // for nlr_buf_t + emit->code_state_start += 1; // for return_value + if (NEED_EXC_HANDLER_UNWIND(emit)) { + emit->code_state_start += 1; + } } + size_t fun_table_off = mp_emit_common_use_const_obj(emit->emit_common, MP_OBJ_FROM_PTR(&mp_fun_table)); + if (emit->do_viper_types) { // Work out size of state (locals plus stack) // n_state counts all stack and locals, even those in registers @@ -455,11 +465,11 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop int num_locals_in_regs = 0; if (CAN_USE_REGS_FOR_LOCALS(emit)) { num_locals_in_regs = scope->num_locals; - if (num_locals_in_regs > REG_LOCAL_NUM) { - num_locals_in_regs = REG_LOCAL_NUM; + if (num_locals_in_regs > MAX_REGS_FOR_LOCAL_VARS) { + num_locals_in_regs = MAX_REGS_FOR_LOCAL_VARS; } - // Need a spot for REG_LOCAL_3 if 4 or more args (see below) - if (scope->num_pos_args >= 4) { + // Need a spot for REG_LOCAL_LAST (see below) + if (scope->num_pos_args >= MAX_REGS_FOR_LOCAL_VARS + 1) { --num_locals_in_regs; } } @@ -483,23 +493,27 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop #endif // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_3, 0); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + #if MICROPY_PERSISTENT_CODE_SAVE + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_QSTR_TABLE, REG_FUN_TABLE, OFFSETOF_MODULE_CONTEXT_QSTR_TABLE); + #endif + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, fun_table_off); // Store function object (passed as first arg) to stack if needed if (NEED_FUN_OBJ(emit)) { ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_PARENT_ARG_1); } - // Put n_args in REG_ARG_1, n_kw in REG_ARG_2, args array in REG_LOCAL_3 + // Put n_args in REG_ARG_1, n_kw in REG_ARG_2, args array in REG_LOCAL_LAST #if N_X86 asm_x86_mov_arg_to_r32(emit->as, 1, REG_ARG_1); asm_x86_mov_arg_to_r32(emit->as, 2, REG_ARG_2); - asm_x86_mov_arg_to_r32(emit->as, 3, REG_LOCAL_3); + asm_x86_mov_arg_to_r32(emit->as, 3, REG_LOCAL_LAST); #else ASM_MOV_REG_REG(emit->as, REG_ARG_1, REG_PARENT_ARG_2); ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_PARENT_ARG_3); - ASM_MOV_REG_REG(emit->as, REG_LOCAL_3, REG_PARENT_ARG_4); + ASM_MOV_REG_REG(emit->as, REG_LOCAL_LAST, REG_PARENT_ARG_4); #endif // Check number of args matches this function, and call mp_arg_check_num_sig if not @@ -514,21 +528,21 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop // Store arguments into locals (reg or stack), converting to native if needed for (int i = 0; i < emit->scope->num_pos_args; i++) { int r = REG_ARG_1; - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_LOCAL_3, i); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_LOCAL_LAST, i); if (emit->local_vtype[i] != VTYPE_PYOBJ) { emit_call_with_imm_arg(emit, MP_F_CONVERT_OBJ_TO_NATIVE, emit->local_vtype[i], REG_ARG_2); r = REG_RET; } - // REG_LOCAL_3 points to the args array so be sure not to overwrite it if it's still needed - if (i < REG_LOCAL_NUM && CAN_USE_REGS_FOR_LOCALS(emit) && (i != 2 || emit->scope->num_pos_args == 3)) { + // REG_LOCAL_LAST points to the args array so be sure not to overwrite it if it's still needed + if (i < MAX_REGS_FOR_LOCAL_VARS && CAN_USE_REGS_FOR_LOCALS(emit) && (i != MAX_REGS_FOR_LOCAL_VARS - 1 || emit->scope->num_pos_args == MAX_REGS_FOR_LOCAL_VARS)) { ASM_MOV_REG_REG(emit->as, reg_local_table[i], r); } else { emit_native_mov_state_reg(emit, LOCAL_IDX_LOCAL_VAR(emit, i), r); } } - // Get 3rd local from the stack back into REG_LOCAL_3 if this reg couldn't be written to above - if (emit->scope->num_pos_args >= 4 && CAN_USE_REGS_FOR_LOCALS(emit)) { - ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_3, LOCAL_IDX_LOCAL_VAR(emit, 2)); + // Get local from the stack back into REG_LOCAL_LAST if this reg couldn't be written to above + if (emit->scope->num_pos_args >= MAX_REGS_FOR_LOCAL_VARS + 1 && CAN_USE_REGS_FOR_LOCALS(emit)) { + ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_LAST, LOCAL_IDX_LOCAL_VAR(emit, MAX_REGS_FOR_LOCAL_VARS - 1)); } emit_native_global_exc_entry(emit); @@ -538,16 +552,13 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop emit->n_state = scope->num_locals + scope->stack_size; if (emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR) { + mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->prelude_ptr_index); + mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->start_offset); + ASM_ENTRY(emit->as, emit->code_state_start); + + // Reset the state size for the state pointed to by REG_GENERATOR_STATE emit->code_state_start = 0; emit->stack_start = SIZEOF_CODE_STATE; - #if N_PRELUDE_AS_BYTES_OBJ - // Load index of prelude bytes object in const_table - mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)(emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1)); - #else - mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->prelude_offset); - #endif - mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->start_offset); - ASM_ENTRY(emit->as, SIZEOF_NLR_BUF); // Put address of code_state into REG_GENERATOR_STATE #if N_X86 @@ -564,8 +575,12 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_GENERATOR_STATE, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_TEMP0, emit->scope->num_pos_args + emit->scope->num_kwonly_args); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONTEXT); + #if MICROPY_PERSISTENT_CODE_SAVE + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_QSTR_TABLE, REG_TEMP0, OFFSETOF_MODULE_CONTEXT_QSTR_TABLE); + #endif + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_TEMP0, fun_table_off); } else { // The locals and stack start after the code_state structure emit->stack_start = emit->code_state_start + SIZEOF_CODE_STATE; @@ -583,38 +598,27 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop #endif // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_3, emit->scope->num_pos_args + emit->scope->num_kwonly_args); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + #if MICROPY_PERSISTENT_CODE_SAVE + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_QSTR_TABLE, REG_FUN_TABLE, OFFSETOF_MODULE_CONTEXT_QSTR_TABLE); + #endif + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, fun_table_off); // Set code_state.fun_bc ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_PARENT_ARG_1); - // Set code_state.ip (offset from start of this function to prelude info) - int code_state_ip_local = emit->code_state_start + OFFSETOF_CODE_STATE_IP; - #if N_PRELUDE_AS_BYTES_OBJ - // Prelude is a bytes object in const_table; store ip = prelude->data - fun_bc->bytecode - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, offsetof(mp_obj_str_t, data) / sizeof(uintptr_t)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_PARENT_ARG_1, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_BYTECODE); - ASM_SUB_REG_REG(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1); - emit_native_mov_state_reg(emit, code_state_ip_local, REG_LOCAL_3); - #else - if (emit->pass == MP_PASS_CODE_SIZE) { - // Commit to the encoding size based on the value of prelude_offset in this pass. - // By using 32768 as the cut-off it is highly unlikely that prelude_offset will - // grow beyond 65535 by the end of thiss pass, and so require the larger encoding. - emit->prelude_offset_uses_u16_encoding = emit->prelude_offset < 32768; + // Set code_state.ip, a pointer to the beginning of the prelude. This pointer is found + // either directly in mp_obj_fun_bc_t.child_table (if there are no children), or in + // mp_obj_fun_bc_t.child_table[num_children] (if num_children > 0). + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_PARENT_ARG_1, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CHILD_TABLE); + if (emit->prelude_ptr_index != 0) { + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_PARENT_ARG_1, REG_PARENT_ARG_1, emit->prelude_ptr_index); } - if (emit->prelude_offset_uses_u16_encoding) { - assert(emit->prelude_offset <= 65535); - emit_native_mov_state_imm_fix_u16_via(emit, code_state_ip_local, emit->prelude_offset, REG_PARENT_ARG_1); - } else { - emit_native_mov_state_imm_fix_word_via(emit, code_state_ip_local, emit->prelude_offset, REG_PARENT_ARG_1); - } - #endif + emit_native_mov_state_reg(emit, emit->code_state_start + OFFSETOF_CODE_STATE_IP, REG_PARENT_ARG_1); // Set code_state.n_state (only works on little endian targets due to n_state being uint16_t) - emit_native_mov_state_imm_via(emit, emit->code_state_start + offsetof(mp_code_state_t, n_state) / sizeof(uintptr_t), emit->n_state, REG_ARG_1); + emit_native_mov_state_imm_via(emit, emit->code_state_start + OFFSETOF_CODE_STATE_N_STATE, emit->n_state, REG_ARG_1); // Put address of code_state into first arg ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, emit->code_state_start); @@ -644,7 +648,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop // cache some locals in registers, but only if no exception handlers if (CAN_USE_REGS_FOR_LOCALS(emit)) { - for (int i = 0; i < REG_LOCAL_NUM && i < scope->num_locals; ++i) { + for (int i = 0; i < MAX_REGS_FOR_LOCAL_VARS && i < scope->num_locals; ++i) { ASM_MOV_REG_LOCAL(emit->as, reg_local_table[i], LOCAL_IDX_LOCAL_VAR(emit, i)); } } @@ -656,55 +660,47 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop emit->local_vtype[id->local_num] = VTYPE_PYOBJ; } } - - if (pass == MP_PASS_EMIT) { - // write argument names as qstr objects - // see comment in corresponding part of emitbc.c about the logic here - for (int i = 0; i < scope->num_pos_args + scope->num_kwonly_args; i++) { - qstr qst = MP_QSTR__star_; - for (int j = 0; j < scope->id_info_len; ++j) { - id_info_t *id = &scope->id_info[j]; - if ((id->flags & ID_FLAG_IS_PARAM) && id->local_num == i) { - qst = id->qst; - break; - } - } - emit->const_table[i] = (mp_uint_t)MP_OBJ_NEW_QSTR(qst); - } - } } - } static inline void emit_native_write_code_info_byte(emit_t *emit, byte val) { mp_asm_base_data(&emit->as->base, 1, val); } -STATIC void emit_native_end_pass(emit_t *emit) { +static inline void emit_native_write_code_info_qstr(emit_t *emit, qstr qst) { + mp_encode_uint(&emit->as->base, mp_asm_base_get_cur_to_write_bytes, mp_emit_common_use_qstr(emit->emit_common, qst)); +} + +STATIC bool emit_native_end_pass(emit_t *emit) { emit_native_global_exc_exit(emit); if (!emit->do_viper_types) { emit->prelude_offset = mp_asm_base_get_code_pos(&emit->as->base); + emit->prelude_ptr_index = emit->emit_common->ct_cur_child; size_t n_state = emit->n_state; size_t n_exc_stack = 0; // exc-stack not needed for native code MP_BC_PRELUDE_SIG_ENCODE(n_state, n_exc_stack, emit->scope, emit_native_write_code_info_byte, emit); - #if MICROPY_PERSISTENT_CODE - size_t n_info = 4; - #else - size_t n_info = 1; - #endif - MP_BC_PRELUDE_SIZE_ENCODE(n_info, emit->n_cell, emit_native_write_code_info_byte, emit); + size_t n_info = emit->n_info; + size_t n_cell = emit->n_cell; + MP_BC_PRELUDE_SIZE_ENCODE(n_info, n_cell, emit_native_write_code_info_byte, emit); - #if MICROPY_PERSISTENT_CODE - mp_asm_base_data(&emit->as->base, 1, emit->scope->simple_name); - mp_asm_base_data(&emit->as->base, 1, emit->scope->simple_name >> 8); - mp_asm_base_data(&emit->as->base, 1, emit->scope->source_file); - mp_asm_base_data(&emit->as->base, 1, emit->scope->source_file >> 8); - #else - mp_asm_base_data(&emit->as->base, 1, 1); - #endif + // bytecode prelude: source info (function and argument qstrs) + size_t info_start = mp_asm_base_get_code_pos(&emit->as->base); + emit_native_write_code_info_qstr(emit, emit->scope->simple_name); + for (int i = 0; i < emit->scope->num_pos_args + emit->scope->num_kwonly_args; i++) { + qstr qst = MP_QSTR__star_; + for (int j = 0; j < emit->scope->id_info_len; ++j) { + id_info_t *id = &emit->scope->id_info[j]; + if ((id->flags & ID_FLAG_IS_PARAM) && id->local_num == i) { + qst = id->qst; + break; + } + } + emit_native_write_code_info_qstr(emit, qst); + } + emit->n_info = mp_asm_base_get_code_pos(&emit->as->base) - info_start; // bytecode prelude: initialise closed over variables size_t cell_start = mp_asm_base_get_code_pos(&emit->as->base); @@ -717,15 +713,6 @@ STATIC void emit_native_end_pass(emit_t *emit) { } emit->n_cell = mp_asm_base_get_code_pos(&emit->as->base) - cell_start; - #if N_PRELUDE_AS_BYTES_OBJ - // Prelude bytes object is after qstr arg names and mp_fun_table - size_t table_off = emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1; - if (emit->pass == MP_PASS_EMIT) { - void *buf = emit->as->base.code_base + emit->prelude_offset; - size_t n = emit->as->base.code_offset - emit->prelude_offset; - emit->const_table[table_off] = (uintptr_t)mp_obj_new_bytes(buf, n); - } - #endif } ASM_END_PASS(emit->as); @@ -734,46 +721,45 @@ STATIC void emit_native_end_pass(emit_t *emit) { assert(emit->stack_size == 0); assert(emit->exc_stack_size == 0); - // Deal with const table accounting - assert(emit->pass <= MP_PASS_STACK_SIZE || (emit->const_table_num_obj == emit->const_table_cur_obj)); - emit->const_table_num_obj = emit->const_table_cur_obj; - if (emit->pass == MP_PASS_CODE_SIZE) { - size_t const_table_alloc = 1 + emit->const_table_num_obj + emit->const_table_cur_raw_code; - size_t nqstr = 0; - if (!emit->do_viper_types) { - // Add room for qstr names of arguments - nqstr = emit->scope->num_pos_args + emit->scope->num_kwonly_args; - const_table_alloc += nqstr; - } - emit->const_table = m_new(mp_uint_t, const_table_alloc); - #if !MICROPY_DYNAMIC_COMPILER - // Store mp_fun_table pointer just after qstrs - // (but in dynamic-compiler mode eliminate dependency on mp_fun_table) - emit->const_table[nqstr] = (mp_uint_t)(uintptr_t)&mp_fun_table; - #endif - - #if MICROPY_PERSISTENT_CODE_SAVE - size_t qstr_link_alloc = emit->qstr_link_cur; - if (qstr_link_alloc > 0) { - emit->qstr_link = m_new(mp_qstr_link_entry_t, qstr_link_alloc); - } - #endif - } - if (emit->pass == MP_PASS_EMIT) { void *f = mp_asm_base_get_code(&emit->as->base); mp_uint_t f_len = mp_asm_base_get_code_size(&emit->as->base); + mp_raw_code_t **children = emit->emit_common->children; + if (!emit->do_viper_types) { + #if MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE + // Executable code cannot be accessed byte-wise on this architecture, so copy + // the prelude to a separate memory region that is byte-wise readable. + void *buf = emit->as->base.code_base + emit->prelude_offset; + size_t n = emit->as->base.code_offset - emit->prelude_offset; + const uint8_t *prelude_ptr = memcpy(m_new(uint8_t, n), buf, n); + #else + // Point to the prelude directly, at the end of the machine code data. + const uint8_t *prelude_ptr = (const uint8_t *)f + emit->prelude_offset; + #endif + + // Store the pointer to the prelude using the child_table. + assert(emit->prelude_ptr_index == emit->emit_common->ct_cur_child); + if (emit->prelude_ptr_index == 0) { + children = (void *)prelude_ptr; + } else { + children = m_renew(mp_raw_code_t *, children, emit->prelude_ptr_index, emit->prelude_ptr_index + 1); + children[emit->prelude_ptr_index] = (void *)prelude_ptr; + } + } + mp_emit_glue_assign_native(emit->scope->raw_code, emit->do_viper_types ? MP_CODE_NATIVE_VIPER : MP_CODE_NATIVE_PY, - f, f_len, emit->const_table, + f, f_len, + children, #if MICROPY_PERSISTENT_CODE_SAVE + emit->emit_common->ct_cur_child, emit->prelude_offset, - emit->const_table_cur_obj, emit->const_table_cur_raw_code, - emit->qstr_link_cur, emit->qstr_link, #endif - emit->scope->num_pos_args, emit->scope->scope_flags, 0); + emit->scope->scope_flags, 0, 0); } + + return true; } STATIC bool emit_native_last_emit_was_return_value(emit_t *emit) { @@ -1165,29 +1151,20 @@ STATIC exc_stack_entry_t *emit_native_pop_exc_stack(emit_t *emit) { return e; } -STATIC void emit_load_reg_with_ptr(emit_t *emit, int reg, mp_uint_t ptr, size_t table_off) { - if (!emit->do_viper_types) { - // Skip qstr names of arguments - table_off += emit->scope->num_pos_args + emit->scope->num_kwonly_args; - } - if (emit->pass == MP_PASS_EMIT) { - emit->const_table[table_off] = ptr; - } +STATIC void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj) { + emit->scope->scope_flags |= MP_SCOPE_FLAG_HASCONSTS; + size_t table_off = mp_emit_common_use_const_obj(emit->emit_common, obj); emit_native_mov_reg_state(emit, REG_TEMP0, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); ASM_LOAD_REG_REG_OFFSET(emit->as, reg, REG_TEMP0, table_off); } -STATIC void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj) { - // First entry is for mp_fun_table - size_t table_off = 1 + emit->const_table_cur_obj++; - emit_load_reg_with_ptr(emit, reg, (mp_uint_t)obj, table_off); -} - -STATIC void emit_load_reg_with_raw_code(emit_t *emit, int reg, mp_raw_code_t *rc) { - // First entry is for mp_fun_table, then constant objects - size_t table_off = 1 + emit->const_table_num_obj + emit->const_table_cur_raw_code++; - emit_load_reg_with_ptr(emit, reg, (mp_uint_t)rc, table_off); +STATIC void emit_load_reg_with_child(emit_t *emit, int reg, mp_raw_code_t *rc) { + size_t table_off = mp_emit_common_alloc_const_child(emit->emit_common, rc); + emit_native_mov_reg_state(emit, REG_TEMP0, LOCAL_IDX_FUN_OBJ(emit)); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CHILD_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, reg, REG_TEMP0, table_off); } STATIC void emit_native_label_assign(emit_t *emit, mp_uint_t l) { @@ -1231,7 +1208,8 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) { if (!(emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR)) { // Set new globals emit_native_mov_reg_state(emit, REG_ARG_1, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_OBJ_FUN_BC_GLOBALS); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_MODULE_CONTEXT_GLOBALS); emit_call(emit, MP_F_NATIVE_SWAP_GLOBALS); // Save old globals (or NULL if globals didn't change) @@ -1262,14 +1240,12 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) { // Wrap everything in an nlr context emit_native_label_assign(emit, nlr_label); - ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_2, LOCAL_IDX_EXC_HANDLER_UNWIND(emit)); ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 0); emit_call(emit, MP_F_NLR_PUSH); #if N_NLR_SETJMP ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 2); emit_call(emit, MP_F_SETJMP); #endif - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_UNWIND(emit), REG_LOCAL_2); ASM_JUMP_IF_REG_NONZERO(emit->as, REG_RET, global_except_label, true); // Clear PC of current code block, and jump there to resume execution @@ -1279,12 +1255,6 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) { // Global exception handler: check for valid exception handler emit_native_label_assign(emit, global_except_label); - #if N_NLR_SETJMP - // Reload REG_FUN_TABLE, since it may be clobbered by longjmp - emit_native_mov_reg_state(emit, REG_LOCAL_1, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_1, REG_LOCAL_1, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_1, emit->scope->num_pos_args + emit->scope->num_kwonly_args); - #endif ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_1, LOCAL_IDX_EXC_HANDLER_PC(emit)); ASM_JUMP_IF_REG_NONZERO(emit->as, REG_LOCAL_1, nlr_label, false); } @@ -1413,11 +1383,7 @@ STATIC void emit_native_import(emit_t *emit, qstr qst, int kind) { STATIC void emit_native_load_const_tok(emit_t *emit, mp_token_kind_t tok) { DEBUG_printf("load_const_tok(tok=%u)\n", tok); if (tok == MP_TOKEN_ELLIPSIS) { - #if MICROPY_PERSISTENT_CODE_SAVE emit_native_load_const_obj(emit, MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj)); - #else - emit_post_push_imm(emit, VTYPE_PYOBJ, (mp_uint_t)MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj)); - #endif } else { emit_native_pre(emit); if (tok == MP_TOKEN_KW_NONE) { @@ -1452,7 +1418,6 @@ STATIC void emit_native_load_const_str(emit_t *emit, qstr qst) { } STATIC void emit_native_load_const_obj(emit_t *emit, mp_obj_t obj) { - emit->scope->scope_flags |= MP_SCOPE_FLAG_HASCONSTS; emit_native_pre(emit); need_reg_single(emit, REG_RET, 0); emit_load_reg_with_object(emit, REG_RET, obj); @@ -1471,7 +1436,7 @@ STATIC void emit_native_load_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { EMIT_NATIVE_VIPER_TYPE_ERROR(emit, MP_ERROR_TEXT("local '%q' used before type known"), qst); } emit_native_pre(emit); - if (local_num < REG_LOCAL_NUM && CAN_USE_REGS_FOR_LOCALS(emit)) { + if (local_num < MAX_REGS_FOR_LOCAL_VARS && CAN_USE_REGS_FOR_LOCALS(emit)) { emit_post_push_reg(emit, vtype, reg_local_table[local_num]); } else { need_reg_single(emit, REG_TEMP0, 0); @@ -1692,7 +1657,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { STATIC void emit_native_store_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { vtype_kind_t vtype; - if (local_num < REG_LOCAL_NUM && CAN_USE_REGS_FOR_LOCALS(emit)) { + if (local_num < MAX_REGS_FOR_LOCAL_VARS && CAN_USE_REGS_FOR_LOCALS(emit)) { emit_pre_pop_reg(emit, &vtype, reg_local_table[local_num]); } else { emit_pre_pop_reg(emit, &vtype, REG_TEMP0); @@ -2485,48 +2450,48 @@ STATIC void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { asm_x86_setcc_r8(emit->as, ops[op_idx], REG_RET); #elif N_THUMB asm_thumb_cmp_rlo_rlo(emit->as, REG_ARG_2, reg_rhs); - #if MICROPY_EMIT_THUMB_ARMV7M - static uint16_t ops[6 + 6] = { - // unsigned - ASM_THUMB_OP_ITE_CC, - ASM_THUMB_OP_ITE_HI, - ASM_THUMB_OP_ITE_EQ, - ASM_THUMB_OP_ITE_LS, - ASM_THUMB_OP_ITE_CS, - ASM_THUMB_OP_ITE_NE, - // signed - ASM_THUMB_OP_ITE_LT, - ASM_THUMB_OP_ITE_GT, - ASM_THUMB_OP_ITE_EQ, - ASM_THUMB_OP_ITE_LE, - ASM_THUMB_OP_ITE_GE, - ASM_THUMB_OP_ITE_NE, - }; - asm_thumb_op16(emit->as, ops[op_idx]); - asm_thumb_mov_rlo_i8(emit->as, REG_RET, 1); - asm_thumb_mov_rlo_i8(emit->as, REG_RET, 0); - #else - static uint16_t ops[6 + 6] = { - // unsigned - ASM_THUMB_CC_CC, - ASM_THUMB_CC_HI, - ASM_THUMB_CC_EQ, - ASM_THUMB_CC_LS, - ASM_THUMB_CC_CS, - ASM_THUMB_CC_NE, - // signed - ASM_THUMB_CC_LT, - ASM_THUMB_CC_GT, - ASM_THUMB_CC_EQ, - ASM_THUMB_CC_LE, - ASM_THUMB_CC_GE, - ASM_THUMB_CC_NE, - }; - asm_thumb_bcc_rel9(emit->as, ops[op_idx], 6); - asm_thumb_mov_rlo_i8(emit->as, REG_RET, 0); - asm_thumb_b_rel12(emit->as, 4); - asm_thumb_mov_rlo_i8(emit->as, REG_RET, 1); - #endif + if (asm_thumb_allow_armv7m(emit->as)) { + static uint16_t ops[6 + 6] = { + // unsigned + ASM_THUMB_OP_ITE_CC, + ASM_THUMB_OP_ITE_HI, + ASM_THUMB_OP_ITE_EQ, + ASM_THUMB_OP_ITE_LS, + ASM_THUMB_OP_ITE_CS, + ASM_THUMB_OP_ITE_NE, + // signed + ASM_THUMB_OP_ITE_LT, + ASM_THUMB_OP_ITE_GT, + ASM_THUMB_OP_ITE_EQ, + ASM_THUMB_OP_ITE_LE, + ASM_THUMB_OP_ITE_GE, + ASM_THUMB_OP_ITE_NE, + }; + asm_thumb_op16(emit->as, ops[op_idx]); + asm_thumb_mov_rlo_i8(emit->as, REG_RET, 1); + asm_thumb_mov_rlo_i8(emit->as, REG_RET, 0); + } else { + static uint16_t ops[6 + 6] = { + // unsigned + ASM_THUMB_CC_CC, + ASM_THUMB_CC_HI, + ASM_THUMB_CC_EQ, + ASM_THUMB_CC_LS, + ASM_THUMB_CC_CS, + ASM_THUMB_CC_NE, + // signed + ASM_THUMB_CC_LT, + ASM_THUMB_CC_GT, + ASM_THUMB_CC_EQ, + ASM_THUMB_CC_LE, + ASM_THUMB_CC_GE, + ASM_THUMB_CC_NE, + }; + asm_thumb_bcc_rel9(emit->as, ops[op_idx], 6); + asm_thumb_mov_rlo_i8(emit->as, REG_RET, 0); + asm_thumb_b_rel12(emit->as, 4); + asm_thumb_mov_rlo_i8(emit->as, REG_RET, 1); + } #elif N_ARM asm_arm_cmp_reg_reg(emit->as, REG_ARG_2, reg_rhs); static uint ccs[6 + 6] = { @@ -2710,33 +2675,46 @@ STATIC void emit_native_unpack_ex(emit_t *emit, mp_uint_t n_left, mp_uint_t n_ri STATIC void emit_native_make_function(emit_t *emit, scope_t *scope, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { // call runtime, with type info for args, or don't support dict/default params, or only support Python objects for them emit_native_pre(emit); + emit_native_mov_reg_state(emit, REG_ARG_2, LOCAL_IDX_FUN_OBJ(emit)); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_2, REG_ARG_2, OFFSETOF_OBJ_FUN_BC_CONTEXT); if (n_pos_defaults == 0 && n_kw_defaults == 0) { need_reg_all(emit); - ASM_MOV_REG_IMM(emit->as, REG_ARG_2, (mp_uint_t)MP_OBJ_NULL); - ASM_MOV_REG_IMM(emit->as, REG_ARG_3, (mp_uint_t)MP_OBJ_NULL); + ASM_MOV_REG_IMM(emit->as, REG_ARG_3, 0); } else { - vtype_kind_t vtype_def_tuple, vtype_def_dict; - emit_pre_pop_reg_reg(emit, &vtype_def_dict, REG_ARG_3, &vtype_def_tuple, REG_ARG_2); - assert(vtype_def_tuple == VTYPE_PYOBJ); - assert(vtype_def_dict == VTYPE_PYOBJ); + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, 2); need_reg_all(emit); } - emit_load_reg_with_raw_code(emit, REG_ARG_1, scope->raw_code); + emit_load_reg_with_child(emit, REG_ARG_1, scope->raw_code); ASM_CALL_IND(emit->as, MP_F_MAKE_FUNCTION_FROM_RAW_CODE); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } STATIC void emit_native_make_closure(emit_t *emit, scope_t *scope, mp_uint_t n_closed_over, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { + // make function emit_native_pre(emit); + emit_native_mov_reg_state(emit, REG_ARG_2, LOCAL_IDX_FUN_OBJ(emit)); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_2, REG_ARG_2, OFFSETOF_OBJ_FUN_BC_CONTEXT); if (n_pos_defaults == 0 && n_kw_defaults == 0) { - emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over); - ASM_MOV_REG_IMM(emit->as, REG_ARG_2, n_closed_over); + need_reg_all(emit); + ASM_MOV_REG_IMM(emit->as, REG_ARG_3, 0); } else { - emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over + 2); - ASM_MOV_REG_IMM(emit->as, REG_ARG_2, 0x100 | n_closed_over); + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, 2 + n_closed_over); + adjust_stack(emit, 2 + n_closed_over); + need_reg_all(emit); } - emit_load_reg_with_raw_code(emit, REG_ARG_1, scope->raw_code); - ASM_CALL_IND(emit->as, MP_F_MAKE_CLOSURE_FROM_RAW_CODE); + emit_load_reg_with_child(emit, REG_ARG_1, scope->raw_code); + ASM_CALL_IND(emit->as, MP_F_MAKE_FUNCTION_FROM_RAW_CODE); + + // make closure + #if REG_ARG_1 != REG_RET + ASM_MOV_REG_REG(emit->as, REG_ARG_1, REG_RET); + #endif + ASM_MOV_REG_IMM(emit->as, REG_ARG_2, n_closed_over); + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over); + if (n_pos_defaults != 0 || n_kw_defaults != 0) { + adjust_stack(emit, -2); + } + ASM_CALL_IND(emit->as, MP_F_NEW_CLOSURE); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } @@ -2781,7 +2759,7 @@ STATIC void emit_native_call_function(emit_t *emit, mp_uint_t n_positional, mp_u } else { assert(vtype_fun == VTYPE_PYOBJ); if (star_flags) { - emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_positional + 2 * n_keyword + 3); // pointer to args + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_positional + 2 * n_keyword + 2); // pointer to args emit_call_with_2_imm_args(emit, MP_F_CALL_METHOD_N_KW_VAR, 0, REG_ARG_1, n_positional | (n_keyword << 8), REG_ARG_2); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } else { @@ -2797,7 +2775,7 @@ STATIC void emit_native_call_function(emit_t *emit, mp_uint_t n_positional, mp_u STATIC void emit_native_call_method(emit_t *emit, mp_uint_t n_positional, mp_uint_t n_keyword, mp_uint_t star_flags) { if (star_flags) { - emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_positional + 2 * n_keyword + 4); // pointer to args + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_positional + 2 * n_keyword + 3); // pointer to args emit_call_with_2_imm_args(emit, MP_F_CALL_METHOD_N_KW_VAR, 1, REG_ARG_1, n_positional | (n_keyword << 8), REG_ARG_2); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } else { diff --git a/py/emitnthumb.c b/py/emitnthumb.c index 1c33e7a68b..844a73ffa8 100644 --- a/py/emitnthumb.c +++ b/py/emitnthumb.c @@ -10,8 +10,6 @@ // Word indices of REG_LOCAL_x in nlr_buf_t #define NLR_BUF_IDX_LOCAL_1 (3) // r4 -#define NLR_BUF_IDX_LOCAL_2 (4) // r5 -#define NLR_BUF_IDX_LOCAL_3 (5) // r6 #define N_THUMB (1) #define EXPORT_FUN(name) emit_native_thumb_##name diff --git a/py/emitnx64.c b/py/emitnx64.c index 4abb3ecad3..1b32286d27 100644 --- a/py/emitnx64.c +++ b/py/emitnx64.c @@ -10,8 +10,6 @@ // Word indices of REG_LOCAL_x in nlr_buf_t #define NLR_BUF_IDX_LOCAL_1 (5) // rbx -#define NLR_BUF_IDX_LOCAL_2 (6) // r12 -#define NLR_BUF_IDX_LOCAL_3 (7) // r13 #define N_X64 (1) #define EXPORT_FUN(name) emit_native_x64_##name diff --git a/py/emitnx86.c b/py/emitnx86.c index f0553f0682..a9050c65d4 100644 --- a/py/emitnx86.c +++ b/py/emitnx86.c @@ -11,8 +11,6 @@ // Word indices of REG_LOCAL_x in nlr_buf_t #define NLR_BUF_IDX_LOCAL_1 (5) // ebx -#define NLR_BUF_IDX_LOCAL_2 (7) // esi -#define NLR_BUF_IDX_LOCAL_3 (6) // edi // x86 needs a table to know how many args a given function has STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = { @@ -56,7 +54,7 @@ STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = { [MP_F_UNPACK_EX] = 3, [MP_F_DELETE_NAME] = 1, [MP_F_DELETE_GLOBAL] = 1, - [MP_F_MAKE_CLOSURE_FROM_RAW_CODE] = 3, + [MP_F_NEW_CLOSURE] = 3, [MP_F_ARG_CHECK_NUM_SIG] = 3, [MP_F_SETUP_CODE_STATE] = 4, [MP_F_SMALL_INT_FLOOR_DIVIDE] = 2, diff --git a/py/emitnxtensa.c b/py/emitnxtensa.c index 34089e90dc..c89b029023 100644 --- a/py/emitnxtensa.c +++ b/py/emitnxtensa.c @@ -10,8 +10,6 @@ // Word indices of REG_LOCAL_x in nlr_buf_t #define NLR_BUF_IDX_LOCAL_1 (8) // a12 -#define NLR_BUF_IDX_LOCAL_2 (9) // a13 -#define NLR_BUF_IDX_LOCAL_3 (10) // a14 #define N_XTENSA (1) #define EXPORT_FUN(name) emit_native_xtensa_##name diff --git a/py/emitnxtensawin.c b/py/emitnxtensawin.c index 38d5db13ea..f6eeff8455 100644 --- a/py/emitnxtensawin.c +++ b/py/emitnxtensawin.c @@ -11,11 +11,8 @@ // Word indices of REG_LOCAL_x in nlr_buf_t #define NLR_BUF_IDX_LOCAL_1 (2 + 4) // a4 -#define NLR_BUF_IDX_LOCAL_2 (2 + 5) // a5 -#define NLR_BUF_IDX_LOCAL_3 (2 + 6) // a6 #define N_NLR_SETJMP (1) -#define N_PRELUDE_AS_BYTES_OBJ (1) #define N_XTENSAWIN (1) #define EXPORT_FUN(name) emit_native_xtensawin_##name #include "py/emitnative.c" diff --git a/py/formatfloat.c b/py/formatfloat.c index 4671f3348b..58e5406eeb 100644 --- a/py/formatfloat.c +++ b/py/formatfloat.c @@ -1,9 +1,11 @@ +// CIRCUITPY: https://github.com/adafruit/circuitpython/pull/6722 +// post v1.19.1. Remove this comment during v1.20 merge. /* * This file is part of the MicroPython project, http://micropython.org/ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/formatfloat.h b/py/formatfloat.h index c433cb8057..9a1643b4dd 100644 --- a/py/formatfloat.h +++ b/py/formatfloat.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/frozenmod.c b/py/frozenmod.c index 57e9d5df9b..61c2f20aa1 100644 --- a/py/frozenmod.c +++ b/py/frozenmod.c @@ -4,8 +4,8 @@ * The MIT License (MIT) * * Copyright (c) 2015 Paul Sokolovsky - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2021 Jim Mussared + * Copyright (c) 2016 Damien P. George + * Copyright (c) 2021 Jim Mussared * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -57,7 +57,7 @@ extern const char mp_frozen_str_content[]; #include "py/emitglue.h" -extern const mp_raw_code_t *const mp_frozen_mpy_content[]; +extern const mp_frozen_module_t *const mp_frozen_mpy_content[]; #endif // MICROPY_MODULE_FROZEN_MPY diff --git a/py/frozenmod.h b/py/frozenmod.h index 0a907b8785..cff6c8616c 100644 --- a/py/frozenmod.h +++ b/py/frozenmod.h @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2015 Paul Sokolovsky - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George + * Copyright (c) 2015 Paul Sokolovsky + * Copyright (c) 2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -27,7 +27,7 @@ #ifndef MICROPY_INCLUDED_PY_FROZENMOD_H #define MICROPY_INCLUDED_PY_FROZENMOD_H -#include "py/lexer.h" +#include "py/builtin.h" enum { MP_FROZEN_NONE, diff --git a/py/gc.c b/py/gc.c index d723272f04..3446f99f63 100644 --- a/py/gc.c +++ b/py/gc.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -412,6 +412,7 @@ void gc_collect_root(void **ptrs, size_t len) { for (size_t i = 0; i < len; i++) { MICROPY_GC_HOOK_LOOP void *ptr = gc_get_ptr(ptrs, i); + // CIRCUITPY changed i PR #1816 gc_mark(ptr); } } @@ -1149,13 +1150,13 @@ void gc_dump_alloc_table(void) { // This code prints "Q" for qstr-pool data, and "q" for qstr-str // data. It can be useful to see how qstrs are being allocated, // but is disabled by default because it is very slow. - for (qstr_pool_t *pool = MP_STATE_VM(last_pool); c == 'h' && pool != NULL; pool = pool->prev) { - if ((qstr_pool_t *)ptr == pool) { + for (const qstr_pool_t *pool = MP_STATE_VM(last_pool); c == 'h' && pool != NULL; pool = pool->prev) { + if ((const qstr_pool_t *)ptr == pool) { c = 'Q'; break; } - for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) { - if ((const byte *)ptr == *q) { + for (const char *const *q = pool->qstrs, *const *q_top = pool->qstrs + pool->len; q < q_top; q++) { + if ((const char *)ptr == *q) { c = 'q'; break; } diff --git a/py/gc.h b/py/gc.h index b9036ff4ba..25d5e52856 100644 --- a/py/gc.h +++ b/py/gc.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/grammar.h b/py/grammar.h index 3b4ceb8c60..285fbded2f 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2020 Damien P. George + * Copyright (c) 2013-2020 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/lexer.c b/py/lexer.c index 0d6de0d522..39e9662f63 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,8 +32,6 @@ #include "py/lexer.h" #include "py/runtime.h" -#include "supervisor/shared/translate/translate.h" - #if MICROPY_ENABLE_COMPILER #define TAB_SIZE (8) @@ -394,7 +392,6 @@ STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) if (is_char(lex, '\\')) { next_char(lex); unichar c = CUR_CHAR(lex); - if (is_raw) { // raw strings allow escaping of quotes, but the backslash is also emitted vstr_add_char(&lex->vstr, '\\'); @@ -476,25 +473,23 @@ STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) } } if (c != MP_LEXER_EOF) { - if (MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) { - if (c < 0x110000 && lex->tok_kind == MP_TOKEN_STRING) { - vstr_add_char(&lex->vstr, c); - } else if (c < 0x100 && lex->tok_kind == MP_TOKEN_BYTES) { - vstr_add_byte(&lex->vstr, c); - } else { - // unicode character out of range - // this raises a generic SyntaxError; could provide more info - lex->tok_kind = MP_TOKEN_INVALID; - } - } else { - // without unicode everything is just added as an 8-bit byte - if (c < 0x100) { - vstr_add_byte(&lex->vstr, c); - } else { - // 8-bit character out of range - // this raises a generic SyntaxError; could provide more info - lex->tok_kind = MP_TOKEN_INVALID; - } + #if MICROPY_PY_BUILTINS_STR_UNICODE + if (c < 0x110000 && lex->tok_kind == MP_TOKEN_STRING) { + // Valid unicode character in a str object. + vstr_add_char(&lex->vstr, c); + } else if (c < 0x100 && lex->tok_kind == MP_TOKEN_BYTES) { + // Valid byte in a bytes object. + vstr_add_byte(&lex->vstr, c); + } + #else + if (c < 0x100) { + // Without unicode everything is just added as an 8-bit byte. + vstr_add_byte(&lex->vstr, c); + } + #endif + else { + // Character out of range; this raises a generic SyntaxError. + lex->tok_kind = MP_TOKEN_INVALID; } } } else { @@ -678,6 +673,7 @@ void mp_lexer_to_next(mp_lexer_t *lex) { skip_whitespace(lex, true); } while (is_string_or_bytes(lex)); + } else if (is_head_of_identifier(lex)) { lex->tok_kind = MP_TOKEN_NAME; diff --git a/py/lexer.h b/py/lexer.h index bcc1c04a68..8295dec0f7 100644 --- a/py/lexer.h +++ b/py/lexer.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -189,24 +189,15 @@ typedef struct _mp_lexer_t { mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader); mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, size_t len, size_t free_len); -void mp_lexer_free(mp_lexer_t *lex); -void mp_lexer_to_next(mp_lexer_t *lex); - -/******************************************************************/ -// platform specific import function; must be implemented for a specific port -// TODO tidy up, rename, or put elsewhere - -typedef enum { - MP_IMPORT_STAT_NO_EXIST, - MP_IMPORT_STAT_DIR, - MP_IMPORT_STAT_FILE, -} mp_import_stat_t; - -mp_import_stat_t mp_import_stat(const char *path); +// If MICROPY_READER_POSIX or MICROPY_READER_VFS aren't enabled then +// this function must be implemented by the port. mp_lexer_t *mp_lexer_new_from_file(const char *filename); #if MICROPY_HELPER_LEXER_UNIX mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd); #endif +void mp_lexer_free(mp_lexer_t *lex); +void mp_lexer_to_next(mp_lexer_t *lex); + #endif // MICROPY_INCLUDED_PY_LEXER_H diff --git a/py/makemoduledefs.py b/py/makemoduledefs.py index aa2e5077ca..9061cd890b 100644 --- a/py/makemoduledefs.py +++ b/py/makemoduledefs.py @@ -1,89 +1,69 @@ -#!/usr/bin/env python - -# This pre-processor parses provided objects' c files for -# MP_REGISTER_MODULE(module_name, obj_module, enabled_define) -# These are used to generate a header with the required entries for -# "mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c +""" +This pre-processor parses a single file containing a list of +MP_REGISTER_MODULE(module_name, obj_module) +These are used to generate a header with the required entries for +"mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c +""" from __future__ import print_function +import sys import re import io -import os import argparse -pattern = re.compile(r"[\n;]\s*MP_REGISTER_MODULE\((.*?),\s*(.*?),\s*(.*?)\);", flags=re.DOTALL) +pattern = re.compile(r"\s*MP_REGISTER_MODULE\((.*?),\s*(.*?)\);", flags=re.DOTALL) -def find_c_file(obj_file, vpath): - """Search vpaths for the c file that matches the provided object_file. +def find_module_registrations(filename): + """Find any MP_REGISTER_MODULE definitions in the provided file. - :param str obj_file: object file to find the matching c file for - :param List[str] vpath: List of base paths, similar to gcc vpath - :return: str path to c file or None - """ - c_file = None - relative_c_file = os.path.splitext(obj_file)[0] + ".c" - relative_c_file = relative_c_file.lstrip("/\\") - for p in vpath: - possible_c_file = os.path.join(p, relative_c_file) - if os.path.exists(possible_c_file): - c_file = possible_c_file - break - - return c_file - - -def find_module_registrations(c_file): - """Find any MP_REGISTER_MODULE definitions in the provided c file. - - :param str c_file: path to c file to check - :return: List[(module_name, obj_module, enabled_define)] + :param str filename: path to file to check + :return: List[(module_name, obj_module)] """ global pattern - if c_file is None: - # No c file to match the object file, skip - return set() - - with io.open(c_file, encoding="utf-8") as c_file_obj: + with io.open(filename, encoding="utf-8") as c_file_obj: return set(re.findall(pattern, c_file_obj.read())) def generate_module_table_header(modules): """Generate header with module table entries for builtin modules. - :param List[(module_name, obj_module, enabled_define)] modules: module defs + :param List[(module_name, obj_module)] modules: module defs :return: None """ # Print header file for all external modules. - mod_defs = [] + mod_defs = set() print("// Automatically generated by makemoduledefs.py.\n") - print('#include "py/mpconfig.h"') - for module_name, obj_module, enabled_define in modules: + for module_name, obj_module in modules: mod_def = "MODULE_DEF_{}".format(module_name.upper()) - mod_defs.append(mod_def) + mod_defs.add(mod_def) + if "," in obj_module: + print( + "ERROR: Call to MP_REGISTER_MODULE({}, {}) should be MP_REGISTER_MODULE({}, {})\n".format( + module_name, obj_module, module_name, obj_module.split(",")[0] + ), + file=sys.stderr, + ) + sys.exit(1) print( ( - "#if ({enabled_define})\n" - " extern const struct _mp_obj_module_t {obj_module};\n" - " #define {mod_def} {{ MP_ROM_QSTR({module_name}), MP_ROM_PTR(&{obj_module}) }},\n" - "#else\n" - " #define {mod_def}\n" - "#endif\n" + "extern const struct _mp_obj_module_t {obj_module};\n" + "#undef {mod_def}\n" + "#define {mod_def} {{ MP_ROM_QSTR({module_name}), MP_ROM_PTR(&{obj_module}) }},\n" ).format( module_name=module_name, obj_module=obj_module, - enabled_define=enabled_define, mod_def=mod_def, ) ) print("\n#define MICROPY_REGISTERED_MODULES \\") - for mod_def in mod_defs: + for mod_def in sorted(mod_defs): print(" {mod_def} \\".format(mod_def=mod_def)) print("// MICROPY_REGISTERED_MODULES") @@ -91,19 +71,10 @@ def generate_module_table_header(modules): def main(): parser = argparse.ArgumentParser() - parser.add_argument( - "--vpath", default=".", help="comma separated list of folders to search for c files in" - ) - parser.add_argument("files", nargs="*", help="list of c files to search") + parser.add_argument("file", nargs=1, help="file with MP_REGISTER_MODULE definitions") args = parser.parse_args() - vpath = [p.strip() for p in args.vpath.split(",")] - - modules = set() - for obj_file in args.files: - c_file = find_c_file(obj_file, vpath) - modules |= find_module_registrations(c_file) - + modules = find_module_registrations(args.file[0]) generate_module_table_header(sorted(modules)) diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py index 974227f2a5..f8b5c82565 100644 --- a/py/makeqstrdefs.py +++ b/py/makeqstrdefs.py @@ -2,7 +2,7 @@ This script processes the output from the C preprocessor and extracts all qstr. Each qstr is transformed into a qstr definition of the form 'Q(...)'. -This script works with Python 2.6, 2.7, 3.3 and 3.4. +This script works with Python 3.x """ from __future__ import print_function @@ -14,20 +14,8 @@ import subprocess import sys import multiprocessing, multiprocessing.dummy -# Python 2/3 compatibility: -# - iterating through bytes is different -# - codepoint2name lives in a different module -import platform -if platform.python_version_tuple()[0] == "2": - bytes_cons = lambda val, enc=None: bytearray(val) - from htmlentitydefs import name2codepoint -elif platform.python_version_tuple()[0] == "3": - bytes_cons = bytes - from html.entities import name2codepoint - - unichr = chr -# end compatibility code +from html.entities import name2codepoint # Blocklist of qstrings that are specially handled in further # processing and should be ignored @@ -67,6 +55,22 @@ del name2codepoint["and"] del name2codepoint["or"] del name2codepoint["not"] +# Extract MP_QSTR_FOO macros. +_MODE_QSTR = "qstr" + +# Extract MP_COMPRESSED_ROM_TEXT("") macros. (Which come from MP_ERROR_TEXT) +_MODE_COMPRESS = "compress" +# Extract MP_REGISTER_MODULE(...) macros. +_MODE_MODULE = "module" + + +def is_c_source(fname): + return os.path.splitext(fname)[1] in [".c"] + + +def is_cxx_source(fname): + return os.path.splitext(fname)[1] in [".cc", ".cp", ".cxx", ".cpp", ".CPP", ".c++", ".C"] + def preprocess(): if any(src in args.dependencies for src in args.changed_sources): @@ -78,9 +82,9 @@ def preprocess(): csources = [] cxxsources = [] for source in sources: - if source.endswith(".cpp"): + if is_cxx_source(source): cxxsources.append(source) - elif source.endswith(".c"): + elif is_c_source(source): csources.append(source) try: os.makedirs(os.path.dirname(args.output[0])) @@ -113,7 +117,7 @@ def write_out(fname, output): if output: for m, r in [("/", "__"), ("\\", "__"), (":", "@"), ("..", "@@")]: fname = fname.replace(m, r) - with open(args.output_dir + "/" + fname + ".qstr", "w") as f: + with open(args.output_dir + "/" + fname + "." + args.mode, "w") as f: f.write("\n".join(output) + "\n") @@ -122,7 +126,7 @@ def qstr_unescape(qstr): if "__" + name + "__" in qstr: continue if "_" + name + "_" in qstr: - qstr = qstr.replace("_" + name + "_", str(unichr(name2codepoint[name]))) + qstr = qstr.replace("_" + name + "_", str(chr(name2codepoint[name]))) return qstr @@ -169,7 +173,7 @@ def cat_together(): hasher = hashlib.md5() all_lines = [] outf = open(args.output_dir + "/out", "wb") - for fname in glob.glob(args.output_dir + "/*.qstr"): + for fname in glob.glob(args.output_dir + "/*." + args.mode): with open(fname, "rb") as f: lines = f.readlines() all_lines += lines @@ -186,8 +190,13 @@ def cat_together(): old_hash = f.read() except IOError: pass + mode_full = "QSTR" + if args.mode == _MODE_COMPRESS: + mode_full = "Compressed data" + elif args.mode == _MODE_MODULE: + mode_full = "Module registrations" if old_hash != new_hash: - print("QSTR updated") + print(mode_full, "updated") try: # rename below might fail if file exists os.remove(args.output_file) @@ -197,12 +206,12 @@ def cat_together(): with open(args.output_file + ".hash", "w") as f: f.write(new_hash) else: - print("QSTR not updated") + print(mode_full, "not updated") if __name__ == "__main__": - if len(sys.argv) != 5: - print("usage: %s command input_filename output_dir output_file" % sys.argv[0]) + if len(sys.argv) < 6: + print("usage: %s command mode input_filename output_dir output_file" % sys.argv[0]) sys.exit(2) class Args: @@ -210,9 +219,45 @@ if __name__ == "__main__": args = Args() args.command = sys.argv[1] - args.input_filename = sys.argv[2] - args.output_dir = sys.argv[3] - args.output_file = sys.argv[4] + + if args.command == "pp": + named_args = { + s: [] + for s in [ + "pp", + "output", + "cflags", + "cxxflags", + "sources", + "changed_sources", + "dependencies", + ] + } + + for arg in sys.argv[1:]: + if arg in named_args: + current_tok = arg + else: + named_args[current_tok].append(arg) + + if not named_args["pp"] or len(named_args["output"]) != 1: + print("usage: %s %s ..." % (sys.argv[0], " ... ".join(named_args))) + sys.exit(2) + + for k, v in named_args.items(): + setattr(args, k, v) + + preprocess() + sys.exit(0) + + args.mode = sys.argv[2] + args.input_filename = sys.argv[3] # Unused for command=cat + args.output_dir = sys.argv[4] + args.output_file = None if len(sys.argv) == 5 else sys.argv[5] # Unused for command=split + + if args.mode not in (_MODE_QSTR, _MODE_COMPRESS, _MODE_MODULE): + print("error: mode %s unrecognised" % sys.argv[2]) + sys.exit(2) try: os.makedirs(args.output_dir) diff --git a/py/malloc.c b/py/malloc.c index 7200285687..c60a5a541b 100644 --- a/py/malloc.c +++ b/py/malloc.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -192,10 +192,11 @@ void *m_realloc_maybe(void *ptr, size_t new_num_bytes, bool allow_move) { } #if MICROPY_MALLOC_USES_ALLOCATED_SIZE -void m_free(void *ptr, size_t num_bytes) { +void m_free(void *ptr, size_t num_bytes) #else -void m_free(void *ptr) { - #endif +void m_free(void *ptr) +#endif +{ free(ptr); #if MICROPY_MEM_STATS MP_STATE_MEM(current_bytes_allocated) -= num_bytes; @@ -239,7 +240,7 @@ STATIC size_t m_tracked_count_links(size_t *nb) { #endif void *m_tracked_calloc(size_t nmemb, size_t size) { - m_tracked_node_t *node = m_malloc_maybe(sizeof(m_tracked_node_t) + nmemb * size, false); + m_tracked_node_t *node = m_malloc_maybe(sizeof(m_tracked_node_t) + nmemb * size); if (node == NULL) { return NULL; } @@ -298,7 +299,7 @@ void m_tracked_free(void *ptr_in) { ); } -#endif +#endif // MICROPY_TRACKED_ALLOC #if MICROPY_MEM_STATS size_t m_get_total_bytes_allocated(void) { diff --git a/py/map.c b/py/map.c index 092adf94ee..b194250cb4 100644 --- a/py/map.c +++ b/py/map.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,8 +33,6 @@ #include "py/misc.h" #include "py/runtime.h" -#include "supervisor/linker.h" - #if MICROPY_DEBUG_VERBOSE // print debugging info #define DEBUG_PRINT (1) #else // don't print debugging info diff --git a/py/misc.h b/py/misc.h index 1ced7c622e..d0e6c3c303 100644 --- a/py/misc.h +++ b/py/misc.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/mkrules.cmake b/py/mkrules.cmake index cb5fdabf6b..d0dc019625 100644 --- a/py/mkrules.cmake +++ b/py/mkrules.cmake @@ -2,13 +2,15 @@ set(MICROPY_GENHDR_DIR "${CMAKE_BINARY_DIR}/genhdr") set(MICROPY_MPVERSION "${MICROPY_GENHDR_DIR}/mpversion.h") -set(MICROPY_MODULEDEFS "${MICROPY_GENHDR_DIR}/moduledefs.h") set(MICROPY_QSTRDEFS_PY "${MICROPY_PY_DIR}/qstrdefs.h") set(MICROPY_QSTRDEFS_LAST "${MICROPY_GENHDR_DIR}/qstr.i.last") set(MICROPY_QSTRDEFS_SPLIT "${MICROPY_GENHDR_DIR}/qstr.split") set(MICROPY_QSTRDEFS_COLLECTED "${MICROPY_GENHDR_DIR}/qstrdefs.collected.h") set(MICROPY_QSTRDEFS_PREPROCESSED "${MICROPY_GENHDR_DIR}/qstrdefs.preprocessed.h") set(MICROPY_QSTRDEFS_GENERATED "${MICROPY_GENHDR_DIR}/qstrdefs.generated.h") +set(MICROPY_MODULEDEFS_SPLIT "${MICROPY_GENHDR_DIR}/moduledefs.split") +set(MICROPY_MODULEDEFS_COLLECTED "${MICROPY_GENHDR_DIR}/moduledefs.collected") +set(MICROPY_MODULEDEFS "${MICROPY_GENHDR_DIR}/moduledefs.h") # Need to do this before extracting MICROPY_CPP_DEF below. Rest of frozen # manifest handling is at the end of this file. @@ -43,6 +45,7 @@ find_package(Python3 REQUIRED COMPONENTS Interpreter) target_sources(${MICROPY_TARGET} PRIVATE ${MICROPY_MPVERSION} ${MICROPY_QSTRDEFS_GENERATED} + ${MICROPY_MODULEDEFS} ) # Command to force the build of another command @@ -62,15 +65,6 @@ add_custom_command( DEPENDS MICROPY_FORCE_BUILD ) -# Generate moduledefs.h - -add_custom_command( - OUTPUT ${MICROPY_MODULEDEFS} - COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makemoduledefs.py --vpath="/" ${MICROPY_SOURCE_QSTR} > ${MICROPY_MODULEDEFS} - DEPENDS ${MICROPY_MPVERSION} - ${MICROPY_SOURCE_QSTR} -) - # Generate qstrs # If any of the dependencies in this rule change then the C-preprocessor step must be run. @@ -79,7 +73,7 @@ add_custom_command( add_custom_command( OUTPUT ${MICROPY_QSTRDEFS_LAST} COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py pp ${CMAKE_C_COMPILER} -E output ${MICROPY_GENHDR_DIR}/qstr.i.last cflags ${MICROPY_CPP_FLAGS} -DNO_QSTR cxxflags ${MICROPY_CPP_FLAGS} -DNO_QSTR sources ${MICROPY_SOURCE_QSTR} - DEPENDS ${MICROPY_MODULEDEFS} + DEPENDS ${MICROPY_MPVERSION} ${MICROPY_SOURCE_QSTR} VERBATIM COMMAND_EXPAND_LISTS @@ -120,6 +114,31 @@ add_custom_command( COMMAND_EXPAND_LISTS ) +# Generate moduledefs.h + +add_custom_command( + OUTPUT ${MICROPY_MODULEDEFS_SPLIT} + COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py split module ${MICROPY_GENHDR_DIR}/qstr.i.last ${MICROPY_GENHDR_DIR}/module _ + COMMAND touch ${MICROPY_MODULEDEFS_SPLIT} + DEPENDS ${MICROPY_QSTRDEFS_LAST} + VERBATIM + COMMAND_EXPAND_LISTS +) + +add_custom_command( + OUTPUT ${MICROPY_MODULEDEFS_COLLECTED} + COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py cat module _ ${MICROPY_GENHDR_DIR}/module ${MICROPY_MODULEDEFS_COLLECTED} + DEPENDS ${MICROPY_MODULEDEFS_SPLIT} + VERBATIM + COMMAND_EXPAND_LISTS +) + +add_custom_command( + OUTPUT ${MICROPY_MODULEDEFS} + COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makemoduledefs.py ${MICROPY_MODULEDEFS_COLLECTED} > ${MICROPY_MODULEDEFS} + DEPENDS ${MICROPY_MODULEDEFS_COLLECTED} +) + # Build frozen code if enabled if(MICROPY_FROZEN_MANIFEST) @@ -159,3 +178,10 @@ if(MICROPY_FROZEN_MANIFEST) VERBATIM ) endif() + +# Update submodules +if(ECHO_SUBMODULES) + # If cmake is run with GIT_SUBMODULES defined on command line, process the port / board + # settings then print the final GIT_SUBMODULES variable as a fatal error and exit. + message(FATAL_ERROR "GIT_SUBMODULES=${GIT_SUBMODULES}") +endif() diff --git a/py/mkrules.mk b/py/mkrules.mk index 09ed82ac5b..74ab8dd78a 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -7,6 +7,24 @@ endif # Extra deps that need to happen before object compilation. OBJ_EXTRA_ORDER_DEPS = +# Generate moduledefs.h. +OBJ_EXTRA_ORDER_DEPS += $(HEADER_BUILD)/moduledefs.h + +ifeq ($(MICROPY_ROM_TEXT_COMPRESSION),1) +# If compression is enabled, trigger the build of compressed.data.h... +OBJ_EXTRA_ORDER_DEPS += $(HEADER_BUILD)/compressed.data.h +# ...and enable the MP_COMPRESSED_ROM_TEXT macro (used by MP_ERROR_TEXT). +CFLAGS += -DMICROPY_ROM_TEXT_COMPRESSION=1 +endif + +# QSTR generation uses the same CFLAGS, with these modifications. +QSTR_GEN_FLAGS = -DNO_QSTR +# Note: := to force evalulation immediately. +QSTR_GEN_CFLAGS := $(CFLAGS) +QSTR_GEN_CFLAGS += $(QSTR_GEN_FLAGS) +QSTR_GEN_CXXFLAGS := $(CXXFLAGS) +QSTR_GEN_CXXFLAGS += $(QSTR_GEN_FLAGS) + # This file expects that OBJ contains a list of all of the object files. # The directory portion of each object file is used to locate the source # and should not contain any ..'s but rather be relative to the top of the @@ -57,12 +75,12 @@ $(Q)$(CXX) $(CXXFLAGS) -c -MD -o $@ $< $(RM) -f $(@:.o=.d) endef -vpath %.c . $(TOP) $(USER_C_MODULES) $(DEVICES_MODULES) -$(BUILD)/%.o: %.c | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/qstrdefs.enum.h +vpath %.c . $(TOP) $(USER_C_MODULES) +$(BUILD)/%.o: %.c $(call compile_c) vpath %.cpp . $(TOP) $(USER_C_MODULES) -$(BUILD)/%.o: %.cpp | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/qstrdefs.enum.h +$(BUILD)/%.o: %.cpp $(call compile_cxx) QSTR_GEN_EXTRA_CFLAGS += -DNO_QSTR -x c @@ -77,8 +95,8 @@ QSTR_GEN_EXTRA_CFLAGS += -I$(BUILD)/tmp vpath %.c . $(TOP) $(USER_C_MODULES) $(DEVICES_MODULES) $(BUILD)/%.pp: %.c - $(STEPECHO) "PreProcess $<" - $(Q)$(CPP) $(CFLAGS) -E -Wp,-C,-dD,-dI -o $@ $< + $(ECHO) "PreProcess $<" + $(Q)$(CPP) $(CFLAGS) -Wp,-C,-dD,-dI -o $@ $< # The following rule uses | to create an order only prerequisite. Order only # prerequisites only get built if they don't exist. They don't cause timestamp @@ -89,20 +107,45 @@ $(BUILD)/%.pp: %.c # the right .o's to get recompiled if the generated.h file changes. Adding # an order-only dependency to all of the .o's will cause the generated .h # to get built before we try to compile any of them. -$(OBJ): | $(HEADER_BUILD)/mpversion.h +$(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h $(OBJ_EXTRA_ORDER_DEPS) # The logic for qstr regeneration (applied by makeqstrdefs.py) is: # - if anything in QSTR_GLOBAL_DEPENDENCIES is newer, then process all source files ($^) # - else, if list of newer prerequisites ($?) is not empty, then process just these ($?) # - else, process all source files ($^) [this covers "make -B" which can set $? to empty] -$(HEADER_BUILD)/qstr.split: $(SRC_QSTR) $(SRC_QSTR_PREPROCESSOR) $(QSTR_GLOBAL_DEPENDENCIES) $(HEADER_BUILD)/moduledefs.h | $(HEADER_BUILD)/mpversion.h $(PY_SRC)/genlast.py +# See more information about this process in docs/develop/qstr.rst. +$(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(QSTR_GLOBAL_REQUIREMENTS) $(STEPECHO) "GEN $@" - $(Q)$(PYTHON) $(PY_SRC)/genlast.py $(HEADER_BUILD)/qstr $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) -- $(SRC_QSTR_PREPROCESSOR) -- $(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py pp $(CPP) output $(HEADER_BUILD)/qstr.i.last cflags $(QSTR_GEN_CFLAGS) cxxflags $(QSTR_GEN_CXXFLAGS) sources $^ dependencies $(QSTR_GLOBAL_DEPENDENCIES) changed_sources $? + +$(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last + $(ECHO) "GEN $@" + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split qstr $< $(HEADER_BUILD)/qstr _ $(Q)$(TOUCH) $@ -$(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.split $(PY_SRC)/makeqstrdefs.py +$(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.split $(STEPECHO) "GEN $@" - $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat - $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED) + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat qstr _ $(HEADER_BUILD)/qstr $@ + +# Module definitions via MP_REGISTER_MODULE. +$(HEADER_BUILD)/moduledefs.split: $(HEADER_BUILD)/qstr.i.last + $(STEPECHO) "GEN $@" + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split module $< $(HEADER_BUILD)/module _ + $(Q)$(TOUCH) $@ + +$(HEADER_BUILD)/moduledefs.collected: $(HEADER_BUILD)/moduledefs.split + $(STEPECHO) "GEN $@" + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat module _ $(HEADER_BUILD)/module $@ + +# Compressed error strings. +$(HEADER_BUILD)/compressed.split: $(HEADER_BUILD)/qstr.i.last + $(STEPECHO) "GEN $@" + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split compress $< $(HEADER_BUILD)/compress _ + $(Q)$(TOUCH) $@ + +$(HEADER_BUILD)/compressed.collected: $(HEADER_BUILD)/compressed.split + $(STEPECHO) "GEN $@" + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat compress _ $(HEADER_BUILD)/compress $@ # $(sort $(var)) removes duplicates # @@ -117,8 +160,6 @@ $(OBJ_DIRS): $(HEADER_BUILD): $(Q)$(MKDIR) -p $@ - $(MKDIR) -p $@ - ifneq ($(MICROPY_MPYCROSS_DEPENDENCY),) # to automatically build mpy-cross, if needed $(MICROPY_MPYCROSS_DEPENDENCY): @@ -135,7 +176,7 @@ endif ifneq ($(FROZEN_MANIFEST),) # to build frozen_content.c from a manifest -$(BUILD)/frozen_content.c: FORCE $(FROZEN_MANIFEST) $(BUILD)/genhdr/qstrdefs.generated.h | $(MICROPY_MPYCROSS_DEPENDENCY) $(TOP)/tools/makemanifest.py +$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h | $(MICROPY_MPYCROSS_DEPENDENCY) $(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) --mpy-tool-flags="$(MPY_TOOL_FLAGS)" $(FROZEN_MANIFEST) endif diff --git a/py/modarray.c b/py/modarray.c index daae34662b..cfed0fbb59 100644 --- a/py/modarray.c +++ b/py/modarray.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,6 +40,6 @@ const mp_obj_module_t mp_module_array = { .globals = (mp_obj_dict_t *)&mp_module_array_globals, }; -MP_REGISTER_MODULE(MP_QSTR_array, mp_module_array, MICROPY_PY_ARRAY); +MP_REGISTER_MODULE(MP_QSTR_array, mp_module_array); #endif diff --git a/py/modbuiltins.c b/py/modbuiltins.c index 072ba8675f..870b19d7b2 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -190,6 +190,7 @@ STATIC mp_obj_t mp_builtin_dir(size_t n_args, const mp_obj_t *args) { // Implemented by probing all possible qstrs with mp_load_method_maybe size_t nqstr = QSTR_TOTAL(); for (size_t i = MP_QSTR_ + 1; i < nqstr; ++i) { + // CIRCUITPY changes #6539 mp_obj_t dest[2] = {}; mp_load_method_protected(args[0], i, dest, true); if (dest[0] != MP_OBJ_NULL) { @@ -484,13 +485,13 @@ STATIC mp_obj_t mp_builtin_round(size_t n_args, const mp_obj_t *args) { return o_in; } + #if !MICROPY_PY_BUILTINS_ROUND_INT + mp_raise_NotImplementedError(NULL); + #else mp_int_t num_dig = mp_obj_get_int(args[1]); if (num_dig >= 0) { return o_in; } - #if !MICROPY_PY_BUILTINS_ROUND_INT - mp_raise_NotImplementedError(NULL); - #else mp_obj_t mult = mp_binary_op(MP_BINARY_OP_POWER, MP_OBJ_NEW_SMALL_INT(10), MP_OBJ_NEW_SMALL_INT(-num_dig)); mp_obj_t half_mult = mp_binary_op(MP_BINARY_OP_FLOOR_DIVIDE, mult, MP_OBJ_NEW_SMALL_INT(2)); @@ -735,6 +736,9 @@ STATIC const mp_rom_map_elem_t mp_module_builtins_globals_table[] = { #endif { MP_ROM_QSTR(MP_QSTR_next), MP_ROM_PTR(&mp_builtin_next_obj) }, { MP_ROM_QSTR(MP_QSTR_oct), MP_ROM_PTR(&mp_builtin_oct_obj) }, + #if MICROPY_PY_IO + { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&mp_builtin_open_obj) }, + #endif { MP_ROM_QSTR(MP_QSTR_ord), MP_ROM_PTR(&mp_builtin_ord_obj) }, { MP_ROM_QSTR(MP_QSTR_pow), MP_ROM_PTR(&mp_builtin_pow_obj) }, { MP_ROM_QSTR(MP_QSTR_print), MP_ROM_PTR(&mp_builtin_print_obj) }, @@ -794,3 +798,5 @@ const mp_obj_module_t mp_module_builtins = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&mp_module_builtins_globals, }; + +MP_REGISTER_MODULE(MP_QSTR_builtins, mp_module_builtins); diff --git a/py/modcmath.c b/py/modcmath.c index a361ab53b4..1418362ad9 100644 --- a/py/modcmath.c +++ b/py/modcmath.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -149,4 +149,6 @@ const mp_obj_module_t mp_module_cmath = { .globals = (mp_obj_dict_t *)&mp_module_cmath_globals, }; -#endif // MICROPY_PY_BUILTINS_FLOAT && MICROPY_PY_CMATH +MP_REGISTER_MODULE(MP_QSTR_cmath, mp_module_cmath); + +#endif // MICROPY_PY_BUILTINS_FLOAT && MICROPY_PY_BUILTINS_COMPLEX && MICROPY_PY_CMATH diff --git a/py/modcollections.c b/py/modcollections.c index 235745f584..a56fe069ea 100644 --- a/py/modcollections.c +++ b/py/modcollections.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -46,4 +46,6 @@ const mp_obj_module_t mp_module_collections = { .globals = (mp_obj_dict_t *)&mp_module_collections_globals, }; +MP_REGISTER_MODULE(MP_QSTR_collections, mp_module_collections); + #endif // MICROPY_PY_COLLECTIONS diff --git a/py/modgc.c b/py/modgc.c index e655cfa93e..c11bcaecd7 100644 --- a/py/modgc.c +++ b/py/modgc.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -115,4 +115,6 @@ const mp_obj_module_t mp_module_gc = { .globals = (mp_obj_dict_t *)&mp_module_gc_globals, }; +MP_REGISTER_MODULE(MP_QSTR_gc, mp_module_gc); + #endif diff --git a/py/modio.c b/py/modio.c index 819a9976a0..836d04fb11 100644 --- a/py/modio.c +++ b/py/modio.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -125,8 +125,7 @@ typedef struct _mp_obj_bufwriter_t { STATIC mp_obj_t bufwriter_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 2, 2, false); size_t alloc = mp_obj_get_int(args[1]); - mp_obj_bufwriter_t *o = m_new_obj_var(mp_obj_bufwriter_t, byte, alloc); - o->base.type = type; + mp_obj_bufwriter_t *o = mp_obj_malloc_var(mp_obj_bufwriter_t, byte, alloc, type); o->stream = args[0]; o->alloc = alloc; o->len = 0; @@ -246,4 +245,6 @@ const mp_obj_module_t mp_module_io = { .globals = (mp_obj_dict_t *)&mp_module_io_globals, }; +MP_REGISTER_MODULE(MP_QSTR_uio, mp_module_io); + #endif diff --git a/py/modmath.c b/py/modmath.c index bf27e68eaf..0567e5dd94 100644 --- a/py/modmath.c +++ b/py/modmath.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2017 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -373,6 +373,11 @@ STATIC const mp_rom_map_elem_t mp_module_math_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_math) }, { MP_ROM_QSTR(MP_QSTR_e), mp_const_float_e }, { MP_ROM_QSTR(MP_QSTR_pi), mp_const_float_pi }, + #if MICROPY_PY_MATH_CONSTANTS + { MP_ROM_QSTR(MP_QSTR_tau), mp_const_float_tau }, + { MP_ROM_QSTR(MP_QSTR_inf), mp_const_float_inf }, + { MP_ROM_QSTR(MP_QSTR_nan), mp_const_float_nan }, + #endif { MP_ROM_QSTR(MP_QSTR_sqrt), MP_ROM_PTR(&mp_math_sqrt_obj) }, { MP_ROM_QSTR(MP_QSTR_pow), MP_ROM_PTR(&mp_math_pow_obj) }, { MP_ROM_QSTR(MP_QSTR_exp), MP_ROM_PTR(&mp_math_exp_obj) }, @@ -432,4 +437,6 @@ const mp_obj_module_t mp_module_math = { .globals = (mp_obj_dict_t *)&mp_module_math_globals, }; +MP_REGISTER_MODULE(MP_QSTR_math, mp_module_math); + #endif // MICROPY_PY_BUILTINS_FLOAT && MICROPY_PY_MATH diff --git a/py/modmicropython.c b/py/modmicropython.c index 42c84e5292..ee4d57d94b 100644 --- a/py/modmicropython.c +++ b/py/modmicropython.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -211,3 +211,5 @@ const mp_obj_module_t mp_module_micropython = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&mp_module_micropython_globals, }; + +MP_REGISTER_MODULE(MP_QSTR_micropython, mp_module_micropython); diff --git a/py/modstruct.c b/py/modstruct.c index 17dfc548ab..a2dfa91655 100644 --- a/py/modstruct.c +++ b/py/modstruct.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * Copyright (c) 2014 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -33,6 +33,7 @@ #include "py/objtuple.h" #include "py/binary.h" #include "py/parsenum.h" + #include "supervisor/shared/translate/translate.h" #if MICROPY_PY_STRUCT @@ -282,4 +283,6 @@ const mp_obj_module_t mp_module_ustruct = { .globals = (mp_obj_dict_t *)&mp_module_struct_globals, }; +MP_REGISTER_MODULE(MP_QSTR_struct, mp_module_ustruct); + #endif diff --git a/py/modsys.c b/py/modsys.c index 0ec5de1645..92779914b0 100644 --- a/py/modsys.c +++ b/py/modsys.c @@ -27,6 +27,7 @@ #include "py/builtin.h" #include "py/objlist.h" +#include "py/objmodule.h" #include "py/objtuple.h" #include "py/objstr.h" #include "py/objint.h" @@ -35,6 +36,7 @@ #include "py/smallint.h" #include "py/runtime.h" #include "py/persistentcode.h" +#include "genhdr/mpversion.h" #if MICROPY_PY_SYS_SETTRACE #include "py/objmodule.h" @@ -84,7 +86,7 @@ STATIC const qstr impl_fields[] = { MP_QSTR_name, MP_QSTR_version, #if MICROPY_PERSISTENT_CODE_LOAD - MP_QSTR_mpy, + MP_QSTR__mpy, #endif }; STATIC MP_DEFINE_ATTRTUPLE( @@ -120,6 +122,25 @@ STATIC mp_obj_t mp_sys_exit(size_t n_args, const mp_obj_t *args) { } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_sys_exit_obj, 0, 1, mp_sys_exit); +STATIC mp_obj_t mp_sys_print_exception(size_t n_args, const mp_obj_t *args) { + #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES + void *stream_obj = &mp_sys_stdout_obj; + if (n_args > 1) { + mp_get_stream_raise(args[1], MP_STREAM_OP_WRITE); + stream_obj = MP_OBJ_TO_PTR(args[1]); + } + + mp_print_t print = {stream_obj, mp_stream_write_adaptor}; + mp_obj_print_exception(&print, args[0]); + #else + (void)n_args; + mp_obj_print_exception(&mp_plat_print, args[0]); + #endif + + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_sys_print_exception_obj, 1, 2, mp_sys_print_exception); + #if MICROPY_PY_SYS_EXC_INFO STATIC mp_obj_t mp_sys_exc_info(void) { mp_obj_t cur_exc = MP_OBJ_FROM_PTR(MP_STATE_VM(cur_exception)); @@ -165,6 +186,25 @@ STATIC mp_obj_t mp_sys_settrace(mp_obj_t obj) { MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_settrace_obj, mp_sys_settrace); #endif // MICROPY_PY_SYS_SETTRACE +#if MICROPY_PY_SYS_ATTR_DELEGATION +STATIC const uint16_t sys_mutable_keys[] = { + #if MICROPY_PY_SYS_PS1_PS2 + MP_QSTR_ps1, + MP_QSTR_ps2, + #endif + #if MICROPY_PY_SYS_TRACEBACKLIMIT + MP_QSTR_tracebacklimit, + #endif + MP_QSTRnull, +}; + +STATIC void mp_module_sys_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + MP_STATIC_ASSERT(MP_ARRAY_SIZE(sys_mutable_keys) == MP_SYS_MUTABLE_NUM + 1); + MP_STATIC_ASSERT(MP_ARRAY_SIZE(MP_STATE_VM(sys_mutable)) == MP_SYS_MUTABLE_NUM); + mp_module_generic_attr(attr, dest, sys_mutable_keys, MP_STATE_VM(sys_mutable)); +} +#endif + STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_sys) }, @@ -223,9 +263,15 @@ STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = { * Extensions to CPython */ + { MP_ROM_QSTR(MP_QSTR_print_exception), MP_ROM_PTR(&mp_sys_print_exception_obj) }, #if MICROPY_PY_SYS_ATEXIT { MP_ROM_QSTR(MP_QSTR_atexit), MP_ROM_PTR(&mp_sys_atexit_obj) }, #endif + + #if MICROPY_PY_SYS_ATTR_DELEGATION + // Delegation of attr lookup. + MP_MODULE_ATTR_DELEGATION_ENTRY(&mp_module_sys_attr), + #endif }; STATIC MP_DEFINE_CONST_DICT(mp_module_sys_globals, mp_module_sys_globals_table); @@ -235,4 +281,6 @@ const mp_obj_module_t mp_module_sys = { .globals = (mp_obj_dict_t *)&mp_module_sys_globals, }; +MP_REGISTER_MODULE(MP_QSTR_usys, mp_module_sys); + #endif diff --git a/py/modthread.c b/py/modthread.c index a3960cddcb..bad94fbf2f 100644 --- a/py/modthread.c +++ b/py/modthread.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd + * Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -30,8 +30,6 @@ #include "py/runtime.h" #include "py/stackctrl.h" -#include "supervisor/shared/translate/translate.h" - #if MICROPY_PY_THREAD #include "py/mpthread.h" @@ -56,8 +54,7 @@ typedef struct _mp_obj_thread_lock_t { } mp_obj_thread_lock_t; STATIC mp_obj_thread_lock_t *mp_obj_new_thread_lock(void) { - mp_obj_thread_lock_t *self = m_new_obj(mp_obj_thread_lock_t); - self->base.type = &mp_type_thread_lock; + mp_obj_thread_lock_t *self = mp_obj_malloc(mp_obj_thread_lock_t, &mp_type_thread_lock); mp_thread_mutex_init(&self->mutex); self->locked = false; return self; @@ -303,4 +300,6 @@ const mp_obj_module_t mp_module_thread = { .globals = (mp_obj_dict_t *)&mp_module_thread_globals, }; +MP_REGISTER_MODULE(MP_QSTR__thread, mp_module_thread); + #endif // MICROPY_PY_THREAD diff --git a/py/moduerrno.c b/py/moduerrno.c index 0ac0503ccc..bbfca93875 100644 --- a/py/moduerrno.c +++ b/py/moduerrno.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George + * Copyright (c) 2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,6 +32,8 @@ #include "supervisor/shared/translate/translate.h" +#if MICROPY_PY_UERRNO + // This list can be defined per port in mpconfigport.h to tailor it to a // specific port's needs. If it's not defined then we provide a default. #ifndef MICROPY_PY_UERRNO_LIST @@ -61,8 +63,6 @@ #endif -#if MICROPY_PY_UERRNO - #if MICROPY_PY_UERRNO_ERRORCODE STATIC const mp_rom_map_elem_t errorcode_table[] = { #define X(e) { MP_ROM_INT(MP_##e), MP_ROM_QSTR(MP_QSTR_##e) }, @@ -105,7 +105,7 @@ const mp_obj_module_t mp_module_uerrno = { .globals = (mp_obj_dict_t *)&mp_module_uerrno_globals, }; -MP_REGISTER_MODULE(MP_QSTR_errno, mp_module_uerrno, MICROPY_PY_UERRNO); +MP_REGISTER_MODULE(MP_QSTR_errno, mp_module_uerrno); qstr mp_errno_to_str(mp_obj_t errno_val) { // Otherwise, return the Exxxx string for that error code @@ -129,52 +129,3 @@ qstr mp_errno_to_str(mp_obj_t errno_val) { } #endif // MICROPY_PY_UERRNO - - -// For commonly encountered errors, return human readable strings, otherwise try errno name -const char *mp_common_errno_to_str(mp_obj_t errno_val, char *buf, size_t len) { - if (!mp_obj_is_small_int(errno_val)) { - return NULL; - } - - const compressed_string_t *desc = NULL; - switch (MP_OBJ_SMALL_INT_VALUE(errno_val)) { - case EPERM: - desc = MP_ERROR_TEXT("Operation not permitted"); - break; - case ENOENT: - desc = MP_ERROR_TEXT("No such file/directory"); - break; - case EIO: - desc = MP_ERROR_TEXT("Input/output error"); - break; - case EACCES: - desc = MP_ERROR_TEXT("Permission denied"); - break; - case EEXIST: - desc = MP_ERROR_TEXT("File exists"); - break; - case ENODEV: - desc = MP_ERROR_TEXT("No such device"); - break; - case EINVAL: - desc = MP_ERROR_TEXT("Invalid argument"); - break; - case ENOSPC: - desc = MP_ERROR_TEXT("No space left on device"); - break; - case EROFS: - desc = MP_ERROR_TEXT("Read-only filesystem"); - break; - } - if (desc != NULL && decompress_length(desc) <= len) { - decompress(desc, buf); - return buf; - } - - const char *msg = ""; - #if MICROPY_PY_UERRNO - msg = qstr_str(mp_errno_to_str(errno_val)); - #endif - return msg[0] != '\0' ? msg : NULL; -} diff --git a/py/mpconfig.h b/py/mpconfig.h index d3842afbe7..db349e451f 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,23 @@ #ifndef MICROPY_INCLUDED_PY_MPCONFIG_H #define MICROPY_INCLUDED_PY_MPCONFIG_H +// Current version of MicroPython +#define MICROPY_VERSION_MAJOR 1 +#define MICROPY_VERSION_MINOR 19 +#define MICROPY_VERSION_MICRO 1 + +// Combined version as a 32-bit number for convenience +#define MICROPY_VERSION ( \ + MICROPY_VERSION_MAJOR << 16 \ + | MICROPY_VERSION_MINOR << 8 \ + | MICROPY_VERSION_MICRO) + +// String version +#define MICROPY_VERSION_STRING \ + MP_STRINGIFY(MICROPY_VERSION_MAJOR) "." \ + MP_STRINGIFY(MICROPY_VERSION_MINOR) "." \ + MP_STRINGIFY(MICROPY_VERSION_MICRO) + // This file contains default configuration settings for MicroPython. // You can override any of the options below using mpconfigport.h file // located in a directory of your port. @@ -330,6 +347,14 @@ #define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE || MICROPY_MODULE_FROZEN_MPY) #endif +// Whether bytecode uses a qstr_table to map internal qstr indices in the bytecode +// to global qstr values in the runtime (behaviour when feature is enabled), or +// just stores global qstr values directly in the bytecode. This must be enabled +// if MICROPY_PERSISTENT_CODE is enabled. +#ifndef MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE +#define MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE (MICROPY_PERSISTENT_CODE) +#endif + // Whether to emit x64 native code #ifndef MICROPY_EMIT_X64 #define MICROPY_EMIT_X64 (0) @@ -388,8 +413,10 @@ // Convenience definition for whether any native emitter is enabled #define MICROPY_EMIT_NATIVE (MICROPY_EMIT_X64 || MICROPY_EMIT_X86 || MICROPY_EMIT_THUMB || MICROPY_EMIT_ARM || MICROPY_EMIT_XTENSA || MICROPY_EMIT_XTENSAWIN) -// Select prelude-as-bytes-object for certain emitters -#define MICROPY_EMIT_NATIVE_PRELUDE_AS_BYTES_OBJ (MICROPY_EMIT_XTENSAWIN) +// Some architectures cannot read byte-wise from executable memory. In this case +// the prelude for a native function (which usually sits after the machine code) +// must be separated and placed somewhere where it can be read byte-wise. +#define MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE (MICROPY_EMIT_XTENSAWIN) // Convenience definition for whether any inline assembler emitter is enabled #define MICROPY_EMIT_INLINE_ASM (MICROPY_EMIT_INLINE_THUMB || MICROPY_EMIT_INLINE_XTENSA) @@ -423,18 +450,17 @@ #define MICROPY_DYNAMIC_COMPILER (0) #endif -// Configure dynamic compiler macros -#if MICROPY_DYNAMIC_COMPILER -#define MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC (mp_dynamic_compiler.py_builtins_str_unicode) -#else -#define MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC MICROPY_PY_BUILTINS_STR_UNICODE -#endif - // Whether to enable constant folding; eg 1+2 rewritten as 3 #ifndef MICROPY_COMP_CONST_FOLDING #define MICROPY_COMP_CONST_FOLDING (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif +// Whether to compile constant tuples immediately to their respective objects; eg (1, True) +// Otherwise the tuple will be built at runtime +#ifndef MICROPY_COMP_CONST_TUPLE +#define MICROPY_COMP_CONST_TUPLE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) +#endif + // Whether to enable optimisations for constant literals, eg OrderedDict #ifndef MICROPY_COMP_CONST_LITERAL #define MICROPY_COMP_CONST_LITERAL (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) @@ -468,11 +494,6 @@ #define MICROPY_COMP_RETURN_IF_EXPR (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -// Whether to include parsing of f-string literals -#ifndef MICROPY_COMP_FSTRING_LITERAL -#define MICROPY_COMP_FSTRING_LITERAL (1) -#endif - /*****************************************************************************/ /* Internal debugging stuff */ @@ -513,6 +534,7 @@ #define MICROPY_DEBUG_VM_STACK_OVERFLOW (0) #endif +// CIRCUITPY // Whether to enable extra instrumentation for valgrind #ifndef MICROPY_DEBUG_VALGRIND #define MICROPY_DEBUG_VALGRIND (0) @@ -530,6 +552,7 @@ #define MICROPY_OPT_COMPUTED_GOTO (0) #endif +// CIRCUITPY // Whether to save trade flash space for speed in MICROPY_OPT_COMPUTED_GOTO. // Costs about 3% speed, saves about 1500 bytes space. In addition to the assumptions // of MICROPY_OPT_COMPUTED_GOTO, also assumes that mp_execute_bytecode is less than @@ -595,6 +618,7 @@ #define MICROPY_HAS_FILE_READER (MICROPY_READER_POSIX || MICROPY_READER_VFS) #endif +// CIRCUITPY // Number of VFS mounts to persist across soft-reset. #ifndef MICROPY_FATFS_NUM_PERSISTENT #define MICROPY_FATFS_NUM_PERSISTENT (0) @@ -632,6 +656,11 @@ #define MICROPY_GC_HOOK_LOOP #endif +// Whether to provide m_tracked_calloc, m_tracked_free functions +#ifndef MICROPY_TRACKED_ALLOC +#define MICROPY_TRACKED_ALLOC (0) +#endif + // Whether to enable finalisers in the garbage collector (ie call __del__) #ifndef MICROPY_ENABLE_FINALISER #define MICROPY_ENABLE_FINALISER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) @@ -654,11 +683,6 @@ #define MICROPY_STACK_CHECK (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -// Whether to measure maximum stack excursion -#ifndef MICROPY_MAX_STACK_USAGE -#define MICROPY_MAX_STACK_USAGE (0) -#endif - // Whether to have an emergency exception buffer #ifndef MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF #define MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF (0) @@ -773,6 +797,7 @@ typedef long long mp_longint_impl_t; #define MICROPY_WARNINGS (0) #endif +// CIRCUITPY // Whether to support chained exceptions #ifndef MICROPY_CPYTHON_EXCEPTION_CHAIN #define MICROPY_CPYTHON_EXCEPTION_CHAIN (0) @@ -854,6 +879,12 @@ typedef double mp_float_t; #define MICROPY_STREAMS_POSIX_API (0) #endif +// Whether modules can use MP_MODULE_ATTR_DELEGATION_ENTRY() to delegate failed +// attribute lookups. +#ifndef MICROPY_MODULE_ATTR_DELEGATION +#define MICROPY_MODULE_ATTR_DELEGATION (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + // Whether to call __init__ when importing builtin modules for the first time #ifndef MICROPY_MODULE_BUILTIN_INIT #define MICROPY_MODULE_BUILTIN_INIT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) @@ -919,6 +950,11 @@ typedef double mp_float_t; #define MICROPY_ENABLE_SCHEDULER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif +// Whether the scheduler supports scheduling static nodes with C callbacks +#ifndef MICROPY_SCHEDULER_STATIC_NODES +#define MICROPY_SCHEDULER_STATIC_NODES (0) +#endif + // Maximum number of entries in the scheduler #ifndef MICROPY_SCHEDULER_DEPTH #define MICROPY_SCHEDULER_DEPTH (4) @@ -939,11 +975,6 @@ typedef double mp_float_t; #define MICROPY_VFS_FAT (0) #endif -// 1 when building C code for native mpy files. 0 otherwise. -#ifndef MICROPY_ENABLE_DYNRUNTIME -#define MICROPY_ENABLE_DYNRUNTIME (0) -#endif - /*****************************************************************************/ /* Fine control over Python builtins, classes, modules, etc */ @@ -1277,6 +1308,11 @@ typedef double mp_float_t; #define MICROPY_PY_MATH (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif +// Whether to provide all math module constants (Python 3.5+), or just pi and e. +#ifndef MICROPY_PY_MATH_CONSTANTS +#define MICROPY_PY_MATH_CONSTANTS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + // Whether to provide special math functions: math.{erf,erfc,gamma,lgamma} #ifndef MICROPY_PY_MATH_SPECIAL_FUNCTIONS #define MICROPY_PY_MATH_SPECIAL_FUNCTIONS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) @@ -1393,6 +1429,11 @@ typedef double mp_float_t; #define MICROPY_PY_SYS_ATEXIT (0) #endif +// Whether to provide sys.{ps1,ps2} mutable attributes, to control REPL prompts +#ifndef MICROPY_PY_SYS_PS1_PS2 +#define MICROPY_PY_SYS_PS1_PS2 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + // Whether to provide "sys.settrace" function #ifndef MICROPY_PY_SYS_SETTRACE #define MICROPY_PY_SYS_SETTRACE (0) @@ -1414,6 +1455,17 @@ typedef double mp_float_t; #define MICROPY_PY_SYS_STDIO_BUFFER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif +// Whether to provide sys.tracebacklimit mutable attribute +#ifndef MICROPY_PY_SYS_TRACEBACKLIMIT +#define MICROPY_PY_SYS_TRACEBACKLIMIT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#endif + +// Whether the sys module supports attribute delegation +// This is enabled automatically when needed by other features +#ifndef MICROPY_PY_SYS_ATTR_DELEGATION +#define MICROPY_PY_SYS_ATTR_DELEGATION (MICROPY_PY_SYS_PS1_PS2 || MICROPY_PY_SYS_TRACEBACKLIMIT) +#endif + // Whether to provide "uerrno" module #ifndef MICROPY_PY_UERRNO #define MICROPY_PY_UERRNO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) @@ -1596,6 +1648,11 @@ typedef double mp_float_t; #define MICROPY_PY_MACHINE_I2C (0) #endif +// Whether the low-level I2C transfer function supports a separate write as the first transfer +#ifndef MICROPY_PY_MACHINE_I2C_TRANSFER_WRITE1 +#define MICROPY_PY_MACHINE_I2C_TRANSFER_WRITE1 (0) +#endif + // Whether to provide the "machine.SoftI2C" class #ifndef MICROPY_PY_MACHINE_SOFTI2C #define MICROPY_PY_MACHINE_SOFTI2C (0) @@ -1610,6 +1667,11 @@ typedef double mp_float_t; #define MICROPY_PY_MACHINE_SOFTSPI (0) #endif +// The default backlog value for socket.listen(backlog) +#ifndef MICROPY_PY_USOCKET_LISTEN_BACKLOG_DEFAULT +#define MICROPY_PY_USOCKET_LISTEN_BACKLOG_DEFAULT (2) +#endif + #ifndef MICROPY_PY_USSL #define MICROPY_PY_USSL (0) #endif @@ -1654,11 +1716,6 @@ typedef double mp_float_t; #define MICROPY_PORT_EXTRA_BUILTINS #endif -// Additional builtin module definitions - see objmodule.c:mp_builtin_module_table for format. -#ifndef MICROPY_PORT_BUILTIN_MODULES -#define MICROPY_PORT_BUILTIN_MODULES -#endif - // Additional constant definitions for the compiler - see compile.c:mp_constants_table. #ifndef MICROPY_PORT_CONSTANTS #define MICROPY_PORT_CONSTANTS @@ -1719,6 +1776,20 @@ typedef double mp_float_t; #define MICROPY_OBJ_BASE_ALIGNMENT #endif +// String used for the banner, and sys.version additional information +#ifndef MICROPY_BANNER_NAME_AND_VERSION +#define MICROPY_BANNER_NAME_AND_VERSION "MicroPython " MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE +#endif + +// String used for the second part of the banner, and sys.implementation._machine +#ifndef MICROPY_BANNER_MACHINE +#ifdef MICROPY_HW_BOARD_NAME +#define MICROPY_BANNER_MACHINE MICROPY_HW_BOARD_NAME " with " MICROPY_HW_MCU_NAME +#else +#define MICROPY_BANNER_MACHINE MICROPY_PY_SYS_PLATFORM " [" MICROPY_PLATFORM_COMPILER "] version" +#endif +#endif + // On embedded platforms, these will typically enable/disable irqs. #ifndef MICROPY_BEGIN_ATOMIC_SECTION #define MICROPY_BEGIN_ATOMIC_SECTION() (0) diff --git a/py/mperrno.h b/py/mperrno.h index f9819169e0..4c9af57ee8 100644 --- a/py/mperrno.h +++ b/py/mperrno.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George + * Copyright (c) 2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/mphal.h b/py/mphal.h index 13aae19a71..0d4b1224e5 100644 --- a/py/mphal.h +++ b/py/mphal.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2015 Damien P. George + * Copyright (c) 2015 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/mpprint.c b/py/mpprint.c index df73587449..527acd471d 100644 --- a/py/mpprint.c +++ b/py/mpprint.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2015 Damien P. George + * Copyright (c) 2013-2015 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/mpprint.h b/py/mpprint.h index 616f21a91e..fbfa3d9914 100644 --- a/py/mpprint.h +++ b/py/mpprint.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/mpstate.c b/py/mpstate.c index b3957cc09a..32f1d60a59 100644 --- a/py/mpstate.c +++ b/py/mpstate.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George + * Copyright (c) 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/mpstate.h b/py/mpstate.h index cf1711b8ad..da0fc00ec3 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -40,11 +40,21 @@ // memory system, runtime and virtual machine. The state is a global // variable, but in the future it is hoped that the state can become local. +enum { + #if MICROPY_PY_SYS_PS1_PS2 + MP_SYS_MUTABLE_PS1, + MP_SYS_MUTABLE_PS2, + #endif + #if MICROPY_PY_SYS_TRACEBACKLIMIT + MP_SYS_MUTABLE_TRACEBACKLIMIT, + #endif + MP_SYS_MUTABLE_NUM, +}; + // This structure contains dynamic configuration for the compiler. #if MICROPY_DYNAMIC_COMPILER typedef struct mp_dynamic_compiler_t { uint8_t small_int_bits; // must be <= host small_int_bits - bool py_builtins_str_unicode; uint8_t native_arch; uint8_t nlr_buf_num_regs; } mp_dynamic_compiler_t; @@ -85,7 +95,7 @@ typedef struct _mp_state_mem_t { // This variable controls auto garbage collection. If set to false then the // GC won't automatically run when gc_alloc can't find enough blocks. But // you can still allocate/free memory and also explicitly call gc_collect. - bool gc_auto_collect_enabled; + uint16_t gc_auto_collect_enabled; #if MICROPY_GC_ALLOC_THRESHOLD size_t gc_alloc_amount; @@ -104,6 +114,7 @@ typedef struct _mp_state_mem_t { mp_thread_mutex_t gc_mutex; #endif + // CIRCUITPY void **permanent_pointers; } mp_state_mem_t; @@ -173,6 +184,11 @@ typedef struct _mp_state_vm_t { // must be initialised after the call to mp_init. mp_obj_list_t mp_sys_path_obj; mp_obj_list_t mp_sys_argv_obj; + + #if MICROPY_PY_SYS_ATTR_DELEGATION + // Contains mutable sys attributes. + mp_obj_t sys_mutable[MP_SYS_MUTABLE_NUM]; + #endif #endif // dictionary for overridden builtins @@ -194,11 +210,23 @@ typedef struct _mp_state_vm_t { vstr_t *repl_line; #endif + #if MICROPY_PY_OS_DUPTERM + mp_obj_t dupterm_objs[MICROPY_PY_OS_DUPTERM]; + #endif + + #if MICROPY_PY_LWIP_SLIP + mp_obj_t lwip_slip_stream; + #endif + #if MICROPY_VFS struct _mp_vfs_mount_t *vfs_cur; struct _mp_vfs_mount_t *vfs_mount_table; #endif + #if MICROPY_PY_BLUETOOTH + mp_obj_t bluetooth; + #endif + // // END ROOT POINTER SECTION //////////////////////////////////////////////////////////// @@ -228,6 +256,16 @@ typedef struct _mp_state_vm_t { #if MICROPY_ENABLE_SCHEDULER volatile int16_t sched_state; + + #if MICROPY_SCHEDULER_STATIC_NODES + // These will usually point to statically allocated memory. They are not + // traced by the GC. They are assumed to be zero'd out before mp_init() is + // called (usually because this struct lives in the BSS). + struct _mp_sched_node_t *sched_head; + struct _mp_sched_node_t *sched_tail; + #endif + + // These index sched_queue. uint8_t sched_len; uint8_t sched_idx; #endif @@ -249,6 +287,7 @@ typedef struct _mp_state_thread_t { // Stack top at the start of program char *stack_top; + // CIRCUITPY #if MICROPY_MAX_STACK_USAGE char *stack_bottom; #endif diff --git a/py/mpthread.h b/py/mpthread.h index fa9e054e3d..e611ef4c11 100644 --- a/py/mpthread.h +++ b/py/mpthread.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd + * Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/mpz.c b/py/mpz.c index b52e05148a..3218d5f392 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/mpz.h b/py/mpz.h index 0fdcf52cff..0de0b31f73 100644 --- a/py/mpz.h +++ b/py/mpz.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/nativeglue.c b/py/nativeglue.c index 29ca77b39e..0e9bbf54eb 100644 --- a/py/nativeglue.c +++ b/py/nativeglue.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George + * Copyright (c) 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -301,9 +301,9 @@ const mp_fun_table_t mp_fun_table = { mp_unpack_ex, mp_delete_name, mp_delete_global, - mp_make_closure_from_raw_code, + mp_obj_new_closure, mp_arg_check_num_sig, - mp_setup_code_state, + mp_setup_code_state_native, mp_small_int_floor_divide, mp_small_int_modulo, mp_native_yield_from, @@ -346,4 +346,8 @@ const mp_fun_table_t mp_fun_table = { &mp_stream_write_obj, }; +#elif MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER + +const int mp_fun_table; + #endif // MICROPY_EMIT_NATIVE diff --git a/py/nativeglue.h b/py/nativeglue.h index e1f0be3f4f..5f7bfe0110 100644 --- a/py/nativeglue.h +++ b/py/nativeglue.h @@ -75,7 +75,7 @@ typedef enum { MP_F_UNPACK_EX, MP_F_DELETE_NAME, MP_F_DELETE_GLOBAL, - MP_F_MAKE_CLOSURE_FROM_RAW_CODE, + MP_F_NEW_CLOSURE, MP_F_ARG_CHECK_NUM_SIG, MP_F_SETUP_CODE_STATE, MP_F_SMALL_INT_FLOOR_DIVIDE, @@ -112,7 +112,7 @@ typedef struct _mp_fun_table_t { void (*set_store)(mp_obj_t self_in, mp_obj_t item); mp_obj_t (*list_append)(mp_obj_t self_in, mp_obj_t arg); mp_obj_t (*dict_store)(mp_obj_t self_in, mp_obj_t key, mp_obj_t value); - mp_obj_t (*make_function_from_raw_code)(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args); + mp_obj_t (*make_function_from_raw_code)(const mp_raw_code_t *rc, const mp_module_context_t *cm, const mp_obj_t *def_args); mp_obj_t (*call_function_n_kw)(mp_obj_t fun_in, size_t n_args_kw, const mp_obj_t *args); mp_obj_t (*call_method_n_kw)(size_t n_args, size_t n_kw, const mp_obj_t *args); mp_obj_t (*call_method_n_kw_var)(bool have_self, size_t n_args_n_kw, const mp_obj_t *args); @@ -129,9 +129,9 @@ typedef struct _mp_fun_table_t { void (*unpack_ex)(mp_obj_t seq, size_t num, mp_obj_t *items); void (*delete_name)(qstr qst); void (*delete_global)(qstr qst); - mp_obj_t (*make_closure_from_raw_code)(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args); + mp_obj_t (*new_closure)(mp_obj_t fun, size_t n_closed_over, const mp_obj_t *closed); void (*arg_check_num_sig)(size_t n_args, size_t n_kw, uint32_t sig); - void (*setup_code_state)(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); + void (*setup_code_state_native)(mp_code_state_native_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); mp_int_t (*small_int_floor_divide)(mp_int_t num, mp_int_t denom); mp_int_t (*small_int_modulo)(mp_int_t dividend, mp_int_t divisor); bool (*yield_from)(mp_obj_t gen, mp_obj_t send_value, mp_obj_t *ret_value); @@ -173,6 +173,12 @@ typedef struct _mp_fun_table_t { const mp_obj_fun_builtin_var_t *stream_write_obj; } mp_fun_table_t; +#if (MICROPY_EMIT_NATIVE && !MICROPY_DYNAMIC_COMPILER) || MICROPY_ENABLE_DYNRUNTIME extern const mp_fun_table_t mp_fun_table; +#elif MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER +// In dynamic-compiler mode eliminate dependency on entries in mp_fun_table. +// This only needs to be an independent pointer, content doesn't matter. +extern const int mp_fun_table; +#endif #endif // MICROPY_INCLUDED_PY_NATIVEGLUE_H diff --git a/py/nlr.c b/py/nlr.c index 6dfd162196..32e2b42c30 100644 --- a/py/nlr.c +++ b/py/nlr.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2017 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/nlr.h b/py/nlr.h index 1fb51f3585..b7b016b1e6 100644 --- a/py/nlr.h +++ b/py/nlr.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -177,7 +177,7 @@ NORETURN void nlr_jump_fail(void *val); #if !MICROPY_NLR_SETJMP #define nlr_push(val) \ - assert(MP_STATE_THREAD(nlr_top) != val),nlr_push(val) + assert(MP_STATE_THREAD(nlr_top) != val), nlr_push(val) /* #define nlr_push(val) \ diff --git a/py/nlrsetjmp.c b/py/nlrsetjmp.c index a93595dc83..960dd86f52 100644 --- a/py/nlrsetjmp.c +++ b/py/nlrsetjmp.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2017 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/nlrthumb.c b/py/nlrthumb.c index 4b20d30c6e..1a1e83956e 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2017 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,6 +36,7 @@ // For reference, arm/thumb callee save regs are: // r4-r11, r13=sp +// CIRCUITPY: added returns_twice __attribute__((naked, returns_twice)) unsigned int nlr_push(nlr_buf_t *nlr) { __asm volatile ( diff --git a/py/nlrx64.c b/py/nlrx64.c index f7e92608d2..df620ed9f7 100644 --- a/py/nlrx64.c +++ b/py/nlrx64.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2017 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/nlrx86.c b/py/nlrx86.c index 0cc8b9e6da..a6c37a6998 100644 --- a/py/nlrx86.c +++ b/py/nlrx86.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2017 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/nlrxtensa.c b/py/nlrxtensa.c index 5aa767aaeb..4123efd180 100644 --- a/py/nlrxtensa.c +++ b/py/nlrxtensa.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014-2017 Damien P. George + * Copyright (c) 2014-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/obj.c b/py/obj.c index c394656016..01efd8ad28 100644 --- a/py/obj.c +++ b/py/obj.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -43,6 +43,13 @@ #include "supervisor/shared/stack.h" #include "supervisor/shared/translate/translate.h" +// Allocates an object and also sets type, for mp_obj_malloc{,_var} macros. +void *mp_obj_malloc_helper(size_t num_bytes, const mp_obj_type_t *type) { + mp_obj_base_t *base = (mp_obj_base_t *)m_malloc(num_bytes); + base->type = type; + return base; +} + const mp_obj_type_t *MICROPY_WRAP_MP_OBJ_GET_TYPE(mp_obj_get_type)(mp_const_obj_t o_in) { #if MICROPY_OBJ_IMMEDIATE_OBJS && MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A @@ -224,11 +231,7 @@ void mp_obj_print_exception_with_limit(const mp_print_t *print, mp_obj_t exc, mp mp_print_str(print, "\n"); } -void mp_obj_print_exception(const mp_print_t *print, mp_obj_t exc) { - mp_obj_print_exception_with_limit(print, exc, 0); -} - -bool PLACE_IN_ITCM(mp_obj_is_true)(mp_obj_t arg) { +bool mp_obj_is_true(mp_obj_t arg) { if (arg == mp_const_false) { return 0; } else if (arg == mp_const_true) { @@ -343,7 +346,7 @@ mp_obj_t mp_obj_equal_not_equal(mp_binary_op_t op, mp_obj_t o1, mp_obj_t o2) { } // Try calling __eq__. - mp_obj_t r = binary_op(MP_BINARY_OP_EQUAL, o1, o2); + mp_obj_t r = type->binary_op(MP_BINARY_OP_EQUAL, o1, o2); if (r != MP_OBJ_NULL) { if (op == MP_BINARY_OP_EQUAL) { return r; @@ -442,10 +445,10 @@ mp_float_t mp_obj_get_float(mp_obj_t arg) { if (!mp_obj_get_float_maybe(arg, &val)) { #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE - mp_raise_TypeError_varg(MP_ERROR_TEXT("can't convert to %q"), MP_QSTR_float); + mp_raise_TypeError(MP_ERROR_TEXT("can't convert to float")); #else - mp_raise_TypeError_varg( - MP_ERROR_TEXT("can't convert %q to %q"), mp_obj_get_type_qstr(arg), MP_QSTR_float); + mp_raise_msg_varg(&mp_type_TypeError, + MP_ERROR_TEXT("can't convert %s to float"), mp_obj_get_type_str(arg)); #endif } @@ -512,7 +515,14 @@ void mp_obj_get_array(mp_obj_t o, size_t *len, mp_obj_t **items) { void mp_obj_get_array_fixed_n(mp_obj_t o, size_t len, mp_obj_t **items) { size_t seq_len; mp_obj_get_array(o, &seq_len, items); - mp_arg_validate_length(seq_len, len, mp_obj_get_type(o)->name); + if (seq_len != len) { + #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE + mp_raise_ValueError(MP_ERROR_TEXT("tuple/list has wrong length")); + #else + mp_raise_msg_varg(&mp_type_ValueError, + MP_ERROR_TEXT("requested length %d but object has length %d"), (int)len, (int)seq_len); + #endif + } } // is_slice determines whether the index is a slice index @@ -521,7 +531,13 @@ size_t mp_get_index(const mp_obj_type_t *type, size_t len, mp_obj_t index, bool if (mp_obj_is_small_int(index)) { i = MP_OBJ_SMALL_INT_VALUE(index); } else if (!mp_obj_get_int_maybe(index, &i)) { - mp_raise_TypeError_varg(MP_ERROR_TEXT("%q must be of type %q, not %q"), MP_QSTR_index, MP_QSTR_int, mp_obj_get_type(index)->name); + #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE + mp_raise_TypeError(MP_ERROR_TEXT("indices must be integers")); + #else + mp_raise_msg_varg(&mp_type_TypeError, + MP_ERROR_TEXT("%q indices must be integers, not %s"), + type->name, mp_obj_get_type_str(index)); + #endif } if (i < 0) { diff --git a/py/obj.h b/py/obj.h index 92732b31dd..9f5314e30d 100644 --- a/py/obj.h +++ b/py/obj.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -142,8 +142,18 @@ static MP_INLINE bool mp_obj_is_immediate_obj(mp_const_obj_t o) { #if MICROPY_PY_BUILTINS_FLOAT #define mp_const_float_e MP_ROM_PTR(&mp_const_float_e_obj) #define mp_const_float_pi MP_ROM_PTR(&mp_const_float_pi_obj) +#if MICROPY_PY_MATH_CONSTANTS +#define mp_const_float_tau MP_ROM_PTR(&mp_const_float_tau_obj) +#define mp_const_float_inf MP_ROM_PTR(&mp_const_float_inf_obj) +#define mp_const_float_nan MP_ROM_PTR(&mp_const_float_nan_obj) +#endif extern const struct _mp_obj_float_t mp_const_float_e_obj; extern const struct _mp_obj_float_t mp_const_float_pi_obj; +#if MICROPY_PY_MATH_CONSTANTS +extern const struct _mp_obj_float_t mp_const_float_tau_obj; +extern const struct _mp_obj_float_t mp_const_float_inf_obj; +extern const struct _mp_obj_float_t mp_const_float_nan_obj; +#endif #define mp_obj_is_float(o) mp_obj_is_type((o), &mp_type_float) mp_float_t mp_obj_float_get(mp_obj_t self_in); @@ -165,6 +175,11 @@ static MP_INLINE bool mp_obj_is_small_int(mp_const_obj_t o) { #if MICROPY_PY_BUILTINS_FLOAT #define mp_const_float_e MP_ROM_PTR((mp_obj_t)(((0x402df854 & ~3) | 2) + 0x80800000)) #define mp_const_float_pi MP_ROM_PTR((mp_obj_t)(((0x40490fdb & ~3) | 2) + 0x80800000)) +#if MICROPY_PY_MATH_CONSTANTS +#define mp_const_float_tau MP_ROM_PTR((mp_obj_t)(((0x40c90fdb & ~3) | 2) + 0x80800000)) +#define mp_const_float_inf MP_ROM_PTR((mp_obj_t)(((0x7f800000 & ~3) | 2) + 0x80800000)) +#define mp_const_float_nan MP_ROM_PTR((mp_obj_t)(((0xffc00000 & ~3) | 2) + 0x80800000)) +#endif static MP_INLINE bool mp_obj_is_float(mp_const_obj_t o) { return (((mp_uint_t)(o)) & 3) == 2 && (((mp_uint_t)(o)) & 0xff800007) != 0x00000006; @@ -229,6 +244,11 @@ static MP_INLINE bool mp_obj_is_immediate_obj(mp_const_obj_t o) { #define mp_const_float_e {((mp_obj_t)((uint64_t)0x4005bf0a8b145769 + 0x8004000000000000))} #define mp_const_float_pi {((mp_obj_t)((uint64_t)0x400921fb54442d18 + 0x8004000000000000))} +#if MICROPY_PY_MATH_CONSTANTS +#define mp_const_float_tau {((mp_obj_t)((uint64_t)0x401921fb54442d18 + 0x8004000000000000))} +#define mp_const_float_inf {((mp_obj_t)((uint64_t)0x7ff0000000000000 + 0x8004000000000000))} +#define mp_const_float_nan {((mp_obj_t)((uint64_t)0xfff8000000000000 + 0x8004000000000000))} +#endif static MP_INLINE bool mp_obj_is_float(mp_const_obj_t o) { return ((uint64_t)(o) & 0xfffc000000000000) != 0; @@ -421,7 +441,9 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t; // param obj_module: mp_obj_module_t instance // param enabled_define: used as `#if (enabled_define) around entry` -#define MP_REGISTER_MODULE(module_name, obj_module, enabled_define) +#ifndef NO_QSTR +#define MP_REGISTER_MODULE(module_name, obj_module) +#endif // Underlying map/hash table implementation (not dict object or map function) @@ -437,8 +459,8 @@ typedef struct _mp_rom_map_elem_t { typedef struct _mp_map_t { size_t all_keys_are_qstrs : 1; - size_t is_fixed : 1; // a fixed array that can't be modified; must also be ordered - size_t is_ordered : 1; // an ordered array + size_t is_fixed : 1; // if set, table is fixed/read-only and can't be modified + size_t is_ordered : 1; / if set, table is an ordered array, not a hash map size_t scanning : 1; // true if we're in the middle of scanning linked dictionaries, // e.g., make_dict_long_lived() size_t used : (8 * sizeof(size_t) - 4); @@ -589,39 +611,6 @@ struct _mp_obj_type_ext { // One of disjoint protocols (interfaces), like mp_stream_p_t, etc. const void *protocol; -}; - -struct _mp_obj_type_t { - // A type is an object so must start with this entry, which points to mp_type_type. - mp_obj_base_t base; - - // Flags associated with this type. - uint16_t flags; - // The name of this type, a qstr. - uint16_t name; - - // A dict mapping qstrs to objects local methods/constants/etc. - struct _mp_obj_dict_t *locals_dict; - - // Corresponds to __new__ and __init__ special methods, to make an instance of the type. - mp_make_new_fun_t make_new; - - // Corresponds to __repr__ and __str__ special methods. - mp_print_fun_t print; - - // Implements load, store and delete attribute. - // - // dest[0] = MP_OBJ_NULL means load - // return: for fail, do nothing - // for fail but continue lookup in locals_dict, dest[1] = MP_OBJ_SENTINEL - // for attr, dest[0] = value - // for method, dest[0] = method, dest[1] = self - // - // dest[0,1] = {MP_OBJ_SENTINEL, MP_OBJ_NULL} means delete - // dest[0,1] = {MP_OBJ_SENTINEL, object} means store - // return: for fail, do nothing - // for success set dest[0] = MP_OBJ_NULL - mp_attr_fun_t attr; // A pointer to the parents of this type: // - 0 parents: pointer is NULL (object is implicitly the single parent) @@ -639,6 +628,7 @@ struct _mp_obj_full_type_t { mp_obj_base_t base; uint16_t flags; uint16_t name; + // A dict mapping qstrs to objects local methods/constants/etc. struct _mp_obj_dict_t *locals_dict; mp_make_new_fun_t make_new; mp_print_fun_t print; @@ -714,9 +704,6 @@ extern const mp_obj_type_t mp_type_fun_builtin_2; extern const mp_obj_type_t mp_type_fun_builtin_3; extern const mp_obj_type_t mp_type_fun_builtin_var; extern const mp_obj_type_t mp_type_fun_bc; -#if MICROPY_EMIT_NATIVE -extern const mp_obj_type_t mp_type_fun_native; -#endif extern const mp_obj_type_t mp_type_module; extern const mp_obj_type_t mp_type_staticmethod; extern const mp_obj_type_t mp_type_classmethod; @@ -801,6 +788,12 @@ extern const struct _mp_obj_exception_t mp_static_GeneratorExit_obj; // General API for objects +// Helper versions of m_new_obj when you need to immediately set base.type. +// Implementing this as a call rather than inline saves 8 bytes per usage. +#define mp_obj_malloc(struct_type, obj_type) ((struct_type *)mp_obj_malloc_helper(sizeof(struct_type), obj_type)) +#define mp_obj_malloc_var(struct_type, var_type, var_num, obj_type) ((struct_type *)mp_obj_malloc_helper(sizeof(struct_type) + sizeof(var_type) * (var_num), obj_type)) +void *mp_obj_malloc_helper(size_t num_bytes, const mp_obj_type_t *type); + // These macros are derived from more primitive ones and are used to // check for more specific object types. // Note: these are kept as macros because inline functions sometimes use much @@ -935,7 +928,6 @@ bool mp_obj_exception_match(mp_obj_t exc, mp_const_obj_t exc_type); void mp_obj_exception_clear_traceback(mp_obj_t self_in); void mp_obj_exception_add_traceback(mp_obj_t self_in, qstr file, size_t line, qstr block); void mp_obj_exception_get_traceback(mp_obj_t self_in, size_t *n, size_t **values); -mp_obj_t mp_obj_exception_get_traceback_obj(mp_obj_t self_in); mp_obj_t mp_obj_exception_get_value(mp_obj_t self_in); mp_obj_t mp_obj_exception_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *args); mp_obj_t mp_alloc_emergency_exception_buf(mp_obj_t size_in); @@ -1011,7 +1003,6 @@ void mp_obj_tuple_del(mp_obj_t self_in); mp_int_t mp_obj_tuple_hash(mp_obj_t self_in); // list -mp_obj_t mp_obj_list_clear(mp_obj_t self_in); mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg); mp_obj_t mp_obj_list_remove(mp_obj_t self_in, mp_obj_t value); void mp_obj_list_get(mp_obj_t self_in, size_t *len, mp_obj_t **items); @@ -1031,7 +1022,7 @@ mp_obj_t mp_obj_dict_get(mp_obj_t self_in, mp_obj_t index); mp_obj_t mp_obj_dict_store(mp_obj_t self_in, mp_obj_t key, mp_obj_t value); mp_obj_t mp_obj_dict_delete(mp_obj_t self_in, mp_obj_t key); mp_obj_t mp_obj_dict_copy(mp_obj_t self_in); -static MP_INLINE mp_map_t *mp_obj_dict_get_map(mp_obj_t dict) { +static inline mp_map_t *mp_obj_dict_get_map(mp_obj_t dict) { return &((mp_obj_dict_t *)MP_OBJ_TO_PTR(dict))->map; } @@ -1091,8 +1082,9 @@ typedef struct _mp_obj_module_t { mp_obj_base_t base; mp_obj_dict_t *globals; } mp_obj_module_t; -mp_obj_dict_t *mp_obj_module_get_globals(mp_obj_t self_in); -void mp_obj_module_set_globals(mp_obj_t self_in, mp_obj_dict_t *globals); +static inline mp_obj_dict_t *mp_obj_module_get_globals(mp_obj_t module) { + return ((mp_obj_module_t *)MP_OBJ_TO_PTR(module))->globals; +} // check if given module object is a package bool mp_obj_is_package(mp_obj_t module); diff --git a/py/objarray.c b/py/objarray.c index c4a9e7cec4..b8ea9992bb 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * Copyright (c) 2014 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -838,8 +838,7 @@ mp_obj_t mp_obj_new_bytearray_of_zeros(size_t n) { // Create bytearray which references specified memory area mp_obj_t mp_obj_new_bytearray_by_ref(size_t n, void *items) { - mp_obj_array_t *o = m_new_obj(mp_obj_array_t); - o->base.type = &mp_type_bytearray; + mp_obj_array_t *o = mp_obj_malloc(mp_obj_array_t, &mp_type_bytearray); o->typecode = BYTEARRAY_TYPECODE; o->free = 0; o->len = n; diff --git a/py/objarray.h b/py/objarray.h index a1bf6abfd1..94c31c9693 100644 --- a/py/objarray.h +++ b/py/objarray.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * Copyright (c) 2014 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/py/objattrtuple.c b/py/objattrtuple.c index 7b664dd0fe..c288727d48 100644 --- a/py/objattrtuple.c +++ b/py/objattrtuple.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2015 Damien P. George + * Copyright (c) 2015 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -71,8 +71,7 @@ STATIC void mp_obj_attrtuple_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { } mp_obj_t mp_obj_new_attrtuple(const qstr *fields, size_t n, const mp_obj_t *items) { - mp_obj_tuple_t *o = m_new_obj_var(mp_obj_tuple_t, mp_obj_t, n + 1); - o->base.type = &mp_type_attrtuple; + mp_obj_tuple_t *o = mp_obj_malloc_var(mp_obj_tuple_t, mp_obj_t, n + 1, &mp_type_attrtuple); o->len = n; for (size_t i = 0; i < n; i++) { o->items[i] = items[i]; diff --git a/py/objbool.c b/py/objbool.c index 13d10ffae3..e450270fd9 100644 --- a/py/objbool.c +++ b/py/objbool.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objboundmeth.c b/py/objboundmeth.c index c460626c4f..edc37462dd 100644 --- a/py/objboundmeth.c +++ b/py/objboundmeth.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -111,8 +111,7 @@ STATIC const mp_obj_type_t mp_type_bound_meth = { }; mp_obj_t mp_obj_new_bound_meth(mp_obj_t meth, mp_obj_t self) { - mp_obj_bound_meth_t *o = m_new_obj(mp_obj_bound_meth_t); - o->base.type = &mp_type_bound_meth; + mp_obj_bound_meth_t *o = mp_obj_malloc(mp_obj_bound_meth_t, &mp_type_bound_meth); o->meth = meth; o->self = self; return MP_OBJ_FROM_PTR(o); diff --git a/py/objcell.c b/py/objcell.c index 2e15e6825a..2702ca5350 100644 --- a/py/objcell.c +++ b/py/objcell.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -64,8 +64,7 @@ STATIC const mp_obj_type_t mp_type_cell = { }; mp_obj_t mp_obj_new_cell(mp_obj_t obj) { - mp_obj_cell_t *o = m_new_obj(mp_obj_cell_t); - o->base.type = &mp_type_cell; + mp_obj_cell_t *o = mp_obj_malloc(mp_obj_cell_t, &mp_type_cell); o->obj = obj; return MP_OBJ_FROM_PTR(o); } diff --git a/py/objclosure.c b/py/objclosure.c index f5dbb705ac..309dd2b826 100644 --- a/py/objclosure.c +++ b/py/objclosure.c @@ -91,8 +91,7 @@ const mp_obj_type_t mp_type_closure = { }; mp_obj_t mp_obj_new_closure(mp_obj_t fun, size_t n_closed_over, const mp_obj_t *closed) { - mp_obj_closure_t *o = m_new_obj_var(mp_obj_closure_t, mp_obj_t, n_closed_over); - o->base.type = &mp_type_closure; + mp_obj_closure_t *o = mp_obj_malloc_var(mp_obj_closure_t, mp_obj_t, n_closed_over, &mp_type_closure); o->fun = fun; o->n_closed = n_closed_over; memcpy(o->closed, closed, n_closed_over * sizeof(mp_obj_t)); diff --git a/py/objcomplex.c b/py/objcomplex.c index 7f4fd621e6..570f9f746f 100644 --- a/py/objcomplex.c +++ b/py/objcomplex.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -169,8 +169,7 @@ const mp_obj_type_t mp_type_complex = { }; mp_obj_t mp_obj_new_complex(mp_float_t real, mp_float_t imag) { - mp_obj_complex_t *o = m_new_obj(mp_obj_complex_t); - o->base.type = &mp_type_complex; + mp_obj_complex_t *o = mp_obj_malloc(mp_obj_complex_t, &mp_type_complex); o->real = real; o->imag = imag; return MP_OBJ_FROM_PTR(o); diff --git a/py/objdeque.c b/py/objdeque.c index 59c4c709fc..621b7b60b0 100644 --- a/py/objdeque.c +++ b/py/objdeque.c @@ -59,8 +59,7 @@ STATIC mp_obj_t deque_make_new(const mp_obj_type_t *type, size_t n_args, size_t mp_raise_ValueError(NULL); } - mp_obj_deque_t *o = m_new_obj(mp_obj_deque_t); - o->base.type = type; + mp_obj_deque_t *o = mp_obj_malloc(mp_obj_deque_t, type); o->alloc = maxlen + 1; o->i_get = o->i_put = 0; o->items = m_new0(mp_obj_t, o->alloc); diff --git a/py/objdict.c b/py/objdict.c index 02aedacdd6..0f5be6627a 100644 --- a/py/objdict.c +++ b/py/objdict.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014-2017 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014-2017 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -529,8 +529,7 @@ STATIC const mp_obj_type_t mp_type_dict_view = { }; STATIC mp_obj_t mp_obj_new_dict_view(mp_obj_t dict, mp_dict_view_kind_t kind) { - mp_obj_dict_view_t *o = m_new_obj(mp_obj_dict_view_t); - o->base.type = &mp_type_dict_view; + mp_obj_dict_view_t *o = mp_obj_malloc(mp_obj_dict_view_t, &mp_type_dict_view); o->dict = dict; o->kind = kind; return MP_OBJ_FROM_PTR(o); diff --git a/py/objenumerate.c b/py/objenumerate.c index c18ac1dd80..2ef553adb5 100644 --- a/py/objenumerate.c +++ b/py/objenumerate.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -54,14 +54,12 @@ STATIC mp_obj_t enumerate_make_new(const mp_obj_type_t *type, size_t n_args, siz MP_ARRAY_SIZE(allowed_args), allowed_args, (mp_arg_val_t *)&arg_vals); // create enumerate object - mp_obj_enumerate_t *o = m_new_obj(mp_obj_enumerate_t); - o->base.type = type; + mp_obj_enumerate_t *o = mp_obj_malloc(mp_obj_enumerate_t, type); o->iter = mp_getiter(arg_vals.iterable.u_obj, NULL); o->cur = arg_vals.start.u_int; #else - (void)n_kw; - mp_obj_enumerate_t *o = m_new_obj(mp_obj_enumerate_t); - o->base.type = type; + mp_arg_check_num(n_args, n_kw, 1, 2, false); + mp_obj_enumerate_t *o = mp_obj_malloc(mp_obj_enumerate_t, type); o->iter = mp_getiter(args[0], NULL); o->cur = n_args > 1 ? mp_obj_get_int(args[1]) : 0; #endif diff --git a/py/objexcept.c b/py/objexcept.c index e05c3ca8d7..6d209e0c34 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014-2016 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014-2016 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -326,40 +326,35 @@ MP_DEFINE_EXCEPTION(ReloadException, BaseException) MP_DEFINE_EXCEPTION(GeneratorExit, BaseException) MP_DEFINE_EXCEPTION(Exception, BaseException) #if MICROPY_PY_ASYNC_AWAIT -MP_DEFINE_EXCEPTION(StopAsyncIteration, Exception) + MP_DEFINE_EXCEPTION(StopAsyncIteration, Exception) #endif -MP_DEFINE_EXCEPTION(StopIteration, Exception) -MP_DEFINE_EXCEPTION(ArithmeticError, Exception) -// MP_DEFINE_EXCEPTION(FloatingPointError, ArithmeticError) -MP_DEFINE_EXCEPTION(OverflowError, ArithmeticError) -MP_DEFINE_EXCEPTION(ZeroDivisionError, ArithmeticError) -MP_DEFINE_EXCEPTION(AssertionError, Exception) -MP_DEFINE_EXCEPTION(AttributeError, Exception) -// MP_DEFINE_EXCEPTION(BufferError, Exception) -// MP_DEFINE_EXCEPTION(EnvironmentError, Exception) use OSError instead -MP_DEFINE_EXCEPTION(EOFError, Exception) -MP_DEFINE_EXCEPTION(ImportError, Exception) -// MP_DEFINE_EXCEPTION(IOError, Exception) use OSError instead -MP_DEFINE_EXCEPTION(LookupError, Exception) -MP_DEFINE_EXCEPTION(IndexError, LookupError) -MP_DEFINE_EXCEPTION(KeyError, LookupError) -MP_DEFINE_EXCEPTION(MemoryError, Exception) -MP_DEFINE_EXCEPTION(NameError, Exception) -/* + MP_DEFINE_EXCEPTION(StopIteration, Exception) + MP_DEFINE_EXCEPTION(ArithmeticError, Exception) + //MP_DEFINE_EXCEPTION(FloatingPointError, ArithmeticError) + MP_DEFINE_EXCEPTION(OverflowError, ArithmeticError) + MP_DEFINE_EXCEPTION(ZeroDivisionError, ArithmeticError) + MP_DEFINE_EXCEPTION(AssertionError, Exception) + MP_DEFINE_EXCEPTION(AttributeError, Exception) + //MP_DEFINE_EXCEPTION(BufferError, Exception) + MP_DEFINE_EXCEPTION(EOFError, Exception) + MP_DEFINE_EXCEPTION(ImportError, Exception) + MP_DEFINE_EXCEPTION(LookupError, Exception) + MP_DEFINE_EXCEPTION(IndexError, LookupError) + MP_DEFINE_EXCEPTION(KeyError, LookupError) + MP_DEFINE_EXCEPTION(MemoryError, Exception) + MP_DEFINE_EXCEPTION(NameError, Exception) + /* MP_DEFINE_EXCEPTION(UnboundLocalError, NameError) */ -MP_DEFINE_EXCEPTION(OSError, Exception) -MP_DEFINE_EXCEPTION(TimeoutError, OSError) -MP_DEFINE_EXCEPTION(ConnectionError, OSError) -MP_DEFINE_EXCEPTION(BrokenPipeError, ConnectionError) -/* + MP_DEFINE_EXCEPTION(OSError, Exception) + /* + MP_DEFINE_EXCEPTION(BlockingIOError, OSError) + MP_DEFINE_EXCEPTION(ChildProcessError, OSError) + MP_DEFINE_EXCEPTION(ConnectionError, OSError) + MP_DEFINE_EXCEPTION(BrokenPipeError, ConnectionError) MP_DEFINE_EXCEPTION(ConnectionAbortedError, ConnectionError) MP_DEFINE_EXCEPTION(ConnectionRefusedError, ConnectionError) MP_DEFINE_EXCEPTION(ConnectionResetError, ConnectionError) - */ -/* - MP_DEFINE_EXCEPTION(BlockingIOError, OSError) - MP_DEFINE_EXCEPTION(ChildProcessError, OSError) MP_DEFINE_EXCEPTION(InterruptedError, OSError) MP_DEFINE_EXCEPTION(IsADirectoryError, OSError) MP_DEFINE_EXCEPTION(NotADirectoryError, OSError) @@ -370,22 +365,22 @@ MP_DEFINE_EXCEPTION(BrokenPipeError, ConnectionError) MP_DEFINE_EXCEPTION(FileNotFoundError, OSError) MP_DEFINE_EXCEPTION(ReferenceError, Exception) */ -MP_DEFINE_EXCEPTION(RuntimeError, Exception) -MP_DEFINE_EXCEPTION(NotImplementedError, RuntimeError) -MP_DEFINE_EXCEPTION(SyntaxError, Exception) -MP_DEFINE_EXCEPTION(IndentationError, SyntaxError) -/* + MP_DEFINE_EXCEPTION(RuntimeError, Exception) + MP_DEFINE_EXCEPTION(NotImplementedError, RuntimeError) + MP_DEFINE_EXCEPTION(SyntaxError, Exception) + MP_DEFINE_EXCEPTION(IndentationError, SyntaxError) + /* MP_DEFINE_EXCEPTION(TabError, IndentationError) */ -// MP_DEFINE_EXCEPTION(SystemError, Exception) -MP_DEFINE_EXCEPTION(TypeError, Exception) + //MP_DEFINE_EXCEPTION(SystemError, Exception) + MP_DEFINE_EXCEPTION(TypeError, Exception) #if MICROPY_EMIT_NATIVE -MP_DEFINE_EXCEPTION(ViperTypeError, TypeError) + MP_DEFINE_EXCEPTION(ViperTypeError, TypeError) #endif -MP_DEFINE_EXCEPTION(ValueError, Exception) + MP_DEFINE_EXCEPTION(ValueError, Exception) #if MICROPY_PY_BUILTINS_STR_UNICODE -MP_DEFINE_EXCEPTION(UnicodeError, ValueError) -// TODO: Implement more UnicodeError subclasses which take arguments + MP_DEFINE_EXCEPTION(UnicodeError, ValueError) + //TODO: Implement more UnicodeError subclasses which take arguments #endif #if CIRCUITPY_ALARM MP_DEFINE_EXCEPTION(DeepSleepRequest, BaseException) @@ -413,7 +408,7 @@ mp_obj_t mp_obj_new_exception(const mp_obj_type_t *exc_type) { mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args, const mp_obj_t *args) { assert(exc_type->make_new == mp_obj_exception_make_new); - return exc_type->make_new(exc_type, n_args, 0, args); + return mp_obj_exception_make_new(exc_type, n_args, 0, args); } #if MICROPY_ERROR_REPORTING != MICROPY_ERROR_REPORTING_NONE @@ -670,172 +665,3 @@ void mp_obj_exception_get_traceback(mp_obj_t self_in, size_t *n, size_t **values *values = self->traceback->data; } } - -#if MICROPY_PY_SYS_EXC_INFO -STATIC const mp_obj_namedtuple_type_t code_type_obj = { - .base = { - .base = { - .type = &mp_type_type - }, - .flags = MP_TYPE_FLAG_EXTENDED, - .name = MP_QSTR_code, - .print = namedtuple_print, - .make_new = namedtuple_make_new, - .parent = &mp_type_tuple, - .attr = namedtuple_attr, - MP_TYPE_EXTENDED_FIELDS( - .unary_op = mp_obj_tuple_unary_op, - .binary_op = mp_obj_tuple_binary_op, - .subscr = mp_obj_tuple_subscr, - .getiter = mp_obj_tuple_getiter, - ), - }, - .n_fields = 15, - .fields = { - MP_QSTR_co_argcount, - MP_QSTR_co_kwonlyargcount, - MP_QSTR_co_nlocals, - MP_QSTR_co_stacksize, - MP_QSTR_co_flags, - MP_QSTR_co_code, - MP_QSTR_co_consts, - MP_QSTR_co_names, - MP_QSTR_co_varnames, - MP_QSTR_co_freevars, - MP_QSTR_co_cellvars, - MP_QSTR_co_filename, - MP_QSTR_co_name, - MP_QSTR_co_firstlineno, - MP_QSTR_co_lnotab, - }, -}; - -STATIC mp_obj_t code_make_new(qstr file, qstr block) { - mp_obj_t elems[15] = { - mp_obj_new_int(0), // co_argcount - mp_obj_new_int(0), // co_kwonlyargcount - mp_obj_new_int(0), // co_nlocals - mp_obj_new_int(0), // co_stacksize - mp_obj_new_int(0), // co_flags - mp_obj_new_bytearray(0, NULL), // co_code - mp_obj_new_tuple(0, NULL), // co_consts - mp_obj_new_tuple(0, NULL), // co_names - mp_obj_new_tuple(0, NULL), // co_varnames - mp_obj_new_tuple(0, NULL), // co_freevars - mp_obj_new_tuple(0, NULL), // co_cellvars - MP_OBJ_NEW_QSTR(file), // co_filename - MP_OBJ_NEW_QSTR(block), // co_name - mp_obj_new_int(1), // co_firstlineno - mp_obj_new_bytearray(0, NULL), // co_lnotab - }; - - return namedtuple_make_new((const mp_obj_type_t *)&code_type_obj, 15, 0, elems); -} - -STATIC const mp_obj_namedtuple_type_t frame_type_obj = { - .base = { - .base = { - .type = &mp_type_type - }, - .name = MP_QSTR_frame, - .print = namedtuple_print, - .make_new = namedtuple_make_new, - .parent = &mp_type_tuple, - .attr = namedtuple_attr, - MP_TYPE_EXTENDED_FIELDS( - .unary_op = mp_obj_tuple_unary_op, - .binary_op = mp_obj_tuple_binary_op, - .subscr = mp_obj_tuple_subscr, - .getiter = mp_obj_tuple_getiter, - ), - }, - .n_fields = 8, - .fields = { - MP_QSTR_f_back, - MP_QSTR_f_builtins, - MP_QSTR_f_code, - MP_QSTR_f_globals, - MP_QSTR_f_lasti, - MP_QSTR_f_lineno, - MP_QSTR_f_locals, - MP_QSTR_f_trace, - }, -}; - -STATIC mp_obj_t frame_make_new(mp_obj_t f_code, int f_lineno) { - mp_obj_t elems[8] = { - mp_const_none, // f_back - mp_obj_new_dict(0), // f_builtins - f_code, // f_code - mp_obj_new_dict(0), // f_globals - mp_obj_new_int(0), // f_lasti - mp_obj_new_int(f_lineno), // f_lineno - mp_obj_new_dict(0), // f_locals - mp_const_none, // f_trace - }; - - return namedtuple_make_new((const mp_obj_type_t *)&frame_type_obj, 8, 0, elems); -} - -STATIC const mp_obj_namedtuple_type_t traceback_type_obj = { - .base = { - .base = { - .type = &mp_type_type - }, - .flags = MP_TYPE_FLAG_EXTENDED, - .name = MP_QSTR_traceback, - .print = namedtuple_print, - .make_new = namedtuple_make_new, - .parent = &mp_type_tuple, - .attr = namedtuple_attr, - MP_TYPE_EXTENDED_FIELDS( - .unary_op = mp_obj_tuple_unary_op, - .binary_op = mp_obj_tuple_binary_op, - .subscr = mp_obj_tuple_subscr, - .getiter = mp_obj_tuple_getiter, - ), - }, - .n_fields = 4, - .fields = { - MP_QSTR_tb_frame, - MP_QSTR_tb_lasti, - MP_QSTR_tb_lineno, - MP_QSTR_tb_next, - }, -}; - -STATIC mp_obj_t traceback_from_values(size_t *values, mp_obj_t tb_next) { - int lineno = values[1]; - - mp_obj_t elems[4] = { - frame_make_new(code_make_new(values[0], values[2]), lineno), - mp_obj_new_int(0), - mp_obj_new_int(lineno), - tb_next, - }; - - return namedtuple_make_new((const mp_obj_type_t *)&traceback_type_obj, 4, 0, elems); -}; - -mp_obj_t mp_obj_exception_get_traceback_obj(mp_obj_t self_in) { - mp_obj_exception_t *self = MP_OBJ_TO_PTR(self_in); - - if (!mp_obj_is_exception_instance(self)) { - return mp_const_none; - } - - size_t n, *values; - mp_obj_exception_get_traceback(self, &n, &values); - if (n == 0) { - return mp_const_none; - } - - mp_obj_t tb_next = mp_const_none; - - for (size_t i = 0; i < n; i += 3) { - tb_next = traceback_from_values(&values[i], tb_next); - } - - return tb_next; -} -#endif diff --git a/py/objexcept.h b/py/objexcept.h index 1230fdf18a..80c06e90e0 100644 --- a/py/objexcept.h +++ b/py/objexcept.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George + * Copyright (c) 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objfilter.c b/py/objfilter.c index c034df28e6..5292cfc550 100644 --- a/py/objfilter.c +++ b/py/objfilter.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,8 +36,7 @@ typedef struct _mp_obj_filter_t { STATIC mp_obj_t filter_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 2, 2, false); - mp_obj_filter_t *o = m_new_obj(mp_obj_filter_t); - o->base.type = type; + mp_obj_filter_t *o = mp_obj_malloc(mp_obj_filter_t, type); o->fun = args[0]; o->iter = mp_getiter(args[1], NULL); return MP_OBJ_FROM_PTR(o); diff --git a/py/objfloat.c b/py/objfloat.c index 7aebcfdeb2..5666f3f5c2 100644 --- a/py/objfloat.c +++ b/py/objfloat.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -59,6 +59,14 @@ typedef struct _mp_obj_float_t { const mp_obj_float_t mp_const_float_e_obj = {{&mp_type_float}, (mp_float_t)M_E}; const mp_obj_float_t mp_const_float_pi_obj = {{&mp_type_float}, (mp_float_t)M_PI}; +#if MICROPY_PY_MATH_CONSTANTS +#ifndef NAN +#error NAN macro is not defined +#endif +const mp_obj_float_t mp_const_float_tau_obj = {{&mp_type_float}, (mp_float_t)(2.0 * M_PI)}; +const mp_obj_float_t mp_const_float_inf_obj = {{&mp_type_float}, (mp_float_t)INFINITY}; +const mp_obj_float_t mp_const_float_nan_obj = {{&mp_type_float}, (mp_float_t)NAN}; +#endif #endif @@ -194,7 +202,8 @@ const mp_obj_type_t mp_type_float = { #if MICROPY_OBJ_REPR != MICROPY_OBJ_REPR_C && MICROPY_OBJ_REPR != MICROPY_OBJ_REPR_D mp_obj_t mp_obj_new_float(mp_float_t value) { - mp_obj_float_t *o = m_new(mp_obj_float_t, 1); + // Don't use mp_obj_malloc here to avoid extra function call overhead. + mp_obj_float_t *o = m_new_obj(mp_obj_float_t); o->base.type = &mp_type_float; o->value = value; return MP_OBJ_FROM_PTR(o); diff --git a/py/objfun.c b/py/objfun.c index 5a02869fcf..eb4a2bfe25 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * Copyright (c) 2014 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -155,19 +155,23 @@ const mp_obj_type_t mp_type_fun_builtin_var = { /******************************************************************************/ /* byte code functions */ -qstr mp_obj_code_get_name(const byte *code_info) { +STATIC qstr mp_obj_code_get_name(const mp_obj_fun_bc_t *fun, const byte *code_info) { MP_BC_PRELUDE_SIZE_DECODE(code_info); - #if MICROPY_PERSISTENT_CODE - return code_info[0] | (code_info[1] << 8); - #else - return mp_decode_uint_value(code_info); + mp_uint_t name = mp_decode_uint_value(code_info); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + name = fun->context->constants.qstr_table[name]; #endif + return name; } +#if MICROPY_EMIT_NATIVE +STATIC const mp_obj_type_t mp_type_fun_native; +#endif + qstr mp_obj_fun_get_name(mp_const_obj_t fun_in) { const mp_obj_fun_bc_t *fun = MP_OBJ_TO_PTR(fun_in); #if MICROPY_EMIT_NATIVE - if (fun->base.type == &mp_type_fun_native) { + if (fun->base.type == &mp_type_fun_native || fun->base.type == &mp_type_native_gen_wrap) { // TODO native functions don't have name stored return MP_QSTR_; } @@ -175,14 +179,14 @@ qstr mp_obj_fun_get_name(mp_const_obj_t fun_in) { const byte *bc = fun->bytecode; MP_BC_PRELUDE_SIG_DECODE(bc); - return mp_obj_code_get_name(bc); + return mp_obj_code_get_name(fun, bc); } #if MICROPY_CPYTHON_COMPAT STATIC void fun_bc_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { (void)kind; mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(o_in); - mp_printf(print, "", mp_obj_fun_get_name(o_in), o); + mp_printf(print, "", mp_obj_fun_get_name(o_in), o); } #endif @@ -217,7 +221,6 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { #define INIT_CODESTATE(code_state, _fun_bc, _n_state, n_args, n_kw, args) \ code_state->fun_bc = _fun_bc; \ - code_state->ip = 0; \ code_state->n_state = _n_state; \ mp_setup_code_state(code_state, n_args, n_kw, args); \ code_state->old_globals = mp_globals_get(); @@ -248,7 +251,7 @@ mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args INIT_CODESTATE(code_state, self, n_state, n_args, n_kw, args); // execute the byte code with the correct globals context - mp_globals_set(self->globals); + mp_globals_set(self->context->module.globals); return code_state; } @@ -293,7 +296,7 @@ STATIC mp_obj_t PLACE_IN_ITCM(fun_bc_call)(mp_obj_t self_in, size_t n_args, size INIT_CODESTATE(code_state, self, n_state, n_args, n_kw, args); // execute the byte code with the correct globals context - mp_globals_set(self->globals); + mp_globals_set(self->context->module.globals); mp_vm_return_kind_t vm_return_kind = mp_execute_bytecode(code_state, MP_OBJ_NULL); mp_globals_set(code_state->old_globals); @@ -366,7 +369,7 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { } if (attr == MP_QSTR___globals__) { mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); - dest[0] = MP_OBJ_FROM_PTR(self->globals); + dest[0] = MP_OBJ_FROM_PTR(self->context->module.globals); } } #endif @@ -387,25 +390,28 @@ const mp_obj_type_t mp_type_fun_bc = { ), }; -mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args_in, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table) { +mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_module_context_t *context, struct _mp_raw_code_t *const *child_table) { size_t n_def_args = 0; size_t n_extra_args = 0; - mp_obj_tuple_t *def_args = MP_OBJ_TO_PTR(def_args_in); - if (def_args_in != MP_OBJ_NULL) { - assert(mp_obj_is_type(def_args_in, &mp_type_tuple)); - n_def_args = def_args->len; - n_extra_args = def_args->len; + mp_obj_tuple_t *def_pos_args = NULL; + mp_obj_t def_kw_args = MP_OBJ_NULL; + if (def_args != NULL && def_args[0] != MP_OBJ_NULL) { + assert(mp_obj_is_type(def_args[0], &mp_type_tuple)); + def_pos_args = MP_OBJ_TO_PTR(def_args[0]); + n_def_args = def_pos_args->len; + n_extra_args = def_pos_args->len; } - if (def_kw_args != MP_OBJ_NULL) { + if (def_args != NULL && def_args[1] != MP_OBJ_NULL) { + assert(mp_obj_is_type(def_args[1], &mp_type_dict)); + def_kw_args = def_args[1]; n_extra_args += 1; } - mp_obj_fun_bc_t *o = m_new_obj_var(mp_obj_fun_bc_t, mp_obj_t, n_extra_args); - o->base.type = &mp_type_fun_bc; - o->globals = mp_globals_get(); + mp_obj_fun_bc_t *o = mp_obj_malloc_var(mp_obj_fun_bc_t, mp_obj_t, n_extra_args, &mp_type_fun_bc); o->bytecode = code; - o->const_table = const_table; - if (def_args != NULL) { - memcpy(o->extra_args, def_args->items, n_def_args * sizeof(mp_obj_t)); + o->context = context; + o->child_table = child_table; + if (def_pos_args != NULL) { + memcpy(o->extra_args, def_pos_args->items, n_def_args * sizeof(mp_obj_t)); } if (def_kw_args != MP_OBJ_NULL) { o->extra_args[n_def_args] = def_kw_args; @@ -420,12 +426,12 @@ mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args_in, mp_obj_t def_kw_args, const byt STATIC mp_obj_t PLACE_IN_ITCM(fun_native_call)(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { MP_STACK_CHECK(); - mp_obj_fun_bc_t *self = self_in; + mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); mp_call_fun_t fun = MICROPY_MAKE_POINTER_CALLABLE((void *)self->bytecode); return fun(self_in, n_args, n_kw, args); } -const mp_obj_type_t mp_type_fun_native = { +STATIC const mp_obj_type_t mp_type_fun_native = { { &mp_type_type }, .flags = MP_TYPE_FLAG_BINDS_SELF | MP_TYPE_FLAG_EXTENDED, .name = MP_QSTR_function, @@ -435,10 +441,10 @@ const mp_obj_type_t mp_type_fun_native = { ), }; -mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const void *fun_data, const mp_uint_t *const_table) { - mp_obj_fun_bc_t *o = mp_obj_new_fun_bc(def_args_in, def_kw_args, (const byte *)fun_data, const_table); +mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) { + mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table)); o->base.type = &mp_type_fun_native; - return o; + return MP_OBJ_FROM_PTR(o); } #endif // MICROPY_EMIT_NATIVE @@ -546,12 +552,11 @@ STATIC const mp_obj_type_t mp_type_fun_asm = { }; mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig) { - mp_obj_fun_asm_t *o = m_new_obj(mp_obj_fun_asm_t); - o->base.type = &mp_type_fun_asm; + mp_obj_fun_asm_t *o = mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm); o->n_args = n_args; o->fun_data = fun_data; o->type_sig = type_sig; - return o; + return MP_OBJ_FROM_PTR(o); } #endif // MICROPY_EMIT_INLINE_ASM diff --git a/py/objfun.h b/py/objfun.h index aae780b310..9de15b8841 100644 --- a/py/objfun.h +++ b/py/objfun.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,13 +26,14 @@ #ifndef MICROPY_INCLUDED_PY_OBJFUN_H #define MICROPY_INCLUDED_PY_OBJFUN_H +#include "py/bc.h" #include "py/obj.h" typedef struct _mp_obj_fun_bc_t { mp_obj_base_t base; - mp_obj_dict_t *globals; // the context within which this function was defined - const byte *bytecode; // bytecode for the function - const mp_uint_t *const_table; // constant table + const mp_module_context_t *context; // context within which this function was defined + struct _mp_raw_code_t *const *child_table; // table of children + const byte *bytecode; // bytecode for the function #if MICROPY_PY_SYS_SETTRACE const struct _mp_raw_code_t *rc; #endif @@ -42,6 +43,9 @@ typedef struct _mp_obj_fun_bc_t { mp_obj_t extra_args[]; } mp_obj_fun_bc_t; +mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_module_context_t *cm, struct _mp_raw_code_t *const *raw_code_table); +mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *cm, struct _mp_raw_code_t *const *raw_code_table); +mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig); void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); #endif // MICROPY_INCLUDED_PY_OBJFUN_H diff --git a/py/objgenerator.c b/py/objgenerator.c index 7c3ec99307..325386a137 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2019 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014-2017 Paul Sokolovsky + * Copyright (c) 2013-2019 Damien P. George + * Copyright (c) 2014-2017 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -65,43 +65,77 @@ typedef struct _mp_obj_gen_instance_t { mp_code_state_t code_state; } mp_obj_gen_instance_t; +STATIC mp_obj_t gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { + // A generating function is just a bytecode function with type mp_type_gen_wrap + mp_obj_fun_bc_t *self_fun = MP_OBJ_TO_PTR(self_in); + + // bytecode prelude: get state size and exception stack size + const uint8_t *ip = self_fun->bytecode; + MP_BC_PRELUDE_SIG_DECODE(ip); + + // allocate the generator object, with room for local stack and exception stack + mp_obj_gen_instance_t *o = mp_obj_malloc_var(mp_obj_gen_instance_t, byte, + n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t), + &mp_type_gen_instance); + + o->pend_exc = mp_const_none; + o->code_state.fun_bc = self_fun; + o->code_state.n_state = n_state; + mp_setup_code_state(&o->code_state, n_args, n_kw, args); + return MP_OBJ_FROM_PTR(o); +} + +const mp_obj_type_t mp_type_gen_wrap = { + { &mp_type_type }, + .flags = MP_TYPE_FLAG_BINDS_SELF, + .name = MP_QSTR_generator, + .call = gen_wrap_call, + .unary_op = mp_generic_unary_op, + #if MICROPY_PY_FUNCTION_ATTRS + .attr = mp_obj_fun_bc_attr, + #endif +}; + /******************************************************************************/ // native generator wrapper #if MICROPY_EMIT_NATIVE +// Based on mp_obj_gen_instance_t. +typedef struct _mp_obj_gen_instance_native_t { + mp_obj_base_t base; + mp_obj_t pend_exc; + mp_code_state_native_t code_state; +} mp_obj_gen_instance_native_t; + STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { - mp_obj_gen_wrap_t *self = MP_OBJ_TO_PTR(self_in); - mp_obj_fun_bc_t *self_fun = (mp_obj_fun_bc_t *)self->fun; + // The state for a native generating function is held in the same struct as a bytecode function + mp_obj_fun_bc_t *self_fun = MP_OBJ_TO_PTR(self_in); - // Determine start of prelude, and extract n_state from it - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wcast-align" - uintptr_t prelude_offset = ((uintptr_t *)self_fun->bytecode)[0]; - #pragma GCC diagnostic pop + // Determine start of prelude. + uintptr_t prelude_ptr_index = ((uintptr_t *)self_fun->bytecode)[0]; + const uint8_t *prelude_ptr; + if (prelude_ptr_index == 0) { + prelude_ptr = (void *)self_fun->child_table; + } else { + prelude_ptr = (void *)self_fun->child_table[prelude_ptr_index]; + } - #if MICROPY_EMIT_NATIVE_PRELUDE_AS_BYTES_OBJ - // Prelude is in bytes object in const_table, at index prelude_offset - mp_obj_str_t *prelude_bytes = MP_OBJ_TO_PTR(self_fun->const_table[prelude_offset]); - prelude_offset = (const byte *)prelude_bytes->data - self_fun->bytecode; - #endif - const uint8_t *ip = self_fun->bytecode + prelude_offset; - size_t n_state, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_args; - MP_BC_PRELUDE_SIG_DECODE_INTO(ip, n_state, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_args); - size_t n_exc_stack = 0; + // Extract n_state from the prelude. + const uint8_t *ip = prelude_ptr; + MP_BC_PRELUDE_SIG_DECODE(ip); - // Allocate the generator object, with room for local stack and exception stack - mp_obj_gen_instance_t *o = m_new_obj_var(mp_obj_gen_instance_t, byte, - n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t)); - o->base.type = &mp_type_gen_instance; + // Allocate the generator object, with room for local stack (exception stack not needed). + mp_obj_gen_instance_native_t *o = mp_obj_malloc_var(mp_obj_gen_instance_native_t, byte, n_state * sizeof(mp_obj_t), &mp_type_gen_instance); // Parse the input arguments and set up the code state o->coroutine_generator = self->coroutine_generator; o->pend_exc = mp_const_none; o->code_state.fun_bc = self_fun; - o->code_state.ip = (const byte *)prelude_offset; + o->code_state.ip = prelude_ptr; o->code_state.n_state = n_state; - mp_setup_code_state(&o->code_state, n_args, n_kw, args); + o->code_state.sp = &o->code_state.state[0] - 1; + mp_setup_code_state_native(&o->code_state, n_args, n_kw, args); // Indicate we are a native function, which doesn't use this variable o->code_state.exc_sp_idx = MP_CODE_STATE_EXC_SP_IDX_SENTINEL; @@ -223,7 +257,13 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ #endif // If the generator is started, allow sending a value. - if (self->code_state.sp == self->code_state.state - 1) { + void *state_start = self->code_state.state - 1; + #if MICROPY_EMIT_NATIVE + if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) { + state_start = ((mp_obj_gen_instance_native_t *)self)->code_state.state - 1; + } + #endif + if (self->code_state.sp == state_start) { if (send_value != mp_const_none) { mp_raise_TypeError(MP_ERROR_TEXT("can't send non-None value to a just-started generator")); } @@ -236,7 +276,7 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ // Set up the correct globals context for the generator and execute it self->code_state.old_globals = mp_globals_get(); - mp_globals_set(self->code_state.fun_bc->globals); + mp_globals_set(self->code_state.fun_bc->context->module.globals); mp_vm_return_kind_t ret_kind; @@ -278,7 +318,14 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ case MP_VM_RETURN_EXCEPTION: { self->code_state.ip = 0; - *ret_val = self->code_state.state[0]; + #if MICROPY_EMIT_NATIVE + if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) { + *ret_val = ((mp_obj_gen_instance_native_t *)self)->code_state.state[0]; + } else + #endif + { + *ret_val = self->code_state.state[0]; + } // PEP479: if StopIteration is raised inside a generator it is replaced with RuntimeError if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(*ret_val)), MP_OBJ_FROM_PTR(&mp_type_StopIteration))) { *ret_val = mp_obj_new_exception_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("generator raised StopIteration")); diff --git a/py/objgenerator.h b/py/objgenerator.h index 4b7f8c1ac5..80bf9cd860 100644 --- a/py/objgenerator.h +++ b/py/objgenerator.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objgetitemiter.c b/py/objgetitemiter.c index 098e7da557..d841212649 100644 --- a/py/objgetitemiter.c +++ b/py/objgetitemiter.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objint.c b/py/objint.c index 6dc675823f..83987d690a 100644 --- a/py/objint.c +++ b/py/objint.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -396,7 +396,7 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i // This is called only with strings whose value doesn't fit in SMALL_INT mp_obj_t mp_obj_new_int_from_str_len(const char **str, size_t len, bool neg, unsigned int base) { - mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("small int overflow")); + mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("No long integer support")); return mp_const_none; } diff --git a/py/objint.h b/py/objint.h index fb491914f2..83f12bac6c 100644 --- a/py/objint.h +++ b/py/objint.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objint_longlong.c b/py/objint_longlong.c index d318eab38e..ba23825abf 100644 --- a/py/objint_longlong.c +++ b/py/objint_longlong.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * Copyright (c) 2014 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -256,8 +256,7 @@ mp_obj_t mp_obj_new_int_from_uint(mp_uint_t value) { } mp_obj_t mp_obj_new_int_from_ll(long long val) { - mp_obj_int_t *o = m_new_obj(mp_obj_int_t); - o->base.type = &mp_type_int; + mp_obj_int_t *o = mp_obj_malloc(mp_obj_int_t, &mp_type_int); o->val = val; return o; } @@ -267,8 +266,7 @@ mp_obj_t mp_obj_new_int_from_ull(unsigned long long val) { if (val >> (sizeof(unsigned long long) * 8 - 1) != 0) { mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("ulonglong too large")); } - mp_obj_int_t *o = m_new_obj(mp_obj_int_t); - o->base.type = &mp_type_int; + mp_obj_int_t *o = mp_obj_malloc(mp_obj_int_t, &mp_type_int); o->val = val; return o; } @@ -276,8 +274,7 @@ mp_obj_t mp_obj_new_int_from_ull(unsigned long long val) { mp_obj_t mp_obj_new_int_from_str_len(const char **str, size_t len, bool neg, unsigned int base) { // TODO this does not honor the given length of the string, but it all cases it should anyway be null terminated // TODO check overflow - mp_obj_int_t *o = m_new_obj(mp_obj_int_t); - o->base.type = &mp_type_int; + mp_obj_int_t *o = mp_obj_malloc(mp_obj_int_t, &mp_type_int); char *endptr; o->val = strtoll(*str, &endptr, base); *str = endptr; diff --git a/py/objint_mpz.c b/py/objint_mpz.c index e147e5f08a..603a5b986c 100644 --- a/py/objint_mpz.c +++ b/py/objint_mpz.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -77,8 +77,7 @@ const mp_obj_int_t mp_sys_maxsize_obj = { #endif mp_obj_int_t *mp_obj_int_new_mpz(void) { - mp_obj_int_t *o = m_new_obj(mp_obj_int_t); - o->base.type = &mp_type_int; + mp_obj_int_t *o = mp_obj_malloc(mp_obj_int_t, &mp_type_int); mpz_init_zero(&o->mpz); return o; } diff --git a/py/objlist.c b/py/objlist.c index 434bca7d0a..f95c322bd5 100644 --- a/py/objlist.c +++ b/py/objlist.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objlist.h b/py/objlist.h index eb005e81cf..79ed6c288c 100644 --- a/py/objlist.h +++ b/py/objlist.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objmap.c b/py/objmap.c index dc1dc1387d..d87105c089 100644 --- a/py/objmap.c +++ b/py/objmap.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,8 +38,7 @@ typedef struct _mp_obj_map_t { STATIC mp_obj_t map_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 2, MP_OBJ_FUN_ARGS_MAX, false); - mp_obj_map_t *o = m_new_obj_var(mp_obj_map_t, mp_obj_t, n_args - 1); - o->base.type = type; + mp_obj_map_t *o = mp_obj_malloc_var(mp_obj_map_t, mp_obj_t, n_args - 1, type); o->n_iters = n_args - 1; o->fun = args[0]; for (size_t i = 0; i < n_args - 1; i++) { diff --git a/py/objmodule.c b/py/objmodule.c index 8f04a44597..965636eb5e 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2019 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014-2015 Paul Sokolovsky + * Copyright (c) 2013-2019 Damien P. George + * Copyright (c) 2014-2015 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,12 +29,16 @@ #include #include -#include "py/gc.h" +#include "py/bc.h" #include "py/objmodule.h" #include "py/runtime.h" #include "py/builtin.h" +#ifndef NO_QSTR +// Only include module definitions when not doing qstr extraction, because the +// qstr extraction stage also generates this module definition header file. #include "genhdr/moduledefs.h" +#endif #if MICROPY_MODULE_BUILTIN_INIT STATIC void mp_module_call_init(mp_obj_t module_name, mp_obj_t module_obj); @@ -63,6 +67,21 @@ STATIC void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kin mp_printf(print, "", module_name); } +STATIC void module_attr_try_delegation(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + #if MICROPY_MODULE_ATTR_DELEGATION + // Delegate lookup to a module's custom attr method (found in last lot of globals dict). + mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in); + mp_map_t *map = &self->globals->map; + if (map->table[map->alloc - 1].key == MP_OBJ_NEW_QSTR(MP_QSTRnull)) { + ((mp_attr_fun_t)MP_OBJ_TO_PTR(map->table[map->alloc - 1].value))(self_in, attr, dest); + } + #else + (void)self_in; + (void)attr; + (void)dest; + #endif +} + STATIC void module_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in); if (dest[0] == MP_OBJ_NULL) { @@ -75,14 +94,17 @@ STATIC void module_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { elem = mp_map_lookup(&self->globals->map, MP_OBJ_NEW_QSTR(MP_QSTR___getattr__), MP_MAP_LOOKUP); if (elem != NULL) { dest[0] = mp_call_function_1(elem->value, MP_OBJ_NEW_QSTR(attr)); + } else { + module_attr_try_delegation(self_in, attr, dest); } #endif + } else { + module_attr_try_delegation(self_in, attr, dest); } } else { // delete/store attribute mp_obj_dict_t *dict = self->globals; if (dict->map.is_fixed) { - mp_map_elem_t *elem = mp_map_lookup(&dict->map, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP); #if MICROPY_CAN_OVERRIDE_BUILTINS if (dict == &mp_module_builtins_globals) { if (MP_STATE_VM(mp_module_builtins_override_dict) == NULL) { @@ -91,13 +113,9 @@ STATIC void module_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { dict = MP_STATE_VM(mp_module_builtins_override_dict); } else #endif - // Return success if the given value is already in the dictionary. This is the case for - // native packages with native submodules. - if (elem != NULL && elem->value == dest[1]) { - dest[0] = MP_OBJ_NULL; // indicate success - return; - } else { + { // can't delete or store to fixed map + module_attr_try_delegation(self_in, attr, dest); return; } } @@ -106,9 +124,7 @@ STATIC void module_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { mp_obj_dict_delete(MP_OBJ_FROM_PTR(dict), MP_OBJ_NEW_QSTR(attr)); } else { // store attribute - mp_obj_t long_lived = MP_OBJ_FROM_PTR(gc_make_long_lived(MP_OBJ_TO_PTR(dest[1]))); - // TODO CPython allows STORE_ATTR to a module, but is this the correct implementation? - mp_obj_dict_store(MP_OBJ_FROM_PTR(dict), MP_OBJ_NEW_QSTR(attr), long_lived); + mp_obj_dict_store(MP_OBJ_FROM_PTR(dict), MP_OBJ_NEW_QSTR(attr), dest[1]); } dest[0] = MP_OBJ_NULL; // indicate success } @@ -131,12 +147,12 @@ mp_obj_t mp_obj_new_module(qstr module_name) { } // create new module object - mp_obj_module_t *o = m_new_ll_obj(mp_obj_module_t); - o->base.type = &mp_type_module; - o->globals = gc_make_long_lived(MP_OBJ_TO_PTR(mp_obj_new_dict(MICROPY_MODULE_DICT_SIZE))); + mp_module_context_t *o = m_new_obj(mp_module_context_t); + o->module.base.type = &mp_type_module; + o->module.globals = MP_OBJ_TO_PTR(mp_obj_new_dict(MICROPY_MODULE_DICT_SIZE)); // store __name__ entry in the module - mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(module_name)); + mp_obj_dict_store(MP_OBJ_FROM_PTR(o->module.globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(module_name)); // store the new module into the slot in the global dict holding all modules el->value = MP_OBJ_FROM_PTR(o); @@ -145,118 +161,12 @@ mp_obj_t mp_obj_new_module(qstr module_name) { return MP_OBJ_FROM_PTR(o); } -mp_obj_dict_t *mp_obj_module_get_globals(mp_obj_t self_in) { - assert(mp_obj_is_type(self_in, &mp_type_module)); - mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in); - return self->globals; -} - -void mp_obj_module_set_globals(mp_obj_t self_in, mp_obj_dict_t *globals) { - assert(mp_obj_is_type(self_in, &mp_type_module)); - mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in); - self->globals = globals; -} - /******************************************************************************/ // Global module table and related functions STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = { - { MP_ROM_QSTR(MP_QSTR___main__), MP_ROM_PTR(&mp_module___main__) }, - { MP_ROM_QSTR(MP_QSTR_builtins), MP_ROM_PTR(&mp_module_builtins) }, - { MP_ROM_QSTR(MP_QSTR_micropython), MP_ROM_PTR(&mp_module_micropython) }, - - #if MICROPY_PY_IO - #if CIRCUITPY - { MP_ROM_QSTR(MP_QSTR_io), MP_ROM_PTR(&mp_module_io) }, - #else - { MP_ROM_QSTR(MP_QSTR_uio), MP_ROM_PTR(&mp_module_io) }, - #endif - #endif - #if MICROPY_PY_COLLECTIONS - { MP_ROM_QSTR(MP_QSTR_collections), MP_ROM_PTR(&mp_module_collections) }, - #endif -// CircuitPython: Now in shared-bindings/, so not defined here. - #if MICROPY_PY_STRUCT - { MP_ROM_QSTR(MP_QSTR_ustruct), MP_ROM_PTR(&mp_module_ustruct) }, - #endif - - #if MICROPY_PY_BUILTINS_FLOAT - #if MICROPY_PY_MATH - { MP_ROM_QSTR(MP_QSTR_math), MP_ROM_PTR(&mp_module_math) }, - #endif - #if MICROPY_PY_BUILTINS_COMPLEX && MICROPY_PY_CMATH - { MP_ROM_QSTR(MP_QSTR_cmath), MP_ROM_PTR(&mp_module_cmath) }, - #endif - #endif - #if MICROPY_PY_SYS - { MP_ROM_QSTR(MP_QSTR_sys), MP_ROM_PTR(&mp_module_sys) }, - #endif - #if MICROPY_PY_GC && MICROPY_ENABLE_GC - { MP_ROM_QSTR(MP_QSTR_gc), MP_ROM_PTR(&mp_module_gc) }, - #endif - #if MICROPY_PY_THREAD - { MP_ROM_QSTR(MP_QSTR__thread), MP_ROM_PTR(&mp_module_thread) }, - #endif - - // extmod modules - - // Modules included in CircuitPython are registered using MP_REGISTER_MODULE, - // and do not have the "u" prefix. - - #if MICROPY_PY_UASYNCIO && !CIRCUITPY - { MP_ROM_QSTR(MP_QSTR__uasyncio), MP_ROM_PTR(&mp_module_uasyncio) }, - #endif - #if MICROPY_PY_UERRNO && !CIRCUITPY - { MP_ROM_QSTR(MP_QSTR_uerrno), MP_ROM_PTR(&mp_module_uerrno) }, - #endif - #if MICROPY_PY_UCTYPES - { MP_ROM_QSTR(MP_QSTR_uctypes), MP_ROM_PTR(&mp_module_uctypes) }, - #endif - #if MICROPY_PY_UZLIB - { MP_ROM_QSTR(MP_QSTR_uzlib), MP_ROM_PTR(&mp_module_uzlib) }, - #endif - #if MICROPY_PY_UJSON && !CIRCUITPY - { MP_ROM_QSTR(MP_QSTR_ujson), MP_ROM_PTR(&mp_module_ujson) }, - #endif - #if MICROPY_PY_URE && !CIRCUITPY - { MP_ROM_QSTR(MP_QSTR_ure), MP_ROM_PTR(&mp_module_ure) }, - #endif - #if MICROPY_PY_UHEAPQ - { MP_ROM_QSTR(MP_QSTR_uheapq), MP_ROM_PTR(&mp_module_uheapq) }, - #endif - #if MICROPY_PY_UTIMEQ - { MP_ROM_QSTR(MP_QSTR_utimeq), MP_ROM_PTR(&mp_module_utimeq) }, - #endif - #if MICROPY_PY_UHASHLIB - { MP_ROM_QSTR(MP_QSTR_hashlib), MP_ROM_PTR(&mp_module_uhashlib) }, - #endif - #if MICROPY_PY_UBINASCII && !CIRCUITPY - { MP_ROM_QSTR(MP_QSTR_ubinascii), MP_ROM_PTR(&mp_module_ubinascii) }, - #endif - #if MICROPY_PY_URANDOM - { MP_ROM_QSTR(MP_QSTR_urandom), MP_ROM_PTR(&mp_module_urandom) }, - #endif - #if MICROPY_PY_USELECT - { MP_ROM_QSTR(MP_QSTR_uselect), MP_ROM_PTR(&mp_module_uselect) }, - #endif - #if MICROPY_PY_FRAMEBUF - { MP_ROM_QSTR(MP_QSTR_framebuf), MP_ROM_PTR(&mp_module_framebuf) }, - #endif - #if MICROPY_PY_BTREE - { MP_ROM_QSTR(MP_QSTR_btree), MP_ROM_PTR(&mp_module_btree) }, - #endif - - // extra builtin modules as defined by a port - MICROPY_PORT_BUILTIN_MODULES - - #ifdef MICROPY_REGISTERED_MODULES // builtin modules declared with MP_REGISTER_MODULE() MICROPY_REGISTERED_MODULES - #endif - - #if defined(MICROPY_DEBUG_MODULES) && defined(MICROPY_PORT_BUILTIN_DEBUG_MODULES) - , MICROPY_PORT_BUILTIN_DEBUG_MODULES - #endif }; MP_DEFINE_CONST_MAP(mp_builtin_module_map, mp_builtin_module_table); @@ -324,3 +234,19 @@ STATIC void mp_module_call_init(mp_obj_t module_name, mp_obj_t module_obj) { } } #endif + +void mp_module_generic_attr(qstr attr, mp_obj_t *dest, const uint16_t *keys, mp_obj_t *values) { + for (size_t i = 0; keys[i] != MP_QSTRnull; ++i) { + if (attr == keys[i]) { + if (dest[0] == MP_OBJ_NULL) { + // load attribute (MP_OBJ_NULL returned for deleted items) + dest[0] = values[i]; + } else { + // delete or store (delete stores MP_OBJ_NULL) + values[i] = dest[1]; + dest[0] = MP_OBJ_NULL; // indicate success + } + return; + } + } +} diff --git a/py/objmodule.h b/py/objmodule.h index 5e54dbf3ab..d11d5bcd74 100644 --- a/py/objmodule.h +++ b/py/objmodule.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2019 Damien P. George + * Copyright (c) 2013-2019 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,9 @@ #include "py/obj.h" +// Place at the very end of a module's globals_table. +#define MP_MODULE_ATTR_DELEGATION_ENTRY(ptr) { MP_ROM_QSTR(MP_QSTRnull), MP_ROM_PTR(ptr) } + extern const mp_map_t mp_builtin_module_map; mp_obj_t mp_module_get_loaded_or_builtin(qstr module_name); @@ -35,4 +38,6 @@ mp_obj_t mp_module_get_loaded_or_builtin(qstr module_name); mp_obj_t mp_module_get_builtin(qstr module_name); #endif +void mp_module_generic_attr(qstr attr, mp_obj_t *dest, const uint16_t *keys, mp_obj_t *values); + #endif // MICROPY_INCLUDED_PY_OBJMODULE_H diff --git a/py/objnamedtuple.c b/py/objnamedtuple.c index 60a0a31847..1139ab2b54 100644 --- a/py/objnamedtuple.c +++ b/py/objnamedtuple.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objnamedtuple.h b/py/objnamedtuple.h index 4ebefb9d3a..b6c37f3909 100644 --- a/py/objnamedtuple.h +++ b/py/objnamedtuple.h @@ -1,10 +1,9 @@ /* - * This file is part of the Micro Python project, http://micropython.org/ + * This file is part of the MicroPython project, http://micropython.org/ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * Copyright (c) 2014 Paul Sokolovsky + * Copyright (c) 2014-2017 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,7 +23,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ - #ifndef MICROPY_INCLUDED_PY_OBJNAMEDTUPLE_H #define MICROPY_INCLUDED_PY_OBJNAMEDTUPLE_H diff --git a/py/objnone.c b/py/objnone.c index 2c33d8902f..9f6960669a 100644 --- a/py/objnone.c +++ b/py/objnone.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objobject.c b/py/objobject.c index b32e5fac0d..32552775aa 100644 --- a/py/objobject.c +++ b/py/objobject.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,8 +38,7 @@ typedef struct _mp_obj_object_t { STATIC mp_obj_t object_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { (void)args; mp_arg_check_num(n_args, n_kw, 0, 0, false); - mp_obj_object_t *o = m_new_obj(mp_obj_object_t); - o->base.type = type; + mp_obj_object_t *o = mp_obj_malloc(mp_obj_object_t, type); return MP_OBJ_FROM_PTR(o); } diff --git a/py/objproperty.c b/py/objproperty.c index f55efc8bdb..b0436e5c1f 100644 --- a/py/objproperty.c +++ b/py/objproperty.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -44,8 +44,7 @@ STATIC mp_obj_t property_make_new(const mp_obj_type_t *type, size_t n_args, size mp_arg_val_t vals[MP_ARRAY_SIZE(allowed_args)]; mp_arg_parse_all_kw_array(n_args, n_kw, args, MP_ARRAY_SIZE(allowed_args), allowed_args, vals); - mp_obj_property_t *o = m_new_obj(mp_obj_property_t); - o->base.type = type; + mp_obj_property_t *o = mp_obj_malloc(mp_obj_property_t, type); o->proxy[0] = vals[ARG_fget].u_obj; o->proxy[1] = vals[ARG_fset].u_obj; o->proxy[2] = vals[ARG_fdel].u_obj; diff --git a/py/objrange.c b/py/objrange.c index 78a5fcd0f8..d83b55d826 100644 --- a/py/objrange.c +++ b/py/objrange.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -97,8 +97,7 @@ STATIC void range_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind STATIC mp_obj_t range_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 1, 3, false); - mp_obj_range_t *o = m_new_obj(mp_obj_range_t); - o->base.type = type; + mp_obj_range_t *o = mp_obj_malloc(mp_obj_range_t, type); o->start = 0; o->step = 1; @@ -173,8 +172,7 @@ STATIC mp_obj_t range_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { if (mp_obj_is_type(index, &mp_type_slice)) { mp_bound_slice_t slice; mp_seq_get_fast_slice_indexes(len, index, &slice); - mp_obj_range_t *o = m_new_obj(mp_obj_range_t); - o->base.type = &mp_type_range; + mp_obj_range_t *o = mp_obj_malloc(mp_obj_range_t, &mp_type_range); o->start = self->start + slice.start * self->step; o->stop = self->start + slice.stop * self->step; o->step = slice.step * self->step; diff --git a/py/objreversed.c b/py/objreversed.c index fe517cc37f..c9b36bf2a4 100644 --- a/py/objreversed.c +++ b/py/objreversed.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George + * Copyright (c) 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -47,8 +47,7 @@ STATIC mp_obj_t reversed_make_new(const mp_obj_type_t *type, size_t n_args, size return mp_call_method_n_kw(0, 0, dest); } - mp_obj_reversed_t *o = m_new_obj(mp_obj_reversed_t); - o->base.type = type; + mp_obj_reversed_t *o = mp_obj_malloc(mp_obj_reversed_t, type); o->seq = args[0]; o->cur_index = mp_obj_get_int(mp_obj_len(args[0])); // start at the end of the sequence diff --git a/py/objset.c b/py/objset.c index 237e4c2471..9557ec76ab 100644 --- a/py/objset.c +++ b/py/objset.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2017 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -176,8 +176,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(set_clear_obj, set_clear); STATIC mp_obj_t set_copy(mp_obj_t self_in) { check_set_or_frozenset(self_in); mp_obj_set_t *self = MP_OBJ_TO_PTR(self_in); - mp_obj_set_t *other = m_new_obj(mp_obj_set_t); - other->base.type = self->base.type; + mp_obj_set_t *other = mp_obj_malloc(mp_obj_set_t, self->base.type); mp_set_init(&other->set, self->set.alloc); other->set.used = self->set.used; memcpy(other->set.table, self->set.table, self->set.alloc * sizeof(mp_obj_t)); @@ -587,8 +586,7 @@ const mp_obj_type_t mp_type_frozenset = { #endif mp_obj_t mp_obj_new_set(size_t n_args, mp_obj_t *items) { - mp_obj_set_t *o = m_new_obj(mp_obj_set_t); - o->base.type = &mp_type_set; + mp_obj_set_t *o = mp_obj_malloc(mp_obj_set_t, &mp_type_set); mp_set_init(&o->set, n_args); for (size_t i = 0; i < n_args; i++) { mp_set_lookup(&o->set, items[i], MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); diff --git a/py/objsingleton.c b/py/objsingleton.c index dfa6876dac..34a6ebd028 100644 --- a/py/objsingleton.c +++ b/py/objsingleton.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objslice.c b/py/objslice.c index 395af727e8..fe272cfa9b 100644 --- a/py/objslice.c +++ b/py/objslice.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -137,8 +137,7 @@ const mp_obj_type_t mp_type_slice = { }; mp_obj_t mp_obj_new_slice(mp_obj_t ostart, mp_obj_t ostop, mp_obj_t ostep) { - mp_obj_slice_t *o = m_new_obj(mp_obj_slice_t); - o->base.type = &mp_type_slice; + mp_obj_slice_t *o = mp_obj_malloc(mp_obj_slice_t, &mp_type_slice); o->start = ostart; o->stop = ostop; o->step = ostep; diff --git a/py/objstr.c b/py/objstr.c index fbb044c65e..72d44d537c 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014-2018 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014-2018 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -197,7 +197,7 @@ STATIC mp_obj_t bytes_make_new(const mp_obj_type_t *type_in, size_t n_args, size (void)type_in; #if MICROPY_CPYTHON_COMPAT - if (n_kw) { + if (n_kw != 0) { mp_arg_error_unimpl_kw(); } #else @@ -1183,7 +1183,7 @@ STATIC vstr_t mp_obj_str_format_helper(const char *str, const char *top, int *ar s++; } if (*s == '0') { - if (!align) { + if (!align && arg_looks_numeric(arg)) { align = '='; } if (!fill) { @@ -2049,8 +2049,7 @@ const mp_obj_str_t mp_const_empty_bytes_obj = {{&mp_type_bytes}, 0, 0, (const by // the data is copied across. This function should only be used if the type is bytes, // or if the type is str and the string data is known to be not interned. mp_obj_t mp_obj_new_str_copy(const mp_obj_type_t *type, const byte *data, size_t len) { - mp_obj_str_t *o = m_new_obj(mp_obj_str_t); - o->base.type = type; + mp_obj_str_t *o = mp_obj_malloc(mp_obj_str_t, type); o->len = len; if (data) { o->hash = qstr_compute_hash(data, len); @@ -2093,8 +2092,7 @@ mp_obj_t mp_obj_new_str_from_vstr(const mp_obj_type_t *type, vstr_t *vstr) { } // make a new str/bytes object - mp_obj_str_t *o = m_new_obj(mp_obj_str_t); - o->base.type = type; + mp_obj_str_t *o = mp_obj_malloc(mp_obj_str_t, type); o->len = vstr->len; o->hash = qstr_compute_hash((byte *)vstr->buf, vstr->len); if (vstr->len + 1 == vstr->alloc) { diff --git a/py/objstr.h b/py/objstr.h index 8031839146..bae32cbffe 100644 --- a/py/objstr.h +++ b/py/objstr.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objstringio.c b/py/objstringio.c index c6d22d6c89..563dd149bc 100644 --- a/py/objstringio.c +++ b/py/objstringio.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014-2017 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014-2017 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -179,8 +179,7 @@ STATIC mp_obj_t stringio___exit__(size_t n_args, const mp_obj_t *args) { STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(stringio___exit___obj, 4, 4, stringio___exit__); STATIC mp_obj_stringio_t *stringio_new(const mp_obj_type_t *type) { - mp_obj_stringio_t *o = m_new_obj(mp_obj_stringio_t); - o->base.type = type; + mp_obj_stringio_t *o = mp_obj_malloc(mp_obj_stringio_t, type); o->pos = 0; o->ref_obj = MP_OBJ_NULL; return o; diff --git a/py/objstringio.h b/py/objstringio.h index 38778f03ae..56738f4e45 100644 --- a/py/objstringio.h +++ b/py/objstringio.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George + * Copyright (c) 2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objstrunicode.c b/py/objstrunicode.c index b3b23d0f01..93277e630d 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014-2016 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014-2016 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objtuple.c b/py/objtuple.c index 87d16905df..9499b086c5 100644 --- a/py/objtuple.c +++ b/py/objtuple.c @@ -34,6 +34,8 @@ #include "supervisor/shared/translate/translate.h" +// type check is done on getiter method to allow tuple, namedtuple, attrtuple +#define mp_obj_is_tuple_compatible(o) (mp_obj_get_type(o)->getiter == mp_obj_tuple_getiter) /******************************************************************************/ /* tuple */ @@ -253,8 +255,7 @@ mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items) { if (n == 0) { return mp_const_empty_tuple; } - mp_obj_tuple_t *o = m_new_obj_var(mp_obj_tuple_t, mp_obj_t, n); - o->base.type = &mp_type_tuple; + mp_obj_tuple_t *o = mp_obj_malloc_var(mp_obj_tuple_t, mp_obj_t, n, &mp_type_tuple); o->len = n; if (items) { for (size_t i = 0; i < n; i++) { diff --git a/py/objtuple.h b/py/objtuple.h index ded265b47e..2d3026fca5 100644 --- a/py/objtuple.h +++ b/py/objtuple.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/objtype.c b/py/objtype.c index 5b04c361e1..30899c2324 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -123,8 +123,7 @@ STATIC mp_obj_instance_t *mp_obj_new_instance(const mp_obj_type_t *class, const mp_obj_type_t **native_base) { size_t num_native_bases = instance_count_native_bases(class, native_base); assert(num_native_bases < 2); - mp_obj_instance_t *o = m_new_obj_var(mp_obj_instance_t, mp_obj_t, num_native_bases); - o->base.type = class; + mp_obj_instance_t *o = mp_obj_malloc_var(mp_obj_instance_t, mp_obj_t, num_native_bases, class); mp_map_init(&o->members, 0); // Initialise the native base-class slot (should be 1 at most) with a valid // object. It doesn't matter which object, so long as it can be uniquely @@ -588,6 +587,7 @@ retry:; } else if (dest[0] != MP_OBJ_NULL) { dest[2] = rhs_in; res = mp_call_method_n_kw(1, 0, dest); + res = op == MP_BINARY_OP_CONTAINS ? mp_obj_new_bool(mp_obj_is_true(res)) : res; } else { // If this was an inplace method, fallback to normal method // https://docs.python.org/3/reference/datamodel.html#object.__iadd__ : diff --git a/py/objzip.c b/py/objzip.c index 91716493d6..69aca29e9a 100644 --- a/py/objzip.c +++ b/py/objzip.c @@ -39,8 +39,7 @@ typedef struct _mp_obj_zip_t { STATIC mp_obj_t zip_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 0, MP_OBJ_FUN_ARGS_MAX, false); - mp_obj_zip_t *o = m_new_obj_var(mp_obj_zip_t, mp_obj_t, n_args); - o->base.type = type; + mp_obj_zip_t *o = mp_obj_malloc_var(mp_obj_zip_t, mp_obj_t, n_args, type); o->n_iters = n_args; for (size_t i = 0; i < n_args; i++) { o->iters[i] = mp_getiter(args[i], NULL); diff --git a/py/parse.c b/py/parse.c index b3be279c5b..03c9b2cd13 100644 --- a/py/parse.c +++ b/py/parse.c @@ -297,6 +297,16 @@ STATIC void *parser_alloc(parser_t *parser, size_t num_bytes) { } #pragma GCC diagnostic pop +#if MICROPY_COMP_CONST_TUPLE +STATIC void parser_free_parse_node_struct(parser_t *parser, mp_parse_node_struct_t *pns) { + mp_parse_chunk_t *chunk = parser->cur_chunk; + if (chunk->data <= (byte *)pns && (byte *)pns < chunk->data + chunk->union_.used) { + size_t num_bytes = sizeof(mp_parse_node_struct_t) + sizeof(mp_parse_node_t) * MP_PARSE_NODE_STRUCT_NUM_NODES(pns); + chunk->union_.used -= num_bytes; + } +} +#endif + STATIC void push_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t arg_i) { if (parser->rule_stack_top >= parser->rule_stack_alloc) { rule_stack_t *rs = m_renew(rule_stack_t, parser->rule_stack, parser->rule_stack_alloc, parser->rule_stack_alloc + MICROPY_ALLOC_PARSE_RULE_INC); @@ -323,6 +333,13 @@ STATIC uint8_t pop_rule(parser_t *parser, size_t *arg_i, size_t *src_line) { return rule_id; } +#if MICROPY_COMP_CONST_TUPLE +STATIC uint8_t peek_rule(parser_t *parser, size_t n) { + assert(parser->rule_stack_top > n); + return parser->rule_stack[parser->rule_stack_top - 1 - n].rule_id; +} +#endif + bool mp_parse_node_is_const_false(mp_parse_node_t pn) { return MP_PARSE_NODE_IS_TOKEN_KIND(pn, MP_TOKEN_KW_FALSE) || (MP_PARSE_NODE_IS_SMALL_INT(pn) && MP_PARSE_NODE_LEAF_SMALL_INT(pn) == 0); @@ -339,18 +356,83 @@ bool mp_parse_node_get_int_maybe(mp_parse_node_t pn, mp_obj_t *o) { return true; } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, RULE_const_object)) { mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn; - #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D - // nodes are 32-bit pointers, but need to extract 64-bit object - *o = (uint64_t)pns->nodes[0] | ((uint64_t)pns->nodes[1] << 32); - #else - *o = (mp_obj_t)pns->nodes[0]; - #endif + *o = mp_parse_node_extract_const_object(pns); return mp_obj_is_int(*o); } else { return false; } } +#if MICROPY_COMP_CONST_TUPLE || MICROPY_COMP_CONST +STATIC bool mp_parse_node_is_const(mp_parse_node_t pn) { + if (MP_PARSE_NODE_IS_SMALL_INT(pn)) { + // Small integer. + return true; + } else if (MP_PARSE_NODE_IS_LEAF(pn)) { + // Possible str, or constant literal. + uintptr_t kind = MP_PARSE_NODE_LEAF_KIND(pn); + if (kind == MP_PARSE_NODE_STRING) { + return true; + } else if (kind == MP_PARSE_NODE_TOKEN) { + uintptr_t arg = MP_PARSE_NODE_LEAF_ARG(pn); + return arg == MP_TOKEN_KW_NONE + || arg == MP_TOKEN_KW_FALSE + || arg == MP_TOKEN_KW_TRUE + || arg == MP_TOKEN_ELLIPSIS; + } + } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, RULE_const_object)) { + // Constant object. + return true; + } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, RULE_atom_paren)) { + // Possible empty tuple. + mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn; + return MP_PARSE_NODE_IS_NULL(pns->nodes[0]); + } + return false; +} + +STATIC mp_obj_t mp_parse_node_convert_to_obj(mp_parse_node_t pn) { + assert(mp_parse_node_is_const(pn)); + if (MP_PARSE_NODE_IS_SMALL_INT(pn)) { + mp_int_t arg = MP_PARSE_NODE_LEAF_SMALL_INT(pn); + #if MICROPY_DYNAMIC_COMPILER + mp_uint_t sign_mask = -((mp_uint_t)1 << (mp_dynamic_compiler.small_int_bits - 1)); + if (!((arg & sign_mask) == 0 || (arg & sign_mask) == sign_mask)) { + // Integer doesn't fit in a small-int, so create a multi-precision int object. + return mp_obj_new_int_from_ll(arg); + } + #endif + return MP_OBJ_NEW_SMALL_INT(arg); + } else if (MP_PARSE_NODE_IS_LEAF(pn)) { + uintptr_t kind = MP_PARSE_NODE_LEAF_KIND(pn); + uintptr_t arg = MP_PARSE_NODE_LEAF_ARG(pn); + if (kind == MP_PARSE_NODE_STRING) { + return MP_OBJ_NEW_QSTR(arg); + } else { + assert(MP_PARSE_NODE_LEAF_KIND(pn) == MP_PARSE_NODE_TOKEN); + switch (arg) { + case MP_TOKEN_KW_NONE: + return mp_const_none; + case MP_TOKEN_KW_FALSE: + return mp_const_false; + case MP_TOKEN_KW_TRUE: + return mp_const_true; + default: + assert(arg == MP_TOKEN_ELLIPSIS); + return MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj); + } + } + } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, RULE_const_object)) { + mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn; + return mp_parse_node_extract_const_object(pns); + } else { + assert(MP_PARSE_NODE_IS_STRUCT_KIND(pn, RULE_atom_paren)); + assert(MP_PARSE_NODE_IS_NULL(((mp_parse_node_struct_t *)pn)->nodes[0])); + return mp_const_empty_tuple; + } +} +#endif + size_t mp_parse_node_extract_list(mp_parse_node_t *pn, size_t pn_kind, mp_parse_node_t **nodes) { if (MP_PARSE_NODE_IS_NULL(*pn)) { *nodes = NULL; @@ -394,9 +476,6 @@ void mp_parse_node_print(const mp_print_t *print, mp_parse_node_t pn, size_t ind case MP_PARSE_NODE_STRING: mp_printf(print, "str(%s)\n", qstr_str(arg)); break; - case MP_PARSE_NODE_BYTES: - mp_printf(print, "bytes(%s)\n", qstr_str(arg)); - break; default: assert(MP_PARSE_NODE_LEAF_KIND(pn) == MP_PARSE_NODE_TOKEN); mp_printf(print, "tok(%u)\n", (uint)arg); @@ -406,11 +485,14 @@ void mp_parse_node_print(const mp_print_t *print, mp_parse_node_t pn, size_t ind // node must be a mp_parse_node_struct_t mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn; if (MP_PARSE_NODE_STRUCT_KIND(pns) == RULE_const_object) { + mp_obj_t obj = mp_parse_node_extract_const_object(pns); #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D - mp_printf(print, "literal const(%016llx)\n", (uint64_t)pns->nodes[0] | ((uint64_t)pns->nodes[1] << 32)); + mp_printf(print, "literal const(%016llx)=", obj); #else - mp_printf(print, "literal const(%p)\n", (mp_obj_t)pns->nodes[0]); + mp_printf(print, "literal const(%p)=", obj); #endif + mp_obj_print_helper(print, obj, PRINT_REPR); + mp_printf(print, "\n"); } else { size_t n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); #if MICROPY_DEBUG_PARSE_RULE_NAME @@ -469,16 +551,28 @@ STATIC mp_parse_node_t make_node_const_object(parser_t *parser, size_t src_line, return (mp_parse_node_t)pn; } -STATIC mp_parse_node_t mp_parse_node_new_small_int_checked(parser_t *parser, mp_obj_t o_val) { - (void)parser; - mp_int_t val = MP_OBJ_SMALL_INT_VALUE(o_val); - #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D - // A parse node is only 32-bits and the small-int value must fit in 31-bits - if (((val ^ (val << 1)) & 0xffffffff80000000) != 0) { - return make_node_const_object(parser, 0, o_val); +// Create a parse node represeting a constant object, possibly optimising the case of +// an integer, by putting the (small) integer value directly in the parse node itself. +STATIC mp_parse_node_t make_node_const_object_optimised(parser_t *parser, size_t src_line, mp_obj_t obj) { + if (mp_obj_is_small_int(obj)) { + mp_int_t val = MP_OBJ_SMALL_INT_VALUE(obj); + #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D + // A parse node is only 32-bits and the small-int value must fit in 31-bits + if (((val ^ (val << 1)) & 0xffffffff80000000) != 0) { + return make_node_const_object(parser, src_line, obj); + } + #endif + #if MICROPY_DYNAMIC_COMPILER + // Check that the integer value fits in target runtime's small-int + mp_uint_t sign_mask = -((mp_uint_t)1 << (mp_dynamic_compiler.small_int_bits - 1)); + if (!((val & sign_mask) == 0 || (val & sign_mask) == sign_mask)) { + return make_node_const_object(parser, src_line, obj); + } + #endif + return mp_parse_node_new_small_int(val); + } else { + return make_node_const_object(parser, src_line, obj); } - #endif - return mp_parse_node_new_small_int(val); } STATIC void push_result_token(parser_t *parser, uint8_t rule_id) { @@ -491,11 +585,7 @@ STATIC void push_result_token(parser_t *parser, uint8_t rule_id) { mp_map_elem_t *elem; if (rule_id == RULE_atom && (elem = mp_map_lookup(&parser->consts, MP_OBJ_NEW_QSTR(id), MP_MAP_LOOKUP)) != NULL) { - if (mp_obj_is_small_int(elem->value)) { - pn = mp_parse_node_new_small_int_checked(parser, elem->value); - } else { - pn = make_node_const_object(parser, lex->tok_line, elem->value); - } + pn = make_node_const_object_optimised(parser, lex->tok_line, elem->value); } else { pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, id); } @@ -505,16 +595,12 @@ STATIC void push_result_token(parser_t *parser, uint8_t rule_id) { #endif } else if (lex->tok_kind == MP_TOKEN_INTEGER) { mp_obj_t o = mp_parse_num_integer(lex->vstr.buf, lex->vstr.len, 0, lex); - if (mp_obj_is_small_int(o)) { - pn = mp_parse_node_new_small_int_checked(parser, o); - } else { - pn = make_node_const_object(parser, lex->tok_line, o); - } + pn = make_node_const_object_optimised(parser, lex->tok_line, o); } else if (lex->tok_kind == MP_TOKEN_FLOAT_OR_IMAG) { mp_obj_t o = mp_parse_num_decimal(lex->vstr.buf, lex->vstr.len, true, false, lex); pn = make_node_const_object(parser, lex->tok_line, o); - } else if (lex->tok_kind == MP_TOKEN_STRING || lex->tok_kind == MP_TOKEN_BYTES) { - // Don't automatically intern all strings/bytes. doc strings (which are usually large) + } else if (lex->tok_kind == MP_TOKEN_STRING) { + // Don't automatically intern all strings. Doc strings (which are usually large) // will be discarded by the compiler, and so we shouldn't intern them. qstr qst = MP_QSTRnull; if (lex->vstr.len <= MICROPY_ALLOC_PARSE_INTERN_STRING_LEN) { @@ -526,14 +612,16 @@ STATIC void push_result_token(parser_t *parser, uint8_t rule_id) { } if (qst != MP_QSTRnull) { // qstr exists, make a leaf node - pn = mp_parse_node_new_leaf(lex->tok_kind == MP_TOKEN_STRING ? MP_PARSE_NODE_STRING : MP_PARSE_NODE_BYTES, qst); + pn = mp_parse_node_new_leaf(MP_PARSE_NODE_STRING, qst); } else { - // not interned, make a node holding a pointer to the string/bytes object - mp_obj_t o = mp_obj_new_str_copy( - lex->tok_kind == MP_TOKEN_STRING ? &mp_type_str : &mp_type_bytes, - (const byte *)lex->vstr.buf, lex->vstr.len); + // not interned, make a node holding a pointer to the string object + mp_obj_t o = mp_obj_new_str_copy(&mp_type_str, (const byte *)lex->vstr.buf, lex->vstr.len); pn = make_node_const_object(parser, lex->tok_line, o); } + } else if (lex->tok_kind == MP_TOKEN_BYTES) { + // make a node holding a pointer to the bytes object + mp_obj_t o = mp_obj_new_bytes((const byte *)lex->vstr.buf, lex->vstr.len); + pn = make_node_const_object(parser, lex->tok_line, o); } else { pn = mp_parse_node_new_leaf(MP_PARSE_NODE_TOKEN, lex->tok_kind); } @@ -557,6 +645,11 @@ STATIC MP_DEFINE_CONST_MAP(mp_constants_map, mp_constants_table); STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t num_args); #if MICROPY_COMP_CONST_FOLDING +#if MICROPY_COMP_CONST_FOLDING_COMPILER_WORKAROUND +// Some versions of the xtensa-esp32-elf-gcc compiler generate wrong code if this +// function is static, so provide a hook for them to work around this problem. +MP_NOINLINE +#endif STATIC bool fold_logical_constants(parser_t *parser, uint8_t rule_id, size_t *num_args) { if (rule_id == RULE_or_test || rule_id == RULE_and_test) { @@ -721,14 +814,14 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { // get the value mp_parse_node_t pn_value = ((mp_parse_node_struct_t *)((mp_parse_node_struct_t *)pn1)->nodes[1])->nodes[0]; - mp_obj_t value; - if (!mp_parse_node_get_int_maybe(pn_value, &value)) { + if (!mp_parse_node_is_const(pn_value)) { mp_obj_t exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, - MP_ERROR_TEXT("constant must be an integer")); + MP_ERROR_TEXT("not a constant")); mp_obj_exception_add_traceback(exc, parser->lexer->source_name, ((mp_parse_node_struct_t *)pn1)->source_line, MP_QSTRnull); nlr_raise(exc); } + mp_obj_t value = mp_parse_node_convert_to_obj(pn_value); // store the value in the table of dynamic constants mp_map_elem_t *elem = mp_map_lookup(&parser->consts, MP_OBJ_NEW_QSTR(id), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); @@ -790,17 +883,65 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { for (size_t i = num_args; i > 0; i--) { pop_result(parser); } - if (mp_obj_is_small_int(arg0)) { - push_result_node(parser, mp_parse_node_new_small_int_checked(parser, arg0)); - } else { - // TODO reuse memory for parse node struct? - push_result_node(parser, make_node_const_object(parser, 0, arg0)); - } + push_result_node(parser, make_node_const_object_optimised(parser, 0, arg0)); return true; } #endif +#if MICROPY_COMP_CONST_TUPLE +STATIC bool build_tuple_from_stack(parser_t *parser, size_t src_line, size_t num_args) { + for (size_t i = num_args; i > 0;) { + mp_parse_node_t pn = peek_result(parser, --i); + if (!mp_parse_node_is_const(pn)) { + return false; + } + } + mp_obj_tuple_t *tuple = MP_OBJ_TO_PTR(mp_obj_new_tuple(num_args, NULL)); + for (size_t i = num_args; i > 0;) { + mp_parse_node_t pn = pop_result(parser); + tuple->items[--i] = mp_parse_node_convert_to_obj(pn); + if (MP_PARSE_NODE_IS_STRUCT(pn)) { + parser_free_parse_node_struct(parser, (mp_parse_node_struct_t *)pn); + } + } + push_result_node(parser, make_node_const_object(parser, src_line, MP_OBJ_FROM_PTR(tuple))); + return true; +} + +STATIC bool build_tuple(parser_t *parser, size_t src_line, uint8_t rule_id, size_t num_args) { + if (rule_id == RULE_testlist_comp) { + if (peek_rule(parser, 0) == RULE_atom_paren) { + // Tuple of the form "(a,)". + return build_tuple_from_stack(parser, src_line, num_args); + } + } + if (rule_id == RULE_testlist_comp_3c) { + assert(peek_rule(parser, 0) == RULE_testlist_comp_3b); + assert(peek_rule(parser, 1) == RULE_testlist_comp); + if (peek_rule(parser, 2) == RULE_atom_paren) { + // Tuple of the form "(a, b)". + if (build_tuple_from_stack(parser, src_line, num_args)) { + parser->rule_stack_top -= 2; // discard 2 rules + return true; + } + } + } + if (rule_id == RULE_testlist_star_expr + || rule_id == RULE_testlist + || rule_id == RULE_subscriptlist) { + // Tuple of the form: + // - x = a, b + // - return a, b + // - for x in a, b: pass + // - x[a, b] + return build_tuple_from_stack(parser, src_line, num_args); + } + + return false; +} +#endif + STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t num_args) { // Simplify and optimise certain rules, to reduce memory usage and simplify the compiler. if (rule_id == RULE_atom_paren) { @@ -857,6 +998,13 @@ STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, } #endif + #if MICROPY_COMP_CONST_TUPLE + if (build_tuple(parser, src_line, rule_id, num_args)) { + // we built a tuple from this rule so return straightaway + return; + } + #endif + mp_parse_node_struct_t *pn = parser_alloc(parser, sizeof(mp_parse_node_struct_t) + sizeof(mp_parse_node_t) * num_args); pn->source_line = src_line; pn->kind_num_nodes = (rule_id & 0xff) | (num_args << 8); diff --git a/py/parse.h b/py/parse.h index 5f1e30c2ff..d9d0e15314 100644 --- a/py/parse.h +++ b/py/parse.h @@ -39,15 +39,13 @@ struct _mp_lexer_t; // - xxxx...xx00: pointer to mp_parse_node_struct_t // - xx...xx0010: an identifier; bits 4 and above are the qstr // - xx...xx0110: a string; bits 4 and above are the qstr holding the value -// - xx...xx1010: a string of bytes; bits 4 and above are the qstr holding the value -// - xx...xx1110: a token; bits 4 and above are mp_token_kind_t +// - xx...xx1010: a token; bits 4 and above are mp_token_kind_t #define MP_PARSE_NODE_NULL (0) #define MP_PARSE_NODE_SMALL_INT (0x1) #define MP_PARSE_NODE_ID (0x02) #define MP_PARSE_NODE_STRING (0x06) -#define MP_PARSE_NODE_BYTES (0x0a) -#define MP_PARSE_NODE_TOKEN (0x0e) +#define MP_PARSE_NODE_TOKEN (0x0a) typedef uintptr_t mp_parse_node_t; // must be pointer size @@ -79,9 +77,20 @@ typedef struct _mp_parse_node_struct_t { static inline mp_parse_node_t mp_parse_node_new_small_int(mp_int_t val) { return (mp_parse_node_t)(MP_PARSE_NODE_SMALL_INT | ((mp_uint_t)val << 1)); } + static inline mp_parse_node_t mp_parse_node_new_leaf(size_t kind, mp_int_t arg) { return (mp_parse_node_t)(kind | ((mp_uint_t)arg << 4)); } + +static inline mp_obj_t mp_parse_node_extract_const_object(mp_parse_node_struct_t *pns) { + #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D + // nodes are 32-bit pointers, but need to extract 64-bit object + return (uint64_t)pns->nodes[0] | ((uint64_t)pns->nodes[1] << 32); + #else + return (mp_obj_t)pns->nodes[0]; + #endif +} + bool mp_parse_node_is_const_false(mp_parse_node_t pn); bool mp_parse_node_is_const_true(mp_parse_node_t pn); bool mp_parse_node_get_int_maybe(mp_parse_node_t pn, mp_obj_t *o); diff --git a/py/persistentcode.c b/py/persistentcode.c index 0431b30a29..e15dcffdea 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -36,8 +36,6 @@ #include "py/objstr.h" #include "py/mpthread.h" -#include "supervisor/shared/translate/translate.h" - #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE #include "py/smallint.h" @@ -50,72 +48,6 @@ #define MPY_FEATURE_ARCH_DYNAMIC MPY_FEATURE_ARCH #endif -#if MICROPY_PERSISTENT_CODE_LOAD || (MICROPY_PERSISTENT_CODE_SAVE && !MICROPY_DYNAMIC_COMPILER) -// The bytecode will depend on the number of bits in a small-int, and -// this function computes that (could make it a fixed constant, but it -// would need to be defined in mpconfigport.h). -STATIC int mp_small_int_bits(void) { - mp_int_t i = MP_SMALL_INT_MAX; - int n = 1; - while (i != 0) { - i >>= 1; - ++n; - } - return n; -} -#endif - -#define QSTR_WINDOW_SIZE (32) - -typedef struct _qstr_window_t { - uint16_t idx; // indexes the head of the window - uint16_t window[QSTR_WINDOW_SIZE]; -} qstr_window_t; - -// Push a qstr to the head of the window, and the tail qstr is overwritten -STATIC void qstr_window_push(qstr_window_t *qw, qstr qst) { - qw->idx = (qw->idx + 1) % QSTR_WINDOW_SIZE; - qw->window[qw->idx] = qst; -} - -// Pull an existing qstr from within the window to the head of the window -STATIC qstr qstr_window_pull(qstr_window_t *qw, size_t idx) { - qstr qst = qw->window[idx]; - if (idx > qw->idx) { - memmove(&qw->window[idx], &qw->window[idx + 1], (QSTR_WINDOW_SIZE - idx - 1) * sizeof(uint16_t)); - qw->window[QSTR_WINDOW_SIZE - 1] = qw->window[0]; - idx = 0; - } - memmove(&qw->window[idx], &qw->window[idx + 1], (qw->idx - idx) * sizeof(uint16_t)); - qw->window[qw->idx] = qst; - return qst; -} - -#if MICROPY_PERSISTENT_CODE_LOAD - -// Access a qstr at the given index, relative to the head of the window (0=head) -STATIC qstr qstr_window_access(qstr_window_t *qw, size_t idx) { - return qstr_window_pull(qw, (qw->idx + QSTR_WINDOW_SIZE - idx) % QSTR_WINDOW_SIZE); -} - -#endif - -#if MICROPY_PERSISTENT_CODE_SAVE - -// Insert a qstr at the head of the window, either by pulling an existing one or pushing a new one -STATIC size_t qstr_window_insert(qstr_window_t *qw, qstr qst) { - for (size_t idx = 0; idx < QSTR_WINDOW_SIZE; ++idx) { - if (qw->window[idx] == qst) { - qstr_window_pull(qw, idx); - return (qw->idx + QSTR_WINDOW_SIZE - idx) % QSTR_WINDOW_SIZE; - } - } - qstr_window_push(qw, qst); - return QSTR_WINDOW_SIZE; -} - -#endif - typedef struct _bytecode_prelude_t { uint n_state; uint n_exc_stack; @@ -126,97 +58,39 @@ typedef struct _bytecode_prelude_t { uint code_info_size; } bytecode_prelude_t; -// ip will point to start of opcodes -// return value will point to simple_name, source_file qstrs -STATIC byte *extract_prelude(const byte **ip, bytecode_prelude_t *prelude) { - MP_BC_PRELUDE_SIG_DECODE(*ip); - prelude->n_state = n_state; - prelude->n_exc_stack = n_exc_stack; - prelude->scope_flags = scope_flags; - prelude->n_pos_args = n_pos_args; - prelude->n_kwonly_args = n_kwonly_args; - prelude->n_def_pos_args = n_def_pos_args; - MP_BC_PRELUDE_SIZE_DECODE(*ip); - byte *ip_info = (byte *)*ip; - *ip += n_info; - *ip += n_cell; - return ip_info; -} - #endif // MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE #if MICROPY_PERSISTENT_CODE_LOAD #include "py/parsenum.h" -STATIC void raise_corrupt_mpy(void) { - mp_raise_RuntimeError(MP_ERROR_TEXT("Corrupt .mpy file")); -} - STATIC int read_byte(mp_reader_t *reader); -STATIC size_t read_uint(mp_reader_t *reader, byte **out); +STATIC size_t read_uint(mp_reader_t *reader); #if MICROPY_EMIT_MACHINE_CODE typedef struct _reloc_info_t { mp_reader_t *reader; - mp_uint_t *const_table; + mp_module_context_t *context; + uint8_t *rodata; + uint8_t *bss; } reloc_info_t; -#if MICROPY_EMIT_THUMB -STATIC void asm_thumb_rewrite_mov(uint8_t *pc, uint16_t val) { - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wcast-align" - // high part - *(uint16_t *)pc = (*(uint16_t *)pc & 0xfbf0) | (val >> 1 & 0x0400) | (val >> 12); - // low part - *(uint16_t *)(pc + 2) = (*(uint16_t *)(pc + 2) & 0x0f00) | (val << 4 & 0x7000) | (val & 0x00ff); - #pragma GCC diagnostic pop -} -#endif - -STATIC void arch_link_qstr(uint8_t *pc, bool is_obj, qstr qst) { - mp_uint_t val = qst; - if (is_obj) { - val = (mp_uint_t)MP_OBJ_NEW_QSTR(qst); - } - #if MICROPY_EMIT_X86 || MICROPY_EMIT_X64 || MICROPY_EMIT_ARM || MICROPY_EMIT_XTENSA || MICROPY_EMIT_XTENSAWIN - pc[0] = val & 0xff; - pc[1] = (val >> 8) & 0xff; - pc[2] = (val >> 16) & 0xff; - pc[3] = (val >> 24) & 0xff; - #elif MICROPY_EMIT_THUMB - if (is_obj) { - // qstr object, movw and movt - asm_thumb_rewrite_mov(pc, val); // movw - asm_thumb_rewrite_mov(pc + 4, val >> 16); // movt - } else { - // qstr number, movw instruction - asm_thumb_rewrite_mov(pc, val); // movw - } - #endif -} - void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) { // Relocate native code reloc_info_t *ri = ri_in; + uint8_t op; uintptr_t *addr_to_adjust = NULL; - - // Read the byte directly so that we don't error on EOF. - mp_uint_t op = ri->reader->readbyte(ri->reader->data); - while (op != 0xff && op != MP_READER_EOF) { + while ((op = read_byte(ri->reader)) != 0xff) { if (op & 1) { // Point to new location to make adjustments - size_t addr = read_uint(ri->reader, NULL); + size_t addr = read_uint(ri->reader); if ((addr & 1) == 0) { // Point to somewhere in text - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wcast-align" addr_to_adjust = &((uintptr_t *)text)[addr >> 1]; - #pragma GCC diagnostic pop } else { // Point to somewhere in rodata - addr_to_adjust = &((uintptr_t *)ri->const_table[1])[addr >> 1]; + addr_to_adjust = &((uintptr_t *)ri->rodata)[addr >> 1]; } } op >>= 1; @@ -225,61 +99,54 @@ void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) { if (op <= 5) { if (op & 1) { // Read in number of adjustments to make - n = read_uint(ri->reader, NULL); + n = read_uint(ri->reader); } op >>= 1; if (op == 0) { // Destination is text dest = reloc_text; + } else if (op == 1) { + // Destination is rodata + dest = (uintptr_t)ri->rodata; } else { - // Destination is rodata (op=1) or bss (op=1 if no rodata, else op=2) - dest = ri->const_table[op]; + // Destination is bss + dest = (uintptr_t)ri->bss; } } else if (op == 6) { + // Destination is qstr_table + dest = (uintptr_t)ri->context->constants.qstr_table; + } else if (op == 7) { + // Destination is obj_table + dest = (uintptr_t)ri->context->constants.obj_table; + } else if (op == 8) { // Destination is mp_fun_table itself dest = (uintptr_t)&mp_fun_table; } else { // Destination is an entry in mp_fun_table - dest = ((uintptr_t *)&mp_fun_table)[op - 7]; + dest = ((uintptr_t *)&mp_fun_table)[op - 9]; } while (n--) { *addr_to_adjust++ += dest; } - op = ri->reader->readbyte(ri->reader->data); } } #endif STATIC int read_byte(mp_reader_t *reader) { - mp_uint_t b = reader->readbyte(reader->data); - if (b == MP_READER_EOF) { - raise_corrupt_mpy(); - } - return b; + return reader->readbyte(reader->data); } STATIC void read_bytes(mp_reader_t *reader, byte *buf, size_t len) { while (len-- > 0) { - mp_uint_t b = reader->readbyte(reader->data); - if (b == MP_READER_EOF) { - raise_corrupt_mpy(); - } - *buf++ = b; + *buf++ = reader->readbyte(reader->data); } } -STATIC size_t read_uint(mp_reader_t *reader, byte **out) { +STATIC size_t read_uint(mp_reader_t *reader) { size_t unum = 0; for (;;) { - mp_uint_t b = reader->readbyte(reader->data); - if (b == MP_READER_EOF) { - raise_corrupt_mpy(); - } - if (out != NULL) { - **out = b; - ++*out; - } + byte b = reader->readbyte(reader->data); unum = (unum << 7) | (b & 0x7f); if ((b & 0x80) == 0) { break; @@ -288,122 +155,89 @@ STATIC size_t read_uint(mp_reader_t *reader, byte **out) { return unum; } -STATIC qstr load_qstr(mp_reader_t *reader, qstr_window_t *qw) { - size_t len = read_uint(reader, NULL); - if (len == 0) { - // static qstr - return read_byte(reader); - } +STATIC qstr load_qstr(mp_reader_t *reader) { + size_t len = read_uint(reader); if (len & 1) { - // qstr in window - return qstr_window_access(qw, len >> 1); + // static qstr + return len >> 1; } len >>= 1; char *str = m_new(char, len); read_bytes(reader, (byte *)str, len); + read_byte(reader); // read and discard null terminator qstr qst = qstr_from_strn(str, len); m_del(char, str, len); - qstr_window_push(qw, qst); return qst; } STATIC mp_obj_t load_obj(mp_reader_t *reader) { byte obj_type = read_byte(reader); - if (obj_type == 'e') { + #if MICROPY_EMIT_MACHINE_CODE + if (obj_type == MP_PERSISTENT_OBJ_FUN_TABLE) { + return MP_OBJ_FROM_PTR(&mp_fun_table); + } else + #endif + if (obj_type == MP_PERSISTENT_OBJ_NONE) { + return mp_const_none; + } else if (obj_type == MP_PERSISTENT_OBJ_FALSE) { + return mp_const_false; + } else if (obj_type == MP_PERSISTENT_OBJ_TRUE) { + return mp_const_true; + } else if (obj_type == MP_PERSISTENT_OBJ_ELLIPSIS) { return MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj); } else { - size_t len = read_uint(reader, NULL); + size_t len = read_uint(reader); + if (len == 0 && obj_type == MP_PERSISTENT_OBJ_BYTES) { + read_byte(reader); // skip null terminator + return mp_const_empty_bytes; + } else if (obj_type == MP_PERSISTENT_OBJ_TUPLE) { + mp_obj_tuple_t *tuple = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL)); + for (size_t i = 0; i < len; ++i) { + tuple->items[i] = load_obj(reader); + } + return MP_OBJ_FROM_PTR(tuple); + } vstr_t vstr; vstr_init_len(&vstr, len); read_bytes(reader, (byte *)vstr.buf, len); - if (obj_type == 's' || obj_type == 'b') { - return mp_obj_new_str_from_vstr(obj_type == 's' ? &mp_type_str : &mp_type_bytes, &vstr); - } else if (obj_type == 'i') { + if (obj_type == MP_PERSISTENT_OBJ_STR || obj_type == MP_PERSISTENT_OBJ_BYTES) { + read_byte(reader); // skip null terminator + return mp_obj_new_str_from_vstr(obj_type == MP_PERSISTENT_OBJ_STR ? &mp_type_str : &mp_type_bytes, &vstr); + } else if (obj_type == MP_PERSISTENT_OBJ_INT) { return mp_parse_num_integer(vstr.buf, vstr.len, 10, NULL); } else { - assert(obj_type == 'f' || obj_type == 'c'); - return mp_parse_num_decimal(vstr.buf, vstr.len, obj_type == 'c', false, NULL); + assert(obj_type == MP_PERSISTENT_OBJ_FLOAT || obj_type == MP_PERSISTENT_OBJ_COMPLEX); + return mp_parse_num_decimal(vstr.buf, vstr.len, obj_type == MP_PERSISTENT_OBJ_COMPLEX, false, NULL); } } } -STATIC void load_prelude_qstrs(mp_reader_t *reader, qstr_window_t *qw, byte *ip) { - qstr simple_name = load_qstr(reader, qw); - ip[0] = simple_name; - ip[1] = simple_name >> 8; - qstr source_file = load_qstr(reader, qw); - ip[2] = source_file; - ip[3] = source_file >> 8; -} - -STATIC void load_prelude(mp_reader_t *reader, qstr_window_t *qw, byte **ip, bytecode_prelude_t *prelude) { - // Read in the prelude header - byte *ip_read = *ip; - read_uint(reader, &ip_read); // read in n_state/etc (is effectively a var-uint) - read_uint(reader, &ip_read); // read in n_info/n_cell (is effectively a var-uint) - - // Prelude header has been read into *ip, now decode and extract values from it - extract_prelude((const byte **)ip, prelude); - - // Load qstrs in prelude - load_prelude_qstrs(reader, qw, ip_read); - ip_read += 4; - - // Read remaining code info - read_bytes(reader, ip_read, *ip - ip_read); -} - -STATIC void load_bytecode(mp_reader_t *reader, qstr_window_t *qw, byte *ip, byte *ip_top) { - while (ip < ip_top) { - *ip = read_byte(reader); - size_t sz; - uint f = mp_opcode_format(ip, &sz, false); - ++ip; - --sz; - if (f == MP_BC_FORMAT_QSTR) { - qstr qst = load_qstr(reader, qw); - *ip++ = qst; - *ip++ = qst >> 8; - sz -= 2; - } else if (f == MP_BC_FORMAT_VAR_UINT) { - while ((*ip++ = read_byte(reader)) & 0x80) { - } - } - read_bytes(reader, ip, sz); - ip += sz; - } -} - -STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { +STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *context) { // Load function kind and data length - size_t kind_len = read_uint(reader, NULL); + size_t kind_len = read_uint(reader); int kind = (kind_len & 3) + MP_CODE_BYTECODE; - size_t fun_data_len = kind_len >> 2; + bool has_children = !!(kind_len & 4); + size_t fun_data_len = kind_len >> 3; #if !MICROPY_EMIT_MACHINE_CODE if (kind != MP_CODE_BYTECODE) { - + mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file")); } #endif uint8_t *fun_data = NULL; - bytecode_prelude_t prelude = {0}; #if MICROPY_EMIT_MACHINE_CODE size_t prelude_offset = 0; - mp_uint_t type_sig = 0; - size_t n_qstr_link = 0; + mp_uint_t native_scope_flags = 0; + mp_uint_t native_n_pos_args = 0; + mp_uint_t native_type_sig = 0; #endif if (kind == MP_CODE_BYTECODE) { // Allocate memory for the bytecode fun_data = m_new(uint8_t, fun_data_len); - - // Load prelude - byte *ip = fun_data; - load_prelude(reader, qw, &ip, &prelude); - // Load bytecode - load_bytecode(reader, qw, ip, fun_data + fun_data_len); + read_bytes(reader, fun_data, fun_data_len); #if MICROPY_EMIT_MACHINE_CODE } else { @@ -412,138 +246,104 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { MP_PLAT_ALLOC_EXEC(fun_data_len, (void **)&fun_data, &fun_alloc); read_bytes(reader, fun_data, fun_data_len); - if (kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER) { - // Parse qstr link table and link native code - n_qstr_link = read_uint(reader, NULL); - for (size_t i = 0; i < n_qstr_link; ++i) { - size_t off = read_uint(reader, NULL); - qstr qst = load_qstr(reader, qw); - uint8_t *dest = fun_data + (off >> 2); - if ((off & 3) == 0) { - // Generic 16-bit link - dest[0] = qst & 0xff; - dest[1] = (qst >> 8) & 0xff; - } else if ((off & 3) == 3) { - // Generic, aligned qstr-object link - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wcast-align" - *(mp_obj_t *)dest = MP_OBJ_NEW_QSTR(qst); - #pragma GCC diagnostic pop - } else { - // Architecture-specific link - arch_link_qstr(dest, (off & 3) == 2, qst); - } - } - } - if (kind == MP_CODE_NATIVE_PY) { - // Extract prelude for later use - prelude_offset = read_uint(reader, NULL); + // Read prelude offset within fun_data, and extract scope flags. + prelude_offset = read_uint(reader); const byte *ip = fun_data + prelude_offset; - byte *ip_info = extract_prelude(&ip, &prelude); - // Load qstrs in prelude - load_prelude_qstrs(reader, qw, ip_info); + MP_BC_PRELUDE_SIG_DECODE(ip); + native_scope_flags = scope_flags; } else { - // Load basic scope info for viper and asm - prelude.scope_flags = read_uint(reader, NULL); - prelude.n_pos_args = 0; - prelude.n_kwonly_args = 0; + // Load basic scope info for viper and asm. + native_scope_flags = read_uint(reader); if (kind == MP_CODE_NATIVE_ASM) { - prelude.n_pos_args = read_uint(reader, NULL); - type_sig = read_uint(reader, NULL); + native_n_pos_args = read_uint(reader); + native_type_sig = read_uint(reader); } } #endif } - size_t n_obj = 0; - size_t n_raw_code = 0; - mp_uint_t *const_table = NULL; + size_t n_children = 0; + mp_raw_code_t **children = NULL; - if (kind != MP_CODE_NATIVE_ASM) { - // Load constant table for bytecode, native and viper - - // Number of entries in constant table - n_obj = read_uint(reader, NULL); - n_raw_code = read_uint(reader, NULL); - - // Allocate constant table - size_t n_alloc = prelude.n_pos_args + prelude.n_kwonly_args + n_obj + n_raw_code; - #if MICROPY_EMIT_MACHINE_CODE - if (kind != MP_CODE_BYTECODE) { - ++n_alloc; // additional entry for mp_fun_table - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { - ++n_alloc; // additional entry for rodata - } - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERBSS) { - ++n_alloc; // additional entry for BSS - } - } - #endif - - const_table = m_new(mp_uint_t, n_alloc); - mp_uint_t *ct = const_table; - - // Load function argument names (initial entries in const_table) - // (viper has n_pos_args=n_kwonly_args=0 so doesn't load any qstrs here) - for (size_t i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) { - *ct++ = (mp_uint_t)MP_OBJ_NEW_QSTR(load_qstr(reader, qw)); + #if MICROPY_EMIT_MACHINE_CODE + // Load optional BSS/rodata for viper. + uint8_t *rodata = NULL; + uint8_t *bss = NULL; + if (kind == MP_CODE_NATIVE_VIPER) { + size_t rodata_size = 0; + if (native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { + rodata_size = read_uint(reader); } - #if MICROPY_EMIT_MACHINE_CODE - if (kind != MP_CODE_BYTECODE) { - // Populate mp_fun_table entry - *ct++ = (mp_uint_t)(uintptr_t)&mp_fun_table; + size_t bss_size = 0; + if (native_scope_flags & MP_SCOPE_FLAG_VIPERBSS) { + bss_size = read_uint(reader); + } - // Allocate and load rodata if needed - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { - size_t size = read_uint(reader, NULL); - uint8_t *rodata = m_new(uint8_t, size); - read_bytes(reader, rodata, size); - *ct++ = (uintptr_t)rodata; + if (rodata_size + bss_size != 0) { + bss_size = (uintptr_t)MP_ALIGN(bss_size, sizeof(uintptr_t)); + uint8_t *data = m_new0(uint8_t, bss_size + rodata_size); + bss = data; + rodata = bss + bss_size; + if (native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { + read_bytes(reader, rodata, rodata_size); } - // Allocate BSS if needed - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERBSS) { - size_t size = read_uint(reader, NULL); - uint8_t *bss = m_new0(uint8_t, size); - *ct++ = (uintptr_t)bss; - } + // Viper code with BSS/rodata should not have any children. + // Reuse the children pointer to reference the BSS/rodata + // memory so that it is not reclaimed by the GC. + assert(!has_children); + children = (void *)data; } - #endif + } + #endif - // Load constant objects and raw code children - for (size_t i = 0; i < n_obj; ++i) { - *ct++ = (mp_uint_t)load_obj(reader); - } - for (size_t i = 0; i < n_raw_code; ++i) { - *ct++ = (mp_uint_t)(uintptr_t)load_raw_code(reader, qw); + // Load children if any. + if (has_children) { + n_children = read_uint(reader); + children = m_new(mp_raw_code_t *, n_children + (kind == MP_CODE_NATIVE_PY)); + for (size_t i = 0; i < n_children; ++i) { + children[i] = load_raw_code(reader, context); } } // Create raw_code and return it mp_raw_code_t *rc = mp_emit_glue_new_raw_code(); if (kind == MP_CODE_BYTECODE) { + const byte *ip = fun_data; + MP_BC_PRELUDE_SIG_DECODE(ip); // Assign bytecode to raw code object mp_emit_glue_assign_bytecode(rc, fun_data, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS fun_data_len, #endif - const_table, + children, #if MICROPY_PERSISTENT_CODE_SAVE - n_obj, n_raw_code, + n_children, #endif - prelude.scope_flags); + scope_flags); #if MICROPY_EMIT_MACHINE_CODE } else { + const uint8_t *prelude_ptr; + #if MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE + if (kind == MP_CODE_NATIVE_PY) { + // Executable code cannot be accessed byte-wise on this architecture, so copy + // the prelude to a separate memory region that is byte-wise readable. + void *buf = fun_data + prelude_offset; + size_t n = fun_data_len - prelude_offset; + prelude_ptr = memcpy(m_new(uint8_t, n), buf, n); + } + #endif + // Relocate and commit code to executable address space - reloc_info_t ri = {reader, const_table}; + reloc_info_t ri = {reader, context, rodata, bss}; #if defined(MP_PLAT_COMMIT_EXEC) - void *opt_ri = (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL; + void *opt_ri = (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL; fun_data = MP_PLAT_COMMIT_EXEC(fun_data, fun_data_len, opt_ri); #else - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRELOC) { + if (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) { #if MICROPY_PERSISTENT_CODE_TRACK_RELOC_CODE // If native code needs relocations then it's not guaranteed that a pointer to // the head of `buf` (containing the machine code) will be retained for the GC @@ -560,55 +360,90 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { } #endif + if (kind == MP_CODE_NATIVE_PY) { + #if !MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE + prelude_ptr = fun_data + prelude_offset; + #endif + if (n_children == 0) { + children = (void *)prelude_ptr; + } else { + children[n_children] = (void *)prelude_ptr; + } + } + // Assign native code to raw code object mp_emit_glue_assign_native(rc, kind, - fun_data, fun_data_len, const_table, + fun_data, fun_data_len, + children, #if MICROPY_PERSISTENT_CODE_SAVE + n_children, prelude_offset, - n_obj, n_raw_code, - n_qstr_link, NULL, #endif - prelude.n_pos_args, prelude.scope_flags, type_sig); + native_scope_flags, native_n_pos_args, native_type_sig + ); #endif } return rc; } -mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader) { +mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *context) { byte header[4]; read_bytes(reader, header, sizeof(header)); - if (header[0] != 'C' + if (header[0] != 'M' || header[1] != MPY_VERSION || MPY_FEATURE_DECODE_FLAGS(header[2]) != MPY_FEATURE_FLAGS - || header[3] > mp_small_int_bits() - || read_uint(reader, NULL) > QSTR_WINDOW_SIZE) { - mp_raise_ValueError(MP_ERROR_TEXT("Incompatible .mpy file. Please update all .mpy files. See http://adafru.it/mpy-update for more info.")); + || header[3] > MP_SMALL_INT_BITS) { + mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file")); } if (MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE) { byte arch = MPY_FEATURE_DECODE_ARCH(header[2]); if (!MPY_FEATURE_ARCH_TEST(arch)) { - mp_raise_ValueError(MP_ERROR_TEXT("incompatible native .mpy architecture")); + mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy arch")); } } - qstr_window_t qw; - qw.idx = 0; - mp_raw_code_t *rc = load_raw_code(reader, &qw); + + size_t n_qstr = read_uint(reader); + size_t n_obj = read_uint(reader); + mp_module_context_alloc_tables(context, n_qstr, n_obj); + + // Load qstrs. + for (size_t i = 0; i < n_qstr; ++i) { + context->constants.qstr_table[i] = load_qstr(reader); + } + + // Load constant objects. + for (size_t i = 0; i < n_obj; ++i) { + context->constants.obj_table[i] = load_obj(reader); + } + + // Load top-level module. + mp_compiled_module_t cm2; + cm2.rc = load_raw_code(reader, context); + cm2.context = context; + + #if MICROPY_PERSISTENT_CODE_SAVE + cm2.has_native = MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE; + cm2.n_qstr = n_qstr; + cm2.n_obj = n_obj; + #endif + reader->close(reader->data); - return rc; + + return cm2; } -mp_raw_code_t *mp_raw_code_load_mem(const byte *buf, size_t len) { +mp_compiled_module_t mp_raw_code_load_mem(const byte *buf, size_t len, mp_module_context_t *context) { mp_reader_t reader; mp_reader_new_mem(&reader, buf, len, 0); - return mp_raw_code_load(&reader); + return mp_raw_code_load(&reader, context); } #if MICROPY_HAS_FILE_READER -mp_raw_code_t *mp_raw_code_load_file(const char *filename) { +mp_compiled_module_t mp_raw_code_load_file(const char *filename, mp_module_context_t *context) { mp_reader_t reader; mp_reader_new_file(&reader, filename); - return mp_raw_code_load(&reader); + return mp_raw_code_load(&reader, context); } #endif // MICROPY_HAS_FILE_READER @@ -635,54 +470,72 @@ STATIC void mp_print_uint(mp_print_t *print, size_t n) { print->print_strn(print->data, (char *)p, buf + sizeof(buf) - p); } -STATIC void save_qstr(mp_print_t *print, qstr_window_t *qw, qstr qst) { +STATIC void save_qstr(mp_print_t *print, qstr qst) { if (qst <= QSTR_LAST_STATIC) { // encode static qstr - byte buf[2] = {0, qst & 0xff}; - mp_print_bytes(print, buf, 2); - return; - } - size_t idx = qstr_window_insert(qw, qst); - if (idx < QSTR_WINDOW_SIZE) { - // qstr found in window, encode index to it - mp_print_uint(print, idx << 1 | 1); + mp_print_uint(print, qst << 1 | 1); return; } size_t len; const byte *str = qstr_data(qst, &len); mp_print_uint(print, len << 1); - mp_print_bytes(print, str, len); + mp_print_bytes(print, str, len + 1); // +1 to store null terminator } STATIC void save_obj(mp_print_t *print, mp_obj_t o) { + #if MICROPY_EMIT_MACHINE_CODE + if (o == MP_OBJ_FROM_PTR(&mp_fun_table)) { + byte obj_type = MP_PERSISTENT_OBJ_FUN_TABLE; + mp_print_bytes(print, &obj_type, 1); + } else + #endif if (mp_obj_is_str_or_bytes(o)) { byte obj_type; if (mp_obj_is_str(o)) { - obj_type = 's'; + obj_type = MP_PERSISTENT_OBJ_STR; } else { - obj_type = 'b'; + obj_type = MP_PERSISTENT_OBJ_BYTES; } size_t len; const char *str = mp_obj_str_get_data(o, &len); mp_print_bytes(print, &obj_type, 1); mp_print_uint(print, len); - mp_print_bytes(print, (const byte *)str, len); - } else if (MP_OBJ_TO_PTR(o) == &mp_const_ellipsis_obj) { - byte obj_type = 'e'; + mp_print_bytes(print, (const byte *)str, len + 1); // +1 to store null terminator + } else if (o == mp_const_none) { + byte obj_type = MP_PERSISTENT_OBJ_NONE; mp_print_bytes(print, &obj_type, 1); + } else if (o == mp_const_false) { + byte obj_type = MP_PERSISTENT_OBJ_FALSE; + mp_print_bytes(print, &obj_type, 1); + } else if (o == mp_const_true) { + byte obj_type = MP_PERSISTENT_OBJ_TRUE; + mp_print_bytes(print, &obj_type, 1); + } else if (MP_OBJ_TO_PTR(o) == &mp_const_ellipsis_obj) { + byte obj_type = MP_PERSISTENT_OBJ_ELLIPSIS; + mp_print_bytes(print, &obj_type, 1); + } else if (mp_obj_is_type(o, &mp_type_tuple)) { + size_t len; + mp_obj_t *items; + mp_obj_tuple_get(o, &len, &items); + byte obj_type = MP_PERSISTENT_OBJ_TUPLE; + mp_print_bytes(print, &obj_type, 1); + mp_print_uint(print, len); + for (size_t i = 0; i < len; ++i) { + save_obj(print, items[i]); + } } else { // we save numbers using a simplistic text representation // TODO could be improved byte obj_type; - if (mp_obj_is_type(o, &mp_type_int)) { - obj_type = 'i'; + if (mp_obj_is_int(o)) { + obj_type = MP_PERSISTENT_OBJ_INT; #if MICROPY_PY_BUILTINS_COMPLEX } else if (mp_obj_is_type(o, &mp_type_complex)) { - obj_type = 'c'; + obj_type = MP_PERSISTENT_OBJ_COMPLEX; #endif } else { assert(mp_obj_is_float(o)); - obj_type = 'f'; + obj_type = MP_PERSISTENT_OBJ_FLOAT; } vstr_t vstr; mp_print_t pr; @@ -695,162 +548,72 @@ STATIC void save_obj(mp_print_t *print, mp_obj_t o) { } } -STATIC void save_prelude_qstrs(mp_print_t *print, qstr_window_t *qw, const byte *ip) { - save_qstr(print, qw, ip[0] | (ip[1] << 8)); // simple_name - save_qstr(print, qw, ip[2] | (ip[3] << 8)); // source_file -} - -STATIC void save_bytecode(mp_print_t *print, qstr_window_t *qw, const byte *ip, const byte *ip_top) { - while (ip < ip_top) { - size_t sz; - uint f = mp_opcode_format(ip, &sz, true); - if (f == MP_BC_FORMAT_QSTR) { - mp_print_bytes(print, ip, 1); - qstr qst = ip[1] | (ip[2] << 8); - save_qstr(print, qw, qst); - ip += 3; - sz -= 3; - } - mp_print_bytes(print, ip, sz); - ip += sz; - } -} - -STATIC void save_raw_code(mp_print_t *print, mp_raw_code_t *rc, qstr_window_t *qstr_window) { +STATIC void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) { // Save function kind and data length - mp_print_uint(print, (rc->fun_data_len << 2) | (rc->kind - MP_CODE_BYTECODE)); + mp_print_uint(print, (rc->fun_data_len << 3) | ((rc->n_children != 0) << 2) | (rc->kind - MP_CODE_BYTECODE)); - bytecode_prelude_t prelude; + // Save function code. + mp_print_bytes(print, rc->fun_data, rc->fun_data_len); - if (rc->kind == MP_CODE_BYTECODE) { - // Extract prelude - const byte *ip = rc->fun_data; - const byte *ip_info = extract_prelude(&ip, &prelude); - - // Save prelude - mp_print_bytes(print, rc->fun_data, ip_info - (const byte *)rc->fun_data); - save_prelude_qstrs(print, qstr_window, ip_info); - ip_info += 4; - mp_print_bytes(print, ip_info, ip - ip_info); - - // Save bytecode - const byte *ip_top = (const byte *)rc->fun_data + rc->fun_data_len; - save_bytecode(print, qstr_window, ip, ip_top); #if MICROPY_EMIT_MACHINE_CODE - } else { - // Save native code - mp_print_bytes(print, rc->fun_data, rc->fun_data_len); - - if (rc->kind == MP_CODE_NATIVE_PY || rc->kind == MP_CODE_NATIVE_VIPER) { - // Save qstr link table for native code - mp_print_uint(print, rc->n_qstr); - for (size_t i = 0; i < rc->n_qstr; ++i) { - mp_print_uint(print, rc->qstr_link[i].off); - save_qstr(print, qstr_window, rc->qstr_link[i].qst); - } - } - - if (rc->kind == MP_CODE_NATIVE_PY) { - // Save prelude size - mp_print_uint(print, rc->prelude_offset); - - // Extract prelude and save qstrs in prelude - const byte *ip = (const byte *)rc->fun_data + rc->prelude_offset; - const byte *ip_info = extract_prelude(&ip, &prelude); - save_prelude_qstrs(print, qstr_window, ip_info); - } else { - // Save basic scope info for viper and asm - mp_print_uint(print, rc->scope_flags & MP_SCOPE_FLAG_ALL_SIG); - prelude.n_pos_args = 0; - prelude.n_kwonly_args = 0; - if (rc->kind == MP_CODE_NATIVE_ASM) { - mp_print_uint(print, rc->n_pos_args); - mp_print_uint(print, rc->type_sig); - } + if (rc->kind == MP_CODE_NATIVE_PY) { + // Save prelude size + mp_print_uint(print, rc->prelude_offset); + } else if (rc->kind == MP_CODE_NATIVE_VIPER || rc->kind == MP_CODE_NATIVE_ASM) { + // Save basic scope info for viper and asm + mp_print_uint(print, rc->scope_flags & MP_SCOPE_FLAG_ALL_SIG); + if (rc->kind == MP_CODE_NATIVE_ASM) { + mp_print_uint(print, rc->n_pos_args); + mp_print_uint(print, rc->type_sig); } + } #endif - } - if (rc->kind != MP_CODE_NATIVE_ASM) { - // Save constant table for bytecode, native and viper - - // Number of entries in constant table - mp_print_uint(print, rc->n_obj); - mp_print_uint(print, rc->n_raw_code); - - const mp_uint_t *const_table = rc->const_table; - - // Save function argument names (initial entries in const_table) - // (viper has n_pos_args=n_kwonly_args=0 so doesn't save any qstrs here) - for (size_t i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) { - mp_obj_t o = (mp_obj_t)*const_table++; - save_qstr(print, qstr_window, MP_OBJ_QSTR_VALUE(o)); - } - - if (rc->kind != MP_CODE_BYTECODE) { - // Skip saving mp_fun_table entry - ++const_table; - } - - // Save constant objects and raw code children - for (size_t i = 0; i < rc->n_obj; ++i) { - save_obj(print, (mp_obj_t)*const_table++); - } - for (size_t i = 0; i < rc->n_raw_code; ++i) { - save_raw_code(print, (mp_raw_code_t *)(uintptr_t)*const_table++, qstr_window); + if (rc->n_children) { + mp_print_uint(print, rc->n_children); + for (size_t i = 0; i < rc->n_children; ++i) { + save_raw_code(print, rc->children[i]); } } } -STATIC bool mp_raw_code_has_native(mp_raw_code_t *rc) { - if (rc->kind != MP_CODE_BYTECODE) { - return true; - } - - const byte *ip = rc->fun_data; - bytecode_prelude_t prelude; - extract_prelude(&ip, &prelude); - - const mp_uint_t *const_table = rc->const_table - + prelude.n_pos_args + prelude.n_kwonly_args - + rc->n_obj; - - for (size_t i = 0; i < rc->n_raw_code; ++i) { - if (mp_raw_code_has_native((mp_raw_code_t *)(uintptr_t)*const_table++)) { - return true; - } - } - - return false; -} - -void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print) { +void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) { // header contains: - // byte 'C' + // byte 'M' // byte version // byte feature flags // byte number of bits in a small int - // uint size of qstr window byte header[4] = { - 'C', + 'M', MPY_VERSION, MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS_DYNAMIC), #if MICROPY_DYNAMIC_COMPILER mp_dynamic_compiler.small_int_bits, #else - mp_small_int_bits(), + MP_SMALL_INT_BITS, #endif }; - if (mp_raw_code_has_native(rc)) { + if (cm->has_native) { header[2] |= MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH_DYNAMIC); } mp_print_bytes(print, header, sizeof(header)); - mp_print_uint(print, QSTR_WINDOW_SIZE); - qstr_window_t qw; - qw.idx = 0; - memset(qw.window, 0, sizeof(qw.window)); - save_raw_code(print, rc, &qw); + // Number of entries in constant table. + mp_print_uint(print, cm->n_qstr); + mp_print_uint(print, cm->n_obj); + + // Save qstrs. + for (size_t i = 0; i < cm->n_qstr; ++i) { + save_qstr(print, cm->context->constants.qstr_table[i]); + } + + // Save constant objects. + for (size_t i = 0; i < cm->n_obj; ++i) { + save_obj(print, (mp_obj_t)cm->context->constants.obj_table[i]); + } + + // Save outer raw code, which will save all its child raw codes. + save_raw_code(print, cm->rc); } #if MICROPY_PERSISTENT_CODE_SAVE_FILE @@ -867,12 +630,12 @@ STATIC void fd_print_strn(void *env, const char *str, size_t len) { (void)ret; } -void mp_raw_code_save_file(mp_raw_code_t *rc, const char *filename) { +void mp_raw_code_save_file(mp_compiled_module_t *cm, const char *filename) { MP_THREAD_GIL_EXIT(); int fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); MP_THREAD_GIL_ENTER(); mp_print_t fd_print = {(void *)(intptr_t)fd, fd_print_strn}; - mp_raw_code_save(rc, &fd_print); + mp_raw_code_save(cm, &fd_print); MP_THREAD_GIL_EXIT(); close(fd); MP_THREAD_GIL_ENTER(); diff --git a/py/persistentcode.h b/py/persistentcode.h index 1c53ca12cf..7ba86c1eb7 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -31,7 +31,7 @@ #include "py/emitglue.h" // The current version of .mpy files -#define MPY_VERSION 5 +#define MPY_VERSION 6 // Macros to encode/decode flags to/from the feature byte #define MPY_FEATURE_ENCODE_FLAGS(flags) (flags) @@ -42,15 +42,11 @@ #define MPY_FEATURE_DECODE_ARCH(feat) ((feat) >> 2) // The feature flag bits encode the compile-time config options that affect -// the generate bytecode. Note: position 0 is now unused -// (formerly MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE). -#define MPY_FEATURE_FLAGS ( \ - ((MICROPY_PY_BUILTINS_STR_UNICODE) << 1) \ - ) +// the generate bytecode. Note: no longer used. +// (formerly MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE and MICROPY_PY_BUILTINS_STR_UNICODE). +#define MPY_FEATURE_FLAGS (0) // This is a version of the flags that can be configured at runtime. -#define MPY_FEATURE_FLAGS_DYNAMIC ( \ - ((MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) << 1) \ - ) +#define MPY_FEATURE_FLAGS_DYNAMIC (0) // Define the host architecture #if MICROPY_EMIT_X86 @@ -67,7 +63,7 @@ #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7EM) #endif #else - #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7M) + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV6M) #endif #define MPY_FEATURE_ARCH_TEST(x) (MP_NATIVE_ARCH_ARMV6M <= (x) && (x) <= MPY_FEATURE_ARCH) #elif MICROPY_EMIT_ARM @@ -102,12 +98,26 @@ enum { MP_NATIVE_ARCH_XTENSAWIN, }; -mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader); -mp_raw_code_t *mp_raw_code_load_mem(const byte *buf, size_t len); -mp_raw_code_t *mp_raw_code_load_file(const char *filename); +enum { + MP_PERSISTENT_OBJ_FUN_TABLE = 0, + MP_PERSISTENT_OBJ_NONE, + MP_PERSISTENT_OBJ_FALSE, + MP_PERSISTENT_OBJ_TRUE, + MP_PERSISTENT_OBJ_ELLIPSIS, + MP_PERSISTENT_OBJ_STR, + MP_PERSISTENT_OBJ_BYTES, + MP_PERSISTENT_OBJ_INT, + MP_PERSISTENT_OBJ_FLOAT, + MP_PERSISTENT_OBJ_COMPLEX, + MP_PERSISTENT_OBJ_TUPLE, +}; -void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print); -void mp_raw_code_save_file(mp_raw_code_t *rc, const char *filename); +mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *ctx); +mp_compiled_module_t mp_raw_code_load_mem(const byte *buf, size_t len, mp_module_context_t *ctx); +mp_compiled_module_t mp_raw_code_load_file(const char *filename, mp_module_context_t *ctx); + +void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print); +void mp_raw_code_save_file(mp_compiled_module_t *cm, const char *filename); void mp_native_relocate(void *reloc, uint8_t *text, uintptr_t reloc_text); diff --git a/py/profile.c b/py/profile.c index 81c80f1168..4e23e9eac4 100644 --- a/py/profile.c +++ b/py/profile.c @@ -27,14 +27,16 @@ #include "py/profile.h" #include "py/bc0.h" #include "py/gc.h" +#include "py/objfun.h" #if MICROPY_PY_SYS_SETTRACE #define prof_trace_cb MP_STATE_THREAD(prof_trace_callback) +#define QSTR_MAP(context, idx) (context->constants.qstr_table[idx]) STATIC uint mp_prof_bytecode_lineno(const mp_raw_code_t *rc, size_t bc) { const mp_bytecode_prelude_t *prelude = &rc->prelude; - return mp_bytecode_get_source_line(prelude->line_info, bc); + return mp_bytecode_get_source_line(prelude->line_info, prelude->line_info_top, bc); } void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelude) { @@ -50,13 +52,14 @@ void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelud MP_BC_PRELUDE_SIZE_DECODE(ip); - prelude->line_info = ip + 4; + prelude->line_info_top = ip + n_info; prelude->opcodes = ip + n_info + n_cell; - qstr block_name = ip[0] | (ip[1] << 8); - qstr source_file = ip[2] | (ip[3] << 8); - prelude->qstr_block_name = block_name; - prelude->qstr_source_file = source_file; + prelude->qstr_block_name_idx = mp_decode_uint_value(ip); + for (size_t i = 0; i < 1 + n_pos_args + n_kwonly_args; ++i) { + ip = mp_decode_uint_skip(ip); + } + prelude->line_info = ip; } /******************************************************************************/ @@ -68,23 +71,20 @@ STATIC void code_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t k const mp_raw_code_t *rc = o->rc; const mp_bytecode_prelude_t *prelude = &rc->prelude; mp_printf(print, - "", - prelude->qstr_block_name, + "", + QSTR_MAP(o->context, prelude->qstr_block_name_idx), o, - prelude->qstr_source_file, + QSTR_MAP(o->context, 0), rc->line_of_definition ); } -STATIC mp_obj_tuple_t *code_consts(const mp_raw_code_t *rc) { - const mp_bytecode_prelude_t *prelude = &rc->prelude; - int start = prelude->n_pos_args + prelude->n_kwonly_args + rc->n_obj; - int stop = prelude->n_pos_args + prelude->n_kwonly_args + rc->n_obj + rc->n_raw_code; - mp_obj_tuple_t *consts = MP_OBJ_TO_PTR(mp_obj_new_tuple(stop - start + 1, NULL)); +STATIC mp_obj_tuple_t *code_consts(const mp_module_context_t *context, const mp_raw_code_t *rc) { + mp_obj_tuple_t *consts = MP_OBJ_TO_PTR(mp_obj_new_tuple(rc->n_children + 1, NULL)); size_t const_no = 0; - for (int i = start; i < stop; ++i) { - mp_obj_t code = mp_obj_new_code((const mp_raw_code_t *)MP_OBJ_TO_PTR(rc->const_table[i])); + for (size_t i = 0; i < rc->n_children; ++i) { + mp_obj_t code = mp_obj_new_code(context, rc->children[i]); if (code == MP_OBJ_NULL) { m_malloc_fail(sizeof(mp_obj_code_t)); } @@ -149,16 +149,16 @@ STATIC void code_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { ); break; case MP_QSTR_co_consts: - dest[0] = MP_OBJ_FROM_PTR(code_consts(rc)); + dest[0] = MP_OBJ_FROM_PTR(code_consts(o->context, rc)); break; case MP_QSTR_co_filename: - dest[0] = MP_OBJ_NEW_QSTR(prelude->qstr_source_file); + dest[0] = MP_OBJ_NEW_QSTR(QSTR_MAP(o->context, 0)); break; case MP_QSTR_co_firstlineno: dest[0] = MP_OBJ_NEW_SMALL_INT(mp_prof_bytecode_lineno(rc, 0)); break; case MP_QSTR_co_name: - dest[0] = MP_OBJ_NEW_QSTR(prelude->qstr_block_name); + dest[0] = MP_OBJ_NEW_QSTR(QSTR_MAP(o->context, prelude->qstr_block_name_idx)); break; case MP_QSTR_co_names: dest[0] = MP_OBJ_FROM_PTR(o->dict_locals); @@ -180,12 +180,13 @@ const mp_obj_type_t mp_type_settrace_codeobj = { .attr = code_attr, }; -mp_obj_t mp_obj_new_code(const mp_raw_code_t *rc) { +mp_obj_t mp_obj_new_code(const mp_module_context_t *context, const mp_raw_code_t *rc) { mp_obj_code_t *o = m_new_obj_maybe(mp_obj_code_t); if (o == NULL) { return MP_OBJ_NULL; } o->base.type = &mp_type_settrace_codeobj; + o->context = context; o->rc = rc; o->dict_locals = mp_locals_get(); // this is a wrong! how to do this properly? o->lnotab = MP_OBJ_NULL; @@ -202,11 +203,11 @@ STATIC void frame_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t const mp_raw_code_t *rc = code->rc; const mp_bytecode_prelude_t *prelude = &rc->prelude; mp_printf(print, - "", + "", frame, - prelude->qstr_source_file, + QSTR_MAP(code->context, 0), frame->lineno, - prelude->qstr_block_name + QSTR_MAP(code->context, prelude->qstr_block_name_idx) ); } @@ -229,7 +230,7 @@ STATIC void frame_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { dest[0] = MP_OBJ_FROM_PTR(o->code); break; case MP_QSTR_f_globals: - dest[0] = MP_OBJ_FROM_PTR(o->code_state->fun_bc->globals); + dest[0] = MP_OBJ_FROM_PTR(o->code_state->fun_bc->context->module.globals); break; case MP_QSTR_f_lasti: dest[0] = MP_OBJ_NEW_SMALL_INT(o->lasti); @@ -258,7 +259,7 @@ mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state) { return MP_OBJ_NULL; } - mp_obj_code_t *code = o->code = MP_OBJ_TO_PTR(mp_obj_new_code(code_state->fun_bc->rc)); + mp_obj_code_t *code = o->code = MP_OBJ_TO_PTR(mp_obj_new_code(code_state->fun_bc->context, code_state->fun_bc->rc)); if (code == NULL) { return MP_OBJ_NULL; } @@ -938,7 +939,7 @@ void mp_prof_print_instr(const byte *ip, mp_code_state_t *code_state) { /* long path */ if (1) { mp_printf(&mp_plat_print, - "@%p:%q:%q+0x%04x:%d", + "@0x%p:%q:%q+0x%04x:%d", ip, prelude->qstr_source_file, prelude->qstr_block_name, diff --git a/py/profile.h b/py/profile.h index 64e207d04f..7f3f914034 100644 --- a/py/profile.h +++ b/py/profile.h @@ -34,7 +34,9 @@ #define mp_prof_is_executing MP_STATE_THREAD(prof_callback_is_executing) typedef struct _mp_obj_code_t { + // TODO this was 4 words mp_obj_base_t base; + const mp_module_context_t *context; const mp_raw_code_t *rc; mp_obj_dict_t *dict_locals; mp_obj_t lnotab; @@ -53,7 +55,7 @@ typedef struct _mp_obj_frame_t { void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelude); -mp_obj_t mp_obj_new_code(const mp_raw_code_t *rc); +mp_obj_t mp_obj_new_code(const mp_module_context_t *mc, const mp_raw_code_t *rc); mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state); // This is the implementation for the sys.settrace diff --git a/py/py.mk b/py/py.mk index c598f44cbd..f50451dc99 100644 --- a/py/py.mk +++ b/py/py.mk @@ -278,9 +278,9 @@ PY_CORE_O += $(PY_BUILD)/translations-$(TRANSLATION).o $(PY_BUILD)/qstr.o: $(HEADER_BUILD)/qstrdefs.generated.h # build a list of registered modules for py/objmodule.c. -$(HEADER_BUILD)/moduledefs.h: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h +$(HEADER_BUILD)/moduledefs.h: $(HEADER_BUILD)/moduledefs.collected @$(ECHO) "GEN $@" - $(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py --vpath="., $(TOP), $(USER_C_MODULES)" $(SRC_QSTR) > $@ + $(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py $< > $@ # Standard C functions like memset need to be compiled with special flags so # the compiler does not optimise these functions in terms of themselves. diff --git a/py/pystack.c b/py/pystack.c index 696b033377..409ca11a7f 100644 --- a/py/pystack.c +++ b/py/pystack.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2017 Damien P. George + * Copyright (c) 2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/pystack.h b/py/pystack.h index 169d58b6f7..4bb717c482 100644 --- a/py/pystack.h +++ b/py/pystack.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2017 Damien P. George + * Copyright (c) 2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/qstr.c b/py/qstr.c index 96e2a79192..19ade1b6a4 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,7 +28,6 @@ #include #include -#include "py/gc.h" #include "py/mpstate.h" #include "py/qstr.h" #include "py/gc.h" @@ -82,11 +81,11 @@ mp_uint_t qstr_compute_hash(const byte *data, size_t len) { #endif #if CIRCUITPY_PRECOMPUTE_QSTR_ATTR == 1 const qstr_attr_t mp_qstr_const_attr[MP_QSTRnumber_of] = { + +const qstr_hash_t mp_qstr_const_hashes[] = { #ifndef NO_QSTR -#define QDEF(id, hash, len, str) { hash, len }, -#define TRANSLATION(id, length, compressed ...) +#define QDEF(id, hash, len, str) hash, #include "genhdr/qstrdefs.generated.h" -#undef TRANSLATION #undef QDEF #endif }; @@ -94,12 +93,21 @@ const qstr_attr_t mp_qstr_const_attr[MP_QSTRnumber_of] = { qstr_attr_t mp_qstr_const_attr[MP_QSTRnumber_of]; #endif +const qstr_len_t mp_qstr_const_lengths[] = { + #ifndef NO_QSTR +#define QDEF(id, hash, len, str) len, + #include "genhdr/qstrdefs.generated.h" +#undef QDEF + #endif +}; + const qstr_pool_t mp_qstr_const_pool = { NULL, // no previous pool 0, // no previous pool MICROPY_ALLOC_QSTR_ENTRIES_INIT, MP_QSTRnumber_of, // corresponds to number of strings in array just below - (qstr_attr_t *)mp_qstr_const_attr, + (qstr_hash_t *)mp_qstr_const_hashes, + (qstr_len_t *)mp_qstr_const_lengths, { #ifndef NO_QSTR #define QDEF(id, hash, len, str) str, @@ -137,17 +145,16 @@ void qstr_init(void) { #endif } -STATIC const char *PLACE_IN_ITCM(find_qstr)(qstr q, qstr_attr_t *attr) { +STATIC const qstr_pool_t *PLACE_IN_ITCM(find_qstr)(qstr *q) { // search pool for this qstr // total_prev_len==0 in the final pool, so the loop will always terminate const qstr_pool_t *pool = MP_STATE_VM(last_pool); - while (q < pool->total_prev_len) { + while (*q < pool->total_prev_len) { pool = pool->prev; } - q -= pool->total_prev_len; - assert(q < pool->len); - *attr = pool->attrs[q]; - return pool->qstrs[q]; + *q -= pool->total_prev_len; + assert(*q < pool->len); + return pool; } // qstr_mutex must be taken while in this function @@ -156,19 +163,14 @@ STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { // make sure we have room in the pool for a new qstr if (MP_STATE_VM(last_pool)->len >= MP_STATE_VM(last_pool)->alloc) { - uint32_t new_pool_length = MP_STATE_VM(last_pool)->alloc * 2; - if (new_pool_length > MICROPY_QSTR_POOL_MAX_ENTRIES) { - new_pool_length = MICROPY_QSTR_POOL_MAX_ENTRIES; - } + size_t new_alloc = MP_STATE_VM(last_pool)->alloc * 2; #ifdef MICROPY_QSTR_EXTRA_POOL // Put a lower bound on the allocation size in case the extra qstr pool has few entries - if (new_pool_length < MICROPY_ALLOC_QSTR_ENTRIES_INIT) { - new_pool_length = MICROPY_ALLOC_QSTR_ENTRIES_INIT; - } + new_alloc = MAX(MICROPY_ALLOC_QSTR_ENTRIES_INIT, new_alloc); #endif mp_uint_t pool_size = sizeof(qstr_pool_t) - + (sizeof(const char *) + sizeof(qstr_attr_t)) * new_pool_length; - qstr_pool_t *pool = (qstr_pool_t *)m_malloc_maybe(pool_size, true); + + (sizeof(const char *) + sizeof(qstr_hash_t) + sizeof(qstr_len_t)) * new_alloc; + qstr_pool_t *pool = (qstr_pool_t *)m_malloc_maybe(pool_size); if (pool == NULL) { // Keep qstr_last_chunk consistent with qstr_pool_t: qstr_last_chunk is not scanned // at garbage collection since it's reachable from a qstr_pool_t. And the caller of @@ -177,12 +179,13 @@ STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { // NULL'd. Otherwise it may become a dangling pointer at the next garbage collection. MP_STATE_VM(qstr_last_chunk) = NULL; QSTR_EXIT(); - m_malloc_fail(new_pool_length); + m_malloc_fail(new_alloc); } - pool->attrs = (qstr_attr_t *)(pool->qstrs + new_pool_length); + pool->hashes = (qstr_hash_t *)(pool->qstrs + new_alloc); + pool->lengths = (qstr_len_t *)(pool->hashes + new_alloc); pool->prev = MP_STATE_VM(last_pool); pool->total_prev_len = MP_STATE_VM(last_pool)->total_prev_len + MP_STATE_VM(last_pool)->len; - pool->alloc = new_pool_length; + pool->alloc = new_alloc; pool->len = 0; MP_STATE_VM(last_pool) = pool; DEBUG_printf("QSTR: allocate new pool of size %d\n", MP_STATE_VM(last_pool)->alloc); @@ -190,8 +193,8 @@ STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { // add the new qstr mp_uint_t at = MP_STATE_VM(last_pool)->len; - MP_STATE_VM(last_pool)->attrs[at].hash = hash; - MP_STATE_VM(last_pool)->attrs[at].len = len; + MP_STATE_VM(last_pool)->hashes[at] = hash; + MP_STATE_VM(last_pool)->lengths[at] = len; MP_STATE_VM(last_pool)->qstrs[at] = q_ptr; MP_STATE_VM(last_pool)->len++; @@ -205,9 +208,9 @@ qstr qstr_find_strn(const char *str, size_t str_len) { // search pools for the data for (const qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL; pool = pool->prev) { - qstr_attr_t *attrs = pool->attrs; for (mp_uint_t at = 0, top = pool->len; at < top; at++) { - if (attrs[at].hash == str_hash && attrs[at].len == str_len && memcmp(pool->qstrs[at], str, str_len) == 0) { + if (pool->hashes[at] == str_hash && pool->lengths[at] == str_len + && memcmp(pool->qstrs[at], str, str_len) == 0) { return pool->total_prev_len + at; } } @@ -283,28 +286,25 @@ qstr qstr_from_strn(const char *str, size_t len) { return q; } -mp_uint_t PLACE_IN_ITCM(qstr_hash)(qstr q) { - qstr_attr_t attr; - find_qstr(q, &attr); - return attr.hash; +mp_uint_t qstr_hash(qstr q) { + const qstr_pool_t *pool = find_qstr(&q); + return pool->hashes[q]; } size_t qstr_len(qstr q) { - qstr_attr_t attr; - find_qstr(q, &attr); - return attr.len; + const qstr_pool_t *pool = find_qstr(&q); + return pool->lengths[q]; } const char *qstr_str(qstr q) { - qstr_attr_t attr; - return find_qstr(q, &attr); + const qstr_pool_t *pool = find_qstr(&q); + return pool->qstrs[q]; } const byte *qstr_data(qstr q, size_t *len) { - qstr_attr_t attr; - const char *qd = find_qstr(q, &attr); - *len = attr.len; - return (byte *)qd; + const qstr_pool_t *pool = find_qstr(&q); + *len = pool->lengths[q]; + return (byte *)pool->qstrs[q]; } void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, size_t *n_total_bytes) { @@ -316,14 +316,14 @@ void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, si for (const qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &CONST_POOL; pool = pool->prev) { *n_pool += 1; *n_qstr += pool->len; - for (const qstr_attr_t *q = pool->attrs, *q_top = pool->attrs + pool->len; q < q_top; q++) { - *n_str_data_bytes += sizeof(*q) + q->len + 1; + for (qstr_len_t *l = pool->lengths, *l_top = pool->lengths + pool->len; l < l_top; l++) { + *n_str_data_bytes += *l + 1; } #if MICROPY_ENABLE_GC - // this counts actual bytes used in heap - *n_total_bytes += gc_nbytes(pool) - sizeof(qstr_attr_t) * pool->alloc; + *n_total_bytes += gc_nbytes(pool); // this counts actual bytes used in heap #else - *n_total_bytes += sizeof(qstr_pool_t) + sizeof(const char *) * pool->alloc; + *n_total_bytes += sizeof(qstr_pool_t) + + (sizeof(const char *) + sizeof(qstr_hash_t) + sizeof(qstr_len_t)) * pool->alloc; #endif } *n_total_bytes += *n_str_data_bytes; @@ -341,3 +341,78 @@ void qstr_dump_data(void) { QSTR_EXIT(); } #endif + +#if MICROPY_ROM_TEXT_COMPRESSION + +#ifdef NO_QSTR + +// If NO_QSTR is set, it means we're doing QSTR extraction. +// So we won't yet have "genhdr/compressed.data.h" + +#else + +// Emit the compressed_string_data string. +#define MP_COMPRESSED_DATA(x) STATIC const char *compressed_string_data = x; +#define MP_MATCH_COMPRESSED(a, b) +#include "genhdr/compressed.data.h" +#undef MP_COMPRESSED_DATA +#undef MP_MATCH_COMPRESSED + +#endif // NO_QSTR + +// This implements the "common word" compression scheme (see makecompresseddata.py) where the most +// common 128 words in error messages are replaced by their index into the list of common words. + +// The compressed string data is delimited by setting high bit in the final char of each word. +// e.g. aaaa<0x80|a>bbbbbb<0x80|b>.... +// This method finds the n'th string. +STATIC const byte *find_uncompressed_string(uint8_t n) { + const byte *c = (byte *)compressed_string_data; + while (n > 0) { + while ((*c & 0x80) == 0) { + ++c; + } + ++c; + --n; + } + return c; +} + +// Given a compressed string in src, decompresses it into dst. +// dst must be large enough (use MP_MAX_UNCOMPRESSED_TEXT_LEN+1). +void mp_decompress_rom_string(byte *dst, const mp_rom_error_text_t src_chr) { + // Skip past the 0xff marker. + const byte *src = (byte *)src_chr + 1; + // Need to add spaces around compressed words, except for the first (i.e. transition from 1<->2). + // 0 = start, 1 = compressed, 2 = regular. + int state = 0; + while (*src) { + if ((byte) * src >= 128) { + if (state != 0) { + *dst++ = ' '; + } + state = 1; + + // High bit set, replace with common word. + const byte *word = find_uncompressed_string(*src & 0x7f); + // The word is terminated by the final char having its high bit set. + while ((*word & 0x80) == 0) { + *dst++ = *word++; + } + *dst++ = (*word & 0x7f); + } else { + // Otherwise just copy one char. + if (state == 1) { + *dst++ = ' '; + } + state = 2; + + *dst++ = *src; + } + ++src; + } + // Add null-terminator. + *dst = 0; +} + +#endif // MICROPY_ROM_TEXT_COMPRESSION diff --git a/py/qstr.h b/py/qstr.h index 7820de2df9..fa634f90b0 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,38 +38,39 @@ // first entry in enum will be MP_QSTRnull=0, which indicates invalid/no qstr enum { #ifndef NO_QSTR -#define QENUM(id) id, - #include "genhdr/qstrdefs.enum.h" -#undef QENUM +#define QDEF(id, hash, len, str) id, + #include "genhdr/qstrdefs.generated.h" +#undef QDEF #endif MP_QSTRnumber_of, // no underscore so it can't clash with any of the above }; typedef size_t qstr; +typedef uint16_t qstr_short_t; -typedef struct _qstr_attr_t { - #if MICROPY_QSTR_BYTES_IN_HASH == 1 - uint8_t hash; - #elif MICROPY_QSTR_BYTES_IN_HASH == 2 - uint16_t hash; - #else - #error unimplemented qstr hash decoding - #endif - #if MICROPY_QSTR_BYTES_IN_LEN == 1 - uint8_t len; - #elif MICROPY_QSTR_BYTES_IN_LEN == 2 - uint16_t len; - #else - #error unimplemented qstr length decoding - #endif -} qstr_attr_t; +#if MICROPY_QSTR_BYTES_IN_HASH == 1 +typedef uint8_t qstr_hash_t; +#elif MICROPY_QSTR_BYTES_IN_HASH == 2 +typedef uint16_t qstr_hash_t; +#else +#error unimplemented qstr hash decoding +#endif + +#if MICROPY_QSTR_BYTES_IN_LEN == 1 +typedef uint8_t qstr_len_t; +#elif MICROPY_QSTR_BYTES_IN_LEN == 2 +typedef uint16_t qstr_len_t; +#else +#error unimplemented qstr length decoding +#endif typedef struct _qstr_pool_t { const struct _qstr_pool_t *prev; size_t total_prev_len; size_t alloc; size_t len; - qstr_attr_t *attrs; + qstr_hash_t *hashes; + qstr_len_t *lengths; const char *qstrs[]; } qstr_pool_t; @@ -91,4 +92,9 @@ const byte *qstr_data(qstr q, size_t *len); void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, size_t *n_total_bytes); void qstr_dump_data(void); +#if MICROPY_ROM_TEXT_COMPRESSION +void mp_decompress_rom_string(byte *dst, mp_rom_error_text_t src); +#define MP_IS_COMPRESSED_ROM_STRING(s) (*(byte *)(s) == 0xff) +#endif + #endif // MICROPY_INCLUDED_PY_QSTR_H diff --git a/py/qstrdefs.h b/py/qstrdefs.h index 70753866bd..e1cebb1db3 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -39,6 +39,10 @@ Q() Q(*) Q(_) Q(/) +#if MICROPY_PY_SYS_PS1_PS2 +Q(>>> ) +Q(... ) +#endif #if MICROPY_PY_BUILTINS_STR_OP_MODULO Q(%#o) Q(%#x) diff --git a/py/reader.c b/py/reader.c index ecc8515662..d68406b1c6 100644 --- a/py/reader.c +++ b/py/reader.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2016 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/reader.h b/py/reader.h index 6d8565d7b7..8511c72ce5 100644 --- a/py/reader.h +++ b/py/reader.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2016 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/repl.c b/py/repl.c index 78367e66da..a07dded50c 100644 --- a/py/repl.c +++ b/py/repl.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013-2015 Damien P. George + * Copyright (c) 2013-2015 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,6 +33,16 @@ #if MICROPY_HELPER_REPL +#if MICROPY_PY_SYS_PS1_PS2 +const char *mp_repl_get_psx(unsigned int entry) { + if (mp_obj_is_str(MP_STATE_VM(sys_mutable)[entry])) { + return mp_obj_str_get_str(MP_STATE_VM(sys_mutable)[entry]); + } else { + return ""; + } +} +#endif + STATIC bool str_startswith_word(const char *str, const char *head) { size_t i; for (i = 0; str[i] && head[i]; i++) { @@ -303,10 +313,7 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print return sizeof(import_str) - 1 - s_len; } } - if (q_first == 0) { - *compl_str = " "; - return s_len ? 0 : 4; - } + return 0; } // 1 match found, or multiple matches with a common prefix diff --git a/py/repl.h b/py/repl.h index 89b64c45f0..9e8f7f1dda 100644 --- a/py/repl.h +++ b/py/repl.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,8 +31,34 @@ #include "py/mpprint.h" #if MICROPY_HELPER_REPL + +#if MICROPY_PY_SYS_PS1_PS2 + +const char *mp_repl_get_psx(unsigned int entry); + +static inline const char *mp_repl_get_ps1(void) { + return mp_repl_get_psx(MP_SYS_MUTABLE_PS1); +} + +static inline const char *mp_repl_get_ps2(void) { + return mp_repl_get_psx(MP_SYS_MUTABLE_PS2); +} + +#else + +static inline const char *mp_repl_get_ps1(void) { + return ">>> "; +} + +static inline const char *mp_repl_get_ps2(void) { + return "... "; +} + +#endif + bool mp_repl_continue_with_input(const char *input); size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print, const char **compl_str); + #endif #endif // MICROPY_INCLUDED_PY_REPL_H diff --git a/py/runtime.c b/py/runtime.c index ebe1590484..2562621cc5 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014-2018 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014-2018 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -25,10 +25,11 @@ * THE SOFTWARE. */ +#include #include #include #include -#include +#include #include "py/parsenum.h" #include "py/compile.h" @@ -67,7 +68,15 @@ void mp_init(void) { // no pending exceptions to start with MP_STATE_THREAD(mp_pending_exception) = MP_OBJ_NULL; #if MICROPY_ENABLE_SCHEDULER - MP_STATE_VM(sched_state) = MP_SCHED_IDLE; + #if MICROPY_SCHEDULER_STATIC_NODES + if (MP_STATE_VM(sched_head) == NULL) { + // no pending callbacks to start with + MP_STATE_VM(sched_state) = MP_SCHED_IDLE; + } else { + // pending callbacks are on the list, eg from before a soft reset + MP_STATE_VM(sched_state) = MP_SCHED_PENDING; + } + #endif MP_STATE_VM(sched_idx) = 0; MP_STATE_VM(sched_len) = 0; #endif @@ -141,6 +150,14 @@ void mp_init(void) { MP_STATE_THREAD(current_code_state) = NULL; #endif + #if MICROPY_PY_SYS_TRACEBACKLIMIT + MP_STATE_VM(sys_mutable[MP_SYS_MUTABLE_TRACEBACKLIMIT]) = MP_OBJ_NEW_SMALL_INT(1000); + #endif + + #if MICROPY_PY_BLUETOOTH + MP_STATE_VM(bluetooth) = MP_OBJ_NULL; + #endif + #if MICROPY_PY_THREAD_GIL mp_thread_mutex_init(&MP_STATE_VM(gil_mutex)); #endif @@ -686,12 +703,11 @@ void PLACE_IN_ITCM(mp_call_prepare_args_n_kw_var)(bool have_self, size_t n_args_ if (have_self) { self = *args++; // may be MP_OBJ_NULL } - uint n_args = n_args_n_kw & 0xff; - uint n_kw = (n_args_n_kw >> 8) & 0xff; - mp_obj_t pos_seq = args[n_args + 2 * n_kw]; // may be MP_OBJ_NULL - mp_obj_t kw_dict = args[n_args + 2 * n_kw + 1]; // may be MP_OBJ_NULL + size_t n_args = n_args_n_kw & 0xff; + size_t n_kw = (n_args_n_kw >> 8) & 0xff; + mp_uint_t star_args = MP_OBJ_SMALL_INT_VALUE(args[n_args + 2 * n_kw]); - DEBUG_OP_printf("call method var (fun=%p, self=%p, n_args=%u, n_kw=%u, args=%p, seq=%p, dict=%p)\n", fun, self, n_args, n_kw, args, pos_seq, kw_dict); + DEBUG_OP_printf("call method var (fun=%p, self=%p, n_args=%u, n_kw=%u, args=%p, map=%u)\n", fun, self, n_args, n_kw, args, star_args); // We need to create the following array of objects: // args[0 .. n_args] unpacked(pos_seq) args[n_args .. n_args + 2 * n_kw] unpacked(kw_dict) @@ -699,19 +715,40 @@ void PLACE_IN_ITCM(mp_call_prepare_args_n_kw_var)(bool have_self, size_t n_args_ // The new args array mp_obj_t *args2; - uint args2_alloc; - uint args2_len = 0; + size_t args2_alloc; + size_t args2_len = 0; + + // Try to get a hint for unpacked * args length + ssize_t list_len = 0; + + if (star_args != 0) { + for (size_t i = 0; i < n_args; i++) { + if ((star_args >> i) & 1) { + mp_obj_t len = mp_obj_len_maybe(args[i]); + if (len != MP_OBJ_NULL) { + // -1 accounts for 1 of n_args occupied by this arg + list_len += mp_obj_get_int(len) - 1; + } + } + } + } // Try to get a hint for the size of the kw_dict - uint kw_dict_len = 0; - if (kw_dict != MP_OBJ_NULL && mp_obj_is_type(kw_dict, &mp_type_dict)) { - kw_dict_len = mp_obj_dict_len(kw_dict); + ssize_t kw_dict_len = 0; + + for (size_t i = 0; i < n_kw; i++) { + mp_obj_t key = args[n_args + i * 2]; + mp_obj_t value = args[n_args + i * 2 + 1]; + if (key == MP_OBJ_NULL && value != MP_OBJ_NULL && mp_obj_is_type(value, &mp_type_dict)) { + // -1 accounts for 1 of n_kw occupied by this arg + kw_dict_len += mp_obj_dict_len(value) - 1; + } } // Extract the pos_seq sequence to the new args array. // Note that it can be arbitrary iterator. - if (pos_seq == MP_OBJ_NULL) { - // no sequence + if (star_args == 0) { + // no star args to unpack // allocate memory for the new array of args args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len); @@ -725,33 +762,11 @@ void PLACE_IN_ITCM(mp_call_prepare_args_n_kw_var)(bool have_self, size_t n_args_ // copy the fixed pos args mp_seq_copy(args2 + args2_len, args, n_args, mp_obj_t); args2_len += n_args; - - } else if (mp_obj_is_type(pos_seq, &mp_type_tuple) || mp_obj_is_type(pos_seq, &mp_type_list)) { - // optimise the case of a tuple and list - - // get the items - size_t len; - mp_obj_t *items; - mp_obj_get_array(pos_seq, &len, &items); - - // allocate memory for the new array of args - args2_alloc = 1 + n_args + len + 2 * (n_kw + kw_dict_len); - args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t)); - - // copy the self - if (self != MP_OBJ_NULL) { - args2[args2_len++] = self; - } - - // copy the fixed and variable position args - mp_seq_cat(args2 + args2_len, args, n_args, items, len, mp_obj_t); - args2_len += n_args + len; - } else { - // generic iterator + // at least one star arg to unpack // allocate memory for the new array of args - args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len) + 3; + args2_alloc = 1 + n_args + list_len + 2 * (n_kw + kw_dict_len); args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t)); // copy the self @@ -759,84 +774,118 @@ void PLACE_IN_ITCM(mp_call_prepare_args_n_kw_var)(bool have_self, size_t n_args_ args2[args2_len++] = self; } - // copy the fixed position args - mp_seq_copy(args2 + args2_len, args, n_args, mp_obj_t); - args2_len += n_args; + for (size_t i = 0; i < n_args; i++) { + mp_obj_t arg = args[i]; + if ((star_args >> i) & 1) { + // star arg + if (mp_obj_is_type(arg, &mp_type_tuple) || mp_obj_is_type(arg, &mp_type_list)) { + // optimise the case of a tuple and list - // extract the variable position args from the iterator - mp_obj_iter_buf_t iter_buf; - mp_obj_t iterable = mp_getiter(pos_seq, &iter_buf); - mp_obj_t item; - while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { - if (args2_len >= args2_alloc) { - args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), args2_alloc * 2 * sizeof(mp_obj_t)); - args2_alloc *= 2; + // get the items + size_t len; + mp_obj_t *items; + mp_obj_get_array(arg, &len, &items); + + // copy the items + assert(args2_len + len <= args2_alloc); + mp_seq_copy(args2 + args2_len, items, len, mp_obj_t); + args2_len += len; + } else { + // generic iterator + + // extract the variable position args from the iterator + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(arg, &iter_buf); + mp_obj_t item; + while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { + if (args2_len >= args2_alloc) { + args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), + args2_alloc * 2 * sizeof(mp_obj_t)); + args2_alloc *= 2; + } + args2[args2_len++] = item; + } + } + } else { + // normal argument + assert(args2_len < args2_alloc); + args2[args2_len++] = arg; } - args2[args2_len++] = item; } } // The size of the args2 array now is the number of positional args. - uint pos_args_len = args2_len; + size_t pos_args_len = args2_len; - // Copy the fixed kw args. - mp_seq_copy(args2 + args2_len, args + n_args, 2 * n_kw, mp_obj_t); - args2_len += 2 * n_kw; + // ensure there is still enough room for kw args + if (args2_len + 2 * (n_kw + kw_dict_len) > args2_alloc) { + size_t new_alloc = args2_len + 2 * (n_kw + kw_dict_len); + args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), + new_alloc * sizeof(mp_obj_t)); + args2_alloc = new_alloc; + } - // Extract (key,value) pairs from kw_dict dictionary and append to args2. - // Note that it can be arbitrary iterator. - if (kw_dict == MP_OBJ_NULL) { - // pass - } else if (mp_obj_is_type(kw_dict, &mp_type_dict)) { - // dictionary - mp_map_t *map = mp_obj_dict_get_map(kw_dict); - assert(args2_len + 2 * map->used <= args2_alloc); // should have enough, since kw_dict_len is in this case hinted correctly above - for (size_t i = 0; i < map->alloc; i++) { - if (mp_map_slot_is_filled(map, i)) { - // the key must be a qstr, so intern it if it's a string - mp_obj_t key = map->table[i].key; - if (!mp_obj_is_qstr(key)) { - key = mp_obj_str_intern_checked(key); + // Copy the kw args. + for (size_t i = 0; i < n_kw; i++) { + mp_obj_t kw_key = args[n_args + i * 2]; + mp_obj_t kw_value = args[n_args + i * 2 + 1]; + if (kw_key == MP_OBJ_NULL) { + // double-star args + if (mp_obj_is_type(kw_value, &mp_type_dict)) { + // dictionary + mp_map_t *map = mp_obj_dict_get_map(kw_value); + // should have enough, since kw_dict_len is in this case hinted correctly above + assert(args2_len + 2 * map->used <= args2_alloc); + for (size_t j = 0; j < map->alloc; j++) { + if (mp_map_slot_is_filled(map, j)) { + // the key must be a qstr, so intern it if it's a string + mp_obj_t key = map->table[j].key; + if (!mp_obj_is_qstr(key)) { + key = mp_obj_str_intern_checked(key); + } + args2[args2_len++] = key; + args2[args2_len++] = map->table[j].value; + } } - args2[args2_len++] = key; - args2[args2_len++] = map->table[i].value; - } - } - } else { - // generic mapping: - // - call keys() to get an iterable of all keys in the mapping - // - call __getitem__ for each key to get the corresponding value + } else { + // generic mapping: + // - call keys() to get an iterable of all keys in the mapping + // - call __getitem__ for each key to get the corresponding value - // get the keys iterable - mp_obj_t dest[3]; - mp_load_method(kw_dict, MP_QSTR_keys, dest); - mp_obj_t iterable = mp_getiter(mp_call_method_n_kw(0, 0, dest), NULL); + // get the keys iterable + mp_obj_t dest[3]; + mp_load_method(kw_value, MP_QSTR_keys, dest); + mp_obj_t iterable = mp_getiter(mp_call_method_n_kw(0, 0, dest), NULL); - mp_obj_t key; - while ((key = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { - // expand size of args array if needed - if (args2_len + 1 >= args2_alloc) { - uint new_alloc = args2_alloc * 2; - if (new_alloc < 4) { - new_alloc = 4; + mp_obj_t key; + while ((key = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { + // expand size of args array if needed + if (args2_len + 1 >= args2_alloc) { + size_t new_alloc = args2_alloc * 2; + args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), new_alloc * sizeof(mp_obj_t)); + args2_alloc = new_alloc; + } + + // the key must be a qstr, so intern it if it's a string + if (!mp_obj_is_qstr(key)) { + key = mp_obj_str_intern_checked(key); + } + + // get the value corresponding to the key + mp_load_method(kw_value, MP_QSTR___getitem__, dest); + dest[2] = key; + mp_obj_t value = mp_call_method_n_kw(1, 0, dest); + + // store the key/value pair in the argument array + args2[args2_len++] = key; + args2[args2_len++] = value; } - args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), new_alloc * sizeof(mp_obj_t)); - args2_alloc = new_alloc; } - - // the key must be a qstr, so intern it if it's a string - if (!mp_obj_is_qstr(key)) { - key = mp_obj_str_intern_checked(key); - } - - // get the value corresponding to the key - mp_load_method(kw_dict, MP_QSTR___getitem__, dest); - dest[2] = key; - mp_obj_t value = mp_call_method_n_kw(1, 0, dest); - - // store the key/value pair in the argument array - args2[args2_len++] = key; - args2[args2_len++] = value; + } else { + // normal kwarg + assert(args2_len + 2 <= args2_alloc); + args2[args2_len++] = kw_key; + args2[args2_len++] = kw_value; } } @@ -1018,8 +1067,7 @@ STATIC const mp_obj_type_t mp_type_checked_fun = { }; STATIC mp_obj_t mp_obj_new_checked_fun(const mp_obj_type_t *type, mp_obj_t fun) { - mp_obj_checked_fun_t *o = m_new_obj(mp_obj_checked_fun_t); - o->base.type = &mp_type_checked_fun; + mp_obj_checked_fun_t *o = mp_obj_malloc(mp_obj_checked_fun_t, &mp_type_checked_fun); o->type = type; o->fun = fun; return MP_OBJ_FROM_PTR(o); @@ -1762,7 +1810,3 @@ NORETURN MP_COLD void mp_raise_recursion_depth(void) { mp_raise_RuntimeError(MP_ERROR_TEXT("maximum recursion depth exceeded")); } #endif - -NORETURN MP_COLD void mp_raise_ZeroDivisionError(void) { - mp_raise_msg(&mp_type_ZeroDivisionError, MP_ERROR_TEXT("division by zero")); -} diff --git a/py/runtime.h b/py/runtime.h index 196874bff9..2a4191fe88 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -63,6 +63,15 @@ typedef struct _mp_arg_t { mp_arg_val_t defval; } mp_arg_t; +struct _mp_sched_node_t; + +typedef void (*mp_sched_callback_t)(struct _mp_sched_node_t *); + +typedef struct _mp_sched_node_t { + mp_sched_callback_t callback; + struct _mp_sched_node_t *next; +} mp_sched_node_t; + // Tables mapping operator enums to qstrs, defined in objtype.c extern const byte mp_unary_op_method_name[]; extern const byte mp_binary_op_method_name[]; @@ -80,6 +89,7 @@ void mp_sched_lock(void); void mp_sched_unlock(void); #define mp_sched_num_pending() (MP_STATE_VM(sched_len)) bool mp_sched_schedule(mp_obj_t function, mp_obj_t arg); +bool mp_sched_schedule_node(mp_sched_node_t *node, mp_sched_callback_t callback); #endif // extra printing method specifically for mp_obj_t's which are integral type diff --git a/py/runtime0.h b/py/runtime0.h index 9af108d8b3..ab048701c4 100644 --- a/py/runtime0.h +++ b/py/runtime0.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/scheduler.c b/py/scheduler.c index 1fd5daeab1..2aa28bd335 100644 --- a/py/scheduler.c +++ b/py/scheduler.c @@ -90,6 +90,24 @@ void mp_handle_pending(bool raise_exc) { // or by the VM's inlined version of that function. void mp_handle_pending_tail(mp_uint_t atomic_state) { MP_STATE_VM(sched_state) = MP_SCHED_LOCKED; + + #if MICROPY_SCHEDULER_STATIC_NODES + // Run all pending C callbacks. + while (MP_STATE_VM(sched_head) != NULL) { + mp_sched_node_t *node = MP_STATE_VM(sched_head); + MP_STATE_VM(sched_head) = node->next; + if (MP_STATE_VM(sched_head) == NULL) { + MP_STATE_VM(sched_tail) = NULL; + } + mp_sched_callback_t callback = node->callback; + node->callback = NULL; + MICROPY_END_ATOMIC_SECTION(atomic_state); + callback(node); + atomic_state = MICROPY_BEGIN_ATOMIC_SECTION(); + } + #endif + + // Run at most one pending Python callback. if (!mp_sched_empty()) { mp_sched_item_t item = MP_STATE_VM(sched_queue)[MP_STATE_VM(sched_idx)]; MP_STATE_VM(sched_idx) = IDX_MASK(MP_STATE_VM(sched_idx) + 1); @@ -99,6 +117,7 @@ void mp_handle_pending_tail(mp_uint_t atomic_state) { } else { MICROPY_END_ATOMIC_SECTION(atomic_state); } + mp_sched_unlock(); } @@ -117,7 +136,11 @@ void mp_sched_unlock(void) { assert(MP_STATE_VM(sched_state) < 0); if (++MP_STATE_VM(sched_state) == 0) { // vm became unlocked - if (MP_STATE_THREAD(mp_pending_exception) != MP_OBJ_NULL || mp_sched_num_pending()) { + if (MP_STATE_THREAD(mp_pending_exception) != MP_OBJ_NULL + #if MICROPY_SCHEDULER_STATIC_NODES + || MP_STATE_VM(sched_head) != NULL + #endif + || mp_sched_num_pending()) { MP_STATE_VM(sched_state) = MP_SCHED_PENDING; } else { MP_STATE_VM(sched_state) = MP_SCHED_IDLE; @@ -146,6 +169,33 @@ bool MICROPY_WRAP_MP_SCHED_SCHEDULE(mp_sched_schedule)(mp_obj_t function, mp_obj return ret; } +#if MICROPY_SCHEDULER_STATIC_NODES +bool mp_sched_schedule_node(mp_sched_node_t *node, mp_sched_callback_t callback) { + mp_uint_t atomic_state = MICROPY_BEGIN_ATOMIC_SECTION(); + bool ret; + if (node->callback == NULL) { + if (MP_STATE_VM(sched_state) == MP_SCHED_IDLE) { + MP_STATE_VM(sched_state) = MP_SCHED_PENDING; + } + node->callback = callback; + node->next = NULL; + if (MP_STATE_VM(sched_tail) == NULL) { + MP_STATE_VM(sched_head) = node; + } else { + MP_STATE_VM(sched_tail)->next = node; + } + MP_STATE_VM(sched_tail) = node; + MICROPY_SCHED_HOOK_SCHEDULED; + ret = true; + } else { + // already scheduled + ret = false; + } + MICROPY_END_ATOMIC_SECTION(atomic_state); + return ret; +} +#endif + #else // MICROPY_ENABLE_SCHEDULER // A variant of this is inlined in the VM at the pending exception check diff --git a/py/scope.c b/py/scope.c index f9308267d3..8fc0943289 100644 --- a/py/scope.c +++ b/py/scope.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,7 +40,7 @@ STATIC const uint8_t scope_simple_name_table[] = { [SCOPE_GEN_EXPR] = MP_QSTR__lt_genexpr_gt_, }; -scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options) { +scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, mp_uint_t emit_options) { // Make sure those qstrs indeed fit in an uint8_t. MP_STATIC_ASSERT(MP_QSTR__lt_module_gt_ <= UINT8_MAX); MP_STATIC_ASSERT(MP_QSTR__lt_lambda_gt_ <= UINT8_MAX); @@ -52,7 +52,6 @@ scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_u scope_t *scope = m_new0(scope_t, 1); scope->kind = kind; scope->pn = pn; - scope->source_file = source_file; if (kind == SCOPE_FUNCTION || kind == SCOPE_CLASS) { assert(MP_PARSE_NODE_IS_STRUCT(pn)); scope->simple_name = MP_PARSE_NODE_LEAF_ARG(((mp_parse_node_struct_t *)pn)->nodes[0]); diff --git a/py/scope.h b/py/scope.h index b189260a37..e7d2a304f7 100644 --- a/py/scope.h +++ b/py/scope.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,6 +32,7 @@ typedef enum { ID_INFO_KIND_UNDECIDED, ID_INFO_KIND_GLOBAL_IMPLICIT, + ID_INFO_KIND_GLOBAL_IMPLICIT_ASSIGNED, ID_INFO_KIND_GLOBAL_EXPLICIT, ID_INFO_KIND_LOCAL, // in a function f, written and only referenced by f ID_INFO_KIND_CELL, // in a function f, read/written by children of f @@ -75,7 +76,6 @@ typedef struct _scope_t { struct _scope_t *next; mp_parse_node_t pn; mp_raw_code_t *raw_code; - uint16_t source_file; // a qstr uint16_t simple_name; // a qstr uint16_t scope_flags; // see runtime0.h uint16_t emit_options; // see emitglue.h @@ -90,7 +90,7 @@ typedef struct _scope_t { id_info_t *id_info; } scope_t; -scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options); +scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, mp_uint_t emit_options); void scope_free(scope_t *scope); id_info_t *scope_find_or_add_id(scope_t *scope, qstr qstr, id_info_kind_t kind); id_info_t *scope_find(scope_t *scope, qstr qstr); diff --git a/py/sequence.c b/py/sequence.c index 42a9a4a0ae..faed558723 100644 --- a/py/sequence.c +++ b/py/sequence.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * Copyright (c) 2014 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/py/showbc.c b/py/showbc.c index 1d02b559b0..f9c334b93b 100644 --- a/py/showbc.c +++ b/py/showbc.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,7 +28,7 @@ #include #include "py/bc0.h" -#include "py/bc.h" +#include "py/emitglue.h" #if MICROPY_DEBUG_PRINTERS @@ -38,80 +38,90 @@ unum = (unum << 7) + (*ip & 0x7f); \ } while ((*ip++ & 0x80) != 0); \ } -#define DECODE_ULABEL do { unum = (ip[0] | (ip[1] << 8)); ip += 2; } while (0) -#define DECODE_SLABEL do { unum = (ip[0] | (ip[1] << 8)) - 0x8000; ip += 2; } while (0) -#if MICROPY_PERSISTENT_CODE +#define DECODE_ULABEL \ + do { \ + if (ip[0] & 0x80) { \ + unum = ((ip[0] & 0x7f) | (ip[1] << 7)); \ + ip += 2; \ + } else { \ + unum = ip[0]; \ + ip += 1; \ + } \ + } while (0) + +#define DECODE_SLABEL \ + do { \ + if (ip[0] & 0x80) { \ + unum = ((ip[0] & 0x7f) | (ip[1] << 7)) - 0x4000; \ + ip += 2; \ + } else { \ + unum = ip[0] - 0x40; \ + ip += 1; \ + } \ + } while (0) + +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE #define DECODE_QSTR \ - qst = ip[0] | ip[1] << 8; \ - ip += 2; -#define DECODE_PTR \ DECODE_UINT; \ - unum = mp_showbc_const_table[unum] -#define DECODE_OBJ \ - DECODE_UINT; \ - unum = mp_showbc_const_table[unum] + qst = qstr_table[unum] #else -#define DECODE_QSTR { \ - qst = 0; \ - do { \ - qst = (qst << 7) + (*ip & 0x7f); \ - } while ((*ip++ & 0x80) != 0); \ -} -#define DECODE_PTR do { \ - ip = (byte *)MP_ALIGN(ip, sizeof(void *)); \ - unum = (uintptr_t)*(void **)ip; \ - ip += sizeof(void *); \ -} while (0) -#define DECODE_OBJ do { \ - ip = (byte *)MP_ALIGN(ip, sizeof(mp_obj_t)); \ - unum = (mp_uint_t)*(mp_obj_t *)ip; \ - ip += sizeof(mp_obj_t); \ -} while (0) +#define DECODE_QSTR \ + DECODE_UINT; \ + qst = unum; #endif -const byte *mp_showbc_code_start; -const mp_uint_t *mp_showbc_const_table; +#define DECODE_PTR \ + DECODE_UINT; \ + unum = (mp_uint_t)(uintptr_t)child_table[unum] -void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *ip, mp_uint_t len, const mp_uint_t *const_table) { - mp_showbc_code_start = ip; +#define DECODE_OBJ \ + DECODE_UINT; \ + unum = (mp_uint_t)obj_table[unum] + +void mp_bytecode_print(const mp_print_t *print, const mp_raw_code_t *rc, const mp_module_constants_t *cm) { + const byte *ip_start = rc->fun_data; + const byte *ip = rc->fun_data; // Decode prelude MP_BC_PRELUDE_SIG_DECODE(ip); MP_BC_PRELUDE_SIZE_DECODE(ip); const byte *code_info = ip; - #if MICROPY_PERSISTENT_CODE - qstr block_name = code_info[0] | (code_info[1] << 8); - qstr source_file = code_info[2] | (code_info[3] << 8); - code_info += 4; - #else qstr block_name = mp_decode_uint(&code_info); - qstr source_file = mp_decode_uint(&code_info); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + block_name = cm->qstr_table[block_name]; + qstr source_file = cm->qstr_table[0]; + #else + qstr source_file = cm->source_file; #endif - mp_printf(print, "File %s, code block '%s' (descriptor: %p, bytecode @%p " UINT_FMT " bytes)\n", - qstr_str(source_file), qstr_str(block_name), descr, mp_showbc_code_start, len); + mp_printf(print, "File %s, code block '%s' (descriptor: %p, bytecode @%p %u bytes)\n", + qstr_str(source_file), qstr_str(block_name), rc, ip_start, (unsigned)rc->fun_data_len); // raw bytecode dump - size_t prelude_size = ip - mp_showbc_code_start + n_info + n_cell; + size_t prelude_size = ip - ip_start + n_info + n_cell; mp_printf(print, "Raw bytecode (code_info_size=%u, bytecode_size=%u):\n", - (unsigned)prelude_size, (unsigned)(len - prelude_size)); - for (mp_uint_t i = 0; i < len; i++) { + (unsigned)prelude_size, (unsigned)(rc->fun_data_len - prelude_size)); + for (size_t i = 0; i < rc->fun_data_len; i++) { if (i > 0 && i % 16 == 0) { mp_printf(print, "\n"); } - mp_printf(print, " %02x", mp_showbc_code_start[i]); + mp_printf(print, " %02x", ip_start[i]); } mp_printf(print, "\n"); // bytecode prelude: arg names (as qstr objects) mp_printf(print, "arg names:"); for (mp_uint_t i = 0; i < n_pos_args + n_kwonly_args; i++) { - mp_printf(print, " %s", qstr_str(MP_OBJ_QSTR_VALUE(const_table[i]))); + qstr qst = mp_decode_uint(&code_info); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + qst = cm->qstr_table[qst]; + #endif + mp_printf(print, " %s", qstr_str(qst)); } mp_printf(print, "\n"); @@ -120,6 +130,7 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i // skip over code_info ip += n_info; + const byte *line_info_top = ip; // bytecode prelude: initialise closed over variables for (size_t i = 0; i < n_cell; ++i) { @@ -132,7 +143,7 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i mp_int_t bc = 0; mp_uint_t source_line = 1; mp_printf(print, " bc=" INT_FMT " line=" UINT_FMT "\n", bc, source_line); - for (const byte *ci = code_info; *ci;) { + for (const byte *ci = code_info; ci < line_info_top;) { if ((ci[0] & 0x80) == 0) { // 0b0LLBBBBB encoding bc += ci[0] & 0x1f; @@ -147,10 +158,14 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i mp_printf(print, " bc=" INT_FMT " line=" UINT_FMT "\n", bc, source_line); } } - mp_bytecode_print2(print, ip, len - prelude_size, const_table); + mp_bytecode_print2(print, ip, rc->fun_data_len - prelude_size, rc->children, cm); } -const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { +const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip_start, const byte *ip, mp_raw_code_t *const *child_table, const mp_module_constants_t *cm) { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + const qstr_short_t *qstr_table = cm->qstr_table; + #endif + const mp_obj_t *obj_table = cm->obj_table; mp_uint_t unum; qstr qst; @@ -309,32 +324,32 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { case MP_BC_JUMP: DECODE_SLABEL; - mp_printf(print, "JUMP " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start)); + mp_printf(print, "JUMP " UINT_FMT, (mp_uint_t)(ip + unum - ip_start)); break; case MP_BC_POP_JUMP_IF_TRUE: DECODE_SLABEL; - mp_printf(print, "POP_JUMP_IF_TRUE " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start)); + mp_printf(print, "POP_JUMP_IF_TRUE " UINT_FMT, (mp_uint_t)(ip + unum - ip_start)); break; case MP_BC_POP_JUMP_IF_FALSE: DECODE_SLABEL; - mp_printf(print, "POP_JUMP_IF_FALSE " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start)); + mp_printf(print, "POP_JUMP_IF_FALSE " UINT_FMT, (mp_uint_t)(ip + unum - ip_start)); break; case MP_BC_JUMP_IF_TRUE_OR_POP: - DECODE_SLABEL; - mp_printf(print, "JUMP_IF_TRUE_OR_POP " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start)); + DECODE_ULABEL; + mp_printf(print, "JUMP_IF_TRUE_OR_POP " UINT_FMT, (mp_uint_t)(ip + unum - ip_start)); break; case MP_BC_JUMP_IF_FALSE_OR_POP: - DECODE_SLABEL; - mp_printf(print, "JUMP_IF_FALSE_OR_POP " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start)); + DECODE_ULABEL; + mp_printf(print, "JUMP_IF_FALSE_OR_POP " UINT_FMT, (mp_uint_t)(ip + unum - ip_start)); break; case MP_BC_SETUP_WITH: DECODE_ULABEL; // loop-like labels are always forward - mp_printf(print, "SETUP_WITH " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start)); + mp_printf(print, "SETUP_WITH " UINT_FMT, (mp_uint_t)(ip + unum - ip_start)); break; case MP_BC_WITH_CLEANUP: @@ -343,18 +358,18 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { case MP_BC_UNWIND_JUMP: DECODE_SLABEL; - mp_printf(print, "UNWIND_JUMP " UINT_FMT " %d", (mp_uint_t)(ip + unum - mp_showbc_code_start), *ip); + mp_printf(print, "UNWIND_JUMP " UINT_FMT " %d", (mp_uint_t)(ip + unum - ip_start), *ip); ip += 1; break; case MP_BC_SETUP_EXCEPT: DECODE_ULABEL; // except labels are always forward - mp_printf(print, "SETUP_EXCEPT " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start)); + mp_printf(print, "SETUP_EXCEPT " UINT_FMT, (mp_uint_t)(ip + unum - ip_start)); break; case MP_BC_SETUP_FINALLY: DECODE_ULABEL; // except labels are always forward - mp_printf(print, "SETUP_FINALLY " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start)); + mp_printf(print, "SETUP_FINALLY " UINT_FMT, (mp_uint_t)(ip + unum - ip_start)); break; case MP_BC_END_FINALLY: @@ -375,12 +390,12 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { case MP_BC_FOR_ITER: DECODE_ULABEL; // the jump offset if iteration finishes; for labels are always forward - mp_printf(print, "FOR_ITER " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start)); + mp_printf(print, "FOR_ITER " UINT_FMT, (mp_uint_t)(ip + unum - ip_start)); break; case MP_BC_POP_EXCEPT_JUMP: DECODE_ULABEL; // these labels are always forward - mp_printf(print, "POP_EXCEPT_JUMP " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start)); + mp_printf(print, "POP_EXCEPT_JUMP " UINT_FMT, (mp_uint_t)(ip + unum - ip_start)); break; case MP_BC_BUILD_TUPLE: @@ -535,12 +550,11 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { return ip; } -void mp_bytecode_print2(const mp_print_t *print, const byte *ip, size_t len, const mp_uint_t *const_table) { - mp_showbc_code_start = ip; - mp_showbc_const_table = const_table; - while (ip < len + mp_showbc_code_start) { - mp_printf(print, "%02u ", (uint)(ip - mp_showbc_code_start)); - ip = mp_bytecode_print_str(print, ip); +void mp_bytecode_print2(const mp_print_t *print, const byte *ip, size_t len, mp_raw_code_t *const *child_table, const mp_module_constants_t *cm) { + const byte *ip_start = ip; + while (ip < ip_start + len) { + mp_printf(print, "%02u ", (uint)(ip - ip_start)); + ip = mp_bytecode_print_str(print, ip_start, ip, child_table, cm); mp_printf(print, "\n"); } } diff --git a/py/smallint.c b/py/smallint.c index 9124b76c13..aa542ca7bf 100644 --- a/py/smallint.c +++ b/py/smallint.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/smallint.h b/py/smallint.h index 2fcd3fb296..584e0018d1 100644 --- a/py/smallint.h +++ b/py/smallint.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -61,6 +61,13 @@ #define MP_SMALL_INT_MAX ((mp_int_t)(~(MP_SMALL_INT_MIN))) +// https://stackoverflow.com/a/4589384/1976323 +// Number of bits in inttype_MAX, or in any (1<state[0] + 1)) #endif -#define TRACE(ip) TRACE_PREFIX; mp_bytecode_print2(&mp_plat_print, ip, 1, code_state->fun_bc->const_table); +#define TRACE(ip) TRACE_PREFIX; mp_bytecode_print2(&mp_plat_print, ip, 1, code_state->fun_bc->child_table, &code_state->fun_bc->context->constants); #else #define TRACE(ip) #endif @@ -64,38 +64,53 @@ do { \ unum = (unum << 7) + (*ip & 0x7f); \ } while ((*ip++ & 0x80) != 0) -#define DECODE_ULABEL size_t ulab = (ip[0] | (ip[1] << 8)); ip += 2 -#define DECODE_SLABEL size_t slab = (ip[0] | (ip[1] << 8)) - 0x8000; ip += 2 -#if MICROPY_PERSISTENT_CODE +#define DECODE_ULABEL \ + size_t ulab; \ + do { \ + if (ip[0] & 0x80) { \ + ulab = ((ip[0] & 0x7f) | (ip[1] << 7)); \ + ip += 2; \ + } else { \ + ulab = ip[0]; \ + ip += 1; \ + } \ + } while (0) + +#define DECODE_SLABEL \ + size_t slab; \ + do { \ + if (ip[0] & 0x80) { \ + slab = ((ip[0] & 0x7f) | (ip[1] << 7)) - 0x4000; \ + ip += 2; \ + } else { \ + slab = ip[0] - 0x40; \ + ip += 1; \ + } \ + } while (0) + +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE #define DECODE_QSTR \ - qstr qst = ip[0] | ip[1] << 8; \ - ip += 2; -#define DECODE_PTR \ DECODE_UINT; \ - void *ptr = (void*)(uintptr_t)code_state->fun_bc->const_table[unum] -#define DECODE_OBJ \ - DECODE_UINT; \ - mp_obj_t obj = (mp_obj_t)code_state->fun_bc->const_table[unum] + qstr qst = qstr_table[unum] #else -#define DECODE_QSTR qstr qst = 0; \ - do { \ - qst = (qst << 7) + (*ip & 0x7f); \ - } while ((*ip++ & 0x80) != 0) -#define DECODE_PTR \ - ip = (byte*)MP_ALIGN(ip, sizeof(void*)); \ - void *ptr = *(void**)ip; \ - ip += sizeof(void*) -#define DECODE_OBJ \ - ip = (byte*)MP_ALIGN(ip, sizeof(mp_obj_t)); \ - mp_obj_t obj = *(mp_obj_t*)ip; \ - ip += sizeof(mp_obj_t) +#define DECODE_QSTR \ + DECODE_UINT; \ + qstr qst = unum; #endif +#define DECODE_PTR \ + DECODE_UINT; \ + void *ptr = (void *)(uintptr_t)code_state->fun_bc->child_table[unum] + +#define DECODE_OBJ \ + DECODE_UINT; \ + mp_obj_t obj = (mp_obj_t)code_state->fun_bc->context->constants.obj_table[unum] + #define PUSH(val) *++sp = (val) #define POP() (*sp--) #define TOP() (*sp) @@ -277,6 +292,9 @@ outer_dispatch_loop: // local variables that are not visible to the exception handler const byte *ip = code_state->ip; mp_obj_t *sp = code_state->sp; + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + const qstr_short_t *qstr_table = code_state->fun_bc->context->constants.qstr_table; + #endif mp_obj_t obj_shared; MICROPY_VM_HOOK_INIT @@ -564,9 +582,9 @@ dispatch_loop: } ENTRY(MP_BC_JUMP_IF_TRUE_OR_POP): { - DECODE_SLABEL; + DECODE_ULABEL; if (mp_obj_is_true(TOP())) { - ip += slab; + ip += ulab; } else { sp--; } @@ -574,11 +592,11 @@ dispatch_loop: } ENTRY(MP_BC_JUMP_IF_FALSE_OR_POP): { - DECODE_SLABEL; + DECODE_ULABEL; if (mp_obj_is_true(TOP())) { sp--; } else { - ip += slab; + ip += ulab; } DISPATCH_WITH_PEND_EXC_CHECK(); } @@ -881,15 +899,15 @@ unwind_jump:; ENTRY(MP_BC_MAKE_FUNCTION): { DECODE_PTR; - PUSH(mp_make_function_from_raw_code(ptr, MP_OBJ_NULL, MP_OBJ_NULL)); + PUSH(mp_make_function_from_raw_code(ptr, code_state->fun_bc->context, NULL)); DISPATCH(); } ENTRY(MP_BC_MAKE_FUNCTION_DEFARGS): { DECODE_PTR; // Stack layout: def_tuple def_dict <- TOS - mp_obj_t def_dict = POP(); - SET_TOP(mp_make_function_from_raw_code(ptr, TOP(), def_dict)); + sp -= 1; + SET_TOP(mp_make_function_from_raw_code(ptr, code_state->fun_bc->context, sp)); DISPATCH(); } @@ -898,7 +916,7 @@ unwind_jump:; size_t n_closed_over = *ip++; // Stack layout: closed_overs <- TOS sp -= n_closed_over - 1; - SET_TOP(mp_make_closure_from_raw_code(ptr, n_closed_over, sp)); + SET_TOP(mp_make_closure_from_raw_code(ptr, code_state->fun_bc->context, n_closed_over, sp)); DISPATCH(); } @@ -907,7 +925,7 @@ unwind_jump:; size_t n_closed_over = *ip++; // Stack layout: def_tuple def_dict closed_overs <- TOS sp -= 2 + n_closed_over - 1; - SET_TOP(mp_make_closure_from_raw_code(ptr, 0x100 | n_closed_over, sp)); + SET_TOP(mp_make_closure_from_raw_code(ptr, code_state->fun_bc->context, 0x100 | n_closed_over, sp)); DISPATCH(); } @@ -953,8 +971,8 @@ unwind_jump:; // unum & 0xff == n_positional // (unum >> 8) & 0xff == n_keyword // We have following stack layout here: - // fun arg0 arg1 ... kw0 val0 kw1 val1 ... seq dict <- TOS - sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe) + 2; + // fun arg0 arg1 ... kw0 val0 kw1 val1 ... bitmap <- TOS + sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe) + 1; #if MICROPY_STACKLESS if (mp_obj_get_type(*sp) == &mp_type_fun_bc) { code_state->ip = ip; @@ -1038,8 +1056,8 @@ unwind_jump:; // unum & 0xff == n_positional // (unum >> 8) & 0xff == n_keyword // We have following stack layout here: - // fun self arg0 arg1 ... kw0 val0 kw1 val1 ... seq dict <- TOS - sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe) + 3; + // fun self arg0 arg1 ... kw0 val0 kw1 val1 ... bitmap <- TOS + sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe) + 2; #if MICROPY_STACKLESS if (mp_obj_get_type(*sp) == &mp_type_fun_bc) { code_state->ip = ip; @@ -1419,23 +1437,20 @@ unwind_loop: const byte *ip = code_state->fun_bc->bytecode; MP_BC_PRELUDE_SIG_DECODE(ip); MP_BC_PRELUDE_SIZE_DECODE(ip); + const byte *line_info_top = ip + n_info; const byte *bytecode_start = ip + n_info + n_cell; - #if !MICROPY_PERSISTENT_CODE - // so bytecode is aligned - bytecode_start = MP_ALIGN(bytecode_start, sizeof(mp_uint_t)); - #endif size_t bc = code_state->ip - bytecode_start; - #if MICROPY_PERSISTENT_CODE - qstr block_name = ip[0] | (ip[1] << 8); - qstr source_file = ip[2] | (ip[3] << 8); - ip += 4; - #else qstr block_name = mp_decode_uint_value(ip); - ip = mp_decode_uint_skip(ip); - qstr source_file = mp_decode_uint_value(ip); - ip = mp_decode_uint_skip(ip); + for (size_t i = 0; i < 1 + n_pos_args + n_kwonly_args; ++i) { + ip = mp_decode_uint_skip(ip); + } + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + block_name = code_state->fun_bc->context->constants.qstr_table[block_name]; + qstr source_file = code_state->fun_bc->context->constants.qstr_table[0]; + #else + qstr source_file = code_state->fun_bc->context->constants.source_file; #endif - size_t source_line = mp_bytecode_get_source_line(ip, bc); + size_t source_line = mp_bytecode_get_source_line(ip, line_info_top, bc); mp_obj_exception_add_traceback(MP_OBJ_FROM_PTR(nlr.ret_val), source_file, source_line, block_name); } diff --git a/py/vmentrytable.h b/py/vmentrytable.h index b270dc9845..8b8781de93 100644 --- a/py/vmentrytable.h +++ b/py/vmentrytable.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013, 2014 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/vstr.c b/py/vstr.c index acc957e778..ef5fe06eb2 100644 --- a/py/vstr.c +++ b/py/vstr.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2014 Paul Sokolovsky + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/py/warning.c b/py/warning.c index 71a3ac59ab..cba21ba6c9 100644 --- a/py/warning.c +++ b/py/warning.c @@ -3,8 +3,8 @@ * * The MIT License (MIT) * - * SPDX-FileCopyrightText: Copyright (c) 2014 Damien P. George - * SPDX-FileCopyrightText: Copyright (c) 2015-2018 Paul Sokolovsky + * Copyright (c) 2014 Damien P. George + * Copyright (c) 2015-2018 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/shared-bindings/_pew/__init__.c b/shared-bindings/_pew/__init__.c index 13b9715bbd..435f50151a 100644 --- a/shared-bindings/_pew/__init__.c +++ b/shared-bindings/_pew/__init__.c @@ -62,5 +62,3 @@ const mp_obj_module_t pew_module = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&pew_module_globals, }; - -MP_REGISTER_MODULE(MP_QSTR__pew, pew_module, CIRCUITPY_PEW); diff --git a/shared-module/keypad/EventQueue.h b/shared-module/keypad/EventQueue.h index b523b16cac..207f3d7c87 100644 --- a/shared-module/keypad/EventQueue.h +++ b/shared-module/keypad/EventQueue.h @@ -24,12 +24,19 @@ * THE SOFTWARE. */ +<<<<<<< HEAD:shared-module/keypad/EventQueue.h #ifndef MICROPY_INCLUDED_SHARED_MODULE_KEYPAD_EVENTQUEUE_H #define MICROPY_INCLUDED_SHARED_MODULE_KEYPAD_EVENTQUEUE_H +======= +#include "mpconfigport.h" + +#define CFG_TUSB_RHPORT0_MODE (OPT_MODE_DEVICE) +>>>>>>> v1.19.1:ports/rp2/tusb_config.h #include "py/obj.h" #include "py/ringbuf.h" +<<<<<<< HEAD:shared-module/keypad/EventQueue.h typedef struct _keypad_eventqueue_obj_t { mp_obj_base_t base; ringbuf_t encoded_events; @@ -39,3 +46,13 @@ typedef struct _keypad_eventqueue_obj_t { bool keypad_eventqueue_record(keypad_eventqueue_obj_t *self, mp_uint_t key_number, bool pressed, mp_obj_t timestamp); #endif // MICROPY_INCLUDED_SHARED_MODULE_KEYPAD_EVENTQUEUE_H +======= +#if MICROPY_HW_USB_MSC +// Board and hardware specific configuration +#define CFG_TUD_MSC (1) +// Set MSC EP buffer size to FatFS block size to avoid partial read/writes (offset arg). +#define CFG_TUD_MSC_BUFSIZE (MICROPY_FATFS_MAX_SS) +#endif + +#endif // MICROPY_INCLUDED_RP2_TUSB_CONFIG_H +>>>>>>> v1.19.1:ports/rp2/tusb_config.h diff --git a/shared-module/sdcardio/SDCard.h b/shared-module/sdcardio/SDCard.h index ff89b89819..994e655a98 100644 --- a/shared-module/sdcardio/SDCard.h +++ b/shared-module/sdcardio/SDCard.h @@ -45,6 +45,7 @@ typedef struct { bool in_cmd25; } sdcardio_sdcard_obj_t; +<<<<<<< HEAD:shared-module/sdcardio/SDCard.h void common_hal_sdcardio_sdcard_construct(sdcardio_sdcard_obj_t *self, busio_spi_obj_t *spi, const mcu_pin_obj_t *cs, int baudrate); void common_hal_sdcardio_sdcard_deinit(sdcardio_sdcard_obj_t *self); void common_hal_sdcardio_sdcard_check_for_deinit(sdcardio_sdcard_obj_t *self); @@ -52,3 +53,36 @@ int common_hal_sdcardio_sdcard_get_blockcount(sdcardio_sdcard_obj_t *self); int common_hal_sdcardio_sdcard_readblocks(sdcardio_sdcard_obj_t *self, uint32_t start_block, mp_buffer_info_t *buf); int common_hal_sdcardio_sdcard_sync(sdcardio_sdcard_obj_t *self); int common_hal_sdcardio_sdcard_writeblocks(sdcardio_sdcard_obj_t *self, uint32_t start_block, mp_buffer_info_t *buf); +======= +static mp_obj_t microbit_repeat_iter_next(mp_obj_t iter_in) { + repeat_iterator_t *iter = (repeat_iterator_t *)iter_in; + iter->index++; + if (iter->index >= mp_obj_get_int(mp_obj_len(iter->iterable))) { + iter->index = 0; + } + return mp_obj_subscr(iter->iterable, MP_OBJ_NEW_SMALL_INT(iter->index), MP_OBJ_SENTINEL); +} + +const mp_obj_type_t microbit_repeat_iterator_type = { + { &mp_type_type }, + .name = MP_QSTR_iterator, + .print = NULL, + .make_new = NULL, + .call = NULL, + .unary_op = NULL, + .binary_op = NULL, + .attr = NULL, + .subscr = NULL, + .getiter = mp_identity_getiter, + .iternext = microbit_repeat_iter_next, + .buffer_p = {NULL}, + MP_OBJ_NULL +}; + +mp_obj_t microbit_repeat_iterator(mp_obj_t iterable) { + repeat_iterator_t *result = mp_obj_malloc(repeat_iterator_t, µbit_repeat_iterator_type); + result->iterable = iterable; + result->index = -1; + return result; +} +>>>>>>> v1.19.1:ports/nrf/boards/microbit/modules/iters.c diff --git a/shared-module/struct/__init__.h b/shared-module/struct/__init__.h index 637080b292..7e27c0474f 100644 --- a/shared-module/struct/__init__.h +++ b/shared-module/struct/__init__.h @@ -31,4 +31,15 @@ char get_fmt_type(const char **fmt); mp_uint_t get_fmt_num(const char **p); mp_uint_t calcsize_items(const char *fmt); +<<<<<<< HEAD:shared-module/struct/__init__.h +======= +// The main sleep implementation for the Windows port. +void msec_sleep(double msec); + +// Define usleep() because some of the unix port's code uses that. +// Mingw and the likes provide a definition of usleep(), note however +// that it's also just Sleep(usec/1000). +#ifdef _MSC_VER +int usleep(__int64 usec); +>>>>>>> v1.19.1:ports/windows/sleep.h #endif diff --git a/shared/readline/readline.c b/shared/readline/readline.c index d8602a9e28..9e1e904417 100644 --- a/shared/readline/readline.c +++ b/shared/readline/readline.c @@ -42,6 +42,10 @@ #define READLINE_HIST_SIZE (MP_ARRAY_SIZE(MP_STATE_PORT(readline_hist))) +// flags for readline_t.auto_indent_state +#define AUTO_INDENT_ENABLED (0x01) +#define AUTO_INDENT_JUST_ADDED (0x02) + enum { ESEQ_NONE, ESEQ_ESC, ESEQ_ESC_BRACKET, ESEQ_ESC_BRACKET_DIGIT, ESEQ_ESC_O }; void readline_init0(void) { @@ -105,6 +109,9 @@ typedef struct _readline_t { size_t cursor_pos; uint8_t utf8_cont_chars; char escape_seq_buf[1]; + #if MICROPY_REPL_AUTO_INDENT + uint8_t auto_indent_state; + #endif const char *prompt; } readline_t; @@ -248,11 +255,37 @@ int readline_process_char(int c) { redraw_step_back = nspace; redraw_from_cursor = true; } + #if MICROPY_REPL_AUTO_INDENT + } else if ((rl.auto_indent_state & AUTO_INDENT_JUST_ADDED) && (c == 9 || c == ' ')) { + // tab/space after auto-indent: disable auto-indent + // - if it's a tab then leave existing indent + // - if it's a space then remove 3 spaces from existing indent + rl.auto_indent_state = 0; + if (c == ' ') { + redraw_step_back = 3; + vstr_cut_tail_bytes(rl.line, 3); + } + #endif #if MICROPY_HELPER_REPL } else if (c == 9) { // tab magic const char *compl_str; - size_t compl_len = mp_repl_autocomplete(rl.line->buf + rl.orig_line_len, rl.cursor_pos - rl.orig_line_len, &mp_plat_print, &compl_str); + size_t compl_len; + if (vstr_len(rl.line) != 0 && unichar_isspace(vstr_str(rl.line)[rl.cursor_pos - 1])) { + // expand tab to 4 spaces if it follows whitespace: + // - includes the case of additional indenting + // - includes the case of indenting the start of a line that's not the first line, + // because a newline will be the previous character + // - doesn't include the case when at the start of the first line, because we still + // want to use auto-complete there + compl_str = " "; + compl_len = 4; + } else { + // try to auto-complete a word + const char *cur_line_buf = vstr_str(rl.line) + rl.orig_line_len; + size_t cur_line_len = rl.cursor_pos - rl.orig_line_len; + compl_len = mp_repl_autocomplete(cur_line_buf, cur_line_len, &mp_plat_print, &compl_str); + } if (compl_len == 0) { // no match } else if (compl_len == (size_t)(-1)) { @@ -504,11 +537,18 @@ redraw: rl.cursor_pos += redraw_step_forward; } + #if MICROPY_REPL_AUTO_INDENT + rl.auto_indent_state &= ~AUTO_INDENT_JUST_ADDED; + #endif + return -1; } #if MICROPY_REPL_AUTO_INDENT STATIC void readline_auto_indent(void) { + if (!(rl.auto_indent_state & AUTO_INDENT_ENABLED)) { + return; + } vstr_t *line = rl.line; if (line->len > 1 && line->buf[line->len - 1] == '\n') { int i; @@ -544,6 +584,7 @@ STATIC void readline_auto_indent(void) { vstr_add_strn(line, " ", 4); mp_hal_stdout_tx_strn(" ", 4); rl.cursor_pos += 4; + rl.auto_indent_state |= AUTO_INDENT_JUST_ADDED; } } } @@ -569,6 +610,10 @@ void readline_init(vstr_t *line, const char *prompt) { rl.prompt = prompt; mp_hal_stdout_tx_str(prompt); #if MICROPY_REPL_AUTO_INDENT + if (vstr_len(line) == 0) { + // start with auto-indent enabled + rl.auto_indent_state = AUTO_INDENT_ENABLED; + } readline_auto_indent(); #endif } diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index aeb92f59d3..417e443097 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -86,8 +86,20 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input nlr.ret_val = NULL; if (nlr_push(&nlr) == 0) { mp_obj_t module_fun; +<<<<<<< HEAD #if CIRCUITPY_ATEXIT if (!(exec_flags & EXEC_FLAG_SOURCE_IS_ATEXIT)) +======= + #if MICROPY_MODULE_FROZEN_MPY + if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) { + // source is a raw_code object, create the function + const mp_frozen_module_t *frozen = source; + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + ctx->module.globals = mp_globals_get(); + ctx->constants = frozen->constants; + module_fun = mp_make_function_from_raw_code(frozen->rc, ctx, NULL); + } else +>>>>>>> v1.19.1 #endif { #if MICROPY_MODULE_FROZEN_MPY @@ -462,9 +474,18 @@ STATIC int pyexec_friendly_repl_process_char(int c) { } else if (ret == CHAR_CTRL_B) { // reset friendly REPL mp_hal_stdout_tx_str("\r\n"); +<<<<<<< HEAD mp_hal_stdout_tx_str(MICROPY_FULL_VERSION_INFO); mp_hal_stdout_tx_str("\r\n"); // mp_hal_stdout_tx_str("Type \"help()\" for more information.\r\n"); +======= + mp_hal_stdout_tx_str(MICROPY_BANNER_NAME_AND_VERSION); + mp_hal_stdout_tx_str("; " MICROPY_BANNER_MACHINE); + mp_hal_stdout_tx_str("\r\n"); + #if MICROPY_PY_BUILTINS_HELP + mp_hal_stdout_tx_str("Type \"help()\" for more information.\r\n"); + #endif +>>>>>>> v1.19.1 goto input_restart; } else if (ret == CHAR_CTRL_C) { // break @@ -493,7 +514,7 @@ STATIC int pyexec_friendly_repl_process_char(int c) { vstr_add_byte(MP_STATE_VM(repl_line), '\n'); repl.cont_line = true; - readline_note_newline("... "); + readline_note_newline(mp_repl_get_ps2()); return 0; } else { @@ -514,7 +535,7 @@ STATIC int pyexec_friendly_repl_process_char(int c) { if (mp_repl_continue_with_input(vstr_null_terminated_str(MP_STATE_VM(repl_line)))) { vstr_add_byte(MP_STATE_VM(repl_line), '\n'); - readline_note_newline("... "); + readline_note_newline(mp_repl_get_ps2()); return 0; } @@ -528,7 +549,7 @@ STATIC int pyexec_friendly_repl_process_char(int c) { vstr_reset(MP_STATE_VM(repl_line)); repl.cont_line = false; repl.paste_mode = false; - readline_init(MP_STATE_VM(repl_line), ">>> "); + readline_init(MP_STATE_VM(repl_line), mp_repl_get_ps1()); return 0; } } @@ -612,10 +633,19 @@ int pyexec_friendly_repl(void) { vstr_init(&line, 32); friendly_repl_reset: +<<<<<<< HEAD mp_hal_stdout_tx_str("\r\n"); mp_hal_stdout_tx_str(MICROPY_FULL_VERSION_INFO); mp_hal_stdout_tx_str("\r\n"); // mp_hal_stdout_tx_str("Type \"help()\" for more information.\r\n"); +======= + mp_hal_stdout_tx_str(MICROPY_BANNER_NAME_AND_VERSION); + mp_hal_stdout_tx_str("; " MICROPY_BANNER_MACHINE); + mp_hal_stdout_tx_str("\r\n"); + #if MICROPY_PY_BUILTINS_HELP + mp_hal_stdout_tx_str("Type \"help()\" for more information.\r\n"); + #endif +>>>>>>> v1.19.1 // to test ctrl-C /* @@ -658,6 +688,7 @@ friendly_repl_reset: } vstr_reset(&line); +<<<<<<< HEAD nlr_buf_t nlr; nlr.ret_val = NULL; @@ -673,6 +704,9 @@ friendly_repl_reset: mp_hal_stdout_tx_str("\r\n"); mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); } +======= + int ret = readline(&line, mp_repl_get_ps1()); +>>>>>>> v1.19.1 mp_parse_input_kind_t parse_input_kind = MP_PARSE_SINGLE_INPUT; if (ret == CHAR_CTRL_A) { @@ -725,7 +759,7 @@ friendly_repl_reset: // got a line with non-zero length, see if it needs continuing while (mp_repl_continue_with_input(vstr_null_terminated_str(&line))) { vstr_add_byte(&line, '\n'); - ret = readline(&line, "... "); + ret = readline(&line, mp_repl_get_ps2()); if (ret == CHAR_CTRL_C) { // cancel everything mp_hal_stdout_tx_str("\r\n"); diff --git a/shared/timeutils/timeutils.h b/shared/timeutils/timeutils.h index bfed13d3a4..f82e57fe96 100644 --- a/shared/timeutils/timeutils.h +++ b/shared/timeutils/timeutils.h @@ -69,6 +69,7 @@ static inline uint64_t timeutils_mktime(mp_uint_t year, mp_int_t month, mp_int_t static inline uint64_t timeutils_seconds_since_epoch(mp_uint_t year, mp_uint_t month, mp_uint_t date, mp_uint_t hour, mp_uint_t minute, mp_uint_t second) { + // TODO this will give incorrect results for dates before 2000/1/1 return timeutils_seconds_since_2000(year, month, date, hour, minute, second) + TIMEUTILS_SECONDS_1970_TO_2000; } diff --git a/tests/basics/builtin_enumerate.py b/tests/basics/builtin_enumerate.py index 29b1dd46c4..fcbf869767 100644 --- a/tests/basics/builtin_enumerate.py +++ b/tests/basics/builtin_enumerate.py @@ -1,3 +1,9 @@ +try: + enumerate +except: + print("SKIP") + raise SystemExit + print(list(enumerate([]))) print(list(enumerate([1, 2, 3]))) print(list(enumerate([1, 2, 3], 5))) diff --git a/tests/basics/class_contains.py b/tests/basics/class_contains.py index b6dd3661cd..5fdb1db4c0 100644 --- a/tests/basics/class_contains.py +++ b/tests/basics/class_contains.py @@ -21,3 +21,27 @@ b = B([1, 2]) print(1 in b) print(2 in b) print(3 in b) + + +class C: + def __contains__(self, arg): + return arg + + +print(C().__contains__(0)) +print(C().__contains__(1)) +print(C().__contains__('')) +print(C().__contains__('foo')) +print(C().__contains__(None)) + +print(0 in C()) +print(1 in C()) +print('' in C()) +print('foo' in C()) +print(None in C()) + +print(0 not in C()) +print(1 not in C()) +print('' not in C()) +print('foo' not in C()) +print(None not in C()) diff --git a/tests/basics/fun_calldblstar4.py b/tests/basics/fun_calldblstar4.py new file mode 100644 index 0000000000..acb332a8c2 --- /dev/null +++ b/tests/basics/fun_calldblstar4.py @@ -0,0 +1,33 @@ +# test calling a function with multiple **args + + +def f(a, b=None, c=None): + print(a, b, c) + + +f(**{"a": 1}, **{"b": 2}) +f(**{"a": 1}, **{"b": 2}, c=3) +f(**{"a": 1}, b=2, **{"c": 3}) + +try: + f(1, **{"b": 2}, **{"b": 3}) +except TypeError: + print("TypeError") + +# test calling a method with multiple **args + + +class A: + def f(self, a, b=None, c=None): + print(a, b, c) + + +a = A() +a.f(**{"a": 1}, **{"b": 2}) +a.f(**{"a": 1}, **{"b": 2}, c=3) +a.f(**{"a": 1}, b=2, **{"c": 3}) + +try: + a.f(1, **{"b": 2}, **{"b": 3}) +except TypeError: + print("TypeError") diff --git a/tests/basics/fun_callstar.py b/tests/basics/fun_callstar.py index a27a288a3c..53d2ece3e1 100644 --- a/tests/basics/fun_callstar.py +++ b/tests/basics/fun_callstar.py @@ -3,10 +3,16 @@ def foo(a, b, c): print(a, b, c) +foo(*(), 1, 2, 3) +foo(*(1,), 2, 3) +foo(*(1, 2), 3) foo(*(1, 2, 3)) foo(1, *(2, 3)) foo(1, 2, *(3,)) foo(1, 2, 3, *()) +foo(*(1,), 2, *(3,)) +foo(*(1, 2), *(3,)) +foo(*(1,), *(2, 3)) # Another sequence type foo(1, 2, *[100]) @@ -29,10 +35,16 @@ class A: print(a, b, c) a = A() +a.foo(*(), 1, 2, 3) +a.foo(*(1,), 2, 3) +a.foo(*(1, 2), 3) a.foo(*(1, 2, 3)) a.foo(1, *(2, 3)) a.foo(1, 2, *(3,)) a.foo(1, 2, 3, *()) +a.foo(*(1,), 2, *(3,)) +a.foo(*(1, 2), *(3,)) +a.foo(*(1,), *(2, 3)) # Another sequence type a.foo(1, 2, *[100]) diff --git a/tests/basics/fun_callstardblstar.py b/tests/basics/fun_callstardblstar.py index f2fd29107e..f395df3333 100644 --- a/tests/basics/fun_callstardblstar.py +++ b/tests/basics/fun_callstardblstar.py @@ -6,6 +6,11 @@ def f(a, b, c, d): f(*(1, 2), **{'c':3, 'd':4}) f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)}) +try: + eval("f(**{'a': 1}, *(2, 3, 4))") +except SyntaxError: + print("SyntaxError") + # test calling a method with *tuple and **dict class A: @@ -15,3 +20,20 @@ class A: a = A() a.f(*(1, 2), **{'c':3, 'd':4}) a.f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)}) + +try: + eval("a.f(**{'a': 1}, *(2, 3, 4))") +except SyntaxError: + print("SyntaxError") + + +# coverage test for arg allocation corner case + +def f2(*args, **kwargs): + print(len(args), len(kwargs)) + + +f2(*iter(range(4)), **{'a': 1}) + +# case where *args is not a tuple/list and takes up most of the memory allocated for **kwargs +f2(*iter(range(100)), **{str(i): i for i in range(100)}) diff --git a/tests/basics/fun_kwvarargs.py b/tests/basics/fun_kwvarargs.py index bdc10fcf14..e9fd0720e9 100644 --- a/tests/basics/fun_kwvarargs.py +++ b/tests/basics/fun_kwvarargs.py @@ -23,3 +23,16 @@ def f4(*vargs, **kwargs): f4(*(1, 2)) f4(kw_arg=3) f4(*(1, 2), kw_arg=3) + + +# test evaluation order of arguments +def f5(*vargs, **kwargs): + print(vargs, kwargs) + + +def print_ret(x): + print(x) + return x + + +f5(*print_ret(["a", "b"]), kw_arg=print_ret(None)) diff --git a/tests/basics/nanbox_smallint.py b/tests/basics/nanbox_smallint.py new file mode 100644 index 0000000000..167b196010 --- /dev/null +++ b/tests/basics/nanbox_smallint.py @@ -0,0 +1,43 @@ +# Test creating small integers without heap allocation in nan-boxing mode. + +import micropython + +try: + # Test for nan-box build by allocating a float while heap is locked. + # This should pass on nan-box builds. + micropython.heap_lock() + float(123) + micropython.heap_unlock() +except: + micropython.heap_unlock() + print("SKIP") + raise SystemExit + +# Check that nan-boxing uses 64-bit floats (eg it's not object representation C). +if float("1e100") == float("inf"): + print("SKIP") + raise SystemExit + +micropython.heap_lock() +print(int("0x80000000")) +micropython.heap_unlock() + +# This is the most positive small integer. +micropython.heap_lock() +print(int("0x3fffffffffff")) +micropython.heap_unlock() + +# This is the most negative small integer. +micropython.heap_lock() +print(int("-0x3fffffffffff") - 1) +micropython.heap_unlock() + +x = 1 +micropython.heap_lock() +print((x << 31) + 1) +micropython.heap_unlock() + +x = 1 +micropython.heap_lock() +print((x << 45) + 1) +micropython.heap_unlock() diff --git a/tests/basics/nanbox_smallint.py.exp b/tests/basics/nanbox_smallint.py.exp new file mode 100644 index 0000000000..aad1f7b8b6 --- /dev/null +++ b/tests/basics/nanbox_smallint.py.exp @@ -0,0 +1,5 @@ +2147483648 +70368744177663 +-70368744177664 +2147483649 +35184372088833 diff --git a/tests/basics/python34.py b/tests/basics/python34.py index 4030db143c..922234d22d 100644 --- a/tests/basics/python34.py +++ b/tests/basics/python34.py @@ -6,27 +6,23 @@ except NameError: print("SKIP") raise SystemExit -# from basics/fun_kwvarargs.py -# test evaluation order of arguments (in 3.4 it's backwards, 3.5 it's fixed) -def f4(*vargs, **kwargs): - print(vargs, kwargs) + def print_ret(x): print(x) return x -f4(*print_ret(['a', 'b']), kw_arg=print_ret(None)) # test evaluation order of dictionary key/value pair (in 3.4 it's backwards) {print_ret(1):print_ret(2)} + # from basics/syntaxerror.py def test_syntax(code): try: exec(code) except SyntaxError: print("SyntaxError") -test_syntax("f(*a, *b)") # can't have multiple * (in 3.5 we can) -test_syntax("f(**a, **b)") # can't have multiple ** (in 3.5 we can) -test_syntax("f(*a, b)") # can't have positional after * + + test_syntax("f(**a, b)") # can't have positional after ** test_syntax("() = []") # can't assign to empty tuple (in 3.6 we can) test_syntax("del ()") # can't delete empty tuple (in 3.6 we can) diff --git a/tests/basics/python34.py.exp b/tests/basics/python34.py.exp index 8480171307..a56c1a50b6 100644 --- a/tests/basics/python34.py.exp +++ b/tests/basics/python34.py.exp @@ -1,14 +1,8 @@ -None -['a', 'b'] -('a', 'b') {'kw_arg': None} 2 1 SyntaxError SyntaxError SyntaxError -SyntaxError -SyntaxError -SyntaxError 3.4 3 4 IndexError('foo',) diff --git a/tests/basics/scope_class.py b/tests/basics/scope_class.py new file mode 100644 index 0000000000..9c519695dc --- /dev/null +++ b/tests/basics/scope_class.py @@ -0,0 +1,77 @@ +# test scoping rules that involve a class + +# the inner A.method should be independent to the local function called method +def test1(): + def method(): + pass + + class A: + def method(): + pass + + print(hasattr(A, "method")) + print(hasattr(A(), "method")) + + +test1() + + +# the inner A.method is a closure and overrides the local function called method +def test2(): + def method(): + return "outer" + + class A: + nonlocal method + + def method(): + return "inner" + + print(hasattr(A, "method")) + print(hasattr(A(), "method")) + return method() # this is actually A.method + + +print(test2()) + + +# a class body will capture external variables by value (not by reference) +def test3(x): + class A: + local = x + + x += 1 + return x, A.local + + +print(test3(42)) + + +# assigning to a variable in a class will implicitly prevent it from closing over a variable +def test4(global_): + class A: + local = global_ # fetches outer global_ + global_ = "global2" # creates class attribute + + global_ += 1 # updates local variable + return global_, A.local, A.global_ + + +global_ = "global" +print(test4(42), global_) + + +# methods within a class can close over variables outside the class +def test5(x): + def closure(): + return x + + class A: + def method(): + return x, closure() + + closure = lambda: x + 1 # change it after A has been created + return A + + +print(test5(42).method()) diff --git a/tests/basics/string_format_cp310.py b/tests/basics/string_format_cp310.py new file mode 100644 index 0000000000..77295330be --- /dev/null +++ b/tests/basics/string_format_cp310.py @@ -0,0 +1,9 @@ +# Python 3.10+ functionality test for {} format string + +def test(fmt, *args): + print('{:8s}'.format(fmt) + '>' + fmt.format(*args) + '<') + +test("{:0s}", "ab") +test("{:06s}", "ab") +test("{:<06s}", "ab") +test("{:>06s}", "ab") diff --git a/tests/basics/string_format_cp310.py.exp b/tests/basics/string_format_cp310.py.exp new file mode 100644 index 0000000000..1c26473d83 --- /dev/null +++ b/tests/basics/string_format_cp310.py.exp @@ -0,0 +1,4 @@ +{:0s} >ab< +{:06s} >ab0000< +{:<06s} >ab0000< +{:>06s} >0000ab< diff --git a/tests/basics/string_format_error.py b/tests/basics/string_format_error.py index 708348d59f..74dc6c52ed 100644 --- a/tests/basics/string_format_error.py +++ b/tests/basics/string_format_error.py @@ -1,7 +1,7 @@ # tests for errors in {} format string try: - '{0:0}'.format('zzz') + '{0:=}'.format('zzz') except (ValueError): print('ValueError') diff --git a/tests/basics/sys1.py b/tests/basics/sys1.py index 15f7771ef5..c8dbfc3541 100644 --- a/tests/basics/sys1.py +++ b/tests/basics/sys1.py @@ -19,8 +19,8 @@ except AttributeError: # Effectively skip subtests print(True) -if hasattr(sys.implementation, 'mpy'): - print(type(sys.implementation.mpy)) +if hasattr(sys.implementation, '_mpy'): + print(type(sys.implementation._mpy)) else: # Effectively skip subtests print(int) diff --git a/tests/basics/sys_tracebacklimit.py b/tests/basics/sys_tracebacklimit.py new file mode 100644 index 0000000000..1ee638967f --- /dev/null +++ b/tests/basics/sys_tracebacklimit.py @@ -0,0 +1,78 @@ +# test sys.tracebacklimit + +try: + try: + import usys as sys + import uio as io + except ImportError: + import sys + import io +except ImportError: + print("SKIP") + raise SystemExit + +try: + sys.tracebacklimit = 1000 +except AttributeError: + print("SKIP") + raise SystemExit + +if hasattr(sys, "print_exception"): + print_exception = sys.print_exception +else: + import traceback + + print_exception = lambda e, f: traceback.print_exception(None, e, sys.exc_info()[2], file=f) + + +def print_exc(e): + buf = io.StringIO() + print_exception(e, buf) + s = buf.getvalue() + for l in s.split("\n"): + # Remove filename. + if l.startswith(" File "): + l = l.split('"') + print(l[0], l[2]) + # uPy and CPy tracebacks differ in that CPy prints a source line for + # each traceback entry. In this case, we know that offending line + # has 4-space indent, so filter it out. + elif not l.startswith(" "): + print(l) + + +def f0(): + raise ValueError("value") + + +def f1(): + f0() + + +def f2(): + f1() + + +def f3(): + f2() + + +def ftop(): + try: + f3() + except ValueError as er: + print_exc(er) + + +ftop() + +for limit in range(4, -2, -1): + print("limit", limit) + sys.tracebacklimit = limit + ftop() + + +# test deleting the attribute +print(hasattr(sys, "tracebacklimit")) +del sys.tracebacklimit +print(hasattr(sys, "tracebacklimit")) diff --git a/tests/basics/sys_tracebacklimit.py.exp b/tests/basics/sys_tracebacklimit.py.exp new file mode 100644 index 0000000000..3647280584 --- /dev/null +++ b/tests/basics/sys_tracebacklimit.py.exp @@ -0,0 +1,42 @@ +Traceback (most recent call last): + File , line 62, in ftop + File , line 57, in f3 + File , line 53, in f2 + File , line 49, in f1 + File , line 45, in f0 +ValueError: value + +limit 4 +Traceback (most recent call last): + File , line 62, in ftop + File , line 57, in f3 + File , line 53, in f2 + File , line 49, in f1 +ValueError: value + +limit 3 +Traceback (most recent call last): + File , line 62, in ftop + File , line 57, in f3 + File , line 53, in f2 +ValueError: value + +limit 2 +Traceback (most recent call last): + File , line 62, in ftop + File , line 57, in f3 +ValueError: value + +limit 1 +Traceback (most recent call last): + File , line 62, in ftop +ValueError: value + +limit 0 +ValueError: value + +limit -1 +ValueError: value + +True +False diff --git a/tests/cmdline/cmd_parsetree.py.exp b/tests/cmdline/cmd_parsetree.py.exp index bee4fc99d1..3049267c0b 100644 --- a/tests/cmdline/cmd_parsetree.py.exp +++ b/tests/cmdline/cmd_parsetree.py.exp @@ -18,7 +18,7 @@ [ 8] literal \.\+ [ 9] \(rule\|expr_stmt\)(5) (n=2) id(d) - bytes(bytes) +[ 9] literal \.\+ [ 10] \(rule\|expr_stmt\)(5) (n=2) id(e) [ 10] literal \.\+ @@ -31,7 +31,7 @@ [ 13] \(rule\|expr_stmt\)(5) (n=2) id(h) [ 13] \(rule\|atom_expr_normal\)(44) (n=2) -[ 13] literal const(\.\+) +[ 13] literal const(\.\+)="fstring: '{}'" [ 13] \(rule\|atom_expr_trailers\)(142) (n=2) [ 13] \(rule\|trailer_period\)(50) (n=1) id(format) @@ -39,50 +39,52 @@ [ 13] \(rule\|arglist\)(164) (n=1) id(b) ---------------- -File cmdline/cmd_parsetree.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_parsetree.py, code block '' (descriptor: \.\+, bytecode @\.\+ 62 bytes) +Raw bytecode (code_info_size=13, bytecode_size=49): + 20 16 01 60 27 22 23 24 24 24 24 24 25 2a 00 5f + 4b 04 16 04 42 3a 51 16 05 10 02 16 06 23 00 16 + 07 23 01 16 08 23 02 16 09 23 03 16 0a 22 80 7b + 16 0b 23 04 14 03 11 06 36 01 16 0c 51 63 arg names: (N_STATE 5) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=4 - bc=9 line=5 - bc=12 line=6 - bc=16 line=7 - bc=22 line=8 - bc=27 line=9 - bc=32 line=10 - bc=37 line=11 - bc=42 line=12 - bc=48 line=13 + bc=7 line=5 + bc=9 line=6 + bc=12 line=7 + bc=16 line=8 + bc=20 line=9 + bc=24 line=10 + bc=28 line=11 + bc=32 line=12 + bc=37 line=13 00 BUILD_TUPLE 0 02 GET_ITER_STACK -03 FOR_ITER 12 -06 STORE_NAME i -09 JUMP 3 -12 LOAD_CONST_NONE -13 STORE_NAME a -16 LOAD_CONST_STRING 'str' -19 STORE_NAME b -22 LOAD_CONST_OBJ \.\+ -24 STORE_NAME c -27 LOAD_CONST_OBJ \.\+ -29 STORE_NAME d -32 LOAD_CONST_OBJ \.\+ -34 STORE_NAME e -37 LOAD_CONST_OBJ \.\+ -39 STORE_NAME f -42 LOAD_CONST_SMALL_INT 123 -45 STORE_NAME g -48 LOAD_CONST_OBJ \.\+ -50 LOAD_METHOD format -53 LOAD_NAME b -56 CALL_METHOD n=1 nkw=0 -58 STORE_NAME h -61 LOAD_CONST_NONE -62 RETURN_VALUE +03 FOR_ITER 9 +05 STORE_NAME i +07 JUMP 3 +09 LOAD_CONST_NONE +10 STORE_NAME a +12 LOAD_CONST_STRING 'str' +14 STORE_NAME b +16 LOAD_CONST_OBJ \.\+='a very long str that will not be interned' +18 STORE_NAME c +20 LOAD_CONST_OBJ \.\+=b'bytes' +22 STORE_NAME d +24 LOAD_CONST_OBJ \.\+=b'a very long bytes that will not be interned' +26 STORE_NAME e +28 LOAD_CONST_OBJ \.\+=123456789012345678901234567890 +30 STORE_NAME f +32 LOAD_CONST_SMALL_INT 123 +35 STORE_NAME g +37 LOAD_CONST_OBJ \.\+="fstring: '{}'" +39 LOAD_METHOD format +41 LOAD_NAME b +43 CALL_METHOD n=1 nkw=0 +45 STORE_NAME h +47 LOAD_CONST_NONE +48 RETURN_VALUE mem: total=\\d\+, current=\\d\+, peak=\\d\+ stack: \\d\+ out of \\d\+ GC: total: \\d\+, used: \\d\+, free: \\d\+ diff --git a/tests/cmdline/cmd_showbc.py b/tests/cmdline/cmd_showbc.py index a960c15c4a..e5874f990a 100644 --- a/tests/cmdline/cmd_showbc.py +++ b/tests/cmdline/cmd_showbc.py @@ -40,7 +40,7 @@ def f(): # slice a = b[::] - # sequenc unpacking + # sequence unpacking a, b = c a, *a = a diff --git a/tests/cmdline/cmd_showbc.py.exp b/tests/cmdline/cmd_showbc.py.exp index 22712b79ee..45a1d169b6 100644 --- a/tests/cmdline/cmd_showbc.py.exp +++ b/tests/cmdline/cmd_showbc.py.exp @@ -1,51 +1,144 @@ -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 63 bytes) +Raw bytecode (code_info_size=18, bytecode_size=45): + 10 20 01 60 20 84 7d 64 60 88 07 64 60 69 20 62 + 64 20 32 00 16 05 32 01 16 05 81 2a 01 53 33 02 + 16 05 32 03 16 05 54 32 04 10 02 34 02 16 02 19 + 02 32 05 16 05 80 10 03 2a 01 1b 04 69 51 63 arg names: (N_STATE 3) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=160 + bc=0 line=4 + bc=0 line=5 + bc=4 line=130 + bc=8 line=133 + bc=8 line=136 + bc=16 line=143 + bc=20 line=146 + bc=20 line=149 + bc=29 line=152 + bc=29 line=153 + bc=31 line=156 + bc=35 line=159 + bc=35 line=160 00 MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ BUILD_TUPLE 1 -\\d\+ LOAD_NULL -\\d\+ MAKE_FUNCTION_DEFARGS \.\+ -\\d\+ STORE_NAME f -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ LOAD_BUILD_CLASS -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ LOAD_CONST_STRING 'Class' -\\d\+ CALL_FUNCTION n=2 nkw=0 -\\d\+ STORE_NAME Class -\\d\+ DELETE_NAME Class -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_CONST_STRING '*' -\\d\+ BUILD_TUPLE 1 -\\d\+ IMPORT_NAME 'sys' -\\d\+ IMPORT_STAR -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): +02 STORE_NAME f +04 MAKE_FUNCTION \.\+ +06 STORE_NAME f +08 LOAD_CONST_SMALL_INT 1 +09 BUILD_TUPLE 1 +11 LOAD_NULL +12 MAKE_FUNCTION_DEFARGS \.\+ +14 STORE_NAME f +16 MAKE_FUNCTION \.\+ +18 STORE_NAME f +20 LOAD_BUILD_CLASS +21 MAKE_FUNCTION \.\+ +23 LOAD_CONST_STRING 'Class' +25 CALL_FUNCTION n=2 nkw=0 +27 STORE_NAME Class +29 DELETE_NAME Class +31 MAKE_FUNCTION \.\+ +33 STORE_NAME f +35 LOAD_CONST_SMALL_INT 0 +36 LOAD_CONST_STRING '*' +38 BUILD_TUPLE 1 +40 IMPORT_NAME 'sys' +42 IMPORT_STAR +43 LOAD_CONST_NONE +44 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 45\[46\] bytes) +Raw bytecode (code_info_size=8\[46\], bytecode_size=370): + a8 12 9\[bf\] 03 05 60 60 26 22 24 64 22 24 25 25 24 + 26 23 63 22 22 25 23 23 2f 6c 25 65 25 25 69 68 + 26 65 27 6a 62 20 23 62 2a 29 69 24 25 28 67 26 ######## -\.\+rg names: +\.\+81 63 +arg names: (N_STATE 22) (N_EXC_STACK 2) (INIT_CELL 14) (INIT_CELL 15) (INIT_CELL 16) bc=0 line=1 + bc=0 line=4 + bc=0 line=7 + bc=6 line=8 + bc=8 line=9 + bc=12 line=10 + bc=16 line=13 + bc=18 line=14 + bc=22 line=15 + bc=27 line=16 + bc=32 line=17 + bc=36 line=18 + bc=42 line=19 + bc=45 line=20 + bc=48 line=23 + bc=50 line=24 + bc=52 line=25 + bc=57 line=26 + bc=60 line=27 + bc=63 line=28 + bc=78 line=29 + bc=90 line=32 + bc=95 line=33 + bc=100 line=36 + bc=105 line=37 + bc=110 line=38 + bc=119 line=41 + bc=127 line=44 + bc=133 line=45 + bc=138 line=48 + bc=145 line=49 + bc=155 line=52 + bc=157 line=55 + bc=157 line=56 + bc=160 line=57 + bc=162 line=60 + bc=172 line=61 + bc=181 line=62 + bc=190 line=65 + bc=194 line=66 + bc=199 line=67 + bc=207 line=68 + bc=214 line=71 + bc=220 line=72 + bc=227 line=73 + bc=237 line=74 + bc=245 line=77 + bc=248 line=78 + bc=253 line=80 + bc=256 line=81 + bc=258 line=82 + bc=264 line=83 + bc=266 line=84 + bc=272 line=85 + bc=277 line=88 + bc=283 line=89 + bc=287 line=92 + bc=291 line=93 + bc=293 line=94 ######## - bc=\\d\+ line=127 + bc=301 line=96 + bc=308 line=98 + bc=311 line=99 + bc=313 line=100 + bc=315 line=101 +######## + bc=321 line=103 + bc=327 line=106 + bc=331 line=107 + bc=337 line=110 + bc=340 line=111 + bc=346 line=114 + bc=346 line=117 + bc=351 line=118 + bc=363 line=121 + bc=363 line=122 + bc=364 line=123 + bc=366 line=126 + bc=368 line=127 00 LOAD_CONST_NONE 01 LOAD_CONST_FALSE 02 BINARY_OP 27 __add__ @@ -60,267 +153,269 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): 15 STORE_FAST 0 16 LOAD_CONST_SMALL_INT 1 17 STORE_FAST 0 -18 LOAD_CONST_SMALL_INT 1 -19 LOAD_CONST_SMALL_INT 2 -20 BUILD_TUPLE 2 -22 STORE_DEREF 14 -24 LOAD_CONST_SMALL_INT 1 -25 LOAD_CONST_SMALL_INT 2 -26 BUILD_LIST 2 -28 STORE_FAST 1 -29 LOAD_CONST_SMALL_INT 1 -30 LOAD_CONST_SMALL_INT 2 -31 BUILD_SET 2 -33 STORE_FAST 2 -34 BUILD_MAP 0 -36 STORE_DEREF 15 -38 BUILD_MAP 1 -40 LOAD_CONST_SMALL_INT 2 -41 LOAD_CONST_SMALL_INT 1 -42 STORE_MAP -43 STORE_FAST 3 -44 LOAD_CONST_STRING 'a' -47 STORE_FAST 4 -48 LOAD_CONST_OBJ \.\+ -\\d\+ STORE_FAST 5 -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ STORE_FAST 6 -\\d\+ LOAD_CONST_SMALL_INT 2 -\\d\+ STORE_FAST 7 -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -58 BINARY_OP 27 __add__ -\\d\+ STORE_FAST 8 -\\d\+ LOAD_FAST 0 -\\d\+ UNARY_OP 1 __neg__ -\\d\+ STORE_FAST 9 -\\d\+ LOAD_FAST 0 -\\d\+ UNARY_OP 3 -\\d\+ STORE_FAST 10 -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ DUP_TOP -\\d\+ ROT_THREE -\\d\+ BINARY_OP 2 __eq__ -\\d\+ JUMP_IF_FALSE_OR_POP \\d\+ -\\d\+ LOAD_FAST 1 -\\d\+ BINARY_OP 2 __eq__ -\\d\+ JUMP \\d\+ -\\d\+ ROT_TWO -\\d\+ POP_TOP -\\d\+ STORE_FAST 10 -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ BINARY_OP 2 __eq__ -\\d\+ JUMP_IF_FALSE_OR_POP \\d\+ -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_FAST 1 -\\d\+ BINARY_OP 2 __eq__ -\\d\+ UNARY_OP 3 -\\d\+ STORE_FAST 10 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_ATTR c -\\d\+ STORE_FAST 11 -\\d\+ LOAD_FAST 11 -\\d\+ LOAD_DEREF 14 -\\d\+ STORE_ATTR c -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_SUBSCR -\\d\+ STORE_FAST 12 -\\d\+ LOAD_FAST 12 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ STORE_SUBSCR -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ DUP_TOP_TWO -\\d\+ LOAD_SUBSCR -\\d\+ LOAD_FAST 12 -\\d\+ BINARY_OP 14 __iadd__ -\\d\+ ROT_THREE -\\d\+ STORE_SUBSCR -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_NONE -\\d\+ LOAD_CONST_NONE -\\d\+ BUILD_SLICE 2 -\\d\+ LOAD_SUBSCR -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 1 -\\d\+ UNPACK_SEQUENCE 2 -\\d\+ STORE_FAST 0 -\\d\+ STORE_DEREF 14 -\\d\+ LOAD_FAST 0 -\\d\+ UNPACK_EX 1 -\\d\+ STORE_FAST 0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_FAST 0 -\\d\+ ROT_TWO -\\d\+ STORE_FAST 0 -\\d\+ STORE_DEREF 14 -\\d\+ LOAD_FAST 1 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_FAST 0 -\\d\+ ROT_THREE -\\d\+ ROT_TWO -\\d\+ STORE_FAST 0 -\\d\+ STORE_DEREF 14 -\\d\+ STORE_FAST 1 -\\d\+ DELETE_FAST 0 -\\d\+ LOAD_FAST 0 -\\d\+ STORE_GLOBAL gl -\\d\+ DELETE_GLOBAL gl -\\d\+ LOAD_FAST 14 -\\d\+ LOAD_FAST 15 -\\d\+ MAKE_CLOSURE \.\+ 2 -\\d\+ LOAD_FAST 2 -\\d\+ GET_ITER -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 14 -\\d\+ LOAD_FAST 15 -\\d\+ MAKE_CLOSURE \.\+ 2 -\\d\+ LOAD_FAST 2 -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 14 -\\d\+ LOAD_FAST 15 -\\d\+ MAKE_CLOSURE \.\+ 2 -\\d\+ LOAD_FAST 2 -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 0 -\\d\+ CALL_FUNCTION n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_CONST_STRING 'b' -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_FUNCTION n=0 nkw=1 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_NULL -\\d\+ CALL_FUNCTION_VAR_KW n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ CALL_METHOD n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_METHOD n=1 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ LOAD_CONST_STRING 'c' -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_METHOD n=0 nkw=1 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ LOAD_FAST 1 -\\d\+ LOAD_NULL -\\d\+ CALL_METHOD_VAR_KW n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_FALSE \\d\+ -\\d\+ LOAD_DEREF 16 -\\d\+ POP_TOP -\\d\+ JUMP \\d\+ -\\d\+ LOAD_GLOBAL y -\\d\+ POP_TOP -\\d\+ JUMP \\d\+ -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_TRUE \\d\+ -\\d\+ JUMP \\d\+ -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_FALSE \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ JUMP_IF_TRUE_OR_POP \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ GET_ITER_STACK -\\d\+ FOR_ITER \\d\+ -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 1 -\\d\+ POP_TOP -\\d\+ JUMP \\d\+ -\\d\+ SETUP_FINALLY \\d\+ -\\d\+ SETUP_EXCEPT \\d\+ -\\d\+ JUMP \\d\+ -\\d\+ JUMP \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_TRUE \\d\+ -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ POP_TOP -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ END_FINALLY -\\d\+ LOAD_CONST_NONE -\\d\+ LOAD_FAST 1 -\\d\+ POP_TOP -\\d\+ END_FINALLY -\\d\+ JUMP \\d\+ -\\d\+ SETUP_EXCEPT \\d\+ -\\d\+ UNWIND_JUMP \\d\+ 1 -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ POP_TOP -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ END_FINALLY -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_TRUE \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ SETUP_WITH \\d\+ -\\d\+ POP_TOP -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ LOAD_CONST_NONE -\\d\+ WITH_CLEANUP -\\d\+ END_FINALLY -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ STORE_DEREF 16 -\\d\+ LOAD_FAST_N 16 -\\d\+ MAKE_CLOSURE \.\+ 1 -\\d\+ STORE_FAST 13 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_CONST_NONE -\\d\+ IMPORT_NAME 'a' -\\d\+ STORE_FAST 0 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_CONST_STRING 'b' -\\d\+ BUILD_TUPLE 1 -\\d\+ IMPORT_NAME 'a' -\\d\+ IMPORT_FROM 'b' -\\d\+ STORE_DEREF 14 -\\d\+ POP_TOP -\\d\+ RAISE_LAST -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ RAISE_OBJ -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+rg names: +18 LOAD_CONST_OBJ \.\+=(1, 2) +20 STORE_DEREF 14 +22 LOAD_CONST_SMALL_INT 1 +23 LOAD_CONST_SMALL_INT 2 +24 BUILD_LIST 2 +26 STORE_FAST 1 +27 LOAD_CONST_SMALL_INT 1 +28 LOAD_CONST_SMALL_INT 2 +29 BUILD_SET 2 +31 STORE_FAST 2 +32 BUILD_MAP 0 +34 STORE_DEREF 15 +36 BUILD_MAP 1 +38 LOAD_CONST_SMALL_INT 2 +39 LOAD_CONST_SMALL_INT 1 +40 STORE_MAP +41 STORE_FAST 3 +42 LOAD_CONST_STRING 'a' +44 STORE_FAST 4 +45 LOAD_CONST_OBJ \.\+=b'a' +47 STORE_FAST 5 +48 LOAD_CONST_SMALL_INT 1 +49 STORE_FAST 6 +50 LOAD_CONST_SMALL_INT 2 +51 STORE_FAST 7 +52 LOAD_FAST 0 +53 LOAD_DEREF 14 +55 BINARY_OP 27 __add__ +56 STORE_FAST 8 +57 LOAD_FAST 0 +58 UNARY_OP 1 __neg__ +59 STORE_FAST 9 +60 LOAD_FAST 0 +61 UNARY_OP 3 +62 STORE_FAST 10 +63 LOAD_FAST 0 +64 LOAD_DEREF 14 +66 DUP_TOP +67 ROT_THREE +68 BINARY_OP 2 __eq__ +69 JUMP_IF_FALSE_OR_POP 75 +71 LOAD_FAST 1 +72 BINARY_OP 2 __eq__ +73 JUMP 77 +75 ROT_TWO +76 POP_TOP +77 STORE_FAST 10 +78 LOAD_FAST 0 +79 LOAD_DEREF 14 +81 BINARY_OP 2 __eq__ +82 JUMP_IF_FALSE_OR_POP 88 +84 LOAD_DEREF 14 +86 LOAD_FAST 1 +87 BINARY_OP 2 __eq__ +88 UNARY_OP 3 +89 STORE_FAST 10 +90 LOAD_DEREF 14 +92 LOAD_ATTR c +94 STORE_FAST 11 +95 LOAD_FAST 11 +96 LOAD_DEREF 14 +98 STORE_ATTR c +100 LOAD_DEREF 14 +102 LOAD_CONST_SMALL_INT 0 +103 LOAD_SUBSCR +104 STORE_FAST 12 +105 LOAD_FAST 12 +106 LOAD_DEREF 14 +108 LOAD_CONST_SMALL_INT 0 +109 STORE_SUBSCR +110 LOAD_DEREF 14 +112 LOAD_CONST_SMALL_INT 0 +113 DUP_TOP_TWO +114 LOAD_SUBSCR +115 LOAD_FAST 12 +116 BINARY_OP 14 __iadd__ +117 ROT_THREE +118 STORE_SUBSCR +119 LOAD_DEREF 14 +121 LOAD_CONST_NONE +122 LOAD_CONST_NONE +123 BUILD_SLICE 2 +125 LOAD_SUBSCR +126 STORE_FAST 0 +127 LOAD_FAST 1 +128 UNPACK_SEQUENCE 2 +130 STORE_FAST 0 +131 STORE_DEREF 14 +133 LOAD_FAST 0 +134 UNPACK_EX 1 +136 STORE_FAST 0 +137 STORE_FAST 0 +138 LOAD_DEREF 14 +140 LOAD_FAST 0 +141 ROT_TWO +142 STORE_FAST 0 +143 STORE_DEREF 14 +145 LOAD_FAST 1 +146 LOAD_DEREF 14 +148 LOAD_FAST 0 +149 ROT_THREE +150 ROT_TWO +151 STORE_FAST 0 +152 STORE_DEREF 14 +154 STORE_FAST 1 +155 DELETE_FAST 0 +157 LOAD_FAST 0 +158 STORE_GLOBAL gl +160 DELETE_GLOBAL gl +162 LOAD_FAST 14 +163 LOAD_FAST 15 +164 MAKE_CLOSURE \.\+ 2 +167 LOAD_FAST 2 +168 GET_ITER +169 CALL_FUNCTION n=1 nkw=0 +171 STORE_FAST 0 +172 LOAD_FAST 14 +173 LOAD_FAST 15 +174 MAKE_CLOSURE \.\+ 2 +177 LOAD_FAST 2 +178 CALL_FUNCTION n=1 nkw=0 +180 STORE_FAST 0 +181 LOAD_FAST 14 +182 LOAD_FAST 15 +183 MAKE_CLOSURE \.\+ 2 +186 LOAD_FAST 2 +187 CALL_FUNCTION n=1 nkw=0 +189 STORE_FAST 0 +190 LOAD_FAST 0 +191 CALL_FUNCTION n=0 nkw=0 +193 POP_TOP +194 LOAD_FAST 0 +195 LOAD_CONST_SMALL_INT 1 +196 CALL_FUNCTION n=1 nkw=0 +198 POP_TOP +199 LOAD_FAST 0 +200 LOAD_CONST_STRING 'b' +202 LOAD_CONST_SMALL_INT 1 +203 CALL_FUNCTION n=0 nkw=1 +206 POP_TOP +207 LOAD_FAST 0 +208 LOAD_DEREF 14 +210 LOAD_CONST_SMALL_INT 1 +211 CALL_FUNCTION_VAR_KW n=1 nkw=0 +213 POP_TOP +214 LOAD_FAST 0 +215 LOAD_METHOD b +217 CALL_METHOD n=0 nkw=0 +219 POP_TOP +220 LOAD_FAST 0 +221 LOAD_METHOD b +223 LOAD_CONST_SMALL_INT 1 +224 CALL_METHOD n=1 nkw=0 +226 POP_TOP +227 LOAD_FAST 0 +228 LOAD_METHOD b +230 LOAD_CONST_STRING 'c' +232 LOAD_CONST_SMALL_INT 1 +233 CALL_METHOD n=0 nkw=1 +236 POP_TOP +237 LOAD_FAST 0 +238 LOAD_METHOD b +240 LOAD_FAST 1 +241 LOAD_CONST_SMALL_INT 1 +242 CALL_METHOD_VAR_KW n=1 nkw=0 +244 POP_TOP +245 LOAD_FAST 0 +246 POP_JUMP_IF_FALSE 253 +248 LOAD_DEREF 16 +250 POP_TOP +251 JUMP 256 +253 LOAD_GLOBAL y +255 POP_TOP +256 JUMP 261 +258 LOAD_DEREF 14 +260 POP_TOP +261 LOAD_FAST 0 +262 POP_JUMP_IF_TRUE 258 +264 JUMP 269 +266 LOAD_DEREF 14 +268 POP_TOP +269 LOAD_FAST 0 +270 POP_JUMP_IF_FALSE 266 +272 LOAD_FAST 0 +273 JUMP_IF_TRUE_OR_POP 276 +275 LOAD_FAST 0 +276 STORE_FAST 0 +277 LOAD_DEREF 14 +279 GET_ITER_STACK +280 FOR_ITER 287 +282 STORE_FAST 0 +283 LOAD_FAST 1 +284 POP_TOP +285 JUMP 280 +287 SETUP_FINALLY 308 +289 SETUP_EXCEPT 300 +291 JUMP 295 +293 JUMP 298 +295 LOAD_FAST 0 +296 POP_JUMP_IF_TRUE 293 +298 POP_EXCEPT_JUMP 307 +300 POP_TOP +301 LOAD_DEREF 14 +303 POP_TOP +304 POP_EXCEPT_JUMP 307 +306 END_FINALLY +307 LOAD_CONST_NONE +308 LOAD_FAST 1 +309 POP_TOP +310 END_FINALLY +311 JUMP 324 +313 SETUP_EXCEPT 320 +315 UNWIND_JUMP 327 1 +318 POP_EXCEPT_JUMP 324 +320 POP_TOP +321 POP_EXCEPT_JUMP 324 +323 END_FINALLY +324 LOAD_FAST 0 +325 POP_JUMP_IF_TRUE 313 +327 LOAD_FAST 0 +328 SETUP_WITH 335 +330 POP_TOP +331 LOAD_DEREF 14 +333 POP_TOP +334 LOAD_CONST_NONE +335 WITH_CLEANUP +336 END_FINALLY +337 LOAD_CONST_SMALL_INT 1 +338 STORE_DEREF 16 +340 LOAD_FAST_N 16 +342 MAKE_CLOSURE \.\+ 1 +345 STORE_FAST 13 +346 LOAD_CONST_SMALL_INT 0 +347 LOAD_CONST_NONE +348 IMPORT_NAME 'a' +350 STORE_FAST 0 +351 LOAD_CONST_SMALL_INT 0 +352 LOAD_CONST_STRING 'b' +354 BUILD_TUPLE 1 +356 IMPORT_NAME 'a' +358 IMPORT_FROM 'b' +360 STORE_DEREF 14 +362 POP_TOP +363 RAISE_LAST +364 LOAD_CONST_SMALL_INT 1 +365 RAISE_OBJ +366 LOAD_CONST_NONE +367 RETURN_VALUE +368 LOAD_CONST_SMALL_INT 1 +369 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 59 bytes) +Raw bytecode (code_info_size=8, bytecode_size=51): + a8 10 0a 05 80 82 34 38 81 57 c0 57 c1 57 c2 57 + c3 57 c4 57 c5 57 c6 57 c7 57 c8 c9 82 57 ca 57 + cb 57 cc 57 cd 57 ce 57 cf 57 26 10 57 26 11 57 + 26 12 26 13 b9 24 13 f2 59 51 63 +arg names: (N_STATE 22) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=133 + bc=0 line=131 + bc=20 line=132 + bc=44 line=133 00 LOAD_CONST_SMALL_INT 1 01 DUP_TOP 02 STORE_FAST 0 @@ -367,28 +462,29 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): 48 POP_TOP 49 LOAD_CONST_NONE 50 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 20 bytes) +Raw bytecode (code_info_size=9, bytecode_size=11): + a1 01 0b 05 06 80 88 40 00 82 2a 01 53 b0 21 00 + 01 c1 51 63 arg names: a (N_STATE 5) (N_EXC_STACK 0) (INIT_CELL 0) -######## - bc=\\d\+ line=139 + bc=0 line=1 + bc=0 line=137 + bc=0 line=139 00 LOAD_CONST_SMALL_INT 2 01 BUILD_TUPLE 1 03 LOAD_NULL 04 LOAD_FAST 0 05 MAKE_CLOSURE_DEFARGS \.\+ 1 -\\d\+ STORE_FAST 1 -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +08 STORE_FAST 1 +09 LOAD_CONST_NONE +10 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 21 bytes) +Raw bytecode (code_info_size=8, bytecode_size=13): + 88 40 0a 05 80 8f 23 23 51 67 59 81 67 59 81 5e + 51 68 59 51 63 arg names: (N_STATE 2) (N_EXC_STACK 0) @@ -409,115 +505,113 @@ arg names: 10 POP_TOP 11 LOAD_CONST_NONE 12 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'Class' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block 'Class' (descriptor: \.\+, bytecode @\.\+ 1\[56\] bytes) +Raw bytecode (code_info_size=\[56\], bytecode_size=10): + 00 \.\+ 11 0f 16 10 10 02 16 11 51 63 arg names: (N_STATE 1) (N_EXC_STACK 0) bc=0 line=1 ######## - bc=12 line=150 + bc=8 line=150 00 LOAD_NAME __name__ -03 STORE_NAME __module__ -06 LOAD_CONST_STRING 'Class' -09 STORE_NAME __qualname__ -12 LOAD_CONST_NONE -13 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +02 STORE_NAME __module__ +04 LOAD_CONST_STRING 'Class' +06 STORE_NAME __qualname__ +08 LOAD_CONST_NONE +09 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 18 bytes) +Raw bytecode (code_info_size=6, bytecode_size=12): + 19 08 05 12 80 9c 12 13 12 14 b0 15 05 36 00 59 + 51 63 arg names: self (N_STATE 4) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=157 00 LOAD_GLOBAL super -\\d\+ LOAD_GLOBAL __class__ -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_SUPER_METHOD f -\\d\+ CALL_METHOD n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +02 LOAD_GLOBAL __class__ +04 LOAD_FAST 0 +05 LOAD_SUPER_METHOD f +07 CALL_METHOD n=0 nkw=0 +09 POP_TOP +10 LOAD_CONST_NONE +11 RETURN_VALUE +File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 28 bytes) +Raw bytecode (code_info_size=9, bytecode_size=19): + c3 40 0c 09 03 03 03 80 3b 53 b2 53 53 4b 0b c3 + 25 01 44 39 25 00 67 59 42 33 51 63 arg names: * * * (N_STATE 9) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=60 -######## 00 LOAD_NULL 01 LOAD_FAST 2 02 LOAD_NULL 03 LOAD_NULL -04 FOR_ITER 20 -07 STORE_FAST 3 -08 LOAD_DEREF 1 -10 POP_JUMP_IF_FALSE 4 -13 LOAD_DEREF 0 -15 YIELD_VALUE -16 POP_TOP -17 JUMP 4 -20 LOAD_CONST_NONE -21 RETURN_VALUE -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +04 FOR_ITER 17 +06 STORE_FAST 3 +07 LOAD_DEREF 1 +09 POP_JUMP_IF_FALSE 4 +11 LOAD_DEREF 0 +13 YIELD_VALUE +14 POP_TOP +15 JUMP 4 +17 LOAD_CONST_NONE +18 RETURN_VALUE +File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 26 bytes) +Raw bytecode (code_info_size=8, bytecode_size=18): + 4b 0c 0a 03 03 03 80 3c 2b 00 b2 5f 4b 0b c3 25 + 01 44 39 25 00 2f 14 42 33 63 arg names: * * * (N_STATE 10) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=61 -######## 00 BUILD_LIST 0 02 LOAD_FAST 2 03 GET_ITER_STACK -04 FOR_ITER 20 -07 STORE_FAST 3 -08 LOAD_DEREF 1 -10 POP_JUMP_IF_FALSE 4 -13 LOAD_DEREF 0 -15 STORE_COMP 20 -17 JUMP 4 -20 RETURN_VALUE -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +04 FOR_ITER 17 +06 STORE_FAST 3 +07 LOAD_DEREF 1 +09 POP_JUMP_IF_FALSE 4 +11 LOAD_DEREF 0 +13 STORE_COMP 20 +15 JUMP 4 +17 RETURN_VALUE +File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 28 bytes) +Raw bytecode (code_info_size=8, bytecode_size=20): + 53 0c 0b 03 03 03 80 3d 2c 00 b2 5f 4b 0d c3 25 + 01 44 39 25 00 25 00 2f 19 42 31 63 arg names: * * * (N_STATE 11) (N_EXC_STACK 0) bc=0 line=1 -######## + bc=0 line=62 00 BUILD_MAP 0 02 LOAD_FAST 2 03 GET_ITER_STACK -04 FOR_ITER 22 -07 STORE_FAST 3 -08 LOAD_DEREF 1 -10 POP_JUMP_IF_FALSE 4 +04 FOR_ITER 19 +06 STORE_FAST 3 +07 LOAD_DEREF 1 +09 POP_JUMP_IF_FALSE 4 +11 LOAD_DEREF 0 13 LOAD_DEREF 0 -15 LOAD_DEREF 0 -17 STORE_COMP 25 -19 JUMP 4 -22 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'closure' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +15 STORE_COMP 25 +17 JUMP 4 +19 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'closure' (descriptor: \.\+, bytecode @\.\+ 20 bytes) +Raw bytecode (code_info_size=8, bytecode_size=12): + 19 0c 0c 03 80 6f 25 23 25 00 81 f2 c1 81 27 00 + 29 00 51 63 arg names: * (N_STATE 4) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=114 + bc=0 line=112 + bc=5 line=113 + bc=8 line=114 00 LOAD_DEREF 0 02 LOAD_CONST_SMALL_INT 1 03 BINARY_OP 27 __add__ @@ -527,16 +621,14 @@ arg names: * 08 DELETE_DEREF 0 10 LOAD_CONST_NONE 11 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 13 bytes) +Raw bytecode (code_info_size=8, bytecode_size=5): + 9a 01 0a 05 03 08 80 8b b1 25 00 f2 63 arg names: * b (N_STATE 4) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=140 + bc=0 line=140 00 LOAD_FAST 1 01 LOAD_DEREF 0 03 BINARY_OP 27 __add__ diff --git a/tests/cmdline/cmd_verbose.py.exp b/tests/cmdline/cmd_verbose.py.exp index 0edd050c22..ae833dbec8 100644 --- a/tests/cmdline/cmd_verbose.py.exp +++ b/tests/cmdline/cmd_verbose.py.exp @@ -1,19 +1,17 @@ -File cmdline/cmd_verbose.py, code block '' (descriptor: \.\+, bytecode \.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): - 08 \.\+ -######## -\.\+63 +File cmdline/cmd_verbose.py, code block '' (descriptor: \.\+, bytecode @\.\+ 12 bytes) +Raw bytecode (code_info_size=4, bytecode_size=8): + 08 04 01 40 11 02 81 34 01 59 51 63 arg names: (N_STATE 2) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=3 00 LOAD_NAME print -03 LOAD_CONST_SMALL_INT 1 -04 CALL_FUNCTION n=1 nkw=0 -06 POP_TOP -07 LOAD_CONST_NONE -08 RETURN_VALUE +02 LOAD_CONST_SMALL_INT 1 +03 CALL_FUNCTION n=1 nkw=0 +05 POP_TOP +06 LOAD_CONST_NONE +07 RETURN_VALUE 1 mem: total=\\d\+, current=\\d\+, peak=\\d\+ stack: \\d\+ out of \\d\+ diff --git a/tests/cmdline/repl_autoindent.py b/tests/cmdline/repl_autoindent.py new file mode 100644 index 0000000000..152c81895f --- /dev/null +++ b/tests/cmdline/repl_autoindent.py @@ -0,0 +1,16 @@ +# tests for autoindent +if 1: +print(1) + + + +if 0: + print(2) +else: + print(3) + +if 0: + print(4) +else: + print(5) + diff --git a/tests/cmdline/repl_autoindent.py.exp b/tests/cmdline/repl_autoindent.py.exp new file mode 100644 index 0000000000..9127a7d31d --- /dev/null +++ b/tests/cmdline/repl_autoindent.py.exp @@ -0,0 +1,22 @@ +MicroPython \.\+ version +Use \.\+ +>>> # tests for autoindent +>>> if 1: +... print(1) +... +... +... +1 +>>> if 0: +...  print(2) +... else: +... print(3) +... +3 +>>> if 0: +... print(4) +... else: +... print(5) +... +5 +>>> diff --git a/tests/cmdline/repl_sys_ps1_ps2.py b/tests/cmdline/repl_sys_ps1_ps2.py new file mode 100644 index 0000000000..4f96057c49 --- /dev/null +++ b/tests/cmdline/repl_sys_ps1_ps2.py @@ -0,0 +1,6 @@ +# test changing ps1/ps2 +import usys +usys.ps1 = "PS1" +usys.ps2 = "PS2" +(1 + +2) diff --git a/tests/cmdline/repl_sys_ps1_ps2.py.exp b/tests/cmdline/repl_sys_ps1_ps2.py.exp new file mode 100644 index 0000000000..e4a802d34d --- /dev/null +++ b/tests/cmdline/repl_sys_ps1_ps2.py.exp @@ -0,0 +1,10 @@ +MicroPython \.\+ version +Use \.\+ +>>> # test changing ps1/ps2 +>>> import usys +>>> usys.ps1 = "PS1" +PS1usys.ps2 = "PS2" +PS1(1 + +PS22) +3 +PS1 diff --git a/tests/cpydiff/syntax_arg_unpacking.py b/tests/cpydiff/syntax_arg_unpacking.py new file mode 100644 index 0000000000..e54832ddb9 --- /dev/null +++ b/tests/cpydiff/syntax_arg_unpacking.py @@ -0,0 +1,23 @@ +""" +categories: Syntax +description: Argument unpacking does not work if the argument being unpacked is the nth or greater argument where n is the number of bits in an MP_SMALL_INT. +cause: The implementation uses an MP_SMALL_INT to flag args that need to be unpacked. +workaround: Use fewer arguments. +""" + + +def example(*args): + print(len(args)) + + +MORE = ["a", "b", "c"] + +# fmt: off +example( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + *MORE, +) +# fmt: on diff --git a/tests/extmod/uasyncio_basic.py b/tests/extmod/uasyncio_basic.py index c88908d99b..20c0a82e47 100644 --- a/tests/extmod/uasyncio_basic.py +++ b/tests/extmod/uasyncio_basic.py @@ -32,18 +32,18 @@ async def main(): print("after sleep") t0 = ticks() - await delay_print(0.02, "short") + await delay_print(0.2, "short") t1 = ticks() - await delay_print(0.04, "long") + await delay_print(0.4, "long") t2 = ticks() await delay_print(-1, "negative") t3 = ticks() print( "took {} {} {}".format( - round(ticks_diff(t1, t0), -1), - round(ticks_diff(t2, t1), -1), - round(ticks_diff(t3, t2), -1), + round(ticks_diff(t1, t0), -2), + round(ticks_diff(t2, t1), -2), + round(ticks_diff(t3, t2), -2), ) ) diff --git a/tests/extmod/uasyncio_basic.py.exp b/tests/extmod/uasyncio_basic.py.exp index 6673978769..478e22abc8 100644 --- a/tests/extmod/uasyncio_basic.py.exp +++ b/tests/extmod/uasyncio_basic.py.exp @@ -3,4 +3,4 @@ after sleep short long negative -took 20 40 0 +took 200 400 0 diff --git a/tests/extmod/uasyncio_gather.py b/tests/extmod/uasyncio_gather.py index 6053873dbc..c081221c9f 100644 --- a/tests/extmod/uasyncio_gather.py +++ b/tests/extmod/uasyncio_gather.py @@ -20,16 +20,30 @@ async def factorial(name, number): return f -async def task(id): +async def task(id, t=0.1): print("start", id) - await asyncio.sleep(0.02) + await asyncio.sleep(t) print("end", id) return id -async def gather_task(): +async def task_loop(id): + print("task_loop start", id) + while True: + await asyncio.sleep(0.1) + print("task_loop loop", id) + + +async def task_raise(id, t=0.1): + print("task_raise start", id) + await asyncio.sleep(t) + print("task_raise raise", id) + raise ValueError(id) + + +async def gather_task(t0, t1): print("gather_task") - await asyncio.gather(task(1), task(2)) + await asyncio.gather(t0, t1) print("gather_task2") @@ -37,19 +51,59 @@ async def main(): # Simple gather with return values print(await asyncio.gather(factorial("A", 2), factorial("B", 3), factorial("C", 4))) + print("====") + # Test return_exceptions, where one task is cancelled and the other finishes normally tasks = [asyncio.create_task(task(1)), asyncio.create_task(task(2))] tasks[0].cancel() print(await asyncio.gather(*tasks, return_exceptions=True)) - # Cancel a multi gather - # TODO doesn't work, Task should not forward cancellation from gather to sub-task - # but rather CancelledError should cancel the gather directly, which will then cancel - # all sub-tasks explicitly - # t = asyncio.create_task(gather_task()) - # await asyncio.sleep(0.01) - # t.cancel() - # await asyncio.sleep(0.01) + print("====") + + # Test return_exceptions, where one task raises an exception and the other finishes normally. + tasks = [asyncio.create_task(task(1)), asyncio.create_task(task_raise(2))] + print(await asyncio.gather(*tasks, return_exceptions=True)) + + print("====") + + # Test case where one task raises an exception and other task keeps running. + tasks = [asyncio.create_task(task_loop(1)), asyncio.create_task(task_raise(2))] + try: + await asyncio.gather(*tasks) + except ValueError as er: + print(repr(er)) + print(tasks[0].done(), tasks[1].done()) + for t in tasks: + t.cancel() + await asyncio.sleep(0.2) + + print("====") + + # Test case where both tasks raise an exception. + # Use t=0 so they raise one after the other, between the gather starting and finishing. + tasks = [asyncio.create_task(task_raise(1, t=0)), asyncio.create_task(task_raise(2, t=0))] + try: + await asyncio.gather(*tasks) + except ValueError as er: + print(repr(er)) + print(tasks[0].done(), tasks[1].done()) + + print("====") + + # Cancel a multi gather. + t = asyncio.create_task(gather_task(task(1), task(2))) + await asyncio.sleep(0.05) + t.cancel() + await asyncio.sleep(0.2) + + # Test edge cases where the gather is cancelled just as tasks are created and ending. + for i in range(1, 4): + print("====") + t = asyncio.create_task(gather_task(task(1, t=0), task(2, t=0))) + for _ in range(i): + await asyncio.sleep(0) + t.cancel() + await asyncio.sleep(0.2) asyncio.run(main()) diff --git a/tests/extmod/uasyncio_gather.py.exp b/tests/extmod/uasyncio_gather.py.exp index 95310bbe1c..a5ea47ab50 100644 --- a/tests/extmod/uasyncio_gather.py.exp +++ b/tests/extmod/uasyncio_gather.py.exp @@ -8,6 +8,47 @@ Task B: factorial(3) = 6 Task C: Compute factorial(4)... Task C: factorial(4) = 24 [2, 6, 24] +==== start 2 end 2 [CancelledError(), 2] +==== +start 1 +task_raise start 2 +end 1 +task_raise raise 2 +[1, ValueError(2,)] +==== +task_loop start 1 +task_raise start 2 +task_loop loop 1 +task_raise raise 2 +ValueError(2,) +False True +==== +task_raise start 1 +task_raise start 2 +task_raise raise 1 +task_raise raise 2 +ValueError(1,) +True True +==== +gather_task +start 1 +start 2 +==== +gather_task +start 1 +start 2 +==== +gather_task +start 1 +start 2 +end 1 +end 2 +==== +gather_task +start 1 +start 2 +end 1 +end 2 diff --git a/tests/extmod/uasyncio_gather_notimpl.py b/tests/extmod/uasyncio_gather_notimpl.py new file mode 100644 index 0000000000..3ebab9bad6 --- /dev/null +++ b/tests/extmod/uasyncio_gather_notimpl.py @@ -0,0 +1,53 @@ +# Test uasyncio.gather() function, features that are not implemented. + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +def custom_handler(loop, context): + print(repr(context["exception"])) + + +async def task(id): + print("task start", id) + await asyncio.sleep(0.01) + print("task end", id) + return id + + +async def gather_task(t0, t1): + print("gather_task start") + await asyncio.gather(t0, t1) + print("gather_task end") + + +async def main(): + loop = asyncio.get_event_loop() + loop.set_exception_handler(custom_handler) + + # Test case where can't wait on a task being gathered. + tasks = [asyncio.create_task(task(1)), asyncio.create_task(task(2))] + gt = asyncio.create_task(gather_task(tasks[0], tasks[1])) + await asyncio.sleep(0) # let the gather start + try: + await tasks[0] # can't await because this task is part of the gather + except RuntimeError as er: + print(repr(er)) + await gt + + print("====") + + # Test case where can't gather on a task being waited. + tasks = [asyncio.create_task(task(1)), asyncio.create_task(task(2))] + asyncio.create_task(gather_task(tasks[0], tasks[1])) + await tasks[0] # wait on this task before the gather starts + await tasks[1] + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_gather_notimpl.py.exp b/tests/extmod/uasyncio_gather_notimpl.py.exp new file mode 100644 index 0000000000..f21614ffbe --- /dev/null +++ b/tests/extmod/uasyncio_gather_notimpl.py.exp @@ -0,0 +1,14 @@ +task start 1 +task start 2 +gather_task start +RuntimeError("can't wait",) +task end 1 +task end 2 +gather_task end +==== +task start 1 +task start 2 +gather_task start +RuntimeError("can't gather",) +task end 1 +task end 2 diff --git a/tests/extmod/uasyncio_heaplock.py b/tests/extmod/uasyncio_heaplock.py index 3a92d36c9f..34a51cd370 100644 --- a/tests/extmod/uasyncio_heaplock.py +++ b/tests/extmod/uasyncio_heaplock.py @@ -29,15 +29,15 @@ async def task(id, n, t): async def main(): - t1 = asyncio.create_task(task(1, 4, 20)) - t2 = asyncio.create_task(task(2, 2, 50)) + t1 = asyncio.create_task(task(1, 4, 100)) + t2 = asyncio.create_task(task(2, 2, 250)) micropython.heap_lock() print("start") - await asyncio.sleep_ms(1) + await asyncio.sleep_ms(5) print("sleep") - await asyncio.sleep_ms(70) + await asyncio.sleep_ms(350) print("finish") micropython.heap_unlock() diff --git a/tests/extmod/uasyncio_wait_for.py b/tests/extmod/uasyncio_wait_for.py index 9612d16204..c636c7dd74 100644 --- a/tests/extmod/uasyncio_wait_for.py +++ b/tests/extmod/uasyncio_wait_for.py @@ -111,6 +111,21 @@ async def main(): await asyncio.sleep(0.01) print(sep) + # When wait_for gets cancelled and the task it's waiting on finishes around the + # same time as the cancellation of the wait_for + for num_sleep in range(1, 5): + t = asyncio.create_task(task_wait_for_cancel(4 + num_sleep, 0, 2)) + for _ in range(num_sleep): + await asyncio.sleep(0) + assert not t.done() + print("cancel wait_for") + t.cancel() + try: + await t + except asyncio.CancelledError as er: + print(repr(er)) + print(sep) + print("finish") diff --git a/tests/extmod/uasyncio_wait_for.py.exp b/tests/extmod/uasyncio_wait_for.py.exp index a4201d31ff..1bbe3d0658 100644 --- a/tests/extmod/uasyncio_wait_for.py.exp +++ b/tests/extmod/uasyncio_wait_for.py.exp @@ -32,4 +32,31 @@ task_wait_for_cancel_ignore cancelled ignore cancel task_catch done ---------- +task_wait_for_cancel start +cancel wait_for +task start 5 +task_wait_for_cancel cancelled +CancelledError() +---------- +task_wait_for_cancel start +task start 6 +cancel wait_for +task end 6 +task_wait_for_cancel cancelled +CancelledError() +---------- +task_wait_for_cancel start +task start 7 +task end 7 +cancel wait_for +task_wait_for_cancel cancelled +CancelledError() +---------- +task_wait_for_cancel start +task start 8 +task end 8 +cancel wait_for +task_wait_for_cancel cancelled +CancelledError() +---------- finish diff --git a/tests/extmod/uasyncio_wait_task.py b/tests/extmod/uasyncio_wait_task.py index 3c79320c9f..e19e29903c 100644 --- a/tests/extmod/uasyncio_wait_task.py +++ b/tests/extmod/uasyncio_wait_task.py @@ -54,8 +54,8 @@ async def main(): print("----") # Create 2 tasks - ts1 = asyncio.create_task(delay_print(0.04, "hello")) - ts2 = asyncio.create_task(delay_print(0.08, "world")) + ts1 = asyncio.create_task(delay_print(0.2, "hello")) + ts2 = asyncio.create_task(delay_print(0.4, "world")) # Time how long the tasks take to finish, they should execute in parallel print("start") @@ -64,7 +64,7 @@ async def main(): t1 = ticks() await ts2 t2 = ticks() - print("took {} {}".format(round(ticks_diff(t1, t0), -1), round(ticks_diff(t2, t1), -1))) + print("took {} {}".format(round(ticks_diff(t1, t0), -2), round(ticks_diff(t2, t1), -2))) # Wait on a task that raises an exception t = asyncio.create_task(task_raise()) diff --git a/tests/extmod/uasyncio_wait_task.py.exp b/tests/extmod/uasyncio_wait_task.py.exp index ee4e70fb4e..04be37f484 100644 --- a/tests/extmod/uasyncio_wait_task.py.exp +++ b/tests/extmod/uasyncio_wait_task.py.exp @@ -5,6 +5,6 @@ task 2 start hello world -took 40 40 +took 200 200 task_raise ValueError diff --git a/tests/extmod/ubinascii_b2a_base64.py b/tests/extmod/ubinascii_b2a_base64.py index 8c63b545a3..34f2f57055 100644 --- a/tests/extmod/ubinascii_b2a_base64.py +++ b/tests/extmod/ubinascii_b2a_base64.py @@ -24,3 +24,6 @@ try: print(binascii.b2a_base64("")) except TypeError: print("TypeError") + +print(binascii.b2a_base64(b"foobar", newline=True)) +print(binascii.b2a_base64(b"foobar", newline=False)) diff --git a/tests/extmod/uctypes_32bit_intbig.py b/tests/extmod/uctypes_32bit_intbig.py index eed36e8774..27d33817cf 100644 --- a/tests/extmod/uctypes_32bit_intbig.py +++ b/tests/extmod/uctypes_32bit_intbig.py @@ -6,7 +6,7 @@ except ImportError: print("SKIP") raise SystemExit -buf = b"12345678abcd" +buf = bytearray(b"12345678abcd") struct = uctypes.struct( uctypes.addressof(buf), {"f32": uctypes.UINT32 | 0, "f64": uctypes.UINT64 | 4}, @@ -30,7 +30,7 @@ print(buf) print("=") -buf = b"12345678abcd" +buf = bytearray(b"12345678abcd") struct = uctypes.struct( uctypes.addressof(buf), {"f32": uctypes.UINT32 | 0, "f64": uctypes.UINT64 | 4}, diff --git a/tests/extmod/uctypes_32bit_intbig.py.exp b/tests/extmod/uctypes_32bit_intbig.py.exp index d1fc1fe350..1b16ddbc32 100644 --- a/tests/extmod/uctypes_32bit_intbig.py.exp +++ b/tests/extmod/uctypes_32bit_intbig.py.exp @@ -1,11 +1,11 @@ -b'\xff\xff\xff\x7f5678abcd' -b'\x00\x00\x00\x805678abcd' -b'\x03\x02\x01\xff5678abcd' -b'\x03\x02\x01\xff\x00\x00\x00\x80\x00\x00\x00\x00' -b'\x03\x02\x01\xff\x00\x00\x00\x00\x01\x00\x00\x00' +bytearray(b'\xff\xff\xff\x7f5678abcd') +bytearray(b'\x00\x00\x00\x805678abcd') +bytearray(b'\x03\x02\x01\xff5678abcd') +bytearray(b'\x03\x02\x01\xff\x00\x00\x00\x80\x00\x00\x00\x00') +bytearray(b'\x03\x02\x01\xff\x00\x00\x00\x00\x01\x00\x00\x00') = -b'\x7f\xff\xff\xff5678abcd' -b'\x80\x00\x00\x005678abcd' -b'\xff\x01\x02\x035678abcd' -b'\xff\x01\x02\x03\x00\x00\x00\x00\x80\x00\x00\x00' -b'\xff\x01\x02\x03\x00\x00\x00\x01\x00\x00\x00\x00' +bytearray(b'\x7f\xff\xff\xff5678abcd') +bytearray(b'\x80\x00\x00\x005678abcd') +bytearray(b'\xff\x01\x02\x035678abcd') +bytearray(b'\xff\x01\x02\x03\x00\x00\x00\x00\x80\x00\x00\x00') +bytearray(b'\xff\x01\x02\x03\x00\x00\x00\x01\x00\x00\x00\x00') diff --git a/tests/extmod/ure_split.py b/tests/extmod/ure_split.py index a8b9c1686c..7e6ef3990f 100644 --- a/tests/extmod/ure_split.py +++ b/tests/extmod/ure_split.py @@ -31,3 +31,13 @@ print(s) r = re.compile(b"x") s = r.split(b"fooxbar") print(s) + +# using ^ +r = re.compile("^ab") +s = r.split("abababcabab") +print(s) + +# using ^ with | +r = re.compile("^ab|cab") +s = r.split("abababcabab") +print(s) diff --git a/tests/extmod/ure_sub.py b/tests/extmod/ure_sub.py index ae6ad28d62..806c389576 100644 --- a/tests/extmod/ure_sub.py +++ b/tests/extmod/ure_sub.py @@ -75,3 +75,7 @@ except TypeError: # Include \ in the sub replacement print(re.sub("b", "\\\\b", "abc")) + +# Using ^, make sure it doesn't repeatedly match +print(re.sub("^ab", "*", "abababcabab")) +print(re.sub("^ab|cab", "*", "abababcabab")) diff --git a/tests/extmod/vfs_fat_finaliser.py b/tests/extmod/vfs_fat_finaliser.py index c2f5966963..91cce2c95b 100644 --- a/tests/extmod/vfs_fat_finaliser.py +++ b/tests/extmod/vfs_fat_finaliser.py @@ -56,6 +56,12 @@ micropython.heap_unlock() # Here we test that the finaliser is actually called during a garbage collection. import gc +# Preallocate global variables, and list of filenames for the test (which may +# in turn allocate new qstrs and/or a new qstr pool). +f = None +n = None +names = ["x%d" % i for i in range(4)] + # Do a large number of single-block allocations to move the GC head forwards, # ensuring that the files are allocated from never-before-used blocks and # therefore couldn't possibly have any references to them left behind on @@ -63,14 +69,13 @@ import gc for i in range(1024): [] -N = 4 -for i in range(N): - n = "x%d" % i +# Run the test: create files without closing them, run GC, then read back files. +for n in names: f = vfs.open(n, "w") f.write(n) f = None # release f without closing - [0, 1, 2, 3, 4, 5] # use up Python stack so f is really gone + sorted([0, 1, 2, 3, 4, 5], key=lambda x: x) # use up Python and C stack so f is really gone gc.collect() # should finalise all N files by closing them -for i in range(N): - with vfs.open("x%d" % i, "r") as f: +for n in names: + with vfs.open(n, "r") as f: print(f.read()) diff --git a/tests/extmod/vfs_lfs_mount.py.exp b/tests/extmod/vfs_lfs_mount.py.exp index 68561b4807..defaad0580 100644 --- a/tests/extmod/vfs_lfs_mount.py.exp +++ b/tests/extmod/vfs_lfs_mount.py.exp @@ -13,4 +13,4 @@ package hello from lfs lfsmod2.py: print("hello from lfs") -OSError(36,) +OSError(30,) diff --git a/tests/extmod/vfs_posix.py b/tests/extmod/vfs_posix.py index f8c4aae406..2a14fc2076 100644 --- a/tests/extmod/vfs_posix.py +++ b/tests/extmod/vfs_posix.py @@ -59,9 +59,10 @@ print(uos.listdir(temp_dir)) vfs = uos.VfsPosix(temp_dir) print(list(i[0] for i in vfs.ilistdir("."))) -# stat, statvfs +# stat, statvfs (statvfs may not exist) print(type(vfs.stat("."))) -print(type(vfs.statvfs("."))) +if hasattr(vfs, "statvfs"): + assert type(vfs.statvfs(".")) is tuple # check types of ilistdir with str/bytes arguments print(type(list(vfs.ilistdir("."))[0][0])) diff --git a/tests/extmod/vfs_posix.py.exp b/tests/extmod/vfs_posix.py.exp index e7d68f38ec..eb9ab43106 100644 --- a/tests/extmod/vfs_posix.py.exp +++ b/tests/extmod/vfs_posix.py.exp @@ -7,7 +7,6 @@ hello ['test2'] ['test2'] - [] diff --git a/tests/float/math_constants.py b/tests/float/math_constants.py new file mode 100644 index 0000000000..2e4c321052 --- /dev/null +++ b/tests/float/math_constants.py @@ -0,0 +1,11 @@ +# Tests various constants of the math module. +try: + import math + from math import exp, cos +except ImportError: + print("SKIP") + raise SystemExit + +print(math.e == exp(1.0)) + +print(cos(math.pi)) diff --git a/tests/float/math_constants_extra.py b/tests/float/math_constants_extra.py new file mode 100644 index 0000000000..dea49aef5a --- /dev/null +++ b/tests/float/math_constants_extra.py @@ -0,0 +1,17 @@ +# Tests constants of the math module available only with MICROPY_PY_MATH_CONSTANTS. +try: + import math + from math import isnan + + math.tau +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + +print(math.tau == 2.0 * math.pi) + +print(math.inf == float("inf")) +print(-math.inf == -float("inf")) + +print(isnan(math.nan)) +print(isnan(-math.nan)) diff --git a/tests/inlineasm/asmpushpop.py b/tests/inlineasm/asmpushpop.py index 74e729dfa2..99566a7558 100644 --- a/tests/inlineasm/asmpushpop.py +++ b/tests/inlineasm/asmpushpop.py @@ -6,4 +6,16 @@ def f(r0, r1, r2): pop({r1, r2}) +@micropython.asm_thumb +def g(): + b(START) + label(SUBROUTINE) + push({lr}) # push return address + mov(r0, 7) + pop({pc}) # return + label(START) + bl(SUBROUTINE) + + print(f(0, 1, 2)) +print(g()) diff --git a/tests/inlineasm/asmpushpop.py.exp b/tests/inlineasm/asmpushpop.py.exp index d00491fd7e..fea32e7d83 100644 --- a/tests/inlineasm/asmpushpop.py.exp +++ b/tests/inlineasm/asmpushpop.py.exp @@ -1 +1,2 @@ 1 +7 diff --git a/tests/micropython/const_alltypes.py b/tests/micropython/const_alltypes.py new file mode 100644 index 0000000000..b5f36edc4f --- /dev/null +++ b/tests/micropython/const_alltypes.py @@ -0,0 +1,26 @@ +# Test constant optimisation, with full range of const types. +# This test will only work when MICROPY_COMP_CONST and MICROPY_COMP_CONST_TUPLE are enabled. + +from micropython import const + +_INT = const(123) +_STR = const("str") +_BYTES = const(b"bytes") +_TUPLE = const((_INT, _STR, _BYTES)) +_TUPLE2 = const((None, False, True, ..., (), _TUPLE)) + +print(_INT) +print(_STR) +print(_BYTES) +print(_TUPLE) +print(_TUPLE2) + +x = _TUPLE +print(x is _TUPLE) +print(x is (_INT, _STR, _BYTES)) + +print(hasattr(globals(), "_INT")) +print(hasattr(globals(), "_STR")) +print(hasattr(globals(), "_BYTES")) +print(hasattr(globals(), "_TUPLE")) +print(hasattr(globals(), "_TUPLE2")) diff --git a/tests/micropython/const_alltypes.py.exp b/tests/micropython/const_alltypes.py.exp new file mode 100644 index 0000000000..77aeffffab --- /dev/null +++ b/tests/micropython/const_alltypes.py.exp @@ -0,0 +1,12 @@ +123 +str +b'bytes' +(123, 'str', b'bytes') +(None, False, True, Ellipsis, (), (123, 'str', b'bytes')) +True +True +False +False +False +False +False diff --git a/tests/micropython/extreme_exc.py b/tests/micropython/extreme_exc.py index 9d2f24745f..ad819e408f 100644 --- a/tests/micropython/extreme_exc.py +++ b/tests/micropython/extreme_exc.py @@ -126,8 +126,8 @@ def main(): ) except Exception as er: e = er - lst[0][0] = None - lst = None + while lst: + lst[0], lst = None, lst[0] # unlink lists to free up heap print(repr(e)[:10]) # raise a deep exception with the heap locked diff --git a/tests/micropython/heapalloc.py b/tests/micropython/heapalloc.py index 99f157105d..e19f8d0255 100644 --- a/tests/micropython/heapalloc.py +++ b/tests/micropython/heapalloc.py @@ -33,6 +33,10 @@ def f3(a, b, c, d): print(x1, x3, x5, x7, x2 + x4 + x6 + x8) +def f4(): + return True, b"bytes", () + + global_var = 1 @@ -45,7 +49,11 @@ def test(): f1(a=i) # keyword arguments f2(i) # default arg (second one) f2(i, i) # 2 args + f1((1, "two", (b"three",))) # use a constant tuple f3(1, 2, 3, 4) # function with lots of local state + for i in 1, "two": # iterate over constant tuple + print(i) + print(f4()) # returns a constant tuple # call test() with heap allocation disabled diff --git a/tests/micropython/heapalloc.py.exp b/tests/micropython/heapalloc.py.exp index c8cffe183f..b8580edc61 100644 --- a/tests/micropython/heapalloc.py.exp +++ b/tests/micropython/heapalloc.py.exp @@ -8,4 +8,8 @@ 1 1 2 1 1 +(1, 'two', (b'three',)) 1 2 3 4 10 +1 +two +(True, b'bytes', ()) diff --git a/tests/micropython/heapalloc_exc_compressed.py b/tests/micropython/heapalloc_exc_compressed.py index 79e423ca0f..aa071d641c 100644 --- a/tests/micropython/heapalloc_exc_compressed.py +++ b/tests/micropython/heapalloc_exc_compressed.py @@ -5,12 +5,10 @@ import micropython # mp_obj_new_exception_msg (decompression can be deferred) # NameError uses mp_obj_new_exception_msg_varg for NameError("name '%q' isn't defined") -# set.pop uses mp_obj_new_exception_msg for KeyError("pop from an empty set") +# `raise 0` uses mp_obj_new_exception_msg for TypeError("exceptions must derive from BaseException") # Tests that deferred decompression works both via print(e) and accessing the message directly via e.args. -a = set() - # First test the regular case (can use heap for allocating the decompression buffer). try: name() @@ -18,8 +16,8 @@ except NameError as e: print(type(e).__name__, e) try: - a.pop() -except KeyError as e: + raise 0 +except TypeError as e: print(type(e).__name__, e) try: @@ -28,8 +26,8 @@ except NameError as e: print(e.args[0]) try: - a.pop() -except KeyError as e: + raise 0 +except TypeError as e: print(e.args[0]) # Then test that it still works when the heap is locked (i.e. in ISR context). @@ -41,8 +39,8 @@ except NameError as e: print(type(e).__name__) try: - a.pop() -except KeyError as e: + raise 0 +except TypeError as e: print(type(e).__name__) micropython.heap_unlock() diff --git a/tests/micropython/heapalloc_exc_compressed.py.exp b/tests/micropython/heapalloc_exc_compressed.py.exp index e00efe088c..c3e6e5dd9f 100644 --- a/tests/micropython/heapalloc_exc_compressed.py.exp +++ b/tests/micropython/heapalloc_exc_compressed.py.exp @@ -1,6 +1,6 @@ -NameError name 'name' is not defined -KeyError pop from empty set -name 'name' is not defined -pop from empty set +NameError name 'name' isn't defined +TypeError exceptions must derive from BaseException +name 'name' isn't defined +exceptions must derive from BaseException NameError -KeyError +TypeError diff --git a/tests/micropython/heapalloc_exc_compressed_emg_exc.py b/tests/micropython/heapalloc_exc_compressed_emg_exc.py index 86ade07862..48ce9dd69e 100644 --- a/tests/micropython/heapalloc_exc_compressed_emg_exc.py +++ b/tests/micropython/heapalloc_exc_compressed_emg_exc.py @@ -9,8 +9,6 @@ try: except AttributeError: pass -a = set() - def test(): micropython.heap_lock() @@ -21,8 +19,8 @@ def test(): print(type(e).__name__, e) try: - a.pop() - except KeyError as e: + raise 0 + except TypeError as e: print(type(e).__name__, e) try: @@ -31,8 +29,8 @@ def test(): print(e.args[0]) try: - a.pop() - except KeyError as e: + raise 0 + except TypeError as e: print(e.args[0]) micropython.heap_unlock() diff --git a/tests/micropython/heapalloc_exc_compressed_emg_exc.py.exp b/tests/micropython/heapalloc_exc_compressed_emg_exc.py.exp index 4293b45091..1fc8109339 100644 --- a/tests/micropython/heapalloc_exc_compressed_emg_exc.py.exp +++ b/tests/micropython/heapalloc_exc_compressed_emg_exc.py.exp @@ -1,4 +1,4 @@ -NameError name 'name' is not defined -KeyError pop from empty set -name 'name' is not defined -pop from empty set +NameError name 'name' isn't defined +TypeError exceptions must derive from BaseException +name 'name' isn't defined +exceptions must derive from BaseException diff --git a/tests/micropython/import_mpy_invalid.py b/tests/micropython/import_mpy_invalid.py index d497e897e1..7bf9417329 100644 --- a/tests/micropython/import_mpy_invalid.py +++ b/tests/micropython/import_mpy_invalid.py @@ -52,7 +52,6 @@ user_files = { "/mod0.mpy": b"", # empty file "/mod1.mpy": b"M", # too short header "/mod2.mpy": b"M\x00\x00\x00", # bad version - "/mod3.mpy": b"M\x00\x00\x00\x7f", # qstr window too large } # create and mount a user filesystem diff --git a/tests/micropython/import_mpy_native_x64.py b/tests/micropython/import_mpy_native.py similarity index 52% rename from tests/micropython/import_mpy_native_x64.py rename to tests/micropython/import_mpy_native.py index c8379e177a..24ea0a17e3 100644 --- a/tests/micropython/import_mpy_native_x64.py +++ b/tests/micropython/import_mpy_native.py @@ -9,7 +9,8 @@ except (ImportError, AttributeError): print("SKIP") raise SystemExit -if not (sys.platform == "linux" and sys.maxsize > 2**32): +mpy_arch = sys.implementation._mpy >> 8 +if mpy_arch == 0: print("SKIP") raise SystemExit @@ -49,59 +50,66 @@ class UserFS: # these are the test .mpy files +valid_header = bytes([77, 6, mpy_arch, 31]) # fmt: off user_files = { # bad architecture - '/mod0.mpy': b'C\x05\xfe\x00\x10', + '/mod0.mpy': b'M\x06\xfc\x1f', # test loading of viper and asm - '/mod1.mpy': ( - b'C\x05\x0a\x1f\x20' # header + '/mod1.mpy': valid_header + ( + b'\x02' # n_qstr + b'\x00' # n_obj - b'\x20' # n bytes, bytecode - b'\x00\x08\x02m\x02m' # prelude + b'\x0emod1.py\x00' # qstr0 = "mod1.py" + b'\x0aouter\x00' # qstr1 = "outer" + + b'\x2c' # 5 bytes, have children, bytecode + b'\x00\x02' # prelude + b'\x01' # simple name (qstr index) b'\x51' # LOAD_CONST_NONE b'\x63' # RETURN_VALUE - b'\x00\x02' # n_obj, n_raw_code + b'\x02' # 2 children - b'\x22' # n bytes, viper code - b'\x00\x00\x00\x00\x00\x00' # dummy machine code - b'\x00\x00' # qstr0 - b'\x01\x0c\x0aprint' # n_qstr, qstr0 - b'\x00\x00\x00' # scope_flags, n_obj, n_raw_code + b'\x42' # 8 bytes, no children, viper code + b'\x00\x00\x00\x00\x00\x00\x00\x00' # dummy machine code + b'\x00' # scope_flags - b'\x23' # n bytes, asm code - b'\x00\x00\x00\x00\x00\x00\x00\x00' # dummy machine code - b'\x00\x00\x00' # scope_flags, n_pos_args, type_sig + b'\x43' # 8 bytes, no children, asm code + b'\x00\x00\x00\x00\x00\x00\x00\x00' # dummy machine code + b'\x00\x00\x00' # scope_flags, n_pos_args, type_sig ), # test loading viper with additional scope flags and relocation - '/mod2.mpy': ( - b'C\x05\x0a\x1f\x20' # header + '/mod2.mpy': valid_header + ( + b'\x02' # n_qstr + b'\x00' # n_obj - b'\x20' # n bytes, bytecode - b'\x00\x08\x02m\x02m' # prelude + b'\x0emod2.py\x00' # qstr0 = "mod2.py" + b'\x0aouter\x00' # qstr1 = "outer" + + b'\x2c' # 5 bytes, have children, bytecode + b'\x00\x02' # prelude + b'\x01' # simple name (qstr index) b'\x51' # LOAD_CONST_NONE b'\x63' # RETURN_VALUE - b'\x00\x01' # n_obj, n_raw_code + b'\x01' # 1 child - b'\x12' # n bytes(=4), viper code - b'\x00\x00\x00\x00' # dummy machine code - b'\x00' # n_qstr - b'\x81\x60' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC - b'\x00\x00' # n_obj, n_raw_code - b'\x06rodata' # rodata, 6 bytes - b'\x04' # bss, 4 bytes - b'\x03\x01\x00' # dummy relocation of rodata + b'\x22' # 4 bytes, no children, viper code + b'\x00\x00\x00\x00' # dummy machine code + b'\x70' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC + b'\x06\x04' # rodata=6 bytes, bss=4 bytes + b'rodata' # rodata content + b'\x03\x01\x00' # dummy relocation of rodata ), } # fmt: on # create and mount a user filesystem uos.mount(UserFS(user_files), "/userfs") -sys.path.append("/userfs") +usys.path.append("/userfs") # import .mpy files from the user filesystem for i in range(len(user_files)): @@ -114,4 +122,4 @@ for i in range(len(user_files)): # unmount and undo path addition uos.umount("/userfs") -sys.path.pop() +usys.path.pop() diff --git a/tests/micropython/import_mpy_native_x64.py.exp b/tests/micropython/import_mpy_native.py.exp similarity index 100% rename from tests/micropython/import_mpy_native_x64.py.exp rename to tests/micropython/import_mpy_native.py.exp diff --git a/tests/micropython/import_mpy_native_gc.py b/tests/micropython/import_mpy_native_gc.py index 58291449ce..bc8dcafdc7 100644 --- a/tests/micropython/import_mpy_native_gc.py +++ b/tests/micropython/import_mpy_native_gc.py @@ -3,7 +3,7 @@ try: import gc, sys, uio, uos - sys.implementation.mpy + sys.implementation._mpy uio.IOBase uos.mount except (ImportError, AttributeError): @@ -46,24 +46,24 @@ class UserFS: # Pre-compiled examples/natmod/features0 example for various architectures, keyed -# by the required value of sys.implementation.mpy. +# by the required value of sys.implementation._mpy. features0_file_contents = { # -march=x64 - 0xA05: b'C\x05\x0a\x1f \x84b\xe9/\x00\x00\x00SH\x8b\x1ds\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dA\x00\x00\x00H\x8b\x7f\x08L\x8bc(A\xff\xd4H\x8d5\x1f\x00\x00\x00H\x89\xc5H\x8b\x05-\x00\x00\x00\x0f\xb78\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x90\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x84@\x12factorial \x00\x00\r \x01"\xa1\x1c\x01\x1e\xff', - # -march=armv7m - 0x1605: b"C\x05\x16\x1f \x84\x12\x1a\xe0\x00\x00\x13\xb5\nK\nJ{D\x9cX\x02!\xe3h\x98G\x03F\x01 3\xb9\x02!#i\x01\x93\x02\xb0\xbd\xe8\x10@\x18GXC\x01;\xf4\xe7\x00\xbfj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\tN\tK~D\xf4X@hgi\xb8G\x05F\x07K\x07I\xf2XyD\x10\x88ck\x98G(F\xb8G h\xf8\xbd6\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x01\x84\x00\x12factorial \x00\x00\r<\x01>\xa18\x01:\xff", + 0x806: b'M\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff', + # -march=armv6m + 0x1006: b"M\x06\x10\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff", } -# Populate other armv7m-derived archs based on armv7m. -for arch in (0x1A05, 0x1E05, 0x2205): - features0_file_contents[arch] = features0_file_contents[0x1605] +# Populate armv7m-derived archs based on armv6m. +for arch in (0x1406, 0x1806, 0x1C06, 0x2006): + features0_file_contents[arch] = features0_file_contents[0x1006] -if sys.implementation.mpy not in features0_file_contents: +if sys.implementation._mpy not in features0_file_contents: print("SKIP") raise SystemExit # These are the test .mpy files. -user_files = {"/features0.mpy": features0_file_contents[sys.implementation.mpy]} +user_files = {"/features0.mpy": features0_file_contents[sys.implementation._mpy]} # Create and mount a user filesystem. uos.mount(UserFS(user_files), "/userfs") diff --git a/tests/micropython/native_gen.py b/tests/micropython/native_gen.py index 7ea45b1497..7e1ee25bcf 100644 --- a/tests/micropython/native_gen.py +++ b/tests/micropython/native_gen.py @@ -25,3 +25,34 @@ def gen2(x): print(list(gen2(3))) + + +# catching an exception from .throw() +@micropython.native +def gen3(): + try: + yield 1 + yield 2 + except Exception as er: + print("caught", repr(er)) + yield 3 + + +g = gen3() +print(next(g)) +print(g.throw(ValueError(42))) + + +# responding to .close() +@micropython.native +def gen4(): + try: + yield 1 + except: + print("raising GeneratorExit") + raise GeneratorExit + + +g = gen4() +print(next(g)) +print(g.close()) diff --git a/tests/micropython/native_gen.py.exp b/tests/micropython/native_gen.py.exp index cc09e309f2..fba4e558cc 100644 --- a/tests/micropython/native_gen.py.exp +++ b/tests/micropython/native_gen.py.exp @@ -2,3 +2,9 @@ 4 5 [0, 1, 2] +1 +caught ValueError(42,) +3 +1 +raising GeneratorExit +None diff --git a/tests/micropython/native_while.py b/tests/micropython/native_while.py new file mode 100644 index 0000000000..ccf0ae0e03 --- /dev/null +++ b/tests/micropython/native_while.py @@ -0,0 +1,13 @@ +# test native while loop + + +@micropython.native +def f(n): + i = 0 + while i < n: + print(i) + i += 1 + + +f(2) +f(4) diff --git a/tests/pybnative/while.py.exp b/tests/micropython/native_while.py.exp similarity index 100% rename from tests/pybnative/while.py.exp rename to tests/micropython/native_while.py.exp diff --git a/tests/perf_bench/bm_chaos.py b/tests/perf_bench/bm_chaos.py index d0f1337db7..e0e9269ebd 100644 --- a/tests/perf_bench/bm_chaos.py +++ b/tests/perf_bench/bm_chaos.py @@ -218,6 +218,10 @@ class Chaosgame(object): ########################################################################### # Benchmark interface +if not hasattr(random, "randrange"): + print("SKIP") + raise SystemExit + bm_params = { (100, 50): (0.25, 100, 50, 50, 50, 1234), (1000, 1000): (0.25, 200, 400, 400, 1000, 1234), diff --git a/tests/perf_bench/core_import_mpy_multi.py b/tests/perf_bench/core_import_mpy_multi.py new file mode 100644 index 0000000000..682c36328a --- /dev/null +++ b/tests/perf_bench/core_import_mpy_multi.py @@ -0,0 +1,85 @@ +# Test performance of importing an .mpy file many times. + +import usys, uio, uos + +if not (hasattr(uio, "IOBase") and hasattr(uos, "mount")): + print("SKIP") + raise SystemExit + +# This is the test.py file that is compiled to test.mpy below. +""" +class A: + def __init__(self, arg): + self.arg = arg + def write(self): + pass + def read(self): + pass +def f(): + print, str, bytes, dict + Exception, ValueError, TypeError + x = "this will be a string object" + x = b"this will be a bytes object" + x = ("const tuple", None, False, True, 1, 2, 3) +result = 123 +""" +file_data = b'M\x06\x00\x1f\x14\x03\x0etest.py\x00\x0f\x02A\x00\x02f\x00\x0cresult\x00/-5#\x82I\x81{\x81w\x82/\x81\x05\x81\x17Iom\x82\x13\x06arg\x00\x05\x1cthis will be a string object\x00\x06\x1bthis will be a bytes object\x00\n\x07\x05\x0bconst tuple\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\x81\\\x10\n\x01\x89\x07d`T2\x00\x10\x024\x02\x16\x022\x01\x16\x03"\x80{\x16\x04Qc\x02\x81d\x00\x08\x02(DD\x11\x05\x16\x06\x10\x02\x16\x072\x00\x16\x082\x01\x16\t2\x02\x16\nQc\x03`\x1a\x08\x08\x12\x13@\xb1\xb0\x18\x13Qc@\t\x08\t\x12` Qc@\t\x08\n\x12``Qc\x82@ \x0e\x03\x80\x08+)##\x12\x0b\x12\x0c\x12\r\x12\x0e*\x04Y\x12\x0f\x12\x10\x12\x11*\x03Y#\x00\xc0#\x01\xc0#\x02\xc0Qc' + + +class File(uio.IOBase): + def __init__(self): + self.off = 0 + + def ioctl(self, request, arg): + return 0 + + def readinto(self, buf): + buf[:] = memoryview(file_data)[self.off : self.off + len(buf)] + self.off += len(buf) + return len(buf) + + +class FS: + def mount(self, readonly, mkfs): + pass + + def chdir(self, path): + pass + + def stat(self, path): + if path == "__injected.mpy": + return tuple(0 for _ in range(10)) + else: + raise OSError(-2) # ENOENT + + def open(self, path, mode): + return File() + + +def mount(): + uos.mount(FS(), "/__remote") + uos.chdir("/__remote") + + +def test(r): + global result + for _ in r: + usys.modules.clear() + module = __import__("__injected") + result = module.result + + +########################################################################### +# Benchmark interface + +bm_params = { + (100, 10): (50,), + (1000, 10): (500,), + (5000, 10): (5000,), +} + + +def bm_setup(params): + (nloop,) = params + mount() + return lambda: test(range(nloop)), lambda: (nloop, result) diff --git a/tests/perf_bench/core_import_mpy_multi.py.exp b/tests/perf_bench/core_import_mpy_multi.py.exp new file mode 100644 index 0000000000..190a18037c --- /dev/null +++ b/tests/perf_bench/core_import_mpy_multi.py.exp @@ -0,0 +1 @@ +123 diff --git a/tests/perf_bench/core_import_mpy_single.py b/tests/perf_bench/core_import_mpy_single.py new file mode 100644 index 0000000000..af3f4a29b2 --- /dev/null +++ b/tests/perf_bench/core_import_mpy_single.py @@ -0,0 +1,135 @@ +# Test performance of importing an .mpy file just once. +# The first import of a module will intern strings that don't already exist, and +# this test should be representative of what happens in a real application. + +import uio, uos + +if not (hasattr(uio, "IOBase") and hasattr(uos, "mount")): + print("SKIP") + raise SystemExit + +# This is the test.py file that is compiled to test.mpy below. +# Many known and unknown names/strings are included to test the linking process. +""" +class A0: + def a0(self): pass + def a1(self): pass + def a2(self): pass + def a3(self): pass +class A1: + def a0(self): pass + def a1(self): pass + def a2(self): pass + def a3(self): pass +def f0(): + __call__, __class__, __delitem__, __enter__, __exit__, __getattr__, __getitem__, + __hash__, __init__, __int__, __iter__, __len__, __main__, __module__, __name__, + __new__, __next__, __qualname__, __repr__, __setitem__, __str__, + ArithmeticError, AssertionError, AttributeError, BaseException, EOFError, Ellipsis, + Exception, GeneratorExit, ImportError, IndentationError, IndexError, KeyError, + KeyboardInterrupt, LookupError, MemoryError, NameError, NoneType, + NotImplementedError, OSError, OverflowError, RuntimeError, StopIteration, + SyntaxError, SystemExit, TypeError, ValueError, ZeroDivisionError, + abs, all, any, append, args, bool, builtins, bytearray, bytecode, bytes, callable, + chr, classmethod, clear, close, const, copy, count, dict, dir, divmod, end, + endswith, eval, exec, extend, find, format, from_bytes, get, getattr, globals, + hasattr, hash, id, index, insert, int, isalpha, isdigit, isinstance, islower, + isspace, issubclass, isupper, items, iter, join, key, keys, len, list, little, + locals, lower, lstrip, main, map, micropython, next, object, open, ord, pop, + popitem, pow, print, range, read, readinto, readline, remove, replace, repr, + reverse, rfind, rindex, round, rsplit, rstrip, self, send, sep, set, setattr, + setdefault, sort, sorted, split, start, startswith, staticmethod, step, stop, str, + strip, sum, super, throw, to_bytes, tuple, type, update, upper, value, values, + write, zip, + name0, name1, name2, name3, name4, name5, name6, name7, name8, name9, + quite_a_long_name0, quite_a_long_name1, quite_a_long_name2, quite_a_long_name3, + quite_a_long_name4, quite_a_long_name5, quite_a_long_name6, quite_a_long_name7, + quite_a_long_name8, quite_a_long_name9, quite_a_long_name10, quite_a_long_name11, +def f1(): + x = "this will be a string object 0" + x = "this will be a string object 1" + x = "this will be a string object 2" + x = "this will be a string object 3" + x = "this will be a string object 4" + x = "this will be a string object 5" + x = "this will be a string object 6" + x = "this will be a string object 7" + x = "this will be a string object 8" + x = "this will be a string object 9" + x = b"this will be a bytes object 0" + x = b"this will be a bytes object 1" + x = b"this will be a bytes object 2" + x = b"this will be a bytes object 3" + x = b"this will be a bytes object 4" + x = b"this will be a bytes object 5" + x = b"this will be a bytes object 6" + x = b"this will be a bytes object 7" + x = b"this will be a bytes object 8" + x = b"this will be a bytes object 9" + x = ("const tuple 0", None, False, True, 1, 2, 3) + x = ("const tuple 1", None, False, True, 1, 2, 3) + x = ("const tuple 2", None, False, True, 1, 2, 3) + x = ("const tuple 3", None, False, True, 1, 2, 3) + x = ("const tuple 4", None, False, True, 1, 2, 3) + x = ("const tuple 5", None, False, True, 1, 2, 3) + x = ("const tuple 6", None, False, True, 1, 2, 3) + x = ("const tuple 7", None, False, True, 1, 2, 3) + x = ("const tuple 8", None, False, True, 1, 2, 3) + x = ("const tuple 9", None, False, True, 1, 2, 3) +result = 123 +""" +file_data = b"M\x06\x00\x1f\x81=\x1e\x0etest.py\x00\x0f\x04A0\x00\x04A1\x00\x04f0\x00\x04f1\x00\x0cresult\x00/-5\x04a0\x00\x04a1\x00\x04a2\x00\x04a3\x00\x13\x15\x17\x19\x1b\x1d\x1f!#%')+1379;=?ACEGIKMOQSUWY[]_acegikmoqsuwy{}\x7f\x81\x01\x81\x03\x81\x05\x81\x07\x81\t\x81\x0b\x81\r\x81\x0f\x81\x11\x81\x13\x81\x15\x81\x17\x81\x19\x81\x1b\x81\x1d\x81\x1f\x81!\x81#\x81%\x81'\x81)\x81+\x81-\x81/\x811\x813\x815\x817\x819\x81;\x81=\x81?\x81A\x81C\x81E\x81G\x81I\x81K\x81M\x81O\x81Q\x81S\x81U\x81W\x81Y\x81[\x81]\x81_\x81a\x81c\x81e\x81g\x81i\x81k\x81m\x81o\x81q\x81s\x81u\x81w\x81y\x81{\x81}\x81\x7f\x82\x01\x82\x03\x82\x05\x82\x07\x82\t\x82\x0b\x82\r\x82\x0f\x82\x11\x82\x13\x82\x15\x82\x17\x82\x19\x82\x1b\x82\x1d\x82\x1f\x82!\x82#\x82%\x82'\x82)\x82+\x82-\x82/\x821\x823\x825\x827\x829\x82;\x82=\x82?\x82A\x82E\x82G\x82I\x82K\nname0\x00\nname1\x00\nname2\x00\nname3\x00\nname4\x00\nname5\x00\nname6\x00\nname7\x00\nname8\x00\nname9\x00$quite_a_long_name0\x00$quite_a_long_name1\x00$quite_a_long_name2\x00$quite_a_long_name3\x00$quite_a_long_name4\x00$quite_a_long_name5\x00$quite_a_long_name6\x00$quite_a_long_name7\x00$quite_a_long_name8\x00$quite_a_long_name9\x00&quite_a_long_name10\x00&quite_a_long_name11\x00\x05\x1ethis will be a string object 0\x00\x05\x1ethis will be a string object 1\x00\x05\x1ethis will be a string object 2\x00\x05\x1ethis will be a string object 3\x00\x05\x1ethis will be a string object 4\x00\x05\x1ethis will be a string object 5\x00\x05\x1ethis will be a string object 6\x00\x05\x1ethis will be a string object 7\x00\x05\x1ethis will be a string object 8\x00\x05\x1ethis will be a string object 9\x00\x06\x1dthis will be a bytes object 0\x00\x06\x1dthis will be a bytes object 1\x00\x06\x1dthis will be a bytes object 2\x00\x06\x1dthis will be a bytes object 3\x00\x06\x1dthis will be a bytes object 4\x00\x06\x1dthis will be a bytes object 5\x00\x06\x1dthis will be a bytes object 6\x00\x06\x1dthis will be a bytes object 7\x00\x06\x1dthis will be a bytes object 8\x00\x06\x1dthis will be a bytes object 9\x00\n\x07\x05\rconst tuple 0\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 1\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 2\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 3\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 4\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 5\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 6\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 7\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 8\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 9\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\x82d\x10\x12\x01i@i@\x84\x18\x84\x1fT2\x00\x10\x024\x02\x16\x02T2\x01\x10\x034\x02\x16\x032\x02\x16\x042\x03\x16\x05\"\x80{\x16\x06Qc\x04\x82\x0c\x00\n\x02($$$\x11\x07\x16\x08\x10\x02\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04@\t\x08\n\x81\x0b Qc@\t\x08\x0b\x81\x0b@Qc@\t\x08\x0c\x81\x0b`QcH\t\n\r\x81\x0b` Qc\x82\x14\x00\x0c\x03h`$$$\x11\x07\x16\x08\x10\x03\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04H\t\n\n\x81\x0b``QcH\t\n\x0b\x81\x0b\x80\x07QcH\t\n\x0c\x81\x0b\x80\x08QcH\t\n\r\x81\x0b\x80\tQc\xa08P:\x04\x80\x0b13///---997799<\x1f%\x1f\"\x1f%)\x1f\"//\x12\x0e\x12\x0f\x12\x10\x12\x11\x12\x12\x12\x13\x12\x14*\x07Y\x12\x15\x12\x16\x12\x17\x12\x18\x12\x19\x12\x1a\x12\x08\x12\x07*\x08Y\x12\x1b\x12\x1c\x12\t\x12\x1d\x12\x1e\x12\x1f*\x06Y\x12 \x12!\x12\"\x12#\x12$\x12%*\x06Y\x12&\x12'\x12(\x12)\x12*\x12+*\x06Y\x12,\x12-\x12.\x12/\x120*\x05Y\x121\x122\x123\x124\x125*\x05Y\x126\x127\x128\x129\x12:*\x05Y\x12;\x12<\x12=\x12>\x12?\x12@\x12A\x12B\x12C\x12D\x12E*\x0bY\x12F\x12G\x12H\x12I\x12J\x12K\x12L\x12M\x12N\x12O\x12P*\x0bY\x12Q\x12R\x12S\x12T\x12U\x12V\x12W\x12X\x12Y\x12Z*\nY\x12[\x12\\\x12]\x12^\x12_\x12`\x12a\x12b\x12c\x12d*\nY\x12e\x12f\x12g\x12h\x12i\x12j\x12k\x12l\x12m\x12n\x12o*\x0bY\x12p\x12q\x12r\x12s\x12t\x12u\x12v\x12w\x12x\x12y\x12z*\x0bY\x12{\x12|\x12}\x12~\x12\x7f\x12\x81\x00\x12\x81\x01\x12\x81\x02\x12\x81\x03\x12\x81\x04*\nY\x12\x81\x05\x12\x81\x06\x12\x81\x07\x12\x81\x08\x12\x81\t\x12\x81\n\x12\x81\x0b\x12\x81\x0c\x12\x81\r\x12\x81\x0e\x12\x81\x0f*\x0bY\x12\x81\x10\x12\x81\x11\x12\x81\x12\x12\x81\x13\x12\x81\x14\x12\x81\x15\x12\x81\x16\x12\x81\x17\x12\x81\x18\x12\x81\x19*\nY\x12\x81\x1a\x12\x81\x1b\x12\x81\x1c\x12\x81\x1d\x12\x81\x1e\x12\x81\x1f\x12\x81 \x12\x81!\x12\x81\"\x12\x81#\x12\x81$*\x0bY\x12\x81%\x12\x81&*\x02Y\x12\x81'\x12\x81(\x12\x81)\x12\x81*\x12\x81+\x12\x81,\x12\x81-\x12\x81.\x12\x81/\x12\x810*\nY\x12\x811\x12\x812\x12\x813\x12\x814*\x04Y\x12\x815\x12\x816\x12\x817\x12\x818*\x04Y\x12\x819\x12\x81:\x12\x81;\x12\x81<*\x04YQc\x87p\x08@\x05\x80###############################\x00\xc0#\x01\xc0#\x02\xc0#\x03\xc0#\x04\xc0#\x05\xc0#\x06\xc0#\x07\xc0#\x08\xc0#\t\xc0#\n\xc0#\x0b\xc0#\x0c\xc0#\r\xc0#\x0e\xc0#\x0f\xc0#\x10\xc0#\x11\xc0#\x12\xc0#\x13\xc0#\x14\xc0#\x15\xc0#\x16\xc0#\x17\xc0#\x18\xc0#\x19\xc0#\x1a\xc0#\x1b\xc0#\x1c\xc0#\x1d\xc0Qc" + + +class File(uio.IOBase): + def __init__(self): + self.off = 0 + + def ioctl(self, request, arg): + return 0 + + def readinto(self, buf): + buf[:] = memoryview(file_data)[self.off : self.off + len(buf)] + self.off += len(buf) + return len(buf) + + +class FS: + def mount(self, readonly, mkfs): + pass + + def chdir(self, path): + pass + + def stat(self, path): + if path == "__injected.mpy": + return tuple(0 for _ in range(10)) + else: + raise OSError(-2) # ENOENT + + def open(self, path, mode): + return File() + + +def mount(): + uos.mount(FS(), "/__remote") + uos.chdir("/__remote") + + +def test(): + global result + module = __import__("__injected") + result = module.result + + +########################################################################### +# Benchmark interface + +bm_params = { + (1, 1): (), +} + + +def bm_setup(params): + mount() + return lambda: test(), lambda: (1, result) diff --git a/tests/perf_bench/core_import_mpy_single.py.exp b/tests/perf_bench/core_import_mpy_single.py.exp new file mode 100644 index 0000000000..190a18037c --- /dev/null +++ b/tests/perf_bench/core_import_mpy_single.py.exp @@ -0,0 +1 @@ +123 diff --git a/tests/perf_bench/core_qstr.py b/tests/perf_bench/core_qstr.py new file mode 100644 index 0000000000..b87e2c0658 --- /dev/null +++ b/tests/perf_bench/core_qstr.py @@ -0,0 +1,21 @@ +# This tests qstr_find_strn() speed when the string being searched for is not found. + + +def test(r): + for _ in r: + str("a string that shouldn't be interned") + + +########################################################################### +# Benchmark interface + +bm_params = { + (100, 10): (400,), + (1000, 10): (4000,), + (5000, 10): (40000,), +} + + +def bm_setup(params): + (nloop,) = params + return lambda: test(range(nloop)), lambda: (nloop // 100, None) diff --git a/tests/perf_bench/core_yield_from.py b/tests/perf_bench/core_yield_from.py new file mode 100644 index 0000000000..2f6930e2b8 --- /dev/null +++ b/tests/perf_bench/core_yield_from.py @@ -0,0 +1,31 @@ +# Test a deep set of "yield from" statements. + + +def recursive_yield_from(depth, iter_): + if depth <= 0: + for i in iter_: + yield i + else: + yield from recursive_yield_from(depth - 1, iter_) + + +def test(n): + global result + result = 0 + for i in recursive_yield_from(10, range(n)): + result += i + + +########################################################################### +# Benchmark interface + +bm_params = { + (100, 10): (2000,), + (1000, 10): (20000,), + (5000, 10): (100000,), +} + + +def bm_setup(params): + (nloop,) = params + return lambda: test(nloop), lambda: (nloop // 100, result) diff --git a/tests/pybnative/while.py b/tests/pybnative/while.py deleted file mode 100644 index 10625e8717..0000000000 --- a/tests/pybnative/while.py +++ /dev/null @@ -1,17 +0,0 @@ -import time, pyb - - -@micropython.native -def f(led, n, d): - led.off() - i = 0 - while i < n: - print(i) - led.toggle() - time.sleep_ms(d) - i += 1 - led.off() - - -f(pyb.LED(1), 2, 150) -f(pyb.LED(2), 4, 50) diff --git a/tests/qemu-arm/native_test.py b/tests/qemu-arm/native_test.py deleted file mode 100644 index 0b58433d92..0000000000 --- a/tests/qemu-arm/native_test.py +++ /dev/null @@ -1,5 +0,0 @@ -import native_frozen_align - -native_frozen_align.native_x(1) -native_frozen_align.native_y(2) -native_frozen_align.native_z(3) diff --git a/tests/qemu-arm/native_test.py.exp b/tests/qemu-arm/native_test.py.exp deleted file mode 100644 index dcf37cd5e2..0000000000 --- a/tests/qemu-arm/native_test.py.exp +++ /dev/null @@ -1,3 +0,0 @@ -2 -3 -4 diff --git a/tests/run-multitests.py b/tests/run-multitests.py index 34389e4292..d8a4a48fa7 100755 --- a/tests/run-multitests.py +++ b/tests/run-multitests.py @@ -63,10 +63,16 @@ class multitest: @staticmethod def get_network_ip(): try: - import network - ip = network.WLAN().ifconfig()[0] + ip = nic.ifconfig()[0] except: - ip = "127.0.0.1" + try: + import network + if hasattr(network, "WLAN"): + ip = network.WLAN().ifconfig()[0] + else: + ip = network.LAN().ifconfig()[0] + except: + ip = HOST_IP return ip {} @@ -76,15 +82,31 @@ multitest.flush() """ # The btstack implementation on Unix generates some spurious output that we -# can't control. +# can't control. Also other platforms may output certain warnings/errors that +# can be safely ignored. IGNORE_OUTPUT_MATCHES = ( "libusb: error ", # It tries to open devices that it doesn't have access to (libusb prints unconditionally). "hci_transport_h2_libusb.c", # Same issue. We enable LOG_ERROR in btstack. "USB Path: ", # Hardcoded in btstack's libusb transport. "hci_number_completed_packet", # Warning from btstack. + "lld_pdu_get_tx_flush_nb HCI packet count mismatch (", # From ESP-IDF, see https://github.com/espressif/esp-idf/issues/5105 ) +def get_host_ip(_ip_cache=[]): + if not _ip_cache: + try: + import socket + + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + _ip_cache.append(s.getsockname()[0]) + s.close() + except: + _ip_cache.append("127.0.0.1") + return _ip_cache[0] + + class PyInstance: def __init__(self): pass @@ -272,6 +294,13 @@ def run_test_on_instances(test_file, num_instances, instances): injected_globals = "" output = [[] for _ in range(num_instances)] + # If the test calls get_network_ip() then inject HOST_IP so that devices can know + # the IP address of the host. Do this lazily to not require a TCP/IP connection + # on the host if it's not needed. + with open(test_file, "rb") as f: + if b"get_network_ip" in f.read(): + injected_globals += "HOST_IP = '" + get_host_ip() + "'\n" + if cmd_args.trace_output: print("TRACE {}:".format("|".join(str(i) for i in instances))) diff --git a/tests/run-perfbench-table.py b/tests/run-perfbench-table.py new file mode 100755 index 0000000000..3d3639e057 --- /dev/null +++ b/tests/run-perfbench-table.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 + +# This file is part of the MicroPython project, http://micropython.org/ +# The MIT License (MIT) +# Copyright (c) 2019 Damien P. George + +import os +import subprocess +import sys +import time +import argparse +from glob import glob +from rich.live import Live +from rich.console import Console +from rich.table import Table + +sys.path.append("../tools") +import pyboard + +# Paths for host executables +if os.name == "nt": + CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3.exe") + MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/windows/micropython.exe") +else: + CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3") + MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/unix/micropython") + +PYTHON_TRUTH = CPYTHON3 + +BENCH_SCRIPT_DIR = "perf_bench/" + + +def compute_stats(lst): + avg = 0 + var = 0 + for x in lst: + avg += x + var += x * x + avg /= len(lst) + var = max(0, var / len(lst) - avg**2) + return avg, var**0.5 + + +def run_script_on_target(target, script, run_command=None): + output = b"" + err = None + + if isinstance(target, pyboard.Pyboard): + # Run via pyboard interface + try: + target.enter_raw_repl() + start_ts = time.monotonic_ns() + output = target.exec_(script) + if run_command: + start_ts = time.monotonic_ns() + output = target.exec_(run_command) + end_ts = time.monotonic_ns() + except pyboard.PyboardError as er: + end_ts = time.monotonic_ns() + err = er + finally: + target.exit_raw_repl() + else: + # Run local executable + try: + if run_command: + script += run_command + start_ts = time.monotonic_ns() + p = subprocess.run( + target, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, input=script + ) + end_ts = time.monotonic_ns() + output = p.stdout + except subprocess.CalledProcessError as er: + end_ts = time.monotonic_ns() + err = er + + return str(output.strip(), "ascii"), err, (end_ts - start_ts) // 1000 + + +def run_feature_test(target, test): + with open("feature_check/" + test + ".py", "rb") as f: + script = f.read() + output, err, _ = run_script_on_target(target, script) + if err is None: + return output + else: + return "CRASH: %r" % err + + +def run_benchmark_on_target(target, script, run_command=None): + output, err, runtime_us = run_script_on_target(target, script, run_command) + if err is None: + time, norm, result = output.split(None, 2) + try: + return int(time), int(norm), result, runtime_us + except ValueError: + return -1, -1, "CRASH: %r" % output, runtime_us + else: + return -1, -1, "CRASH: %r" % err, runtime_us + + +def run_benchmarks(console, target, param_n, param_m, n_average, test_list): + skip_complex = run_feature_test(target, "complex") != "complex" + skip_native = run_feature_test(target, "native_check") != "native" + + table = Table(show_header=True) + table.add_column("Test") + table.add_column("Time", justify="right") + table.add_column("Score", justify="right") + table.add_column("Ref Time", justify="right") + + live = Live(table, console=console) + live.start() + + for test_file in sorted(test_list): + # print(test_file + ": ", end="") + + # Check if test should be skipped + skip = ( + skip_complex + and test_file.find("bm_fft") != -1 + or skip_native + and test_file.find("viper_") != -1 + ) + if skip: + print("skip") + table.add_row(test_file, *(["skip"] * 6)) + continue + + # Create test script + with open(test_file, "rb") as f: + test_script = f.read() + with open(BENCH_SCRIPT_DIR + "benchrun.py", "rb") as f: + test_script += f.read() + bm_run = b"bm_run(%u, %u)\n" % (param_n, param_m) + + # Write full test script if needed + if 0: + with open("%s.full" % test_file, "wb") as f: + f.write(test_script) + + # Run MicroPython a given number of times + times = [] + runtimes = [] + scores = [] + error = None + result_out = None + for _ in range(n_average): + self_time, norm, result, runtime_us = run_benchmark_on_target( + target, test_script, bm_run + ) + if self_time < 0 or norm < 0: + error = result + break + if result_out is None: + result_out = result + elif result != result_out: + error = "FAIL self" + break + times.append(self_time) + runtimes.append(runtime_us) + scores.append(1e6 * norm / self_time) + + # Check result against truth if needed + if error is None and result_out != "None": + _, _, result_exp, _ = run_benchmark_on_target(PYTHON_TRUTH, test_script, bm_run) + if result_out != result_exp: + error = "FAIL truth" + + if error is not None: + print(test_file, error) + if error == "no matching params": + table.add_row(test_file, *([None] * 3)) + else: + table.add_row(test_file, *(["error"] * 3)) + else: + t_avg, t_sd = compute_stats(times) + r_avg, r_sd = compute_stats(runtimes) + s_avg, s_sd = compute_stats(scores) + # print( + # "{:.2f} {:.4f} {:.2f} {:.4f} {:.2f} {:.4f}".format( + # t_avg, 100 * t_sd / t_avg, s_avg, 100 * s_sd / s_avg, r_avg, 100 * r_sd / r_avg + # ) + # ) + table.add_row( + test_file, + f"{t_avg:.2f}±{100 * t_sd / t_avg:.1f}%", + f"{s_avg:.2f}±{100 * s_sd / s_avg:.1f}%", + f"{r_avg:.2f}±{100 * r_sd / r_avg:.1f}%", + ) + if 0: + print(" times: ", times) + print(" scores:", scores) + + live.update(table, refresh=True) + live.stop() + + +def parse_output(filename): + with open(filename) as f: + params = f.readline() + n, m, _ = params.strip().split() + n = int(n.split("=")[1]) + m = int(m.split("=")[1]) + data = [] + for l in f: + if l.find(": ") != -1 and l.find(": skip") == -1 and l.find("CRASH: ") == -1: + name, values = l.strip().split(": ") + values = tuple(float(v) for v in values.split()) + data.append((name,) + values) + return n, m, data + + +def compute_diff(file1, file2, diff_score): + # Parse output data from previous runs + n1, m1, d1 = parse_output(file1) + n2, m2, d2 = parse_output(file2) + + # Print header + if diff_score: + print("diff of scores (higher is better)") + else: + print("diff of microsecond times (lower is better)") + if n1 == n2 and m1 == m2: + hdr = "N={} M={}".format(n1, m1) + else: + hdr = "N={} M={} vs N={} M={}".format(n1, m1, n2, m2) + print( + "{:24} {:>10} -> {:>10} {:>10} {:>7}% (error%)".format( + hdr, file1, file2, "diff", "diff" + ) + ) + + # Print entries + while d1 and d2: + if d1[0][0] == d2[0][0]: + # Found entries with matching names + entry1 = d1.pop(0) + entry2 = d2.pop(0) + name = entry1[0].rsplit("/")[-1] + av1, sd1 = entry1[1 + 2 * diff_score], entry1[2 + 2 * diff_score] + av2, sd2 = entry2[1 + 2 * diff_score], entry2[2 + 2 * diff_score] + sd1 *= av1 / 100 # convert from percent sd to absolute sd + sd2 *= av2 / 100 # convert from percent sd to absolute sd + av_diff = av2 - av1 + sd_diff = (sd1**2 + sd2**2) ** 0.5 + percent = 100 * av_diff / av1 + percent_sd = 100 * sd_diff / av1 + print( + "{:24} {:10.2f} -> {:10.2f} : {:+10.2f} = {:+7.3f}% (+/-{:.2f}%)".format( + name, av1, av2, av_diff, percent, percent_sd + ) + ) + elif d1[0][0] < d2[0][0]: + d1.pop(0) + else: + d2.pop(0) + + +def main(): + cmd_parser = argparse.ArgumentParser(description="Run benchmarks for MicroPython") + cmd_parser.add_argument( + "-t", "--diff-time", action="store_true", help="diff time outputs from a previous run" + ) + cmd_parser.add_argument( + "-s", "--diff-score", action="store_true", help="diff score outputs from a previous run" + ) + cmd_parser.add_argument( + "-p", "--pyboard", action="store_true", help="run tests via pyboard.py" + ) + cmd_parser.add_argument( + "-d", "--device", default="/dev/ttyACM0", help="the device for pyboard.py" + ) + cmd_parser.add_argument("-a", "--average", default="8", help="averaging number") + cmd_parser.add_argument( + "--emit", default="bytecode", help="MicroPython emitter to use (bytecode or native)" + ) + cmd_parser.add_argument("N", nargs=1, help="N parameter (approximate target CPU frequency)") + cmd_parser.add_argument("M", nargs=1, help="M parameter (approximate target heap in kbytes)") + cmd_parser.add_argument("files", nargs="*", help="input test files") + args = cmd_parser.parse_args() + + if args.diff_time or args.diff_score: + compute_diff(args.N[0], args.M[0], args.diff_score) + sys.exit(0) + + # N, M = 50, 25 # esp8266 + # N, M = 100, 100 # pyboard, esp32 + # N, M = 1000, 1000 # PC + N = int(args.N[0]) + M = int(args.M[0]) + n_average = int(args.average) + + if args.pyboard: + target = pyboard.Pyboard(args.device) + target.enter_raw_repl() + else: + target = [MICROPYTHON, "-X", "emit=" + args.emit] + + if len(args.files) == 0: + tests_skip = ("benchrun.py",) + if M <= 25: + # These scripts are too big to be compiled by the target + tests_skip += ("bm_chaos.py", "bm_hexiom.py", "misc_raytrace.py") + tests = sorted( + BENCH_SCRIPT_DIR + test_file + for test_file in os.listdir(BENCH_SCRIPT_DIR) + if test_file.endswith(".py") and test_file not in tests_skip + ) + else: + tests = sorted(args.files) + + console = Console() + print("N={} M={} n_average={}".format(N, M, n_average)) + + run_benchmarks(console, target, N, M, n_average, tests) + + if isinstance(target, pyboard.Pyboard): + target.exit_raw_repl() + target.close() + + +if __name__ == "__main__": + main() diff --git a/tests/run-perfbench.py b/tests/run-perfbench.py index 3d3639e057..a2e9e8079a 100755 --- a/tests/run-perfbench.py +++ b/tests/run-perfbench.py @@ -7,16 +7,14 @@ import os import subprocess import sys -import time import argparse from glob import glob -from rich.live import Live -from rich.console import Console -from rich.table import Table sys.path.append("../tools") import pyboard +prepare_script_for_target = __import__("run-tests").prepare_script_for_target + # Paths for host executables if os.name == "nt": CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3.exe") @@ -41,7 +39,7 @@ def compute_stats(lst): return avg, var**0.5 -def run_script_on_target(target, script, run_command=None): +def run_script_on_target(target, script): output = b"" err = None @@ -49,72 +47,53 @@ def run_script_on_target(target, script, run_command=None): # Run via pyboard interface try: target.enter_raw_repl() - start_ts = time.monotonic_ns() output = target.exec_(script) - if run_command: - start_ts = time.monotonic_ns() - output = target.exec_(run_command) - end_ts = time.monotonic_ns() except pyboard.PyboardError as er: - end_ts = time.monotonic_ns() err = er - finally: - target.exit_raw_repl() else: # Run local executable try: - if run_command: - script += run_command - start_ts = time.monotonic_ns() p = subprocess.run( target, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, input=script ) - end_ts = time.monotonic_ns() output = p.stdout except subprocess.CalledProcessError as er: - end_ts = time.monotonic_ns() err = er - return str(output.strip(), "ascii"), err, (end_ts - start_ts) // 1000 + return str(output.strip(), "ascii"), err def run_feature_test(target, test): with open("feature_check/" + test + ".py", "rb") as f: script = f.read() - output, err, _ = run_script_on_target(target, script) + output, err = run_script_on_target(target, script) if err is None: return output else: return "CRASH: %r" % err -def run_benchmark_on_target(target, script, run_command=None): - output, err, runtime_us = run_script_on_target(target, script, run_command) +def run_benchmark_on_target(target, script): + output, err = run_script_on_target(target, script) if err is None: + if output == "SKIP": + return -1, -1, "SKIP" time, norm, result = output.split(None, 2) try: - return int(time), int(norm), result, runtime_us + return int(time), int(norm), result except ValueError: - return -1, -1, "CRASH: %r" % output, runtime_us + return -1, -1, "CRASH: %r" % output else: - return -1, -1, "CRASH: %r" % err, runtime_us + return -1, -1, "CRASH: %r" % err -def run_benchmarks(console, target, param_n, param_m, n_average, test_list): +def run_benchmarks(args, target, param_n, param_m, n_average, test_list): skip_complex = run_feature_test(target, "complex") != "complex" skip_native = run_feature_test(target, "native_check") != "native" - - table = Table(show_header=True) - table.add_column("Test") - table.add_column("Time", justify="right") - table.add_column("Score", justify="right") - table.add_column("Ref Time", justify="right") - - live = Live(table, console=console) - live.start() + target_had_error = False for test_file in sorted(test_list): - # print(test_file + ": ", end="") + print(test_file + ": ", end="") # Check if test should be skipped skip = ( @@ -124,8 +103,7 @@ def run_benchmarks(console, target, param_n, param_m, n_average, test_list): and test_file.find("viper_") != -1 ) if skip: - print("skip") - table.add_row(test_file, *(["skip"] * 6)) + print("SKIP") continue # Create test script @@ -133,24 +111,30 @@ def run_benchmarks(console, target, param_n, param_m, n_average, test_list): test_script = f.read() with open(BENCH_SCRIPT_DIR + "benchrun.py", "rb") as f: test_script += f.read() - bm_run = b"bm_run(%u, %u)\n" % (param_n, param_m) + test_script += b"bm_run(%u, %u)\n" % (param_n, param_m) # Write full test script if needed if 0: with open("%s.full" % test_file, "wb") as f: f.write(test_script) + # Process script through mpy-cross if needed + if isinstance(target, pyboard.Pyboard) or args.via_mpy: + crash, test_script_target = prepare_script_for_target(args, script_text=test_script) + if crash: + print("CRASH:", test_script_target) + continue + else: + test_script_target = test_script + # Run MicroPython a given number of times times = [] - runtimes = [] scores = [] error = None result_out = None for _ in range(n_average): - self_time, norm, result, runtime_us = run_benchmark_on_target( - target, test_script, bm_run - ) - if self_time < 0 or norm < 0: + time, norm, result = run_benchmark_on_target(target, test_script_target) + if time < 0 or norm < 0: error = result break if result_out is None: @@ -158,43 +142,41 @@ def run_benchmarks(console, target, param_n, param_m, n_average, test_list): elif result != result_out: error = "FAIL self" break - times.append(self_time) - runtimes.append(runtime_us) - scores.append(1e6 * norm / self_time) + times.append(time) + scores.append(1e6 * norm / time) # Check result against truth if needed if error is None and result_out != "None": - _, _, result_exp, _ = run_benchmark_on_target(PYTHON_TRUTH, test_script, bm_run) + test_file_expected = test_file + ".exp" + if os.path.isfile(test_file_expected): + # Expected result is given by a file, so read that in + with open(test_file_expected) as f: + result_exp = f.read().strip() + else: + # Run CPython to work out the expected result + _, _, result_exp = run_benchmark_on_target(PYTHON_TRUTH, test_script) if result_out != result_exp: error = "FAIL truth" if error is not None: - print(test_file, error) - if error == "no matching params": - table.add_row(test_file, *([None] * 3)) - else: - table.add_row(test_file, *(["error"] * 3)) + if not error.startswith("SKIP"): + target_had_error = True + print(error) else: t_avg, t_sd = compute_stats(times) - r_avg, r_sd = compute_stats(runtimes) s_avg, s_sd = compute_stats(scores) - # print( - # "{:.2f} {:.4f} {:.2f} {:.4f} {:.2f} {:.4f}".format( - # t_avg, 100 * t_sd / t_avg, s_avg, 100 * s_sd / s_avg, r_avg, 100 * r_sd / r_avg - # ) - # ) - table.add_row( - test_file, - f"{t_avg:.2f}±{100 * t_sd / t_avg:.1f}%", - f"{s_avg:.2f}±{100 * s_sd / s_avg:.1f}%", - f"{r_avg:.2f}±{100 * r_sd / r_avg:.1f}%", + print( + "{:.2f} {:.4f} {:.2f} {:.4f}".format( + t_avg, 100 * t_sd / t_avg, s_avg, 100 * s_sd / s_avg + ) ) if 0: print(" times: ", times) print(" scores:", scores) - live.update(table, refresh=True) - live.stop() + sys.stdout.flush() + + return target_had_error def parse_output(filename): @@ -205,7 +187,7 @@ def parse_output(filename): m = int(m.split("=")[1]) data = [] for l in f: - if l.find(": ") != -1 and l.find(": skip") == -1 and l.find("CRASH: ") == -1: + if l.find(": ") != -1 and l.find(": SKIP") == -1 and l.find("CRASH: ") == -1: name, values = l.strip().split(": ") values = tuple(float(v) for v in values.split()) data.append((name,) + values) @@ -227,7 +209,7 @@ def compute_diff(file1, file2, diff_score): else: hdr = "N={} M={} vs N={} M={}".format(n1, m1, n2, m2) print( - "{:24} {:>10} -> {:>10} {:>10} {:>7}% (error%)".format( + "{:26} {:>10} -> {:>10} {:>10} {:>7}% (error%)".format( hdr, file1, file2, "diff", "diff" ) ) @@ -248,7 +230,7 @@ def compute_diff(file1, file2, diff_score): percent = 100 * av_diff / av1 percent_sd = 100 * sd_diff / av1 print( - "{:24} {:10.2f} -> {:10.2f} : {:+10.2f} = {:+7.3f}% (+/-{:.2f}%)".format( + "{:26} {:10.2f} -> {:10.2f} : {:+10.2f} = {:+7.3f}% (+/-{:.2f}%)".format( name, av1, av2, av_diff, percent, percent_sd ) ) @@ -276,6 +258,8 @@ def main(): cmd_parser.add_argument( "--emit", default="bytecode", help="MicroPython emitter to use (bytecode or native)" ) + cmd_parser.add_argument("--via-mpy", action="store_true", help="compile code to .mpy first") + cmd_parser.add_argument("--mpy-cross-flags", default="", help="flags to pass to mpy-cross") cmd_parser.add_argument("N", nargs=1, help="N parameter (approximate target CPU frequency)") cmd_parser.add_argument("M", nargs=1, help="M parameter (approximate target heap in kbytes)") cmd_parser.add_argument("files", nargs="*", help="input test files") @@ -293,6 +277,8 @@ def main(): n_average = int(args.average) if args.pyboard: + if not args.mpy_cross_flags: + args.mpy_cross_flags = "-march=armv7m" target = pyboard.Pyboard(args.device) target.enter_raw_repl() else: @@ -311,15 +297,17 @@ def main(): else: tests = sorted(args.files) - console = Console() print("N={} M={} n_average={}".format(N, M, n_average)) - run_benchmarks(console, target, N, M, n_average, tests) + target_had_error = run_benchmarks(args, target, N, M, n_average, tests) if isinstance(target, pyboard.Pyboard): target.exit_raw_repl() target.close() + if target_had_error: + sys.exit(1) + if __name__ == "__main__": main() diff --git a/tests/run-tests.py b/tests/run-tests.py index c756729ff9..45b4ebf536 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -203,16 +203,8 @@ def run_micropython(pyb, args, test_file, is_special=False): rm_f(mpy_filename) else: - # run on pyboard - pyb.enter_raw_repl() - try: - output_mupy = pyb.execfile(test_file) - except pyboard.PyboardError as e: - had_crash = True - if not is_special and e.args[0] == "exception": - output_mupy = e.args[1] + e.args[2] + b"CRASH" - else: - output_mupy = bytes(e.args[0], "ascii") + b"\nCRASH" + # run via pyboard interface + had_crash, output_mupy = run_script_on_remote_target(pyb, args, test_file, is_special) # canonical form for all ports/platforms is to use \n for end-of-line output_mupy = output_mupy.replace(b"\r\n", b"\n") @@ -436,7 +428,6 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): if upy_float_precision < 64: skip_tests.add("float/float_divmod.py") # tested by float/float_divmod_relaxed.py instead skip_tests.add("float/float2int_doubleprec_intbig.py") - skip_tests.add("float/float_format_ints_doubleprec.py") skip_tests.add("float/float_parse_doubleprec.py") if not has_complex: @@ -492,6 +483,10 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): for t in tests: if t.startswith("basics/io_"): skip_tests.add(t) + elif args.target == "renesas-ra": + skip_tests.add( + "extmod/utime_time_ns.py" + ) # RA fsp rtc function doesn't support nano sec info elif args.target == "qemu-arm": skip_tests.add("misc/print_exception.py") # requires sys stdfiles @@ -520,6 +515,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_tests.add("basics/del_local.py") # requires checking for unbound local skip_tests.add("basics/exception_chain.py") # raise from is not supported skip_tests.add("basics/scope_implicit.py") # requires checking for unbound local + skip_tests.add("basics/sys_tracebacklimit.py") # requires traceback info skip_tests.add("basics/try_finally_return2.py") # requires raise_varargs skip_tests.add("basics/unboundlocal.py") # requires checking for unbound local skip_tests.update( @@ -538,6 +534,9 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): "misc/print_exception.py" ) # because native doesn't have proper traceback info skip_tests.add("misc/sys_exc_info.py") # sys.exc_info() is not supported for native + skip_tests.add("misc/sys_settrace_features.py") # sys.settrace() not supported + skip_tests.add("misc/sys_settrace_generator.py") # sys.settrace() not supported + skip_tests.add("misc/sys_settrace_loop.py") # sys.settrace() not supported skip_tests.add( "micropython/emg_exc.py" ) # because native doesn't have proper traceback info @@ -548,6 +547,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): "micropython/opt_level_lineno.py" ) # native doesn't have proper traceback info skip_tests.add("micropython/schedule.py") # native code doesn't check pending events + skip_tests.add("stress/bytecode_limit.py") # bytecode specific test def run_one_test(test_file): test_file = test_file.replace("\\", "/") @@ -571,7 +571,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): is_endian = test_name.endswith("_endian") is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig") is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray") - is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset") + is_set_type = test_name.startswith(("set_", "frozenset")) or test_name.endswith("_set") is_slice = test_name.find("slice") != -1 or test_name in misc_slice_tests is_async = test_name.startswith(("async_", "uasyncio_")) is_const = test_name.startswith("const") @@ -818,7 +818,16 @@ the last matching regex is used: "unix", "qemu-arm", ) - EXTERNAL_TARGETS = ("pyboard", "wipy", "esp8266", "esp32", "minimal", "nrf") + EXTERNAL_TARGETS = ( + "pyboard", + "wipy", + "esp8266", + "esp32", + "minimal", + "nrf", + "renesas-ra", + "rp2", + ) if args.target in LOCAL_TARGETS or args.list_tests: pyb = None elif args.target in EXTERNAL_TARGETS: @@ -826,6 +835,18 @@ the last matching regex is used: sys.path.append(base_path("../tools")) import pyboard + if not args.mpy_cross_flags: + if args.target == "esp8266": + args.mpy_cross_flags = "-march=xtensa" + elif args.target == "esp32": + args.mpy_cross_flags = "-march=xtensawin" + elif args.target == "rp2": + args.mpy_cross_flags = "-march=armv6m" + elif args.target == "pyboard": + args.mpy_cross_flags = "-march=armv7emsp" + else: + args.mpy_cross_flags = "-march=armv7m" + pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password) pyb.enter_raw_repl() else: @@ -842,7 +863,11 @@ the last matching regex is used: ) if args.target == "pyboard": # run pyboard tests - test_dirs += ("float", "stress", "pyb", "pybnative", "inlineasm") + test_dirs += ("float", "stress", "pyb", "inlineasm") + elif args.target in ("renesas-ra"): + test_dirs += ("float", "inlineasm", "renesas-ra") + elif args.target == "rp2": + test_dirs += ("float", "stress", "inlineasm") elif args.target in ("esp8266", "esp32", "minimal", "nrf"): test_dirs += ("float",) elif args.target == "wipy": diff --git a/tests/stress/bytecode_limit.py b/tests/stress/bytecode_limit.py new file mode 100644 index 0000000000..8cca413cf2 --- /dev/null +++ b/tests/stress/bytecode_limit.py @@ -0,0 +1,36 @@ +# Test the limits of bytecode generation. + +body = " with f()()() as a:\n try:\n f()()()\n except Exception:\n pass\n" + +# Test overflow of jump offset. +for n in (430, 431, 432, 433): + try: + exec("cond = 0\nif cond:\n" + body * n + "else:\n print('cond false')\n") + except MemoryError: + print("SKIP") + raise SystemExit + except RuntimeError: + print("RuntimeError") + +# Test changing size of code info (source line/bytecode mapping) due to changing +# bytecode size in the final passes. This test is very specific to how the +# code info is encoded, and how jump offsets shrink in the final passes. This +# test should fail if the bytecode emitter doesn't correctly handle shrinking of +# the code info section. +exec( + """ +x = 0 +if x: +""" + + body * 13 + + """ +x = [1 if x else 123] + + + + + + +print(x) +""" +) diff --git a/tests/stress/bytecode_limit.py.exp b/tests/stress/bytecode_limit.py.exp new file mode 100644 index 0000000000..74ab06c09b --- /dev/null +++ b/tests/stress/bytecode_limit.py.exp @@ -0,0 +1,5 @@ +cond false +cond false +RuntimeError +RuntimeError +[123] diff --git a/tests/stress/fun_call_limit.py b/tests/stress/fun_call_limit.py new file mode 100644 index 0000000000..b802aadd55 --- /dev/null +++ b/tests/stress/fun_call_limit.py @@ -0,0 +1,36 @@ +# Test the limit of the number of arguments to a function call. +# This currently tests the case of *args after many positional args. + + +def f(*args): + return len(args) + + +def test(n): + pos_args = ",".join(str(i) for i in range(n)) + s = "f({}, *(100, 101), 102, 103)".format(pos_args) + try: + return eval(s) + except SyntaxError: + return "SyntaxError" + + +# If the port has at least 32-bits then this test should pass. +print(test(29)) + +# This test should fail on all ports (overflows a small int). +print(test(70)) + +# Check that there is a correct transition to the limit of too many args before *args. +reached_limit = False +for i in range(30, 70): + result = test(i) + if reached_limit: + if result != "SyntaxError": + print("FAIL") + else: + if result == "SyntaxError": + reached_limit = True + else: + if result != i + 4: + print("FAIL") diff --git a/tests/stress/fun_call_limit.py.exp b/tests/stress/fun_call_limit.py.exp new file mode 100644 index 0000000000..53d2b28043 --- /dev/null +++ b/tests/stress/fun_call_limit.py.exp @@ -0,0 +1,2 @@ +33 +SyntaxError diff --git a/tests/thread/stress_aes.py b/tests/thread/stress_aes.py index 199fe3c88c..0ff5544aa8 100644 --- a/tests/thread/stress_aes.py +++ b/tests/thread/stress_aes.py @@ -271,7 +271,11 @@ def thread_entry(n_loop): if __name__ == "__main__": import sys - if sys.platform == "rp2": + if hasattr(sys, "settrace"): + # Builds with sys.settrace enabled are slow, so make the test short. + n_thread = 2 + n_loop = 2 + elif sys.platform == "rp2": n_thread = 1 n_loop = 2 elif sys.platform in ("esp32", "pyboard"): diff --git a/tests/unix/extra_coverage.py b/tests/unix/extra_coverage.py index 8ea27cbf2f..bb22485026 100644 --- a/tests/unix/extra_coverage.py +++ b/tests/unix/extra_coverage.py @@ -89,6 +89,9 @@ try: except ZeroDivisionError: print("ZeroDivisionError") +# test importing various objects +import frzmpy3 + # test for MP_QSTR_NULL regression from frzqstr import returns_NULL diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index c750ca0e95..c61b4ad5d8 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -17,6 +17,25 @@ abc # GC 0x0 0x0 +# tracked allocation +m_tracked_head = 0 +0 1 +1 1 +2 1 +3 1 +4 1 +5 1 +6 1 +7 1 +0 1 +1 1 +2 1 +3 1 +4 1 +5 1 +6 1 +7 1 +m_tracked_head = 0 # vstr tests sts diff --git a/tools/autobuild/build-boards.sh b/tools/autobuild/build-boards.sh index 4b5259b667..9caadc1b07 100755 --- a/tools/autobuild/build-boards.sh +++ b/tools/autobuild/build-boards.sh @@ -30,8 +30,8 @@ function build_board { dest=$dest_dir/$descr$fw_tag.$ext if [ -r $build_dir/firmware.$ext ]; then mv $build_dir/firmware.$ext $dest - else - # esp32 has micropython.elf and micropython.map + elif [ -r $build_dir/micropython.$ext ]; then + # esp32 has micropython.elf, etc mv $build_dir/micropython.$ext $dest fi done @@ -93,7 +93,7 @@ function build_esp32_boards { else if [ $mcu != esp32 ]; then # build esp32-s2/s3/c3 based boards with IDF v4.4+ - build_board $board_json $fw_tag $dest_dir bin elf map + build_board $board_json $fw_tag $dest_dir bin elf map uf2 fi fi done @@ -103,6 +103,10 @@ function build_mimxrt_boards { build_boards modmimxrt.c $1 $2 bin hex } +function build_renesas_ra_boards { + build_boards ra_it.c $1 $2 hex +} + function build_rp2_boards { build_boards modrp2.c $1 $2 uf2 } diff --git a/tools/ci.sh b/tools/ci.sh new file mode 100755 index 0000000000..f92db66d70 --- /dev/null +++ b/tools/ci.sh @@ -0,0 +1,699 @@ +#!/bin/bash + +if which nproc > /dev/null; then + MAKEOPTS="-j$(nproc)" +else + MAKEOPTS="-j$(sysctl -n hw.ncpu)" +fi + +######################################################################################## +# general helper functions + +function ci_gcc_arm_setup { + sudo apt-get install gcc-arm-none-eabi libnewlib-arm-none-eabi + arm-none-eabi-gcc --version +} + +######################################################################################## +# code formatting + +function ci_code_formatting_setup { + sudo apt-add-repository --yes --update ppa:pybricks/ppa + sudo apt-get install uncrustify + pip3 install black + uncrustify --version + black --version +} + +function ci_code_formatting_run { + tools/codeformat.py -v +} + +######################################################################################## +# commit formatting + +function ci_commit_formatting_run { + git remote add upstream https://github.com/micropython/micropython.git + git fetch --depth=100 upstream master + # For a PR, upstream/master..HEAD ends with a merge commit into master, exlude that one. + tools/verifygitlog.py -v upstream/master..HEAD --no-merges +} + +######################################################################################## +# code size + +function ci_code_size_setup { + sudo apt-get update + sudo apt-get install gcc-multilib + gcc --version + ci_gcc_arm_setup +} + +function ci_code_size_build { + # starts off at either the ref/pull/N/merge FETCH_HEAD, or the current branch HEAD + git checkout -b pull_request # save the current location + git remote add upstream https://github.com/micropython/micropython.git + git fetch --depth=100 upstream master + # build reference, save to size0 + # ignore any errors with this build, in case master is failing + git checkout `git merge-base --fork-point upstream/master pull_request` + git show -s + tools/metrics.py clean bm + tools/metrics.py build bm | tee ~/size0 || true + # build PR/branch, save to size1 + git checkout pull_request + git log upstream/master..HEAD + tools/metrics.py clean bm + tools/metrics.py build bm | tee ~/size1 +} + +######################################################################################## +# .mpy file format + +function ci_mpy_format_setup { + sudo pip3 install pyelftools +} + +function ci_mpy_format_test { + # Test mpy-tool.py dump feature on bytecode + python2 ./tools/mpy-tool.py -xd ports/minimal/frozentest.mpy + python3 ./tools/mpy-tool.py -xd ports/minimal/frozentest.mpy + + # Test mpy-tool.py dump feature on native code + make -C examples/natmod/features1 + ./tools/mpy-tool.py -xd examples/natmod/features1/features1.mpy +} + +######################################################################################## +# ports/cc3200 + +function ci_cc3200_setup { + ci_gcc_arm_setup +} + +function ci_cc3200_build { + make ${MAKEOPTS} -C ports/cc3200 BTARGET=application BTYPE=release + make ${MAKEOPTS} -C ports/cc3200 BTARGET=bootloader BTYPE=release +} + +######################################################################################## +# ports/esp32 + +function ci_esp32_setup_helper { + pip3 install pyelftools + git clone https://github.com/espressif/esp-idf.git + git -C esp-idf checkout $1 + git -C esp-idf submodule update --init \ + components/bt/host/nimble/nimble \ + components/esp_wifi \ + components/esptool_py/esptool \ + components/lwip/lwip \ + components/mbedtls/mbedtls + if [ -d esp-idf/components/bt/controller/esp32 ]; then + git -C esp-idf submodule update --init \ + components/bt/controller/lib_esp32 \ + components/bt/controller/lib_esp32c3_family + else + git -C esp-idf submodule update --init \ + components/bt/controller/lib + fi + ./esp-idf/install.sh +} + +function ci_esp32_idf402_setup { + ci_esp32_setup_helper v4.0.2 +} + +function ci_esp32_idf44_setup { + ci_esp32_setup_helper v4.4 +} + +function ci_esp32_build { + source esp-idf/export.sh + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/esp32 submodules + make ${MAKEOPTS} -C ports/esp32 \ + USER_C_MODULES=../../../examples/usercmodule/micropython.cmake \ + FROZEN_MANIFEST=$(pwd)/ports/esp32/boards/manifest_test.py + if [ -d $IDF_PATH/components/esp32c3 ]; then + make ${MAKEOPTS} -C ports/esp32 BOARD=GENERIC_C3 + fi + if [ -d $IDF_PATH/components/esp32s2 ]; then + make ${MAKEOPTS} -C ports/esp32 BOARD=GENERIC_S2 + fi + if [ -d $IDF_PATH/components/esp32s3 ]; then + make ${MAKEOPTS} -C ports/esp32 BOARD=GENERIC_S3 + fi + + # Test building native .mpy with xtensawin architecture. + ci_native_mpy_modules_build xtensawin +} + +######################################################################################## +# ports/esp8266 + +function ci_esp8266_setup { + sudo pip install pyserial esptool==3.3.1 + wget https://github.com/jepler/esp-open-sdk/releases/download/2018-06-10/xtensa-lx106-elf-standalone.tar.gz + zcat xtensa-lx106-elf-standalone.tar.gz | tar x + # Remove this esptool.py so pip version is used instead + rm xtensa-lx106-elf/bin/esptool.py +} + +function ci_esp8266_path { + echo $(pwd)/xtensa-lx106-elf/bin +} + +function ci_esp8266_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/esp8266 submodules + make ${MAKEOPTS} -C ports/esp8266 + make ${MAKEOPTS} -C ports/esp8266 BOARD=GENERIC_512K + make ${MAKEOPTS} -C ports/esp8266 BOARD=GENERIC_1M +} + +######################################################################################## +# ports/javascript + +function ci_javascript_setup { + git clone https://github.com/emscripten-core/emsdk.git + (cd emsdk && ./emsdk install latest && ./emsdk activate latest) +} + +function ci_javascript_build { + source emsdk/emsdk_env.sh + make ${MAKEOPTS} -C ports/javascript +} + +function ci_javascript_run_tests { + # This port is very slow at running, so only run a few of the tests. + (cd tests && MICROPY_MICROPYTHON=../ports/javascript/node_run.sh ./run-tests.py -j1 basics/builtin_*.py) +} + +######################################################################################## +# ports/mimxrt + +function ci_mimxrt_setup { + ci_gcc_arm_setup +} + +function ci_mimxrt_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/mimxrt submodules + make ${MAKEOPTS} -C ports/mimxrt BOARD=MIMXRT1020_EVK + make ${MAKEOPTS} -C ports/mimxrt BOARD=TEENSY40 +} + +######################################################################################## +# ports/nrf + +function ci_nrf_setup { + ci_gcc_arm_setup +} + +function ci_nrf_build { + ports/nrf/drivers/bluetooth/download_ble_stack.sh s140_nrf52_6_1_1 + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/nrf submodules + make ${MAKEOPTS} -C ports/nrf BOARD=pca10040 + make ${MAKEOPTS} -C ports/nrf BOARD=microbit + make ${MAKEOPTS} -C ports/nrf BOARD=pca10056 SD=s140 + make ${MAKEOPTS} -C ports/nrf BOARD=pca10090 +} + +######################################################################################## +# ports/powerpc + +function ci_powerpc_setup { + sudo apt-get update + sudo apt-get install gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross +} + +function ci_powerpc_build { + make ${MAKEOPTS} -C ports/powerpc UART=potato + make ${MAKEOPTS} -C ports/powerpc UART=lpc_serial +} + +######################################################################################## +# ports/qemu-arm + +function ci_qemu_arm_setup { + ci_gcc_arm_setup + sudo apt-get update + sudo apt-get install qemu-system + qemu-system-arm --version +} + +function ci_qemu_arm_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/qemu-arm CFLAGS_EXTRA=-DMP_ENDIANNESS_BIG=1 + make ${MAKEOPTS} -C ports/qemu-arm clean + make ${MAKEOPTS} -C ports/qemu-arm -f Makefile.test test + make ${MAKEOPTS} -C ports/qemu-arm -f Makefile.test clean + make ${MAKEOPTS} -C ports/qemu-arm -f Makefile.test BOARD=sabrelite test +} + +######################################################################################## +# ports/renesas-ra + +function ci_renesas_ra_setup { + ci_gcc_arm_setup +} + +function ci_renesas_ra_board_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/renesas-ra submodules + make ${MAKEOPTS} -C ports/renesas-ra BOARD=RA4M1_CLICKER + make ${MAKEOPTS} -C ports/renesas-ra BOARD=RA6M2_EK +} + +######################################################################################## +# ports/rp2 + +function ci_rp2_setup { + ci_gcc_arm_setup +} + +function ci_rp2_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/rp2 submodules + make ${MAKEOPTS} -C ports/rp2 + make ${MAKEOPTS} -C ports/rp2 clean + make ${MAKEOPTS} -C ports/rp2 USER_C_MODULES=../../examples/usercmodule/micropython.cmake + make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO submodules + make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO +} + +######################################################################################## +# ports/samd + +function ci_samd_setup { + ci_gcc_arm_setup +} + +function ci_samd_build { + make ${MAKEOPTS} -C ports/samd submodules + make ${MAKEOPTS} -C ports/samd +} + +######################################################################################## +# ports/stm32 + +function ci_stm32_setup { + ci_gcc_arm_setup + pip3 install pyelftools + pip3 install pyhy +} + +function ci_stm32_pyb_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/stm32 submodules + git submodule update --init lib/btstack + git submodule update --init lib/mynewt-nimble + make ${MAKEOPTS} -C ports/stm32 BOARD=PYBV11 MICROPY_PY_NETWORK_WIZNET5K=5200 MICROPY_PY_CC3K=1 USER_C_MODULES=../../examples/usercmodule + make ${MAKEOPTS} -C ports/stm32 BOARD=PYBD_SF2 + make ${MAKEOPTS} -C ports/stm32 BOARD=PYBD_SF6 NANBOX=1 MICROPY_BLUETOOTH_NIMBLE=0 MICROPY_BLUETOOTH_BTSTACK=1 + make ${MAKEOPTS} -C ports/stm32/mboot BOARD=PYBV10 CFLAGS_EXTRA='-DMBOOT_FSLOAD=1 -DMBOOT_VFS_LFS2=1' + make ${MAKEOPTS} -C ports/stm32/mboot BOARD=PYBD_SF6 + make ${MAKEOPTS} -C ports/stm32/mboot BOARD=STM32F769DISC CFLAGS_EXTRA='-DMBOOT_ADDRESS_SPACE_64BIT=1 -DMBOOT_SDCARD_ADDR=0x100000000ULL -DMBOOT_SDCARD_BYTE_SIZE=0x400000000ULL -DMBOOT_FSLOAD=1 -DMBOOT_VFS_FAT=1' + + # Test building native .mpy with armv7emsp architecture. + git submodule update --init lib/berkeley-db-1.xx + ci_native_mpy_modules_build armv7emsp +} + +function ci_stm32_nucleo_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/stm32 submodules + git submodule update --init lib/mynewt-nimble + + # Test building various MCU families, some with additional options. + make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_F091RC + make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_H743ZI COPT=-O2 CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1' + make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_L073RZ + make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_L476RG DEBUG=1 + + # Test building a board with mboot packing enabled (encryption, signing, compression). + make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_WB55 USE_MBOOT=1 MBOOT_ENABLE_PACKING=1 + make ${MAKEOPTS} -C ports/stm32/mboot BOARD=NUCLEO_WB55 USE_MBOOT=1 MBOOT_ENABLE_PACKING=1 + # Test mboot_pack_dfu.py created a valid file, and that its unpack-dfu command works. + BOARD_WB55=ports/stm32/boards/NUCLEO_WB55 + BUILD_WB55=ports/stm32/build-NUCLEO_WB55 + python3 ports/stm32/mboot/mboot_pack_dfu.py -k $BOARD_WB55/mboot_keys.h unpack-dfu $BUILD_WB55/firmware.pack.dfu $BUILD_WB55/firmware.unpack.dfu + diff $BUILD_WB55/firmware.unpack.dfu $BUILD_WB55/firmware.dfu + # Test unpack-dfu command works without a secret key + tail -n +2 $BOARD_WB55/mboot_keys.h > $BOARD_WB55/mboot_keys_no_sk.h + python3 ports/stm32/mboot/mboot_pack_dfu.py -k $BOARD_WB55/mboot_keys_no_sk.h unpack-dfu $BUILD_WB55/firmware.pack.dfu $BUILD_WB55/firmware.unpack_no_sk.dfu + diff $BUILD_WB55/firmware.unpack.dfu $BUILD_WB55/firmware.unpack_no_sk.dfu +} + +######################################################################################## +# ports/teensy + +function ci_teensy_setup { + ci_gcc_arm_setup +} + +function ci_teensy_build { + make ${MAKEOPTS} -C ports/teensy +} + +######################################################################################## +# ports/unix + +CI_UNIX_OPTS_SYS_SETTRACE=( + MICROPY_PY_BTREE=0 + MICROPY_PY_FFI=0 + MICROPY_PY_USSL=0 + CFLAGS_EXTRA="-DMICROPY_PY_SYS_SETTRACE=1" +) + +CI_UNIX_OPTS_SYS_SETTRACE_STACKLESS=( + MICROPY_PY_BTREE=0 + MICROPY_PY_FFI=0 + MICROPY_PY_USSL=0 + CFLAGS_EXTRA="-DMICROPY_STACKLESS=1 -DMICROPY_STACKLESS_STRICT=1 -DMICROPY_PY_SYS_SETTRACE=1" +) + +CI_UNIX_OPTS_QEMU_MIPS=( + CROSS_COMPILE=mips-linux-gnu- + VARIANT=coverage + MICROPY_STANDALONE=1 + LDFLAGS_EXTRA="-static" +) + +CI_UNIX_OPTS_QEMU_ARM=( + CROSS_COMPILE=arm-linux-gnueabi- + VARIANT=coverage + MICROPY_STANDALONE=1 +) + +function ci_unix_build_helper { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/unix "$@" submodules + make ${MAKEOPTS} -C ports/unix "$@" deplibs + make ${MAKEOPTS} -C ports/unix "$@" +} + +function ci_unix_build_ffi_lib_helper { + $1 $2 -shared -o tests/unix/ffi_lib.so tests/unix/ffi_lib.c +} + +function ci_unix_run_tests_helper { + make -C ports/unix "$@" test +} + +function ci_unix_run_tests_full_helper { + variant=$1 + shift + if [ $variant = standard ]; then + micropython=micropython + else + micropython=micropython-$variant + fi + make -C ports/unix VARIANT=$variant "$@" test_full + (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/$micropython ./run-multitests.py multi_net/*.py) + (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/$micropython ./run-perfbench.py 1000 1000) +} + +function ci_native_mpy_modules_build { + if [ "$1" = "" ]; then + arch=x64 + else + arch=$1 + fi + make -C examples/natmod/features1 ARCH=$arch + make -C examples/natmod/features2 ARCH=$arch + make -C examples/natmod/features3 ARCH=$arch + make -C examples/natmod/btree ARCH=$arch + make -C examples/natmod/framebuf ARCH=$arch + make -C examples/natmod/uheapq ARCH=$arch + make -C examples/natmod/urandom ARCH=$arch + make -C examples/natmod/ure ARCH=$arch + make -C examples/natmod/uzlib ARCH=$arch +} + +function ci_native_mpy_modules_32bit_build { + ci_native_mpy_modules_build x86 +} + +function ci_unix_minimal_build { + make ${MAKEOPTS} -C ports/unix VARIANT=minimal +} + +function ci_unix_minimal_run_tests { + (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython-minimal ./run-tests.py -e exception_chain -e self_type_check -e subclass_native_init -d basics) +} + +function ci_unix_standard_build { + ci_unix_build_helper VARIANT=standard + ci_unix_build_ffi_lib_helper gcc +} + +function ci_unix_standard_run_tests { + ci_unix_run_tests_full_helper standard +} + +function ci_unix_dev_build { + ci_unix_build_helper VARIANT=dev +} + +function ci_unix_dev_run_tests { + ci_unix_run_tests_helper VARIANT=dev +} + +function ci_unix_coverage_setup { + sudo pip3 install setuptools + sudo pip3 install pyelftools + gcc --version + python3 --version +} + +function ci_unix_coverage_build { + ci_unix_build_helper VARIANT=coverage + ci_unix_build_ffi_lib_helper gcc +} + +function ci_unix_coverage_run_tests { + ci_unix_run_tests_full_helper coverage +} + +function ci_unix_coverage_run_mpy_merge_tests { + mptop=$(pwd) + outdir=$(mktemp -d) + allmpy=() + + # Compile a selection of tests to .mpy and execute them, collecting the output. + # None of the tests should SKIP. + for inpy in $mptop/tests/basics/[acdel]*.py; do + test=$(basename $inpy .py) + echo $test + outmpy=$outdir/$test.mpy + $mptop/mpy-cross/mpy-cross -o $outmpy $inpy + (cd $outdir && $mptop/ports/unix/micropython-coverage -m $test >> out-individual) + allmpy+=($outmpy) + done + + # Merge all the tests into one .mpy file, and then execute it. + python3 $mptop/tools/mpy-tool.py --merge -o $outdir/merged.mpy ${allmpy[@]} + (cd $outdir && $mptop/ports/unix/micropython-coverage -m merged > out-merged) + + # Make sure the outputs match. + diff $outdir/out-individual $outdir/out-merged && /bin/rm -rf $outdir +} + +function ci_unix_coverage_run_native_mpy_tests { + MICROPYPATH=examples/natmod/features2 ./ports/unix/micropython-coverage -m features2 + (cd tests && ./run-natmodtests.py "$@" extmod/{btree*,framebuf*,uheapq*,urandom*,ure*,uzlib*}.py) +} + +function ci_unix_32bit_setup { + sudo dpkg --add-architecture i386 + sudo apt-get update + sudo apt-get install gcc-multilib g++-multilib libffi-dev:i386 + sudo pip3 install setuptools + sudo pip3 install pyelftools + gcc --version + python2 --version + python3 --version +} + +function ci_unix_coverage_32bit_build { + ci_unix_build_helper VARIANT=coverage MICROPY_FORCE_32BIT=1 + ci_unix_build_ffi_lib_helper gcc -m32 +} + +function ci_unix_coverage_32bit_run_tests { + ci_unix_run_tests_full_helper coverage MICROPY_FORCE_32BIT=1 +} + +function ci_unix_coverage_32bit_run_native_mpy_tests { + ci_unix_coverage_run_native_mpy_tests --arch x86 +} + +function ci_unix_nanbox_build { + # Use Python 2 to check that it can run the build scripts + ci_unix_build_helper PYTHON=python2 VARIANT=nanbox CFLAGS_EXTRA="-DMICROPY_PY_MATH_CONSTANTS=1" + ci_unix_build_ffi_lib_helper gcc -m32 +} + +function ci_unix_nanbox_run_tests { + ci_unix_run_tests_full_helper nanbox PYTHON=python2 +} + +function ci_unix_float_build { + ci_unix_build_helper VARIANT=standard CFLAGS_EXTRA="-DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_FLOAT" + ci_unix_build_ffi_lib_helper gcc +} + +function ci_unix_float_run_tests { + # TODO get this working: ci_unix_run_tests_full_helper standard CFLAGS_EXTRA="-DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_FLOAT" + ci_unix_run_tests_helper CFLAGS_EXTRA="-DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_FLOAT" +} + +function ci_unix_clang_setup { + sudo apt-get install clang + clang --version +} + +function ci_unix_stackless_clang_build { + make ${MAKEOPTS} -C mpy-cross CC=clang + make ${MAKEOPTS} -C ports/unix submodules + make ${MAKEOPTS} -C ports/unix CC=clang CFLAGS_EXTRA="-DMICROPY_STACKLESS=1 -DMICROPY_STACKLESS_STRICT=1" +} + +function ci_unix_stackless_clang_run_tests { + ci_unix_run_tests_helper CC=clang +} + +function ci_unix_float_clang_build { + make ${MAKEOPTS} -C mpy-cross CC=clang + make ${MAKEOPTS} -C ports/unix submodules + make ${MAKEOPTS} -C ports/unix CC=clang CFLAGS_EXTRA="-DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_FLOAT" +} + +function ci_unix_float_clang_run_tests { + ci_unix_run_tests_helper CC=clang +} + +function ci_unix_settrace_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/unix "${CI_UNIX_OPTS_SYS_SETTRACE[@]}" +} + +function ci_unix_settrace_run_tests { + ci_unix_run_tests_full_helper standard "${CI_UNIX_OPTS_SYS_SETTRACE[@]}" +} + +function ci_unix_settrace_stackless_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/unix "${CI_UNIX_OPTS_SYS_SETTRACE_STACKLESS[@]}" +} + +function ci_unix_settrace_stackless_run_tests { + ci_unix_run_tests_full_helper standard "${CI_UNIX_OPTS_SYS_SETTRACE_STACKLESS[@]}" +} + +function ci_unix_macos_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/unix submodules + #make ${MAKEOPTS} -C ports/unix deplibs + make ${MAKEOPTS} -C ports/unix + # check for additional compiler errors/warnings + make ${MAKEOPTS} -C ports/unix VARIANT=dev submodules + make ${MAKEOPTS} -C ports/unix VARIANT=dev + make ${MAKEOPTS} -C ports/unix VARIANT=coverage submodules + make ${MAKEOPTS} -C ports/unix VARIANT=coverage +} + +function ci_unix_macos_run_tests { + # Issues with macOS tests: + # - import_pkg7 has a problem with relative imports + # - urandom_basic has a problem with getrandbits(0) + (cd tests && ./run-tests.py --exclude 'import_pkg7.py' --exclude 'urandom_basic.py') +} + +function ci_unix_qemu_mips_setup { + sudo apt-get update + sudo apt-get install gcc-mips-linux-gnu g++-mips-linux-gnu + sudo apt-get install qemu-user + qemu-mips --version +} + +function ci_unix_qemu_mips_build { + # qemu-mips on GitHub Actions will seg-fault if not linked statically + ci_unix_build_helper "${CI_UNIX_OPTS_QEMU_MIPS[@]}" +} + +function ci_unix_qemu_mips_run_tests { + # Issues with MIPS tests: + # - (i)listdir does not work, it always returns the empty list (it's an issue with the underlying C call) + # - ffi tests do not work + file ./ports/unix/micropython-coverage + (cd tests && MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py --exclude 'vfs_posix.py' --exclude 'ffi_(callback|float|float2).py') +} + +function ci_unix_qemu_arm_setup { + sudo apt-get update + sudo apt-get install gcc-arm-linux-gnueabi g++-arm-linux-gnueabi + sudo apt-get install qemu-user + qemu-arm --version +} + +function ci_unix_qemu_arm_build { + ci_unix_build_helper "${CI_UNIX_OPTS_QEMU_ARM[@]}" + ci_unix_build_ffi_lib_helper arm-linux-gnueabi-gcc +} + +function ci_unix_qemu_arm_run_tests { + # Issues with ARM tests: + # - (i)listdir does not work, it always returns the empty list (it's an issue with the underlying C call) + export QEMU_LD_PREFIX=/usr/arm-linux-gnueabi + file ./ports/unix/micropython-coverage + (cd tests && MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py --exclude 'vfs_posix.py') +} + +######################################################################################## +# ports/windows + +function ci_windows_setup { + sudo apt-get install gcc-mingw-w64 +} + +function ci_windows_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/windows CROSS_COMPILE=i686-w64-mingw32- +} + +######################################################################################## +# ports/zephyr + +ZEPHYR_DOCKER_VERSION=v0.21.0 +ZEPHYR_SDK_VERSION=0.13.2 +ZEPHYR_VERSION=v3.0.0 + +function ci_zephyr_setup { + docker pull zephyrprojectrtos/ci:${ZEPHYR_DOCKER_VERSION} + docker run --name zephyr-ci -d -it \ + -v "$(pwd)":/micropython \ + -e ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-${ZEPHYR_SDK_VERSION} \ + -e ZEPHYR_TOOLCHAIN_VARIANT=zephyr \ + -e ZEPHYR_BASE=/zephyrproject/zephyr \ + -w /micropython/ports/zephyr \ + zephyrprojectrtos/ci:${ZEPHYR_DOCKER_VERSION} + docker ps -a +} + +function ci_zephyr_install { + docker exec zephyr-ci west init --mr ${ZEPHYR_VERSION} /zephyrproject + docker exec -w /zephyrproject zephyr-ci west update + docker exec -w /zephyrproject zephyr-ci west zephyr-export +} + +function ci_zephyr_build { + docker exec zephyr-ci west build -p auto -b qemu_x86 -- -DCONF_FILE=prj_minimal.conf + docker exec zephyr-ci west build -p auto -b qemu_x86 + docker exec zephyr-ci west build -p auto -b frdm_k64f + docker exec zephyr-ci west build -p auto -b mimxrt1050_evk + docker exec zephyr-ci west build -p auto -b nucleo_wb55rg # for bluetooth +} diff --git a/tools/codeformat.py b/tools/codeformat.py index f76e5f0681..c4150d863d 100644 --- a/tools/codeformat.py +++ b/tools/codeformat.py @@ -197,8 +197,7 @@ def main(): cmd_parser.add_argument("-c", action="store_true", help="Format C code only") cmd_parser.add_argument("-p", action="store_true", help="Format Python code only") cmd_parser.add_argument("-v", action="store_true", help="Enable verbose output") - cmd_parser.add_argument("--dry-run", action="store_true", help="Print, don't act") - cmd_parser.add_argument("files", nargs="+", help="Run on specific globs") + cmd_parser.add_argument("files", nargs="*", help="Run on specific globs") args = cmd_parser.parse_args() if args.dry_run: @@ -217,11 +216,6 @@ def main(): if os.path.splitext(file)[1].lower() in exts: yield file - def bindings_files(): - for file in lang_files(C_EXTS): - if file.startswith("shared-bindings/") or "/bindings/" in file: - yield file - # Run tool on N files at a time (to avoid making the command line too long). def batch(cmd, files, N=200, check=False): while True: diff --git a/tools/merge_micropython.py b/tools/merge_micropython.py index ce3736ddb7..a6d3efcc0d 100644 --- a/tools/merge_micropython.py +++ b/tools/merge_micropython.py @@ -1,6 +1,6 @@ """ This is a helper script for merging in new versions of MicroPython. You *must* -evaluate it's correctness and adapt it for each MP version. This is committed +evaluate its correctness and adapt it for each MP version. This is committed in the repo more for reference than "fire and forget" use. """ @@ -11,22 +11,24 @@ from io import StringIO out_buf = StringIO() ports_to_delete = [ - "mimxrt", - "powerpc", - "samd", - "javascript", - "stm32", - "esp32", + "bare-arm", "cc3200", + "embed", + "esp32", + "esp8266", + "javascript", + "mimxrt", + "minimal", + "pic16bit", + "powerpc", + "qemu-arm", + "renesas-ra", + "rp2", + "samd", + "stm32", "teensy", "windows", "zephyr", - "minimal", - "esp8266", - "pic16bit", - "qemu-arm", - "bare-arm", - "rp2", ] for p in ports_to_delete: try: diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 02ea656c99..18d2b65c41 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -1,7 +1,4 @@ #!/usr/bin/env python3 - -# SPDX-FileCopyrightText: Copyright (c) 2016-2019 Damien P. George -# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) # # SPDX-License-Identifier: MIT @@ -10,26 +7,53 @@ from __future__ import print_function import platform if platform.python_version_tuple()[0] == "2": - str_cons = lambda val, enc=None: val + from binascii import hexlify as hexlify_py2 + + str_cons = lambda val, enc=None: str(val) bytes_cons = lambda val, enc=None: bytearray(val) is_str_type = lambda o: type(o) is str is_bytes_type = lambda o: type(o) is bytearray is_int_type = lambda o: type(o) is int or type(o) is long + + def hexlify_to_str(b): + x = hexlify_py2(b) + return ":".join(x[i : i + 2] for i in range(0, len(x), 2)) + else: + from binascii import hexlify + str_cons = str bytes_cons = bytes is_str_type = lambda o: type(o) is str is_bytes_type = lambda o: type(o) is bytes is_int_type = lambda o: type(o) is int + + def hexlify_to_str(b): + return str(hexlify(b, ":"), "ascii") + + # end compatibility code import sys import struct -from collections import namedtuple sys.path.append(sys.path[0] + "/../py") import makeqstrdata as qstrutil +# Threshold of str length below which it will be turned into a qstr when freezing. +# This helps to reduce frozen code size because qstrs are more efficient to encode +# as objects than full mp_obj_str_t instances. +PERSISTENT_STR_INTERN_THRESHOLD = 25 + + +class MPYReadError(Exception): + def __init__(self, filename, msg): + self.filename = filename + self.msg = msg + + def __str__(self): + return "%s: %s" % (self.filename, self.msg) + class FreezeError(Exception): def __init__(self, rawcode, msg): @@ -41,7 +65,7 @@ class FreezeError(Exception): class Config: - MPY_VERSION = 5 + MPY_VERSION = 6 MICROPY_LONGINT_IMPL_NONE = 0 MICROPY_LONGINT_IMPL_LONGLONG = 1 MICROPY_LONGINT_IMPL_MPZ = 2 @@ -50,33 +74,6 @@ class Config: config = Config() -class QStrType: - def __init__(self, str): - self.str = str - self.qstr_esc = qstrutil.qstr_escape(self.str) - self.qstr_id = "MP_QSTR_" + self.qstr_esc - - -# Initialise global list of qstrs with static qstrs -global_qstrs = [None] # MP_QSTRnull should never be referenced -for n in qstrutil.static_qstr_list: - global_qstrs.append(QStrType(n)) - - -class QStrWindow: - def __init__(self, size): - self.window = [] - self.size = size - - def push(self, val): - self.window = [val] + self.window[: self.size - 1] - - def access(self, idx): - val = self.window[idx] - self.window = [val] + self.window[:idx] + self.window[idx + 1 :] - return val - - MP_CODE_BYTECODE = 2 MP_CODE_NATIVE_PY = 3 MP_CODE_NATIVE_VIPER = 4 @@ -94,6 +91,22 @@ MP_NATIVE_ARCH_ARMV7EMDP = 8 MP_NATIVE_ARCH_XTENSA = 9 MP_NATIVE_ARCH_XTENSAWIN = 10 +MP_PERSISTENT_OBJ_FUN_TABLE = 0 +MP_PERSISTENT_OBJ_NONE = 1 +MP_PERSISTENT_OBJ_FALSE = 2 +MP_PERSISTENT_OBJ_TRUE = 3 +MP_PERSISTENT_OBJ_ELLIPSIS = 4 +MP_PERSISTENT_OBJ_STR = 5 +MP_PERSISTENT_OBJ_BYTES = 6 +MP_PERSISTENT_OBJ_INT = 7 +MP_PERSISTENT_OBJ_FLOAT = 8 +MP_PERSISTENT_OBJ_COMPLEX = 9 +MP_PERSISTENT_OBJ_TUPLE = 10 + +MP_SCOPE_FLAG_VIPERRELOC = 0x10 +MP_SCOPE_FLAG_VIPERRODATA = 0x20 +MP_SCOPE_FLAG_VIPERBSS = 0x40 + MP_BC_MASK_EXTRA_BYTE = 0x9E MP_BC_FORMAT_BYTE = 0 @@ -101,32 +114,280 @@ MP_BC_FORMAT_QSTR = 1 MP_BC_FORMAT_VAR_UINT = 2 MP_BC_FORMAT_OFFSET = 3 -# extra byte if caching enabled: -MP_BC_LOAD_NAME = 0x11 -MP_BC_LOAD_GLOBAL = 0x12 -MP_BC_LOAD_ATTR = 0x13 -MP_BC_STORE_ATTR = 0x18 +mp_unary_op_method_name = ( + "__pos__", + "__neg__", + "__invert__", + "", +) + +mp_binary_op_method_name = ( + "__lt__", + "__gt__", + "__eq__", + "__le__", + "__ge__", + "__ne__", + "", + "", + "", + "__ior__", + "__ixor__", + "__iand__", + "__ilshift__", + "__irshift__", + "__iadd__", + "__isub__", + "__imul__", + "__imatmul__", + "__ifloordiv__", + "__itruediv__", + "__imod__", + "__ipow__", + "__or__", + "__xor__", + "__and__", + "__lshift__", + "__rshift__", + "__add__", + "__sub__", + "__mul__", + "__matmul__", + "__floordiv__", + "__truediv__", + "__mod__", + "__pow__", +) -# this function mirrors that in py/bc.c -def mp_opcode_format(bytecode, ip, count_var_uint): +class Opcode: + # fmt: off + # Load, Store, Delete, Import, Make, Build, Unpack, Call, Jump, Exception, For, sTack, Return, Yield, Op + MP_BC_BASE_RESERVED = (0x00) # ---------------- + MP_BC_BASE_QSTR_O = (0x10) # LLLLLLSSSDDII--- + MP_BC_BASE_VINT_E = (0x20) # MMLLLLSSDDBBBBBB + MP_BC_BASE_VINT_O = (0x30) # UUMMCCCC-------- + MP_BC_BASE_JUMP_E = (0x40) # J-JJJJJEEEEF---- + MP_BC_BASE_BYTE_O = (0x50) # LLLLSSDTTTTTEEFF + MP_BC_BASE_BYTE_E = (0x60) # --BREEEYYI------ + MP_BC_LOAD_CONST_SMALL_INT_MULTI = (0x70) # LLLLLLLLLLLLLLLL + # = (0x80) # LLLLLLLLLLLLLLLL + # = (0x90) # LLLLLLLLLLLLLLLL + # = (0xa0) # LLLLLLLLLLLLLLLL + MP_BC_LOAD_FAST_MULTI = (0xb0) # LLLLLLLLLLLLLLLL + MP_BC_STORE_FAST_MULTI = (0xc0) # SSSSSSSSSSSSSSSS + MP_BC_UNARY_OP_MULTI = (0xd0) # OOOOOOO + MP_BC_BINARY_OP_MULTI = (0xd7) # OOOOOOOOO + # = (0xe0) # OOOOOOOOOOOOOOOO + # = (0xf0) # OOOOOOOOOO------ + + MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM = 64 + MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS = 16 + MP_BC_LOAD_FAST_MULTI_NUM = 16 + MP_BC_STORE_FAST_MULTI_NUM = 16 + MP_BC_UNARY_OP_MULTI_NUM = 4 # MP_UNARY_OP_NUM_BYTECODE + MP_BC_BINARY_OP_MULTI_NUM = 35 # MP_BINARY_OP_NUM_BYTECODE + + MP_BC_LOAD_CONST_FALSE = (MP_BC_BASE_BYTE_O + 0x00) + MP_BC_LOAD_CONST_NONE = (MP_BC_BASE_BYTE_O + 0x01) + MP_BC_LOAD_CONST_TRUE = (MP_BC_BASE_BYTE_O + 0x02) + MP_BC_LOAD_CONST_SMALL_INT = (MP_BC_BASE_VINT_E + 0x02) # signed var-int + MP_BC_LOAD_CONST_STRING = (MP_BC_BASE_QSTR_O + 0x00) # qstr + MP_BC_LOAD_CONST_OBJ = (MP_BC_BASE_VINT_E + 0x03) # ptr + MP_BC_LOAD_NULL = (MP_BC_BASE_BYTE_O + 0x03) + + MP_BC_LOAD_FAST_N = (MP_BC_BASE_VINT_E + 0x04) # uint + MP_BC_LOAD_DEREF = (MP_BC_BASE_VINT_E + 0x05) # uint + MP_BC_LOAD_NAME = (MP_BC_BASE_QSTR_O + 0x01) # qstr + MP_BC_LOAD_GLOBAL = (MP_BC_BASE_QSTR_O + 0x02) # qstr + MP_BC_LOAD_ATTR = (MP_BC_BASE_QSTR_O + 0x03) # qstr + MP_BC_LOAD_METHOD = (MP_BC_BASE_QSTR_O + 0x04) # qstr + MP_BC_LOAD_SUPER_METHOD = (MP_BC_BASE_QSTR_O + 0x05) # qstr + MP_BC_LOAD_BUILD_CLASS = (MP_BC_BASE_BYTE_O + 0x04) + MP_BC_LOAD_SUBSCR = (MP_BC_BASE_BYTE_O + 0x05) + + MP_BC_STORE_FAST_N = (MP_BC_BASE_VINT_E + 0x06) # uint + MP_BC_STORE_DEREF = (MP_BC_BASE_VINT_E + 0x07) # uint + MP_BC_STORE_NAME = (MP_BC_BASE_QSTR_O + 0x06) # qstr + MP_BC_STORE_GLOBAL = (MP_BC_BASE_QSTR_O + 0x07) # qstr + MP_BC_STORE_ATTR = (MP_BC_BASE_QSTR_O + 0x08) # qstr + MP_BC_STORE_SUBSCR = (MP_BC_BASE_BYTE_O + 0x06) + + MP_BC_DELETE_FAST = (MP_BC_BASE_VINT_E + 0x08) # uint + MP_BC_DELETE_DEREF = (MP_BC_BASE_VINT_E + 0x09) # uint + MP_BC_DELETE_NAME = (MP_BC_BASE_QSTR_O + 0x09) # qstr + MP_BC_DELETE_GLOBAL = (MP_BC_BASE_QSTR_O + 0x0a) # qstr + + MP_BC_DUP_TOP = (MP_BC_BASE_BYTE_O + 0x07) + MP_BC_DUP_TOP_TWO = (MP_BC_BASE_BYTE_O + 0x08) + MP_BC_POP_TOP = (MP_BC_BASE_BYTE_O + 0x09) + MP_BC_ROT_TWO = (MP_BC_BASE_BYTE_O + 0x0a) + MP_BC_ROT_THREE = (MP_BC_BASE_BYTE_O + 0x0b) + + MP_BC_UNWIND_JUMP = (MP_BC_BASE_JUMP_E + 0x00) # signed relative bytecode offset; then a byte + MP_BC_JUMP = (MP_BC_BASE_JUMP_E + 0x02) # signed relative bytecode offset + MP_BC_POP_JUMP_IF_TRUE = (MP_BC_BASE_JUMP_E + 0x03) # signed relative bytecode offset + MP_BC_POP_JUMP_IF_FALSE = (MP_BC_BASE_JUMP_E + 0x04) # signed relative bytecode offset + MP_BC_JUMP_IF_TRUE_OR_POP = (MP_BC_BASE_JUMP_E + 0x05) # unsigned relative bytecode offset + MP_BC_JUMP_IF_FALSE_OR_POP = (MP_BC_BASE_JUMP_E + 0x06) # unsigned relative bytecode offset + MP_BC_SETUP_WITH = (MP_BC_BASE_JUMP_E + 0x07) # unsigned relative bytecode offset + MP_BC_SETUP_EXCEPT = (MP_BC_BASE_JUMP_E + 0x08) # unsigned relative bytecode offset + MP_BC_SETUP_FINALLY = (MP_BC_BASE_JUMP_E + 0x09) # unsigned relative bytecode offset + MP_BC_POP_EXCEPT_JUMP = (MP_BC_BASE_JUMP_E + 0x0a) # unsigned relative bytecode offset + MP_BC_FOR_ITER = (MP_BC_BASE_JUMP_E + 0x0b) # unsigned relative bytecode offset + MP_BC_WITH_CLEANUP = (MP_BC_BASE_BYTE_O + 0x0c) + MP_BC_END_FINALLY = (MP_BC_BASE_BYTE_O + 0x0d) + MP_BC_GET_ITER = (MP_BC_BASE_BYTE_O + 0x0e) + MP_BC_GET_ITER_STACK = (MP_BC_BASE_BYTE_O + 0x0f) + + MP_BC_BUILD_TUPLE = (MP_BC_BASE_VINT_E + 0x0a) # uint + MP_BC_BUILD_LIST = (MP_BC_BASE_VINT_E + 0x0b) # uint + MP_BC_BUILD_MAP = (MP_BC_BASE_VINT_E + 0x0c) # uint + MP_BC_STORE_MAP = (MP_BC_BASE_BYTE_E + 0x02) + MP_BC_BUILD_SET = (MP_BC_BASE_VINT_E + 0x0d) # uint + MP_BC_BUILD_SLICE = (MP_BC_BASE_VINT_E + 0x0e) # uint + MP_BC_STORE_COMP = (MP_BC_BASE_VINT_E + 0x0f) # uint + MP_BC_UNPACK_SEQUENCE = (MP_BC_BASE_VINT_O + 0x00) # uint + MP_BC_UNPACK_EX = (MP_BC_BASE_VINT_O + 0x01) # uint + + MP_BC_RETURN_VALUE = (MP_BC_BASE_BYTE_E + 0x03) + MP_BC_RAISE_LAST = (MP_BC_BASE_BYTE_E + 0x04) + MP_BC_RAISE_OBJ = (MP_BC_BASE_BYTE_E + 0x05) + MP_BC_RAISE_FROM = (MP_BC_BASE_BYTE_E + 0x06) + MP_BC_YIELD_VALUE = (MP_BC_BASE_BYTE_E + 0x07) + MP_BC_YIELD_FROM = (MP_BC_BASE_BYTE_E + 0x08) + + MP_BC_MAKE_FUNCTION = (MP_BC_BASE_VINT_O + 0x02) # uint + MP_BC_MAKE_FUNCTION_DEFARGS = (MP_BC_BASE_VINT_O + 0x03) # uint + MP_BC_MAKE_CLOSURE = (MP_BC_BASE_VINT_E + 0x00) # uint; extra byte + MP_BC_MAKE_CLOSURE_DEFARGS = (MP_BC_BASE_VINT_E + 0x01) # uint; extra byte + MP_BC_CALL_FUNCTION = (MP_BC_BASE_VINT_O + 0x04) # uint + MP_BC_CALL_FUNCTION_VAR_KW = (MP_BC_BASE_VINT_O + 0x05) # uint + MP_BC_CALL_METHOD = (MP_BC_BASE_VINT_O + 0x06) # uint + MP_BC_CALL_METHOD_VAR_KW = (MP_BC_BASE_VINT_O + 0x07) # uint + + MP_BC_IMPORT_NAME = (MP_BC_BASE_QSTR_O + 0x0b) # qstr + MP_BC_IMPORT_FROM = (MP_BC_BASE_QSTR_O + 0x0c) # qstr + MP_BC_IMPORT_STAR = (MP_BC_BASE_BYTE_E + 0x09) + # fmt: on + + # Create sets of related opcodes. + ALL_OFFSET_SIGNED = ( + MP_BC_UNWIND_JUMP, + MP_BC_JUMP, + MP_BC_POP_JUMP_IF_TRUE, + MP_BC_POP_JUMP_IF_FALSE, + ) + + # Create a dict mapping opcode value to opcode name. + mapping = ["unknown" for _ in range(256)] + for op_name in list(locals()): + if op_name.startswith("MP_BC_"): + mapping[locals()[op_name]] = op_name[len("MP_BC_") :] + for i in range(MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM): + name = "LOAD_CONST_SMALL_INT %d" % (i - MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS) + mapping[MP_BC_LOAD_CONST_SMALL_INT_MULTI + i] = name + for i in range(MP_BC_LOAD_FAST_MULTI_NUM): + mapping[MP_BC_LOAD_FAST_MULTI + i] = "LOAD_FAST %d" % i + for i in range(MP_BC_STORE_FAST_MULTI_NUM): + mapping[MP_BC_STORE_FAST_MULTI + i] = "STORE_FAST %d" % i + for i in range(MP_BC_UNARY_OP_MULTI_NUM): + mapping[MP_BC_UNARY_OP_MULTI + i] = "UNARY_OP %d %s" % (i, mp_unary_op_method_name[i]) + for i in range(MP_BC_BINARY_OP_MULTI_NUM): + mapping[MP_BC_BINARY_OP_MULTI + i] = "BINARY_OP %d %s" % (i, mp_binary_op_method_name[i]) + + def __init__(self, offset, fmt, opcode_byte, arg, extra_arg): + self.offset = offset + self.fmt = fmt + self.opcode_byte = opcode_byte + self.arg = arg + self.extra_arg = extra_arg + + +# This definition of a small int covers all possible targets, in the sense that every +# target can encode as a small int, an integer that passes this test. The minimum is set +# by MICROPY_OBJ_REPR_B on a 16-bit machine, where there are 14 bits for the small int. +def mp_small_int_fits(i): + return -0x2000 <= i <= 0x1FFF + + +def mp_encode_uint(val, signed=False): + encoded = bytearray([val & 0x7F]) + val >>= 7 + while val != 0 and val != -1: + encoded.insert(0, 0x80 | (val & 0x7F)) + val >>= 7 + if signed: + if val == -1 and encoded[0] & 0x40 == 0: + encoded.insert(0, 0xFF) + elif val == 0 and encoded[0] & 0x40 != 0: + encoded.insert(0, 0x80) + return encoded + + +def mp_opcode_decode(bytecode, ip): opcode = bytecode[ip] ip_start = ip f = (0x000003A4 >> (2 * ((opcode) >> 4))) & 3 - if f == MP_BC_FORMAT_QSTR: - ip += 3 - else: - extra_byte = (opcode & MP_BC_MASK_EXTRA_BYTE) == 0 + ip += 1 + arg = None + extra_arg = None + if f in (MP_BC_FORMAT_QSTR, MP_BC_FORMAT_VAR_UINT): + arg = bytecode[ip] & 0x7F + if opcode == Opcode.MP_BC_LOAD_CONST_SMALL_INT and arg & 0x40 != 0: + arg |= -1 << 7 + while bytecode[ip] & 0x80 != 0: + ip += 1 + arg = arg << 7 | bytecode[ip] & 0x7F ip += 1 - if f == MP_BC_FORMAT_VAR_UINT: - if count_var_uint: - while bytecode[ip] & 0x80 != 0: - ip += 1 - ip += 1 - elif f == MP_BC_FORMAT_OFFSET: + elif f == MP_BC_FORMAT_OFFSET: + if bytecode[ip] & 0x80 == 0: + arg = bytecode[ip] + ip += 1 + if opcode in Opcode.ALL_OFFSET_SIGNED: + arg -= 0x40 + else: + arg = bytecode[ip] & 0x7F | bytecode[ip + 1] << 7 ip += 2 - ip += extra_byte - return f, ip - ip_start + if opcode in Opcode.ALL_OFFSET_SIGNED: + arg -= 0x4000 + if opcode & MP_BC_MASK_EXTRA_BYTE == 0: + extra_arg = bytecode[ip] + ip += 1 + return f, ip - ip_start, arg, extra_arg + + +def mp_opcode_encode(opcode): + overflow = False + encoded = bytearray([opcode.opcode_byte]) + if opcode.fmt in (MP_BC_FORMAT_QSTR, MP_BC_FORMAT_VAR_UINT): + signed = opcode.opcode_byte == Opcode.MP_BC_LOAD_CONST_SMALL_INT + encoded.extend(mp_encode_uint(opcode.arg, signed)) + elif opcode.fmt == MP_BC_FORMAT_OFFSET: + is_signed = opcode.opcode_byte in Opcode.ALL_OFFSET_SIGNED + + # The -2 accounts for this jump opcode taking 2 bytes (at least). + bytecode_offset = opcode.target.offset - opcode.offset - 2 + + # Check if the bytecode_offset is small enough to use a 1-byte encoding. + if (is_signed and -64 <= bytecode_offset <= 63) or ( + not is_signed and bytecode_offset <= 127 + ): + # Use a 1-byte jump offset. + if is_signed: + bytecode_offset += 0x40 + overflow = not (0 <= bytecode_offset <= 0x7F) + encoded.append(bytecode_offset & 0x7F) + else: + bytecode_offset -= 1 + if is_signed: + bytecode_offset += 0x4000 + overflow = not (0 <= bytecode_offset <= 0x7FFF) + encoded.append(0x80 | (bytecode_offset & 0x7F)) + encoded.append((bytecode_offset >> 7) & 0xFF) + if opcode.extra_arg is not None: + encoded.append(opcode.extra_arg) + return overflow, encoded def read_prelude_sig(read_byte): @@ -168,6 +429,21 @@ def read_prelude_size(read_byte): return I, C +# See py/bc.h:MP_BC_PRELUDE_SIZE_ENCODE macro. +def encode_prelude_size(I, C): + # Encode bit-wise as: xIIIIIIC + encoded = bytearray() + while True: + z = (I & 0x3F) << 1 | (C & 1) + C >>= 1 + I >>= 6 + if C | I: + z |= 0x80 + encoded.append(z) + if not C | I: + return encoded + + def extract_prelude(bytecode, ip): def local_read_byte(): b = bytecode[ip_ref[0]] @@ -175,6 +451,8 @@ def extract_prelude(bytecode, ip): return b ip_ref = [ip] # to close over ip in Python 2 and 3 + + # Read prelude signature. ( n_state, n_exc_stack, @@ -183,18 +461,358 @@ def extract_prelude(bytecode, ip): n_kwonly_args, n_def_pos_args, ) = read_prelude_sig(local_read_byte) - n_info, n_cell = read_prelude_size(local_read_byte) - ip = ip_ref[0] - ip2 = ip - ip = ip2 + n_info + n_cell - # ip now points to first opcode - # ip2 points to simple_name qstr - return ip, ip2, (n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args) + offset_prelude_size = ip_ref[0] + + # Read prelude size. + n_info, n_cell = read_prelude_size(local_read_byte) + + offset_source_info = ip_ref[0] + + # Extract simple_name and argument qstrs (var uints). + args = [] + for arg_num in range(1 + n_pos_args + n_kwonly_args): + value = 0 + while True: + b = local_read_byte() + value = (value << 7) | (b & 0x7F) + if b & 0x80 == 0: + break + args.append(value) + + offset_line_info = ip_ref[0] + offset_closure_info = offset_source_info + n_info + offset_opcodes = offset_source_info + n_info + n_cell + + return ( + offset_prelude_size, + offset_source_info, + offset_line_info, + offset_closure_info, + offset_opcodes, + (n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args), + (n_info, n_cell), + args, + ) + + +class QStrType: + def __init__(self, str): + self.str = str + self.qstr_esc = qstrutil.qstr_escape(self.str) + self.qstr_id = "MP_QSTR_" + self.qstr_esc + + +class GlobalQStrList: + def __init__(self): + # Initialise global list of qstrs with static qstrs + self.qstrs = [None] # MP_QSTRnull should never be referenced + for n in qstrutil.static_qstr_list: + self.qstrs.append(QStrType(n)) + + def add(self, s): + q = QStrType(s) + self.qstrs.append(q) + return q + + def get_by_index(self, i): + return self.qstrs[i] + + def find_by_str(self, s): + for q in self.qstrs: + if q is not None and q.str == s: + return q + return None class MPFunTable: - pass + def __repr__(self): + return "mp_fun_table" + + +class CompiledModule: + def __init__( + self, + mpy_source_file, + mpy_segments, + header, + qstr_table, + obj_table, + raw_code, + qstr_table_file_offset, + obj_table_file_offset, + raw_code_file_offset, + escaped_name, + ): + self.mpy_source_file = mpy_source_file + self.mpy_segments = mpy_segments + self.source_file = qstr_table[0] + self.header = header + self.qstr_table = qstr_table + self.obj_table = obj_table + self.raw_code = raw_code + self.qstr_table_file_offset = qstr_table_file_offset + self.obj_table_file_offset = obj_table_file_offset + self.raw_code_file_offset = raw_code_file_offset + self.escaped_name = escaped_name + + def hexdump(self): + with open(self.mpy_source_file, "rb") as f: + WIDTH = 16 + COL_OFF = "\033[0m" + COL_TABLE = ( + ("", ""), # META + ("\033[0;31m", "\033[0;91m"), # QSTR + ("\033[0;32m", "\033[0;92m"), # OBJ + ("\033[0;34m", "\033[0;94m"), # CODE + ) + cur_col = "" + cur_col_index = 0 + offset = 0 + segment_index = 0 + while True: + data = bytes_cons(f.read(WIDTH)) + if not data: + break + + # Print out the hex dump of this line of data. + line_hex = cur_col + line_chr = cur_col + line_comment = "" + for i in range(len(data)): + # Determine the colour of the data, if any, and the line comment. + while segment_index < len(self.mpy_segments): + if offset + i == self.mpy_segments[segment_index].start: + cur_col = COL_TABLE[self.mpy_segments[segment_index].kind][ + cur_col_index + ] + cur_col_index = 1 - cur_col_index + line_hex += cur_col + line_chr += cur_col + line_comment += " %s%s%s" % ( + cur_col, + self.mpy_segments[segment_index].name, + COL_OFF, + ) + if offset + i == self.mpy_segments[segment_index].end: + cur_col = "" + line_hex += COL_OFF + line_chr += COL_OFF + segment_index += 1 + else: + break + + # Add to the hex part of the line. + if i % 2 == 0: + line_hex += " " + line_hex += "%02x" % data[i] + + # Add to the characters part of the line. + if 0x20 <= data[i] <= 0x7E: + line_chr += "%s" % chr(data[i]) + else: + line_chr += "." + + # Print out this line. + if cur_col: + line_hex += COL_OFF + line_chr += COL_OFF + pad = " " * ((WIDTH - len(data)) * 5 // 2) + print("%08x:%s%s %s %s" % (offset, line_hex, pad, line_chr, line_comment)) + offset += WIDTH + + def disassemble(self): + print("mpy_source_file:", self.mpy_source_file) + print("source_file:", self.source_file.str) + print("header:", hexlify_to_str(self.header)) + print("qstr_table[%u]:" % len(self.qstr_table)) + for q in self.qstr_table: + print(" %s" % q.str) + print("obj_table:", self.obj_table) + self.raw_code.disassemble() + + def freeze(self, compiled_module_index): + print() + print("/" * 80) + print("// frozen module %s" % self.escaped_name) + print("// - original source file: %s" % self.mpy_source_file) + print("// - frozen file name: %s" % self.source_file.str) + print("// - .mpy header: %s" % ":".join("%02x" % b for b in self.header)) + print() + + self.raw_code.freeze() + print() + + self.freeze_constants() + + print() + print("static const mp_frozen_module_t frozen_module_%s = {" % self.escaped_name) + print(" .constants = {") + if len(self.qstr_table): + print( + " .qstr_table = (qstr_short_t *)&const_qstr_table_data_%s," + % self.escaped_name + ) + else: + print(" .qstr_table = NULL,") + if len(self.obj_table): + print(" .obj_table = (mp_obj_t *)&const_obj_table_data_%s," % self.escaped_name) + else: + print(" .obj_table = NULL,") + print(" },") + print(" .rc = &raw_code_%s," % self.raw_code.escaped_name) + print("};") + + def freeze_constant_obj(self, obj_name, obj): + global const_str_content, const_int_content, const_obj_content + + if isinstance(obj, MPFunTable): + return "&mp_fun_table" + elif obj is None: + return "MP_ROM_NONE" + elif obj is False: + return "MP_ROM_FALSE" + elif obj is True: + return "MP_ROM_TRUE" + elif obj is Ellipsis: + return "MP_ROM_PTR(&mp_const_ellipsis_obj)" + elif is_str_type(obj) or is_bytes_type(obj): + if len(obj) == 0: + if is_str_type(obj): + return "MP_ROM_QSTR(MP_QSTR_)" + else: + return "MP_ROM_PTR(&mp_const_empty_bytes_obj)" + if is_str_type(obj): + q = global_qstrs.find_by_str(obj) + if q: + return "MP_ROM_QSTR(%s)" % q.qstr_id + obj = bytes_cons(obj, "utf8") + obj_type = "mp_type_str" + else: + obj_type = "mp_type_bytes" + print( + 'static const mp_obj_str_t %s = {{&%s}, %u, %u, (const byte*)"%s"};' + % ( + obj_name, + obj_type, + qstrutil.compute_hash(obj, config.MICROPY_QSTR_BYTES_IN_HASH), + len(obj), + "".join(("\\x%02x" % b) for b in obj), + ) + ) + const_str_content += len(obj) + const_obj_content += 4 * 4 + return "MP_ROM_PTR(&%s)" % obj_name + elif is_int_type(obj): + if mp_small_int_fits(obj): + # Encode directly as a small integer object. + return "MP_ROM_INT(%d)" % obj + elif config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_NONE: + raise FreezeError(self, "target does not support long int") + elif config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_LONGLONG: + # TODO + raise FreezeError(self, "freezing int to long-long is not implemented") + elif config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_MPZ: + neg = 0 + if obj < 0: + obj = -obj + neg = 1 + bits_per_dig = config.MPZ_DIG_SIZE + digs = [] + z = obj + while z: + digs.append(z & ((1 << bits_per_dig) - 1)) + z >>= bits_per_dig + ndigs = len(digs) + digs = ",".join(("%#x" % d) for d in digs) + print( + "static const mp_obj_int_t %s = {{&mp_type_int}, " + "{.neg=%u, .fixed_dig=1, .alloc=%u, .len=%u, .dig=(uint%u_t*)(const uint%u_t[]){%s}}};" + % (obj_name, neg, ndigs, ndigs, bits_per_dig, bits_per_dig, digs) + ) + const_int_content += (digs.count(",") + 1) * bits_per_dig // 8 + const_obj_content += 4 * 4 + return "MP_ROM_PTR(&%s)" % obj_name + elif type(obj) is float: + macro_name = "%s_macro" % obj_name + print( + "#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B" + ) + print( + "static const mp_obj_float_t %s = {{&mp_type_float}, (mp_float_t)%.16g};" + % (obj_name, obj) + ) + print("#define %s MP_ROM_PTR(&%s)" % (macro_name, obj_name)) + print("#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C") + n = struct.unpack(">= bits_per_dig - ndigs = len(digs) - digs = ",".join(("%#x" % d) for d in digs) - print( - "STATIC const mp_obj_int_t %s = {{&mp_type_int}, " - "{.neg=%u, .fixed_dig=1, .alloc=%u, .len=%u, .dig=(uint%u_t*)(const uint%u_t[]){%s}}};" - % (obj_name, neg, ndigs, ndigs, bits_per_dig, bits_per_dig, digs) - ) - elif type(obj) is float: - print( - "#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B" - ) - print( - "STATIC const mp_obj_float_t %s = {{&mp_type_float}, (mp_float_t)%.16g};" - % (obj_name, obj) - ) - print("#endif") - elif type(obj) is complex: - print( - "STATIC const mp_obj_complex_t %s = {{&mp_type_complex}, (mp_float_t)%.16g, (mp_float_t)%.16g};" - % (obj_name, obj.real, obj.imag) - ) - else: - raise FreezeError(self, "freezing of object %r is not implemented" % (obj,)) - - # generate constant table, if it has any entries - const_table_len = len(self.qstrs) + len(self.objs) + len(self.raw_codes) - if const_table_len: - print( - "STATIC const mp_rom_obj_t const_table_data_%s[%u] = {" - % (self.escaped_name, const_table_len) - ) - for qst in self.qstrs: - print(" MP_ROM_QSTR(%s)," % global_qstrs[qst].qstr_id) - for i in range(len(self.objs)): - if self.objs[i] is MPFunTable: - print(" &mp_fun_table,") - elif type(self.objs[i]) is float: - print( - "#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B" - ) - print(" MP_ROM_PTR(&const_obj_%s_%u)," % (self.escaped_name, i)) - print("#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C") - n = struct.unpack("> 8,") - print(" ", self.source_file.qstr_id, "& 0xff,", self.source_file.qstr_id, ">> 8,") - print(" ", end="") - for i in range(self.ip2 + 4, self.ip): - print(" 0x%02x," % self.bytecode[i], end="") - print() - ip = self.ip - while ip < len(self.bytecode): - f, sz = mp_opcode_format(self.bytecode, ip, True) - if f == 1: - qst = self._unpack_qstr(ip + 1).qstr_id - extra = "" if sz == 3 else " 0x%02x," % self.bytecode[ip + 3] - print(" ", "0x%02x," % self.bytecode[ip], qst, "& 0xff,", qst, ">> 8,", extra) + def disassemble(self): + bc = self.fun_data + print("simple_name:", self.simple_name.str) + print(" raw bytecode:", len(bc), hexlify_to_str(bc)) + print(" prelude:", self.prelude_signature) + print(" args:", [self.qstr_table[i].str for i in self.names[1:]]) + print(" line info:", hexlify_to_str(bc[self.offset_line_info : self.offset_opcodes])) + ip = self.offset_opcodes + while ip < len(bc): + fmt, sz, arg, _ = mp_opcode_decode(bc, ip) + if bc[ip] == Opcode.MP_BC_LOAD_CONST_OBJ: + arg = repr(self.obj_table[arg]) + if fmt == MP_BC_FORMAT_QSTR: + arg = self.qstr_table[arg].str + elif fmt in (MP_BC_FORMAT_VAR_UINT, MP_BC_FORMAT_OFFSET): + pass else: - print(" ", "".join("0x%02x, " % self.bytecode[ip + i] for i in range(sz))) + arg = "" + print( + " %-11s %s %s" % (hexlify_to_str(bc[ip : ip + sz]), Opcode.mapping[bc[ip]], arg) + ) ip += sz + self.disassemble_children() + + def freeze(self): + # generate bytecode data + bc = self.fun_data + print( + "// frozen bytecode for file %s, scope %s" + % (self.qstr_table[0].str, self.escaped_name) + ) + print("static const byte fun_data_%s[%u] = {" % (self.escaped_name, len(bc))) + + print(" ", end="") + for b in bc[: self.offset_source_info]: + print("0x%02x," % b, end="") + print(" // prelude") + + print(" ", end="") + for b in bc[self.offset_source_info : self.offset_line_info]: + print("0x%02x," % b, end="") + print(" // names: %s" % ", ".join(self.qstr_table[i].str for i in self.names)) + + print(" ", end="") + for b in bc[self.offset_line_info : self.offset_opcodes]: + print("0x%02x," % b, end="") + print(" // code info") + + ip = self.offset_opcodes + while ip < len(bc): + fmt, sz, arg, _ = mp_opcode_decode(bc, ip) + opcode_name = Opcode.mapping[bc[ip]] + if fmt == MP_BC_FORMAT_QSTR: + opcode_name += " " + repr(self.qstr_table[arg].str) + elif fmt in (MP_BC_FORMAT_VAR_UINT, MP_BC_FORMAT_OFFSET): + opcode_name += " %u" % arg + print( + " %s, // %s" % (",".join("0x%02x" % b for b in bc[ip : ip + sz]), opcode_name) + ) + ip += sz + print("};") - self.freeze_constants() - self.freeze_module() + self.freeze_children() + self.freeze_raw_code() + + global bc_content + bc_content += len(bc) class RawCodeNative(RawCode): def __init__( self, - code_kind, + parent_name, + qstr_table, + kind, fun_data, prelude_offset, - prelude, - qstr_links, - qstrs, - objs, - raw_codes, + scope_flags, + n_pos_args, type_sig, ): super(RawCodeNative, self).__init__( - code_kind, fun_data, prelude_offset, qstrs, objs, raw_codes + parent_name, qstr_table, fun_data, prelude_offset, kind ) - self.prelude = prelude - self.qstr_links = qstr_links + + if kind in (MP_CODE_NATIVE_VIPER, MP_CODE_NATIVE_ASM): + self.scope_flags = scope_flags + self.n_pos_args = n_pos_args + self.type_sig = type_sig if config.native_arch in ( MP_NATIVE_ARCH_X86, @@ -484,322 +1055,328 @@ class RawCodeNative(RawCode): # ARMVxxM -- two byte align. self.fun_data_attributes += " __attribute__ ((aligned (2)))" - def _asm_thumb_rewrite_mov(self, pc, val): - print(" (%u & 0xf0) | (%s >> 12)," % (self.bytecode[pc], val), end="") - print(" (%u & 0xfb) | (%s >> 9 & 0x04)," % (self.bytecode[pc + 1], val), end="") - print(" (%s & 0xff)," % (val,), end="") - print(" (%u & 0x07) | (%s >> 4 & 0x70)," % (self.bytecode[pc + 3], val)) + def disassemble(self): + fun_data = self.fun_data + print("simple_name:", self.simple_name.str) + print( + " raw data:", + len(fun_data), + hexlify_to_str(fun_data[:32]), + "..." if len(fun_data) > 32 else "", + ) + if self.code_kind != MP_CODE_NATIVE_PY: + return + print(" prelude:", self.prelude_signature) + print(" args:", [self.qstr_table[i].str for i in self.names[1:]]) + print(" line info:", fun_data[self.offset_line_info : self.offset_opcodes]) + ip = 0 + while ip < self.prelude_offset: + sz = 16 + print(" ", hexlify_to_str(fun_data[ip : min(ip + sz, self.prelude_offset)])) + ip += sz + self.disassemble_children() - def _link_qstr(self, pc, kind, qst): - if kind == 0: - # Generic 16-bit link - print(" %s & 0xff, %s >> 8," % (qst, qst)) - return 2 - else: - # Architecture-specific link - is_obj = kind == 2 - if is_obj: - qst = "((uintptr_t)MP_OBJ_NEW_QSTR(%s))" % qst - if config.native_arch in ( - MP_NATIVE_ARCH_X86, - MP_NATIVE_ARCH_X64, - MP_NATIVE_ARCH_ARMV6, - MP_NATIVE_ARCH_XTENSA, - MP_NATIVE_ARCH_XTENSAWIN, - ): - print( - " %s & 0xff, (%s >> 8) & 0xff, (%s >> 16) & 0xff, %s >> 24," - % (qst, qst, qst, qst) - ) - return 4 - elif MP_NATIVE_ARCH_ARMV6M <= config.native_arch <= MP_NATIVE_ARCH_ARMV7EMDP: - if is_obj: - # qstr object, movw and movt - self._asm_thumb_rewrite_mov(pc, qst) - self._asm_thumb_rewrite_mov(pc + 4, "(%s >> 16)" % qst) - return 8 - else: - # qstr number, movw instruction - self._asm_thumb_rewrite_mov(pc, qst) - return 4 - else: - assert 0 - - def freeze(self, parent_name): - if self.prelude[2] & ~0x0F: + def freeze(self): + if self.scope_flags & ~0x0F: raise FreezeError("unable to freeze code with relocations") - self.freeze_children(parent_name) - # generate native code data print() - if self.code_kind == MP_CODE_NATIVE_PY: - print( - "// frozen native code for file %s, scope %s%s" - % (self.source_file.str, parent_name, self.simple_name.str) - ) - elif self.code_kind == MP_CODE_NATIVE_VIPER: - print("// frozen viper code for scope %s" % (parent_name,)) - else: - print("// frozen assembler code for scope %s" % (parent_name,)) print( - "STATIC const byte fun_data_%s[%u] %s = {" - % (self.escaped_name, len(self.bytecode), self.fun_data_attributes) + "// frozen native code for file %s, scope %s" + % (self.qstr_table[0].str, self.escaped_name) + ) + print( + "static const byte fun_data_%s[%u] %s = {" + % (self.escaped_name, len(self.fun_data), self.fun_data_attributes) ) - if self.code_kind == MP_CODE_NATIVE_PY: - i_top = self.prelude_offset - else: - i_top = len(self.bytecode) + i_top = len(self.fun_data) i = 0 qi = 0 while i < i_top: - if qi < len(self.qstr_links) and i == self.qstr_links[qi][0]: - # link qstr - qi_off, qi_kind, qi_val = self.qstr_links[qi] - qst = global_qstrs[qi_val].qstr_id - i += self._link_qstr(i, qi_kind, qst) - qi += 1 - else: - # copy machine code (max 16 bytes) - i16 = min(i + 16, i_top) - if qi < len(self.qstr_links): - i16 = min(i16, self.qstr_links[qi][0]) - print(" ", end="") - for ii in range(i, i16): - print(" 0x%02x," % self.bytecode[ii], end="") - print() - i = i16 - - if self.code_kind == MP_CODE_NATIVE_PY: + # copy machine code (max 16 bytes) + i16 = min(i + 16, i_top) print(" ", end="") - for i in range(self.prelude_offset, self.ip2): - print(" 0x%02x," % self.bytecode[i], end="") - print() - - print(" ", self.simple_name.qstr_id, "& 0xff,", self.simple_name.qstr_id, ">> 8,") - print(" ", self.source_file.qstr_id, "& 0xff,", self.source_file.qstr_id, ">> 8,") - - print(" ", end="") - for i in range(self.ip2 + 4, self.ip): - print(" 0x%02x," % self.bytecode[i], end="") + for ii in range(i, i16): + print(" 0x%02x," % self.fun_data[ii], end="") print() + i = i16 print("};") - self.freeze_constants() - self.freeze_module(self.qstr_links, self.type_sig) + prelude_ptr = None + if self.code_kind == MP_CODE_NATIVE_PY: + prelude_ptr = "fun_data_%s_prelude_macro" % self.escaped_name + print("#if MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE") + n = len(self.fun_data) - self.prelude_offset + print("static const byte fun_data_%s_prelude[%u] = {" % (self.escaped_name, n), end="") + for i in range(n): + print(" 0x%02x," % self.fun_data[self.prelude_offset + i], end="") + print("};") + print("#define %s &fun_data_%s_prelude[0]" % (prelude_ptr, self.escaped_name)) + print("#else") + print( + "#define %s &fun_data_%s[%u]" + % (prelude_ptr, self.escaped_name, self.prelude_offset) + ) + print("#endif") + + self.freeze_children(prelude_ptr) + self.freeze_raw_code(prelude_ptr, self.type_sig) -class BytecodeBuffer: - def __init__(self, size): - self.buf = bytearray(size) - self.idx = 0 +class MPYSegment: + META = 0 + QSTR = 1 + OBJ = 2 + CODE = 3 - def is_full(self): - return self.idx == len(self.buf) - - def append(self, b): - self.buf[self.idx] = b - self.idx += 1 + def __init__(self, kind, name, start, end): + self.kind = kind + self.name = name + self.start = start + self.end = end -def read_byte(f, out=None): - b = bytes_cons(f.read(1))[0] - if out is not None: - out.append(b) - return b +class MPYReader: + def __init__(self, filename, fileobj): + self.filename = filename + self.fileobj = fileobj + + def tell(self): + return self.fileobj.tell() + + def read_byte(self): + return bytes_cons(self.fileobj.read(1))[0] + + def read_bytes(self, n): + return bytes_cons(self.fileobj.read(n)) + + def read_uint(self): + i = 0 + while True: + b = self.read_byte() + i = (i << 7) | (b & 0x7F) + if b & 0x80 == 0: + break + return i -def read_uint(f, out=None): - i = 0 - while True: - b = read_byte(f, out) - i = (i << 7) | (b & 0x7F) - if b & 0x80 == 0: - break - return i - - -def read_qstr(f, qstr_win): - ln = read_uint(f) - if ln == 0: - # static qstr - return bytes_cons(f.read(1))[0] +def read_qstr(reader, segments): + start_pos = reader.tell() + ln = reader.read_uint() if ln & 1: - # qstr in table - return qstr_win.access(ln >> 1) + # static qstr + q = global_qstrs.get_by_index(ln >> 1) + segments.append(MPYSegment(MPYSegment.META, q.str, start_pos, start_pos)) + return q ln >>= 1 - data = str_cons(f.read(ln), "utf8") - global_qstrs.append(QStrType(data)) - qstr_win.push(len(global_qstrs) - 1) - return len(global_qstrs) - 1 + start_pos = reader.tell() + data = str_cons(reader.read_bytes(ln), "utf8") + reader.read_byte() # read and discard null terminator + segments.append(MPYSegment(MPYSegment.QSTR, data, start_pos, reader.tell())) + return global_qstrs.add(data) -def read_obj(f): - obj_type = f.read(1) - if obj_type == b"e": +def read_obj(reader, segments): + obj_type = reader.read_byte() + if obj_type == MP_PERSISTENT_OBJ_FUN_TABLE: + return MPFunTable() + elif obj_type == MP_PERSISTENT_OBJ_NONE: + return None + elif obj_type == MP_PERSISTENT_OBJ_FALSE: + return False + elif obj_type == MP_PERSISTENT_OBJ_TRUE: + return True + elif obj_type == MP_PERSISTENT_OBJ_ELLIPSIS: return Ellipsis + elif obj_type == MP_PERSISTENT_OBJ_TUPLE: + ln = reader.read_uint() + return tuple(read_obj(reader, segments) for _ in range(ln)) else: - buf = f.read(read_uint(f)) - if obj_type == b"s": - return str_cons(buf, "utf8") - elif obj_type == b"b": - return bytes_cons(buf) - elif obj_type == b"i": - return int(str_cons(buf, "ascii"), 10) - elif obj_type == b"f": - return float(str_cons(buf, "ascii")) - elif obj_type == b"c": - return complex(str_cons(buf, "ascii")) + ln = reader.read_uint() + start_pos = reader.tell() + buf = reader.read_bytes(ln) + if obj_type in (MP_PERSISTENT_OBJ_STR, MP_PERSISTENT_OBJ_BYTES): + reader.read_byte() # read and discard null terminator + if obj_type == MP_PERSISTENT_OBJ_STR: + obj = str_cons(buf, "utf8") + if len(obj) < PERSISTENT_STR_INTERN_THRESHOLD: + if not global_qstrs.find_by_str(obj): + global_qstrs.add(obj) + elif obj_type == MP_PERSISTENT_OBJ_BYTES: + obj = buf + elif obj_type == MP_PERSISTENT_OBJ_INT: + obj = int(str_cons(buf, "ascii"), 10) + elif obj_type == MP_PERSISTENT_OBJ_FLOAT: + obj = float(str_cons(buf, "ascii")) + elif obj_type == MP_PERSISTENT_OBJ_COMPLEX: + obj = complex(str_cons(buf, "ascii")) else: - assert 0 + raise MPYReadError(reader.filename, "corrupt .mpy file") + segments.append(MPYSegment(MPYSegment.OBJ, obj, start_pos, reader.tell())) + return obj -def read_prelude(f, bytecode, qstr_win): - ( - n_state, - n_exc_stack, - scope_flags, - n_pos_args, - n_kwonly_args, - n_def_pos_args, - ) = read_prelude_sig(lambda: read_byte(f, bytecode)) - n_info, n_cell = read_prelude_size(lambda: read_byte(f, bytecode)) - read_qstr_and_pack(f, bytecode, qstr_win) # simple_name - read_qstr_and_pack(f, bytecode, qstr_win) # source_file - for _ in range(n_info - 4 + n_cell): - read_byte(f, bytecode) - return n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args - - -def read_qstr_and_pack(f, bytecode, qstr_win): - qst = read_qstr(f, qstr_win) - bytecode.append(qst & 0xFF) - bytecode.append(qst >> 8) - - -def read_bytecode(file, bytecode, qstr_win): - while not bytecode.is_full(): - op = read_byte(file, bytecode) - f, sz = mp_opcode_format(bytecode.buf, bytecode.idx - 1, False) - sz -= 1 - if f == MP_BC_FORMAT_QSTR: - read_qstr_and_pack(file, bytecode, qstr_win) - sz -= 2 - elif f == MP_BC_FORMAT_VAR_UINT: - while read_byte(file, bytecode) & 0x80: - pass - for _ in range(sz): - read_byte(file, bytecode) - - -def read_raw_code(f, qstr_win): - kind_len = read_uint(f) +def read_raw_code(reader, parent_name, qstr_table, obj_table, segments): + # Read raw code header. + kind_len = reader.read_uint() kind = (kind_len & 3) + MP_CODE_BYTECODE - fun_data_len = kind_len >> 2 - fun_data = BytecodeBuffer(fun_data_len) + has_children = (kind_len >> 2) & 1 + fun_data_len = kind_len >> 3 + + # Read the body of the raw code. + file_offset = reader.tell() + fun_data = reader.read_bytes(fun_data_len) + segments_len = len(segments) if kind == MP_CODE_BYTECODE: - prelude = read_prelude(f, fun_data, qstr_win) - read_bytecode(f, fun_data, qstr_win) + # Create bytecode raw code. + rc = RawCodeBytecode(parent_name, qstr_table, obj_table, fun_data) else: - fun_data.buf[:] = f.read(fun_data_len) - - qstr_links = [] - if kind in (MP_CODE_NATIVE_PY, MP_CODE_NATIVE_VIPER): - # load qstr link table - n_qstr_link = read_uint(f) - for _ in range(n_qstr_link): - off = read_uint(f) - qst = read_qstr(f, qstr_win) - qstr_links.append((off >> 2, off & 3, qst)) - - type_sig = 0 + # Create native raw code. + native_scope_flags = 0 + native_n_pos_args = 0 + native_type_sig = 0 if kind == MP_CODE_NATIVE_PY: - prelude_offset = read_uint(f) - _, name_idx, prelude = extract_prelude(fun_data.buf, prelude_offset) - fun_data.idx = name_idx # rewind to where qstrs are in prelude - read_qstr_and_pack(f, fun_data, qstr_win) # simple_name - read_qstr_and_pack(f, fun_data, qstr_win) # source_file + prelude_offset = reader.read_uint() else: - prelude_offset = None - scope_flags = read_uint(f) - n_pos_args = 0 - if kind == MP_CODE_NATIVE_ASM: - n_pos_args = read_uint(f) - type_sig = read_uint(f) - prelude = (None, None, scope_flags, n_pos_args, 0) + prelude_offset = 0 + native_scope_flags = reader.read_uint() + if kind == MP_CODE_NATIVE_VIPER: + # Read any additional sections for native viper. + if native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA: + rodata_size = reader.read_uint() + if native_scope_flags & MP_SCOPE_FLAG_VIPERBSS: + bss_size = reader.read_uint() + if native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA: + reader.read_bytes(rodata_size) + if native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC: + while True: + op = reader.read_byte() + if op == 0xFF: + break + if op & 1: + addr = reader.read_uint() + op >>= 1 + if op <= 5 and op & 1: + n = reader.read_uint() + else: + assert kind == MP_CODE_NATIVE_ASM + native_n_pos_args = reader.read_uint() + native_type_sig = reader.read_uint() - qstrs = [] - objs = [] - raw_codes = [] - if kind != MP_CODE_NATIVE_ASM: - # load constant table - n_obj = read_uint(f) - n_raw_code = read_uint(f) - qstrs = [read_qstr(f, qstr_win) for _ in range(prelude[3] + prelude[4])] - if kind != MP_CODE_BYTECODE: - objs.append(MPFunTable) - objs.extend([read_obj(f) for _ in range(n_obj)]) - raw_codes = [read_raw_code(f, qstr_win) for _ in range(n_raw_code)] - - if kind == MP_CODE_BYTECODE: - return RawCodeBytecode(fun_data.buf, qstrs, objs, raw_codes) - else: - return RawCodeNative( + rc = RawCodeNative( + parent_name, + qstr_table, kind, - fun_data.buf, + fun_data, prelude_offset, - prelude, - qstr_links, - qstrs, - objs, - raw_codes, - type_sig, + native_scope_flags, + native_n_pos_args, + native_type_sig, ) + # Add a segment for the raw code data. + segments.insert( + segments_len, + MPYSegment(MPYSegment.CODE, rc.simple_name.str, file_offset, file_offset + fun_data_len), + ) + + # Read children, if there are any. + rc.children = [] + if has_children: + # Make a pretty parent name (otherwise all identifiers will include _lt_module_gt_). + if not rc.escaped_name.endswith("_lt_module_gt_"): + parent_name = rc.escaped_name + + # Read all the child raw codes. + n_children = reader.read_uint() + for _ in range(n_children): + rc.children.append(read_raw_code(reader, parent_name, qstr_table, obj_table, segments)) + + return rc + def read_mpy(filename): - with open(filename, "rb") as f: - header = bytes_cons(f.read(4)) - if header[0] != ord("C"): - raise Exception("not a valid CircuitPython .mpy file") + with open(filename, "rb") as fileobj: + reader = MPYReader(filename, fileobj) + segments = [] + + # Read and verify the header. + header = reader.read_bytes(4) + if header[0] != ord("M"): + raise MPYReadError(filename, "not a valid .mpy file") if header[1] != config.MPY_VERSION: - raise Exception("incompatible .mpy version") + raise MPYReadError(filename, "incompatible .mpy version") feature_byte = header[2] - qw_size = read_uint(f) - config.MICROPY_PY_BUILTINS_STR_UNICODE = (feature_byte & 2) != 0 mpy_native_arch = feature_byte >> 2 if mpy_native_arch != MP_NATIVE_ARCH_NONE: if config.native_arch == MP_NATIVE_ARCH_NONE: config.native_arch = mpy_native_arch elif config.native_arch != mpy_native_arch: - raise Exception("native architecture mismatch") + raise MPYReadError(filename, "native architecture mismatch") config.mp_small_int_bits = header[3] - qstr_win = QStrWindow(qw_size) - rc = read_raw_code(f, qstr_win) - rc.mpy_source_file = filename - rc.qstr_win_size = qw_size - return rc + + # Read number of qstrs, and number of objects. + n_qstr = reader.read_uint() + n_obj = reader.read_uint() + + # Read qstrs and construct qstr table. + qstr_table_file_offset = reader.tell() + qstr_table = [] + for i in range(n_qstr): + qstr_table.append(read_qstr(reader, segments)) + + # Read objects and construct object table. + obj_table_file_offset = reader.tell() + obj_table = [] + for i in range(n_obj): + obj_table.append(read_obj(reader, segments)) + + # Compute the compiled-module escaped name. + cm_escaped_name = qstr_table[0].str.replace("/", "_")[:-3] + + # Read the outer raw code, which will in turn read all its children. + raw_code_file_offset = reader.tell() + raw_code = read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments) + + # Create the outer-level compiled module representing the whole .mpy file. + return CompiledModule( + filename, + segments, + header, + qstr_table, + obj_table, + raw_code, + qstr_table_file_offset, + obj_table_file_offset, + raw_code_file_offset, + cm_escaped_name, + ) -def dump_mpy(raw_codes): - for rc in raw_codes: - rc.dump() +def hexdump_mpy(compiled_modules): + for cm in compiled_modules: + cm.hexdump() -def freeze_mpy(base_qstrs, raw_codes): +def disassemble_mpy(compiled_modules): + for cm in compiled_modules: + cm.disassemble() + + +def freeze_mpy(base_qstrs, compiled_modules): # add to qstrs new = {} - for q in global_qstrs: + for q in global_qstrs.qstrs: # don't add duplicates if q is None or q.qstr_esc in base_qstrs or q.qstr_esc in new: continue - new[q.qstr_esc] = (len(new), q.qstr_esc, q.str) + new[q.qstr_esc] = (len(new), q.qstr_esc, q.str, bytes_cons(q.str, "utf8")) new = sorted(new.values(), key=lambda x: x[0]) - print('#include "py/bc0.h"') print('#include "py/mpconfig.h"') print('#include "py/objint.h"') print('#include "py/objstr.h"') @@ -835,7 +1412,7 @@ def freeze_mpy(base_qstrs, raw_codes): print("#endif") print() - if new: + if len(new) > 0: print("enum {") for i in range(len(new)): if i == 0: @@ -844,24 +1421,36 @@ def freeze_mpy(base_qstrs, raw_codes): print(" MP_QSTR_%s," % new[i][1]) print("};") + # As in qstr.c, set so that the first dynamically allocated pool is twice this size; must be <= the len + qstr_pool_alloc = min(len(new), 10) + + global bc_content, const_str_content, const_int_content, const_obj_content, const_table_qstr_content, const_table_ptr_content, raw_code_count, raw_code_content + qstr_content = 0 + bc_content = 0 + const_str_content = 0 + const_int_content = 0 + const_obj_content = 0 + const_table_qstr_content = 0 + const_table_ptr_content = 0 + raw_code_count = 0 + raw_code_content = 0 + print() - print("const qstr_attr_t mp_qstr_frozen_const_attr[] = {") + print("const qstr_hash_t mp_qstr_frozen_const_hashes[] = {") qstr_size = {"metadata": 0, "data": 0} - for _, _, qstr in new: - qbytes = qstrutil.bytes_cons(qstr, "utf8") - print( - " {%d, %d}," - % (qstrutil.compute_hash(qbytes, config.MICROPY_QSTR_BYTES_IN_HASH), len(qbytes)) - ) + for _, _, _, qbytes in new: + qhash = qstrutil.compute_hash(qbytes, config.MICROPY_QSTR_BYTES_IN_HASH) + print(" %d," % qhash) + print("};") + print() + print("const qstr_len_t mp_qstr_frozen_const_lengths[] = {") + for _, _, _, qbytes in new: + print(" %d," % len(qbytes)) qstr_size["metadata"] += ( config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH ) qstr_size["data"] += len(qbytes) print("};") - - # As in qstr.c, set so that the first dynamically allocated pool is twice this size; must be <= the len - qstr_pool_alloc = min(len(new), 10) - print() print("extern const qstr_pool_t mp_qstr_const_pool;") print("const qstr_pool_t mp_qstr_frozen_const_pool = {") @@ -869,36 +1458,54 @@ def freeze_mpy(base_qstrs, raw_codes): print(" MP_QSTRnumber_of, // previous pool size") print(" %u, // allocated entries" % qstr_pool_alloc) print(" %u, // used entries" % len(new)) - print(" (qstr_attr_t *)mp_qstr_frozen_const_attr,") + print(" (qstr_hash_t *)mp_qstr_frozen_const_hashes,") + print(" (qstr_len_t *)mp_qstr_frozen_const_lengths,") print(" {") - for _, _, qstr in new: - print(' "%s",' % qstrutil.escape_bytes(qstr)) + for _, _, qstr, qbytes in new: + print(' "%s",' % qstrutil.escape_bytes(qstr, qbytes)) + qstr_content += ( + config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH + len(qbytes) + 1 + ) print(" },") print("};") - for rc in raw_codes: - rc.freeze(rc.source_file.str.replace("/", "_")[:-3] + "_") + # Freeze all modules. + for idx, cm in enumerate(compiled_modules): + cm.freeze(idx) + # Print separator, separating individual modules from global data structures. + print() + print("/" * 80) + print("// collection of all frozen modules") + + # Define the string of frozen module names. print() print("const char mp_frozen_names[] = {") - print("#ifdef MP_FROZEN_STR_NAMES") + print(" #ifdef MP_FROZEN_STR_NAMES") # makemanifest.py might also include some frozen string content. - print("MP_FROZEN_STR_NAMES") - print("#endif") - for rc in raw_codes: - module_name = rc.source_file.str - print('"%s\\0"' % module_name) - print('"\\0"};') - - print("const mp_raw_code_t *const mp_frozen_mpy_content[] = {") - for rc in raw_codes: - print(" &raw_code_%s," % rc.escaped_name) + print(" MP_FROZEN_STR_NAMES") + print(" #endif") + mp_frozen_mpy_names_content = 1 + for cm in compiled_modules: + module_name = cm.source_file.str + print(' "%s\\0"' % module_name) + mp_frozen_mpy_names_content += len(cm.source_file.str) + 1 + print(' "\\0"') print("};") + # Define the array of pointers to frozen module content. + print() + print("const mp_frozen_module_t *const mp_frozen_mpy_content[] = {") + for cm in compiled_modules: + print(" &frozen_module_%s," % cm.escaped_name) + print("};") + mp_frozen_mpy_content_size = len(compiled_modules * 4) + # If a port defines MICROPY_FROZEN_LIST_ITEM then list all modules wrapped in that macro. + print() print("#ifdef MICROPY_FROZEN_LIST_ITEM") - for rc in raw_codes: - module_name = rc.source_file.str + for cm in compiled_modules: + module_name = cm.source_file.str if module_name.endswith("/__init__.py"): short_name = module_name[: -len("/__init__.py")] else: @@ -906,47 +1513,189 @@ def freeze_mpy(base_qstrs, raw_codes): print('MICROPY_FROZEN_LIST_ITEM("%s", "%s")' % (short_name, module_name)) print("#endif") + print() + print("/*") + print("byte sizes:") + print("qstr content: %d unique, %d bytes" % (len(new), qstr_content)) + print("bc content: %d" % bc_content) + print("const str content: %d" % const_str_content) + print("const int content: %d" % const_int_content) + print("const obj content: %d" % const_obj_content) + print( + "const table qstr content: %d entries, %d bytes" + % (const_table_qstr_content, const_table_qstr_content * 4) + ) + print( + "const table ptr content: %d entries, %d bytes" + % (const_table_ptr_content, const_table_ptr_content * 4) + ) + print("raw code content: %d * 4 = %d" % (raw_code_count, raw_code_content)) + print("mp_frozen_mpy_names_content: %d" % mp_frozen_mpy_names_content) + print("mp_frozen_mpy_content_size: %d" % mp_frozen_mpy_content_size) + print( + "total: %d" + % ( + qstr_content + + bc_content + + const_str_content + + const_int_content + + const_obj_content + + const_table_qstr_content * 4 + + const_table_ptr_content * 4 + + raw_code_content + + mp_frozen_mpy_names_content + + mp_frozen_mpy_content_size + ) + ) + print("*/") -def merge_mpy(raw_codes, output_file): - assert len(raw_codes) <= 31 # so var-uints all fit in 1 byte + +def adjust_bytecode_qstr_obj_indices(bytecode_in, qstr_table_base, obj_table_base): + # Expand bytcode to a list of opcodes. + opcodes = [] + labels = {} + ip = 0 + while ip < len(bytecode_in): + fmt, sz, arg, extra_arg = mp_opcode_decode(bytecode_in, ip) + opcode = Opcode(ip, fmt, bytecode_in[ip], arg, extra_arg) + labels[ip] = opcode + opcodes.append(opcode) + ip += sz + if fmt == MP_BC_FORMAT_OFFSET: + opcode.arg += ip + + # Link jump opcodes to their destination. + for opcode in opcodes: + if opcode.fmt == MP_BC_FORMAT_OFFSET: + opcode.target = labels[opcode.arg] + + # Adjust bytcode as required. + for opcode in opcodes: + if opcode.fmt == MP_BC_FORMAT_QSTR: + opcode.arg += qstr_table_base + elif opcode.opcode_byte == Opcode.MP_BC_LOAD_CONST_OBJ: + opcode.arg += obj_table_base + + # Write out new bytecode. + offset_changed = True + while offset_changed: + offset_changed = False + overflow = False + bytecode_out = b"" + for opcode in opcodes: + ip = len(bytecode_out) + if opcode.offset != ip: + offset_changed = True + opcode.offset = ip + opcode_overflow, encoded_opcode = mp_opcode_encode(opcode) + if opcode_overflow: + overflow = True + bytecode_out += encoded_opcode + + if overflow: + raise Exception("bytecode overflow") + + return bytecode_out + + +def rewrite_raw_code(rc, qstr_table_base, obj_table_base): + if rc.code_kind != MP_CODE_BYTECODE: + raise Exception("can only rewrite bytecode") + + source_info = bytearray() + for arg in rc.names: + source_info.extend(mp_encode_uint(qstr_table_base + arg)) + + closure_info = rc.fun_data[rc.offset_closure_info : rc.offset_opcodes] + + bytecode_in = memoryview(rc.fun_data)[rc.offset_opcodes :] + bytecode_out = adjust_bytecode_qstr_obj_indices(bytecode_in, qstr_table_base, obj_table_base) + + prelude_signature = rc.fun_data[: rc.offset_prelude_size] + prelude_size = encode_prelude_size(len(source_info), len(closure_info)) + + fun_data = prelude_signature + prelude_size + source_info + closure_info + bytecode_out + + output = mp_encode_uint(len(fun_data) << 3 | bool(len(rc.children)) << 2) + output += fun_data + + if rc.children: + output += mp_encode_uint(len(rc.children)) + for child in rc.children: + output += rewrite_raw_code(child, qstr_table_base, obj_table_base) + + return output + + +def merge_mpy(compiled_modules, output_file): merged_mpy = bytearray() - if len(raw_codes) == 1: - with open(raw_codes[0].mpy_source_file, "rb") as f: + if len(compiled_modules) == 1: + with open(compiled_modules[0].mpy_source_file, "rb") as f: merged_mpy.extend(f.read()) else: - header = bytearray(5) - header[0] = ord("C") + main_cm_idx = None + for idx, cm in enumerate(compiled_modules): + if cm.header[2]: + # Must use qstr_table and obj_table from this raw_code + if main_cm_idx is not None: + raise Exception("can't merge files when more than one contains native code") + main_cm_idx = idx + if main_cm_idx is not None: + # Shift main_cm to front of list. + compiled_modules.insert(0, compiled_modules.pop(main_cm_idx)) + + header = bytearray(4) + header[0] = ord("M") header[1] = config.MPY_VERSION - header[2] = config.native_arch << 2 | config.MICROPY_PY_BUILTINS_STR_UNICODE << 1 + header[2] = config.native_arch << 2 header[3] = config.mp_small_int_bits - header[4] = 32 # qstr_win_size merged_mpy.extend(header) + n_qstr = 0 + n_obj = 0 + for cm in compiled_modules: + n_qstr += len(cm.qstr_table) + n_obj += len(cm.obj_table) + merged_mpy.extend(mp_encode_uint(n_qstr)) + merged_mpy.extend(mp_encode_uint(n_obj)) + + # Copy verbatim the qstr and object tables from all compiled modules. + def copy_section(file, offset, offset2): + with open(file, "rb") as f: + f.seek(offset) + merged_mpy.extend(f.read(offset2 - offset)) + + for cm in compiled_modules: + copy_section(cm.mpy_source_file, cm.qstr_table_file_offset, cm.obj_table_file_offset) + for cm in compiled_modules: + copy_section(cm.mpy_source_file, cm.obj_table_file_offset, cm.raw_code_file_offset) + bytecode = bytearray() - bytecode_len = 6 + len(raw_codes) * 5 + 2 - bytecode.append(bytecode_len << 2) # kind and length - bytecode.append(0b00000000) # signature prelude - bytecode.append(0b00001000) # size prelude - bytecode.extend(b"\x00\x01") # MP_QSTR_ - bytecode.extend(b"\x00\x01") # MP_QSTR_ - for idx in range(len(raw_codes)): + bytecode.append(0b00000000) # prelude signature + bytecode.append(0b00000010) # prelude size (n_info=1, n_cell=0) + bytecode.extend(b"\x00") # simple_name: qstr index 0 (will use source filename) + for idx in range(len(compiled_modules)): bytecode.append(0x32) # MP_BC_MAKE_FUNCTION bytecode.append(idx) # index raw code bytecode.extend(b"\x34\x00\x59") # MP_BC_CALL_FUNCTION, 0 args, MP_BC_POP_TOP bytecode.extend(b"\x51\x63") # MP_BC_LOAD_NONE, MP_BC_RETURN_VALUE - bytecode.append(0) # n_obj - bytecode.append(len(raw_codes)) # n_raw_code - + merged_mpy.extend(mp_encode_uint(len(bytecode) << 3 | 1 << 2)) # length, has_children merged_mpy.extend(bytecode) + merged_mpy.extend(mp_encode_uint(len(compiled_modules))) # n_children - for rc in raw_codes: - with open(rc.mpy_source_file, "rb") as f: - f.read(4) # skip header - read_uint(f) # skip qstr_win_size - data = f.read() # read rest of mpy file - merged_mpy.extend(data) + qstr_table_base = 0 + obj_table_base = 0 + for cm in compiled_modules: + if qstr_table_base == 0 and obj_table_base == 0: + with open(cm.mpy_source_file, "rb") as f: + f.seek(cm.raw_code_file_offset) + merged_mpy.extend(f.read()) + else: + merged_mpy.extend(rewrite_raw_code(cm.raw_code, qstr_table_base, obj_table_base)) + qstr_table_base += len(cm.qstr_table) + obj_table_base += len(cm.obj_table) if output_file is None: sys.stdout.buffer.write(merged_mpy) @@ -956,10 +1705,17 @@ def merge_mpy(raw_codes, output_file): def main(): + global global_qstrs + import argparse cmd_parser = argparse.ArgumentParser(description="A tool to work with MicroPython .mpy files.") - cmd_parser.add_argument("-d", "--dump", action="store_true", help="dump contents of files") + cmd_parser.add_argument( + "-x", "--hexdump", action="store_true", help="output an annotated hex dump of files" + ) + cmd_parser.add_argument( + "-d", "--disassemble", action="store_true", help="output disassembled contents of files" + ) cmd_parser.add_argument("-f", "--freeze", action="store_true", help="freeze files") cmd_parser.add_argument( "--merge", action="store_true", help="merge multiple .mpy files into one" @@ -993,26 +1749,41 @@ def main(): # set config values for qstrs, and get the existing base set of qstrs if args.qstr_header: - qcfgs, base_qstrs, _ = qstrutil.parse_input_headers([args.qstr_header]) + qcfgs, base_qstrs = qstrutil.parse_input_headers([args.qstr_header]) config.MICROPY_QSTR_BYTES_IN_LEN = int(qcfgs["BYTES_IN_LEN"]) config.MICROPY_QSTR_BYTES_IN_HASH = int(qcfgs["BYTES_IN_HASH"]) else: config.MICROPY_QSTR_BYTES_IN_LEN = 1 config.MICROPY_QSTR_BYTES_IN_HASH = 1 - base_qstrs = {} + base_qstrs = list(qstrutil.static_qstr_list) - raw_codes = [read_mpy(file) for file in args.files] + # Create initial list of global qstrs. + global_qstrs = GlobalQStrList() - if args.dump: - dump_mpy(raw_codes) - elif args.freeze: + # Load all .mpy files. + try: + compiled_modules = [read_mpy(file) for file in args.files] + except MPYReadError as er: + print(er, file=sys.stderr) + sys.exit(1) + + if args.hexdump: + hexdump_mpy(compiled_modules) + + if args.disassemble: + if args.hexdump: + print() + disassemble_mpy(compiled_modules) + + if args.freeze: try: - freeze_mpy(base_qstrs, raw_codes) + freeze_mpy(base_qstrs, compiled_modules) except FreezeError as er: print(er, file=sys.stderr) sys.exit(1) - elif args.merge: - merged_mpy = merge_mpy(raw_codes, args.output) + + if args.merge: + merge_mpy(compiled_modules, args.output) if __name__ == "__main__": diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index 54df98ef07..3b50aef8c4 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -35,22 +35,22 @@ sys.path.append(os.path.dirname(__file__) + "/../py") import makeqstrdata as qstrutil # MicroPython constants -MPY_VERSION = 5 +MPY_VERSION = 6 +MP_CODE_BYTECODE = 2 +MP_CODE_NATIVE_VIPER = 4 MP_NATIVE_ARCH_X86 = 1 MP_NATIVE_ARCH_X64 = 2 +MP_NATIVE_ARCH_ARMV6M = 4 MP_NATIVE_ARCH_ARMV7M = 5 MP_NATIVE_ARCH_ARMV7EMSP = 7 MP_NATIVE_ARCH_ARMV7EMDP = 8 MP_NATIVE_ARCH_XTENSA = 9 MP_NATIVE_ARCH_XTENSAWIN = 10 -MP_CODE_BYTECODE = 2 -MP_CODE_NATIVE_VIPER = 4 -MP_SCOPE_FLAG_VIPERRELOC = 0x20 -MP_SCOPE_FLAG_VIPERRODATA = 0x40 -MP_SCOPE_FLAG_VIPERBSS = 0x80 -MICROPY_PY_BUILTINS_STR_UNICODE = 2 +MP_PERSISTENT_OBJ_STR = 5 +MP_SCOPE_FLAG_VIPERRELOC = 0x10 +MP_SCOPE_FLAG_VIPERRODATA = 0x20 +MP_SCOPE_FLAG_VIPERBSS = 0x40 MP_SMALL_INT_BITS = 31 -QSTR_WINDOW_SIZE = 32 # ELF constants R_386_32 = 1 @@ -75,6 +75,7 @@ R_ARM_THM_JUMP24 = 30 R_X86_64_GOTPCREL = 9 R_X86_64_REX_GOTPCRELX = 42 R_386_GOT32X = 43 +R_XTENSA_PDIFF32 = 59 ################################################################################ # Architecture configuration @@ -84,7 +85,14 @@ def asm_jump_x86(entry): return struct.pack("> 11 == 0 or b_off >> 11 == -1, b_off + return struct.pack("> 1 & 0x07FF)) + + +def asm_jump_thumb2(entry): b_off = entry - 4 if b_off >> 11 == 0 or b_off >> 11 == -1: # Signed value fits in 12 bits @@ -104,72 +112,73 @@ def asm_jump_xtensa(entry): class ArchData: - def __init__(self, name, mpy_feature, qstr_entry_size, word_size, arch_got, asm_jump): + def __init__(self, name, mpy_feature, word_size, arch_got, asm_jump, *, separate_rodata=False): self.name = name self.mpy_feature = mpy_feature - self.qstr_entry_size = qstr_entry_size + self.qstr_entry_size = 2 self.word_size = word_size self.arch_got = arch_got self.asm_jump = asm_jump - self.separate_rodata = name == "EM_XTENSA" and qstr_entry_size == 4 + self.separate_rodata = separate_rodata ARCH_DATA = { "x86": ArchData( "EM_386", - MP_NATIVE_ARCH_X86 << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, - 2, + MP_NATIVE_ARCH_X86 << 2, 4, (R_386_PC32, R_386_GOT32, R_386_GOT32X), asm_jump_x86, ), "x64": ArchData( "EM_X86_64", - MP_NATIVE_ARCH_X64 << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, - 2, + MP_NATIVE_ARCH_X64 << 2, 8, (R_X86_64_GOTPCREL, R_X86_64_REX_GOTPCRELX), asm_jump_x86, ), - "armv7m": ArchData( + "armv6m": ArchData( "EM_ARM", - MP_NATIVE_ARCH_ARMV7M << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, - 2, + MP_NATIVE_ARCH_ARMV6M << 2, 4, (R_ARM_GOT_BREL,), - asm_jump_arm, + asm_jump_thumb, + ), + "armv7m": ArchData( + "EM_ARM", + MP_NATIVE_ARCH_ARMV7M << 2, + 4, + (R_ARM_GOT_BREL,), + asm_jump_thumb2, ), "armv7emsp": ArchData( "EM_ARM", - MP_NATIVE_ARCH_ARMV7EMSP << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, - 2, + MP_NATIVE_ARCH_ARMV7EMSP << 2, 4, (R_ARM_GOT_BREL,), - asm_jump_arm, + asm_jump_thumb2, ), "armv7emdp": ArchData( "EM_ARM", - MP_NATIVE_ARCH_ARMV7EMDP << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, - 2, + MP_NATIVE_ARCH_ARMV7EMDP << 2, 4, (R_ARM_GOT_BREL,), - asm_jump_arm, + asm_jump_thumb2, ), "xtensa": ArchData( "EM_XTENSA", - MP_NATIVE_ARCH_XTENSA << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, - 2, + MP_NATIVE_ARCH_XTENSA << 2, 4, (R_XTENSA_32, R_XTENSA_PLT), asm_jump_xtensa, ), "xtensawin": ArchData( "EM_XTENSA", - MP_NATIVE_ARCH_XTENSAWIN << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, - 4, + MP_NATIVE_ARCH_XTENSAWIN << 2, 4, (R_XTENSA_32, R_XTENSA_PLT), asm_jump_xtensa, + separate_rodata=True, ), } @@ -421,10 +430,12 @@ def populate_got(env): # Create a relocation for each GOT entry for got_entry in got_list: - if got_entry.name == "mp_fun_table": - dest = "mp_fun_table" + if got_entry.name in ("mp_native_qstr_table", "mp_native_obj_table", "mp_fun_table"): + dest = got_entry.name elif got_entry.name.startswith("mp_fun_table+0x"): dest = int(got_entry.name.split("+")[1], 16) // env.arch.word_size + elif got_entry.sec_name == ".external.mp_fun_table": + dest = got_entry.sym.mp_fun_table_offset elif got_entry.sec_name.startswith(".text"): dest = ".text" elif got_entry.sec_name.startswith(".rodata"): @@ -564,9 +575,9 @@ def do_relocation_text(env, text_addr, r): reloc = addr - r_offset reloc_type = "xtensa_l32r" - elif env.arch.name == "EM_XTENSA" and r_info_type == R_XTENSA_DIFF32: + elif env.arch.name == "EM_XTENSA" and r_info_type in (R_XTENSA_DIFF32, R_XTENSA_PDIFF32): if s.section.name.startswith(".text"): - # it looks like R_XTENSA_DIFF32 into .text is already correctly relocated + # it looks like R_XTENSA_[P]DIFF32 into .text is already correctly relocated return assert 0 @@ -740,24 +751,26 @@ def link_objects(env, native_qstr_vals_len, native_qstr_objs_len): env.lit_section = Section("LIT", bytearray(lit_size), env.arch.word_size) env.sections.insert(1, env.lit_section) - # Create section to contain mp_native_qstr_val_table - env.qstr_val_section = Section( - ".text.QSTR_VAL", + # Create section to contain mp_native_qstr_table + env.qstr_table_section = Section( + ".external.qstr_table", bytearray(native_qstr_vals_len * env.arch.qstr_entry_size), env.arch.qstr_entry_size, ) env.sections.append(env.qstr_val_section) - # Create section to contain mp_native_qstr_obj_table - env.qstr_obj_section = Section( - ".text.QSTR_OBJ", bytearray(native_qstr_objs_len * env.arch.word_size), env.arch.word_size + # Create section to contain mp_native_obj_table + env.obj_table_section = Section( + ".external.obj_table", + bytearray(native_qstr_objs_len * env.arch.word_size), + env.arch.word_size, ) env.sections.append(env.qstr_obj_section) # Resolve unknown symbols mp_fun_table_sec = Section(".external.mp_fun_table", b"", 0) fun_table = { - key: 68 + idx + key: 67 + idx for idx, key in enumerate( [ "mp_type_type", @@ -782,10 +795,10 @@ def link_objects(env, native_qstr_vals_len, native_qstr_objs_len): pass elif sym.name == "mp_fun_table": sym.section = Section(".external", b"", 0) - elif sym.name == "mp_native_qstr_val_table": - sym.section = env.qstr_val_section - elif sym.name == "mp_native_qstr_obj_table": - sym.section = env.qstr_obj_section + elif sym.name == "mp_native_qstr_table": + sym.section = env.qstr_table_section + elif sym.name == "mp_native_obj_table": + sym.section = env.obj_table_section elif sym.name in env.known_syms: sym.resolved = env.known_syms[sym.name] else: @@ -860,11 +873,12 @@ class MPYOutput: def write_qstr(self, s): if s in qstrutil.static_qstr_list: - self.write_bytes(bytes([0, qstrutil.static_qstr_list.index(s) + 1])) + self.write_uint((qstrutil.static_qstr_list.index(s) + 1) << 1 | 1) else: s = bytes(s, "ascii") self.write_uint(len(s) << 1) self.write_bytes(s) + self.write_bytes(b"\x00") def write_reloc(self, base, offset, dest, n): need_offset = not (base == self.prev_base and offset == self.prev_offset + 1) @@ -905,20 +919,27 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): out.open(fmpy) # MPY: header - out.write_bytes( - bytearray( - [ - ord("C"), - MPY_VERSION, - env.arch.mpy_feature, - MP_SMALL_INT_BITS, - QSTR_WINDOW_SIZE, - ] - ) - ) + out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS])) + + # MPY: n_qstr + out.write_uint(1 + len(native_qstr_vals)) + + # MPY: n_obj + out.write_uint(len(native_qstr_objs)) + + # MPY: qstr table + out.write_qstr(fmpy) # filename + for q in native_qstr_vals: + out.write_qstr(q) + + # MPY: object table + for q in native_qstr_objs: + out.write_bytes(bytearray([MP_PERSISTENT_OBJ_STR])) + out.write_uint(len(q)) + out.write_bytes(bytes(q, "utf8") + b"\x00") # MPY: kind/len - out.write_uint(len(env.full_text) << 2 | (MP_CODE_NATIVE_VIPER - MP_CODE_BYTECODE)) + out.write_uint(len(env.full_text) << 3 | (MP_CODE_NATIVE_VIPER - MP_CODE_BYTECODE)) # MPY: machine code out.write_bytes(env.full_text) @@ -942,20 +963,16 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): scope_flags |= MP_SCOPE_FLAG_VIPERBSS out.write_uint(scope_flags) - # MPY: n_obj - out.write_uint(0) - - # MPY: n_raw_code - out.write_uint(0) - - # MPY: rodata and/or bss + # MPY: bss and/or rodata if len(env.full_rodata): rodata_const_table_idx = 1 out.write_uint(len(env.full_rodata)) out.write_bytes(env.full_rodata) if len(env.full_bss): - bss_const_table_idx = bool(env.full_rodata) + 1 + bss_const_table_idx = 2 out.write_uint(len(env.full_bss)) + if len(env.full_rodata): + out.write_bytes(env.full_rodata) # MPY: relocation information prev_kind = None @@ -969,10 +986,14 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): kind = 0 elif isinstance(kind, str) and kind.startswith(".bss"): kind = bss_const_table_idx - elif kind == "mp_fun_table": + elif kind == "mp_native_qstr_table": kind = 6 + elif kind == "mp_native_obj_table": + kind = 7 + elif kind == "mp_fun_table": + kind = 8 else: - kind = 7 + kind + kind = 9 + kind assert addr % env.arch.word_size == 0, addr offset = addr // env.arch.word_size if kind == prev_kind and base == prev_base and offset == prev_offset + 1: @@ -1014,18 +1035,14 @@ def do_preprocess(args): for i, q in enumerate(static_qstrs): print("#define %s (%u)" % (q, i + 1), file=f) for i, q in enumerate(sorted(qstr_vals)): - print("#define %s (mp_native_qstr_val_table[%d])" % (q, i), file=f) + print("#define %s (mp_native_qstr_table[%d])" % (q, i + 1), file=f) for i, q in enumerate(sorted(qstr_objs)): print( - "#define MP_OBJ_NEW_QSTR_%s ((mp_obj_t)mp_native_qstr_obj_table[%d])" % (q, i), + "#define MP_OBJ_NEW_QSTR_%s ((mp_obj_t)mp_native_obj_table[%d])" % (q, i), file=f, ) - if args.arch == "xtensawin": - qstr_type = "uint32_t" # esp32 can only read 32-bit values from IRAM - else: - qstr_type = "uint16_t" - print("extern const {} mp_native_qstr_val_table[];".format(qstr_type), file=f) - print("extern const mp_uint_t mp_native_qstr_obj_table[];", file=f) + print("extern const uint16_t mp_native_qstr_table[];", file=f) + print("extern const mp_uint_t mp_native_obj_table[];", file=f) def do_link(args): diff --git a/tools/pyboard.py b/tools/pyboard.py index a43a4d0574..d2813ad5ad 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -472,7 +472,10 @@ class Pyboard: ) self.exec_(cmd, data_consumer=stdout_write_bytes) - def fs_get(self, src, dest, chunk_size=256): + def fs_get(self, src, dest, chunk_size=256, progress_callback=None): + if progress_callback: + src_size = int(self.exec_("import os\nprint(os.stat('%s')[6])" % src)) + written = 0 self.exec_("f=open('%s','rb')\nr=f.read" % src) with open(dest, "wb") as f: while True: @@ -488,9 +491,15 @@ class Pyboard: if not data: break f.write(data) + if progress_callback: + written += len(data) + progress_callback(written, src_size) self.exec_("f.close()") - def fs_put(self, src, dest, chunk_size=256): + def fs_put(self, src, dest, chunk_size=256, progress_callback=None): + if progress_callback: + src_size = os.path.getsize(src) + written = 0 self.exec_("f=open('%s','wb')\nw=f.write" % dest) with open(src, "rb") as f: while True: @@ -501,6 +510,9 @@ class Pyboard: self.exec_("w(b" + repr(data) + ")") else: self.exec_("w(" + repr(data) + ")") + if progress_callback: + written += len(data) + progress_callback(written, src_size) self.exec_("f.close()") def fs_mkdir(self, dir): @@ -527,7 +539,7 @@ def execfile(filename, device="/dev/ttyACM0", baudrate=115200, user="micro", pas pyb.close() -def filesystem_command(pyb, args): +def filesystem_command(pyb, args, progress_callback=None): def fname_remote(src): if src.startswith(":"): src = src[1:] @@ -560,7 +572,7 @@ def filesystem_command(pyb, args): src = fname_remote(src) dest2 = fname_cp_dest(src, dest) print(fmt % (src, dest2)) - op(src, dest2) + op(src, dest2, progress_callback=progress_callback) else: op = { "ls": pyb.fs_ls, diff --git a/tools/uf2families.json b/tools/uf2families.json new file mode 100644 index 0000000000..fafae82a60 --- /dev/null +++ b/tools/uf2families.json @@ -0,0 +1,192 @@ +[ + { + "id": "0x16573617", + "short_name": "ATMEGA32", + "description": "Microchip (Atmel) ATmega32" + }, + { + "id": "0x1851780a", + "short_name": "SAML21", + "description": "Microchip (Atmel) SAML21" + }, + { + "id": "0x1b57745f", + "short_name": "NRF52", + "description": "Nordic NRF52" + }, + { + "id": "0x1c5f21b0", + "short_name": "ESP32", + "description": "ESP32" + }, + { + "id": "0x1e1f432d", + "short_name": "STM32L1", + "description": "ST STM32L1xx" + }, + { + "id": "0x202e3a91", + "short_name": "STM32L0", + "description": "ST STM32L0xx" + }, + { + "id": "0x21460ff0", + "short_name": "STM32WL", + "description": "ST STM32WLxx" + }, + { + "id": "0x2abc77ec", + "short_name": "LPC55", + "description": "NXP LPC55xx" + }, + { + "id": "0x300f5633", + "short_name": "STM32G0", + "description": "ST STM32G0xx" + }, + { + "id": "0x31d228c6", + "short_name": "GD32F350", + "description": "GD32F350" + }, + { + "id": "0x04240bdf", + "short_name": "STM32L5", + "description": "ST STM32L5xx" + }, + { + "id": "0x4c71240a", + "short_name": "STM32G4", + "description": "ST STM32G4xx" + }, + { + "id": "0x4fb2d5bd", + "short_name": "MIMXRT10XX", + "description": "NXP i.MX RT10XX" + }, + { + "id": "0x53b80f00", + "short_name": "STM32F7", + "description": "ST STM32F7xx" + }, + { + "id": "0x55114460", + "short_name": "SAMD51", + "description": "Microchip (Atmel) SAMD51" + }, + { + "id": "0x57755a57", + "short_name": "STM32F4", + "description": "ST STM32F401" + }, + { + "id": "0x5a18069b", + "short_name": "FX2", + "description": "Cypress FX2" + }, + { + "id": "0x5d1a0a2e", + "short_name": "STM32F2", + "description": "ST STM32F2xx" + }, + { + "id": "0x5ee21072", + "short_name": "STM32F1", + "description": "ST STM32F103" + }, + { + "id": "0x621e937a", + "short_name": "NRF52833", + "description": "Nordic NRF52833" + }, + { + "id": "0x647824b6", + "short_name": "STM32F0", + "description": "ST STM32F0xx" + }, + { + "id": "0x68ed2b88", + "short_name": "SAMD21", + "description": "Microchip (Atmel) SAMD21" + }, + { + "id": "0x6b846188", + "short_name": "STM32F3", + "description": "ST STM32F3xx" + }, + { + "id": "0x6d0922fa", + "short_name": "STM32F407", + "description": "ST STM32F407" + }, + { + "id": "0x6db66082", + "short_name": "STM32H7", + "description": "ST STM32H7xx" + }, + { + "id": "0x70d16653", + "short_name": "STM32WB", + "description": "ST STM32WBxx" + }, + { + "id": "0x7eab61ed", + "short_name": "ESP8266", + "description": "ESP8266" + }, + { + "id": "0x7f83e793", + "short_name": "KL32L2", + "description": "NXP KL32L2x" + }, + { + "id": "0x8fb060fe", + "short_name": "STM32F407VG", + "description": "ST STM32F407VG" + }, + { + "id": "0xada52840", + "short_name": "NRF52840", + "description": "Nordic NRF52840" + }, + { + "id": "0xbfdd4eee", + "short_name": "ESP32S2", + "description": "ESP32-S2" + }, + { + "id": "0xc47e5767", + "short_name": "ESP32S3", + "description": "ESP32-S3" + }, + { + "id": "0xd42ba06c", + "short_name": "ESP32C3", + "description": "ESP32-C3" + }, + { + "id": "0x2b88d29c", + "short_name": "ESP32C2", + "description": "ESP32-C2" + }, + { + "id": "0x332726f6", + "short_name": "ESP32H2", + "description": "ESP32-H2" + }, + { + "id": "0xe48bff56", + "short_name": "RP2040", + "description": "Raspberry Pi RP2040" + }, + { + "id": "0x00ff6919", + "short_name": "STM32L4", + "description": "ST STM32L4xx" + }, + { + "id": "0x9af03e33", + "short_name": "GD32VF103", + "description": "GigaDevice GD32VF103" + } +] diff --git a/tools/uncrustify.cfg b/tools/uncrustify.cfg index a92530fd93..3812cd3182 100644 --- a/tools/uncrustify.cfg +++ b/tools/uncrustify.cfg @@ -372,7 +372,7 @@ sp_inside_square = ignore # ignore/add/remove/force sp_inside_square_oc_array = ignore # ignore/add/remove/force # Add or remove space after ',', i.e. 'a,b' vs. 'a, b'. -sp_after_comma = ignore # ignore/add/remove/force +sp_after_comma = add # ignore/add/remove/force # Add or remove space before ','. # diff --git a/tools/upip.py b/tools/upip.py index 70afe36a45..a62d04f66b 100644 --- a/tools/upip.py +++ b/tools/upip.py @@ -41,10 +41,6 @@ def op_split(path): return (head, r[1]) -def op_basename(path): - return op_split(path)[1] - - # Expects *file* name def _makedirs(name, mode=0o777): ret = False @@ -205,7 +201,6 @@ def install_pkg(pkg_spec, install_path): assert len(packages) == 1 package_url = packages[0]["url"] print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url)) - package_fname = op_basename(package_url) f1 = url_open(package_url) try: f2 = uzlib.DecompIO(f1, gzdict_sz) diff --git a/tools/verifygitlog.py b/tools/verifygitlog.py index 24171f9c68..09c10c80d6 100755 --- a/tools/verifygitlog.py +++ b/tools/verifygitlog.py @@ -69,7 +69,8 @@ def verify(sha): # Message body lines. for line in raw_body[2:]: - if len(line) >= 76: + # Long lines with URLs are exempt from the line length rule. + if len(line) >= 76 and "://" not in line: error("Message lines should be 75 or less characters: " + line) if not raw_body[-1].startswith("Signed-off-by: ") or "@" not in raw_body[-1]: