diff --git a/.gitmodules b/.gitmodules index 9d14ad6da1..aeadf19b95 100644 --- a/.gitmodules +++ b/.gitmodules @@ -92,9 +92,6 @@ [submodule "frozen/circuitpython-stage"] path = frozen/circuitpython-stage url = https://github.com/python-ugame/circuitpython-stage.git -[submodule "ports/cxd56/spresense-exported-sdk"] - path = ports/cxd56/spresense-exported-sdk - url = https://github.com/sonydevworld/spresense-exported-sdk.git [submodule "frozen/Adafruit_CircuitPython_SD"] path = frozen/Adafruit_CircuitPython_SD url = https://github.com/adafruit/Adafruit_CircuitPython_SD.git diff --git a/LICENSE b/LICENSE index 5b5c37f7d1..2b9a64b89a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013-2021 Damien P. George +Copyright (c) 2013-2022 Damien P. George Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docs/differences/python_35.rst b/docs/differences/python_35.rst new file mode 100644 index 0000000000..84c38c9cc1 --- /dev/null +++ b/docs/differences/python_35.rst @@ -0,0 +1,181 @@ +.. _python_35: + +Python 3.5 +========== + +Below is a list of finalised/accepted PEPs for Python 3.5 grouped into their impact to MicroPython. + + +----------------------------------------------------------------------------------------------------------+---------------+ + | **Extensions to the syntax:** | **Status** | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 448 `_ | additional unpacking generalizations | | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 465 `_ | a new matrix multiplication operator | Completed | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 492 `_ | coroutines with async and await syntax | Completed | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | **Extensions and changes to runtime:** | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 461 `_ | % formatting for binary strings | Completed | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 475 `_ | retrying system calls that fail with EINTR | Completed | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 479 `_ | change StopIteration handling inside generators | Completed | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | **Standard library changes:** | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 471 `_ | os.scandir() | | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 485 `_ | math.isclose(), a function for testing | Completed | + | | approximate equality | | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | **Miscellaneous changes:** | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 441 `_ | improved Python zip application support | | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 486 `_ | make the Python Laucher aware of virtual | | + | | environments | | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 484 `_ | type hints (advisory only) | In Progress | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 488 `_ | elimination of PYO files | Not relevant | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + | `PEP 489 `_ | redesigning extension module loading | | + +--------------------------------------------------------+-------------------------------------------------+---------------+ + + +Other Language Changes: + + +-----------------------------------------------------------------------------------------------------------+---------------+ + | Added the *namereplace* error handlers. The *backslashreplace* error handlers now work with decoding and | | + | translating. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | Property docstrings are now writable. This is especially useful for collections.namedtuple() docstrings | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | Circular imports involving relative imports are now supported. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + + +New Modules: + +* `typing `_ + +* `zipzap `_ + + +Changes to built-in modules: + + +-----------------------------------------------------------------------------------------------------------+---------------+ + | `collections `_ | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The *OrderedDict* class is now implemented in C, which makes it 4 to 100 times faster. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | *OrderedDict.items()* , *OrderedDict.keys()* , *OrderedDict.values()* views now support reversed() | | + | iteration. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The deque class now defines *index()*, *insert()*, and *copy()*, and supports the + and * operators. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | Docstrings produced by namedtuple() can now be updated. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The UserString class now implements the *__getnewargs__()*, *__rmod__()*, *casefold()*, *format_map()*, | | + | *isprintable()*, and *maketrans()* methods to match the corresponding methods of str. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | `heapq `_ | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | Element comparison in *merge()* can now be customized by passing a key function in a new optional key | | + | keyword argument, and a new optional *reverse* keyword argument can be used to reverse element comparison | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | `io `_ | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | A new *BufferedIOBase.readinto1()* method, that uses at most one call to the underlying raw stream's | | + | *RawIOBase.read()* or *RawIOBase.readinto()* methods | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | `json `_ | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | JSON decoder now raises JSONDecodeError instead of ValueError to provide better context information about | | + | the error. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | `math `_ | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | Two new constants have been added to the math module: *inf* and *nan*. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | A new function *isclose()* provides a way to test for approximate equality. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | A new *gcd()* function has been added. The *fractions.gcd()* function is now deprecated. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | `os `_ | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The new *scandir()* function returning an iterator of DirEntry objects has been added. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The *urandom()* function now uses the *getrandom()* syscall on Linux 3.17 or newer, and *getentropy()* on | | + | OpenBSD 5.6 and newer, removing the need to use /dev/urandom and avoiding failures due to potential file | | + | descriptor exhaustion. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | New *get_blocking()* and *set_blocking()* functions allow getting and setting a file descriptor's blocking| | + | mode (O_NONBLOCK.) | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | There is a new *os.path.commonpath()* function returning the longest common sub-path of each passed | | + | pathname | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | `re `_ | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | References and conditional references to groups with fixed length are now allowed in lookbehind assertions| | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The number of capturing groups in regular expressions is no longer limited to 100. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The *sub()* and *subn()* functions now replace unmatched groups with empty strings instead of raising an | | + | exception. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The *re.error* exceptions have new attributes, msg, pattern, pos, lineno, and colno, that provide better | | + | context information about the error | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | `socket `_ | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | Functions with timeouts now use a monotonic clock, instead of a system clock. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | A new *socket.sendfile()* method allows sending a file over a socket by using the high-performance | | + | *os.sendfile()* function on UNIX, resulting in uploads being from 2 to 3 times faster than when using | | + | plain *socket.send()* | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The *socket.sendall()* method no longer resets the socket timeout every time bytes are received or sent. | | + | The socket timeout is now the maximum total duration to send all data. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The backlog argument of the *socket.listen()* method is now optional. By default it is set to SOMAXCONN or| | + | to 128, whichever is less. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | `ssl `_ | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | Memory BIO Support | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | Application-Layer Protocol Negotiation Support | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | There is a new *SSLSocket.version()* method to query the actual protocol version in use. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The SSLSocket class now implements a *SSLSocket.sendfile()* method. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The *SSLSocket.send()* method now raises either the *ssl.SSLWantReadError* or *ssl.SSLWantWriteError* | | + | exception on a non-blocking socket if the operation would block. Previously, it would return 0. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The *cert_time_to_seconds()* function now interprets the input time as UTC and not as local time, per RFC | | + | 5280. Additionally, the return value is always an int. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | New *SSLObject.shared_ciphers()* and *SSLSocket.shared_ciphers()* methods return the list of ciphers sent | | + | by the client during the handshake. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The *SSLSocket.do_handshake()*, *SSLSocket.read()*, *SSLSocket.shutdown()*, and *SSLSocket.write()* | | + | methods of the SSLSocket class no longer reset the socket timeout every time bytes are received or sent. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The *match_hostname()* function now supports matching of IP addresses. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | `sys `_ | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | A new *set_coroutine_wrapper()* function allows setting a global hook that will be called whenever a | | + | coroutine object is created by an async def function. A corresponding *get_coroutine_wrapper()* can be | | + | used to obtain a currently set wrapper. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | A new *is_finalizing()* function can be used to check if the Python interpreter is shutting down. | | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | `time `_ | + +-----------------------------------------------------------------------------------------------------------+---------------+ + | The *monotonic()* function is now always available | | + +-----------------------------------------------------------------------------------------------------------+---------------+ diff --git a/docs/differences/python_36.rst b/docs/differences/python_36.rst new file mode 100644 index 0000000000..4ea8742243 --- /dev/null +++ b/docs/differences/python_36.rst @@ -0,0 +1,191 @@ +.. _python_36: + +Python 3.6 +========== + +Python 3.6 beta 1 was released on 12 Sep 2016, and a summary of the new features can be found here: + + +-----------------------------------------------------------------------------------------------------------+--------------+ + | **New Syntax Features:** | **Status** | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 498 `_ | Literal String Formatting | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 515 `_ | Underscores in Numeric Literals | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 525 `_ | Asynchronous Generators | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 526 `_ | Syntax for Variable Annotations (provisional) | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 530 `_ | Asynchronous Comprehensions | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | **New Built-in Features:** | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 468 `_ | Preserving the order of *kwargs* in a function | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 487 `_ | Simpler customization of class creation | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 520 `_ | Preserving Class Attribute Definition Order | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | **Standard Library Changes:** | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 495 `_ | Local Time Disambiguation | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 506 `_ | Adding A Secrets Module To The Standard Library | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 519 `_ | Adding a file system path protocol | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | **CPython internals:** | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 509 `_ | Add a private version to dict | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 523 `_ | Adding a frame evaluation API to CPython | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | **Linux/Window Changes** | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 524 `_ | Make os.urandom() blocking on Linux | | + | | (during system startup) | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 528 `_ | Change Windows console encoding to UTF-8 | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + | `PEP 529 `_ | Change Windows filesystem encoding to UTF-8 | | + +--------------------------------------------------------+--------------------------------------------------+--------------+ + +Other Language Changes: + + +-------------------------------------------------------------------------------------------------------------+---------------+ + | A *global* or *nonlocal* statement must now textually appear before the first use of the affected name in | | + | the same scope. Previously this was a SyntaxWarning. | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + | It is now possible to set a special method to None to indicate that the corresponding operation is not | | + | available. For example, if a class sets *__iter__()* to *None* , the class is not iterable. | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + | Long sequences of repeated traceback lines are now abbreviated as *[Previous line repeated {count} more | | + | times]* | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + | Import now raises the new exception *ModuleNotFoundError* when it cannot find a module. Code that currently | | + | checks for ImportError (in try-except) will still work. | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + | Class methods relying on zero-argument *super()* will now work correctly when called from metaclass methods | | + | during class creation. | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + +Changes to built-in modules: + + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `array `_ | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | Exhausted iterators of *array.array* will now stay exhausted even if the iterated array is extended. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `binascii `_ | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The b2a_base64() function now accepts an optional newline keyword argument to control whether the newline | | + | character is appended to the return value | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `cmath `_ | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The new cmath.tau (τ) constant has been added | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | New constants: *cmath.inf* and *cmath.nan* to match *math.inf* and *math.nan* , and also *cmath.infj* and | | + | *cmath.nanj* to match the format used by complex repr | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `collections `_ | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The new Collection abstract base class has been added to represent sized iterable container classes | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The new *Reversible* abstract base class represents iterable classes that also provide the *__reversed__()* | | + | method. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The new *AsyncGenerator* abstract base class represents asynchronous generators. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The *namedtuple()* function now accepts an optional keyword argument module, which, when specified, is used | | + | for the *__module__* attribute of the returned named tuple class. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The verbose and rename arguments for *namedtuple()* are now keyword-only. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | Recursive *collections.deque* instances can now be pickled. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `hashlib `_ | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | BLAKE2 hash functions were added to the module. *blake2b()* and *blake2s()* are always available and support | | + | the full feature set of BLAKE2. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The SHA-3 hash functions *sha3_224()*, *sha3_256()*, *sha3_384()*, *sha3_512()*, and *SHAKE* hash functions | | + | *shake_128()* and *shake_256()* were added. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The password-based key derivation function *scrypt()* is now available with OpenSSL 1.1.0 and newer. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `json `_ | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | *json.load()* and *json.loads()* now support binary input. Encoded JSON should be represented using either | | + | UTF-8, UTF-16, or UTF-32. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `math `_ | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The new math.tau (τ) constant has been added | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `os `_ | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | A new *close()* method allows explicitly closing a *scandir()* iterator. The *scandir()* iterator now | | + | supports the context manager protocol. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | On Linux, *os.urandom()* now blocks until the system urandom entropy pool is initialized to increase the | | + | security. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The Linux *getrandom()* syscall (get random bytes) is now exposed as the new *os.getrandom()* function. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `re `_ | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | Added support of modifier spans in regular expressions. Examples: *'(?i:p)ython'* matches 'python' and | | + | 'Python', but not 'PYTHON'; *'(?i)g(?-i:v)r'* matches *'GvR'* and *'gvr'*, but not *'GVR'* . | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | Match object groups can be accessed by *__getitem__*, which is equivalent to *group()*. So *mo['name']* is | | + | now equivalent to *mo.group('name')*. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | Match objects now support index-like objects as group indices. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `socket `_ | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The *ioctl()* function now supports the *SIO_LOOPBACK_FAST_PATH* control code. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The *getsockopt()* constants *SO_DOMAIN* , *SO_PROTOCOL*, *SO_PEERSEC* , and *SO_PASSSEC* are now supported. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The *setsockopt()* now supports the *setsockopt(level, optname, None, optlen: int)* form. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The socket module now supports the address family *AF_ALG* to interface with Linux Kernel crypto API. | | + | *ALG_*, *SOL_ALG* and *sendmsg_afalg()* were added. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | New Linux constants *TCP_USER_TIMEOUT* and *TCP_CONGESTION* were added. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `ssl `_ | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | ssl supports OpenSSL 1.1.0. The minimum recommend version is 1.0.2. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | 3DES has been removed from the default cipher suites and ChaCha20 Poly1305 cipher suites have been added. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | *SSLContext* has better default configuration for options and ciphers. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | SSL session can be copied from one client-side connection to another with the new *SSLSession* class. TLS | | + | session resumption can speed up the initial handshake, reduce latency and improve performance. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The new *get_ciphers()* method can be used to get a list of enabled ciphers in order of cipher priority. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | All constants and flags have been converted to *IntEnum* and *IntFlags*. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | Server and client-side specific TLS protocols for *SSLContext* were added. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | Added *SSLContext.post_handshake_auth* to enable and *ssl.SSLSocket.verify_client_post_handshake()* to | | + | initiate TLS 1.3 post-handshake authentication. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `struct `_ | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | now supports IEEE 754 half-precision floats via the 'e' format specifier. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `sys `_ | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The new *getfilesystemencodeerrors()* function returns the name of the error mode used to convert between | | + | Unicode filenames and bytes filenames. | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | `zlib `_ | | + +--------------------------------------------------------------------------------------------------------------+----------------+ + | The *compress()* and *decompress()* functions now accept keyword arguments | | + +--------------------------------------------------------------------------------------------------------------+----------------+ diff --git a/docs/differences/python_37.rst b/docs/differences/python_37.rst new file mode 100644 index 0000000000..c46678e931 --- /dev/null +++ b/docs/differences/python_37.rst @@ -0,0 +1,95 @@ +.. _python_37: + +Python 3.7 +========== + +New Features: + + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | **Features:** | **Status** | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 538 `_ | Coercing the legacy C locale to a UTF-8 based | | + | | locale | | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 539 `_ | A New C-API for Thread-Local Storage in CPython | | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 540 `_ | UTF-8 mode | | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 552 `_ | Deterministic pyc | | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 553 `_ | Built-in breakpoint() | | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 557 `_ | Data Classes | | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 560 `_ | Core support for typing module and generic types | | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 562 `_ | Module __getattr__ and __dir__ | Partially done | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 563 `_ | Postponed Evaluation of Annotations | | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 564 `_ | Time functions with nanosecond resolution | | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 565 `_ | Show DeprecationWarning in __main__ | | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + | `PEP 567 `_ | Context Variables | | + +--------------------------------------------------------+--------------------------------------------------+----------------+ + +Other Language Changes: + + +----------------------------------------------------------------------------------------------------------+----------------+ + | async and await are now reserved keywords | Completed | + +----------------------------------------------------------------------------------------------------------+----------------+ + | dict objects must preserve insertion-order | | + +----------------------------------------------------------------------------------------------------------+----------------+ + | More than 255 arguments can now be passed to a function; a function can now have more than 255 parameters| | + +----------------------------------------------------------------------------------------------------------+----------------+ + | bytes.fromhex() and bytearray.fromhex() now ignore all ASCII whitespace, not only spaces | | + +----------------------------------------------------------------------------------------------------------+----------------+ + | str, bytes, and bytearray gained support for the new isascii() method, which can be used to test if a | | + | string or bytes contain only the ASCII characters | | + +----------------------------------------------------------------------------------------------------------+----------------+ + | ImportError now displays module name and module __file__ path whenfrom ... import ... fails | | + +----------------------------------------------------------------------------------------------------------+----------------+ + | Circular imports involving absolute imports with binding a submodule to a name are now supported | | + +----------------------------------------------------------------------------------------------------------+----------------+ + | object.__format__(x, '') is now equivalent to str(x) rather than format(str(self), '') | | + +----------------------------------------------------------------------------------------------------------+----------------+ + | In order to better support dynamic creation of stack traces, types.TracebackType can now be instantiated | | + | from Python code, and the tb_next attribute on tracebacks is now writable | | + +----------------------------------------------------------------------------------------------------------+----------------+ + | When using the -m switch, sys.path[0] is now eagerly expanded to the full starting directory path, rather| | + | than being left as the empty directory (which allows imports from the current working directory at the | | + | time when an import occurs) | | + +----------------------------------------------------------------------------------------------------------+----------------+ + | The new -X importtime option or the PYTHONPROFILEIMPORTTIME environment variable can be used to show the | | + | timing of each module import | | + +----------------------------------------------------------------------------------------------------------+----------------+ + +Changes to built-in modules: + + +------------------------------------------------------------------------------------------------------------+----------------+ + | `asyncio `_ | | + +------------------------------------------------------------------------------------------------------------+----------------+ + | asyncio (many, may need a separate ticket) | | + +------------------------------------------------------------------------------------------------------------+----------------+ + | `gc `_ | | + +------------------------------------------------------------------------------------------------------------+----------------+ + | New features include *gc.freeze()*, *gc.unfreeze()*, *gc-get_freeze_count* | | + +------------------------------------------------------------------------------------------------------------+----------------+ + | `math `_ | | + +------------------------------------------------------------------------------------------------------------+----------------+ + | math.remainder() added to implement IEEE 754-style remainder | | + +------------------------------------------------------------------------------------------------------------+----------------+ + | `re `_ | | + +------------------------------------------------------------------------------------------------------------+----------------+ + | A number of tidy up features including better support for splitting on empty strings and copy support for | | + | compiled expressions and match objects | | + +------------------------------------------------------------------------------------------------------------+----------------+ + | `sys `_ | | + +------------------------------------------------------------------------------------------------------------+----------------+ + | sys.breakpointhook() added. sys.get(/set)_coroutine_origin_tracking_depth() added | | + +------------------------------------------------------------------------------------------------------------+----------------+ + | `time `_ | | + +------------------------------------------------------------------------------------------------------------+----------------+ + | Mostly updates to support nanosecond resolution in PEP564, see above | | + +------------------------------------------------------------------------------------------------------------+----------------+ diff --git a/docs/differences/python_38.rst b/docs/differences/python_38.rst new file mode 100644 index 0000000000..47840a8b40 --- /dev/null +++ b/docs/differences/python_38.rst @@ -0,0 +1,118 @@ +.. _python_38: + +Python 3.8 +========== + +Python 3.8.0 (final) was released on the 14 October 2019. The Features for 3.8 +are defined in `PEP 569 `_ and +a detailed description of the changes can be found in What's New in `Python +3.8. `_ + + +--------------------------------------------------------+---------------------------------------------------+---------------+ + | **Features:** | Status | + +--------------------------------------------------------+---------------------------------------------------+---------------+ + | `PEP 570 `_ | Positional-only arguments | | + +--------------------------------------------------------+---------------------------------------------------+---------------+ + | `PEP 572 `_ | Assignment Expressions | | + +--------------------------------------------------------+---------------------------------------------------+---------------+ + | `PEP 574 `_ | Pickle protocol 5 with out-of-band data | | + +--------------------------------------------------------+---------------------------------------------------+---------------+ + | `PEP 578 `_ | Runtime audit hooks | | + +--------------------------------------------------------+---------------------------------------------------+---------------+ + | `PEP 587 `_ | Python Initialization Configuration | | + +--------------------------------------------------------+---------------------------------------------------+---------------+ + | `PEP 590 `_ | Vectorcall: a fast calling protocol for CPython | | + +--------------------------------------------------------+---------------------------------------------------+---------------+ + | **Miscellaneous** | + +------------------------------------------------------------------------------------------------------------+---------------+ + | f-strings support = for self-documenting expressions and debugging | Completed | + +------------------------------------------------------------------------------------------------------------+---------------+ + +Other Language Changes: + + +------------------------------------------------------------------------------------------------------------+-------------+ + | A *continue* statement was illegal in the *finally* clause due to a problem with the implementation. In | Completed | + | Python 3.8 this restriction was lifted | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | The *bool*, *int* , and *fractions.Fraction* types now have an *as_integer_ratio()* method like that found | | + | in *float* and *decimal.Decimal* | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Constructors of *int*, *float* and *complex* will now use the *__index__()* special method, if available | | + | and the corresponding method *__int__()*, *__float__()* or *__complex__()* is not available | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Added support of *\N{name}* escapes in regular expressions | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Dict and dictviews are now iterable in reversed insertion order using *reversed()* | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | The syntax allowed for keyword names in function calls was further restricted. In particular, | | + | f((keyword)=arg) is no longer allowed | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Generalized iterable unpacking in yield and return statements no longer requires enclosing parentheses | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | When a comma is missed in code such as [(10, 20) (30, 40)], the compiler displays a SyntaxWarning with a | | + | helpful suggestion | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Arithmetic operations between subclasses of *datetime.date* or *datetime.datetime* and *datetime.timedelta*| | + | objects now return an instance of the subclass, rather than the base class | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | When the Python interpreter is interrupted by *Ctrl-C (SIGINT)* and the resulting *KeyboardInterrupt* | | + | exception is not caught, the Python process now exits via a SIGINT signal or with the correct exit code | | + | such that the calling process can detect that it died due to a *Ctrl-C* | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Some advanced styles of programming require updating the *types.CodeType* object for an existing function | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | For integers, the three-argument form of the pow() function now permits the exponent to be negative in the | | + | case where the base is relatively prime to the modulus | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Dict comprehensions have been synced-up with dict literals so that the key is computed first and the value | | + | second | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | The *object.__reduce__()* method can now return a tuple from two to six elements long | | + +------------------------------------------------------------------------------------------------------------+-------------+ + +Changes to built-in modules: + + +------------------------------------------------------------------------------------------------------------+-------------+ + | `asyncio` | + +------------------------------------------------------------------------------------------------------------+-------------+ + | *asyncio.run()* has graduated from the provisional to stable API | Completed | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Running *python -m asyncio* launches a natively async REPL | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | The exception *asyncio.CancelledError* now inherits from *BaseException* rather than *Exception* and no | Completed | + | longer inherits from *concurrent.futures.CancelledError* | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Added *asyncio.Task.get_coro()* for getting the wrapped coroutine within an *asyncio.Task* | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Asyncio tasks can now be named, either by passing the name keyword argument to *asyncio.create_task()* or | | + | the *create_task()* event loop method, or by calling the *set_name()* method on the task object | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Added support for Happy Eyeballs to *asyncio.loop.create_connection()*. To specify the behavior, two new | | + | parameters have been added: *happy_eyeballs_delay* and interleave. | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | `gc` | + +------------------------------------------------------------------------------------------------------------+-------------+ + | *get_objects()* can now receive an optional generation parameter indicating a generation to get objects | | + | from. (Note, though, that while *gc* is a built-in, *get_objects()* is not implemented for MicroPython) | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | `math` | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Added new function *math.dist()* for computing Euclidean distance between two points | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Expanded the *math.hypot()* function to handle multiple dimensions | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Added new function, *math.prod()*, as analogous function to *sum()* that returns the product of a "start" | | + | value (default: 1) times an iterable of numbers | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Added two new combinatoric functions *math.perm()* and *math.comb()* | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Added a new function *math.isqrt()* for computing accurate integer square roots without conversion to | | + | floating point | | + +------------------------------------------------------------------------------------------------------------+-------------+ + | The function *math.factorial()* no longer accepts arguments that are not int-like | Completed | + +------------------------------------------------------------------------------------------------------------+-------------+ + | `sys` | + +------------------------------------------------------------------------------------------------------------+-------------+ + | Add new *sys.unraisablehook()* function which can be overridden to control how "unraisable exceptions" | | + | are handled | | + +------------------------------------------------------------------------------------------------------------+-------------+ diff --git a/docs/differences/python_39.rst b/docs/differences/python_39.rst new file mode 100644 index 0000000000..6852dd635e --- /dev/null +++ b/docs/differences/python_39.rst @@ -0,0 +1,121 @@ +.. _python_39: + +Python 3.9 +========== + +Python 3.9.0 (final) was released on the 5th October 2020. The Features for 3.9 are +defined in `PEP 596 `_ +and a detailed description of the changes can be found in +`What's New in Python 3.9 `_ + + +--------------------------------------------------------+----------------------------------------------------+--------------+ + | **Features:** | | **Status** | + +--------------------------------------------------------+----------------------------------------------------+--------------+ + | `PEP 573 `_ | fast access to module state from methods of C | | + | | extension types | | + +--------------------------------------------------------+----------------------------------------------------+--------------+ + | `PEP 584 `_ | union operators added to dict | | + +--------------------------------------------------------+----------------------------------------------------+--------------+ + | `PEP 585 `_ | type hinting generics in standard collections | | + +--------------------------------------------------------+----------------------------------------------------+--------------+ + | `PEP 593 `_ | flexible function and variable annotations | | + +--------------------------------------------------------+----------------------------------------------------+--------------+ + | `PEP 602 `_ | CPython adopts an annual release cycle. Instead of | | + | | annual, aiming for two month release cycle | | + +--------------------------------------------------------+----------------------------------------------------+--------------+ + | `PEP 614 `_ | relaxed grammar restrictions on decorators | | + +--------------------------------------------------------+----------------------------------------------------+--------------+ + | `PEP 615 `_ | the IANA Time Zone Database is now present in the | | + | | standard library in the zoneinfo module | | + +--------------------------------------------------------+----------------------------------------------------+--------------+ + | `PEP 616 `_ | string methods to remove prefixes and suffixes | | + +--------------------------------------------------------+----------------------------------------------------+--------------+ + | `PEP 617 `_ | CPython now uses a new parser based on PEG | | + +--------------------------------------------------------+----------------------------------------------------+--------------+ + +Other Language Changes: + + +-------------------------------------------------------------------------------------------------------------+---------------+ + | *__import__()* now raises *ImportError* instead of *ValueError* | Completed | + +-------------------------------------------------------------------------------------------------------------+---------------+ + | Python now gets the absolute path of the script filename specified on the command line (ex: *python3* | | + | *script.py*): the *__file__* attribute of the *__main__* module became an absolute path, rather than a | | + | relative path | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + | By default, for best performance, the errors argument is only checked at the first encoding/decoding error | | + | and the encoding argument is sometimes ignored for empty strings | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + | *"".replace("", s, n)* now returns *s* instead of an empty string for all non-zero n. It is now consistent | | + | with *"".replace("", s)* | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + | Any valid expression can now be used as a decorator. Previously, the grammar was much more restrictive | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + | Parallel running of *aclose()* / *asend()* / *athrow()* is now prohibited, and *ag_running* now reflects | | + | the actual running status of the async generator | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + | Unexpected errors in calling the *__iter__* method are no longer masked by TypeError in the in operator and | | + | functions contains(), indexOf() and countOf() of the operator module | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + | Unparenthesized lambda expressions can no longer be the expression part in an if clause in comprehensions | | + | and generator expressions | | + +-------------------------------------------------------------------------------------------------------------+---------------+ + +Changes to built-in modules: + + +---------------------------------------------------------------------------------------------------------------+---------------+ + | `asyncio` | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Due to significant security concerns, the reuse_address parameter of *asyncio.loop.create_datagram_endpoint()*| | + | is no longer supported | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Added a new coroutine *shutdown_default_executor()* that schedules a shutdown for the default executor that | | + | waits on the *ThreadPoolExecutor* to finish closing. Also, *asyncio.run()* has been updated to use the new | | + | coroutine. | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Added *asyncio.PidfdChildWatcher*, a Linux-specific child watcher implementation that polls process file | | + | descriptors | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | added a new *coroutine asyncio.to_thread()* | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | When cancelling the task due to a timeout, *asyncio.wait_for()* will now wait until the cancellation is | | + | complete also in the case when timeout is <= 0, like it does with positive timeouts | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | *asyncio* now raises *TyperError* when calling incompatible methods with an *ssl.SSLSocket* socket | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | `gc` | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Garbage collection does not block on resurrected objects | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Added a new function *gc.is_finalized()* to check if an object has been finalized by the garbage collector | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | `math` | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Expanded the *math.gcd()* function to handle multiple arguments. Formerly, it only supported two arguments | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Added *math.lcm()*: return the least common multiple of specified arguments | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Added *math.nextafter()*: return the next floating-point value after x towards y | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Added *math.ulp()*: return the value of the least significant bit of a float | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | `os` | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Exposed the Linux-specific *os.pidfd_open()* and *os.P_PIDFD* | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | The *os.unsetenv()* function is now also available on Windows | Completed | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | The *os.putenv()* and *os.unsetenv()* functions are now always available | Completed | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Added *os.waitstatus_to_exitcode()* function: convert a wait status to an exit code | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | `random` | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Added a new *random.Random.randbytes* method: generate random bytes | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | `sys` | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Added a new *sys.platlibdir* attribute: name of the platform-specific library directory | | + +---------------------------------------------------------------------------------------------------------------+---------------+ + | Previously, *sys.stderr* was block-buffered when non-interactive. Now stderr defaults to always being | | + | line-buffered | | + +---------------------------------------------------------------------------------------------------------------+---------------+ diff --git a/docs/library/framebuf.rst b/docs/library/framebuf.rst index 42b5c9e732..421066b82d 100644 --- a/docs/library/framebuf.rst +++ b/docs/library/framebuf.rst @@ -21,11 +21,11 @@ For example:: import framebuf # FrameBuffer needs 2 bytes for every RGB565 pixel - fbuf = framebuf.FrameBuffer(bytearray(10 * 100 * 2), 10, 100, framebuf.RGB565) + fbuf = framebuf.FrameBuffer(bytearray(100 * 10 * 2), 100, 10, framebuf.RGB565) fbuf.fill(0) fbuf.text('MicroPython!', 0, 0, 0xffff) - fbuf.hline(0, 10, 96, 0xffff) + fbuf.hline(0, 9, 96, 0xffff) Constructors ------------ diff --git a/docs/library/random.rst b/docs/library/random.rst new file mode 100644 index 0000000000..dd8b47c80f --- /dev/null +++ b/docs/library/random.rst @@ -0,0 +1,82 @@ +:mod:`random` -- generate random numbers +======================================== + +.. module:: random + :synopsis: random numbers + +This module implements a pseudo-random number generator (PRNG). + +|see_cpython_module| :mod:`python:random` . + +.. note:: + + The following notation is used for intervals: + + - () are open interval brackets and do not include their endpoints. + For example, (0, 1) means greater than 0 and less than 1. + In set notation: (0, 1) = {x | 0 < x < 1}. + + - [] are closed interval brackets which include all their limit points. + For example, [0, 1] means greater than or equal to 0 and less than + or equal to 1. + In set notation: [0, 1] = {x | 0 <= x <= 1}. + +.. note:: + + The :func:`randrange`, :func:`randint` and :func:`choice` functions are only + available if the ``MICROPY_PY_URANDOM_EXTRA_FUNCS`` configuration option is + enabled. + + +Functions for integers +---------------------- + +.. function:: getrandbits(n) + + Return an integer with *n* random bits (0 <= n <= 32). + +.. function:: randint(a, b) + + Return a random integer in the range [*a*, *b*]. + +.. function:: randrange(stop) + randrange(start, stop) + randrange(start, stop[, step]) + + The first form returns a random integer from the range [0, *stop*). + The second form returns a random integer from the range [*start*, *stop*). + The third form returns a random integer from the range [*start*, *stop*) in + steps of *step*. For instance, calling ``randrange(1, 10, 2)`` will + return odd numbers between 1 and 9 inclusive. + + +Functions for floats +-------------------- + +.. function:: random() + + Return a random floating point number in the range [0.0, 1.0). + +.. function:: uniform(a, b) + + Return a random floating point number N such that *a* <= N <= *b* for *a* <= *b*, + and *b* <= N <= *a* for *b* < *a*. + + +Other Functions +--------------- + +.. function:: seed(n=None, /) + + Initialise the random number generator module with the seed *n* which should + be an integer. When no argument (or ``None``) is passed in it will (if + supported by the port) initialise the PRNG with a true random number + (usually a hardware generated random number). + + The ``None`` case only works if ``MICROPY_PY_URANDOM_SEED_INIT_FUNC`` is + enabled by the port, otherwise it raises ``ValueError``. + +.. function:: choice(sequence) + + Chooses and returns one item at random from *sequence* (tuple, list or + any object that supports the subscript operation). diff --git a/docs/library/stm.rst b/docs/library/stm.rst new file mode 100644 index 0000000000..a181d6044c --- /dev/null +++ b/docs/library/stm.rst @@ -0,0 +1,104 @@ +.. currentmodule:: stm + +:mod:`stm` --- functionality specific to STM32 MCUs +=================================================== + +.. module:: stm + :synopsis: functionality specific to STM32 MCUs + +This module provides functionality specific to STM32 microcontrollers, including +direct access to peripheral registers. + +Memory access +------------- + +The module exposes three objects used for raw memory access. + +.. data:: mem8 + + Read/write 8 bits of memory. + +.. data:: mem16 + + Read/write 16 bits of memory. + +.. data:: mem32 + + Read/write 32 bits of memory. + +Use subscript notation ``[...]`` to index these objects with the address of +interest. + +These memory objects can be used in combination with the peripheral register +constants to read and write registers of the MCU hardware peripherals, as well +as all other areas of address space. + + +Peripheral register constants +----------------------------- + +The module defines constants for registers which are generated from CMSIS header +files, and the constants available depend on the microcontroller series that is +being compiled for. Examples of some constants include: + +.. data:: GPIOA + + Base address of the GPIOA peripheral. + +.. data:: GPIOB + + Base address of the GPIOB peripheral. + +.. data:: GPIO_BSRR + + Offset of the GPIO bit set/reset register. + +.. data:: GPIO_IDR + + Offset of the GPIO input data register. + +.. data:: GPIO_ODR + + Offset of the GPIO output data register. + +Constants that are named after a peripheral, like ``GPIOA``, are the absolute +address of that peripheral. Constants that have a prefix which is the name of a +peripheral, like ``GPIO_BSRR``, are relative offsets of the register. Accessing +peripheral registers requires adding the absolute base address of the peripheral +and the relative register offset. For example ``GPIOA + GPIO_BSRR`` is the +full, absolute address of the ``GPIOA->BSRR`` register. + +Example use: + +.. code-block:: python3 + + # set PA2 high + stm.mem32[stm.GPIOA + stm.GPIO_BSRR] = 1 << 2 + + # read PA3 + value = (stm.mem32[stm.GPIOA + stm.GPIO_IDR] >> 3) & 1 + + +Functions specific to STM32WBxx MCUs +------------------------------------ + +These functions are available on STM32WBxx microcontrollers, and interact with +the second CPU, the RF core. + +.. function:: rfcore_status() + + Returns the status of the second CPU as an integer (the first word of device + info table). + +.. function:: rfcore_fw_version(id) + + Get the version of the firmware running on the second CPU. Pass in 0 for + *id* to get the FUS version, and 1 to get the WS version. + + Returns a 5-tuple with the full version number. + +.. function:: rfcore_sys_hci(ogf, ocf, data, timeout_ms=0) + + Execute a HCI command on the SYS channel. The execution is synchronous. + + Returns a bytes object with the result of the SYS command. diff --git a/docs/library/sys.rst b/docs/library/sys.rst index aad01edbb2..a0675f0f50 100644 --- a/docs/library/sys.rst +++ b/docs/library/sys.rst @@ -80,6 +80,14 @@ Constants A mutable list of directories to search for imported modules. + .. admonition:: Difference to CPython + :class: attention + + On MicroPython, an entry with the value ``".frozen"`` will indicate that import + should search :term:`frozen modules ` at that point in the search. + If no frozen module is found then search will *not* look for a directory called + ``.frozen``, instead it will continue with the next entry in ``sys.path``. + .. data:: platform The platform that CircuitPython is running on. For OS/RTOS ports, this is diff --git a/docs/reference/glossary.rst b/docs/reference/glossary.rst deleted file mode 100644 index 5aae70af0a..0000000000 --- a/docs/reference/glossary.rst +++ /dev/null @@ -1,175 +0,0 @@ -Glossary -======== - -.. glossary:: - - baremetal - A system without a (full-fledged) operating system, for example an - :term:`MCU`-based system. When running on a baremetal system, - MicroPython effectively functions like a small operating system, - running user programs and providing a command interpreter - (:term:`REPL`). - - buffer protocol - Any Python object that can be automatically converted into bytes, such - as ``bytes``, ``bytearray``, ``memoryview`` and ``str`` objects, which - all implement the "buffer protocol". - - board - Typically this refers to a printed circuit board (PCB) containing a - :term:`microcontroller ` and supporting components. - MicroPython firmware is typically provided per-board, as the firmware - contains both MCU-specific functionality but also board-level - functionality such as drivers or pin names. - - bytecode - A compact representation of a Python program that generated by - compiling the Python source code. This is what the VM actually - executes. Bytecode is typically generated automatically at runtime and - is invisible to the user. Note that while :term:`CPython` and - MicroPython both use bytecode, the format is different. You can also - pre-compile source code offline using the :term:`cross-compiler`. - - callee-owned tuple - This is a MicroPython-specific construct where, for efficiency - reasons, some built-in functions or methods may re-use the same - underlying tuple object to return data. This avoids having to allocate - a new tuple for every call, and reduces heap fragmentation. Programs - should not hold references to callee-owned tuples and instead only - extract data from them (or make a copy). - - CircuitPython - A variant of MicroPython developed by `Adafruit Industries - `_. - - CPython - CPython is the reference implementation of the Python programming - language, and the most well-known one. It is, however, one of many - implementations (including Jython, IronPython, PyPy, and MicroPython). - While MicroPython's implementation differs substantially from CPython, - it aims to maintain as much compatibility as possible. - - cross-compiler - Also known as ``mpy-cross``. This tool runs on your PC and converts a - :term:`.py file` containing MicroPython code into a :term:`.mpy file` - containing MicroPython bytecode. This means it loads faster (the board - doesn't have to compile the code), and uses less space on flash (the - bytecode is more space efficient). - - driver - A MicroPython library that implements support for a particular - component, such as a sensor or display. - - FFI - Acronym for Foreign Function Interface. A mechanism used by the - :term:`MicroPython Unix port` to access operating system functionality. - This is not available on :term:`baremetal` ports. - - filesystem - Most MicroPython ports and boards provide a filesystem stored in flash - that is available to user code via the standard Python file APIs such - as ``open()``. Some boards also make this internal filesystem - accessible to the host via USB mass-storage. - - frozen module - A Python module that has been cross compiled and bundled into the - firmware image. This reduces RAM requirements as the code is executed - directly from flash. - - Garbage Collector - A background process that runs in Python (and MicroPython) to reclaim - unused memory in the :term:`heap`. - - GPIO - General-purpose input/output. The simplest means to control electrical - signals (commonly referred to as "pins") on a microcontroller. GPIO - typically allows pins to be either input or output, and to set or get - their digital value (logical "0" or "1"). MicroPython abstracts GPIO - access using the :class:`machine.Pin` and :class:`machine.Signal` - classes. - - GPIO port - A group of :term:`GPIO` pins, usually based on hardware properties of - these pins (e.g. controllable by the same register). - - heap - A region of RAM where MicroPython stores dynamic data. It is managed - automatically by the :term:`Garbage Collector`. Different MCUs and - boards have vastly different amounts of RAM available for the heap, so - this will affect how complex your program can be. - - interned string - An optimisation used by MicroPython to improve the efficiency of - working with strings. An interned string is referenced by its (unique) - identity rather than its address and can therefore be quickly compared - just by its identifier. It also means that identical strings can be - de-duplicated in memory. String interning is almost always invisible to - the user. - - MCU - Microcontroller. Microcontrollers usually have much less resources - than a desktop, laptop, or phone, but are smaller, cheaper and - require much less power. MicroPython is designed to be small and - optimized enough to run on an average modern microcontroller. - - MicroPython port - MicroPython supports different :term:`boards `, RTOSes, and - OSes, and can be relatively easily adapted to new systems. MicroPython - with support for a particular system is called a "port" to that - system. Different ports may have widely different functionality. This - documentation is intended to be a reference of the generic APIs - available across different ports ("MicroPython core"). Note that some - ports may still omit some APIs described here (e.g. due to resource - constraints). Any such differences, and port-specific extensions - beyond the MicroPython core functionality, would be described in the - separate port-specific documentation. - - MicroPython Unix port - The unix port is one of the major :term:`MicroPython ports - `. It is intended to run on POSIX-compatible - operating systems, like Linux, MacOS, FreeBSD, Solaris, etc. It also - serves as the basis of Windows port. The Unix port is very useful for - quick development and testing of the MicroPython language and - machine-independent features. It can also function in a similar way to - :term:`CPython`'s ``python`` executable. - - .mpy file - The output of the :term:`cross-compiler`. A compiled form of a - :term:`.py file` that contains MicroPython bytecode instead of Python - source code. - - native - Usually refers to "native code", i.e. machine code for the target - microcontroller (such as ARM Thumb, Xtensa, x86/x64). The ``@native`` - decorator can be applied to a MicroPython function to generate native - code instead of bytecode for that function, which will likely be - faster but use more RAM. - - port - Usually short for :term:`MicroPython port`, but could also refer to - :term:`GPIO port`. - - .py file - A file containing Python source code. - - REPL - An acronym for "Read, Eval, Print, Loop". This is the interactive - Python prompt, useful for debugging or testing short snippets of code. - Most MicroPython boards make a REPL available over a UART, and this is - typically accessible on a host PC via USB. - - stream - Also known as a "file-like object". An Python object which provides - sequential read-write access to the underlying data. A stream object - implements a corresponding interface, which consists of methods like - ``read()``, ``write()``, ``readinto()``, ``seek()``, ``flush()``, - ``close()``, etc. A stream is an important concept in MicroPython; - many I/O objects implement the stream interface, and thus can be used - consistently and interchangeably in different contexts. For more - information on streams in MicroPython, see the `io` module. - - UART - Acronym for "Universal Asynchronous Receiver/Transmitter". This is a - peripheral that sends data over a pair of pins (TX & RX). Many boards - include a way to make at least one of the UARTs available to a host PC - as a serial port over USB. diff --git a/docs/rp2/tutorial/pio.rst b/docs/rp2/tutorial/pio.rst new file mode 100644 index 0000000000..9981aed832 --- /dev/null +++ b/docs/rp2/tutorial/pio.rst @@ -0,0 +1,123 @@ +Programmable IO +=============== + +The RP2040 has hardware support for standard communication protocols like I2C, +SPI and UART. For protocols where there is no hardware support, or where there +is a requirement of custom I/O behaviour, Programmable Input Output (PIO) comes +into play. Also, some MicroPython applications make use of a technique called +bit banging in which pins are rapidly turned on and off to transmit data. This +can make the entire process slow as the processor concentrates on bit banging +rather than executing other logic. However, PIO allows bit banging to happen +in the background while the CPU is executing the main work. + +Along with the two central Cortex-M0+ processing cores, the RP2040 has two PIO +blocks each of which has four independent state machines. These state machines +can transfer data to/from other entities using First-In-First-Out (FIFO) buffers, +which allow the state machine and main processor to work independently yet also +synchronise their data. Each FIFO has four words (each of 32 bits) which can be +linked to the DMA to transfer larger amounts of data. + +All PIO instructions follow a common pattern:: + + .side() [] + +The side-set ``.side(...)`` and delay ``[...]`` parts are both optional, and if +specified allow the instruction to perform more than one operation. This keeps +PIO programs small and efficient. + +There are nine instructions which perform the following tasks: + +- ``jmp()`` transfers control to a different part of the code +- ``wait()`` pauses until a particular action happens +- ``in_()`` shifts the bits from a source (scratch register or set of pins) to the + input shift register +- ``out()`` shifts the bits from the output shift register to a destination +- ``push()`` sends data to the RX FIFO +- ``pull()`` receives data from the TX FIFO +- ``mov()`` moves data from a source to a destination +- ``irq()`` sets or clears an IRQ flag +- ``set()`` writes a literal value to a destination + +The instruction modifiers are: + +- ``.side()`` sets the side-set pins at the start of the instruction +- ``[]`` delays for a certain number of cycles after execution of the instruction + +There are also directives: + +- ``wrap_target()`` specifies where the program execution will get continued from +- ``wrap()`` specifies the instruction where the control flow of the program will + get wrapped from +- ``label()`` sets a label for use with ``jmp()`` instructions +- ``word()`` emits a raw 16-bit value which acts as an instruction in the program + +An example +---------- + +Take the ``pio_1hz.py`` example for a simple understanding of how to use the PIO +and state machines. Below is the code for reference. + +.. code-block:: python3 + + # Example using PIO to blink an LED and raise an IRQ at 1Hz. + + import time + from machine import Pin + import rp2 + + + @rp2.asm_pio(set_init=rp2.PIO.OUT_LOW) + def blink_1hz(): + # Cycles: 1 + 1 + 6 + 32 * (30 + 1) = 1000 + irq(rel(0)) + set(pins, 1) + set(x, 31) [5] + label("delay_high") + nop() [29] + jmp(x_dec, "delay_high") + + # Cycles: 1 + 7 + 32 * (30 + 1) = 1000 + set(pins, 0) + set(x, 31) [6] + label("delay_low") + nop() [29] + jmp(x_dec, "delay_low") + + + # Create the StateMachine with the blink_1hz program, outputting on Pin(25). + sm = rp2.StateMachine(0, blink_1hz, freq=2000, set_base=Pin(25)) + + # Set the IRQ handler to print the millisecond timestamp. + sm.irq(lambda p: print(time.ticks_ms())) + + # Start the StateMachine. + sm.active(1) + +This creates an instance of class :class:`rp2.StateMachine` which runs the +``blink_1hz`` program at 2000Hz, and connects to pin 25. The ``blink_1hz`` +program uses the PIO to blink an LED connected to this pin at 1Hz, and also +raises an IRQ as the LED turns on. This IRQ then calls the ``lambda`` function +which prints out a millisecond timestamp. + +The ``blink_1hz`` program is a PIO assembler routine. It connects to a single +pin which is configured as an output and starts out low. The instructions do +the following: + +- ``irq(rel(0))`` raises the IRQ associated with the state machine. +- The LED is turned on via the ``set(pins, 1)`` instruction. +- The value 31 is put into register X, and then there is a delay for 5 more + cycles, specified by the ``[5]``. +- The ``nop() [29]`` instruction waits for 30 cycles. +- The ``jmp(x_dec, "delay_high")`` will keep looping to the ``delay_high`` label + as long as the register X is non-zero, and will also post-decrement X. Since + X starts with the value 31 this jump will happen 31 times, so the ``nop() [29]`` + runs 32 times in total (note there is also one instruction cycle taken by the + ``jmp`` for each of these 32 loops). +- ``set(pins, 0)`` will turn the LED off by setting pin 25 low. +- Another 32 loops of ``nop() [29]`` and ``jmp(...)`` will execute. +- Because ``wrap_target()`` and ``wrap()`` are not specified, their default will + be used and execution of the program will wrap around from the bottom to the + top. This wrapping does not cost any execution cycles. + +The entire routine takes exactly 2000 cycles of the state machine. Setting the +frequency of the state machine to 2000Hz makes the LED blink at 1Hz. diff --git a/extmod/extmod.cmake b/extmod/extmod.cmake index c6b45b0d3e..67f7d8fd39 100644 --- a/extmod/extmod.cmake +++ b/extmod/extmod.cmake @@ -10,11 +10,13 @@ set(MICROPY_SOURCE_EXTMOD ${MICROPY_EXTMOD_DIR}/machine_i2c.c ${MICROPY_EXTMOD_DIR}/machine_mem.c ${MICROPY_EXTMOD_DIR}/machine_pulse.c + ${MICROPY_EXTMOD_DIR}/machine_pwm.c ${MICROPY_EXTMOD_DIR}/machine_signal.c ${MICROPY_EXTMOD_DIR}/machine_spi.c ${MICROPY_EXTMOD_DIR}/modbluetooth.c ${MICROPY_EXTMOD_DIR}/modbtree.c ${MICROPY_EXTMOD_DIR}/modframebuf.c + ${MICROPY_EXTMOD_DIR}/modnetwork.c ${MICROPY_EXTMOD_DIR}/modonewire.c ${MICROPY_EXTMOD_DIR}/moduasyncio.c ${MICROPY_EXTMOD_DIR}/modubinascii.c @@ -23,9 +25,11 @@ set(MICROPY_SOURCE_EXTMOD ${MICROPY_EXTMOD_DIR}/moduhashlib.c ${MICROPY_EXTMOD_DIR}/moduheapq.c ${MICROPY_EXTMOD_DIR}/modujson.c + ${MICROPY_EXTMOD_DIR}/moduplatform.c ${MICROPY_EXTMOD_DIR}/modurandom.c ${MICROPY_EXTMOD_DIR}/modure.c ${MICROPY_EXTMOD_DIR}/moduselect.c + ${MICROPY_EXTMOD_DIR}/modusocket.c ${MICROPY_EXTMOD_DIR}/modussl_axtls.c ${MICROPY_EXTMOD_DIR}/modussl_mbedtls.c ${MICROPY_EXTMOD_DIR}/modutimeq.c diff --git a/extmod/modonewire.c b/extmod/modonewire.c index a963c17ebb..ba7405fbae 100644 --- a/extmod/modonewire.c +++ b/extmod/modonewire.c @@ -9,6 +9,8 @@ #include "py/obj.h" #include "py/mphal.h" +#if MICROPY_PY_ONEWIRE + /******************************************************************************/ // Low-level 1-Wire routines @@ -139,3 +141,5 @@ const mp_obj_module_t mp_module_onewire = { .base = { &mp_type_module }, .globals = (mp_obj_dict_t *)&onewire_module_globals, }; + +#endif // MICROPY_PY_ONEWIRE diff --git a/extmod/moduplatform.c b/extmod/moduplatform.c new file mode 100644 index 0000000000..820feb312a --- /dev/null +++ b/extmod/moduplatform.c @@ -0,0 +1,146 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2021 Ibrahim Abdelkader + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include "py/runtime.h" +#include "py/objtuple.h" +#include "py/objstr.h" +#include "py/mphal.h" +#include "genhdr/mpversion.h" + +#if MICROPY_PY_UPLATFORM + +// platform - Access to underlying platform's identifying data + +// TODO: Add more architectures, compilers and libraries. +// See: https://sourceforge.net/p/predef/wiki/Home/ + +#if defined(__ARM_ARCH) +#define PLATFORM_ARCH "arm" +#elif defined(__x86_64__) || defined(_WIN64) +#define PLATFORM_ARCH "x86_64" +#elif defined(__i386__) || defined(_M_IX86) +#define PLATFORM_ARCH "x86" +#elif defined(__xtensa__) || defined(_M_IX86) +#define PLATFORM_ARCH "xtensa" +#else +#define PLATFORM_ARCH "" +#endif + +#if defined(__GNUC__) +#define PLATFORM_COMPILER \ + "GCC " \ + MP_STRINGIFY(__GNUC__) "." \ + MP_STRINGIFY(__GNUC_MINOR__) "." \ + MP_STRINGIFY(__GNUC_PATCHLEVEL__) +#elif defined(__ARMCC_VERSION) +#define PLATFORM_COMPILER \ + "ARMCC " \ + MP_STRINGIFY((__ARMCC_VERSION / 1000000)) "." \ + MP_STRINGIFY((__ARMCC_VERSION / 10000 % 100)) "." \ + MP_STRINGIFY((__ARMCC_VERSION % 10000)) +#elif defined(_MSC_VER) +#if defined(_WIN64) +#define COMPILER_BITS "64 bit" +#elif defined(_M_IX86) +#define COMPILER_BITS "32 bit" +#else +#define COMPILER_BITS "" +#endif +#define PLATFORM_COMPILER \ + "MSC v." MP_STRINGIFY(_MSC_VER) " " COMPILER_BITS +#else +#define PLATFORM_COMPILER "" +#endif + +#if defined(__GLIBC__) +#define PLATFORM_LIBC_LIB "glibc" +#define PLATFORM_LIBC_VER \ + MP_STRINGIFY(__GLIBC__) "." \ + MP_STRINGIFY(__GLIBC_MINOR__) +#elif defined(__NEWLIB__) +#define PLATFORM_LIBC_LIB "newlib" +#define PLATFORM_LIBC_VER _NEWLIB_VERSION +#else +#define PLATFORM_LIBC_LIB "" +#define PLATFORM_LIBC_VER "" +#endif + +#if defined(__linux) +#define PLATFORM_SYSTEM "Linux" +#elif defined(__unix__) +#define PLATFORM_SYSTEM "Unix" +#elif defined(__CYGWIN__) +#define PLATFORM_SYSTEM "Cygwin" +#elif defined(_WIN32) +#define PLATFORM_SYSTEM "Windows" +#else +#define PLATFORM_SYSTEM "MicroPython" +#endif + +#ifndef MICROPY_PLATFORM_VERSION +#define MICROPY_PLATFORM_VERSION "" +#endif + +STATIC const MP_DEFINE_STR_OBJ(info_platform_obj, PLATFORM_SYSTEM "-" MICROPY_VERSION_STRING "-" \ + PLATFORM_ARCH "-" MICROPY_PLATFORM_VERSION "-with-" PLATFORM_LIBC_LIB "" PLATFORM_LIBC_VER); +STATIC const MP_DEFINE_STR_OBJ(info_python_compiler_obj, PLATFORM_COMPILER); +STATIC const MP_DEFINE_STR_OBJ(info_libc_lib_obj, PLATFORM_LIBC_LIB); +STATIC const MP_DEFINE_STR_OBJ(info_libc_ver_obj, PLATFORM_LIBC_VER); +STATIC const mp_rom_obj_tuple_t info_libc_tuple_obj = { + {&mp_type_tuple}, 2, {MP_ROM_PTR(&info_libc_lib_obj), MP_ROM_PTR(&info_libc_ver_obj)} +}; + +STATIC mp_obj_t platform_platform(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { + return MP_OBJ_FROM_PTR(&info_platform_obj); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_KW(platform_platform_obj, 0, platform_platform); + +STATIC mp_obj_t platform_python_compiler(void) { + return MP_OBJ_FROM_PTR(&info_python_compiler_obj); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(platform_python_compiler_obj, platform_python_compiler); + +STATIC mp_obj_t platform_libc_ver(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { + return MP_OBJ_FROM_PTR(&info_libc_tuple_obj); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_KW(platform_libc_ver_obj, 0, platform_libc_ver); + +STATIC const mp_rom_map_elem_t modplatform_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uplatform) }, + { MP_ROM_QSTR(MP_QSTR_platform), MP_ROM_PTR(&platform_platform_obj) }, + { MP_ROM_QSTR(MP_QSTR_python_compiler), MP_ROM_PTR(&platform_python_compiler_obj) }, + { MP_ROM_QSTR(MP_QSTR_libc_ver), MP_ROM_PTR(&platform_libc_ver_obj) }, +}; + +STATIC MP_DEFINE_CONST_DICT(modplatform_globals, modplatform_globals_table); + +const mp_obj_module_t mp_module_uplatform = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&modplatform_globals, +}; + +#endif // MICROPY_PY_UPLATFORM diff --git a/extmod/modure.c b/extmod/modure.c index cca74a7267..c65eddb988 100644 --- a/extmod/modure.c +++ b/extmod/modure.c @@ -477,11 +477,16 @@ MP_REGISTER_MODULE(MP_QSTR_re, mp_module_ure, MICROPY_PY_URE); // only if module is enabled by config setting. #define re1_5_fatal(x) assert(!x) + #include "lib/re1.5/compilecode.c" -#if MICROPY_PY_URE_DEBUG -#include "lib/re1.5/dumpcode.c" -#endif #include "lib/re1.5/recursiveloop.c" #include "lib/re1.5/charclass.c" +#if MICROPY_PY_URE_DEBUG +// Make sure the output print statements go to the same output as other Python output. +#define printf(...) mp_printf(&mp_plat_print, __VA_ARGS__) +#include "lib/re1.5/dumpcode.c" +#undef printf +#endif + #endif // MICROPY_PY_URE diff --git a/extmod/uasyncio/core.py b/extmod/uasyncio/core.py index 12833cf0cd..c3ce3ccafa 100644 --- a/extmod/uasyncio/core.py +++ b/extmod/uasyncio/core.py @@ -36,7 +36,7 @@ class SingletonGenerator: self.state = None self.exc = StopIteration() - def __iter__(self): + def __await__(self): return self def __next__(self): diff --git a/extmod/uasyncio/funcs.py b/extmod/uasyncio/funcs.py index 93f4fd256c..0ce48b015c 100644 --- a/extmod/uasyncio/funcs.py +++ b/extmod/uasyncio/funcs.py @@ -66,7 +66,7 @@ async def gather(*aws, return_exceptions=False): # # cancel all waiting tasks # raise er ts[i] = await ts[i] - except Exception as er: + except (core.CancelledError, Exception) as er: if return_exceptions: ts[i] = er else: diff --git a/extmod/uasyncio/task.py b/extmod/uasyncio/task.py index 26df7b1725..cd75a14a09 100644 --- a/extmod/uasyncio/task.py +++ b/extmod/uasyncio/task.py @@ -130,7 +130,7 @@ class Task: self.ph_next = None # Paring heap self.ph_rightmost_parent = None # Paring heap - def __iter__(self): + def __await__(self): if not self.state: # Task finished, signal that is has been await'ed on. self.state = False diff --git a/extmod/vfs_posix_file.c b/extmod/vfs_posix_file.c index 8837fdbbd6..a984ace4a2 100644 --- a/extmod/vfs_posix_file.c +++ b/extmod/vfs_posix_file.c @@ -17,6 +17,8 @@ #ifdef _WIN32 #define fsync _commit +#else +#include #endif typedef struct _mp_obj_vfs_posix_file_t { @@ -180,6 +182,32 @@ STATIC mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_ return 0; case MP_STREAM_GET_FILENO: return o->fd; + #if MICROPY_PY_USELECT + case MP_STREAM_POLL: { + #ifdef _WIN32 + mp_raise_NotImplementedError(MP_ERROR_TEXT("poll on file not available on win32")); + #else + mp_uint_t ret = 0; + uint8_t pollevents = 0; + if (arg & MP_STREAM_POLL_RD) { + pollevents |= POLLIN; + } + if (arg & MP_STREAM_POLL_WR) { + pollevents |= POLLOUT; + } + struct pollfd pfd = { .fd = o->fd, .events = pollevents }; + if (poll(&pfd, 1, 0) > 0) { + if (pfd.revents & POLLIN) { + ret |= MP_STREAM_POLL_RD; + } + if (pfd.revents & POLLOUT) { + ret |= MP_STREAM_POLL_WR; + } + } + return ret; + #endif + } + #endif default: *errcode = EINVAL; return MP_STREAM_ERROR; diff --git a/lib/pico-sdk b/lib/pico-sdk new file mode 160000 index 0000000000..fc10a97c38 --- /dev/null +++ b/lib/pico-sdk @@ -0,0 +1 @@ +Subproject commit fc10a97c386f65c1a44c68684fe52a56aaf50df0 diff --git a/lib/stm32lib b/lib/stm32lib new file mode 160000 index 0000000000..302c52794d --- /dev/null +++ b/lib/stm32lib @@ -0,0 +1 @@ +Subproject commit 302c52794d2f579903f4e49cbad1f5d3a7f401ad diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot index 8daa071c57..f1c2b725d9 100644 --- a/locale/circuitpython.pot +++ b/locale/circuitpython.pot @@ -2874,6 +2874,10 @@ msgstr "" msgid "can't load with '%q' index" msgstr "" +#: py/builtinimport.c +msgid "can't perform relative import" +msgstr "" + #: py/objgenerator.c msgid "can't send non-None value to a just-started generator" msgstr "" @@ -2936,10 +2940,6 @@ msgstr "" msgid "cannot import name %q" msgstr "" -#: py/builtinimport.c -msgid "cannot perform relative import" -msgstr "" - #: extmod/moductypes.c msgid "cannot unambiguously get sizeof scalar" msgstr "" @@ -3630,7 +3630,7 @@ msgstr "" msgid "matrix is not positive definite" msgstr "" -#: shared-bindings/wifi/Radio.c +#: ports/espressif/common-hal/wifi/Radio.c msgid "max_connections must be between 0 and 10" msgstr "" @@ -4010,6 +4010,10 @@ msgstr "" msgid "pixel_shader must be displayio.Palette or displayio.ColorConverter" msgstr "" +#: extmod/vfs_posix_file.c +msgid "poll on file not available on win32" +msgstr "" + #: shared-module/vectorio/Polygon.c msgid "polygon can only be registered in one parent" msgstr "" @@ -4055,6 +4059,7 @@ msgstr "" #: ports/espressif/boards/crumpspace_crumps2/mpconfigboard.h #: ports/espressif/boards/electroniccats_bastwifi/mpconfigboard.h #: ports/espressif/boards/espressif_esp32c3_devkitm_1_n4/mpconfigboard.h +#: ports/espressif/boards/espressif_esp32s2_devkitc_1_n4r2/mpconfigboard.h #: ports/espressif/boards/espressif_esp32s3_box/mpconfigboard.h #: ports/espressif/boards/espressif_esp32s3_devkitc_1_n8/mpconfigboard.h #: ports/espressif/boards/espressif_esp32s3_devkitc_1_n8r2/mpconfigboard.h diff --git a/main.c b/main.c index b4537b1d69..2a0a8777c1 100644 --- a/main.c +++ b/main.c @@ -162,9 +162,6 @@ STATIC void start_mp(supervisor_allocation *heap, bool first_run) { mp_obj_list_init((mp_obj_list_t *)mp_sys_path, 0); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script) mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_)); - // Frozen modules are in their own pseudo-dir, e.g., ".frozen". - // Prioritize .frozen over /lib. - mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_FROZEN_FAKE_DIR_QSTR)); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_lib)); mp_obj_list_init((mp_obj_list_t *)mp_sys_argv, 0); diff --git a/mpy-cross/README.md b/mpy-cross/README.md index 60f9c593c7..3d88814d64 100644 --- a/mpy-cross/README.md +++ b/mpy-cross/README.md @@ -23,10 +23,7 @@ by the target MicroPython runtime (eg onto a pyboard's filesystem), and then imported like any other Python module using `import foo`. Different target runtimes may require a different format of the compiled -bytecode, and such options can be passed to the cross compiler. For example, -the unix port of MicroPython requires the following: - - $ ./mpy-cross -mcache-lookup-bc foo.py +bytecode, and such options can be passed to the cross compiler. If the Python code contains `@native` or `@viper` annotations, then you must specify `-march` to match the target architecture. diff --git a/mpy-cross/main.c b/mpy-cross/main.c index c120d6f2db..c00829be10 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -87,7 +87,6 @@ STATIC int usage(char **argv) { "Target specific options:\n" "-msmall-int-bits=number : set the maximum bits used to encode a small-int\n" "-mno-unicode : don't support unicode in compiled strings\n" - "-mcache-lookup-bc : cache map lookups in the bytecode\n" "-march= : set architecture for native emitter; x86, x64, armv6, armv7m, armv7em, armv7emsp, armv7emdp, xtensa, xtensawin\n" "\n" "Implementation specific options:\n", argv[0] @@ -172,8 +171,6 @@ MP_NOINLINE int main_(int argc, char **argv) { #ifdef _WIN32 set_fmode_binary(); #endif - mp_obj_list_init(mp_sys_path, 0); - mp_obj_list_init(mp_sys_argv, 0); #if MICROPY_EMIT_NATIVE // Set default emitter options @@ -184,7 +181,6 @@ MP_NOINLINE int main_(int argc, char **argv) { // set default compiler configuration mp_dynamic_compiler.small_int_bits = 31; - mp_dynamic_compiler.opt_cache_map_lookup_in_bytecode = 0; mp_dynamic_compiler.py_builtins_str_unicode = 1; #if defined(__i386__) mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_X86; @@ -243,10 +239,6 @@ MP_NOINLINE int main_(int argc, char **argv) { return usage(argv); } // TODO check that small_int_bits is within range of host's capabilities - } else if (strcmp(argv[a], "-mno-cache-lookup-bc") == 0) { - mp_dynamic_compiler.opt_cache_map_lookup_in_bytecode = 0; - } else if (strcmp(argv[a], "-mcache-lookup-bc") == 0) { - mp_dynamic_compiler.opt_cache_map_lookup_in_bytecode = 1; } else if (strcmp(argv[a], "-mno-unicode") == 0) { mp_dynamic_compiler.py_builtins_str_unicode = 0; } else if (strcmp(argv[a], "-municode") == 0) { diff --git a/mpy-cross/mpconfigport.h b/mpy-cross/mpconfigport.h index 50c77a1186..d91e1d6d6e 100644 --- a/mpy-cross/mpconfigport.h +++ b/mpy-cross/mpconfigport.h @@ -36,8 +36,6 @@ #define MICROPY_COMP_TRIPLE_TUPLE_ASSIGN (1) #define MICROPY_COMP_RETURN_IF_EXPR (1) -#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (0) - #define MICROPY_READER_POSIX (1) #define MICROPY_ENABLE_RUNTIME (0) #define MICROPY_ENABLE_GC (1) diff --git a/ports/unix/Makefile b/ports/unix/Makefile index 344ebdb423..9857ec5f88 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -13,6 +13,9 @@ include ../../py/mkenv.mk -include mpconfigport.mk include $(VARIANT_DIR)/mpconfigvariant.mk +# Use the default frozen manifest, variants may override this. +FROZEN_MANIFEST ?= variants/manifest.py + # This should be configured by the mpconfigvariant.mk PROG ?= micropython @@ -39,7 +42,7 @@ CFLAGS += $(INC) $(CWARN) -std=gnu99 -DUNIX $(CFLAGS_MOD) $(COPT) -I$(VARIANT_DI # Debugging/Optimization ifdef DEBUG -COPT ?= -O0 +COPT ?= -Og else COPT ?= -Os COPT += -DNDEBUG @@ -256,12 +259,14 @@ SRC_QSTR += $(SRC_C) $(SRC_CXX) $(SHARED_SRC_C) $(EXTMOD_SRC_C) # SRC_QSTR SRC_QSTR_AUTO_DEPS += -ifneq ($(FROZEN_MPY_DIR),) +ifneq ($(FROZEN_MANIFEST),) +# To use frozen code create a manifest.py file with a description of files to +# freeze, then invoke make with FROZEN_MANIFEST=manifest.py (be sure to build from scratch). CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool CFLAGS += -DMICROPY_MODULE_FROZEN_MPY CFLAGS += -DMICROPY_MODULE_FROZEN_STR CFLAGS += -DMPZ_DIG_SIZE=16 # force 16 bits to work on both 32 and 64 bit archs -MPY_CROSS_FLAGS += -mcache-lookup-bc +CFLAGS += -DMICROPY_MODULE_FROZEN_STR endif HASCPP17 = $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 7) @@ -273,9 +278,7 @@ endif CXXFLAGS += $(filter-out -Wmissing-prototypes -Wold-style-definition -std=gnu99 -std=gnu11,$(CFLAGS) $(CXXFLAGS_MOD)) ifeq ($(MICROPY_FORCE_32BIT),1) -RUN_TESTS_MPY_CROSS_FLAGS = --mpy-cross-flags='-mcache-lookup-bc -march=x86' -else -RUN_TESTS_MPY_CROSS_FLAGS = --mpy-cross-flags='-mcache-lookup-bc' +RUN_TESTS_MPY_CROSS_FLAGS = --mpy-cross-flags='-march=x86' endif ifeq ($(CROSS_COMPILE),arm-linux-gnueabi-) diff --git a/ports/unix/coverage.c b/ports/unix/coverage.c index 16287fdc01..179181dc83 100644 --- a/ports/unix/coverage.c +++ b/ports/unix/coverage.c @@ -161,7 +161,7 @@ STATIC void pairheap_test(size_t nops, int *ops) { mp_pairheap_init_node(pairheap_lt, &node[i]); } mp_pairheap_t *heap = mp_pairheap_new(pairheap_lt); - printf("create:"); + mp_printf(&mp_plat_print, "create:"); for (size_t i = 0; i < nops; ++i) { if (ops[i] >= 0) { heap = mp_pairheap_push(pairheap_lt, heap, &node[ops[i]]); @@ -175,13 +175,13 @@ STATIC void pairheap_test(size_t nops, int *ops) { ; } } - printf("\npop all:"); + mp_printf(&mp_plat_print, "\npop all:"); while (!mp_pairheap_is_empty(pairheap_lt, heap)) { mp_printf(&mp_plat_print, " %d", mp_pairheap_peek(pairheap_lt, heap) - &node[0]); ; heap = mp_pairheap_pop(pairheap_lt, heap); } - printf("\n"); + mp_printf(&mp_plat_print, "\n"); } // function to run extra tests for things that can't be checked by scripts diff --git a/ports/unix/main.c b/ports/unix/main.c index c1cf6a7ffa..5ebcf9193b 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -494,16 +494,10 @@ MP_NOINLINE int main_(int argc, char **argv) { char *home = getenv("HOME"); char *path = getenv("MICROPYPATH"); if (path == NULL) { - #ifdef MICROPY_PY_SYS_PATH_DEFAULT path = MICROPY_PY_SYS_PATH_DEFAULT; - #else - path = "~/.micropython/lib:/usr/lib/micropython"; - #endif } - size_t path_num = 2; // [0] is for current dir (or base dir of the script) - // [1] is for frozen files. - size_t builtin_path_count = path_num; - if (*path == ':') { + size_t path_num = 1; // [0] is for current dir (or base dir of the script) + if (*path == PATHLIST_SEP_CHAR) { path_num++; } for (char *p = path; p != NULL; p = strchr(p, PATHLIST_SEP_CHAR)) { @@ -516,11 +510,9 @@ MP_NOINLINE int main_(int argc, char **argv) { mp_obj_t *path_items; mp_obj_list_get(mp_sys_path, &path_num, &path_items); path_items[0] = MP_OBJ_NEW_QSTR(MP_QSTR_); - // Frozen modules are in their own pseudo-dir, e.g., ".frozen". - path_items[1] = MP_OBJ_NEW_QSTR(MP_FROZEN_FAKE_DIR_QSTR); { char *p = path; - for (mp_uint_t i = builtin_path_count; i < path_num; i++) { + for (mp_uint_t i = 1; i < path_num; i++) { char *p1 = strchr(p, PATHLIST_SEP_CHAR); if (p1 == NULL) { p1 = p + strlen(p); @@ -661,7 +653,7 @@ MP_NOINLINE int main_(int argc, char **argv) { break; } - // Set base dir of the script as first entry in sys.path + // Set base dir of the script as first entry in sys.path. char *p = strrchr(basedir, '/'); path_items[0] = mp_obj_new_str_via_qstr(basedir, p - basedir); free(pathbuf); diff --git a/ports/unix/modos.c b/ports/unix/modos.c index c7386180ae..6241dfa6c3 100644 --- a/ports/unix/modos.c +++ b/ports/unix/modos.c @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -48,6 +49,29 @@ #define USE_STATFS 1 #endif +#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) +#if __GLIBC_PREREQ(2, 25) +#include +#define _HAVE_GETRANDOM +#endif +#endif + +STATIC mp_obj_t mod_os_urandom(mp_obj_t num) { + mp_int_t n = mp_obj_get_int(num); + vstr_t vstr; + vstr_init_len(&vstr, n); + #ifdef _HAVE_GETRANDOM + RAISE_ERRNO(getrandom(vstr.buf, n, 0), errno); + #else + int fd = open("/dev/urandom", O_RDONLY); + RAISE_ERRNO(fd, errno); + RAISE_ERRNO(read(fd, vstr.buf, n), errno); + close(fd); + #endif + return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_os_urandom_obj, mod_os_urandom); + STATIC mp_obj_t mod_os_stat(mp_obj_t path_in) { struct stat sb; const char *path = mp_obj_str_get_str(path_in); @@ -308,6 +332,7 @@ STATIC const mp_rom_map_elem_t mp_module_os_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uos) }, { MP_ROM_QSTR(MP_QSTR_errno), MP_ROM_PTR(&mod_os_errno_obj) }, { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&mod_os_stat_obj) }, + { MP_ROM_QSTR(MP_QSTR_urandom), MP_ROM_PTR(&mod_os_urandom_obj) }, #if MICROPY_PY_OS_STATVFS { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&mod_os_statvfs_obj) }, #endif diff --git a/ports/unix/mpconfigport.h b/ports/unix/mpconfigport.h index da3c71d776..c54fede813 100644 --- a/ports/unix/mpconfigport.h +++ b/ports/unix/mpconfigport.h @@ -80,10 +80,14 @@ #endif #define MICROPY_STREAMS_POSIX_API (1) #define MICROPY_OPT_COMPUTED_GOTO (1) -#ifndef MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE -#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (1) +#ifndef MICROPY_OPT_LOAD_ATTR_FAST_PATH +#define MICROPY_OPT_LOAD_ATTR_FAST_PATH (1) +#endif +#ifndef MICROPY_OPT_MAP_LOOKUP_CACHE +#define MICROPY_OPT_MAP_LOOKUP_CACHE (1) #endif #define MICROPY_MODULE_WEAK_LINKS (1) +#define MICROPY_MODULE_OVERRIDE_MAIN_IMPORT (1) #define MICROPY_CAN_OVERRIDE_BUILTINS (1) #define MICROPY_VFS_POSIX_FILE (1) #define MICROPY_PY_FUNCTION_ATTRS (1) @@ -107,6 +111,8 @@ #define MICROPY_PY_ARRAY_SLICE_ASSIGN (1) #define MICROPY_PY_BUILTINS_SLICE_ATTRS (1) #define MICROPY_PY_BUILTINS_SLICE_INDICES (1) +#define MICROPY_PY_SYS_ATEXIT (1) +#define MICROPY_PY_SYS_PATH_ARGV_DEFAULTS (0) #define MICROPY_PY_SYS_EXIT (1) #if MICROPY_PY_SYS_SETTRACE #define MICROPY_PERSISTENT_CODE_SAVE (1) @@ -119,6 +125,9 @@ #define MICROPY_PY_SYS_PLATFORM "linux" #endif #endif +#ifndef MICROPY_PY_SYS_PATH_DEFAULT +#define MICROPY_PY_SYS_PATH_DEFAULT ".frozen:~/.micropython/lib:/usr/lib/micropython" +#endif #define MICROPY_PY_SYS_MAXSIZE (1) #define MICROPY_PY_SYS_STDFILES (1) #define MICROPY_PY_SYS_EXC_INFO (1) diff --git a/ports/unix/variants/coverage/mpconfigvariant.h b/ports/unix/variants/coverage/mpconfigvariant.h index 2ad0d94bc0..f6ebc2087b 100644 --- a/ports/unix/variants/coverage/mpconfigvariant.h +++ b/ports/unix/variants/coverage/mpconfigvariant.h @@ -51,7 +51,6 @@ #define MICROPY_PY_MATH_FACTORIAL (1) #define MICROPY_PY_URANDOM_EXTRA_FUNCS (1) #define MICROPY_PY_IO_BUFFEREDWRITER (1) -#define MICROPY_PY_IO_RESOURCE_STREAM (1) #define MICROPY_PY_UASYNCIO (1) #define MICROPY_PY_URE_DEBUG (1) #define MICROPY_PY_URE_MATCH_GROUPS (1) diff --git a/ports/unix/variants/coverage/mpconfigvariant.mk b/ports/unix/variants/coverage/mpconfigvariant.mk index 7e8aabef5b..59f443e760 100644 --- a/ports/unix/variants/coverage/mpconfigvariant.mk +++ b/ports/unix/variants/coverage/mpconfigvariant.mk @@ -12,15 +12,13 @@ CFLAGS += \ LDFLAGS += -fprofile-arcs -ftest-coverage +FROZEN_MANIFEST ?= $(VARIANT_DIR)/manifest.py USER_C_MODULES = $(TOP)/examples/usercmodule MICROPY_VFS_FAT = 1 MICROPY_VFS_LFS1 = 1 MICROPY_VFS_LFS2 = 1 -FROZEN_DIR=variants/coverage/frzstr -FROZEN_MPY_DIR=variants/coverage/frzmpy - SRC_QRIO := $(patsubst ../../%,%,$(wildcard ../../shared-bindings/qrio/*.c ../../shared-module/qrio/*.c ../../lib/quirc/lib/*.c)) SRC_C += $(SRC_QRIO) diff --git a/ports/unix/variants/minimal/mpconfigvariant.h b/ports/unix/variants/minimal/mpconfigvariant.h index a959a95059..09dc37d1c3 100644 --- a/ports/unix/variants/minimal/mpconfigvariant.h +++ b/ports/unix/variants/minimal/mpconfigvariant.h @@ -55,7 +55,8 @@ #define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_NONE) #define MICROPY_STREAMS_NON_BLOCK (0) #define MICROPY_OPT_COMPUTED_GOTO (0) -#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (0) +#define MICROPY_OPT_LOAD_ATTR_FAST_PATH (0) +#define MICROPY_OPT_MAP_LOOKUP_CACHE (0) #define MICROPY_CAN_OVERRIDE_BUILTINS (0) #define MICROPY_BUILTIN_METHOD_CHECK_SELF_ARG (0) #define MICROPY_CPYTHON_COMPAT (0) @@ -87,6 +88,9 @@ #define MICROPY_PY_SYS (1) #define MICROPY_PY_SYS_EXIT (0) #define MICROPY_PY_SYS_PLATFORM "linux" +#ifndef MICROPY_PY_SYS_PATH_DEFAULT +#define MICROPY_PY_SYS_PATH_DEFAULT ".frozen:~/.micropython/lib:/usr/lib/micropython" +#endif #define MICROPY_PY_SYS_MAXSIZE (0) #define MICROPY_PY_SYS_STDFILES (0) #define MICROPY_PY_CMATH (0) diff --git a/py/bc.c b/py/bc.c index ccf503631d..33b94c4a9f 100644 --- a/py/bc.c +++ b/py/bc.c @@ -314,24 +314,10 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw // The following table encodes the number of bytes that a specific opcode // takes up. Some opcodes have an extra byte, defined by MP_BC_MASK_EXTRA_BYTE. -// There are 4 special opcodes that have an extra byte only when -// MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE is enabled (and they take a qstr): -// MP_BC_LOAD_NAME -// MP_BC_LOAD_GLOBAL -// MP_BC_LOAD_ATTR -// MP_BC_STORE_ATTR uint mp_opcode_format(const byte *ip, size_t *opcode_size, bool count_var_uint) { uint f = MP_BC_FORMAT(*ip); const byte *ip_start = ip; if (f == MP_BC_FORMAT_QSTR) { - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { - if (*ip == MP_BC_LOAD_NAME - || *ip == MP_BC_LOAD_GLOBAL - || *ip == MP_BC_LOAD_ATTR - || *ip == MP_BC_STORE_ATTR) { - ip += 1; - } - } ip += 3; } else { int extra_byte = (*ip & MP_BC_MASK_EXTRA_BYTE) == 0; diff --git a/py/builtin.h b/py/builtin.h index 2c0437d403..4fa4b08f9d 100644 --- a/py/builtin.h +++ b/py/builtin.h @@ -119,13 +119,8 @@ extern const mp_obj_module_t mp_module_utimeq; extern const mp_obj_module_t mp_module_machine; extern const mp_obj_module_t mp_module_framebuf; extern const mp_obj_module_t mp_module_btree; -extern const mp_obj_module_t ulab_user_cmodule; -extern mp_obj_module_t ulab_fft_module; -extern mp_obj_module_t ulab_filter_module; -extern mp_obj_module_t ulab_linalg_module; -extern mp_obj_module_t ulab_numerical_module; -extern mp_obj_module_t ulab_poly_module; - +extern const mp_obj_module_t mp_module_ubluetooth; +extern const mp_obj_module_t mp_module_uplatform; extern const char MICROPY_PY_BUILTINS_HELP_TEXT[]; diff --git a/py/builtinhelp.c b/py/builtinhelp.c index fc32de5abe..453c9bb118 100644 --- a/py/builtinhelp.c +++ b/py/builtinhelp.c @@ -69,10 +69,10 @@ STATIC void mp_help_add_from_map(mp_obj_t list, const mp_map_t *map) { #if MICROPY_MODULE_FROZEN STATIC void mp_help_add_from_names(mp_obj_t list, const char *name) { while (*name) { - size_t l = strlen(name); + size_t len = strlen(name); // name should end in '.py' and we strip it off - mp_obj_list_append(list, mp_obj_new_str(name, l - 3)); - name += l + 1; + mp_obj_list_append(list, mp_obj_new_str(name, len - 3)); + name += len + 1; } } #endif @@ -92,12 +92,9 @@ STATIC void mp_help_print_modules(void) { mp_help_add_from_map(list, &mp_builtin_module_map); - #if MICROPY_MODULE_FROZEN_STR - mp_help_add_from_names(list, mp_frozen_str_names); - #endif - - #if MICROPY_MODULE_FROZEN_MPY - mp_help_add_from_names(list, mp_frozen_mpy_names); + #if MICROPY_MODULE_FROZEN + extern const char mp_frozen_names[]; + mp_help_add_from_names(list, mp_frozen_names); #endif // sort the list so it's printed in alphabetical order diff --git a/py/builtinimport.c b/py/builtinimport.c index ca4b848407..dbacc3d657 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -5,6 +5,7 @@ * * SPDX-FileCopyrightText: Copyright (c) 2013-2019 Damien P. George * Copyright (c) 2014 Paul Sokolovsky + * Copyright (c) 2021 Jim Mussared * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -50,7 +51,11 @@ #if MICROPY_ENABLE_EXTERNAL_IMPORT -#define PATH_SEP_CHAR '/' +// Must be a string of one byte. +#define PATH_SEP_CHAR "/" + +// Virtual sys.path entry that maps to the frozen modules. +#define MP_FROZEN_PATH_PREFIX ".frozen/" bool mp_obj_is_package(mp_obj_t module) { mp_obj_t dest[2]; @@ -58,31 +63,33 @@ bool mp_obj_is_package(mp_obj_t module) { return dest[0] != MP_OBJ_NULL; } -// Stat either frozen or normal module by a given path -// (whatever is available, if at all). -STATIC mp_import_stat_t mp_import_stat_any(const char *path) { +// Wrapper for mp_import_stat (which is provided by the port, and typically +// uses mp_vfs_import_stat) to also search frozen modules. Given an exact +// path to a file or directory (e.g. "foo/bar", foo/bar.py" or "foo/bar.mpy"), +// will return whether the path is a file, directory, or doesn't exist. +STATIC mp_import_stat_t stat_path_or_frozen(const char *path) { #if MICROPY_MODULE_FROZEN - if (strncmp(MP_FROZEN_FAKE_DIR_SLASH, - path, - MP_FROZEN_FAKE_DIR_SLASH_LENGTH) == 0) { - mp_import_stat_t st = mp_frozen_stat(path + MP_FROZEN_FAKE_DIR_SLASH_LENGTH); - if (st != MP_IMPORT_STAT_NO_EXIST) { - return st; - } + // Only try and load as a frozen module if it starts with .frozen/. + const int frozen_path_prefix_len = strlen(MP_FROZEN_PATH_PREFIX); + if (strncmp(path, MP_FROZEN_PATH_PREFIX, frozen_path_prefix_len) == 0) { + return mp_find_frozen_module(path + frozen_path_prefix_len, NULL, NULL); } #endif return mp_import_stat(path); } +// Given a path to a .py file, try and find this path as either a .py or .mpy +// in either the filesystem or frozen modules. STATIC mp_import_stat_t stat_file_py_or_mpy(vstr_t *path) { - mp_import_stat_t stat = mp_import_stat_any(vstr_null_terminated_str(path)); + mp_import_stat_t stat = stat_path_or_frozen(vstr_null_terminated_str(path)); if (stat == MP_IMPORT_STAT_FILE) { return stat; } #if MICROPY_PERSISTENT_CODE_LOAD + // Didn't find .py -- try the .mpy instead by inserting an 'm' into the '.py'. vstr_ins_byte(path, path->len - 2, 'm'); - stat = mp_import_stat_any(vstr_null_terminated_str(path)); + stat = stat_path_or_frozen(vstr_null_terminated_str(path)); if (stat == MP_IMPORT_STAT_FILE) { return stat; } @@ -91,8 +98,10 @@ STATIC mp_import_stat_t stat_file_py_or_mpy(vstr_t *path) { return MP_IMPORT_STAT_NO_EXIST; } +// Given an import path (e.g. "foo/bar"), try and find "foo/bar" (a directory) +// or "foo/bar.(m)py" in either the filesystem or frozen modules. STATIC mp_import_stat_t stat_dir_or_file(vstr_t *path) { - mp_import_stat_t stat = mp_import_stat_any(vstr_null_terminated_str(path)); + mp_import_stat_t stat = stat_path_or_frozen(vstr_null_terminated_str(path)); DEBUG_printf("stat %s: %d\n", vstr_str(path), stat); if (stat == MP_IMPORT_STAT_DIR) { return stat; @@ -103,40 +112,41 @@ STATIC mp_import_stat_t stat_dir_or_file(vstr_t *path) { return stat_file_py_or_mpy(path); } -STATIC mp_import_stat_t find_file(const char *file_str, uint file_len, vstr_t *dest) { +// Given a top-level module, try and find it in each of the sys.path entries +// via stat_dir_or_file. +STATIC mp_import_stat_t stat_top_level_dir_or_file(qstr mod_name, vstr_t *dest) { + DEBUG_printf("stat_top_level_dir_or_file: '%s'\n", qstr_str(mod_name)); #if MICROPY_PY_SYS - // extract the list of paths size_t path_num; mp_obj_t *path_items; mp_obj_list_get(mp_sys_path, &path_num, &path_items); - if (path_num == 0) { - #endif - // mp_sys_path is empty, so just use the given file name - vstr_add_strn(dest, file_str, file_len); - return stat_dir_or_file(dest); - #if MICROPY_PY_SYS -} else { - // go through each path looking for a directory or file - for (size_t i = 0; i < path_num; i++) { - vstr_reset(dest); - size_t p_len; - const char *p = mp_obj_str_get_data(path_items[i], &p_len); - if (p_len > 0) { - vstr_add_strn(dest, p, p_len); - vstr_add_char(dest, PATH_SEP_CHAR); + if (path_num > 0) { + // go through each path looking for a directory or file + for (size_t i = 0; i < path_num; i++) { + vstr_reset(dest); + size_t p_len; + const char *p = mp_obj_str_get_data(path_items[i], &p_len); + if (p_len > 0) { + vstr_add_strn(dest, p, p_len); + vstr_add_char(dest, PATH_SEP_CHAR[0]); + } + vstr_add_str(dest, qstr_str(mod_name)); + mp_import_stat_t stat = stat_dir_or_file(dest); + if (stat != MP_IMPORT_STAT_NO_EXIST) { + return stat; + } } - vstr_add_strn(dest, file_str, file_len); - mp_import_stat_t stat = stat_dir_or_file(dest); - if (stat != MP_IMPORT_STAT_NO_EXIST) { - return stat; - } - } - // could not find a directory or file - return MP_IMPORT_STAT_NO_EXIST; -} + // could not find a directory or file + return MP_IMPORT_STAT_NO_EXIST; + } #endif + + // mp_sys_path is empty (or not enabled), so just stat the given path + // directly. + vstr_add_str(dest, qstr_str(mod_name)); + return stat_dir_or_file(dest); } #if MICROPY_MODULE_FROZEN_STR || MICROPY_ENABLE_COMPILER @@ -193,19 +203,17 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, co STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { #if MICROPY_MODULE_FROZEN || MICROPY_ENABLE_COMPILER || (MICROPY_PERSISTENT_CODE_LOAD && MICROPY_HAS_FILE_READER) - char *file_str = vstr_null_terminated_str(file); + const char *file_str = vstr_null_terminated_str(file); #endif - #if MICROPY_MODULE_FROZEN || MICROPY_MODULE_FROZEN_MPY - if (strncmp(MP_FROZEN_FAKE_DIR_SLASH, - file_str, - MP_FROZEN_FAKE_DIR_SLASH_LENGTH) == 0) { - // If we support frozen modules (either as str or mpy) then try to find the - // requested filename in the list of frozen module filenames. - #if MICROPY_MODULE_FROZEN - void *modref; - int frozen_type = mp_find_frozen_module(file_str + MP_FROZEN_FAKE_DIR_SLASH_LENGTH, file->len - MP_FROZEN_FAKE_DIR_SLASH_LENGTH, &modref); - #endif + // If we support frozen modules (either as str or mpy) then try to find the + // requested filename in the list of frozen module filenames. + #if MICROPY_MODULE_FROZEN + void *modref; + int frozen_type; + const int frozen_path_prefix_len = strlen(MP_FROZEN_PATH_PREFIX); + if (strncmp(file_str, MP_FROZEN_PATH_PREFIX, frozen_path_prefix_len) == 0) { + mp_find_frozen_module(file_str + frozen_path_prefix_len, &frozen_type, &modref); // If we support frozen str modules and the compiler is enabled, and we // found the filename in the list of frozen files, then load and execute it. @@ -220,13 +228,12 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { // its data) in the list of frozen files, execute it. #if MICROPY_MODULE_FROZEN_MPY if (frozen_type == MP_FROZEN_MPY) { - do_execute_raw_code(module_obj, modref, file_str); + do_execute_raw_code(module_obj, modref, file_str + frozen_path_prefix_len); return; } #endif - } - #endif // MICROPY_MODULE_FROZEN || MICROPY_MODULE_FROZEN_MPY + #endif // MICROPY_MODULE_FROZEN // If we support loading .mpy files then check if the file extension is of // the correct format and, if so, load and execute the file. @@ -251,15 +258,212 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { #endif } -STATIC void chop_component(const char *start, const char **end) { - const char *p = *end; - while (p > start) { +// Convert a relative (to the current module) import, going up "level" levels, +// into an absolute import. +STATIC void evaluate_relative_import(mp_int_t level, const char **module_name, size_t *module_name_len) { + // What we want to do here is to take the name of the current module, + // remove trailing components, and concatenate the passed-in + // module name. + // For example, level=3, module_name="foo.bar", __name__="a.b.c.d" --> "a.foo.bar" + // "Relative imports use a module's __name__ attribute to determine that + // module's position in the package hierarchy." + // http://legacy.python.org/dev/peps/pep-0328/#relative-imports-and-name + + mp_obj_t current_module_name_obj = mp_obj_dict_get(MP_OBJ_FROM_PTR(mp_globals_get()), MP_OBJ_NEW_QSTR(MP_QSTR___name__)); + assert(current_module_name_obj != MP_OBJ_NULL); + + #if MICROPY_MODULE_OVERRIDE_MAIN_IMPORT && MICROPY_CPYTHON_COMPAT + if (MP_OBJ_QSTR_VALUE(current_module_name_obj) == MP_QSTR___main__) { + // This is a module loaded by -m command-line switch (e.g. unix port), + // and so its __name__ has been set to "__main__". Get its real name + // that we stored during import in the __main__ attribute. + current_module_name_obj = mp_obj_dict_get(MP_OBJ_FROM_PTR(mp_globals_get()), MP_OBJ_NEW_QSTR(MP_QSTR___main__)); + } + #endif + + // If we have a __path__ in the globals dict, then we're a package. + bool is_pkg = mp_map_lookup(&mp_globals_get()->map, MP_OBJ_NEW_QSTR(MP_QSTR___path__), MP_MAP_LOOKUP); + + #if DEBUG_PRINT + DEBUG_printf("Current module/package: "); + mp_obj_print_helper(MICROPY_DEBUG_PRINTER, current_module_name_obj, PRINT_REPR); + DEBUG_printf(", is_package: %d", is_pkg); + DEBUG_printf("\n"); + #endif + + size_t current_module_name_len; + const char *current_module_name = mp_obj_str_get_data(current_module_name_obj, ¤t_module_name_len); + + const char *p = current_module_name + current_module_name_len; + if (is_pkg) { + // If we're evaluating relative to a package, then take off one fewer + // level (i.e. the relative search starts inside the package, rather + // than as a sibling of the package). + --level; + } + + // Walk back 'level' dots (or run out of path). + while (level && p > current_module_name) { if (*--p == '.') { - *end = p; - return; + --level; } } - *end = p; + + // We must have some component left over to import from. + if (p == current_module_name) { + mp_raise_msg(&mp_type_ImportError, MP_ERROR_TEXT("can't perform relative import")); + } + + // New length is len("."). Note: might be one byte + // more than we need if module_name is empty (for the extra . we will + // append). + uint new_module_name_len = (size_t)(p - current_module_name) + 1 + *module_name_len; + char *new_mod = mp_local_alloc(new_module_name_len); + memcpy(new_mod, current_module_name, p - current_module_name); + + // Only append "." if there was one). + if (*module_name_len != 0) { + new_mod[p - current_module_name] = '.'; + memcpy(new_mod + (p - current_module_name) + 1, *module_name, *module_name_len); + } else { + --new_module_name_len; + } + + // Copy into a QSTR. + qstr new_mod_q = qstr_from_strn(new_mod, new_module_name_len); + mp_local_free(new_mod); + + DEBUG_printf("Resolved base name for relative import: '%s'\n", qstr_str(new_mod_q)); + *module_name = qstr_str(new_mod_q); + *module_name_len = new_module_name_len; +} + +// Load a module at the specified absolute path, possibly as a submodule of the given outer module. +// full_mod_name: The full absolute path to this module (e.g. "foo.bar.baz"). +// level_mod_name: The final component of the path (e.g. "baz"). +// outer_module_obj: The parent module (we need to store this module as an +// attribute on it) (or MP_OBJ_NULL for top-level). +// path: The filesystem path where we found the parent module +// (or empty for a top level module). +// override_main: Whether to set the __name__ to "__main__" (and use __main__ +// for the actual path). +STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, mp_obj_t outer_module_obj, vstr_t *path, bool override_main) { + mp_import_stat_t stat = MP_IMPORT_STAT_NO_EXIST; + + // Exact-match of built-in (or already-loaded) takes priority. + mp_obj_t module_obj = mp_module_get_loaded_or_builtin(full_mod_name); + + // Even if we find the module, go through the motions of searching for it + // because we may actually be in the process of importing a sub-module. + // So we need to (re-)find the correct path to be finding the sub-module + // on the next iteration of process_import_at_level. + + if (outer_module_obj == MP_OBJ_NULL) { + DEBUG_printf("Searching for top-level module\n"); + + // First module in the dotted-name; search for a directory or file + // relative to all the locations in sys.path. + stat = stat_top_level_dir_or_file(full_mod_name, path); + + // If the module "foo" doesn't exist on the filesystem, and it's not a + // builtin, try and find "ufoo" as a built-in. (This feature was + // formerly known as "weak links"). + #if MICROPY_MODULE_WEAK_LINKS + if (stat == MP_IMPORT_STAT_NO_EXIST && module_obj == MP_OBJ_NULL) { + char *umodule_buf = vstr_str(path); + umodule_buf[0] = 'u'; + strcpy(umodule_buf + 1, qstr_str(level_mod_name)); + qstr umodule_name = qstr_from_str(umodule_buf); + module_obj = mp_module_get_builtin(umodule_name); + } + #endif + } else { + DEBUG_printf("Searching for sub-module\n"); + + // Add the current part of the module name to the path. + vstr_add_char(path, PATH_SEP_CHAR[0]); + vstr_add_str(path, qstr_str(level_mod_name)); + + // Because it's not top level, we already know which path the parent was found in. + stat = stat_dir_or_file(path); + } + DEBUG_printf("Current path: %.*s\n", (int)vstr_len(path), vstr_str(path)); + + if (module_obj == MP_OBJ_NULL) { + // Not a built-in and not already-loaded. + + if (stat == MP_IMPORT_STAT_NO_EXIST) { + // And the file wasn't found -- fail. + #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE + mp_raise_msg(&mp_type_ImportError, MP_ERROR_TEXT("module not found")); + #else + mp_raise_msg_varg(&mp_type_ImportError, MP_ERROR_TEXT("no module named '%q'"), full_mod_name); + #endif + } + + // Not a built-in but found on the filesystem, try and load it. + + DEBUG_printf("Found path: %.*s\n", (int)vstr_len(path), vstr_str(path)); + + // Prepare for loading from the filesystem. Create a new shell module. + module_obj = mp_obj_new_module(full_mod_name); + + #if MICROPY_MODULE_OVERRIDE_MAIN_IMPORT + // If this module is being loaded via -m on unix, then + // override __name__ to "__main__". Do this only for *modules* + // however - packages never have their names replaced, instead + // they're -m'ed using a special __main__ submodule in them. (This all + // apparently is done to not touch the package name itself, which is + // important for future imports). + if (override_main && stat != MP_IMPORT_STAT_DIR) { + mp_obj_module_t *o = MP_OBJ_TO_PTR(module_obj); + mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR___main__)); + #if MICROPY_CPYTHON_COMPAT + // Store module as "__main__" in the dictionary of loaded modules (returned by sys.modules). + mp_obj_dict_store(MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_loaded_modules_dict)), MP_OBJ_NEW_QSTR(MP_QSTR___main__), module_obj); + // Store real name in "__main__" attribute. Need this for + // resolving relative imports later. "__main__ was chosen + // semi-randonly, to reuse existing qstr's. + mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___main__), MP_OBJ_NEW_QSTR(full_mod_name)); + #endif + } + #endif // MICROPY_MODULE_OVERRIDE_MAIN_IMPORT + + if (stat == MP_IMPORT_STAT_DIR) { + // Directory -- execute "path/__init__.py". + DEBUG_printf("%.*s is dir\n", (int)vstr_len(path), vstr_str(path)); + // Store the __path__ attribute onto this module. + // https://docs.python.org/3/reference/import.html + // "Specifically, any module that contains a __path__ attribute is considered a package." + mp_store_attr(module_obj, MP_QSTR___path__, mp_obj_new_str(vstr_str(path), vstr_len(path))); + size_t orig_path_len = path->len; + vstr_add_str(path, PATH_SEP_CHAR "__init__.py"); + if (stat_file_py_or_mpy(path) == MP_IMPORT_STAT_FILE) { + do_load(module_obj, path); + } else { + // No-op. Nothing to load. + // mp_warning("%s is imported as namespace package", vstr_str(&path)); + } + // Remove /__init__.py suffix. + path->len = orig_path_len; + } else { // MP_IMPORT_STAT_FILE + // File -- execute "path.(m)py". + do_load(module_obj, path); + // Note: This should be the last component in the import path. If + // there are remaining components then it's an ImportError + // because the current path(the module that was just loaded) is + // not a package. This will be caught on the next iteration + // because the file will not exist. + } + } + + if (outer_module_obj != MP_OBJ_NULL && VERIFY_PTR(MP_OBJ_TO_PTR(outer_module_obj))) { + // If it's a sub-module (not a built-in one), then make it available on + // the parent module. + mp_store_attr(outer_module_obj, level_mod_name, module_obj); + } + + return module_obj; } mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { @@ -267,14 +471,28 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { DEBUG_printf("__import__:\n"); for (size_t i = 0; i < n_args; i++) { DEBUG_printf(" "); - mp_obj_print(args[i], PRINT_REPR); + mp_obj_print_helper(MICROPY_DEBUG_PRINTER, args[i], PRINT_REPR); DEBUG_printf("\n"); } #endif - mp_obj_t module_name = args[0]; + // This is the import path, with any leading dots stripped. + // "import foo.bar" --> module_name="foo.bar" + // "from foo.bar import baz" --> module_name="foo.bar" + // "from . import foo" --> module_name="" + // "from ...foo.bar import baz" --> module_name="foo.bar" + mp_obj_t module_name_obj = args[0]; + + // These are the imported names. + // i.e. "from foo.bar import baz, zap" --> fromtuple=("baz", "zap",) + // Note: There's a special case on the Unix port, where this is set to mp_const_false which means that it's __main__. mp_obj_t fromtuple = mp_const_none; + + // Level is the number of leading dots in a relative import. + // i.e. "from . import foo" --> level=1 + // i.e. "from ...foo.bar import baz" --> level=3 mp_int_t level = 0; + if (n_args >= 4) { fromtuple = args[3]; if (n_args >= 5) { @@ -285,226 +503,64 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { } } - size_t mod_len; - const char *mod_str = mp_obj_str_get_data(module_name, &mod_len); + size_t module_name_len; + const char *module_name = mp_obj_str_get_data(module_name_obj, &module_name_len); if (level != 0) { - // What we want to do here is to take name of current module, - // chop trailing components, and concatenate with passed-in - // module name, thus resolving relative import name into absolute. - // This even appears to be correct per - // http://legacy.python.org/dev/peps/pep-0328/#relative-imports-and-name - // "Relative imports use a module's __name__ attribute to determine that - // module's position in the package hierarchy." - level--; - mp_obj_t this_name_q = mp_obj_dict_get(MP_OBJ_FROM_PTR(mp_globals_get()), MP_OBJ_NEW_QSTR(MP_QSTR___name__)); - assert(this_name_q != MP_OBJ_NULL); - #if MICROPY_CPYTHON_COMPAT - if (MP_OBJ_QSTR_VALUE(this_name_q) == MP_QSTR___main__) { - // This is a module run by -m command-line switch, get its real name from backup attribute - this_name_q = mp_obj_dict_get(MP_OBJ_FROM_PTR(mp_globals_get()), MP_OBJ_NEW_QSTR(MP_QSTR___main__)); - } - #endif - mp_map_t *globals_map = &mp_globals_get()->map; - mp_map_elem_t *elem = mp_map_lookup(globals_map, MP_OBJ_NEW_QSTR(MP_QSTR___path__), MP_MAP_LOOKUP); - bool is_pkg = (elem != NULL); - - #if DEBUG_PRINT - DEBUG_printf("Current module/package: "); - mp_obj_print(this_name_q, PRINT_REPR); - DEBUG_printf(", is_package: %d", is_pkg); - DEBUG_printf("\n"); - #endif - - size_t this_name_l; - const char *this_name = mp_obj_str_get_data(this_name_q, &this_name_l); - - const char *p = this_name + this_name_l; - if (!is_pkg) { - // We have module, but relative imports are anchored at package, so - // go there. - chop_component(this_name, &p); - } - - while (level--) { - chop_component(this_name, &p); - } - - // We must have some component left over to import from - if (p == this_name) { - mp_raise_ImportError(MP_ERROR_TEXT("cannot perform relative import")); - } - - uint new_mod_l = (mod_len == 0 ? (size_t)(p - this_name) : (size_t)(p - this_name) + 1 + mod_len); - char *new_mod = mp_local_alloc(new_mod_l); - memcpy(new_mod, this_name, p - this_name); - if (mod_len != 0) { - new_mod[p - this_name] = '.'; - memcpy(new_mod + (p - this_name) + 1, mod_str, mod_len); - } - - qstr new_mod_q = qstr_from_strn(new_mod, new_mod_l); - mp_local_free(new_mod); - DEBUG_printf("Resolved base name for relative import: '%s'\n", qstr_str(new_mod_q)); - module_name = MP_OBJ_NEW_QSTR(new_mod_q); - mod_str = qstr_str(new_mod_q); - mod_len = new_mod_l; + // Turn "foo.bar" into ".foo.bar". + evaluate_relative_import(level, &module_name, &module_name_len); } - if (mod_len == 0) { + if (module_name_len == 0) { mp_raise_ValueError(NULL); } - // check if module already exists - qstr module_name_qstr = mp_obj_str_get_qstr(module_name); - mp_obj_t module_obj = mp_module_get(module_name_qstr); - if (module_obj != MP_OBJ_NULL) { - DEBUG_printf("Module already loaded\n"); - // If it's not a package, return module right away - char *p = strchr(mod_str, '.'); - if (p == NULL) { - return module_obj; - } - // If fromlist is not empty, return leaf module - if (fromtuple != mp_const_none) { - return module_obj; - } - // Otherwise, we need to return top-level package - qstr pkg_name = qstr_from_strn(mod_str, p - mod_str); - return mp_module_get(pkg_name); - } - DEBUG_printf("Module not yet loaded\n"); + DEBUG_printf("Starting module search for '%s'\n", module_name); - uint last = 0; VSTR_FIXED(path, MICROPY_ALLOC_PATH_MAX) - module_obj = MP_OBJ_NULL; mp_obj_t top_module_obj = MP_OBJ_NULL; mp_obj_t outer_module_obj = MP_OBJ_NULL; - uint i; - for (i = 1; i <= mod_len; i++) { - if (i == mod_len || mod_str[i] == '.') { - // create a qstr for the module name up to this depth - qstr mod_name = qstr_from_strn(mod_str, i); - DEBUG_printf("Processing module: %s\n", qstr_str(mod_name)); - DEBUG_printf("Previous path: =%.*s=\n", vstr_len(&path), vstr_str(&path)); - // find the file corresponding to the module name - mp_import_stat_t stat; - if (vstr_len(&path) == 0) { - // first module in the dotted-name; search for a directory or file - DEBUG_printf("Find file =%.*s=\n", vstr_len(&path), vstr_str(&path)); - stat = find_file(mod_str, i, &path); - } else { - // latter module in the dotted-name; append to path - vstr_add_char(&path, PATH_SEP_CHAR); - vstr_add_strn(&path, mod_str + last, i - last); - stat = stat_dir_or_file(&path); - } - DEBUG_printf("Current path: %.*s\n", vstr_len(&path), vstr_str(&path)); + // Search for the end of each component. + size_t current_component_start = 0; + for (size_t i = 1; i <= module_name_len; i++) { + if (i == module_name_len || module_name[i] == '.') { + // The module name up to this depth (e.g. foo.bar.baz). + qstr full_mod_name = qstr_from_strn(module_name, i); + // The current level name (e.g. baz). + qstr level_mod_name = qstr_from_strn(module_name + current_component_start, i - current_component_start); - if (stat == MP_IMPORT_STAT_NO_EXIST) { - // This is just the module name after the previous . - qstr current_module_name = qstr_from_strn(mod_str + last, i - last); - mp_map_elem_t *el = NULL; - if (outer_module_obj == MP_OBJ_NULL) { - el = mp_map_lookup((mp_map_t *)&mp_builtin_module_map, - MP_OBJ_NEW_QSTR(current_module_name), - MP_MAP_LOOKUP); - } else { - el = mp_map_lookup(&((mp_obj_module_t *)outer_module_obj)->globals->map, - MP_OBJ_NEW_QSTR(current_module_name), - MP_MAP_LOOKUP); - } + DEBUG_printf("Processing module: '%s' at level '%s'\n", qstr_str(full_mod_name), qstr_str(level_mod_name)); + DEBUG_printf("Previous path: =%.*s=\n", (int)vstr_len(&path), vstr_str(&path)); - if (el != NULL && mp_obj_is_type(el->value, &mp_type_module)) { - module_obj = el->value; - mp_module_call_init(mod_name, module_obj); - } else { - // couldn't find the file, so fail - #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE - mp_raise_ImportError(MP_ERROR_TEXT("module not found")); - #else - mp_raise_msg_varg(&mp_type_ImportError, - MP_ERROR_TEXT("no module named '%q'"), mod_name); - #endif - } - } else { - // found the file, so get the module - module_obj = mp_module_get(mod_name); - } + #if MICROPY_MODULE_OVERRIDE_MAIN_IMPORT + // On unix, if this is being loaded via -m (magic mp_const_false), + // then handle that if it's the final component. + bool override_main = (i == module_name_len && fromtuple == mp_const_false); + #else + bool override_main = false; + #endif - if (module_obj == MP_OBJ_NULL) { - // module not already loaded, so load it! + // Import this module. + mp_obj_t module_obj = process_import_at_level(full_mod_name, level_mod_name, outer_module_obj, &path, override_main); - module_obj = mp_obj_new_module(mod_name); - - // if args[3] (fromtuple) has magic value False, set up - // this module for command-line "-m" option (set module's - // name to __main__ instead of real name). Do this only - // for *modules* however - packages never have their names - // replaced, instead they're -m'ed using a special __main__ - // submodule in them. (This all apparently is done to not - // touch package name itself, which is important for future - // imports). - if (i == mod_len && fromtuple == mp_const_false && stat != MP_IMPORT_STAT_DIR) { - mp_obj_module_t *o = MP_OBJ_TO_PTR(module_obj); - mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR___main__)); - #if MICROPY_CPYTHON_COMPAT - // Store module as "__main__" in the dictionary of loaded modules (returned by sys.modules). - mp_obj_dict_store(MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_loaded_modules_dict)), MP_OBJ_NEW_QSTR(MP_QSTR___main__), module_obj); - // Store real name in "__main__" attribute. Chosen semi-randonly, to reuse existing qstr's. - mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___main__), MP_OBJ_NEW_QSTR(mod_name)); - #endif - } - - if (stat == MP_IMPORT_STAT_DIR) { - DEBUG_printf("%.*s is dir\n", vstr_len(&path), vstr_str(&path)); - // https://docs.python.org/3/reference/import.html - // "Specifically, any module that contains a __path__ attribute is considered a package." - mp_store_attr(module_obj, MP_QSTR___path__, mp_obj_new_str(vstr_str(&path), vstr_len(&path))); - size_t orig_path_len = path.len; - vstr_add_char(&path, PATH_SEP_CHAR); - vstr_add_str(&path, "__init__.py"); - if (stat_file_py_or_mpy(&path) != MP_IMPORT_STAT_FILE) { - // mp_warning("%s is imported as namespace package", vstr_str(&path)); - } else { - do_load(module_obj, &path); - } - path.len = orig_path_len; - } else { // MP_IMPORT_STAT_FILE - do_load(module_obj, &path); - // This should be the last component in the import path. If there are - // remaining components then it's an ImportError because the current path - // (the module that was just loaded) is not a package. This will be caught - // on the next iteration because the file will not exist. - } - - // Loading a module thrashes the heap significantly so we explicitly clean up - // afterwards. - gc_collect(); - } - if (outer_module_obj != MP_OBJ_NULL && VERIFY_PTR(MP_OBJ_TO_PTR(outer_module_obj))) { - qstr s = qstr_from_strn(mod_str + last, i - last); - mp_store_attr(outer_module_obj, s, module_obj); - // The above store can cause a dictionary rehash and new allocation. So, - // lets make sure the globals dictionary is still long lived. - mp_obj_module_set_globals(outer_module_obj, - make_dict_long_lived(mp_obj_module_get_globals(outer_module_obj), 10)); - } + // Set this as the parent module, and remember the top-level module if it's the first. outer_module_obj = module_obj; if (top_module_obj == MP_OBJ_NULL) { top_module_obj = module_obj; } - last = i + 1; + + current_component_start = i + 1; } } - // If fromlist is not empty, return leaf module if (fromtuple != mp_const_none) { - return module_obj; + // If fromtuple is not empty, return leaf module + return outer_module_obj; + } else { + // Otherwise, we need to return top-level package + return top_module_obj; } - // Otherwise, we need to return top-level package - return top_module_obj; } #else // MICROPY_ENABLE_EXTERNAL_IMPORT @@ -517,17 +573,19 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { // Check if module already exists, and return it if it does qstr module_name_qstr = mp_obj_str_get_qstr(args[0]); - mp_obj_t module_obj = mp_module_get(module_name_qstr); + mp_obj_t module_obj = mp_module_get_loaded_or_builtin(module_name_qstr); if (module_obj != MP_OBJ_NULL) { return module_obj; } #if MICROPY_MODULE_WEAK_LINKS // Check if there is a weak link to this module - module_obj = mp_module_search_umodule(qstr_str(module_name_qstr)); + char umodule_buf[MICROPY_ALLOC_PATH_MAX]; + umodule_buf[0] = 'u'; + strcpy(umodule_buf + 1, args[0]); + qstr umodule_name_qstr = qstr_from_str(umodule_buf); + module_obj = mp_module_get_loaded_or_builtin(umodule_name_qstr); if (module_obj != MP_OBJ_NULL) { - // Found weak-linked module - mp_module_call_init(module_name_qstr, module_obj); return module_obj; } #endif diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h index 2893df46ec..982dc2681a 100644 --- a/py/circuitpy_mpconfig.h +++ b/py/circuitpy_mpconfig.h @@ -541,5 +541,16 @@ void supervisor_run_background_tasks_if_tick(void); #define USB_MIDI_EP_NUM_IN (0) #endif +#ifndef MICROPY_WRAP_MP_MAP_LOOKUP +#define MICROPY_WRAP_MP_MAP_LOOKUP PLACE_IN_ITCM +#endif + +#ifndef MICROPY_WRAP_MP_BINARY_OP +#define MICROPY_WRAP_MP_BINARY_OP PLACE_IN_ITCM +#endif + +#ifndef MICROPY_WRAP_MP_EXECUTE_BYTECODE +#define MICROPY_WRAP_MP_EXECUTE_BYTECODE PLACE_IN_ITCM +#endif #endif // __INCLUDED_MPCONFIG_CIRCUITPY_H diff --git a/py/compile.c b/py/compile.c index bdedc874e3..e5f341a656 100644 --- a/py/compile.c +++ b/py/compile.c @@ -61,6 +61,12 @@ typedef enum { #undef DEF_RULE_NC } pn_kind_t; +// Whether a mp_parse_node_struct_t that has pns->kind == PN_testlist_comp +// corresponds to a list comprehension or generator. +#define MP_PARSE_NODE_TESTLIST_COMP_HAS_COMP_FOR(pns) \ + (MP_PARSE_NODE_STRUCT_NUM_NODES(pns) == 2 && \ + MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[1], PN_comp_for)) + #define NEED_METHOD_TABLE MICROPY_EMIT_NATIVE #if NEED_METHOD_TABLE @@ -319,25 +325,13 @@ STATIC void compile_delete_id(compiler_t *comp, qstr qst) { } } -STATIC void c_tuple(compiler_t *comp, mp_parse_node_t pn, mp_parse_node_struct_t *pns_list) { - int total = 0; - if (!MP_PARSE_NODE_IS_NULL(pn)) { - compile_node(comp, pn); - total += 1; - } - if (pns_list != NULL) { - int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns_list); - for (int i = 0; i < n; i++) { - compile_node(comp, pns_list->nodes[i]); - } - total += n; - } - EMIT_ARG(build, total, MP_EMIT_BUILD_TUPLE); -} - STATIC void compile_generic_tuple(compiler_t *comp, mp_parse_node_struct_t *pns) { // a simple tuple expression - c_tuple(comp, MP_PARSE_NODE_NULL, pns); + size_t num_nodes = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); + for (size_t i = 0; i < num_nodes; i++) { + compile_node(comp, pns->nodes[i]); + } + EMIT_ARG(build, num_nodes, MP_EMIT_BUILD_TUPLE); } STATIC void c_if_cond(compiler_t *comp, mp_parse_node_t pn, bool jump_if, int label) { @@ -454,21 +448,14 @@ STATIC void c_assign_atom_expr(compiler_t *comp, mp_parse_node_struct_t *pns, as compile_syntax_error(comp, (mp_parse_node_t)pns, MP_ERROR_TEXT("can't assign to expression")); } -// we need to allow for a caller passing in 1 initial node (node_head) followed by an array of nodes (nodes_tail) -STATIC void c_assign_tuple(compiler_t *comp, mp_parse_node_t node_head, uint num_tail, mp_parse_node_t *nodes_tail) { - uint num_head = (node_head == MP_PARSE_NODE_NULL) ? 0 : 1; - +STATIC void c_assign_tuple(compiler_t *comp, uint num_tail, mp_parse_node_t *nodes_tail) { // look for star expression uint have_star_index = -1; - if (num_head != 0 && MP_PARSE_NODE_IS_STRUCT_KIND(node_head, PN_star_expr)) { - EMIT_ARG(unpack_ex, 0, num_tail); - have_star_index = 0; - } for (uint i = 0; i < num_tail; i++) { if (MP_PARSE_NODE_IS_STRUCT_KIND(nodes_tail[i], PN_star_expr)) { if (have_star_index == (uint)-1) { - EMIT_ARG(unpack_ex, num_head + i, num_tail - i - 1); - have_star_index = num_head + i; + EMIT_ARG(unpack_ex, i, num_tail - i - 1); + have_star_index = i; } else { compile_syntax_error(comp, nodes_tail[i], MP_ERROR_TEXT("multiple *x in assignment")); return; @@ -476,17 +463,10 @@ STATIC void c_assign_tuple(compiler_t *comp, mp_parse_node_t node_head, uint num } } if (have_star_index == (uint)-1) { - EMIT_ARG(unpack_sequence, num_head + num_tail); - } - if (num_head != 0) { - if (0 == have_star_index) { - c_assign(comp, ((mp_parse_node_struct_t *)node_head)->nodes[0], ASSIGN_STORE); - } else { - c_assign(comp, node_head, ASSIGN_STORE); - } + EMIT_ARG(unpack_sequence, num_tail); } for (uint i = 0; i < num_tail; i++) { - if (num_head + i == have_star_index) { + if (i == have_star_index) { c_assign(comp, ((mp_parse_node_struct_t *)nodes_tail[i])->nodes[0], ASSIGN_STORE); } else { c_assign(comp, nodes_tail[i], ASSIGN_STORE); @@ -528,7 +508,7 @@ STATIC void c_assign(compiler_t *comp, mp_parse_node_t pn, assign_kind_t assign_ if (assign_kind != ASSIGN_STORE) { goto cannot_assign; } - c_assign_tuple(comp, MP_PARSE_NODE_NULL, MP_PARSE_NODE_STRUCT_NUM_NODES(pns), pns->nodes); + c_assign_tuple(comp, MP_PARSE_NODE_STRUCT_NUM_NODES(pns), pns->nodes); break; case PN_atom_paren: @@ -553,13 +533,13 @@ STATIC void c_assign(compiler_t *comp, mp_parse_node_t pn, assign_kind_t assign_ } if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) { // empty list, assignment allowed - c_assign_tuple(comp, MP_PARSE_NODE_NULL, 0, NULL); + c_assign_tuple(comp, 0, NULL); } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_comp)) { pns = (mp_parse_node_struct_t *)pns->nodes[0]; goto testlist_comp; } else { // brackets around 1 item - c_assign_tuple(comp, pns->nodes[0], 0, NULL); + c_assign_tuple(comp, 1, pns->nodes); } break; @@ -570,27 +550,10 @@ STATIC void c_assign(compiler_t *comp, mp_parse_node_t pn, assign_kind_t assign_ testlist_comp: // lhs is a sequence - if (MP_PARSE_NODE_IS_STRUCT(pns->nodes[1])) { - mp_parse_node_struct_t *pns2 = (mp_parse_node_struct_t *)pns->nodes[1]; - if (MP_PARSE_NODE_STRUCT_KIND(pns2) == PN_testlist_comp_3b) { - // sequence of one item, with trailing comma - assert(MP_PARSE_NODE_IS_NULL(pns2->nodes[0])); - c_assign_tuple(comp, pns->nodes[0], 0, NULL); - } else if (MP_PARSE_NODE_STRUCT_KIND(pns2) == PN_testlist_comp_3c) { - // sequence of many items - uint n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns2); - c_assign_tuple(comp, pns->nodes[0], n, pns2->nodes); - } else if (MP_PARSE_NODE_STRUCT_KIND(pns2) == PN_comp_for) { - goto cannot_assign; - } else { - // sequence with 2 items - goto sequence_with_2_items; - } - } else { - // sequence with 2 items - sequence_with_2_items: - c_assign_tuple(comp, MP_PARSE_NODE_NULL, 2, pns->nodes); + if (MP_PARSE_NODE_TESTLIST_COMP_HAS_COMP_FOR(pns)) { + goto cannot_assign; } + c_assign_tuple(comp, MP_PARSE_NODE_STRUCT_NUM_NODES(pns), pns->nodes); return; } return; @@ -993,32 +956,11 @@ STATIC void c_del_stmt(compiler_t *comp, mp_parse_node_t pn) { } else { assert(MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_testlist_comp)); mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn; - // TODO perhaps factorise testlist_comp code with other uses of PN_testlist_comp - - if (MP_PARSE_NODE_IS_STRUCT(pns->nodes[1])) { - mp_parse_node_struct_t *pns1 = (mp_parse_node_struct_t *)pns->nodes[1]; - if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_testlist_comp_3b) { - // sequence of one item, with trailing comma - assert(MP_PARSE_NODE_IS_NULL(pns1->nodes[0])); - c_del_stmt(comp, pns->nodes[0]); - } else if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_testlist_comp_3c) { - // sequence of many items - int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns1); - c_del_stmt(comp, pns->nodes[0]); - for (int i = 0; i < n; i++) { - c_del_stmt(comp, pns1->nodes[i]); - } - } else if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_comp_for) { - goto cannot_delete; - } else { - // sequence with 2 items - goto sequence_with_2_items; - } - } else { - // sequence with 2 items - sequence_with_2_items: - c_del_stmt(comp, pns->nodes[0]); - c_del_stmt(comp, pns->nodes[1]); + if (MP_PARSE_NODE_TESTLIST_COMP_HAS_COMP_FOR(pns)) { + goto cannot_delete; + } + for (size_t i = 0; i < MP_PARSE_NODE_STRUCT_NUM_NODES(pns); ++i) { + c_del_stmt(comp, pns->nodes[i]); } } } else { @@ -2502,31 +2444,16 @@ STATIC void compile_comprehension(compiler_t *comp, mp_parse_node_struct_t *pns, STATIC void compile_atom_paren(compiler_t *comp, mp_parse_node_struct_t *pns) { if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) { // an empty tuple - c_tuple(comp, MP_PARSE_NODE_NULL, NULL); + EMIT_ARG(build, 0, MP_EMIT_BUILD_TUPLE); } else { assert(MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_comp)); pns = (mp_parse_node_struct_t *)pns->nodes[0]; - assert(!MP_PARSE_NODE_IS_NULL(pns->nodes[1])); - if (MP_PARSE_NODE_IS_STRUCT(pns->nodes[1])) { - mp_parse_node_struct_t *pns2 = (mp_parse_node_struct_t *)pns->nodes[1]; - if (MP_PARSE_NODE_STRUCT_KIND(pns2) == PN_testlist_comp_3b) { - // tuple of one item, with trailing comma - assert(MP_PARSE_NODE_IS_NULL(pns2->nodes[0])); - c_tuple(comp, pns->nodes[0], NULL); - } else if (MP_PARSE_NODE_STRUCT_KIND(pns2) == PN_testlist_comp_3c) { - // tuple of many items - c_tuple(comp, pns->nodes[0], pns2); - } else if (MP_PARSE_NODE_STRUCT_KIND(pns2) == PN_comp_for) { - // generator expression - compile_comprehension(comp, pns, SCOPE_GEN_EXPR); - } else { - // tuple with 2 items - goto tuple_with_2_items; - } + if (MP_PARSE_NODE_TESTLIST_COMP_HAS_COMP_FOR(pns)) { + // generator expression + compile_comprehension(comp, pns, SCOPE_GEN_EXPR); } else { - // tuple with 2 items - tuple_with_2_items: - c_tuple(comp, MP_PARSE_NODE_NULL, pns); + // tuple with N items + compile_generic_tuple(comp, pns); } } } @@ -2537,31 +2464,13 @@ STATIC void compile_atom_bracket(compiler_t *comp, mp_parse_node_struct_t *pns) EMIT_ARG(build, 0, MP_EMIT_BUILD_LIST); } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_comp)) { mp_parse_node_struct_t *pns2 = (mp_parse_node_struct_t *)pns->nodes[0]; - if (MP_PARSE_NODE_IS_STRUCT(pns2->nodes[1])) { - mp_parse_node_struct_t *pns3 = (mp_parse_node_struct_t *)pns2->nodes[1]; - if (MP_PARSE_NODE_STRUCT_KIND(pns3) == PN_testlist_comp_3b) { - // list of one item, with trailing comma - assert(MP_PARSE_NODE_IS_NULL(pns3->nodes[0])); - compile_node(comp, pns2->nodes[0]); - EMIT_ARG(build, 1, MP_EMIT_BUILD_LIST); - } else if (MP_PARSE_NODE_STRUCT_KIND(pns3) == PN_testlist_comp_3c) { - // list of many items - compile_node(comp, pns2->nodes[0]); - compile_generic_all_nodes(comp, pns3); - EMIT_ARG(build, 1 + MP_PARSE_NODE_STRUCT_NUM_NODES(pns3), MP_EMIT_BUILD_LIST); - } else if (MP_PARSE_NODE_STRUCT_KIND(pns3) == PN_comp_for) { - // list comprehension - compile_comprehension(comp, pns2, SCOPE_LIST_COMP); - } else { - // list with 2 items - goto list_with_2_items; - } + if (MP_PARSE_NODE_TESTLIST_COMP_HAS_COMP_FOR(pns2)) { + // list comprehension + compile_comprehension(comp, pns2, SCOPE_LIST_COMP); } else { - // list with 2 items - list_with_2_items: - compile_node(comp, pns2->nodes[0]); - compile_node(comp, pns2->nodes[1]); - EMIT_ARG(build, 2, MP_EMIT_BUILD_LIST); + // list with N items + compile_generic_all_nodes(comp, pns2); + EMIT_ARG(build, MP_PARSE_NODE_STRUCT_NUM_NODES(pns2), MP_EMIT_BUILD_LIST); } } else { // list with 1 item diff --git a/py/dynruntime.h b/py/dynruntime.h index 0a7eb20397..eee88e395b 100644 --- a/py/dynruntime.h +++ b/py/dynruntime.h @@ -141,7 +141,7 @@ static inline mp_obj_t mp_obj_cast_to_native_base_dyn(mp_obj_t self_in, mp_const if (MP_OBJ_FROM_PTR(self_type) == native_type) { return self_in; } - mp_parent_t parent = mp_type_get_parent_slot(self_type); + const void *parent = mp_type_get_parent_slot(self_type); if (parent != native_type) { // The self_in object is not a direct descendant of native_type, so fail the cast. // This is a very simple version of mp_obj_is_subclass_fast that could be improved. diff --git a/py/dynruntime.mk b/py/dynruntime.mk index cb5ab845eb..db06d41e73 100644 --- a/py/dynruntime.mk +++ b/py/dynruntime.mk @@ -46,7 +46,6 @@ ifeq ($(ARCH),x86) # x86 CROSS = CFLAGS += -m32 -fno-stack-protector -MPY_CROSS_FLAGS += -mcache-lookup-bc MICROPY_FLOAT_IMPL ?= double else ifeq ($(ARCH),x64) @@ -54,7 +53,6 @@ else ifeq ($(ARCH),x64) # x64 CROSS = CFLAGS += -fno-stack-protector -MPY_CROSS_FLAGS += -mcache-lookup-bc MICROPY_FLOAT_IMPL ?= double else ifeq ($(ARCH),armv7m) diff --git a/py/emitbc.c b/py/emitbc.c index 1d798faf68..8f7b1d5b72 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -560,9 +560,6 @@ void mp_emit_bc_load_global(emit_t *emit, qstr qst, int kind) { MP_STATIC_ASSERT(MP_BC_LOAD_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_BC_LOAD_GLOBAL); (void)qst; emit_write_bytecode_byte_qstr(emit, 1, MP_BC_LOAD_NAME + kind, qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { - emit_write_bytecode_raw_byte(emit, 0); - } } void mp_emit_bc_load_method(emit_t *emit, qstr qst, bool is_super) { @@ -596,9 +593,6 @@ void mp_emit_bc_attr(emit_t *emit, qstr qst, int kind) { } emit_write_bytecode_byte_qstr(emit, -2, MP_BC_STORE_ATTR, qst); } - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { - emit_write_bytecode_raw_byte(emit, 0); - } } void mp_emit_bc_store_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) { diff --git a/py/emitnative.c b/py/emitnative.c index c5d9fecb4c..5946fcd341 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1588,6 +1588,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { int reg_base = REG_ARG_1; int reg_index = REG_ARG_2; emit_pre_pop_reg_flexible(emit, &vtype_base, ®_base, reg_index, reg_index); + need_reg_single(emit, REG_RET, 0); switch (vtype_base) { case VTYPE_PTR8: { // pointer to 8-bit memory @@ -1651,6 +1652,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { int reg_index = REG_ARG_2; emit_pre_pop_reg_flexible(emit, &vtype_index, ®_index, REG_ARG_1, REG_ARG_1); emit_pre_pop_reg(emit, &vtype_base, REG_ARG_1); + need_reg_single(emit, REG_RET, 0); if (vtype_index != VTYPE_INT && vtype_index != VTYPE_UINT) { EMIT_NATIVE_VIPER_TYPE_ERROR(emit, MP_ERROR_TEXT("can't load with '%q' index"), vtype_to_qstr(vtype_index)); diff --git a/py/frozenmod.c b/py/frozenmod.c index 6f8191d311..57e9d5df9b 100644 --- a/py/frozenmod.c +++ b/py/frozenmod.c @@ -5,6 +5,7 @@ * * Copyright (c) 2015 Paul Sokolovsky * SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George + * SPDX-FileCopyrightText: Copyright (c) 2021 Jim Mussared * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,6 +32,13 @@ #include "py/lexer.h" #include "py/frozenmod.h" +#if MICROPY_MODULE_FROZEN + +// Null-separated frozen file names. All string-type entries are listed first, +// followed by mpy-type entries. Use mp_frozen_str_sizes to determine how +// many string entries. +extern const char mp_frozen_names[]; + #if MICROPY_MODULE_FROZEN_STR #ifndef MICROPY_MODULE_FROZEN_LEXER @@ -39,129 +47,89 @@ mp_lexer_t *MICROPY_MODULE_FROZEN_LEXER(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len); #endif -extern const char mp_frozen_str_names[]; +// Size in bytes of each string entry, followed by a zero (terminator). extern const uint32_t mp_frozen_str_sizes[]; +// Null-separated string content. extern const char mp_frozen_str_content[]; - -// str_len is length of str. *len is set on on output to size of content -const char *mp_find_frozen_str(const char *str, size_t str_len, size_t *len) { - // If the frozen module pseudo dir (e.g., ".frozen/") is a prefix of str, remove it. - if (strncmp(str, MP_FROZEN_FAKE_DIR_SLASH, MP_FROZEN_FAKE_DIR_SLASH_LENGTH) == 0) { - str = str + MP_FROZEN_FAKE_DIR_SLASH_LENGTH; - str_len = str_len - MP_FROZEN_FAKE_DIR_SLASH_LENGTH; - } - - const char *name = mp_frozen_str_names; - - size_t offset = 0; - for (int i = 0; *name != 0; i++) { - size_t l = strlen(name); - if (l == str_len && !memcmp(str, name, l)) { - *len = mp_frozen_str_sizes[i]; - return mp_frozen_str_content + offset; - } - name += l + 1; - offset += mp_frozen_str_sizes[i] + 1; - } - return NULL; -} - -STATIC mp_lexer_t *mp_lexer_frozen_str(const char *str, size_t str_len) { - size_t file_len; - const char *content = mp_find_frozen_str(str, str_len, &file_len); - - if (content == NULL) { - return NULL; - } - - qstr source = qstr_from_strn(str, str_len); - mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(source, content, file_len, 0); - return lex; -} -#endif +#endif // MICROPY_MODULE_FROZEN_STR #if MICROPY_MODULE_FROZEN_MPY #include "py/emitglue.h" -extern const char mp_frozen_mpy_names[]; extern const mp_raw_code_t *const mp_frozen_mpy_content[]; -STATIC const mp_raw_code_t *mp_find_frozen_mpy(const char *str, size_t str_len) { - const char *name = mp_frozen_mpy_names; - for (size_t i = 0; *name != 0; i++) { - size_t l = strlen(name); - if (l == str_len && !memcmp(str, name, l)) { - return mp_frozen_mpy_content[i]; - } - name += l + 1; - } - return NULL; -} +#endif // MICROPY_MODULE_FROZEN_MPY -#endif - -#if MICROPY_MODULE_FROZEN - -STATIC mp_import_stat_t mp_frozen_stat_helper(const char *name, const char *str) { +// Search for "str" as a frozen entry, returning the stat result +// (no-exist/file/dir), as well as the type (none/str/mpy) and data. +// frozen_type can be NULL if its value isn't needed (and then data is assumed to be NULL). +mp_import_stat_t mp_find_frozen_module(const char *str, int *frozen_type, void **data) { size_t len = strlen(str); + const char *name = mp_frozen_names; + + if (frozen_type != NULL) { + *frozen_type = MP_FROZEN_NONE; + } + + // Count the number of str lengths we have to find how many str entries. + size_t num_str = 0; + #if MICROPY_MODULE_FROZEN_STR && MICROPY_MODULE_FROZEN_MPY + for (const uint32_t *s = mp_frozen_str_sizes; *s != 0; ++s) { + ++num_str; + } + #endif + + for (size_t i = 0; *name != 0; i++) { + size_t entry_len = strlen(name); + if (entry_len >= len && memcmp(str, name, len) == 0) { + // Query is a prefix of the current entry. + if (entry_len == len) { + // Exact match --> file. + + if (frozen_type != NULL) { + #if MICROPY_MODULE_FROZEN_STR + if (i < num_str) { + *frozen_type = MP_FROZEN_STR; + // Use the size table to figure out where this index starts. + size_t offset = 0; + for (size_t j = 0; j < i; ++j) { + offset += mp_frozen_str_sizes[j] + 1; + } + size_t content_len = mp_frozen_str_sizes[i]; + const char *content = &mp_frozen_str_content[offset]; + + // Note: str & len have been updated by find_frozen_entry to strip + // the ".frozen/" prefix (to avoid this being a distinct qstr to + // the original path QSTR in frozen_content.c). + qstr source = qstr_from_strn(str, len); + mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(source, content, content_len, 0); + *data = lex; + } + #endif + + #if MICROPY_MODULE_FROZEN_MPY + if (i >= num_str) { + *frozen_type = MP_FROZEN_MPY; + // Load the corresponding index as a raw_code, taking + // into account any string entries to offset by. + *data = (void *)mp_frozen_mpy_content[i - num_str]; + } + #endif + } - for (int i = 0; *name != 0; i++) { - size_t l = strlen(name); - if (l >= len && !memcmp(str, name, len)) { - if (name[len] == 0) { return MP_IMPORT_STAT_FILE; } else if (name[len] == '/') { + // Matches up to directory separator, this is a valid + // directory path. return MP_IMPORT_STAT_DIR; } } - name += l + 1; + // Skip null separator. + name += entry_len + 1; } - return MP_IMPORT_STAT_NO_EXIST; -} - -mp_import_stat_t mp_frozen_stat(const char *str) { - mp_import_stat_t stat; - - #if MICROPY_MODULE_FROZEN_STR - stat = mp_frozen_stat_helper(mp_frozen_str_names, str); - if (stat != MP_IMPORT_STAT_NO_EXIST) { - return stat; - } - #endif - - #if MICROPY_MODULE_FROZEN_MPY - stat = mp_frozen_stat_helper(mp_frozen_mpy_names, str); - if (stat != MP_IMPORT_STAT_NO_EXIST) { - return stat; - } - #endif return MP_IMPORT_STAT_NO_EXIST; } -int mp_find_frozen_module(const char *str, size_t len, void **data) { - // If the frozen module pseudo dir (e.g., ".frozen/") is a prefix of str, remove it. - if (strncmp(str, MP_FROZEN_FAKE_DIR_SLASH, MP_FROZEN_FAKE_DIR_SLASH_LENGTH) == 0) { - str = str + MP_FROZEN_FAKE_DIR_SLASH_LENGTH; - len = len - MP_FROZEN_FAKE_DIR_SLASH_LENGTH; - } - - #if MICROPY_MODULE_FROZEN_STR - mp_lexer_t *lex = mp_lexer_frozen_str(str, len); - if (lex != NULL) { - *data = lex; - return MP_FROZEN_STR; - } - #endif - #if MICROPY_MODULE_FROZEN_MPY - const mp_raw_code_t *rc = mp_find_frozen_mpy(str, len); - if (rc != NULL) { - *data = (void *)rc; - return MP_FROZEN_MPY; - } - #endif - return MP_FROZEN_NONE; -} - -#endif +#endif // MICROPY_MODULE_FROZEN diff --git a/py/frozenmod.h b/py/frozenmod.h index 51c86f29f8..0a907b8785 100644 --- a/py/frozenmod.h +++ b/py/frozenmod.h @@ -35,18 +35,6 @@ enum { MP_FROZEN_MPY, }; -// Frozen modules are in a pseudo-directory, so sys.path can control how they're found. -#define MP_FROZEN_FAKE_DIR ".frozen" -#define MP_FROZEN_FAKE_DIR_LENGTH (sizeof(MP_FROZEN_FAKE_DIR) - 1) - -#define MP_FROZEN_FAKE_DIR_SLASH (MP_FROZEN_FAKE_DIR "/") -#define MP_FROZEN_FAKE_DIR_SLASH_LENGTH (sizeof(MP_FROZEN_FAKE_DIR_SLASH) - 1) - -// This should match MP_FROZEN_FAKE_DIR. -#define MP_FROZEN_FAKE_DIR_QSTR MP_QSTR__dot_frozen - -int mp_find_frozen_module(const char *str, size_t len, void **data); -const char *mp_find_frozen_str(const char *str, size_t str_len, size_t *len); -mp_import_stat_t mp_frozen_stat(const char *str); +mp_import_stat_t mp_find_frozen_module(const char *str, int *frozen_type, void **data); #endif // MICROPY_INCLUDED_PY_FROZENMOD_H diff --git a/py/gc.c b/py/gc.c index 69ab969da7..826540d353 100644 --- a/py/gc.c +++ b/py/gc.c @@ -238,6 +238,7 @@ STATIC void gc_mark_subtree(size_t block) { // Start with the block passed in the argument. size_t sp = 0; for (;;) { + MICROPY_GC_HOOK_LOOP // work out number of consecutive blocks in the chain starting with this one size_t n_blocks = 0; do { @@ -247,6 +248,7 @@ STATIC void gc_mark_subtree(size_t block) { // check this block's children void **ptrs = (void **)PTR_FROM_BLOCK(block); for (size_t i = n_blocks * BYTES_PER_BLOCK / sizeof(void *); i > 0; i--, ptrs++) { + MICROPY_GC_HOOK_LOOP void *ptr = *ptrs; if (VERIFY_PTR(ptr)) { // Mark and push this pointer @@ -280,6 +282,7 @@ STATIC void gc_deal_with_stack_overflow(void) { // scan entire memory looking for blocks which have been marked but not their children for (size_t block = 0; block < MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB; block++) { + MICROPY_GC_HOOK_LOOP // trace (again) if mark bit set if (ATB_GET_KIND(block) == AT_MARK) { gc_mark_subtree(block); @@ -295,6 +298,7 @@ STATIC void gc_sweep(void) { // free unmarked heads and their tails int free_tail = 0; for (size_t block = 0; block < MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB; block++) { + MICROPY_GC_HOOK_LOOP switch (ATB_GET_KIND(block)) { case AT_HEAD: #if MICROPY_ENABLE_FINALISER @@ -407,6 +411,7 @@ static void *gc_get_ptr(void **ptrs, int i) { void gc_collect_root(void **ptrs, size_t len) { for (size_t i = 0; i < len; i++) { + MICROPY_GC_HOOK_LOOP void *ptr = gc_get_ptr(ptrs, i); gc_mark(ptr); } diff --git a/py/lexer.c b/py/lexer.c index ce48adbfa5..196f9a2644 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -365,9 +365,16 @@ STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) // (MicroPython limitation) note: this is completely unaware of // Python syntax and will not handle any expression containing '}' or ':'. // e.g. f'{"}"}' or f'{foo({})}'. - while (!is_end(lex) && !is_char_or(lex, ':', '}')) { + unsigned int nested_bracket_level = 0; + while (!is_end(lex) && (nested_bracket_level != 0 || !is_char_or(lex, ':', '}'))) { + unichar c = CUR_CHAR(lex); + if (c == '[' || c == '{') { + nested_bracket_level += 1; + } else if (c == ']' || c == '}') { + nested_bracket_level -= 1; + } // like the default case at the end of this function, stay 8-bit clean - vstr_add_byte(&lex->fstring_args, CUR_CHAR(lex)); + vstr_add_byte(&lex->fstring_args, c); next_char(lex); } if (lex->fstring_args.buf[lex->fstring_args.len - 1] == '=') { diff --git a/py/map.c b/py/map.c index dc5d4b061a..092adf94ee 100644 --- a/py/map.c +++ b/py/map.c @@ -42,6 +42,27 @@ #define DEBUG_printf(...) (void)0 #endif +#if MICROPY_OPT_MAP_LOOKUP_CACHE +// MP_STATE_VM(map_lookup_cache) provides a cache of index to the last known +// position of that index in any map. On a cache hit, this allows +// short-circuiting the full linear search in the case of an ordered map +// (i.e. all builtin modules and objects' locals dicts), and computation of +// the hash (and potentially some linear probing) in the case of a regular +// map. Note the same cache is shared across all maps. + +// Gets the index into the cache for this index. Shift down by two to remove +// mp_obj_t tag bits. +#define MAP_CACHE_OFFSET(index) ((((uintptr_t)(index)) >> 2) % MICROPY_OPT_MAP_LOOKUP_CACHE_SIZE) +// Gets the map cache entry for the corresponding index. +#define MAP_CACHE_ENTRY(index) (MP_STATE_VM(map_lookup_cache)[MAP_CACHE_OFFSET(index)]) +// Retrieve the mp_obj_t at the location suggested by the cache. +#define MAP_CACHE_GET(map, index) (&(map)->table[MAP_CACHE_ENTRY(index) % (map)->alloc]) +// Update the cache for this index. +#define MAP_CACHE_SET(index, pos) MAP_CACHE_ENTRY(index) = (pos) & 0xff; +#else +#define MAP_CACHE_SET(index, pos) +#endif + // This table of sizes is used to control the growth of hash tables. // The first set of sizes are chosen so the allocation fits exactly in a // 4-word GC block, and it's not so important for these small values to be @@ -134,10 +155,22 @@ STATIC void mp_map_rehash(mp_map_t *map) { // - returns slot, with key non-null and value=MP_OBJ_NULL if it was added // MP_MAP_LOOKUP_REMOVE_IF_FOUND behaviour: // - returns NULL if not found, else the slot if was found in with key null and value non-null -mp_map_elem_t *PLACE_IN_ITCM(mp_map_lookup)(mp_map_t * map, mp_obj_t index, mp_map_lookup_kind_t lookup_kind) { +mp_map_elem_t *MICROPY_WRAP_MP_MAP_LOOKUP(mp_map_lookup)(mp_map_t * map, mp_obj_t index, mp_map_lookup_kind_t lookup_kind) { // If the map is a fixed array then we must only be called for a lookup assert(!map->is_fixed || lookup_kind == MP_MAP_LOOKUP); + #if MICROPY_OPT_MAP_LOOKUP_CACHE + // Try the cache for lookup or add-if-not-found. + if (lookup_kind != MP_MAP_LOOKUP_REMOVE_IF_FOUND && map->alloc) { + mp_map_elem_t *slot = MAP_CACHE_GET(map, index); + // Note: Just comparing key for value equality will have false negatives, but + // these will be handled by the regular path below. + if (slot->key == index) { + return slot; + } + } + #endif + // Work out if we can compare just pointers bool compare_only_ptrs = map->all_keys_are_qstrs; if (compare_only_ptrs) { @@ -174,6 +207,7 @@ mp_map_elem_t *PLACE_IN_ITCM(mp_map_lookup)(mp_map_t * map, mp_obj_t index, mp_m elem->value = value; } #endif + MAP_CACHE_SET(index, elem - map->table); return elem; } } @@ -256,6 +290,7 @@ mp_map_elem_t *PLACE_IN_ITCM(mp_map_lookup)(mp_map_t * map, mp_obj_t index, mp_m } // keep slot->value so that caller can access it if needed } + MAP_CACHE_SET(index, pos); return slot; } diff --git a/py/mkrules.cmake b/py/mkrules.cmake index 9d08017931..cb5fdabf6b 100644 --- a/py/mkrules.cmake +++ b/py/mkrules.cmake @@ -10,6 +10,15 @@ set(MICROPY_QSTRDEFS_COLLECTED "${MICROPY_GENHDR_DIR}/qstrdefs.collected.h") set(MICROPY_QSTRDEFS_PREPROCESSED "${MICROPY_GENHDR_DIR}/qstrdefs.preprocessed.h") set(MICROPY_QSTRDEFS_GENERATED "${MICROPY_GENHDR_DIR}/qstrdefs.generated.h") +# Need to do this before extracting MICROPY_CPP_DEF below. Rest of frozen +# manifest handling is at the end of this file. +if(MICROPY_FROZEN_MANIFEST) + target_compile_definitions(${MICROPY_TARGET} PUBLIC + MICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool + MICROPY_MODULE_FROZEN_MPY=\(1\) + ) +endif() + # Provide defaults for preprocessor flags if not already defined if(NOT MICROPY_CPP_FLAGS) get_target_property(MICROPY_CPP_INC ${MICROPY_TARGET} INCLUDE_DIRECTORIES) @@ -120,10 +129,7 @@ if(MICROPY_FROZEN_MANIFEST) ${MICROPY_FROZEN_CONTENT} ) - target_compile_definitions(${MICROPY_TARGET} PUBLIC - MICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool - MICROPY_MODULE_FROZEN_MPY=\(1\) - ) + # Note: target_compile_definitions already added earlier. if(NOT MICROPY_LIB_DIR) set(MICROPY_LIB_DIR ${MICROPY_DIR}/../micropython-lib) diff --git a/py/mkrules.mk b/py/mkrules.mk index b121260c07..d306cfabeb 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -125,46 +125,18 @@ $(MICROPY_MPYCROSS_DEPENDENCY): $(MAKE) -C $(dir $@) endif +ifneq ($(FROZEN_DIR),) +$(error Support for FROZEN_DIR was removed. Please use manifest.py instead, see https://docs.micropython.org/en/latest/reference/manifest.html) +endif + +ifneq ($(FROZEN_MPY_DIR),) +$(error Support for FROZEN_MPY_DIR was removed. Please use manifest.py instead, see https://docs.micropython.org/en/latest/reference/manifest.html) +endif + ifneq ($(FROZEN_MANIFEST),) # to build frozen_content.c from a manifest $(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h | $(MICROPY_MPYCROSS_DEPENDENCY) $(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) --mpy-tool-flags="$(MPY_TOOL_FLAGS)" $(FROZEN_MANIFEST) - -ifneq ($(FROZEN_DIR),) -$(error FROZEN_DIR cannot be used in conjunction with FROZEN_MANIFEST) -endif - -ifneq ($(FROZEN_MPY_DIR),) -$(error FROZEN_MPY_DIR cannot be used in conjunction with FROZEN_MANIFEST) -endif -endif - -ifneq ($(FROZEN_DIR),) -$(info Warning: FROZEN_DIR is deprecated in favour of FROZEN_MANIFEST) -$(BUILD)/frozen.c: $(wildcard $(FROZEN_DIR)/*) $(HEADER_BUILD) $(FROZEN_EXTRA_DEPS) - $(STEPECHO) "Generating $@" - $(Q)$(MAKE_FROZEN) $(FROZEN_DIR) > $@ -endif - -ifneq ($(FROZEN_MPY_DIRS),) -# Copy all the modules and single python files to freeze to a common area, omitting top-level dirs (the repo names). -# Do any preprocessing necessary: currently, this adds version information, removes examples, and -# non-library .py files in the modules (setup.py and conf.py) -# Then compile .mpy files from all the .py files, placing them in the same directories as the .py files. -$(BUILD)/frozen_mpy: $(FROZEN_MPY_DIRS) - $(ECHO) FREEZE $(FROZEN_MPY_DIRS) - $(Q)$(MKDIR) -p $@ - $(Q)$(PREPROCESS_FROZEN_MODULES) -o $@ $(FROZEN_MPY_DIRS) - $(Q)$(CD) $@ && \ -$(FIND) -L . -type f -name '*.py' | sed 's=^\./==' | \ -xargs -n1 "$(abspath $(MICROPY_MPYCROSS_DEPENDENCY))" $(MPY_CROSS_FLAGS) - -# to build frozen_mpy.c from all .mpy files -# You need to define MPY_TOOL_LONGINT_IMPL in mpconfigport.mk -# if the default will not work (mpz is the default). -$(BUILD)/frozen_mpy.c: $(BUILD)/frozen_mpy $(BUILD)/genhdr/qstrdefs.generated.h $(TOP)/tools/mpy-tool.py - $(STEPECHO) "Creating $@" - $(Q)$(MPY_TOOL) $(MPY_TOOL_LONGINT_IMPL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(shell $(FIND) -L $(BUILD)/frozen_mpy -type f -name '*.mpy') > $@ endif ifneq ($(PROG),) @@ -220,27 +192,6 @@ clean: $(RM) -rf $(BUILD) $(CLEAN_EXTRA) .PHONY: clean -# Clean every non-git file from FROZEN_DIR/FROZEN_MPY_DIR, but making a backup. -# We run rmdir below to avoid empty backup dir (it will silently fail if backup -# is non-empty). -clean-frozen: - if [ -n "$(FROZEN_MPY_DIR)" ]; then \ - backup_dir=$(FROZEN_MPY_DIR).$$(date +%Y%m%dT%H%M%S); mkdir $$backup_dir; \ - cd $(FROZEN_MPY_DIR); git status --ignored -u all -s . | awk ' {print $$2}' \ - | xargs --no-run-if-empty cp --parents -t ../$$backup_dir; \ - rmdir ../$$backup_dir 2>/dev/null || true; \ - git clean -d -f .; \ - fi - - if [ -n "$(FROZEN_DIR)" ]; then \ - backup_dir=$(FROZEN_DIR).$$(date +%Y%m%dT%H%M%S); mkdir $$backup_dir; \ - cd $(FROZEN_DIR); git status --ignored -u all -s . | awk ' {print $$2}' \ - | xargs --no-run-if-empty cp --parents -t ../$$backup_dir; \ - rmdir ../$$backup_dir 2>/dev/null || true; \ - git clean -d -f .; \ - fi -.PHONY: clean-frozen - print-cfg: $(ECHO) "PY_SRC = $(PY_SRC)" $(ECHO) "BUILD = $(BUILD)" diff --git a/py/modbuiltins.c b/py/modbuiltins.c index e56b43047a..ebdbd52dff 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -786,6 +786,7 @@ STATIC const mp_rom_map_elem_t mp_module_builtins_globals_table[] = { // Extra builtins as defined by a port MICROPY_PORT_BUILTINS + MICROPY_PORT_EXTRA_BUILTINS }; MP_DEFINE_CONST_DICT(mp_module_builtins_globals, mp_module_builtins_globals_table); diff --git a/py/modio.c b/py/modio.c index 23374578cf..819a9976a0 100644 --- a/py/modio.c +++ b/py/modio.c @@ -212,50 +212,6 @@ STATIC const mp_obj_type_t mp_type_bufwriter = { }; #endif // MICROPY_PY_IO_BUFFEREDWRITER -#if MICROPY_PY_IO_RESOURCE_STREAM -STATIC mp_obj_t resource_stream(mp_obj_t package_in, mp_obj_t path_in) { - VSTR_FIXED(path_buf, MICROPY_ALLOC_PATH_MAX); - size_t len; - - // As an extension to pkg_resources.resource_stream(), we support - // package parameter being None, the path_in is interpreted as a - // raw path. - if (package_in != mp_const_none) { - // Pass "True" as sentinel value in fromlist to force returning of leaf module - mp_obj_t pkg = mp_import_name(mp_obj_str_get_qstr(package_in), mp_const_true, MP_OBJ_NEW_SMALL_INT(0)); - - mp_obj_t dest[2]; - mp_load_method_maybe(pkg, MP_QSTR___path__, dest); - if (dest[0] == MP_OBJ_NULL) { - mp_raise_TypeError(NULL); - } - - const char *path = mp_obj_str_get_data(dest[0], &len); - vstr_add_strn(&path_buf, path, len); - vstr_add_byte(&path_buf, '/'); - } - - const char *path = mp_obj_str_get_data(path_in, &len); - vstr_add_strn(&path_buf, path, len); - - size_t file_len; - const char *data = mp_find_frozen_str(path_buf.buf, path_buf.len, &file_len); - if (data != NULL) { - mp_obj_stringio_t *o = m_new_obj(mp_obj_stringio_t); - o->base.type = &mp_type_bytesio; - o->vstr = m_new_obj(vstr_t); - vstr_init_fixed_buf(o->vstr, file_len + 1, (char *)data); - o->vstr->len = file_len; - o->pos = 0; - return MP_OBJ_FROM_PTR(o); - } - - mp_obj_t path_out = mp_obj_new_str(path_buf.buf, path_buf.len); - return mp_builtin_open(1, &path_out, (mp_map_t *)&mp_const_empty_map); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_2(resource_stream_obj, resource_stream); -#endif - STATIC const mp_rom_map_elem_t mp_module_io_globals_table[] = { #if CIRCUITPY { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_io) }, @@ -268,9 +224,6 @@ STATIC const mp_rom_map_elem_t mp_module_io_globals_table[] = { #if MICROPY_PY_IO_IOBASE { MP_ROM_QSTR(MP_QSTR_IOBase), MP_ROM_PTR(&mp_type_iobase) }, #endif - #if MICROPY_PY_IO_RESOURCE_STREAM - { MP_ROM_QSTR(MP_QSTR_resource_stream), MP_ROM_PTR(&resource_stream_obj) }, - #endif #if MICROPY_PY_IO_FILEIO { MP_ROM_QSTR(MP_QSTR_FileIO), MP_ROM_PTR(&mp_type_fileio) }, #if MICROPY_CPYTHON_COMPAT diff --git a/py/modsys.c b/py/modsys.c index 6016570933..0ec5de1645 100644 --- a/py/modsys.c +++ b/py/modsys.c @@ -158,7 +158,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_atexit_obj, mp_sys_atexit); #endif #if MICROPY_PY_SYS_SETTRACE -// settrace(tracefunc): Set the system’s trace function. +// settrace(tracefunc): Set the system's trace function. STATIC mp_obj_t mp_sys_settrace(mp_obj_t obj) { return mp_prof_settrace(obj); } diff --git a/py/mpconfig.h b/py/mpconfig.h index 98a11c15ad..d66d615996 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -50,6 +50,31 @@ #define CIRCUITPY 0 #endif +// Disable all optional features (i.e. minimal port). +#define MICROPY_CONFIG_ROM_LEVEL_MINIMUM (0) +// Only enable core features (constrained flash, e.g. STM32L072) +#define MICROPY_CONFIG_ROM_LEVEL_CORE_FEATURES (10) +// Enable most common features (small on-device flash, e.g. STM32F411) +#define MICROPY_CONFIG_ROM_LEVEL_BASIC_FEATURES (20) +// Enable convenience features (medium on-device flash, e.g. STM32F405) +#define MICROPY_CONFIG_ROM_LEVEL_EXTRA_FEATURES (30) +// Enable all common features (large/external flash, rp2, unix) +#define MICROPY_CONFIG_ROM_LEVEL_FULL_FEATURES (40) +// Enable everything (e.g. coverage) +#define MICROPY_CONFIG_ROM_LEVEL_EVERYTHING (50) + +// Ports/boards should set this, but default to level=core. +#ifndef MICROPY_CONFIG_ROM_LEVEL +#define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_CORE_FEATURES) +#endif + +// Helper macros for "have at least this level". +#define MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES (MICROPY_CONFIG_ROM_LEVEL >= MICROPY_CONFIG_ROM_LEVEL_CORE_FEATURES) +#define MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_BASIC_FEATURES (MICROPY_CONFIG_ROM_LEVEL >= MICROPY_CONFIG_ROM_LEVEL_BASIC_FEATURES) +#define MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES (MICROPY_CONFIG_ROM_LEVEL >= MICROPY_CONFIG_ROM_LEVEL_EXTRA_FEATURES) +#define MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES (MICROPY_CONFIG_ROM_LEVEL >= MICROPY_CONFIG_ROM_LEVEL_FULL_FEATURES) +#define MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING (MICROPY_CONFIG_ROM_LEVEL >= MICROPY_CONFIG_ROM_LEVEL_EVERYTHING) + // Any options not explicitly set in mpconfigport.h will get default // values below. @@ -144,7 +169,7 @@ // Support automatic GC when reaching allocation threshold, // configurable by gc.threshold(). #ifndef MICROPY_GC_ALLOC_THRESHOLD -#define MICROPY_GC_ALLOC_THRESHOLD (1) +#define MICROPY_GC_ALLOC_THRESHOLD (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Number of bytes to allocate initially when creating new chunks to store @@ -243,7 +268,11 @@ // Number of bytes used to store qstr hash #ifndef MICROPY_QSTR_BYTES_IN_HASH +#if MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES #define MICROPY_QSTR_BYTES_IN_HASH (2) +#else +#define MICROPY_QSTR_BYTES_IN_HASH (1) +#endif #endif // Avoid using C stack when making Python function calls. C stack still @@ -385,7 +414,7 @@ // Whether to include the compiler #ifndef MICROPY_ENABLE_COMPILER -#define MICROPY_ENABLE_COMPILER (1) +#define MICROPY_ENABLE_COMPILER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether the compiler is dynamically configurable (ie at runtime) @@ -396,49 +425,47 @@ // Configure dynamic compiler macros #if MICROPY_DYNAMIC_COMPILER -#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC (mp_dynamic_compiler.opt_cache_map_lookup_in_bytecode) #define MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC (mp_dynamic_compiler.py_builtins_str_unicode) #else -#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE #define MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC MICROPY_PY_BUILTINS_STR_UNICODE #endif // Whether to enable constant folding; eg 1+2 rewritten as 3 #ifndef MICROPY_COMP_CONST_FOLDING -#define MICROPY_COMP_CONST_FOLDING (1) +#define MICROPY_COMP_CONST_FOLDING (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to enable optimisations for constant literals, eg OrderedDict #ifndef MICROPY_COMP_CONST_LITERAL -#define MICROPY_COMP_CONST_LITERAL (1) +#define MICROPY_COMP_CONST_LITERAL (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to enable lookup of constants in modules; eg module.CONST #ifndef MICROPY_COMP_MODULE_CONST -#define MICROPY_COMP_MODULE_CONST (0) +#define MICROPY_COMP_MODULE_CONST (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to enable constant optimisation; id = const(value) #ifndef MICROPY_COMP_CONST -#define MICROPY_COMP_CONST (1) +#define MICROPY_COMP_CONST (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to enable optimisation of: a, b = c, d // Costs 124 bytes (Thumb2) #ifndef MICROPY_COMP_DOUBLE_TUPLE_ASSIGN -#define MICROPY_COMP_DOUBLE_TUPLE_ASSIGN (1) +#define MICROPY_COMP_DOUBLE_TUPLE_ASSIGN (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to enable optimisation of: a, b, c = d, e, f // Requires MICROPY_COMP_DOUBLE_TUPLE_ASSIGN and costs 68 bytes (Thumb2) #ifndef MICROPY_COMP_TRIPLE_TUPLE_ASSIGN -#define MICROPY_COMP_TRIPLE_TUPLE_ASSIGN (0) +#define MICROPY_COMP_TRIPLE_TUPLE_ASSIGN (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to enable optimisation of: return a if b else c // Costs about 80 bytes (Thumb2) and saves 2 bytes of bytecode for each use #ifndef MICROPY_COMP_RETURN_IF_EXPR -#define MICROPY_COMP_RETURN_IF_EXPR (0) +#define MICROPY_COMP_RETURN_IF_EXPR (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to include parsing of f-string literals @@ -511,23 +538,36 @@ #define MICROPY_OPT_COMPUTED_GOTO_SAVE_SPACE (0) #endif -// Whether to cache result of map lookups in LOAD_NAME, LOAD_GLOBAL, LOAD_ATTR, -// STORE_ATTR bytecodes. Uses 1 byte extra RAM for each of these opcodes and -// uses a bit of extra code ROM, but greatly improves lookup speed. -#ifndef MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE -#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (0) +// Optimise the fast path for loading attributes from instance types. Increases +// Thumb2 code size by about 48 bytes. +#ifndef MICROPY_OPT_LOAD_ATTR_FAST_PATH +#define MICROPY_OPT_LOAD_ATTR_FAST_PATH (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + +// Use extra RAM to cache map lookups by remembering the likely location of +// the index. Avoids the hash computation on unordered maps, and avoids the +// linear search on ordered (especially in-ROM) maps. Can provide a +10-15% +// performance improvement on benchmarks involving lots of attribute access +// or dictionary lookup. +#ifndef MICROPY_OPT_MAP_LOOKUP_CACHE +#define MICROPY_OPT_MAP_LOOKUP_CACHE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + +// How much RAM (in bytes) to use for the map lookup cache. +#ifndef MICROPY_OPT_MAP_LOOKUP_CACHE_SIZE +#define MICROPY_OPT_MAP_LOOKUP_CACHE_SIZE (128) #endif // Whether to use fast versions of bitwise operations (and, or, xor) when the // arguments are both positive. Increases Thumb2 code size by about 250 bytes. #ifndef MICROPY_OPT_MPZ_BITWISE -#define MICROPY_OPT_MPZ_BITWISE (0) +#define MICROPY_OPT_MPZ_BITWISE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether math.factorial is large, fast and recursive (1) or small and slow (0). #ifndef MICROPY_OPT_MATH_FACTORIAL -#define MICROPY_OPT_MATH_FACTORIAL (0) +#define MICROPY_OPT_MATH_FACTORIAL (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif /*****************************************************************************/ @@ -537,7 +577,7 @@ // When disabled, only importing of built-in modules is supported // When enabled, a port must implement mp_import_stat (among other things) #ifndef MICROPY_ENABLE_EXTERNAL_IMPORT -#define MICROPY_ENABLE_EXTERNAL_IMPORT (1) +#define MICROPY_ENABLE_EXTERNAL_IMPORT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to use the POSIX reader for importing files @@ -587,9 +627,14 @@ #define MICROPY_ENABLE_GC (0) #endif +// Hook to run code during time consuming garbage collector operations +#ifndef MICROPY_GC_HOOK_LOOP +#define MICROPY_GC_HOOK_LOOP +#endif + // Whether to enable finalisers in the garbage collector (ie call __del__) #ifndef MICROPY_ENABLE_FINALISER -#define MICROPY_ENABLE_FINALISER (0) +#define MICROPY_ENABLE_FINALISER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to enable a separate allocator for the Python stack. @@ -606,7 +651,7 @@ // Whether to check C stack usage. C stack used for calling Python functions, // etc. Not checking means segfault on overflow. #ifndef MICROPY_STACK_CHECK -#define MICROPY_STACK_CHECK (0) +#define MICROPY_STACK_CHECK (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to measure maximum stack excursion @@ -626,7 +671,7 @@ // Whether to provide the mp_kbd_exception object, and micropython.kbd_intr function #ifndef MICROPY_KBD_EXCEPTION -#define MICROPY_KBD_EXCEPTION (0) +#define MICROPY_KBD_EXCEPTION (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Prefer to raise KeyboardInterrupt asynchronously (from signal or interrupt @@ -637,7 +682,7 @@ // Whether to include REPL helper function #ifndef MICROPY_HELPER_REPL -#define MICROPY_HELPER_REPL (0) +#define MICROPY_HELPER_REPL (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Allow enabling debug prints after each REPL line @@ -647,7 +692,7 @@ // Whether to include emacs-style readline behavior in REPL #ifndef MICROPY_REPL_EMACS_KEYS -#define MICROPY_REPL_EMACS_KEYS (0) +#define MICROPY_REPL_EMACS_KEYS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to include emacs-style word movement/kill readline behavior in REPL. @@ -667,7 +712,7 @@ // Whether to implement auto-indent in REPL #ifndef MICROPY_REPL_AUTO_INDENT -#define MICROPY_REPL_AUTO_INDENT (0) +#define MICROPY_REPL_AUTO_INDENT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether port requires event-driven REPL functions @@ -696,7 +741,7 @@ typedef long long mp_longint_impl_t; // Whether to include information in the byte code to determine source // line number (increases RAM usage, but doesn't slow byte code execution) #ifndef MICROPY_ENABLE_SOURCE_LINE -#define MICROPY_ENABLE_SOURCE_LINE (0) +#define MICROPY_ENABLE_SOURCE_LINE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to include doc strings (increases RAM usage) @@ -714,7 +759,13 @@ typedef long long mp_longint_impl_t; #define MICROPY_ERROR_REPORTING_DETAILED (3) #ifndef MICROPY_ERROR_REPORTING +#if MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES +#define MICROPY_ERROR_REPORTING (MICROPY_ERROR_REPORTING_DETAILED) +#elif MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES #define MICROPY_ERROR_REPORTING (MICROPY_ERROR_REPORTING_NORMAL) +#else +#define MICROPY_ERROR_REPORTING (MICROPY_ERROR_REPORTING_TERSE) +#endif #endif // Whether issue warnings during compiling/execution @@ -770,7 +821,7 @@ typedef double mp_float_t; // TODO: Originally intended as generic category to not // add bunch of once-off options. May need refactoring later #ifndef MICROPY_CPYTHON_COMPAT -#define MICROPY_CPYTHON_COMPAT (1) +#define MICROPY_CPYTHON_COMPAT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Perform full checks as done by CPython. Disabling this @@ -779,12 +830,12 @@ typedef double mp_float_t; // grave issues (in other words, only user app should be, // affected, not system). #ifndef MICROPY_FULL_CHECKS -#define MICROPY_FULL_CHECKS (1) +#define MICROPY_FULL_CHECKS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether POSIX-semantics non-blocking streams are supported #ifndef MICROPY_STREAMS_NON_BLOCK -#define MICROPY_STREAMS_NON_BLOCK (0) +#define MICROPY_STREAMS_NON_BLOCK (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide stream functions with POSIX-like signatures @@ -795,17 +846,23 @@ typedef double mp_float_t; // Whether to call __init__ when importing builtin modules for the first time #ifndef MICROPY_MODULE_BUILTIN_INIT -#define MICROPY_MODULE_BUILTIN_INIT (0) +#define MICROPY_MODULE_BUILTIN_INIT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support module-level __getattr__ (see PEP 562) #ifndef MICROPY_MODULE_GETATTR -#define MICROPY_MODULE_GETATTR (1) +#define MICROPY_MODULE_GETATTR (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether module weak links are supported #ifndef MICROPY_MODULE_WEAK_LINKS -#define MICROPY_MODULE_WEAK_LINKS (0) +#define MICROPY_MODULE_WEAK_LINKS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + +// Whether to enable importing foo.py with __name__ set to '__main__' +// Used by the unix port for the -m flag. +#ifndef MICROPY_MODULE_OVERRIDE_MAIN_IMPORT +#define MICROPY_MODULE_OVERRIDE_MAIN_IMPORT (0) #endif // Whether frozen modules are supported in the form of strings @@ -825,7 +882,7 @@ typedef double mp_float_t; // Whether you can override builtins in the builtins module #ifndef MICROPY_CAN_OVERRIDE_BUILTINS -#define MICROPY_CAN_OVERRIDE_BUILTINS (0) +#define MICROPY_CAN_OVERRIDE_BUILTINS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to check that the "self" argument of a builtin method has the @@ -834,7 +891,7 @@ typedef double mp_float_t; // list.append([], 1). Without this check such calls will have undefined // behaviour (usually segfault) if the first argument is the wrong type. #ifndef MICROPY_BUILTIN_METHOD_CHECK_SELF_ARG -#define MICROPY_BUILTIN_METHOD_CHECK_SELF_ARG (1) +#define MICROPY_BUILTIN_METHOD_CHECK_SELF_ARG (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to use internally defined errno's (otherwise system provided ones) @@ -849,7 +906,7 @@ typedef double mp_float_t; // Support for internal scheduler #ifndef MICROPY_ENABLE_SCHEDULER -#define MICROPY_ENABLE_SCHEDULER (0) +#define MICROPY_ENABLE_SCHEDULER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Maximum number of entries in the scheduler @@ -884,41 +941,41 @@ typedef double mp_float_t; // inheritance makes some C functions inherently recursive, and adds a bit of // code overhead. #ifndef MICROPY_MULTIPLE_INHERITANCE -#define MICROPY_MULTIPLE_INHERITANCE (1) +#define MICROPY_MULTIPLE_INHERITANCE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to implement attributes on functions #ifndef MICROPY_PY_FUNCTION_ATTRS -#define MICROPY_PY_FUNCTION_ATTRS (0) +#define MICROPY_PY_FUNCTION_ATTRS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support the descriptors __get__, __set__, __delete__ // This costs some code size and makes load/store/delete of instance // attributes slower for the classes that use this feature #ifndef MICROPY_PY_DESCRIPTORS -#define MICROPY_PY_DESCRIPTORS (0) +#define MICROPY_PY_DESCRIPTORS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support class __delattr__ and __setattr__ methods // This costs some code size and makes store/delete of instance // attributes slower for the classes that use this feature #ifndef MICROPY_PY_DELATTR_SETATTR -#define MICROPY_PY_DELATTR_SETATTR (0) +#define MICROPY_PY_DELATTR_SETATTR (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Support for async/await/async for/async with #ifndef MICROPY_PY_ASYNC_AWAIT -#define MICROPY_PY_ASYNC_AWAIT (1) +#define MICROPY_PY_ASYNC_AWAIT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Support for literal string interpolation, f-strings (see PEP 498, Python 3.6+) #ifndef MICROPY_PY_FSTRINGS -#define MICROPY_PY_FSTRINGS (0) +#define MICROPY_PY_FSTRINGS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Support for assignment expressions with := (see PEP 572, Python 3.8+) #ifndef MICROPY_PY_ASSIGN_EXPR -#define MICROPY_PY_ASSIGN_EXPR (1) +#define MICROPY_PY_ASSIGN_EXPR (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Non-standard .pend_throw() method for generators, allowing for @@ -927,7 +984,7 @@ typedef double mp_float_t; // to generator's .send() or .__next__(). (This is useful to implement // async schedulers.) #ifndef MICROPY_PY_GENERATOR_PEND_THROW -#define MICROPY_PY_GENERATOR_PEND_THROW (1) +#define MICROPY_PY_GENERATOR_PEND_THROW (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Issue a warning when comparing str and bytes objects @@ -937,7 +994,7 @@ typedef double mp_float_t; // Whether str object is proper unicode #ifndef MICROPY_PY_BUILTINS_STR_UNICODE -#define MICROPY_PY_BUILTINS_STR_UNICODE (0) +#define MICROPY_PY_BUILTINS_STR_UNICODE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to check for valid UTF-8 when converting bytes to str @@ -947,42 +1004,42 @@ typedef double mp_float_t; // Whether str.center() method provided #ifndef MICROPY_PY_BUILTINS_STR_CENTER -#define MICROPY_PY_BUILTINS_STR_CENTER (0) +#define MICROPY_PY_BUILTINS_STR_CENTER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether str.count() method provided #ifndef MICROPY_PY_BUILTINS_STR_COUNT -#define MICROPY_PY_BUILTINS_STR_COUNT (1) +#define MICROPY_PY_BUILTINS_STR_COUNT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether str % (...) formatting operator provided #ifndef MICROPY_PY_BUILTINS_STR_OP_MODULO -#define MICROPY_PY_BUILTINS_STR_OP_MODULO (1) +#define MICROPY_PY_BUILTINS_STR_OP_MODULO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether str.partition()/str.rpartition() method provided #ifndef MICROPY_PY_BUILTINS_STR_PARTITION -#define MICROPY_PY_BUILTINS_STR_PARTITION (0) +#define MICROPY_PY_BUILTINS_STR_PARTITION (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether str.splitlines() method provided #ifndef MICROPY_PY_BUILTINS_STR_SPLITLINES -#define MICROPY_PY_BUILTINS_STR_SPLITLINES (0) +#define MICROPY_PY_BUILTINS_STR_SPLITLINES (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support bytearray object #ifndef MICROPY_PY_BUILTINS_BYTEARRAY -#define MICROPY_PY_BUILTINS_BYTEARRAY (1) +#define MICROPY_PY_BUILTINS_BYTEARRAY (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to support dict.fromkeys() class method #ifndef MICROPY_PY_BUILTINS_DICT_FROMKEYS -#define MICROPY_PY_BUILTINS_DICT_FROMKEYS (1) +#define MICROPY_PY_BUILTINS_DICT_FROMKEYS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to support memoryview object #ifndef MICROPY_PY_BUILTINS_MEMORYVIEW -#define MICROPY_PY_BUILTINS_MEMORYVIEW (0) +#define MICROPY_PY_BUILTINS_MEMORYVIEW (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support memoryview.itemsize attribute @@ -992,39 +1049,39 @@ typedef double mp_float_t; // Whether to support set object #ifndef MICROPY_PY_BUILTINS_SET -#define MICROPY_PY_BUILTINS_SET (1) +#define MICROPY_PY_BUILTINS_SET (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to support slice subscript operators and slice object #ifndef MICROPY_PY_BUILTINS_SLICE -#define MICROPY_PY_BUILTINS_SLICE (1) +#define MICROPY_PY_BUILTINS_SLICE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to support slice attribute read access, // i.e. slice.start, slice.stop, slice.step #ifndef MICROPY_PY_BUILTINS_SLICE_ATTRS -#define MICROPY_PY_BUILTINS_SLICE_ATTRS (0) +#define MICROPY_PY_BUILTINS_SLICE_ATTRS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support the .indices(len) method on slice objects #ifndef MICROPY_PY_BUILTINS_SLICE_INDICES -#define MICROPY_PY_BUILTINS_SLICE_INDICES (0) +#define MICROPY_PY_BUILTINS_SLICE_INDICES (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support frozenset object #ifndef MICROPY_PY_BUILTINS_FROZENSET -#define MICROPY_PY_BUILTINS_FROZENSET (0) +#define MICROPY_PY_BUILTINS_FROZENSET (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support property object #ifndef MICROPY_PY_BUILTINS_PROPERTY -#define MICROPY_PY_BUILTINS_PROPERTY (1) +#define MICROPY_PY_BUILTINS_PROPERTY (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to implement the start/stop/step attributes (readback) on // the "range" builtin type. Rarely used, and costs ~60 bytes (x86). #ifndef MICROPY_PY_BUILTINS_RANGE_ATTRS -#define MICROPY_PY_BUILTINS_RANGE_ATTRS (1) +#define MICROPY_PY_BUILTINS_RANGE_ATTRS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to support binary ops [only (in)equality is defined] between range @@ -1042,7 +1099,7 @@ typedef double mp_float_t; // Whether to support rounding of integers (incl bignum); eg round(123,-1)=120 #ifndef MICROPY_PY_BUILTINS_ROUND_INT -#define MICROPY_PY_BUILTINS_ROUND_INT (0) +#define MICROPY_PY_BUILTINS_ROUND_INT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support complete set of special methods for user @@ -1051,7 +1108,7 @@ typedef double mp_float_t; // "Reverse" methods are controlled by // MICROPY_PY_REVERSE_SPECIAL_METHODS below. #ifndef MICROPY_PY_ALL_SPECIAL_METHODS -#define MICROPY_PY_ALL_SPECIAL_METHODS (0) +#define MICROPY_PY_ALL_SPECIAL_METHODS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support all inplace arithmetic operarion methods @@ -1064,17 +1121,17 @@ typedef double mp_float_t; // (__radd__, etc.). Additionally gated by // MICROPY_PY_ALL_SPECIAL_METHODS. #ifndef MICROPY_PY_REVERSE_SPECIAL_METHODS -#define MICROPY_PY_REVERSE_SPECIAL_METHODS (0) +#define MICROPY_PY_REVERSE_SPECIAL_METHODS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support compile function #ifndef MICROPY_PY_BUILTINS_COMPILE -#define MICROPY_PY_BUILTINS_COMPILE (0) +#define MICROPY_PY_BUILTINS_COMPILE (MICROPY_ENABLE_COMPILER && MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support enumerate function(type) #ifndef MICROPY_PY_BUILTINS_ENUMERATE -#define MICROPY_PY_BUILTINS_ENUMERATE (1) +#define MICROPY_PY_BUILTINS_ENUMERATE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to support eval and exec functions @@ -1085,43 +1142,43 @@ typedef double mp_float_t; // Whether to support the Python 2 execfile function #ifndef MICROPY_PY_BUILTINS_EXECFILE -#define MICROPY_PY_BUILTINS_EXECFILE (0) +#define MICROPY_PY_BUILTINS_EXECFILE (MICROPY_ENABLE_COMPILER && MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support filter function(type) #ifndef MICROPY_PY_BUILTINS_FILTER -#define MICROPY_PY_BUILTINS_FILTER (1) +#define MICROPY_PY_BUILTINS_FILTER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to support reversed function(type) #ifndef MICROPY_PY_BUILTINS_REVERSED -#define MICROPY_PY_BUILTINS_REVERSED (1) +#define MICROPY_PY_BUILTINS_REVERSED (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to define "NotImplemented" special constant #ifndef MICROPY_PY_BUILTINS_NOTIMPLEMENTED -#define MICROPY_PY_BUILTINS_NOTIMPLEMENTED (0) +#define MICROPY_PY_BUILTINS_NOTIMPLEMENTED (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide the built-in input() function. The implementation of this // uses shared/readline, so can only be enabled if the port uses this readline. #ifndef MICROPY_PY_BUILTINS_INPUT -#define MICROPY_PY_BUILTINS_INPUT (0) +#define MICROPY_PY_BUILTINS_INPUT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support min/max functions #ifndef MICROPY_PY_BUILTINS_MIN_MAX -#define MICROPY_PY_BUILTINS_MIN_MAX (1) +#define MICROPY_PY_BUILTINS_MIN_MAX (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Support for calls to pow() with 3 integer arguments #ifndef MICROPY_PY_BUILTINS_POW3 -#define MICROPY_PY_BUILTINS_POW3 (0) +#define MICROPY_PY_BUILTINS_POW3 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide the help function #ifndef MICROPY_PY_BUILTINS_HELP -#define MICROPY_PY_BUILTINS_HELP (0) +#define MICROPY_PY_BUILTINS_HELP (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Use this to configure the help text shown for help(). It should be a @@ -1132,17 +1189,17 @@ typedef double mp_float_t; // Add the ability to list the available modules when executing help('modules') #ifndef MICROPY_PY_BUILTINS_HELP_MODULES -#define MICROPY_PY_BUILTINS_HELP_MODULES (0) +#define MICROPY_PY_BUILTINS_HELP_MODULES (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to set __file__ for imported modules #ifndef MICROPY_PY___FILE__ -#define MICROPY_PY___FILE__ (1) +#define MICROPY_PY___FILE__ (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to provide mem-info related functions in micropython module #ifndef MICROPY_PY_MICROPYTHON_MEM_INFO -#define MICROPY_PY_MICROPYTHON_MEM_INFO (0) +#define MICROPY_PY_MICROPYTHON_MEM_INFO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide "micropython.stack_use" function @@ -1159,13 +1216,13 @@ typedef double mp_float_t; // underlying code is shared with "bytearray" builtin type, so to // get real savings, it should be disabled too. #ifndef MICROPY_PY_ARRAY -#define MICROPY_PY_ARRAY (1) +#define MICROPY_PY_ARRAY (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to support slice assignments for array (and bytearray). // This is rarely used, but adds ~0.5K of code. #ifndef MICROPY_PY_ARRAY_SLICE_ASSIGN -#define MICROPY_PY_ARRAY_SLICE_ASSIGN (0) +#define MICROPY_PY_ARRAY_SLICE_ASSIGN (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support nonstandard typecodes "O", "P" and "S" @@ -1177,22 +1234,22 @@ typedef double mp_float_t; // Whether to support attrtuple type (MicroPython extension) // It provides space-efficient tuples with attribute access #ifndef MICROPY_PY_ATTRTUPLE -#define MICROPY_PY_ATTRTUPLE (1) +#define MICROPY_PY_ATTRTUPLE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to provide "collections" module #ifndef MICROPY_PY_COLLECTIONS -#define MICROPY_PY_COLLECTIONS (1) +#define MICROPY_PY_COLLECTIONS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to provide "ucollections.deque" type #ifndef MICROPY_PY_COLLECTIONS_DEQUE -#define MICROPY_PY_COLLECTIONS_DEQUE (0) +#define MICROPY_PY_COLLECTIONS_DEQUE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide "collections.OrderedDict" type #ifndef MICROPY_PY_COLLECTIONS_ORDEREDDICT -#define MICROPY_PY_COLLECTIONS_ORDEREDDICT (0) +#define MICROPY_PY_COLLECTIONS_ORDEREDDICT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide the _asdict function for namedtuple @@ -1202,22 +1259,22 @@ typedef double mp_float_t; // Whether to provide "math" module #ifndef MICROPY_PY_MATH -#define MICROPY_PY_MATH (1) +#define MICROPY_PY_MATH (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to provide special math functions: math.{erf,erfc,gamma,lgamma} #ifndef MICROPY_PY_MATH_SPECIAL_FUNCTIONS -#define MICROPY_PY_MATH_SPECIAL_FUNCTIONS (0) +#define MICROPY_PY_MATH_SPECIAL_FUNCTIONS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide math.factorial function #ifndef MICROPY_PY_MATH_FACTORIAL -#define MICROPY_PY_MATH_FACTORIAL (0) +#define MICROPY_PY_MATH_FACTORIAL (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide math.isclose function #ifndef MICROPY_PY_MATH_ISCLOSE -#define MICROPY_PY_MATH_ISCLOSE (0) +#define MICROPY_PY_MATH_ISCLOSE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide fix for atan2 Inf handling. @@ -1242,12 +1299,12 @@ typedef double mp_float_t; // Whether to provide "cmath" module #ifndef MICROPY_PY_CMATH -#define MICROPY_PY_CMATH (0) +#define MICROPY_PY_CMATH (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide "gc" module #ifndef MICROPY_PY_GC -#define MICROPY_PY_GC (1) +#define MICROPY_PY_GC (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to return number of collected objects from gc.collect() @@ -1257,28 +1314,17 @@ typedef double mp_float_t; // Whether to provide "io" module #ifndef MICROPY_PY_IO -#define MICROPY_PY_IO (1) +#define MICROPY_PY_IO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to provide "io.IOBase" class to support user streams #ifndef MICROPY_PY_IO_IOBASE -#define MICROPY_PY_IO_IOBASE (0) -#endif - -// Whether to provide "uio.resource_stream()" function with -// the semantics of CPython's pkg_resources.resource_stream() -// (allows to access binary resources in frozen source packages). -// Note that the same functionality can be achieved in "pure -// Python" by prepocessing binary resources into Python source -// and bytecode-freezing it (with a simple helper module available -// e.g. in micropython-lib). -#ifndef MICROPY_PY_IO_RESOURCE_STREAM -#define MICROPY_PY_IO_RESOURCE_STREAM (0) +#define MICROPY_PY_IO_IOBASE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide "io.FileIO" class #ifndef MICROPY_PY_IO_FILEIO -#define MICROPY_PY_IO_FILEIO (0) +#define MICROPY_PY_IO_FILEIO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide "io.BytesIO" class @@ -1293,17 +1339,22 @@ typedef double mp_float_t; // Whether to provide "struct" module #ifndef MICROPY_PY_STRUCT -#define MICROPY_PY_STRUCT (1) +#define MICROPY_PY_STRUCT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to provide "sys" module #ifndef MICROPY_PY_SYS -#define MICROPY_PY_SYS (1) +#define MICROPY_PY_SYS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) +#endif + +// Whether to initialise "sys.path" and "sys.argv" to their defaults in mp_init() +#ifndef MICROPY_PY_SYS_PATH_ARGV_DEFAULTS +#define MICROPY_PY_SYS_PATH_ARGV_DEFAULTS (MICROPY_PY_SYS) #endif // Whether to provide "sys.maxsize" constant #ifndef MICROPY_PY_SYS_MAXSIZE -#define MICROPY_PY_SYS_MAXSIZE (0) +#define MICROPY_PY_SYS_MAXSIZE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide "sys.modules" dictionary @@ -1339,18 +1390,18 @@ typedef double mp_float_t; // Whether to provide sys.{stdin,stdout,stderr} objects #ifndef MICROPY_PY_SYS_STDFILES -#define MICROPY_PY_SYS_STDFILES (0) +#define MICROPY_PY_SYS_STDFILES (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide sys.{stdin,stdout,stderr}.buffer object // This is implemented per-port #ifndef MICROPY_PY_SYS_STDIO_BUFFER -#define MICROPY_PY_SYS_STDIO_BUFFER (0) +#define MICROPY_PY_SYS_STDIO_BUFFER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide "uerrno" module #ifndef MICROPY_PY_UERRNO -#define MICROPY_PY_UERRNO (0) +#define MICROPY_PY_UERRNO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide the uerrno.errorcode dict @@ -1360,7 +1411,7 @@ typedef double mp_float_t; // Whether to provide "uselect" module (baremetal implementation) #ifndef MICROPY_PY_USELECT -#define MICROPY_PY_USELECT (0) +#define MICROPY_PY_USELECT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to enable the select() function in the "uselect" module (baremetal @@ -1406,11 +1457,11 @@ typedef double mp_float_t; // Extended modules #ifndef MICROPY_PY_UASYNCIO -#define MICROPY_PY_UASYNCIO (0) +#define MICROPY_PY_UASYNCIO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif #ifndef MICROPY_PY_UCTYPES -#define MICROPY_PY_UCTYPES (0) +#define MICROPY_PY_UCTYPES (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide SHORT, INT, LONG, etc. types in addition to @@ -1420,11 +1471,11 @@ typedef double mp_float_t; #endif #ifndef MICROPY_PY_UZLIB -#define MICROPY_PY_UZLIB (0) +#define MICROPY_PY_UZLIB (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif #ifndef MICROPY_PY_UJSON -#define MICROPY_PY_UJSON (0) +#define MICROPY_PY_UJSON (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to support the "separators" argument to dump, dumps @@ -1437,7 +1488,7 @@ typedef double mp_float_t; #endif #ifndef MICROPY_PY_URE -#define MICROPY_PY_URE (0) +#define MICROPY_PY_URE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif #ifndef MICROPY_PY_URE_DEBUG @@ -1453,20 +1504,20 @@ typedef double mp_float_t; #endif #ifndef MICROPY_PY_URE_SUB -#define MICROPY_PY_URE_SUB (0) +#define MICROPY_PY_URE_SUB (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif #ifndef MICROPY_PY_UHEAPQ -#define MICROPY_PY_UHEAPQ (0) +#define MICROPY_PY_UHEAPQ (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -// Optimized heap queue for relative timestamps +// Optimized heap queue for relative timestamps (only used by uasyncio v2) #ifndef MICROPY_PY_UTIMEQ #define MICROPY_PY_UTIMEQ (0) #endif #ifndef MICROPY_PY_UHASHLIB -#define MICROPY_PY_UHASHLIB (0) +#define MICROPY_PY_UHASHLIB (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif #ifndef MICROPY_PY_UHASHLIB_MD5 @@ -1495,25 +1546,70 @@ typedef double mp_float_t; #endif #ifndef MICROPY_PY_UBINASCII -#define MICROPY_PY_UBINASCII (0) +#define MICROPY_PY_UBINASCII (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Depends on MICROPY_PY_UZLIB #ifndef MICROPY_PY_UBINASCII_CRC32 -#define MICROPY_PY_UBINASCII_CRC32 (0) +#define MICROPY_PY_UBINASCII_CRC32 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif #ifndef MICROPY_PY_URANDOM -#define MICROPY_PY_URANDOM (0) +#define MICROPY_PY_URANDOM (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to include: randrange, randint, choice, random, uniform #ifndef MICROPY_PY_URANDOM_EXTRA_FUNCS -#define MICROPY_PY_URANDOM_EXTRA_FUNCS (0) +#define MICROPY_PY_URANDOM_EXTRA_FUNCS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + +#ifndef MICROPY_PY_MACHINE +#define MICROPY_PY_MACHINE (0) +#endif + +// Whether to include: bitstream +#ifndef MICROPY_PY_MACHINE_BITSTREAM +#define MICROPY_PY_MACHINE_BITSTREAM (0) +#endif + +// Whether to include: time_pulse_us +#ifndef MICROPY_PY_MACHINE_PULSE +#define MICROPY_PY_MACHINE_PULSE (0) +#endif + +#ifndef MICROPY_PY_MACHINE_I2C +#define MICROPY_PY_MACHINE_I2C (0) +#endif + +// Whether to provide the "machine.SoftI2C" class +#ifndef MICROPY_PY_MACHINE_SOFTI2C +#define MICROPY_PY_MACHINE_SOFTI2C (0) +#endif + +#ifndef MICROPY_PY_MACHINE_SPI +#define MICROPY_PY_MACHINE_SPI (0) +#endif + +// Whether to provide the "machine.SoftSPI" class +#ifndef MICROPY_PY_MACHINE_SOFTSPI +#define MICROPY_PY_MACHINE_SOFTSPI (0) +#endif + +#ifndef MICROPY_PY_USSL +#define MICROPY_PY_USSL (0) +#endif + +// Whether to add finaliser code to ussl objects +#ifndef MICROPY_PY_USSL_FINALISER +#define MICROPY_PY_USSL_FINALISER (0) +#endif + +#ifndef MICROPY_PY_UWEBSOCKET +#define MICROPY_PY_UWEBSOCKET (0) #endif #ifndef MICROPY_PY_FRAMEBUF -#define MICROPY_PY_FRAMEBUF (0) +#define MICROPY_PY_FRAMEBUF (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif #ifndef MICROPY_PY_BTREE @@ -1523,6 +1619,12 @@ typedef double mp_float_t; #ifndef MICROPY_HW_ENABLE_USB #define MICROPY_HW_ENABLE_USB (0) #endif + +// Whether to provide the low-level "_onewire" module +#ifndef MICROPY_PY_ONEWIRE +#define MICROPY_PY_ONEWIRE (0) +#endif + /*****************************************************************************/ /* Hooks for a port to add builtins */ @@ -1531,6 +1633,12 @@ typedef double mp_float_t; #define MICROPY_PORT_BUILTINS #endif +// Additional builtin function definitions for extension by command-line, boards or variants. +// See modbuiltins.c:mp_module_builtins_globals_table for format. +#ifndef MICROPY_PORT_EXTRA_BUILTINS +#define MICROPY_PORT_EXTRA_BUILTINS +#endif + // Additional builtin module definitions - see objmodule.c:mp_builtin_module_table for format. #ifndef MICROPY_PORT_BUILTIN_MODULES #define MICROPY_PORT_BUILTIN_MODULES @@ -1549,6 +1657,30 @@ typedef double mp_float_t; /*****************************************************************************/ /* Hooks for a port to wrap functions with attributes */ +#ifndef MICROPY_WRAP_MP_BINARY_OP +#define MICROPY_WRAP_MP_BINARY_OP(f) f +#endif + +#ifndef MICROPY_WRAP_MP_EXECUTE_BYTECODE +#define MICROPY_WRAP_MP_EXECUTE_BYTECODE(f) f +#endif + +#ifndef MICROPY_WRAP_MP_LOAD_GLOBAL +#define MICROPY_WRAP_MP_LOAD_GLOBAL(f) f +#endif + +#ifndef MICROPY_WRAP_MP_LOAD_NAME +#define MICROPY_WRAP_MP_LOAD_NAME(f) f +#endif + +#ifndef MICROPY_WRAP_MP_MAP_LOOKUP +#define MICROPY_WRAP_MP_MAP_LOOKUP(f) f +#endif + +#ifndef MICROPY_WRAP_MP_OBJ_GET_TYPE +#define MICROPY_WRAP_MP_OBJ_GET_TYPE(f) f +#endif + #ifndef MICROPY_WRAP_MP_SCHED_EXCEPTION #define MICROPY_WRAP_MP_SCHED_EXCEPTION(f) f #endif diff --git a/py/mpstate.h b/py/mpstate.h index 964a04f300..bfbc29d55a 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -44,7 +44,6 @@ #if MICROPY_DYNAMIC_COMPILER typedef struct mp_dynamic_compiler_t { uint8_t small_int_bits; // must be <= host small_int_bits - bool opt_cache_map_lookup_in_bytecode; bool py_builtins_str_unicode; uint8_t native_arch; uint8_t nlr_buf_num_regs; @@ -165,9 +164,12 @@ typedef struct _mp_state_vm_t { // dictionary for the __main__ module mp_obj_dict_t dict_main; - // these two lists must be initialised per port, after the call to mp_init + #if MICROPY_PY_SYS + // If MICROPY_PY_SYS_PATH_ARGV_DEFAULTS is not enabled then these two lists + // must be initialised after the call to mp_init. mp_obj_list_t mp_sys_path_obj; mp_obj_list_t mp_sys_argv_obj; + #endif // dictionary for overridden builtins #if MICROPY_CAN_OVERRIDE_BUILTINS @@ -230,6 +232,11 @@ typedef struct _mp_state_vm_t { // This is a global mutex used to make the VM/runtime thread-safe. mp_thread_mutex_t gil_mutex; #endif + + #if MICROPY_OPT_MAP_LOOKUP_CACHE + // See mp_map_lookup. + uint8_t map_lookup_cache[MICROPY_OPT_MAP_LOOKUP_CACHE_SIZE]; + #endif } mp_state_vm_t; // This structure holds state that is specific to a given thread. diff --git a/py/mpz.c b/py/mpz.c index b871b76b7c..b52e05148a 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -717,6 +717,7 @@ void mpz_set(mpz_t *dest, const mpz_t *src) { void mpz_set_from_int(mpz_t *z, mp_int_t val) { if (val == 0) { + z->neg = 0; z->len = 0; return; } @@ -903,10 +904,6 @@ bool mpz_is_even(const mpz_t *z) { #endif int mpz_cmp(const mpz_t *z1, const mpz_t *z2) { - // to catch comparison of -0 with +0 - if (z1->len == 0 && z2->len == 0) { - return 0; - } int cmp = (int)z2->neg - (int)z1->neg; if (cmp != 0) { return cmp; @@ -1056,7 +1053,9 @@ void mpz_neg_inpl(mpz_t *dest, const mpz_t *z) { if (dest != z) { mpz_set(dest, z); } - dest->neg = 1 - dest->neg; + if (dest->len) { + dest->neg = 1 - dest->neg; + } } /* computes dest = ~z (= -z - 1) @@ -1152,7 +1151,7 @@ void mpz_add_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs) { dest->len = mpn_sub(dest->dig, lhs->dig, lhs->len, rhs->dig, rhs->len); } - dest->neg = lhs->neg; + dest->neg = lhs->neg & !!dest->len; } /* computes dest = lhs - rhs @@ -1176,7 +1175,9 @@ void mpz_sub_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs) { dest->len = mpn_sub(dest->dig, lhs->dig, lhs->len, rhs->dig, rhs->len); } - if (neg) { + if (dest->len == 0) { + dest->neg = 0; + } else if (neg) { dest->neg = 1 - lhs->neg; } else { dest->neg = lhs->neg; @@ -1488,14 +1489,16 @@ void mpz_divmod_inpl(mpz_t *dest_quo, mpz_t *dest_rem, const mpz_t *lhs, const m mpz_need_dig(dest_quo, lhs->len + 1); // +1 necessary? memset(dest_quo->dig, 0, (lhs->len + 1) * sizeof(mpz_dig_t)); + dest_quo->neg = 0; dest_quo->len = 0; mpz_need_dig(dest_rem, lhs->len + 1); // +1 necessary? mpz_set(dest_rem, lhs); mpn_div(dest_rem->dig, &dest_rem->len, rhs->dig, rhs->len, dest_quo->dig, &dest_quo->len); + dest_rem->neg &= !!dest_rem->len; // check signs and do Python style modulo if (lhs->neg != rhs->neg) { - dest_quo->neg = 1; + dest_quo->neg = !!dest_quo->len; if (!mpz_is_zero(dest_rem)) { mpz_t mpzone; mpz_init_from_int(&mpzone, -1); diff --git a/py/mpz.h b/py/mpz.h index ba2db4c308..0fdcf52cff 100644 --- a/py/mpz.h +++ b/py/mpz.h @@ -91,6 +91,7 @@ typedef int8_t mpz_dbl_dig_signed_t; #define MPZ_NUM_DIG_FOR_LL ((sizeof(long long) * 8 + MPZ_DIG_SIZE - 1) / MPZ_DIG_SIZE) typedef struct _mpz_t { + // Zero has neg=0, len=0. Negative zero is not allowed. size_t neg : 1; size_t fixed_dig : 1; size_t alloc : (8 * sizeof(size_t) - 2); @@ -119,7 +120,7 @@ static inline bool mpz_is_zero(const mpz_t *z) { return z->len == 0; } static inline bool mpz_is_neg(const mpz_t *z) { - return z->len != 0 && z->neg != 0; + return z->neg != 0; } int mpz_cmp(const mpz_t *lhs, const mpz_t *rhs); diff --git a/py/obj.c b/py/obj.c index 45f8c96859..cc7f9006e8 100644 --- a/py/obj.c +++ b/py/obj.c @@ -43,7 +43,7 @@ #include "supervisor/shared/stack.h" #include "supervisor/shared/translate.h" -const mp_obj_type_t *mp_obj_get_type(mp_const_obj_t o_in) { +const mp_obj_type_t *MICROPY_WRAP_MP_OBJ_GET_TYPE(mp_obj_get_type)(mp_const_obj_t o_in) { #if MICROPY_OBJ_IMMEDIATE_OBJS && MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A if (mp_obj_is_obj(o_in)) { diff --git a/py/obj.h b/py/obj.h index 823922ad1a..b31320f7e0 100644 --- a/py/obj.h +++ b/py/obj.h @@ -617,6 +617,7 @@ struct _mp_obj_type_t { // // dest[0] = MP_OBJ_NULL means load // return: for fail, do nothing + // for fail but continue lookup in locals_dict, dest[1] = MP_OBJ_SENTINEL // for attr, dest[0] = value // for method, dest[0] = method, dest[1] = self // diff --git a/py/objfun.h b/py/objfun.h index 1571ce7922..aae780b310 100644 --- a/py/objfun.h +++ b/py/objfun.h @@ -39,8 +39,6 @@ typedef struct _mp_obj_fun_bc_t { // the following extra_args array is allocated space to take (in order): // - values of positional default args (if any) // - a single slot for default kw args dict (if it has them) - // - a single slot for var args tuple (if it takes them) - // - a single slot for kw args dict (if it takes them) mp_obj_t extra_args[]; } mp_obj_fun_bc_t; diff --git a/py/objmodule.c b/py/objmodule.c index 238b24be4b..1403a4d2fe 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -36,6 +36,10 @@ #include "genhdr/moduledefs.h" +#if MICROPY_MODULE_BUILTIN_INIT +STATIC void mp_module_call_init(mp_obj_t module_name, mp_obj_t module_obj); +#endif + STATIC void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { (void)kind; mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in); @@ -279,47 +283,56 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = { MP_DEFINE_CONST_MAP(mp_builtin_module_map, mp_builtin_module_table); -// returns MP_OBJ_NULL if not found -mp_obj_t mp_module_get(qstr module_name) { - mp_map_t *mp_loaded_modules_map = &MP_STATE_VM(mp_loaded_modules_dict).map; - // lookup module - mp_map_elem_t *el = mp_map_lookup(mp_loaded_modules_map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP); +// Tries to find a loaded module, otherwise attempts to load a builtin, otherwise MP_OBJ_NULL. +mp_obj_t mp_module_get_loaded_or_builtin(qstr module_name) { + // First try loaded modules. + mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_loaded_modules_dict).map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP); - if (el == NULL) { - // module not found, look for builtin module names - el = mp_map_lookup((mp_map_t *)&mp_builtin_module_map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP); - if (el == NULL) { + if (!elem) { + #if MICROPY_MODULE_WEAK_LINKS + return mp_module_get_builtin(module_name); + #else + // Otherwise try builtin. + elem = mp_map_lookup((mp_map_t *)&mp_builtin_module_map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP); + if (!elem) { return MP_OBJ_NULL; } - mp_module_call_init(module_name, el->value); + + #if MICROPY_MODULE_BUILTIN_INIT + // If found, it's a newly loaded built-in, so init it. + mp_module_call_init(MP_OBJ_NEW_QSTR(module_name), elem->value); + #endif + #endif } - // module found, return it - return el->value; -} - -void mp_module_register(qstr qst, mp_obj_t module) { - mp_map_t *mp_loaded_modules_map = &MP_STATE_VM(mp_loaded_modules_dict).map; - mp_map_lookup(mp_loaded_modules_map, MP_OBJ_NEW_QSTR(qst), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = module; + return elem->value; } #if MICROPY_MODULE_WEAK_LINKS -// Search for u"foo" in built-in modules, return MP_OBJ_NULL if not found -mp_obj_t mp_module_search_umodule(const char *module_str) { - for (size_t i = 0; i < MP_ARRAY_SIZE(mp_builtin_module_table); ++i) { - const mp_map_elem_t *entry = (const mp_map_elem_t *)&mp_builtin_module_table[i]; - const char *key = qstr_str(MP_OBJ_QSTR_VALUE(entry->key)); - if (key[0] == 'u' && strcmp(&key[1], module_str) == 0) { - return (mp_obj_t)entry->value; - } - +// Tries to find a loaded module, otherwise attempts to load a builtin, otherwise MP_OBJ_NULL. +mp_obj_t mp_module_get_builtin(qstr module_name) { + // Try builtin. + mp_map_elem_t *elem = mp_map_lookup((mp_map_t *)&mp_builtin_module_map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP); + if (!elem) { + return MP_OBJ_NULL; } - return MP_OBJ_NULL; + + #if MICROPY_MODULE_BUILTIN_INIT + // If found, it's a newly loaded built-in, so init it. + mp_module_call_init(MP_OBJ_NEW_QSTR(module_name), elem->value); + #endif + + return elem->value; } #endif #if MICROPY_MODULE_BUILTIN_INIT -void mp_module_call_init(qstr module_name, mp_obj_t module_obj) { +STATIC void mp_module_register(mp_obj_t module_name, mp_obj_t module) { + mp_map_t *mp_loaded_modules_map = &MP_STATE_VM(mp_loaded_modules_dict).map; + mp_map_lookup(mp_loaded_modules_map, module_name, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = module; +} + +STATIC void mp_module_call_init(mp_obj_t module_name, mp_obj_t module_obj) { // Look for __init__ and call it if it exists mp_obj_t dest[2]; mp_load_method_maybe(module_obj, MP_QSTR___init__, dest); diff --git a/py/objmodule.h b/py/objmodule.h index b9106f524e..5e54dbf3ab 100644 --- a/py/objmodule.h +++ b/py/objmodule.h @@ -30,18 +30,9 @@ extern const mp_map_t mp_builtin_module_map; -mp_obj_t mp_module_get(qstr module_name); -void mp_module_register(qstr qstr, mp_obj_t module); - -mp_obj_t mp_module_search_umodule(const char *module_str); - -#if MICROPY_MODULE_BUILTIN_INIT -void mp_module_call_init(qstr module_name, mp_obj_t module_obj); -#else -static inline void mp_module_call_init(qstr module_name, mp_obj_t module_obj) { - (void)module_name; - (void)module_obj; -} +mp_obj_t mp_module_get_loaded_or_builtin(qstr module_name); +#if MICROPY_MODULE_WEAK_LINKS +mp_obj_t mp_module_get_builtin(qstr module_name); #endif #endif // MICROPY_INCLUDED_PY_OBJMODULE_H diff --git a/py/objtype.c b/py/objtype.c index cf94d300b9..d247987ce2 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -616,6 +616,7 @@ STATIC void mp_obj_instance_load_attr(mp_obj_t self_in, qstr attr, mp_obj_t *des assert(mp_obj_is_instance_type(mp_obj_get_type(self_in))); mp_obj_instance_t *self = MP_OBJ_TO_PTR(self_in); + // Note: This is fast-path'ed in the VM for the MP_BC_LOAD_ATTR operation. mp_map_elem_t *elem = mp_map_lookup(&self->members, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP); if (elem != NULL) { // object member, always treated as a value diff --git a/py/parse.c b/py/parse.c index f5f3e774da..dee662b4ec 100644 --- a/py/parse.c +++ b/py/parse.c @@ -802,9 +802,11 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { #endif STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t num_args) { - // optimise away parenthesis around an expression if possible + // Simplify and optimise certain rules, to reduce memory usage and simplify the compiler. if (rule_id == RULE_atom_paren) { - // there should be just 1 arg for this rule + // Remove parenthesis around a single expression if possible. + // This atom_paren rule always has a single argument, and after this + // optimisation that argument is either NULL or testlist_comp. mp_parse_node_t pn = peek_result(parser, 0); if (MP_PARSE_NODE_IS_NULL(pn)) { // need to keep parenthesis for () @@ -814,6 +816,34 @@ STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, // parenthesis around a single expression, so it's just the expression return; } + } else if (rule_id == RULE_testlist_comp) { + // The testlist_comp rule can be the sole argument to either atom_parent + // or atom_bracket, for (...) and [...] respectively. + assert(num_args == 2); + mp_parse_node_t pn = peek_result(parser, 0); + if (MP_PARSE_NODE_IS_STRUCT(pn)) { + mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn; + if (MP_PARSE_NODE_STRUCT_KIND(pns) == RULE_testlist_comp_3b) { + // tuple of one item, with trailing comma + pop_result(parser); + --num_args; + } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == RULE_testlist_comp_3c) { + // tuple of many items, convert testlist_comp_3c to testlist_comp + pop_result(parser); + assert(pn == peek_result(parser, 0)); + pns->kind_num_nodes = rule_id | MP_PARSE_NODE_STRUCT_NUM_NODES(pns) << 8; + return; + } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == RULE_comp_for) { + // generator expression + } else { + // tuple with 2 items + } + } else { + // tuple with 2 items + } + } else if (rule_id == RULE_testlist_comp_3c) { + // steal first arg of outer testlist_comp rule + ++num_args; } #if MICROPY_COMP_CONST_FOLDING @@ -833,6 +863,10 @@ STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, for (size_t i = num_args; i > 0; i--) { pn->nodes[i - 1] = pop_result(parser); } + if (rule_id == RULE_testlist_comp_3c) { + // need to push something non-null to replace stolen first arg of testlist_comp + push_result_node(parser, (mp_parse_node_t)pn); + } push_result_node(parser, (mp_parse_node_t)pn); } diff --git a/py/persistentcode.h b/py/persistentcode.h index 8f459999c2..1c53ca12cf 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -41,16 +41,15 @@ #define MPY_FEATURE_ENCODE_ARCH(arch) ((arch) << 2) #define MPY_FEATURE_DECODE_ARCH(feat) ((feat) >> 2) -// The feature flag bits encode the compile-time config options that -// affect the generate bytecode. +// The feature flag bits encode the compile-time config options that affect +// the generate bytecode. Note: position 0 is now unused +// (formerly MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE). #define MPY_FEATURE_FLAGS ( \ - ((MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) << 0) \ - | ((MICROPY_PY_BUILTINS_STR_UNICODE) << 1) \ + ((MICROPY_PY_BUILTINS_STR_UNICODE) << 1) \ ) // This is a version of the flags that can be configured at runtime. #define MPY_FEATURE_FLAGS_DYNAMIC ( \ - ((MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) << 0) \ - | ((MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) << 1) \ + ((MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) << 1) \ ) // Define the host architecture diff --git a/py/profile.c b/py/profile.c index 7d1d7ed659..81c80f1168 100644 --- a/py/profile.c +++ b/py/profile.c @@ -540,9 +540,6 @@ STATIC const byte *mp_prof_opcode_decode(const byte *ip, const mp_uint_t *const_ instruction->qstr_opname = MP_QSTR_LOAD_NAME; instruction->arg = qst; instruction->argobj = MP_OBJ_NEW_QSTR(qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { - instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT(*ip++); - } break; case MP_BC_LOAD_GLOBAL: @@ -550,9 +547,6 @@ STATIC const byte *mp_prof_opcode_decode(const byte *ip, const mp_uint_t *const_ instruction->qstr_opname = MP_QSTR_LOAD_GLOBAL; instruction->arg = qst; instruction->argobj = MP_OBJ_NEW_QSTR(qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { - instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT(*ip++); - } break; case MP_BC_LOAD_ATTR: @@ -560,9 +554,6 @@ STATIC const byte *mp_prof_opcode_decode(const byte *ip, const mp_uint_t *const_ instruction->qstr_opname = MP_QSTR_LOAD_ATTR; instruction->arg = qst; instruction->argobj = MP_OBJ_NEW_QSTR(qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { - instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT(*ip++); - } break; case MP_BC_LOAD_METHOD: @@ -618,9 +609,6 @@ STATIC const byte *mp_prof_opcode_decode(const byte *ip, const mp_uint_t *const_ instruction->qstr_opname = MP_QSTR_STORE_ATTR; instruction->arg = qst; instruction->argobj = MP_OBJ_NEW_QSTR(qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { - instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT(*ip++); - } break; case MP_BC_STORE_SUBSCR: diff --git a/py/py.mk b/py/py.mk index db92ef3c02..9e2e522cf0 100644 --- a/py/py.mk +++ b/py/py.mk @@ -224,19 +224,6 @@ ifneq ($(FROZEN_MANIFEST),) PY_O += $(BUILD)/$(BUILD)/frozen_content.o endif -# object file for frozen files -ifneq ($(FROZEN_DIR),) -PY_O += $(BUILD)/frozen.o -endif - -# Combine old singular FROZEN_MPY_DIR with new multiple value form. -FROZEN_MPY_DIRS += $(FROZEN_MPY_DIR) - -# object file for frozen bytecode (frozen .mpy files) -ifneq ($(FROZEN_MPY_DIRS),) -PY_O += $(BUILD)/frozen_mpy.o -endif - # Sources that may contain qstrings SRC_QSTR_IGNORE = py/nlr% SRC_QSTR_EMITNATIVE = py/emitn% diff --git a/py/qstr.c b/py/qstr.c index d7b05930f7..41176ec5ab 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -169,6 +169,12 @@ STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { + (sizeof(const char *) + sizeof(qstr_attr_t)) * new_pool_length; qstr_pool_t *pool = (qstr_pool_t *)m_malloc_maybe(pool_size, true); if (pool == NULL) { + // Keep qstr_last_chunk consistent with qstr_pool_t: qstr_last_chunk is not scanned + // at garbage collection since it's reachable from a qstr_pool_t. And the caller of + // this function expects q_ptr to be stored in a qstr_pool_t so it can be reached + // by the collector. If qstr_pool_t allocation failed, qstr_last_chunk needs to be + // NULL'd. Otherwise it may become a dangling pointer at the next garbage collection. + MP_STATE_VM(qstr_last_chunk) = NULL; QSTR_EXIT(); m_malloc_fail(new_pool_length); } diff --git a/py/qstrdefs.h b/py/qstrdefs.h index 6337399d16..02b87f4ec3 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -60,6 +60,10 @@ Q() Q() Q(utf-8) +#if MICROPY_MODULE_FROZEN +Q(.frozen) +#endif + #if MICROPY_ENABLE_PYSTACK Q(pystack exhausted) #endif diff --git a/py/runtime.c b/py/runtime.c index 1554a70231..bf44234434 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -126,6 +126,15 @@ void mp_init(void) { sizeof(MP_STATE_VM(fs_user_mount)) - MICROPY_FATFS_NUM_PERSISTENT); #endif + #if MICROPY_PY_SYS_PATH_ARGV_DEFAULTS + mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), 0); + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script) + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); + #endif + mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), 0); + #endif + #if MICROPY_PY_SYS_ATEXIT MP_STATE_VM(sys_exitfunc) = mp_const_none; #endif @@ -157,7 +166,7 @@ void mp_deinit(void) { #endif } -mp_obj_t mp_load_name(qstr qst) { +mp_obj_t MICROPY_WRAP_MP_LOAD_NAME(mp_load_name)(qstr qst) { // logic: search locals, globals, builtins DEBUG_OP_printf("load name %s\n", qstr_str(qst)); // If we're at the outer scope (locals == globals), dispatch to load_global right away @@ -170,7 +179,7 @@ mp_obj_t mp_load_name(qstr qst) { return mp_load_global(qst); } -mp_obj_t mp_load_global(qstr qst) { +mp_obj_t MICROPY_WRAP_MP_LOAD_GLOBAL(mp_load_global)(qstr qst) { // logic: search globals, builtins DEBUG_OP_printf("load global %s\n", qstr_str(qst)); mp_map_elem_t *elem = mp_map_lookup(&mp_globals_get()->map, MP_OBJ_NEW_QSTR(qst), MP_MAP_LOOKUP); @@ -310,7 +319,7 @@ mp_obj_t mp_unary_op(mp_unary_op_t op, mp_obj_t arg) { } } -mp_obj_t PLACE_IN_ITCM(mp_binary_op)(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { +mp_obj_t MICROPY_WRAP_MP_BINARY_OP(mp_binary_op)(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { DEBUG_OP_printf("binary " UINT_FMT " %q %p %p\n", op, mp_binary_op_method_name[op], lhs, rhs); // TODO correctly distinguish inplace operators for mutable objects @@ -1103,6 +1112,10 @@ void mp_load_method_maybe(mp_obj_t obj, qstr attr, mp_obj_t *dest) { dest[0] = MP_OBJ_NULL; dest[1] = MP_OBJ_NULL; + // Note: the specific case of obj being an instance type is fast-path'ed in the VM + // for the MP_BC_LOAD_ATTR opcode. Instance types handle type->attr and look up directly + // in their member's map. + // get the type const mp_obj_type_t *type = mp_obj_get_type(obj); @@ -1124,7 +1137,14 @@ void mp_load_method_maybe(mp_obj_t obj, qstr attr, mp_obj_t *dest) { if (attr_fun != NULL) { // this type can do its own load, so call it attr_fun(obj, attr, dest); - return; + + // If type->attr has set dest[1] = MP_OBJ_SENTINEL, we should proceed + // with lookups below (i.e. in locals_dict). If not, return right away. + if (dest[1] != MP_OBJ_SENTINEL) { + return; + } + // Clear the fail flag set by type->attr so it's like it never ran. + dest[1] = MP_OBJ_NULL; } if (type->locals_dict != NULL) { // generic method lookup @@ -1135,6 +1155,7 @@ void mp_load_method_maybe(mp_obj_t obj, qstr attr, mp_obj_t *dest) { if (elem != NULL) { mp_convert_member_lookup(obj, type, elem->value, dest); } + return; } } @@ -1426,8 +1447,10 @@ mp_obj_t mp_make_raise_obj(mp_obj_t o) { // create and return a new exception instance by calling o // TODO could have an option to disable traceback, then builtin exceptions (eg TypeError) // could have const instances in ROM which we return here instead - return mp_call_function_n_kw(o, 0, 0, NULL); - } else if (mp_obj_is_exception_instance(o)) { + o = mp_call_function_n_kw(o, 0, 0, NULL); + } + + if (mp_obj_is_exception_instance(o)) { // o is an instance of an exception, so use it as the exception return o; } else { diff --git a/py/showbc.c b/py/showbc.c index e9b43c29e3..1d02b559b0 100644 --- a/py/showbc.c +++ b/py/showbc.c @@ -98,8 +98,8 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i // raw bytecode dump size_t prelude_size = ip - mp_showbc_code_start + n_info + n_cell; - mp_printf(print, "Raw bytecode (code_info_size=" UINT_FMT ", bytecode_size=" UINT_FMT "):\n", - prelude_size, len - prelude_size); + mp_printf(print, "Raw bytecode (code_info_size=%u, bytecode_size=%u):\n", + (unsigned)prelude_size, (unsigned)(len - prelude_size)); for (mp_uint_t i = 0; i < len; i++) { if (i > 0 && i % 16 == 0) { mp_printf(print, "\n"); @@ -208,25 +208,16 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { case MP_BC_LOAD_NAME: DECODE_QSTR; mp_printf(print, "LOAD_NAME %s", qstr_str(qst)); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { - mp_printf(print, " (cache=%u)", *ip++); - } break; case MP_BC_LOAD_GLOBAL: DECODE_QSTR; mp_printf(print, "LOAD_GLOBAL %s", qstr_str(qst)); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { - mp_printf(print, " (cache=%u)", *ip++); - } break; case MP_BC_LOAD_ATTR: DECODE_QSTR; mp_printf(print, "LOAD_ATTR %s", qstr_str(qst)); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { - mp_printf(print, " (cache=%u)", *ip++); - } break; case MP_BC_LOAD_METHOD: @@ -270,9 +261,6 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { case MP_BC_STORE_ATTR: DECODE_QSTR; mp_printf(print, "STORE_ATTR %s", qstr_str(qst)); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { - mp_printf(print, " (cache=%u)", *ip++); - } break; case MP_BC_STORE_SUBSCR: @@ -531,7 +519,8 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { } else if (ip[-1] < MP_BC_STORE_FAST_MULTI + 16) { mp_printf(print, "STORE_FAST " UINT_FMT, (mp_uint_t)ip[-1] - MP_BC_STORE_FAST_MULTI); } else if (ip[-1] < MP_BC_UNARY_OP_MULTI + MP_UNARY_OP_NUM_BYTECODE) { - mp_printf(print, "UNARY_OP " UINT_FMT, (mp_uint_t)ip[-1] - MP_BC_UNARY_OP_MULTI); + mp_uint_t op = ip[-1] - MP_BC_UNARY_OP_MULTI; + mp_printf(print, "UNARY_OP " UINT_FMT " %s", op, qstr_str(mp_unary_op_method_name[op])); } else if (ip[-1] < MP_BC_BINARY_OP_MULTI + MP_BINARY_OP_NUM_BYTECODE) { mp_uint_t op = ip[-1] - MP_BC_BINARY_OP_MULTI; mp_printf(print, "BINARY_OP " UINT_FMT " %s", op, qstr_str(mp_binary_op_method_name[op])); diff --git a/py/vm.c b/py/vm.c index 9ad67bb2e0..63eb7f46d9 100644 --- a/py/vm.c +++ b/py/vm.c @@ -182,30 +182,13 @@ #define TRACE_TICK(current_ip, current_sp, is_exception) #endif // MICROPY_PY_SYS_SETTRACE -#if MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE -static inline mp_map_elem_t *mp_map_cached_lookup(mp_map_t *map, qstr qst, uint8_t *idx_cache) { - size_t idx = *idx_cache; - mp_obj_t key = MP_OBJ_NEW_QSTR(qst); - mp_map_elem_t *elem = NULL; - if (idx < map->alloc && map->table[idx].key == key) { - elem = &map->table[idx]; - } else { - elem = mp_map_lookup(map, key, MP_MAP_LOOKUP); - if (elem != NULL) { - *idx_cache = (elem - &map->table[0]) & 0xff; - } - } - return elem; -} -#endif - // fastn has items in reverse order (fastn[0] is local[0], fastn[-1] is local[1], etc) // sp points to bottom of stack which grows up // returns: // MP_VM_RETURN_NORMAL, sp valid, return value in *sp // MP_VM_RETURN_YIELD, ip, sp valid, yielded value in *sp // MP_VM_RETURN_EXCEPTION, exception in state[0] -mp_vm_return_kind_t PLACE_IN_ITCM(mp_execute_bytecode)(mp_code_state_t *code_state, volatile mp_obj_t inject_exc) { +mp_vm_return_kind_t MICROPY_WRAP_MP_EXECUTE_BYTECODE(mp_execute_bytecode)(mp_code_state_t *code_state, volatile mp_obj_t inject_exc) { #define SELECTIVE_EXC_IP (0) #if SELECTIVE_EXC_IP #define MARK_EXC_IP_SELECTIVE() { code_state->ip = ip; } /* stores ip 1 byte past last opcode */ @@ -373,84 +356,46 @@ dispatch_loop: goto load_check; } - #if !MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE ENTRY(MP_BC_LOAD_NAME): { MARK_EXC_IP_SELECTIVE(); DECODE_QSTR; PUSH(mp_load_name(qst)); DISPATCH(); } - #else - ENTRY(MP_BC_LOAD_NAME): { - MARK_EXC_IP_SELECTIVE(); - DECODE_QSTR; - mp_map_elem_t *elem = mp_map_cached_lookup(&mp_locals_get()->map, qst, (uint8_t*)ip); - mp_obj_t obj; - if (elem != NULL) { - obj = elem->value; - } else { - obj = mp_load_name(qst); - } - PUSH(obj); - ip++; - DISPATCH(); - } - #endif - #if !MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE ENTRY(MP_BC_LOAD_GLOBAL): { MARK_EXC_IP_SELECTIVE(); DECODE_QSTR; PUSH(mp_load_global(qst)); DISPATCH(); } - #else - ENTRY(MP_BC_LOAD_GLOBAL): { - MARK_EXC_IP_SELECTIVE(); - DECODE_QSTR; - mp_map_elem_t *elem = mp_map_cached_lookup(&mp_globals_get()->map, qst, (uint8_t*)ip); - mp_obj_t obj; - if (elem != NULL) { - obj = elem->value; - } else { - obj = mp_load_global(qst); - } - PUSH(obj); - ip++; - DISPATCH(); - } - #endif - #if !MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE - ENTRY(MP_BC_LOAD_ATTR): { - FRAME_UPDATE(); - MARK_EXC_IP_SELECTIVE(); - DECODE_QSTR; - SET_TOP(mp_load_attr(TOP(), qst)); - DISPATCH(); - } - #else ENTRY(MP_BC_LOAD_ATTR): { FRAME_UPDATE(); MARK_EXC_IP_SELECTIVE(); DECODE_QSTR; mp_obj_t top = TOP(); + mp_obj_t obj; + #if MICROPY_OPT_LOAD_ATTR_FAST_PATH + // For the specific case of an instance type, it implements .attr + // and forwards to its members map. Attribute lookups on instance + // types are extremely common, so avoid all the other checks and + // calls that normally happen first. mp_map_elem_t *elem = NULL; if (mp_obj_is_instance_type(mp_obj_get_type(top))) { mp_obj_instance_t *self = MP_OBJ_TO_PTR(top); - elem = mp_map_cached_lookup(&self->members, qst, (uint8_t*)ip); + elem = mp_map_lookup(&self->members, MP_OBJ_NEW_QSTR(qst), MP_MAP_LOOKUP); } - mp_obj_t obj; - if (elem != NULL) { + if (elem) { obj = elem->value; - } else { + } else + #endif + { obj = mp_load_attr(top, qst); } SET_TOP(obj); - ip++; DISPATCH(); } - #endif ENTRY(MP_BC_LOAD_METHOD): { MARK_EXC_IP_SELECTIVE(); @@ -506,7 +451,6 @@ dispatch_loop: DISPATCH(); } - #if !MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE ENTRY(MP_BC_STORE_ATTR): { FRAME_UPDATE(); MARK_EXC_IP_SELECTIVE(); @@ -515,32 +459,6 @@ dispatch_loop: sp -= 2; DISPATCH(); } - #else - // This caching code works with MICROPY_PY_BUILTINS_PROPERTY and/or - // MICROPY_PY_DESCRIPTORS enabled because if the attr exists in - // self->members then it can't be a property or have descriptors. A - // consequence of this is that we can't use MP_MAP_LOOKUP_ADD_IF_NOT_FOUND - // in the fast-path below, because that store could override a property. - ENTRY(MP_BC_STORE_ATTR): { - FRAME_UPDATE(); - MARK_EXC_IP_SELECTIVE(); - DECODE_QSTR; - mp_map_elem_t *elem = NULL; - mp_obj_t top = TOP(); - if (mp_obj_is_instance_type(mp_obj_get_type(top)) && sp[-1] != MP_OBJ_NULL) { - mp_obj_instance_t *self = MP_OBJ_TO_PTR(top); - elem = mp_map_cached_lookup(&self->members, qst, (uint8_t*)ip); - } - if (elem != NULL) { - elem->value = sp[-1]; - } else { - mp_store_attr(sp[0], qst, sp[-1]); - } - sp -= 2; - ip++; - DISPATCH(); - } - #endif ENTRY(MP_BC_STORE_SUBSCR): MARK_EXC_IP_SELECTIVE(); diff --git a/shared/libc/string0.c b/shared/libc/string0.c index fee2c017f8..0d22c2d99b 100644 --- a/shared/libc/string0.c +++ b/shared/libc/string0.c @@ -25,7 +25,7 @@ */ #include -#include +#include #ifndef likely #define likely(x) __builtin_expect((x), 1) @@ -68,6 +68,13 @@ void *memcpy(void *dst, const void *src, size_t n) { return dst; } +void *__memcpy_chk(void *dest, const void *src, size_t len, size_t slen) { + if (len > slen) { + return NULL; + } + return memcpy(dest, src, len); +} + void *memmove(void *dest, const void *src, size_t n) { if (src < dest && (uint8_t*)dest < (const uint8_t*)src + n) { // need to copy backwards diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 7b44bb15dc..3c42459aba 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -55,21 +55,21 @@ int pyexec_system_exit = 0; STATIC bool repl_display_debugging_info = 0; #endif -#define EXEC_FLAG_PRINT_EOF (1) -#define EXEC_FLAG_ALLOW_DEBUGGING (2) -#define EXEC_FLAG_IS_REPL (4) -#define EXEC_FLAG_SOURCE_IS_RAW_CODE (8) -#define EXEC_FLAG_SOURCE_IS_VSTR (16) -#define EXEC_FLAG_SOURCE_IS_FILENAME (32) -#define EXEC_FLAG_SOURCE_IS_READER (64) -#define EXEC_FLAG_SOURCE_IS_ATEXIT (128) +#define EXEC_FLAG_PRINT_EOF (1 << 0) +#define EXEC_FLAG_ALLOW_DEBUGGING (1 << 1) +#define EXEC_FLAG_IS_REPL (1 << 2) +#define EXEC_FLAG_SOURCE_IS_RAW_CODE (1 << 3) +#define EXEC_FLAG_SOURCE_IS_VSTR (1 << 4) +#define EXEC_FLAG_SOURCE_IS_FILENAME (1 << 5) +#define EXEC_FLAG_SOURCE_IS_READER (1 << 6) +#define EXEC_FLAG_SOURCE_IS_ATEXIT (1 << 7) // parses, compiles and executes the code in the lexer // frees the lexer before returning // EXEC_FLAG_PRINT_EOF prints 2 EOF chars: 1 after normal output, 1 after exception output // EXEC_FLAG_ALLOW_DEBUGGING allows debugging info to be printed after executing the code // EXEC_FLAG_IS_REPL is used for REPL inputs (flag passed on to mp_compile) -STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input_kind, int exec_flags, pyexec_result_t *result) { +STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input_kind, mp_uint_t exec_flags, pyexec_result_t *result) { int ret = 0; #if MICROPY_REPL_INFO uint32_t start = 0; @@ -166,6 +166,7 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input if (exec_flags & EXEC_FLAG_PRINT_EOF) { mp_hal_stdout_tx_strn("\x04", 1); } + // check for SystemExit if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type((mp_obj_t)nlr.ret_val)), MP_OBJ_FROM_PTR(&mp_type_SystemExit))) { // at the moment, the value of SystemExit is unused @@ -745,7 +746,7 @@ int pyexec_file(const char *filename, pyexec_result_t *result) { int pyexec_file_if_exists(const char *filename, pyexec_result_t *result) { #if MICROPY_MODULE_FROZEN - if (mp_frozen_stat(filename) == MP_IMPORT_STAT_FILE) { + if (mp_find_frozen_module(filename, NULL, NULL) == MP_IMPORT_STAT_FILE) { return pyexec_frozen_module(filename, result); } #endif @@ -758,7 +759,8 @@ int pyexec_file_if_exists(const char *filename, pyexec_result_t *result) { #if MICROPY_MODULE_FROZEN int pyexec_frozen_module(const char *name, pyexec_result_t *result) { void *frozen_data; - int frozen_type = mp_find_frozen_module(name, strlen(name), &frozen_data); + int frozen_type; + mp_find_frozen_module(name, &frozen_type, &frozen_data); switch (frozen_type) { #if MICROPY_MODULE_FROZEN_STR diff --git a/shared/upytesthelper/upytesthelper.c b/shared/upytesthelper/upytesthelper.c index 326172be65..ce60732424 100644 --- a/shared/upytesthelper/upytesthelper.c +++ b/shared/upytesthelper/upytesthelper.c @@ -94,6 +94,9 @@ void upytest_execute_test(const char *src) { gc_init(heap_start, heap_end); mp_init(); mp_obj_list_init(mp_sys_path, 0); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); + #endif mp_obj_list_init(mp_sys_argv, 0); nlr_buf_t nlr; diff --git a/tests/basics/int_big_cmp.py b/tests/basics/int_big_cmp.py index 7cb7412bdf..d7394d3bc8 100644 --- a/tests/basics/int_big_cmp.py +++ b/tests/basics/int_big_cmp.py @@ -1,10 +1,13 @@ # test bignum comparisons i = 1 << 65 +cases = (0, 1, -1, i, -i, i + 1, -(i + 1)) -print(i == 0) -print(i != 0) -print(i < 0) -print(i > 0) -print(i <= 0) -print(i >= 0) +for lhs in cases: + for rhs in cases: + print("{} == {} = {}".format(lhs, rhs, lhs == rhs)) + print("{} != {} = {}".format(lhs, rhs, lhs != rhs)) + print("{} < {} = {}".format(lhs, rhs, lhs < rhs)) + print("{} > {} = {}".format(lhs, rhs, lhs > rhs)) + print("{} <= {} = {}".format(lhs, rhs, lhs <= rhs)) + print("{} >= {} = {}".format(lhs, rhs, lhs >= rhs)) diff --git a/tests/basics/int_big_zeroone.py b/tests/basics/int_big_zeroone.py index 81636724a5..e97369a2db 100644 --- a/tests/basics/int_big_zeroone.py +++ b/tests/basics/int_big_zeroone.py @@ -1,4 +1,4 @@ -# test [0,-0,1,-1] edge cases of bignum +# test [0,1,-1] edge cases of bignum import skip_if skip_if.no_bigint() @@ -16,7 +16,7 @@ print([~c for c in cases]) print([c >> 1 for c in cases]) print([c << 1 for c in cases]) -# comparison of 0/-0/+0 +# comparison of 0 print(long_zero == 0) print(long_neg_zero == 0) print(long_one - 1 == 0) @@ -29,3 +29,40 @@ print(long_neg_zero < 1) print(long_neg_zero < -1) print(long_neg_zero > 1) print(long_neg_zero > -1) + +# generate zeros that involve negative numbers +large = 1 << 70 +large_plus_one = large + 1 +zeros = ( + large - large, + -large + large, + large + -large, + -(large - large), + large - large_plus_one + 1, + -large & (large - large), + -large ^ -large, + -large * (large - large), + (large - large) // -large, + -large // -large_plus_one, + -(large + large) % large, + (large + large) % -large, + -(large + large) % -large, +) +print(zeros) + +# compute arithmetic operations that may have problems with -0 +# (this checks that -0 is never generated in the zeros tuple) +cases = (0, 1, -1) + zeros +for lhs in cases: + print("-{} = {}".format(lhs, -lhs)) + print("~{} = {}".format(lhs, ~lhs)) + print("{} >> 1 = {}".format(lhs, lhs >> 1)) + print("{} << 1 = {}".format(lhs, lhs << 1)) + for rhs in cases: + print("{} == {} = {}".format(lhs, rhs, lhs == rhs)) + print("{} + {} = {}".format(lhs, rhs, lhs + rhs)) + print("{} - {} = {}".format(lhs, rhs, lhs - rhs)) + print("{} * {} = {}".format(lhs, rhs, lhs * rhs)) + print("{} | {} = {}".format(lhs, rhs, lhs | rhs)) + print("{} & {} = {}".format(lhs, rhs, lhs & rhs)) + print("{} ^ {} = {}".format(lhs, rhs, lhs ^ rhs)) diff --git a/tests/basics/string_fstring.py b/tests/basics/string_fstring.py index 4f7225fcad..7e8a97fd30 100644 --- a/tests/basics/string_fstring.py +++ b/tests/basics/string_fstring.py @@ -22,6 +22,13 @@ def foo(a, b): return f'{x}{y}{a}{b}' print(foo(7, 8)) +# ':' character within {...} that should not be interpreted as format specifiers. +print(f"a{[0,1,2][0:2]}") +print(f"a{[0,15,2][0:2][-1]:04x}") + +# Nested '{' and '}' characters. +print(f"a{ {0,1,2}}") + # PEP-0498 specifies that '\\' and '#' must be disallowed explicitly, whereas # MicroPython relies on the syntax error as a result of the substitution. diff --git a/tests/basics/subclass_native_exc_new.py b/tests/basics/subclass_native_exc_new.py new file mode 100644 index 0000000000..c1bd89a6fa --- /dev/null +++ b/tests/basics/subclass_native_exc_new.py @@ -0,0 +1,39 @@ +# test subclassing exceptions and providing __new__ + + +class Dummy(BaseException): + pass + + +class GoodException(BaseException): + def __new__(cls, *args, **kwargs): + print("GoodException __new__") + return Dummy(*args, **kwargs) + + +class BadException(BaseException): + def __new__(cls, *args, **kwargs): + print("BadException __new__") + return 1 + + +try: + raise GoodException("good message") +except BaseException as good: + print(type(good), good.args[0]) + +try: + raise BadException("bad message") +except Exception as bad: + # Should be TypeError 'exceptions must derive from BaseException' + print(type(bad), bad.args[0]) + +try: + + def gen(): + yield + + gen().throw(BadException) +except Exception as genbad: + # Should be TypeError 'exceptions must derive from BaseException' + print(type(genbad), genbad.args[0]) diff --git a/tests/basics/subclass_native_exc_new.py.exp b/tests/basics/subclass_native_exc_new.py.exp new file mode 100644 index 0000000000..65709b2ccf --- /dev/null +++ b/tests/basics/subclass_native_exc_new.py.exp @@ -0,0 +1,6 @@ +GoodException __new__ + good message +BadException __new__ + exceptions must derive from BaseException +BadException __new__ + exceptions must derive from BaseException diff --git a/tests/basics/sys_path.py b/tests/basics/sys_path.py new file mode 100644 index 0000000000..6456e24019 --- /dev/null +++ b/tests/basics/sys_path.py @@ -0,0 +1,16 @@ +# test sys.path + +try: + import usys as sys +except ImportError: + import sys + +# check that this script was executed from a file of the same name +if "__file__" not in globals() or "sys_path.py" not in __file__: + print("SKIP") + raise SystemExit + +# test that sys.path[0] is the directory containing this script +with open(sys.path[0] + "/sys_path.py") as f: + for _ in range(4): + print(f.readline()) diff --git a/tests/cmdline/cmd_parsetree.py.exp b/tests/cmdline/cmd_parsetree.py.exp index cc8ba82c05..bee4fc99d1 100644 --- a/tests/cmdline/cmd_parsetree.py.exp +++ b/tests/cmdline/cmd_parsetree.py.exp @@ -78,11 +78,11 @@ arg names: 45 STORE_NAME g 48 LOAD_CONST_OBJ \.\+ 50 LOAD_METHOD format -53 LOAD_NAME b (cache=0) -57 CALL_METHOD n=1 nkw=0 -59 STORE_NAME h -62 LOAD_CONST_NONE -63 RETURN_VALUE +53 LOAD_NAME b +56 CALL_METHOD n=1 nkw=0 +58 STORE_NAME h +61 LOAD_CONST_NONE +62 RETURN_VALUE mem: total=\\d\+, current=\\d\+, peak=\\d\+ stack: \\d\+ out of \\d\+ GC: total: \\d\+, used: \\d\+, free: \\d\+ diff --git a/tests/cmdline/cmd_showbc.py b/tests/cmdline/cmd_showbc.py index 3195d41e84..a960c15c4a 100644 --- a/tests/cmdline/cmd_showbc.py +++ b/tests/cmdline/cmd_showbc.py @@ -15,9 +15,9 @@ def f(): c = [1, 2] d = {1, 2} e = {} - f = {1: 2} - g = "a" - h = b"a" + f = {1:2} + g = 'a' + h = b'a' # unary/binary ops i = 1 @@ -59,7 +59,7 @@ def f(): # comprehensions a = (b for c in d if e) a = [b for c in d if e] - a = {b: b for c in d if e} + a = {b:b for c in d if e} # function calls a() @@ -108,10 +108,8 @@ def f(): # closed over variables x = 1 - def closure(): - nonlocal x - a = x + 1 + nonlocal x; a = x + 1 x = 1 del x @@ -128,14 +126,12 @@ def f(): return return 1 - # function with lots of locals def f(): l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = l10 = 1 m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = m10 = 2 l10 + m10 - # functions with default args def f(a=1): pass @@ -143,19 +139,16 @@ def f(a=1): def f(b=2): return b + a - # function which yields def f(): yield yield 1 yield from 1 - # class class Class: pass - # delete name del Class diff --git a/tests/cmdline/cmd_showbc.py.exp b/tests/cmdline/cmd_showbc.py.exp index e13bf0abed..22712b79ee 100644 --- a/tests/cmdline/cmd_showbc.py.exp +++ b/tests/cmdline/cmd_showbc.py.exp @@ -1,13 +1,13 @@ File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): ######## -\.\+51 63 +\.\+63 arg names: (N_STATE 3) (N_EXC_STACK 0) bc=0 line=1 ######## - bc=\\d\+ line=167 + bc=\\d\+ line=160 00 MAKE_FUNCTION \.\+ \\d\+ STORE_NAME f \\d\+ MAKE_FUNCTION \.\+ @@ -45,7 +45,7 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): (INIT_CELL 16) bc=0 line=1 ######## - bc=\\d\+ line=129 + bc=\\d\+ line=127 00 LOAD_CONST_NONE 01 LOAD_CONST_FALSE 02 BINARY_OP 27 __add__ @@ -92,10 +92,10 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): 58 BINARY_OP 27 __add__ \\d\+ STORE_FAST 8 \\d\+ LOAD_FAST 0 -\\d\+ UNARY_OP 1 +\\d\+ UNARY_OP 1 __neg__ \\d\+ STORE_FAST 9 \\d\+ LOAD_FAST 0 -\\d\+ UNARY_OP 3 +\\d\+ UNARY_OP 3 \\d\+ STORE_FAST 10 \\d\+ LOAD_FAST 0 \\d\+ LOAD_DEREF 14 @@ -116,14 +116,14 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): \\d\+ LOAD_DEREF 14 \\d\+ LOAD_FAST 1 \\d\+ BINARY_OP 2 __eq__ -\\d\+ UNARY_OP 3 +\\d\+ UNARY_OP 3 \\d\+ STORE_FAST 10 \\d\+ LOAD_DEREF 14 -\\d\+ LOAD_ATTR c (cache=0) +\\d\+ LOAD_ATTR c \\d\+ STORE_FAST 11 \\d\+ LOAD_FAST 11 \\d\+ LOAD_DEREF 14 -\\d\+ STORE_ATTR c (cache=0) +\\d\+ STORE_ATTR c \\d\+ LOAD_DEREF 14 \\d\+ LOAD_CONST_SMALL_INT 0 \\d\+ LOAD_SUBSCR @@ -233,7 +233,7 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): \\d\+ LOAD_DEREF 16 \\d\+ POP_TOP \\d\+ JUMP \\d\+ -\\d\+ LOAD_GLOBAL y (cache=0) +\\d\+ LOAD_GLOBAL y \\d\+ POP_TOP \\d\+ JUMP \\d\+ \\d\+ LOAD_DEREF 14 @@ -320,7 +320,7 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): (N_EXC_STACK 0) bc=0 line=1 ######## - bc=\\d\+ line=136 + bc=\\d\+ line=133 00 LOAD_CONST_SMALL_INT 1 01 DUP_TOP 02 STORE_FAST 0 @@ -376,7 +376,7 @@ arg names: a (N_EXC_STACK 0) (INIT_CELL 0) ######## - bc=\\d\+ line=143 + bc=\\d\+ line=139 00 LOAD_CONST_SMALL_INT 2 01 BUILD_TUPLE 1 03 LOAD_NULL @@ -393,9 +393,9 @@ arg names: (N_STATE 2) (N_EXC_STACK 0) bc=0 line=1 - bc=0 line=149 - bc=3 line=150 - bc=6 line=151 + bc=0 line=144 + bc=3 line=145 + bc=6 line=146 00 LOAD_CONST_NONE 01 YIELD_VALUE 02 POP_TOP @@ -418,13 +418,13 @@ arg names: (N_EXC_STACK 0) bc=0 line=1 ######## - bc=13 line=156 -00 LOAD_NAME __name__ (cache=0) -04 STORE_NAME __module__ -07 LOAD_CONST_STRING 'Class' -10 STORE_NAME __qualname__ -13 LOAD_CONST_NONE -14 RETURN_VALUE + bc=12 line=150 +00 LOAD_NAME __name__ +03 STORE_NAME __module__ +06 LOAD_CONST_STRING 'Class' +09 STORE_NAME __qualname__ +12 LOAD_CONST_NONE +13 RETURN_VALUE File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): ######## @@ -433,9 +433,9 @@ arg names: self (N_STATE 4) (N_EXC_STACK 0) bc=0 line=1 - bc=0 line=164 -00 LOAD_GLOBAL super (cache=0) -\\d\+ LOAD_GLOBAL __class__ (cache=0) + bc=0 line=157 +00 LOAD_GLOBAL super +\\d\+ LOAD_GLOBAL __class__ \\d\+ LOAD_FAST 0 \\d\+ LOAD_SUPER_METHOD f \\d\+ CALL_METHOD n=0 nkw=0 @@ -517,7 +517,7 @@ arg names: * (N_EXC_STACK 0) bc=0 line=1 ######## - bc=\\d\+ line=116 + bc=\\d\+ line=114 00 LOAD_DEREF 0 02 LOAD_CONST_SMALL_INT 1 03 BINARY_OP 27 __add__ @@ -536,7 +536,7 @@ arg names: * b (N_EXC_STACK 0) bc=0 line=1 ######## - bc=\\d\+ line=144 + bc=\\d\+ line=140 00 LOAD_FAST 1 01 LOAD_DEREF 0 03 BINARY_OP 27 __add__ diff --git a/tests/cmdline/cmd_verbose.py.exp b/tests/cmdline/cmd_verbose.py.exp index a2fdf1f00d..0edd050c22 100644 --- a/tests/cmdline/cmd_verbose.py.exp +++ b/tests/cmdline/cmd_verbose.py.exp @@ -8,12 +8,12 @@ arg names: (N_EXC_STACK 0) bc=0 line=1 bc=0 line=3 -00 LOAD_NAME print (cache=0) -04 LOAD_CONST_SMALL_INT 1 -05 CALL_FUNCTION n=1 nkw=0 -07 POP_TOP -08 LOAD_CONST_NONE -09 RETURN_VALUE +00 LOAD_NAME print +03 LOAD_CONST_SMALL_INT 1 +04 CALL_FUNCTION n=1 nkw=0 +06 POP_TOP +07 LOAD_CONST_NONE +08 RETURN_VALUE 1 mem: total=\\d\+, current=\\d\+, peak=\\d\+ stack: \\d\+ out of \\d\+ diff --git a/tests/cpydiff/core_fstring_concat.py b/tests/cpydiff/core_fstring_concat.py index fd83527b5c..c2bdb4e666 100644 --- a/tests/cpydiff/core_fstring_concat.py +++ b/tests/cpydiff/core_fstring_concat.py @@ -1,12 +1,13 @@ """ categories: Core -description: f-strings don't support concatenation with adjacent literals if the adjacent literals contain braces +description: f-strings don't support concatenation with adjacent literals if the adjacent literals contain braces or are f-strings cause: MicroPython is optimised for code space. -workaround: Use the + operator between literal strings when either is an f-string +workaround: Use the + operator between literal strings when either or both are f-strings """ -x = 1 -print("aa" f"{x}") -print(f"{x}" "ab") -print("a{}a" f"{x}") -print(f"{x}" "a{}b") +x, y = 1, 2 +print("aa" f"{x}") # works +print(f"{x}" "ab") # works +print("a{}a" f"{x}") # fails +print(f"{x}" "a{}b") # fails +print(f"{x}" f"{y}") # fails diff --git a/tests/cpydiff/core_fstring_parser.py b/tests/cpydiff/core_fstring_parser.py index 6917f3cfa4..22bbc5866e 100644 --- a/tests/cpydiff/core_fstring_parser.py +++ b/tests/cpydiff/core_fstring_parser.py @@ -1,9 +1,9 @@ """ categories: Core -description: f-strings cannot support expressions that require parsing to resolve nested braces +description: f-strings cannot support expressions that require parsing to resolve unbalanced nested braces and brackets cause: MicroPython is optimised for code space. -workaround: Only use simple expressions inside f-strings +workaround: Always use balanced braces and brackets in expressions inside f-strings """ -f'{"hello {} world"}' -f"{repr({})}" +print(f'{"hello { world"}') +print(f'{"hello ] world"}') diff --git a/tests/extmod/uasyncio_gather.py b/tests/extmod/uasyncio_gather.py index 0e2948b07c..6053873dbc 100644 --- a/tests/extmod/uasyncio_gather.py +++ b/tests/extmod/uasyncio_gather.py @@ -22,8 +22,9 @@ async def factorial(name, number): async def task(id): print("start", id) - await asyncio.sleep(0.2) + await asyncio.sleep(0.02) print("end", id) + return id async def gather_task(): @@ -36,12 +37,17 @@ async def main(): # Simple gather with return values print(await asyncio.gather(factorial("A", 2), factorial("B", 3), factorial("C", 4))) + # Test return_exceptions, where one task is cancelled and the other finishes normally + tasks = [asyncio.create_task(task(1)), asyncio.create_task(task(2))] + tasks[0].cancel() + print(await asyncio.gather(*tasks, return_exceptions=True)) + # Cancel a multi gather # TODO doesn't work, Task should not forward cancellation from gather to sub-task # but rather CancelledError should cancel the gather directly, which will then cancel # all sub-tasks explicitly # t = asyncio.create_task(gather_task()) - # await asyncio.sleep(0.1) + # await asyncio.sleep(0.01) # t.cancel() # await asyncio.sleep(0.01) diff --git a/tests/extmod/uasyncio_gather.py.exp b/tests/extmod/uasyncio_gather.py.exp index a37578d7eb..95310bbe1c 100644 --- a/tests/extmod/uasyncio_gather.py.exp +++ b/tests/extmod/uasyncio_gather.py.exp @@ -8,3 +8,6 @@ Task B: factorial(3) = 6 Task C: Compute factorial(4)... Task C: factorial(4) = 24 [2, 6, 24] +start 2 +end 2 +[CancelledError(), 2] diff --git a/tests/extmod/uselect_poll_udp.py b/tests/extmod/uselect_poll_udp.py index f6be262ee0..2a56a122b5 100644 --- a/tests/extmod/uselect_poll_udp.py +++ b/tests/extmod/uselect_poll_udp.py @@ -5,7 +5,9 @@ try: except ImportError: try: import socket, select - except ImportError: + + select.poll # Raises AttributeError for CPython implementations without poll() + except (ImportError, AttributeError): print("SKIP") raise SystemExit diff --git a/tests/import/mpy_native.py b/tests/import/mpy_native.py deleted file mode 100644 index db7881df43..0000000000 --- a/tests/import/mpy_native.py +++ /dev/null @@ -1,125 +0,0 @@ -# test importing of .mpy files with native code (x64 only) - -import sys, uio - -try: - uio.IOBase - import uos - - uos.mount -except (ImportError, AttributeError): - print("SKIP") - raise SystemExit - -if not (sys.platform == "linux" and sys.maxsize > 2**32): - print("SKIP") - raise SystemExit - - -class UserFile(uio.IOBase): - def __init__(self, data): - self.data = data - self.pos = 0 - - def read(self): - return self.data - - def readinto(self, buf): - n = 0 - while n < len(buf) and self.pos < len(self.data): - buf[n] = self.data[self.pos] - n += 1 - self.pos += 1 - return n - - def ioctl(self, req, arg): - return 0 - - -class UserFS: - def __init__(self, files): - self.files = files - - def mount(self, readonly, mksfs): - pass - - def umount(self): - pass - - def stat(self, path): - if path in self.files: - return (32768, 0, 0, 0, 0, 0, 0, 0, 0, 0) - raise OSError - - def open(self, path, mode): - return UserFile(self.files[path]) - - -# these are the test .mpy files -user_files = { - # bad architecture - "/mod0.mpy": b"C\x05\xff\x00\x10", - # test loading of viper and asm - "/mod1.mpy": ( - b"C\x05\x0b\x1f\x20" # header - b"\x20" # n bytes, bytecode - b"\x00\x08\x02m\x02m" # prelude - b"\x51" # LOAD_CONST_NONE - b"\x63" # RETURN_VALUE - b"\x00\x02" # n_obj, n_raw_code - b"\x22" # n bytes, viper code - b"\x00\x00\x00\x00\x00\x00" # dummy machine code - b"\x00\x00" # qstr0 - b"\x01\x0c\x0aprint" # n_qstr, qstr0 - b"\x00\x00\x00" # scope_flags, n_obj, n_raw_code - b"\x23" # n bytes, asm code - b"\x00\x00\x00\x00\x00\x00\x00\x00" # dummy machine code - b"\x00\x00\x00" # scope_flags, n_pos_args, type_sig - ), - # test loading viper with truncated data - "/mod2.mpy": ( - b"C\x05\x0b\x1f\x20" # header - b"\x20" # n bytes, bytecode - b"\x00\x08\x02m\x02m" # prelude - b"\x51" # LOAD_CONST_NONE - b"\x63" # RETURN_VALUE - b"\x00\x01" # n_obj, n_raw_code - b"\x12" # n bytes(=4), viper code - ), - # test loading viper with additional scope flags and relocation - "/mod3.mpy": ( - b"C\x05\x0b\x1f\x20" # header - b"\x20" # n bytes, bytecode - b"\x00\x08\x02m\x02m" # prelude - b"\x51" # LOAD_CONST_NONE - b"\x63" # RETURN_VALUE - b"\x00\x01" # n_obj, n_raw_code - b"\x12" # n bytes(=4), viper code - b"\x00\x00\x00\x00" # dummy machine code - b"\x00" # n_qstr - b"\x81\x60" # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC (0xe0 encoded over two bytes) - b"\x00\x00" # n_obj, n_raw_code - b"\x06rodata" # rodata, 6 bytes - b"\x04" # bss, 4 bytes - b"\x03\x01\x00" # dummy relocation of rodata - ), -} - -# create and mount a user filesystem -uos.mount(UserFS(user_files), "/userfs") -sys.path.append("/userfs") - -# import .mpy files from the user filesystem -for i in range(len(user_files)): - mod = "mod%u" % i - try: - __import__(mod) - print(mod, "OK") - except ValueError as er: - print(mod, "ValueError", er) - except RuntimeError as er: - print(mod, "RuntimeError", er) - -# unmount and undo path addition -uos.umount("/userfs") -sys.path.pop() diff --git a/tests/import/mpy_native.py.exp b/tests/import/mpy_native.py.exp deleted file mode 100644 index 8ebfa81864..0000000000 --- a/tests/import/mpy_native.py.exp +++ /dev/null @@ -1,4 +0,0 @@ -mod0 ValueError incompatible native .mpy architecture -mod1 OK -mod2 RuntimeError Corrupt .mpy file -mod3 OK diff --git a/tests/io/resource_stream.py b/tests/io/resource_stream.py deleted file mode 100644 index 5656205b69..0000000000 --- a/tests/io/resource_stream.py +++ /dev/null @@ -1,15 +0,0 @@ -import uio -import sys - -try: - uio.resource_stream -except AttributeError: - print("SKIP") - raise SystemExit - -buf = uio.resource_stream("data", "file2") -print(buf.read()) - -# resource_stream(None, ...) look ups from current dir, hence sys.path[0] hack -buf = uio.resource_stream(None, sys.path[0] + "/data/file2") -print(buf.read()) diff --git a/tests/io/resource_stream.py.exp b/tests/io/resource_stream.py.exp deleted file mode 100644 index 75404a347a..0000000000 --- a/tests/io/resource_stream.py.exp +++ /dev/null @@ -1,2 +0,0 @@ -1234 -1234 diff --git a/tests/micropython/const.py b/tests/micropython/const.py index 1faf22be9a..1c805a45f9 100644 --- a/tests/micropython/const.py +++ b/tests/micropython/const.py @@ -1,4 +1,5 @@ # test constant optimisation +# This test will only work when MICROPY_COMP_CONST is enabled. from micropython import const diff --git a/tests/micropython/import_mpy_native_gc.py b/tests/micropython/import_mpy_native_gc.py index 9c00f7984d..58291449ce 100644 --- a/tests/micropython/import_mpy_native_gc.py +++ b/tests/micropython/import_mpy_native_gc.py @@ -48,8 +48,8 @@ class UserFS: # Pre-compiled examples/natmod/features0 example for various architectures, keyed # by the required value of sys.implementation.mpy. features0_file_contents = { - # -march=x64 -mcache-lookup-bc - 0xB05: b'C\x05\x0b\x1f \x84b\xe9/\x00\x00\x00SH\x8b\x1ds\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dA\x00\x00\x00H\x8b\x7f\x08H\x8bk(\xff\xd5H\x8d5 \x00\x00\x00I\x89\xc4H\x8b\x05.\x00\x00\x00\x0f\xb78\xffShL\x89\xe7\xff\xd5H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x90\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x84@\x12factorial \x00\x00\r \x01"\xa1\x1c\x01\x1e\xff', + # -march=x64 + 0xA05: b'C\x05\x0a\x1f \x84b\xe9/\x00\x00\x00SH\x8b\x1ds\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dA\x00\x00\x00H\x8b\x7f\x08L\x8bc(A\xff\xd4H\x8d5\x1f\x00\x00\x00H\x89\xc5H\x8b\x05-\x00\x00\x00\x0f\xb78\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x90\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x84@\x12factorial \x00\x00\r \x01"\xa1\x1c\x01\x1e\xff', # -march=armv7m 0x1605: b"C\x05\x16\x1f \x84\x12\x1a\xe0\x00\x00\x13\xb5\nK\nJ{D\x9cX\x02!\xe3h\x98G\x03F\x01 3\xb9\x02!#i\x01\x93\x02\xb0\xbd\xe8\x10@\x18GXC\x01;\xf4\xe7\x00\xbfj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\tN\tK~D\xf4X@hgi\xb8G\x05F\x07K\x07I\xf2XyD\x10\x88ck\x98G(F\xb8G h\xf8\xbd6\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x01\x84\x00\x12factorial \x00\x00\r<\x01>\xa18\x01:\xff", } diff --git a/tests/micropython/import_mpy_native_x64.py b/tests/micropython/import_mpy_native_x64.py index c0203cb61c..54fecc059a 100644 --- a/tests/micropython/import_mpy_native_x64.py +++ b/tests/micropython/import_mpy_native_x64.py @@ -9,7 +9,7 @@ except (ImportError, AttributeError): print("SKIP") raise SystemExit -if not (sys.platform == "linux" and sys.maxsize > 2**32): +if not (sys.platform == "linux" and sys.maxsize > 2 ** 32): print("SKIP") raise SystemExit @@ -52,11 +52,11 @@ class UserFS: # fmt: off user_files = { # bad architecture - '/mod0.mpy': b'C\x05\xff\x00\x10', + '/mod0.mpy': b'C\x05\xfe\x00\x10', # test loading of viper and asm '/mod1.mpy': ( - b'C\x05\x0b\x1f\x20' # header + b'C\x05\x0a\x1f\x20' # header b'\x20' # n bytes, bytecode b'\x00\x08\x02m\x02m' # prelude @@ -78,7 +78,7 @@ user_files = { # test loading viper with additional scope flags and relocation '/mod2.mpy': ( - b'C\x05\x0b\x1f\x20' # header + b'C\x05\x0a\x1f\x20' # header b'\x20' # n bytes, bytecode b'\x00\x08\x02m\x02m' # prelude diff --git a/tests/micropython/viper_subscr_multi.py b/tests/micropython/viper_subscr_multi.py new file mode 100644 index 0000000000..1561e5534d --- /dev/null +++ b/tests/micropython/viper_subscr_multi.py @@ -0,0 +1,20 @@ +# test viper with multiple subscripts in a single expression + + +@micropython.viper +def f1(b: ptr8): + b[0] += b[1] + + +@micropython.viper +def f2(b: ptr8, i: int): + b[0] += b[i] + + +b = bytearray(b"\x01\x02") +f1(b) +print(b) + +b = bytearray(b"\x01\x02") +f2(b, 1) +print(b) diff --git a/tests/micropython/viper_subscr_multi.py.exp b/tests/micropython/viper_subscr_multi.py.exp new file mode 100644 index 0000000000..a2c298bb16 --- /dev/null +++ b/tests/micropython/viper_subscr_multi.py.exp @@ -0,0 +1,2 @@ +bytearray(b'\x03\x02') +bytearray(b'\x03\x02') diff --git a/tests/misc/non_compliant.py b/tests/misc/non_compliant.py index f745c30e8e..cff1894106 100644 --- a/tests/misc/non_compliant.py +++ b/tests/misc/non_compliant.py @@ -54,8 +54,8 @@ except NotImplementedError: # str(...) with keywords not implemented try: str(b"abc", encoding="utf8") -except NotImplementedError: - print("NotImplementedError") +except TypeError: + print("TypeError") # str.rsplit(None, n) not implemented try: diff --git a/tests/misc/non_compliant.py.exp b/tests/misc/non_compliant.py.exp index 8518828ec3..9348d5959e 100644 --- a/tests/misc/non_compliant.py.exp +++ b/tests/misc/non_compliant.py.exp @@ -5,7 +5,7 @@ NotImplementedError NotImplementedError TypeError, ValueError NotImplementedError -NotImplementedError +TypeError NotImplementedError NotImplementedError NotImplementedError diff --git a/tests/perf_bench/bm_fft.py b/tests/perf_bench/bm_fft.py index fb79a9fd28..9a2d03d11b 100644 --- a/tests/perf_bench/bm_fft.py +++ b/tests/perf_bench/bm_fft.py @@ -15,7 +15,7 @@ def transform_radix2(vector, inverse): # Initialization n = len(vector) - levels = int(math.log2(n)) + levels = int(math.log(n) / math.log(2)) coef = (2 if inverse else -2) * cmath.pi / n exptable = [cmath.rect(1, i * coef) for i in range(n // 2)] vector = [vector[reverse(i, levels)] for i in range(n)] # Copy with bit-reversed permutation diff --git a/tests/run-tests.py b/tests/run-tests.py index 34372590a2..f7fdb66282 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -112,7 +112,7 @@ def run_micropython(pyb, args, test_file, is_special=False): def get(required=False): rv = b"" while True: - ready = select.select([emulator], [], [], 0.02) + ready = select.select([emulator], [], [emulator], 0.02) if ready[0] == [emulator]: rv += os.read(emulator, 1024) else: @@ -150,9 +150,10 @@ def run_micropython(pyb, args, test_file, is_special=False): p.kill() except ProcessLookupError: pass - os.close(emulator) os.close(subterminal) + os.close(emulator) else: + print("subprocess", args + [test_file]) output_mupy = subprocess.check_output( args + [test_file], stderr=subprocess.STDOUT ) @@ -757,9 +758,7 @@ the last matching regex is used: cmd_parser.add_argument( "--via-mpy", action="store_true", help="compile .py files to .mpy first" ) - cmd_parser.add_argument( - "--mpy-cross-flags", default="-mcache-lookup-bc", help="flags to pass to mpy-cross" - ) + cmd_parser.add_argument("--mpy-cross-flags", default="", help="flags to pass to mpy-cross") cmd_parser.add_argument( "--keep-path", action="store_true", help="do not clear MICROPYPATH when running tests" ) @@ -871,7 +870,7 @@ the last matching regex is used: if not args.keep_path: # clear search path to make sure tests use only builtin modules and those in extmod - os.environ["MICROPYPATH"] = os.pathsep + base_path("../extmod") + os.environ["MICROPYPATH"] = os.pathsep + ".frozen" + os.pathsep + base_path("../extmod") try: os.makedirs(args.result_dir, exist_ok=True) diff --git a/tests/unix/extra_coverage.py b/tests/unix/extra_coverage.py index b4808993a7..8ea27cbf2f 100644 --- a/tests/unix/extra_coverage.py +++ b/tests/unix/extra_coverage.py @@ -89,12 +89,6 @@ try: except ZeroDivisionError: print("ZeroDivisionError") -# test loading a resource from a frozen string -import uio - -buf = uio.resource_stream("frzstr_pkg2", "mod.py") -print(buf.read(21)) - # test for MP_QSTR_NULL regression from frzqstr import returns_NULL diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index 4cce80a412..d3a2f651ee 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -44,10 +44,10 @@ ime utime utimeq -argv byteorder exc_info exit -getsizeof implementation maxsize modules -path platform stderr stdin -stdout version version_info +argv atexit byteorder exc_info +exit getsizeof implementation maxsize +modules path platform stderr +stdin stdout version version_info ementation # attrtuple (start=1, stop=2, step=3) @@ -165,15 +165,14 @@ cpp None frzstr1 frzstr1.py frzmpy1 -.frozen/frzmpy1.py +frzmpy1.py frzstr_pkg1.__init__ frzstr_pkg1/__init__.py 1 frzmpy_pkg1.__init__ -.frozen/frzmpy_pkg1/__init__.py 1 +frzmpy_pkg1/__init__.py 1 frzstr_pkg2.mod 1 frzmpy_pkg2.mod 1 ZeroDivisionError -b'# test frozen package' NULL diff --git a/tools/autobuild/build-boards.sh b/tools/autobuild/build-boards.sh new file mode 100755 index 0000000000..4b5259b667 --- /dev/null +++ b/tools/autobuild/build-boards.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# +# The functions in this file can be run independently to build boards. +# For example: +# +# $ source build-boards.sh +# $ MICROPY_AUTOBUILD_MAKE=make build_rp2_boards -latest /tmp + +function build_board { + # check/get parameters + if [ $# -lt 4 ]; then + echo "usage: $0 " + return 1 + fi + + board_json=$1 + fw_tag=$2 + dest_dir=$3 + shift + shift + shift + + board=$(echo $board_json | awk -F '/' '{ print $2 }') + descr=$(cat $board_json | python3 -c "import json,sys; print(json.load(sys.stdin).get('id', '$board'))") + build_dir=/tmp/micropython-build-$board + + echo "building $descr $board" + $MICROPY_AUTOBUILD_MAKE BOARD=$board BUILD=$build_dir && ( + for ext in $@; do + dest=$dest_dir/$descr$fw_tag.$ext + if [ -r $build_dir/firmware.$ext ]; then + mv $build_dir/firmware.$ext $dest + else + # esp32 has micropython.elf and micropython.map + mv $build_dir/micropython.$ext $dest + fi + done + ) + rm -rf $build_dir +} + +function build_boards { + # check/get parameters + if [ $# -lt 4 ]; then + echo "usage: $0 " + return 1 + fi + + check_file=$1 + shift + + # check we are in the correct directory + if [ ! -r $check_file ]; then + echo "must be in directory containing $check_file" + return 1 + fi + + # build all boards that have a board.json file + for board_json in $(find boards/ -name board.json | sort); do + build_board $board_json $@ + done +} + +function build_esp32_boards { + # check/get parameters + if [ $# != 2 ]; then + echo "usage: $0 " + return 1 + fi + + fw_tag=$1 + dest_dir=$2 + + # check we are in the correct directory + if [ ! -r modesp32.c ]; then + echo "must be in esp32 directory" + return 1 + fi + + # build the boards, based on the IDF version + for board_json in $(find boards/ -name board.json | sort); do + mcu=$(cat $board_json | python3 -c "import json,sys; print(json.load(sys.stdin).get('mcu', 'unknown'))") + if idf.py --version | grep -q v4.2; then + if [ $mcu = esp32 ]; then + # build standard esp32-based boards with IDF v4.2 + if echo $board_json | grep -q GENERIC; then + # traditionally, GENERIC and GENERIC_SPIRAM boards used manifest_release.py + MICROPY_AUTOBUILD_MAKE="$MICROPY_AUTOBUILD_MAKE FROZEN_MANIFEST=$(pwd)/boards/manifest_release.py" build_board $board_json $fw_tag $dest_dir bin elf map + else + build_board $board_json $fw_tag $dest_dir bin elf map + fi + fi + else + if [ $mcu != esp32 ]; then + # build esp32-s2/s3/c3 based boards with IDF v4.4+ + build_board $board_json $fw_tag $dest_dir bin elf map + fi + fi + done +} + +function build_mimxrt_boards { + build_boards modmimxrt.c $1 $2 bin hex +} + +function build_rp2_boards { + build_boards modrp2.c $1 $2 uf2 +} + +function build_samd_boards { + build_boards samd_soc.c $1 $2 uf2 +} + +function build_stm32_boards { + build_boards modpyb.c $1 $2 dfu hex +} diff --git a/tools/autobuild/build-downloads.py b/tools/autobuild/build-downloads.py new file mode 100755 index 0000000000..0f532e8bf0 --- /dev/null +++ b/tools/autobuild/build-downloads.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +import glob +import json +import os +import sys + + +def main(repo_path, output_path): + boards_index = [] + board_ids = set() + + for board_json in glob.glob(os.path.join(repo_path, "ports/*/boards/*/board.json")): + # Relative path to the board directory (e.g. "ports/stm32/boards/PYBV11"). + board_dir = os.path.dirname(board_json) + # Relative path to the port (e.g. "ports/stm32") + port_dir = os.path.dirname(os.path.dirname(board_dir)) + + with open(board_json, "r") as f: + blob = json.load(f) + + # Use "id" if specified, otherwise default to board dir (e.g. "PYBV11"). + # We allow boards to override ID for the historical build names. + blob["id"] = blob.get("id", os.path.basename(board_dir)) + + # Check for duplicate board IDs. + if blob["id"] in board_ids: + print("Duplicate board ID: '{}'".format(blob["id"]), file=sys.stderr) + board_ids.add(blob["id"]) + + # Add in default fields. + blob["port"] = os.path.basename(port_dir) + blob["build"] = os.path.basename(board_dir) + boards_index.append(blob) + + # Create the board markdown, which is the concatenation of the + # default "board.md" file (if exists), as well as any flashing + # instructions. + board_markdown = os.path.join(board_dir, "board.md") + with open(os.path.join(output_path, blob["id"] + ".md"), "w") as f: + if os.path.exists(board_markdown): + with open(board_markdown, "r") as fin: + f.write(fin.read()) + + if blob["deploy"]: + f.write("\n\n## Installation instructions\n") + for deploy in blob["deploy"]: + with open(os.path.join(board_dir, deploy), "r") as fin: + f.write(fin.read()) + + # Write the full index for the website to load. + with open(os.path.join(output_path, "index.json"), "w") as f: + json.dump(boards_index, f, indent=4, sort_keys=True) + f.write("\n") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) diff --git a/tools/codeformat.py b/tools/codeformat.py index c98b34b17a..edefbc8ddc 100644 --- a/tools/codeformat.py +++ b/tools/codeformat.py @@ -40,7 +40,6 @@ PATHS = [ # C "main.c", "devices/**/*.[ch]", - "drivers/bus/*.[ch]", "extmod/*.[ch]", "shared/netutils/*.[ch]", "shared/timeutils/*.[ch]", diff --git a/tools/dfu.py b/tools/dfu.py index 4dcb4fdca7..244629d60b 100644 --- a/tools/dfu.py +++ b/tools/dfu.py @@ -25,7 +25,7 @@ def consume(fmt, data, names): def cstring(string): - return string.split("\0", 1)[0] + return string.split(b"\0", 1)[0] def compute_crc(data): diff --git a/tools/makemanifest.py b/tools/makemanifest.py index 7897a83c6e..8cdc3eb774 100644 --- a/tools/makemanifest.py +++ b/tools/makemanifest.py @@ -201,8 +201,6 @@ def freeze_internal(kind, path, script, opt): if not os.path.isdir(path): raise FreezeError("freeze path must be a directory: {}".format(path)) if script is None and kind == KIND_AS_STR: - if any(f[0] == KIND_AS_STR for f in manifest_list): - raise FreezeError("can only freeze one str directory") manifest_list.append((KIND_AS_STR, path, script, opt)) elif script is None or isinstance(script, str) and script.find(".") == -1: # Recursively search `path` for files to freeze, optionally restricted @@ -235,6 +233,73 @@ def freeze_internal(kind, path, script, opt): manifest_list.append((kind, path, script, opt)) +# Formerly make-frozen.py. +# This generates: +# - MP_FROZEN_STR_NAMES macro +# - mp_frozen_str_sizes +# - mp_frozen_str_content +def generate_frozen_str_content(paths): + def module_name(f): + return f + + modules = [] + output = [b"#include \n"] + + for path in paths: + root = path.rstrip("/") + root_len = len(root) + + for dirpath, dirnames, filenames in os.walk(root): + for f in filenames: + fullpath = dirpath + "/" + f + st = os.stat(fullpath) + modules.append((path, fullpath[root_len + 1 :], st)) + + output.append(b"#define MP_FROZEN_STR_NAMES \\\n") + for _path, f, st in modules: + m = module_name(f) + output.append(b'"%s\\0" \\\n' % m.encode()) + output.append(b"\n") + + output.append(b"const uint32_t mp_frozen_str_sizes[] = { ") + + for _path, f, st in modules: + output.append(b"%d, " % st.st_size) + output.append(b"0 };\n") + + output.append(b"const char mp_frozen_str_content[] = {\n") + for path, f, st in modules: + data = open(path + "/" + f, "rb").read() + + # We need to properly escape the script data to create a C string. + # When C parses hex characters of the form \x00 it keeps parsing the hex + # data until it encounters a non-hex character. Thus one must create + # strings of the form "data\x01" "abc" to properly encode this kind of + # data. We could just encode all characters as hex digits but it's nice + # to be able to read the resulting C code as ASCII when possible. + + data = bytearray(data) # so Python2 extracts each byte as an integer + esc_dict = {ord("\n"): b"\\n", ord("\r"): b"\\r", ord('"'): b'\\"', ord("\\"): b"\\\\"} + output.append(b'"') + break_str = False + for c in data: + try: + output.append(esc_dict[c]) + except KeyError: + if 32 <= c <= 126: + if break_str: + output.append(b'" "') + break_str = False + output.append(chr(c).encode()) + else: + output.append(b"\\x%02x" % c) + break_str = True + output.append(b'\\0"\n') + + output.append(b'"\\0"\n};\n\n') + return b"".join(output) + + def main(): # Parse arguments import argparse @@ -264,7 +329,6 @@ def main(): sys.exit(1) # Get paths to tools - MAKE_FROZEN = VARS["MPY_DIR"] + "/tools/make-frozen.py" MPY_CROSS = VARS["MPY_DIR"] + "/mpy-cross/mpy-cross" if sys.platform == "win32": MPY_CROSS += ".exe" @@ -327,10 +391,7 @@ def main(): return # Freeze paths as strings - res, output_str = system([sys.executable, MAKE_FROZEN] + str_paths) - if res != 0: - print("error freezing strings {}: {}".format(str_paths, output_str)) - sys.exit(1) + output_str = generate_frozen_str_content(str_paths) # Freeze .mpy files if mpy_files: @@ -347,7 +408,7 @@ def main(): ) if res != 0: print("error freezing mpy {}:".format(mpy_files)) - print(str(output_mpy, "utf8")) + print(output_mpy.decode()) sys.exit(1) else: output_mpy = ( @@ -356,8 +417,8 @@ def main(): b"const qstr_pool_t mp_qstr_frozen_const_pool = {\n" b" (qstr_pool_t*)&mp_qstr_const_pool, MP_QSTRnumber_of, 0, 0\n" b"};\n" - b'const char mp_frozen_mpy_names[1] = {"\\0"};\n' - b"const mp_raw_code_t *const mp_frozen_mpy_content[1] = {NULL};\n" + b'const char mp_frozen_names[] = { MP_FROZEN_STR_NAMES "\\0"};\n' + b"const mp_raw_code_t *const mp_frozen_mpy_content[] = {NULL};\n" ) # Generate output diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 05a9db48da..32b064d35f 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -113,14 +113,6 @@ def mp_opcode_format(bytecode, ip, count_var_uint): ip_start = ip f = (0x000003A4 >> (2 * ((opcode) >> 4))) & 3 if f == MP_BC_FORMAT_QSTR: - if config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE: - if ( - opcode == MP_BC_LOAD_NAME - or opcode == MP_BC_LOAD_GLOBAL - or opcode == MP_BC_LOAD_ATTR - or opcode == MP_BC_STORE_ATTR - ): - ip += 1 ip += 3 else: extra_byte = (opcode & MP_BC_MASK_EXTRA_BYTE) == 0 @@ -421,10 +413,7 @@ class RawCodeBytecode(RawCode): "// frozen bytecode for file %s, scope %s%s" % (self.source_file.str, parent_name, self.simple_name.str) ) - print("STATIC ", end="") - if not config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE: - print("const ", end="") - print("byte fun_data_%s[%u] = {" % (self.escaped_name, len(self.bytecode))) + print("STATIC const byte fun_data_%s[%u] = {" % (self.escaped_name, len(self.bytecode))) print(" ", end="") for i in range(self.ip2): print(" 0x%02x," % self.bytecode[i], end="") @@ -779,7 +768,6 @@ def read_mpy(filename): raise Exception("incompatible .mpy version") feature_byte = header[2] qw_size = read_uint(f) - config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE = (feature_byte & 1) != 0 config.MICROPY_PY_BUILTINS_STR_UNICODE = (feature_byte & 2) != 0 mpy_native_arch = feature_byte >> 2 if mpy_native_arch != MP_NATIVE_ARCH_NONE: @@ -818,14 +806,6 @@ def freeze_mpy(base_qstrs, raw_codes): print('#include "py/nativeglue.h"') print() - print( - "#if MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE != %u" - % config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE - ) - print('#error "incompatible MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE"') - print("#endif") - print() - print("#if MICROPY_LONGINT_IMPL != %u" % config.MICROPY_LONGINT_IMPL) print('#error "incompatible MICROPY_LONGINT_IMPL"') print("#endif") @@ -899,7 +879,11 @@ def freeze_mpy(base_qstrs, raw_codes): rc.freeze(rc.source_file.str.replace("/", "_")[:-3] + "_") print() - print("const char mp_frozen_mpy_names[] = {") + print("const char mp_frozen_names[] = {") + print("#ifdef MP_FROZEN_STR_NAMES") + # makemanifest.py might also include some frozen string content. + print("MP_FROZEN_STR_NAMES") + print("#endif") for rc in raw_codes: module_name = rc.source_file.str print('"%s\\0"' % module_name) @@ -933,11 +917,7 @@ def merge_mpy(raw_codes, output_file): header = bytearray(5) header[0] = ord("C") header[1] = config.MPY_VERSION - header[2] = ( - config.native_arch << 2 - | config.MICROPY_PY_BUILTINS_STR_UNICODE << 1 - | config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE - ) + header[2] = config.native_arch << 2 | config.MICROPY_PY_BUILTINS_STR_UNICODE << 1 header[3] = config.mp_small_int_bits header[4] = 32 # qstr_win_size merged_mpy.extend(header) diff --git a/tools/mpy_cross_all.py b/tools/mpy_cross_all.py index 9c217400fb..1a2c612943 100755 --- a/tools/mpy_cross_all.py +++ b/tools/mpy_cross_all.py @@ -11,13 +11,13 @@ import os.path argparser = argparse.ArgumentParser(description="Compile all .py files to .mpy recursively") argparser.add_argument("-o", "--out", help="output directory (default: input dir)") argparser.add_argument("--target", help="select MicroPython target config") -argparser.add_argument( - "-mcache-lookup-bc", action="store_true", help="cache map lookups in the bytecode" -) argparser.add_argument("dir", help="input directory") args = argparser.parse_args() -TARGET_OPTS = {"unix": "-mcache-lookup-bc", "baremetal": ""} +TARGET_OPTS = { + "unix": "", + "baremetal": "", +} args.dir = args.dir.rstrip("/") diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index daa9eabca4..54df98ef07 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -48,7 +48,6 @@ MP_CODE_NATIVE_VIPER = 4 MP_SCOPE_FLAG_VIPERRELOC = 0x20 MP_SCOPE_FLAG_VIPERRODATA = 0x40 MP_SCOPE_FLAG_VIPERBSS = 0x80 -MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE = 1 MICROPY_PY_BUILTINS_STR_UNICODE = 2 MP_SMALL_INT_BITS = 31 QSTR_WINDOW_SIZE = 32 @@ -118,9 +117,7 @@ class ArchData: ARCH_DATA = { "x86": ArchData( "EM_386", - MP_NATIVE_ARCH_X86 << 2 - | MICROPY_PY_BUILTINS_STR_UNICODE - | MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE, + MP_NATIVE_ARCH_X86 << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, 2, 4, (R_386_PC32, R_386_GOT32, R_386_GOT32X), @@ -128,9 +125,7 @@ ARCH_DATA = { ), "x64": ArchData( "EM_X86_64", - MP_NATIVE_ARCH_X64 << 2 - | MICROPY_PY_BUILTINS_STR_UNICODE - | MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE, + MP_NATIVE_ARCH_X64 << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, 2, 8, (R_X86_64_GOTPCREL, R_X86_64_REX_GOTPCRELX), diff --git a/tools/upip.py b/tools/upip.py index 95cd8d5cc7..70afe36a45 100644 --- a/tools/upip.py +++ b/tools/upip.py @@ -192,9 +192,13 @@ def fatal(msg, exc=None): def install_pkg(pkg_spec, install_path): - data = get_pkg_metadata(pkg_spec) + package = pkg_spec.split("==") + data = get_pkg_metadata(package[0]) - latest_ver = data["info"]["version"] + if len(package) == 1: + latest_ver = data["info"]["version"] + else: + latest_ver = package[1] packages = data["releases"][latest_ver] del data gc.collect() @@ -258,6 +262,8 @@ def get_install_path(): if install_path is None: # sys.path[0] is current module's path install_path = sys.path[1] + if install_path == ".frozen": + install_path = sys.path[2] install_path = expandhome(install_path) return install_path @@ -277,11 +283,11 @@ upip - Simple PyPI package manager for MicroPython Usage: micropython -m upip install [-p ] ... | -r import upip; upip.install(package_or_list, []) -If is not given, packages will be installed into sys.path[1] -(can be set from MICROPYPATH environment variable, if current system -supports that).""" +If isn't given, packages will be installed to sys.path[1], or +sys.path[2] if the former is .frozen (path can be set from MICROPYPATH +environment variable if supported).""" ) - print("Current value of sys.path[1]:", sys.path[1]) + print("Default install path:", get_install_path()) print( """\