djsundog
/
circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #6069 from gamblor21/uzlib-module 

zlib Module
This commit is contained in:
Dan Halbert 2022-04-06 12:06:44 -04:00 committed by GitHub
parent 1a329cfd11 8ed7b114cd
commit 2693a4cfe1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 326 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/library/zlib.rst
View File

@ -5,6 +5,7 @@
.. module:: zlib
:synopsis: zlib decompression
:noindex:
|see_cpython_module| :mod:`cpython:zlib`.
@ -26,6 +27,7 @@ Functions
CPython and is ignored.
.. class:: DecompIO(stream, wbits=0, /)
:noindex:
Create a ``stream`` wrapper which allows transparent decompression of
compressed data in another *stream*. This allows to process compressed

2
lib/uzlib/uzlib.h
View File

@ -83,6 +83,8 @@ typedef struct {
} TINF_TREE;
struct uzlib_uncomp {
/* Point to the CircuitPython object that owns this decompression stream */
void *self;
/* Pointer to the next byte in the input buffer */
const unsigned char *source;
/* Pointer to the next byte past the input buffer (source_limit = source + len) */

2
locale/circuitpython.pot
View File

@ -3019,7 +3019,7 @@ msgstr ""
msgid "complex values not supported"
msgstr ""
#: extmod/moduzlib.c
#: extmod/moduzlib.c shared-module/zlib/DecompIO.c
msgid "compression header"
msgstr ""

1
ports/atmel-samd/boards/cp_sapling_m0_spiflash/mpconfigboard.mk
View File

@ -13,3 +13,4 @@ EXTERNAL_FLASH_DEVICES = AT25DF081A
CIRCUITPY_AUDIOIO = 0
CIRCUITPY_AUDIOBUSIO = 0
CIRCUITPY_ZLIB = 0

1
ports/atmel-samd/boards/matrixportal_m4/mpconfigboard.mk
View File

@ -16,6 +16,7 @@ CIRCUITPY_PARALLELDISPLAY = 0
CIRCUITPY_SDCARDIO = 0
CIRCUITPY_SHARPDISPLAY = 0
CIRCUITPY_TRACEBACK = 0
CIRCUITPY_ZLIB=0
# Include these Python libraries in firmware.
FROZEN_MPY_DIRS += $(TOP)/frozen/Adafruit_CircuitPython_Requests

1
ports/atmel-samd/mpconfigport.mk
View File

@ -46,6 +46,7 @@ CIRCUITPY_SYNTHIO ?= 0
CIRCUITPY_TOUCHIO_USE_NATIVE ?= 1
CIRCUITPY_ULAB = 0
CIRCUITPY_VECTORIO = 0
CIRCUITPY_ZLIB = 0
# TODO: In CircuitPython 8.0, turn this back on, after `busio.OneWire` is removed.
# We'd like a smoother transition, but we can't afford the space to have both

1
ports/nrf/boards/bluemicro833/mpconfigboard.mk
View File

@ -42,6 +42,7 @@ CIRCUITPY_TRACEBACK = 0
CIRCUITPY_ULAB = 0
CIRCUITPY_USB_MIDI = 0
CIRCUITPY_VECTORIO = 0
CIRCUITPY_ZLIB = 0
MICROPY_PY_ASYNC_AWAIT = 0

1
ports/nrf/boards/pca10100/mpconfigboard.mk
View File

@ -32,6 +32,7 @@ CIRCUITPY_SYNTHIO = 0
CIRCUITPY_ULAB = 0
CIRCUITPY_USB_MIDI = 0
CIRCUITPY_VECTORIO = 0
CIRCUITPY_ZLIB = 0
MICROPY_PY_ASYNC_AWAIT = 0

1
ports/nrf/boards/simmel/mpconfigboard.mk
View File

@ -42,6 +42,7 @@ CIRCUITPY_ULAB = 0
CIRCUITPY_USB_CDC = 0
CIRCUITPY_USB_MIDI = 0
CIRCUITPY_WATCHDOG = 1
CIRCUITPY_ZLIB = 0
# Enable micropython.native
#CIRCUITPY_ENABLE_MPY_NATIVE = 1

1
ports/stm/boards/meowbit_v121/mpconfigboard.mk
View File

@ -26,5 +26,6 @@ CIRCUITPY_BLEIO_HCI = 0
CIRCUITPY_GIFIO = 0
CIRCUITPY_ULAB = 0
CIRCUITPY_STAGE = 1
CIRCUITPY_ZLIB = 0
FROZEN_MPY_DIRS += $(TOP)/frozen/circuitpython-stage/meowbit

1
ports/stm/boards/thunderpack_v11/mpconfigboard.mk
View File

@ -21,3 +21,4 @@ CIRCUITPY_BITMAPTOOLS = 0
CIRCUITPY_BLEIO_HCI = 0
CIRCUITPY_VECTORIO = 0
CIRCUITPY_ULAB = 0
CIRCUITPY_ZLIB = 0

1
ports/stm/boards/thunderpack_v12/mpconfigboard.mk
View File

@ -14,6 +14,7 @@ EXTERNAL_FLASH_DEVICES = GD25Q16C
CIRCUITPY_NVM = 1
CIRCUITPY_BITMAPTOOLS = 0
CIRCUITPY_BLEIO_HCI = 0
CIRCUITPY_ZLIB = 0
MCU_SERIES = F4
MCU_VARIANT = STM32F411xE

7
ports/unix/variants/coverage/mpconfigvariant.mk
View File

@ -36,6 +36,7 @@ SRC_BITMAP := \
shared-bindings/rainbowio/__init__.c \
shared-bindings/traceback/__init__.c \
shared-bindings/util.c \
shared-bindings/zlib/__init__.c \
shared-module/aesio/aes.c \
shared-module/aesio/__init__.c \
shared-module/bitmaptools/__init__.c \
@ -44,7 +45,8 @@ SRC_BITMAP := \
shared-module/displayio/ColorConverter.c \
shared-module/displayio/ColorConverter.c \
shared-module/rainbowio/__init__.c \
shared-module/traceback/__init__.c
shared-module/traceback/__init__.c \
shared-module/zlib/__init__.c \
$(info $(SRC_BITMAP))
SRC_C += $(SRC_BITMAP)
@ -55,7 +57,8 @@ CFLAGS += \
-DCIRCUITPY_DISPLAYIO_UNIX=1 \
-DCIRCUITPY_GIFIO=1 \
-DCIRCUITPY_RAINBOWIO=1 \
-DCIRCUITPY_TRACEBACK=1
-DCIRCUITPY_TRACEBACK=1 \
-DCIRCUITPY_ZLIB=1
SRC_C += coverage.c
SRC_CXX += coveragecpp.cpp

15
py/circuitpy_defns.mk
View File

@ -331,6 +331,9 @@ endif
ifeq ($(CIRCUITPY_USTACK),1)
SRC_PATTERNS += ustack/%
endif
ifeq ($(CIRCUITPY_ZLIB),1)
SRC_PATTERNS += zlib/%
endif
ifeq ($(CIRCUITPY_VIDEOCORE),1)
SRC_PATTERNS += videocore/%
endif
@ -597,6 +600,7 @@ SRC_SHARED_MODULE_ALL = \
usb/core/__init__.c \
usb/core/Device.c \
ustack/__init__.c \
zlib/__init__.c \
vectorio/Circle.c \
vectorio/Polygon.c \
vectorio/Rectangle.c \
@ -653,6 +657,17 @@ SRC_MOD += $(addprefix lib/protomatter/src/, \
$(BUILD)/lib/protomatter/src/core.o: CFLAGS += -include "shared-module/rgbmatrix/allocator.h" -DCIRCUITPY -Wno-missing-braces -Wno-missing-prototypes
endif
ifeq ($(CIRCUITPY_ZLIB),1)
SRC_MOD += $(addprefix lib/uzlib/, \
tinflate.c \
tinfzlib.c \
tinfgzip.c \
adler32.c \
crc32.c \
)
$(BUILD)/lib/uzlib/tinflate.o: CFLAGS += -Wno-missing-braces -Wno-missing-prototypes
endif
# All possible sources are listed here, and are filtered by SRC_PATTERNS.
SRC_SHARED_MODULE_INTERNAL = \
$(filter $(SRC_PATTERNS), \

4
py/circuitpy_mpconfig.mk
View File

@ -450,6 +450,10 @@ CFLAGS += -DUSB_NUM_ENDPOINT_PAIRS=$(USB_NUM_ENDPOINT_PAIRS)
CIRCUITPY_USTACK ?= 0
CFLAGS += -DCIRCUITPY_USTACK=$(CIRCUITPY_USTACK)
# for decompressing utlities
CIRCUITPY_ZLIB ?= 1
CFLAGS += -DCIRCUITPY_ZLIB=$(CIRCUITPY_ZLIB)
# ulab numerics library
CIRCUITPY_ULAB ?= $(CIRCUITPY_FULL_BUILD)
CFLAGS += -DCIRCUITPY_ULAB=$(CIRCUITPY_ULAB)

95
shared-bindings/zlib/__init__.c Normal file
View File

@ -0,0 +1,95 @@
/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2022 Mark Komus
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <stdint.h>
#include <assert.h>
#include <string.h>
#include "py/obj.h"
#include "py/runtime.h"
#include "py/stream.h"
#include "py/mperrno.h"
#include "py/builtin.h"
#include "py/objtuple.h"
#include "py/binary.h"
#include "py/parsenum.h"
#include "shared-bindings/zlib/__init__.h"
#include "supervisor/shared/translate.h"
//| """zlib decompression functionality
//|
//| The `zlib` module allows limited functionality similar to the CPython zlib library.
//| This module allows to decompress binary data compressed with DEFLATE algorithm
//| (commonly used in zlib library and gzip archiver). Compression is not yet implemented."""
//|
//| def zlib_decompress(data: bytes, wbits: Optional[int] = 0, bufsize: Optional[int] = 0) -> bytes:
//| """Return decompressed *data* as bytes. *wbits* is DEFLATE dictionary window
//| size used during compression (8-15, the dictionary size is power of 2 of
//| that value). Additionally, if value is positive, *data* is assumed to be
//| zlib stream (with zlib header). Otherwise, if it's negative, it's assumed
//| to be raw DEFLATE stream.
//|
//| The wbits parameter controls the size of the history buffer (or “window size”), and what header
//| and trailer format is expected.
//|
//| Common wbits values:
//|
//| * To decompress deflate format, use wbits = -15
//| * To decompress zlib format, use wbits = 15
//| * To decompress gzip format, use wbits = 31
//|
//| :param bytes data: data to be decompressed
//| :param int wbits: DEFLATE dictionary window size used during compression. See above.
//| :param int bufsize: ignored for compatibility with CPython only
//| """
//| ...
//|
STATIC mp_obj_t zlib_decompress(size_t n_args, const mp_obj_t *args) {
bool is_zlib = true;
if (n_args > 1 && MP_OBJ_SMALL_INT_VALUE(args[1]) < 0) {
is_zlib = false;
}
return common_hal_zlib_decompress(args[0], is_zlib);
}
STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(zlib_decompress_obj, 1, 3, zlib_decompress);
STATIC const mp_rom_map_elem_t zlib_globals_table[] = {
{ MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_zlib) },
{ MP_ROM_QSTR(MP_QSTR_decompress), MP_ROM_PTR(&zlib_decompress_obj) },
};
STATIC MP_DEFINE_CONST_DICT(zlib_globals, zlib_globals_table);
const mp_obj_module_t zlib_module = {
.base = { &mp_type_module },
.globals = (mp_obj_dict_t *)&zlib_globals,
};
MP_REGISTER_MODULE(MP_QSTR_zlib, zlib_module, CIRCUITPY_ZLIB);

32
shared-bindings/zlib/__init__.h Normal file
View File

@ -0,0 +1,32 @@
/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2022 Mark Komus
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef MICROPY_INCLUDED_SHARED_BINDINGS_ZLIB___INIT___H
#define MICROPY_INCLUDED_SHARED_BINDINGS_ZLIB___INIT___H
mp_obj_t common_hal_zlib_decompress(mp_obj_t data, bool is_zlib);
#endif // MICROPY_INCLUDED_SHARED_BINDINGS_ZLIB___INIT___H

100
shared-module/zlib/__init__.c Normal file
View File

@ -0,0 +1,100 @@
/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2022 Mark Komus
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <stdint.h>
#include <assert.h>
#include <string.h>
#include "py/obj.h"
#include "py/runtime.h"
#include "py/stream.h"
#include "py/mperrno.h"
#include "py/builtin.h"
#include "py/objtuple.h"
#include "py/binary.h"
#include "py/parsenum.h"
#include "shared-bindings/zlib/__init__.h"
#define UZLIB_CONF_PARANOID_CHECKS (1)
#include "lib/uzlib/tinf.h"
#if 0 // print debugging info
#define DEBUG_printf DEBUG_printf
#else // don't print debugging info
#define DEBUG_printf(...) (void)0
#endif
mp_obj_t common_hal_zlib_decompress(mp_obj_t data, bool is_zlib) {
mp_buffer_info_t bufinfo;
mp_get_buffer_raise(data, &bufinfo, MP_BUFFER_READ);
TINF_DATA *decomp = m_new_obj(TINF_DATA);
memset(decomp, 0, sizeof(*decomp));
DEBUG_printf("sizeof(TINF_DATA)=" UINT_FMT "\n", sizeof(*decomp));
uzlib_uncompress_init(decomp, NULL, 0);
mp_uint_t dest_buf_size = (bufinfo.len + 15) & ~15;
byte *dest_buf = m_new(byte, dest_buf_size);
decomp->dest = dest_buf;
decomp->dest_limit = dest_buf + dest_buf_size;
DEBUG_printf("zlib: Initial out buffer: " UINT_FMT " bytes\n", decomp->destSize);
decomp->source = bufinfo.buf;
decomp->source_limit = (unsigned char *)bufinfo.buf + bufinfo.len;
int st;
if (is_zlib) {
st = uzlib_zlib_parse_header(decomp);
if (st < 0) {
goto error;
}
}
while (1) {
st = uzlib_uncompress_chksum(decomp);
if (st < 0) {
goto error;
}
if (st == TINF_DONE) {
break;
}
size_t offset = decomp->dest - dest_buf;
dest_buf = m_renew(byte, dest_buf, dest_buf_size, dest_buf_size + 256);
dest_buf_size += 256;
decomp->dest = dest_buf + offset;
decomp->dest_limit = dest_buf + offset + 256;
}
mp_uint_t final_sz = decomp->dest - dest_buf;
DEBUG_printf("zlib: Resizing from " UINT_FMT " to final size: " UINT_FMT " bytes\n", dest_buf_size, final_sz);
dest_buf = (byte *)m_renew(byte, dest_buf, dest_buf_size, final_sz);
mp_obj_t res = mp_obj_new_bytearray_by_ref(final_sz, dest_buf);
m_del_obj(TINF_DATA, decomp);
return res;
error:
mp_raise_type_arg(&mp_type_ValueError, MP_OBJ_NEW_SMALL_INT(st));
}

60
tests/circuitpython/zlib_decompress.py Normal file
View File

@ -0,0 +1,60 @@
try:
import zlib
except ImportError:
try:
import zlib as zlib
except ImportError:
print("SKIP")
raise SystemExit
PATTERNS = [
# Packed results produced by CPy's zlib.compress()
(b"0", b"x\x9c3\x00\x00\x001\x001"),
(b"a", b"x\x9cK\x04\x00\x00b\x00b"),
(b"0" * 100, b"x\x9c30\xa0=\x00\x00\xb3q\x12\xc1"),
(
bytes(range(64)),
b"x\x9cc`dbfaec\xe7\xe0\xe4\xe2\xe6\xe1\xe5\xe3\x17\x10\x14\x12\x16\x11\x15\x13\x97\x90\x94\x92\x96\x91\x95\x93WPTRVQUS\xd7\xd0\xd4\xd2\xd6\xd1\xd5\xd370426153\xb7\xb0\xb4\xb2\xb6\xb1\xb5\xb3\x07\x00\xaa\xe0\x07\xe1",
),
(b"hello", b"x\x01\x01\x05\x00\xfa\xffhello\x06,\x02\x15"), # compression level 0
# adaptive/dynamic huffman tree
(
b"13371813150|13764518736|12345678901",
b"x\x9c\x05\xc1\x81\x01\x000\x04\x04\xb1\x95\\\x1f\xcfn\x86o\x82d\x06Qq\xc8\x9d\xc5X}<e\xb5g\x83\x0f\x89X\x07\xab",
),
# dynamic Huffman tree with "case 17" (repeat code for 3-10 times)
(
b">I}\x00\x951D>I}\x00\x951D>I}\x00\x951D>I}\x00\x951D",
b"x\x9c\x05\xc11\x01\x00\x00\x00\x010\x95\x14py\x84\x12C_\x9bR\x8cV\x8a\xd1J1Z)F\x1fw`\x089",
),
]
for unpacked, packed in PATTERNS:
assert zlib.decompress(packed) == unpacked
print(unpacked)
# Raw DEFLATE bitstream
v = b"\xcbH\xcd\xc9\xc9\x07\x00"
exp = b"hello"
out = zlib.decompress(v, -15)
assert out == exp
print(exp)
# Even when you ask CPython zlib.compress to produce Raw DEFLATE stream,
# it returns it with adler2 and oriignal size appended, as if it was a
# zlib stream. Make sure there're no random issues decompressing such.
v = b"\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x05\x00\x00\x00"
out = zlib.decompress(v, -15)
assert out == exp
# this should error
try:
zlib.decompress(b"abc")
except Exception:
print("Exception")
# invalid block type
try:
zlib.decompress(b"\x07", -15) # final-block, block-type=3 (invalid)
except Exception as er:
print("Exception")

2
tests/unix/extra_coverage.py.exp
View File

@ -43,7 +43,7 @@ ulab.scipy.linalg ulab.scipy.optimize
ulab.scipy.signal ulab.scipy.special
ulab.utils uos urandom ure
uselect ustruct utime utimeq
uzlib
uzlib zlib
ime
utime utimeq