djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

extmod/moddeflate: Add deflate module providing the DeflateIO class.

Browse Source 
This provides similar functionality to the former zlib.DecompIO and
especially CPython's gzip.GzipFile for both compression and decompression.

This class can be used directly, and also can be used from Python to
implement (via io.BytesIO) zlib.decompress and zlib.compress, as well as
gzip.GzipFile.

Enable/disable this on all ports/boards that zlib was previously configured
for.

This work was funded through GitHub Sponsors.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
This commit is contained in:
Jim Mussared 2023-06-26 13:52:10 +10:00 committed by Damien George
parent e6c290c3d1
commit 3533924c36
12 changed files with 434 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
extmod/extmod.cmake
View File

@ -23,6 +23,7 @@ set(MICROPY_SOURCE_EXTMOD
${MICROPY_EXTMOD_DIR}/modbinascii.c
${MICROPY_EXTMOD_DIR}/modcryptolib.c
${MICROPY_EXTMOD_DIR}/moductypes.c
${MICROPY_EXTMOD_DIR}/moddeflate.c
${MICROPY_EXTMOD_DIR}/modhashlib.c
${MICROPY_EXTMOD_DIR}/modheapq.c
${MICROPY_EXTMOD_DIR}/modjson.c

1
extmod/extmod.mk
View File

@ -16,6 +16,7 @@ SRC_EXTMOD_C += \
extmod/modbluetooth.c \
extmod/modbtree.c \
extmod/modcryptolib.c \
extmod/moddeflate.c \
extmod/modframebuf.c \
extmod/modhashlib.c \
extmod/modheapq.c \

6
extmod/modbinascii.c
View File

@ -170,8 +170,8 @@ STATIC mp_obj_t mod_binascii_b2a_base64(size_t n_args, const mp_obj_t *pos_args,
}
STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_binascii_b2a_base64_obj, 1, mod_binascii_b2a_base64);
#if 0 // MICROPY_PY_BINASCII_CRC32
#include "lib/uzlib/crc32.c"
#if MICROPY_PY_BINASCII_CRC32 && MICROPY_PY_DEFLATE
#include "lib/uzlib/uzlib.h"
STATIC mp_obj_t mod_binascii_crc32(size_t n_args, const mp_obj_t *args) {
mp_buffer_info_t bufinfo;
@ -191,7 +191,7 @@ STATIC const mp_rom_map_elem_t mp_module_binascii_globals_table[] = {
#endif
{ MP_ROM_QSTR(MP_QSTR_a2b_base64), MP_ROM_PTR(&mod_binascii_a2b_base64_obj) },
{ MP_ROM_QSTR(MP_QSTR_b2a_base64), MP_ROM_PTR(&mod_binascii_b2a_base64_obj) },
#if 0 // MICROPY_PY_BINASCII_CRC32
#if MICROPY_PY_BINASCII_CRC32 && MICROPY_PY_DEFLATE
{ MP_ROM_QSTR(MP_QSTR_crc32), MP_ROM_PTR(&mod_binascii_crc32_obj) },
#endif
};

404
extmod/moddeflate.c Normal file
View File

@ -0,0 +1,404 @@
/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2023 Jim Mussared
*
* Based on extmod/modzlib.c
* Copyright (c) 2014-2016 Paul Sokolovsky
* Copyright (c) 2021-2023 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <stdio.h>
#include <string.h>
#include "py/runtime.h"
#include "py/stream.h"
#include "py/mperrno.h"
#if MICROPY_PY_DEFLATE
#include "lib/uzlib/uzlib.h"
#if 0 // print debugging info
#define DEBUG_printf DEBUG_printf
#else // don't print debugging info
#define DEBUG_printf(...) (void)0
#endif
typedef enum {
DEFLATEIO_FORMAT_MIN = 0,
DEFLATEIO_FORMAT_AUTO = DEFLATEIO_FORMAT_MIN, // Read mode this means auto-detect zlib/gzip, write mode this means RAW.
DEFLATEIO_FORMAT_RAW = 1,
DEFLATEIO_FORMAT_ZLIB = 2,
DEFLATEIO_FORMAT_GZIP = 3,
DEFLATEIO_FORMAT_MAX = DEFLATEIO_FORMAT_GZIP,
} deflateio_format_t;
const int DEFLATEIO_DEFAULT_WBITS = 8;
typedef struct {
void *window;
uzlib_uncomp_t decomp;
bool eof;
} mp_obj_deflateio_read_t;
#if MICROPY_PY_DEFLATE_COMPRESS
typedef struct {
void *window;
size_t input_len;
uint32_t input_checksum;
uzlib_lz77_state_t lz77;
} mp_obj_deflateio_write_t;
#endif
typedef struct {
mp_obj_base_t base;
mp_obj_t stream;
uint8_t format : 2;
uint8_t window_bits : 4;
bool close : 1;
mp_obj_deflateio_read_t *read;
#if MICROPY_PY_DEFLATE_COMPRESS
mp_obj_deflateio_write_t *write;
#endif
} mp_obj_deflateio_t;
STATIC int deflateio_read_stream(void *data) {
mp_obj_deflateio_t *self = data;
const mp_stream_p_t *stream = mp_get_stream(self->stream);
int err;
byte c;
mp_uint_t out_sz = stream->read(self->stream, &c, 1, &err);
if (out_sz == MP_STREAM_ERROR) {
mp_raise_OSError(err);
}
if (out_sz == 0) {
mp_raise_type(&mp_type_EOFError);
}
return c;
}
STATIC bool deflateio_init_read(mp_obj_deflateio_t *self) {
if (self->read) {
return true;
}
mp_get_stream_raise(self->stream, MP_STREAM_OP_READ);
self->read = m_new_obj(mp_obj_deflateio_read_t);
memset(&self->read->decomp, 0, sizeof(self->read->decomp));
self->read->decomp.source_read_data = self;
self->read->decomp.source_read_cb = deflateio_read_stream;
self->read->eof = false;
// Don't modify self->window_bits as it may also be used for write.
int wbits = self->window_bits;
// Parse the header if we're in NONE/ZLIB/GZIP modes.
if (self->format != DEFLATEIO_FORMAT_RAW) {
int header_wbits = wbits;
int header_type = uzlib_parse_zlib_gzip_header(&self->read->decomp, &header_wbits);
if ((self->format == DEFLATEIO_FORMAT_ZLIB && header_type != UZLIB_HEADER_ZLIB) || (self->format == DEFLATEIO_FORMAT_GZIP && header_type != UZLIB_HEADER_GZIP)) {
return false;
}
if (wbits == 0 && header_wbits < 15) {
// If the header specified something lower than the default, then
// use that instead.
wbits = header_wbits;
}
}
if (wbits == 0) {
wbits = DEFLATEIO_DEFAULT_WBITS;
}
size_t window_len = 1 << wbits;
self->read->window = m_new(uint8_t, window_len);
uzlib_uncompress_init(&self->read->decomp, self->read->window, window_len);
return true;
}
#if MICROPY_PY_DEFLATE_COMPRESS
STATIC void deflateio_out_byte(void *data, uint8_t b) {
mp_obj_deflateio_t *self = data;
const mp_stream_p_t *stream = mp_get_stream(self->stream);
int err;
mp_uint_t ret = stream->write(self->stream, &b, 1, &err);
if (ret == MP_STREAM_ERROR) {
mp_raise_OSError(err);
}
}
STATIC bool deflateio_init_write(mp_obj_deflateio_t *self) {
if (self->write) {
return true;
}
const mp_stream_p_t *stream = mp_get_stream_raise(self->stream, MP_STREAM_OP_WRITE);
self->write = m_new_obj(mp_obj_deflateio_write_t);
self->write->input_len = 0;
int wbits = self->window_bits;
if (wbits == 0) {
wbits = DEFLATEIO_DEFAULT_WBITS;
}
size_t window_len = 1 << wbits;
self->write->window = m_new(uint8_t, window_len);
uzlib_lz77_init(&self->write->lz77, self->write->window, window_len);
self->write->lz77.dest_write_data = self;
self->write->lz77.dest_write_cb = deflateio_out_byte;
// Write header if needed.
mp_uint_t ret = 0;
int err;
if (self->format == DEFLATEIO_FORMAT_ZLIB) {
// -----CMF------ ----------FLG---------------
// CINFO(5) CM(3) FLEVEL(2) FDICT(1) FCHECK(5)
uint8_t buf[] = { 0x08, 0x80 }; // CM=2 (deflate), FLEVEL=2 (default), FDICT=0 (no dictionary)
buf[0] |= MAX(wbits - 8, 1) << 4; // base-2 logarithm of the LZ77 window size, minus eight.
buf[1] |= 31 - ((buf[0] * 256 + buf[1]) % 31); // (CMF*256 + FLG) % 31 == 0.
ret = stream->write(self->stream, buf, sizeof(buf), &err);
self->write->input_checksum = 1; // ADLER32
} else if (self->format == DEFLATEIO_FORMAT_GZIP) {
// ID1(8) ID2(8) CM(8) ---FLG--- MTIME(32) XFL(8) OS(8)
// FLG: x x x FCOMMENT FNAME FEXTRA FHCRC FTEXT
uint8_t buf[] = { 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03 }; // MTIME=0, XFL=4 (fastest), OS=3 (unix)
ret = stream->write(self->stream, buf, sizeof(buf), &err);
self->write->input_checksum = ~0; // CRC32
}
if (ret == MP_STREAM_ERROR) {
return false;
}
// Write starting block.
uzlib_start_block(&self->write->lz77);
return true;
}
#endif
STATIC mp_obj_t deflateio_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args_in) {
// args: stream, format=NONE, wbits=0, close=False
mp_arg_check_num(n_args, n_kw, 1, 4, false);
mp_int_t format = n_args > 1 ? mp_obj_get_int(args_in[1]) : DEFLATEIO_FORMAT_AUTO;
mp_int_t wbits = n_args > 2 ? mp_obj_get_int(args_in[2]) : 0;
if (format < DEFLATEIO_FORMAT_MIN || format > DEFLATEIO_FORMAT_MAX) {
mp_raise_ValueError(MP_ERROR_TEXT("format"));
}
if (wbits != 0 && (wbits < 5 || wbits > 15)) {
mp_raise_ValueError(MP_ERROR_TEXT("wbits"));
}
mp_obj_deflateio_t *self = mp_obj_malloc(mp_obj_deflateio_t, type);
self->stream = args_in[0];
self->format = format;
self->window_bits = wbits;
self->read = NULL;
#if MICROPY_PY_DEFLATE_COMPRESS
self->write = NULL;
#endif
self->close = n_args > 3 ? mp_obj_is_true(args_in[3]) : false;
return MP_OBJ_FROM_PTR(self);
}
STATIC mp_uint_t deflateio_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) {
mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(o_in);
if (self->stream == MP_OBJ_NULL || !deflateio_init_read(self)) {
*errcode = MP_EINVAL;
return MP_STREAM_ERROR;
}
if (self->read->eof) {
return 0;
}
self->read->decomp.dest = buf;
self->read->decomp.dest_limit = (uint8_t *)buf + size;
int st = uzlib_uncompress_chksum(&self->read->decomp);
if (st == UZLIB_DONE) {
self->read->eof = true;
}
if (st < 0) {
DEBUG_printf("uncompress error=" INT_FMT "\n", st);
*errcode = MP_EINVAL;
return MP_STREAM_ERROR;
}
return self->read->decomp.dest - (uint8_t *)buf;
}
#if MICROPY_PY_DEFLATE_COMPRESS
STATIC mp_uint_t deflateio_write(mp_obj_t self_in, const void *buf, mp_uint_t size, int *errcode) {
mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(self_in);
if (self->stream == MP_OBJ_NULL || !deflateio_init_write(self)) {
*errcode = MP_EINVAL;
return MP_STREAM_ERROR;
}
self->write->input_len += size;
if (self->format == DEFLATEIO_FORMAT_ZLIB) {
self->write->input_checksum = uzlib_adler32(buf, size, self->write->input_checksum);
} else if (self->format == DEFLATEIO_FORMAT_GZIP) {
self->write->input_checksum = uzlib_crc32(buf, size, self->write->input_checksum);
}
uzlib_lz77_compress(&self->write->lz77, buf, size);
return size;
}
static inline void put_le32(char *buf, uint32_t value) {
buf[0] = value & 0xff;
buf[1] = value >> 8 & 0xff;
buf[2] = value >> 16 & 0xff;
buf[3] = value >> 24 & 0xff;
}
static inline void put_be32(char *buf, uint32_t value) {
buf[3] = value & 0xff;
buf[2] = value >> 8 & 0xff;
buf[1] = value >> 16 & 0xff;
buf[0] = value >> 24 & 0xff;
}
#endif
STATIC mp_uint_t deflateio_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, int *errcode) {
if (request == MP_STREAM_CLOSE) {
mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(self_in);
mp_uint_t ret = 0;
if (self->stream != MP_OBJ_NULL) {
#if MICROPY_PY_DEFLATE_COMPRESS
if (self->write) {
uzlib_finish_block(&self->write->lz77);
const mp_stream_p_t *stream = mp_get_stream(self->stream);
// Write footer if needed.
if (self->format == DEFLATEIO_FORMAT_ZLIB || self->format == DEFLATEIO_FORMAT_GZIP) {
char footer[8];
size_t footer_len;
if (self->format == DEFLATEIO_FORMAT_ZLIB) {
put_be32(&footer[0], self->write->input_checksum);
footer_len = 4;
} else { // DEFLATEIO_FORMAT_GZIP
put_le32(&footer[0], ~self->write->input_checksum);
put_le32(&footer[4], self->write->input_len);
footer_len = 8;
}
if (stream->write(self->stream, footer, footer_len, errcode) == MP_STREAM_ERROR) {
ret = MP_STREAM_ERROR;
}
}
}
#endif
// Only close the stream if required. e.g. when using io.BytesIO
// it needs to stay open so that getvalue() can be called.
if (self->close) {
mp_stream_close(self->stream);
}
// Either way, free the reference to the stream.
self->stream = MP_OBJ_NULL;
}
return ret;
} else {
*errcode = MP_EINVAL;
return MP_STREAM_ERROR;
}
}
STATIC const mp_stream_p_t deflateio_stream_p = {
.read = deflateio_read,
#if MICROPY_PY_DEFLATE_COMPRESS
.write = deflateio_write,
#endif
.ioctl = deflateio_ioctl,
};
#if !MICROPY_ENABLE_DYNRUNTIME
STATIC const mp_rom_map_elem_t deflateio_locals_dict_table[] = {
{ MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
{ MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
{ MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) },
#if MICROPY_PY_DEFLATE_COMPRESS
{ MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) },
#endif
{ MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) },
{ MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) },
{ MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) },
};
STATIC MP_DEFINE_CONST_DICT(deflateio_locals_dict, deflateio_locals_dict_table);
STATIC MP_DEFINE_CONST_OBJ_TYPE(
deflateio_type,
MP_QSTR_DeflateIO,
MP_TYPE_FLAG_NONE,
make_new, deflateio_make_new,
protocol, &deflateio_stream_p,
locals_dict, &deflateio_locals_dict
);
STATIC const mp_rom_map_elem_t mp_module_deflate_globals_table[] = {
{ MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_deflate) },
{ MP_ROM_QSTR(MP_QSTR_DeflateIO), MP_ROM_PTR(&deflateio_type) },
{ MP_ROM_QSTR(MP_QSTR_AUTO), MP_ROM_INT(DEFLATEIO_FORMAT_AUTO) },
{ MP_ROM_QSTR(MP_QSTR_RAW), MP_ROM_INT(DEFLATEIO_FORMAT_RAW) },
{ MP_ROM_QSTR(MP_QSTR_ZLIB), MP_ROM_INT(DEFLATEIO_FORMAT_ZLIB) },
{ MP_ROM_QSTR(MP_QSTR_GZIP), MP_ROM_INT(DEFLATEIO_FORMAT_GZIP) },
};
STATIC MP_DEFINE_CONST_DICT(mp_module_deflate_globals, mp_module_deflate_globals_table);
const mp_obj_module_t mp_module_deflate = {
.base = { &mp_type_module },
.globals = (mp_obj_dict_t *)&mp_module_deflate_globals,
};
MP_REGISTER_MODULE(MP_QSTR_deflate, mp_module_deflate);
#endif // !MICROPY_ENABLE_DYNRUNTIME
// Source files #include'd here to make sure they're compiled in
// only if the module is enabled.
#include "lib/uzlib/tinflate.c"
#include "lib/uzlib/header.c"
#include "lib/uzlib/adler32.c"
#include "lib/uzlib/crc32.c"
#if MICROPY_PY_DEFLATE_COMPRESS
#include "lib/uzlib/lz77.c"
#endif
#endif // MICROPY_PY_DEFLATE

1
ports/cc3200/mpconfigport.h
View File

@ -110,6 +110,7 @@
#define MICROPY_PY_THREAD_GIL (1)
#define MICROPY_PY_BINASCII (1)
#define MICROPY_PY_UCTYPES (0)
#define MICROPY_PY_DEFLATE (0)
#define MICROPY_PY_JSON (1)
#define MICROPY_PY_RE (1)
#define MICROPY_PY_HEAPQ (0)

1
ports/qemu-arm/mpconfigport.h
View File

@ -40,6 +40,7 @@
#define MICROPY_PY_BINASCII (1)
#define MICROPY_PY_RANDOM (1)
#define MICROPY_PY_UCTYPES (1)
#define MICROPY_PY_DEFLATE (1)
#define MICROPY_PY_JSON (1)
#define MICROPY_PY_OS (1)
#define MICROPY_PY_RE (1)

1
ports/samd/mpconfigport.h
View File

@ -91,6 +91,7 @@
#define MICROPY_PY_UCTYPES (1)
#define MICROPY_PY_HEAPQ (1)
#define MICROPY_PY_RANDOM (1)
#define MICROPY_PY_DEFLATE (1)
#define MICROPY_PY_ASYNCIO (1)
#define MICROPY_PY_MACHINE_RTC (1)
#ifndef MICROPY_PY_MACHINE_ADC

1
ports/stm32/boards/NUCLEO_G474RE/mpconfigboard.h
View File

@ -10,6 +10,7 @@
#define MICROPY_HW_HAS_FLASH (0) // QSPI extflash not mounted
#define MICROPY_PY_ASYNCIO (0)
#define MICROPY_PY_DEFLATE (0)
#define MICROPY_PY_BINASCII (0)
#define MICROPY_PY_HASHLIB (0)
#define MICROPY_PY_JSON (0)

2
ports/windows/mpconfigport.h
View File

@ -144,6 +144,8 @@
#define MICROPY_PY_TIME_INCLUDEFILE "ports/unix/modtime.c"
#define MICROPY_PY_ERRNO (1)
#define MICROPY_PY_UCTYPES (1)
#define MICROPY_PY_DEFLATE (1)
#define MICROPY_PY_DEFLATE_COMPRESS (1)
#define MICROPY_PY_JSON (1)
#define MICROPY_PY_RE (1)
#define MICROPY_PY_HEAPQ (1)

1
ports/windows/msvc/sources.props
View File

@ -10,6 +10,7 @@
<PyExtModSource Include="$(PyBaseDir)extmod\modasyncio.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\modbinascii.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\moductypes.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\moddeflate.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\modhashlib.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\modheapq.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\modjson.c" />

12
py/mpconfig.h
View File

@ -1555,6 +1555,16 @@ typedef double mp_float_t;
#define MICROPY_PY_UCTYPES_NATIVE_C_TYPES (1)
#endif
// Whether to provide "deflate" module (decompression-only by default)
#ifndef MICROPY_PY_DEFLATE
#define MICROPY_PY_DEFLATE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)
#endif
// Whether to provide compression support in "deflate" module
#ifndef MICROPY_PY_DEFLATE_COMPRESS
#define MICROPY_PY_DEFLATE_COMPRESS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES)
#endif
#ifndef MICROPY_PY_JSON
#define MICROPY_PY_JSON (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)
#endif
@ -1629,7 +1639,7 @@ typedef double mp_float_t;
#define MICROPY_PY_BINASCII (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)
#endif
// Depends on MICROPY_PY_ZLIB
// Depends on MICROPY_PY_DEFLATE
#ifndef MICROPY_PY_BINASCII_CRC32
#define MICROPY_PY_BINASCII_CRC32 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)
#endif

13
tests/unix/extra_coverage.py.exp
View File

@ -53,12 +53,13 @@ port
builtins micropython _asyncio _thread
array binascii btree cexample
cmath collections cppexample cryptolib
errno example_package ffi
framebuf gc hashlib heapq
io json machine math
os random re select
socket ssl struct sys
termios time uctypes websocket
deflate errno example_package
ffi framebuf gc hashlib
heapq io json machine
math os random re
select socket ssl struct
sys termios time uctypes
websocket
me
micropython machine math