djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

extmod/uzlib/: Update uzlib to v2.0.

Browse Source 
New API supporting stream decompression.
This commit is contained in:
Paul Sokolovsky 2016-08-17 06:00:28 +03:00
parent bb19e7b94b
commit ff1c2b03a9
5 changed files with 326 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
extmod/uzlib/adler32.c
View File

@ -41,12 +41,12 @@
#define A32_BASE 65521
#define A32_NMAX 5552
unsigned int tinf_adler32(const void *data, unsigned int length)
unsigned int uzlib_adler32(const void *data, unsigned int length, unsigned int prev_sum /* 1 */)
{
const unsigned char *buf = (const unsigned char *)data;
unsigned int s1 = 1;
unsigned int s2 = 0;
unsigned int s1 = prev_sum & 0xffff;
unsigned int s2 = prev_sum >> 16;
while (length > 0)
{

63
extmod/uzlib/crc32.c Normal file
View File

@ -0,0 +1,63 @@
/*
* CRC32 checksum
*
* Copyright (c) 1998-2003 by Joergen Ibsen / Jibz
* All Rights Reserved
*
* http://www.ibsensoftware.com/
*
* This software is provided 'as-is', without any express
* or implied warranty. In no event will the authors be
* held liable for any damages arising from the use of
* this software.
*
* Permission is granted to anyone to use this software
* for any purpose, including commercial applications,
* and to alter it and redistribute it freely, subject to
* the following restrictions:
*
* 1. The origin of this software must not be
* misrepresented; you must not claim that you
* wrote the original software. If you use this
* software in a product, an acknowledgment in
* the product documentation would be appreciated
* but is not required.
*
* 2. Altered source versions must be plainly marked
* as such, and must not be misrepresented as
* being the original software.
*
* 3. This notice may not be removed or altered from
* any source distribution.
*/
/*
* CRC32 algorithm taken from the zlib source, which is
* Copyright (C) 1995-1998 Jean-loup Gailly and Mark Adler
*/
#include "tinf.h"
static const unsigned int tinf_crc32tab[16] = {
0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190,
0x6b6b51f4, 0x4db26158, 0x5005713c, 0xedb88320, 0xf00f9344,
0xd6d6a3e8, 0xcb61b38c, 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278,
0xbdbdf21c
};
/* crc is previous value for incremental computation, 0xffffffff initially */
unsigned int uzlib_crc32(const void *data, unsigned int length, unsigned int crc)
{
const unsigned char *buf = (const unsigned char *)data;
unsigned int i;
for (i = 0; i < length; ++i)
{
crc ^= buf[i];
crc = tinf_crc32tab[crc & 0x0f] ^ (crc >> 4);
crc = tinf_crc32tab[crc & 0x0f] ^ (crc >> 4);
}
// return value suitable for passing in next time, for final value invert it
return crc/* ^ 0xffffffff*/;
}

78
extmod/uzlib/tinf.h
View File

@ -5,7 +5,7 @@
* All Rights Reserved
* http://www.ibsensoftware.com/
*
* Copyright (c) 2014 by Paul Sokolovsky
* Copyright (c) 2014-2016 by Paul Sokolovsky
*/
#ifndef TINF_H_INCLUDED
@ -26,9 +26,17 @@
extern "C" {
#endif
/* ok status, more data produced */
#define TINF_OK 0
/* end of compressed stream reached */
#define TINF_DONE 1
#define TINF_DATA_ERROR (-3)
#define TINF_DEST_OVERFLOW (-4)
#define TINF_CHKSUM_ERROR (-4)
/* checksum types */
#define TINF_CHKSUM_NONE 0
#define TINF_CHKSUM_ADLER 1
#define TINF_CHKSUM_CRC 2
/* data structures */
@ -40,6 +48,10 @@ typedef struct {
struct TINF_DATA;
typedef struct TINF_DATA {
const unsigned char *source;
/* If source above is NULL, this function will be used to read
next byte from source stream */
unsigned char (*readSource)(struct TINF_DATA *data);
unsigned int tag;
unsigned int bitcount;
@ -51,49 +63,51 @@ typedef struct TINF_DATA {
unsigned char *dest;
/* Remaining bytes in buffer */
unsigned int destRemaining;
/* Argument is the allocation size which didn't fit into buffer. Note that
exact mimumum size to grow buffer by is lastAlloc - destRemaining. But
growing by this exact size is ineficient, as the next allocation will
fail again. */
int (*destGrow)(struct TINF_DATA *data, unsigned int lastAlloc);
/* Accumulating checksum */
unsigned int checksum;
char checksum_type;
int btype;
int bfinal;
unsigned int curlen;
int lzOff;
unsigned char *dict_ring;
unsigned int dict_size;
unsigned int dict_idx;
TINF_TREE ltree; /* dynamic length/symbol tree */
TINF_TREE dtree; /* dynamic distance tree */
} TINF_DATA;
#define TINF_PUT(d, c) \
{ \
*d->dest++ = c; \
if (d->dict_ring) { d->dict_ring[d->dict_idx++] = c; if (d->dict_idx == d->dict_size) d->dict_idx = 0; } \
}
/* low-level API */
unsigned char TINFCC uzlib_get_byte(TINF_DATA *d);
/* Step 1: Allocate TINF_DATA structure */
/* Step 2: Set destStart, destSize, and destGrow fields */
/* Step 3: Set source field */
/* Step 4: Call tinf_uncompress_dyn() */
/* Step 5: In response to destGrow callback, update destStart and destSize fields */
/* Step 6: When tinf_uncompress_dyn() returns, buf.dest points to a byte past last uncompressed byte */
/* Decompression API */
int TINFCC tinf_uncompress_dyn(TINF_DATA *d);
int TINFCC tinf_zlib_uncompress_dyn(TINF_DATA *d, unsigned int sourceLen);
void TINFCC uzlib_init(void);
void TINFCC uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen);
int TINFCC uzlib_uncompress(TINF_DATA *d);
int TINFCC uzlib_uncompress_chksum(TINF_DATA *d);
/* high-level API */
int TINFCC uzlib_zlib_parse_header(TINF_DATA *d);
int TINFCC uzlib_gzip_parse_header(TINF_DATA *d);
void TINFCC tinf_init(void);
/* Compression API */
int TINFCC tinf_uncompress(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen);
void TINFCC uzlib_compress(void *data, const uint8_t *src, unsigned slen);
int TINFCC tinf_gzip_uncompress(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen);
/* Checksum API */
int TINFCC tinf_zlib_uncompress(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen);
unsigned int TINFCC tinf_adler32(const void *data, unsigned int length);
unsigned int TINFCC tinf_crc32(const void *data, unsigned int length);
/* compression API */
void TINFCC tinf_compress(void *data, const uint8_t *src, unsigned slen);
/* prev_sum is previous value for incremental computation, 1 initially */
uint32_t TINFCC uzlib_adler32(const void *data, unsigned int length, uint32_t prev_sum);
/* crc is previous value for incremental computation, 0xffffffff initially */
uint32_t TINFCC uzlib_crc32(const void *data, unsigned int length, uint32_t crc);
#ifdef __cplusplus
} /* extern "C" */

381
extmod/uzlib/tinflate.c
View File

@ -5,7 +5,7 @@
* All Rights Reserved
* http://www.ibsensoftware.com/
*
* Copyright (c) 2014 by Paul Sokolovsky
* Copyright (c) 2014-2016 by Paul Sokolovsky
*
* This software is provided 'as-is', without any express
* or implied warranty. In no event will the authors be
@ -32,6 +32,7 @@
* any source distribution.
*/
#include <assert.h>
#include "tinf.h"
/* --------------------------------------------------- *
@ -89,21 +90,6 @@ const unsigned char clcidx[] = {
* -- utility functions -- *
* ----------------------- */
/* Execute callback to grow destination buffer */
static int tinf_grow_dest_buf(TINF_DATA *d, unsigned int lastAlloc)
{
unsigned int oldsize = d->dest - d->destStart;
/* This will update only destStart and destSize */
if (!d->destGrow)
{
return TINF_DEST_OVERFLOW;
}
d->destGrow(d, lastAlloc);
d->dest = d->destStart + oldsize;
d->destRemaining = d->destSize - oldsize;
return 0;
}
#ifdef RUNTIME_BITS_TABLES
/* build extra bits and base tables */
static void tinf_build_bits_base(unsigned char *bits, unsigned short *base, int delta, int first)
@ -180,6 +166,34 @@ static void tinf_build_tree(TINF_TREE *t, const unsigned char *lengths, unsigned
* -- decode functions -- *
* ---------------------- */
unsigned char uzlib_get_byte(TINF_DATA *d)
{
if (d->source) {
return *d->source++;
}
return d->readSource(d);
}
uint32_t tinf_get_le_uint32(TINF_DATA *d)
{
uint32_t val = 0;
int i;
for (i = 4; i--;) {
val = val >> 8 | uzlib_get_byte(d) << 24;
}
return val;
}
uint32_t tinf_get_be_uint32(TINF_DATA *d)
{
uint32_t val = 0;
int i;
for (i = 4; i--;) {
val = val << 8 | uzlib_get_byte(d);
}
return val;
}
/* get one bit from source stream */
static int tinf_getbit(TINF_DATA *d)
{
@ -189,7 +203,7 @@ static int tinf_getbit(TINF_DATA *d)
if (!d->bitcount--)
{
/* load next tag */
d->tag = *d->source++;
d->tag = uzlib_get_byte(d);
d->bitcount = 7;
}
@ -318,113 +332,83 @@ static void tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt)
/* given a stream and two trees, inflate a block of data */
static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt)
{
while (1)
{
int sym = tinf_decode_symbol(d, lt);
if (d->curlen == 0) {
unsigned int offs;
int dist;
int sym = tinf_decode_symbol(d, lt);
//printf("huff sym: %02x\n", sym);
/* check for end of block */
if (sym == 256)
{
return TINF_OK;
}
/* literal byte */
if (sym < 256) {
TINF_PUT(d, sym);
return TINF_OK;
}
if (sym < 256)
{
if (d->destRemaining == 0)
{
int res = tinf_grow_dest_buf(d, 1);
if (res) return res;
}
/* end of block */
if (sym == 256) {
return TINF_DONE;
}
*d->dest++ = sym;
d->destRemaining--;
/* substring from sliding dictionary */
sym -= 257;
/* possibly get more bits from length code */
d->curlen = tinf_read_bits(d, length_bits[sym], length_base[sym]);
} else {
dist = tinf_decode_symbol(d, dt);
/* possibly get more bits from distance code */
offs = tinf_read_bits(d, dist_bits[dist], dist_base[dist]);
if (d->dict_ring) {
d->lzOff = d->dict_idx - offs;
if (d->lzOff < 0) {
d->lzOff += d->dict_size;
}
} else {
d->lzOff = -offs;
}
}
unsigned int length, offs, i;
int dist;
sym -= 257;
/* possibly get more bits from length code */
length = tinf_read_bits(d, length_bits[sym], length_base[sym]);
dist = tinf_decode_symbol(d, dt);
/* possibly get more bits from distance code */
offs = tinf_read_bits(d, dist_bits[dist], dist_base[dist]);
if (d->destRemaining < length)
{
int res = tinf_grow_dest_buf(d, length);
if (res) return res;
}
/* copy match */
for (i = 0; i < length; ++i)
{
d->dest[i] = d->dest[(int)(i - offs)];
}
d->dest += length;
d->destRemaining -= length;
}
}
/* copy next byte from dict substring */
if (d->dict_ring) {
TINF_PUT(d, d->dict_ring[d->lzOff]);
if (++d->lzOff == d->dict_size) {
d->lzOff = 0;
}
} else {
d->dest[0] = d->dest[d->lzOff];
d->dest++;
}
d->curlen--;
return TINF_OK;
}
/* inflate an uncompressed block of data */
static int tinf_inflate_uncompressed_block(TINF_DATA *d)
{
unsigned int length, invlength;
unsigned int i;
if (d->curlen == 0) {
unsigned int length, invlength;
/* get length */
length = d->source[1];
length = 256*length + d->source[0];
/* get length */
length = uzlib_get_byte(d) + 256 * uzlib_get_byte(d);
/* get one's complement of length */
invlength = uzlib_get_byte(d) + 256 * uzlib_get_byte(d);
/* check length */
if (length != (~invlength & 0x0000ffff)) return TINF_DATA_ERROR;
/* get one's complement of length */
invlength = d->source[3];
invlength = 256*invlength + d->source[2];
/* increment length to properly return TINF_DONE below, without
producing data at the same time */
d->curlen = length + 1;
/* check length */
if (length != (~invlength & 0x0000ffff)) return TINF_DATA_ERROR;
/* make sure we start next block on a byte boundary */
d->bitcount = 0;
}
if (d->destRemaining < length)
{
int res = tinf_grow_dest_buf(d, length);
if (res) return res;
}
if (--d->curlen == 0) {
return TINF_DONE;
}
d->source += 4;
/* copy block */
for (i = length; i; --i) *d->dest++ = *d->source++;
d->destRemaining -= length;
/* make sure we start next block on a byte boundary */
d->bitcount = 0;
return TINF_OK;
}
/* inflate a block of data compressed with fixed huffman trees */
static int tinf_inflate_fixed_block(TINF_DATA *d)
{
/* build fixed huffman trees */
tinf_build_fixed_trees(&d->ltree, &d->dtree);
/* decode block using fixed trees */
return tinf_inflate_block_data(d, &d->ltree, &d->dtree);
}
/* inflate a block of data compressed with dynamic huffman trees */
static int tinf_inflate_dynamic_block(TINF_DATA *d)
{
/* decode trees from stream */
tinf_decode_trees(d, &d->ltree, &d->dtree);
/* decode block using decoded trees */
return tinf_inflate_block_data(d, &d->ltree, &d->dtree);
unsigned char c = uzlib_get_byte(d);
TINF_PUT(d, c);
return TINF_OK;
}
/* ---------------------- *
@ -432,7 +416,7 @@ static int tinf_inflate_dynamic_block(TINF_DATA *d)
* ---------------------- */
/* initialize global (static) data */
void tinf_init(void)
void uzlib_init(void)
{
#ifdef RUNTIME_BITS_TABLES
/* build extra bits and base tables */
@ -445,72 +429,117 @@ void tinf_init(void)
#endif
}
/* inflate stream from source to dest */
int tinf_uncompress(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen)
/* initialize decompression structure */
void uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen)
{
(void)sourceLen;
TINF_DATA d;
int res;
/* initialise data */
d.source = (const unsigned char *)source;
d.destStart = (unsigned char *)dest;
d.destRemaining = *destLen;
d.destSize = *destLen;
res = tinf_uncompress_dyn(&d);
*destLen = d.dest - d.destStart;
return res;
}
/* inflate stream from source to dest */
int tinf_uncompress_dyn(TINF_DATA *d)
{
int bfinal;
/* initialise data */
d->bitcount = 0;
d->dest = d->destStart;
d->destRemaining = d->destSize;
do {
unsigned int btype;
int res;
/* read final block flag */
bfinal = tinf_getbit(d);
/* read block type (2 bits) */
btype = tinf_read_bits(d, 2, 0);
/* decompress block */
switch (btype)
{
case 0:
/* decompress uncompressed block */
res = tinf_inflate_uncompressed_block(d);
break;
case 1:
/* decompress block with fixed huffman trees */
res = tinf_inflate_fixed_block(d);
break;
case 2:
/* decompress block with dynamic huffman trees */
res = tinf_inflate_dynamic_block(d);
break;
default:
return TINF_DATA_ERROR;
}
if (res != TINF_OK) return TINF_DATA_ERROR;
} while (!bfinal);
return TINF_OK;
d->bfinal = 0;
d->btype = -1;
d->dict_size = dictLen;
d->dict_ring = dict;
d->dict_idx = 0;
d->curlen = 0;
}
/* inflate next byte of compressed stream */
int uzlib_uncompress(TINF_DATA *d)
{
do {
int res;
/* start a new block */
if (d->btype == -1) {
next_blk:
/* read final block flag */
d->bfinal = tinf_getbit(d);
/* read block type (2 bits) */
d->btype = tinf_read_bits(d, 2, 0);
//printf("Started new block: type=%d final=%d\n", d->btype, d->bfinal);
if (d->btype == 1) {
/* build fixed huffman trees */
tinf_build_fixed_trees(&d->ltree, &d->dtree);
} else if (d->btype == 2) {
/* decode trees from stream */
tinf_decode_trees(d, &d->ltree, &d->dtree);
}
}
/* process current block */
switch (d->btype)
{
case 0:
/* decompress uncompressed block */
res = tinf_inflate_uncompressed_block(d);
break;
case 1:
case 2:
/* decompress block with fixed/dyanamic huffman trees */
/* trees were decoded previously, so it's the same routine for both */
res = tinf_inflate_block_data(d, &d->ltree, &d->dtree);
break;
default:
return TINF_DATA_ERROR;
}
if (res == TINF_DONE && !d->bfinal) {
/* the block has ended (without producing more data), but we
can't return without data, so start procesing next block */
goto next_blk;
}
if (res != TINF_OK) {
return res;
}
} while (--d->destSize);
return TINF_OK;
}
int uzlib_uncompress_chksum(TINF_DATA *d)
{
int res;
unsigned char *data = d->dest;
res = uzlib_uncompress(d);
if (res < 0) return res;
switch (d->checksum_type) {
case TINF_CHKSUM_ADLER:
d->checksum = uzlib_adler32(data, d->dest - data, d->checksum);
break;
case TINF_CHKSUM_CRC:
d->checksum = uzlib_crc32(data, d->dest - data, d->checksum);
break;
}
if (res == TINF_DONE) {
unsigned int val;
switch (d->checksum_type) {
case TINF_CHKSUM_ADLER:
val = tinf_get_be_uint32(d);
if (d->checksum != val) {
return TINF_CHKSUM_ERROR;
}
break;
case TINF_CHKSUM_CRC:
val = tinf_get_le_uint32(d);
if (~d->checksum != val) {
return TINF_CHKSUM_ERROR;
}
// Uncompressed size. TODO: Check
val = tinf_get_le_uint32(d);
break;
}
}
return res;
}

53
extmod/uzlib/tinfzlib.c
View File

@ -6,6 +6,8 @@
*
* http://www.ibsensoftware.com/
*
* Copyright (c) 2014-2016 by Paul Sokolovsky
*
* This software is provided 'as-is', without any express
* or implied warranty. In no event will the authors be
* held liable for any damages arising from the use of
@ -33,35 +35,14 @@
#include "tinf.h"
int tinf_zlib_uncompress(void *dest, unsigned int *destLen,
const void *source, unsigned int sourceLen)
int uzlib_zlib_parse_header(TINF_DATA *d)
{
TINF_DATA d;
int res;
/* initialise data */
d.source = (const unsigned char *)source;
d.destStart = (unsigned char *)dest;
d.destRemaining = *destLen;
res = tinf_zlib_uncompress_dyn(&d, sourceLen);
*destLen = d.dest - d.destStart;
return res;
}
int tinf_zlib_uncompress_dyn(TINF_DATA *d, unsigned int sourceLen)
{
unsigned int a32;
int res;
unsigned char cmf, flg;
/* -- get header bytes -- */
cmf = d->source[0];
flg = d->source[1];
cmf = uzlib_get_byte(d);
flg = uzlib_get_byte(d);
/* -- check format -- */
@ -77,25 +58,9 @@ int tinf_zlib_uncompress_dyn(TINF_DATA *d, unsigned int sourceLen)
/* check there is no preset dictionary */
if (flg & 0x20) return TINF_DATA_ERROR;
/* -- get adler32 checksum -- */
/* initialize for adler32 checksum */
d->checksum_type = TINF_CHKSUM_ADLER;
d->checksum = 1;
a32 = d->source[sourceLen - 4];
a32 = 256*a32 + d->source[sourceLen - 3];
a32 = 256*a32 + d->source[sourceLen - 2];
a32 = 256*a32 + d->source[sourceLen - 1];
d->source += 2;
/* -- inflate -- */
res = tinf_uncompress_dyn(d);
if (res != TINF_OK) return res;
/* -- check adler32 checksum -- */
if (a32 != tinf_adler32(d->destStart, d->dest - d->destStart)) return TINF_DATA_ERROR;
return TINF_OK;
return cmf >> 4;
}