py/unicode: Clean up utf8 funcs and provide non-utf8 inline versions.

This patch provides inline versions of the utf8 helper functions for the
case when unicode is disabled (MICROPY_PY_BUILTINS_STR_UNICODE set to 0).
This saves code size.

The unichar_charlen function is also renamed to utf8_charlen to match the
other utf8 helper functions, and the signature of this function is adjusted
for consistency (const char* -> const byte*, mp_uint_t -> size_t).
This commit is contained in:
Damien George 2018-02-14 18:19:22 +11:00
parent 49e0dd54e6
commit 19aee9438a
5 changed files with 21 additions and 22 deletions

View File

@ -121,8 +121,15 @@ typedef uint32_t unichar;
typedef uint unichar;
#endif
#if MICROPY_PY_BUILTINS_STR_UNICODE
unichar utf8_get_char(const byte *s);
const byte *utf8_next_char(const byte *s);
size_t utf8_charlen(const byte *str, size_t len);
#else
static inline unichar utf8_get_char(const byte *s) { return *s; }
static inline const byte *utf8_next_char(const byte *s) { return s + 1; }
static inline size_t utf8_charlen(const byte *str, size_t len) { (void)str; return len; }
#endif
bool unichar_isspace(unichar c);
bool unichar_isalpha(unichar c);
@ -135,7 +142,6 @@ bool unichar_islower(unichar c);
unichar unichar_tolower(unichar c);
unichar unichar_toupper(unichar c);
mp_uint_t unichar_xdigit_value(unichar c);
mp_uint_t unichar_charlen(const char *str, mp_uint_t len);
#define UTF8_IS_NONASCII(ch) ((ch) & 0x80)
#define UTF8_IS_CONT(ch) (((ch) & 0xC0) == 0x80)

View File

@ -346,7 +346,7 @@ STATIC mp_obj_t mp_builtin_ord(mp_obj_t o_in) {
const char *str = mp_obj_str_get_data(o_in, &len);
#if MICROPY_PY_BUILTINS_STR_UNICODE
if (MP_OBJ_IS_STR(o_in)) {
len = unichar_charlen(str, len);
len = utf8_charlen((const byte*)str, len);
if (len == 1) {
return mp_obj_new_int(utf8_get_char((const byte*)str));
}

View File

@ -1704,7 +1704,7 @@ STATIC mp_obj_t str_count(size_t n_args, const mp_obj_t *args) {
// if needle_len is zero then we count each gap between characters as an occurrence
if (needle_len == 0) {
return MP_OBJ_NEW_SMALL_INT(unichar_charlen((const char*)start, end - start) + 1);
return MP_OBJ_NEW_SMALL_INT(utf8_charlen(start, end - start) + 1);
}
// count the occurrences

View File

@ -104,7 +104,7 @@ STATIC mp_obj_t uni_unary_op(mp_unary_op_t op, mp_obj_t self_in) {
case MP_UNARY_OP_BOOL:
return mp_obj_new_bool(str_len != 0);
case MP_UNARY_OP_LEN:
return MP_OBJ_NEW_SMALL_INT(unichar_charlen((const char *)str_data, str_len));
return MP_OBJ_NEW_SMALL_INT(utf8_charlen(str_data, str_len));
default:
return MP_OBJ_NULL; // op not supported
}

View File

@ -67,9 +67,9 @@ STATIC const uint8_t attr[] = {
AT_LO, AT_LO, AT_LO, AT_PR, AT_PR, AT_PR, AT_PR, 0
};
// TODO: Rename to str_get_char
unichar utf8_get_char(const byte *s) {
#if MICROPY_PY_BUILTINS_STR_UNICODE
unichar utf8_get_char(const byte *s) {
unichar ord = *s++;
if (!UTF8_IS_NONASCII(ord)) return ord;
ord &= 0x7F;
@ -80,22 +80,14 @@ unichar utf8_get_char(const byte *s) {
ord = (ord << 6) | (*s++ & 0x3F);
}
return ord;
#else
return *s;
#endif
}
// TODO: Rename to str_next_char
const byte *utf8_next_char(const byte *s) {
#if MICROPY_PY_BUILTINS_STR_UNICODE
++s;
while (UTF8_IS_CONT(*s)) {
++s;
}
return s;
#else
return s + 1;
#endif
}
mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr) {
@ -109,21 +101,18 @@ mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr) {
return i;
}
// TODO: Rename to str_charlen
mp_uint_t unichar_charlen(const char *str, mp_uint_t len) {
#if MICROPY_PY_BUILTINS_STR_UNICODE
mp_uint_t charlen = 0;
for (const char *top = str + len; str < top; ++str) {
size_t utf8_charlen(const byte *str, size_t len) {
size_t charlen = 0;
for (const byte *top = str + len; str < top; ++str) {
if (!UTF8_IS_CONT(*str)) {
++charlen;
}
}
return charlen;
#else
return len;
#endif
}
#endif
// Be aware: These unichar_is* functions are actually ASCII-only!
bool unichar_isspace(unichar c) {
return c < 128 && (attr[c] & FL_SPACE) != 0;
@ -183,6 +172,8 @@ mp_uint_t unichar_xdigit_value(unichar c) {
return n;
}
#if MICROPY_PY_BUILTINS_STR_UNICODE
bool utf8_check(const byte *p, size_t len) {
uint8_t need = 0;
const byte *end = p + len;
@ -210,3 +201,5 @@ bool utf8_check(const byte *p, size_t len) {
}
return need == 0; // no pending fragments allowed
}
#endif