circuitpython/tests/float/string_format_modulo2_intbi...

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

24 lines
804 B
Python
Raw Normal View History

# test formatting floats with large precision, that it doesn't overflow the buffer
2021-03-15 09:57:36 -04:00
def test(num, num_str):
2021-03-15 09:57:36 -04:00
if num == float("inf") or num == 0.0 and num_str != "0.0":
# skip numbers that overflow or underflow the FP precision
return
2021-03-15 09:57:36 -04:00
for kind in ("e", "f", "g"):
# check precision either side of the size of the buffer (32 bytes)
for prec in range(23, 36, 2):
2021-03-15 09:57:36 -04:00
fmt = "%." + "%d" % prec + kind
s = fmt % num
check = abs(float(s) - num)
if num > 1:
check /= num
if check > 1e-6:
2021-03-15 09:57:36 -04:00
print("FAIL", num_str, fmt, s, len(s), check)
# check most powers of 10, making sure to include exponents with 3 digits
for e in range(-101, 102):
num = pow(10, e)
2021-03-15 09:57:36 -04:00
test(num, "1e%d" % e)