djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tests/run-multitests.py: Support outputting test metrics.

Browse Source 
If a multitest calls `multitest.output_metric(...)` then that output will
be collected separately, not considered as part of the test verification
output, and instead be printed at the end.  This is useful for tests that
want to output performance/timing metrics that may change from one run to
the next.

Signed-off-by: Damien George <damien@micropython.org>
This commit is contained in:
Damien George 2023-03-17 13:55:35 +11:00
parent 31e7a0587d
commit af42634866
1 changed files with 16 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
tests/run-multitests.py
View File

@ -93,6 +93,9 @@ class multitest:
@staticmethod @staticmethod
def expect_reboot(resume, delay_ms=0): def expect_reboot(resume, delay_ms=0):
print("WAIT_FOR_REBOOT", resume, delay_ms) print("WAIT_FOR_REBOOT", resume, delay_ms)
@staticmethod
def output_metric(data):
print("OUTPUT_METRIC", data)
{} {}
@ -312,6 +315,7 @@ def run_test_on_instances(test_file, num_instances, instances):
skip = False skip = False
injected_globals = "" injected_globals = ""
output = [[] for _ in range(num_instances)] output = [[] for _ in range(num_instances)]
output_metrics = []
# If the test calls get_network_ip() then inject HOST_IP so that devices can know # If the test calls get_network_ip() then inject HOST_IP so that devices can know
# the IP address of the host. Do this lazily to not require a TCP/IP connection # the IP address of the host. Do this lazily to not require a TCP/IP connection
@ -400,6 +404,8 @@ def run_test_on_instances(test_file, num_instances, instances):
for instance2 in instances: for instance2 in instances:
if instance2 is not instance: if instance2 is not instance:
instance2.write(bytes(out, "ascii") + b"\r\n") instance2.write(bytes(out, "ascii") + b"\r\n")
elif out.startswith("OUTPUT_METRIC "):
output_metrics.append(out.split(" ", 1)[1])
else: else:
output[idx].append(out) output[idx].append(out)
if err is not None: if err is not None:
@ -421,7 +427,7 @@ def run_test_on_instances(test_file, num_instances, instances):
output_str += "--- instance{} ---\n".format(idx) output_str += "--- instance{} ---\n".format(idx)
output_str += "\n".join(lines) + "\n" output_str += "\n".join(lines) + "\n"
return error, skip, output_str return error, skip, output_str, output_metrics
def wait_for_reboot(instance, extra_timeout_ms=0): def wait_for_reboot(instance, extra_timeout_ms=0):
@ -481,7 +487,9 @@ def run_tests(test_files, instances_truth, instances_test):
sys.stdout.flush() sys.stdout.flush()
# Run test on test instances # Run test on test instances
error, skip, output_test = run_test_on_instances(test_file, num_instances, instances_test) error, skip, output_test, output_metrics = run_test_on_instances(
test_file, num_instances, instances_test
)
if not skip: if not skip:
# Check if truth exists in a file, and read it in # Check if truth exists in a file, and read it in
@ -491,7 +499,7 @@ def run_tests(test_files, instances_truth, instances_test):
output_truth = f.read() output_truth = f.read()
else: else:
# Run test on truth instances to get expected output # Run test on truth instances to get expected output
_, _, output_truth = run_test_on_instances( _, _, output_truth, _ = run_test_on_instances(
test_file, num_instances, instances_truth test_file, num_instances, instances_truth
) )
@ -520,6 +528,11 @@ def run_tests(test_files, instances_truth, instances_test):
print("### DIFF ###") print("### DIFF ###")
print_diff(output_truth, output_test) print_diff(output_truth, output_test)
# Print test output metrics, if there are any.
if output_metrics:
for metric in output_metrics:
print(test_file, ": ", metric, sep="")
if cmd_args.show_output: if cmd_args.show_output:
print() print()