From 252aacdddf1aae89e3ebe91ca72a3ad666728e37 Mon Sep 17 00:00:00 2001 From: Scott Shawcroft Date: Wed, 27 Jun 2018 11:17:13 -0700 Subject: [PATCH] Analysis fixes and long lived tweaks. --- .gitignore | 1 + py/gc.c | 0 py/gc_long_lived.c | 2 +- tools/analyze_heap_dump.py | 69 +++++++++++++++++++++------------- tools/output_gc_until_repl.txt | 2 +- 5 files changed, 45 insertions(+), 29 deletions(-) mode change 100644 => 100755 py/gc.c mode change 100644 => 100755 py/gc_long_lived.c mode change 100644 => 100755 tools/analyze_heap_dump.py diff --git a/.gitignore b/.gitignore index 09ffb164d1..aa4ff6dab2 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,4 @@ TAGS ################# *.orig +*.DS_Store diff --git a/py/gc.c b/py/gc.c old mode 100644 new mode 100755 diff --git a/py/gc_long_lived.c b/py/gc_long_lived.c old mode 100644 new mode 100755 index d34fde5d9b..e30b389faf --- a/py/gc_long_lived.c +++ b/py/gc_long_lived.c @@ -121,7 +121,7 @@ mp_obj_t make_obj_long_lived(mp_obj_t obj, uint8_t max_depth){ } else if (MP_OBJ_IS_TYPE(obj, &mp_type_property)) { mp_obj_property_t *prop = MP_OBJ_TO_PTR(obj); return MP_OBJ_FROM_PTR(make_property_long_lived(prop, max_depth)); - } else if (MP_OBJ_IS_TYPE(obj, &mp_type_str)) { + } else if (MP_OBJ_IS_TYPE(obj, &mp_type_str) || MP_OBJ_IS_TYPE(obj, &mp_type_bytes)) { mp_obj_str_t *str = MP_OBJ_TO_PTR(obj); return MP_OBJ_FROM_PTR(make_str_long_lived(str)); } else if (MP_OBJ_IS_TYPE(obj, &mp_type_type)) { diff --git a/tools/analyze_heap_dump.py b/tools/analyze_heap_dump.py old mode 100644 new mode 100755 index 55342ac693..fa3efbe207 --- a/tools/analyze_heap_dump.py +++ b/tools/analyze_heap_dump.py @@ -135,8 +135,11 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont ram_start = symbols["_srelocate"][0] ram_end = symbols["_estack"][0] ram_length = ram_end - ram_start + # print(ram_length, "ram length") + # print(len(ram_dump) // ram_length, "snapshots") if analyze_snapshots == "all": - snapshots = range(len(ram_dump) // ram_length - 1, -1, -1) + #snapshots = range(len(ram_dump) // ram_length - 1, -1, -1) + snapshots = range(4576, -1, -1) elif analyze_snapshots == "last": snapshots = range(len(ram_dump) // ram_length - 1, len(ram_dump) // ram_length - 2, -1) for snapshot_num in snapshots: @@ -167,16 +170,16 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont mp_state_ctx = symbols["mp_state_ctx"][0] manual_symbol_map["mp_state_ctx+20"] = "mp_state_ctx.vm.last_pool" last_pool = load_pointer(mp_state_ctx + 20) # (gdb) p &mp_state_ctx.vm.last_pool - manual_symbol_map["mp_state_ctx+88"] = "mp_state_ctx.vm.dict_main.map.table" - dict_main_table = load_pointer(mp_state_ctx + 88) # (gdb) p &mp_state_ctx.vm.dict_main.map.table - manual_symbol_map["mp_state_ctx+68"] = "mp_state_ctx.vm.mp_loaded_modules_dict.map.table" - imports_table = load_pointer(mp_state_ctx + 68) # (gdb) p &mp_state_ctx.vm.mp_loaded_modules_dict.map.table + manual_symbol_map["mp_state_ctx+104"] = "mp_state_ctx.vm.dict_main.map.table" + dict_main_table = load_pointer(mp_state_ctx + 104) # (gdb) p &mp_state_ctx.vm.dict_main.map.table + manual_symbol_map["mp_state_ctx+84"] = "mp_state_ctx.vm.mp_loaded_modules_dict.map.table" + imports_table = load_pointer(mp_state_ctx + 84) # (gdb) p &mp_state_ctx.vm.mp_loaded_modules_dict.map.table - manual_symbol_map["mp_state_ctx+104"] = "mp_state_ctx.vm.mp_sys_path_obj.items" - manual_symbol_map["mp_state_ctx+120"] = "mp_state_ctx.vm.mp_sys_argv_obj.items" + manual_symbol_map["mp_state_ctx+120"] = "mp_state_ctx.vm.mp_sys_path_obj.items" + manual_symbol_map["mp_state_ctx+136"] = "mp_state_ctx.vm.mp_sys_argv_obj.items" for i in range(READLINE_HIST_SIZE): - manual_symbol_map["mp_state_ctx+{}".format(128 + i * 4)] = "mp_state_ctx.vm.readline_hist[{}]".format(i) + manual_symbol_map["mp_state_ctx+{}".format(144 + i * 4)] = "mp_state_ctx.vm.readline_hist[{}]".format(i) tuple_type = symbols["mp_type_tuple"][0] type_type = symbols["mp_type_type"][0] @@ -214,8 +217,8 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont pool_start = heap_start + total_byte_len - pool_length - pool_shift pool = heap[-pool_length-pool_shift:] - total_height = 65 * 18 - total_width = (pool_length // (64 * 16)) * 90 + total_height = 128 * 18 + total_width = (pool_length // (128 * 16)) * 85 map_element_blocks = [dict_main_table, imports_table] string_blocks = [] @@ -255,7 +258,7 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont block_data[address] = data for k in range(len(data) // 4): word = struct.unpack_from(">= 3 + if qstr_index > total_prev_len + alloc: + return "invalid" while pool_ptr != 0: if pool_ptr > ram_start: if pool_ptr in block_data: @@ -492,7 +500,10 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont offset = len(data) continue offset += 2 + qstr_len + 1 - qstrs_in_chunk += " " + data[offset - qstr_len - 1: offset - 1].decode("utf-8") + try: + qstrs_in_chunk += " " + data[offset - qstr_len - 1: offset - 1].decode("utf-8") + except UnicodeDecodeError: + qstrs_in_chunk += " " + "░"*qstr_len printable_qstrs = "" for i in range(len(qstrs_in_chunk)): c = qstrs_in_chunk[i] @@ -515,20 +526,28 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont # First render the graph of objects on the heap. if draw_heap_ownership: ownership_graph.layout(prog="dot") - fn = os.path.join(output_directory, "heap_ownership{:04d}.png".format(snapshot_num)) + fn = os.path.join(output_directory, "heap_ownership{:04d}.svg".format(snapshot_num)) print(fn) ownership_graph.draw(fn) + # Clear edge positioning from ownership graph layout. + if draw_heap_ownership: + for edge in ownership_graph.iteredges(): + del edge.attr["pos"] + else: + for edge in ownership_graph.edges(): + ownership_graph.delete_edge(edge) + # Second, render the heap layout in memory order. - for node in ownership_graph: + for node in ownership_graph.nodes(): try: address = int(node.name) except ValueError: - ownership_graph.remove_node(node) + ownership_graph.remove_node(node.name) continue block = (address - pool_start) // 16 - x = block // 64 - y = 64 - block % 64 + x = block // 128 + y = 128 - block % 128 try: height = float(node.attr["height"]) except: @@ -538,11 +557,6 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont # print(hex(address), block, len(block_data[address]), x, y, height) node.attr["pos"] = "{},{}".format(x * 80, (y - (height - 0.25) * 2) * 18) # in inches - # Clear edge positioning from ownership graph layout. - if draw_heap_ownership: - for edge in ownership_graph.iteredges(): - del edge.attr["pos"] - # Reformat block nodes so they are the correct size and do not have keys in them. for block in sorted(map_element_blocks): try: @@ -565,9 +579,9 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont else: #print(" {}, {}".format(format(key), format(value))) cells.append((key, "")) - if value in block_data: - edge = ownership_graph.get_edge(block, value) - edge.attr["tailport"] = str(key) + # if value in block_data: + # edge = ownership_graph.get_edge(block, value) + # edge.attr["tailport"] = str(key) rows = "" for i in range(len(cells) // 2): rows += "{}{}".format( @@ -586,6 +600,7 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont if draw_heap_layout: fn = os.path.join(output_directory, "heap_layout{:04d}.png".format(snapshot_num)) print(fn) + #ownership_graph.write(fn+".dot") ownership_graph.draw(fn) if __name__ == "__main__": diff --git a/tools/output_gc_until_repl.txt b/tools/output_gc_until_repl.txt index 0e99626353..81711deeb8 100644 --- a/tools/output_gc_until_repl.txt +++ b/tools/output_gc_until_repl.txt @@ -19,7 +19,7 @@ append binary memory ram.bin &_srelocate &_estack continue end -break main.c:164 +break main.c:179 continue