diff --git a/docs/commands/index.md b/docs/commands/index.md index dfba16710..e16843b76 100644 --- a/docs/commands/index.md +++ b/docs/commands/index.md @@ -266,6 +266,7 @@ - [mallocng-meta](musl/mallocng-meta.md) - Print out information about a mallocng group given the address of its meta. - [mallocng-slot-start](musl/mallocng-slot-start.md) - Dump information about a mallocng slot, given its start address. - [mallocng-slot-user](musl/mallocng-slot-user.md) - Dump information about a mallocng slot, given its user address. +- [mallocng-visualize-slots](musl/mallocng-visualize-slots.md) - Visualize slots in a group. diff --git a/docs/commands/musl/mallocng-visualize-slots.md b/docs/commands/musl/mallocng-visualize-slots.md new file mode 100644 index 000000000..26fdd1917 --- /dev/null +++ b/docs/commands/musl/mallocng-visualize-slots.md @@ -0,0 +1,26 @@ + +# mallocng-visualize-slots + +```text +usage: mallocng-visualize-slots [-h] address [count] + +``` + +Visualize slots in a group. + +**Alias:** ng-vis +### Positional arguments + +|Positional Argument|Help| +| :--- | :--- | +|address|Address which is inside some slot.| +|count|The amount of slots to visualize. (default: 10)| + +### Optional arguments + +|Short|Long|Help| +| :--- | :--- | :--- | +|-h|--help|show this help message and exit| + + + diff --git a/docs/configuration/heap.md b/docs/configuration/heap.md index db236277f..b275164a8 100644 --- a/docs/configuration/heap.md +++ b/docs/configuration/heap.md @@ -86,6 +86,17 @@ commands will search the heap to try to find the correct meta/group. ---------- +## **ng-vis-count** + + +Default count for ng-vis. + + + +**Default:** 10 + +---------- + ## **resolve-heap-via-heuristic** === "GDB" diff --git a/pwndbg/commands/mallocng.py b/pwndbg/commands/mallocng.py index c3eade9f6..c8cc9e307 100644 --- a/pwndbg/commands/mallocng.py +++ b/pwndbg/commands/mallocng.py @@ -5,6 +5,8 @@ Commands that help with debugging musl's allocator, mallocng. from __future__ import annotations import argparse +import string +from typing import List from typing import Optional import pwndbg @@ -34,225 +36,32 @@ commands will search the heap to try to find the correct meta/group. scope=pwndbg.lib.config.Scope.heap, ) +state_alloc_color = C.BLUE +state_alloc_color_alt = C.CYAN +state_freed_color = C.RED +state_freed_color_alt = C.LIGHT_RED +state_avail_color = C.GRAY +state_avail_color_alt = C.LIGHT_GRAY -@pwndbg.commands.Command( - "Gives a quick explanation of musl's mallocng allocator.", - category=CommandCategory.MUSL, - aliases=["ng-explain"], -) -def mallocng_explain() -> None: - txt = ( - C.bold("mallocng") - + ' is a slab allocator. The "unit of allocation" is called a ' - + C.bold("slot") - + "\n" - ) - txt += '(the equivalent of glibc\'s "chunk"). Slots are in 0x10 granularity and\n' - txt += ( - "alignment. The slots are organized into objects called " + C.bold('"groups"') + " (the \n" - ) - txt += "slabs). Each group is composed of slots of the same size. If a group is big\n" - txt += "it is allocated using mmap, otherwise it is allocated as a slot of a larger\n" - txt += "group.\n\n" - - txt += "Each group has some associated metadata. This metadata is stored in a separate\n" - txt += "object called " + C.bold('"meta"') + ". Metas are allocated separately from groups in\n" - txt += C.bold('"meta areas"') + " to make it harder to reach them during exploitation.\n\n" - - txt += "Here are the definitions of group, meta and meta_area.\n\n" - - txt += C.bold("struct group {\n") - txt += " // the metadata of this group\n" - txt += C.bold(" struct meta *meta;\n") - txt += " unsigned char active_idx:5;\n" - txt += " char pad[UNIT - sizeof(struct meta *) - 1];\n" - txt += " // start of the slots array\n" - txt += C.bold(" unsigned char storage[];\n") - txt += C.bold("};\n\n") - - txt += C.bold("struct meta {\n") - txt += " // doubly linked list connecting meta's\n" - txt += C.bold(" struct meta *prev, *next;\n") - txt += " // which group is this metadata for\n" - txt += C.bold(" struct group *mem;\n") - txt += " // slot bitmap\n" - txt += " // avail - slots which have not yet been allocated\n" - txt += " // freed - free slots\n" - txt += C.bold(" volatile int avail_mask, freed_mask;\n") - txt += " uintptr_t last_idx:5;\n" - txt += " uintptr_t freeable:1;\n" - txt += " // describes the size of the slots\n" - txt += C.bold(" uintptr_t sizeclass:6;\n") - txt += " // if this group was mmaped, how many pages did we use?\n" - txt += " uintptr_t maplen:8*sizeof(uintptr_t)-12;\n" - txt += C.bold("};\n\n") - - txt += C.bold("struct meta_area {\n") - txt += " uint64_t check;\n" - txt += " struct meta_area *next;\n" - txt += " int nslots;\n" - txt += " // start of the meta array\n" - txt += C.bold(" struct meta slots[];\n") - txt += C.bold("};\n\n") - - txt += ( - "Two other important definitions are " + C.bold("IB") + " and " + C.bold("UNIT") + ".\n\n" - ) - - txt += "// the aforementioned slot alignment.\n" - txt += C.bold("#define UNIT 16\n") - txt += "// the size of the in-band metadata.\n" - txt += C.bold("#define IB 4\n\n") - - txt += "The allocator state is stored in the global `ctx` variable which is of\n" - txt += "type `struct malloc_context`. It is accessible through the __malloc_context\n" - txt += "symbol.\n\n" - - txt += C.bold("struct malloc_context {\n") - txt += C.bold(" uint64_t secret;\n") - txt += "#ifndef PAGESIZE\n" - txt += " size_t pagesize;\n" - txt += "#endif\n" - txt += " int init_done;\n" - txt += " unsigned mmap_counter;\n" - txt += C.bold(" struct meta *free_meta_head;\n") - txt += C.bold(" struct meta *avail_meta;\n") - txt += " size_t avail_meta_count, avail_meta_area_count, meta_alloc_shift;\n" - txt += C.bold(" struct meta_area *meta_area_head, *meta_area_tail;\n") - txt += C.bold(" unsigned char *avail_meta_areas;\n") - txt += ' // the "active" group for each sizeclass\n' - txt += " // it will be picked for allocation\n" - txt += C.bold(" struct meta *active[48];\n") - txt += " size_t usage_by_class[48];\n" - txt += " uint8_t unmap_seq[32], bounces[32];\n" - txt += " uint8_t seq;\n" - txt += " uintptr_t brk;\n" - txt += C.bold("};\n\n") - - txt += "Here is a diagram of how these components interact.\n\n" - - diag = """+-malloc_context--+ -| | -| free_meta_head |-----------------------> Points to a free meta which is connected -| avail_meta |---------------+ to other free meta's via a doubly linked list. -| meta_area_head |------------+ | -| active[48] |---+ | +-> Points to a not-yet-allocated meta. -| | | | When it gets allocated, the next -|-----------------+ | 1/48 | meta in the meta_area gets selected - | | i.e. avail_meta++ . - Each size class has | +-------------------------------------------+ - an "active" group. +-------+ | - v | - +-meta--+ +-meta--+ +-meta--+ | - | | | | | | | - ... <---| prev |<------| prev |------>| prev |------> ... | - ... --->| next |------>| next |<------| next |<------ ... | - | mem | +->| mem |-+ | mem | | - | | | | | | | | v - +-------+ | +-------+ | +-------+ +-meta_area----------------+ - | | (yes these metas) | | - | | (are in some meta_area) | check (ctx.secret) | - +---------------------+ | | next |----> ... - | v | nslots | - | +-group----------------------------------------+ | meta0 | - | | | | Meta objects are | - +-| meta (8) | active_idx (1) | pad (7) | | meta1 stored here. | - | slot0 | | | - | | | ... | - | | | | - | slot1 Slots contain the actual | | meta(nslots-1) | - | user data. | | | - | | +--------------------------+ - | slot2 | - | | - | ... | - | | - | slot(cnt-1) | - | | - | | - +----------------------------------------------+ -""" - - txt += diag - - txt += f""" -### What slots look like - -Unfortunately, musl doesn't provide a struct which describes the -slot's in-band metadata. It does however use consistent variable -names to describe the values saved in slots, so we will use those -as well. Check the {C.bold('enframe()')} function in the source, it is very -important. - -{C.bold('idx')} is the index of the slot within its group. The {C.bold("stride")} of -a group is (generally) determined by the sizeclass as -{C.bold("UNIT * size_classes[meta.sizeclass]")}. {C.bold("start")} is the starting -address of the slot (the slot0, slot1, ... in the above diagram). -The start of a slot with index i is {C.bold("group.storage + i * stride")}. -The "nominal size" is the amount of memory the user requested with -their malloc() call, in the source it is also referred to as {C.bold("n")}. - -For every slot in a group, the memory in [start - IB, start) contains -some metadata that we will call the "start header". For this reason, -the {C.bold("end")} of a slot is calculated as {C.bold("start + stride - IB")}. The -{C.bold("slack")} of a slot is calculated as {C.bold("(stride - n - IB) / UNIT")} and -describes the amount of unused memory within a slot. - -To prevent double-frees and exploitation attempts, the mallocng -allocator performs "cycling" i.e. the actual start of user data -(the pointer returned by malloc) can be at some offset from the -{C.bold("start")} of the slot. The start of user data is called {C.bold("p")} and it -is also UNIT aligned. We will call the distance between {C.bold("p")} and -{C.bold("start")} the "cyclic offset" ({C.bold("off")} in code). When calculating -the cyclic offset, mallocng ensures {C.bold("off <= slack")}. - -If a slot is in fact cycled, then that is stored in the start -header as {C.bold("off = *(uint16_t*)(start-2)")} and {C.bold("start[-3] = 7 << 5")}. -The {C.bold("start[-3]")} field acts as a flag. - -For every slot, the memory in [p - IB, p) contains some metadata. -We will call this the "p header". If the slot is not cycled i.e. -{C.bold("start == p")}, then [start - IB, start) will contain the p header -fields and start[-3] >> 5 will *not* be 7. - -The value in {C.bold("*(uint16_t*)(p-2)")} is the {C.bold("offset")} from the slot's -{C.bold("start")} to the start of the group (divided by UNIT). The value -in {C.bold("p[-4]")} is either 0 or 1 and describes if a "big offset" should -be used. It is usually zero and gets set to one only in some cases -in aligned_alloc(). If it is 1, the offset is to be calculated as -{C.bold("*(uint32_t *)(p - 8)")}. - -{C.bold("p[-3]")} contains multiple pieces of information. If {C.bold("p[-3] == 0xFF")} -the slot is freed. Otherwise, the lower 5 bits of p[-3] describe -the index of the slot in its group: {C.bold("idx = p[-3] & 31")}. The top -3 bits desribed the {C.bold("reserved")} area size. This is the memory -between the end of user memory and {C.bold("end")} i.e. {C.bold("reserved = end - p - n")}. -We will call the value {C.bold("p[-3] >> 5")}, "hdr reserved" for "reserved as -specified in the p header". It can happen however, that the value -{C.bold("reserved = end - p - n")} is large and so doesn't fit in the three -bits in p[-3]. In this case "hdr reserved" will be strictly 5, which -denotes that we need to look at the slot's footer to read the actual -value of {C.bold("reserved")}. As a special case, if {C.bold("p[-3] >> 5 == 6")} that -doesn't describe the reserved size at all, but specifies that there -is a group nested inside this slot. {C.bold("p[-3] >> 5")} should never be 7, -contrary to {C.bold("start[-3] >> 5")}. - -The "footer" of a slot is the third and final area of a slot's -memory where metadata is contained. This is the [end - 4, end) -area. It only contains the reserved size as -{C.bold("reserved = *(const uint32_t *)(end-4)")} when {C.bold("p[-3] >> 5 == 5")}. +def get_slot_color(state: mallocng.SlotState, last_color: str = "") -> str: + match state: + case mallocng.SlotState.ALLOCATED: + if last_color == state_alloc_color: + return state_alloc_color_alt + return state_alloc_color + case mallocng.SlotState.FREED: + if last_color == state_freed_color: + return state_freed_color_alt + return state_freed_color + case mallocng.SlotState.AVAIL: + if last_color == state_avail_color: + return state_avail_color_alt + return state_avail_color -All of the above is only generally true for allocated slots. Mallocng -ensures {C.bold("p[-3] = 0xFF")} and {C.bold("*(uint16_t *)(p - 2) = 0")} for freed slots, -which makes the start of the slot's group (and thus meta) unreachable. -Only in this case does {C.bold("p[-3] >> 5")} become 7. Available slots, -i.e. those that haven't been allocated nor freed yet (but are ready -for allocation), have almost no guarantees on their data and -metadata contents. -""" - print(txt) +def get_colored_slot_state(ss: mallocng.SlotState) -> str: + return C.colorize(ss.value, get_slot_color(ss)) def dump_group(group: mallocng.Group) -> str: @@ -347,16 +156,6 @@ def dump_meta(meta: mallocng.Meta) -> str: return output -def get_colored_slot_state(ss: mallocng.SlotState) -> str: - match ss: - case mallocng.SlotState.ALLOCATED: - return C.green(ss.value) - case mallocng.SlotState.FREED: - return C.red(ss.value) - case mallocng.SlotState.AVAIL: - return C.blue(ss.value) - - def dump_grouped_slot(gslot: mallocng.GroupedSlot, all: bool) -> str: pp = PropertyPrinter() @@ -506,8 +305,7 @@ def smart_dump_slot( try: slot.preload() except pwndbg.dbg_mod.Error as e: - print(message.error(f"Error while reading slot: {e}")) - return "" + return message.error(f"Error while reading slot: {e}") successful_preload: bool = True err_msg = "" @@ -540,15 +338,15 @@ def smart_dump_slot( # could be possible in exploitation I suppose). return dump_slot(slot, all, True, False) + output = "" + if not (slot._pn3 == 0xFF or slot._offset == 0): # If the group/meta read failed because the slot is freed/avail, # we won't throw an error. This is just a heuristic check for # better UX. I'm using the private fields for the check so we # don't accidentally cause an exception here if we are bordering # unreadable memory. - print(err_msg) - - output = "" + output += err_msg + "\n" if gslot is None: if not search_on_fail: @@ -995,3 +793,495 @@ def mallocng_find( return print(smart_dump_slot(slot, all, grouped_slot), end="") + + +VALID_CHARS = list(map(ord, set(string.printable) - set("\t\r\n\x0c\x0b"))) + + +def bin_ascii(bs: bytearray): + return "".join(chr(c) if c in VALID_CHARS else "." for c in bs) + + +vis_cyclic_offset_color = C.YELLOW +vis_offset_color = C.LIGHT_YELLOW +vis_cycled_mark_color = C.PURPLE +vis_pn3_reserved_color = C.LIGHT_CYAN +vis_big_offset_check_color = C.BLACK +vis_ftr_reserved_color = C.GREEN + + +def colorize_pointer( + address: int, ptrvalue: int, state: mallocng.SlotState, slot: mallocng.Slot +) -> str: + ptrsize = pwndbg.aglib.typeinfo.ptrsize + out = f"{ptrvalue:0{ptrsize * 2}x}" + + if state != mallocng.SlotState.ALLOCATED: + # Nothing to do. + return out + + # Are we in user data? + if slot.p <= address < slot.p + slot.nominal_size: + # Yes, bold the parts that are. + boldable_bytes = min(slot.p + slot.nominal_size - address, ptrsize) + plain_part = out[: (-2 * boldable_bytes)] + bold_part = C.bold(out[(-2 * boldable_bytes) :]) + out = plain_part + bold_part + + # Are we in the p header of this slot? + if address == slot.p - ptrsize: + offset_part = C.colorize(out[:4], vis_offset_color) + pn3_part = C.colorize(out[4:6], vis_pn3_reserved_color) + big_offset_part = C.colorize(out[6:8], vis_big_offset_check_color) + plain_part = out[8:] + + out = offset_part + pn3_part + big_offset_part + plain_part + + # Are we in the footer of this slot? + if address == slot.start + slot.meta.stride - ptrsize: + # Highlight ftr reserved if it is used. + if slot.reserved_in_header == 5: + plain_part = out[:8] + ftr_reserved_part = C.colorize(out[8:], vis_ftr_reserved_color) + out = plain_part + ftr_reserved_part + + return out + + +def colorize_start_header_line(shline: str, state: mallocng.SlotState, slot: mallocng.Slot) -> str: + if state != mallocng.SlotState.ALLOCATED: + # Nothing to do. + return shline + + splitline = shline.split("0x", maxsplit=3) + assert len(splitline) == 4 + rightvalplus = splitline[3] + + if slot.start != slot.p: + # A cycled slot. The offset has completely different meaning + # than in p header. The hdr_res has kinda~ different meaning. + offset_part = C.colorize(rightvalplus[:4], vis_cyclic_offset_color) + sorpn3 = C.colorize(rightvalplus[4:6], vis_cycled_mark_color) + else: + offset_part = C.colorize(rightvalplus[:4], vis_offset_color) + sorpn3 = C.colorize(rightvalplus[4:6], vis_pn3_reserved_color) + + big_offset_part = C.colorize(rightvalplus[6:8], vis_big_offset_check_color) + plain_part = rightvalplus[8:] + + out = ( + f"{splitline[0]}0x{splitline[1]}0x{splitline[2]}0x" + + offset_part + + sorpn3 + + big_offset_part + + plain_part + ) + return out + + +def line_decoration(addr: int, slot_state: mallocng.SlotState, slot: mallocng.Slot) -> str: + """ + Maybe append extra clarification to a line. + + Currently only appends to p headers. + """ + if slot_state != mallocng.SlotState.ALLOCATED: + return "" + if addr != slot.p - 2 * pwndbg.aglib.typeinfo.ptrsize: + return "" + + return " " + C.colorize( + f"{slot.idx} + ({slot.reserved_in_header} << 5)", vis_pn3_reserved_color + ) + + +default_vis_count = pwndbg.config.add_param( + "ng-vis-count", + 10, + "default count for ng-vis", + param_class=pwndbg.lib.config.PARAM_UINTEGER, + scope=pwndbg.lib.config.Scope.heap, +) + +parser = argparse.ArgumentParser( + description="""Visualize slots in a group.""", +) +parser.add_argument( + "address", + type=int, + help="Address which is inside some slot.", +) +parser.add_argument( + "count", + type=int, + default=default_vis_count, + nargs="?", # Optional + help="The amount of slots to visualize.", +) + + +@pwndbg.commands.Command( + parser, + category=CommandCategory.MUSL, + aliases=["ng-vis"], +) +@pwndbg.commands.OnlyWhenRunning +def mallocng_visualize_slots(address: int, count: int = default_vis_count): + ptrsize = pwndbg.aglib.typeinfo.ptrsize + + if ptrsize != 8: + print(message.error("This command only works on architectures where a pointer is 64 bits.")) + return + + if not memory.is_readable_address(address): + print(message.error(f"Address {address:#x} not readable.")) + return + + if not ng.init_if_needed(): + print(message.error("Couldn't find the allocator, aborting the command.")) + return + + first_grouped_slot, first_slot = ng.find_slot(address, False, False) + + if first_slot is None: + print(message.info("No slot found containing that address.")) + return + + group: mallocng.Group = first_grouped_slot.group + meta: mallocng.Meta = first_grouped_slot.meta + first_idx: int = first_grouped_slot.idx + + print("group @ " + C.memory.get(group.addr)) + print("meta @ " + C.memory.get(meta.addr)) + + if first_idx + count >= meta.cnt: + if count != default_vis_count: + # If the default was passed, no need to warn the user. + print( + message.info( + f"Clamping count to {meta.cnt - first_idx} to not go over end of group." + ) + ) + + count = meta.cnt - first_idx + + cyc_offset_part = C.colorize("cyclic offset", vis_cyclic_offset_color) + cycled_mark_part = C.colorize("cycled mark", vis_cycled_mark_color) + offset_part = C.colorize("offset", vis_offset_color) + pn3_part = C.colorize("p[-3] = idx + (hdr reserved << 5)", vis_pn3_reserved_color) + big_offset_part = C.colorize("big offset mark", vis_big_offset_check_color) + ftr_reserved_part = C.colorize("ftr reserved", vis_ftr_reserved_color) + + legend = ( + "LEGEND: " + + cyc_offset_part + + "; " + + cycled_mark_part + + "; " + + offset_part + + "; " + + pn3_part + + "; " + + big_offset_part + + "; " + + ftr_reserved_part + + "\n" + ) + legend += ( + "LEGEND: " + + C.colorize("allo", state_alloc_color) + + C.colorize("cated", state_alloc_color_alt) + + "; " + + C.colorize("fr", state_freed_color) + + C.colorize("eed", state_freed_color_alt) + + "; " + + C.colorize("avai", state_avail_color) + + C.colorize("lable", state_avail_color_alt) + + "\n" + ) + print(legend) + + out: List[str] = [] # List of lines. + last_color = "nothing" + + # Add the line before the start of the first slot, to include its start header. + shline_addr = group.at_index(first_idx) - 2 * ptrsize + shline_bytes = pwndbg.aglib.memory.read(shline_addr, ptrsize * 2) + leftptr = pwndbg.aglib.arch.unpack(shline_bytes[:ptrsize]) + rightptr = pwndbg.aglib.arch.unpack(shline_bytes[ptrsize:]) + out.append( + f"{shline_addr:#x}\t0x{leftptr:0{ptrsize * 2}x}\t0x{rightptr:0{ptrsize * 2}x}\t{bin_ascii(shline_bytes)}" + ) + + # Iterate over slots + for idx in range(first_idx, first_idx + count): + start_address = group.at_index(idx) + next_start_address = start_address + meta.stride + + if idx == first_idx: + slot = first_slot + else: + try: + slot = mallocng.Slot.from_start(start_address) + slot.preload() + slot.set_group(group) + # Probably redundant, but just in case. + slot.preload_meta_dependants() + except pwndbg.dbg_mod.Error as e: + print( + message.error( + f"Error while reading slot {idx} @ {C.memory.get(start_address)}: {e}" + ) + ) + return + + slot_state: mallocng.SlotState = meta.slotstate_at_index(idx) + cur_slot_color = get_slot_color(slot_state, last_color) + + # Colorize the previous line which contains our start header. + out[-1] = colorize_start_header_line(out[-1], slot_state, slot) + line_decoration( + start_address - 2 * ptrsize, slot_state, slot + ) + + # Make the output line by line (advance 0x10 bytes at a time). + cur_address = start_address + while cur_address < next_start_address: + line_bytes = pwndbg.aglib.memory.read(cur_address, ptrsize * 2) + leftptr = pwndbg.aglib.arch.unpack(line_bytes[:ptrsize]) + rightptr = pwndbg.aglib.arch.unpack(line_bytes[ptrsize:]) + + line_out = f"{cur_address:#x}" + line_out += "\t0x" + colorize_pointer(cur_address, leftptr, slot_state, slot) + line_out += "\t0x" + colorize_pointer(cur_address + ptrsize, rightptr, slot_state, slot) + line_out += f"\t{bin_ascii(line_bytes)}" + + line_out = pwndbg.color.colorize(line_out, cur_slot_color) + line_out += line_decoration(cur_address, slot_state, slot) + + out.append(line_out) + + cur_address += 2 * ptrsize + + last_color = cur_slot_color + + print("\n".join(out)) + + +@pwndbg.commands.Command( + "Gives a quick explanation of musl's mallocng allocator.", + category=CommandCategory.MUSL, + aliases=["ng-explain"], +) +def mallocng_explain() -> None: + txt = ( + C.bold("mallocng") + + ' is a slab allocator. The "unit of allocation" is called a ' + + C.bold("slot") + + "\n" + ) + txt += '(the equivalent of glibc\'s "chunk"). Slots are in 0x10 granularity and\n' + txt += ( + "alignment. The slots are organized into objects called " + C.bold('"groups"') + " (the \n" + ) + txt += "slabs). Each group is composed of slots of the same size. If a group is big\n" + txt += "it is allocated using mmap, otherwise it is allocated as a slot of a larger\n" + txt += "group.\n\n" + + txt += "Each group has some associated metadata. This metadata is stored in a separate\n" + txt += "object called " + C.bold('"meta"') + ". Metas are allocated separately from groups in\n" + txt += C.bold('"meta areas"') + " to make it harder to reach them during exploitation.\n\n" + + txt += "Here are the definitions of group, meta and meta_area.\n\n" + + txt += C.bold("struct group {\n") + txt += " // the metadata of this group\n" + txt += C.bold(" struct meta *meta;\n") + txt += " unsigned char active_idx:5;\n" + txt += " char pad[UNIT - sizeof(struct meta *) - 1];\n" + txt += " // start of the slots array\n" + txt += C.bold(" unsigned char storage[];\n") + txt += C.bold("};\n\n") + + txt += C.bold("struct meta {\n") + txt += " // doubly linked list connecting meta's\n" + txt += C.bold(" struct meta *prev, *next;\n") + txt += " // which group is this metadata for\n" + txt += C.bold(" struct group *mem;\n") + txt += " // slot bitmap\n" + txt += " // avail - slots which have not yet been allocated\n" + txt += " // freed - free slots\n" + txt += C.bold(" volatile int avail_mask, freed_mask;\n") + txt += " uintptr_t last_idx:5;\n" + txt += " uintptr_t freeable:1;\n" + txt += " // describes the size of the slots\n" + txt += C.bold(" uintptr_t sizeclass:6;\n") + txt += " // if this group was mmaped, how many pages did we use?\n" + txt += " uintptr_t maplen:8*sizeof(uintptr_t)-12;\n" + txt += C.bold("};\n\n") + + txt += C.bold("struct meta_area {\n") + txt += " uint64_t check;\n" + txt += " struct meta_area *next;\n" + txt += " int nslots;\n" + txt += " // start of the meta array\n" + txt += C.bold(" struct meta slots[];\n") + txt += C.bold("};\n\n") + + txt += ( + "Two other important definitions are " + C.bold("IB") + " and " + C.bold("UNIT") + ".\n\n" + ) + + txt += "// the aforementioned slot alignment.\n" + txt += C.bold("#define UNIT 16\n") + txt += "// the size of the in-band metadata.\n" + txt += C.bold("#define IB 4\n\n") + + txt += "The allocator state is stored in the global `ctx` variable which is of\n" + txt += "type `struct malloc_context`. It is accessible through the __malloc_context\n" + txt += "symbol.\n\n" + + txt += C.bold("struct malloc_context {\n") + txt += C.bold(" uint64_t secret;\n") + txt += "#ifndef PAGESIZE\n" + txt += " size_t pagesize;\n" + txt += "#endif\n" + txt += " int init_done;\n" + txt += " unsigned mmap_counter;\n" + txt += C.bold(" struct meta *free_meta_head;\n") + txt += C.bold(" struct meta *avail_meta;\n") + txt += " size_t avail_meta_count, avail_meta_area_count, meta_alloc_shift;\n" + txt += C.bold(" struct meta_area *meta_area_head, *meta_area_tail;\n") + txt += C.bold(" unsigned char *avail_meta_areas;\n") + txt += ' // the "active" group for each sizeclass\n' + txt += " // it will be picked for allocation\n" + txt += C.bold(" struct meta *active[48];\n") + txt += " size_t usage_by_class[48];\n" + txt += " uint8_t unmap_seq[32], bounces[32];\n" + txt += " uint8_t seq;\n" + txt += " uintptr_t brk;\n" + txt += C.bold("};\n\n") + + txt += "Here is a diagram of how these components interact.\n\n" + + diag = """+-malloc_context--+ +| | +| free_meta_head |-----------------------> Points to a free meta which is connected +| avail_meta |---------------+ to other free meta's via a doubly linked list. +| meta_area_head |------------+ | +| active[48] |---+ | +-> Points to a not-yet-allocated meta. +| | | | When it gets allocated, the next +|-----------------+ | 1/48 | meta in the meta_area gets selected + | | i.e. avail_meta++ . + Each size class has | +-------------------------------------------+ + an "active" group. +-------+ | + v | + +-meta--+ +-meta--+ +-meta--+ | + | | | | | | | + ... <---| prev |<------| prev |------>| prev |------> ... | + ... --->| next |------>| next |<------| next |<------ ... | + | mem | +->| mem |-+ | mem | | + | | | | | | | | v + +-------+ | +-------+ | +-------+ +-meta_area----------------+ + | | (yes these metas) | | + | | (are in some meta_area) | check (ctx.secret) | + +---------------------+ | | next |----> ... + | v | nslots | + | +-group----------------------------------------+ | meta0 | + | | | | Meta objects are | + +-| meta (8) | active_idx (1) | pad (7) | | meta1 stored here. | + | slot0 | | | + | | | ... | + | | | | + | slot1 Slots contain the actual | | meta(nslots-1) | + | user data. | | | + | | +--------------------------+ + | slot2 | + | | + | ... | + | | + | slot(cnt-1) | + | | + | | + +----------------------------------------------+ +""" + + txt += diag + + txt += f""" +### What slots look like + +Unfortunately, musl doesn't provide a struct which describes the +slot's in-band metadata. It does however use consistent variable +names to describe the values saved in slots, so we will use those +as well. Check the {C.bold('enframe()')} function in the source, it is very +important. + +{C.bold('idx')} is the index of the slot within its group. The {C.bold("stride")} of +a group is (generally) determined by the sizeclass as +{C.bold("UNIT * size_classes[meta.sizeclass]")}. {C.bold("start")} is the starting +address of the slot (the slot0, slot1, ... in the above diagram). +The start of a slot with index i is {C.bold("group.storage + i * stride")}. +The "nominal size" is the amount of memory the user requested with +their malloc() call, in the source it is also referred to as {C.bold("n")}. + +For every slot in a group, the memory in [start - IB, start) contains +some metadata that we will call the "start header". For this reason, +the {C.bold("end")} of a slot is calculated as {C.bold("start + stride - IB")}. The +{C.bold("slack")} of a slot is calculated as {C.bold("(stride - n - IB) / UNIT")} and +describes the amount of unused memory within a slot. + +To prevent double-frees and exploitation attempts, the mallocng +allocator performs "cycling" i.e. the actual start of user data +(the pointer returned by malloc) can be at some offset from the +{C.bold("start")} of the slot. The start of user data is called {C.bold("p")} and it +is also UNIT aligned. We will call the distance between {C.bold("p")} and +{C.bold("start")} the "cyclic offset" ({C.bold("off")} in code). When calculating +the cyclic offset, mallocng ensures {C.bold("off <= slack")}. + +If a slot is in fact cycled, then that is stored in the start +header as {C.bold("off = *(uint16_t*)(start-2)")} and {C.bold("start[-3] = 7 << 5")}. +The {C.bold("start[-3]")} field acts as a flag. + +For every slot, the memory in [p - IB, p) contains some metadata. +We will call this the "p header". If the slot is not cycled i.e. +{C.bold("start == p")}, then [start - IB, start) will contain the p header +fields and start[-3] >> 5 will *not* be 7. + +The value in {C.bold("*(uint16_t*)(p-2)")} is the {C.bold("offset")} from the slot's +{C.bold("start")} to the start of the group (divided by UNIT). The value +in {C.bold("p[-4]")} is either 0 or 1 and describes if a "big offset" should +be used. It is usually zero and gets set to one only in some cases +in aligned_alloc(). If it is 1, the offset is to be calculated as +{C.bold("*(uint32_t *)(p - 8)")}. + +{C.bold("p[-3]")} contains multiple pieces of information. If {C.bold("p[-3] == 0xFF")} +the slot is freed. Otherwise, the lower 5 bits of p[-3] describe +the index of the slot in its group: {C.bold("idx = p[-3] & 31")}. The top +3 bits desribed the {C.bold("reserved")} area size. This is the memory +between the end of user memory and {C.bold("end")} i.e. {C.bold("reserved = end - p - n")}. + +We will call the value {C.bold("p[-3] >> 5")}, "hdr reserved" for "reserved as +specified in the p header". It can happen however, that the value +{C.bold("reserved = end - p - n")} is large and so doesn't fit in the three +bits in p[-3]. In this case "hdr reserved" will be strictly 5, which +denotes that we need to look at the slot's footer to read the actual +value of {C.bold("reserved")}. As a special case, if {C.bold("p[-3] >> 5 == 6")} that +doesn't describe the reserved size at all, but specifies that there +is a group nested inside this slot. {C.bold("p[-3] >> 5")} should never be 7, +contrary to {C.bold("start[-3] >> 5")}. + +The "footer" of a slot is the third and final area of a slot's +memory where metadata is contained. This is the [end - 4, end) +area. It only contains the reserved size as +{C.bold("reserved = *(const uint32_t *)(end-4)")} when {C.bold("p[-3] >> 5 == 5")}. + +All of the above is only generally true for allocated slots. Mallocng +ensures {C.bold("p[-3] = 0xFF")} and {C.bold("*(uint16_t *)(p - 2) = 0")} for freed slots, +which makes the start of the slot's group (and thus meta) unreachable. +Only in this case does {C.bold("p[-3] >> 5")} become 7. Available slots, +i.e. those that haven't been allocated nor freed yet (but are ready +for allocation), have almost no guarantees on their data and +metadata contents. +""" + + print(txt) diff --git a/tests/library/dbg/tests/test_mallocng.py b/tests/library/dbg/tests/test_mallocng.py index a47a25676..82be2df8c 100644 --- a/tests/library/dbg/tests/test_mallocng.py +++ b/tests/library/dbg/tests/test_mallocng.py @@ -5,10 +5,6 @@ from typing import List import pytest -import pwndbg -import pwndbg.color as color -import pwndbg.dbg - from ....host import Controller from . import break_at_sym from . import get_binary @@ -27,6 +23,8 @@ re_addr = r"0x[0-9a-fA-F]{1,12}" "binary", [HEAP_MALLOCNG_DYN, HEAP_MALLOCNG_STATIC], ids=["dynamic", "static"] ) async def test_mallocng_slot_user(ctrl: Controller, binary: str): + import pwndbg.color as color + await launch_to(ctrl, binary, "break_here") # Get out of the break_here() function. await ctrl.finish() @@ -177,6 +175,8 @@ async def test_mallocng_slot_user(ctrl: Controller, binary: str): "binary", [HEAP_MALLOCNG_DYN, HEAP_MALLOCNG_STATIC], ids=["dynamic", "static"] ) async def test_mallocng_slot_start(ctrl: Controller, binary: str): + import pwndbg.color as color + await launch_to(ctrl, binary, "break_here") await ctrl.finish() @@ -203,6 +203,8 @@ async def test_mallocng_slot_start(ctrl: Controller, binary: str): "binary", [HEAP_MALLOCNG_DYN, HEAP_MALLOCNG_STATIC], ids=["dynamic", "static"] ) async def test_mallocng_group(ctrl: Controller, binary: str): + import pwndbg.color as color + await launch_to(ctrl, binary, "break_here") await ctrl.finish() @@ -272,6 +274,8 @@ async def test_mallocng_group(ctrl: Controller, binary: str): "binary", [HEAP_MALLOCNG_DYN, HEAP_MALLOCNG_STATIC], ids=["dynamic", "static"] ) async def test_mallocng_meta(ctrl: Controller, binary: str): + import pwndbg.color as color + await launch_to(ctrl, binary, "break_here") await ctrl.finish() @@ -292,7 +296,9 @@ async def test_mallocng_meta(ctrl: Controller, binary: str): "binary", [HEAP_MALLOCNG_DYN, HEAP_MALLOCNG_STATIC], ids=["dynamic", "static"] ) async def test_mallocng_malloc_context(ctrl: Controller, binary: str): - await launch_to(ctrl, binary, "main") + import pwndbg.color as color + + await ctrl.launch(binary) # Check that we do not find it at the first program instruction if binary == HEAP_MALLOCNG_DYN: @@ -300,7 +306,6 @@ async def test_mallocng_malloc_context(ctrl: Controller, binary: str): # __malloc_context by simply looking up the symbol. So we only # check this for the dynamically linked binary. - await ctrl.execute("starti") # This is at _dlstart - the heap is uninitialized at this point. ctx_out = color.strip(await ctrl.execute_and_capture("ng-ctx")) @@ -330,6 +335,9 @@ async def test_mallocng_malloc_context(ctrl: Controller, binary: str): "binary", [HEAP_MALLOCNG_DYN, HEAP_MALLOCNG_STATIC], ids=["dynamic", "static"] ) async def test_mallocng_find(ctrl: Controller, binary: str): + import pwndbg + import pwndbg.color as color + await launch_to(ctrl, binary, "break_here") await ctrl.finish() @@ -374,6 +382,8 @@ async def test_mallocng_find(ctrl: Controller, binary: str): "binary", [HEAP_MALLOCNG_DYN, HEAP_MALLOCNG_STATIC], ids=["dynamic", "static"] ) async def test_mallocng_metaarea(ctrl: Controller, binary: str): + import pwndbg.color as color + await launch_to(ctrl, binary, "break_here") await ctrl.finish() @@ -398,3 +408,76 @@ async def test_mallocng_metaarea(ctrl: Controller, binary: str): for i in range(len(expected_out)): assert re.match(expected_out[i], meta_area_out[i]) + + +@pwndbg_test +@pytest.mark.parametrize( + "binary", [HEAP_MALLOCNG_DYN, HEAP_MALLOCNG_STATIC], ids=["dynamic", "static"] +) +async def test_mallocng_vis(ctrl: Controller, binary: str): + import pwndbg.color as color + + await launch_to(ctrl, binary, "break_here") + + break_at_sym("break_here") + await ctrl.cont() + await ctrl.cont() + await ctrl.finish() + + vis_out = color.strip(await ctrl.execute_and_capture("ng-vis buffer1")).splitlines() + + expected_out = [ + f"group @ {re_addr}", + f"meta @ {re_addr}", + "LEGEND: .*", + "LEGEND: .*", + "", + rf"{re_addr}0\t0x[0-9a-fA-F]{{16}}\t0x0000ff0000000009\t................", + rf"{re_addr}0\t0x0a0a0a0a0a0a0a0a\t0x0a0a0a0a0a0a0a0a\t................", + rf"{re_addr}0\t0x0a0a0a0a0a0a0a0a\t0x0a0a0a0a0a0a0a0a\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000ff000000000c\t................", + rf"{re_addr}0\t0x0b0b0b0b0b0b0b0b\t0x0b0b0b0b0b0b0b0b\t................", + rf"{re_addr}0\t0x0b0b0b0b0b0b0b0b\t0x0b0b0b0b0b0b0b0b\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0006a2000000000c\t................ 2 \+ \(5 << 5\)", + rf"{re_addr}0\t0x0c0c0c0c0c0c0c0c\t0x0c0c0c0c0c0c0c0c\t................", + rf"{re_addr}0\t0x0c0c0c0c0c0c0c0c\t0x0c0c0c0c0c0c0c0c\t................", + rf"{re_addr}0\t0x0000000000000000\t0x000000000000000c\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + rf"{re_addr}0\t0x0000000000000000\t0x0000000000000000\t................", + ] + + assert len(expected_out) == len(vis_out) + + for i in range(len(expected_out)): + assert re.match(expected_out[i], vis_out[i]) + + # Make sure ng-vis properly resolves anywhere inside the slot. + # The stride of the group is 0x30. + vis_out2 = color.strip(await ctrl.execute_and_capture("ng-vis buffer1+0x2F")).splitlines() + assert vis_out == vis_out2 + + # Step over the free(buffer3) + await ctrl.execute("next") + # Check that the output is not the same anymore since the group got freed. + # (Now the outer group will be printed.) + vis_out3 = color.strip(await ctrl.execute_and_capture("ng-vis buffer1")).splitlines() + assert len(vis_out3) > len(vis_out)