improved glibc 2.42 support (#3464)

* improved glibc 2.42 support

* fixed glibc < 2.42 bug

* removed uneeded import

* removed unused variable `e`

* readded default TCACHE_FILL_COUNT

* added comment for GLIBC 2.42 related changes

* started adding tests for GLIBC 2.42

* fixed spacing for linter

* skiping all gdb tests for GLIBC 2.42 for now

* added GLIBC 2.42 support to ptmalloc2.py

* started adding ubuntu 26.04 tests

* added skips for GLIBC 2.42

* added missing import

* more skipped tests in try_free

* rerun try_free tests for 2.42

* downgraded pytest-rerunfailures to 15.0

* added pytest-rerunfailures to pyenv.nix

* removed all testing related changes

* removed heap_bugs hyperlink

* improved glibc 2.42 implementation

+ new tcache_small_bins property
+ updated tcache_perthread_struct
+ updated malloc_par struct

* tcache_small_bins now "caches" self.mp locally

* reduced redundant tcache_small_bins

* fixed heap heuristic for glibc 2.42

* fixed thread_cache for glibc 2.42

* removed duplicate tcache symbol from binary

* improved tcache symbol lookup

* changed objfile_endswith search name

* hopefully fixed mypy error

* hopefully fixed mypy error

* fixed Exception in thread_cache

Co-authored-by: Disconnect3d <dominik.b.czarnota@gmail.com>

* implemented feedback

* implemented some more feedback

* implemented correct feedback

* enforce bool existence again in multithreaded()

* fixed linting ...

* reversed restrictive lookup_symbol_addr

---------

Co-authored-by: Disconnect3d <dominik.b.czarnota@gmail.com>

Dockerfile

@@ -24,7 +24,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
         locales vim && \
-    localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 && \
+    localedef -i en_US -c -f UTF-8 en_US.UTF-8 && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # setup.sh needs scripts/common.sh

pwndbg/aglib/heap/ptmalloc.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import copy
 import importlib
+import os
 import sys
 import types
 from collections import OrderedDict
@@ -1104,6 +1105,18 @@ class GlibcMemoryAllocator(pwndbg.aglib.heap.heap.MemoryAllocator, Generic[TheTy
     def tcache_entry(self) -> TheType | None:
         raise NotImplementedError()
 
+    @property
+    @pwndbg.lib.cache.cache_until("objfile")
+    def tcache_small_bins(self) -> int | None:
+        if not self.has_tcache():
+            return None
+        mp = self.mp
+        if "tcache_small_bins" in mp.type.keys():
+            return int(mp["tcache_small_bins"])
+        elif "tcache_bins" in mp.type.keys():
+            return int(mp["tcache_bins"])
+        return None
+
     @property
     def mallinfo(self) -> TheType | None:
         raise NotImplementedError()
@@ -1156,7 +1169,10 @@ class GlibcMemoryAllocator(pwndbg.aglib.heap.heap.MemoryAllocator, Generic[TheTy
         """Is malloc operating within a multithreaded environment."""
         addr = pwndbg.aglib.symbol.lookup_symbol_addr("__libc_multiple_threads")
         if addr:
-            return pwndbg.aglib.memory.s32(addr) > 0
+            return pwndbg.aglib.memory.u32(addr) > 0
+        # glibc 2.42 replaced __libc_multiple_threads with __libc_single_threaded
+        elif addr := pwndbg.aglib.symbol.lookup_symbol_addr("__libc_single_threaded"):
+            return pwndbg.aglib.memory.u32(addr) == 0
         return len(pwndbg.dbg.selected_inferior().threads()) > 1
 
     def _request2size(self, req: int) -> int:
@@ -1252,8 +1268,11 @@ class GlibcMemoryAllocator(pwndbg.aglib.heap.heap.MemoryAllocator, Generic[TheTy
         if tcache is None:
             return None
 
-        if pwndbg.glibc.get_version() >= (2, 42) and not hasattr(
+        # this will break expected output during tests, so we skip it
-            GlibcMemoryAllocator.tcachebins, "tcache_2_42_warning_issued"
+        if (
+            pwndbg.glibc.get_version() >= (2, 42)
+            and not not hasattr(GlibcMemoryAllocator.tcachebins, "tcache_2_42_warning_issued")
+            and os.environ.get("PWNDBG_IN_TEST") is None
         ):
             print(
                 message.warn(
@@ -1263,7 +1282,11 @@ class GlibcMemoryAllocator(pwndbg.aglib.heap.heap.MemoryAllocator, Generic[TheTy
             )
             setattr(GlibcMemoryAllocator.tcachebins, "tcache_2_42_warning_issued", True)
 
-        counts = tcache["counts"]
+        # counts was renamed to num_slots in newer version of GLIBC 2.42
+        try:
+            counts = tcache["num_slots"]
+        except Exception:
+            counts = tcache["counts"]
         entries = tcache["entries"]
 
         num_tcachebins = entries.type.sizeof // entries.type.target().sizeof
@@ -1278,7 +1301,7 @@ class GlibcMemoryAllocator(pwndbg.aglib.heap.heap.MemoryAllocator, Generic[TheTy
             size = self._request2size(tidx2usize(i))
             count = int(counts[i])
             if pwndbg.glibc.get_version() >= (2, 42):
-                count = pwndbg.aglib.heap.structs.TCACHE_FILL_COUNT - count
+                count = int(self.mp["tcache_count"]) - count
             chain = pwndbg.chain.get(
                 int(entries[i]),
                 offset=self.tcache_next_offset,
@@ -1576,7 +1599,10 @@ class DebugSymsHeap(GlibcMemoryAllocator[pwndbg.dbg_mod.Type, pwndbg.dbg_mod.Val
         return self._main_arena
 
     def has_tcache(self) -> bool:
-        return self.mp is not None and "tcache_bins" in self.mp.type.keys()
+        # tcache_bins was renamed to tcache_small_bins in GLIBC 2.42
+        return self.mp is not None and any(
+            x in self.mp.type.keys() for x in ["tcache_bins", "tcache_small_bins"]
+        )
 
     @property
     def thread_arena(self) -> Arena | None:
@@ -1598,36 +1624,40 @@ class DebugSymsHeap(GlibcMemoryAllocator[pwndbg.dbg_mod.Type, pwndbg.dbg_mod.Val
         """Locate a thread's tcache struct. If it doesn't have one, use the main
         thread's tcache.
         """
-        if self.has_tcache():
+        if not self.has_tcache():
-            if self.multithreaded:
+            print(message.warn("This version of GLIBC was not compiled with tcache support."))
-                tcache_addr = pwndbg.aglib.memory.read_pointer_width(
+            return None
-                    pwndbg.aglib.symbol.lookup_symbol_addr("tcache", prefer_static=True)
-                )
-                if tcache_addr == 0:
-                    # This thread doesn't have a tcache yet
-                    return None
-                tcache = tcache_addr
-            else:
-                tcache = self.main_arena.heaps[0].start + pwndbg.aglib.arch.ptrsize * 2
 
-            try:
+        tcache_ptr = pwndbg.aglib.symbol.lookup_symbol_addr(
-                self._thread_cache = pwndbg.aglib.memory.get_typed_pointer_value(
+            "tcache",
-                    self.tcache_perthread_struct, tcache
+            prefer_static=True,
-                )
+        )
-                self._thread_cache["entries"].fetch_lazy()
+        if not tcache_ptr:
-            except Exception:
+            tcache_ptr = pwndbg.aglib.symbol.lookup_symbol_addr("tcache", prefer_static=True)
-                print(
-                    message.error(
-                        "Error fetching tcache. GDB cannot access "
-                        "thread-local variables unless you compile with -lpthread."
-                    )
-                )
-                return None
 
-            return self._thread_cache
+        if tcache_ptr and (tcache_addr := pwndbg.aglib.memory.read_pointer_width(tcache_ptr)):
+            tcache = tcache_addr
+        elif not self.multithreaded:
+            tcache = self.main_arena.heaps[0].start + pwndbg.aglib.arch.ptrsize * 2
+        else:
+            # This thread doesn't have a tcache yet
+            return None
 
-        print(message.warn("This version of GLIBC was not compiled with tcache support."))
+        try:
-        return None
+            self._thread_cache = pwndbg.aglib.memory.get_typed_pointer_value(
+                self.tcache_perthread_struct, tcache
+            )
+            self._thread_cache["entries"].fetch_lazy()
+        except Exception:
+            print(
+                message.error(
+                    "Error fetching tcache. Cannot access "
+                    "thread-local variables unless you compile with -lpthread."
+                )
+            )
+            return None
+
+        return self._thread_cache
 
     @property
     def mp(self) -> pwndbg.dbg_mod.Value | None:
@@ -1721,7 +1751,9 @@ class DebugSymsHeap(GlibcMemoryAllocator[pwndbg.dbg_mod.Type, pwndbg.dbg_mod.Val
         addr = pwndbg.aglib.symbol.lookup_symbol_addr("__libc_malloc_initialized")
         if addr is None:
             addr = pwndbg.aglib.symbol.lookup_symbol_addr("__malloc_initialized")
-        assert addr is not None, "Could not find __libc_malloc_initialized or __malloc_initialized"
+        # fallback for GLIBC 2.42 as __malloc_initialized was removed
+        if addr is None:
+            return int(self.mp["sbrk_base"]) != 0
         return pwndbg.aglib.memory.s32(addr) > 0
 
 

pwndbg/aglib/heap/structs.py

@@ -63,6 +63,7 @@ DEFAULT_MMAP_THRESHOLD = 128 * 1024
 DEFAULT_TRIM_THRESHOLD = 128 * 1024
 DEFAULT_PAGE_SIZE = 4096
 TCACHE_FILL_COUNT = 7
+MAX_TCACHE_SMALL_SIZE = (TCACHE_SMALL_BINS - 1) * MALLOC_ALIGN + MINSIZE - SIZE_SZ
 
 
 class c_pvoid(PTR):
@@ -585,8 +586,8 @@ class c_tcache_perthread_struct_2_42(Structure):
     """
 
     _fields_ = [
-        ("counts", ctypes.c_uint16 * TCACHE_MAX_BINS),
+        ("num_slots", ctypes.c_uint16 * TCACHE_MAX_BINS),
-        ("entries", c_pvoid * TCACHE_SMALL_BINS),
+        ("entries", c_pvoid * TCACHE_MAX_BINS),
     ]
 
 
@@ -949,12 +950,88 @@ class c_malloc_par_2_35(Structure):
     ]
 
 
+class c_malloc_par_2_42(Structure):
+    """
+    This class represents the malloc_par struct for GLIBC >= 2.42 as a ctypes struct.
+
+    https://elixir.bootlin.com/glibc/glibc-2.42/source/malloc/malloc.c#L1864
+
+    struct malloc_par
+    {
+      /* Tunable parameters */
+      unsigned long trim_threshold;
+      INTERNAL_SIZE_T top_pad;
+      INTERNAL_SIZE_T mmap_threshold;
+      INTERNAL_SIZE_T arena_test;
+      INTERNAL_SIZE_T arena_max;
+
+      /* Transparent Large Page support.  */
+      INTERNAL_SIZE_T thp_pagesize;
+      /* A value different than 0 means to align mmap allocation to hp_pagesize
+        add hp_flags on flags.  */
+      INTERNAL_SIZE_T hp_pagesize;
+      int hp_flags;
+
+      /* Memory map support */
+      int n_mmaps;
+      int n_mmaps_max;
+      int max_n_mmaps;
+      /* the mmap_threshold is dynamic, until the user sets
+        it manually, at which point we need to disable any
+        dynamic behavior. */
+      int no_dyn_threshold;
+
+      /* Statistics */
+      INTERNAL_SIZE_T mmapped_mem;
+      INTERNAL_SIZE_T max_mmapped_mem;
+
+      /* First address handed out by MORECORE/sbrk.  */
+      char *sbrk_base;
+
+    #if USE_TCACHE
+      /* Maximum number of small buckets to use.  */
+      size_t tcache_small_bins;
+      size_t tcache_max_bytes;
+      /* Maximum number of chunks in each bucket.  */
+      size_t tcache_count;
+      /* Maximum number of chunks to remove from the unsorted list, which
+        aren't used to prefill the cache.  */
+      size_t tcache_unsorted_limit;
+    #endif
+    };
+    """
+
+    _fields_ = [
+        ("trim_threshold", c_size_t),
+        ("top_pad", c_size_t),
+        ("mmap_threshold", c_size_t),
+        ("arena_test", c_size_t),
+        ("arena_max", c_size_t),
+        ("thp_pagesize", c_size_t),
+        ("hp_pagesize", c_size_t),
+        ("hp_flags", ctypes.c_int32),
+        ("n_mmaps", ctypes.c_int32),
+        ("n_mmaps_max", ctypes.c_int32),
+        ("max_n_mmaps", ctypes.c_int32),
+        ("no_dyn_threshold", ctypes.c_int32),
+        ("mmapped_mem", c_size_t),
+        ("max_mmapped_mem", c_size_t),
+        ("sbrk_base", c_pvoid),
+        ("tcache_small_bins", c_size_t),
+        ("tcache_max_bytes", c_size_t),
+        ("tcache_count", c_size_t),
+        ("tcache_unsorted_limit", c_size_t),
+    ]
+
+
 class MallocPar(CStruct2GDB):
     """
     This class represents the malloc_par struct with interface compatible with `pwndbg.dbg_mod.Value`.
     """
 
-    if GLIBC_VERSION >= (2, 35):
+    if GLIBC_VERSION >= (2, 42):
+        _c_struct = c_malloc_par_2_42
+    elif GLIBC_VERSION >= (2, 35):
         _c_struct = c_malloc_par_2_35
     elif GLIBC_VERSION >= (2, 26):
         _c_struct = c_malloc_par_2_26
@@ -993,7 +1070,14 @@ DEFAULT_MP_.arena_test = 2 if pwndbg.aglib.arch.ptrsize == 4 else 8
 if (MallocPar._c_struct != c_malloc_par_2_23) and (MallocPar._c_struct != c_malloc_par_2_12):
     # the only difference between 2.23 and the rest is the lack of tcache
     DEFAULT_MP_.tcache_count = TCACHE_FILL_COUNT
-    DEFAULT_MP_.tcache_bins = TCACHE_SMALL_BINS
+    if MallocPar._c_struct == c_malloc_par_2_42:
-    DEFAULT_MP_.tcache_max_bytes = (TCACHE_SMALL_BINS - 1) * MALLOC_ALIGN + MINSIZE - SIZE_SZ
+        DEFAULT_MP_.tcache_small_bins = TCACHE_SMALL_BINS
+        DEFAULT_MP_.tcache_max_bytes = (
+            MAX_TCACHE_SMALL_SIZE + SIZE_SZ + MALLOC_ALIGN_MASK
+        ) & ~MALLOC_ALIGN_MASK + 1
+
+    else:
+        DEFAULT_MP_.tcache_bins = TCACHE_SMALL_BINS
+        DEFAULT_MP_.tcache_max_bytes = MAX_TCACHE_SMALL_SIZE
 if MallocPar._c_struct == c_malloc_par_2_12:
     DEFAULT_MP_.pagesize = DEFAULT_PAGE_SIZE

pwndbg/commands/ptmalloc2.py

@@ -1337,7 +1337,7 @@ def try_free(addr: str | int) -> None:
         and "key" in allocator.tcache_entry.keys()
     ):
         tc_idx = (chunk_size_unmasked - chunk_minsize + malloc_alignment - 1) // malloc_alignment
-        if allocator.mp is not None and tc_idx < int(allocator.mp["tcache_bins"]):
+        if allocator.mp is not None and tc_idx < allocator.tcache_small_bins:
             print(message.notice("Tcache checks"))
             e = addr + 2 * size_sz
             e += allocator.tcache_entry.keys().index("key") * ptr_size
@@ -1354,7 +1354,12 @@ def try_free(addr: str | int) -> None:
 
             # May be an array, and tc_idx may be negative, so always cast to a
             # pointer before we index into it.
-            counts = allocator.get_tcache()["counts"]
+            # counts was renamed to num_slots in newer version of GLIBC 2.42
+            tcache = allocator.get_tcache()
+            try:
+                counts = tcache["num_slots"]
+            except Exception:
+                counts = tcache["counts"]
             if int(counts.address.cast(counts.type.target().pointer())[tc_idx]) < int(
                 allocator.mp["tcache_count"]
             ):

tests/binaries/host/heap_bins.native.c

@@ -24,7 +24,7 @@ const size_t largebin_size = LARGEBIN_SIZE;
 const size_t largebin_count = LARGEBIN_COUNT;
 
 int break_id = 0;
-void *tcache[TCACHE_COUNT];
+void *tcachebin[TCACHE_COUNT];
 void *fastbin[FASTBIN_COUNT];
 void *smallbin[SMALLBIN_COUNT + TCACHE_COUNT];
 void *largebin[LARGEBIN_COUNT];
@@ -39,7 +39,7 @@ void alloc_chunks()
 {
     void *padding;
     for (int i = 0; i < TCACHE_COUNT; i++)
-        tcache[i] = malloc(TCACHE_SIZE);
+        tcachebin[i] = malloc(TCACHE_SIZE);
     for (int i = 0; i < FASTBIN_COUNT; i++)
         fastbin[i] = malloc(FASTBIN_SIZE);
     for (int i = 0; i < SMALLBIN_COUNT + TCACHE_COUNT; i++)
@@ -61,7 +61,7 @@ void alloc_chunks()
 void tcache_test()
 {
     for (int i = 0; i < TCACHE_COUNT; i++)
-        free(tcache[i]);
+        free(tcachebin[i]);
     breakpoint();
     return;
 }