Integrate Debugger-agnostic tests into the test pipelines (#3215)

* Add dbg tests to test suite

* Fix 0

* Check for OSError in OpportunisticTerminalControl

* Split tests tasks

* Fix go tests in LLDB

* Update TLS tests to handle LLDB failing to resolve %GS on LLDB

* Disable go-based x86 test for windbg commands on LLDB

* Fix listening to new modules being loaded in LLDB

* Force LLDB tests to run in series

Parallel execution is broken, anyway

* Fix mallocng tests in LLDB

* ptmalloc2: Always cast tcache counts to pointer during try-free

* Catch LLDB_INVALID_ADDRESS in LLDB symbol lookup

* Handle the binary formatting from LLDB in `test_vis_heap_chunks`

* Split GDB and DBG GDB Nix tests, remove DBG LLDB Nix tests

* Replace ParamSpec in type param list with explicit use

* Add mising dependencies in Ubuntu test targets

* Revert "Add mising dependencies in Ubuntu test targets"

This reverts commit bd56a6b9dc.

* Disable -fcf-protection in test binaries

* Disable LLDB tests on Ubuntu 22.04

We don't seem to even officially support it for pwndbg-lldb

.github/workflows/docker.yml

@@ -29,5 +29,15 @@ jobs:
     - name: Docker Build ${{ matrix.images }}
       run: docker compose build ${{ matrix.images }}
 
-    - name: Test on ${{ matrix.images }}
-      run: docker compose run ${{ matrix.images }} ./tests.sh -d gdb -g gdb
+    - name: Run GDB Tests on ${{ matrix.images }}
+      run: |
+        docker compose run ${{ matrix.images }} ./tests.sh -d gdb -g gdb
+    
+    - name: Run DBG Tests on GDB on ${{ matrix.images }}
+      run: |
+        docker compose run ${{ matrix.images }} ./tests.sh -d gdb -g dbg
+
+    - name: Run DBG Tests on LLDB on ${{ matrix.images }}
+      run: |
+        docker compose run ${{ matrix.images }} ./tests.sh -d lldb -g dbg
+        

.github/workflows/tests.yml

@@ -52,11 +52,16 @@ jobs:
       run: |
         ./unit-tests.sh
 
-    - name: Run tests
+    - name: Run GDB Tests
       if: matrix.type == 'tests'
       run: |
         ./tests.sh --nix -d gdb -g gdb
 
+    - name: Run DBG Tests on GDB
+      if: matrix.type == 'tests'
+      run: |
+        ./tests.sh --nix -d gdb -g dbg
+
     - name: Run cross-arch tests
       if: matrix.type == 'qemu-user-tests'
       run: |
@@ -112,10 +117,21 @@ jobs:
         ./.venv/bin/pip freeze
 
     # We set `kernel.yama.ptrace_scope=0` for `attachp` command tests
-    - name: Run tests
+    - name: Run GDB Tests
       run: |
         sudo sysctl -w kernel.yama.ptrace_scope=0
         ./tests.sh -d gdb -g gdb
+  
+    - name: Run DBG Tests on GDB
+      run: |
+        sudo sysctl -w kernel.yama.ptrace_scope=0
+        ./tests.sh -d gdb -g dbg
+    
+    - name: Run DBG Tests on LLDB
+      if: matrix.os != 'ubuntu-22.04'
+      run: |
+        sudo sysctl -w kernel.yama.ptrace_scope=0
+        ./tests.sh -d lldb -g dbg
 
   qemu-user-tests:
     runs-on: [ubuntu-24.04]

pwndbg/commands/ptmalloc2.py

@@ -1352,7 +1352,12 @@ def try_free(addr: str | int) -> None:
                 )
                 errors_found += 1
 
-            if int(allocator.get_tcache()["counts"][tc_idx]) < int(allocator.mp["tcache_count"]):
+            # May be an array, and tc_idx may be negative, so always cast to a
+            # pointer before we index into it.
+            counts = allocator.get_tcache()["counts"]
+            if int(counts.address.cast(counts.type.target().pointer())[tc_idx]) < int(
+                allocator.mp["tcache_count"]
+            ):
                 print(message.success("Using tcache_put"))
                 if errors_found == 0:
                     returned_before_error = True

pwndbg/dbg/lldb/__init__.py

@@ -1249,6 +1249,12 @@ class LLDBProcess(pwndbg.dbg_mod.Process):
             return None
 
         sym_addr = ctx.symbol.addr.GetLoadAddress(self.target)
+        if sym_addr == lldb.LLDB_INVALID_ADDRESS:
+            # Unlikely. LLDB can return some truly insane matches sometimes. In
+            # this case, we could not find the load address of the symbol we
+            # matched. Act as if we found nothing.
+            return None
+
         assert (
             sym_addr <= address
         ), f"LLDB returned an out-of-range address {sym_addr:#x} for a requested symbol with address {address:#x}"

pwndbg/dbg/lldb/repl/io.py

@@ -66,7 +66,7 @@ class OpportunisticTerminalControl:
         if fd == -1:
             try:
                 fd = os.open("/dev/tty", os.O_RDWR)
-            except (FileNotFoundError, PermissionError):
+            except (FileNotFoundError, PermissionError, OSError):
                 # Flop and die.
                 self.supported = False
                 return
@@ -403,7 +403,7 @@ class IODriverPseudoTerminal(IODriver):
                     termios.tcsetwinsize(self.manager, size)  # novm
 
                 signal.signal(signal.SIGWINCH, handle_sigwinch)
-            except FileNotFoundError:
+            except (FileNotFoundError, PermissionError, OSError):
                 print(
                     "warning: no terminal device in /dev/tty, expect no support for terminal sizes"
                 )

pwndbg/dbg/lldb/repl/proc.py

@@ -537,9 +537,8 @@ class ProcessDriver:
         self.listener = lldb.SBListener("pwndbg.dbg.lldb.repl.proc.ProcessDriver")
         assert self.listener.IsValid()
 
-        self.listener.StartListeningForEventClass(
-            target.GetDebugger(),
-            lldb.SBTarget.GetBroadcasterClassName(),
+        self.listener.StartListeningForEvents(
+            target.broadcaster,
             lldb.SBTarget.eBroadcastBitModulesLoaded,
         )
 

tests/binaries/host/makefile

@@ -31,6 +31,9 @@ else
 CFLAGS         +=      -O1
 endif
 
+# LLDB does not support PLT symbolication with -fcf-protection :(
+CFLAGS         += -fcf-protection=none
+
 PWD=$(shell pwd)
 # Apparently we don't have this version? :(
 #GLIBC=/glibc_versions/2.29/tcache_x64

tests/library/dbg/tests/__init__.py

@@ -7,14 +7,17 @@ from typing import Any
 from typing import Callable
 from typing import Concatenate
 from typing import Coroutine
+from typing import ParamSpec
 
 from .... import host
 from ....host import Controller
 
 BINARIES_PATH = os.environ.get("TEST_BINARIES_ROOT")
 
+T = ParamSpec("T")
 
-def pwndbg_test[**T](
+
+def pwndbg_test(
     test: Callable[Concatenate[Controller, T], Coroutine[Any, Any, None]],
 ) -> Callable[T, None]:
     @functools.wraps(test)

tests/library/dbg/tests/heap/test_vis_heap_chunks.py

@@ -42,8 +42,8 @@ async def test_vis_heap_chunk_command(ctrl: Controller) -> None:
         from pwndbg.commands.ptmalloc2 import bin_ascii
 
         first, second = (await ctrl.execute_and_capture(f"x/16xb {gdb_symbol}")).splitlines()
-        first = [int(v, 16) for v in first.split(":")[1].split("\t")[1:]]
-        second = [int(v, 16) for v in second.split(":")[1].split("\t")[1:]]
+        first = [int(v, 16) for v in first.split(":")[1].split()]
+        second = [int(v, 16) for v in second.split(":")[1].split()]
 
         return bin_ascii(first + second)
 

tests/library/dbg/tests/test_command_tls.py

@@ -17,6 +17,11 @@ TLS_I386_BINARY = get_binary("tls.i386.out")
 async def test_tls_address_and_command(ctrl: Controller, binary: str):
     import pwndbg.aglib.tls
     import pwndbg.aglib.vmmap
+    from pwndbg.dbg import DebuggerType
+
+    if pwndbg.dbg.name() == DebuggerType.LLDB and binary == TLS_I386_BINARY:
+        pytest.skip("TLS commands are flaky in LLDB on i386")
+        return
 
     await launch_to(ctrl, binary, "break_here")
 

tests/library/dbg/tests/test_go.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import pytest
+
 from ....host import Controller
 from . import get_binary
 from . import pwndbg_test

tests/library/dbg/tests/test_windbg.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import re
 
+import pytest
+
 from ....host import Controller
 from . import get_binary
 from . import pwndbg_test
@@ -323,6 +325,13 @@ async def test_windbg_commands_x86(ctrl: Controller) -> None:
     like dq, dw, db, ds etc.
     """
     import pwndbg
+    from pwndbg.dbg import DebuggerType
+
+    if pwndbg.dbg.name() == DebuggerType.LLDB:
+        pytest.skip(
+            "LLDB does not properly support Go, and fails to resolve expressions such as `$esp`"
+        )
+        return
 
     await ctrl.launch(X86_BINARY)
 

tests/tests.py

@@ -48,12 +48,20 @@ def main():
         )
         sys.exit(1)
 
+    force_serial = False
     match args.driver:
         case Driver.GDB:
             host = get_gdb_host(args, local_pwndbg_root)
         case Driver.LLDB:
             host = get_lldb_host(args, local_pwndbg_root)
 
+            # LLDB does not properly support having its tests run in parallel,
+            # so we forcibly disable it, for now.
+            print(
+                "WARNING: LLDB tests always run in series, even when parallel execution is requested."
+            )
+            force_serial = True
+
     # Handle the case in which the user only wants the collection to run.
     if args.collect_only:
         for test in host.collect():
@@ -62,7 +70,12 @@ def main():
 
     # Actually run the tests.
     run_tests_and_print_stats(
-        host, args.test_name_filter, args.pdb, args.serial, args.verbose, coverage_out
+        host,
+        args.test_name_filter,
+        args.pdb,
+        force_serial or args.serial,
+        args.verbose,
+        coverage_out,
     )