Add nix tests (#2557)

* Fix terminal width to 80 columns in qemu-tests - same as PR #2444

* Add the ability to run tests with nix on ubuntu

* setup-dev.sh: add --install-only option

* bump poetry.lock

* workflows: add nix tests on ubuntu

* workflows: enable --nix flag in tests

* workflows: enable pwndbg-dev in tests

* setup-dev: install libc 32bit dbg symbols

* workflows: bump timeout 40min

* tests: fix link

* steup-dev: fix lint

* tests: fix tmpdir

.github/workflows/tests.yml

@@ -15,6 +15,65 @@ on:
       - '!docs/**'
       - '!*.md'
 jobs:
+  tests-using-nix:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04]
+        type: [qemu-user-tests, qemu-tests, tests]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 40
+    env:
+      TMPDIR: /tmp
+    steps:
+    - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9  # @v3
+    - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72  # @v30
+      with:
+        nix_path: nixpkgs=channel:nixos-unstable
+
+    - name: Install dependencies
+      run: |
+        ./setup-dev.sh --install-only
+
+    - name: Build pwndbg
+      run: |
+        nix build '.#pwndbg-dev' --accept-flake-config -o result
+
+    # We set `kernel.yama.ptrace_scope=0` for `attachp` command tests
+    - name: Setup
+      run: |
+        sudo sysctl -w kernel.yama.ptrace_scope=0
+        mkdir .cov
+
+    - name: Run tests
+      if: matrix.type == 'tests'
+      run: |
+        ./tests.sh --nix --cov
+
+    - name: Run qemu-user-tests
+      if: matrix.type == 'qemu-user-tests'
+      run: |
+        ./qemu-tests.sh --nix --cov
+
+    - name: Set up cache for QEMU images
+      if: matrix.type == 'qemu-tests'
+      id: qemu-cache
+      uses: actions/cache@v3
+      with:
+        path: ./tests/qemu-tests/images
+        key: ${{ matrix.os }}-cache-qemu-images
+
+    - name: Download QEMU images
+      if: matrix.type == 'qemu-tests'
+      run: |
+        ./tests/qemu-tests/download_images.sh
+
+    - name: Run qemu-tests
+      if: matrix.type == 'qemu-tests'
+      working-directory: ./tests/qemu-tests
+      run: |
+        ./tests.sh --nix --cov
+
   tests:
     strategy:
       fail-fast: false

pyproject.toml

@@ -173,6 +173,8 @@ typing-extensions = "^4.12.0"
 unicorn = "^2.1.1"
 requests = "^2.32.3"
 pt = {git = "https://github.com/martinradev/gdb-pt-dump", rev = "50227bda0b6332e94027f811a15879588de6d5cb"}
+# Newer versions of bcrypt break NIX. Who need bcrypt: pwntools->paramiko->bcrypt
+bcrypt="4.2.0"
 
 [tool.poetry.group.lldb]
 optional = true

setup-dev.sh

@@ -6,6 +6,29 @@ echo "# Install testing tools."
 echo "# Only works with Ubuntu / APT or Arch / Pacman."
 echo "# --------------------------------------"
 
+help_and_exit() {
+    echo "Usage: ./setup-dev.sh [--install-only]"
+    echo "  --install-only              install only distro dependencies without installing python-venv"
+    exit 1
+}
+
+USE_INSTALL_ONLY=0
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --install-only)
+            USE_INSTALL_ONLY=1
+            ;;
+        -h | --help)
+            help_and_exit
+            ;;
+        *)
+            help_and_exit
+            ;;
+    esac
+    shift
+done
+
 hook_script_path=".git/hooks/pre-push"
 hook_script=$(
     cat << 'EOF'
@@ -92,6 +115,7 @@ install_apt() {
         gcc \
         libc6-dev \
         curl \
+        wget \
         build-essential \
         gdb \
         gdb-multiarch \
@@ -107,6 +131,13 @@ install_apt() {
         gcc-mips-linux-gnu \
         gcc-mips64-linux-gnuabi64
 
+    # Some tests require i386 libc/ld, eg: test_smallbins_sizes_32bit_big
+    if uname -m | grep -q x86_64; then
+        sudo dpkg --add-architecture i386
+        sudo apt-get update
+        sudo apt-get install -y libc6-dbg:i386 libgcc-s1:i386
+    fi
+
     if [[ "$1" != "" ]]; then
         sudo apt install shfmt
     fi
@@ -147,6 +178,7 @@ EOF
         gcc \
         glibc-debug \
         curl \
+        wget \
         base-devel \
         gdb \
         parallel
@@ -169,6 +201,7 @@ install_dnf() {
         nasm \
         gcc \
         curl \
+        wget \
         gdb \
         parallel \
         qemu-system-arm \
@@ -210,6 +243,21 @@ install_jemalloc() {
 
 }
 
+configure_venv() {
+    if [[ -z "${PWNDBG_VENV_PATH}" ]]; then
+        PWNDBG_VENV_PATH="./.venv"
+    fi
+    echo "Using virtualenv from path: ${PWNDBG_VENV_PATH}"
+
+    source "${PWNDBG_VENV_PATH}/bin/activate"
+    ~/.local/bin/poetry install --with dev
+
+    # Create a developer marker file
+    DEV_MARKER_PATH="${PWNDBG_VENV_PATH}/dev.marker"
+    touch "${DEV_MARKER_PATH}"
+    echo "Developer marker created at ${DEV_MARKER_PATH}"
+}
+
 if linux; then
     distro=$(
         . /etc/os-release
@@ -248,16 +296,8 @@ if linux; then
     esac
 
     install_jemalloc
-    if [[ -z "${PWNDBG_VENV_PATH}" ]]; then
-        PWNDBG_VENV_PATH="./.venv"
-    fi
-    echo "Using virtualenv from path: ${PWNDBG_VENV_PATH}"
 
-    source "${PWNDBG_VENV_PATH}/bin/activate"
-    ~/.local/bin/poetry install --with dev
-
-    # Create a developer marker file
-    DEV_MARKER_PATH="${PWNDBG_VENV_PATH}/dev.marker"
-    touch "${DEV_MARKER_PATH}"
-    echo "Developer marker created at ${DEV_MARKER_PATH}"
+    if [ $USE_INSTALL_ONLY -eq 0 ]; then
+        configure_venv
+    fi
 fi

tests/gdb-tests/tests/utils.py

@@ -6,6 +6,7 @@ import re
 import subprocess
 
 gdb_init_path = os.environ.get("GDB_INIT_PATH", "../gdbinit.py")
+gdb_bin_path = os.environ.get("GDB_BIN_PATH", "gdb")
 
 
 def run_gdb_with_script(
@@ -23,12 +24,13 @@ def run_gdb_with_script(
     pybefore = ([pybefore] if isinstance(pybefore, str) else pybefore) or []
     pyafter = ([pyafter] if isinstance(pyafter, str) else pyafter) or []
 
-    command = ["gdb", "--silent", "--nx", "--nh"]
+    command = [gdb_bin_path, "--silent", "--nx", "--nh"]
 
     for cmd in pybefore:
-        command += ["--eval-command", cmd]
+        command += ["--init-eval-command", cmd]
 
-    command += ["--command", gdb_init_path]
+    if gdb_init_path:
+        command += ["--command", gdb_init_path]
 
     if binary:
         command += [binary]

tests/qemu-tests/conftest.py

@@ -54,8 +54,11 @@ def qemu_assembly_run():
             ]
         )
 
-        gdb.execute(f"target remote :{QEMU_PORT}")
+        os.environ["PWNDBG_IN_TEST"] = "1"
+        os.environ["COLUMNS"] = "80"
         gdb.execute("set exception-verbose on")
+        gdb.execute("set width 80")
+        gdb.execute(f"target remote :{QEMU_PORT}")
 
         global _start_binary_called
         # if _start_binary_called:
@@ -97,8 +100,11 @@ def qemu_start_binary():
             ]
         )
 
-        gdb.execute(f"target remote :{QEMU_PORT}")
+        os.environ["PWNDBG_IN_TEST"] = "1"
+        os.environ["COLUMNS"] = "80"
         gdb.execute("set exception-verbose on")
+        gdb.execute("set width 80")
+        gdb.execute(f"target remote :{QEMU_PORT}")
 
         global _start_binary_called
         # if _start_binary_called:

tests/qemu-tests/tests.sh

@@ -24,10 +24,11 @@ EOF
 fi
 
 help_and_exit() {
-    echo "Usage: ./tests.sh [-p|--pdb] [-c|--cov] [--gdb-port=<port>] [-Q|--preserve-qemu-image] [<test-name-filter>]"
+    echo "Usage: ./tests.sh [-p|--pdb] [-c|--cov] [--nix] [--gdb-port=<port>] [-Q|--preserve-qemu-image] [<test-name-filter>]"
     echo "  -p,  --pdb                  enable pdb (Python debugger) post mortem debugger on failed tests"
     echo "  -c,  --cov                  enable codecov"
     echo "  -v,  --verbose              display all test output instead of just failing test output"
+    echo "  --nix                       run tests using built for nix environment"
     echo "  --gdb-port=<port>           specify debug port for gdb/QEMU (Default: 1234)"
     echo "  --collect-only              only show the output of test collection, don't run any tests"
     echo "  -Q,  --preserve-qemu-image  don't kill QEMU image after failed tests"
@@ -54,6 +55,7 @@ VERBOSE=0
 COLLECT_ONLY=0
 PRESERVE_QEMU_IMAGE=0
 GDB_PORT=1234
+RUN_IN_NIX=0
 
 while [[ $# -gt 0 ]]; do
     case $1 in
@@ -68,6 +70,9 @@ while [[ $# -gt 0 ]]; do
         -v | --verbose)
             VERBOSE=1
             ;;
+        --nix)
+            RUN_IN_NIX=1
+            ;;
         --collect-only)
             COLLECT_ONLY=1
             ;;
@@ -106,18 +111,29 @@ else
     fi
 fi
 
-gdb_load_pwndbg=(--command "$GDB_INIT_PATH" -ex "set exception-verbose on")
 run_gdb() {
     local arch="$1"
     shift
 
-    if [[ "${arch}" == x86_64 ]]; then
-        GDB=gdb
+    if [ $RUN_IN_NIX -eq 1 ]; then
+        gdb_load_pwndbg=()
+
+        GDB="$ROOT_DIR/result/bin/pwndbg"
+        if [ ! -x "$GDB" ]; then
+            echo "ERROR: No nix-compatible pwndbg found. Run nix build .#pwndbg-dev"
+            exit 1
+        fi
     else
-        GDB=gdb-multiarch
+        gdb_load_pwndbg=(--command "$GDB_INIT_PATH")
+
+        if [[ "${arch}" == x86_64 ]]; then
+            GDB=gdb
+        else
+            GDB=gdb-multiarch
+        fi
     fi
 
-    $GDB --silent --nx --nh "${gdb_load_pwndbg[@]}" "$@" -ex "quit" 2> /dev/null
+    $GDB --silent --nx --nh "${gdb_load_pwndbg[@]}" -ex "set exception-verbose on" "$@" -ex "quit" 2> /dev/null
     return $?
 }
 

tests/tests.py

@@ -4,6 +4,7 @@ import argparse
 import concurrent.futures
 import os
 import re
+import shutil
 import subprocess
 import sys
 import time
@@ -84,11 +85,11 @@ def make_binaries(test_dir: str):
 
 
 def run_gdb(
-    gdb_binary: str, gdb_args: List[str], env=None, capture_output=True
+    gdb_path: str, gdb_args: List[str], env=None, capture_output=True
 ) -> CompletedProcess[str]:
     env = os.environ if env is None else env
     return subprocess.run(
-        [gdb_binary, "--silent", "--nx", "--nh"] + gdb_args + ["--eval-command", "quit"],
+        [gdb_path, "--silent", "--nx", "--nh"] + gdb_args + ["--eval-command", "quit"],
         env=env,
         capture_output=capture_output,
         text=True,
@@ -98,18 +99,20 @@ def run_gdb(
 def get_tests_list(
     collect_only: bool,
     test_name_filter: str,
-    gdb_binary: str,
+    gdb_path: str,
     gdbinit_path: str,
     test_dir_path: str,
 ) -> List[str]:
     # NOTE: We run tests under GDB sessions and because of some cleanup/tests dependencies problems
     # we decided to run each test in a separate GDB session
-    gdb_args = ["--init-command", gdbinit_path, "--command", "pytests_collect.py"]
+    gdb_args = ["--command", "pytests_collect.py"]
+    if gdbinit_path:
+        gdb_args.extend(["--init-command", gdbinit_path])
 
     env = os.environ.copy()
     env["TESTS_PATH"] = os.path.join(os.path.dirname(os.path.realpath(__file__)), test_dir_path)
 
-    result = run_gdb(gdb_binary, gdb_args, env=env)
+    result = run_gdb(gdb_path, gdb_args, env=env)
     tests_collect_output = result.stdout
 
     if result.returncode == 1:
@@ -130,9 +133,12 @@ TEST_RETURN_TYPE = Tuple[CompletedProcess[str], str, float]
 
 
 def run_test(
-    test_case: str, args: argparse.Namespace, gdb_binary: str, gdbinit_path: str, port: int = None
+    test_case: str, args: argparse.Namespace, gdb_path: str, gdbinit_path: str, port: int = None
 ) -> TEST_RETURN_TYPE:
-    gdb_args = ["--init-command", gdbinit_path, "--command", "pytests_launcher.py"]
+    gdb_args = ["--command", "pytests_launcher.py"]
+    if gdbinit_path:
+        gdb_args.extend(["--init-command", gdbinit_path])
+
     if args.cov:
         print("Running with coverage")
         gdb_args = [
@@ -154,7 +160,7 @@ def run_test(
         env["QEMU_PORT"] = str(port)
 
     started_at = time.time()
-    result = run_gdb(gdb_binary, gdb_args, env=env, capture_output=not args.serial)
+    result = run_gdb(gdb_path, gdb_args, env=env, capture_output=not args.serial)
     duration = time.time() - started_at
     return result, test_case, duration
 
@@ -194,10 +200,9 @@ class TestStats:
 def run_tests_and_print_stats(
     tests_list: List[str],
     args: argparse.Namespace,
-    gdb_binary: str,
+    gdb_path: str,
     gdbinit_path: str,
     test_dir_path: str,
-    ports: List[int] = [],
 ):
     start = time.time()
     test_results: List[TEST_RETURN_TYPE] = []
@@ -205,7 +210,7 @@ def run_tests_and_print_stats(
 
     if args.serial:
         test_results = [
-            run_test(test, args, gdb_binary, gdbinit_path, reserve_port()) for test in tests_list
+            run_test(test, args, gdb_path, gdbinit_path, reserve_port()) for test in tests_list
         ]
     else:
         print("")
@@ -213,7 +218,7 @@ def run_tests_and_print_stats(
         with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
             for test in tests_list:
                 executor.submit(
-                    run_test, test, args, gdb_binary, gdbinit_path, reserve_port()
+                    run_test, test, args, gdb_path, gdbinit_path, reserve_port()
                 ).add_done_callback(
                     lambda future: stats.handle_test_result(future.result(), args, test_dir_path)
                 )
@@ -266,7 +271,7 @@ def parse_args():
     parser.add_argument(
         "--nix",
         action="store_true",
-        help="run tests using gdbinit.py built for nix environment",
+        help="run tests using built for nix environment",
     )
     parser.add_argument(
         "--collect-only",
@@ -292,31 +297,37 @@ def main():
     if args.pdb:
         print("Will run tests in serial and with Python debugger")
         args.serial = True
+
     if args.nix:
-        gdbinit_path = os.path.join(root_dir, "result/share/pwndbg/gdbinit.py")
-        if not os.path.exists(gdbinit_path):
-            print("ERROR: No nix-compatible gdbinit.py found. Run nix build .#pwndbg-dev")
+        gdbinit_path = ""
+        gdb_path = os.path.join(root_dir, "result/bin/pwndbg")
+        if not os.path.exists(gdb_path):
+            print("ERROR: No nix-compatible pwndbg found. Run nix build .#pwndbg-dev")
             sys.exit(1)
-        os.environ["GDB_INIT_PATH"] = gdbinit_path
     else:
         gdbinit_path = os.path.join(root_dir, "gdbinit.py")
+        gdb_binary = "gdb"
+        if args.type == "cross-arch":
+            gdb_binary = "gdb-multiarch"
+        gdb_path = shutil.which(gdb_binary)
+
+    os.environ["GDB_INIT_PATH"] = gdbinit_path
+    os.environ["GDB_BIN_PATH"] = gdb_path
 
     test_dir_path = TEST_FOLDER_NAME[args.type]
 
     if args.type == "gdb":
-        gdb_binary = "gdb"
         ensure_zig_path()
         make_binaries(test_dir_path)
     elif args.type == "cross-arch":
-        gdb_binary = "gdb-multiarch"
         make_binaries(test_dir_path)
     else:
         raise NotImplementedError(args.type)
 
     tests_list = get_tests_list(
-        args.collect_only, args.test_name_filter, gdb_binary, gdbinit_path, test_dir_path
+        args.collect_only, args.test_name_filter, gdb_path, gdbinit_path, test_dir_path
     )
-    run_tests_and_print_stats(tests_list, args, gdb_binary, gdbinit_path, test_dir_path)
+    run_tests_and_print_stats(tests_list, args, gdb_path, gdbinit_path, test_dir_path)
 
 
 if __name__ == "__main__":