Cleanup testing scripts a bit (#3041)

* rename qemu-tests to crossarch tests

* rename kernel tests and add kernel-tests.sh script

* update workflows

* set ptrace, update contributing docs

* update kernel tests to work on ubuntu docker

.github/workflows/docker.yml

@@ -1,4 +1,4 @@
-name: Docker build
+name: Test Docker
 on:
   pull_request:
     paths:

.github/workflows/tests.yml

@@ -54,26 +54,27 @@ jobs:
     - name: Run cross-arch tests
       if: matrix.type == 'qemu-user-tests'
       run: |
-        ./qemu-tests.sh --nix --cov
+        ./crossarch-tests.sh --nix --cov
 
     - name: Set up cache for QEMU images
       if: matrix.type == 'qemu-system-tests'
       id: qemu-cache
       uses: actions/cache@v3
       with:
-        path: ./tests/qemu-tests/images
+        path: ./tests/qemu-tests/kimages
         key: ${{ matrix.os }}-cache-qemu-images
 
     - name: Download QEMU images
       if: matrix.type == 'qemu-system-tests'
       run: |
-        ./tests/qemu-tests/download_images.sh
+        # Would happen in the next step as well, but this way
+        # it looks nicer in the CI.
+        ./tests/qemu-tests/download-kernel-images.sh
 
     - name: Run kernel tests
       if: matrix.type == 'qemu-system-tests'
-      working-directory: ./tests/qemu-tests
       run: |
-        ./tests.sh --nix --cov
+        ./kernel-tests.sh --nix --cov
 
   tests:
     strategy:
@@ -142,7 +143,7 @@ jobs:
 
     - name: Run cross-architecture tests
       run: |
-        ./qemu-tests.sh --cov
+        ./crossarch-tests.sh --cov
 
     - name: Process coverage data
       run: |
@@ -174,19 +175,18 @@ jobs:
       id: qemu-cache
       uses: actions/cache@v3
       with:
-        path: ./tests/qemu-tests/images
+        path: ./tests/qemu-tests/kimages
         key: ${{ matrix.os }}-cache-qemu-images
 
     - name: Download images
       run: |
-        ./tests/qemu-tests/download_images.sh
+        ./tests/qemu-tests/download-kernel-images.sh
 
     # We set `kernel.yama.ptrace_scope=0` for `gdb-pt-dump`
     - name: Run kernel tests
-      working-directory: ./tests/qemu-tests
       run: |
         sudo sysctl -w kernel.yama.ptrace_scope=0
-        ./tests.sh --cov
+        ./kernel-tests.sh --cov
 
     - name: Process coverage data
       run: |

.gitignore

@@ -82,7 +82,7 @@ tests/**/binaries/div_zero_binary/binary
 !tests/**/binaries/*.go
 
 # QEMU test images
-tests/qemu-tests/images
+tests/qemu-tests/kimages
 
 # VS Code files
 .vscode/

crossarch-tests.sh

@@ -2,6 +2,9 @@
 
 source "$(dirname "$0")/scripts/common.sh"
 
-(cd tests && $UV_RUN_TEST python3 tests.py -t cross-arch $@)
+cd "${PWNDBG_ABS_PATH}/tests"
+
+$UV_RUN_TEST python3 tests.py -t cross-arch $@
+
 exit_code=$?
 exit $exit_code

docs/contributing/index.md

@@ -34,7 +34,9 @@ To run the tests in the same environment as the testing CI, you can use the foll
 # General (x86_64) test suite
 docker compose run --rm --build ubuntu24.04-mount ./tests.sh
 # Cross-architecture tests
-docker compose run --rm --build ubuntu24.04-mount ./qemu-tests.sh
+docker compose run --rm --build ubuntu24.04-mount ./crossarch-tests.sh
+# Kernel tests (x86_64 and aarch64)
+docker compose run --rm --build ubuntu24.04-mount ./kernel-tests.sh
 # Unit tests
 docker compose run --rm --build ubuntu24.04-mount ./unit-tests.sh
 ```
@@ -42,12 +44,10 @@ This comes in handy particularly for cross-architecture tests because the docker
 
 Remove the `-mount` if you want the tests to run from a clean slate (no files are mounted, meaning all binaries are recompiled each time).
 
-If you wish to focus on some failing tests, you can filter the tests to run by providing an argument to the script, such as `<docker..> ./tests.sh heap`, which will only run tests that contain "heap" in the name. See `./tests.sh --help` for more information and other options. You can also do this with the cross-arch tests.
+If you wish to focus on some failing tests, you can filter the tests to run by providing an argument to the script, such as `<docker..> ./tests.sh heap`, which will only run tests that contain "heap" in the name. See `./tests.sh --help` for more information and other options. You can also do this with the cross-arch and kernel tests.
 
 If you want to, you may also [run the tests with nix](#running-tests-with-nix) or [run them bare](#running-without-docker).
 
-TODO: Create a script for running kernel tests instead of running them with `./tests/qemu-tests/tests.sh`.
-
 #### Running tests with nix
 You will need to build a nix-compatible `gdbinit.py` file, which you can do with
 ```{.bash .copy}
@@ -65,24 +65,13 @@ The commands are analogous to the docker commands.
 # General (x86_64) test suite
 ./tests.sh
 # Cross-architecture tests
-./qemu-tests.sh
+./crossarch-tests.sh
+# Kernel tests (x86_64 and aarch64)
+./kernel-tests.sh
 # Unit tests
 ./unit-tests.sh
 ```
 
-To run the kernel tests you will need to install the appropriate qemu-system packages for your distribution. Then download the kernel images with
-```{.bash .copy}
-./tests/qemu-tests/download_images.sh
-```
-set ptrace_scope to zero with
-```{.bash .copy}
-echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
-```
-and run the kernel tests with
-```{.bash .copy}
-cd ./tests/qemu-tests/ && ./tests.sh
-```
-
 ## Updating Documentation
 All the documentation is written in markdown files in the `./docs/` folder. The docs are built into a website using [mkdocs](https://www.mkdocs.org/) (you may see the configuration in `./mkdocs.yml`), pushed to the gh-pages branch, and published via [github pages](https://pages.github.com/). All of this happens in the CI.
 

kernel-tests.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+source "$(dirname "$0")/scripts/common.sh"
+
+cd "${PWNDBG_ABS_PATH}/tests/qemu-tests"
+
+# Check if we have correct ptrace_scope
+ptrace_scope=$(cat /proc/sys/kernel/yama/ptrace_scope)
+if [[ $ptrace_scope -ne 0 && $(id -u) -ne 0 ]]; then
+    echo "Setting ptrace_scope to zero..."
+    echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+fi
+
+# Check if we need to download kernel images
+VMLINUX_LIST=($(basename -a "${TESTING_KERNEL_IMAGES_DIR}"/vmlinux*))
+
+if [ ! -d "$TESTING_KERNEL_IMAGES_DIR" ] || [ "$VMLINUX_LIST" = "vmlinux*" ]; then
+    echo "No kernel images found. Downloading to ${TESTING_KERNEL_IMAGES_DIR}..."
+    echo "(This may take some time.)"
+    echo "(You can always run the download yourself with ./tests/qemu-tests/download-kernel-images.sh .)"
+    echo ""
+    ./download-kernel-images.sh
+    echo "Download finished."
+fi
+
+echo "Running tests..."
+./system-tests.sh $@
+
+exit_code=$?
+exit $exit_code

scripts/common.sh

@@ -4,6 +4,8 @@ _COMMON_ABS_DIR=$(realpath "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")
 # dirname of a directory gives the parent directory.
 PWNDBG_ABS_PATH=$(dirname $_COMMON_ABS_DIR)
 
+TESTING_KERNEL_IMAGES_DIR="${PWNDBG_ABS_PATH}/tests/qemu-tests/kimages"
+
 if [[ -z "${PWNDBG_VENV_PATH}" ]]; then
     PWNDBG_VENV_PATH="${PWNDBG_ABS_PATH}/.venv"
 fi

setup-dev.sh

@@ -144,7 +144,8 @@ install_apt() {
         parallel \
         qemu-system-x86 \
         qemu-system-arm \
-        qemu-user
+        qemu-user \
+        iproute2
 
     # Some tests require i386 libc/ld, eg: test_smallbins_sizes_32bit_big
     if uname -m | grep -q x86_64; then
@@ -198,6 +199,8 @@ EOF
         gdb \
         parallel
 
+    # FIXME: add the necessary deps for testing
+
     command -v go &> /dev/null || sudo pacman -S --noconfirm go
 
     download_zig_binary

tests.sh

@@ -8,6 +8,9 @@ glibc_version=$(ldd --version | sed -n '1s/([^)]*)//g; s/.* \([0-9]\+\.[0-9]\+\)
 echo "glibc version: $glibc_version"
 
 # Run integration tests
-(cd tests && $UV_RUN_TEST python3 tests.py $@)
+cd "${PWNDBG_ABS_PATH}/tests"
+
+$UV_RUN_TEST python3 tests.py $@
+
 exit_code=$?
 exit $exit_code

tests/qemu-tests/download-kernel-images.sh

@@ -2,16 +2,17 @@
 
 set -o errexit
 
-SCRIPT_ABS_DIR="$(dirname "$(realpath "$0")")"
+source "$(dirname "$0")/../../scripts/common.sh"
-OUT_DIR="${SCRIPT_ABS_DIR}/images"
+
+OUT_DIR=$TESTING_KERNEL_IMAGES_DIR
 URL=${URL:-"https://github.com/pwndbg/linux-exploit-dev-env/releases/latest/download"}
 
 mkdir -p "${OUT_DIR}"
 
 download() {
     local file="$1"
-    hash_old=$(grep "${file}" "${OUT_DIR}/hashsums.txt.old" || true)
+    hash_old=$(grep "${file}" "${OUT_DIR}/hashsums.txt.old" 2> /dev/null || true)
-    hash_new=$(grep "${file}" "${OUT_DIR}/hashsums.txt")
+    hash_new=$(grep "${file}" "${OUT_DIR}/hashsums.txt" 2> /dev/null)
     # only download file if it doesn't exist or its hashsum has changed
     if [ ! -f "${OUT_DIR}/${file}" ] || [ "${hash_new}" != "${hash_old}" ]; then
         wget --no-verbose --show-progress --progress=bar:force:noscroll "${URL}/${file}" -O "${OUT_DIR}/${file}"

tests/qemu-tests/run-qemu-system.sh

@@ -6,10 +6,7 @@ ARCH=""
 KERNEL_TYPE=""
 CMDLINE=""
 
-SCRIPT_ABS_DIR="$(dirname "$(realpath "$0")")"
+KERNEL_LIST=($(basename -a "${TESTING_KERNEL_IMAGES_DIR}"/vmlinux* | sed "s/vmlinux-//"))
-IMAGE_DIR="${SCRIPT_ABS_DIR}/images"
-
-KERNEL_LIST=($(basename -a "${IMAGE_DIR}"/vmlinux* | sed "s/vmlinux-//"))
 GDB_PORT=1234
 help_and_exit() {
     echo "Usage: $0 [options] [-- other qemu options]"
@@ -70,8 +67,8 @@ elif [ "$ARCH" == "x86_64" ]; then
     QEMU_ARGS=()
 fi
 
-KERNEL=$(echo ${IMAGE_DIR}/*Image-${KERNEL_NAME})
+KERNEL=$(echo ${TESTING_KERNEL_IMAGES_DIR}/*Image-${KERNEL_NAME})
-ROOTFS=$(echo ${IMAGE_DIR}/*-${ARCH}.img)
+ROOTFS=$(echo ${TESTING_KERNEL_IMAGES_DIR}/*-${ARCH}.img)
 
 QEMU_ARGS+=(
     -kernel $KERNEL

tests/qemu-tests/system-tests.sh

@@ -5,21 +5,19 @@ set -o pipefail
 
 source "$(dirname "$0")/../../scripts/common.sh"
 
-ROOT_DIR="$(readlink -f ../../)"
+ROOT_DIR=$PWNDBG_ABS_PATH
 GDB_INIT_PATH="$ROOT_DIR/gdbinit.py"
 COVERAGERC_PATH="$ROOT_DIR/pyproject.toml"
 
-CWD=$(dirname -- "$0")
+VMLINUX_LIST=($(basename -a "${TESTING_KERNEL_IMAGES_DIR}"/vmlinux*))
-IMAGE_DIR="${CWD}/images"
-VMLINUX_LIST=($(basename -a "${IMAGE_DIR}"/vmlinux*))
 
-# Ensure we have images directory and directories inside
+# Ensure we have kimages directory and directories inside
-if [ ! -d "$IMAGE_DIR" ]; then
+if [ ! -d "$TESTING_KERNEL_IMAGES_DIR" ]; then
-    echo "ERROR: The '$IMAGE_DIR' directory does not exist. Please run ./download_images.sh first"
+    echo "ERROR: The '$TESTING_KERNEL_IMAGES_DIR' directory does not exist. Please run ./download-kernel-images.sh first"
     exit 1
 fi
 if [ "${VMLINUX_LIST}" = "vmlinux*" ]; then
-    echo "ERROR: The '$IMAGE_DIR' directory does not contain any kernel images. Please run ./download_images.sh first"
+    echo "ERROR: The '$TESTING_KERNEL_IMAGES_DIR' directory does not contain any kernel images. Please run ./download-kernel-images.sh first"
     exit 1
 fi
 
@@ -36,7 +34,7 @@ EOF
 fi
 
 help_and_exit() {
-    echo "Usage: ./tests.sh [-p|--pdb] [-c|--cov] [--nix] [--gdb-port=<port>] [-Q|--preserve-qemu-image] [<test-name-filter>]"
+    echo "Usage: ./system-tests.sh [-p|--pdb] [-c|--cov] [--nix] [--gdb-port=<port>] [-Q|--preserve-qemu-image] [<test-name-filter>]"
     echo "  -p,  --pdb                  enable pdb (Python debugger) post mortem debugger on failed tests"
     echo "  -c,  --cov                  enable codecov"
     echo "  -v,  --verbose              display all test output instead of just failing test output"
@@ -109,16 +107,16 @@ done
 
 # Test if the port is already listening, possibly by other qemu instance. This
 # can cause unexpected test failures.
-NETSTAT=$(which netstat)
+NETSTAT=$(which netstat 2> /dev/null)
 if [[ -z "${NETSTAT}" ]]; then
-    NETSTAT=$(which ss)
+    NETSTAT=$(which ss 2> /dev/null)
 fi
 if [[ -z "${NETSTAT}" ]]; then
-    echo "WARNING: netstat/ss not found. Cannot check if port ${GDB_PORT} is already bound." >&2
+    echo "ERROR: netstat/ss not found. Cannot check if port ${GDB_PORT} is already bound." >&2
     exit 1
 else
     if [[ $(${NETSTAT} -tuln 2> /dev/null | grep ":${GDB_PORT}" | grep -c LISTEN) -ne 0 ]]; then
-        echo "WARNING: Port ${GDB_PORT} appears already bound. Please specify a different port with --gdb-port=<port>" >&2
+        echo "ERROR: Port ${GDB_PORT} appears already bound. Please specify a different port with --gdb-port=<port>" >&2
         exit 1
     fi
 fi
@@ -170,7 +168,7 @@ init_gdb() {
     local kernel_version="$2"
     local arch="$3"
 
-    gdb_connect_qemu=(-ex "file ${IMAGE_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}")
+    gdb_connect_qemu=(-ex "file ${TESTING_KERNEL_IMAGES_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}")
     # using 'rest_init' instead of 'start_kernel' to make sure that kernel
     # initialization has progressed sufficiently for testing purposes
     gdb_args=("${gdb_connect_qemu[@]}" -ex 'break *rest_init' -ex 'continue')
@@ -183,7 +181,7 @@ run_test() {
     local kernel_version="$3"
     local arch="$4"
 
-    gdb_connect_qemu=(-ex "file ${IMAGE_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}")
+    gdb_connect_qemu=(-ex "file ${TESTING_KERNEL_IMAGES_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}")
     gdb_args=("${gdb_connect_qemu[@]}" --command ../pytests_launcher.py)
     if [ ${RUN_CODECOV} -ne 0 ]; then
         gdb_args=(-ex 'py import coverage;coverage.process_startup()' "${gdb_args[@]}")
@@ -247,7 +245,7 @@ test_system() {
 
     # NOTE: If you run simultaneous tests or left an image lying around via -Q, this
     # will hang due to failure to obtain lock. But will see the error message...
-    "${CWD}/run_qemu_system.sh" --kernel="${kernel_type}-${kernel_version}-${arch}" --gdb-port="${GDB_PORT}" -- "${qemu_args[@]}" > /dev/null &
+    "./run-qemu-system.sh" --kernel="${kernel_type}-${kernel_version}-${arch}" --gdb-port="${GDB_PORT}" -- "${qemu_args[@]}" > /dev/null &
     QEMU_PID=$!
     init_gdb "${kernel_type}" "${kernel_version}" "${arch}"
     start=$(date +%s)