diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index da56175cb..83a35115f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,4 @@ -name: Docker build +name: Test Docker on: pull_request: paths: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f39bcf2ed..fc12d0ca1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,26 +54,27 @@ jobs: - name: Run cross-arch tests if: matrix.type == 'qemu-user-tests' run: | - ./qemu-tests.sh --nix --cov + ./crossarch-tests.sh --nix --cov - name: Set up cache for QEMU images if: matrix.type == 'qemu-system-tests' id: qemu-cache uses: actions/cache@v3 with: - path: ./tests/qemu-tests/images + path: ./tests/qemu-tests/kimages key: ${{ matrix.os }}-cache-qemu-images - name: Download QEMU images if: matrix.type == 'qemu-system-tests' run: | - ./tests/qemu-tests/download_images.sh + # Would happen in the next step as well, but this way + # it looks nicer in the CI. + ./tests/qemu-tests/download-kernel-images.sh - name: Run kernel tests if: matrix.type == 'qemu-system-tests' - working-directory: ./tests/qemu-tests run: | - ./tests.sh --nix --cov + ./kernel-tests.sh --nix --cov tests: strategy: @@ -142,7 +143,7 @@ jobs: - name: Run cross-architecture tests run: | - ./qemu-tests.sh --cov + ./crossarch-tests.sh --cov - name: Process coverage data run: | @@ -174,19 +175,18 @@ jobs: id: qemu-cache uses: actions/cache@v3 with: - path: ./tests/qemu-tests/images + path: ./tests/qemu-tests/kimages key: ${{ matrix.os }}-cache-qemu-images - name: Download images run: | - ./tests/qemu-tests/download_images.sh + ./tests/qemu-tests/download-kernel-images.sh # We set `kernel.yama.ptrace_scope=0` for `gdb-pt-dump` - name: Run kernel tests - working-directory: ./tests/qemu-tests run: | sudo sysctl -w kernel.yama.ptrace_scope=0 - ./tests.sh --cov + ./kernel-tests.sh --cov - name: Process coverage data run: | diff --git a/.gitignore b/.gitignore index 27c404c28..fd50b380c 100644 --- a/.gitignore +++ b/.gitignore @@ -82,7 +82,7 @@ tests/**/binaries/div_zero_binary/binary !tests/**/binaries/*.go # QEMU test images -tests/qemu-tests/images +tests/qemu-tests/kimages # VS Code files .vscode/ diff --git a/qemu-tests.sh b/crossarch-tests.sh similarity index 54% rename from qemu-tests.sh rename to crossarch-tests.sh index 96a5ab2d0..b065e293f 100755 --- a/qemu-tests.sh +++ b/crossarch-tests.sh @@ -2,6 +2,9 @@ source "$(dirname "$0")/scripts/common.sh" -(cd tests && $UV_RUN_TEST python3 tests.py -t cross-arch $@) +cd "${PWNDBG_ABS_PATH}/tests" + +$UV_RUN_TEST python3 tests.py -t cross-arch $@ + exit_code=$? exit $exit_code diff --git a/docs/contributing/index.md b/docs/contributing/index.md index dec4d8616..bcd6951bb 100644 --- a/docs/contributing/index.md +++ b/docs/contributing/index.md @@ -34,7 +34,9 @@ To run the tests in the same environment as the testing CI, you can use the foll # General (x86_64) test suite docker compose run --rm --build ubuntu24.04-mount ./tests.sh # Cross-architecture tests -docker compose run --rm --build ubuntu24.04-mount ./qemu-tests.sh +docker compose run --rm --build ubuntu24.04-mount ./crossarch-tests.sh +# Kernel tests (x86_64 and aarch64) +docker compose run --rm --build ubuntu24.04-mount ./kernel-tests.sh # Unit tests docker compose run --rm --build ubuntu24.04-mount ./unit-tests.sh ``` @@ -42,12 +44,10 @@ This comes in handy particularly for cross-architecture tests because the docker Remove the `-mount` if you want the tests to run from a clean slate (no files are mounted, meaning all binaries are recompiled each time). -If you wish to focus on some failing tests, you can filter the tests to run by providing an argument to the script, such as ` ./tests.sh heap`, which will only run tests that contain "heap" in the name. See `./tests.sh --help` for more information and other options. You can also do this with the cross-arch tests. +If you wish to focus on some failing tests, you can filter the tests to run by providing an argument to the script, such as ` ./tests.sh heap`, which will only run tests that contain "heap" in the name. See `./tests.sh --help` for more information and other options. You can also do this with the cross-arch and kernel tests. If you want to, you may also [run the tests with nix](#running-tests-with-nix) or [run them bare](#running-without-docker). -TODO: Create a script for running kernel tests instead of running them with `./tests/qemu-tests/tests.sh`. - #### Running tests with nix You will need to build a nix-compatible `gdbinit.py` file, which you can do with ```{.bash .copy} @@ -65,24 +65,13 @@ The commands are analogous to the docker commands. # General (x86_64) test suite ./tests.sh # Cross-architecture tests -./qemu-tests.sh +./crossarch-tests.sh +# Kernel tests (x86_64 and aarch64) +./kernel-tests.sh # Unit tests ./unit-tests.sh ``` -To run the kernel tests you will need to install the appropriate qemu-system packages for your distribution. Then download the kernel images with -```{.bash .copy} -./tests/qemu-tests/download_images.sh -``` -set ptrace_scope to zero with -```{.bash .copy} -echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope -``` -and run the kernel tests with -```{.bash .copy} -cd ./tests/qemu-tests/ && ./tests.sh -``` - ## Updating Documentation All the documentation is written in markdown files in the `./docs/` folder. The docs are built into a website using [mkdocs](https://www.mkdocs.org/) (you may see the configuration in `./mkdocs.yml`), pushed to the gh-pages branch, and published via [github pages](https://pages.github.com/). All of this happens in the CI. diff --git a/kernel-tests.sh b/kernel-tests.sh new file mode 100755 index 000000000..dee036662 --- /dev/null +++ b/kernel-tests.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/scripts/common.sh" + +cd "${PWNDBG_ABS_PATH}/tests/qemu-tests" + +# Check if we have correct ptrace_scope +ptrace_scope=$(cat /proc/sys/kernel/yama/ptrace_scope) +if [[ $ptrace_scope -ne 0 && $(id -u) -ne 0 ]]; then + echo "Setting ptrace_scope to zero..." + echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope +fi + +# Check if we need to download kernel images +VMLINUX_LIST=($(basename -a "${TESTING_KERNEL_IMAGES_DIR}"/vmlinux*)) + +if [ ! -d "$TESTING_KERNEL_IMAGES_DIR" ] || [ "$VMLINUX_LIST" = "vmlinux*" ]; then + echo "No kernel images found. Downloading to ${TESTING_KERNEL_IMAGES_DIR}..." + echo "(This may take some time.)" + echo "(You can always run the download yourself with ./tests/qemu-tests/download-kernel-images.sh .)" + echo "" + ./download-kernel-images.sh + echo "Download finished." +fi + +echo "Running tests..." +./system-tests.sh $@ + +exit_code=$? +exit $exit_code diff --git a/scripts/common.sh b/scripts/common.sh index 1cc5c2de3..9365cf4fc 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -4,6 +4,8 @@ _COMMON_ABS_DIR=$(realpath "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)") # dirname of a directory gives the parent directory. PWNDBG_ABS_PATH=$(dirname $_COMMON_ABS_DIR) +TESTING_KERNEL_IMAGES_DIR="${PWNDBG_ABS_PATH}/tests/qemu-tests/kimages" + if [[ -z "${PWNDBG_VENV_PATH}" ]]; then PWNDBG_VENV_PATH="${PWNDBG_ABS_PATH}/.venv" fi diff --git a/setup-dev.sh b/setup-dev.sh index 2a34bcf01..ab0d65363 100755 --- a/setup-dev.sh +++ b/setup-dev.sh @@ -144,7 +144,8 @@ install_apt() { parallel \ qemu-system-x86 \ qemu-system-arm \ - qemu-user + qemu-user \ + iproute2 # Some tests require i386 libc/ld, eg: test_smallbins_sizes_32bit_big if uname -m | grep -q x86_64; then @@ -198,6 +199,8 @@ EOF gdb \ parallel + # FIXME: add the necessary deps for testing + command -v go &> /dev/null || sudo pacman -S --noconfirm go download_zig_binary diff --git a/tests.sh b/tests.sh index e8e4aa35a..3aea172fa 100755 --- a/tests.sh +++ b/tests.sh @@ -8,6 +8,9 @@ glibc_version=$(ldd --version | sed -n '1s/([^)]*)//g; s/.* \([0-9]\+\.[0-9]\+\) echo "glibc version: $glibc_version" # Run integration tests -(cd tests && $UV_RUN_TEST python3 tests.py $@) +cd "${PWNDBG_ABS_PATH}/tests" + +$UV_RUN_TEST python3 tests.py $@ + exit_code=$? exit $exit_code diff --git a/tests/qemu-tests/download_images.sh b/tests/qemu-tests/download-kernel-images.sh similarity index 77% rename from tests/qemu-tests/download_images.sh rename to tests/qemu-tests/download-kernel-images.sh index dfa81db0b..737a6bf41 100755 --- a/tests/qemu-tests/download_images.sh +++ b/tests/qemu-tests/download-kernel-images.sh @@ -2,16 +2,17 @@ set -o errexit -SCRIPT_ABS_DIR="$(dirname "$(realpath "$0")")" -OUT_DIR="${SCRIPT_ABS_DIR}/images" +source "$(dirname "$0")/../../scripts/common.sh" + +OUT_DIR=$TESTING_KERNEL_IMAGES_DIR URL=${URL:-"https://github.com/pwndbg/linux-exploit-dev-env/releases/latest/download"} mkdir -p "${OUT_DIR}" download() { local file="$1" - hash_old=$(grep "${file}" "${OUT_DIR}/hashsums.txt.old" || true) - hash_new=$(grep "${file}" "${OUT_DIR}/hashsums.txt") + hash_old=$(grep "${file}" "${OUT_DIR}/hashsums.txt.old" 2> /dev/null || true) + hash_new=$(grep "${file}" "${OUT_DIR}/hashsums.txt" 2> /dev/null) # only download file if it doesn't exist or its hashsum has changed if [ ! -f "${OUT_DIR}/${file}" ] || [ "${hash_new}" != "${hash_old}" ]; then wget --no-verbose --show-progress --progress=bar:force:noscroll "${URL}/${file}" -O "${OUT_DIR}/${file}" diff --git a/tests/qemu-tests/run_qemu_system.sh b/tests/qemu-tests/run-qemu-system.sh similarity index 89% rename from tests/qemu-tests/run_qemu_system.sh rename to tests/qemu-tests/run-qemu-system.sh index c0d691e9a..bba41608e 100755 --- a/tests/qemu-tests/run_qemu_system.sh +++ b/tests/qemu-tests/run-qemu-system.sh @@ -6,10 +6,7 @@ ARCH="" KERNEL_TYPE="" CMDLINE="" -SCRIPT_ABS_DIR="$(dirname "$(realpath "$0")")" -IMAGE_DIR="${SCRIPT_ABS_DIR}/images" - -KERNEL_LIST=($(basename -a "${IMAGE_DIR}"/vmlinux* | sed "s/vmlinux-//")) +KERNEL_LIST=($(basename -a "${TESTING_KERNEL_IMAGES_DIR}"/vmlinux* | sed "s/vmlinux-//")) GDB_PORT=1234 help_and_exit() { echo "Usage: $0 [options] [-- other qemu options]" @@ -70,8 +67,8 @@ elif [ "$ARCH" == "x86_64" ]; then QEMU_ARGS=() fi -KERNEL=$(echo ${IMAGE_DIR}/*Image-${KERNEL_NAME}) -ROOTFS=$(echo ${IMAGE_DIR}/*-${ARCH}.img) +KERNEL=$(echo ${TESTING_KERNEL_IMAGES_DIR}/*Image-${KERNEL_NAME}) +ROOTFS=$(echo ${TESTING_KERNEL_IMAGES_DIR}/*-${ARCH}.img) QEMU_ARGS+=( -kernel $KERNEL diff --git a/tests/qemu-tests/tests.sh b/tests/qemu-tests/system-tests.sh similarity index 86% rename from tests/qemu-tests/tests.sh rename to tests/qemu-tests/system-tests.sh index 2c1dd2caf..a990584c3 100755 --- a/tests/qemu-tests/tests.sh +++ b/tests/qemu-tests/system-tests.sh @@ -5,21 +5,19 @@ set -o pipefail source "$(dirname "$0")/../../scripts/common.sh" -ROOT_DIR="$(readlink -f ../../)" +ROOT_DIR=$PWNDBG_ABS_PATH GDB_INIT_PATH="$ROOT_DIR/gdbinit.py" COVERAGERC_PATH="$ROOT_DIR/pyproject.toml" -CWD=$(dirname -- "$0") -IMAGE_DIR="${CWD}/images" -VMLINUX_LIST=($(basename -a "${IMAGE_DIR}"/vmlinux*)) +VMLINUX_LIST=($(basename -a "${TESTING_KERNEL_IMAGES_DIR}"/vmlinux*)) -# Ensure we have images directory and directories inside -if [ ! -d "$IMAGE_DIR" ]; then - echo "ERROR: The '$IMAGE_DIR' directory does not exist. Please run ./download_images.sh first" +# Ensure we have kimages directory and directories inside +if [ ! -d "$TESTING_KERNEL_IMAGES_DIR" ]; then + echo "ERROR: The '$TESTING_KERNEL_IMAGES_DIR' directory does not exist. Please run ./download-kernel-images.sh first" exit 1 fi if [ "${VMLINUX_LIST}" = "vmlinux*" ]; then - echo "ERROR: The '$IMAGE_DIR' directory does not contain any kernel images. Please run ./download_images.sh first" + echo "ERROR: The '$TESTING_KERNEL_IMAGES_DIR' directory does not contain any kernel images. Please run ./download-kernel-images.sh first" exit 1 fi @@ -36,7 +34,7 @@ EOF fi help_and_exit() { - echo "Usage: ./tests.sh [-p|--pdb] [-c|--cov] [--nix] [--gdb-port=] [-Q|--preserve-qemu-image] []" + echo "Usage: ./system-tests.sh [-p|--pdb] [-c|--cov] [--nix] [--gdb-port=] [-Q|--preserve-qemu-image] []" echo " -p, --pdb enable pdb (Python debugger) post mortem debugger on failed tests" echo " -c, --cov enable codecov" echo " -v, --verbose display all test output instead of just failing test output" @@ -109,16 +107,16 @@ done # Test if the port is already listening, possibly by other qemu instance. This # can cause unexpected test failures. -NETSTAT=$(which netstat) +NETSTAT=$(which netstat 2> /dev/null) if [[ -z "${NETSTAT}" ]]; then - NETSTAT=$(which ss) + NETSTAT=$(which ss 2> /dev/null) fi if [[ -z "${NETSTAT}" ]]; then - echo "WARNING: netstat/ss not found. Cannot check if port ${GDB_PORT} is already bound." >&2 + echo "ERROR: netstat/ss not found. Cannot check if port ${GDB_PORT} is already bound." >&2 exit 1 else if [[ $(${NETSTAT} -tuln 2> /dev/null | grep ":${GDB_PORT}" | grep -c LISTEN) -ne 0 ]]; then - echo "WARNING: Port ${GDB_PORT} appears already bound. Please specify a different port with --gdb-port=" >&2 + echo "ERROR: Port ${GDB_PORT} appears already bound. Please specify a different port with --gdb-port=" >&2 exit 1 fi fi @@ -170,7 +168,7 @@ init_gdb() { local kernel_version="$2" local arch="$3" - gdb_connect_qemu=(-ex "file ${IMAGE_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}") + gdb_connect_qemu=(-ex "file ${TESTING_KERNEL_IMAGES_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}") # using 'rest_init' instead of 'start_kernel' to make sure that kernel # initialization has progressed sufficiently for testing purposes gdb_args=("${gdb_connect_qemu[@]}" -ex 'break *rest_init' -ex 'continue') @@ -183,7 +181,7 @@ run_test() { local kernel_version="$3" local arch="$4" - gdb_connect_qemu=(-ex "file ${IMAGE_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}") + gdb_connect_qemu=(-ex "file ${TESTING_KERNEL_IMAGES_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}") gdb_args=("${gdb_connect_qemu[@]}" --command ../pytests_launcher.py) if [ ${RUN_CODECOV} -ne 0 ]; then gdb_args=(-ex 'py import coverage;coverage.process_startup()' "${gdb_args[@]}") @@ -247,7 +245,7 @@ test_system() { # NOTE: If you run simultaneous tests or left an image lying around via -Q, this # will hang due to failure to obtain lock. But will see the error message... - "${CWD}/run_qemu_system.sh" --kernel="${kernel_type}-${kernel_version}-${arch}" --gdb-port="${GDB_PORT}" -- "${qemu_args[@]}" > /dev/null & + "./run-qemu-system.sh" --kernel="${kernel_type}-${kernel_version}-${arch}" --gdb-port="${GDB_PORT}" -- "${qemu_args[@]}" > /dev/null & QEMU_PID=$! init_gdb "${kernel_type}" "${kernel_version}" "${arch}" start=$(date +%s)