Cleanup testing scripts a bit (#3041)

* rename qemu-tests to crossarch tests

* rename kernel tests and add kernel-tests.sh script

* update workflows

* set ptrace, update contributing docs

* update kernel tests to work on ubuntu docker
pull/3027/head
k4lizen 6 months ago committed by GitHub
parent 6502675f2b
commit effa016b33
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -1,4 +1,4 @@
name: Docker build
name: Test Docker
on:
pull_request:
paths:

@ -54,26 +54,27 @@ jobs:
- name: Run cross-arch tests
if: matrix.type == 'qemu-user-tests'
run: |
./qemu-tests.sh --nix --cov
./crossarch-tests.sh --nix --cov
- name: Set up cache for QEMU images
if: matrix.type == 'qemu-system-tests'
id: qemu-cache
uses: actions/cache@v3
with:
path: ./tests/qemu-tests/images
path: ./tests/qemu-tests/kimages
key: ${{ matrix.os }}-cache-qemu-images
- name: Download QEMU images
if: matrix.type == 'qemu-system-tests'
run: |
./tests/qemu-tests/download_images.sh
# Would happen in the next step as well, but this way
# it looks nicer in the CI.
./tests/qemu-tests/download-kernel-images.sh
- name: Run kernel tests
if: matrix.type == 'qemu-system-tests'
working-directory: ./tests/qemu-tests
run: |
./tests.sh --nix --cov
./kernel-tests.sh --nix --cov
tests:
strategy:
@ -142,7 +143,7 @@ jobs:
- name: Run cross-architecture tests
run: |
./qemu-tests.sh --cov
./crossarch-tests.sh --cov
- name: Process coverage data
run: |
@ -174,19 +175,18 @@ jobs:
id: qemu-cache
uses: actions/cache@v3
with:
path: ./tests/qemu-tests/images
path: ./tests/qemu-tests/kimages
key: ${{ matrix.os }}-cache-qemu-images
- name: Download images
run: |
./tests/qemu-tests/download_images.sh
./tests/qemu-tests/download-kernel-images.sh
# We set `kernel.yama.ptrace_scope=0` for `gdb-pt-dump`
- name: Run kernel tests
working-directory: ./tests/qemu-tests
run: |
sudo sysctl -w kernel.yama.ptrace_scope=0
./tests.sh --cov
./kernel-tests.sh --cov
- name: Process coverage data
run: |

2
.gitignore vendored

@ -82,7 +82,7 @@ tests/**/binaries/div_zero_binary/binary
!tests/**/binaries/*.go
# QEMU test images
tests/qemu-tests/images
tests/qemu-tests/kimages
# VS Code files
.vscode/

@ -2,6 +2,9 @@
source "$(dirname "$0")/scripts/common.sh"
(cd tests && $UV_RUN_TEST python3 tests.py -t cross-arch $@)
cd "${PWNDBG_ABS_PATH}/tests"
$UV_RUN_TEST python3 tests.py -t cross-arch $@
exit_code=$?
exit $exit_code

@ -34,7 +34,9 @@ To run the tests in the same environment as the testing CI, you can use the foll
# General (x86_64) test suite
docker compose run --rm --build ubuntu24.04-mount ./tests.sh
# Cross-architecture tests
docker compose run --rm --build ubuntu24.04-mount ./qemu-tests.sh
docker compose run --rm --build ubuntu24.04-mount ./crossarch-tests.sh
# Kernel tests (x86_64 and aarch64)
docker compose run --rm --build ubuntu24.04-mount ./kernel-tests.sh
# Unit tests
docker compose run --rm --build ubuntu24.04-mount ./unit-tests.sh
```
@ -42,12 +44,10 @@ This comes in handy particularly for cross-architecture tests because the docker
Remove the `-mount` if you want the tests to run from a clean slate (no files are mounted, meaning all binaries are recompiled each time).
If you wish to focus on some failing tests, you can filter the tests to run by providing an argument to the script, such as `<docker..> ./tests.sh heap`, which will only run tests that contain "heap" in the name. See `./tests.sh --help` for more information and other options. You can also do this with the cross-arch tests.
If you wish to focus on some failing tests, you can filter the tests to run by providing an argument to the script, such as `<docker..> ./tests.sh heap`, which will only run tests that contain "heap" in the name. See `./tests.sh --help` for more information and other options. You can also do this with the cross-arch and kernel tests.
If you want to, you may also [run the tests with nix](#running-tests-with-nix) or [run them bare](#running-without-docker).
TODO: Create a script for running kernel tests instead of running them with `./tests/qemu-tests/tests.sh`.
#### Running tests with nix
You will need to build a nix-compatible `gdbinit.py` file, which you can do with
```{.bash .copy}
@ -65,24 +65,13 @@ The commands are analogous to the docker commands.
# General (x86_64) test suite
./tests.sh
# Cross-architecture tests
./qemu-tests.sh
./crossarch-tests.sh
# Kernel tests (x86_64 and aarch64)
./kernel-tests.sh
# Unit tests
./unit-tests.sh
```
To run the kernel tests you will need to install the appropriate qemu-system packages for your distribution. Then download the kernel images with
```{.bash .copy}
./tests/qemu-tests/download_images.sh
```
set ptrace_scope to zero with
```{.bash .copy}
echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
```
and run the kernel tests with
```{.bash .copy}
cd ./tests/qemu-tests/ && ./tests.sh
```
## Updating Documentation
All the documentation is written in markdown files in the `./docs/` folder. The docs are built into a website using [mkdocs](https://www.mkdocs.org/) (you may see the configuration in `./mkdocs.yml`), pushed to the gh-pages branch, and published via [github pages](https://pages.github.com/). All of this happens in the CI.

@ -0,0 +1,30 @@
#!/usr/bin/env bash
source "$(dirname "$0")/scripts/common.sh"
cd "${PWNDBG_ABS_PATH}/tests/qemu-tests"
# Check if we have correct ptrace_scope
ptrace_scope=$(cat /proc/sys/kernel/yama/ptrace_scope)
if [[ $ptrace_scope -ne 0 && $(id -u) -ne 0 ]]; then
echo "Setting ptrace_scope to zero..."
echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
fi
# Check if we need to download kernel images
VMLINUX_LIST=($(basename -a "${TESTING_KERNEL_IMAGES_DIR}"/vmlinux*))
if [ ! -d "$TESTING_KERNEL_IMAGES_DIR" ] || [ "$VMLINUX_LIST" = "vmlinux*" ]; then
echo "No kernel images found. Downloading to ${TESTING_KERNEL_IMAGES_DIR}..."
echo "(This may take some time.)"
echo "(You can always run the download yourself with ./tests/qemu-tests/download-kernel-images.sh .)"
echo ""
./download-kernel-images.sh
echo "Download finished."
fi
echo "Running tests..."
./system-tests.sh $@
exit_code=$?
exit $exit_code

@ -4,6 +4,8 @@ _COMMON_ABS_DIR=$(realpath "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")
# dirname of a directory gives the parent directory.
PWNDBG_ABS_PATH=$(dirname $_COMMON_ABS_DIR)
TESTING_KERNEL_IMAGES_DIR="${PWNDBG_ABS_PATH}/tests/qemu-tests/kimages"
if [[ -z "${PWNDBG_VENV_PATH}" ]]; then
PWNDBG_VENV_PATH="${PWNDBG_ABS_PATH}/.venv"
fi

@ -144,7 +144,8 @@ install_apt() {
parallel \
qemu-system-x86 \
qemu-system-arm \
qemu-user
qemu-user \
iproute2
# Some tests require i386 libc/ld, eg: test_smallbins_sizes_32bit_big
if uname -m | grep -q x86_64; then
@ -198,6 +199,8 @@ EOF
gdb \
parallel
# FIXME: add the necessary deps for testing
command -v go &> /dev/null || sudo pacman -S --noconfirm go
download_zig_binary

@ -8,6 +8,9 @@ glibc_version=$(ldd --version | sed -n '1s/([^)]*)//g; s/.* \([0-9]\+\.[0-9]\+\)
echo "glibc version: $glibc_version"
# Run integration tests
(cd tests && $UV_RUN_TEST python3 tests.py $@)
cd "${PWNDBG_ABS_PATH}/tests"
$UV_RUN_TEST python3 tests.py $@
exit_code=$?
exit $exit_code

@ -2,16 +2,17 @@
set -o errexit
SCRIPT_ABS_DIR="$(dirname "$(realpath "$0")")"
OUT_DIR="${SCRIPT_ABS_DIR}/images"
source "$(dirname "$0")/../../scripts/common.sh"
OUT_DIR=$TESTING_KERNEL_IMAGES_DIR
URL=${URL:-"https://github.com/pwndbg/linux-exploit-dev-env/releases/latest/download"}
mkdir -p "${OUT_DIR}"
download() {
local file="$1"
hash_old=$(grep "${file}" "${OUT_DIR}/hashsums.txt.old" || true)
hash_new=$(grep "${file}" "${OUT_DIR}/hashsums.txt")
hash_old=$(grep "${file}" "${OUT_DIR}/hashsums.txt.old" 2> /dev/null || true)
hash_new=$(grep "${file}" "${OUT_DIR}/hashsums.txt" 2> /dev/null)
# only download file if it doesn't exist or its hashsum has changed
if [ ! -f "${OUT_DIR}/${file}" ] || [ "${hash_new}" != "${hash_old}" ]; then
wget --no-verbose --show-progress --progress=bar:force:noscroll "${URL}/${file}" -O "${OUT_DIR}/${file}"

@ -6,10 +6,7 @@ ARCH=""
KERNEL_TYPE=""
CMDLINE=""
SCRIPT_ABS_DIR="$(dirname "$(realpath "$0")")"
IMAGE_DIR="${SCRIPT_ABS_DIR}/images"
KERNEL_LIST=($(basename -a "${IMAGE_DIR}"/vmlinux* | sed "s/vmlinux-//"))
KERNEL_LIST=($(basename -a "${TESTING_KERNEL_IMAGES_DIR}"/vmlinux* | sed "s/vmlinux-//"))
GDB_PORT=1234
help_and_exit() {
echo "Usage: $0 [options] [-- other qemu options]"
@ -70,8 +67,8 @@ elif [ "$ARCH" == "x86_64" ]; then
QEMU_ARGS=()
fi
KERNEL=$(echo ${IMAGE_DIR}/*Image-${KERNEL_NAME})
ROOTFS=$(echo ${IMAGE_DIR}/*-${ARCH}.img)
KERNEL=$(echo ${TESTING_KERNEL_IMAGES_DIR}/*Image-${KERNEL_NAME})
ROOTFS=$(echo ${TESTING_KERNEL_IMAGES_DIR}/*-${ARCH}.img)
QEMU_ARGS+=(
-kernel $KERNEL

@ -5,21 +5,19 @@ set -o pipefail
source "$(dirname "$0")/../../scripts/common.sh"
ROOT_DIR="$(readlink -f ../../)"
ROOT_DIR=$PWNDBG_ABS_PATH
GDB_INIT_PATH="$ROOT_DIR/gdbinit.py"
COVERAGERC_PATH="$ROOT_DIR/pyproject.toml"
CWD=$(dirname -- "$0")
IMAGE_DIR="${CWD}/images"
VMLINUX_LIST=($(basename -a "${IMAGE_DIR}"/vmlinux*))
VMLINUX_LIST=($(basename -a "${TESTING_KERNEL_IMAGES_DIR}"/vmlinux*))
# Ensure we have images directory and directories inside
if [ ! -d "$IMAGE_DIR" ]; then
echo "ERROR: The '$IMAGE_DIR' directory does not exist. Please run ./download_images.sh first"
# Ensure we have kimages directory and directories inside
if [ ! -d "$TESTING_KERNEL_IMAGES_DIR" ]; then
echo "ERROR: The '$TESTING_KERNEL_IMAGES_DIR' directory does not exist. Please run ./download-kernel-images.sh first"
exit 1
fi
if [ "${VMLINUX_LIST}" = "vmlinux*" ]; then
echo "ERROR: The '$IMAGE_DIR' directory does not contain any kernel images. Please run ./download_images.sh first"
echo "ERROR: The '$TESTING_KERNEL_IMAGES_DIR' directory does not contain any kernel images. Please run ./download-kernel-images.sh first"
exit 1
fi
@ -36,7 +34,7 @@ EOF
fi
help_and_exit() {
echo "Usage: ./tests.sh [-p|--pdb] [-c|--cov] [--nix] [--gdb-port=<port>] [-Q|--preserve-qemu-image] [<test-name-filter>]"
echo "Usage: ./system-tests.sh [-p|--pdb] [-c|--cov] [--nix] [--gdb-port=<port>] [-Q|--preserve-qemu-image] [<test-name-filter>]"
echo " -p, --pdb enable pdb (Python debugger) post mortem debugger on failed tests"
echo " -c, --cov enable codecov"
echo " -v, --verbose display all test output instead of just failing test output"
@ -109,16 +107,16 @@ done
# Test if the port is already listening, possibly by other qemu instance. This
# can cause unexpected test failures.
NETSTAT=$(which netstat)
NETSTAT=$(which netstat 2> /dev/null)
if [[ -z "${NETSTAT}" ]]; then
NETSTAT=$(which ss)
NETSTAT=$(which ss 2> /dev/null)
fi
if [[ -z "${NETSTAT}" ]]; then
echo "WARNING: netstat/ss not found. Cannot check if port ${GDB_PORT} is already bound." >&2
echo "ERROR: netstat/ss not found. Cannot check if port ${GDB_PORT} is already bound." >&2
exit 1
else
if [[ $(${NETSTAT} -tuln 2> /dev/null | grep ":${GDB_PORT}" | grep -c LISTEN) -ne 0 ]]; then
echo "WARNING: Port ${GDB_PORT} appears already bound. Please specify a different port with --gdb-port=<port>" >&2
echo "ERROR: Port ${GDB_PORT} appears already bound. Please specify a different port with --gdb-port=<port>" >&2
exit 1
fi
fi
@ -170,7 +168,7 @@ init_gdb() {
local kernel_version="$2"
local arch="$3"
gdb_connect_qemu=(-ex "file ${IMAGE_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}")
gdb_connect_qemu=(-ex "file ${TESTING_KERNEL_IMAGES_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}")
# using 'rest_init' instead of 'start_kernel' to make sure that kernel
# initialization has progressed sufficiently for testing purposes
gdb_args=("${gdb_connect_qemu[@]}" -ex 'break *rest_init' -ex 'continue')
@ -183,7 +181,7 @@ run_test() {
local kernel_version="$3"
local arch="$4"
gdb_connect_qemu=(-ex "file ${IMAGE_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}")
gdb_connect_qemu=(-ex "file ${TESTING_KERNEL_IMAGES_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}")
gdb_args=("${gdb_connect_qemu[@]}" --command ../pytests_launcher.py)
if [ ${RUN_CODECOV} -ne 0 ]; then
gdb_args=(-ex 'py import coverage;coverage.process_startup()' "${gdb_args[@]}")
@ -247,7 +245,7 @@ test_system() {
# NOTE: If you run simultaneous tests or left an image lying around via -Q, this
# will hang due to failure to obtain lock. But will see the error message...
"${CWD}/run_qemu_system.sh" --kernel="${kernel_type}-${kernel_version}-${arch}" --gdb-port="${GDB_PORT}" -- "${qemu_args[@]}" > /dev/null &
"./run-qemu-system.sh" --kernel="${kernel_type}-${kernel_version}-${arch}" --gdb-port="${GDB_PORT}" -- "${qemu_args[@]}" > /dev/null &
QEMU_PID=$!
init_gdb "${kernel_type}" "${kernel_version}" "${arch}"
start=$(date +%s)
Loading…
Cancel
Save