From 427bf8c96ea75163981ecefecdbffa4ea84ffdd5 Mon Sep 17 00:00:00 2001 From: intrigus-lgtm <60750685+intrigus-lgtm@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:44:16 +0100 Subject: [PATCH] Port gdb-tests from bash to python (#1916) * [WIP] Port gdb-tests from bash to python * Use threads instead of processes * Port gdb tests to python * Linting * Fix coverage "again" * Remove bash tests --------- Co-authored-by: intrigus --- tests.sh | 9 +- tests/gdb-tests/tests.py | 182 +++++++++++++++++++++++++++++++++ tests/gdb-tests/tests.sh | 216 --------------------------------------- 3 files changed, 183 insertions(+), 224 deletions(-) create mode 100644 tests/gdb-tests/tests.py delete mode 100755 tests/gdb-tests/tests.sh diff --git a/tests.sh b/tests.sh index dec798ab6..86e7a272b 100755 --- a/tests.sh +++ b/tests.sh @@ -1,14 +1,7 @@ #!/bin/bash -# Check some basic test dependencies -if ! command -v env_parallel &> /dev/null; then - echo 'Error: The `env_parallel` command could not be found. You should run `setup-dev.sh` to install development dependencies.' - echo '(Alternatively, run ./tests.sh with `--serial` to skip using parallel test running. However, if `env_parallel` is missing, it is likely that other dependencies like the `zig` compiler are also missing)' - exit -fi - # Run integration tests -(cd tests/gdb-tests && ./tests.sh $@) +(cd tests/gdb-tests && python3 tests.py $@) # Run unit tests # coverage run -m pytest tests/unit-tests diff --git a/tests/gdb-tests/tests.py b/tests/gdb-tests/tests.py new file mode 100644 index 000000000..2bc4f7a03 --- /dev/null +++ b/tests/gdb-tests/tests.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import argparse +import concurrent.futures +import os +import re +import subprocess +import time +from subprocess import CompletedProcess +from typing import Tuple + +ROOT_DIR = os.path.realpath("../../") +GDB_INIT_PATH = os.path.join(ROOT_DIR, "gdbinit.py") +COVERAGERC_PATH = os.path.join(ROOT_DIR, "pyproject.toml") + + +def ensureZigPath(): + if "ZIGPATH" not in os.environ: + # If ZIGPATH is not set, set it to $pwd/.zig + # In Docker environment this should by default be set to /opt/zig + os.environ["ZIGPATH"] = os.path.join(ROOT_DIR, ".zig") + print(f'ZIGPATH set to {os.environ["ZIGPATH"]}') + + +def makeBinaries(): + try: + subprocess.check_call(["make", "all"], cwd="./tests/binaries") + except subprocess.CalledProcessError: + exit(1) + + +def run_gdb(gdb_args: list[str], env=None, capture_output=True) -> CompletedProcess[str]: + env = os.environ if env is None else env + return subprocess.run( + ["gdb", "--silent", "--nx", "--nh"] + gdb_args + ["--eval-command", "quit"], + env=env, + capture_output=capture_output, + text=True, + ) + + +def getTestsList(collect_only: bool, test_name_filter: str) -> list[str]: + # NOTE: We run tests under GDB sessions and because of some cleanup/tests dependencies problems + # we decided to run each test in a separate GDB session + gdb_args = ["--init-command", GDB_INIT_PATH, "--command", "pytests_collect.py"] + result = run_gdb(gdb_args) + TESTS_COLLECT_OUTPUT = result.stdout + + if result.returncode == 1: + print(TESTS_COLLECT_OUTPUT) + exit(1) + elif collect_only == 1: + print(TESTS_COLLECT_OUTPUT) + exit(0) + + # Extract the test names from the output using regex + pattern = re.compile(r"tests/.*::.*") + matches = pattern.findall(TESTS_COLLECT_OUTPUT) + TESTS_LIST = [match for match in matches if re.search(test_name_filter, match)] + return TESTS_LIST + + +def run_test(test_case: str, args: argparse.Namespace) -> Tuple[CompletedProcess[str], str]: + gdb_args = ["--init-command", GDB_INIT_PATH, "--command", "pytests_launcher.py"] + if args.cov: + print("Running with coverage") + gdb_args = [ + "-ex", + "py import sys;print(sys.path);import coverage;coverage.process_startup();", + ] + gdb_args + env = os.environ.copy() + env["LC_ALL"] = "C.UTF-8" + env["LANG"] = "C.UTF-8" + env["LC_CTYPE"] = "C.UTF-8" + env["SRC_DIR"] = ROOT_DIR + env["COVERAGE_FILE"] = os.path.join(ROOT_DIR, ".cov/coverage") + env["COVERAGE_PROCESS_START"] = COVERAGERC_PATH + if args.pdb: + env["USE_PDB"] = "1" + env["PWNDBG_LAUNCH_TEST"] = test_case + env["PWNDBG_DISABLE_COLORS"] = "1" + result = run_gdb(gdb_args, env=env, capture_output=not args.serial) + return (result, test_case) + + +def run_tests_and_print_stats(tests_list: list[str], args: argparse.Namespace): + start = time.time() + test_results: list[Tuple[CompletedProcess[str], str]] = [] + + def handle_parallel_test_result(test_result: Tuple[CompletedProcess[str], str]): + test_results.append(test_result) + (process, _) = test_result + content = process.stdout + + # Extract the test name and result using regex + testname = re.search(r"^(tests/[^ ]+)", content, re.MULTILINE)[0] + result = re.search( + r"(\x1b\[3.m(PASSED|FAILED|SKIPPED|XPASS|XFAIL)\x1b\[0m)", content, re.MULTILINE + )[0] + + (_, testname) = testname.split("::") + print(f"{testname:<70} {result}") + + # Only show the output of failed tests unless the verbose flag was used + if args.verbose or "FAIL" in result: + print("") + print(content) + + if args.serial: + test_results = [run_test(test, args) for test in tests_list] + else: + print("") + print("Running tests in parallel") + with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: + for test in tests_list: + executor.submit(run_test, test, args).add_done_callback( + lambda future: handle_parallel_test_result(future.result()) + ) + + end = time.time() + seconds = int(end - start) + print(f"Tests completed in {seconds} seconds") + + failed_tests = [(process, _) for (process, _) in test_results if process.returncode != 0] + num_tests_failed = len(failed_tests) + num_tests_passed_or_skipped = len(tests_list) - num_tests_failed + + print("") + print("*********************************") + print("********* TESTS SUMMARY *********") + print("*********************************") + print(f"Tests passed or skipped: {num_tests_passed_or_skipped}") + print(f"Tests failed: {num_tests_failed}") + + if num_tests_failed != 0: + print("") + print( + f"Failing tests: {' '.join([failed_test_name for _, failed_test_name in failed_tests])}" + ) + exit(1) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Run tests.") + parser.add_argument( + "-p", + "--pdb", + action="store_true", + help="enable pdb (Python debugger) post mortem debugger on failed tests", + ) + parser.add_argument("-c", "--cov", action="store_true", help="enable codecov") + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="display all test output instead of just failing test output", + ) + parser.add_argument( + "-s", "--serial", action="store_true", help="run tests one at a time instead of in parallel" + ) + parser.add_argument( + "--collect-only", + action="store_true", + help="only show the output of test collection, don't run any tests", + ) + parser.add_argument( + "test_name_filter", nargs="?", help="run only tests that match the regex", default=".*" + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + if args.cov: + print("Will run codecov") + if args.pdb: + print("Will run tests in serial and with Python debugger") + args.serial = True + ensureZigPath() + makeBinaries() + tests: list[str] = getTestsList(args.collect_only, args.test_name_filter) + run_tests_and_print_stats(tests, args) diff --git a/tests/gdb-tests/tests.sh b/tests/gdb-tests/tests.sh deleted file mode 100755 index 93e8258f7..000000000 --- a/tests/gdb-tests/tests.sh +++ /dev/null @@ -1,216 +0,0 @@ -#!/bin/bash - -#set -o errexit -set -o pipefail - -# env_parallel will fail if there are too many environment variables, so we need -# to use `--session` or `--record-env`, and only `--record-env` is supported on -# the version of `parallel` on Ubuntu 20.04 and earlier. The directory also -# needs to be created for CI -mkdir -p ~/.parallel -. $(which env_parallel.bash) - -# Workaround for Ubuntu 20.04 CI. If no aliases are defined -# `env_parallel --record-env` will have non-zero exit code for older versions of -# `parallel`, so we define a dummy alias here -alias __dummy=foo -env_parallel --record-env - -ROOT_DIR="$(readlink -f ../../)" -GDB_INIT_PATH="$ROOT_DIR/gdbinit.py" -COVERAGERC_PATH="$ROOT_DIR/pyproject.toml" - -help_and_exit() { - echo "Usage: ./tests.sh [-p|--pdb] [-c|--cov] []" - echo " -p, --pdb enable pdb (Python debugger) post mortem debugger on failed tests" - echo " -c, --cov enable codecov" - echo " -v, --verbose display all test output instead of just failing test output" - echo " -k, --keep don't delete the temporary files containing the command output" - echo " -s, --serial run tests one at a time instead of in parallel" - echo " --collect-only only show the output of test collection, don't run any tests" - echo " run only tests that match the regex" - exit 1 -} - -if [[ $# -gt 3 ]]; then - help_and_exit -fi - -USE_PDB=0 -TEST_NAME_FILTER="" -RUN_CODECOV=0 -KEEP=0 -SERIAL=0 -VERBOSE=0 -COLLECT_ONLY=0 - -while [[ $# -gt 0 ]]; do - case $1 in - -p | --pdb) - USE_PDB=1 - SERIAL=1 - echo "Will run tests in serial and with Python debugger" - shift - ;; - -c | --cov) - echo "Will run codecov" - RUN_CODECOV=1 - shift - ;; - -v | --verbose) - VERBOSE=1 - shift - ;; - -k | --keep) - KEEP=1 - shift - ;; - -s | --serial) - SERIAL=1 - shift - ;; - --collect-only) - COLLECT_ONLY=1 - shift - ;; - -h | --help) - help_and_exit - ;; - *) - if [[ ! -z "${TEST_NAME_FILTER}" ]]; then - help_and_exit - fi - TEST_NAME_FILTER="$1" - shift - ;; - esac -done - -if [[ -z "$ZIGPATH" ]]; then - # If ZIGPATH is not set, set it to $pwd/.zig - # In Docker environment this should by default be set to /opt/zig - export ZIGPATH="$ROOT_DIR/.zig" -fi -echo "ZIGPATH set to $ZIGPATH" - -(cd ./tests/binaries && make all) || exit 1 - -run_gdb() { - gdb --silent --nx --nh "$@" --eval-command quit -} - -# NOTE: We run tests under GDB sessions and because of some cleanup/tests dependencies problems -# we decided to run each test in a separate GDB session -gdb_args=(--init-command $GDB_INIT_PATH --command pytests_collect.py) -TESTS_COLLECT_OUTPUT=$(run_gdb "${gdb_args[@]}") - -if [ $? -eq 1 ]; then - echo -E "$TESTS_COLLECT_OUTPUT" - exit 1 -elif [ $COLLECT_ONLY -eq 1 ]; then - echo "$TESTS_COLLECT_OUTPUT" - exit 0 -fi - -TESTS_LIST=($(echo -E "$TESTS_COLLECT_OUTPUT" | grep -o "tests/.*::.*" | grep "${TEST_NAME_FILTER}")) - -run_test() { - test_case="$1" - - gdb_args=(--init-command $GDB_INIT_PATH --command pytests_launcher.py) - if [ ${RUN_CODECOV} -ne 0 ]; then - gdb_args=(-ex 'py import coverage;coverage.process_startup()' "${gdb_args[@]}") - fi - SRC_DIR=$ROOT_DIR \ - COVERAGE_FILE=$ROOT_DIR/.cov/coverage \ - COVERAGE_PROCESS_START=$COVERAGERC_PATH \ - USE_PDB="${USE_PDB}" \ - PWNDBG_LAUNCH_TEST="${test_case}" \ - PWNDBG_DISABLE_COLORS=1 \ - run_gdb "${gdb_args[@]}" - retval=$? - - if [ "$SERIAL" -ne 1 ]; then - exit $retval - fi -} - -parse_output_file() { - output_file="$1" - - read -r testname result < <( - grep -Po '(^tests/[^ ]+)|(\x1b\[3.m(PASSED|FAILED|SKIPPED|XPASS|XFAIL)\x1b\[0m)' "$output_file" \ - | tr '\n' ' ' \ - | cut -d ' ' -f 1,2 - ) - testfile=${testname%::*} - testname=${testname#*::} - - printf '%-70s %s\n' $testname $result - - # Only show the output of failed tests unless the verbose flag was used - if [[ $VERBOSE -eq 1 || "$result" =~ FAIL ]]; then - echo "" - cat "$output_file" - echo "" - fi - - if [[ $KEEP -ne 1 ]]; then - # Delete the temporary file created by `parallel` - rm "$output_file" - else - echo "$output_file" - fi -} - -start=$(date +%s) - -if [ $SERIAL -eq 1 ]; then - for t in "${TESTS_LIST[@]}"; do - run_test "$t" - done -else - JOBLOG_PATH="$(mktemp)" - echo "" - echo -n "Running tests in parallel and using a joblog in $JOBLOG_PATH" - - if [[ $KEEP -ne 1 ]]; then - echo " (use --keep it to persist it)" - else - echo "" - fi - - # The `--env _` is required when using `--record-env` - env_parallel --env _ --output-as-files --joblog $JOBLOG_PATH run_test ::: "${TESTS_LIST[@]}" | env_parallel --env _ parse_output_file {} -fi - -end=$(date +%s) -seconds=$((end - start)) -echo "Tests completed in ${seconds} seconds" - -# TODO: This doesn't work with serial -# The seventh column in the joblog is the exit value and the tenth is the test name -FAILED_TESTS=($(awk '$7 == "1" { print $10 }' "${JOBLOG_PATH}")) - -num_tests_failed=${#FAILED_TESTS[@]} -num_tests_passed_or_skipped=$((${#TESTS_LIST[@]} - $num_tests_failed)) - -echo "" -echo "*********************************" -echo "********* TESTS SUMMARY *********" -echo "*********************************" -echo "Tests passed or skipped: ${num_tests_passed_or_skipped}" -echo "Tests failed: ${num_tests_failed}" - -if [ "${num_tests_failed}" -ne 0 ]; then - echo "" - echo "Failing tests: ${FAILED_TESTS[@]}" - exit 1 -fi - -if [[ $KEEP -ne 1 ]]; then - # Delete the temporary joblog file - rm "${JOBLOG_PATH}" -else - echo "Not removing the ${JOBLOG_PATH} since --keep was passed" -fi