#!/bin/bash ROOT_DIR="$(readlink -f ../../)" GDB_INIT_PATH="$ROOT_DIR/gdbinit.py" COVERAGERC_PATH="$ROOT_DIR/pyproject.toml" help_and_exit() { echo "Usage: ./tests.sh [-p|--pdb] [-c|--cov] []" echo " -p, --pdb enable pdb (Python debugger) post mortem debugger on failed tests" echo " -c, --cov enable codecov" echo " run only tests that match the regex" exit 1 } if [[ $# -gt 3 ]]; then help_and_exit fi USE_PDB=0 TEST_NAME_FILTER="" RUN_CODECOV=0 while [[ $# -gt 0 ]]; do case $1 in -p | --pdb) USE_PDB=1 echo "Will run tests with Python debugger" shift ;; -c | --cov) echo "Will run codecov" RUN_CODECOV=1 shift ;; -h | --help) help_and_exit ;; *) if [[ ! -z "${TEST_NAME_FILTER}" ]]; then help_and_exit fi TEST_NAME_FILTER="$1" shift ;; esac done if [[ -z "$ZIGPATH" ]]; then # If ZIGPATH is not set, set it to $pwd/.zig # In Docker environment this should by default be set to /opt/zig export ZIGPATH="$ROOT_DIR/.zig" fi echo "ZIGPATH set to $ZIGPATH" (cd ./tests/binaries && make clean && make all) || exit 1 run_gdb() { gdb --silent --nx --nh "$@" --eval-command quit } # NOTE: We run tests under GDB sessions and because of some cleanup/tests dependencies problems # we decided to run each test in a separate GDB session gdb_args=(--command $GDB_INIT_PATH --command pytests_collect.py) TESTS_COLLECT_OUTPUT=$(run_gdb "${gdb_args[@]}") if [ $? -eq 1 ]; then echo -E "$TESTS_COLLECT_OUTPUT" exit 1 fi TESTS_LIST=($(echo -E "$TESTS_COLLECT_OUTPUT" | grep -o "tests/.*::.*" | grep "${TEST_NAME_FILTER}")) run_test() { test_case="$1" gdb_args=(--command $GDB_INIT_PATH --command pytests_launcher.py) if [ ${RUN_CODECOV} -ne 0 ]; then gdb_args=(-ex 'py import coverage;coverage.process_startup()' "${gdb_args[@]}") fi SRC_DIR=$ROOT_DIR \ COVERAGE_FILE=$ROOT_DIR/.cov/coverage \ COVERAGE_PROCESS_START=$COVERAGERC_PATH \ USE_PDB="${USE_PDB}" \ PWNDBG_LAUNCH_TEST="${test_case}" \ PWNDBG_DISABLE_COLORS=1 \ run_gdb "${gdb_args[@]}" exit $? } parse_output_file() { #echo $1 output_file="$1" read -r testname result < <(grep -Po '(^tests/[^ ]+)|(\x1b\[3.m(PASSED|FAILED|SKIPPED)\x1b\[0m)' "$output_file" | tr '\n' ' ' | cut -d ' ' -f 1,2) testfile=${testname%::*} testname=${testname#*::} printf '%-70s %s\n' $testname $result # Only show the output of failed tests if [[ "$result" =~ FAILED ]]; then echo "" cat "$output_file" echo "" fi # Delete the temporary file created by `parallel` rm "$output_file" } JOBLOG_PATH="$(mktemp)" echo "" echo "Joblog: $JOBLOG_PATH" . $(which env_parallel.bash) start=$(date +%s) env_parallel --output-as-files --joblog $JOBLOG_PATH run_test ::: "${TESTS_LIST[@]}" | env_parallel parse_output_file {} end=$(date +%s) seconds=$((end - start)) echo "Tests completed in ${seconds} seconds" # The seventh column in the joblog is the exit value and the tenth is the test name FAILED_TESTS=($(awk '$7 == "1" { print $10 }' "${JOBLOG_PATH}")) num_tests_failed=${#FAILED_TESTS[@]} num_tests_passed_or_skipped=$((${#TESTS_LIST[@]} - $num_tests_failed)) echo "" echo "*********************************" echo "********* TESTS SUMMARY *********" echo "*********************************" echo "Tests passed or skipped: ${num_tests_passed_or_skipped}" echo "Tests failed: ${num_tests_failed}" if [ "${num_tests_failed}" -ne 0 ]; then echo "" echo "Failing tests: ${FAILED_TESTS[@]}" exit 1 fi