ZIGPATH cleanup (#3175)

* add ziglang to uv

* ZIGPATH cleanup

* bump uv2nix and pyproject-nix & nix fmt

* fix darwin version

* guard makefile with PWNDBG_NO_UV

* clean

* pass zigcc to makefile

* pass zigcc to makefile

* nix-ci install uv

* fix uv detection

* fix uv detection

* fix uv detection
pull/3184/head
patryk4815 4 months ago committed by GitHub
parent 72dc157686
commit 12237f4c0b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -25,7 +25,6 @@ jobs:
timeout-minutes: 40
env:
TMPDIR: /tmp
PWNDBG_NO_UV: 1
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v3
- uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # @v30
@ -36,6 +35,9 @@ jobs:
run: |
./setup-dev.sh --install-only
- name: install uv
run: nix profile install --inputs-from . nixpkgs#uv
- name: Build pwndbg
run: |
nix build '.#pwndbg-dev' --accept-flake-config -o result

@ -16,7 +16,6 @@ WORKDIR /pwndbg
ENV PIP_NO_CACHE_DIR=true
ENV LANG=en_US.utf8
ENV TZ=America/New_York
ENV ZIGPATH=/opt/zig
ENV PWNDBG_VENV_PATH=/venv
ENV UV_PROJECT_ENVIRONMENT=/venv

@ -16,7 +16,6 @@ WORKDIR /pwndbg
ENV PIP_NO_CACHE_DIR=true
ENV LANG=en_US.utf8
ENV TZ=America/New_York
ENV ZIGPATH=/opt/zig
ENV PWNDBG_VENV_PATH=/venv
ENV UV_PROJECT_ENVIRONMENT=/venv

@ -16,7 +16,6 @@ WORKDIR /pwndbg
ENV PIP_NO_CACHE_DIR=true
ENV LANG=en_US.utf8
ENV TZ=America/New_York
ENV ZIGPATH=/opt/zig
ENV PWNDBG_VENV_PATH=/venv
ENV UV_PROJECT_ENVIRONMENT=/venv

@ -49,11 +49,11 @@
]
},
"locked": {
"lastModified": 1743085397,
"narHash": "sha256-mCJgxAltNx9uzYTpaSNr6yQtDMXnRykXL87L2bLmsPo=",
"lastModified": 1753063596,
"narHash": "sha256-el1vFxDk6DR2hKGYnMfQHR7+K4aMiJDKQRMP3gdh+ZI=",
"owner": "pyproject-nix",
"repo": "pyproject.nix",
"rev": "af4c3ccf8cffcd49626b0455defb0f6b22cc1910",
"rev": "cac90713492f23be5f1072bae88406890b9c68f6",
"type": "github"
},
"original": {
@ -80,11 +80,11 @@
]
},
"locked": {
"lastModified": 1743267007,
"narHash": "sha256-A5lFzCjO3kBnpUewPaHoM1f6qgubDqw7bgIGSi5i0JE=",
"lastModified": 1753067819,
"narHash": "sha256-vjLTZCgMWvfep8eKZL25T8pBb4MyBjswfyAlHTyJaoY=",
"owner": "pyproject-nix",
"repo": "uv2nix",
"rev": "ede084fd69a0b656acb1ac20e6609385a3f967ba",
"rev": "5b7b7a8808c031fa8e13d8e949a2af3c2c12a1c6",
"type": "github"
},
"original": {

@ -63,7 +63,6 @@ in
curl
parallel
qemu
zig # version match setup-dev.sh
go
# for onegadget command
@ -85,7 +84,6 @@ in
export PWNDBG_NO_AUTOUPDATE=1
export PWNDBG_NO_UV=1
export PWNDBG_VENV_PATH="${pyEnv}"
export ZIGPATH="${pkgs.lib.getBin pkgs.zig}/bin/"
export REPO_ROOT=$(git rev-parse --show-toplevel)
'';
};

@ -79,49 +79,47 @@ let
python3
];
cmakeFlags =
[
# Projects with tablegen-like tools.
"-DLLVM_ENABLE_PROJECTS=${
lib.concatStringsSep ";" (
[
"llvm"
"clang"
"clang-tools-extra"
"lldb"
]
++ lib.optionals (lib.versionAtLeast release_version "16") [
"mlir"
]
)
}"
]
# LLDB test suite requires libc++ on darwin, but we need compile only lldb-tblgen
# These flags are needed only for evaluating the CMake file.
++ lib.optionals stdenv.hostPlatform.isDarwin [
"-DLLDB_INCLUDE_TESTS=OFF"
"-DLIBXML2_INCLUDE_DIR=/non-existent"
]
++ devExtraCmakeFlags;
cmakeFlags = [
# Projects with tablegen-like tools.
"-DLLVM_ENABLE_PROJECTS=${
lib.concatStringsSep ";" (
[
"llvm"
"clang"
"clang-tools-extra"
"lldb"
]
++ lib.optionals (lib.versionAtLeast release_version "16") [
"mlir"
]
)
}"
]
# LLDB test suite requires libc++ on darwin, but we need compile only lldb-tblgen
# These flags are needed only for evaluating the CMake file.
++ lib.optionals stdenv.hostPlatform.isDarwin [
"-DLLDB_INCLUDE_TESTS=OFF"
"-DLIBXML2_INCLUDE_DIR=/non-existent"
]
++ devExtraCmakeFlags;
# List of tablegen targets.
ninjaFlags =
[
"clang-tblgen"
"llvm-tblgen"
"lldb-tblgen"
]
++ lib.optionals (lib.versionAtLeast release_version "15") [
"clang-tidy-confusable-chars-gen"
]
++ lib.optionals (lib.versionAtLeast release_version "16") [
"mlir-tblgen"
]
++
lib.optionals ((lib.versionAtLeast release_version "15") && (lib.versionOlder release_version "20"))
[
"clang-pseudo-gen" # Removed in LLVM 20 @ ed8f78827895050442f544edef2933a60d4a7935.
];
ninjaFlags = [
"clang-tblgen"
"llvm-tblgen"
"lldb-tblgen"
]
++ lib.optionals (lib.versionAtLeast release_version "15") [
"clang-tidy-confusable-chars-gen"
]
++ lib.optionals (lib.versionAtLeast release_version "16") [
"mlir-tblgen"
]
++
lib.optionals ((lib.versionAtLeast release_version "15") && (lib.versionOlder release_version "20"))
[
"clang-pseudo-gen" # Removed in LLVM 20 @ ed8f78827895050442f544edef2933a60d4a7935.
];
installPhase = ''
mkdir -p $out

@ -10,13 +10,12 @@
}:
let
lib = pkgs.lib;
extraPackags =
[
python3.pkgs.pwntools # ref: https://github.com/pwndbg/pwndbg/blob/2023.07.17/pwndbg/wrappers/checksec.py#L8
]
++ lib.optionals pkgs.stdenv.isLinux [
python3.pkgs.ropper # ref: https://github.com/pwndbg/pwndbg/blob/2023.07.17/pwndbg/commands/ropper.py#L30
];
extraPackags = [
python3.pkgs.pwntools # ref: https://github.com/pwndbg/pwndbg/blob/2023.07.17/pwndbg/wrappers/checksec.py#L8
]
++ lib.optionals pkgs.stdenv.isLinux [
python3.pkgs.ropper # ref: https://github.com/pwndbg/pwndbg/blob/2023.07.17/pwndbg/commands/ropper.py#L30
];
pyEnv = import ./pyenv.nix {
inherit

@ -10,7 +10,7 @@
let
lib = pkgs.lib;
hacks = pkgs.callPackage inputs.pyproject-nix.build.hacks { };
workspace = inputs.uv2nix.lib.workspace.loadWorkspace { workspaceRoot = "${inputs.self}"; };
workspace = inputs.uv2nix.lib.workspace.loadWorkspace { workspaceRoot = ./..; };
pyprojectOverlay = workspace.mkPyprojectOverlay {
sourcePreference = "sdist";
@ -111,8 +111,7 @@ let
pkgName:
prev.${pkgName}.overrideAttrs (old: {
nativeBuildInputs =
old.nativeBuildInputs
++ final.resolveBuildSystem (lib.genAttrs pydeps (name: [ ]));
old.nativeBuildInputs ++ final.resolveBuildSystem (lib.genAttrs pydeps (name: [ ]));
})
));
@ -136,6 +135,9 @@ let
uv = dummy;
gdb-for-pwndbg = dummy;
lldb-for-pwndbg = dummy;
ziglang = prev.ziglang.override {
sourcePreference = "wheel";
};
psutil = pkgs.callPackage (
{
@ -247,11 +249,12 @@ let
cp -rf ${readlineStatic.dev}/include/readline/*.h ./readline/
cp -rf ${readlineStatic.out}/lib/*.a ./readline/
'';
buildInputs =
[ ncurses ]
++ lib.optionals isCross [
python3
];
buildInputs = [
ncurses
]
++ lib.optionals isCross [
python3
];
}
)
) { };
@ -284,6 +287,12 @@ let
baseSet = pkgs.callPackage inputs.pyproject-nix.build.packages {
python = python3;
stdenv = pkgs.stdenv.override {
targetPlatform = pkgs.stdenv.targetPlatform // {
# See https://en.wikipedia.org/wiki/MacOS_version_history#Releases for more background on version numbers.
darwinSdkVersion = "13.0";
};
};
};
pythonSet = baseSet.overrideScope overlays;
editablePythonSet = pythonSet.overrideScope (

@ -28,6 +28,8 @@ dependencies = [
"rich>=13.7.1,<14",
# Optional? only for qemu-system vmmap
"pt",
# Optional? only for 'cymbol' command
"ziglang==0.14.1",
]
[project.optional-dependencies]

@ -21,7 +21,15 @@ if [[ "$PWNDBG_NO_UV" == "1" ]]; then
UV_RUN_MYPY=""
else
# We are going to use uv.
UV="${PWNDBG_VENV_PATH}/bin/uv"
if [ -x "${PWNDBG_VENV_PATH}/bin/uv" ]; then
UV="${PWNDBG_VENV_PATH}/bin/uv"
elif command -v uv > /dev/null 2>&1; then
echo "Warning: Falling back to 'uv' found in PATH." >&2
UV="$(command -v uv)"
else
echo "Error: 'uv' binary not found." >&2
UV="${PWNDBG_VENV_PATH}/bin/uv"
fi
UV_RUN="${UV} run"
UV_RUN_TEST="${UV_RUN} --group dev --group tests --all-extras"
UV_RUN_LINT="${UV_RUN} --group lint"

@ -81,56 +81,7 @@ osx() {
uname | grep -iqs Darwin
}
set_zigpath() {
if [[ -z "$ZIGPATH" ]]; then
# If ZIGPATH is not set, set it
# In Docker environment this should by default be set to /opt/zig (APT) or /usr/bin (Pacman)
export ZIGPATH="$1"
fi
echo "ZIGPATH set to $ZIGPATH"
}
download_zig_binary() {
# Install zig to current directory
# We use zig to compile some test binaries as it is much easier than with gcc
TARGET_ZIG_VERSION="0.14.1"
ZIG_TAR_URL="https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz"
ZIG_TAR_SHA256="24aeeec8af16c381934a6cd7d95c807a8cb2cf7df9fa40d359aa884195c4716c"
if command -v "${ZIGPATH}"/zig &> /dev/null; then
ZIG_VERSION=$("$ZIGPATH/zig" version)
if [ "${ZIG_VERSION}" = "${TARGET_ZIG_VERSION}" ]; then
echo "Zig is already installed. Skipping build and install."
return
else
echo "Old version of Zig installed (${ZIG_VERSION}). Installing version ${TARGET_ZIG_VERSION}."
fi
fi
echo "Downloading and installing Zig..."
curl --output /tmp/zig.tar.xz "${ZIG_TAR_URL}"
ACTUAL_SHA256=$(sha256sum /tmp/zig.tar.xz | cut -d' ' -f1)
if [ "${ACTUAL_SHA256}" != "${ZIG_TAR_SHA256}" ]; then
echo "Zig binary checksum mismatch"
echo "Expected: ${ZIG_TAR_SHA256}"
echo "Actual: ${ACTUAL_SHA256}"
exit 1
fi
tar -C /tmp -xJf /tmp/zig.tar.xz
# Delete previous installation
rm -rf "${ZIGPATH}"
mv /tmp/zig-* ${ZIGPATH} &> /dev/null || true
echo "Zig installed to ${ZIGPATH}"
}
install_apt() {
set_zigpath "$(pwd)/.zig"
sudo apt-get update || true
sudo apt-get install -y \
nasm \
@ -159,13 +110,9 @@ install_apt() {
fi
command -v go &> /dev/null || sudo apt-get install -y golang
download_zig_binary
}
install_pacman() {
set_zigpath "$(pwd)/.zig"
# add debug repo for glibc-debug if it doesn't already exist
if ! grep -q "\[core-debug\]" /etc/pacman.conf; then
cat << EOF | sudo tee -a /etc/pacman.conf
@ -202,13 +149,9 @@ EOF
# FIXME: add the necessary deps for testing
command -v go &> /dev/null || sudo pacman -S --noconfirm go
download_zig_binary
}
install_dnf() {
set_zigpath "$(pwd)/.zig"
sudo dnf upgrade || true
sudo dnf install -y \
nasm \
@ -225,8 +168,6 @@ install_dnf() {
if [[ "$1" != "" ]]; then
sudo dnf install shfmt
fi
download_zig_binary
}
install_jemalloc() {

@ -1,5 +1,4 @@
ZIGPATH ?= ../../../.zig
ZIGCC = $(ZIGPATH)/zig cc
ZIGCC = uv run python3 -m ziglang cc
CC = gcc
DEBUG = 1

@ -1,7 +1,6 @@
.PHONY: all
ZIGPATH ?= ../../../.zig
ZIGCC = $(ZIGPATH)/zig cc
ZIGCC = uv run python3 -m ziglang cc
ARCHS = aarch64 arm riscv64 mips32 mipsel32 mips64 loongarch64 s390x powerpc32 powerpc64

@ -13,13 +13,13 @@ from typing import Tuple
import gdb
import pytest
import ziglang
from pwndbg.lib import tempfile
_start_binary_called = False
QEMU_PORT: str | None = None
ZIGPATH = os.environ.get("ZIGPATH")
COMPILATION_TARGETS_TYPE = Literal[
"aarch64",
@ -124,10 +124,6 @@ def qemu_assembly_run():
The `path` is returned from `make_elf_from_assembly` (provided by pwntools)
"""
if ZIGPATH is None:
raise Exception("ZIGPATH not defined")
PATH_TO_ZIG = os.path.join(ZIGPATH, "zig")
ensure_qemu_port()
qemu: subprocess.Popen = None
@ -154,7 +150,7 @@ def qemu_assembly_run():
# Build the binary with Zig
compile_process = subprocess.run(
[
PATH_TO_ZIG,
os.path.join(os.path.dirname(ziglang.__file__), "zig"),
"cc",
*extra_cli_args,
f"--target={zig_target}",

@ -14,6 +14,8 @@ import time
from enum import Enum
from pathlib import Path
import ziglang
from .host import TestHost
from .host import TestResult
from .host import TestStatus
@ -38,7 +40,6 @@ def main():
# building tests, even if the user has requested a nix-compatible test.
#
# Ideally, however, we would build the test targets as part of `nix verify`.
ensure_zig_path(local_pwndbg_root)
make_all(local_pwndbg_root / args.group.binary_dir())
if not args.driver.can_run(args.group):
@ -314,14 +315,6 @@ def parse_args():
return parser.parse_args()
def ensure_zig_path(local_pwndbg_root: Path):
if "ZIGPATH" not in os.environ:
# If ZIGPATH is not set, set it to $pwd/.zig
# In Docker environment this should by default be set to /opt/zig
os.environ["ZIGPATH"] = str(local_pwndbg_root / ".zig")
print(f'[+] ZIGPATH set to {os.environ["ZIGPATH"]}')
def make_all(path: Path, jobs: int = multiprocessing.cpu_count()):
"""
Build the binaries for a given test group.
@ -331,7 +324,15 @@ def make_all(path: Path, jobs: int = multiprocessing.cpu_count()):
print(f"[+] make -C {path} -j{jobs} all")
try:
subprocess.check_call(["make", f"-j{jobs}", "all"], cwd=str(path))
subprocess.check_call(
[
"make",
f"-j{jobs}",
"ZIGCC=" + os.path.join(os.path.dirname(ziglang.__file__), "zig") + " cc",
"all",
],
cwd=str(path),
)
except subprocess.CalledProcessError:
sys.exit(1)

@ -1309,6 +1309,7 @@ dependencies = [
{ name = "typing-extensions" },
{ name = "unicorn" },
{ name = "uv" },
{ name = "ziglang" },
]
[package.optional-dependencies]
@ -1380,6 +1381,7 @@ requires-dist = [
{ name = "typing-extensions", specifier = ">=4.12.0,<5" },
{ name = "unicorn", specifier = ">=2.1.3,<3" },
{ name = "uv", specifier = ">=0.7.6" },
{ name = "ziglang", specifier = "==0.14.1" },
]
provides-extras = ["lldb", "gdb"]
@ -2282,6 +2284,23 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166, upload-time = "2024-01-06T02:10:55.763Z" },
]
[[package]]
name = "ziglang"
version = "0.14.1"
source = { registry = "https://pypi.org/simple" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/16/a8/6989b2d0f61f6b33ec7f711b48089fc9d1a53787ce657e81b0b941d22f59/ziglang-0.14.1-py3-none-macosx_12_0_arm64.whl", hash = "sha256:eae7d46746b8e80a4d1a8369b09f7a6aace973ee39207b1b63cfb6bb086e84dc", size = 82418439, upload-time = "2025-05-26T11:43:58.726Z" },
{ url = "https://files.pythonhosted.org/packages/fa/15/ff2df54335b247df74776a65904a704402904fd503fc185d386e0b03f27c/ziglang-0.14.1-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:7b880229c41e822c0388318de89dffe034a146ce44a7342b7e74b167329a67b9", size = 86646805, upload-time = "2025-05-26T11:44:20.565Z" },
{ url = "https://files.pythonhosted.org/packages/e8/5c/552ade5873cd6133bba25cc991a9ceb2261fcba0fbb8b69a528629e7488a/ziglang-0.14.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:479a36e94a9da3fc1fad01951de35600c3d11642182fe36c2aeeb031ceab7a89", size = 86910843, upload-time = "2025-05-26T11:44:43.285Z" },
{ url = "https://files.pythonhosted.org/packages/9d/ef/494f128ea7ec3345850bbd75d2cb7b987006fb5601c37543e8fa83e5e3d4/ziglang-0.14.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:6eb9d4d759b292c83810dbee2e9e8e3fbfbf01d864e6e9811bae711fd74e1c2f", size = 83184072, upload-time = "2025-05-26T11:44:55.631Z" },
{ url = "https://files.pythonhosted.org/packages/94/84/2ca431d4f7984a260b5115a5ab130c1459d0b0ed08c5ae7d4093e52cb4a3/ziglang-0.14.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:7994b27f3cbfcedea43f9b7552e38b45857bdf0e9a45065474092dd74e7048cf", size = 80233732, upload-time = "2025-05-26T11:45:07.741Z" },
{ url = "https://files.pythonhosted.org/packages/74/ea/c59e5a0368bb85eded8df8893f6e7a72d20293268f4ad5b2ce6aabe8cf8e/ziglang-0.14.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:2c9dbee405ad83a062df3569949f24b59e938c1b85aa26674e30b515e654fef4", size = 81624139, upload-time = "2025-05-26T11:45:19.882Z" },
{ url = "https://files.pythonhosted.org/packages/cf/b6/2e9673067d0e25a4c0681e33f1c6213f36e9b1d3ad327ea58f061765a6ef/ziglang-0.14.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:ad2c7c3a77cff522971fc303da2b656c4ed675fe17cdf5ff74ba7e1b594a4780", size = 89083283, upload-time = "2025-05-26T11:45:31.61Z" },
{ url = "https://files.pythonhosted.org/packages/8b/2e/a66ac42e58db4349d75e77644f2717aecccade7bc4f2f924cd114a05ef2e/ziglang-0.14.1-py3-none-win32.whl", hash = "sha256:3d68104704f850c52e2baec1e12caf58e097e954a40880b247da8fb4fa500550", size = 85479685, upload-time = "2025-05-26T11:45:43.203Z" },
{ url = "https://files.pythonhosted.org/packages/9d/53/b15661c6f4442c0e1ec1223c17f7a47e5cc108cb171bbdadbc675479582d/ziglang-0.14.1-py3-none-win_amd64.whl", hash = "sha256:e4f7e089a44d5ce34181853a90cdb8456e63c6640f5d44b844a117055326c375", size = 83574233, upload-time = "2025-05-26T11:45:55.268Z" },
{ url = "https://files.pythonhosted.org/packages/83/55/ba6235dfcaf5c64524615786031b69af2323ac55242433fc01089badf7bf/ziglang-0.14.1-py3-none-win_arm64.whl", hash = "sha256:f9e13d3e3778a850acf20115b0193a9ae0569c5ca2f71c8e690c4110746b6993", size = 79290101, upload-time = "2025-05-26T11:46:06.54Z" },
]
[[package]]
name = "zipp"
version = "3.21.0"

Loading…
Cancel
Save