Setup.sh use virtualenv (#1780)

* Change setup.sh to create & use Python virtualenv

The `setup.sh` script now creates a `.venv` directory during execution and installs all dependencies into that directory. Then, `gdbinit.py` will adds the proper `site-packages` directory as the first item of `sys.path`.

Fixes #1634.

Dockerfile

@@ -16,6 +16,7 @@ WORKDIR /pwndbg
 ENV LANG en_US.utf8
 ENV TZ=America/New_York
 ENV ZIGPATH=/opt/zig
+ENV PWNDBG_VENV_PATH=/venv
 
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
     echo $TZ > /etc/timezone && \
@@ -43,4 +44,3 @@ RUN echo "source /pwndbg/gdbinit.py" >> ~/.gdbinit.py
 ADD . /pwndbg/
 
 RUN git submodule update --init --recursive
-

gdbinit.py

@@ -4,11 +4,10 @@ import locale
 import os
 import sys
 import time
+from glob import glob
 from os import environ
 from os import path
 
-import pkg_resources
-
 _profiler = cProfile.Profile()
 
 _start_time = None
@@ -16,73 +15,31 @@ if environ.get("PWNDBG_PROFILE") == "1":
     _start_time = time.time()
     _profiler.enable()
 
-# Allow users to use packages from a virtualenv
-# That's not 100% supported, but they do it on their own,
-# so we will warn them if the GDB's Python is not virtualenv's Python
-virtual_env = environ.get("VIRTUAL_ENV")
-
-if virtual_env:
-    py_exe_matches = sys.executable.startswith(virtual_env)
-
-    if not py_exe_matches:
-        venv_warning = int(environ.get("PWNDBG_VENV_WARNING", 1))
-        if venv_warning:
-            print(
-                f"""WARNING: Pwndbg/GDB run in virtualenv with which it may not work correctly ***
-  Detected Python virtual environment: VIRTUAL_ENV='{virtual_env}'
-  while GDB is built with different Python binary: {sys.executable}
-  Assuming that you installed Pwndbg dependencies into the virtual environment
-  If this is not true, this may cause import errors or other issues in Pwndbg
-  If all works for you, you can suppress this warning and all further prints
-  by setting `export PWNDBG_VENV_WARNING=0` (e.g. in ~/.bashrc or ~/.zshrc etc.)"""
-            )
-            venv_warn = print
-        else:
-            venv_warn = lambda *a, **kw: None
-
-        possible_site_packages = glob.glob(
-            path.join(virtual_env, "lib", "python*", "site-packages")
-        )
-
-        if len(possible_site_packages) > 1:
-            venv_warn("*** Found multiple site packages in virtualenv:")
-            for site_pkg in possible_site_packages:
-                venv_warn(f"    - {site_pkg}")
-
-            virtualenv_site_packages = possible_site_packages[-1]
-            venv_warn(f"*** Using the last one: {virtualenv_site_packages}")
-
-        elif len(possible_site_packages) == 1:
-            virtualenv_site_packages = possible_site_packages[-1]
-            venv_warn(f"*** Using the only site packages dir found: {virtualenv_site_packages}")
+directory, file = path.split(__file__)
+directory = path.expanduser(directory)
+directory = path.abspath(directory)
 
-        else:
-            guessed_python_directory = f"python{sys.version_info.major}.{sys.version_info.minor}"
-            virtualenv_site_packages = path.join(
-                virtual_env, "lib", guessed_python_directory, "site-packages"
-            )
-            venv_warn(
-                f"***  Not found site-packages in virtualenv, using guessed site packages Python dir: {virtualenv_site_packages}"
-            )
+# Get virtualenv's site-packages path
+venv_path = os.environ.get("PWNDBG_VENV_PATH")
+if not venv_path:
+    venv_path = os.path.join(directory, ".venv")
 
-        venv_warn("  Added detected virtualenv's Python site packages to sys.path")
-        venv_warn("")
-        sys.path.append(virtualenv_site_packages)
+if not os.path.exists(venv_path):
+    print(f"Cannot find Pwndbg virtualenv directory: {venv_path}: please re-run setup.sh")
+    sys.exit(1)
 
+site_pkgs_path = glob(os.path.join(venv_path, "lib/*/site-packages"))[0]
 
-directory, file = path.split(__file__)
-directory = path.expanduser(directory)
-directory = path.abspath(directory)
+# Set virtualenv's bin path (needed for utility tools like ropper, pwntools etc)
+bin_path = os.path.join(venv_path, "bin")
+os.environ["PATH"] = bin_path + os.pathsep + os.environ.get("PATH")
 
 # Add gdb-pt-dump directory to sys.path so it can be imported
 gdbpt = path.join(directory, "gdb-pt-dump")
 sys.path.append(directory)
+sys.path.append(site_pkgs_path)
 sys.path.append(gdbpt)
 
-# Add the dir where Pwntools binaries might be into PATH
-pwntools_bin_dir = os.path.join(pkg_resources.get_distribution("pwntools").location, "bin")
-os.environ["PATH"] = os.environ.get("PATH") + os.pathsep + pwntools_bin_dir
-
 # warn if the user has different encoding than utf-8
 encoding = locale.getpreferredencoding()
 

requirements.txt

@@ -1,3 +1,4 @@
+setuptools-rust; python_version < '3.7'
 capstone==5.0.0rc4
 psutil==5.9.5
 pwntools==4.10.0

setup-dev.sh

@@ -173,5 +173,10 @@ if linux; then
             ;;
     esac
 
-    python3 -m pip install -r dev-requirements.txt
+    if [[ -z "${PWNDBG_VENV_PATH}" ]]; then
+        PWNDBG_VENV_PATH="./.venv"
+    fi
+    echo "Using virtualenv from path: ${PWNDBG_VENV_PATH}"
+    PYTHON=${PWNDBG_VENV_PATH}/bin/python
+    ${PYTHON} -m pip install -r dev-requirements.txt
 fi

setup.sh

@@ -20,7 +20,7 @@ osx() {
 
 install_apt() {
     sudo apt-get update || true
-    sudo apt-get install -y git gdb gdbserver python3-dev python3-pip python3-setuptools libglib2.0-dev libc6-dbg
+    sudo apt-get install -y git gdb gdbserver python3-dev python3-venv python3-pip python3-setuptools libglib2.0-dev libc6-dbg
 
     if uname -m | grep x86_64 > /dev/null; then
         sudo dpkg --add-architecture i386 || true
@@ -99,7 +99,6 @@ for arg in "$@"; do
 done
 
 PYTHON=''
-INSTALLFLAGS=''
 
 # Check for the presence of the initializer line in the user's ~/.gdbinit file
 if [ -z "$UPDATE_MODE" ] && grep -q '^[^#]*source.*pwndbg/gdbinit.py' ~/.gdbinit; then
@@ -112,12 +111,6 @@ if [ -z "$UPDATE_MODE" ] && grep -q '^[^#]*source.*pwndbg/gdbinit.py' ~/.gdbinit
     fi
 fi
 
-if osx || [ -n "$USER_MODE" ]; then
-    INSTALLFLAGS="--user"
-else
-    PYTHON="sudo "
-fi
-
 if linux; then
     distro=$(grep "^ID=" /etc/os-release | cut -d'=' -f2 | sed -e 's/"//g')
 
@@ -179,23 +172,20 @@ if ! osx; then
     PYTHON+="${PYVER}"
 fi
 
-# Find the Python site-packages that we need to use so that
-# GDB can find the files once we've installed them.
-if linux && [ -z "$INSTALLFLAGS" ]; then
-    SITE_PACKAGES=$(gdb -batch -q --nx -ex 'pi import site; print(site.getsitepackages()[0])')
-    INSTALLFLAGS="--target ${SITE_PACKAGES}"
+# Create Python virtualenv
+if [[ -z "${PWNDBG_VENV_PATH}" ]]; then
+    PWNDBG_VENV_PATH="./.venv"
 fi
+echo "Creating virtualenv in path: ${PWNDBG_VENV_PATH}"
 
-# Make sure that pip is available
-if ! ${PYTHON} -m pip -V; then
-    ${PYTHON} -m ensurepip ${INSTALLFLAGS} --upgrade
-fi
+${PYTHON} -m venv -- ${PWNDBG_VENV_PATH}
+PYTHON=${PWNDBG_VENV_PATH}/bin/python
 
 # Upgrade pip itself
-${PYTHON} -m pip install ${INSTALLFLAGS} --upgrade pip
+${PYTHON} -m pip install --upgrade pip
 
-# Install Python dependencies
-${PYTHON} -m pip install ${INSTALLFLAGS} -Ur requirements.txt
+# Create Python virtual environment and install dependencies in it
+${PWNDBG_VENV_PATH}/bin/pip install -Ur ./requirements.txt
 
 if [ -z "$UPDATE_MODE" ]; then
     # Comment old configs out