diff --git a/Dockerfile b/Dockerfile
index 7d5c9e9f0..84ca3e804 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,10 +25,13 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
     rm -rf /var/lib/apt/lists/* && \
     localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 && \
     apt-get update && \
-    apt-get install -y vim git
+    apt-get install -y vim
 
-ADD . /pwndbg/
-RUN git submodule update --init --recursive
+ADD ./setup.sh /pwndbg/
+ADD ./poetry.lock /pwndbg/
+ADD ./pyproject.toml /pwndbg/
+ADD ./dev-requirements.txt /pwndbg/
+RUN touch README.md && mkdir pwndbg && touch pwndbg/empty.py && mkdir gdb-pt-dump && touch gdb-pt-dump/empty.py
 
 # The `git submodule` is commented because it refreshes all the sub-modules in the project
 # but at this time we only need the essentials for the set up. It will execute at the end.
@@ -40,3 +43,7 @@ ADD ./setup-dev.sh /pwndbg/
 RUN ./setup-dev.sh
 
 RUN echo "source /pwndbg/gdbinit.py" >> ~/.gdbinit.py
+
+ADD . /pwndbg/
+
+RUN git submodule update --init --recursive
diff --git a/Dockerfile.arch b/Dockerfile.arch
index 82405efd8..1e582720b 100644
--- a/Dockerfile.arch
+++ b/Dockerfile.arch
@@ -21,11 +21,14 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
     echo $TZ > /etc/timezone && \
     pacman -Syu --noconfirm && \
     localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 && \
-    pacman -S --noconfirm vim git && \
+    pacman -S --noconfirm vim && \
     pacman -Scc --noconfirm
 
-ADD . /pwndbg/
-RUN git submodule update --init --recursive
+ADD ./setup.sh /pwndbg/
+ADD ./pyproject.toml /pwndbg/
+ADD ./poetry.lock /pwndbg/
+ADD ./dev-requirements.txt /pwndbg/
+RUN touch README.md && mkdir pwndbg && touch pwndbg/empty.py && mkdir gdb-pt-dump && touch gdb-pt-dump/empty.py
 
 # The `git submodule` is commented because it refreshes all the sub-modules in the project
 # but at this time we only need the essentials for the set up. It will execute at the end.
@@ -40,3 +43,7 @@ RUN echo "source /pwndbg/gdbinit.py" >> ~/.gdbinit.py && \
     echo "PYTHON_MINOR=$(python3 -c "import sys;print(sys.version_info.minor)")" >> /root/.bashrc && \
     echo "PYTHON_PATH=\"/usr/local/lib/python3.${PYTHON_MINOR}/dist-packages/bin\"" >> /root/.bashrc && \
     echo "export PATH=$PATH:$PYTHON_PATH" >> /root/.bashrc
+
+ADD . /pwndbg/
+
+RUN git submodule update --init --recursive
diff --git a/setup.sh b/setup.sh
index 0635be7fa..7adceee40 100755
--- a/setup.sh
+++ b/setup.sh
@@ -191,6 +191,17 @@ ${PYTHON} -m pip install --upgrade pip
 # Create Python virtual environment and install dependencies in it
 ${PWNDBG_VENV_PATH}/bin/pip install -U .
 
+# pyproject.toml install itself "pwndbg"/"gdb-pt-dump" into site-packages, for "caching" dockerfile we need remove it
+PYTHON_VERSION=$(ls "${PWNDBG_VENV_PATH}/lib/")
+CHECK_PATH="${PWNDBG_VENV_PATH}/lib/${PYTHON_VERSION}/site-packages/pwndbg/empty.py"
+if [ -f "$CHECK_PATH" ]; then
+    rm -rf "$(dirname "$CHECK_PATH")"
+fi
+CHECK_PATH="${PWNDBG_VENV_PATH}/lib/${PYTHON_VERSION}/site-packages/gdb-pt-dump/empty.py"
+if [ -f "$CHECK_PATH" ]; then
+    rm -rf "$(dirname "$CHECK_PATH")"
+fi
+
 if [ -z "$UPDATE_MODE" ]; then
     # Comment old configs out
     if grep -q '^[^#]*source.*pwndbg/gdbinit.py' ~/.gdbinit; then