diff --git a/.github/actions/deps/apt_packages/action.yml b/.github/actions/deps/apt_packages/action.yml
new file mode 100644
index 00000000000..789cf50cc90
--- /dev/null
+++ b/.github/actions/deps/apt_packages/action.yml
@@ -0,0 +1,46 @@
+name: Install apt packages, caching the downloaded files
+
+inputs:
+  packages:
+    description: 'The apt packages to install, separated by spaces'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Name the cache
+      id: cache-name
+      run: echo "prefix=apt-$ImageOS-$(echo "$PACKAGES" | sha256sum | cut -c1-16)-" >> $GITHUB_OUTPUT
+      shell: bash
+      env:
+        PACKAGES: ${{ inputs.packages }}
+    - name: Restore downloaded packages
+      id: restore
+      uses: actions/cache/restore@v5
+      with:
+        path: ~/apt-archives
+        key: ${{ steps.cache-name.outputs.prefix }}
+        restore-keys: ${{ steps.cache-name.outputs.prefix }}
+    - name: Install packages
+      id: install
+      # apt only downloads the packages that are not already in ~/apt-archives, and
+      # autoclean drops versions the mirror no longer has. The cache key names the
+      # .deb files kept, so it changes whenever a package is updated.
+      run: |
+        mkdir -p ~/apt-archives/partial
+        sudo apt-get update
+        sudo apt-get install -y -o Dir::Cache::Archives="$HOME/apt-archives" $PACKAGES
+        sudo apt-get autoclean -o Dir::Cache::Archives="$HOME/apt-archives"
+        sudo rm -rf ~/apt-archives/partial ~/apt-archives/lock
+        sudo chown -R "$(id -un)" ~/apt-archives
+        echo "key=$PREFIX$(ls ~/apt-archives | sha256sum | cut -c1-16)" >> $GITHUB_OUTPUT
+      shell: bash
+      env:
+        PACKAGES: ${{ inputs.packages }}
+        PREFIX: ${{ steps.cache-name.outputs.prefix }}
+    - name: Save downloaded packages
+      if: github.event_name == 'push' && steps.restore.outputs.cache-matched-key != steps.install.outputs.key
+      uses: actions/cache/save@v5
+      with:
+        path: ~/apt-archives
+        key: ${{ steps.install.outputs.key }}
diff --git a/.github/actions/deps/ports/zephyr-cp/action.yml b/.github/actions/deps/ports/zephyr-cp/action.yml
index c63d9397e47..516499b0c8f 100644
--- a/.github/actions/deps/ports/zephyr-cp/action.yml
+++ b/.github/actions/deps/ports/zephyr-cp/action.yml
@@ -9,24 +9,29 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Get Linux build dependencies
+    - name: Show CPU model
       if: runner.os == 'Linux'
-      run: |
-        echo "--- cpu model ---"
-        grep "model name" /proc/cpuinfo | head -1
-        sudo apt-get update
-        sudo apt-get install -y libusb-1.0-0-dev libudev-dev pkg-config mtools
+      run: grep "model name" /proc/cpuinfo | head -1
       shell: bash
-    - name: Get SDL2 for native_sim
+    - name: Get Linux build dependencies
+      if: runner.os == 'Linux'
+      uses: ./.github/actions/deps/apt_packages
+      with:
+        # The last four are the packages action-zephyr-setup@v1 installs, so its apt-get has nothing to download.
+        packages: libusb-1.0-0-dev libudev-dev pkg-config mtools ccache gperf libc6-dev-i386 g++-multilib
+    - name: Prepare for 32-bit SDL2
       if: runner.os == 'Linux' && inputs.sdl == 'true'
       run: |
         sudo dpkg --add-architecture i386
-        sudo apt-get update
         # We have to hold python3 so the following install works. See https://github.com/actions/runner-images/issues/13803
         sudo apt-mark hold python3
-        sudo apt-get install -y libsdl2-dev:i386 libsdl2-image-dev:i386
         echo "PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" >> $GITHUB_ENV
       shell: bash
+    - name: Get SDL2 for native_sim
+      if: runner.os == 'Linux' && inputs.sdl == 'true'
+      uses: ./.github/actions/deps/apt_packages
+      with:
+        packages: libsdl2-dev:i386 libsdl2-image-dev:i386
     - name: Setup Zephyr project
       uses: zephyrproject-rtos/action-zephyr-setup@v1
       with:
diff --git a/.github/workflows/build-mpy-cross.yml b/.github/workflows/build-mpy-cross.yml
index 9469cf332e0..a338632cfea 100644
--- a/.github/workflows/build-mpy-cross.yml
+++ b/.github/workflows/build-mpy-cross.yml
@@ -45,14 +45,14 @@ jobs:
 
     - name: Install toolchain (aarch64)
       if: matrix.mpy-cross == 'static-aarch64'
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gcc-aarch64-linux-gnu
+      uses: ./.github/actions/deps/apt_packages
+      with:
+        packages: gcc-aarch64-linux-gnu
     - name: Install toolchain (mingw)
       if: matrix.mpy-cross == 'static-mingw'
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gcc-mingw-w64-x86-64-win32
+      uses: ./.github/actions/deps/apt_packages
+      with:
+        packages: gcc-mingw-w64-x86-64-win32
 
     - name: Build mpy-cross.${{ matrix.mpy-cross }}
       run: make -C mpy-cross -j$(nproc) -f Makefile.${{ matrix.mpy-cross }}