Merge pull request #78 from LASTRADA-Software/feature/distributed-compilation

Add distributed compilation: dispatch cache misses to worker nodes

Author: christianparpart

.github/workflows/build.yml

@@ -668,7 +668,7 @@ jobs:
         run: cmake --preset gcc-release -DFASTCACHED_BUILD_TESTS=OFF -DFASTCACHED_PACKAGE_ROOT_PREFIX=ON -DFASTCACHED_REQUIRE_EXACT_VERSION=${{ env.FASTCACHED_REQUIRE_EXACT_VERSION }}
 
       - name: "Build"
-        run: cmake --build --preset gcc-release --target fastcached fastcache-cc
+        run: cmake --build --preset gcc-release --target fastcached fastcache-cc fastcache-compile-node
 
       - name: "Package"
         working-directory: out/build/gcc-release
@@ -710,11 +710,15 @@ jobs:
 
           # The binaries, the units and the config must each land in the one
           # location that works; /usr/etc or /lib would silently never be read.
-          for path in ./usr/bin/fastcached ./usr/bin/fastcache-cc \
+          for path in ./usr/bin/fastcached ./usr/bin/fastcache-cc ./usr/bin/fastcache-compile-node \
                       ./usr/lib/systemd/system/fastcached.service \
                       ./usr/lib/systemd/user/fastcached.service \
                       ./usr/lib/sysusers.d/fastcached.conf \
                       ./usr/lib/tmpfiles.d/fastcached.conf \
+                      ./usr/lib/systemd/system/fastcache-compile-node.service \
+                      ./usr/lib/systemd/system/fastcache-compile-node.socket \
+                      ./usr/lib/sysusers.d/fastcache-node.conf \
+                      ./etc/fastcached/compile-node.env \
                       ./etc/fastcached/fastcached.yaml; do
             grep -qx "$path" /tmp/deb-paths.txt || { echo "::error::missing $path"; exit 1; }
           done
@@ -739,17 +743,26 @@ jobs:
           rpm -qlp "$rpmfile" > /tmp/rpm-paths.txt
           rpm -qcp "$rpmfile" > /tmp/rpm-config.txt
 
-          for path in /usr/bin/fastcached /usr/bin/fastcache-cc \
+          for path in /usr/bin/fastcached /usr/bin/fastcache-cc /usr/bin/fastcache-compile-node \
                       /usr/lib/systemd/system/fastcached.service \
                       /usr/lib/systemd/user/fastcached.service \
                       /usr/lib/sysusers.d/fastcached.conf \
                       /usr/lib/tmpfiles.d/fastcached.conf \
+                      /usr/lib/systemd/system/fastcache-compile-node.service \
+                      /usr/lib/systemd/system/fastcache-compile-node.socket \
+                      /usr/lib/sysusers.d/fastcache-node.conf \
+                      /etc/fastcached/compile-node.env \
                       /etc/fastcached/fastcached.yaml; do
             grep -qx "$path" /tmp/rpm-paths.txt || { echo "::error::missing $path"; exit 1; }
           done
 
-          grep -qx '/etc/fastcached/fastcached.yaml' /tmp/rpm-config.txt \
-            || { echo "::error::config not marked %config"; exit 1; }
+          # Both, and for the same reason: an upgrade that reset either one is a
+          # silent regression. compile-node.env is what makes the worker work at
+          # all, so resetting it un-registers the worker with no error anywhere.
+          for path in /etc/fastcached/fastcached.yaml /etc/fastcached/compile-node.env; do
+            grep -qx "$path" /tmp/rpm-config.txt \
+              || { echo "::error::$path not marked %config"; exit 1; }
+          done
 
       # Install for real and drive traffic through it. The GitHub runner has a
       # live systemd, so this exercises the actual unit — the service account,
@@ -803,6 +816,75 @@ jobs:
           sudo systemctl stop fastcached
           test "$(systemctl show -p Result --value fastcached)" = success
 
+      - name: "Socket-activate the compile worker on real systemd"
+        run: |
+          set -euo pipefail
+
+          # The account the sysusers.d entry must have created. Without it the
+          # unit fails to start with "user does not exist", which says nothing
+          # about which package forgot to create it.
+          id fastcache-node >/dev/null \
+            || { echo "::error::the fastcache-node service account was not created"; exit 1; }
+
+          # A scheduler that is not there. The worker must still start, warn and
+          # keep retrying -- a fleet where the scheduler restarts must not turn
+          # into a fleet of dead workers.
+          printf 'FASTCACHE_NODE_ARGS=--scheduler=127.0.0.1:6675 --advertise=127.0.0.1:6676 --toolchain=/usr/bin/g++\n' \
+            | sudo tee /etc/fastcached/compile-node.env >/dev/null
+
+          # Enabling the SOCKET is what enables the worker; the service carries
+          # no [Install] on purpose.
+          sudo systemctl enable --now fastcache-compile-node.socket
+          systemctl is-active --quiet fastcache-compile-node.socket \
+            || { echo "::error::the socket unit did not start"; exit 1; }
+
+          # Nothing should be running yet -- that is the whole point of
+          # activation. An already-active service here means the socket was
+          # bypassed and the unit is just an ordinary daemon.
+          systemctl is-active --quiet fastcache-compile-node \
+            && { echo "::error::the service started without a connection"; exit 1; }
+
+          # One connection activates it.
+          timeout 10 bash -c 'exec 3<>/dev/tcp/127.0.0.1/6676' || true
+
+          # Waited for by its OUTPUT, not by is-active. Type=simple is reported
+          # active the moment systemd forks, while this worker still has to walk
+          # its toolchain's include tree to compute a fingerprint -- seconds of
+          # real work. Polling is-active therefore answers "has it started",
+          # which is not the question; the first version of this step asked it
+          # and read an empty journal.
+          for _ in $(seq 1 90); do
+            sudo journalctl -u fastcache-compile-node --no-pager > /tmp/node-log.txt 2>/dev/null || true
+            grep -q 'compile node ready' /tmp/node-log.txt && break
+            sleep 1
+          done
+          cat /tmp/node-log.txt
+          grep -q 'compile node ready' /tmp/node-log.txt || {
+            echo "::error::a connection did not bring the worker up"
+            systemctl status fastcache-compile-node --no-pager || true; exit 1; }
+
+          # It must have ADOPTED the socket rather than bound its own. Binding
+          # would have failed against the socket unit already holding the port,
+          # so a running service that did not adopt is a service listening
+          # somewhere nobody was told about.
+          grep -q 'adopted a socket-activated listener' /tmp/node-log.txt \
+            || { echo "::error::the worker did not adopt the activated socket"; exit 1; }
+
+          test "$(systemctl show -p User --value fastcache-compile-node)" = fastcache-node
+
+          # The stop path, on real systemd and on an ADOPTED descriptor. This is
+          # the assertion that matters most here: POSIX does not unblock a parked
+          # accept() when the listener is closed, so a worker without a receive
+          # timeout hangs until systemd escalates to SIGKILL -- and that bug has
+          # already shipped once, on the ordinary bind path. `Result=success`
+          # distinguishes a clean exit from a killed one; `is-system-running`
+          # would not.
+          sudo systemctl stop fastcache-compile-node
+          test "$(systemctl show -p Result --value fastcache-compile-node)" = success \
+            || { echo "::error::the worker did not stop cleanly"
+                 sudo journalctl -u fastcache-compile-node --no-pager; exit 1; }
+          sudo systemctl stop fastcache-compile-node.socket
+
       - name: "Verify an upgrade keeps local config edits"
         working-directory: out/build/gcc-release
         run: |
@@ -818,8 +900,13 @@ jobs:
           set -euo pipefail
           sudo apt-get purge -y fastcached
           test ! -e /usr/lib/systemd/system/fastcached.service
+          test ! -e /usr/lib/systemd/system/fastcache-compile-node.service
+          test ! -e /usr/lib/systemd/system/fastcache-compile-node.socket
           # A leftover wants-symlink makes systemd log a failure on every boot.
           test ! -e /etc/systemd/system/multi-user.target.wants/fastcached.service
+          # The socket is the unit that gets enabled, so it is the one that
+          # leaves a symlink behind if the postrm forgets it.
+          test ! -e /etc/systemd/system/sockets.target.wants/fastcache-compile-node.socket
 
       - name: "Upload packages"
         uses: actions/upload-artifact@v4
@@ -1001,15 +1088,15 @@ jobs:
             -DFASTCACHED_REQUIRE_EXACT_VERSION=${{ env.FASTCACHED_REQUIRE_EXACT_VERSION }}
 
       - name: "Build"
-        run: cmake --build out/build/macos-package --target fastcached fastcache-cc
+        run: cmake --build out/build/macos-package --target fastcached fastcache-cc fastcache-compile-node
 
       - name: "Verify the binaries are redistributable"
         # The regression that would otherwise ship silently: everything builds,
         # every test passes, and the package refuses to launch on any machine
         # without the build host's Homebrew prefix.
         run: |
           set -euo pipefail
-          for tool in fastcached fastcache-cc; do
+          for tool in fastcached fastcache-cc fastcache-compile-node; do
             binary="out/build/macos-package/target/$tool"
             echo "== $tool"; otool -L "$binary"
             strays="$(otool -L "$binary" | tail -n +2 | awk '{print $1}' | grep -v '^/usr/lib/' || true)"
@@ -1157,7 +1244,7 @@ jobs:
         run: cmake --preset cl-release -DFASTCACHED_BUILD_TESTS=OFF -DFASTCACHED_REQUIRE_EXACT_VERSION=${{ env.FASTCACHED_REQUIRE_EXACT_VERSION }}
 
       - name: "Build"
-        run: cmake --build --preset cl-release --target fastcached fastcache-cc
+        run: cmake --build --preset cl-release --target fastcached fastcache-cc fastcache-compile-node
 
       # On failure CPack only says "check wix.log", which lives in the build
       # tree and is otherwise discarded with the runner — so the actual WiX