From 1a999c7548d2abe5f997f18a5e9a592c7a496fee Mon Sep 17 00:00:00 2001 From: Mike Kell Date: Sat, 3 Oct 2026 17:25:22 -0400 Subject: [PATCH] Fix 9 bugs found by dev-container test suite; harden test harness - ssh.socket: preserve address families when rewriting ListenStream (Ubuntu 24.04 dual entries -> EADDRINUSE) - ufw: declare ufw-docker-logging-deny chain in after.rules (iptables-restore failed at line 47) - caddy: seed Caddyfile with comment + :80 health block (empty file -> adaptation EOF crash-loop) - mariadb: wait for authenticated SELECT 1 instead of ping (first-boot race, ERROR 1045) - create: CREATE USER host '\%' -> '%' (ERROR 1133, site provisioning never worked) - wp-cli: pass WORDPRESS_DB_* and WORDPRESS_CONFIG_EXTRA env (getenv_docker fallbacks) - wp-cli: add missing 'wp' prefix to all four invocations (exit 127) - compose: escape \ as \$\ so interpolation keeps PHP intact - devcontainer: ship daemon.json (overlay2) and document docker-storage volume for nested dockerd - smoke.sh: retry windows, no false-pass curl checks, credentialed DB check, full-env wp checks - README: podman workflow, troubleshooting, bugs-caught section --- .devcontainer/Dockerfile | 5 ++++ .devcontainer/daemon.json | 4 +++ README.md | 54 +++++++++++++++++++++++++++++++++++++++ tests/smoke.sh | 46 +++++++++++++++++++++++++-------- wp-engine.sh | 51 ++++++++++++++++++++++++++++++------ 5 files changed, 141 insertions(+), 19 deletions(-) create mode 100644 .devcontainer/daemon.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index aca259c..2c6950c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -48,6 +48,11 @@ RUN systemctl set-default multi-user.target \ sys-fs-fuse-connections.mount \ 2>/dev/null || true +# Docker-in-Docker: Docker 29's default containerd snapshotter fails to mount +# nested overlay layers (EINVAL on overlay-on-overlay). The classic overlay2 +# graphdriver (same as the official docker:dind image) works on kernels >= 5.11. +COPY daemon.json /etc/docker/daemon.json + # Container-friendly init: systemd wants SIGRTMIN+3 to shut down. STOPSIGNAL SIGRTMIN+3 diff --git a/.devcontainer/daemon.json b/.devcontainer/daemon.json new file mode 100644 index 0000000..3e64fba --- /dev/null +++ b/.devcontainer/daemon.json @@ -0,0 +1,4 @@ +{ + "storage-driver": "overlay2", + "features": { "containerd-snapshotter": false } +} diff --git a/README.md b/README.md index 25c73a4..2d609e9 100644 --- a/README.md +++ b/README.md @@ -544,6 +544,26 @@ STAGE=all bash tests/smoke.sh STAGE=cleanup bash tests/smoke.sh ``` +**Option C — Podman (verified working on this workstation):** + +```bash +# One-time machine setup: rootful, enough RAM for MariaDB, repo drive shared +podman machine init --now --rootful -m 6144 --cpus 4 -v "D:\:/mnt/d" + +podman build -t wp-engine-dev .devcontainer +# Named volume REQUIRED: nested dockerd can't use the container's own overlay +# as upperdir ("not supported as upperdir"); the volume binds real ext4 instead. +podman volume create docker-storage +podman run --rm -d --name wp-engine-dev --privileged \ + -p 80:80 -p 443:443 -p 81:81 -p 8080:8080 -p 3000:3000 -p 48222:48222 \ + -v docker-storage:/var/lib/docker \ + -v "D:/programming/Servers/SecureServer:/workspaces/SecureServer" wp-engine-dev + +# run the suites inside it +podman exec wp-engine-dev bash -c 'cd /workspaces/SecureServer && STAGE=static bash tests/smoke.sh' +podman exec wp-engine-dev bash -c 'STAGE=all bash /workspaces/SecureServer/tests/smoke.sh' +``` + ### 14.3 Notes & caveats - **First full run pulls ~1–2 GB** of images (Docker CE, WordPress, MariaDB, Grafana, Prometheus, …) and takes several minutes. @@ -554,9 +574,43 @@ STAGE=cleanup bash tests/smoke.sh --- +### 14.4 Troubleshooting the Podman-on-Windows setup + +Hard-won fixes from bringing this environment up (recorded so the next machine doesn't have to rediscover them): + +1. **`podman machine init` fails with `failed to pull … machine-os: The system cannot find the path specified`** — known bug in Podman ≤ 6.0.2 ([podman#29639](https://github.com/podman-container-tools/podman/issues/29639)). Download the machine-os blob manually from `quay.io/podman/machine-os` (auth token → manifest → layer blob, verify SHA256) and place it at the cache path podman expects: + `%USERPROFILE%\.local\share\containers\podman\machine\wsl\cache\.tar.zst` — the next `podman machine init` treats it as a cache hit. +2. **Containers fail with `crun: controller 'pids' is not available under …/non-systemd/…`** — the machine booted with WSL's init instead of systemd, so cgroups aren't managed. Append `[boot] systemd=true` to `/etc/wsl.conf` inside the machine (`wsl -d podman-machine-default -u root`), then `wsl --terminate podman-machine-default` + `podman machine start`. +3. **`machine start` fails with `ssh error: machine not in running state`** — usually a stale `win-sshproxy.exe` from a previous attempt holding the SSH port, which triggers a port reassignment and leaves sshd listening on the old port while the config says the new one. Fix: kill all `win-sshproxy`/`gvproxy` processes, `podman machine stop`, `wsl --terminate podman-machine-default`, verify both show *Stopped*, then `podman machine start`. +4. **`CreateFile \\.\pipe\docker_engine: All pipe instances are busy`** — Podman Desktop is holding the Docker-compatible pipe; Podman clients are unaffected. A clean machine stop/start cycle usually reclaims it. +5. **Machine dies mid-run (e.g., during a Docker CE install)** — WSL OOM. Ensure `%USERPROFILE%\.wslconfig` has a swap (e.g. `swap=8GB` under `[wsl2]`). +6. **`machine start` reports an SSH transition error but `podman version` shows a healthy Server** — verify with `podman ps` / `podman run quay.io/podman/hello`; if those work, the machine is usable. + +--- + +### 14.5 Bugs the suite caught (fixed) + +Every one of these broke `wp-engine.sh` on stock Ubuntu 24.04 and would have failed on a real host — exactly what the harness exists for: + +1. **SSH socket port rewrite** — Ubuntu 24.04's `ssh.socket` lists per-family entries (`0.0.0.0:22`, `[::]:22`); the blanket `sed` produced two identical `ListenStream=48222` lines → systemd `EADDRINUSE` → bootstrap aborted. Fixed with family-preserving substitutions. +2. **Missing `ufw-docker-logging-deny` chain declaration** in the `/etc/ufw/after.rules` DOCKER-USER block → `iptables-restore` error at line 47 → `ufw enable` failed. +3. **Empty `Caddyfile`** (`touch`) → `caddy run` fails adaptation with `EOF` and crash-loops. Now seeded with a comment + `:80` health-respond block. +4. **MariaDB first-boot race** — `mariadb-admin ping` reports success before root-password init completes; the first `create` got `ERROR 1045`. Replaced with an authenticated `SELECT 1` wait loop. +5. **`CREATE USER ... '@'\%'`** created the user with a literal `\%` host while `GRANT` targeted `%` → `ERROR 1133`. (This one meant site provisioning never worked at all.) +6. **WP-CLI containers missing `WORDPRESS_DB_*` env** — `wp-config.php` falls back to host `mysql` → "Error establishing a database connection". +7. **Missing `wp` binary prefix** in all four WP-CLI invocations → `exec: core: not found` (exit 127). +8. **WP-CLI containers missing `WORDPRESS_CONFIG_EXTRA`** — `wp redis enable` fell back to `127.0.0.1`. +9. **Compose `$VAR` interpolation ate `$_SERVER`** in the injected PHP block (`isset(['HTTP_X_FORWARDED_PROTO'])`) — broke HTTPS detection behind the proxy on every site. Fixed with `$$` escaping. + +Harness fixes (in `tests/smoke.sh`): Grafana/proxy assertions got 60 s retry windows, the `:80` check no longer false-passes on `curl` exit quirks, the DB check uses root credentials from `core/.env`, and the `wp core is-installed` check passes the site's full env (multi-line `WORDPRESS_CONFIG_EXTRA` included). + +--- + ## Repository Contents + + ``` wp-engine.sh # The installer/provisioner script (executable) README.md # This file diff --git a/tests/smoke.sh b/tests/smoke.sh index 20f8894..857d207 100755 --- a/tests/smoke.sh +++ b/tests/smoke.sh @@ -137,15 +137,28 @@ stage_bootstrap() { assert_cmd "node_exporter container running" docker inspect -f '{{.State.Running}}' node_exporter assert_cmd "cadvisor container running" docker inspect -f '{{.State.Running}}' cadvisor assert_cmd "grafana container running" docker inspect -f '{{.State.Running}}' grafana - assert_cmd "grafana health endpoint" curl -fsS http://localhost:3000/api/health assert_cmd "proxy access log created" test -f "${BASE_DIR}/proxy/logs/access.log" - local code - code="$(curl -sS -o /dev/null -w '%{http_code}' http://localhost/ 2>/dev/null || echo 000)" - if [ "${code}" != "000" ]; then + # Grafana needs a few seconds after its container starts + local gh="" i + for i in $(seq 1 30); do + gh="$(curl -fsS --max-time 5 http://localhost:3000/api/health 2>/dev/null || true)" + [ -n "${gh}" ] && break + sleep 2 + done + if [ -n "${gh}" ]; then pass "grafana health endpoint"; else fail "grafana health endpoint"; fi + + # Proxy must answer on :80 (give caddy a moment to bind) + local code="" + for i in $(seq 1 30); do + code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 http://localhost/ 2>/dev/null || true)" + [ -n "${code}" ] && [ "${code}" != "000" ] && break + sleep 2 + done + if [ -n "${code}" ] && [ "${code}" != "000" ]; then pass "proxy answers on :80 (HTTP ${code})" else - fail "proxy answers on :80" + fail "proxy answers on :80 (HTTP ${code:-000})" fi if ss -tln 2>/dev/null | grep -q ":${SSH_PORT} "; then @@ -177,21 +190,32 @@ stage_site() { assert_cmd "site directory created" test -d "${BASE_DIR}/sites/${TEST_SLUG}" assert_cmd "wp container running" docker inspect -f '{{.State.Running}}' "${TEST_SLUG}_wp" assert_cmd "wp-config.php written" test -f "${BASE_DIR}/sites/${TEST_SLUG}/data/wp/wp-config.php" + local dbroot + dbroot="$(grep '^DB_ROOT_PASSWORD=' "${BASE_DIR}/core/.env" 2>/dev/null | cut -d= -f2-)" assert_contains "database wp_${slug} exists" "wp_${slug}" \ - "$(docker exec core_mariadb mariadb -N -e 'SHOW DATABASES' 2>/dev/null || true)" + "$(docker exec core_mariadb mariadb -u root -p"${dbroot}" -N -e 'SHOW DATABASES' 2>/dev/null || true)" assert_contains "Caddyfile route appended" "${TEST_DOMAIN}" \ "$(cat "${BASE_DIR}/proxy/Caddyfile" 2>/dev/null || true)" # Site must answer through the ingress proxy (200 or any redirect is fine) local code - code="$(curl -sS -o /dev/null -w '%{http_code}' -H "Host: ${TEST_DOMAIN}" http://localhost/ 2>/dev/null || echo 000)" - case "${code}" in + code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 -H "Host: ${TEST_DOMAIN}" http://localhost/ 2>/dev/null || true)" + case "${code:-000}" in 200|301|302|303|307|308) pass "site answers via proxy (HTTP ${code})" ;; - *) fail "site answers via proxy (HTTP ${code})" ;; + *) fail "site answers via proxy (HTTP ${code:-000})" ;; esac - # WordPress reports itself installed - if docker run --rm --user 33:33 --network shared-backend \ + assert_contains "site env carries WP config (redis etc.)" "WP_REDIS_HOST" \ + "$(docker exec "${TEST_SLUG}_wp" printenv WORDPRESS_CONFIG_EXTRA 2>/dev/null || true)" + + # WordPress reports itself installed (wp-config needs DB + extra config env) + local -a wpdb_env=() + local vname vval + for vname in WORDPRESS_DB_HOST WORDPRESS_DB_NAME WORDPRESS_DB_USER WORDPRESS_DB_PASSWORD WORDPRESS_CONFIG_EXTRA; do + vval="$(docker exec "${TEST_SLUG}_wp" printenv "${vname}" 2>/dev/null || true)" + [ -n "${vval}" ] && wpdb_env+=(-e "${vname}=${vval}") + done + if docker run --rm --user 33:33 --network shared-backend "${wpdb_env[@]}" \ -v "${BASE_DIR}/sites/${TEST_SLUG}/data/wp:/var/www/html" \ wordpress:cli wp core is-installed >/dev/null 2>&1; then pass "wp core is-installed" diff --git a/wp-engine.sh b/wp-engine.sh index 4b6111f..8eb440f 100755 --- a/wp-engine.sh +++ b/wp-engine.sh @@ -122,7 +122,14 @@ run_host_bootstrap() { echo " [3/6] Hardening SSH on Port ${SSH_PORT}" echo "==========================================================" if systemctl is-active --quiet ssh.socket; then - sed -i -E "s/^ListenStream=.*/ListenStream=${SSH_PORT}/" /lib/systemd/system/ssh.socket || true + # Ubuntu 24.04 lists per-family sockets (0.0.0.0:22 and [::]:22); rewriting + # both to a bare port would make systemd bind the same address twice and + # fail with EADDRINUSE. Preserve/normalize each form explicitly. + sed -i -E \ + -e "s/^ListenStream=([0-9.]+):[0-9]+$/ListenStream=\1:${SSH_PORT}/" \ + -e "s/^ListenStream=\[::\]:[0-9]+$/ListenStream=[::]:${SSH_PORT}/" \ + -e "s/^ListenStream=[0-9]+$/ListenStream=${SSH_PORT}/" \ + /lib/systemd/system/ssh.socket || true systemctl daemon-reload systemctl restart ssh.socket fi @@ -160,6 +167,7 @@ EOF *filter :ufw-user-forward - [0:0] :DOCKER-USER - [0:0] +:ufw-docker-logging-deny - [0:0] -A DOCKER-USER -j ufw-user-forward -A DOCKER-USER -j RETURN -s 10.0.0.0/8 -A DOCKER-USER -j RETURN -s 172.16.0.0/12 @@ -289,7 +297,14 @@ networks: EOF elif [ "${PROXY_TYPE}" = "caddy" ]; then mkdir -p "${PROXY_DIR}/data" "${PROXY_DIR}/config" - touch "${PROXY_DIR}/Caddyfile" + # Seed with a comment: an EMPTY file makes `caddy run` fail adaptation with + # "EOF" and crash-loop. A comment-only Caddyfile validates cleanly. + cat <<'EOF' > "${PROXY_DIR}/Caddyfile" +# Managed by wp-engine.sh — site blocks are appended below this line. +:80 { + respond /_ingress/health 200 +} +EOF cat < "${PROXY_DIR}/docker-compose.yml" services: ingress: @@ -498,7 +513,9 @@ EOF compose_exec "${CORE_DIR}" up -d echo "==> Waiting for MariaDB health status..." - until cli_exec exec core_mariadb mariadb-admin ping -u root -p"${CORE_DB_ROOT_PW}" --silent >/dev/null 2>&1; do + # NOTE: `mariadb-admin ping` reports success while the server is still in + # first-boot initialization (auth not yet ready). Verify authenticated access. + until cli_exec exec core_mariadb mariadb -u root -p"${CORE_DB_ROOT_PW}" -e 'SELECT 1' >/dev/null 2>&1; do sleep 2 done echo "==> Core database and cache online." @@ -549,7 +566,7 @@ provision_site() { echo "==> Provisioning database credentials..." cli_exec exec -i core_mariadb mariadb -u root -p"${DB_ROOT_PASSWORD}" < Running automated WP-CLI bootstrap..." + # wp-config.php reads WORDPRESS_DB_* via getenv_docker() with a fallback host + # of 'mysql' — one-off WP-CLI containers must receive these explicitly. + local -a WP_CLI_ENV=( + -e "WORDPRESS_DB_HOST=core_mariadb:3306" + -e "WORDPRESS_DB_NAME=${DB_NAME}" + -e "WORDPRESS_DB_USER=${DB_USER}" + -e "WORDPRESS_DB_PASSWORD=${DB_PASS}" + -e "WORDPRESS_CONFIG_EXTRA=$(cli_exec exec "${SITE_SLUG}_wp" printenv WORDPRESS_CONFIG_EXTRA)" + ) + cli_exec run --rm \ --user 33:33 \ --network "${BACKEND_NETWORK}" \ + "${WP_CLI_ENV[@]}" \ -v "${SITE_DIR}/data/wp:/var/www/html" \ wordpress:cli \ + wp \ core install \ --url="https://${DOMAIN}" \ --title="${SITE_SLUG}" \ @@ -695,22 +724,28 @@ EOF cli_exec run --rm \ --user 33:33 \ --network "${BACKEND_NETWORK}" \ + "${WP_CLI_ENV[@]}" \ -v "${SITE_DIR}/data/wp:/var/www/html" \ wordpress:cli \ + wp \ rewrite structure '/%postname%/' --hard cli_exec run --rm \ --user 33:33 \ --network "${BACKEND_NETWORK}" \ + "${WP_CLI_ENV[@]}" \ -v "${SITE_DIR}/data/wp:/var/www/html" \ wordpress:cli \ + wp \ plugin install redis-cache --activate cli_exec run --rm \ --user 33:33 \ --network "${BACKEND_NETWORK}" \ + "${WP_CLI_ENV[@]}" \ -v "${SITE_DIR}/data/wp:/var/www/html" \ wordpress:cli \ + wp \ redis enable chown -R 33:33 "${SITE_DIR}/data/wp/wp-content"