emi
de8e847f9f
All checks were successful
Build + push backend image (Forgejo registry) / build-push (push) Successful in 1m23s
Build + push frontend image (Forgejo registry) / build-push (push) Successful in 1m22s
Sync infra to hosts / sync-beta (push) Successful in 9s
Sync infra to hosts / sync-prod (push) Successful in 8s
secrets-guard / encrypted (push) Successful in 6s
shell-lint / shellcheck (push) Successful in 9s
Deploy backend to beta VPS / deploy (push) Successful in 2m4s
Deploy frontend to beta VPS / deploy (push) Successful in 1m8s
Adds .forgejo/workflows/shell-lint.yml (pinned shellcheck v0.11.0 + sha256, -x, default severity, fail on any finding, not path-filtered) and drives all 26 scripts to zero findings. Two defects shellcheck cannot see: render-secrets.sh left DECRYPTED vault contents in /tmp whenever a sops decrypt failed -- the caller's set -e aborted the function before either cleanup ran. Now removed on every exit path, with `|| return 1` on both sops calls so a decrypt failure can never write a partial /etc/thermograph.env regardless of the caller's shell options. Explicitly not a `trap ... RETURN`: such a trap set in a sourced function persists into the caller's shell and re-fires when the caller's next `.`/source completes, where the function-local tmp is unset -- fatal and silent under deploy.sh's set -u. The file now records that reasoning. autoscale.sh ran `set -eu` without pipefail while piping docker stats into awk, so a failed left side was swallowed and the loop autoscaled on empty input. Promoted to pipefail with a missed sample treated as a skip; verified busybox ash in docker:27-cli supports it. Also: capture-fixtures.sh's `jq . || cat` ran cat after jq had consumed stdin, silently writing truncated fixtures; deploy.sh/deploy-stack.sh `# shellcheck source=` paths corrected for the monorepo layout.
100 lines
3.7 KiB
Bash
Executable file
100 lines
3.7 KiB
Bash
Executable file
#!/bin/sh
|
|
# Autoscaler for the stack's `web` service: scale replicas between
|
|
# MIN_REPLICAS and MAX_REPLICAS on sustained per-task CPU.
|
|
#
|
|
# Runs as a Swarm service on the manager with the docker socket mounted (see
|
|
# thermograph-stack.yml). Every web task is placed on this node today, so
|
|
# node-local `docker stats` sees them all — when a second app node exists,
|
|
# this needs a per-node reader or a metrics-based signal instead; that's the
|
|
# documented upgrade path, not a today problem.
|
|
#
|
|
# Semantics (deliberately boring):
|
|
# - Sample avg CPU% per web task every POLL_SECONDS (docker stats CPUPerc:
|
|
# 100 = one full host core).
|
|
# - UP_SAMPLES consecutive samples above SCALE_UP_CPU -> scale +1.
|
|
# - DOWN_SAMPLES consecutive samples below SCALE_DOWN_CPU -> scale -1.
|
|
# (Down is ~7x slower than up on defaults: flap-averse by construction.)
|
|
# - COOLDOWN_SECONDS after any change: samples are ignored entirely.
|
|
# - Clamped to [MIN_REPLICAS, MAX_REPLICAS]; scaling waits for convergence
|
|
# (--detach=false), so a stuck rollout blocks further changes rather than
|
|
# stacking them.
|
|
# pipefail: without it a `docker stats` failure on the left of the avg_cpu
|
|
# pipe is silently swallowed by awk's exit 0 and the loop trusts the output.
|
|
# (Runs under busybox ash in docker:*-cli, which supports pipefail.)
|
|
# shellcheck disable=SC3040 # pipefail is not POSIX, but the busybox ash this runs under has it
|
|
set -euo pipefail
|
|
|
|
STACK_NAME="${STACK_NAME:-thermograph}"
|
|
SERVICE="${STACK_NAME}_web"
|
|
MIN="${MIN_REPLICAS:-1}"
|
|
MAX="${MAX_REPLICAS:-3}"
|
|
UP_AT="${SCALE_UP_CPU:-220}"
|
|
DOWN_AT="${SCALE_DOWN_CPU:-60}"
|
|
POLL="${POLL_SECONDS:-15}"
|
|
UP_N="${UP_SAMPLES:-3}"
|
|
DOWN_N="${DOWN_SAMPLES:-20}"
|
|
COOLDOWN="${COOLDOWN_SECONDS:-180}"
|
|
|
|
up_hits=0
|
|
down_hits=0
|
|
last_change=0
|
|
|
|
log() { echo "[autoscale] $(date -u +%H:%M:%S) $*"; }
|
|
|
|
replicas() {
|
|
docker service inspect "$SERVICE" \
|
|
--format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null || echo ""
|
|
}
|
|
|
|
avg_cpu() {
|
|
# Mean CPUPerc across this node's web tasks, as an integer percent.
|
|
docker stats --no-stream --format '{{.Name}} {{.CPUPerc}}' 2>/dev/null \
|
|
| awk -v svc="$SERVICE" '
|
|
index($1, svc".") == 1 {
|
|
gsub(/%/, "", $2); sum += $2; n++
|
|
}
|
|
END { if (n > 0) printf "%d", sum / n; else print "" }'
|
|
}
|
|
|
|
log "watching $SERVICE: min=$MIN max=$MAX up>@${UP_AT}%x${UP_N} down<@${DOWN_AT}%x${DOWN_N} poll=${POLL}s cooldown=${COOLDOWN}s"
|
|
|
|
while :; do
|
|
sleep "$POLL"
|
|
|
|
now=$(date +%s)
|
|
if [ $((now - last_change)) -lt "$COOLDOWN" ]; then
|
|
continue
|
|
fi
|
|
|
|
cur=$(replicas)
|
|
[ -n "$cur" ] || { log "service $SERVICE not found; waiting"; continue; }
|
|
# With pipefail, a transient `docker stats` failure makes avg_cpu return
|
|
# non-zero; treat that as a missed sample (don't trust partial output, and
|
|
# don't let set -e kill the autoscaler over a daemon hiccup).
|
|
cpu=$(avg_cpu) || cpu=""
|
|
[ -n "$cpu" ] || continue # no running tasks visible this sample
|
|
|
|
if [ "$cpu" -gt "$UP_AT" ]; then
|
|
up_hits=$((up_hits + 1)); down_hits=0
|
|
elif [ "$cpu" -lt "$DOWN_AT" ]; then
|
|
down_hits=$((down_hits + 1)); up_hits=0
|
|
else
|
|
up_hits=0; down_hits=0
|
|
fi
|
|
|
|
if [ "$up_hits" -ge "$UP_N" ] && [ "$cur" -lt "$MAX" ]; then
|
|
target=$((cur + 1))
|
|
log "avg cpu ${cpu}% > ${UP_AT}% x${UP_N}: scaling $cur -> $target"
|
|
if docker service scale --detach=false "$SERVICE=$target"; then
|
|
last_change=$(date +%s)
|
|
fi
|
|
up_hits=0; down_hits=0
|
|
elif [ "$down_hits" -ge "$DOWN_N" ] && [ "$cur" -gt "$MIN" ]; then
|
|
target=$((cur - 1))
|
|
log "avg cpu ${cpu}% < ${DOWN_AT}% x${DOWN_N}: scaling $cur -> $target"
|
|
if docker service scale --detach=false "$SERVICE=$target"; then
|
|
last_change=$(date +%s)
|
|
fi
|
|
up_hits=0; down_hits=0
|
|
fi
|
|
done
|