dc4ee9a8db
Two prod-readiness hardening changes: DB tuning scales with the container budget. Replace the fixed 8 GB 20-tuning.sql with 20-tuning.sh, which derives shared_buffers (25%), effective_cache_size (75%), work_mem, maintenance_work_mem and duckdb.max_memory (50%) from the DB_MEMORY the compose db service now passes in. The ratios reproduce the historical 8 GB tuning exactly and scale linearly, so the 48 GB prod box (db_memory 16g) gets shared_buffers 4 GB / duckdb 8 GB with no separate edit. Beta/local (8g default) are unchanged. Docs that told operators to raise the tuning by hand are updated. Boot ordering for the self-hosted archive. On an openmeteo host, install a docker.service drop-in (Wants/After rclone-om.service) so Docker starts after the object-storage mount is ready on every boot — the restart-policy containers never bind an empty mount point. rclone-om is Type=notify, so After waits for the mount to actually be ready. Cleaned up when openmeteo is toggled off.
300 lines
12 KiB
HCL
300 lines
12 KiB
HCL
locals {
|
|
# THERMOGRAPH_DATABASE_URL is built from the same password compose uses for the db
|
|
# container, so the app always matches the database it initialized.
|
|
database_url = "postgresql+asyncpg://thermograph:${var.postgres_password}@db:5432/thermograph"
|
|
|
|
# A Secure cookie is only sent over HTTPS, so enable it only where Caddy terminates
|
|
# TLS (domain set). A plain-HTTP dev box (domain "") would otherwise drop its login
|
|
# cookie and no one could stay signed in.
|
|
cookie_secure = var.domain != "" ? "1" : "0"
|
|
|
|
# Public base URL: the domain over HTTPS, else the raw host:port for a Caddy-less box.
|
|
base_url = var.domain != "" ? "https://${var.domain}" : "http://${var.host}:${var.app_port}"
|
|
|
|
# Layer the self-hosted Open-Meteo overlay on hosts that self-host the archive.
|
|
effective_compose_files = var.openmeteo ? concat(var.compose_files, ["docker-compose.openmeteo.yml"]) : var.compose_files
|
|
|
|
# `-f a -f b` for the compose invocations (dev layers the dev overlay).
|
|
compose_flags = join(" ", [for f in local.effective_compose_files : "-f ${f}"])
|
|
|
|
# Hash the local compose files so a compose edit re-triggers the remote deploy.
|
|
compose_files_sha = join(",", [for f in local.effective_compose_files : filesha256("${var.repo_root}/${f}")])
|
|
|
|
# Rendered /etc/thermograph.env (sensitive — carries every secret).
|
|
env_content = templatefile("${path.module}/templates/thermograph.env.tftpl", {
|
|
app_port = var.app_port
|
|
postgres_password = var.postgres_password
|
|
database_url = local.database_url
|
|
auth_secret = var.auth_secret
|
|
workers = var.workers
|
|
app_cpus = var.app_cpus
|
|
db_cpus = var.db_cpus
|
|
db_memory = var.db_memory
|
|
base = "/"
|
|
base_url = local.base_url
|
|
cookie_secure = local.cookie_secure
|
|
openmeteo = var.openmeteo
|
|
om_data_dir = var.om_data_dir
|
|
vapid_private_key = var.vapid_private_key
|
|
vapid_public_key = var.vapid_public_key
|
|
vapid_contact = var.vapid_contact
|
|
google_verify = var.google_verify
|
|
bing_verify = var.bing_verify
|
|
mail_backend = var.mail_backend
|
|
smtp_host = var.smtp_host
|
|
smtp_port = var.smtp_port
|
|
smtp_user = var.smtp_user
|
|
smtp_password = var.smtp_password
|
|
smtp_starttls = var.smtp_starttls
|
|
mail_from = var.mail_from
|
|
mail_reply_to = var.mail_reply_to
|
|
discord_webhook = var.discord_webhook
|
|
discord_public_key = var.discord_public_key
|
|
discord_app_id = var.discord_app_id
|
|
discord_bot_token = var.discord_bot_token
|
|
discord_client_secret = var.discord_client_secret
|
|
})
|
|
|
|
# Caddyfile is only meaningful on a host with a public domain.
|
|
caddy_content = var.domain != "" ? templatefile("${path.module}/templates/Caddyfile.tftpl", {
|
|
domain = var.domain
|
|
port = var.app_port
|
|
}) : "# No public domain on this host; Caddy is not managed here.\n"
|
|
|
|
# systemd unit that keeps the object-storage bucket rclone-mounted at om_data_dir,
|
|
# so the Open-Meteo containers read the ERA5 .om archive from it. Only installed on
|
|
# openmeteo hosts; --allow-other lets the container (root) read the FUSE mount.
|
|
rclone_unit = <<-UNIT
|
|
[Unit]
|
|
Description=rclone mount ERA5 archive (Thermograph)
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=notify
|
|
ExecStartPre=/bin/mkdir -p ${var.om_data_dir}
|
|
ExecStart=/usr/bin/rclone mount ${var.om_bucket_remote} ${var.om_data_dir} --config /etc/rclone/rclone.conf --vfs-cache-mode full --vfs-cache-max-size ${var.om_vfs_cache_max} --dir-cache-time 12h --allow-other --umask 000
|
|
ExecStop=/bin/fusermount -u ${var.om_data_dir}
|
|
Restart=on-failure
|
|
RestartSec=5
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
UNIT
|
|
|
|
rclone_unit_content = var.openmeteo ? local.rclone_unit : "# openmeteo disabled on this host\n"
|
|
}
|
|
|
|
resource "null_resource" "host" {
|
|
# Re-provision when the rendered env, the compose files, the branch, sizing, or the
|
|
# Caddyfile change. sha256 of the secret-bearing env is unwrapped with nonsensitive()
|
|
# (a one-way hash leaks nothing) so plans stay readable.
|
|
triggers = {
|
|
env_sha = nonsensitive(sha256(local.env_content))
|
|
compose_sha = local.compose_files_sha
|
|
compose_flags = local.compose_flags
|
|
caddy_sha = sha256(local.caddy_content)
|
|
branch = var.git_branch
|
|
app_dir = var.app_dir
|
|
sizing = "${var.workers}/${var.app_cpus}/${var.db_cpus}/${var.db_memory}"
|
|
# Re-provision when the archive self-hosting config changes. The rclone.conf is
|
|
# hashed (nonsensitive on a one-way digest) so a credential rotation redeploys.
|
|
openmeteo = "${var.openmeteo}/${var.om_data_dir}/${var.om_bucket_remote}/${var.om_vfs_cache_max}"
|
|
om_conf = var.openmeteo ? nonsensitive(sha256(var.om_rclone_conf)) : "off"
|
|
}
|
|
|
|
connection {
|
|
type = "ssh"
|
|
host = var.host
|
|
user = var.ssh_user
|
|
private_key = file(pathexpand(var.ssh_private_key_path))
|
|
timeout = "5m"
|
|
}
|
|
|
|
# Push the rendered secrets via `content` so they are never written to local disk.
|
|
provisioner "file" {
|
|
content = local.env_content
|
|
destination = "/tmp/thermograph.env"
|
|
}
|
|
|
|
provisioner "file" {
|
|
content = local.caddy_content
|
|
destination = "/tmp/thermograph.Caddyfile"
|
|
}
|
|
|
|
# rclone config (bucket credentials) + the mount unit. Pushed via `content` so the
|
|
# secret never touches local disk; harmless placeholders on non-openmeteo hosts.
|
|
provisioner "file" {
|
|
content = var.openmeteo ? var.om_rclone_conf : "# openmeteo disabled on this host\n"
|
|
destination = "/tmp/thermograph.rclone.conf"
|
|
}
|
|
|
|
provisioner "file" {
|
|
content = local.rclone_unit_content
|
|
destination = "/tmp/rclone-om.service"
|
|
}
|
|
|
|
# 1. Host setup: Docker + compose plugin, ufw firewall, and (domain hosts) Caddy.
|
|
provisioner "remote-exec" {
|
|
inline = [
|
|
<<-EOT
|
|
set -eu
|
|
echo "[${var.name}] setup: docker, compose plugin, firewall"
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
curl -fsSL https://get.docker.com | sudo sh
|
|
fi
|
|
sudo usermod -aG docker "$(id -un)" || true
|
|
if ! sudo docker compose version >/dev/null 2>&1; then
|
|
sudo apt-get update -y
|
|
sudo apt-get install -y docker-compose-plugin
|
|
fi
|
|
if ! command -v ufw >/dev/null 2>&1; then
|
|
sudo apt-get update -y
|
|
sudo apt-get install -y ufw
|
|
fi
|
|
sudo ufw allow 22/tcp
|
|
sudo ufw allow 80/tcp
|
|
sudo ufw allow 443/tcp
|
|
DOMAIN='${var.domain}'
|
|
if [ -z "$DOMAIN" ]; then
|
|
sudo ufw allow ${var.app_port}/tcp
|
|
fi
|
|
sudo ufw --force enable
|
|
if [ -n "$DOMAIN" ]; then
|
|
if ! command -v caddy >/dev/null 2>&1; then
|
|
sudo apt-get install -y debian-keyring debian-archive-keyring apt-transport-https curl
|
|
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
|
|
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list >/dev/null
|
|
sudo apt-get update -y
|
|
sudo apt-get install -y caddy
|
|
fi
|
|
sudo install -m 0644 /tmp/thermograph.Caddyfile /etc/caddy/Caddyfile
|
|
sudo systemctl reload caddy || sudo systemctl restart caddy
|
|
fi
|
|
rm -f /tmp/thermograph.Caddyfile
|
|
EOT
|
|
]
|
|
}
|
|
|
|
# 1b. Self-hosted archive: rclone-mount the ERA5 bucket before compose up so
|
|
# open-meteo-api can read .om from object storage. Skipped on non-openmeteo hosts.
|
|
provisioner "remote-exec" {
|
|
inline = [
|
|
<<-EOT
|
|
set -eu
|
|
if [ "${var.openmeteo}" != "true" ]; then
|
|
rm -f /tmp/thermograph.rclone.conf /tmp/rclone-om.service
|
|
# Undo any prior openmeteo setup so a toggled-off host doesn't wait on a mount.
|
|
if [ -e /etc/systemd/system/docker.service.d/10-wait-rclone.conf ]; then
|
|
sudo rm -f /etc/systemd/system/docker.service.d/10-wait-rclone.conf
|
|
sudo systemctl disable --now rclone-om >/dev/null 2>&1 || true
|
|
sudo systemctl daemon-reload
|
|
fi
|
|
echo "[${var.name}] openmeteo: disabled"
|
|
exit 0
|
|
fi
|
|
echo "[${var.name}] openmeteo: rclone mount ${var.om_bucket_remote} -> ${var.om_data_dir}"
|
|
if ! command -v rclone >/dev/null 2>&1; then
|
|
curl -fsSL https://rclone.org/install.sh | sudo bash
|
|
fi
|
|
# FUSE allow_other so the container (root) can read a mount owned by this user.
|
|
if ! grep -q '^user_allow_other' /etc/fuse.conf 2>/dev/null; then
|
|
echo user_allow_other | sudo tee -a /etc/fuse.conf >/dev/null
|
|
fi
|
|
sudo install -d -m 0755 /etc/rclone
|
|
sudo install -m 0600 /tmp/thermograph.rclone.conf /etc/rclone/rclone.conf
|
|
sudo install -m 0644 /tmp/rclone-om.service /etc/systemd/system/rclone-om.service
|
|
rm -f /tmp/thermograph.rclone.conf /tmp/rclone-om.service
|
|
sudo install -d -m 0755 ${var.om_data_dir}
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable rclone-om
|
|
sudo systemctl restart rclone-om
|
|
# Wait for the mount before compose bind-mounts it.
|
|
i=0
|
|
while [ "$i" -lt 30 ]; do
|
|
if mountpoint -q ${var.om_data_dir}; then break; fi
|
|
i=$((i + 1)); sleep 2
|
|
done
|
|
if ! mountpoint -q ${var.om_data_dir}; then
|
|
echo "[${var.name}] RCLONE MOUNT FAILED" >&2
|
|
sudo systemctl status rclone-om --no-pager || true
|
|
exit 1
|
|
fi
|
|
# Order Docker after the mount on every boot, so the restart-policy containers
|
|
# never bind an empty mount point. rclone-om is Type=notify, so `After` waits
|
|
# until the mount is actually ready — not merely that the unit was launched.
|
|
sudo install -d -m 0755 /etc/systemd/system/docker.service.d
|
|
printf '[Unit]\nWants=rclone-om.service\nAfter=rclone-om.service\n' \
|
|
| sudo tee /etc/systemd/system/docker.service.d/10-wait-rclone.conf >/dev/null
|
|
sudo systemctl daemon-reload
|
|
echo "[${var.name}] openmeteo: mounted at ${var.om_data_dir}"
|
|
EOT
|
|
]
|
|
}
|
|
|
|
# 2. Sync the checkout to the host's branch (cloning first on a fresh box).
|
|
provisioner "remote-exec" {
|
|
inline = [
|
|
<<-EOT
|
|
set -eu
|
|
echo "[${var.name}] code: ${var.app_dir} -> origin/${var.git_branch}"
|
|
sudo mkdir -p ${var.app_dir}
|
|
sudo chown -R "$(id -un)":"$(id -gn)" ${var.app_dir}
|
|
if [ ! -d ${var.app_dir}/.git ]; then
|
|
git clone ${var.repo_url} ${var.app_dir}
|
|
fi
|
|
cd ${var.app_dir}
|
|
git fetch --prune origin ${var.git_branch}
|
|
git reset --hard origin/${var.git_branch}
|
|
EOT
|
|
]
|
|
}
|
|
|
|
# 3. Install /etc/thermograph.env, then bring the stack up. docker runs as root
|
|
# (sources the env file in the same shell) so it never depends on the docker
|
|
# group membership taking effect in this session.
|
|
provisioner "remote-exec" {
|
|
inline = [
|
|
<<-EOT
|
|
set -eu
|
|
echo "[${var.name}] deploy: install env + docker compose up"
|
|
sudo install -m 0640 -o root -g root /tmp/thermograph.env /etc/thermograph.env
|
|
rm -f /tmp/thermograph.env
|
|
sudo bash -c 'set -a; . /etc/thermograph.env; set +a; cd ${var.app_dir} && docker compose ${local.compose_flags} up -d --build'
|
|
EOT
|
|
]
|
|
}
|
|
|
|
# 4. Health check the app on loopback.
|
|
provisioner "remote-exec" {
|
|
inline = [
|
|
<<-EOT
|
|
set -eu
|
|
echo "[${var.name}] health: http://127.0.0.1:${var.app_port}/"
|
|
ok=0
|
|
i=0
|
|
while [ "$i" -lt 30 ]; do
|
|
if curl -fsS -o /dev/null "http://127.0.0.1:${var.app_port}/"; then ok=1; break; fi
|
|
i=$((i + 1))
|
|
sleep 2
|
|
done
|
|
if [ "$ok" != 1 ]; then
|
|
echo "[${var.name}] HEALTH CHECK FAILED" >&2
|
|
sudo bash -c 'cd ${var.app_dir} && docker compose ${local.compose_flags} ps; docker compose ${local.compose_flags} logs --tail=50 app' || true
|
|
exit 1
|
|
fi
|
|
echo "[${var.name}] OK: serving on 127.0.0.1:${var.app_port}"
|
|
EOT
|
|
]
|
|
}
|
|
}
|
|
|
|
output "host" {
|
|
description = "Address this module manages."
|
|
value = var.host
|
|
}
|
|
|
|
output "role" {
|
|
description = "Role of this host."
|
|
value = var.role
|
|
}
|