thermograph/infra/ops/iceberg.sh

#!/usr/bin/env bash
# iceberg -- run READ-ONLY SQL against the Thermograph Iceberg lake (LAN dev / beta / prod).
#
#   infra/ops/iceberg.sh dev  -c "show tables"
#   infra/ops/iceberg.sh prod -c "select count(*) from era5_daily"
#   infra/ops/iceberg.sh beta -json -c "select * from era5_daily limit 3"
#   echo "select 1" | infra/ops/iceberg.sh prod -f -
#
# Why an ephemeral container instead of a query service: the lake is one shared
# warehouse in Contabo object storage (s3://era5-thermograph/iceberg), readable
# from every environment, so the environments differ only in WHERE the engine
# runs -- a throwaway `docker run` of a small DuckDB image on the target box
# (vps1 for dev, vps2 for beta/prod -- the SSH target comes from
# env-topology.sh, never a hardcoded table; beta and prod are two Swarm stacks
# on the SAME vps2 box now). No daemon, no listening port, no tunnel (prod's
# overlay cannot be tunnelled), and no container names to resolve, so prod
# redeploys cannot break it.
#
# There is no catalog service: a table's current state is its newest metadata
# JSON under <warehouse>/<table>/metadata/, resolved at call time and exposed
# as a view named after the table directory.
#
# Safety: before user SQL runs, the session pins external access to the lake
# prefixes (iceberg/ metadata + era5/ data files) and locks the configuration
# -- anything else, notably backups/ and all local files, is refused by
# DuckDB itself, and the lockdown cannot be SET away. The provisioned S3
# keypair is read-write, so a write *inside* those prefixes is still possible
# until a scoped read-only keypair exists (see infra/ops/README.md).
#
# Credentials are resolved at call time, never stored here: the target host's
# /etc/thermograph.env (THERMOGRAPH_LAKE_S3_*) wins if present; otherwise the
# calling machine decrypts the SOPS vault and feeds the two values to the
# remote shell over stdin (never argv, so never visible in `ps`).
#
# Any extra arguments are passed straight through to the duckdb CLI, so -c,
# -json, -csv, -line, -f etc. all work. With `-f -`, stdin is forwarded and
# read as piped SQL. A single bare argument is treated as `-c` SQL.
set -euo pipefail

SELF_DIR=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=infra/deploy/env-topology.sh
. "$SELF_DIR/../deploy/env-topology.sh"

KEYFILE="${THERMOGRAPH_AGENT_KEY:-$HOME/.ssh/thermograph_agent_ed25519}"

usage() {
  sed -n '2,8p' "$0" | sed 's/^# \{0,1\}//' >&2
  echo "environments: dev | beta | prod" >&2
  exit 2
}

env_name="${1:-}"
[ -n "$env_name" ] || usage
shift

case "$env_name" in
  dev|beta|prod) ;;
  -h|--help) usage ;;
  *) echo "iceberg: unknown environment '$env_name' (want: dev|beta|prod)" >&2; exit 2 ;;
esac

# SSH target comes from env-topology.sh, not a hardcoded table: beta and prod
# are two Swarm stacks on the SAME box (vps2) now, so a stale beta entry here
# would silently point a beta query at the wrong host.
thermograph_topology "$env_name"
ssh_target="$TG_SSH_TARGET"

[ $# -gt 0 ] || usage

# The engine image: duckdb CLI with the httpfs + iceberg extensions installed
# at build time, so a query needs no downloads. Built on the target host the
# first time it's needed; the tag is a hash of this Dockerfile, so editing it
# here rolls every host forward automatically on the next call.
dockerfile=$(cat <<'DOCKERFILE'
FROM debian:bookworm-slim
RUN apt-get update \
 && apt-get install -y --no-install-recommends ca-certificates curl \
 && rm -rf /var/lib/apt/lists/*
RUN curl -fsSL https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.gz \
      | gunzip > /usr/local/bin/duckdb \
 && chmod +x /usr/local/bin/duckdb
RUN duckdb -c "INSTALL httpfs; INSTALL iceberg;"
COPY <<'ENTRY' /usr/local/bin/iceberg-query
#!/bin/sh
# Entrypoint of the thermograph-iceberg image; run by iceberg.sh, not directly.
set -eu
: "${LAKE_S3_ACCESS_KEY:?}" "${LAKE_S3_SECRET_KEY:?}"
WAREHOUSE="${LAKE_WAREHOUSE:-s3://era5-thermograph/iceberg}"
ENDPOINT="${LAKE_S3_ENDPOINT:-eu2.contabostorage.com}"
# Prefixes a query may touch. era5/ is needed because era5_daily was built
# with add_files over the existing hive parquet: its metadata lives in the
# warehouse but its data files stay in place under era5/daily/.
ALLOWED="${LAKE_ALLOWED_DIRS:-$WAREHOUSE/,s3://era5-thermograph/era5/}"

esc() { printf %s "$1" | sed "s/'/''/g"; }
SECRET="CREATE SECRET lake (TYPE s3, KEY_ID '$(esc "$LAKE_S3_ACCESS_KEY")', SECRET '$(esc "$LAKE_S3_SECRET_KEY")', ENDPOINT '$ENDPOINT', REGION 'default', URL_STYLE 'path');"

# One view per table dir, over its newest metadata JSON (numeric version wins,
# so v10 beats v9 and 00010-... beats 00009-...). grep drops statement noise
# like CREATE SECRET's "true" row; an empty warehouse yields no views.
VIEWS=$(duckdb -batch -noheader -list \
  -cmd "LOAD httpfs; $SECRET" \
  -c "SELECT 'CREATE VIEW \"' || tbl || '\" AS SELECT * FROM iceberg_scan(''' || file || ''');'
      FROM (SELECT file,
                   split_part(replace(file, '$WAREHOUSE/', ''), '/', 1) AS tbl,
                   row_number() OVER (
                     PARTITION BY split_part(replace(file, '$WAREHOUSE/', ''), '/', 1)
                     ORDER BY coalesce(try_cast(regexp_extract(parse_filename(file), '[0-9]+') AS BIGINT), -1) DESC,
                              file DESC) AS rn
            FROM glob('$WAREHOUSE/*/metadata/*.metadata.json'))
      WHERE rn = 1;" | grep '^CREATE VIEW ' || true)

# Lock the session down BEFORE user SQL: external access is pinned to the
# allowed prefixes, local files are unreachable, and the config can't be
# un-SET. Views bind lazily, so reads still work under the lockdown.
allowed_sql=
oifs=$IFS; IFS=,
for d in $ALLOWED; do allowed_sql="$allowed_sql${allowed_sql:+, }'$d'"; done
IFS=$oifs

INIT="LOAD httpfs; LOAD iceberg; $SECRET $VIEWS
SET allowed_directories=[$allowed_sql];
SET enable_external_access=false;
SET lock_configuration=true;"

# A single bare argument is the SQL itself (dbq.sh calling convention).
if [ $# -eq 1 ] && [ "${1#-}" = "$1" ]; then set -- -c "$1"; fi

# duckdb has no "-f - means stdin" convention; hand it the real thing.
i=0; n=$#; prev=
while [ "$i" -lt "$n" ]; do
  a=$1; shift
  if [ "$prev" = "-f" ] && [ "$a" = "-" ]; then a=/dev/stdin; fi
  prev=$a
  set -- "$@" "$a"
  i=$((i+1))
done

# The .output pair silences the init's own result rows (CREATE SECRET prints
# a "true"); errors still reach stderr, and user SQL output is untouched.
exec duckdb -batch -cmd ".output /dev/null" -cmd "$INIT" -cmd ".output" "$@"
ENTRY
RUN chmod +x /usr/local/bin/iceberg-query
DOCKERFILE
)

tag="thermograph-iceberg:$(printf '%s' "$dockerfile" | sha256sum | cut -c1-12)"
b64=$(printf '%s' "$dockerfile" | base64 -w0)

# Quote the duckdb arguments so they survive the remote shell intact.
args=$(printf '%q ' "$@")

# Credentials: env override first, then a call-time sops decrypt of the vault
# on the calling machine. Either may come up empty here -- the remote side
# still gets a chance to fill them from its own /etc/thermograph.env.
ak="${THERMOGRAPH_LAKE_S3_ACCESS_KEY:-}"
sk="${THERMOGRAPH_LAKE_S3_SECRET_KEY:-}"
if [ -z "$ak" ] || [ -z "$sk" ]; then
  repo_root=$(cd -- "$(dirname -- "$0")/../.." && pwd)
  vault="${THERMOGRAPH_LAKE_VAULT:-$repo_root/infra/deploy/secrets/prod.yaml}"
  sops_bin=$(command -v sops || echo "$HOME/.local/bin/sops")
  if [ -r "$vault" ] && [ -x "$sops_bin" ]; then
    ak=$("$sops_bin" -d --extract '["THERMOGRAPH_LAKE_S3_ACCESS_KEY"]' "$vault" 2>/dev/null || true)
    sk=$("$sops_bin" -d --extract '["THERMOGRAPH_LAKE_S3_SECRET_KEY"]' "$vault" 2>/dev/null || true)
  fi
fi

# The engine image is (re)built on the target host only when its tag is
# missing, then run with the credentials passed via environment -- docker
# inherits them from the remote shell, which read them from stdin, so they
# never appear on a command line. The rest of stdin flows through to duckdb.
remote=$(cat <<REMOTE
set -eu
IFS= read -r ak; IFS= read -r sk
if [ -r /etc/thermograph.env ]; then
  hak=\$(sed -n 's/^THERMOGRAPH_LAKE_S3_ACCESS_KEY=//p' /etc/thermograph.env | tail -n1)
  hsk=\$(sed -n 's/^THERMOGRAPH_LAKE_S3_SECRET_KEY=//p' /etc/thermograph.env | tail -n1)
  if [ -n "\$hak" ] && [ -n "\$hsk" ]; then ak=\$hak sk=\$hsk; fi
fi
if [ -z "\$ak" ] || [ -z "\$sk" ]; then
  echo "iceberg: no lake credentials (vault decrypt failed on the caller and no THERMOGRAPH_LAKE_S3_* in /etc/thermograph.env)" >&2
  exit 1
fi
export LAKE_S3_ACCESS_KEY="\$ak" LAKE_S3_SECRET_KEY="\$sk"
docker image inspect $tag >/dev/null 2>&1 \
  || echo $b64 | base64 -d | docker build -q -t $tag - >/dev/null
exec docker run --rm -i -e LAKE_S3_ACCESS_KEY -e LAKE_S3_SECRET_KEY $tag iceberg-query $args
REMOTE
)

# Stdin is forwarded only for the documented `-f -` form. Anything else gets
# just the credential lines -- unconditionally forwarding an idle stdin would
# leave the local pipeline waiting on input the query never reads.
want_stdin=false
prev=
for a in "$@"; do
  if [ "$prev" = "-f" ] && [ "$a" = "-" ]; then want_stdin=true; fi
  prev=$a
done

feed() {
  printf '%s\n%s\n' "$ak" "$sk" || true
  if $want_stdin; then cat || true; fi
}

if [ -z "$ssh_target" ]; then
  feed | bash -c "$remote"
else
  feed | ssh -i "$KEYFILE" -o StrictHostKeyChecking=no -o ConnectTimeout=15 \
    "$ssh_target" "$remote"
fi