From 67d1734e99e7050db1cc17d28c7d665a082f375e Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 19:34:30 +0000 Subject: [PATCH] ops/iceberg.sh: read-only Iceberg lake queries across all environments (#23) --- infra/ops/README.md | 89 ++++++++++++++++--- infra/ops/iceberg.sh | 200 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 275 insertions(+), 14 deletions(-) create mode 100755 infra/ops/iceberg.sh diff --git a/infra/ops/README.md b/infra/ops/README.md index 0f4db3b..f7aeb7d 100644 --- a/infra/ops/README.md +++ b/infra/ops/README.md @@ -46,20 +46,81 @@ GRANT CONNECT ON DATABASE thermograph TO thermograph_ro; GRANT pg_read_all_data TO thermograph_ro; ``` -## Iceberg (planned) +## `iceberg.sh` — the Iceberg lake, any environment -**Implementation spec: [`ICEBERG-HANDOFF.md`](./ICEBERG-HANDOFF.md)** — the -contract, environment constraints, acceptance criteria and anti-goals for the -Iceberg worker. +```sh +infra/ops/iceberg.sh dev -c "show tables" +infra/ops/iceberg.sh prod -c "select count(*) from era5_daily" +infra/ops/iceberg.sh beta -json -c "select * from era5_daily limit 3" +echo "select 1" | infra/ops/iceberg.sh prod -f - +``` -Iceberg lands as a sibling in this directory following the same three -conventions, so the query surface stays uniform: +Same environments and argument shape as `dbq.sh` (spec: +[`ICEBERG-HANDOFF.md`](./ICEBERG-HANDOFF.md)). Extra args pass straight +through to the duckdb CLI (`-json`, `-csv`, `-line`, `-c`, `-f`, …); with +`-f -`, stdin is forwarded and read as piped SQL; a single bare argument is +treated as `-c` SQL. -1. **Env-keyed dispatch** — `iceberg.sh ""`, same argument - shape as `dbq.sh`. -2. **Run the engine where the data is reachable** — the catalog/warehouse - dictates it (a Trino/DuckDB/pyiceberg invocation, containerised the same way), - rather than exposing new network endpoints. -3. **A dedicated read-only identity** — the Iceberg equivalent of - `thermograph_ro`, so the same "read-only enforced by the system, not by - convention" property holds. +### Why an ephemeral DuckDB container instead of a query service + +The lake is **one shared warehouse** in Contabo object storage +(`s3://era5-thermograph/iceberg`), reachable from every environment with the +same credentials — so unlike Postgres there is no per-env database to reach, +and the environments differ only in *where the engine runs*: a throwaway +`docker run` of a small DuckDB image (duckdb CLI + `httpfs`/`iceberg` +extensions pre-installed) on the target box — locally for LAN dev, over SSH +for beta/prod. The image is built on the host the first time it's needed from +a Dockerfile embedded in the script; its tag is a hash of that Dockerfile, so +editing the script rolls every host forward on the next call. + +A container per query means no daemon, no listening port, no tunnel (prod's +overlay cannot be tunnelled), no firewall or Swarm changes — `sudo ss -ltnp` +is identical before and after a query. It also never execs into an app +container and resolves no container names, so prod redeploys can't break it. + +There is no catalog service to run or expose: an Iceberg table's current +state is fully described by the newest metadata JSON under +`//metadata/`. `iceberg.sh` resolves that at call time +(numeric metadata version, newest wins) and exposes each table directory as a +view of the same name — which is also how it keeps reading fresh snapshots of +a table that is still being loaded. + +Credentials are resolved at call time and never stored in the repo: the +target host's `/etc/thermograph.env` (`THERMOGRAPH_LAKE_S3_*`) wins if +present; otherwise the calling machine decrypts the SOPS vault +(`infra/deploy/secrets/prod.yaml`) and hands the two values to the remote +shell over stdin — never argv, so never visible in `ps`, and never written to +a file on the target. + +### Safety + +Every session locks itself down *before* user SQL runs: + +```sql +SET allowed_directories=['s3://era5-thermograph/iceberg/', 's3://era5-thermograph/era5/']; +SET enable_external_access=false; +SET lock_configuration=true; +``` + +Both prefixes are required for reads: `era5_daily` was built with pyiceberg +`add_files` over the existing hive parquet, so its metadata lives under +`iceberg/` while its data files stay in place under `era5/daily/`. DuckDB +itself then refuses everything else — the live `backups/` prefix, all local +files — and the lockdown cannot be SET away by query text: + +``` +$ infra/ops/iceberg.sh prod -c "COPY (SELECT 1) TO 's3://era5-thermograph/backups/x.csv'" +Permission Error: Cannot access file "s3://era5-thermograph/backups/x.csv" - file system operations are disabled by configuration +$ infra/ops/iceberg.sh dev -c "SET enable_external_access=true" +Invalid Input Error: Cannot change configuration option "enable_external_access" - the configuration has been locked +``` + +**Known gap:** the only provisioned keypair for the bucket is read-write, so +a raw `COPY TO` targeting a path *inside* the two allowed prefixes is not +refused — the DuckDB iceberg extension cannot write Iceberg tables, but it +could still clobber raw objects under `iceberg/` or `era5/`. Path-based +restriction cannot both allow reading a prefix and forbid writing it; closing +the gap needs a scoped read-only keypair (the object-storage equivalent of +`thermograph_ro`). When the operator mints one, render it as +`THERMOGRAPH_LAKE_S3_*` on the hosts or swap it into the vault — +`iceberg.sh` needs no code change. diff --git a/infra/ops/iceberg.sh b/infra/ops/iceberg.sh new file mode 100755 index 0000000..693e54d --- /dev/null +++ b/infra/ops/iceberg.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# iceberg -- run READ-ONLY SQL against the Thermograph Iceberg lake (LAN dev / beta / prod). +# +# infra/ops/iceberg.sh dev -c "show tables" +# infra/ops/iceberg.sh prod -c "select count(*) from era5_daily" +# infra/ops/iceberg.sh beta -json -c "select * from era5_daily limit 3" +# echo "select 1" | infra/ops/iceberg.sh prod -f - +# +# Why an ephemeral container instead of a query service: the lake is one shared +# warehouse in Contabo object storage (s3://era5-thermograph/iceberg), readable +# from every environment, so the environments differ only in WHERE the engine +# runs -- a throwaway `docker run` of a small DuckDB image on the target box. +# No daemon, no listening port, no tunnel (prod's overlay cannot be tunnelled), +# and no container names to resolve, so prod redeploys cannot break it. +# +# There is no catalog service: a table's current state is its newest metadata +# JSON under /
/metadata/, resolved at call time and exposed +# as a view named after the table directory. +# +# Safety: before user SQL runs, the session pins external access to the lake +# prefixes (iceberg/ metadata + era5/ data files) and locks the configuration +# -- anything else, notably backups/ and all local files, is refused by +# DuckDB itself, and the lockdown cannot be SET away. The provisioned S3 +# keypair is read-write, so a write *inside* those prefixes is still possible +# until a scoped read-only keypair exists (see infra/ops/README.md). +# +# Credentials are resolved at call time, never stored here: the target host's +# /etc/thermograph.env (THERMOGRAPH_LAKE_S3_*) wins if present; otherwise the +# calling machine decrypts the SOPS vault and feeds the two values to the +# remote shell over stdin (never argv, so never visible in `ps`). +# +# Any extra arguments are passed straight through to the duckdb CLI, so -c, +# -json, -csv, -line, -f etc. all work. With `-f -`, stdin is forwarded and +# read as piped SQL. A single bare argument is treated as `-c` SQL. +set -euo pipefail + +KEYFILE="${THERMOGRAPH_AGENT_KEY:-$HOME/.ssh/thermograph_agent_ed25519}" + +usage() { + sed -n '2,8p' "$0" | sed 's/^# \{0,1\}//' >&2 + echo "environments: dev | beta | prod" >&2 + exit 2 +} + +env_name="${1:-}" +[ -n "$env_name" ] || usage +shift + +case "$env_name" in + dev) ssh_target= ;; + beta) ssh_target=agent@75.119.132.91 ;; + prod) ssh_target=agent@169.58.46.181 ;; + -h|--help) usage ;; + *) echo "iceberg: unknown environment '$env_name' (want: dev|beta|prod)" >&2; exit 2 ;; +esac + +[ $# -gt 0 ] || usage + +# The engine image: duckdb CLI with the httpfs + iceberg extensions installed +# at build time, so a query needs no downloads. Built on the target host the +# first time it's needed; the tag is a hash of this Dockerfile, so editing it +# here rolls every host forward automatically on the next call. +dockerfile=$(cat <<'DOCKERFILE' +FROM debian:bookworm-slim +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +RUN curl -fsSL https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.gz \ + | gunzip > /usr/local/bin/duckdb \ + && chmod +x /usr/local/bin/duckdb +RUN duckdb -c "INSTALL httpfs; INSTALL iceberg;" +COPY <<'ENTRY' /usr/local/bin/iceberg-query +#!/bin/sh +# Entrypoint of the thermograph-iceberg image; run by iceberg.sh, not directly. +set -eu +: "${LAKE_S3_ACCESS_KEY:?}" "${LAKE_S3_SECRET_KEY:?}" +WAREHOUSE="${LAKE_WAREHOUSE:-s3://era5-thermograph/iceberg}" +ENDPOINT="${LAKE_S3_ENDPOINT:-eu2.contabostorage.com}" +# Prefixes a query may touch. era5/ is needed because era5_daily was built +# with add_files over the existing hive parquet: its metadata lives in the +# warehouse but its data files stay in place under era5/daily/. +ALLOWED="${LAKE_ALLOWED_DIRS:-$WAREHOUSE/,s3://era5-thermograph/era5/}" + +esc() { printf %s "$1" | sed "s/'/''/g"; } +SECRET="CREATE SECRET lake (TYPE s3, KEY_ID '$(esc "$LAKE_S3_ACCESS_KEY")', SECRET '$(esc "$LAKE_S3_SECRET_KEY")', ENDPOINT '$ENDPOINT', REGION 'default', URL_STYLE 'path');" + +# One view per table dir, over its newest metadata JSON (numeric version wins, +# so v10 beats v9 and 00010-... beats 00009-...). grep drops statement noise +# like CREATE SECRET's "true" row; an empty warehouse yields no views. +VIEWS=$(duckdb -batch -noheader -list \ + -cmd "LOAD httpfs; $SECRET" \ + -c "SELECT 'CREATE VIEW \"' || tbl || '\" AS SELECT * FROM iceberg_scan(''' || file || ''');' + FROM (SELECT file, + split_part(replace(file, '$WAREHOUSE/', ''), '/', 1) AS tbl, + row_number() OVER ( + PARTITION BY split_part(replace(file, '$WAREHOUSE/', ''), '/', 1) + ORDER BY coalesce(try_cast(regexp_extract(parse_filename(file), '[0-9]+') AS BIGINT), -1) DESC, + file DESC) AS rn + FROM glob('$WAREHOUSE/*/metadata/*.metadata.json')) + WHERE rn = 1;" | grep '^CREATE VIEW ' || true) + +# Lock the session down BEFORE user SQL: external access is pinned to the +# allowed prefixes, local files are unreachable, and the config can't be +# un-SET. Views bind lazily, so reads still work under the lockdown. +allowed_sql= +oifs=$IFS; IFS=, +for d in $ALLOWED; do allowed_sql="$allowed_sql${allowed_sql:+, }'$d'"; done +IFS=$oifs + +INIT="LOAD httpfs; LOAD iceberg; $SECRET $VIEWS +SET allowed_directories=[$allowed_sql]; +SET enable_external_access=false; +SET lock_configuration=true;" + +# A single bare argument is the SQL itself (dbq.sh calling convention). +if [ $# -eq 1 ] && [ "${1#-}" = "$1" ]; then set -- -c "$1"; fi + +# duckdb has no "-f - means stdin" convention; hand it the real thing. +i=0; n=$#; prev= +while [ "$i" -lt "$n" ]; do + a=$1; shift + if [ "$prev" = "-f" ] && [ "$a" = "-" ]; then a=/dev/stdin; fi + prev=$a + set -- "$@" "$a" + i=$((i+1)) +done + +# The .output pair silences the init's own result rows (CREATE SECRET prints +# a "true"); errors still reach stderr, and user SQL output is untouched. +exec duckdb -batch -cmd ".output /dev/null" -cmd "$INIT" -cmd ".output" "$@" +ENTRY +RUN chmod +x /usr/local/bin/iceberg-query +DOCKERFILE +) + +tag="thermograph-iceberg:$(printf '%s' "$dockerfile" | sha256sum | cut -c1-12)" +b64=$(printf '%s' "$dockerfile" | base64 -w0) + +# Quote the duckdb arguments so they survive the remote shell intact. +args=$(printf '%q ' "$@") + +# Credentials: env override first, then a call-time sops decrypt of the vault +# on the calling machine. Either may come up empty here -- the remote side +# still gets a chance to fill them from its own /etc/thermograph.env. +ak="${THERMOGRAPH_LAKE_S3_ACCESS_KEY:-}" +sk="${THERMOGRAPH_LAKE_S3_SECRET_KEY:-}" +if [ -z "$ak" ] || [ -z "$sk" ]; then + repo_root=$(cd -- "$(dirname -- "$0")/../.." && pwd) + vault="${THERMOGRAPH_LAKE_VAULT:-$repo_root/infra/deploy/secrets/prod.yaml}" + sops_bin=$(command -v sops || echo "$HOME/.local/bin/sops") + if [ -r "$vault" ] && [ -x "$sops_bin" ]; then + ak=$("$sops_bin" -d --extract '["THERMOGRAPH_LAKE_S3_ACCESS_KEY"]' "$vault" 2>/dev/null || true) + sk=$("$sops_bin" -d --extract '["THERMOGRAPH_LAKE_S3_SECRET_KEY"]' "$vault" 2>/dev/null || true) + fi +fi + +# The engine image is (re)built on the target host only when its tag is +# missing, then run with the credentials passed via environment -- docker +# inherits them from the remote shell, which read them from stdin, so they +# never appear on a command line. The rest of stdin flows through to duckdb. +remote=$(cat <&2 + exit 1 +fi +export LAKE_S3_ACCESS_KEY="\$ak" LAKE_S3_SECRET_KEY="\$sk" +docker image inspect $tag >/dev/null 2>&1 \ + || echo $b64 | base64 -d | docker build -q -t $tag - >/dev/null +exec docker run --rm -i -e LAKE_S3_ACCESS_KEY -e LAKE_S3_SECRET_KEY $tag iceberg-query $args +REMOTE +) + +# Stdin is forwarded only for the documented `-f -` form. Anything else gets +# just the credential lines -- unconditionally forwarding an idle stdin would +# leave the local pipeline waiting on input the query never reads. +want_stdin=false +prev= +for a in "$@"; do + if [ "$prev" = "-f" ] && [ "$a" = "-" ]; then want_stdin=true; fi + prev=$a +done + +feed() { + printf '%s\n%s\n' "$ak" "$sk" || true + if $want_stdin; then cat || true; fi +} + +if [ -z "$ssh_target" ]; then + feed | bash -c "$remote" +else + feed | ssh -i "$KEYFILE" -o StrictHostKeyChecking=no -o ConnectTimeout=15 \ + "$ssh_target" "$remote" +fi