Merge pull request 'Promote main to release: Iceberg-backed lake query' (#46) from main into release

backend/Dockerfile

@@ -53,10 +53,12 @@ RUN useradd --system --create-home --uid 10001 thermograph \
     && chown -R thermograph:thermograph /app /state
 
 USER thermograph
-# Bake DuckDB's httpfs extension AS THE RUNTIME USER — extensions install to
-# the invoking user's ~/.duckdb, and a root-time bake leaves uid 10001 unable
-# to LOAD it (seen live: prod lake /query 500'd on exactly this).
-RUN python -c "import duckdb; duckdb.connect().execute('INSTALL httpfs')"
+# Bake DuckDB's httpfs + iceberg extensions AS THE RUNTIME USER — extensions
+# install to the invoking user's ~/.duckdb, and a root-time bake leaves uid
+# 10001 unable to LOAD them (seen live: prod lake /query 500'd on exactly
+# this). iceberg powers the era5_daily view via table metadata instead of a
+# bucket-wide glob LIST.
+RUN python -c "import duckdb; c = duckdb.connect(); c.execute('INSTALL httpfs'); c.execute('INSTALL iceberg')"
 WORKDIR /app
 
 ENV PORT=8137 \

backend/lake_app.py

@@ -135,34 +135,51 @@ _FORBIDDEN = re.compile(
 
 
 def _connect():
-    """A fresh in-memory DuckDB with the lake views registered. Per-request:
-    connections are cheap, and a throwaway one means a query can't leave state
-    behind for the next. Returns None when no lake is configured."""
+    """A fresh in-memory DuckDB with the lake views registered. Returns None
+    when no lake is configured.
+
+    Bucket mode reads era5_daily through the ICEBERG table, not a hive glob:
+    a glob over era5/daily/ must LIST every object under it (~1k per tile —
+    minutes at a few hundred tiles, seen live timing out /query at 390),
+    while Iceberg gets the file list from table metadata in a handful of
+    reads. Local mode (tests, small trees) keeps the plain hive glob — no
+    Iceberg table exists in the fixtures and listing a local dir is free."""
     import duckdb  # noqa: PLC0415 - only the lake role pays the import
 
     mode, cfg = _source()
+    if mode == "none":
+        return None
+    con = duckdb.connect()
     if mode == "local":
         daily_glob = os.path.join(era5lake.local_dir(), era5lake.PREFIX,
                                   "daily/*/*/*/*.parquet")
+        con.execute(f"""
+            CREATE VIEW era5_daily AS
+            SELECT * FROM read_parquet('{daily_glob}', hive_partitioning=1)""")
         manifest_src = os.path.join(era5lake.local_dir(), era5lake.MANIFEST_KEY)
-    elif mode == "bucket":
-        daily_glob = f"s3://{cfg['bucket']}/{era5lake.PREFIX}/daily/*/*/*/*.parquet"
-        manifest_src = f"s3://{cfg['bucket']}/{era5lake.MANIFEST_KEY}"
-    else:
-        return None
-    con = duckdb.connect()
-    if mode == "bucket":
-        con.execute("LOAD httpfs")   # baked into the image at build time
-        host = cfg["endpoint"].split("://", 1)[-1]
-        con.execute(f"SET s3_endpoint='{host}'")
-        con.execute("SET s3_url_style='path'")
-        con.execute(f"SET s3_region='{cfg['region']}'")
-        con.execute("SET s3_access_key_id=?", [cfg["access_key"]])
-        con.execute("SET s3_secret_access_key=?", [cfg["secret_key"]])
-    con.execute(f"""
-        CREATE VIEW era5_daily AS
-        SELECT * FROM read_parquet('{daily_glob}', hive_partitioning=1)""")
-    con.execute(f"CREATE VIEW manifest AS SELECT * FROM read_parquet('{manifest_src}')")
+        con.execute(
+            f"CREATE VIEW manifest AS SELECT * FROM read_parquet('{manifest_src}')")
+        return con
+
+    con.execute("LOAD httpfs")   # both baked into the image at build time,
+    con.execute("LOAD iceberg")  # as the runtime user (see Dockerfile)
+    host = cfg["endpoint"].split("://", 1)[-1]
+    con.execute(f"SET s3_endpoint='{host}'")
+    con.execute("SET s3_url_style='path'")
+    con.execute(f"SET s3_region='{cfg['region']}'")
+    con.execute("SET s3_access_key_id=?", [cfg["access_key"]])
+    con.execute("SET s3_secret_access_key=?", [cfg["secret_key"]])
+    # Newest metadata JSON wins (one single-page LIST of metadata/ — cheap);
+    # pyiceberg names them NNNNN-uuid.metadata.json, so lexicographic max is
+    # the numeric max.
+    meta = con.execute(
+        f"SELECT max(file) FROM glob('s3://{cfg['bucket']}/iceberg/era5_daily/"
+        f"metadata/*.metadata.json')").fetchone()[0]
+    if meta is None:
+        return None   # lake bucket exists but the Iceberg table doesn't yet
+    con.execute(f"CREATE VIEW era5_daily AS SELECT * FROM iceberg_scan('{meta}')")
+    con.execute(f"""CREATE VIEW manifest AS SELECT * FROM
+        read_parquet('s3://{cfg['bucket']}/{era5lake.MANIFEST_KEY}')""")
     return con
 
 

infra/deploy/forgejo/README.md

@@ -107,6 +107,25 @@ See that script's header for exactly what it replaces (the pre-Forgejo GitHub
 self-hosted runner on this same machine) and why it registers with two
 labels where there used to be two separate runners.
 
+`config.yaml`'s `runner.capacity` is raised from the tool's default of 1 to 8
+(override with `CAPACITY=`) — a single PR push fires `pr-build`,
+`secrets-guard`, and `shell-lint` simultaneously (3 independent workflows,
+no `needs:` between them), so capacity 1 serializes work that could run in
+parallel, and even capacity 3 (an earlier, undocumented hand-tune) leaves
+`build-backend`/`build-frontend`/`validate-observability` queued behind
+those three before they get a slot. The desktop has 16 cores / 34GB free
+today; 8 concurrent jobs is comfortable headroom without starving LAN dev's
+own compose stack.
+
+**Adding more runner capacity should mean raising this number, or adding a
+second runner on the desktop itself — not putting a runner on prod or
+beta.** `container.docker_host: automount` gives job containers the *host's*
+Docker socket; on prod or beta that would mean any CI job has root-equivalent
+access to whatever else is running there (the live app stack, or Forgejo
+itself). An earlier revision of this stack actually did run the runner as a
+Swarm-hosted container on beta and was deliberately reverted to the desktop
+for this reason — see the note at the top of `docker-stack.yml`.
+
 ## Custom CI job image (`ci-runner/`)
 
 `ci-runner/Dockerfile` still bases on `node:20-bookworm` — Node is a hard

infra/deploy/forgejo/register-lan-runner.sh

@@ -62,11 +62,15 @@ echo "==> Registering with $FORGEJO_URL (label: $LABELS)"
   --name "thermograph-lan-$(hostname -s)" \
   --labels "$LABELS"
 
-echo "==> Runner config (docker.sock automount, so docker:-labeled jobs like"
-echo "    build-push.yml can actually run 'docker build/push' -- generated"
-echo "    config only overridden on the one setting that matters here)"
+echo "==> Runner config (docker.sock automount so docker:-labeled jobs like"
+echo "    build-push.yml can actually run 'docker build/push'; capacity raised"
+echo "    from the default of 1 -- a single PR push fires 3+ independent"
+echo "    workflows (pr-build, secrets-guard, shell-lint) simultaneously, so"
+echo "    anything less than that serializes jobs that could run in parallel)"
+CAPACITY="${CAPACITY:-8}"
 ./forgejo-runner generate-config \
-  | sed 's/docker_host: "-"/docker_host: "automount"/' \
+  | sed -e 's/docker_host: "-"/docker_host: "automount"/' \
+        -e "s/capacity: 1/capacity: ${CAPACITY}/" \
   > "${RUNNER_DIR}/config.yaml"
 
 echo "==> systemd --user unit"