From 81c579ddcb5f48c249e91e047eb9c073de1b408e Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 19:05:21 +0000 Subject: [PATCH 1/2] infra/forgejo: raise LAN runner concurrency to 8 (#43) --- infra/deploy/forgejo/README.md | 19 +++++++++++++++++++ infra/deploy/forgejo/register-lan-runner.sh | 12 ++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/infra/deploy/forgejo/README.md b/infra/deploy/forgejo/README.md index 8ad737d..6f91009 100644 --- a/infra/deploy/forgejo/README.md +++ b/infra/deploy/forgejo/README.md @@ -107,6 +107,25 @@ See that script's header for exactly what it replaces (the pre-Forgejo GitHub self-hosted runner on this same machine) and why it registers with two labels where there used to be two separate runners. +`config.yaml`'s `runner.capacity` is raised from the tool's default of 1 to 8 +(override with `CAPACITY=`) — a single PR push fires `pr-build`, +`secrets-guard`, and `shell-lint` simultaneously (3 independent workflows, +no `needs:` between them), so capacity 1 serializes work that could run in +parallel, and even capacity 3 (an earlier, undocumented hand-tune) leaves +`build-backend`/`build-frontend`/`validate-observability` queued behind +those three before they get a slot. The desktop has 16 cores / 34GB free +today; 8 concurrent jobs is comfortable headroom without starving LAN dev's +own compose stack. + +**Adding more runner capacity should mean raising this number, or adding a +second runner on the desktop itself — not putting a runner on prod or +beta.** `container.docker_host: automount` gives job containers the *host's* +Docker socket; on prod or beta that would mean any CI job has root-equivalent +access to whatever else is running there (the live app stack, or Forgejo +itself). An earlier revision of this stack actually did run the runner as a +Swarm-hosted container on beta and was deliberately reverted to the desktop +for this reason — see the note at the top of `docker-stack.yml`. + ## Custom CI job image (`ci-runner/`) `ci-runner/Dockerfile` still bases on `node:20-bookworm` — Node is a hard diff --git a/infra/deploy/forgejo/register-lan-runner.sh b/infra/deploy/forgejo/register-lan-runner.sh index c35660d..0f96f97 100755 --- a/infra/deploy/forgejo/register-lan-runner.sh +++ b/infra/deploy/forgejo/register-lan-runner.sh @@ -62,11 +62,15 @@ echo "==> Registering with $FORGEJO_URL (label: $LABELS)" --name "thermograph-lan-$(hostname -s)" \ --labels "$LABELS" -echo "==> Runner config (docker.sock automount, so docker:-labeled jobs like" -echo " build-push.yml can actually run 'docker build/push' -- generated" -echo " config only overridden on the one setting that matters here)" +echo "==> Runner config (docker.sock automount so docker:-labeled jobs like" +echo " build-push.yml can actually run 'docker build/push'; capacity raised" +echo " from the default of 1 -- a single PR push fires 3+ independent" +echo " workflows (pr-build, secrets-guard, shell-lint) simultaneously, so" +echo " anything less than that serializes jobs that could run in parallel)" +CAPACITY="${CAPACITY:-8}" ./forgejo-runner generate-config \ - | sed 's/docker_host: "-"/docker_host: "automount"/' \ + | sed -e 's/docker_host: "-"/docker_host: "automount"/' \ + -e "s/capacity: 1/capacity: ${CAPACITY}/" \ > "${RUNNER_DIR}/config.yaml" echo "==> systemd --user unit" From 0f1b3a168c6500680e1153becc82717628a6b2a5 Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 19:09:22 +0000 Subject: [PATCH 2/2] Lake /query reads era5_daily through Iceberg metadata (#44) --- backend/Dockerfile | 10 +++++--- backend/lake_app.py | 59 +++++++++++++++++++++++++++++---------------- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 5641a0c..ab97519 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -53,10 +53,12 @@ RUN useradd --system --create-home --uid 10001 thermograph \ && chown -R thermograph:thermograph /app /state USER thermograph -# Bake DuckDB's httpfs extension AS THE RUNTIME USER — extensions install to -# the invoking user's ~/.duckdb, and a root-time bake leaves uid 10001 unable -# to LOAD it (seen live: prod lake /query 500'd on exactly this). -RUN python -c "import duckdb; duckdb.connect().execute('INSTALL httpfs')" +# Bake DuckDB's httpfs + iceberg extensions AS THE RUNTIME USER — extensions +# install to the invoking user's ~/.duckdb, and a root-time bake leaves uid +# 10001 unable to LOAD them (seen live: prod lake /query 500'd on exactly +# this). iceberg powers the era5_daily view via table metadata instead of a +# bucket-wide glob LIST. +RUN python -c "import duckdb; c = duckdb.connect(); c.execute('INSTALL httpfs'); c.execute('INSTALL iceberg')" WORKDIR /app ENV PORT=8137 \ diff --git a/backend/lake_app.py b/backend/lake_app.py index 37eb721..c6a6100 100644 --- a/backend/lake_app.py +++ b/backend/lake_app.py @@ -135,34 +135,51 @@ _FORBIDDEN = re.compile( def _connect(): - """A fresh in-memory DuckDB with the lake views registered. Per-request: - connections are cheap, and a throwaway one means a query can't leave state - behind for the next. Returns None when no lake is configured.""" + """A fresh in-memory DuckDB with the lake views registered. Returns None + when no lake is configured. + + Bucket mode reads era5_daily through the ICEBERG table, not a hive glob: + a glob over era5/daily/ must LIST every object under it (~1k per tile — + minutes at a few hundred tiles, seen live timing out /query at 390), + while Iceberg gets the file list from table metadata in a handful of + reads. Local mode (tests, small trees) keeps the plain hive glob — no + Iceberg table exists in the fixtures and listing a local dir is free.""" import duckdb # noqa: PLC0415 - only the lake role pays the import mode, cfg = _source() + if mode == "none": + return None + con = duckdb.connect() if mode == "local": daily_glob = os.path.join(era5lake.local_dir(), era5lake.PREFIX, "daily/*/*/*/*.parquet") + con.execute(f""" + CREATE VIEW era5_daily AS + SELECT * FROM read_parquet('{daily_glob}', hive_partitioning=1)""") manifest_src = os.path.join(era5lake.local_dir(), era5lake.MANIFEST_KEY) - elif mode == "bucket": - daily_glob = f"s3://{cfg['bucket']}/{era5lake.PREFIX}/daily/*/*/*/*.parquet" - manifest_src = f"s3://{cfg['bucket']}/{era5lake.MANIFEST_KEY}" - else: - return None - con = duckdb.connect() - if mode == "bucket": - con.execute("LOAD httpfs") # baked into the image at build time - host = cfg["endpoint"].split("://", 1)[-1] - con.execute(f"SET s3_endpoint='{host}'") - con.execute("SET s3_url_style='path'") - con.execute(f"SET s3_region='{cfg['region']}'") - con.execute("SET s3_access_key_id=?", [cfg["access_key"]]) - con.execute("SET s3_secret_access_key=?", [cfg["secret_key"]]) - con.execute(f""" - CREATE VIEW era5_daily AS - SELECT * FROM read_parquet('{daily_glob}', hive_partitioning=1)""") - con.execute(f"CREATE VIEW manifest AS SELECT * FROM read_parquet('{manifest_src}')") + con.execute( + f"CREATE VIEW manifest AS SELECT * FROM read_parquet('{manifest_src}')") + return con + + con.execute("LOAD httpfs") # both baked into the image at build time, + con.execute("LOAD iceberg") # as the runtime user (see Dockerfile) + host = cfg["endpoint"].split("://", 1)[-1] + con.execute(f"SET s3_endpoint='{host}'") + con.execute("SET s3_url_style='path'") + con.execute(f"SET s3_region='{cfg['region']}'") + con.execute("SET s3_access_key_id=?", [cfg["access_key"]]) + con.execute("SET s3_secret_access_key=?", [cfg["secret_key"]]) + # Newest metadata JSON wins (one single-page LIST of metadata/ — cheap); + # pyiceberg names them NNNNN-uuid.metadata.json, so lexicographic max is + # the numeric max. + meta = con.execute( + f"SELECT max(file) FROM glob('s3://{cfg['bucket']}/iceberg/era5_daily/" + f"metadata/*.metadata.json')").fetchone()[0] + if meta is None: + return None # lake bucket exists but the Iceberg table doesn't yet + con.execute(f"CREATE VIEW era5_daily AS SELECT * FROM iceberg_scan('{meta}')") + con.execute(f"""CREATE VIEW manifest AS SELECT * FROM + read_parquet('s3://{cfg['bucket']}/{era5lake.MANIFEST_KEY}')""") return con