From 92e74c585ab677a8d5bac79055f53485a0256f40 Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 00:53:48 +0000 Subject: [PATCH 01/10] frontend: rewrite the SSR content service in Go (#28) Ports frontend/ (Jinja2/FastAPI, ~1180 LOC) to Go with html/template. No climate math, no DB, no auth -- every route fetches from the backend's /content/* API. Verified with a golden-HTML diff, not just unit tests: both the Python original and the Go rewrite were run against the same committed fixtures and every route compared byte-for-byte, confirmed programmatically. That process caught defects unit tests alone missed, since map[string]any has no compile-time field check: - Render-context keys were snake_case throughout while the templates read PascalCase fields. A missing map key doesn't error, it silently renders empty -- title, meta description, canonical URL, OpenGraph tags, and the homepage's entire ranked list were blank on every page despite every route returning 200. Fixed by renaming every key to match each template's own documented field contract, and passing API structs straight through wherever their fields already matched (removes a whole layer of future drift risk). - Three pages 500'd: ToolHref needed a composed href, not a bare "lat,lon" fragment; the records table needed the raw API struct. - JSON-LD was double-encoded: ") + if err != nil { + t.Fatal(err) + } + var buf strings.Builder + if err := tmpl.Execute(&buf, nil); err != nil { + t.Fatal(err) + } + want := "" + if got := buf.String(); got != want { + t.Errorf("jscomment through a real template:\n got %q\nwant %q", got, want) + } +} + +func TestHeadVerifyHTML(t *testing.T) { + if headVerifyHTML("", "") != "" { + t.Error("no tokens -> empty") + } + got := headVerifyHTML("gtok", "btok") + want := "\n " + + "" + if string(got) != want { + t.Errorf("head_verify: %q", got) + } + // Attribute escaping matches markupsafe. + if !strings.Contains(string(headVerifyHTML(`a"b`, "")), "a"b") { + t.Error("token not attribute-escaped") + } +} + +func TestToJSON(t *testing.T) { + // u builds a backslash-u escape the way json.dumps/htmlsafe_json_dumps + // emit them (lowercase hex, four digits). + u := func(r rune) string { return fmt.Sprintf(`\u%04x`, r) } + cases := []struct { + in, want string + }{ + {"Feels-like temperature", `"Feels-like temperature"`}, + // Jinja's htmlsafe |tojson escapes the HTML-unsafe <, >, &, '. + {`a&'c`, `"a` + u('<') + `b` + u('>') + u('&') + u('\'') + `c"`}, + // ...and ensure_ascii turns non-ASCII into escapes. + {"50°F", `"50` + u('°') + `F"`}, + {"±7", `"` + u('±') + `7"`}, + } + for _, c := range cases { + got, err := toJSON(c.in) + if err != nil { + t.Fatal(err) + } + if string(got) != c.want { + t.Errorf("toJSON(%q) = %s, want %s", c.in, got, c.want) + } + } +} + +func TestFuncMapHelpers(t *testing.T) { + fm := FuncMap(testConfig()) + temp := fm["temp"].(func(any, any) template.HTML) + // Jinja's {{ temp(-10) }} literal (int) and a nullable payload pointer. + if got := temp("C", -10); got != `-23°C` { + t.Errorf("temp int literal: %q", got) + } + if got := temp("", (*float64)(nil)); got != "—" { + t.Errorf("temp nil pointer: %q", got) + } + tc := fm["temp_class"].(func(any) string) + if tc(nil) != "none" { + t.Error("temp_class nil") + } + ord := fm["ordinal"].(func(any) string) + if ord(89.1) != "89th" { + t.Error("ordinal") + } +} diff --git a/frontend/server/internal/content/funcmap.go b/frontend/server/internal/content/funcmap.go new file mode 100644 index 0000000..c1ec52c --- /dev/null +++ b/frontend/server/internal/content/funcmap.go @@ -0,0 +1,177 @@ +package content + +import ( + "fmt" + "html/template" + "strings" + "unicode/utf16" + + "thermograph/frontend/internal/config" + "thermograph/frontend/internal/format" +) + +// FuncMap builds the template helper set (the Jinja globals/filters +// content.py registered on its Environment). Pass it to render.New at boot — +// html/template resolves names at parse time. +// +// Unit-dependent helpers (temp, temp_bare) take the active unit as their +// FIRST argument: the Python scoped it in a ContextVar, but html/template +// FuncMaps are engine-global, so per-request state must arrive through the +// call. Every page context carries the active unit under the "unit" key +// (same value as "unit_default"), so template call sites are +// {{temp $.unit .high_f}} where Jinja had {{ temp(m.high_f) }}. +func FuncMap(cfg config.Config) template.FuncMap { + return template.FuncMap{ + // temp / temp_bare / temp_class accept the loosely-typed values the + // templates hand them: nullable *float64 payload fields, plain + // float64s, and int literals (Jinja's {{ temp(-10) }}). + "temp": func(unit, f any) template.HTML { + return format.Temp(unitArg(unit), floatArg(f)) + }, + "temp_bare": func(unit, f any) template.HTML { + return format.TempBare(unitArg(unit), floatArg(f)) + }, + "temp_class": func(f any) string { + return format.TempClass(floatArg(f)) + }, + // The `ordinal` filter: {{ c.percentile|ordinal }} -> {{ordinal .percentile}}. + "ordinal": format.PctOrdinal, + // Search-engine ownership-verification tags, from env (empty + // when unset). No backend dependency — the same two variables the + // Python read, resolved once into config. + "head_verify": func() template.HTML { + return headVerifyHTML(cfg.GoogleVerify, cfg.BingVerify) + }, + // Jinja's |tojson (glossary_term.html.j2's inline JSON-LD): JSON with + // the HTML-unsafe characters escaped so it can sit inside . html/template's + // contextual escaper treats . html/template's + // contextual escaper treats . html/template's + // contextual escaper treats + + +{{end}} +{{define "base_tail" -}} + + + +{{end}} +{{/* Shared breadcrumb nav (each Jinja page repeated this line verbatim; the + output is identical, so it is factored here). The separator is emitted + BEFORE every non-first crumb — same byte stream as Jinja's + `{% if not loop.last %}` after-each-but-last. Crumb.Href is *string + (terminal crumb: null). */}} +{{define "breadcrumb" -}} + +{{end}} \ No newline at end of file diff --git a/frontend/server/internal/render/templates/city.html.tmpl b/frontend/server/internal/render/templates/city.html.tmpl new file mode 100644 index 0000000..1166acd --- /dev/null +++ b/frontend/server/internal/render/templates/city.html.tmpl @@ -0,0 +1,101 @@ +{{/* Ported from templates/city.html.j2. Extra data (built by the handler + inside unit_scope(default_unit), as content.py's city_page did): + .Breadcrumb; .City (needs .Slug); .Display/.Name string; + .YearRange [2]int; .NYears int; + .Months []{Name, Slug string; HighF, LowF, RangeLoF, RangeHiF *float64; + High, Low, Precip template.HTML; + Bar *{Left, Width string; C1, C2 string}} + — Bar.Left/Width are STRINGS pre-formatted to Python's repr of + round(x, 1) ("36.0", not "36"): Go's %v drops the ".0" and the golden + diff would catch it; + .Warmest/.Coldest/.Wettest *month (entries of .Months, or nil); + .Records contentapi.AllTimeRecords (raw floats; .Fmt.Temp in-template); + .Today *{Date string; Cards []{Label, Grade, Cls string; + Value template.HTML; Percentile *float64}}; + .Flavor *{Extract string; URL string}; .Event *{Text string; URL string}; + .ToolHref string (full "{base}/#{lat:.5f},{lon:.5f}" href — composed + inline, html/template's fragment urlEscaper would %-escape the comma); + .CompareURL string ("{base}/compare#loc={lat:.4f},{lon:.4f}"); + .JSONLDStr template.JS (compacted raw payload bytes — the |safe site; + trusted backend-owned JSON-LD, never re-marshaled through a Go map). */}}{{template "base_head_start" .}}{{template "base_head_end" .}}{{template "base_body_start" .}}{{template "breadcrumb" .}} +
+

{{.Display}} climate

+

Average temperatures, precipitation and all-time records for {{.Display}}, + with every day graded against ~{{.NYears}} years of local climate history + ({{index .YearRange 0}}–{{index .YearRange 1}}).{{if and .Warmest .Coldest}} Its warmest month is + {{.Warmest.Name}}, averaging a high of {{.Warmest.High}}, and its coldest is + {{.Coldest.Name}}, with an average low of {{.Coldest.Low}}.{{end}}

+ +{{if .Flavor}}

{{.Flavor.Extract}}{{if .Flavor.URL}} + via Wikipedia{{end}}

+{{end}} +{{if .Today}}
+

How today compares

+

Latest recorded day: {{.Today.Date}}. Each reading placed on {{.Display}}'s + own ±7-day seasonal distribution.

+
+{{range .Today.Cards}}
+ {{.Label}} + {{.Value}} +{{if .Percentile}}{{.Grade}} · {{ordinal .Percentile}} pct +{{else}}{{.Grade}}{{end}}
+{{end}}
+

Open {{.Name}} in the live weather grader →

+
+{{end}} +{{if .Event}}
+

Notable weather in {{.Name}}

+

{{.Event.Text}}{{if .Event.URL}} Read more →{{end}}

+
+{{end}} +
+

{{.Name}} average temperatures by month

+

Typical daily high and low, and average precipitation, for each month in {{.Display}}, + based on {{index .YearRange 0}}–{{index .YearRange 1}} records. +{{if .Wettest}}The wettest month is {{.Wettest.Name}} ({{.Wettest.Precip}} on an average day).{{end}}

+
+ + + +{{range .Months}} + + + + + +{{end}} +
MonthAvg highAvg lowAvg precip
{{.Name}}{{.High}}{{.Low}}{{.Precip}}
+
+ + +

Each bar spans the 10th-percentile daily low to the 90th-percentile daily + high (the range most days fall within), coloured cold → hot on a shared + {{.Fmt.Temp -10.0}} to {{.Fmt.Temp 115.0}} scale, so the whole year's rhythm reads at a glance.

+
+ +
+

Thinking of visiting {{.Name}}?

+

Trips live and die by the weather. See how {{.Name}}'s comfort stacks up against where + you live. {{.Name}} is pre-loaded, just add your city.

+

Compare {{.Name}}'s comfort with your city →

+
+ +{{if or .Records.Tmax .Records.Tmin}}
+

Record extremes

+
    +{{if .Records.Tmax}}
  • Hottest day on record: {{.Fmt.Temp .Records.Tmax.Max}} on {{.Records.Tmax.MaxDate}}
    • {{end}}{{if .Records.Tmin}}
  • Coldest day on record: {{.Fmt.Temp .Records.Tmin.Min}} on {{.Records.Tmin.MinDate}}
    • {{end}}
+

All {{.Name}} weather records →

+
+{{end}} +

Percentiles compare each day to {{.Display}}'s own history, so grades are + relative to this location: a “Near Record” day here isn't the same temperature as elsewhere. + How the grades work →

+
+{{template "base_body_end" .}}{{template "base_scripts_default" .}}{{template "base_tail" .}} \ No newline at end of file diff --git a/frontend/server/internal/render/templates/glossary.html.tmpl b/frontend/server/internal/render/templates/glossary.html.tmpl new file mode 100644 index 0000000..4c730b2 --- /dev/null +++ b/frontend/server/internal/render/templates/glossary.html.tmpl @@ -0,0 +1,10 @@ +{{/* Ported from templates/glossary.html.j2. Extra data: .Breadcrumb, + .Terms []{Slug, Term, Short string} (glossary.yaml file order). */}}{{template "base_head_start" .}}{{template "base_head_end" .}}{{template "base_body_start" .}}{{template "breadcrumb" .}}
+

Weather & climate glossary

+

Plain-language definitions of the terms Thermograph uses to grade the weather.

+{{range .Terms}}
+

{{.Term}}

+

{{.Short}}

+
+{{end}}
+{{template "base_body_end" .}}{{template "base_scripts_default" .}}{{template "base_tail" .}} \ No newline at end of file diff --git a/frontend/server/internal/render/templates/glossary_term.html.tmpl b/frontend/server/internal/render/templates/glossary_term.html.tmpl new file mode 100644 index 0000000..a1fd2d4 --- /dev/null +++ b/frontend/server/internal/render/templates/glossary_term.html.tmpl @@ -0,0 +1,21 @@ +{{/* Ported from templates/glossary_term.html.j2. Extra data: + .Breadcrumb; .Term string; .Others []{Slug, Term string}; + .JSONLDStr template.JS — the handler builds the WHOLE DefinedTerm object + (the Jinja original interpolated `term|tojson` / `page_description|tojson` + / base_url inline; Go's JS-context escaping differs byte-wise from + markupsafe's tojson, so the handler reproduces Python htmlsafe-tojson — + ensure_ascii \uXXXX escapes plus <,>,&,' → <,>,&,' — + and template.JS passes it through untouched); + .Body template.HTML — glossary yaml `body` with {base} substituted. This + is the Jinja `| safe` site: the content is trusted, committed copy from + frontend/content/glossary.yaml, not user or API input. */}}{{template "base_head_start" .}}{{template "base_head_end" .}}{{template "base_body_start" .}}{{template "breadcrumb" .}} +{{template "base_body_end" .}}{{template "base_scripts_default" .}}{{template "base_tail" .}} \ No newline at end of file diff --git a/frontend/server/internal/render/templates/home.html.tmpl b/frontend/server/internal/render/templates/home.html.tmpl new file mode 100644 index 0000000..2ce3918 --- /dev/null +++ b/frontend/server/internal/render/templates/home.html.tmpl @@ -0,0 +1,182 @@ +{{/* Ported from templates/home.html.j2. + + The Weekly view — the product itself — with the distribution surfaces built + around it. Everything structural is server-rendered so crawlers and no-JS + readers get a complete page; app.js hydrates the live numbers in place. + + The interactive controls below (#find-btn, #date-input, #panel, #results, …) + are app.js's DOM contract, carried over verbatim from the old static + frontend/index.html. Do not rename these ids. + + Section / BrandTag ("p") / MainClass ("") come from the route handler's + render data, not the template — the SEO pages already pass Section that way. + + Extra data: .Unusual *contentapi.HomeUnusual; .Stale bool; + .Ranked []contentapi.HomeRanked (Lat/Lon are json.Number, printed raw); + .Cities []contentapi.HomeCity; + .JSONLDStr template.JS — _HOME_JSONLD + "url" appended, serialized like + Python's plain json.dumps: ", "/": " separators, ensure_ascii=True. */}}{{template "base_head_start" .}} +{{template "base_head_end" .}} +{{template "base_body_start" .}}{{/* --- Hero ------------------------------------------------------------ + The grade card is the LCP element: its frame is server-rendered and its + slots are sized here, so hydrating the live numbers causes no layout + shift. Band colour is always paired with the band word — colour is never + the only signal. */}}
+
+

How unusual is your weather?

+

Any day, anywhere on Earth, graded against 45 years of that place's own history.

+
+ +
{{/* --cat carries the band colour, the same convention .normal-card uses, + so the band token stays the single source of truth. */}} +
+

+ {{- if .Unusual -}} + Today in {{.Unusual.Display}} + {{- else -}} + Today + {{- end -}} +

+

{{if .Unusual}}{{.Unusual.Grade}}{{else}}—{{end}}

+

+ {{- if .Unusual -}} + {{.Unusual.MetricLabel}} in the {{ordinal .Unusual.Percentile}} percentile for {{.Unusual.WindowLabel}} + {{- else -}} + Pick a place to see how unusual its weather is today. + {{- end -}} +

+{{if and .Unusual .Unusual.IsDefault}}

the most unusual weather we're tracking{{if .Stale}} · as of {{.Unusual.Date}}{{end}}

+{{end}}
+
{{/* app.js hides the locate button outside a secure context, where the + browser refuses geolocation and it could only ever fail. */}} + + +
+ +

See your week ↓

+
+
+ +
+
+ + +
+
+ + + What do the grades mean? → +
+
+ +
+
+

Find a location to begin.

+

Thermograph fetches decades of daily highs, lows, and precipitation + for your ~4 sq mi cell, builds a ±7-day seasonal distribution for each + day of the year, and grades recent weather by where it falls on that distribution.

+
+ +
+ +{{/* Sits outside #panel on purpose: app.js rewrites results.innerHTML + wholesale on every grade, so anything inside it would be destroyed. */}}

+ Free. No ads. No tracking. No account needed. + How we work → +

+ +{{/* --- Unusual right now ---------------------------------------------- + Scroll-snap row, no JS carousel. Both tails are represented whenever a + cold-tail city qualifies — two-sided honesty is product identity. */}}{{if .Ranked}}
+

Unusual right now

+ +

See all records →

+
+{{end}} +{{/* --- How it works ---------------------------------------------------- */}}
+

How it works

+
    +
  1. We keep 45 years of climate history (ERA5 reanalysis) for your exact ~2-mile grid square.
    2. +
  2. Today gets compared to the same two weeks of the year, every year back to 1980.
    3. +
  3. The result is a percentile and a plain-language grade, from Near Record cold to Near Record hot.
    4. +
+

+ Full methodology → + · Data: ERA5 via Open-Meteo · NASA POWER · MET Norway +

+
+ +{{/* --- Explore --------------------------------------------------------- */}}
+

Explore

{{/* Each card takes a colour from the grade ramp rather than a decorative + palette, so the section reads as part of the product's own scale. + Calendar carries the ramp itself — it IS the heatmap. */}} +
+ + Calendar + + Two years of your days, each colored by how unusual it was. + + + Compare + Rank any places by comfort for a trip or a move. + + + Day + Any date since 1980, anywhere: what was normal, what happened. + + + Alerts + Free percentile alerts: get pinged when your weather goes statistically weird. + +
+
+ +{{/* --- City hub links -------------------------------------------------- + Real HTML links funnelling homepage authority into the ~1000-page + /climate surface. */}}
+

Climate context for 1,000 cities

+ +

Browse all cities →

+
+{{template "base_body_end" .}} + {{/* The records strip renders real temperatures server-side as .temp spans, so + it needs the same converter the SEO pages use or they'd stay in °F while + the toggle says °C. Both import units.js, which the module loader dedupes. */}} + +{{template "base_tail" .}} \ No newline at end of file diff --git a/frontend/server/internal/render/templates/hub.html.tmpl b/frontend/server/internal/render/templates/hub.html.tmpl new file mode 100644 index 0000000..082fd0c --- /dev/null +++ b/frontend/server/internal/render/templates/hub.html.tmpl @@ -0,0 +1,80 @@ +{{/* Ported from templates/hub.html.j2. Extra data: .Breadcrumb; + .NCities/.NCountries int; .Groups — MUST be the ordered + []contentapi.HubCountry slice from the hub payload, never a Go map + (template range over a map sorts keys; Python's dict kept the backend's + order, and the golden diff would catch the reorder). */}}{{template "base_head_start" .}}{{template "base_head_end" .}}{{template "base_body_start" .}}{{template "breadcrumb" .}}
+

City climates

+

Average temperatures by month, all-time records, and how today compares, for + {{.NCities}} cities across {{.NCountries}} countries. Every day is graded against that + location's own ~45 years of climate history.

+ +
+ +
+ + +
+{{range .Groups}}
+ {{.Country}} {{len .Cities}} + +
+{{end}}
+
+ + +{{template "base_body_end" .}}{{template "base_scripts_default" .}}{{template "base_tail" .}} \ No newline at end of file diff --git a/frontend/server/internal/render/templates/month.html.tmpl b/frontend/server/internal/render/templates/month.html.tmpl new file mode 100644 index 0000000..515d9a0 --- /dev/null +++ b/frontend/server/internal/render/templates/month.html.tmpl @@ -0,0 +1,60 @@ +{{/* Ported from templates/month.html.j2. Extra data (all display values are + pre-formatted by the handler inside the city's unit scope, as in Python): + .Breadcrumb; .City (needs .Slug); .Display/.Name/.MonthName string; + .YearRange [2]int; .AvgHigh/.AvgLow template.HTML (temp spans); + .AvgHighCls/.AvgLowCls string (temp_class names); + .Stats []{Label string; Value template.HTML}; + .Records []{Label, HighTag, LowTag string; High, Low template.HTML; + HighDate, LowDate string}; + .ToolHref string — the full "{base}/#{lat:.5f},{lon:.5f}" href, built by + the handler: composed inline (Jinja's {{ base }}/#{{ tool_hash }}) the + fragment would hit html/template's urlEscaper and %-escape the comma; + .CompareURL string; .Prev/.Next {Slug, Name string}. */}}{{template "base_head_start" .}}{{template "base_head_end" .}}{{template "base_body_start" .}}{{template "breadcrumb" .}} +
+

Weather in {{.Display}} in {{.MonthName}}

+

In an average {{.MonthName}}, {{.Name}} sees a daily high around + {{.AvgHigh}} and a low around + {{.AvgLow}}, based on + {{index .YearRange 0}}–{{index .YearRange 1}} of local climate records.

+ + + +{{range .Stats}} +{{end}} +
{{.Label}}{{.Value}}
+ +{{if .Records}}

{{.MonthName}} records

+

The most extreme {{.MonthName}} on record for each metric, across + {{index .YearRange 0}}–{{index .YearRange 1}}. For rainfall, the wettest and driest are by + total {{.MonthName}} accumulation.

+
+{{range .Records}}
+

{{.Label}}

+
+ {{.HighTag}} + {{.High}} + {{.HighDate}} +
+
+ {{.LowTag}} + {{.Low}} + {{.LowDate}} +
+
+{{end}}
+{{end}} +

See {{.Name}}'s live weather grade →

+ +
+

Visiting {{.Name}} in {{.MonthName}}? See how its comfort stacks up against + where you live.

+ Compare {{.Name}} with your city → +
+ +

+ ‹ {{.Prev.Name}} · + {{.Name}} climate overview · + {{.Next.Name}} › +

+
+{{template "base_body_end" .}}{{template "base_scripts_default" .}}{{template "base_tail" .}} \ No newline at end of file diff --git a/frontend/server/internal/render/templates/privacy.html.tmpl b/frontend/server/internal/render/templates/privacy.html.tmpl new file mode 100644 index 0000000..5db4099 --- /dev/null +++ b/frontend/server/internal/render/templates/privacy.html.tmpl @@ -0,0 +1,31 @@ +{{/* Ported from templates/privacy.html.j2. Extra data: .Breadcrumb. */}}{{template "base_head_start" .}}{{template "base_head_end" .}}{{template "base_body_start" .}}{{template "breadcrumb" .}}
+

Privacy

+

Thermograph is free, has no ads, and needs no account. It is built so that + there is very little about you to collect in the first place.

+ +

Your location

+

Thermograph never looks up your location from your IP address. The only way the site + learns where you are is if you press Use my location, which asks your browser for + permission. You can decline, and picking a place on the map works just as well.

+

Whichever way you choose a place, the coordinates are sent to the server only as part of + the ordinary request for that grid square's climate data, and are not logged beyond the + normal web-server request handling every site does. The place you picked is remembered in + your own browser's local storage, not on the server, so clearing your browser data + forgets it.

+ +

Analytics

+

There is no analytics library, no cookies for tracking, and no per-visitor identifier. + The server keeps simple aggregate counts (how many people opened a page or tapped a + button, and which site referred them) with nothing tying any of it to an individual.

+ +

Email

+

If you sign up for the monthly digest, your address is used for that digest and nothing + else. It is never sold or shared, and every message can unsubscribe you.

+ +

Accounts

+

An account is optional and only exists to hold weather alerts you have asked for. It + stores your email address and the places you are watching.

+ +

Questions: see About & methodology.

+
+{{template "base_body_end" .}}{{template "base_scripts_default" .}}{{template "base_tail" .}} \ No newline at end of file diff --git a/frontend/server/internal/render/templates/records.html.tmpl b/frontend/server/internal/render/templates/records.html.tmpl new file mode 100644 index 0000000..b8a4930 --- /dev/null +++ b/frontend/server/internal/render/templates/records.html.tmpl @@ -0,0 +1,95 @@ +{{/* Ported from templates/records.html.j2. Extra data (formatted by the + handler inside unit_for_country scope, as in Python): + .Breadcrumb; .City (needs .Slug); .Display/.Name string; .YearRange [2]int; + .Rows []{Label string; High, Low template.HTML; HighDate, LowDate string} + — Low is either the formatted metric value or the "N-day dry spell" + text, LowDate already "—"-defaulted (content.py records_page); + .Monthly []{Name, Slug string; High, Low side} and + .Seasonal []{Name, Span string; High, Low side} where a side is + {Warm, Cold *{Cls string; Txt template.HTML; Date string}}; + .Hemisphere string; .AllTime contentapi.AllTimeRecords (raw floats — + .Fmt.Temp runs in the template, like Jinja's temp()); + .ToolHref string (full "{base}/#{lat:.5f},{lon:.5f}" — composed inline the + fragment comma would be %-escaped by html/template's urlEscaper); + .JSONLDStr template.JS (compacted raw payload bytes — the |safe site; + trusted backend-owned JSON-LD, never re-marshaled through a Go map). */}} +{{/* A metric's two extremes — ▲ warmest and ▼ coldest it has ever been — as tinted + chips with the date each occurred. Used for the daytime-high and overnight-low + columns of the month and season tables. (Jinja macro `extremes`.) */}} +{{define "extremes" -}} +{{if and .Warm .Cold -}} +
{{.Warm.Txt}}{{.Warm.Date}}
+
{{.Cold.Txt}}{{.Cold.Date}}
+{{else -}} +—{{end}}{{end}}{{template "base_head_start" .}}{{template "base_head_end" .}}{{template "base_body_start" .}}{{template "breadcrumb" .}} +
+

{{.Display}} weather records

+

Record high and low temperatures for {{.Display}}, by month, by season, and + all-time, with the dates they occurred, across {{index .YearRange 0}}–{{index .YearRange 1}} of daily + climate history.{{if and .AllTime.Tmax .AllTime.Tmin}} Its hottest day on record reached + {{.Fmt.Temp .AllTime.Tmax.Max}} on {{.AllTime.Tmax.MaxDate}}; its coldest fell to + {{.Fmt.Temp .AllTime.Tmin.Min}} on {{.AllTime.Tmin.MinDate}}.{{end}}

+ +
+

Records by month

+

The warmest (▲) and coldest (▼) both the daytime high and the overnight low have ever been in + each calendar month, so each column shows its record high and its record low. Tap a + month for its full averages and typical range.

+
+ + + +{{range .Monthly}} + + + + +{{end}} +
MonthDaytime highOvernight low
{{.Name}}{{template "extremes" .High}}{{template "extremes" .Low}}
+
+
+ +
+

Records by season

+

The warmest (▲) and coldest (▼) both the daytime high and the overnight low have reached in each + meteorological season ({{.Hemisphere}} Hemisphere, each a three-month span).

+
+ + + +{{range .Seasonal}} + + + + +{{end}} +
SeasonDaytime highOvernight low
{{.Name}} ({{.Span}}){{template "extremes" .High}}{{template "extremes" .Low}}
+
+
+ +
+

All-time records

+

The most extreme value {{.Name}} has recorded for each metric across its entire history.

+
+{{range .Rows}}
+

{{.Label}}

+
+ {{if eq .Label "Precip"}}Wettest{{else}}Highest{{end}} + {{.High}} + {{.HighDate}} +
+
+ {{if eq .Label "Precip"}}Driest{{else}}Lowest{{end}} + {{.Low}} + {{.LowDate}} +
+
+{{end}}
+

For rainfall, the “driest” figure is the longest run of consecutive + days without measurable rain, dated to when that dry spell began.

+
+ +

Grade {{.Name}}'s weather now →

+

‹ Back to {{.Name}} climate

+
+{{template "base_body_end" .}}{{template "base_scripts_default" .}}{{template "base_tail" .}} \ No newline at end of file diff --git a/frontend/server/main.go b/frontend/server/main.go new file mode 100644 index 0000000..914bcf0 --- /dev/null +++ b/frontend/server/main.go @@ -0,0 +1,174 @@ +// thermograph-frontend: the SSR frontend service — server-rendered content +// pages (climate hub / per-city / month / records / glossary / about), the +// interactive tool's SPA shells, and every static asset. All climate data +// comes from the backend's content API over HTTP (internal/contentapi); this +// process holds no data of its own. +// +// Go port of app.py: configuration, the route wiring (internal/content for +// the content.py routes, internal/handlers for the SPA shells + static +// assets), and the process lifecycle uvicorn used to own (listen, access +// logs, graceful shutdown). +package main + +import ( + "context" + "errors" + "log/slog" + "mime" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "thermograph/frontend/internal/config" + "thermograph/frontend/internal/content" + "thermograph/frontend/internal/contentapi" + "thermograph/frontend/internal/contentdata" + "thermograph/frontend/internal/handlers" + "thermograph/frontend/internal/render" +) + +func main() { + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + slog.SetDefault(logger) + + // Fail loud at boot on bad configuration or content (missing backend URL, + // garbage TTL, malformed glossary/pages YAML) — the Python raised at + // import for the same cases: a missing backend URL or a bad content edit + // should break the boot, not silently 500 on the first request. + cfg, err := config.Load() + if err != nil { + fatal(logger, "config", err) + } + + client := contentapi.New(contentapi.Options{ + BaseURL: cfg.APIBaseInternal, + BasePrefix: cfg.Base, // backend runs under the same THERMOGRAPH_BASE + APIVersion: cfg.APIVersion, + TTL: cfg.SSRCacheTTL, + }) + + glossary, err := contentdata.LoadGlossary(cfg.ContentDir) + if err != nil { + fatal(logger, "glossary", err) + } + pages, err := contentdata.LoadPages(cfg.ContentDir) + if err != nil { + fatal(logger, "pages", err) + } + + // Templates parse once, at boot, with the full FuncMap (html/template + // resolves function names at parse time) — the Go analogue of the Jinja + // environment's auto_reload=False. + engine, err := render.New(content.FuncMap(cfg)) + if err != nil { + fatal(logger, "templates", err) + } + + // The PWA manifest is served as a static file; register its media type so + // the file server labels it correctly (not in Go's default mime table). + if err := mime.AddExtensionType(".webmanifest", "application/manifest+json"); err != nil { + fatal(logger, "mime", err) + } + + // app.py-layer routes: SPA shells + static assets + the bare-BASE redirect. + app := handlers.New(handlers.Options{ + Base: cfg.Base, + StaticDir: cfg.StaticDir, + GoogleVerify: cfg.GoogleVerify, + BingVerify: cfg.BingVerify, + Log: logger, + }) + + // content.py-layer routes. content.New takes a *log.Logger; bridge it into + // slog so its lines (IndexNow boot warning, render failures) land on + // stdout structured like everything else. + contentHandlers, err := content.New(cfg, client, engine, glossary, pages, + slog.NewLogLogger(logger.Handler(), slog.LevelWarn)) + if err != nil { + fatal(logger, "content", err) + } + + mux := http.NewServeMux() + + // Liveness probe: the process is up and serving. Deliberately does no I/O + // — no call to the backend — so it stays cheap and reliable for a tight + // healthcheck interval. Readiness (backend reachable) is a separate concern. + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ok"}`)) + }) + + // Register order does not matter for precedence (most-specific pattern + // wins), but content registration performs the eager IndexNow key fetch — + // same point in boot as the Python's content.register(app). Its lazy + // fallback route needs the static handler to delegate non-key requests to. + contentHandlers.Register(mux, app.Static()) + app.Register(mux) + + srv := &http.Server{ + Addr: net.JoinHostPort("", cfg.Port), // 0.0.0.0:PORT, like uvicorn --host 0.0.0.0 + Handler: accessLog(logger, mux), + ReadHeaderTimeout: 10 * time.Second, + } + + // Graceful shutdown on SIGINT/SIGTERM: stop accepting, drain in-flight + // requests, then exit — what uvicorn did for the Python process. + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + errCh := make(chan error, 1) + go func() { + logger.Info("thermograph-frontend listening", "port", cfg.Port, "base", cfg.Base) + errCh <- srv.ListenAndServe() + }() + + select { + case err := <-errCh: + if !errors.Is(err, http.ErrServerClosed) { + fatal(logger, "serve", err) + } + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + logger.Error("shutdown", "err", err) + os.Exit(1) + } + logger.Info("shut down cleanly") + } +} + +func fatal(logger *slog.Logger, stage string, err error) { + logger.Error(stage, "err", err) + os.Exit(1) +} + +// accessLog is the uvicorn access log's replacement: one structured line per +// request on stdout. +func accessLog(logger *slog.Logger, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK} + next.ServeHTTP(rec, r) + logger.Info("request", + "method", r.Method, + "path", r.URL.Path, + "status", rec.status, + "dur_ms", time.Since(start).Milliseconds(), + ) + }) +} + +// statusRecorder captures the response status for the access log. +type statusRecorder struct { + http.ResponseWriter + status int +} + +func (rec *statusRecorder) WriteHeader(status int) { + rec.status = status + rec.ResponseWriter.WriteHeader(status) +} diff --git a/infra/deploy/stack/env-entrypoint.sh b/infra/deploy/stack/env-entrypoint.sh index 7af8173..da353d7 100755 --- a/infra/deploy/stack/env-entrypoint.sh +++ b/infra/deploy/stack/env-entrypoint.sh @@ -34,9 +34,14 @@ if [ -f "$ENV_FILE" ]; then done < "$ENV_FILE" fi -# Hand off: the backend image has a real entrypoint script; the frontend -# image is plain-CMD (docker passes that CMD to us as $@ when the stack -# overrides the entrypoint) -- exec whichever this image actually is. +# Hand off: the backend image has a real entrypoint script and takes that +# branch. Anything else needs an explicit `command:` in the stack yml's +# service block -- overriding `entrypoint:` here drops the image's own CMD +# entirely (Docker/Swarm does not merge them), so `$@` is empty unless the +# stack file supplies one. The frontend service does exactly that. The +# uvicorn fallback below is what ran, silently, whenever that expectation +# didn't hold; it is Python-specific and kept only as a loud, familiar-looking +# failure for a misconfigured non-Python image, not a real handoff path. if [ -x /app/deploy/entrypoint.sh ]; then exec /app/deploy/entrypoint.sh "$@" elif [ "$#" -gt 0 ]; then diff --git a/infra/deploy/stack/thermograph-stack.yml b/infra/deploy/stack/thermograph-stack.yml index 8bbc3c2..561b687 100644 --- a/infra/deploy/stack/thermograph-stack.yml +++ b/infra/deploy/stack/thermograph-stack.yml @@ -238,6 +238,12 @@ services: frontend: image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required} entrypoint: ["/host/env-entrypoint.sh"] + # REQUIRED, not cosmetic: overriding `entrypoint:` with no `command:` drops + # the image's own CMD entirely (Docker/Swarm semantics, not merged) -- + # env-entrypoint.sh then sees zero args and falls through to its hardcoded + # `exec uvicorn app:app` fallback, which no longer exists in this (Go) + # image. Without this line the frontend task exits 127 on every deploy. + command: ["/usr/local/bin/thermograph-frontend"] environment: THERMOGRAPH_BASE: / PORT: 8080 diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index f4d7330..2b13bd5 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -8,8 +8,8 @@ # FRONTEND_IMAGE_TAG. This replaces the earlier Stage-4 model where both # containers shared the single emi/thermograph/app image and # THERMOGRAPH_SERVICE_ROLE picked the process; the split Dockerfiles now start -# the right process directly (frontend `uvicorn app:app`, backend -# entrypoint.sh), so a backend deploy and a frontend deploy are fully +# the right process directly (frontend the thermograph-frontend Go binary, +# backend entrypoint.sh), so a backend deploy and a frontend deploy are fully # independent -- deploy.sh rolls one service without touching the other's tag. # Both get their own loopback-published port since prod/beta's host Caddy # path-splits directly to each; backend also gets a reverse-proxy fallback to @@ -256,9 +256,9 @@ services: restart: unless-stopped frontend: - # Frontend's OWN image, published by thermograph-frontend's build-push.yml - # from its own Dockerfile (which starts `uvicorn app:app` directly -- no - # THERMOGRAPH_SERVICE_ROLE process-picking anymore). Independent + # Frontend's OWN image, published by frontend-build-push.yml from + # frontend/Dockerfile (which starts the thermograph-frontend Go binary + # directly -- no THERMOGRAPH_SERVICE_ROLE process-picking). Independent # FRONTEND_IMAGE_TAG so a frontend deploy never disturbs the backend's tag. image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:-local} # Its own register() fetches the IndexNow key from backend at boot -- must -- 2.45.2 From 979653f407a3836665a51b7777fe8c248d501c56 Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 04:03:26 +0000 Subject: [PATCH 02/10] deploy.sh: fix the daemon-binary probe, which always dropped daemon (#33) docker compose config --images daemon does not filter to the named service on this host's Compose v5.3.1 -- it prints every service's image, one per line, in file order, so `| head -1` was silently grabbing db's image (timescaledb) instead of daemon's. The probe then always found no /usr/local/bin/thermograph-daemon in a Postgres image and dropped daemon from every backend deploy, regardless of what the real backend image contained. Fixed by building the image reference directly from the same vars docker-compose.yml's daemon.image: already interpolates, instead of going through docker compose config at all. Reproduced against beta directly: the old sequence selected the wrong image; the new construction resolves correctly and the binary probe passes. --- infra/deploy/deploy.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/infra/deploy/deploy.sh b/infra/deploy/deploy.sh index 487779e..51748eb 100755 --- a/infra/deploy/deploy.sh +++ b/infra/deploy/deploy.sh @@ -247,9 +247,17 @@ fi # next deploy of a tag that has it picks it up with no further action. case " ${TARGETS[*]} " in *" daemon "*) - daemon_img="$(docker compose config --images daemon 2>/dev/null | head -1 || true)" - if [ -n "$daemon_img" ] \ - && ! docker run --rm --entrypoint sh "$daemon_img" -c 'test -x /usr/local/bin/thermograph-daemon' 2>/dev/null; then + # Built directly from the same vars docker-compose.yml's `daemon.image:` + # interpolates (REGISTRY_HOST/BACKEND_IMAGE_PATH/BACKEND_IMAGE_TAG), + # NOT via `docker compose config --images daemon`: that command does not + # actually filter to the named service (confirmed live on Compose + # v5.3.1 -- it prints every service's image, one per line, in file + # order) so `| head -1` silently grabbed db's image instead. The probe + # then always found no daemon binary in a Postgres image and dropped + # daemon from EVERY backend deploy, regardless of what the real backend + # image contained -- reproduced and confirmed against beta directly. + daemon_img="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG}" + if ! docker run --rm --entrypoint sh "$daemon_img" -c 'test -x /usr/local/bin/thermograph-daemon' 2>/dev/null; then echo "==> $daemon_img predates the daemon binary; rolling without the daemon service this run" kept=() for t in "${TARGETS[@]}"; do -- 2.45.2 From da82abda27d04e60a020947930bcfc0f252c064a Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 06:25:06 +0000 Subject: [PATCH 03/10] frontend: fetch City() concurrently with the page's primary API call (#37) MonthPage and RecordsPage each made two backend calls sequentially (CityMonth/CityRecords, then City) where the second never depended on the first's result. fetchWithCity launches both concurrently via goroutines and a WaitGroup. Verified live against a stub with an injected 400ms delay on both endpoints: city page (1 call) and month/records pages (2 calls each) all cost ~0.404s now, not double for the two-call pages. Error priority preserved exactly: primary's error wins even when City also fails, matching the old sequential code. One trade-off: City() is now always launched even on a request about to 404 from primary, costing one extra cheap lookup on that rare path. Tests include a deterministic concurrency proof via rendezvous channels (the old sequential code would deadlock this test, not just run it slower). Full suite green under -race -count=2. --- .../server/internal/content/content_test.go | 111 ++++++++++++++++++ frontend/server/internal/content/pages.go | 67 ++++++++--- 2 files changed, 164 insertions(+), 14 deletions(-) diff --git a/frontend/server/internal/content/content_test.go b/frontend/server/internal/content/content_test.go index ef90a28..1d199ea 100644 --- a/frontend/server/internal/content/content_test.go +++ b/frontend/server/internal/content/content_test.go @@ -19,6 +19,7 @@ import ( "path/filepath" "strings" "testing" + "time" "thermograph/frontend/internal/config" "thermograph/frontend/internal/contentapi" @@ -474,6 +475,116 @@ func TestMonthCtx(t *testing.T) { } } +// --- fetchWithCity ----------------------------------------------------------- + +func TestFetchWithCityHappyPath(t *testing.T) { + api := &fakeAPI{ + city: func(slug, _ string) (*contentapi.CityPayload, error) { + return &contentapi.CityPayload{City: contentapi.CityInfo{Slug: slug, Name: "Testville"}}, nil + }, + } + primary, city, err := fetchWithCity(api, "testville", func() (string, error) { return "primary-value", nil }) + if err != nil { + t.Fatal(err) + } + if primary != "primary-value" { + t.Errorf("primary = %q", primary) + } + if city.Slug != "testville" || city.Name != "Testville" { + t.Errorf("city = %+v", city) + } +} + +// When only primary fails, its error must win even though City() also ran +// (and, here, succeeded) -- matching the old sequential code's behavior, +// where City() was never even called once primary had already failed. +func TestFetchWithCityPrimaryErrorWins(t *testing.T) { + primaryErr := notFound("no such month") + api := &fakeAPI{ + city: func(slug, _ string) (*contentapi.CityPayload, error) { + return &contentapi.CityPayload{City: contentapi.CityInfo{Slug: slug}}, nil + }, + } + _, _, err := fetchWithCity(api, "testville", func() (string, error) { return "", primaryErr }) + if err != primaryErr { + t.Errorf("err = %v, want the primary error", err) + } +} + +// When primary succeeds but City() fails, City's error must surface -- same +// as the old sequential code (it called City() second and returned whatever +// that call did). +func TestFetchWithCityCityErrorSurfaces(t *testing.T) { + cityErr := notFound("no such city") + api := &fakeAPI{ + city: func(_, _ string) (*contentapi.CityPayload, error) { return nil, cityErr }, + } + _, _, err := fetchWithCity(api, "testville", func() (string, error) { return "primary-value", nil }) + if err != cityErr { + t.Errorf("err = %v, want the city error", err) + } +} + +// When BOTH fail, primary's error still wins -- the same priority as the +// single-failure case above, just with both goroutines erroring. +func TestFetchWithCityBothErrorPrimaryWins(t *testing.T) { + primaryErr, cityErr := notFound("primary"), notFound("city") + api := &fakeAPI{ + city: func(_, _ string) (*contentapi.CityPayload, error) { return nil, cityErr }, + } + _, _, err := fetchWithCity(api, "testville", func() (string, error) { return "", primaryErr }) + if err != primaryErr { + t.Errorf("err = %v, want the primary error (city error must not win)", err) + } +} + +// Proves the two calls actually run CONCURRENTLY rather than sequentially -- +// deterministically, via a rendezvous, not by timing (which would be flaky +// under CI load). Each fake blocks until BOTH have started; if fetchWithCity +// ran them sequentially, the second would never start until the first +// returned, and this would deadlock and fail on the test's own timeout. +func TestFetchWithCityRunsConcurrently(t *testing.T) { + started := make(chan struct{}, 2) + release := make(chan struct{}) + rendezvous := func() { + started <- struct{}{} + <-release + } + + api := &fakeAPI{ + city: func(_, _ string) (*contentapi.CityPayload, error) { + rendezvous() + return &contentapi.CityPayload{}, nil + }, + } + + done := make(chan struct{}) + go func() { + fetchWithCity(api, "testville", func() (string, error) { + rendezvous() + return "", nil + }) + close(done) + }() + + // Both goroutines must reach the rendezvous before either can proceed -- + // only possible if they were launched concurrently. + for range 2 { + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for both calls to start -- they are not running concurrently") + } + } + close(release) + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("fetchWithCity did not return after release") + } +} + func TestMonthCtxUnknownLabelIsError(t *testing.T) { h := newHandlers(t, fixtureAPI(t)) j := loadFixture[contentapi.MonthPayload](t, "month") diff --git a/frontend/server/internal/content/pages.go b/frontend/server/internal/content/pages.go index 43eaedf..a508df0 100644 --- a/frontend/server/internal/content/pages.go +++ b/frontend/server/internal/content/pages.go @@ -7,6 +7,7 @@ import ( "html/template" "net/http" "strings" + "sync" "thermograph/frontend/internal/contentapi" "thermograph/frontend/internal/contentdata" @@ -165,6 +166,49 @@ func (h *Handlers) cityCtx(j *contentapi.CityPayload) (map[string]any, format.Un return ctx, unit, nil } +// fetchWithCity runs a page's primary API call and the shared City lookup +// concurrently instead of sequentially: both are independent single-slug +// fetches (City never depends on primary's result), so waiting for one to +// fully round-trip before even starting the other was pure latency with +// nothing to show for it. Concurrent instead cuts that leg to roughly +// whichever call is slower. +// +// If primary fails, its error wins even when City also fails or hasn't +// finished — matching the old sequential code's behavior exactly (it never +// even called City() once primary had already failed). The one real +// trade-off: City() is now ALWAYS launched, even on a request that's about +// to 404 from primary, so an invalid slug costs one extra (wasted, cheap) +// backend lookup it previously skipped. Worth it: a bad slug is the rare +// path; a good one is the common path this speeds up. +func fetchWithCity[T any](api API, slug string, primary func() (T, error)) (T, contentapi.CityInfo, error) { + var ( + pVal T + pErr error + cPay *contentapi.CityPayload + cErr error + group sync.WaitGroup + ) + group.Add(2) + go func() { + defer group.Done() + pVal, pErr = primary() + }() + go func() { + defer group.Done() + cPay, cErr = api.City(slug, "") // no origin: the Python called city(slug) bare here too + }() + group.Wait() + + if pErr != nil { + return pVal, contentapi.CityInfo{}, pErr + } + if cErr != nil { + var zero T + return zero, contentapi.CityInfo{}, cErr + } + return pVal, cPay.City, nil +} + // --- month page -------------------------------------------------------------- // MonthPage renders /climate/{slug}/{month} (content.py's month_page). @@ -176,17 +220,14 @@ func (h *Handlers) cityCtx(j *contentapi.CityPayload) (map[string]any, format.Un // Jinja iterated (label, value) tuples). func (h *Handlers) MonthPage(w http.ResponseWriter, r *http.Request) { slug, month := r.PathValue("slug"), r.PathValue("month") - j, err := h.api.CityMonth(slug, month) + j, city, err := fetchWithCity(h.api, slug, func() (*contentapi.MonthPayload, error) { + return h.api.CityMonth(slug, month) + }) if err != nil { h.apiError(w, err) return } - cp, err := h.api.City(slug, "") // no origin: the Python called city(slug) bare here - if err != nil { - h.apiError(w, err) - return - } - ctx, unit, err := h.monthCtx(j, cp.City) + ctx, unit, err := h.monthCtx(j, city) if err != nil { h.serverError(w, err) return @@ -308,17 +349,15 @@ func fmtPeriodSide(unit format.Unit, s contentapi.PeriodSide) map[string]any { // temperature metrics, nil otherwise, as the Python set them. func (h *Handlers) RecordsPage(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") - j, err := h.api.CityRecords(slug, origin(r)) + o := origin(r) + j, city, err := fetchWithCity(h.api, slug, func() (*contentapi.RecordsPayload, error) { + return h.api.CityRecords(slug, o) + }) if err != nil { h.apiError(w, err) return } - cp, err := h.api.City(slug, "") // no origin, same as month_page - if err != nil { - h.apiError(w, err) - return - } - ctx, unit, err := h.recordsCtx(j, cp.City) + ctx, unit, err := h.recordsCtx(j, city) if err != nil { h.serverError(w, err) return -- 2.45.2 From dcf15ea5724c4deae8cff90c02adc0d4df30a0e9 Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 18:56:46 +0000 Subject: [PATCH 04/10] infra/forgejo: add resource limits and a leaner CI job image (#38) --- infra/deploy/forgejo/README.md | 46 +++++++++++++++++++++ infra/deploy/forgejo/ci-runner/Dockerfile | 46 +++++++++++++++++++++ infra/deploy/forgejo/docker-stack.yml | 8 ++++ infra/deploy/forgejo/register-lan-runner.sh | 2 +- 4 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 infra/deploy/forgejo/ci-runner/Dockerfile diff --git a/infra/deploy/forgejo/README.md b/infra/deploy/forgejo/README.md index 0325386..8ad737d 100644 --- a/infra/deploy/forgejo/README.md +++ b/infra/deploy/forgejo/README.md @@ -41,6 +41,15 @@ Do this **before** the DNS + Caddy step below — Caddy's reverse_proxy target (`127.0.0.1:3080`) needs the `forgejo` service actually listening first, or its first health check just fails harmlessly until it is. +This stack has **no auto-deploy trigger** — nothing in `.forgejo/workflows/` +redeploys it on push. A change to `docker-stack.yml` only takes effect once +someone re-runs `docker stack deploy` by hand on the manager (prod). + +`db`/`forgejo` both carry `resources.limits` (defaults: db 1 CPU/1g, forgejo 2 +CPU/2g — several times observed steady-state usage), overridable with +`FORGEJO_DB_CPUS`/`FORGEJO_DB_MEMORY`/`FORGEJO_CPUS`/`FORGEJO_MEMORY` env vars +before `docker stack deploy`, same convention as the app stack. + ## DNS + TLS: reusing beta's existing Caddy, not a second reverse proxy Forgejo is pinned to beta (`role=forge`) — but beta is also **today's live @@ -98,6 +107,43 @@ See that script's header for exactly what it replaces (the pre-Forgejo GitHub self-hosted runner on this same machine) and why it registers with two labels where there used to be two separate runners. +## Custom CI job image (`ci-runner/`) + +`ci-runner/Dockerfile` still bases on `node:20-bookworm` — Node is a hard +requirement, not leftover: Forgejo's runner executes `actions/checkout@v4` +(used by every workflow) as `node dist/index.js` *inside the job container*, +regardless of whether the workflow itself uses npm/node. (v1 of this image +tried a Node-free Debian-slim base and broke every job's checkout step — +`node: executable file not found` — within a minute of going live; reverted +immediately.) What it actually fixes: every `docker`-labeled build-push job +currently re-installs the Docker CLI on each run (`apt-get install +docker.io`), which pulls in the classic builder rather than BuildKit (the +classic builder mishandles `COPY --chown=` group resolution — a real +bug hit during the frontend Go rewrite). `ci-runner` adds `docker-ce-cli` + +`docker-buildx-plugin` (BuildKit) on top of the same Node base, plus +`git`/`python3`/`python3-yaml` for the other jobs that need them +(`shell-lint`, `observability-validate`). + +Current tag: `git.thermograph.org/emi/thermograph/ci-runner:v2` (`v1` is +broken — do not register any runner against it). Rebuild/push (requires a PAT +with `write:package` scope — the embedded git-remote token lacks it, same +requirement documented in `backend-build-push.yml`): + +```bash +docker build -t git.thermograph.org/emi/thermograph/ci-runner:vN deploy/forgejo/ci-runner +docker push git.thermograph.org/emi/thermograph/ci-runner:vN +``` + +`register-lan-runner.sh`'s `LABELS` default points at the current tag, so +fresh registrations pick it up automatically. The live runner is cut over by +editing the `labels` array in `~/forgejo-runner/.runner` on the desktop (same +runner id/token, no re-registration needed) and restarting the service — +**verify a real job runs green under the new image before relying on it**, +same way v1's break was caught. Only after that verification should the +now-redundant `apt-get install docker.io` / `python3-yaml` steps be removed +from the workflows that had them — removing them first would break every job +still running on the stock `node:20-bookworm` image. + ## Why Postgres here and not the Thermograph app's TimescaleDB Separate instance, separate network (`forgejo_net`, not the app's compose diff --git a/infra/deploy/forgejo/ci-runner/Dockerfile b/infra/deploy/forgejo/ci-runner/Dockerfile new file mode 100644 index 0000000..e4259e0 --- /dev/null +++ b/infra/deploy/forgejo/ci-runner/Dockerfile @@ -0,0 +1,46 @@ +# Job-container image for the LAN Forgejo Actions runner's `docker`-labeled +# jobs (see ../register-lan-runner.sh) — used by every backend/frontend +# build-push, deploy, and validation workflow that needs `docker build`. +# +# Base stays node:20-bookworm, NOT a Node-free slim image (v1 of this file +# tried that and broke every job: `actions/checkout@v4` is a JS action that +# Forgejo's runner execs as `node dist/index.js` *inside the job container*, +# so a Node runtime is a hard requirement regardless of whether the workflow +# itself uses npm/node — this repo's own workflows don't, but the checkout +# step every one of them starts with does). +# +# What this image actually fixes: every workflow using the `docker` label +# currently re-provisions the Docker CLI on each run via `apt-get install +# docker.io` — that step costs ~15-20s per job and, more importantly, +# installs the CLASSIC Docker builder rather than BuildKit, which mishandles +# `COPY --chown=` group resolution on images without a matching system +# group (a real bug hit during the frontend Go rewrite). Installing +# docker-buildx-plugin here makes BuildKit the default, closing that bug +# class, and skips the per-job install entirely. +# +# Build/push (manual — see ../README.md for when to rebuild): +# docker build -t git.thermograph.org/emi/thermograph/ci-runner:vN \ +# infra/deploy/forgejo/ci-runner +# docker push git.thermograph.org/emi/thermograph/ci-runner:vN +# +# Cutting over the live runner to a new tag means editing the `labels` array +# in ~/forgejo-runner/.runner on the runner host directly (same runner +# id/token, no re-registration needed) and updating register-lan-runner.sh's +# LABELS default so future re-provisioning picks it up too. +FROM node:20-bookworm + +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends \ + ca-certificates curl gnupg git python3 python3-yaml \ + && install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc \ + && chmod a+r /etc/apt/keyrings/docker.asc \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" \ + > /etc/apt/sources.list.d/docker.list \ + && apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends docker-ce-cli docker-buildx-plugin \ + && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/docker.list + +RUN docker --version && docker buildx version && git --version \ + && node --version \ + && python3 -c "import yaml; print('PyYAML', yaml.__version__)" diff --git a/infra/deploy/forgejo/docker-stack.yml b/infra/deploy/forgejo/docker-stack.yml index 9d94cbd..9891c3b 100644 --- a/infra/deploy/forgejo/docker-stack.yml +++ b/infra/deploy/forgejo/docker-stack.yml @@ -48,6 +48,10 @@ services: deploy: placement: constraints: [node.labels.role == forge] + resources: + limits: + cpus: "${FORGEJO_DB_CPUS:-1}" + memory: ${FORGEJO_DB_MEMORY:-1g} restart_policy: condition: on-failure @@ -131,6 +135,10 @@ services: deploy: placement: constraints: [node.labels.role == forge] + resources: + limits: + cpus: "${FORGEJO_CPUS:-2}" + memory: ${FORGEJO_MEMORY:-2g} restart_policy: condition: on-failure diff --git a/infra/deploy/forgejo/register-lan-runner.sh b/infra/deploy/forgejo/register-lan-runner.sh index 71f1c90..c35660d 100755 --- a/infra/deploy/forgejo/register-lan-runner.sh +++ b/infra/deploy/forgejo/register-lan-runner.sh @@ -26,7 +26,7 @@ set -euo pipefail FORGEJO_URL="${1:?usage: $0 }" TOKEN="${2:?}" RUNNER_DIR="${RUNNER_DIR:-$HOME/forgejo-runner}" -LABELS="${LABELS:-docker:docker://node:20-bookworm,thermograph-lan}" +LABELS="${LABELS:-docker:docker://git.thermograph.org/emi/thermograph/ci-runner:v2,thermograph-lan}" echo "==> Stopping and disabling the old GitHub Actions runner service, if present" systemctl --user stop github-actions-runner 2>/dev/null || true -- 2.45.2 From 81c579ddcb5f48c249e91e047eb9c073de1b408e Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 19:05:21 +0000 Subject: [PATCH 05/10] infra/forgejo: raise LAN runner concurrency to 8 (#43) --- infra/deploy/forgejo/README.md | 19 +++++++++++++++++++ infra/deploy/forgejo/register-lan-runner.sh | 12 ++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/infra/deploy/forgejo/README.md b/infra/deploy/forgejo/README.md index 8ad737d..6f91009 100644 --- a/infra/deploy/forgejo/README.md +++ b/infra/deploy/forgejo/README.md @@ -107,6 +107,25 @@ See that script's header for exactly what it replaces (the pre-Forgejo GitHub self-hosted runner on this same machine) and why it registers with two labels where there used to be two separate runners. +`config.yaml`'s `runner.capacity` is raised from the tool's default of 1 to 8 +(override with `CAPACITY=`) — a single PR push fires `pr-build`, +`secrets-guard`, and `shell-lint` simultaneously (3 independent workflows, +no `needs:` between them), so capacity 1 serializes work that could run in +parallel, and even capacity 3 (an earlier, undocumented hand-tune) leaves +`build-backend`/`build-frontend`/`validate-observability` queued behind +those three before they get a slot. The desktop has 16 cores / 34GB free +today; 8 concurrent jobs is comfortable headroom without starving LAN dev's +own compose stack. + +**Adding more runner capacity should mean raising this number, or adding a +second runner on the desktop itself — not putting a runner on prod or +beta.** `container.docker_host: automount` gives job containers the *host's* +Docker socket; on prod or beta that would mean any CI job has root-equivalent +access to whatever else is running there (the live app stack, or Forgejo +itself). An earlier revision of this stack actually did run the runner as a +Swarm-hosted container on beta and was deliberately reverted to the desktop +for this reason — see the note at the top of `docker-stack.yml`. + ## Custom CI job image (`ci-runner/`) `ci-runner/Dockerfile` still bases on `node:20-bookworm` — Node is a hard diff --git a/infra/deploy/forgejo/register-lan-runner.sh b/infra/deploy/forgejo/register-lan-runner.sh index c35660d..0f96f97 100755 --- a/infra/deploy/forgejo/register-lan-runner.sh +++ b/infra/deploy/forgejo/register-lan-runner.sh @@ -62,11 +62,15 @@ echo "==> Registering with $FORGEJO_URL (label: $LABELS)" --name "thermograph-lan-$(hostname -s)" \ --labels "$LABELS" -echo "==> Runner config (docker.sock automount, so docker:-labeled jobs like" -echo " build-push.yml can actually run 'docker build/push' -- generated" -echo " config only overridden on the one setting that matters here)" +echo "==> Runner config (docker.sock automount so docker:-labeled jobs like" +echo " build-push.yml can actually run 'docker build/push'; capacity raised" +echo " from the default of 1 -- a single PR push fires 3+ independent" +echo " workflows (pr-build, secrets-guard, shell-lint) simultaneously, so" +echo " anything less than that serializes jobs that could run in parallel)" +CAPACITY="${CAPACITY:-8}" ./forgejo-runner generate-config \ - | sed 's/docker_host: "-"/docker_host: "automount"/' \ + | sed -e 's/docker_host: "-"/docker_host: "automount"/' \ + -e "s/capacity: 1/capacity: ${CAPACITY}/" \ > "${RUNNER_DIR}/config.yaml" echo "==> systemd --user unit" -- 2.45.2 From ca84e0ce95f090bf9865012e0db3b897772611d9 Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 23:20:16 +0000 Subject: [PATCH 06/10] =?UTF-8?q?Promote=20dev=20=E2=86=92=20main=20(front?= =?UTF-8?q?end=20QA=20batch=20=E2=86=92=20beta)=20(#71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 ++ backend/api/content_routes.py | 18 +++++++- backend/data/climate.py | 29 ++++++++++++- backend/data/grading.py | 29 +++++++------ backend/notifications/push.py | 13 +++--- backend/tests/api/test_content_routes.py | 22 ++++++++++ backend/tests/data/test_climate.py | 43 +++++++++++++++++++ backend/tests/data/test_grading.py | 6 ++- backend/tests/notifications/test_push.py | 52 +++++++++++++++++++++-- backend/tests/web/test_api.py | 12 ++++++ backend/web/app.py | 18 +++++++- frontend/content.py | 18 +++++++- frontend/static/app.js | 11 +++-- frontend/static/calendar.js | 6 +-- frontend/static/chart.js | 10 +++-- frontend/static/day.js | 18 +++++--- frontend/static/push-client.js | 34 +++++++++++++-- frontend/static/shared.js | 24 +++++++++-- frontend/tests/unit/test_rendering.py | 45 ++++++++++++++++++++ infra/.claude/skills/key-gaps/SKILL.md | 14 +++++- infra/.claude/skills/key-gaps/key_gaps.py | 2 +- 21 files changed, 374 insertions(+), 54 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5aee0f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ + +# Claude Code worktrees. These are other sessions' checkouts living inside this +# repo; `git add -A` will otherwise commit them as embedded git repositories. +.claude/worktrees/ diff --git a/backend/api/content_routes.py b/backend/api/content_routes.py index cd682b1..ef01df2 100644 --- a/backend/api/content_routes.py +++ b/backend/api/content_routes.py @@ -6,6 +6,7 @@ pattern web/app.py's own endpoints use. """ import hashlib import os +from urllib.parse import urlsplit from fastapi import APIRouter, HTTPException, Request, Response @@ -84,9 +85,24 @@ def _load_history(cell): return history +# The configured public origin is the source of truth for the scheme. Caddy +# terminates TLS and reverse-proxies plain HTTP, so x-forwarded-proto (and +# request.url.scheme) read "http" even though the public site is HTTPS-only -- +# and this origin is folded into the jsonld "url" the payloads carry. Trusting +# the forwarded scheme makes that url (and the canonical/og the frontend builds +# from the same origin it forwards here) emit http://, which 308-redirects to +# https://. So a request that arrives on the configured public host is answered +# with the configured public scheme; localhost/LAN dev never matches that host +# and keeps the observed scheme, leaving plain-HTTP development unaffected. +_PUBLIC = urlsplit(os.environ.get("THERMOGRAPH_BASE_URL", "https://thermograph.org")) + + def _origin(request: Request) -> str: - proto = request.headers.get("x-forwarded-proto") or request.url.scheme host = request.headers.get("host") or request.url.netloc + if _PUBLIC.scheme and _PUBLIC.netloc and host == _PUBLIC.netloc: + proto = _PUBLIC.scheme + else: + proto = request.headers.get("x-forwarded-proto") or request.url.scheme return f"{proto}://{host}" diff --git a/backend/data/climate.py b/backend/data/climate.py index b1c5e00..2bb1b2f 100644 --- a/backend/data/climate.py +++ b/backend/data/climate.py @@ -937,9 +937,29 @@ def _fetch_recent_forecast(cell: dict) -> pl.DataFrame: .sort("date")) +def _om_local_today(payload: dict) -> "datetime.date | None": + """The calendar date it is *now* at the cell, from the UTC offset Open-Meteo + reports for a ``timezone=auto`` request. None when the offset is absent, which + tells the caller to skip the in-progress-day guard rather than guess a date.""" + offset = payload.get("utc_offset_seconds") + if offset is None: + return None + return (datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=int(offset))).date() + + def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame: """Fallback recent+forecast bundle from the Open-Meteo forecast API (the former - primary): recent past + forward days in one call.""" + primary): recent past + forward days in one call. + + The cell's in-progress *local* day is dropped from the bundle. Open-Meteo's + daily high/low for today aggregates only the hours elapsed so far, so grading it + reads a still-unfolding day as complete and produces spurious extremes (a cool + morning served as a record-low high, seen live at the 1st percentile). This is + the same failure the MET Norway path guards against with its diurnal-coverage + gate (see _metno_to_frame); the fallback needs the equivalent. Past days are + complete and future days are whole-day forecasts, so only today is excluded — + the day lands in the record once it is over.""" params = { "latitude": cell["center_lat"], "longitude": cell["center_lon"], @@ -952,7 +972,12 @@ def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame: "forecast_days": FORECAST_DAYS, } r = _request(FORECAST_URL, params, 60, phase="recent_forecast_fetch") - return _to_frame(r.json()["daily"]) + payload = r.json() + df = _to_frame(payload["daily"]) + local_today = _om_local_today(payload) + if local_today is not None: + df = df.filter(pl.col("date") != local_today) + return df def _load_recent_forecast(cell: dict) -> pl.DataFrame: diff --git a/backend/data/grading.py b/backend/data/grading.py index dfd34df..a160eb1 100644 --- a/backend/data/grading.py +++ b/backend/data/grading.py @@ -279,8 +279,10 @@ def all_time_records(df: pl.DataFrame) -> dict: def longest_dry_streak(df: pl.DataFrame) -> tuple[int, str | None]: - """Longest run of consecutive days without measurable rain, and the ISO date the - streak began. A dry day is precip < RAIN_THRESHOLD; null precip counts as dry.""" + """Longest run of consecutive days without any rain, and the ISO date the streak + began. A dry day has no rain at all (precip <= 0); null precip counts as dry. The + threshold matches _grade_precip's dry/rain split — any measurable rain, however + slight, is a rain day that breaks the streak (not the 0.01" rain-frequency line).""" if "precip" not in df.columns: return (0, None) d = df.select(["date", "precip"]).sort("date") @@ -288,7 +290,7 @@ def longest_dry_streak(df: pl.DataFrame) -> tuple[int, str | None]: best_len, best_start = 0, None cur_len, cur_start = 0, None for dt, p in zip(dates, precips): - wet = p is not None and not (isinstance(p, float) and np.isnan(p)) and p >= RAIN_THRESHOLD + wet = p is not None and not (isinstance(p, float) and np.isnan(p)) and p > 0 if wet: cur_len, cur_start = 0, None else: @@ -305,11 +307,12 @@ def longest_dry_streak(df: pl.DataFrame) -> tuple[int, str | None]: def _band_stats(samples: np.ndarray) -> dict | None: if samples.size == 0: return None - # Percentiles for the chart's nested "normal" fan, matching the 9 tier bounds: - # p40-p60 is the Normal band; p25/p75, p10/p90 and p1/p99 mark the successive - # Below/Above Normal, Low/High, Very Low/High and Near-Record edges. - p1, p10, p25, p40, p50, p60, p75, p90, p99 = np.percentile( - samples, [1, 10, 25, 40, 50, 60, 75, 90, 99] + # Percentiles for the chart's nested "normal" fan. p40-p60 is the Normal band; + # p25/p75, p10/p90 and p1/p99 mark the successive Below/Above Normal, Low/High, + # Very Low/High and Near-Record edges. p95 additionally splits the rain fan's top + # region into Very Heavy (90-95) and Severe (95-99); unused by the temperature fan. + p1, p10, p25, p40, p50, p60, p75, p90, p95, p99 = np.percentile( + samples, [1, 10, 25, 40, 50, 60, 75, 90, 95, 99] ) return { "p1": round(float(p1), 1), @@ -320,6 +323,7 @@ def _band_stats(samples: np.ndarray) -> dict | None: "p60": round(float(p60), 1), "p75": round(float(p75), 1), "p90": round(float(p90), 1), + "p95": round(float(p95), 1), "p99": round(float(p99), 1), } @@ -352,13 +356,14 @@ def _grade_precip(samples: np.ndarray, value) -> dict | None: def dry_streaks(dates, precips) -> dict[str, int]: - """Map each date (ISO string) to days since the last measurable rain, walking a - chronological precip series. Missing precip counts as a dry day. `dates` is an - iterable of ``datetime.date`` (a polars Date column's ``.to_list()``).""" + """Map each date (ISO string) to days since the last day with any rain, walking a + chronological precip series. Any measurable rain (precip > 0) resets the count, so + this matches _grade_precip's dry/rain split; missing precip counts as a dry day. + `dates` is an iterable of ``datetime.date`` (a polars Date column's ``.to_list()``).""" out: dict[str, int] = {} streak = 0 for d, p in zip(dates, precips): - wet = p is not None and not (isinstance(p, float) and np.isnan(p)) and p >= RAIN_THRESHOLD + wet = p is not None and not (isinstance(p, float) and np.isnan(p)) and p > 0 streak = 0 if wet else streak + 1 out[_as_date(d).isoformat()] = streak return out diff --git a/backend/notifications/push.py b/backend/notifications/push.py index 594607f..d2807e7 100644 --- a/backend/notifications/push.py +++ b/backend/notifications/push.py @@ -37,7 +37,7 @@ log = logging.getLogger("thermograph.push") _DATA_DIR = paths.DATA_DIR _VAPID_PATH = os.environ.get("THERMOGRAPH_VAPID_FILE") or os.path.join(_DATA_DIR, "vapid.json") # The VAPID "sub" claim — a contact the push service can reach about our traffic. -_CONTACT = os.environ.get("THERMOGRAPH_VAPID_CONTACT", "mailto:admin@thermograph.app") +_CONTACT = os.environ.get("THERMOGRAPH_VAPID_CONTACT", "mailto:admin@thermograph.org") # pywebpush 2.0.0 forwards `timeout` straight to requests.post with no default of # its own — omit the kwarg here and the send blocks with NO timeout at all (the @@ -174,10 +174,13 @@ def send(subscription_info: dict, payload: dict) -> str: return "ok" except WebPushException as e: status = getattr(getattr(e, "response", None), "status_code", None) - if status in (404, 410): - return "gone" # endpoint retired — caller should delete it - # 401/403 = VAPID key mismatch, etc. Record it where it's visible (the errors - # JSONL / dashboard), not just the system journal. + if status in (401, 403, 404, 410): + # 404/410 = endpoint retired; 401/403 = VAPID mismatch (e.g. after a key + # rotation) — the row was minted under a key we can no longer sign for and + # will never authenticate again. All are permanently dead: caller deletes. + return "gone" + # Anything else (rate limits, 5xx, malformed payload) may be transient — keep + # the row and record it where it's visible (the errors JSONL / dashboard). log.warning("web push failed (status=%s): %s", status, e) audit.log_event("error", {"phase": "push", "status": status, "endpoint": (subscription_info.get("endpoint") or "")[:120]}) diff --git a/backend/tests/api/test_content_routes.py b/backend/tests/api/test_content_routes.py index 236601a..e172b9b 100644 --- a/backend/tests/api/test_content_routes.py +++ b/backend/tests/api/test_content_routes.py @@ -118,6 +118,28 @@ def test_records_payload_shape(client): assert body["canonical_path"] == "/climate/testville/records" +# --- https origin (jsonld url) ----------------------------------------------- + +def test_public_host_jsonld_url_is_https(client): + """The public site is HTTPS-only behind Caddy, which proxies plain HTTP, so + x-forwarded-proto reads "http". A request on the configured public host + (THERMOGRAPH_BASE_URL defaults to https://thermograph.org) must still fold an + https:// origin into the payload's jsonld url; otherwise the frontend renders + an http:// canonical/JSON-LD that 308-redirects.""" + import json as _json + hdr = {"host": "thermograph.org", "x-forwarded-proto": "http"} + for path in ("city/testville", "city/testville/records"): + s = _json.dumps(client.get(f"{BASE}/{path}", headers=hdr).json()) + assert "https://thermograph.org/thermograph/climate/testville" in s + assert "http://thermograph.org" not in s + + +def test_non_public_host_jsonld_keeps_forwarded_scheme(client): + hdr = {"host": "192.168.1.10:8000", "x-forwarded-proto": "http"} + body = client.get(f"{BASE}/city/testville", headers=hdr).json() + assert body["jsonld"]["@graph"][0]["url"].startswith("http://192.168.1.10:8000/") + + # --- 404s -------------------------------------------------------------------- def test_unknown_slug_is_404(client, counts): diff --git a/backend/tests/data/test_climate.py b/backend/tests/data/test_climate.py index fab8bad..de90ff3 100644 --- a/backend/tests/data/test_climate.py +++ b/backend/tests/data/test_climate.py @@ -208,6 +208,49 @@ def test_recent_forecast_falls_back_to_open_meteo(monkeypatch, tmp_path): assert df.height == om.height +def test_recent_forecast_om_drops_the_in_progress_local_day(monkeypatch): + """The Open-Meteo fallback must not grade the cell's in-progress local day: its + daily high/low is only a partial aggregate of the hours elapsed so far (a cool + morning would read as a record-low high). Past days and future forecast days + survive; today — per the UTC offset Open-Meteo reports for a timezone=auto + request — is dropped, matching the MET path's diurnal-coverage gate.""" + offset = 3 * 3600 # UTC+3, e.g. Europe/Vilnius (the reported Ringaudai incident) + local_today = (datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=offset)).date() + days = [local_today + datetime.timedelta(days=n) for n in (-2, -1, 0, 1)] + daily = { + "time": [d.isoformat() for d in days], + "temperature_2m_max": [80.0, 82.0, 61.0, 84.0], # today's 61 is the partial value + "temperature_2m_min": [60.0, 61.0, 57.0, 62.0], + "precipitation_sum": [0.0, 0.0, 0.0, 0.0], + } + payload = {"utc_offset_seconds": offset, "daily": daily} + + class Resp: + def json(self): return payload + monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) + + got = climate._fetch_recent_forecast_om( + {"center_lat": 54.9, "center_lon": 23.8})["date"].to_list() + assert local_today not in got # partial today dropped + assert local_today - datetime.timedelta(days=1) in got # yesterday kept + assert local_today + datetime.timedelta(days=1) in got # tomorrow's forecast kept + assert len(got) == 3 + + +def test_recent_forecast_om_keeps_all_days_without_a_utc_offset(monkeypatch): + """No UTC offset reported -> skip the in-progress-day guard rather than guess a + date, so the bundle passes through as before (only the usual null-day filter).""" + payload = {"daily": _om_daily(3)} # no utc_offset_seconds + + class Resp: + def json(self): return payload + monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) + + df = climate._fetch_recent_forecast_om({"center_lat": 1.0, "center_lon": 2.0}) + assert df.height == climate._to_frame(_om_daily(3)).height + + def test_recent_forecast_serves_stale_cache_when_all_sources_fail(monkeypatch, tmp_path): """With every source down (the NASA + MET Norway primary and the Open-Meteo fallback), an existing (stale) cache is served rather than failing — and it is NOT diff --git a/backend/tests/data/test_grading.py b/backend/tests/data/test_grading.py index cfe7d28..a2f512a 100644 --- a/backend/tests/data/test_grading.py +++ b/backend/tests/data/test_grading.py @@ -113,7 +113,9 @@ def test_dry_streaks_walk(): dates = [datetime.date(2024, 1, 1) + datetime.timedelta(days=i) for i in range(5)] precips = [0.5, 0.0, float("nan"), 0.02, 0.005] out = grading.dry_streaks(dates, precips) - assert list(out.values()) == [0, 1, 2, 0, 1] # NaN counts as dry + # Any rain > 0 breaks the streak (matching the dry/rain grading split), so the + # 0.005" trace day resets to 0; only exact 0.0 and NaN count as dry. + assert list(out.values()) == [0, 1, 2, 0, 0] # ---- range + day grading over a synthetic record -------------------------------- @@ -141,7 +143,7 @@ def test_grade_day_normals_and_departure(history): result = grading.grade_day(history, target, {"tmax": row["tmax"], "tmin": row["tmin"], "precip": row["precip"]}) assert set(result["normals"]["tmax"]) == {"p1", "p10", "p25", "p40", "p50", "p60", - "p75", "p90", "p99"} + "p75", "p90", "p95", "p99"} expected = max(abs(result["tmax"]["percentile"] - 50), abs(result["tmin"]["percentile"] - 50)) assert result["departure"] == round(expected, 1) diff --git a/backend/tests/notifications/test_push.py b/backend/tests/notifications/test_push.py index 422583f..5450778 100644 --- a/backend/tests/notifications/test_push.py +++ b/backend/tests/notifications/test_push.py @@ -1,11 +1,16 @@ """VAPID key resolution: the atomic first-writer-wins claim of the keypair file (push.py's `_claim_file`), and that `_load()` caches whatever that settles on -rather than its own local generation when it loses the race. +rather than its own local generation when it loses the race. Also `send()`'s +mapping of push-service responses onto 'ok'/'gone'/'error'. -No network — `send()`'s pywebpush call isn't exercised here (that's notify.py's -`test_push_dispatched_on_new_notification` etc., which stub `push.send` itself).""" +No network — where `send()` is exercised the pywebpush call is stubbed; the +dispatch path (notify.py's `test_push_dispatched_on_new_notification` etc.) stubs +`push.send` itself.""" import json +import pytest +from pywebpush import WebPushException + from notifications import push @@ -94,3 +99,44 @@ def test_load_prefers_env_over_file(monkeypatch, tmp_path): result = push._load() assert result == {"private_key": "priv-env", "public_key": "pub-env"} + + +# --- send() response mapping ------------------------------------------------ +_SUB = {"endpoint": "https://push.example.com/ep-1", "keys": {"p256dh": "BKEY", "auth": "YXV0aA"}} + + +class _Resp: + def __init__(self, status_code): + self.status_code = status_code + + +def _raise_status(status): + def _webpush(**kwargs): + raise WebPushException("boom", response=_Resp(status)) + return _webpush + + +@pytest.mark.parametrize("status", [401, 403, 404, 410]) +def test_send_prunes_permanently_dead_endpoints(monkeypatch, status): + # 404/410 = endpoint retired; 401/403 = VAPID key mismatch (e.g. after a key + # rotation). All are permanently dead, so send() reports 'gone' and the caller + # deletes the row — otherwise a rotated key leaves dead subscriptions forever. + monkeypatch.setattr(push, "webpush", _raise_status(status)) + assert push.send(_SUB, {"hello": "world"}) == "gone" + + +@pytest.mark.parametrize("status", [429, 500, 502]) +def test_send_keeps_row_on_transient_failure(monkeypatch, status): + # Rate limits / 5xx may recover; keep the row and report it as an error. + monkeypatch.setattr(push, "webpush", _raise_status(status)) + assert push.send(_SUB, {"hello": "world"}) == "error" + + +def test_send_ok(monkeypatch): + monkeypatch.setattr(push, "webpush", lambda **kwargs: None) + assert push.send(_SUB, {"hello": "world"}) == "ok" + + +def test_default_contact_is_the_org_domain(): + # The VAPID "sub" contact must be a domain we actually own; .app was a typo. + assert push._CONTACT == "mailto:admin@thermograph.org" diff --git a/backend/tests/web/test_api.py b/backend/tests/web/test_api.py index f6893fa..1048001 100644 --- a/backend/tests/web/test_api.py +++ b/backend/tests/web/test_api.py @@ -71,6 +71,18 @@ def test_api_version_reports_backend_contract(client): assert body["backend_version"] == "2" +def test_malformed_date_is_rejected(client): + # A malformed or non-calendar date must be a 422, not a 500: fromisoformat + # raises ValueError on these and the handler used to leak it as a crash. + for bad in ("notadate", "2026-13-40", "2026-02-30"): + for route in ("grade", "day"): + r = client.get(f"/thermograph/api/v2/{route}", params={**Q, "date": bad}) + assert r.status_code == 422, (route, bad, r.status_code) + # A valid date still works. + assert client.get("/thermograph/api/v2/grade", + params={**Q, "date": "2026-07-11"}).status_code == 200 + + def test_day_detail_and_ladders(client, history): r = client.get("/thermograph/api/v2/day", params=Q) assert r.status_code == 200 diff --git a/backend/web/app.py b/backend/web/app.py index b8a5d35..5949033 100644 --- a/backend/web/app.py +++ b/backend/web/app.py @@ -79,6 +79,20 @@ def _weather_fetch_error(e) -> HTTPException: return HTTPException(status_code=502, detail=f"weather data fetch failed: {e}") +def _parse_target_date(date: str | None, default: datetime.date) -> datetime.date: + """Parse the ``date`` query param (YYYY-MM-DD) into a date, falling back to + ``default`` when it's absent. A malformed or non-calendar date (e.g. 2026-13-40) + is a client error, not a crash: surface it as a 422 rather than let + fromisoformat's ValueError become a 500.""" + if not date: + return default + try: + return datetime.date.fromisoformat(date[:10]) + except ValueError: + raise HTTPException(status_code=422, + detail=f"invalid date: {date!r} (expected YYYY-MM-DD)") + + # --- background neighbor warming --------------------------------------------- # /cell?neighbors=1 asks the server to warm the 8 grid cells around the request # (the frontend used to fire 8 staggered prefetch requests, mirroring grid.py's @@ -492,7 +506,7 @@ def api_grade( description="days to grade after the target (observed, or forecast when future; " "the forecast reaches ~7 days out so future days cap there)"), ): - target = datetime.date.fromisoformat(date[:10]) if date else datetime.date.today() + target = _parse_target_date(date, datetime.date.today()) cell = grid.snap(lat, lon) with audit.RunAudit( @@ -580,7 +594,7 @@ def api_day( ) as run: history, cache_meta, _ = _fetch_history(run, cell) last = history["date"].max() - target = datetime.date.fromisoformat(date[:10]) if date else last + target = _parse_target_date(date, last) run.set(target_date=target.isoformat()) def build(place): diff --git a/frontend/content.py b/frontend/content.py index 6981b25..7295362 100644 --- a/frontend/content.py +++ b/frontend/content.py @@ -13,6 +13,7 @@ import hashlib import json import logging import os +from urllib.parse import urlsplit import httpx from fastapi import HTTPException, Request, Response @@ -56,13 +57,28 @@ _log = logging.getLogger(__name__) # --- helpers ------------------------------------------------------------- +# The configured public origin is the source of truth for the scheme. Caddy +# terminates TLS and reverse-proxies plain HTTP, so x-forwarded-proto (and +# request.url.scheme) read "http" even though the public site is HTTPS-only. +# Trusting that scheme makes every canonical / og:url / og:image / sitemap +# emit http://, which 308-redirects to https:// -- self-conflicting canonicals +# and a sitemap full of redirecting URLs. So a request that arrives on the +# configured public host is answered with the configured public scheme (https on +# prod/beta, both of which set THERMOGRAPH_BASE_URL per host); localhost/LAN dev +# never matches it and keeps the observed scheme, so plain-HTTP dev is unchanged. +_PUBLIC = urlsplit(os.environ.get("THERMOGRAPH_BASE_URL", "https://thermograph.org")) + + def _origin(request: Request) -> str: # x-forwarded-host takes precedence over host: when reached via backend's # internal proxy fallback (no Caddy in front -- see _proxy_to_frontend in # backend/web/app.py), Host is the internal hop's own address, not the # browser-facing one. - proto = request.headers.get("x-forwarded-proto") or request.url.scheme host = request.headers.get("x-forwarded-host") or request.headers.get("host") or request.url.netloc + if _PUBLIC.scheme and _PUBLIC.netloc and host == _PUBLIC.netloc: + proto = _PUBLIC.scheme + else: + proto = request.headers.get("x-forwarded-proto") or request.url.scheme return f"{proto}://{host}" diff --git a/frontend/static/app.js b/frontend/static/app.js index 47fa6be..4b21646 100644 --- a/frontend/static/app.js +++ b/frontend/static/app.js @@ -8,7 +8,7 @@ import { W, PW, H, PL, PR, plotTop, plotBot, xLabY, setChartWidth, chartPalette, tempChart, precipChart, dryChart, attachChartHover } from "./chart.js"; import { track } from "./digest.js"; import { TIER_COLORS, SCALE_TEMP, drynessColor, pctOrd, esc, todayISO, placeLabel, - tierKeySegs, GUIDE_LINK, fmtPrecip, fmtWind, fmtHumid } from "./shared.js"; + tierKeySegs, GUIDE_LINK, fmtPrecip, fmtPrecipTier, fmtWind, fmtHumid } from "./shared.js"; let selected = null; // {lat, lon} @@ -323,11 +323,14 @@ function precipCell(d, isFc, isToday) { const dd = ` data-date="${d.date}"`; const cls = "rd-c" + (isFc ? " rd-fc" : isToday ? " rd-today" : ""); if (!g) return `—`; - if (g.value === 0 && d.dsr != null && d.dsr > 0) { - return `${d.dsr}d`; + // Only a genuinely dry day (no rain at all) labels the streak; any rain day — + // down to sub-0.01" trace that rounds to 0 — shows its depth, tinted by tier. + if (g.class === "dry" && d.dsr != null && d.dsr > 0) { + return `${d.dsr}d`; } const col = TIER_COLORS[g.class] || ""; - return `${cRain(g.value)}`; + const amt = g.class && g.class !== "dry" ? fmtPrecipTier(g.value, g.class, false) : cRain(g.value); + return `${amt}`; } // Compact "graded days" table — a ROW per metric, a COLUMN per day (newest first). diff --git a/frontend/static/calendar.js b/frontend/static/calendar.js index ed3f848..c2b19bd 100644 --- a/frontend/static/calendar.js +++ b/frontend/static/calendar.js @@ -7,7 +7,7 @@ import { uv } from "./account.js"; // header sign-in entry + notification bell import { TTL, chunkedFetch, prefetchViews } from "./cache.js"; import { initFindButton, setFindLabel } from "./mappicker.js"; import { TIER_COLORS, PRECIP_COLORS, SCALE_TEMP, SCALE_RAIN, drynessColor, pctOrd, - placeLabel, fmtPrecip, fmtWind, fmtHumid, MONTHS, isoOfDate, monthStart, monthEnd, + placeLabel, fmtPrecip, fmtPrecipTier, fmtWind, fmtHumid, MONTHS, isoOfDate, monthStart, monthEnd, CHUNK_MONTHS, buildChunks, clickOpensPicker, metricBuckets, distStrip, seasonFilterDropdown, seasonSummaryText, syncSeasonChecks, applySeasonMonthChange, initSeasonExpand, allMonths, monthsToMask, maskToMonths, @@ -42,7 +42,7 @@ const SKY_WORDS = [ // The shared weather summary, adapted to the calendar's compact day records // (dsr included, so a long-dry no-rain day reads "dry" rather than "clear"). const weatherType = (rec) => - wxType(rec.tmax && rec.tmax.v, rec.precip && rec.precip.v, rec.dsr); + wxType(rec.tmax && rec.tmax.v, rec.precip && rec.precip.v, rec.dsr, rec.precip && rec.precip.c); let selected = null; // {lat, lon} let data = null; // last /api/v2/calendar response @@ -704,7 +704,7 @@ function attachHover(byDate) { ["humid", line("humid", "Humid", rec.humid, fmtHumid, tempColor(rec.humid))], ["wind", line("wind", "Wind", rec.wind, fmtWind, tempColor(rec.wind))], ["gust", line("gust", "Gust", rec.gust, fmtWind, tempColor(rec.gust))], - ["precip", line("precip", "Precip", rec.precip, fmtPrecip, precipColor)], + ["precip", line("precip", "Precip", rec.precip, (v) => fmtPrecipTier(v, rec.precip && rec.precip.c), precipColor)], ]; if (dsrStr) { const rc = metric === "dsr" ? " tt-r-active" : ""; diff --git a/frontend/static/chart.js b/frontend/static/chart.js index d48dfae..6facfcd 100644 --- a/frontend/static/chart.js +++ b/frontend/static/chart.js @@ -64,7 +64,7 @@ const PCT_FALLBACK = { p1: ["p1", "p10", "min", "p50"], p10: ["p10", "p50"], p25: ["p25", "p30", "p50"], p40: ["p40", "p30", "p50"], p60: ["p60", "p70", "p50"], p75: ["p75", "p70", "p50"], - p90: ["p90", "p50"], p99: ["p99", "p90", "max", "p50"], + p90: ["p90", "p50"], p95: ["p95", "p90", "p50"], p99: ["p99", "p90", "max", "p50"], }; const pget = (o, k) => { for (const kk of (PCT_FALLBACK[k] || [k])) if (o && o[kk] != null) return o[kk]; @@ -93,8 +93,12 @@ const TEMP_FAN = [ ["p90", "p99", "very-hot"], ["p75", "p90", "hot"], ["p60", "p75", "warm"], ["p40", "p60", "normal"], ["p25", "p40", "cool"], ["p10", "p25", "cold"], ["p1", "p10", "very-cold"], ]; +// Eight rain-intensity tiers (post-split): Trace / Light / Brisk / Typical / Heavy +// (60-90) / Very Heavy (90-95) / Severe (95-99) / Extreme. Each band maps a rain-day +// percentile range to its calendar tier colour; the p75 vertex inside the single +// Heavy tier keeps the fan following the p75 contour. const RAIN_FAN = [ - ["p90", "p99", "wet-8"], ["p75", "p90", "wet-7"], ["p60", "p75", "wet-6"], + ["p95", "p99", "wet-8"], ["p90", "p95", "wet-7"], ["p75", "p90", "wet-6"], ["p60", "p75", "wet-6"], ["p40", "p60", "wet-5"], ["p25", "p40", "wet-4"], ["p10", "p25", "wet-3"], ["p1", "p10", "wet-2"], ]; @@ -212,7 +216,7 @@ export function precipChart(days, n, xFor, C) { getPct: (d) => (d.precip ? d.precip.percentile : null), // Rain days take their intensity-tier color; no-rain days warm with the dry // streak (tan→red) so a dry spell reads as dry, matching the Dry chart. - dotColor: (d) => (d.precip && d.precip.value > 0 ? (TIER_COLORS[d.precip.class] || C.precip) : (drynessColor(d.dsr) || C.precip)), + dotColor: (d) => (d.precip && d.precip.class && d.precip.class !== "dry" ? (TIER_COLORS[d.precip.class] || C.precip) : (drynessColor(d.dsr) || C.precip)), fmtAxis: (v) => fmtPrecip(v, false), fmtLabel: (v) => fmtPrecip(v, false), labelOn: (v) => v > 0, // only label rain days; dry days rest on the baseline diff --git a/frontend/static/day.js b/frontend/static/day.js index c07aec3..3760323 100644 --- a/frontend/static/day.js +++ b/frontend/static/day.js @@ -6,8 +6,8 @@ import { fmtTemp, onUnitChange } from "./units.js"; import { uv } from "./account.js"; // header sign-in entry + notification bell, and the API-version resolver import { getJSON, TTL, prefetchViews } from "./cache.js"; import { initFindButton, setFindLabel } from "./mappicker.js"; -import { TIER_COLORS, PRECIP_COLORS, DRY_COLOR, pctOrd, fmtPrecip, fmtWind, fmtHumid, - todayISO, weatherType, placeLabel } from "./shared.js"; +import { TIER_COLORS, PRECIP_COLORS, DRY_COLOR, pctOrd, fmtPrecip, fmtPrecipTier, fmtWind, fmtHumid, + todayISO, isoOfDate, weatherType, placeLabel } from "./shared.js"; // Color a tier the same way the calendar cell would be colored for it. const tierColor = (c) => (c === "dry" ? DRY_COLOR : TIER_COLORS[c] || PRECIP_COLORS[c] || ""); @@ -44,7 +44,9 @@ function stepDay(delta) { if (!curDate) return; const d = new Date(curDate + "T00:00:00"); d.setDate(d.getDate() + delta); - const iso = d.toISOString().slice(0, 10); + // Format in LOCAL time — curDate was parsed as local midnight, so toISOString() + // (UTC) would shift the result a day for UTC± viewers and desync the today guard. + const iso = isoOfDate(d); if (iso > todayISO()) return; // don't step past today curDate = iso; fetchDay(); @@ -115,7 +117,8 @@ function render(data) { // Weather summary from the observed high + precip (omitted for days with no obs yet). const th = d.metrics.tmax.obs ? d.metrics.tmax.obs.value : null; const pr = d.metrics.precip.obs ? d.metrics.precip.obs.value : null; - const wt = th != null || pr != null ? weatherType(th, pr) : null; + const prCls = d.metrics.precip.obs ? d.metrics.precip.obs.class : null; + const wt = th != null || pr != null ? weatherType(th, pr, undefined, prCls) : null; dayHead.innerHTML = `

${nice}

${wt ? `

${wt.icon} ${wt.text}

` : ""} @@ -155,11 +158,14 @@ function ladderCard(title, m, fmt, kind) { if (!m || !m.ladder) return ""; const obs = m.obs; const activeClass = obs ? obs.class : null; + // The observed precip depth reads "trace" for a sub-0.01" rain day (rounds to 0 + // but is graded a rain tier); ladder tier ranges keep the plain numeric formatter. + const obsFmt = kind === "precip" ? ((v) => fmtPrecipTier(v, activeClass)) : fmt; let summary; if (obs) { const pct = obs.percentile == null ? "" : ` · ${pctOrd(obs.percentile)} pct`; - summary = `${fmt(obs.value)} + summary = `${obsFmt(obs.value)} ${obs.grade}${pct}`; } else { summary = `No observation for this day yet`; @@ -182,7 +188,7 @@ function ladderCard(title, m, fmt, kind) { else if (t.lo == null) val = `< ${fmt(t.hi)}`; // bottom tier: strictly below p1 else val = fmtRange(fmt, t.lo, t.hi); const marker = t.c === activeClass && obs - ? `◀ ${bareVal(fmt, obs.value)}` : ""; + ? `◀ ${bareVal(obsFmt, obs.value)}` : ""; return `
${t.label} diff --git a/frontend/static/push-client.js b/frontend/static/push-client.js index 8e61f8b..f9081b1 100644 --- a/frontend/static/push-client.js +++ b/frontend/static/push-client.js @@ -36,6 +36,21 @@ async function registration() { return navigator.serviceWorker.ready; } +// Does an existing subscription's baked-in applicationServerKey still match the +// server's current VAPID public key? After a key rotation it won't: the browser +// keeps signing pushes the server can no longer authenticate, so we must replace +// the subscription. `options.applicationServerKey` is an ArrayBuffer of the raw +// key bytes (or null on browsers that don't expose it — treat that as a mismatch +// and re-subscribe rather than leaving a possibly-stale subscription in place). +function applicationServerKeyMatches(sub, serverKey) { + const current = sub.options && sub.options.applicationServerKey; + if (!current) return false; + const a = new Uint8Array(current); + if (a.length !== serverKey.length) return false; + for (let i = 0; i < a.length; i++) if (a[i] !== serverKey[i]) return false; + return true; +} + // Is this device currently subscribed? (a PushSubscription exists locally) export async function isEnabled() { if (!supported()) return false; @@ -60,13 +75,24 @@ export async function enable() { const reg = await registration(); let sub = await reg.pushManager.getSubscription(); + + // Always resolve the server's current VAPID key: it's needed to subscribe, and + // — when a subscription already exists — to detect a key rotation. A subscription + // minted under an old key silently stops receiving pushes, so on a mismatch we + // drop it and re-subscribe with the new key. Matching key = happy path, untouched. + const keyRes = await apiFetch(uv("push/vapid-key")); + if (!keyRes.ok) throw new Error("Couldn't fetch the server key."); + const { key } = await keyRes.json(); + const serverKey = urlB64ToUint8Array(key); + + if (sub && !applicationServerKeyMatches(sub, serverKey)) { + await sub.unsubscribe(); + sub = null; + } if (!sub) { - const res = await apiFetch(uv("push/vapid-key")); - if (!res.ok) throw new Error("Couldn't fetch the server key."); - const { key } = await res.json(); sub = await reg.pushManager.subscribe({ userVisibleOnly: true, - applicationServerKey: urlB64ToUint8Array(key), + applicationServerKey: serverKey, }); } const res = await apiFetch(uv("push/subscribe"), { method: "POST", json: sub.toJSON() }); diff --git a/frontend/static/shared.js b/frontend/static/shared.js index 423c5ea..a6e5224 100644 --- a/frontend/static/shared.js +++ b/frontend/static/shared.js @@ -257,7 +257,10 @@ export const pctOrd = (n) => { if (!Number.isFinite(v)) return "—"; return ord(Math.min(99, Math.max(1, Math.round(v)))); }; -export const todayISO = () => new Date().toISOString().slice(0, 10); +// Today in the viewer's LOCAL zone (see isoOfDate below). A UTC date rolls a day +// early/late for UTC± viewers past local midnight, putting "today" and the +// date-picker max out of reach of the day they're actually living in. +export const todayISO = () => isoOfDate(new Date()); export const esc = (s) => String(s).replace(/&/g, "&").replace(//g, ">"); // The heading label for a graded response: the resolved place name, or the // cell-center coordinates while (or if) no name resolves. @@ -464,13 +467,28 @@ export const WX_ICONS = { snow: WX(``), }; +// A graded precip depth for display, given its grade class. Reanalysis "trace" +// rain — any measurable precip below 0.01" — is graded as a rain tier but rounds to +// 0.00" / 0 mm, so its depth prints as "trace" rather than a bone-dry "0.00" that +// would contradict the rain grade (and the now-reset dry streak). Genuinely dry +// days (class "dry") and real, roundable depths format as usual. +export function fmtPrecipTier(v, cls, withUnit = true) { + if (cls && cls !== "dry" && v != null && parseFloat(fmtPrecip(v, false)) === 0) + return "trace"; + return fmtPrecip(v, withUnit); +} + // Plain-language "what was the day like" descriptor from the raw values: a // temperature word (from the daily high, °F) crossed with a sky/precip word // (precip in inches — falling as snow at/below freezing). `dsr` is optional: // when a long dry streak is known, a no-rain day reads "dry" instead of // "clear" (the calendar passes it; the day page doesn't track streaks). -export function weatherType(t, p, dsr) { - const wet = p != null && p >= 0.01; +// `precipCls` is the precip grade class when known: a rain tier means it rained +// even when the rounded depth reads 0 (trace), so it decides wet/dry ahead of the +// depth — matching the dry/rain grading split (precip > 0). Without it, any +// positive depth counts as wet. +export function weatherType(t, p, dsr, precipCls) { + const wet = precipCls != null ? precipCls !== "dry" : (p != null && p > 0); const freezing = t != null && t <= 34; const temp = t == null ? "" : t >= 95 ? "Scorching" : t >= 85 ? "Hot" : t >= 72 ? "Warm" : diff --git a/frontend/tests/unit/test_rendering.py b/frontend/tests/unit/test_rendering.py index 496580e..a10f9f8 100644 --- a/frontend/tests/unit/test_rendering.py +++ b/frontend/tests/unit/test_rendering.py @@ -29,6 +29,51 @@ def test_sitemap_lists_city_urls(client): assert f"/climate/{SLUG}/records" in r.text +# The public site is HTTPS-only behind Caddy, which terminates TLS and proxies +# plain HTTP -- so x-forwarded-proto / request.url.scheme read "http". A request +# arriving on the configured public host (THERMOGRAPH_BASE_URL defaults to +# https://thermograph.org in the test env) must still emit https:// in every +# frontend-built absolute URL, or canonicals self-conflict and the sitemap lists +# redirecting URLs. +_PUBLIC = {"host": "thermograph.org", "x-forwarded-proto": "http"} + + +def test_public_host_canonical_and_og_are_https(client): + b = client.get(f"{B}/climate/{SLUG}", headers=_PUBLIC).text + assert f'https://thermograph.org{B}/climate/{SLUG}" in body + assert "http://thermograph.org" not in body + + +def test_public_host_robots_sitemap_is_https(client): + body = client.get(f"{B}/robots.txt", headers=_PUBLIC).text + assert f"Sitemap: https://thermograph.org{B}/sitemap.xml" in body + + +def test_non_public_host_keeps_forwarded_scheme(client): + # A LAN/dev host never matches the configured public host, so the observed + # (plain-http) scheme is preserved -- http development is unaffected. + b = client.get(f"{B}/climate/{SLUG}", + headers={"host": "192.168.1.10:8000", "x-forwarded-proto": "http"}).text + assert f' Date: Sat, 25 Jul 2026 07:08:54 +0000 Subject: [PATCH 08/10] docs: rewrite the agent context layer to match the live system (#81) --- CLAUDE.md | 98 ++++++++++++++---- README.md | 2 +- backend/CLAUDE.md | 207 ++++++++++--------------------------- frontend/CLAUDE.md | 217 ++++++++++----------------------------- infra/CLAUDE.md | 82 +++++++++------ infra/README.md | 77 +++++++------- infra/docker-compose.yml | 6 +- infra/docker-stack.yml | 173 ------------------------------- observability/CLAUDE.md | 87 ++++++++-------- 9 files changed, 322 insertions(+), 627 deletions(-) delete mode 100644 infra/docker-stack.yml diff --git a/CLAUDE.md b/CLAUDE.md index d6cdf31..2395be1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,27 +1,81 @@ # thermograph monorepo — agent instructions -Reunified monorepo (2026-07-22): `backend/`, `frontend/`, `infra/`, -`observability/` — each a former split repo subtree-merged with history. -`thermograph-docs` remains a separate repo; cross-cutting decision docs and -operator runbooks belong there, not here. +One repo, four domains: `backend/`, `frontend/`, `infra/`, `observability/`. +Reunified 2026-07-22 from four split repos, with history, via subtree merges. +`thermograph-docs` is deliberately a separate repo — cross-cutting decision +records and operator runbooks go there, not here. -- **Each domain keeps its own `CLAUDE.md`** — read the one for the domain you - touch. They still use repo-era wording ("this repo", sibling-repo names); - mentally map `thermograph-backend` → `backend/` etc., and fix wording as you - touch files near it. -- **CI lives only in root `.forgejo/workflows/`**, path-filtered per domain - (see README). Never re-add workflows under a domain's own `.forgejo/` — they - are inert there and become a trap. -- Images are `emi/thermograph/backend` and `emi/thermograph/frontend` - (`sha-<12hex>`). Deploy = `infra/deploy/deploy.sh` over SSH, per-service. +**Rule for this file and every domain `CLAUDE.md`: only statements that would +break CI if they became false, or that name a file that exists.** Background, +history and rationale belong in `thermograph-docs`. These files are read before +every change, so a stale one is a correctness bug, not a documentation bug. + +## Branches and environments + +| Branch | Deploys to | Workflow | +|---|---|---| +| feature branch | nothing | PR into `dev` | +| `dev` | LAN dev | `*-deploy-dev.yml` — **currently inert**, see below | +| `main` | beta (beta.thermograph.org) | `*-deploy.yml` | +| `release` | prod (thermograph.org) | `*-deploy-prod.yml` | + +`dev`, `main` and `release` are protected: **everything is a PR**, for humans and +agents alike. Promotion is one PR per hop, `dev` → `main` → `release`. + +The two `*-deploy-dev.yml` workflows are inert — the LAN box's +`~/thermograph-dev` is still a split-era `thermograph-infra` checkout, so the +monorepo path they call does not exist there. Use `infra/`'s `make dev-up` +locally instead. + +## The deploy contract + +One entry point, two modes, one contract: + +``` +SERVICE=backend|frontend|all BACKEND_IMAGE_TAG=sha-<12hex> FRONTEND_IMAGE_TAG=sha-<12hex> \ + /opt/thermograph/infra/deploy/deploy.sh +``` + +`deploy.sh` resets the host checkout, renders secrets from the SOPS vault, then +either rolls compose services or — if `/etc/thermograph/deploy-mode` contains +`stack` — execs `infra/deploy/stack/deploy-stack.sh`. + +- **prod runs Swarm.** Its stack is `infra/deploy/stack/thermograph-stack.yml` + (db, web, worker, lake, daemon, frontend, autoscaler, autoscaler-lake). + `deploy-stack.sh` also offers `STACK_TEST=1`: a full parallel rehearsal on + throwaway volumes and ports that cannot touch live data. +- **beta and LAN dev run compose**, from `infra/docker-compose.yml` + (db, backend, lake, daemon, frontend). +- Each service's live tag is persisted host-side, so a single-service roll never + disturbs the sibling's running tag. + +Images are `emi/thermograph/backend` and `emi/thermograph/frontend`, tagged +`sha-<12hex>` on every push and by semver on `v*.*.*` tags. They build and deploy +**independently** — a backend change ships without a frontend deploy and vice +versa. That independence is the point of the FE/BE split and survived +reunification. + +## Rules that bind across domains + +- **CI lives only in root `.forgejo/workflows/`**, path-filtered per domain. + Never add workflows under a domain's own `.forgejo/` — they are inert there and + become a trap. +- **Secrets only via the SOPS vault** (`infra/deploy/secrets/`). Never hand-edit + `/etc/thermograph.env` on a host — it is a rendered artifact. `secrets-guard` + CI rejects plaintext. `seed-from-live.sh` reads production secrets and is + explicitly **not** for an agent to run. +- FE and BE ship out of lockstep, so the `/api/version` contract and + `PAYLOAD_VER` discipline are load-bearing — see `backend/CLAUDE.md`. +- Anything named "prefetch" must never spend the Open-Meteo quota. + Nominatim ≤ 1 req/s. - The compose project name is pinned (`name: thermograph` in - `infra/docker-compose.yml`); LAN dev overrides via - `COMPOSE_PROJECT_NAME=thermograph-dev`. Don't remove either half. -- Cross-domain rules that still bind: FE/BE ship independently → keep the - `/api/version` contract and `PAYLOAD_VER` discipline (see - `backend/CLAUDE.md`); anything named "prefetch" must never spend the - Open-Meteo quota; Nominatim ≤ 1 req/s; secrets only via the SOPS vault - (`infra/deploy/secrets/`). + `infra/docker-compose.yml`); LAN dev overrides with + `COMPOSE_PROJECT_NAME=thermograph-dev`. Don't remove either half — the pinned + name is what makes the Swarm stack's external volume names line up. - `CUTOVER-NOTES.md` is the source of truth for what is and isn't live yet. -- Commits/PRs: concise and technical; never mention AI/assistants/automated - authorship. + +## Commits & PRs + +Describe only the substance of the change; concise and technical. Never mention +AI, Claude, assistants or automated authorship anywhere — no trailers, +co-authors, emoji, or "as requested" narration. diff --git a/README.md b/README.md index 3b3ff55..e268e31 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ for: **per-domain images, per-domain deploys, and an async FE/BE contract**. |---|---|---| | `backend/` | FastAPI graded-climate API, accounts, notifications (Discord bot, push, mail), data pipeline | `backend-build-push` → image `emi/thermograph/backend`; `backend-deploy[-prod\|-dev]` | | `frontend/` | Public client: static JS/CSS + SSR pages | `frontend-*` mirrors of the above; image `emi/thermograph/frontend` | -| `infra/` | Compose, deploy scripts, terraform, SOPS secrets vault, ops cron | `infra-sync` (host checkout + secrets render), `secrets-guard`, `ops-cron` | +| `infra/` | Compose (beta, LAN dev) + the Swarm stack (prod), deploy scripts, terraform, SOPS secrets vault, ops cron | `infra-sync` (host checkout + secrets render), `secrets-guard`, `ops-cron` | | `observability/` | Loki + Grafana + Alloy stack | `observability-validate` | `thermograph-docs` deliberately **stays its own repo** (ADRs + runbooks, no diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 0bd2165..ccc1b68 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -1,161 +1,66 @@ -# thermograph-backend — agent instructions +# backend/ — agent instructions -## What this repo is +The Thermograph **API service**: a FastAPI app serving the graded-climate API +(`/api/v2/...`), user accounts, notifications, and the SSR-content JSON API the +frontend renders pages from. It owns the climate data pipeline and the +derived-payload cache. It serves **no HTML/CSS/JS** — that is `frontend/`. -The Thermograph **API service**: FastAPI app serving the graded-climate API -(`/api/v2/...`), user accounts (`accounts/`, fastapi-users + Postgres via -alembic), push/email/Discord notifications (`notifications/`), and the -SSR-content JSON API (`api/content_routes.py` + `api/content_payloads.py`) that -the separate `thermograph-frontend`'s server-rendered pages consume. It owns -the climate data pipeline (`data/` — grid snapping, Open-Meteo fetch + parquet -cache, percentile grading/scoring, places/cities) and the derived-payload -cache/ETag store (`data/store.py`, `api/payloads.py`). It does **not** serve -any HTML/CSS/JS itself — that moved to `thermograph-frontend` (repo-split -Stage 7a); this process is a pure JSON API + DB-backed service, launched as -`app:app` (a shim re-exporting `web.app:app`, kept stable for systemd/CI/`run.sh` -callers that still expect that target). +Read the root `CLAUDE.md` first for the branch model, deploy contract and image +names. -## Split topology +## Build & verify -Split out of the `emi/thermograph` monorepo (see -`MONOREPO-CUTOVER-PLAN.md` at the workspace root during the transition). Sibling -repos: -- **`thermograph-frontend`** — the client the public sees: static - JS/CSS/HTML + `frontend_ssr`'s server-rendered climate hub/city/month pages, - calling this repo's `/api/v2/...` and `/content/...` JSON endpoints. -- **`thermograph-infra`** — deploy/terraform/Swarm-Compose plumbing: - `docker-compose.yml`, `deploy/deploy.sh`, secrets (SOPS-encrypted - `deploy/secrets/*.yaml`), Terraform for prod/beta provisioning. This repo has - no deploy scripts or terraform of its own — it only builds an image and hands - infra a tag. -- **`thermograph-docs`** — non-code planning layer: architecture decision - records and operator runbooks. Nothing here should duplicate that; if you're - writing a cross-cutting decision doc or a step-by-step operator runbook, - it belongs there, not in this repo's README/CLAUDE.md. -- **`thermograph-observability`** — monitoring/metrics stack, separate from - this repo's own lightweight in-process `core/metrics.py`. +- `make test` — builds a py3.12 venv and runs the hermetic pytest suite + (`scripts/test.sh`; pass `ARGS=...`). Tests are hermetic per `tests/conftest.py`: + no real Open-Meteo/Nominatim/GeoNames calls, throwaway SQLite, notifier thread + disabled. +- `make smoke` — boots the built image plus a throwaway db and asserts + `/healthz` + `/api/version`. +- CI runs this suite **inside the built image** (`build.yml`), so the exact + interpreter and deps that ship are what gets tested. -## Branch, PR & deploy flow +`app.py` at the root is a one-line re-export shim for `web/app.py`, kept stable +for systemd/CI callers. Entry target is `app:app`. -- `build-push.yml` builds **this repo's own image** - (`git.thermograph.org/emi/thermograph-backend/app`, tagged `sha-<12hex>` on - every push to `dev`/`main`/`release` and additionally by semver on `v*.*.*` - tags) — no more shared monorepo image. `build.yml` (push/PR to `main`) is a - build-only sanity check (`docker build`), not a boot/health check — repeated - attempts at a live boot+healthz check hit this runner's own quirks (bridge-IP - reachability, non-unique `$$`, squatted host ports) without ever going - reliably green; the image's actual boot is verified by hand (`docker run` + - alembic + `/healthz` 200 + a real `/api/v2/place` 200) instead. -- **`main` → beta**: `deploy.yml` SSHes to beta and runs - `thermograph-infra`'s `/opt/thermograph/deploy/deploy.sh` with - `SERVICE=backend` + `BACKEND_IMAGE_TAG=sha-<12hex>` (must match - build-push.yml's 12-hex SHA tag exactly — Actions expressions have no - substring function for the full 40-char SHA). `deploy.sh` rolls only the - `backend` compose service (`--no-deps`) and persists the tag in - `deploy/.image-tags.env` host-side, so a backend-only deploy never touches - the frontend's running container or tag. -- **`release` → prod**: `deploy-prod.yml` is the same shape against a - completely separate secret set (`PROD_SSH_HOST/USER/KEY/PORT`) so a beta - credential leak can't reach prod. Also `SERVICE=backend` + - `BACKEND_IMAGE_TAG`. -- The frontend deploys itself the same way, independently — that - independence (a backend change ships without a frontend deploy and vice - versa) is the entire point of the FE/BE CI-CD split. -- No `dev`-branch LAN deploy path exists in this repo (the monorepo's - `deploy-dev.yml`/`docker-compose.dev.yml` never made it across the split — - known gap, see the cutover plan if resurrecting it). +## Contracts that break silently if you change them -## Run / test locally - -There is **no Makefile in this repo** — the monorepo's app-level `make` -targets (`lan-run`, `test`, `venv`, ...) haven't been given a new home yet -(tracked as a gap in the cutover plan). Until that lands, drive it directly: - -```bash -python3 -m venv .venv && .venv/bin/pip install -r requirements.txt -r requirements-dev.txt -.venv/bin/uvicorn app:app --host 0.0.0.0 --port 8137 # serve -.venv/bin/python -m pytest tests # test -``` - -The monorepo's Makefile preferred `uv venv --python 3.12 .venv` (falling back -to plain `python3 -m venv` only if `uv` isn't installed) — prefer `uv` here too -if it's on the box. There is no committed `.venv` in this repo; if one exists -locally and looks broken (import errors, wrong Python), just `rm -rf .venv` and -recreate rather than debugging it — it's gitignored, disposable state, not -something the repo depends on being a specific way. - -Tests are hermetic (`tests/conftest.py`): no real Open-Meteo/Nominatim/GeoNames -calls, a throwaway SQLite for accounts + the derived-payload store, and the -notifier thread disabled. `THERMOGRAPH_FRONTEND_BASE_INTERNAL` is set to an -unreachable placeholder in conftest — `web/app.py` fails loud at import if it's -unset in a real run (it's the internal URL back to `frontend_ssr` for the -non-API HTML routes this process no longer serves itself). - -## API version contract - -- Everything real lives under **`/api/v2/...`**; `/api/` and `/api/v1/` stay - mounted as aliases of the same handlers (they always were — not new). -- **`GET /api/version`** (`web/app.py`) is the capability/negotiation endpoint - a split-out frontend can call at boot: `{backend_version, min_frontend, - payload_ver}`, driven by two module constants next to it — - `API_CONTRACT_VERSION` (bump only on a breaking change to any `/api/v{N}` - route or payload shape) and `MIN_SUPPORTED_FRONTEND` (oldest frontend - contract version this backend still serves correctly). A genuine breaking - change ships as a new `v3` router mounted alongside `v2` re-registering only - the changed handlers, `v2` kept working, `API_CONTRACT_VERSION` bumped in the - same PR. -- **`PAYLOAD_VER`** (`api/payloads.py`, currently `"p2"`) is a *separate* - concern from URL versioning — it's the cache/ETag invalidation token baked - into every derived-store validity token (`history_token()`). Bump it - whenever a response payload's shape changes (new metric, renamed/removed - field) so one bump atomically orphans every pre-upgrade cached row instead - of a stale row's `history_end` happening to still validate under the old - shape. -- Both endpoints (`/healthz`, `/api/version`) are deliberately I/O-free so they - stay cheap under tight healthcheck intervals. - -## Live cross-repo contracts (don't break silently) - -- **`/cell` bundle + ETag/If-None-Match**: every graded payload (`grade`, - `calendar`, `day`) is cached in SQLite by `(kind, cell_id, key)` keyed to a - validity token that only advances when the underlying history actually - changes (`_etag_for` in `web/app.py`, `history_token`/`PAYLOAD_VER` in - `api/payloads.py`). The same token doubles as a weak ETag; a client sending - `If-None-Match` gets an empty 304 without the payload being rebuilt or even - loaded. `expose_headers=["ETag"]` in the CORS middleware matters as much as - `allow_origins` — without it the frontend's `cache.js` reading - `res.headers.get("ETag")` cross-origin silently gets `null` and never - revalidates correctly. Don't change the ETag derivation or the CORS - `expose_headers` list without checking the frontend's cache layer. -- **Percentile/unit parity with `thermograph-frontend`'s `static/shared.js`**: - the frontend's percentile-to-ordinal display logic explicitly mirrors this - repo's `data/grading.py::pct_ordinal()` (floors an empirical percentile into - 1..99 — a rank against the sample can round to 0 or 100, and "100th - percentile" is never shown). `TEMP_BANDS`/`RAIN_BANDS` tier boundaries in - `data/grading.py` are the source of truth for the tier names/thresholds - (Near Record / High / Above Normal / Normal / Below Normal / Low, and the - five rain-intensity tiers) — changing them without a corresponding frontend - update produces a client that draws differently-colored tiers than the - labels the API returns. -- **SSR content API** (`api/content_routes.py`): a separate ETag'd JSON surface - (`/content/hub`, `/content/city/{slug}`, `/content/city/{slug}/month/{month}`, - `/content/city/{slug}/records`, `/content/home`, `/content/sitemap`, - `/content/indexnow-key`) that `frontend_ssr` calls in-process-free (no shared - Python import) to render pages. Same caching pattern as the main API; keep - payload shape changes coordinated with whatever consumes it on the frontend - side. +- **`GET /api/version`** → `{backend_version, min_frontend, payload_ver}`, driven + by `API_CONTRACT_VERSION` and `MIN_SUPPORTED_FRONTEND` in `web/app.py`. A + breaking change ships as a new `v3` router mounted **alongside** `v2`, + re-registering only changed handlers, with the constant bumped in the same PR. + `/api` and `/api/v1` stay mounted as aliases. +- **`PAYLOAD_VER`** (`api/payloads.py`) is separate from URL versioning — it is + the cache/ETag invalidation token. Bump it whenever a response payload's shape + changes, so one bump atomically orphans every pre-upgrade cached row. +- **ETag / `If-None-Match`** — every graded payload is cached against a validity + token that doubles as a weak ETag. `expose_headers=["ETag"]` in the CORS + middleware matters as much as `allow_origins`: without it the frontend's + `cache.js` reads `null` cross-origin and never revalidates. Don't change the + ETag derivation or that list without checking `frontend/static/cache.js`. +- **`data/grading.py::pct_ordinal()`** is mirrored by `frontend`'s + `shared.js::pctOrd()` — floor a percentile into `1..99`, never 0 or 100. + `TEMP_BANDS`/`RAIN_BANDS` are the source of truth for tier names and + thresholds; changing them without the frontend produces tiers drawn in colours + that disagree with the labels the API returns. +- **Fahrenheit country set** — `api/content_payloads.py`'s `F_COUNTRIES` must stay + identical to `frontend`'s `format.py::F_COUNTRIES` and `static/units.js`'s + `F_REGIONS`. There is a test asserting this. +- **`/healthz` and `/api/version` are deliberately I/O-free** so they stay cheap + under tight healthcheck intervals. ## Layout -`accounts/` (fastapi-users models/schemas/db, `alembic/` migrations), -`api/` (versioned payload builders + SSR content routes), `core/` (metrics, -audit logging, a singleton helper), `data/` (grid, climate fetch/cache, grading/ -scoring, places/cities, the derived-payload store), `notifications/` (push, -email, Discord bot + interactions + linking, digest, scheduler), `web/app.py` -(the actual FastAPI app; `app.py` at the repo root is a one-line re-export -shim). `paths.py` resolves every filesystem location (climate parquet cache, -accounts DB, logs, bundled `cities.json`/`cities_flavor.json`) from the repo -root — don't reintroduce `__file__`-relative paths in a module, it breaks the -moment a module moves. `deploy/entrypoint.sh` is the container entrypoint -(alembic migrate-then-serve, with a Swarm-secrets-as-files shim); actual deploy -orchestration lives in `thermograph-infra`, not here. +`accounts/` (fastapi-users + alembic), `api/` (payload builders + SSR content +routes), `core/` (metrics, audit, singleton helper), `data/` (grid, climate +fetch/cache, grading/scoring, places/cities, derived-payload store), +`notifications/` (push, email, Discord bot, digest, scheduler), `web/app.py` +(the real app), `daemon/`. + +`paths.py` resolves every filesystem location from the repo root — don't +reintroduce `__file__`-relative paths in a module; it breaks the moment a module +moves. `deploy/entrypoint.sh` is the container entrypoint (alembic migrate, then +serve, with a Swarm-secrets-as-files shim). + +## Commits & PRs + +Concise and technical. Never mention AI, assistants or automated authorship. diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md index 17ea9fa..5542457 100644 --- a/frontend/CLAUDE.md +++ b/frontend/CLAUDE.md @@ -1,181 +1,72 @@ -# Thermograph frontend — agent instructions +# frontend/ — agent instructions -This repo is the **SSR + static-asset service** split out of the `emi/thermograph` -monorepo (repo-split Stage 7). It has no climate data, no DB, and does no -polars/compute work — everything comes from the backend's content API over -HTTP. It is one of four sibling repos in `thermograph-repos/`: +The **SSR + static-asset service**: server-rendered crawlable pages, the +interactive tool's SPA shells, and every static asset. No climate data, no DB, +no compute — everything comes from `backend/`'s content API over HTTP. -- **`thermograph-backend`** — FastAPI API + grading/scoring/grid compute + DB. - This repo's only dependency. -- **`thermograph-frontend`** (this repo) — SSR content pages + the interactive - tool's SPA shells + every static asset. -- **`thermograph-infra`** — Terraform, `docker-compose*.yml`, `deploy/` (incl. - `deploy.sh`, the SOPS secrets vault), Caddy config. No app code. -- **`thermograph-docs`** — architecture decision records + operator runbooks. - No code. +Read the root `CLAUDE.md` first for the branch model, deploy contract and image +names. -See `MONOREPO-CUTOVER-PLAN.md` (one level up, in `thermograph-repos/`) for the -full split status and the remaining gap list before the monorepo can be -archived. +## Build & verify -## What this repo is +- `make test` — whole suite (`scripts/test.sh`). +- `make test-unit` — hermetic unit tier: SSR rendering fed committed fixtures, + no Docker. **This is the tier CI runs.** +- `make test-integration` — pulls and runs the real backend image and tests the + live contract. Local only; CI does not run it. +- `make backend-up` / `make backend-down` — a local backend container for dev. +- `make capture-fixtures` — refresh `tests/fixtures/*.json` from a live backend. -- **`content.py`** — server-rendered, crawlable pages (climate hub, per-city, - month, records, glossary, about, privacy) + `robots.txt` + `sitemap.xml`. - Every route fetches its data from the backend's content API - (`api_client.py`) instead of computing in-process; `content_payloads.py` on - the backend owns `page_title`/`canonical_path`/`breadcrumb`/`jsonld`, not - this repo. -- **`static/*.js`** — the interactive tool: `app.js` (map/search/graded - results + inline SVG chart), `calendar.js`/`day.js`/`score.js`/`compare.js` - (SPA shells served by `app.py`'s `_page()`), `account.js` (auth), `cache.js` - (IndexedDB response cache + `/cell` bundle prefetch), `shared.js` (format - helpers shared across views). -- **`static/style.css`** — the single hand-written stylesheet; all design - tokens live here (see `DESIGN.md`). -- **`templates/*.html.j2`** — Jinja templates `content.py` renders. -- **`content/*.yaml`** — structured SSR copy (glossary, static-page SEO meta), - loaded by `content_loader.py`. Committed here as a starter copy extracted - alongside the split; real cross-repo copy vendoring (a pinned - `thermograph-copy` checkout at build time) is deferred, unbuilt follow-up — - don't assume it exists. +## Running it -## Branch/PR + deploy flow - -- Work happens on feature branches; PR into `main`. (The monorepo's - `dev`→`main`→`release` three-stage promotion and its LAN `deploy-dev.yml` - path do **not** exist in this repo yet — a documented gap, see the cutover - plan §4. Don't assume a `dev` branch here.) -- **`.forgejo/workflows/build-push.yml`** — on push to `dev`/`main`/`release` - or a `v*.*.*` tag: builds THIS repo's own `Dockerfile` and pushes - `git.thermograph.org/emi/thermograph-frontend/app`, tagged `sha-<12 hex>` - (every push) and the semver tag (tag pushes only). Backend publishes its own - separate image the same way — the two are no longer one shared - `emi/thermograph/app` image. -- **`.forgejo/workflows/build.yml`** — push/PR to `main`: proves the Dockerfile - builds. It is a **build check only**, not a boot/health check — booting the - real app crashes at import without a reachable backend (see API-version - section below), so a standalone boot check would need to check out - `thermograph-backend` too. Not yet built; flagged, not silently skipped. -- **`.forgejo/workflows/deploy.yml`** — push to `main`: SSH to beta, run - `SERVICE=frontend FRONTEND_IMAGE_TAG=sha-<12 hex> /opt/thermograph/deploy/deploy.sh` - (that script lives in `thermograph-infra`). Rolls **only** the frontend - container (`--no-deps`); backend is untouched and deployed independently by - its own repo's workflow. `deploy.sh` retries the image pull for ~5 min - (Forgejo has no cross-workflow `needs:`, so this deploy can race ahead of - `build-push.yml`) and waits for a healthy backend before declaring the roll - OK (frontend's boot fetches the IndexNow key from backend — see below). -- **`.forgejo/workflows/deploy-prod.yml`** — push to `release`: same shape, - targets prod via its own `PROD_SSH_*` secret set (fully separate from beta's, - so a beta credential leak can't touch prod). Nothing else deploys to prod; - there is no release-triggered Terraform apply from this repo. -- The `SERVICE=frontend` + `FRONTEND_IMAGE_TAG=sha-<12 hex>` pair is the entire - contract into `thermograph-infra/deploy/deploy.sh`: it persists each - service's live tag in `deploy/.image-tags.env` (host-side, untracked) so a - frontend-only roll never disturbs backend's currently-running tag, and - vice versa. - -## How to run / test - -There is no `run.sh`/`Makefile` in this repo yet (the monorepo's `make -lan-run`/`make run`/`make stop`/`venv` targets have no home here — see the -cutover plan's gap list). Run directly: +`THERMOGRAPH_API_BASE_INTERNAL` is **required** — `api_client.py` raises at +import if unset. ```bash -python3 -m venv .venv && .venv/bin/pip install -r requirements.txt -THERMOGRAPH_API_BASE_INTERNAL=http://127.0.0.1:8137 \ -THERMOGRAPH_BASE=/thermograph \ +THERMOGRAPH_API_BASE_INTERNAL=http://127.0.0.1:8137 THERMOGRAPH_BASE=/thermograph \ .venv/bin/uvicorn app:app --host 0.0.0.0 --port 8080 ``` -`THERMOGRAPH_API_BASE_INTERNAL` is **required** — `api_client.py` raises -`RuntimeError` at import if it's unset. Other env vars: `THERMOGRAPH_BASE` -(default `/thermograph`; the Dockerfile sets `/` for the deployed clean-root -topology), `THERMOGRAPH_API_VERSION` (default `v2`, see below), -`THERMOGRAPH_API_BASE_PUBLIC` (browser-facing backend origin for asset URLs -when frontend and backend are cross-origin; empty = same-origin, today's -default), `THERMOGRAPH_SSR_CACHE_TTL` (default 600s, the content-API response -cache in `api_client.py`), `THERMOGRAPH_GOOGLE_VERIFY`/`THERMOGRAPH_BING_VERIFY` -(search-console `` tags). +Other env: `THERMOGRAPH_BASE` (default `/thermograph`; the Dockerfile sets `/`), +`THERMOGRAPH_API_VERSION` (default `v2`), `THERMOGRAPH_API_BASE_PUBLIC` +(browser-facing backend origin when cross-origin; empty = same-origin, today's +default), `THERMOGRAPH_SSR_CACHE_TTL` (default 600s). -**Known gap — `tests/conftest.py` does not run standalone in this repo.** It -still assumes the pre-split layout: a sibling `backend/` checkout with its own -`tests/conftest.py` (`make_history`/`make_recent`) and an `api.content_payloads` -module, neither of which exists here. It was extracted verbatim as reference -material for the real fix (a genuine cross-repo contract-test job, or a -self-contained set of fixtures owned in this repo) — not yet built. Do **not** -assume `pytest tests` passes here until that lands; `.forgejo/workflows/build.yml` -only proves the image builds, for the same reason. There is also no -`requirements-dev.txt` in this repo yet (pytest/httpx test deps aren't pinned). +## Layout -## API-version pinning contract +- **`content.py`** — SSR routes (climate hub, city, month, records, glossary, + about, privacy) + `robots.txt` + `sitemap.xml`. Each fetches from the backend + via `api_client.py`; the backend's `content_payloads.py` owns `page_title` / + `canonical_path` / `breadcrumb` / `jsonld`, not this domain. +- **`static/*.js`** — `app.js` (map/search/results + inline SVG chart), + `calendar.js`/`day.js`/`score.js`/`compare.js` (SPA shells), `account.js` + (auth), `cache.js` (IndexedDB cache + `/cell` bundle prefetch), `shared.js`. +- **`static/style.css`** — the single hand-written stylesheet; design tokens live + here, documented in `DESIGN.md`. +- **`templates/*.html.j2`**, **`content/*.yaml`** (structured SSR copy, loaded by + `content_loader.py`). -Every backend call in this repo goes through a single pinned constant instead -of scattered `api/v2/...` literals: +## Contracts with the backend -- **Python (SSR):** `api_client.py`'s module-level `API_VERSION` (default - `"v2"`, overridable via `THERMOGRAPH_API_VERSION`). Every path builder - (`hub()`, `sitemap()`, `indexnow_key()`, `home()`, `city()`, `city_month()`, - `city_records()`) reads it. -- **JS (interactive tool):** `static/account.js`'s exported `API_VERSION` - constant and the `uv(path)` helper — every fetch in `cache.js`, `account.js` - itself, etc. is built by wrapping the path in `uv(...)` rather than - hardcoding a version. - -**Bump only in lockstep with a verified backend `/api/version` check** — the -backend exposes `GET {BASE}/api/version` → `{backend_version, min_frontend, -payload_ver}` (`web/app.py`'s `API_CONTRACT_VERSION`/`MIN_SUPPORTED_FRONTEND`). -Before bumping this repo's `API_VERSION`, confirm the target backend's -`backend_version` actually supports it and its `min_frontend` doesn't already -exclude the version you're moving *away* from. - -**A v3 cutover would work like this:** backend mounts a new `v3 = APIRouter()` -alongside `v2`, re-registering only the changed handlers (v2 keeps serving -old clients); backend bumps `API_CONTRACT_VERSION` to `"3"` in that same PR. -Only once that's deployed and verified does this repo bump `API_VERSION` to -`"v3"` in both `api_client.py` and `account.js` — one PR, both pins together, -never one without the other. `v2` (and the `/api`, `/api/v1` aliases) stay -mounted and working until no client depends on them. - -The frontend's own boot is **resilient to the backend being down**: the -`indexnow_key()` fetch in `content.py`'s `register()` is tried once eagerly -(so the common case still serves `/.txt` as a plain static route), but a -failure is caught, logged, and falls back to a lazy per-request lookup -instead of crashing boot — asynchronous frontend/backend deploys depend on -this (a briefly-unreachable backend at frontend boot must be survivable, not -fatal). - -## Cross-repo contracts this repo depends on - -These are **live contracts with the backend** — a change on either side -without the other breaks something silently, not loudly: - -- **The `/cell` bundle + ETag/`If-None-Match`** — `static/cache.js` fetches - `/api/v2/cell` once per view-set and slices it for calendar/day/score/compare - instead of one request per view; conditional refetch is via `If-None-Match` - against the stored `ETag`, so an unchanged payload costs an empty 304. The - URL map (which slice serves which view) must stay in sync with - `thermograph-backend`'s route/payload shape. -- **`shared.js`'s `pctOrd()` must mirror `thermograph-backend`'s - `data/grading.py`'s `pct_ordinal()`** byte-for-byte in behavior: floor a - percentile into `1..99` (never round to 100/0 — "100th percentile" reads as - measurement error, not "as extreme as it has ever been"). Every percentile - shown anywhere (Day page, calendar tooltip, chart, city pages, homepage - strip) goes through one of these two functions; if they diverge, the same - reading says two different things on two different surfaces. -- **Unit/region logic** (`format.py`'s `F_COUNTRIES` / `static/units.js`'s - `F_REGIONS` / backend's `api/content_payloads.py`'s `F_COUNTRIES`) — the set - of Fahrenheit-using country codes must stay identical across all three; - backend has a test asserting this. - -## Design & visual verification - -Design tokens + conventions are documented in `DESIGN.md` (source of truth: -`static/style.css`). To see a change rendered, see `DESIGN.md`'s `make shots` -section (`tools/shoot.py`). +- **API version is pinned in exactly two places** — `api_client.py`'s + `API_VERSION` (Python) and `static/account.js`'s exported `API_VERSION` plus the + `uv(path)` helper (JS). Never hardcode `api/v2/...` anywhere else. Bump both in + one PR, and only after the target backend's `/api/version` confirms it serves + that version and its `min_frontend` doesn't exclude the one you're leaving. +- **`shared.js::pctOrd()` must mirror `backend`'s `data/grading.py::pct_ordinal()`** + in behaviour — floor into `1..99`, never 0 or 100. Every percentile on every + surface goes through one of the two; if they diverge, the same reading says two + different things in two places. +- **`/cell` bundle + ETag** — `cache.js` fetches `/api/v2/cell` once per view-set + and slices it, revalidating with `If-None-Match`. The slice→view map must track + the backend's payload shape. +- **Fahrenheit country set** — `format.py::F_COUNTRIES` and `static/units.js`'s + `F_REGIONS` must stay identical to the backend's `F_COUNTRIES`. +- **Boot must survive an unreachable backend.** `content.py`'s `register()` tries + the IndexNow-key fetch once, catches failure, and falls back to a lazy + per-request lookup. Asynchronous FE/BE deploys depend on this — don't make it + fatal. ## Commits & PRs -Describe only the substance of the change; concise, technical. Never mention -AI/Claude/assistants or automated authorship anywhere — no trailers, -co-authors, emoji, or "as requested"/"per the agent" narration. +Concise and technical. Never mention AI, assistants or automated authorship. diff --git a/infra/CLAUDE.md b/infra/CLAUDE.md index e10bb85..ffa2576 100644 --- a/infra/CLAUDE.md +++ b/infra/CLAUDE.md @@ -1,42 +1,58 @@ -# thermograph-infra — agent instructions +# infra/ — agent instructions -How and where the already-built Thermograph app images run. This repo owns -Terraform, the SOPS+age secrets vault, compose files, deploy scripts, host -provisioning, and the ops cron (DB backup + IndexNow). The app repos -(`thermograph-backend`, `thermograph-frontend`) own building and testing the -images; this repo never checks out app source. The old monorepo -(`emi/thermograph`) is archived — do not point anything at it. +How and where the already-built app images run: Terraform, the SOPS+age secrets +vault, compose and Swarm stack files, deploy scripts, host provisioning, mail, +and the ops cron (DB backup + IndexNow). Read the root `CLAUDE.md` first — it +owns the branch model, the deploy contract, and which environment runs which +orchestrator. -## The four machines +## The machines -Same as ACCESS.md: **dev machine** (operator's box, LAN dev server, CI runner), +Per `ACCESS.md`: **dev** (operator's box — LAN dev server and CI runner), **prod** (`169.58.46.181`, thermograph.org, `agent` user, passwordless sudo), -**beta** (`75.119.132.91`, beta.thermograph.org + Forgejo, `agent` user). Hosts' -`/opt/thermograph` (and LAN's `~/thermograph-dev`) are checkouts of THIS repo. +**beta** (`75.119.132.91`, beta.thermograph.org + Forgejo + Grafana/Loki, +`agent` user). All four are on a WireGuard mesh. Hosts' `/opt/thermograph` is a +checkout of **this monorepo**, not an infra-only repo. -## Deploys +## Deploy paths -- `deploy/deploy.sh` — the one deploy path (prod/beta): resets this repo's - checkout, renders secrets from the SOPS vault, pulls the per-service image - tags (`BACKEND_IMAGE_TAG`/`FRONTEND_IMAGE_TAG`, persisted in untracked - `deploy/.image-tags.env`), rolls the target service, health-checks via - container healthchecks. Invoked over SSH by the app repos' deploy workflows. -- `deploy/deploy-dev.sh` — thin LAN-dev wrapper around deploy.sh (dev compose - overlay, `~/thermograph-dev`, branch `dev`). -- Branch model: see README — infra `main` is live on prod+beta, `dev` on LAN; - `release` is currently unused. App code is env-staged via image tags; infra - is not. +- **`deploy/deploy.sh`** — the single entry point for beta and prod. Resets the + host checkout to `BRANCH` (default `main`), renders secrets, then routes: + if `/etc/thermograph/deploy-mode` says `stack` it execs + `deploy/stack/deploy-stack.sh`, otherwise it rolls compose services. Per-service + tags persist in untracked `deploy/.image-tags.env` (compose) and + `deploy/.stack-image-tags.env` (stack) so the two never mix. +- **`deploy/stack/deploy-stack.sh`** — the Swarm path, live on prod. `backend` + rolls web **and** worker; `frontend` rolls frontend; `all` runs a full + `docker stack deploy`. Start-first, health-gated, auto-rollback. + `STACK_TEST=1` rehearses the whole thing under stack name `thermograph-test` + on throwaway volumes and ports `18137`/`18080`. +- **`deploy/deploy-dev.sh`** — thin LAN-dev wrapper (dev compose overlay, + `~/thermograph-dev`). Its CI trigger is inert; see root `CLAUDE.md`. +- **`Makefile`** — compose orchestration only: `up`, `down`, `db-up`, `dev-up`, + `om-up`, `om-backfill`. + +Volumes are the reason the compose project name is pinned: compose creates +`thermograph_pgdata`/`_appdata`/`_applogs`, and the Swarm stack declares those +same names as `external: true` at the same mount paths. ## Rules -- **Never run `terraform apply` casually** — no tfstate exists anywhere (never - persisted), so an apply would attempt full re-provisioning of live hosts. - Terraform here is executable documentation until state is bootstrapped. -- Secrets: only via the SOPS vault (`deploy/secrets/*.yaml`, `sops edit` + - commit + deploy). Never hand-edit `/etc/thermograph.env` on a SOPS-enabled - host — it's a rendered artifact. `secrets-guard` CI rejects plaintext. -- The ops cron (`.forgejo/workflows/ops-cron.yml`) is THE prod backup — it uses - this repo's `PROD_SSH_*` Actions secrets. If you touch it, verify a dump - actually lands in `agent@prod:~/thermograph-backups/`. -- Commits/PRs: concise and technical; never mention AI/assistants/automated - authorship. +- **Never run `terraform apply` casually.** No tfstate is persisted anywhere, so + an apply would attempt full re-provisioning of live hosts. Terraform here is + executable documentation until state is bootstrapped. +- **Secrets: SOPS vault only** (`deploy/secrets/*.yaml`, `sops edit` → commit → + deploy). Never hand-edit `/etc/thermograph.env` — it is a rendered artifact. + `secrets-guard` CI rejects plaintext. `deploy/secrets/seed-from-live.sh` reads + production secrets and is **not** for an agent to run. +- **The ops cron (`.forgejo/workflows/ops-cron.yml`, at the repo root) is THE + prod backup.** It uses `PROD_SSH_*`, not `SSH_*` — an earlier revision reused + `SSH_*` and silently dumped beta while prod had no backup at all. If you touch + it, verify a dump actually lands in `agent@prod:~/thermograph-backups/`. +- Shell here runs as root over SSH against live hosts with no test suite in + front of it. `shell-lint` CI (pinned shellcheck) is the only guard — keep the + tree at zero findings. + +## Commits & PRs + +Concise and technical. Never mention AI, assistants or automated authorship. diff --git a/infra/README.md b/infra/README.md index d6f2e55..fdd28e8 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,46 +1,49 @@ -# thermograph-infra +# infra/ Infrastructure for [Thermograph](https://thermograph.org): Terraform host -provisioning, the SOPS+age secrets vault, Docker Swarm/WireGuard networking, -Forgejo, Caddy, and the deploy scripts that run the already-built app image on -each host. Extracted from the app monorepo (`emi/thermograph`) — the app repo -owns building and testing the app; this repo owns running it. +provisioning, the SOPS+age secrets vault, WireGuard/Swarm networking, Forgejo, +Caddy, mail, and the deploy scripts that run the already-built app images on each +host. This is a domain of the `emi/thermograph` monorepo — hosts' `/opt/thermograph` +is a checkout of the whole monorepo, and `infra/` never builds app source; it only +runs published images. - **`terraform/`** — provisions/configures hosts (SSH-driven by default; an - optional GCP-creating module is scaffolded, no live resources yet) and - triggers each deploy. See `terraform/README.md`. -- **`deploy/secrets/`** — the git-native SOPS+age secrets vault (every app - secret, encrypted at rest, rendered at deploy time). See - `deploy/secrets/README.md`. -- **`deploy/swarm/`, `deploy/forgejo/`** — the 3-node WireGuard/Swarm cluster - that hosts Forgejo (git + CI + registry); does not run the app itself. See - `ACCESS.md` and the READMEs under each directory. -- **`deploy/deploy.sh`** — pulls the pinned app image (`IMAGE_TAG`) and rolls - the compose stack; invoked by Terraform and by the app repo's - `.forgejo/workflows/deploy.yml` over SSH. -- **`docker-compose*.yml`, `docker-stack.yml`** — how the app image runs - (compose in production today; `docker-stack.yml` is a design record for a - possible future Swarm-based app deploy, not currently live). + optional GCP-creating module is scaffolded, no live resources yet). See + `terraform/README.md`. No tfstate is persisted anywhere — treat `apply` as + executable documentation, not a routine operation. +- **`deploy/secrets/`** — the git-native SOPS+age secrets vault (every app secret, + encrypted at rest, rendered at deploy time). See `deploy/secrets/README.md`. +- **`deploy/swarm/`, `deploy/forgejo/`** — the WireGuard/Swarm cluster hosting + Forgejo (git + CI + registry). See `ACCESS.md`. +- **`deploy/deploy.sh`** — the single deploy entry point for beta and prod. + Takes `SERVICE=backend|frontend|all` plus `BACKEND_IMAGE_TAG`/`FRONTEND_IMAGE_TAG`, + resets the host checkout, renders secrets, and routes to the right orchestrator. +- **`deploy/stack/`** — the **Swarm** path, live on **prod**: + `thermograph-stack.yml` (db, web, worker, lake, daemon, frontend, autoscaler, + autoscaler-lake), `deploy-stack.sh`, `autoscale.sh`, and the LB. Rolling updates + are start-first, health-gated, with auto-rollback. `STACK_TEST=1` rehearses the + whole stack on throwaway volumes and ports. +- **`docker-compose*.yml`** — the **compose** path, live on **beta** and LAN dev + (db, backend, lake, daemon, frontend). `docker-compose.dev.yml` is the LAN + overlay; `docker-compose.openmeteo.yml` is the self-hosted Open-Meteo overlay. -The app's own source, `Dockerfile`, and build/test CI stay in the app repo — -this repo never checks out app source; hosts only pull tagged images from the -registry. See `ACCESS.md` for host access and the Swarm/Forgejo topology, and -`terraform/README.md` for the day-to-day `plan`/`apply` workflow. +Which path a host takes is decided by `/etc/thermograph/deploy-mode`: the string +`stack` makes `deploy.sh` exec `deploy/stack/deploy-stack.sh`; anything else is +compose. The workflows never need to know which mode a host runs. ## Branches & how changes reach each environment -- **`main`** — what **prod and beta** run: their `/opt/thermograph` checkouts - `git reset --hard origin/main` at the start of every deploy (`deploy/deploy.sh`). - A merge to `main` reaches those hosts on the next app deploy (or a by-hand - `deploy.sh` run); there is no separate infra deploy trigger. -- **`dev`** — what **LAN dev** runs: `~/thermograph-dev` resets to it via - `deploy/deploy-dev.sh`. Keep it fast-forwarded to `main` (infra changes are not - environment-staged today; the branches exist so LAN dev *can* trail or lead - when needed). -- **`release`** — currently consumed by nothing (prod tracks `main`, not - `release`). It exists to mirror the app repos' dev→main→release promotion - shape if per-environment infra staging is ever wanted; until then, treat - `main` as live-everywhere. +- **`main`** — what **prod and beta** run. `infra-sync.yml` fires on a push to + `main` touching `infra/**`, fast-forwards each host's `/opt/thermograph` checkout + and re-renders `/etc/thermograph.env` from the vault. It deliberately does + **not** roll any service: image tags are the app domains' axis, not infra's. A + compose or stack change that must recreate containers takes effect on the next + app deploy, or a by-hand `SERVICE=all … deploy/deploy.sh`. +- **`dev`** — what LAN dev would run via `deploy/deploy-dev.sh`. The CI trigger + for this is currently inert (the LAN box still holds a split-era checkout); use + `make dev-up` locally. +- **`release`** — consumed by app deploys only. Both hosts track infra via `main`; + prod's *app images* are staged by `release`, but its checkout follows `main`. -Note the asymmetry with the app repos: app code IS environment-staged -(dev→main→release maps to LAN→beta→prod via image tags), infra is not. +Note the asymmetry with the app domains: app code IS environment-staged +(`dev`→`main`→`release` maps to LAN→beta→prod via image tags); infra is not. diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 2b13bd5..228dff2 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -46,8 +46,10 @@ services: # to an exact minor (e.g. 2.17.2-pg18) before any host of this stack could ever # replicate with another — a floating tag risks two hosts landing on different # extension minors, which blocks a physical replica and risks compressed-chunk - # corruption on restore. docker-stack.yml (the Swarm interim stack) REQUIRES an - # exact pin for exactly this reason; use the SAME tag on both once you set one. + # corruption on restore. The Swarm path (deploy/stack/) enforces this a + # different way: deploy-stack.sh resolves TIMESCALEDB_IMAGE to the digest of + # whatever db is ALREADY running -- including this compose stack's + # thermograph-db-1 -- so the image under an existing volume can never drift. image: timescale/timescaledb:${TIMESCALEDB_TAG:-latest-pg18} environment: POSTGRES_USER: thermograph diff --git a/infra/docker-stack.yml b/infra/docker-stack.yml deleted file mode 100644 index 437148b..0000000 --- a/infra/docker-stack.yml +++ /dev/null @@ -1,173 +0,0 @@ -# Docker Swarm stack for the hop-1 interim cutover — see -# thermograph-docs/runbooks/hop1-forgejo-registry-cutover.md (which this file implements) and -# thermograph-docs/architecture/repo-topology-and-infrastructure.md §6/§9. -# -# Distinct from docker-compose.yml (today's plain-compose deploy, unaffected by -# this file): Swarm pulls a pre-built image from the registry (IMAGE_TAG) rather -# than building in place, and needs Swarm-specific keys (deploy:, networks:, -# secrets:, no host-published ports on the overlay-fronted services). -# -# docker stack deploy -c docker-stack.yml thermograph -# -# `docker stack deploy` only interpolates from the invoking shell's environment, -# not a .env file — export the required vars first (or `set -a; . ./stack.env; -# set +a; docker stack deploy ...`). Required shell vars: IMAGE_TAG (the tag CI -# pushed to the registry), TIMESCALEDB_TAG (an EXACT pinned minor — see the db -# service below, hazard #7). POSTGRES_PASSWORD is NOT a shell var here: unlike -# docker-compose.yml (which interpolates it into THERMOGRAPH_DATABASE_URL at -# compose time), this file reads it as the `postgres_password` Swarm secret — -# POSTGRES_PASSWORD_FILE for db (native support in the postgres/timescaledb -# image), and the chunk-3 entrypoint shim builds THERMOGRAPH_DATABASE_URL for -# app/worker from the same secret file at container start. All `postgres_password` -# / `thermograph_*` secrets below must already exist in the Swarm (`docker secret -# create -`) before the first deploy — Track B step 7 in -# thermograph-docs/runbooks/implementation-handoff.md. -# -# Migrations are NOT run inline by app/worker here (RUN_MIGRATIONS=0) — the -# runbook's Stage F brings the schema to head via a one-shot task BEFORE scaling -# app up, so multiple replicas never race Alembic and nothing runs DDL against a -# still-read-only standby mid-cutover (hazard #9): -# -# docker run --rm --network thermograph_internal \ -# -e THERMOGRAPH_DATABASE_URL=... migrate - -services: - db: - # Pin the EXACT TimescaleDB minor — never latest-pg18 here. A floating tag can - # give two hosts different extension minors, which blocks a physical replica - # (a newer .so over an older catalog won't start) and risks compressed-chunk - # corruption on timescaledb_post_restore() (hazard #7). Get the exact X.Y.Z - # from the source database: SELECT extversion FROM pg_extension WHERE - # extname='timescaledb' — use the SAME tag docker-compose.yml's TIMESCALEDB_TAG - # is pinned to, on every host that could ever replicate with this one. - image: timescale/timescaledb:${TIMESCALEDB_TAG:?set TIMESCALEDB_TAG to an exact pinned minor, e.g. 2.17.2-pg18 -- not latest-pg18} - environment: - POSTGRES_USER: thermograph - POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password - POSTGRES_DB: thermograph - DB_MEMORY: ${DB_MEMORY:-8g} - volumes: - - pgdata:/var/lib/postgresql - - ./deploy/db/init:/docker-entrypoint-initdb.d - networks: - - internal - secrets: - - postgres_password - deploy: - # Pin to the labelled DB node so replication/IO never crosses the slow WG - # uplink (hazard #14): `docker node update --label-add db=true ` once, - # on whichever node holds pgdata (Track B). - placement: - constraints: ["node.labels.db == true"] - resources: - limits: - cpus: "${DB_CPUS:-2}" - memory: ${DB_MEMORY:-8g} - restart_policy: - condition: on-failure - # No published port: reachable only as db:5432 on the `internal` overlay — - # Postgres must never be public (design doc §6). - - # Stateless, freely-replicable web tier. ROLE=web means _should_run_notifier() - # is False unconditionally (web/app.py) — it never starts the notifier or the - # worker scheduler even if it would otherwise win leader election. - app: - image: ${IMAGE_TAG:?set IMAGE_TAG to the image CI pushed, e.g. forge.example/thermograph/app:sha-abc123} - environment: - THERMOGRAPH_BASE: / - PORT: 8137 - WORKERS: ${WORKERS:-4} - THERMOGRAPH_ROLE: web - RUN_MIGRATIONS: "0" - volumes: - - appdata:/app/data - - applogs:/app/logs - networks: - - internal - secrets: - - postgres_password - - thermograph_auth_secret - - thermograph_vapid_private_key - - thermograph_vapid_public_key - deploy: - replicas: 1 # single replica this hop -- homepage.json now lives in - # Postgres (chunk 4), but the notifier/scheduler split (chunk - # 2/5) is what actually lets this go multi-replica in Phase 2 - resources: - limits: - cpus: "${APP_CPUS:-4}" - restart_policy: - condition: on-failure - update_config: - order: start-first # new task must pass the image's HEALTHCHECK (now - # GET /healthz) before the old one is stopped - # No `ports:` — 127.0.0.1:8137:8137 (docker-compose.yml) has no Swarm - # equivalent: Swarm's routing mesh publishes on 0.0.0.0, which would expose - # the plaintext app un-fronted (hazard #6). Only the host's Caddy reaches - # `app`, over the `internal` overlay network — see the Caddyfile templates' - # health-gated reverse_proxy. - - # Owns the notifier + worker scheduler (chunks 1/2/5). Exactly one replica — - # the leader-election guard is belt-and-suspenders, not a substitute for it. - worker: - image: ${IMAGE_TAG:?set IMAGE_TAG to the image CI pushed, e.g. forge.example/thermograph/app:sha-abc123} - environment: - THERMOGRAPH_BASE: / - WORKERS: "1" - THERMOGRAPH_ROLE: worker - # Cluster-wide advisory lock, not the flock: the flock only arbitrates - # workers on ONE host, so under Swarm it guards nothing across replicas. - THERMOGRAPH_SINGLETON_PG: "1" - RUN_MIGRATIONS: "0" - volumes: - - appdata:/app/data - - applogs:/app/logs - networks: - - internal - secrets: - - postgres_password - - thermograph_auth_secret - - thermograph_vapid_private_key - - thermograph_vapid_public_key - - thermograph_discord_webhook - deploy: - replicas: 1 - resources: - limits: - cpus: "${WORKER_CPUS:-1}" - restart_policy: - condition: on-failure - # No `ports:` — the worker serves no public traffic; GET /healthz on its own - # container port is for the image's own HEALTHCHECK (Swarm task health) only. - -networks: - internal: - driver: overlay - driver_opts: - # VXLAN-over-WireGuard double encapsulation needs a lower MTU than the - # ~1450 overlay default, or large payloads (a /cell bundle, the homepage - # feed) silently stall while small packets (health checks) keep passing - # (hazard #13). Validate with a real large-payload transfer, not just a - # ping, once the mesh is up (Track B). - com.docker.network.driver.mtu: "1370" - -volumes: - pgdata: {} - appdata: {} - applogs: {} - -# Declared here, provisioned externally (docker secret create - < file) — -# never by this stack, and never committed. See Stage 0 of the cutover runbook -# for which values are continuity-critical (AUTH_SECRET, VAPID, POSTGRES_PASSWORD -# must be the EXISTING live values, not freshly generated ones). -secrets: - postgres_password: - external: true - thermograph_auth_secret: - external: true - thermograph_vapid_private_key: - external: true - thermograph_vapid_public_key: - external: true - thermograph_discord_webhook: - external: true diff --git a/observability/CLAUDE.md b/observability/CLAUDE.md index dcca01e..1485024 100644 --- a/observability/CLAUDE.md +++ b/observability/CLAUDE.md @@ -1,55 +1,52 @@ -# thermograph-observability — agent instructions +# observability/ — agent instructions -The **logging/metrics stack** for the Thermograph fleet: Loki + Grafana on beta, -with a Grafana Alloy agent on every node (prod, beta, dev) shipping container and -app logs over the WireGuard mesh. Grafana is fronted by beta's Caddy at -`dashboard.thermograph.org` (Google SSO, pre-provisioned users only). +The **logging stack** for the fleet: Loki + Grafana on beta, with a Grafana Alloy +agent on every node (prod, beta, dev) shipping container and app logs over the +WireGuard mesh. Grafana is fronted by beta's Caddy at +**`dashboard.thermograph.org`** (Google SSO, pre-provisioned users only) — use +that hostname everywhere, never `grafana.thermograph.org`. -This is **operational infra config, not application code** — there is no build. -It deploys by hand: `docker compose up -d` on beta for Loki+Grafana, and the -Alloy agent (`alloy/docker-compose.agent.yml`) on each node. See the README for -the full per-node procedure. +This is operational config, not application code: there is **no build** and no +deploy automation. It ships by hand — `docker compose up -d` on beta for +Loki+Grafana, and `alloy/docker-compose.agent.yml` on each node. Read the root +`CLAUDE.md` first; the branch model there applies here too. ## Layout -- `docker-compose.yml` — the Loki + Grafana stack (runs on beta). -- `loki/config.yml` — Loki config (mesh-only, filesystem storage). -- `grafana/provisioning/` — datasource (Loki) + dashboard provider, auto-loaded - at startup. `grafana/dashboards/*.json` — the dashboards themselves. -- `grafana/provisioning/alerting/` — the alert rules, the Discord contact point - and the notification policy, also auto-loaded at startup. Same rule as the - dashboards: **the repo is the only durable path**, UI edits get overwritten. - Alerts go to Discord `#ops-alerts`, never email — beta's Grafana relays SMTP - through prod's Postfix, so email dies exactly when prod does. Every rule is - LogQL (there is no Prometheus anywhere in the fleet). Thresholds were derived - from real Loki data and the working is in the comments beside each rule — - re-derive before changing a number rather than guessing. -- `alloy/config.alloy` + `alloy/docker-compose.agent.yml` — the per-node log - shipper. The node name comes from `ALLOY_NODE` (set per host). -- `caddy-grafana.conf` — the Caddy vhost for `dashboard.thermograph.org` (lives - in beta's Caddy config, kept here for reference). -- `.env.example` — copy to `.env` on beta before `docker compose up`. `.env` is - gitignored; never commit real OAuth secrets or the admin password. +- `docker-compose.yml` — the Loki + Grafana stack (beta). +- `loki/config.yml` — mesh-only, filesystem storage. +- `grafana/provisioning/` — datasource + dashboard provider, auto-loaded at + startup. `grafana/dashboards/*.json` — the dashboards. +- `grafana/provisioning/alerting/` — alert rules, the Discord contact point, and + the notification policy. +- `alloy/config.alloy` + `alloy/docker-compose.agent.yml` — the per-node shipper. + Node name comes from `ALLOY_NODE`, set per host. +- `caddy-grafana.conf` — the beta Caddy vhost, kept here for reference. +- `.env.example` — copy to `.env` on beta. `.env` is gitignored; never commit + OAuth secrets or the admin password. -## Conventions +## Rules +- **The repo is the only durable path.** Dashboards and alerting are provisioned + from this directory at startup; edits made through Grafana's UI or API are + overwritten on the next provision. Changes go through a PR. - **Every artifact that ships to a node is CI-validated** - (`.forgejo/workflows/observability-validate.yml`): both compose files parse, all - dashboard JSON is valid, and the Loki/provisioning YAML parses. Keep new - dashboards as valid JSON and new config as valid YAML or CI fails. The alerting - config gets a stricter third step — every rule's `condition` must name a refId - that exists, every policy must route to a receiver that exists, and a literal - Discord webhook URL in the repo is a hard failure. (A rule pointing at a missing - refId is valid YAML, provisions cleanly, and then never fires; that is precisely - the silent-no-op this whole domain exists to prevent.) The Alloy config is still - not CI-validated (needs the `alloy` binary). -- The public hostname is **`dashboard.thermograph.org`** everywhere — not - `grafana.thermograph.org`. Match it in any new comment/config. -- Secrets (OAuth client id/secret, admin password) live only in the host `.env`, - never in the repo. + (`.forgejo/workflows/observability-validate.yml`, at the repo root): both + compose files parse, all dashboard JSON is valid, the Loki and provisioning + YAML parse, and the Alloy config is checked with the pinned `alloy` binary + (v1.9.1, matching what the fleet runs). +- **Alerting gets a stricter third step.** Every rule's `condition` must name a + refId that exists, every policy must route to a receiver that exists, and a + literal Discord webhook URL in the repo is a hard failure. A rule pointing at a + missing refId is valid YAML, provisions cleanly, and then never fires — that + silent no-op is what this check exists to prevent. +- **Alerts go to Discord `#ops-alerts`, never email.** Beta's Grafana relays SMTP + through prod's Postfix, so email dies exactly when prod does. +- **Every rule is LogQL** — there is no Prometheus anywhere in the fleet. + Thresholds were derived from real Loki data and the working is in the comments + beside each rule; re-derive before changing a number rather than guessing. +- Secrets (OAuth client id/secret, admin password) live only in the host `.env`. -## Branching +## Commits & PRs -Single `main` branch, no protection. Open a PR against `main`; the validator -gates it. Deploying the change to beta / the nodes is a separate manual step -(this repo has no deploy automation — a known gap). +Concise and technical. Never mention AI, assistants or automated authorship. -- 2.45.2 From 7307a7aadc258bcfe0dda2da4bfc186c35363be1 Mon Sep 17 00:00:00 2001 From: emi Date: Sat, 25 Jul 2026 07:09:11 +0000 Subject: [PATCH 09/10] guardrails: enforce live-host and secrets policy with hooks (#82) --- .claude/hooks/README.md | 84 +++++++++++++++++ .claude/hooks/lint-after-edit.sh | 34 +++++++ .claude/hooks/prod-guard.sh | 156 +++++++++++++++++++++++++++++++ .claude/hooks/secrets-guard.sh | 34 +++++++ .claude/settings.json | 42 +++++++++ 5 files changed, 350 insertions(+) create mode 100644 .claude/hooks/README.md create mode 100755 .claude/hooks/lint-after-edit.sh create mode 100755 .claude/hooks/prod-guard.sh create mode 100755 .claude/hooks/secrets-guard.sh create mode 100644 .claude/settings.json diff --git a/.claude/hooks/README.md b/.claude/hooks/README.md new file mode 100644 index 0000000..0075e86 --- /dev/null +++ b/.claude/hooks/README.md @@ -0,0 +1,84 @@ +# .claude/hooks — enforcement, not advice + +`CLAUDE.md` files can only ask. These hooks are the part that enforces, and they +travel with the repo so a session started anywhere in this checkout gets them. + +They are wired up in `.claude/settings.json`. + +| Hook | Event | What it does | +|---|---|---| +| `prod-guard.sh` | PreToolUse | Live-host commands: reads run unprompted, mutations ask first | +| `secrets-guard.sh` | PreToolUse | Denies any direct write to `infra/deploy/secrets/*.yaml` | +| `lint-after-edit.sh` | PostToolUse | shellchecks an edited `*.sh` and feeds findings back immediately | + +## Why prod-guard exists + +`agent` has passwordless sudo on prod and beta, and the operator's global settings +allow `Bash(ssh prod:*)` outright. Nothing about `ssh prod 'docker service rm …'` +goes through a pull request, so CI cannot see it — before this hook, any session in +any directory could delete production with no prompt. + +**It classifies by allowlist, not blocklist.** Only commands positively recognised as +read-only are allowed through; everything else asks. A blocklist of dangerous verbs +is wrong by construction — the first destructive verb nobody thought of sails +straight through. Being wrong in the "ask" direction costs a keystroke; being wrong +the other way costs production. + +Beta is guarded as strictly as prod: it serves beta.thermograph.org *and* hosts +Forgejo, so a destructive command there takes out git, CI and the registry at once. +The LAN dev box is deliberately unguarded. + +## Changing the classifier + +`is_readonly()` in `prod-guard.sh` splits a command on `&&`, `||`, `;` and `|`, then +checks each segment's leading word. Redirection to a file, command substitution, +`sed -i`, and mutating `docker`/`git`/`systemctl` subcommands all count as writes. + +If you add a command to the allowlist, **re-run the test matrix**. There is a +non-obvious failure mode this already hit once: the loop is fed by +`printf '%s\n' | sed`, and dropping that trailing newline makes `read` return +non-zero on the only line, so the loop body never runs and *every* command +classifies as read-only — a silent, total bypass that still looks like it works. + +Test by piping a payload straight in: + +```bash +echo '{"tool_name":"Bash","tool_input":{"command":"ssh prod '\''rm -rf /'\''"}}' \ + | .claude/hooks/prod-guard.sh +# expect: permissionDecision "ask" +``` + +Exit 0 with no output means "no opinion" and the normal permission flow proceeds. +That is also what happens if `jq` is missing or the script errors, so a bug here +fails toward the prompt rather than toward silent execution. + +## lint-after-edit needs shellcheck on PATH + +It exits quietly when shellcheck is absent — a missing local tool must not break +editing, and `shell-lint` CI is still the backstop. v0.11.0 (the version CI pins) is +installed at `~/.local/bin/shellcheck`; if you move machines, reinstall it: + +```bash +VER=v0.11.0 +curl -fsSL "https://github.com/koalaman/shellcheck/releases/download/${VER}/shellcheck-${VER}.linux.x86_64.tar.xz" \ + | tar -xJ --strip-components=1 -C ~/.local/bin "shellcheck-${VER}/shellcheck" +``` + +Keep it matched to `shell-lint.yml`'s pin. A drifted shellcheck grows *new* warnings +and fails CI on an unrelated push, which is why that workflow pins the version and +its sha256 rather than using `apt-get install`. + +## The global copy + +`~/.claude/settings.json` runs `prod-guard.sh` from `~/.claude/hooks/` as well, so a +session started **outside** this repo is covered too — otherwise the guard would be +trivially bypassed by working from `~/Code`. That copy is a mirror; this one is +canonical. If you change the classifier here, copy it over: + +```bash +cp .claude/hooks/prod-guard.sh ~/.claude/hooks/prod-guard.sh +``` + +The blanket `Bash(ssh prod:*)` / `Bash(ssh beta:*)` allow rules were removed from +global settings at the same time, so the hook is the only thing granting live-host +access. If it is missing or broken, nothing allows the command and you get a prompt. diff --git a/.claude/hooks/lint-after-edit.sh b/.claude/hooks/lint-after-edit.sh new file mode 100755 index 0000000..304cdea --- /dev/null +++ b/.claude/hooks/lint-after-edit.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# PostToolUse: shellcheck a shell script the moment it is edited. +# +# The scripts under infra/deploy/ run as root over SSH against live hosts with no +# test suite in front of them — a quoting bug or an unset-variable typo is found on +# the host, or not at all. shell-lint.yml already guards this, but only after a +# push: a five-minute round trip to learn about a missing quote, by which point the +# context that produced it is gone. +# +# Findings are fed back to the model rather than blocking the edit. The file is +# already written; the useful move is to fix it now, in the same turn. +# +# Matches shell-lint.yml's pinned shellcheck when one is on PATH. If shellcheck is +# not installed this exits quietly — CI is still the backstop, and a missing local +# tool must not break editing. +set -uo pipefail + +payload=$(cat) +command -v jq >/dev/null 2>&1 || exit 0 +command -v shellcheck >/dev/null 2>&1 || exit 0 + +path=$(printf '%s' "$payload" | jq -r '.tool_response.filePath // .tool_input.file_path // ""') +[ -n "$path" ] || exit 0 +case "$path" in *.sh) ;; *) exit 0 ;; esac +[ -f "$path" ] || exit 0 + +if out=$(shellcheck -f gcc "$path" 2>&1); then + exit 0 +fi + +# Keep the feedback small: findings only, capped. +out=$(printf '%s' "$out" | head -30) +jq -n --arg p "$path" --arg o "$out" \ + '{decision:"block", reason:("shellcheck findings in \($p) — shell-lint CI will fail on these, so fix them now:\n\($o)")}' diff --git a/.claude/hooks/prod-guard.sh b/.claude/hooks/prod-guard.sh new file mode 100755 index 0000000..5d81dda --- /dev/null +++ b/.claude/hooks/prod-guard.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# PreToolUse guard for commands that reach a live host. +# +# Policy: reads stay friction-free, mutations ask first. +# +# The estate's widest privilege is that `agent` has passwordless sudo on prod and +# beta, and the global settings allow `ssh prod:*` outright — so before this hook +# existed, any session in any directory could roll, delete or restart production +# with no prompt. CI cannot help here: nothing about `ssh prod 'docker service rm'` +# goes through a pull request. +# +# Classification is an ALLOWLIST of read-only commands, not a blocklist of +# dangerous ones. A blocklist is wrong by construction: the first destructive verb +# nobody thought of sails straight through. Anything this script does not +# positively recognise as a read becomes an "ask", which is the safe direction to +# be wrong in. +# +# Exit 0 with no output = "no opinion", and the normal permission flow proceeds. +# That is also what happens if this script errors or jq is missing, so a bug here +# fails toward the prompt rather than toward silent execution. +# +# Beta is guarded as strictly as prod: it serves beta.thermograph.org AND hosts +# Forgejo, so a destructive command there can take out the git server, CI and the +# container registry at once. +set -uo pipefail + +payload=$(cat) +command -v jq >/dev/null 2>&1 || exit 0 + +tool=$(printf '%s' "$payload" | jq -r '.tool_name // ""') + +# An ssh target naming a live host, matched as a whole token after any `user@`. +# Backslashes are doubled because awk -v processes escape sequences in the value +# before the regex ever sees it; a single \. arrives as a bare dot (and warns). +readonly HOST_TOKEN_RE='^(prod|beta|169\\.58\\.46\\.181|75\\.119\\.132\\.91|10\\.10\\.0\\.[12])$' + +emit() { # $1=allow|ask|deny $2=reason + jq -n --arg d "$1" --arg r "$2" \ + '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:$d,permissionDecisionReason:$r}}' + exit 0 +} + +# --- is this shell command read-only? ---------------------------------------- +# Returns 0 only when every segment leads with a recognised read-only command. +is_readonly() { + local cmd="$1" seg w1 w2 stripped + + # Discard the harmless redirections that appear in ordinary read commands, + # then treat any surviving redirect as a write. + stripped=${cmd//2>&1/} + stripped=${stripped//&>\/dev\/null/} + stripped=${stripped//2>\/dev\/null/} + stripped=${stripped//>\/dev\/null/} + stripped=${stripped//> \/dev\/null/} + case "$stripped" in *'>'*) return 1 ;; esac + + # Command substitution can hide anything; refuse to reason about it. + # shellcheck disable=SC2016 # single quotes are the point: these are literal + # `$(` and backtick characters being searched for inside the command text, not + # expansions to perform. Expanding them here would defeat the check entirely. + case "$cmd" in *'$('*|*'`'*) return 1 ;; esac + + while IFS= read -r seg; do + seg="${seg#"${seg%%[![:space:]]*}"}" + [ -z "$seg" ] && continue + w1=$(printf '%s' "$seg" | awk '{print $1}') + w2=$(printf '%s' "$seg" | awk '{print $2}') + case "$w1" in + cat|ls|head|tail|wc|grep|egrep|fgrep|rg|find|stat|df|du|free|uptime|whoami) ;; + id|hostname|date|ps|env|printenv|pwd|which|uname|lsblk|ip|ss|netstat) ;; + pg_isready|echo|true|sort|uniq|cut|tr|awk|jq|nproc|readlink|realpath|test) ;; + sed) case "$seg" in *' -i'*) return 1 ;; esac ;; + curl) case "$seg" in *' -X'*|*' -d'*|*'--data'*|*' -T'*|*' -F'*|*'--upload'*) return 1 ;; esac ;; + journalctl) ;; + systemctl) + case "$w2" in status|show|is-active|is-enabled|is-failed|list-units|cat) ;; *) return 1 ;; esac ;; + docker) + case "$w2" in + ps|logs|inspect|stats|images|version|info|top|port|events|history|diff) ;; + service) case "$seg" in *'service ls'*|*'service ps'*|*'service inspect'*|*'service logs'*) ;; *) return 1 ;; esac ;; + stack) case "$seg" in *'stack ls'*|*'stack ps'*|*'stack services'*) ;; *) return 1 ;; esac ;; + compose) case "$seg" in *'compose ps'*|*'compose logs'*|*'compose config'*|*'compose top'*|*'compose images'*) ;; *) return 1 ;; esac ;; + node) case "$seg" in *'node ls'*|*'node inspect'*) ;; *) return 1 ;; esac ;; + volume) case "$seg" in *'volume ls'*|*'volume inspect'*) ;; *) return 1 ;; esac ;; + *) return 1 ;; + esac ;; + git) + case "$w2" in status|log|diff|show|branch|remote|rev-parse|describe|blame|ls-files|config) ;; *) return 1 ;; esac ;; + *) return 1 ;; + esac + # NOTE: the trailing newline matters. Without it `read` returns non-zero on the + # final (only) line, the loop body never runs, and every command classifies as + # read-only — a silent false negative that allows anything. + done < <(printf '%s\n' "$cmd" | sed 's/&&/\n/g; s/||/\n/g; s/;/\n/g; s/|/\n/g') + return 0 +} + +# --- find an ssh target and return what would run on it ----------------------- +# Prints "\t" when a live host is addressed, nothing +# otherwise. The remote command is empty for a bare login shell. +ssh_target_of() { + printf '%s' "$1" | awk -v re="$HOST_TOKEN_RE" ' + { + for (i = 1; i <= NF; i++) { + t = $i; sub(/^[^@]*@/, "", t) + if (t ~ re) { + out = "" + for (j = i + 1; j <= NF; j++) out = out (out == "" ? "" : " ") $j + printf "%s\t%s\n", t, out + exit + } + } + }' +} + +case "$tool" in + Bash) + cmd=$(printf '%s' "$payload" | jq -r '.tool_input.command // ""') + case "$cmd" in *ssh*) ;; *) exit 0 ;; esac + found=$(ssh_target_of "$cmd") + [ -n "$found" ] || exit 0 # no live host addressed + host=${found%%$'\t'*} + remote=${found#*$'\t'} + remote=$(printf '%s' "$remote" | sed -E "s/^'(.*)'$/\1/; s/^\"(.*)\"$/\1/") + [ -z "$remote" ] && exit 0 # bare login shell + if is_readonly "$remote"; then + emit allow "read-only command on ${host}" + fi + emit ask "This mutates ${host}, a live host where the agent user has passwordless sudo. Command: ${remote}" + ;; + + mcp__centralis__run_on_host) + env=$(printf '%s' "$payload" | jq -r '.tool_input.env // ""') + cmd=$(printf '%s' "$payload" | jq -r '.tool_input.command // ""') + case "$env" in prod|beta) ;; *) exit 0 ;; esac + if is_readonly "$cmd"; then + emit allow "read-only command on ${env}" + fi + emit ask "run_on_host would mutate ${env}. Command: ${cmd}" + ;; + + mcp__centralis__sql_query) + env=$(printf '%s' "$payload" | jq -r '.tool_input.env // ""') + write=$(printf '%s' "$payload" | jq -r '.tool_input.write // false') + [ "$write" = "true" ] || exit 0 + case "$env" in prod|beta) ;; *) exit 0 ;; esac + sql=$(printf '%s' "$payload" | jq -r '.tool_input.sql // ""' | tr '\n' ' ' | cut -c1-300) + emit ask "Write query against the ${env} database. sql_query escalates to the app role and has no confirmation of its own. SQL: ${sql}" + ;; + + mcp__centralis__rollback_to|mcp__centralis__promote|mcp__centralis__secrets_rotate) + emit ask "${tool#mcp__centralis__} changes production state directly." + ;; + + *) exit 0 ;; +esac diff --git a/.claude/hooks/secrets-guard.sh b/.claude/hooks/secrets-guard.sh new file mode 100755 index 0000000..eed51f1 --- /dev/null +++ b/.claude/hooks/secrets-guard.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# PreToolUse guard: never write infra/deploy/secrets/*.yaml directly. +# +# Every file matching that glob is SOPS-encrypted, and secrets-guard.yml fails the +# build if one is not. This is the same rule enforced one step earlier: a direct +# Write or Edit produces plaintext, and by the time CI catches it the secret has +# already been written to disk and probably committed. +# +# The supported path is `sops edit `, which decrypts to a temp file, opens an +# editor and re-encrypts on save — the plaintext never touches the working tree. +# +# Denies rather than asks: there is no legitimate reason to hand-write one of these, +# so a prompt would only be an invitation to click through. +set -uo pipefail + +payload=$(cat) +command -v jq >/dev/null 2>&1 || exit 0 + +path=$(printf '%s' "$payload" | jq -r '.tool_input.file_path // ""') +[ -n "$path" ] || exit 0 + +case "$path" in + */infra/deploy/secrets/*.yaml) + jq -n --arg p "$path" '{ + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: ("Refusing to write \($p) directly — every file under infra/deploy/secrets/ is SOPS-encrypted and a direct write would land plaintext. Use `sops edit \($p)` instead, which re-encrypts on save. (secrets-guard.yml would fail the build on this anyway, but only after the secret was already on disk.)") + } + }' + exit 0 + ;; + *) exit 0 ;; +esac diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..11fcff0 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "hooks": { + "PreToolUse": [ + { + "matcher": "Bash|mcp__centralis__run_on_host|mcp__centralis__sql_query|mcp__centralis__rollback_to|mcp__centralis__promote|mcp__centralis__secrets_rotate", + "hooks": [ + { + "type": "command", + "command": "\"${CLAUDE_PROJECT_DIR:-.}/.claude/hooks/prod-guard.sh\"", + "timeout": 10, + "statusMessage": "Checking live-host access" + } + ] + }, + { + "matcher": "Write|Edit", + "hooks": [ + { + "type": "command", + "command": "\"${CLAUDE_PROJECT_DIR:-.}/.claude/hooks/secrets-guard.sh\"", + "timeout": 10, + "statusMessage": "Checking SOPS vault" + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "Write|Edit", + "hooks": [ + { + "type": "command", + "command": "\"${CLAUDE_PROJECT_DIR:-.}/.claude/hooks/lint-after-edit.sh\"", + "timeout": 30, + "statusMessage": "shellcheck" + } + ] + } + ] + } +} -- 2.45.2 From dde2cdd6e2cdabd061bbd2b1e01d669ea3828f13 Mon Sep 17 00:00:00 2001 From: emi Date: Sat, 25 Jul 2026 07:16:08 +0000 Subject: [PATCH 10/10] infra: mirror LAN dev secrets under $HOME for snap-confined Docker (#85) --- infra/.gitignore | 4 ++++ infra/deploy/deploy-dev.sh | 9 +++++++++ infra/deploy/render-secrets.sh | 17 +++++++++++++++++ infra/docker-compose.dev.yml | 17 +++++++++++++++++ 4 files changed, 47 insertions(+) diff --git a/infra/.gitignore b/infra/.gitignore index c3ac61f..c6b008f 100644 --- a/infra/.gitignore +++ b/infra/.gitignore @@ -27,3 +27,7 @@ __pycache__/ deploy/.image-tags.env deploy/.deploy.lock deploy/.stack-image-tags.env + +# LAN dev's mirror of the rendered secrets, for snap-confined Docker (see +# deploy-dev.sh / render-secrets.sh) -- plaintext, re-rendered every deploy. +deploy/dev-secrets.env diff --git a/infra/deploy/deploy-dev.sh b/infra/deploy/deploy-dev.sh index 08c501f..8149d8b 100755 --- a/infra/deploy/deploy-dev.sh +++ b/infra/deploy/deploy-dev.sh @@ -72,6 +72,15 @@ export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-thermograph-dev}" # genuinely needs, and nothing else. export THERMOGRAPH_SECRETS_SKIP_COMMON=1 +# (1b) Docker on this box is the snap package, confined to $HOME (plus a short +# allowlist) -- it silently can't see /etc/thermograph.env at all, so +# env_file: /etc/thermograph.env resolves to nothing for every container no +# matter how correctly it's rendered. Mirror the render to a path under +# $APP_DIR too; docker-compose.dev.yml points env_file at this copy instead of +# /etc/thermograph.env. Untracked (see infra/.gitignore); re-rendered on every +# deploy, never committed. +export THERMOGRAPH_SECRETS_ENV_FILE_MIRROR="$APP_DIR/infra/deploy/dev-secrets.env" + # (2) The age key is read from wherever it is READABLE, not necessarily # /etc/thermograph/age.key. render-secrets.sh falls back to `sudo cat` for a # root-owned 0400 key, and this box has no passwordless sudo -- under the Forgejo diff --git a/infra/deploy/render-secrets.sh b/infra/deploy/render-secrets.sh index b5de8fc..b3b5b3c 100755 --- a/infra/deploy/render-secrets.sh +++ b/infra/deploy/render-secrets.sh @@ -127,6 +127,23 @@ render_thermograph_secrets() { echo "!! cannot write /etc/thermograph.env (need file write access or passwordless sudo)" >&2 rc=1 fi + + # Optional second copy under $HOME, for a host whose docker CLI cannot read + # /etc/thermograph.env at all -- the LAN dev box's snap-packaged Docker is + # confined to $HOME (plus a short allowlist) and silently treats anything + # under /etc as nonexistent, so env_file: /etc/thermograph.env resolves to + # nothing for every container even though the file is right there and the + # shell that rendered it can read it fine. deploy-dev.sh sets this; + # prod/beta (apt-installed Docker, no snap confinement) leave it unset and + # this is a no-op for them. + if [ "$rc" = 0 ] && [ -n "${THERMOGRAPH_SECRETS_ENV_FILE_MIRROR:-}" ]; then + if ! mkdir -p "$(dirname "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR")" \ + || ! install -m 0600 "$tmp" "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR"; then + echo "!! cannot write mirror at $THERMOGRAPH_SECRETS_ENV_FILE_MIRROR" >&2 + rc=1 + fi + fi + rm -f "$tmp" return "$rc" } diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index 9c12a9b..86f9c53 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -27,12 +27,23 @@ # prod's backend=4 / frontend=2 / db=2 allocation. `!reset` drops the base # value (both the top-level `cpus:` and the Swarm-style `deploy.resources` # block the base file carries for parity). +# 4. backend/daemon/lake get a second env_file entry pointing at a copy of the +# render under $APP_DIR (deploy-dev.sh sets THERMOGRAPH_SECRETS_ENV_FILE_MIRROR +# to produce it). This box's Docker is the snap package, confined to $HOME -- +# it can't see /etc/thermograph.env at all (not a permissions error, it just +# doesn't exist as far as snap-confined Docker is concerned), so the base +# file's env_file entry silently loads nothing here. compose appends env_file +# lists across overlays (last-wins on duplicate keys), so this is additive: +# prod/beta never set the mirror var, so they only ever get the base entry. services: backend: ports: !override - "8137:8137" cpus: !reset null deploy: !reset null + env_file: + - path: ./deploy/dev-secrets.env + required: false frontend: ports: !reset null cpus: !reset null @@ -42,6 +53,9 @@ services: daemon: cpus: !reset null deploy: !reset null + env_file: + - path: ./deploy/dev-secrets.env + required: false db: cpus: !reset null deploy: !reset null @@ -53,3 +67,6 @@ services: # the default DB_MEMORY (raise DB_MEMORY to give dev more); shm_size stays (it's # required shared memory for parallel query, not a limit). mem_limit: !reset null + env_file: + - path: ./deploy/dev-secrets.env + required: false -- 2.45.2