package content import ( "net/http" "strings" ) // RobotsTxt is the port of content.py's robots_txt — the body is built the // same way, byte for byte. func (h *Handlers) RobotsTxt(w http.ResponseWriter, r *http.Request) { base := h.cfg.Base baseURL := origin(r) + base body := "User-agent: *\n" + "Allow: /\n" + "Disallow: " + base + "/api/\n" + "Disallow: " + base + "/alerts\n" + "Sitemap: " + baseURL + "/sitemap.xml\n" w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Write([]byte(body)) } // SitemapXML is the port of content.py's sitemap_xml: one line per // backend sitemap entry, pinned to the per-process boot date. // ~2.35 MB and crawled repeatedly — the short Cache-Control saves every // crawler hit within the window a full Sitemap() call plus the string build, // on top of the response body itself (the backend client's TTL cache absorbs // the rest). func (h *Handlers) SitemapXML(w http.ResponseWriter, r *http.Request) { baseURL := origin(r) + h.cfg.Base entries, err := h.api.Sitemap() if err != nil { h.apiError(w, err) return } var b strings.Builder b.WriteString(`` + "\n") b.WriteString(``) for _, e := range entries { b.WriteString("\n") b.WriteString(baseURL) b.WriteString(e.Path) b.WriteString("") b.WriteString(h.bootDate) b.WriteString("") b.WriteString(e.Changefreq) b.WriteString("") b.WriteString(e.Priority) b.WriteString("") } b.WriteString("\n") w.Header().Set("Content-Type", "application/xml") w.Header().Set("Cache-Control", "public, max-age=300") w.Write([]byte(b.String())) } // registerIndexNow wires the IndexNow key file. IndexNow verification works // by serving a file at /.txt — the key isn't just response *content*, // it's baked into the route *path*, which the mux needs at registration // time. That used to mean an eager, unretried key fetch at boot — so if the // backend was unreachable (down, mid-restart, a network blip) the frontend // would never finish booting at all. That's exactly the coupling the // repo-split removed: frontend and backend deploy asynchronously, so // "backend happens to be briefly unreachable" must be a normal, survivable // condition at frontend boot, not a crash. // // So: try the eager fetch once (preserving the plain static route for the // overwhelmingly common case where the backend IS reachable at boot). If it // fails, log a warning and fall back to a route that lazily (re)fetches the // key on each request via the client's TTL cache — once the backend comes // back up the correct key starts being served automatically, no frontend // restart required, and no request ever gets a permanently wrong/empty key. // // Mux mechanics differ from Starlette here: Go's patterns cannot express the // Python's "/{token}.txt" suffix wildcard, so the lazy fallback claims the // whole single-segment slot ({BASE}/{token}) and forwards anything that // isn't a .txt request to the static file server it displaced (explicit page // routes still win on specificity). func (h *Handlers) registerIndexNow(mux *http.ServeMux, static http.Handler) { base := h.cfg.Base key, err := h.api.IndexNowKey() if err == nil { mux.HandleFunc("GET "+base+"/"+key+".txt", func(w http.ResponseWriter, r *http.Request) { writeKey(w, key) }) return } h.log.Printf("indexnow_key fetch failed at boot (backend unreachable?) -- "+ "continuing boot without it; falling back to lazy per-request lookup at "+ "/.txt so the frontend doesn't depend on backend liveness to start up: %v", err) mux.HandleFunc("GET "+base+"/{token}", func(w http.ResponseWriter, r *http.Request) { token := r.PathValue("token") if !strings.HasSuffix(token, ".txt") { // Not a key-file request: this pattern displaced the static // subtree for single-segment paths, so hand it back. if static != nil { static.ServeHTTP(w, r) return } http.NotFound(w, r) return } key, err := h.api.IndexNowKey() if err != nil { writeDetail(w, http.StatusServiceUnavailable, "backend unavailable") return } if strings.TrimSuffix(token, ".txt") != key { writeDetail(w, http.StatusNotFound, "Not Found") return } writeKey(w, key) }) } func writeKey(w http.ResponseWriter, key string) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Write([]byte(key + "\n")) }