From f5d9f33b5ba826430fe4d8767008dd45277801ad Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Thu, 16 Jul 2026 13:38:59 -0700 Subject: [PATCH] Auto-submit IndexNow on deploy when the URL set changes (#130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hook the production deploy (deploy/deploy.sh) to ping IndexNow after the health check passes, so new/changed pages reach Bing/DuckDuckGo/Yandex without a manual run. Best-effort — wrapped so it can never fail a deploy — and it sources /etc/thermograph.env so its key matches the one the running service serves. To avoid re-blasting ~14k URLs on every code push, indexnow.py gains a --if-changed mode gated by a signature of the URL set (persisted to data/indexnow_state.txt): code-only deploys skip; a deploy that adds or removes a city submits. `make indexnow` still forces a full submit. --- indexnow.py | 44 +++++++++++++++++++++++++++++++++++++++++-- tests/test_content.py | 10 ++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/indexnow.py b/indexnow.py index 7f2d6e9..90e3b0a 100644 --- a/indexnow.py +++ b/indexnow.py @@ -16,6 +16,7 @@ Usage (run after a content rebuild, or once to seed a fresh site): cd backend && python indexnow.py https://thermograph.org # or: make indexnow URL=https://thermograph.org """ +import hashlib import logging import os import secrets @@ -28,6 +29,7 @@ log = logging.getLogger("thermograph.indexnow") _DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "data")) _KEY_PATH = os.environ.get("THERMOGRAPH_INDEXNOW_KEY_FILE") or os.path.join(_DATA_DIR, "indexnow_key.txt") +_STATE_PATH = os.environ.get("THERMOGRAPH_INDEXNOW_STATE_FILE") or os.path.join(_DATA_DIR, "indexnow_state.txt") _ENDPOINT = "https://api.indexnow.org/indexnow" # a shared endpoint; it fans out to all participants _BATCH = 10000 # IndexNow's max URLs per request @@ -104,13 +106,51 @@ def submit_all(site_base_url: str, **kw) -> dict: key_location=f"{site}/{key()}.txt", **kw) +def url_signature() -> str: + """A stable fingerprint of the indexable URL set — changes only when pages are + added or removed (e.g. a new city), not on code-only deploys. Used to skip + resubmitting an unchanged set (see the --if-changed CLI flag).""" + import content + return hashlib.sha1("\n".join(content.public_paths()).encode()).hexdigest() + + +def _read_state() -> str: + try: + with open(_STATE_PATH, encoding="utf-8") as f: + return f.read().strip() + except OSError: + return "" + + +def _write_state(sig: str) -> None: + try: + os.makedirs(_DATA_DIR, exist_ok=True) + with open(_STATE_PATH, "w", encoding="utf-8") as f: + f.write(sig) + except OSError: + log.warning("could not persist IndexNow state to %s", _STATE_PATH) + + if __name__ == "__main__": import sys - arg = sys.argv[1].strip() if len(sys.argv) > 1 else "" - base = arg or os.environ.get("THERMOGRAPH_BASE_URL", "https://thermograph.org") + # Usage: indexnow.py [--if-changed] [BASE_URL] + # --if-changed submit only when the URL set changed since the last run + # (used by the deploy hook so code-only deploys don't resubmit). + argv = sys.argv[1:] + if_changed = "--if-changed" in argv + positional = [a for a in argv if not a.startswith("-") and a.strip()] + base = (positional[0] if positional else "") or os.environ.get("THERMOGRAPH_BASE_URL", "https://thermograph.org") logging.basicConfig(level=logging.INFO) + + sig = url_signature() + if if_changed and sig == _read_state(): + print("IndexNow: URL set unchanged since last submission — skipping.") + sys.exit(0) + print(f"IndexNow key: {key()} (served at {base.rstrip('/')}/{key()}.txt)") result = submit_all(base) print(f"Submitted {result['submitted']}/{result['total']} URLs " f"in {result['batches']} batch(es); statuses={result['statuses']}") + if result["total"] and result["submitted"] == result["total"]: + _write_state(sig) # only remember a fully-successful submission diff --git a/tests/test_content.py b/tests/test_content.py index a2994ae..83b1579 100644 --- a/tests/test_content.py +++ b/tests/test_content.py @@ -231,6 +231,16 @@ def test_indexnow_submit_all_builds_payload(monkeypatch): assert all(len(c[1]["urlList"]) <= 10000 for c in calls) # batched under the IndexNow cap +def test_indexnow_if_changed_state(monkeypatch, tmp_path): + import indexnow + monkeypatch.setattr(indexnow, "_STATE_PATH", str(tmp_path / "state.txt")) + sig = indexnow.url_signature() + assert len(sig) == 40 and sig == indexnow.url_signature() # stable sha1 of the URL set + assert indexnow._read_state() == "" # first deploy → would submit + indexnow._write_state(sig) + assert indexnow._read_state() == sig # unchanged → deploy would skip + + def test_search_verification_meta(client, monkeypatch): monkeypatch.setenv("THERMOGRAPH_GOOGLE_VERIFY", "gtok123") monkeypatch.setenv("THERMOGRAPH_BING_VERIFY", "btok456")