23615a1085
Faster search-engine indexing for the ~14k climate/city/record URLs:
- IndexNow (backend/indexnow.py): instantly notify Bing/DuckDuckGo/Yandex of new
or changed URLs. Per-host key resolved env → gitignored file → generated (mirrors
push.py), served at /{key}.txt, and echoed in submissions. `submit_all()` + a
`make indexnow` CLI push every indexable URL, batched under the 10k cap. Google
doesn't use IndexNow, so it stays on the sitemap.
- Sitemap: replace the per-request today() <lastmod> (which churns every fetch and
trains crawlers to ignore lastmod) with a stable content-build date; factor the
URL list into public_paths() shared with IndexNow so the two never drift.
- Search-console verification: render google-site-verification + msvalidate.01
<meta> tags from env into every page's <head> — the SEO pages via base.html.j2
and the static pages (incl. the homepage Google verifies) via _page().
- Docs: env example documents the three new vars; .gitignore ignores the key.
Tests: key-file route, stable single-date lastmod, IndexNow payload/batching, and
verification meta on both SEO and static pages.
116 lines
4.6 KiB
Python
116 lines
4.6 KiB
Python
"""IndexNow — instantly notify Bing, DuckDuckGo, Yandex, Seznam and friends of new
|
||
or changed URLs so they (re)crawl within minutes instead of days.
|
||
|
||
Note: Google does NOT use IndexNow. For Google, submit the sitemap in Search
|
||
Console; this module covers everyone else (and DuckDuckGo, which is powered by
|
||
Bing's index).
|
||
|
||
A per-host key authorises submissions: it is served as a plain-text file at the
|
||
site root (``/{key}.txt``, wired up in content.register) and echoed in every
|
||
submission so the search engine can confirm we own the host. Key resolution order
|
||
mirrors push.py's VAPID handling: env var → gitignored file → generated once and
|
||
persisted.
|
||
|
||
Usage (run after a content rebuild, or once to seed a fresh site):
|
||
|
||
cd backend && python indexnow.py https://thermograph.org
|
||
# or: make indexnow URL=https://thermograph.org
|
||
"""
|
||
import logging
|
||
import os
|
||
import secrets
|
||
import threading
|
||
from urllib.parse import urlparse
|
||
|
||
import httpx
|
||
|
||
log = logging.getLogger("thermograph.indexnow")
|
||
|
||
_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "data"))
|
||
_KEY_PATH = os.environ.get("THERMOGRAPH_INDEXNOW_KEY_FILE") or os.path.join(_DATA_DIR, "indexnow_key.txt")
|
||
_ENDPOINT = "https://api.indexnow.org/indexnow" # a shared endpoint; it fans out to all participants
|
||
_BATCH = 10000 # IndexNow's max URLs per request
|
||
|
||
_lock = threading.Lock()
|
||
_key = None
|
||
|
||
|
||
def key() -> str:
|
||
"""The IndexNow key (env → file → generate-and-persist), cached for the process."""
|
||
global _key
|
||
if _key:
|
||
return _key
|
||
with _lock:
|
||
if _key:
|
||
return _key
|
||
env = os.environ.get("THERMOGRAPH_INDEXNOW_KEY", "").strip()
|
||
if env:
|
||
_key = env
|
||
return _key
|
||
try:
|
||
with open(_KEY_PATH, encoding="utf-8") as f:
|
||
k = f.read().strip()
|
||
if k:
|
||
_key = k
|
||
return _key
|
||
except OSError:
|
||
pass
|
||
_key = secrets.token_hex(16) # 32 hex chars — within IndexNow's 8–128 range
|
||
try:
|
||
os.makedirs(_DATA_DIR, exist_ok=True)
|
||
with open(_KEY_PATH, "w", encoding="utf-8") as f:
|
||
f.write(_key)
|
||
os.chmod(_KEY_PATH, 0o600)
|
||
except OSError:
|
||
log.warning("could not persist IndexNow key to %s; using an in-memory key", _KEY_PATH)
|
||
return _key
|
||
|
||
|
||
def submit(urls, host: str, key_location: str, scheme: str = "https", timeout: float = 30) -> dict:
|
||
"""POST ``urls`` (deduped, in batches of 10k) to IndexNow. Every URL must be on
|
||
``host``; ``key_location`` is the public URL of our key file. Returns a summary."""
|
||
urls = list(dict.fromkeys(urls))
|
||
if not urls:
|
||
return {"submitted": 0, "total": 0, "batches": 0}
|
||
k = key()
|
||
submitted, statuses = 0, []
|
||
for i in range(0, len(urls), _BATCH):
|
||
batch = urls[i:i + _BATCH]
|
||
payload = {"host": host, "key": k, "keyLocation": key_location, "urlList": batch}
|
||
try:
|
||
r = httpx.post(_ENDPOINT, json=payload, timeout=timeout,
|
||
headers={"Content-Type": "application/json; charset=utf-8"})
|
||
except httpx.HTTPError as e:
|
||
log.warning("IndexNow request failed: %s", e)
|
||
statuses.append("error")
|
||
continue
|
||
statuses.append(r.status_code)
|
||
if r.status_code in (200, 202):
|
||
submitted += len(batch)
|
||
else:
|
||
log.warning("IndexNow batch rejected: %s %s", r.status_code, r.text[:200])
|
||
return {"submitted": submitted, "total": len(urls),
|
||
"batches": len(statuses), "statuses": statuses, "key_location": key_location}
|
||
|
||
|
||
def submit_all(site_base_url: str, **kw) -> dict:
|
||
"""Submit every indexable page. ``site_base_url`` is the public origin + base
|
||
path, e.g. ``https://thermograph.org`` (root) or ``https://host/thermograph``."""
|
||
import content # lazy: avoids a content ↔ indexnow import cycle
|
||
site = site_base_url.rstrip("/")
|
||
parsed = urlparse(site)
|
||
urls = [site + path for path in content.public_paths()]
|
||
return submit(urls, host=parsed.hostname, scheme=parsed.scheme or "https",
|
||
key_location=f"{site}/{key()}.txt", **kw)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import sys
|
||
|
||
arg = sys.argv[1].strip() if len(sys.argv) > 1 else ""
|
||
base = arg or os.environ.get("THERMOGRAPH_BASE_URL", "https://thermograph.org")
|
||
logging.basicConfig(level=logging.INFO)
|
||
print(f"IndexNow key: {key()} (served at {base.rstrip('/')}/{key()}.txt)")
|
||
result = submit_all(base)
|
||
print(f"Submitted {result['submitted']}/{result['total']} URLs "
|
||
f"in {result['batches']} batch(es); statuses={result['statuses']}")
|