220 lines
9 KiB
Python
220 lines
9 KiB
Python
"""IndexNow — instantly notify Bing, DuckDuckGo, Yandex, Seznam and friends of new
|
||
or changed URLs so they (re)crawl within minutes instead of days.
|
||
|
||
Note: Google does NOT use IndexNow. For Google, submit the sitemap in Search
|
||
Console; this module covers everyone else (and DuckDuckGo, which is powered by
|
||
Bing's index).
|
||
|
||
A per-host key authorises submissions: it is served as a plain-text file at the
|
||
site root (``/{key}.txt``, wired up in content.register) and echoed in every
|
||
submission so the search engine can confirm we own the host. Key resolution order
|
||
mirrors push.py's VAPID handling: env var → gitignored file → generated once and
|
||
persisted.
|
||
|
||
Usage (run after a content rebuild, or once to seed a fresh site):
|
||
|
||
cd backend && python indexnow.py https://thermograph.org
|
||
# or: make indexnow URL=https://thermograph.org
|
||
"""
|
||
import hashlib
|
||
import logging
|
||
import os
|
||
import secrets
|
||
import threading
|
||
from urllib.parse import urlparse
|
||
|
||
import httpx
|
||
|
||
from api import sitemap
|
||
import paths
|
||
|
||
log = logging.getLogger("thermograph.indexnow")
|
||
|
||
_DATA_DIR = paths.DATA_DIR
|
||
_KEY_PATH = os.environ.get("THERMOGRAPH_INDEXNOW_KEY_FILE") or os.path.join(_DATA_DIR, "indexnow_key.txt")
|
||
_STATE_PATH = os.environ.get("THERMOGRAPH_INDEXNOW_STATE_FILE") or os.path.join(_DATA_DIR, "indexnow_state.txt")
|
||
_ENDPOINT = "https://api.indexnow.org/indexnow" # a shared endpoint; it fans out to all participants
|
||
_BATCH = 10000 # IndexNow's max URLs per request
|
||
|
||
# One shared client instead of a bare httpx.post per batch — same reused-client
|
||
# pattern as web/app.py's _frontend_client and notifications/discord.py's _client.
|
||
# 30s default matches submit()'s own prior per-call default; still overridable
|
||
# per call (submit_all's fan-out passes its own `timeout` through unchanged).
|
||
_client = httpx.Client(timeout=30)
|
||
|
||
_lock = threading.Lock()
|
||
_key = None
|
||
|
||
|
||
def _claim_file(value: str) -> str:
|
||
"""Persist a freshly generated key as `_KEY_PATH`'s content, atomic
|
||
first-writer-wins — same race, same fix as push.py's VAPID `_claim_file`
|
||
(see its docstring): on a cold /state volume several workers can hit the
|
||
missing-file branch at once, and without this each would generate and cache
|
||
its OWN key, diverging from the file and from each other (the key served at
|
||
/{key}.txt would then randomly mismatch whichever key a given worker signed
|
||
a submission with). Write to a private temp file, then claim the real path
|
||
with a hard link (atomic; a loser gets EEXIST with no half-written window);
|
||
a loser reads back the winner's file instead of keeping its own generation."""
|
||
tmp_path = f"{_KEY_PATH}.{os.getpid()}.tmp"
|
||
try:
|
||
os.makedirs(_DATA_DIR, exist_ok=True)
|
||
with open(tmp_path, "w", encoding="utf-8") as f:
|
||
f.write(value)
|
||
os.chmod(tmp_path, 0o600)
|
||
except OSError:
|
||
log.warning("could not persist IndexNow key to %s; using an in-memory key", _KEY_PATH)
|
||
return value
|
||
try:
|
||
try:
|
||
os.link(tmp_path, _KEY_PATH)
|
||
return value
|
||
except FileExistsError:
|
||
try:
|
||
with open(_KEY_PATH, encoding="utf-8") as f:
|
||
winner = f.read().strip()
|
||
if winner:
|
||
return winner
|
||
except OSError:
|
||
pass
|
||
return value
|
||
except OSError:
|
||
log.warning("could not persist IndexNow key to %s; using an in-memory key", _KEY_PATH)
|
||
return value
|
||
finally:
|
||
try:
|
||
os.unlink(tmp_path)
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def key() -> str:
|
||
"""The IndexNow key (env → file → generate-and-persist), cached for the process."""
|
||
global _key
|
||
if _key:
|
||
return _key
|
||
with _lock:
|
||
if _key:
|
||
return _key
|
||
env = os.environ.get("THERMOGRAPH_INDEXNOW_KEY", "").strip()
|
||
if env:
|
||
_key = env
|
||
return _key
|
||
try:
|
||
with open(_KEY_PATH, encoding="utf-8") as f:
|
||
k = f.read().strip()
|
||
if k:
|
||
_key = k
|
||
return _key
|
||
except OSError:
|
||
pass
|
||
# Cache whatever _claim_file settles on — see its docstring: our own
|
||
# generation if we won the race to create the file, the winner's key
|
||
# read back from disk if we lost it.
|
||
_key = _claim_file(secrets.token_hex(16)) # 32 hex chars — within IndexNow's 8–128 range
|
||
return _key
|
||
|
||
|
||
def submit(urls, host: str, key_location: str, scheme: str = "https", timeout: float = 30) -> dict:
|
||
"""POST ``urls`` (deduped, in batches of 10k) to IndexNow. Every URL must be on
|
||
``host``; ``key_location`` is the public URL of our key file. Returns a summary."""
|
||
urls = list(dict.fromkeys(urls))
|
||
if not urls:
|
||
return {"submitted": 0, "total": 0, "batches": 0}
|
||
k = key()
|
||
submitted, statuses = 0, []
|
||
for i in range(0, len(urls), _BATCH):
|
||
batch = urls[i:i + _BATCH]
|
||
payload = {"host": host, "key": k, "keyLocation": key_location, "urlList": batch}
|
||
try:
|
||
r = _client.post(_ENDPOINT, json=payload, timeout=timeout,
|
||
headers={"Content-Type": "application/json; charset=utf-8"})
|
||
except httpx.HTTPError as e:
|
||
log.warning("IndexNow request failed: %s", e)
|
||
statuses.append("error")
|
||
continue
|
||
statuses.append(r.status_code)
|
||
if r.status_code in (200, 202):
|
||
submitted += len(batch)
|
||
else:
|
||
log.warning("IndexNow batch rejected: %s %s", r.status_code, r.text[:200])
|
||
return {"submitted": submitted, "total": len(urls),
|
||
"batches": len(statuses), "statuses": statuses, "key_location": key_location}
|
||
|
||
|
||
def submit_all(site_base_url: str, **kw) -> dict:
|
||
"""Submit every indexable page. ``site_base_url`` is the public origin + base
|
||
path, e.g. ``https://thermograph.org`` (root) or ``https://host/thermograph``."""
|
||
site = site_base_url.rstrip("/")
|
||
parsed = urlparse(site)
|
||
urls = [site + path for path in sitemap.public_paths()]
|
||
return submit(urls, host=parsed.hostname, scheme=parsed.scheme or "https",
|
||
key_location=f"{site}/{key()}.txt", **kw)
|
||
|
||
|
||
def url_signature() -> str:
|
||
"""A stable fingerprint of the indexable URL set — changes only when pages are
|
||
added or removed (e.g. a new city), not on code-only deploys. Used to skip
|
||
resubmitting an unchanged set (see the --if-changed CLI flag)."""
|
||
return hashlib.sha1("\n".join(sitemap.public_paths()).encode()).hexdigest()
|
||
|
||
|
||
def _read_state() -> str:
|
||
try:
|
||
with open(_STATE_PATH, encoding="utf-8") as f:
|
||
return f.read().strip()
|
||
except OSError:
|
||
return ""
|
||
|
||
|
||
def _write_state(sig: str) -> None:
|
||
try:
|
||
os.makedirs(_DATA_DIR, exist_ok=True)
|
||
with open(_STATE_PATH, "w", encoding="utf-8") as f:
|
||
f.write(sig)
|
||
except OSError:
|
||
log.warning("could not persist IndexNow state to %s", _STATE_PATH)
|
||
|
||
|
||
def submit_if_changed(base_url: str | None = None) -> dict | None:
|
||
"""Submit every indexable page, but only when the URL set changed since the
|
||
last successful submission — the shared logic behind the CLI's --if-changed
|
||
flag and the worker scheduler's periodic ping (notifications/scheduler.py), so
|
||
a code-only deploy or an unremarkable scheduled tick never resubmits. Returns
|
||
the submit_all() result, or None when skipped (unchanged, or a partial/failed
|
||
submission that leaves the prior state in place so the next attempt retries)."""
|
||
base = base_url or os.environ.get("THERMOGRAPH_BASE_URL", "https://thermograph.org")
|
||
sig = url_signature()
|
||
if sig == _read_state():
|
||
return None
|
||
result = submit_all(base)
|
||
if result["total"] and result["submitted"] == result["total"]:
|
||
_write_state(sig) # only remember a fully-successful submission
|
||
return result
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import sys
|
||
|
||
# Usage: indexnow.py [--if-changed] [BASE_URL]
|
||
# --if-changed submit only when the URL set changed since the last run
|
||
# (used by the deploy hook so code-only deploys don't resubmit).
|
||
argv = sys.argv[1:]
|
||
if_changed = "--if-changed" in argv
|
||
positional = [a for a in argv if not a.startswith("-") and a.strip()]
|
||
base = (positional[0] if positional else "") or os.environ.get("THERMOGRAPH_BASE_URL", "https://thermograph.org")
|
||
logging.basicConfig(level=logging.INFO)
|
||
|
||
print(f"IndexNow key: {key()} (served at {base.rstrip('/')}/{key()}.txt)")
|
||
if if_changed:
|
||
result = submit_if_changed(base)
|
||
if result is None:
|
||
print("IndexNow: URL set unchanged since last submission — skipping.")
|
||
sys.exit(0)
|
||
else:
|
||
result = submit_all(base)
|
||
print(f"Submitted {result['submitted']}/{result['total']} URLs "
|
||
f"in {result['batches']} batch(es); statuses={result['statuses']}")
|
||
if result["total"] and result["submitted"] == result["total"] and not if_changed:
|
||
sig = url_signature()
|
||
_write_state(sig) # only remember a fully-successful submission
|