thermograph/tests/test_content.py

"""Tests for the crawlable SEO content pages: the city set, robots/sitemap, and
that the server-rendered pages contain the stats in the HTML (with the weather
layer faked, like test_api.py)."""
import re

import pytest
from fastapi.testclient import TestClient

import app as appmod
import cities
import climate
import grading

# BASE defaults to /thermograph in tests (THERMOGRAPH_BASE unset).
B = "/thermograph"
SLUG = "london-england-gb"   # present in cities.json


@pytest.fixture
def client(monkeypatch, history, recent):
    monkeypatch.setattr(climate, "load_cached_history", lambda cell: history.clone())
    monkeypatch.setattr(climate, "get_recent_forecast", lambda cell: recent.clone())
    monkeypatch.setattr(climate, "get_history", lambda cell: (history.clone(), {"cached": True}))
    return TestClient(appmod.app)


def test_city_set_slugs_unique_and_lookup():
    slugs = cities.all_slugs()
    assert len(slugs) == len(set(slugs)) >= 100
    assert cities.get(SLUG) is not None
    assert cities.get("does-not-exist") is None


def test_robots_txt(client):
    r = client.get(f"{B}/robots.txt")
    assert r.status_code == 200
    assert "Sitemap:" in r.text and "/sitemap.xml" in r.text
    assert "Disallow: /thermograph/api/" in r.text


def test_sitemap_lists_city_urls(client):
    r = client.get(f"{B}/sitemap.xml")
    assert r.status_code == 200
    assert "<urlset" in r.text
    assert f"/climate/{SLUG}</loc>" in r.text
    assert f"/climate/{SLUG}/july</loc>" in r.text
    assert f"/climate/{SLUG}/records</loc>" in r.text


def test_city_page_renders_stats_in_html(client):
    r = client.get(f"{B}/climate/{SLUG}")
    assert r.status_code == 200
    b = r.text
    assert "London" in b and "climate" in b.lower()
    assert 'rel="canonical"' in b and f"/climate/{SLUG}" in b
    assert '"@type":"Dataset"' in b
    assert "average temperatures by month" in b.lower()
    # a real number rendered (°F appears in the normals/today text)
    assert "°F" in b


def _jsonld_breadcrumbs(html: str) -> list[dict]:
    import json, re
    m = re.search(r'<script type="application/ld\+json">(.*?)</script>', html, re.S)
    assert m, "no JSON-LD block on page"
    graph = json.loads(m.group(1))["@graph"]
    return [n for n in graph if n["@type"] == "BreadcrumbList"]


@pytest.mark.parametrize("path", [f"/climate/{SLUG}", f"/climate/{SLUG}/records"])
def test_breadcrumb_jsonld_items_valid_for_google(client, path):
    # Google: every ListItem except the last must carry an ``item`` URL, so
    # unlinked intermediate crumbs (the country) must not appear in JSON-LD.
    (bc,) = _jsonld_breadcrumbs(client.get(B + path).text)
    els = bc["itemListElement"]
    assert [e["position"] for e in els] == list(range(1, len(els) + 1))
    for e in els[:-1]:
        assert e["item"].startswith("http"), f"crumb {e['name']!r} missing item"


def test_city_404(client):
    assert client.get(f"{B}/climate/nope-not-a-city").status_code == 404


def test_curated_events_have_valid_slugs():
    import city_events
    slugs = set(cities.all_slugs())
    assert len(city_events.EVENTS) >= 20
    assert all(s in slugs for s in city_events.EVENTS), \
        [s for s in city_events.EVENTS if s not in slugs]


def test_curated_event_renders(client):
    b = client.get(f"{B}/climate/new-orleans-louisiana-us").text
    assert "Notable weather in New Orleans" in b
    assert "Hurricane Katrina" in b


def test_uncurated_city_has_no_event_section(client):
    b = client.get(f"{B}/climate/shanghai-cn").text
    assert "city-event" not in b     # falls back to the flavor blurb


def test_city_travel_cta_prefills_compare(client):
    b = client.get(f"{B}/climate/{SLUG}").text
    assert "Thinking of visiting" in b
    assert "/compare#loc=" in b            # the city is pre-loaded on the compare page


def test_city_blurb_renders_with_attribution(client, monkeypatch):
    monkeypatch.setattr(cities, "flavor",
                        lambda slug: {"extract": "Testville is a lovely place to test.",
                                      "url": "https://en.wikipedia.org/wiki/Testville", "title": "Testville"})
    b = client.get(f"{B}/climate/{SLUG}").text
    assert "Testville is a lovely place to test." in b
    assert "via Wikipedia" in b            # CC BY-SA attribution link


def test_month_and_records(client):
    assert client.get(f"{B}/climate/{SLUG}/july").status_code == 200
    assert client.get(f"{B}/climate/{SLUG}/records").status_code == 200
    assert client.get(f"{B}/climate/{SLUG}/notamonth").status_code == 404


def test_month_records_cover_every_metric_high_and_low(client):
    # The month page's records show a record high AND low for each metric present, not
    # just warmest/coldest temperature — one card per metric with both extremes. (The
    # test fixture carries tmax/tmin/precip; real cities add feels/humidity/wind/gust.)
    b = client.get(f"{B}/climate/{SLUG}/july").text
    assert "July records" in b
    for label in ("High", "Low", "Precip"):
        assert f'class="rc-metric">{label}<' in b
    # Every card carries a labelled high and low row (temp metrics use Highest/Lowest).
    n = b.count('rc-row rc-high')
    assert n >= 3 and b.count('rc-row rc-low') == n
    assert "Highest" in b and "Lowest" in b
    # Precip's extremes are the wettest/driest whole month by total accumulation.
    assert "Wettest" in b and "Driest" in b


def test_records_page_has_monthly_and_seasonal(client):
    b = client.get(f"{B}/climate/{SLUG}/records").text
    # Monthly records: all 12 months, each linking to its month page.
    assert "Records by month" in b
    for month in ("January", "July", "December"):
        assert month in b
    assert f"/climate/{SLUG}/january" in b
    # Seasonal records: labelled for London's Northern Hemisphere, with month spans.
    assert "Records by season" in b
    assert "Northern Hemisphere" in b
    assert 'Winter <span class="season-span">(Dec–Feb)</span>' in b
    assert 'Summer <span class="season-span">(Jun–Aug)</span>' in b
    # All-time table still present, plus structured data and real °F values.
    assert "All-time records" in b
    assert '"@type":"Dataset"' in b and "°F" in b
    # Title/description advertise the new coverage.
    assert "monthly" in b.lower() and "seasonal" in b.lower()


def test_records_seasons_flip_in_southern_hemisphere(client):
    # São Paulo is south of the equator, so Dec–Feb is summer, not winter.
    b = client.get(f"{B}/climate/sao-paulo-br/records").text
    assert "Southern Hemisphere" in b
    assert 'Summer <span class="season-span">(Dec–Feb)</span>' in b
    assert 'Winter <span class="season-span">(Jun–Aug)</span>' in b


def test_climate_pages_are_colour_coded(client):
    # Temperatures wear the diverging cold→hot palette (a heat map, not a plain grid),
    # and the city page carries the monthly temperature-range strip.
    city = client.get(f"{B}/climate/{SLUG}").text
    assert "t-cell t-" in city              # tinted temperature cells in the normals table
    assert "range-fill" in city and "--c1:var(--" in city   # the monthly range strip's gradient bars
    recs = client.get(f"{B}/climate/{SLUG}/records").text
    assert "t-inline t-" in recs            # tinted record chips
    month = client.get(f"{B}/climate/{SLUG}/july").text
    assert "t-inline t-" in month           # tinted hero numbers + records


def test_records_show_both_extremes_per_metric(client):
    # Each metric column shows BOTH its record warmest (▲) and coldest (▼) — so the
    # daytime high carries its record-low high, and the overnight low its record-high low.
    b = client.get(f"{B}/climate/{SLUG}/records").text
    assert "Daytime high" in b and "Overnight low" in b
    assert "▲" in b and "▼" in b
    # 12 months × 2 metric columns × 2 extremes = 48 chips just in the monthly table.
    assert b.count('class="rec-ext"') >= 48


def test_hub_glossary_about(client):
    assert client.get(f"{B}/climate").status_code == 200
    assert client.get(f"{B}/glossary").status_code == 200
    assert client.get(f"{B}/glossary/percentile").status_code == 200
    assert client.get(f"{B}/glossary/not-a-term").status_code == 404
    assert client.get(f"{B}/about").status_code == 200


def test_title_name_is_short_and_unique():
    # Titles lead with the bare city name so the payload ("… in July") survives
    # SERP truncation — display_name()'s region+country is what pushed it off the
    # end. The exception is a name that recurs: those must still be told apart,
    # and every rendered title must stay unique across the whole city set.
    all_cities = cities.all_cities()
    titles = [cities.title_name(c) for c in all_cities]
    assert len(titles) == len(set(titles)), "two cities share a title"
    # The overwhelming majority pay no disambiguation cost at all.
    assert sum(1 for t in titles if "," not in t) > 0.9 * len(titles)

    by_slug = {c["slug"]: cities.title_name(c) for c in all_cities}
    assert by_slug["seattle-washington-us"] == "Seattle"
    # Same name, different countries -> country code is enough.
    assert by_slug["london-england-gb"] == "London, GB"
    assert by_slug["london-ontario-ca"] == "London, CA"
    # Same name AND same country -> the country code would collide, so fall back
    # to admin1. Without this, both Columbuses would render "Columbus, US".
    assert by_slug["columbus-ohio-us"] == "Columbus, Ohio"
    assert by_slug["columbus-georgia-us"] == "Columbus, Georgia"


def test_page_titles_lead_with_city_and_payload(client):
    for path, must_start, payload in (
        (f"{B}/climate/{SLUG}", "London, GB climate", "how unusual it is now"),
        (f"{B}/climate/{SLUG}/july", "London, GB in July", "normal weather"),
        (f"{B}/climate/{SLUG}/records", "London, GB weather records", "hottest & coldest"),
    ):
        b = client.get(path).text
        title = re.search(r"<title>(.*?)</title>", b, re.S).group(1)
        title = title.replace("&amp;", "&")
        assert title.startswith(must_start), title
        assert payload in title, title
        # The region+country the old template spent its width on is gone.
        assert "England, United Kingdom" not in title
        # Whatever else happens, the city and the page's subject are inside the
        # first ~40 chars, so a SERP cut can no longer remove the payload.
        assert len(must_start) <= 40
        # og:title mirrors <title> via base.html.j2's self.title().
        og = re.search(r'<meta property="og:title" content="(.*?)"', b, re.S).group(1)
        assert og == re.search(r"<title>(.*?)</title>", b, re.S).group(1)


def test_meta_descriptions_lead_with_real_numbers(client):
    for path in (f"{B}/climate/{SLUG}", f"{B}/climate/{SLUG}/july",
                 f"{B}/climate/{SLUG}/records"):
        b = client.get(path).text
        desc = re.search(r'<meta name="description" content="(.*?)"', b, re.S).group(1)
        assert len(desc) <= 155, f"{len(desc)}: {desc}"
        # A real temperature, not a generic "averages, records and rainfall".
        assert re.search(r"-?\d+°F", desc), desc
        assert desc.endswith("years of local history."), desc
    # The records description names the dates the extremes were set.
    rec = client.get(f"{B}/climate/{SLUG}/records").text
    desc = re.search(r'<meta name="description" content="(.*?)"', rec, re.S).group(1)
    assert re.search(r"\([A-Z][a-z]{2} \d{4}\)", desc), desc


def test_climate_pages_carry_convertible_temps(client):
    # Every temperature is a client-convertible span (default °F text so crawlers
    # and no-JS visitors still see a real value); the old inline "(NN°C)" dual form
    # is gone (the toggle now supplies Celsius).
    import re
    for path in (f"{B}/climate/{SLUG}", f"{B}/climate/{SLUG}/july", f"{B}/climate/{SLUG}/records"):
        b = client.get(path).text
        assert 'class="temp" data-temp-f=' in b
        assert "°F" in b
        assert re.search(r"\d+°F \(-?\d+°C\)", b) is None   # no leftover dual-unit strings


def test_seo_pages_load_unit_and_account_scripts(client):
    # Header parity with the interactive pages: the °F/°C toggle, account/bell, and
    # the temperature converter are all loaded.
    b = client.get(f"{B}/climate/{SLUG}").text
    for src in (f"{B}/units.js", f"{B}/account.js", f"{B}/climate.js"):
        assert f'src="{src}"' in b


def test_indexnow_key_file_served(client):
    import indexnow
    k = indexnow.key()
    r = client.get(f"{B}/{k}.txt")
    assert r.status_code == 200
    assert r.text.strip() == k
    assert r.headers["content-type"].startswith("text/plain")


def test_sitemap_lastmod_is_stable(client):
    import datetime
    import re
    body = client.get(f"{B}/sitemap.xml").text
    mods = set(re.findall(r"<lastmod>([^<]+)</lastmod>", body))
    assert len(mods) == 1                        # one build date, not a per-request churn
    datetime.date.fromisoformat(next(iter(mods)))   # a valid ISO date


def test_indexnow_submit_all_builds_payload(monkeypatch):
    import indexnow

    calls = []

    class _Resp:
        status_code = 200
        text = "ok"

    monkeypatch.setattr(indexnow.httpx, "post",
                        lambda url, json=None, **kw: (calls.append((url, json)), _Resp())[1])
    res = indexnow.submit_all("https://thermograph.org")
    assert res["submitted"] == res["total"] > 100
    first = calls[0][1]
    assert first["host"] == "thermograph.org"
    assert first["keyLocation"] == f"https://thermograph.org/{indexnow.key()}.txt"
    assert first["urlList"][0] == "https://thermograph.org/"
    assert all(len(c[1]["urlList"]) <= 10000 for c in calls)   # batched under the IndexNow cap


def test_indexnow_if_changed_state(monkeypatch, tmp_path):
    import indexnow
    monkeypatch.setattr(indexnow, "_STATE_PATH", str(tmp_path / "state.txt"))
    sig = indexnow.url_signature()
    assert len(sig) == 40 and sig == indexnow.url_signature()   # stable sha1 of the URL set
    assert indexnow._read_state() == ""                         # first deploy → would submit
    indexnow._write_state(sig)
    assert indexnow._read_state() == sig                        # unchanged → deploy would skip


def test_search_verification_meta(client, monkeypatch):
    monkeypatch.setenv("THERMOGRAPH_GOOGLE_VERIFY", "gtok123")
    monkeypatch.setenv("THERMOGRAPH_BING_VERIFY", "btok456")
    seo = client.get(f"{B}/climate/{SLUG}").text          # Jinja SEO page
    home = client.get(f"{B}/").text                        # static homepage via _page()
    for b in (seo, home):
        assert '<meta name="google-site-verification" content="gtok123">' in b
        assert '<meta name="msvalidate.01" content="btok456">' in b


# The brand lockup is the site-wide "go home" affordance. The static tool pages
# and the Jinja pages build their headers separately, so this is easy to add to
# one and forget on the other — assert every page carries it.
BRAND_PAGES = ["/", "/calendar", "/day", "/compare", "/legend", "/alerts",
               "/about", "/privacy", "/climate", "/glossary"]


@pytest.mark.parametrize("path", BRAND_PAGES)
def test_brand_lockup_links_home(client, path):
    html = client.get(f"{B}{path}").text
    brand = html.split('<div class="brand">', 1)[1].split("</header>", 1)[0]
    head = brand.split("</h1>")[0] if "<h1" in brand else brand.split("</p>")[0]
    # The link must wrap the whole lockup, so clicking the mark works too, not
    # just the word.
    assert "<a href=" in head, f"{path}: brand is not a link"
    href = head.split('<a href="', 1)[1].split('"', 1)[0]
    assert href in ("./", f"{B}/"), f"{path}: brand links to {href!r}, not home"
    assert '<span class="logo">' in head.split("<a href=", 1)[1], \
        f"{path}: the mark sits outside the home link"


# Percentiles are shown on the Day page, the calendar tooltip, the chart, the
# city pages and the homepage strip. They all have to say the same thing about
# the same reading, so the rule lives in grading.pct_ordinal() and every surface
# defers to it (the frontend mirrors it as pctOrd in shared.js).
def test_pct_ordinal_rule():
    assert grading.pct_ordinal(66.4) == "66th"
    assert grading.pct_ordinal(1.2) == "1st"
    assert grading.pct_ordinal(22) == "22nd"
    assert grading.pct_ordinal(3) == "3rd"
    assert grading.pct_ordinal(11) == "11th"
    # Floored into 1..99: an empirical percentile is a rank against the sample,
    # so "100th percentile" reads as a measurement error.
    assert grading.pct_ordinal(99.6) == "99th"
    assert grading.pct_ordinal(100) == "99th"
    assert grading.pct_ordinal(0) == "1st"
    assert grading.pct_ordinal(None) == "—"


def test_city_page_percentiles_are_real_ordinals(client):
    """The city page used to render `{{ c.percentile }}th pct` against a float,
    producing "97.1th pct" and "1th pct" on every one of the ~1000 city pages."""
    html = client.get(f"{B}/climate/{SLUG}").text
    assert "th pct" in html or "st pct" in html or "nd pct" in html or "rd pct" in html
    # No float ordinals, no impossible ones, no "1th"/"2th"/"3th".
    for bad in re.findall(r"\d+\.\d+(?:st|nd|rd|th)|100th|\b0th|\b\d*[123]th", html):
        raise AssertionError(f"malformed percentile ordinal: {bad!r}")