Repo-split Stage 1: sever web/'s reverse imports (#9)
This commit is contained in:
parent
7b591439c1
commit
241b87c7b3
19 changed files with 124 additions and 117 deletions
|
|
@ -17,7 +17,7 @@ from data import grid
|
||||||
from notifications import push
|
from notifications import push
|
||||||
from accounts.db import get_async_session, get_read_session
|
from accounts.db import get_async_session, get_read_session
|
||||||
from accounts.models import Notification, PushSubscription, Subscription
|
from accounts.models import Notification, PushSubscription, Subscription
|
||||||
from web.schemas import (
|
from accounts.schemas import (
|
||||||
NotificationList,
|
NotificationList,
|
||||||
NotificationOut,
|
NotificationOut,
|
||||||
PushSubscriptionIn,
|
PushSubscriptionIn,
|
||||||
|
|
|
||||||
0
api/__init__.py
Normal file
0
api/__init__.py
Normal file
|
|
@ -33,7 +33,7 @@ from data import grading
|
||||||
from data import grid
|
from data import grid
|
||||||
from data import store
|
from data import store
|
||||||
import paths
|
import paths
|
||||||
from web.views import OBS_COLS
|
from api.payloads import OBS_COLS
|
||||||
|
|
||||||
# data/ is the only writable path under the hardened systemd unit
|
# data/ is the only writable path under the hardened systemd unit
|
||||||
# (ReadWritePaths=/opt/thermograph/data …), so the feed lives there when it's a
|
# (ReadWritePaths=/opt/thermograph/data …), so the feed lives there when it's a
|
||||||
26
api/sitemap.py
Normal file
26
api/sitemap.py
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
"""The indexable-page URL list — shared by the sitemap, IndexNow submission, and
|
||||||
|
(eventually) the frontend's own sitemap.xml, so all three never drift apart.
|
||||||
|
"""
|
||||||
|
from data import cities
|
||||||
|
|
||||||
|
MONTHS = ["january", "february", "march", "april", "may", "june",
|
||||||
|
"july", "august", "september", "october", "november", "december"]
|
||||||
|
|
||||||
|
|
||||||
|
def sitemap_entries() -> list[tuple[str, str, str]]:
|
||||||
|
"""Every indexable page as (path, changefreq, priority). Paths are relative to
|
||||||
|
BASE."""
|
||||||
|
entries = [("/", "daily", "1.0")]
|
||||||
|
for p in ("/climate", "/about", "/glossary", "/calendar", "/compare", "/legend"):
|
||||||
|
entries.append((p, "weekly", "0.6"))
|
||||||
|
for slug in cities.all_slugs():
|
||||||
|
entries.append((f"/climate/{slug}", "daily", "0.8"))
|
||||||
|
entries.append((f"/climate/{slug}/records", "monthly", "0.5"))
|
||||||
|
for m in MONTHS:
|
||||||
|
entries.append((f"/climate/{slug}/{m}", "monthly", "0.5"))
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def public_paths() -> list[str]:
|
||||||
|
"""BASE-relative paths of every indexable page — for IndexNow submission."""
|
||||||
|
return [p for p, _, _ in sitemap_entries()]
|
||||||
|
|
@ -11,7 +11,7 @@ lives in two tables managed by Alembic (``backend/alembic/versions``):
|
||||||
forecast bundle (includes *future* dates), fully rewritten each refresh.
|
forecast bundle (includes *future* dates), fully rewritten each refresh.
|
||||||
* ``climate_sync`` — per-cell freshness (epoch seconds), replacing the parquet
|
* ``climate_sync`` — per-cell freshness (epoch seconds), replacing the parquet
|
||||||
file mtimes that used to drive the topup cadence, the forecast TTL, and the
|
file mtimes that used to drive the topup cadence, the forecast TTL, and the
|
||||||
``recent_stamp`` token embedded in derived-payload validity (see web/views.py).
|
``recent_stamp`` token embedded in derived-payload validity (see api/payloads.py).
|
||||||
|
|
||||||
This module is the Postgres backend only. ``climate.py`` keeps the parquet cache
|
This module is the Postgres backend only. ``climate.py`` keeps the parquet cache
|
||||||
as the backend when ``THERMOGRAPH_DATABASE_URL`` is *not* a Postgres URL (dev,
|
as the backend when ``THERMOGRAPH_DATABASE_URL`` is *not* a Postgres URL (dev,
|
||||||
|
|
@ -44,7 +44,7 @@ COLS = ("date", "tmax", "tmin", "precip", "wind", "gust", "humid", "fmax", "fmin
|
||||||
|
|
||||||
# Explicit polars schema so a frame read back from Postgres has the exact dtypes
|
# Explicit polars schema so a frame read back from Postgres has the exact dtypes
|
||||||
# the parquet path produced (pl.Date + Float64), even when the result is empty —
|
# the parquet path produced (pl.Date + Float64), even when the result is empty —
|
||||||
# so grading.py / scoring.py / web/views.py are untouched by the backend switch.
|
# so grading.py / scoring.py / api/payloads.py are untouched by the backend switch.
|
||||||
_SCHEMA: dict[str, pl.DataType] = {
|
_SCHEMA: dict[str, pl.DataType] = {
|
||||||
"date": pl.Date,
|
"date": pl.Date,
|
||||||
**{c: pl.Float64 for c in COLS if c != "date"},
|
**{c: pl.Float64 for c in COLS if c != "date"},
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ from urllib.parse import urlparse
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
|
from api import sitemap
|
||||||
import paths
|
import paths
|
||||||
|
|
||||||
log = logging.getLogger("thermograph.indexnow")
|
log = logging.getLogger("thermograph.indexnow")
|
||||||
|
|
@ -100,10 +101,9 @@ def submit(urls, host: str, key_location: str, scheme: str = "https", timeout: f
|
||||||
def submit_all(site_base_url: str, **kw) -> dict:
|
def submit_all(site_base_url: str, **kw) -> dict:
|
||||||
"""Submit every indexable page. ``site_base_url`` is the public origin + base
|
"""Submit every indexable page. ``site_base_url`` is the public origin + base
|
||||||
path, e.g. ``https://thermograph.org`` (root) or ``https://host/thermograph``."""
|
path, e.g. ``https://thermograph.org`` (root) or ``https://host/thermograph``."""
|
||||||
from web import content # lazy: avoids a content ↔ indexnow import cycle
|
|
||||||
site = site_base_url.rstrip("/")
|
site = site_base_url.rstrip("/")
|
||||||
parsed = urlparse(site)
|
parsed = urlparse(site)
|
||||||
urls = [site + path for path in content.public_paths()]
|
urls = [site + path for path in sitemap.public_paths()]
|
||||||
return submit(urls, host=parsed.hostname, scheme=parsed.scheme or "https",
|
return submit(urls, host=parsed.hostname, scheme=parsed.scheme or "https",
|
||||||
key_location=f"{site}/{key()}.txt", **kw)
|
key_location=f"{site}/{key()}.txt", **kw)
|
||||||
|
|
||||||
|
|
@ -112,8 +112,7 @@ def url_signature() -> str:
|
||||||
"""A stable fingerprint of the indexable URL set — changes only when pages are
|
"""A stable fingerprint of the indexable URL set — changes only when pages are
|
||||||
added or removed (e.g. a new city), not on code-only deploys. Used to skip
|
added or removed (e.g. a new city), not on code-only deploys. Used to skip
|
||||||
resubmitting an unchanged set (see the --if-changed CLI flag)."""
|
resubmitting an unchanged set (see the --if-changed CLI flag)."""
|
||||||
from web import content
|
return hashlib.sha1("\n".join(sitemap.public_paths()).encode()).hexdigest()
|
||||||
return hashlib.sha1("\n".join(content.public_paths()).encode()).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _read_state() -> str:
|
def _read_state() -> str:
|
||||||
|
|
|
||||||
14
migrate.py
14
migrate.py
|
|
@ -24,7 +24,7 @@ import time
|
||||||
from data import climate
|
from data import climate
|
||||||
from data import grid
|
from data import grid
|
||||||
from data import store
|
from data import store
|
||||||
from web import views
|
from api import payloads
|
||||||
|
|
||||||
|
|
||||||
def migrate() -> int:
|
def migrate() -> int:
|
||||||
|
|
@ -52,11 +52,11 @@ def migrate() -> int:
|
||||||
skipped += 1
|
skipped += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
token = views.history_token(history)
|
token = payloads.history_token(history)
|
||||||
start_ts, end_ts = views.cal_span(history, None, None, 24)
|
start_ts, end_ts = payloads.cal_span(history, None, None, 24)
|
||||||
cal_key = views.calendar_key(start_ts, end_ts, 24)
|
cal_key = payloads.calendar_key(start_ts, end_ts, 24)
|
||||||
last = history["date"].max()
|
last = history["date"].max()
|
||||||
day_key = views.day_key(last)
|
day_key = payloads.day_key(last)
|
||||||
|
|
||||||
have_cal = store.get_payload("calendar", cell_id, cal_key, token) is not None
|
have_cal = store.get_payload("calendar", cell_id, cal_key, token) is not None
|
||||||
have_day = store.get_payload("day", cell_id, day_key, token) is not None
|
have_day = store.get_payload("day", cell_id, day_key, token) is not None
|
||||||
|
|
@ -71,10 +71,10 @@ def migrate() -> int:
|
||||||
|
|
||||||
if not have_cal:
|
if not have_cal:
|
||||||
store.put_payload("calendar", cell_id, cal_key, token,
|
store.put_payload("calendar", cell_id, cal_key, token,
|
||||||
views.build_calendar(cell, history, start_ts, end_ts, 24, place))
|
payloads.build_calendar(cell, history, start_ts, end_ts, 24, place))
|
||||||
if not have_day:
|
if not have_day:
|
||||||
store.put_payload("day", cell_id, day_key, token,
|
store.put_payload("day", cell_id, day_key, token,
|
||||||
views.build_day(cell, history, last, place))
|
payloads.build_day(cell, history, last, place))
|
||||||
built += 1
|
built += 1
|
||||||
print(f" {cell_id}: materialized ({place or 'no label'})")
|
print(f" {cell_id}: materialized ({place or 'no label'})")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ import time
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from web import homepage
|
from api import homepage
|
||||||
|
|
||||||
# The webhook URL IS the credential — env only, never the repo. Unset => disabled.
|
# The webhook URL IS the credential — env only, never the repo. Unset => disabled.
|
||||||
WEBHOOK_URL = os.environ.get("THERMOGRAPH_DISCORD_WEBHOOK", "").strip()
|
WEBHOOK_URL = os.environ.get("THERMOGRAPH_DISCORD_WEBHOOK", "").strip()
|
||||||
|
|
|
||||||
|
|
@ -21,9 +21,9 @@ import os
|
||||||
from nacl.exceptions import BadSignatureError
|
from nacl.exceptions import BadSignatureError
|
||||||
from nacl.signing import VerifyKey
|
from nacl.signing import VerifyKey
|
||||||
|
|
||||||
|
from api import homepage
|
||||||
from data import cities
|
from data import cities
|
||||||
from notifications import discord
|
from notifications import discord
|
||||||
from web import homepage
|
|
||||||
|
|
||||||
# App's Ed25519 public key (Developer Portal -> General Information). Unset =>
|
# App's Ed25519 public key (Developer Portal -> General Information). Unset =>
|
||||||
# every request fails verification, which is the safe default for an unconfigured
|
# every request fails verification, which is the safe default for an unconfigured
|
||||||
|
|
|
||||||
|
|
@ -29,16 +29,16 @@ import time
|
||||||
import polars as pl
|
import polars as pl
|
||||||
from sqlalchemy import delete, select
|
from sqlalchemy import delete, select
|
||||||
|
|
||||||
|
from api import homepage
|
||||||
|
from api.payloads import OBS_COLS
|
||||||
from data import climate
|
from data import climate
|
||||||
from notifications import discord
|
from notifications import discord
|
||||||
from data import grading
|
from data import grading
|
||||||
from data import grid
|
from data import grid
|
||||||
from web import homepage
|
|
||||||
from core import metrics
|
from core import metrics
|
||||||
from notifications import push
|
from notifications import push
|
||||||
from accounts.db import sync_session_maker
|
from accounts.db import sync_session_maker
|
||||||
from accounts.models import AccessToken, Notification, PushSubscription, Subscription, User
|
from accounts.models import AccessToken, Notification, PushSubscription, Subscription, User
|
||||||
from web.views import OBS_COLS
|
|
||||||
|
|
||||||
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
|
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
|
||||||
BASE = f"/{_BASE}" if _BASE else ""
|
BASE = f"/{_BASE}" if _BASE else ""
|
||||||
|
|
|
||||||
|
|
@ -9,17 +9,17 @@ import polars as pl
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from data import climate
|
from data import climate
|
||||||
from web import views
|
from api import payloads
|
||||||
|
|
||||||
CELL = {"id": "1642_-4223", "center_lat": 47.6087, "center_lon": -122.29377}
|
CELL = {"id": "1642_-4223", "center_lat": 47.6087, "center_lon": -122.29377}
|
||||||
|
|
||||||
|
|
||||||
def test_views_and_migrate_import_without_the_web_stack():
|
def test_payloads_and_migrate_import_without_the_web_stack():
|
||||||
"""migrate.py must stay runnable offline: importing the payload layer may
|
"""migrate.py must stay runnable offline: importing the payload layer may
|
||||||
not construct the FastAPI app or start the places-index download."""
|
not construct the FastAPI app or start the places-index download."""
|
||||||
code = ("import sys; from web import views; import migrate; "
|
code = ("import sys; from api import payloads; import migrate; "
|
||||||
"assert 'fastapi' not in sys.modules, 'views/migrate pulled in FastAPI'; "
|
"assert 'fastapi' not in sys.modules, 'payloads/migrate pulled in FastAPI'; "
|
||||||
"assert 'web.app' not in sys.modules, 'views/migrate imported the web app'; "
|
"assert 'web.app' not in sys.modules, 'payloads/migrate imported the web app'; "
|
||||||
"from data import places; assert places._load_started is False")
|
"from data import places; assert places._load_started is False")
|
||||||
backend = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
backend = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
subprocess.run([sys.executable, "-c", code], cwd=backend, check=True)
|
subprocess.run([sys.executable, "-c", code], cwd=backend, check=True)
|
||||||
|
|
@ -32,23 +32,23 @@ def test_views_and_migrate_import_without_the_web_stack():
|
||||||
|
|
||||||
def test_cache_identity_formats_are_pinned(history):
|
def test_cache_identity_formats_are_pinned(history):
|
||||||
t = datetime.date(2026, 6, 15)
|
t = datetime.date(2026, 6, 15)
|
||||||
assert views.grade_key(t, 14, 7) == "2026-06-15:14:7"
|
assert payloads.grade_key(t, 14, 7) == "2026-06-15:14:7"
|
||||||
assert views.calendar_key(t, datetime.date(2026, 6, 20), 24) == "2026-06-15:2026-06-20:24"
|
assert payloads.calendar_key(t, datetime.date(2026, 6, 20), 24) == "2026-06-15:2026-06-20:24"
|
||||||
assert views.day_key(t) == "2026-06-15"
|
assert payloads.day_key(t) == "2026-06-15"
|
||||||
assert views.forecast_key(t, 7) == "2026-06-15:7"
|
assert payloads.forecast_key(t, 7) == "2026-06-15:7"
|
||||||
assert views.history_token(history) == f"{views.PAYLOAD_VER}:{views.hist_end(history)}"
|
assert payloads.history_token(history) == f"{payloads.PAYLOAD_VER}:{payloads.hist_end(history)}"
|
||||||
|
|
||||||
|
|
||||||
def test_recent_token_composes_history_and_stamp(history, monkeypatch):
|
def test_recent_token_composes_history_and_stamp(history, monkeypatch):
|
||||||
monkeypatch.setattr(climate, "recent_stamp", lambda cid: "stamp")
|
monkeypatch.setattr(climate, "recent_stamp", lambda cid: "stamp")
|
||||||
assert views.recent_token(history, "1_2") == f"{views.history_token(history)}:stamp"
|
assert payloads.recent_token(history, "1_2") == f"{payloads.history_token(history)}:stamp"
|
||||||
|
|
||||||
|
|
||||||
def test_day_token_expires_hourly_only_beyond_the_archive(history):
|
def test_day_token_expires_hourly_only_beyond_the_archive(history):
|
||||||
last = history["date"].max()
|
last = history["date"].max()
|
||||||
assert views.day_token(history, last) == views.history_token(history)
|
assert payloads.day_token(history, last) == payloads.history_token(history)
|
||||||
future = views.day_token(history, last + datetime.timedelta(days=1))
|
future = payloads.day_token(history, last + datetime.timedelta(days=1))
|
||||||
assert future.startswith(views.history_token(history) + ":h")
|
assert future.startswith(payloads.history_token(history) + ":h")
|
||||||
|
|
||||||
|
|
||||||
# ---- cal_span clamping ---------------------------------------------------------
|
# ---- cal_span clamping ---------------------------------------------------------
|
||||||
|
|
@ -60,27 +60,27 @@ def _hist(start, end):
|
||||||
|
|
||||||
|
|
||||||
def test_cal_span_defaults_to_months_back_same_day_of_month():
|
def test_cal_span_defaults_to_months_back_same_day_of_month():
|
||||||
start_ts, end_ts = views.cal_span(_hist("2020-01-01", "2026-06-29"), None, None, 24)
|
start_ts, end_ts = payloads.cal_span(_hist("2020-01-01", "2026-06-29"), None, None, 24)
|
||||||
assert end_ts == datetime.date(2026, 6, 29)
|
assert end_ts == datetime.date(2026, 6, 29)
|
||||||
assert start_ts == datetime.date(2024, 6, 29)
|
assert start_ts == datetime.date(2024, 6, 29)
|
||||||
|
|
||||||
|
|
||||||
def test_cal_span_clamps_to_the_record():
|
def test_cal_span_clamps_to_the_record():
|
||||||
h = _hist("2025-03-01", "2026-06-29") # record shorter than the 2-year cap
|
h = _hist("2025-03-01", "2026-06-29") # record shorter than the 2-year cap
|
||||||
start_ts, end_ts = views.cal_span(h, "2020-01-01", None, 24)
|
start_ts, end_ts = payloads.cal_span(h, "2020-01-01", None, 24)
|
||||||
assert start_ts == datetime.date(2025, 3, 1) # can't start before the record
|
assert start_ts == datetime.date(2025, 3, 1) # can't start before the record
|
||||||
_, end_ts = views.cal_span(h, None, "2030-01-01", 24)
|
_, end_ts = payloads.cal_span(h, None, "2030-01-01", 24)
|
||||||
assert end_ts == datetime.date(2026, 6, 29) # nor end past it
|
assert end_ts == datetime.date(2026, 6, 29) # nor end past it
|
||||||
|
|
||||||
|
|
||||||
def test_cal_span_caps_at_two_years():
|
def test_cal_span_caps_at_two_years():
|
||||||
start_ts, end_ts = views.cal_span(_hist("2018-01-01", "2026-06-29"),
|
start_ts, end_ts = payloads.cal_span(_hist("2018-01-01", "2026-06-29"),
|
||||||
"2018-01-01", "2026-06-29", 24)
|
"2018-01-01", "2026-06-29", 24)
|
||||||
assert (end_ts - start_ts).days == views.CAL_MAX_SPAN_DAYS - 1
|
assert (end_ts - start_ts).days == payloads.CAL_MAX_SPAN_DAYS - 1
|
||||||
|
|
||||||
|
|
||||||
def test_cal_span_never_inverts():
|
def test_cal_span_never_inverts():
|
||||||
start_ts, end_ts = views.cal_span(_hist("2020-01-01", "2026-06-29"),
|
start_ts, end_ts = payloads.cal_span(_hist("2020-01-01", "2026-06-29"),
|
||||||
"2026-06-01", "2021-01-01", 24)
|
"2026-06-01", "2021-01-01", 24)
|
||||||
assert start_ts == end_ts
|
assert start_ts == end_ts
|
||||||
|
|
||||||
|
|
@ -88,9 +88,9 @@ def test_cal_span_never_inverts():
|
||||||
def test_months_before_clamps_month_end_and_leap():
|
def test_months_before_clamps_month_end_and_leap():
|
||||||
"""Calendar-aware month subtraction (replaces pandas DateOffset): land on the
|
"""Calendar-aware month subtraction (replaces pandas DateOffset): land on the
|
||||||
same day-of-month, clamped to the target month's last valid day."""
|
same day-of-month, clamped to the target month's last valid day."""
|
||||||
assert views._months_before(datetime.date(2026, 3, 31), 1) == datetime.date(2026, 2, 28)
|
assert payloads._months_before(datetime.date(2026, 3, 31), 1) == datetime.date(2026, 2, 28)
|
||||||
assert views._months_before(datetime.date(2024, 3, 31), 1) == datetime.date(2024, 2, 29)
|
assert payloads._months_before(datetime.date(2024, 3, 31), 1) == datetime.date(2024, 2, 29)
|
||||||
assert views._months_before(datetime.date(2026, 1, 15), 14) == datetime.date(2024, 11, 15)
|
assert payloads._months_before(datetime.date(2026, 1, 15), 14) == datetime.date(2024, 11, 15)
|
||||||
|
|
||||||
|
|
||||||
def test_attach_dry_streaks_prefers_the_fresher_source_on_shared_dates():
|
def test_attach_dry_streaks_prefers_the_fresher_source_on_shared_dates():
|
||||||
|
|
@ -100,7 +100,7 @@ def test_attach_dry_streaks_prefers_the_fresher_source_on_shared_dates():
|
||||||
hist = pl.DataFrame({"date": [d], "precip": [1.0]}) # archive: it rained (streak resets)
|
hist = pl.DataFrame({"date": [d], "precip": [1.0]}) # archive: it rained (streak resets)
|
||||||
rec = pl.DataFrame({"date": [d], "precip": [0.0]}) # fresher: dry (streak counts)
|
rec = pl.DataFrame({"date": [d], "precip": [0.0]}) # fresher: dry (streak counts)
|
||||||
graded = [{"date": d.isoformat()}]
|
graded = [{"date": d.isoformat()}]
|
||||||
views._attach_dry_streaks(graded, hist, rec) # archive first, recent last
|
payloads._attach_dry_streaks(graded, hist, rec) # archive first, recent last
|
||||||
assert graded[0]["dsr"] == 1 # recent (dry) won
|
assert graded[0]["dsr"] == 1 # recent (dry) won
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -108,7 +108,7 @@ def test_attach_dry_streaks_prefers_the_fresher_source_on_shared_dates():
|
||||||
|
|
||||||
def test_build_grade_window_and_shape(history, recent):
|
def test_build_grade_window_and_shape(history, recent):
|
||||||
target = datetime.date.today()
|
target = datetime.date.today()
|
||||||
payload = views.build_grade(CELL, target, 14, history, recent,
|
payload = payloads.build_grade(CELL, target, 14, history, recent,
|
||||||
{"cached": True}, "Testville")
|
{"cached": True}, "Testville")
|
||||||
assert payload["target_date"] == target.isoformat()
|
assert payload["target_date"] == target.isoformat()
|
||||||
days = [d["date"] for d in payload["recent"]]
|
days = [d["date"] for d in payload["recent"]]
|
||||||
|
|
@ -119,7 +119,7 @@ def test_build_grade_window_and_shape(history, recent):
|
||||||
|
|
||||||
def test_build_day_pulls_future_obs_from_recent(history, recent):
|
def test_build_day_pulls_future_obs_from_recent(history, recent):
|
||||||
today = datetime.date.today()
|
today = datetime.date.today()
|
||||||
payload = views.build_day(CELL, history, today, "Testville", recent=recent)
|
payload = payloads.build_day(CELL, history, today, "Testville", recent=recent)
|
||||||
assert payload["detail"]["date"] == today.isoformat()
|
assert payload["detail"]["date"] == today.isoformat()
|
||||||
assert payload["detail"]["metrics"]["tmax"]["obs"] is not None
|
assert payload["detail"]["metrics"]["tmax"]["obs"] is not None
|
||||||
|
|
||||||
|
|
@ -129,14 +129,14 @@ def test_build_day_survives_recent_fetch_failure(history, monkeypatch):
|
||||||
raise RuntimeError("upstream down")
|
raise RuntimeError("upstream down")
|
||||||
monkeypatch.setattr(climate, "get_recent_forecast", boom)
|
monkeypatch.setattr(climate, "get_recent_forecast", boom)
|
||||||
today = datetime.date.today()
|
today = datetime.date.today()
|
||||||
payload = views.build_day(CELL, history, today, "Testville")
|
payload = payloads.build_day(CELL, history, today, "Testville")
|
||||||
assert payload["detail"]["metrics"]["tmax"]["obs"] is None # climatology only
|
assert payload["detail"]["metrics"]["tmax"]["obs"] is None # climatology only
|
||||||
assert payload["detail"]["metrics"]["tmax"]["ladder"] is not None
|
assert payload["detail"]["metrics"]["tmax"]["ladder"] is not None
|
||||||
|
|
||||||
|
|
||||||
def test_build_forecast_only_future_days(history, recent):
|
def test_build_forecast_only_future_days(history, recent):
|
||||||
today = datetime.date.today()
|
today = datetime.date.today()
|
||||||
payload = views.build_forecast(CELL, 7, history, recent, today, None)
|
payload = payloads.build_forecast(CELL, 7, history, recent, today, None)
|
||||||
days = [d["date"] for d in payload["recent"]]
|
days = [d["date"] for d in payload["recent"]]
|
||||||
assert days == sorted(days, reverse=True) # furthest-out first
|
assert days == sorted(days, reverse=True) # furthest-out first
|
||||||
assert min(days) > today.isoformat()
|
assert min(days) > today.isoformat()
|
||||||
|
|
@ -167,10 +167,10 @@ def _full_history(years=45, seed=5):
|
||||||
|
|
||||||
def test_build_score_shape():
|
def test_build_score_shape():
|
||||||
hist = _full_history()
|
hist = _full_history()
|
||||||
payload = views.build_score(CELL, hist, "Testville")
|
payload = payloads.build_score(CELL, hist, "Testville")
|
||||||
assert payload["api_version"] == "v2"
|
assert payload["api_version"] == "v2"
|
||||||
assert payload["cell"] == CELL and payload["place"] == "Testville"
|
assert payload["cell"] == CELL and payload["place"] == "Testville"
|
||||||
assert payload["latest"] == views.hist_end(hist)
|
assert payload["latest"] == payloads.hist_end(hist)
|
||||||
s = payload["scores"]
|
s = payload["scores"]
|
||||||
assert set(s["slices"]) == {"annual", "djf", "mam", "jja", "son"}
|
assert set(s["slices"]) == {"annual", "djf", "mam", "jja", "son"}
|
||||||
ann = s["slices"]["annual"]
|
ann = s["slices"]["annual"]
|
||||||
|
|
@ -180,7 +180,7 @@ def test_build_score_shape():
|
||||||
|
|
||||||
|
|
||||||
def test_score_key_is_the_version():
|
def test_score_key_is_the_version():
|
||||||
assert views.score_key() == views.SCORE_VER
|
assert payloads.score_key() == payloads.SCORE_VER
|
||||||
# Score payloads are history-only, so they ride the plain history token.
|
# Score payloads are history-only, so they ride the plain history token.
|
||||||
hist = _full_history()
|
hist = _full_history()
|
||||||
assert views.history_token(hist) == f"{views.PAYLOAD_VER}:{views.hist_end(hist)}"
|
assert payloads.history_token(hist) == f"{payloads.PAYLOAD_VER}:{payloads.hist_end(hist)}"
|
||||||
|
|
@ -8,9 +8,9 @@ import pytest
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from nacl.signing import SigningKey
|
from nacl.signing import SigningKey
|
||||||
|
|
||||||
|
from api import homepage
|
||||||
from web import app as appmod
|
from web import app as appmod
|
||||||
from notifications import discord_interactions as di
|
from notifications import discord_interactions as di
|
||||||
from web import homepage
|
|
||||||
|
|
||||||
B = "/thermograph"
|
B = "/thermograph"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -212,15 +212,15 @@ def test_cell_neighbors_flag_enqueues_warming(client, monkeypatch):
|
||||||
def test_warm_cell_materializes_history_slices(client, tmp_store, monkeypatch, history):
|
def test_warm_cell_materializes_history_slices(client, tmp_store, monkeypatch, history):
|
||||||
from web import app as appmod
|
from web import app as appmod
|
||||||
from data import grid
|
from data import grid
|
||||||
from web import views
|
from api import payloads
|
||||||
cell = grid.snap(-33.87, 151.21)
|
cell = grid.snap(-33.87, 151.21)
|
||||||
appmod._warm_cell(cell)
|
appmod._warm_cell(cell)
|
||||||
token = views.history_token(history)
|
token = payloads.history_token(history)
|
||||||
start_ts, end_ts = views.cal_span(history, None, None, 24)
|
start_ts, end_ts = payloads.cal_span(history, None, None, 24)
|
||||||
assert tmp_store.get_payload("calendar", cell["id"],
|
assert tmp_store.get_payload("calendar", cell["id"],
|
||||||
views.calendar_key(start_ts, end_ts, 24), token) is not None
|
payloads.calendar_key(start_ts, end_ts, 24), token) is not None
|
||||||
last = history["date"].max()
|
last = history["date"].max()
|
||||||
assert tmp_store.get_payload("day", cell["id"], views.day_key(last), token) is not None
|
assert tmp_store.get_payload("day", cell["id"], payloads.day_key(last), token) is not None
|
||||||
|
|
||||||
|
|
||||||
def test_warm_cell_never_fetches_upstream(client, monkeypatch):
|
def test_warm_cell_never_fetches_upstream(client, monkeypatch):
|
||||||
|
|
|
||||||
|
|
@ -13,12 +13,12 @@ import time
|
||||||
import pytest
|
import pytest
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from api import homepage
|
||||||
from web import app as appmod
|
from web import app as appmod
|
||||||
from data import cities
|
from data import cities
|
||||||
from data import climate
|
from data import climate
|
||||||
from data import store
|
from data import store
|
||||||
from web import content
|
from web import content
|
||||||
from web import homepage
|
|
||||||
|
|
||||||
B = "/thermograph"
|
B = "/thermograph"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -12,10 +12,10 @@ so this is an optimization, not a hard dependency.
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
from api import homepage
|
||||||
from data import cities
|
from data import cities
|
||||||
from data import climate
|
from data import climate
|
||||||
from data import grid
|
from data import grid
|
||||||
from web import homepage
|
|
||||||
|
|
||||||
|
|
||||||
def main(limit: int | None = None, pace: float = 2.0) -> None:
|
def main(limit: int | None = None, pace: float = 2.0) -> None:
|
||||||
|
|
|
||||||
72
web/app.py
72
web/app.py
|
|
@ -33,8 +33,8 @@ from data import places
|
||||||
from core import singleton
|
from core import singleton
|
||||||
from data import store
|
from data import store
|
||||||
from accounts import users
|
from accounts import users
|
||||||
from web import views
|
from api import payloads
|
||||||
from web.schemas import UserCreate, UserRead, UserUpdate
|
from accounts.schemas import UserCreate, UserRead, UserUpdate
|
||||||
|
|
||||||
FRONTEND_DIR = paths.FRONTEND_DIR
|
FRONTEND_DIR = paths.FRONTEND_DIR
|
||||||
|
|
||||||
|
|
@ -95,17 +95,17 @@ def _warm_cell(cell: dict) -> None:
|
||||||
if history is None or history.is_empty():
|
if history is None or history.is_empty():
|
||||||
return
|
return
|
||||||
place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"])
|
place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"])
|
||||||
token = views.history_token(history)
|
token = payloads.history_token(history)
|
||||||
start_ts, end_ts = views.cal_span(history, None, None, 24)
|
start_ts, end_ts = payloads.cal_span(history, None, None, 24)
|
||||||
cal_key = views.calendar_key(start_ts, end_ts, 24)
|
cal_key = payloads.calendar_key(start_ts, end_ts, 24)
|
||||||
if store.get_payload("calendar", cell["id"], cal_key, token) is None:
|
if store.get_payload("calendar", cell["id"], cal_key, token) is None:
|
||||||
store.put_payload("calendar", cell["id"], cal_key, token,
|
store.put_payload("calendar", cell["id"], cal_key, token,
|
||||||
views.build_calendar(cell, history, start_ts, end_ts, 24, place))
|
payloads.build_calendar(cell, history, start_ts, end_ts, 24, place))
|
||||||
last = history["date"].max()
|
last = history["date"].max()
|
||||||
day_key = views.day_key(last)
|
day_key = payloads.day_key(last)
|
||||||
if store.get_payload("day", cell["id"], day_key, token) is None:
|
if store.get_payload("day", cell["id"], day_key, token) is None:
|
||||||
store.put_payload("day", cell["id"], day_key, token,
|
store.put_payload("day", cell["id"], day_key, token,
|
||||||
views.build_day(cell, history, last, place))
|
payloads.build_day(cell, history, last, place))
|
||||||
|
|
||||||
|
|
||||||
def _enqueue_neighbor_warming(cell: dict) -> None:
|
def _enqueue_neighbor_warming(cell: dict) -> None:
|
||||||
|
|
@ -230,8 +230,8 @@ async def revalidate_static(request, call_next):
|
||||||
# from what those return, so a cached payload expires exactly when its inputs
|
# from what those return, so a cached payload expires exactly when its inputs
|
||||||
# change and never before. The same tokens double as ETags: a client sending
|
# change and never before. The same tokens double as ETags: a client sending
|
||||||
# If-None-Match gets an empty 304 without the payload even being loaded.
|
# If-None-Match gets an empty 304 without the payload even being loaded.
|
||||||
# The payloads themselves are assembled in views.py, shared with the offline
|
# The payloads themselves are assembled in api/payloads.py, shared with the
|
||||||
# migrate script.
|
# offline migrate script.
|
||||||
|
|
||||||
def _etag_for(kind: str, cell_id: str, key: str, token: str) -> str:
|
def _etag_for(kind: str, cell_id: str, key: str, token: str) -> str:
|
||||||
"""Deterministic weak ETag from a payload's identity + validity token. Weak
|
"""Deterministic weak ETag from a payload's identity + validity token. Weak
|
||||||
|
|
@ -379,8 +379,8 @@ def api_grade(
|
||||||
history, cache_meta, recent = _fetch_history(run, cell, recent_too=True)
|
history, cache_meta, recent = _fetch_history(run, cell, recent_too=True)
|
||||||
return _cached_response(
|
return _cached_response(
|
||||||
request, run, "grade", cell,
|
request, run, "grade", cell,
|
||||||
views.grade_key(target, days, after), views.recent_token(history, cell["id"]),
|
payloads.grade_key(target, days, after), payloads.recent_token(history, cell["id"]),
|
||||||
lambda place: views.build_grade(cell, target, days, history, recent,
|
lambda place: payloads.build_grade(cell, target, days, history, recent,
|
||||||
cache_meta, place, run, after=after))
|
cache_meta, place, run, after=after))
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -412,10 +412,10 @@ def api_calendar(
|
||||||
cell_id=cell["id"],
|
cell_id=cell["id"],
|
||||||
) as run:
|
) as run:
|
||||||
history, cache_meta, _ = _fetch_history(run, cell)
|
history, cache_meta, _ = _fetch_history(run, cell)
|
||||||
start_ts, end_ts = views.cal_span(history, start, end, months)
|
start_ts, end_ts = payloads.cal_span(history, start, end, months)
|
||||||
|
|
||||||
def build(place):
|
def build(place):
|
||||||
payload = views.build_calendar(cell, history, start_ts, end_ts, months, place, run)
|
payload = payloads.build_calendar(cell, history, start_ts, end_ts, months, place, run)
|
||||||
full = not cache_meta.get("cached", False)
|
full = not cache_meta.get("cached", False)
|
||||||
run.set(run_type="full" if full else "partial",
|
run.set(run_type="full" if full else "partial",
|
||||||
history_source="fetch" if full else "cache")
|
history_source="fetch" if full else "cache")
|
||||||
|
|
@ -424,8 +424,8 @@ def api_calendar(
|
||||||
# Compare loads several cells at once, so a transient reverse-geocode miss
|
# Compare loads several cells at once, so a transient reverse-geocode miss
|
||||||
# is most likely here; cache_placeless=False keeps that miss un-persisted.
|
# is most likely here; cache_placeless=False keeps that miss un-persisted.
|
||||||
return _cached_response(request, run, "calendar", cell,
|
return _cached_response(request, run, "calendar", cell,
|
||||||
views.calendar_key(start_ts, end_ts, months),
|
payloads.calendar_key(start_ts, end_ts, months),
|
||||||
views.history_token(history), build,
|
payloads.history_token(history), build,
|
||||||
cache_placeless=False)
|
cache_placeless=False)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -457,14 +457,14 @@ def api_day(
|
||||||
run.set(target_date=target.isoformat())
|
run.set(target_date=target.isoformat())
|
||||||
|
|
||||||
def build(place):
|
def build(place):
|
||||||
payload = views.build_day(cell, history, target, place, run)
|
payload = payloads.build_day(cell, history, target, place, run)
|
||||||
full = not cache_meta.get("cached", False)
|
full = not cache_meta.get("cached", False)
|
||||||
run.set(run_type="full" if full else "partial",
|
run.set(run_type="full" if full else "partial",
|
||||||
history_source="fetch" if full else "cache")
|
history_source="fetch" if full else "cache")
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
return _cached_response(request, run, "day", cell,
|
return _cached_response(request, run, "day", cell,
|
||||||
views.day_key(target), views.day_token(history, target),
|
payloads.day_key(target), payloads.day_token(history, target),
|
||||||
build)
|
build)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -488,14 +488,14 @@ def api_score(
|
||||||
history, cache_meta, _ = _fetch_history(run, cell)
|
history, cache_meta, _ = _fetch_history(run, cell)
|
||||||
|
|
||||||
def build(place):
|
def build(place):
|
||||||
payload = views.build_score(cell, history, place, run)
|
payload = payloads.build_score(cell, history, place, run)
|
||||||
full = not cache_meta.get("cached", False)
|
full = not cache_meta.get("cached", False)
|
||||||
run.set(run_type="full" if full else "partial",
|
run.set(run_type="full" if full else "partial",
|
||||||
history_source="fetch" if full else "cache")
|
history_source="fetch" if full else "cache")
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
return _cached_response(request, run, "score", cell,
|
return _cached_response(request, run, "score", cell,
|
||||||
views.score_key(), views.history_token(history), build)
|
payloads.score_key(), payloads.history_token(history), build)
|
||||||
|
|
||||||
|
|
||||||
def api_forecast(
|
def api_forecast(
|
||||||
|
|
@ -520,13 +520,13 @@ def api_forecast(
|
||||||
today = datetime.date.today()
|
today = datetime.date.today()
|
||||||
|
|
||||||
def build(place):
|
def build(place):
|
||||||
payload = views.build_forecast(cell, days, history, fc, today, place, run)
|
payload = payloads.build_forecast(cell, days, history, fc, today, place, run)
|
||||||
run.set(run_type="partial")
|
run.set(run_type="partial")
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
return _cached_response(request, run, "forecast", cell,
|
return _cached_response(request, run, "forecast", cell,
|
||||||
views.forecast_key(today, days),
|
payloads.forecast_key(today, days),
|
||||||
views.recent_token(history, cell["id"]), build)
|
payloads.recent_token(history, cell["id"]), build)
|
||||||
|
|
||||||
|
|
||||||
def api_cell(
|
def api_cell(
|
||||||
|
|
@ -590,31 +590,31 @@ def api_cell(
|
||||||
return {"etag": etag, "data": data}
|
return {"etag": etag, "data": data}
|
||||||
|
|
||||||
slices = {}
|
slices = {}
|
||||||
hist_token = views.history_token(history)
|
hist_token = payloads.history_token(history)
|
||||||
# Calendar: the default last-24-months span — what the calendar view (and
|
# Calendar: the default last-24-months span — what the calendar view (and
|
||||||
# the cross-view prefetch) requests first.
|
# the cross-view prefetch) requests first.
|
||||||
start_ts, end_ts = views.cal_span(history, None, None, 24)
|
start_ts, end_ts = payloads.cal_span(history, None, None, 24)
|
||||||
slices["calendar"] = slice_for(
|
slices["calendar"] = slice_for(
|
||||||
"calendar", views.calendar_key(start_ts, end_ts, 24), hist_token,
|
"calendar", payloads.calendar_key(start_ts, end_ts, 24), hist_token,
|
||||||
lambda: views.build_calendar(cell, history, start_ts, end_ts, 24, place, run))
|
lambda: payloads.build_calendar(cell, history, start_ts, end_ts, 24, place, run))
|
||||||
|
|
||||||
if prefetch:
|
if prefetch:
|
||||||
# Latest archived day: its observation comes from history alone, so
|
# Latest archived day: its observation comes from history alone, so
|
||||||
# it's buildable without the (skipped) recent bundle.
|
# it's buildable without the (skipped) recent bundle.
|
||||||
slices["day"] = slice_for(
|
slices["day"] = slice_for(
|
||||||
"day", views.day_key(last), hist_token,
|
"day", payloads.day_key(last), hist_token,
|
||||||
lambda: views.build_day(cell, history, last, place, run))
|
lambda: payloads.build_day(cell, history, last, place, run))
|
||||||
else:
|
else:
|
||||||
rf_token = views.recent_token(history, cid)
|
rf_token = payloads.recent_token(history, cid)
|
||||||
slices["grade"] = slice_for(
|
slices["grade"] = slice_for(
|
||||||
"grade", views.grade_key(today, 14, 14), rf_token,
|
"grade", payloads.grade_key(today, 14, 14), rf_token,
|
||||||
lambda: views.build_grade(cell, today, 14, history, recent, cache_meta, place, run, after=14))
|
lambda: payloads.build_grade(cell, today, 14, history, recent, cache_meta, place, run, after=14))
|
||||||
slices["forecast"] = slice_for(
|
slices["forecast"] = slice_for(
|
||||||
"forecast", views.forecast_key(today, 7), rf_token,
|
"forecast", payloads.forecast_key(today, 7), rf_token,
|
||||||
lambda: views.build_forecast(cell, 7, history, recent, today, place, run))
|
lambda: payloads.build_forecast(cell, 7, history, recent, today, place, run))
|
||||||
slices["day"] = slice_for(
|
slices["day"] = slice_for(
|
||||||
"day", views.day_key(today), views.day_token(history, today),
|
"day", payloads.day_key(today), payloads.day_token(history, today),
|
||||||
lambda: views.build_day(cell, history, today, place, run, recent=recent))
|
lambda: payloads.build_day(cell, history, today, place, run, recent=recent))
|
||||||
|
|
||||||
# The bundle's identity is the combination of its slices' identities.
|
# The bundle's identity is the combination of its slices' identities.
|
||||||
etag = _etag_for("cell", cid, "bundle" + (":p" if prefetch else ""),
|
etag = _etag_for("cell", cid, "bundle" + (":p" if prefetch else ""),
|
||||||
|
|
|
||||||
|
|
@ -20,15 +20,16 @@ from fastapi.responses import PlainTextResponse
|
||||||
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
||||||
from markupsafe import Markup
|
from markupsafe import Markup
|
||||||
|
|
||||||
|
from api import homepage
|
||||||
|
from api import sitemap
|
||||||
|
from api.payloads import OBS_COLS
|
||||||
from data import cities
|
from data import cities
|
||||||
from data import city_events
|
from data import city_events
|
||||||
from data import climate
|
from data import climate
|
||||||
from data import grading
|
from data import grading
|
||||||
from data import grid
|
from data import grid
|
||||||
from web import content_loader
|
from web import content_loader
|
||||||
from web import homepage
|
|
||||||
import paths
|
import paths
|
||||||
from web.views import OBS_COLS
|
|
||||||
|
|
||||||
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
|
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
|
||||||
BASE = f"/{_BASE}" if _BASE else ""
|
BASE = f"/{_BASE}" if _BASE else ""
|
||||||
|
|
@ -336,25 +337,6 @@ def robots_txt(request: Request) -> Response:
|
||||||
return PlainTextResponse(body)
|
return PlainTextResponse(body)
|
||||||
|
|
||||||
|
|
||||||
def _sitemap_entries() -> list[tuple[str, str, str]]:
|
|
||||||
"""Every indexable page as (path, changefreq, priority). Paths are relative to
|
|
||||||
BASE. Shared by the sitemap and IndexNow so the two never drift."""
|
|
||||||
entries = [("/", "daily", "1.0")]
|
|
||||||
for p in ("/climate", "/about", "/glossary", "/calendar", "/compare", "/legend"):
|
|
||||||
entries.append((p, "weekly", "0.6"))
|
|
||||||
for slug in cities.all_slugs():
|
|
||||||
entries.append((f"/climate/{slug}", "daily", "0.8"))
|
|
||||||
entries.append((f"/climate/{slug}/records", "monthly", "0.5"))
|
|
||||||
for m in MONTHS:
|
|
||||||
entries.append((f"/climate/{slug}/{m}", "monthly", "0.5"))
|
|
||||||
return entries
|
|
||||||
|
|
||||||
|
|
||||||
def public_paths() -> list[str]:
|
|
||||||
"""BASE-relative paths of every indexable page — for IndexNow submission."""
|
|
||||||
return [p for p, _, _ in _sitemap_entries()]
|
|
||||||
|
|
||||||
|
|
||||||
@functools.lru_cache(maxsize=1)
|
@functools.lru_cache(maxsize=1)
|
||||||
def _content_lastmod() -> str:
|
def _content_lastmod() -> str:
|
||||||
"""A stable 'content last built' date for <lastmod> — the newest mtime of the
|
"""A stable 'content last built' date for <lastmod> — the newest mtime of the
|
||||||
|
|
@ -372,7 +354,7 @@ def sitemap_xml(request: Request) -> Response:
|
||||||
lastmod = _content_lastmod()
|
lastmod = _content_lastmod()
|
||||||
parts = ['<?xml version="1.0" encoding="UTF-8"?>',
|
parts = ['<?xml version="1.0" encoding="UTF-8"?>',
|
||||||
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">']
|
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">']
|
||||||
for path, cf, pr in _sitemap_entries():
|
for path, cf, pr in sitemap.sitemap_entries():
|
||||||
parts.append(
|
parts.append(
|
||||||
f"<url><loc>{base_url}{path}</loc><lastmod>{lastmod}</lastmod>"
|
f"<url><loc>{base_url}{path}</loc><lastmod>{lastmod}</lastmod>"
|
||||||
f"<changefreq>{cf}</changefreq><priority>{pr}</priority></url>"
|
f"<changefreq>{cf}</changefreq><priority>{pr}</priority></url>"
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue