"""Backfill sports_teams and match_level from Wikipedia.

The two factors that decide whether a broadcast is a blockbuster -- who played
and what stage it was -- are missing from 60%/70% of the real-match rows in
global_sports, and absent entirely from a raw EPG. They are, however, public
knowledge: a tournament's Wikipedia page lists every fixture with its date,
teams and stage.

Flow, mirroring the one proven in crystalcast/index.php:

    suggest_url(event, season)   -> a best-guess Wikipedia page, pre-filled for
                                    the analyst rather than asked for cold
    fetch_wikitext(url)          -> ?action=raw, not rendered HTML
    compact_wikitext(text)       -> keep section headers + match-template lines
    parse_fixtures(client, ...)  -> Claude turns that into structured fixtures

Nothing is committed without a human looking at it: the caller previews the
fixtures, edits them, and only then confirms. Results are written to the
`crystal` application database -- NEVER to tvviewers, which stays read-only.

The same fixtures serve both directions: historical rows (so the engine can
learn what a marquee fixture is worth) and future schedule rows (so it can
recognise one). Enriching only the forecast side would give the model the
question without ever having taught it the answer.
"""
from __future__ import annotations

import json
import os
import re
import time
import urllib.parse
import urllib.request

WIKI_API = "https://en.wikipedia.org/w/api.php"
UA = "GSIQ-Crystal2/1.0 (broadcast audience estimation; contact denson.joseph@gmail.com)"

# Claude's parse target. Forcing the shape here is what makes the output usable
# without any downstream repair.
FIXTURE_SCHEMA = {
    "type": "object",
    "properties": {
        "event_label": {"type": "string",
                        "description": "The tournament as the page names it, e.g. "
                                       "'2026 ICC Men's T20 World Cup'"},
        "fixtures": {
            "type": "array",
            "items": {
                "type": "object",
                "properties": {
                    "match_date": {"type": "string",
                                   "description": "ISO date YYYY-MM-DD, or empty if unknown"},
                    "match_time": {"type": "string",
                                   "description": "Local start time HH:MM if the page gives one, else empty"},
                    "team1": {"type": "string"},
                    "team2": {"type": "string"},
                    "stage": {"type": "string",
                              "description": "One of GROUP, LEAGUE, QUALIFIER, ROUND_OF_32, "
                                             "ROUND_OF_16, QUARTER_FINAL, SEMI_FINAL, FINAL, "
                                             "THIRD_PLACE, SUPER_8, SUPER_4, PLAYOFF, "
                                             "ELIMINATOR, REGULAR, WARMUP, or UNKNOWN"},
                    "match_no": {"type": "string", "description": "Match number if given, else empty"},
                },
                "required": ["match_date", "match_time", "team1", "team2", "stage", "match_no"],
                "additionalProperties": False,
            },
        },
    },
    "required": ["event_label", "fixtures"],
    "additionalProperties": False,
}

PROMPT = """Below is the compacted wikitext of a Wikipedia page for a sports tournament.

Extract every individual match/fixture you can find. For each one give:
- match_date  : the calendar date it was played (YYYY-MM-DD). Use the season year \
{season} if the wikitext gives only a day and month. Empty string if genuinely absent.
- match_time  : the local start time as HH:MM if the page states one, else empty string.
- team1, team2: the two competing teams, as full names (not abbreviations) where possible.
- stage       : which round of the tournament, using the controlled vocabulary in the schema.
- match_no    : the match number if the page numbers them, else empty string.

Also give event_label: the tournament's own name as this page states it.

Rules:
- Only real fixtures between two teams. Skip standings tables, squad lists, venue
  lists, broadcast-rights tables, and prose.
- Do not invent fixtures. If the page has none, return an empty list.
- A match listed with a date range or "TBD" gets an empty match_date rather than a guess.

Event as our EPG names it: {event}
Season: {season}

WIKITEXT:
{wikitext}
"""


# --------------------------------------------------------------- fetching
def _get(url: str, timeout: int = 30) -> str:
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read().decode("utf-8", errors="replace")


def find_page(event: str, season: int | None) -> str | None:
    """Search Wikipedia for the tournament's page title."""
    q = f"{season} {event}" if season else event
    url = (f"{WIKI_API}?action=query&list=search&srsearch="
           f"{urllib.parse.quote(q)}&srlimit=5&format=json")
    try:
        hits = json.loads(_get(url)).get("query", {}).get("search", [])
    except Exception:
        return None
    if not hits:
        return None
    # Prefer a title containing the season year -- "2024 ICC Men's T20 World Cup"
    # beats the evergreen "ICC Men's T20 World Cup" page, which lists no fixtures.
    if season:
        for h in hits:
            if str(season) in h["title"]:
                return h["title"]
    return hits[0]["title"]


def suggest_url(event: str, season: int | None) -> str:
    """A pre-filled guess for the analyst to accept or correct.

    Always returns something: an empty box makes the analyst do the search, and
    a wrong guess is cheaper to fix than a blank one is to fill.
    """
    title = find_page(event, season)
    if not title:
        q = f"{season} {event}" if season else event
        return "https://en.wikipedia.org/wiki/" + urllib.parse.quote(q.replace(" ", "_"))
    return "https://en.wikipedia.org/wiki/" + urllib.parse.quote(title.replace(" ", "_"))


def fetch_wikitext(url_or_title: str, max_chars: int = 400_000) -> str | None:
    """Raw wikitext is far smaller and more structured than rendered HTML."""
    m = re.match(r"^https?://([a-z]+)\.(?:m\.)?wikipedia\.org/wiki/([^?#]+)", url_or_title, re.I)
    if m:
        lang, title = m.group(1), urllib.parse.unquote(m.group(2))
        api = f"https://{lang}.wikipedia.org/w/api.php"
    elif url_or_title.startswith("http"):
        # Some other site: fetch it as-is and let the caller's filter cope.
        try:
            return _get(url_or_title)[:max_chars]
        except Exception:
            return None
    else:
        api, title = WIKI_API, url_or_title

    url = (f"{api}?action=parse&page={urllib.parse.quote(title)}"
           f"&prop=wikitext&formatversion=2&format=json&redirects=1")
    try:
        data = json.loads(_get(url))
    except Exception:
        return None
    text = data.get("parse", {}).get("wikitext")
    return text[:max_chars] if text else None


def compact_wikitext(text: str, max_lines: int = 2500, max_chars: int = 60_000) -> str:
    """Throw away everything that is not a fixture before the model sees it.

    Keeps section headers (they carry the stage: "Semi-finals", "Final"), any
    line declaring a match template, and any line carrying team/date/time/score
    parameters. Falls back to a looser date+versus filter when the page does not
    use match templates at all, so non-template pages still yield something.
    """
    if not text:
        return ""
    # Match templates declare their fixture as named parameters, one per line.
    # Selecting on the PARAMETER is far tighter than selecting on the template:
    # a cricket page opens {{cr|IND}} for every flag icon, so a template-name
    # filter drags in the whole infobox, while `| team1 =` appears once per real
    # fixture and nowhere else.
    WANTED = re.compile(r"^\|\s*(team1|team2|date|time|round|stage|score1|score2|"
                        r"home ?team|away ?team|venue|aet|penalties)\s*=", re.I)
    keep = []
    for raw in text.split("\n"):
        s = raw.strip()
        if re.match(r"^=+\s*(.+?)\s*=+$", s):
            keep.append("SECTION: " + s.strip("= ").strip())
            continue
        if not WANTED.match(s):
            continue
        if re.match(r"^\|\s*team1\s*=", s, re.I) or re.match(r"^\|\s*home ?team\s*=", s, re.I):
            keep.append("--- match ---")
        # Strip refs and wiki markup noise; the model only needs the value.
        s = re.sub(r"<ref[^>]*>.*?</ref>|<ref[^>]*/>", "", s, flags=re.S)
        keep.append(s[:200])
        if len(keep) >= max_lines:
            break
    out = "\n".join(keep)
    # Section headers alone are not fixtures. Require real match content before
    # accepting this pass, or a page whose fixtures live on sub-pages returns a
    # tidy-looking list of headings and the model finds nothing in it.
    if "--- match ---" in out:
        return out[:max_chars]

    # No match templates -- keep lines that carry both a date and a versus/time.
    keep = []
    for raw in text.split("\n"):
        s = re.sub(r"\s+", " ", raw.strip())
        if not (8 <= len(s) <= 300):
            continue
        has_date = bool(re.search(r"\b(\d{1,2}\s+)?(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
                                  r"[a-z]*\.?\s*\d{0,2}(,?\s*\d{4})?\b", s, re.I)
                        or re.search(r"\d{4}-\d{2}-\d{2}|\d{1,2}[/.]\d{1,2}[/.]\d{2,4}", s))
        has_match = bool(re.search(r"\bvs?\.?\b|\bv\b|–|—", s, re.I)
                         or re.search(r"\d{1,2}[:.]\d{2}", s))
        if has_date and has_match:
            keep.append(s)
        if len(keep) >= 400:
            break
    out = "\n".join(keep)
    return out[:16_000] if len(out) > 40 else ""


# --------------------------------------------------------------- parsing
def _client():
    """Anthropic client.

    Key resolution order: the environment first (what a developer shell has),
    then worker/config.json. The config route matters because the worker and the
    enrichment runner are started by Apache/systemd, neither of which inherits an
    interactive shell's environment.
    """
    import anthropic
    key = os.environ.get("ANTHROPIC_API_KEY") or ""
    if not key:
        from pathlib import Path
        cfg_path = Path(__file__).resolve().parent.parent / "worker" / "config.json"
        if cfg_path.exists():
            key = json.loads(cfg_path.read_text()).get("anthropic_api_key") or ""
    if not key:
        raise RuntimeError(
            "No Anthropic API key. Set ANTHROPIC_API_KEY, or add "
            '"anthropic_api_key" to worker/config.json.')
    return anthropic.Anthropic(api_key=key)


def parse_fixtures(client, event: str, season: int | None, wikitext: str) -> tuple[list[dict], str]:
    """Have Claude turn compacted wikitext into structured fixtures."""
    msg = client.messages.create(
        model="claude-opus-5",
        max_tokens=16000,
        thinking={"type": "adaptive"},
        output_config={"effort": "medium",
                       "format": {"type": "json_schema", "schema": FIXTURE_SCHEMA}},
        messages=[{"role": "user", "content": PROMPT.format(
            event=event, season=season or "unknown", wikitext=wikitext)}],
    )
    if msg.stop_reason == "refusal":
        return [], ""
    text = next((b.text for b in msg.content if b.type == "text"), "")
    try:
        data = json.loads(text)
    except json.JSONDecodeError:
        return [], ""
    return data.get("fixtures", []), data.get("event_label", "")


def enrich_from_url(client, event: str, season: int | None, url: str,
                    sleep: float = 0.3) -> dict:
    """One event, one page. Network-and-AI only; no DB writes.

    Returns {status, url, title, fixtures, label, note} so the caller can show
    the analyst exactly what happened for each event it looked up.
    """
    if not url:
        return {"status": "no_url", "url": "", "fixtures": [], "label": "", "note": "no page given"}
    wt = fetch_wikitext(url)
    if not wt:
        return {"status": "no_page", "url": url, "fixtures": [], "label": "",
                "note": "could not read that page"}
    time.sleep(sleep)                       # be polite to Wikipedia
    compact = compact_wikitext(wt)
    if not compact:
        return {"status": "no_fixtures", "url": url, "fixtures": [], "label": "",
                "note": "page has no fixture-shaped content"}
    try:
        fx, label = parse_fixtures(client, event, season, compact)
    except Exception as exc:                # noqa: BLE001 — surface, don't crash the batch
        return {"status": "error", "url": url, "fixtures": [], "label": "",
                "note": f"{type(exc).__name__}: {exc}"[:300]}
    return {"status": "ok" if fx else "no_fixtures", "url": url, "fixtures": fx,
            "label": label, "note": f"{len(fx)} fixtures read"}


# ------------------------------------------------------- legacy entry point
def enrich_event_season(client, event: str, season: int | None,
                        sleep: float = 0.5) -> tuple[list[dict], str, str]:
    """Search-then-parse, used by the historical backfill runner."""
    title = find_page(event, season)
    if not title:
        return [], "no_page", "no Wikipedia search hit"
    time.sleep(sleep)
    wt = fetch_wikitext(title)
    if not wt:
        return [], "no_wikitext", title
    fx, _ = parse_fixtures(client, event, season, compact_wikitext(wt))
    return fx, ("ok" if fx else "no_fixtures"), title