#!/usr/bin/env python
"""
Extract Methodology + Data Sources prose content from the ortho-atlas HTML
export into a structured JSON file used by /clinical/methodology and
/clinical/data-sources route pages.

Usage:
    python scripts/extract-atlas-pages.py

Inputs:
    ../../methodology.html
    ../../data-sources.html

Output:
    src/data/clinical/atlas-pages.json
"""

from __future__ import annotations

import json
import re
import sys
from html.parser import HTMLParser
from pathlib import Path


ROOT = Path(__file__).resolve().parents[2]
SITE = Path(__file__).resolve().parents[1]
OUTPUT = SITE / "src" / "data" / "clinical" / "atlas-pages.json"


class SectionParser(HTMLParser):
    """Walk a single HTML file and produce a linear list of content nodes.

    Each node is one of:
      {"type": "h1"|"h2"|"h3", "text": "..."}
      {"type": "p",           "text": "..."}
      {"type": "ul"|"ol",     "items": ["...", ...]}
      {"type": "table",       "rows": [["...", ...], ...]}

    We ignore nav chrome, header, footer, scripts, and styles.
    """

    SKIP_CONTAINERS = {"script", "style", "nav", "header", "footer", "form"}

    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.nodes: list[dict] = []
        self._stack: list[str] = []
        self._current_tag: str | None = None
        self._buffer: list[str] = []
        self._list_items: list[str] | None = None
        self._list_type: str | None = None
        self._in_list_item = False
        self._in_table = False
        self._table_rows: list[list[str]] = []
        self._current_row: list[str] | None = None
        self._cell_buffer: list[str] | None = None
        self._skip_depth = 0

    def _flush_text(self, tag: str) -> None:
        text = re.sub(r"\s+", " ", "".join(self._buffer)).strip()
        if text:
            self.nodes.append({"type": tag, "text": text})
        self._buffer = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if tag in self.SKIP_CONTAINERS:
            self._skip_depth += 1
            return
        if self._skip_depth:
            return

        if tag in {"h1", "h2", "h3", "h4"}:
            self._current_tag = tag
            self._buffer = []
        elif tag == "p":
            self._current_tag = "p"
            self._buffer = []
        elif tag in {"ul", "ol"}:
            self._list_items = []
            self._list_type = tag
        elif tag == "li" and self._list_items is not None:
            self._in_list_item = True
            self._buffer = []
        elif tag == "table":
            self._in_table = True
            self._table_rows = []
        elif tag == "tr" and self._in_table:
            self._current_row = []
        elif tag in {"td", "th"} and self._current_row is not None:
            self._cell_buffer = []
        elif tag == "br" and self._buffer is not None:
            self._buffer.append(" ")

    def handle_endtag(self, tag: str) -> None:
        if tag in self.SKIP_CONTAINERS:
            self._skip_depth = max(0, self._skip_depth - 1)
            return
        if self._skip_depth:
            return

        if tag in {"h1", "h2", "h3", "h4"} and self._current_tag == tag:
            self._flush_text(tag if tag != "h4" else "h3")
            self._current_tag = None
        elif tag == "p" and self._current_tag == "p":
            self._flush_text("p")
            self._current_tag = None
        elif tag in {"ul", "ol"} and self._list_items is not None:
            if self._list_items:
                self.nodes.append({"type": self._list_type or "ul", "items": self._list_items})
            self._list_items = None
            self._list_type = None
        elif tag == "li" and self._in_list_item:
            text = re.sub(r"\s+", " ", "".join(self._buffer)).strip()
            if text and self._list_items is not None:
                self._list_items.append(text)
            self._in_list_item = False
            self._buffer = []
        elif tag == "table" and self._in_table:
            if self._table_rows:
                self.nodes.append({"type": "table", "rows": self._table_rows})
            self._in_table = False
            self._table_rows = []
        elif tag == "tr" and self._in_table and self._current_row is not None:
            if self._current_row:
                self._table_rows.append(self._current_row)
            self._current_row = None
        elif tag in {"td", "th"} and self._current_row is not None and self._cell_buffer is not None:
            self._current_row.append(
                re.sub(r"\s+", " ", "".join(self._cell_buffer)).strip(),
            )
            self._cell_buffer = None

    def handle_data(self, data: str) -> None:
        if self._skip_depth:
            return
        if self._cell_buffer is not None:
            self._cell_buffer.append(data)
        elif self._in_list_item or self._current_tag:
            self._buffer.append(data)


def parse_file(path: Path) -> list[dict]:
    parser = SectionParser()
    parser.feed(path.read_text(encoding="utf-8"))
    return parser.nodes


def group_into_sections(nodes: list[dict]) -> list[dict]:
    """Group a linear node list into a list of top-level sections keyed by h2.

    Structure:
      [
        { "title": "H2 heading", "blocks": [ <node>, <node> ...] },
        ...
      ]
    Anything before the first h2 is placed under a leading "_intro" section.
    """
    sections: list[dict] = []
    current = {"title": None, "blocks": []}
    for node in nodes:
        if node.get("type") == "h2":
            if current["blocks"]:
                sections.append(current)
            current = {"title": node["text"], "blocks": []}
        else:
            current["blocks"].append(node)
    if current["blocks"]:
        sections.append(current)
    return sections


def find_h1(nodes: list[dict]) -> str | None:
    for node in nodes:
        if node.get("type") == "h1":
            # Skip the top "Global Orthopaedic Atlas" site header.
            text = node.get("text", "")
            if text and "Global Orthopaedic Atlas" not in text:
                return text
    for node in nodes:
        if node.get("type") == "h1":
            return node.get("text")
    return None


def main() -> int:
    methodology_src = ROOT / "methodology.html"
    sources_src = ROOT / "data-sources.html"
    if not methodology_src.exists():
        print(f"missing: {methodology_src}", file=sys.stderr)
        return 1
    if not sources_src.exists():
        print(f"missing: {sources_src}", file=sys.stderr)
        return 1

    methodology_nodes = parse_file(methodology_src)
    sources_nodes = parse_file(sources_src)

    payload = {
        "generatedAt": __import__("datetime")
        .datetime.utcnow()
        .strftime("%Y-%m-%dT%H:%M:%SZ"),
        "methodology": {
            "title": find_h1(methodology_nodes) or "Methodology",
            "sections": group_into_sections(methodology_nodes),
        },
        "dataSources": {
            "title": find_h1(sources_nodes) or "Data Sources & References",
            "sections": group_into_sections(sources_nodes),
        },
    }

    OUTPUT.parent.mkdir(parents=True, exist_ok=True)
    OUTPUT.write_text(
        json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8",
    )
    print(
        f"wrote {OUTPUT.relative_to(SITE)}: "
        f"methodology {len(payload['methodology']['sections'])} sections, "
        f"data-sources {len(payload['dataSources']['sections'])} sections",
    )
    return 0


if __name__ == "__main__":
    sys.exit(main())
