#!/usr/bin/env python3
"""Read a frozen public URL cohort; save serial Lighthouse and delivered-HTML observations.

Requires Python 3, curl, Node/npm and Google Chrome. No forms are submitted.
Usage: python3 website-evidence-audit.py cohort.json output-directory
Lighthouse is pinned, with its standard mobile simulated-throttling configuration.
Full reports remain in output-directory/raw; publish only reviewed summaries.
"""
import hashlib
import json
import re
import subprocess
import sys
from datetime import datetime, timezone
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import urljoin, urlparse

VERSION = "13.5.0"
AUDITS = ["document-title", "meta-description", "is-crawlable", "canonical", "html-has-lang", "image-alt", "link-name", "button-name", "label", "color-contrast", "heading-order"]

class Document(HTMLParser):
    def __init__(self):
        super().__init__(); self.links = []; self.headings = []; self.title = ""; self.description = ""; self.robots = ""; self.canonical = ""; self.lang = ""; self.text = []; self.active = None; self.skip = 0
    def handle_starttag(self, tag, attributes):
        a = dict(attributes)
        if tag in ("script", "style"): self.skip += 1
        if tag == "html": self.lang = a.get("lang", "")
        if tag == "title": self.active = "title"
        if tag == "h1": self.active = "h1"; self.headings.append("")
        if tag == "meta" and a.get("name", "").lower() == "description": self.description = a.get("content", "")
        if tag == "meta" and a.get("name", "").lower() == "robots": self.robots = a.get("content", "")
        if tag == "link" and a.get("rel", "").lower() == "canonical": self.canonical = a.get("href", "")
        if tag == "a" and a.get("href"): self.links.append({"href": a["href"], "text": ""}); self.active = "a"
    def handle_data(self, data):
        if self.skip: return
        if data.strip(): self.text.append(data.strip())
        if self.active == "title": self.title += data
        if self.active == "h1" and self.headings: self.headings[-1] += data
        if self.active == "a" and self.links: self.links[-1]["text"] += data
    def handle_endtag(self, tag):
        if tag in ("script", "style"): self.skip = max(0, self.skip - 1)
        if tag in ("title", "h1", "a"): self.active = None

def fetch(url, destination):
    result = subprocess.run(["curl", "--location", "--silent", "--show-error", "--compressed", "--max-time", "40", "--max-filesize", "10000000", "--proto", "=https", "--proto-redir", "=https", "--user-agent", "DardoResearch/1.0 (+https://dardo.studio/en/studio/)", "--output", str(destination), "--write-out", "%{http_code}\n%{url_effective}", url], capture_output=True, text=True, timeout=45)
    parts = result.stdout.splitlines()
    return {"status": int(parts[0]) if parts and parts[0].isdigit() else None, "finalUrl": parts[1] if len(parts) > 1 else url, "error": result.stderr.strip() or None}

def inspect_html(site, raw):
    path = raw / (site["id"] + ".html")
    response = fetch(site["url"], path)
    if not path.exists() or response["status"] != 200: return {**response, "observed": False}
    payload = path.read_bytes(); doc = Document(); doc.feed(payload.decode("utf-8", "replace"))
    contacts = []
    for link in doc.links:
        href = urljoin(response["finalUrl"], link["href"])
        if href.startswith(("mailto:", "tel:")): channel = href.split(":")[0]
        elif urlparse(href).hostname in ("wa.me", "api.whatsapp.com", "web.whatsapp.com", "www.whatsapp.com"): channel = "whatsapp"
        elif urlparse(href).scheme in ("http", "https") and re.search(r"contact|contacto|cont[aá]ct|cotiza|hablemos|escr[ií]ben|inquiry|quote", link["text"] + " " + urlparse(href).path, re.I): channel = "contact-link"
        else: continue
        # Retain public contact page URLs, but not phone numbers, emails or chat payloads.
        contacts.append({"channel": channel, "url": href.split("?")[0] if channel == "contact-link" else None})
    (raw / (site["id"] + "-text.txt")).write_text("\n".join(doc.text))
    return {**response, "observed": True, "sha256": hashlib.sha256(payload).hexdigest(), "title": re.sub(r"\s+", " ", doc.title).strip(), "descriptionPresent": bool(doc.description.strip()), "htmlLang": doc.lang, "h1Count": len(doc.headings), "canonical": doc.canonical, "robots": doc.robots, "contactChannels": sorted({x["channel"] for x in contacts}), "contactPageLinks": sorted({x["url"] for x in contacts if x["url"]})}

def inspect_lighthouse(site, raw):
    path = raw / (site["id"] + ".lighthouse.json")
    command = ["npm", "exec", "--yes", f"--package=lighthouse@{VERSION}", "--", "lighthouse", site["url"], "--quiet", "--chrome-flags=--headless --no-first-run", "--only-categories=performance,accessibility,seo", "--output=json", f"--output-path={path}"]
    try:
        result = subprocess.run(command, capture_output=True, text=True, timeout=150)
        (raw / (site["id"] + "-lighthouse.log")).write_text(result.stderr)
    except subprocess.TimeoutExpired:
        return {"observed": False, "error": "audit timeout after 150 seconds"}
    if not path.exists(): return {"observed": False, "error": "Lighthouse did not produce a report"}
    body = path.read_bytes(); data = json.loads(body)
    if data.get("runtimeError"): return {"observed": False, "error": data["runtimeError"].get("code"), "fetchTime": data.get("fetchTime"), "sha256": hashlib.sha256(body).hexdigest()}
    audits = data["audits"]
    return {"observed": True, "version": data["lighthouseVersion"], "fetchTime": data["fetchTime"], "finalUrl": data.get("finalDisplayedUrl", data.get("finalUrl")), "sha256": hashlib.sha256(body).hexdigest(), "warnings": data.get("runWarnings", []), "environment": data.get("environment"), "settings": data["configSettings"], "scores": {key: value["score"] for key, value in data["categories"].items()}, "metrics": {key: audits.get(key, {}).get("numericValue") for key in ["first-contentful-paint", "largest-contentful-paint", "total-blocking-time", "cumulative-layout-shift", "speed-index"]}, "audits": {key: {"score": audits[key].get("score"), "mode": audits[key].get("scoreDisplayMode")} for key in AUDITS if key in audits}}

def main():
    cohort_path, output = Path(sys.argv[1]), Path(sys.argv[2])
    cohort = json.loads(cohort_path.read_text()); raw = output / "raw"; raw.mkdir(parents=True, exist_ok=True)
    result_path = output / "observations.json"
    observations = json.loads(result_path.read_text()) if result_path.exists() else []
    completed = {row["id"] for row in observations}
    for site in cohort["sites"]:
        if site["id"] in completed: continue
        print(f"START {len(observations)+1}/{len(cohort['sites'])} {site['name']}", flush=True)
        row = {**site, "observedAt": datetime.now(timezone.utc).isoformat()}
        try: row["html"] = inspect_html(site, raw)
        except Exception as error: row["html"] = {"observed": False, "error": type(error).__name__}
        row["lighthouse"] = inspect_lighthouse(site, raw)
        observations.append(row)
        result_path.write_text(json.dumps(observations, ensure_ascii=False, indent=2) + "\n")
        print(f"DONE {site['id']} html={row['html'].get('status')} lighthouse={row['lighthouse']['observed']}", flush=True)

if __name__ == "__main__": main()