import csv, datetime, urllib.request
from urllib.robotparser import RobotFileParser

# Documented dedicated crawlers only (sources in DECISIONS.md)
BOTS = ["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User",
        "PerplexityBot","Perplexity-User","Google-Extended","Googlebot","bingbot","YandexBot",
        "CCBot","Bytespider","Applebot-Extended","Meta-ExternalAgent",
        "KimiBot","Kimi-SearchBot","Kimi-User","MistralAI-User"]
UA = {"User-Agent": "Mozilla/5.0 (research audit; https://seo-for-llm.com)"}
TODAY = datetime.date.today().isoformat()

def get(url):
    try:
        r = urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=10)
        return r.status, r.read(500000).decode("utf-8", "ignore")
    except Exception as e:
        return getattr(e, "code", "erreur"), ""

def is_text(body):
    return bool(body.strip()) and not body.lstrip().lower().startswith(("<!doctype", "<html", "<"))

def check(robots_txt, bot):
    rp = RobotFileParser(); rp.parse(robots_txt.splitlines())
    return rp.can_fetch(bot, "/")

domains = [l.strip() for l in open("sites.txt") if l.strip() and not l.startswith("#")]
rows, stats, ok_sites, llms_yes = [], {b: 0 for b in BOTS}, 0, 0
for d in domains:
    st, txt = get(f"https://{d}/robots.txt")
    ls, lb = get(f"https://{d}/llms.txt")
    llms = "oui" if ls == 200 and is_text(lb) else "non"
    if st == 200 and is_text(txt):
        ok_sites += 1; llms_yes += (llms == "oui")
        for b in BOTS:
            allowed = check(txt, b)
            rows.append([d, st, b, "autorise" if allowed else "bloque",
                         "oui" if b.lower() in txt.lower() else "non", llms, TODAY])
            stats[b] += (not allowed)
    else:
        rows.append([d, st, "-", "non_recuperable", "", llms, TODAY])
with open("resultats.csv", "w", newline="") as f:
    w = csv.writer(f)
    w.writerow(["domaine","http_robots","bot","statut","cite_explicitement","llms_txt","date_test"])
    w.writerows(rows)
with open("stats.csv", "w", newline="") as f:
    w = csv.writer(f)
    w.writerow(["bot","sites_bloquant","sites_analysables","pourcentage","date_test"])
    for b in BOTS: w.writerow([b, stats[b], ok_sites, round(100*stats[b]/ok_sites,1) if ok_sites else "", TODAY])
    w.writerow(["llms.txt present", llms_yes, ok_sites, round(100*llms_yes/ok_sites,1) if ok_sites else "", TODAY])
print(f"analysables: {ok_sites}/{len(domains)}")