#!/usr/bin/env python3
"""Survey of 100 public AI / MCP repositories — which ignore rules are in place. Aggregate counts only.

  python3 tools/repo_survey.py            take the sample, count, write evidence/repo_survey.json
  python3 tools/repo_survey.py --page     also refresh the figures inside docs/report/index.html

What it does, and what it does not do:
  * The sample: public, non-archived, non-fork repositories with at least MIN_STARS stars that were pushed
    to in the last 30 days, found through GitHub's public search API for four topics. The four lists are
    merged in turn (first of each, second of each, ...) until 100 distinct repositories are reached.
  * For each repository it reads two public files from the default branch, .gitignore and .dockerignore,
    and asks only whether five other paths exist (Dockerfile, .env, .env.example, .mcp.json,
    .claude/settings.local.json). The contents of those five are never downloaded.
  * It writes counts. No repository name, owner or address is written to any output file, and none is
    printed. Nothing is reported to anyone; no issue, comment or pull request is opened.
  * No key, no account. Standard library plus curl for the transfers.
Reading the numbers: an ignore rule that is absent is not a leak. The file it would cover may not exist,
may be ignored by a global rule, or may be kept out by habit. "Pattern present" is a line-by-line match
of the common spellings listed in RULES below, not a full implementation of gitignore semantics.
The sample changes every day, so a later run gives somewhat different counts.
"""
from __future__ import annotations

import concurrent.futures
import datetime
import json
import re
import subprocess
import sys
import time
import urllib.parse
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
TOPICS = ["mcp", "ai-agents", "claude-code", "llm"]
MIN_STARS = 50
SIZE = 100
UA = "repo-survey/1.0"
RULES = {
    "env": re.compile(r"^(\*\*/)?/?(\.env(\*|\.\*|\.local)?|\*\.env|\.env\..*)/?$"),
    "claude_local": re.compile(r"(^|/)\.claude(/|$)|settings\.local\.json|(^|/)\.claude/\*"),
    "git_dir": re.compile(r"^(\*\*/)?/?\.git/?$"),
    "keys": re.compile(r"\*\.(pem|key|p12|pfx)$|(^|/)id_rsa|secrets?(\.|/|$)|credentials"),
}


def curl(args: list[str], timeout: int = 20) -> subprocess.CompletedProcess:
    return subprocess.run(["curl", "-sS", "-L", "--max-time", str(timeout), "-A", UA, *args], capture_output=True, text=True, timeout=timeout + 5)


def sample() -> list[dict]:
    since = (datetime.date.today() - datetime.timedelta(days=30)).isoformat()
    lists = []
    for topic in TOPICS:
        q = urllib.parse.quote(f"topic:{topic} pushed:>{since} stars:>={MIN_STARS} archived:false fork:false")
        res = curl(["-f", "-H", "Accept: application/vnd.github+json", f"https://api.github.com/search/repositories?q={q}&sort=stars&order=desc&per_page=60"])
        if res.returncode != 0:
            sys.exit(f"repo survey: the search for topic {topic} was not reached ({res.stderr.strip()[:80]})")
        lists.append([{"full": r["full_name"], "branch": r["default_branch"], "lang": r.get("language") or "other"} for r in json.loads(res.stdout)["items"]])
        time.sleep(7)   # the public search API allows 10 requests a minute
    picked, seen = [], set()
    for i in range(60):
        for lst in lists:
            if i < len(lst) and lst[i]["full"] not in seen and len(picked) < SIZE:
                seen.add(lst[i]["full"])
                picked.append(lst[i])
    return picked


def lines(text: str) -> list[str]:
    return [ln.strip() for ln in text.splitlines() if ln.strip() and not ln.strip().startswith(("#", "!"))]


def look(repo: dict) -> dict:
    base = f"https://raw.githubusercontent.com/{repo['full']}/{urllib.parse.quote(repo['branch'], safe='')}/"

    def text(path: str):
        r = curl(["-f", base + path])
        return r.stdout if r.returncode == 0 else None

    def exists(path: str) -> bool:
        r = curl(["-o", "/dev/null", "-I", "-w", "%{http_code}", base + path])
        return r.stdout.strip() == "200"

    gi, di = text(".gitignore"), text(".dockerignore")
    gl, dl = lines(gi or ""), lines(di or "")
    return {
        "lang": repo["lang"],
        "gitignore": gi is not None,
        "gi_env": any(RULES["env"].search(x) for x in gl),
        "gi_claude": any(RULES["claude_local"].search(x) for x in gl),
        "gi_keys": any(RULES["keys"].search(x) for x in gl),
        "dockerfile": exists("Dockerfile"),
        "dockerignore": di is not None,
        "di_env": any(RULES["env"].search(x) for x in dl),
        "di_git": any(RULES["git_dir"].search(x) for x in dl),
        "env_example": exists(".env.example"),
        "env_committed": exists(".env"),
        "mcp_json": exists(".mcp.json"),
        "claude_local_committed": exists(".claude/settings.local.json"),
        "claude_dir": exists(".claude/settings.json"),
    }


def main() -> int:
    repos = sample()
    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
        rows = list(ex.map(look, repos))
    n = len(rows)
    c = lambda key, among=None: sum(1 for r in (among if among is not None else rows) if r[key])
    docker = [r for r in rows if r["dockerfile"]]
    claude = [r for r in rows if r["claude_dir"] or r["claude_local_committed"]]
    langs: dict[str, int] = {}
    for r in rows:
        langs[r["lang"]] = langs.get(r["lang"], 0) + 1
    out = {
        "taken_at": datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
        "sample": {"size": n, "topics": TOPICS, "min_stars": MIN_STARS, "pushed_within_days": 30,
                   "languages": dict(sorted(langs.items(), key=lambda kv: -kv[1])[:6])},
        "gitignore": {"present": c("gitignore"), "env_pattern": c("gi_env"), "claude_local_pattern": c("gi_claude"), "key_file_pattern": c("gi_keys")},
        "docker": {"with_dockerfile": len(docker), "with_dockerignore": c("dockerignore", docker), "dockerignore_env_pattern": c("di_env", docker),
                   "dockerignore_git_pattern": c("di_git", docker)},
        "files": {"env_example": c("env_example"), "env_at_root_of_default_branch": c("env_committed"), "mcp_json": c("mcp_json"),
                  "claude_settings_shared": c("claude_dir"), "claude_settings_local_at_default_branch": c("claude_local_committed"),
                  "uses_claude_folder": len(claude), "uses_claude_folder_and_ignores_local": c("gi_claude", claude)},
        "method": "tools/repo_survey.py — counts only; no repository is named; five paths are checked for existence without downloading them",
    }
    (ROOT / "evidence").mkdir(exist_ok=True)
    (ROOT / "evidence" / "repo_survey.json").write_text(json.dumps(out, indent=2) + "\n", encoding="utf-8")
    print(json.dumps(out, indent=2))
    page = ROOT / "docs" / "report" / "index.html"
    if "--page" in sys.argv and page.exists():
        s = page.read_text(encoding="utf-8")
        s2 = re.sub(r"/\*SURVEY\*/.*?/\*END\*/", lambda _: "/*SURVEY*/const S=" + json.dumps(out) + ";/*END*/", s, count=1, flags=re.S)
        page.write_text(s2, encoding="utf-8")
    return 0


if __name__ == "__main__":
    sys.exit(main())