#!/usr/bin/env python3
# Copyright: 2026, Maximiliano Curia <maxy@debian.org>
#
# License: ISC
#  Permission to use, copy, modify, and/or distribute this software for any
#  purpose with or without fee is hereby granted, provided that the above
#  copyright notice and this permission notice appear in all copies.
#  .
#  THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
#  REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
#  AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
#  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
#  LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
#  OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
#  PERFORMANCE OF THIS SOFTWARE.

"""Verify decopy licence anchors against codesearch.debian.net.

Each anchor decopy adds to ``LICENSES_RES`` is scanned against *all* input text,
so a short/generic anchor can mislabel unrelated files. This tool queries Debian
Code Search -- the broadest corpus of "text that is NOT this licence but might
trip its anchor" -- and reports whether each anchor's hits are concentrated in
licence-bearing files. It is an offline audit aid (like
``audit-license-coverage.py``); it is NOT part of the test suite and must never
run in CI.

It sources the real anchor patterns from ``decopy.res`` so it cannot drift from
the matchers it audits. The primary unit is each ``LICENSES_RES`` entry (the hot
path); ``*_SUBS`` sub-anchors only fire behind a gate, so they are secondary --
a stray result on a sub whose gate is clean is ``WEAK-behind-gate``, not a live
bug.

Politeness: one request at a time, never concurrent; a sleep
between live requests; on-disk response cache so re-runs do not re-hit the
service; a descriptive User-Agent; honours HTTP 429 / Retry-After. The v1 API
needs an ``x-dcs-apikey`` header (key from
https://codesearch.debian.net/apikeys/ via Salsa login), supplied via
``$DCS_API_KEY`` or ``--key-file PATH`` (no default; the path is never hardcoded
and the key is never logged).

Run from the repository root, e.g. verify a batch's anchors::

    # gates + standalone rules (primary) and their behind-gate subs (secondary)
    scripts/verify-anchors-codesearch.py \\
        --licenses CmuMach Ssh BoehmGc Adobe-Display-PostScript Adobe-Utopia \\
                   ssh-keyscan OpenVision \\
        --subs CMU_MACH_SUBS SSH_SUBS BOEHM_SUBS

    # every LICENSES_RES anchor added since a baseline commit
    scripts/verify-anchors-codesearch.py --new-since ac025827

    # offline: show what would be queried, no network
    scripts/verify-anchors-codesearch.py --licenses CmuMach --list

Caveat: decopy anchors are written against ``clean_comments`` output (newlines,
tabs, comment-continuation prefixes, banners and quotes collapsed to single
spaces), but Code Search searches the RAW source. The default plain query joins
words with ``\\s+`` and so *under*-matches comment-wrapped text (it cannot bridge a
``\\n * `` / ``# `` wrap or a stripped quote), making a low/zero-hit PASS weak
evidence. Code Search *does* match across newlines, so ``--comment-aware`` widens
each gap to tolerate those characters and recovers the real occurrences (favouring
recall; it can over-bridge). Use ``--comment-aware`` for a faithful check; pass
``--query`` to override a single anchor's phrase or ``--max-query-len`` to shorten.
"""

from __future__ import annotations

import argparse
import ast
import contextlib
import hashlib
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass, field
from pathlib import Path

try:
    import regex as rex
except ImportError:
    import re as rex  # type: ignore[no-redef]

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))

from decopy import res  # noqa: E402
from decopy.re_types import ReSub, ReSubGroup  # noqa: E402

API_URL = "https://codesearch.debian.net/api/v1/search"
USER_AGENT = "decopy-anchor-verify (maxy@debian.org)"
DEFAULT_CACHE = ROOT / ".claude" / "dcs-cache"
BASELINE_REF = "ac025827"

DEFAULT_SLEEP = 1.3  # seconds between LIVE requests (>= 1 req/s with margin)
TRIGRAM_MIN = 3  # Code Search needs a literal run of at least this length
THIN_LEN = 24  # anchors with a shorter longest literal run are flagged risky
MAX_QUERY_LEN = 64  # cap the derived query so it tends to fit one source line
# Above this, an anchor is broad-by-design (a common licence phrase); flag it
# BROAD rather than downloading the (potentially multi-GB) full result set.
MAX_RESPONSE_BYTES = 25_000_000
TINY_TOTAL = 5  # a handful of total hits -> verdict downgraded to manual review

# decopy matches on clean_comments() output, where newlines, tabs and comment
# punctuation are collapsed to single spaces. Code Search searches the RAW source,
# where the same gap between two words is often a line wrap with a comment
# continuation prefix (" * ", "# ", "// ", ";; ", "> "), a banner ("==="/"***") or
# a stripped quote. In --comment-aware mode each inter-word gap becomes this class
# so the query matches the same comment-wrapped text decopy would. Code Search
# matches across newlines, so this recovers occurrences a plain/\\s+ query misses.
# It deliberately favours recall (it can over-bridge), so it is opt-in.
COMMENT_GAP = r'[\s"*#;>%!/=~|-]+'

# A hit counts as licence-context if its path looks like a licence/notice file or
# its surrounding lines carry licence wording.
LICENCE_PATH_RE = rex.compile(
    r"(copying|licen[sc]e|copyright|notice|readme|\.spdx|\.license|debian/)",
    rex.IGNORECASE,
)
LICENCE_CTX_RE = rex.compile(
    r"(permission|redistribut|warranty|royalty|copyright|granted|trademark|"
    r"provided that|as is|disclaim|free of charge|all rights reserved)",
    rex.IGNORECASE,
)

# Escapes that introduce a character class / anchor (they end a literal run);
# any other escaped char is a literal (e.g. \( \. \+ \)).
_CLASS_ESCAPES = set("sSdDwWbBAZGnrtfv0123456789")


@dataclass
class Anchor:
    label: str
    kind: str  # "primary" (LICENSES_RES) or "sub" (behind a gate)
    pattern: str
    run: str = ""
    query: str = ""
    match_mode: str = "regexp"
    note: str = ""


@dataclass
class Result:
    anchor: Anchor
    total: int = 0
    licence: int = 0
    stray: int = 0
    strays: list[str] = field(default_factory=list)
    verdict: str = ""


def load_key(path: Path | None) -> str:
    env = os.environ.get("DCS_API_KEY")
    if env:
        return env.strip()
    if path is not None:
        return path.read_text().strip()
    msg = (
        "No API key: set $DCS_API_KEY or pass --key-file PATH "
        "(get one at https://codesearch.debian.net/apikeys/)."
    )
    raise SystemExit(msg)


def longest_literal_run(pattern: str) -> str:
    """Return the longest run of literal characters in a regex pattern.

    Character classes, groups, quantifiers and class-escapes break a run;
    ``\\x`` for a non-class ``x`` contributes the literal ``x``. The result is
    the trigram seed Code Search indexes on -- an empty/short result means the
    pattern is too generic to search (and so too generic to trust).
    """
    runs: list[str] = []
    cur: list[str] = []
    i, n = 0, len(pattern)
    in_class = False

    def flush() -> None:
        if cur:
            runs.append("".join(cur))
            cur.clear()

    while i < n:
        ch = pattern[i]
        if in_class:
            if ch == "]":
                in_class = False
            i += 1
            continue
        if ch == "\\":
            nxt = pattern[i + 1] if i + 1 < n else ""
            if nxt and nxt not in _CLASS_ESCAPES:
                cur.append(nxt)
            else:
                flush()
            i += 2
            continue
        if ch == "[":
            flush()
            in_class = True
            i += 1
            continue
        if ch in "()|^$.*+?{}":
            flush()
            i += 1
            continue
        cur.append(ch)
        i += 1
    flush()

    trimmed = [r.strip(" \t:;,.\"'`-") for r in runs]
    return max(trimmed, key=len) if trimmed else ""


def build_query(
    run: str, max_len: int, *, literal: bool, comment_aware: bool = False
) -> tuple[str, str]:
    """Turn a literal run into a (query, match_mode) pair.

    Truncated on a word boundary to ``max_len``. In regexp mode each whitespace
    gap becomes ``\\s+`` (tolerant of varied spacing within a source line), or
    ``COMMENT_GAP`` when ``comment_aware`` (also tolerant of comment-continuation
    prefixes, banners, stripped quotes and line wraps); the words are regex-escaped
    so punctuation kept from the anchor is literal.
    """
    words = run.split()
    out: list[str] = []
    length = 0
    for w in words:
        add = len(w) + (1 if out else 0)
        if out and length + add > max_len:
            break
        out.append(w)
        length += add
    if not out:
        return run[:max_len], "literal"
    if literal:
        return " ".join(out), "literal"
    gap = COMMENT_GAP if comment_aware else r"\s+"
    return gap.join(rex.escape(w) for w in out), "regexp"


def relicense_patterns(src: str) -> set[str]:
    """Extract every ``LICENSES_RES`` ReLicense pattern (arg 0) from source.

    Uses ``ast`` so it works on any checked-out revision without importing it
    (implicit string concatenation is already merged by the parser).
    """
    tree = ast.parse(src)
    patterns: set[str] = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Assign) and node.targets:
            target = getattr(node.targets[0], "id", None)
            value: ast.expr | None = node.value
        elif isinstance(node, ast.AnnAssign):
            target = getattr(node.target, "id", None)
            value = node.value
        else:
            continue
        if target != "LICENSES_RES" or not isinstance(value, ast.Tuple):
            continue
        for elt in value.elts:
            if (
                isinstance(elt, ast.Call)
                and getattr(elt.func, "id", None) == "ReLicense"
                and elt.args
            ):
                with contextlib.suppress(ValueError, SyntaxError):
                    patterns.add(ast.literal_eval(elt.args[0]))
    return patterns


def _anchors_new_since(ref: str) -> list[Anchor]:
    baseline_src = subprocess.run(  # noqa: S603
        ["git", "show", f"{ref}:decopy/res.py"],  # noqa: S607
        cwd=ROOT,
        capture_output=True,
        text=True,
        check=True,
    ).stdout
    old = relicense_patterns(baseline_src)
    seen: set[str] = set()
    anchors: list[Anchor] = []
    for entry in res.LICENSES_RES:
        if entry.re not in old and entry.re not in seen:
            seen.add(entry.re)
            anchors.append(
                Anchor(label=entry.license, kind="primary", pattern=entry.re)
            )
    return anchors


def _anchors_by_name(names: list[str]) -> list[Anchor]:
    by_name: dict[str, list[str]] = {}
    for entry in res.LICENSES_RES:
        by_name.setdefault(entry.license, []).append(entry.re)
    anchors: list[Anchor] = []
    for name in names:
        pats = by_name.get(name)
        if not pats:
            print(f"warning: no LICENSES_RES entry named {name!r}", file=sys.stderr)
            continue
        for j, pat in enumerate(pats):
            label = name if len(pats) == 1 else f"{name}#{j + 1}"
            anchors.append(Anchor(label=label, kind="primary", pattern=pat))
    return anchors


def _flatten_subs(tup: tuple[ReSub | ReSubGroup, ...]) -> list[ReSub]:
    """Flatten a *_SUBS tuple, expanding any ReSubGroup into its member ReSubs."""
    out: list[ReSub] = []
    for sub in tup:
        if isinstance(sub, ReSubGroup):
            out.extend(sub.subs)
        else:
            out.append(sub)
    return out


def _anchors_from_subs(sub_names: list[str]) -> list[Anchor]:
    anchors: list[Anchor] = []
    for sub_name in sub_names:
        tup = getattr(res, sub_name, None)
        if tup is None:
            print(f"warning: no SUBS tuple named {sub_name!r}", file=sys.stderr)
            continue
        anchors.extend(
            Anchor(
                label=f"{sub_name}:{sub.repl}",
                kind="sub",
                pattern=sub.pattern.pattern,
            )
            for sub in _flatten_subs(tup)
        )
    return anchors


def collect_anchors(args: argparse.Namespace) -> list[Anchor]:
    anchors: list[Anchor] = []
    if args.new_since:
        anchors += _anchors_new_since(args.new_since)
    anchors += _anchors_by_name(list(args.licenses) + list(args.positional))
    anchors += _anchors_from_subs(args.subs)

    for a in anchors:
        a.run = longest_literal_run(a.pattern)
        if len(a.run) < TRIGRAM_MIN:
            a.note = "no-trigram"
            continue
        a.query, a.match_mode = build_query(
            a.run,
            args.max_query_len,
            literal=args.literal,
            comment_aware=args.comment_aware,
        )
        if len(a.run) < THIN_LEN:
            a.note = "thin"
    return anchors


class DcsClient:
    def __init__(
        self, key: str, cache_dir: Path, sleep: float, *, refresh: bool
    ) -> None:
        self.key = key
        self.cache_dir = cache_dir
        self.sleep = sleep
        self.refresh = refresh
        self._last_live = 0.0
        cache_dir.mkdir(parents=True, exist_ok=True)

    def _cache_path(self, query: str, match_mode: str) -> Path:
        digest = hashlib.sha256(f"{match_mode}\x00{query}".encode()).hexdigest()
        return self.cache_dir / f"{digest[:20]}.json"

    def search(self, query: str, match_mode: str) -> list[dict] | dict:
        path = self._cache_path(query, match_mode)
        if path.exists() and not self.refresh:
            return json.loads(path.read_text())

        params = urllib.parse.urlencode({"query": query, "match_mode": match_mode})
        req = urllib.request.Request(  # noqa: S310
            f"{API_URL}?{params}",
            headers={"User-Agent": USER_AGENT, "x-dcs-apikey": self.key},
        )
        for attempt in range(4):
            wait = self.sleep - (time.monotonic() - self._last_live)
            if wait > 0:
                time.sleep(wait)
            try:
                with urllib.request.urlopen(req, timeout=120) as resp:  # noqa: S310
                    raw = resp.read(MAX_RESPONSE_BYTES + 1)
                # An anchor matching tens of MB of results is broad-by-design (a
                # common licence phrase), not a thin-anchor FP risk: don't download
                # gigabytes — cache a marker and flag it BROAD.
                if len(raw) > MAX_RESPONSE_BYTES:
                    data: list[dict] | dict = {"broad": True}
                else:
                    data = json.loads(raw.decode())
            except urllib.error.HTTPError as exc:
                self._last_live = time.monotonic()
                if exc.code == 429 or exc.code >= 500:  # noqa: PLR2004
                    retry_after = exc.headers.get("Retry-After")
                    back = float(retry_after) if retry_after else 2.0 * (attempt + 1)
                    print(
                        f"  HTTP {exc.code}; backing off {back:.0f}s", file=sys.stderr
                    )
                    time.sleep(back)
                    continue
                body = exc.read().decode(errors="replace")[:120]
                msg = f"HTTP {exc.code} from Code Search: {body}"
                raise SystemExit(msg) from exc
            self._last_live = time.monotonic()
            path.write_text(json.dumps(data))
            return data
        msg = "Code Search kept failing after retries; aborting."
        raise SystemExit(msg)


def classify(results: list[dict]) -> tuple[int, int, int, list[str]]:
    licence = stray = 0
    strays: list[str] = []
    for r in results:
        ctx = " ".join(
            [
                *r.get("context_before", []),
                r.get("context", ""),
                *r.get("context_after", []),
            ]
        )
        if LICENCE_PATH_RE.search(r.get("path", "")) or LICENCE_CTX_RE.search(ctx):
            licence += 1
        else:
            stray += 1
            strays.append(r.get("path", "?"))
    return len(results), licence, stray, strays


def verdict(anchor: Anchor, total: int, stray: int) -> str:
    if anchor.note == "no-trigram":
        return "RED:no-trigram"
    if stray == 0:
        return "PASS"
    if total <= TINY_TOTAL:
        return "WEAK:tiny(review)"
    return "WEAK-behind-gate" if anchor.kind == "sub" else "REVIEW"


def main() -> None:
    ap = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0])
    ap.add_argument("positional", nargs="*", help="LICENSES_RES group/licence names")
    ap.add_argument(
        "--licenses",
        nargs="+",
        default=[],
        help="LICENSES_RES names (primary anchors)",
    )
    ap.add_argument(
        "--subs",
        nargs="+",
        default=[],
        help="*_SUBS tuple names (behind-gate anchors)",
    )
    ap.add_argument(
        "--new-since",
        nargs="?",
        const=BASELINE_REF,
        help=f"verify LICENSES_RES entries added since REF (default {BASELINE_REF})",
    )
    ap.add_argument(
        "--list", action="store_true", help="show queries; do not hit network"
    )
    ap.add_argument("--refresh", action="store_true", help="ignore cache; re-query")
    ap.add_argument(
        "--literal", action="store_true", help="literal match_mode (no \\s+)"
    )
    ap.add_argument(
        "--comment-aware",
        action="store_true",
        help="inter-word gaps tolerate comment-continuation/line-wrap (recall++)",
    )
    ap.add_argument("--query", help="override the derived query (single anchor only)")
    ap.add_argument("--max-query-len", type=int, default=MAX_QUERY_LEN)
    ap.add_argument("--sleep", type=float, default=DEFAULT_SLEEP)
    ap.add_argument(
        "--key-file",
        type=Path,
        default=None,
        help="file holding the API key (or set $DCS_API_KEY)",
    )
    ap.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE)
    ap.add_argument(
        "--show-strays", type=int, default=5, help="sample stray paths to print"
    )
    args = ap.parse_args()

    anchors = collect_anchors(args)
    if not anchors:
        msg = "No anchors selected. Pass names, --licenses, --subs or --new-since."
        raise SystemExit(msg)
    if args.query:
        if len(anchors) != 1:
            msg = "--query needs exactly one selected anchor."
            raise SystemExit(msg)
        anchors[0].query = args.query
        anchors[0].match_mode = "literal" if args.literal else "regexp"
        if anchors[0].note != "thin":
            anchors[0].note = ""

    mode = "literal" if args.literal else "regexp"
    if args.comment_aware and not args.literal:
        mode += "+comment-aware"
    print(
        f"# {len(anchors)} anchor(s); mode={mode} "
        f"max_query_len={args.max_query_len} sleep={args.sleep}s"
    )
    if args.list:
        for a in anchors:
            tag = f" [{a.note}]" if a.note else ""
            print(f"{a.kind:8} {a.label:32} run={len(a.run):3} q={a.query!r}{tag}")
        return

    client = DcsClient(
        load_key(args.key_file), args.cache_dir, args.sleep, refresh=args.refresh
    )
    run_anchors(client, anchors, args.show_strays)


def run_anchors(client: DcsClient, anchors: list[Anchor], show_strays: int) -> None:
    results: list[Result] = []
    for a in anchors:
        if a.note == "no-trigram":
            r = Result(anchor=a, verdict="RED:no-trigram")
        else:
            hits = client.search(a.query, a.match_mode)
            if isinstance(hits, dict):  # BROAD marker: too many results to download
                cap = MAX_RESPONSE_BYTES // 1_000_000
                r = Result(anchor=a, verdict=f"BROAD:>{cap}MB")
            else:
                total, lic, stray, strays = classify(hits)
                r = Result(a, total, lic, stray, strays, verdict(a, total, stray))
        results.append(r)
        tag = f" [{a.note}]" if a.note else ""
        print(
            f"{r.verdict:18} {a.kind:8} {a.label:32}"
            f" total={r.total:4} licence={r.licence:4} stray={r.stray:4}{tag}"
        )
        for p in r.strays[:show_strays]:
            print(f"      stray: {p}")

    # BROAD anchors are broad-by-design (a common licence phrase), not an FP risk.
    weak = [r for r in results if not r.verdict.startswith(("PASS", "BROAD"))]
    broad = sum(r.verdict.startswith("BROAD") for r in results)
    print(
        f"\n# {len(results) - len(weak) - broad} PASS, {broad} BROAD (by design), "
        f"{len(weak)} need review"
    )
    for r in weak:
        print(f"#   {r.verdict}: {r.anchor.label}")


if __name__ == "__main__":
    main()
