#!/usr/bin/env python3
# Copyright: 2026, Maximiliano Curia <maxy@debian.org>
#
# License: ISC
#  Permission to use, copy, modify, and/or distribute this software for any
#  purpose with or without fee is hereby granted, provided that the above
#  copyright notice and this permission notice appear in all copies.
#  .
#  THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
#  REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
#  AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
#  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
#  LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
#  OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
#  PERFORMANCE OF THIS SOFTWARE.

"""Rank licence / exception usage across Debian via codesearch.debian.net.

An offline curation aid for deciding which short licences and which exceptions
are worth embedding as full text (TODO items 2 and 3). For each candidate it
asks Debian Code Search how many distinct source packages carry its
`SPDX-License-Identifier:` tag (licences) or `WITH <id>` exception, and prints a
ranked table flagging those at or above a package threshold (default 100).

It is NOT part of the test suite and must never run in CI (like
verify-anchors-codesearch.py / audit-license-coverage.py). Its human-reviewed
output guides the curated `_EMBEDDED_LICENSE_IDS` / exception list in
scripts/emit-license-data.py; the runtime itself never touches the network.

CAVEAT - the `SPDX-License-Identifier:` signal is biased and undercounts. It only
sees packages that write an SPDX tag, which skews heavily toward tag-adopting
ecosystems (Rust/Cargo, Go, modern JS) and misses traditional C/Perl projects
that ship a COPYING file with no tag. In practice ubiquitous licences like
ISC/Zlib/X11 score far below their real prevalence while MIT (tagged by most
Rust crates) is inflated.

`--text-mode` queries a different signal instead: a distinctive, non-template
substring of the licence's own text (via spdx_template_as_re.py), matched with
a whitespace-tolerant regex so re-wrapped copies still count. This recovers
usage the tag misses regardless of tagging. It is not strictly better
(a licence whose wording varies a lot in the wild, or whose text is largely
shared with a sibling licence, can score lower than the tag - MIT and
BSD-3-Clause commonly have no distinctive-enough window at all), so run both
modes and compare rather than trusting either alone.

Politeness (identical to verify-anchors-codesearch.py): one request at a time, a
sleep between live requests, an on-disk response cache, a descriptive
User-Agent, and honouring HTTP 429 / Retry-After. The v1 API needs an
`x-dcs-apikey` header (key from https://codesearch.debian.net/apikeys/ via Salsa
login) supplied via `$DCS_API_KEY` or `--key-file PATH` (never hardcoded).

CAUTION with `--all`: a few hundred requests at the default 1.3s spacing can
still be enough load that individual requests start stalling well before any
HTTP 429 comes back. The cache makes a run resumable, but prefer a larger
`--sleep`, a `--limit`-ed slice at a time, and spacing separate `--all` runs
out over time rather than back-to-back, out of consideration for the service.

Examples::

    # rank a few licence ids by package count
    scripts/rank-license-usage-codesearch.py --key-file KEY MIT Zlib ISC Unlicense

    # rank every SPDX licence not already in common-licenses (long; cached)
    scripts/rank-license-usage-codesearch.py --key-file KEY --all

    # same, but the distinctive-text-window signal instead of the SPDX tag
    scripts/rank-license-usage-codesearch.py --key-file KEY --text-mode --all

    # rank exceptions
    scripts/rank-license-usage-codesearch.py --key-file KEY --exceptions --all

    # offline: show what would be queried, no network
    scripts/rank-license-usage-codesearch.py --all --list
"""

from __future__ import annotations

import argparse
import json
import sys
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Literal

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(Path(__file__).resolve().parent))

from dcs_client import (  # noqa: E402
    DEFAULT_SLEEP,
    DcsClient,
    DcsConfig,
    SearchError,
    SearchOutcome,
    SearchState,
    load_key,
)
from spdx_template_as_re import (  # noqa: E402
    distinctive_window,
    template_literal_spans,
    window_to_regexp_query,
)

from decopy.licensing.data.common_licenses import COMMON_LICENSES  # noqa: E402
from decopy.licensing.spdx import canonical_license_name  # noqa: E402

LICENSES_DIR = ROOT / "tests" / "testdata" / "licenses"
EXCEPTIONS_DIR = ROOT / "tests" / "testdata" / "exceptions"

USER_AGENT = "decopy-usage-rank (+https://salsa.debian.org/debian/decopy)"
DEFAULT_CACHE = ROOT / ".cache" / "dcs-cache"
DEFAULT_THRESHOLD = 5  # distinct source packages, tag-mode; low on purpose -
# the tag can undercount real usage by orders of magnitude, so this flags
# "worth a --text-mode follow-up", not "definitely popular".


@dataclass
class Candidate:
    label: str  # decopy DEP-5 name, for the human-facing table
    query: str
    match_mode: Literal["literal", "regexp"] = "literal"


@dataclass
class Result:
    candidate: Candidate
    state: SearchState = SearchState.HITS
    packages: int = 0  # only meaningful when state is HITS
    hits: int = 0


def license_candidates(spdx_ids: list[str]) -> list[Candidate]:
    return [
        Candidate(
            label=canonical_license_name(spdx_id),
            query=f"SPDX-License-Identifier: {spdx_id}",
        )
        for spdx_id in spdx_ids
    ]


def exception_candidates(exception_ids: list[str]) -> list[Candidate]:
    return [
        Candidate(label=exception_id, query=f"WITH {exception_id}")
        for exception_id in exception_ids
    ]


def all_license_ids(max_chars: int | None = None) -> list[str]:
    """SPDX ids whose decopy name is not already a common-licences reference.

    ``max_chars`` additionally drops ids whose ``licenseText`` is longer than
    that (a long popular licence is a candidate for a common-licences
    addition, not for embedding a copy in every debian/copyright).
    """
    ids = []
    for path in LICENSES_DIR.glob("*.json"):
        if canonical_license_name(path.stem) in COMMON_LICENSES:
            continue
        if max_chars is not None:
            text = json.loads(path.read_text(encoding="utf-8")).get("licenseText", "")
            if len(text) > max_chars:
                continue
        ids.append(path.stem)
    return sorted(ids)


def all_exception_ids() -> list[str]:
    return sorted(p.stem for p in EXCEPTIONS_DIR.glob("*.json"))


def common_license_ids() -> set[str]:
    """SPDX ids already covered by a COMMON_LICENSES reference."""
    return {
        p.stem
        for p in LICENSES_DIR.glob("*.json")
        if canonical_license_name(p.stem) in COMMON_LICENSES
    }


def _load_template(spdx_id: str) -> str:
    path = LICENSES_DIR / f"{spdx_id}.json"
    template = json.loads(path.read_text(encoding="utf-8")).get(
        "standardLicenseTemplate", ""
    )
    return template if isinstance(template, str) else ""


def _text_corpus(ids: list[str]) -> dict[str, str]:
    corpus: dict[str, str] = {}
    for spdx_id in ids:
        template = _load_template(spdx_id)
        if template:
            corpus[spdx_id] = " ".join(template_literal_spans(template))
    return corpus


def text_license_candidates(
    spdx_ids: list[str],
) -> tuple[list[Candidate], list[str]]:
    """(candidates, skipped) where each query is a distinctive text window.

    Unlike license_candidates() (an SPDX-tag search), this searches for the
    licence's own text so untagged copies count too. The collision corpus is
    scoped to this ranking batch plus every COMMON_LICENSES-covered id, not
    the whole SPDX corpus - see distinctive_window()'s docstring for why. Ids
    with no distinctive-enough window are returned in the skipped list (a
    query, not a print) for the caller to report.
    """
    corpus = _text_corpus(sorted(set(spdx_ids) | common_license_ids()))
    candidates: list[Candidate] = []
    skipped: list[str] = []
    for spdx_id in spdx_ids:
        spans = template_literal_spans(_load_template(spdx_id))
        others = [text for other, text in corpus.items() if other != spdx_id]
        window = distinctive_window(spans, others)
        if window is None:
            skipped.append(spdx_id)
            continue
        candidates.append(
            Candidate(
                label=canonical_license_name(spdx_id),
                query=window_to_regexp_query(window),
                match_mode="regexp",
            )
        )
    return candidates, skipped


def count_packages(hits: list[dict]) -> int:
    """Distinct source packages across the hits.

    A hit without a ``package`` field is counted by its path instead, and a
    note is printed so a path-count is never silently reported as a
    package-count.
    """
    keys: set[str] = set()
    fell_back = False
    for hit in hits:
        package = hit.get("package")
        if package:
            keys.add(package)
        else:
            keys.add(hit.get("path", ""))
            fell_back = True
    if fell_back:
        note = "  note: some hits had no package field; counted by path"
        print(note, file=sys.stderr)
    return len(keys)


def _result_for(candidate: Candidate, outcome: SearchOutcome) -> Result:
    if outcome.state is SearchState.HITS:
        return Result(
            candidate,
            state=SearchState.HITS,
            packages=count_packages(outcome.hits),
            hits=len(outcome.hits),
        )
    return Result(candidate, state=outcome.state)


Search = Callable[[str, str], SearchOutcome]


def fetch_results(
    search: Search, candidates: list[Candidate], *, progress: bool
) -> list[Result]:
    results: list[Result] = []
    for i, candidate in enumerate(candidates, 1):
        if progress:
            print(f"  [{i}/{len(candidates)}] {candidate.label}", file=sys.stderr)
        try:
            outcome = search(candidate.query, candidate.match_mode)
        except SearchError as exc:
            # One candidate exhausting its retries should not discard every
            # other already-fetched result; skip it and keep going. A fatal
            # error (bad key/query) is SystemExit, which propagates and aborts.
            print(f"  giving up on {candidate.label}: {exc}", file=sys.stderr)
            continue
        results.append(_result_for(candidate, outcome))
    return results


def _sort_key(result: Result) -> tuple[int, int, int]:
    # Pending candidates sort last; broad ones first; then by package count.
    return (
        0 if result.state is SearchState.PENDING else 1,
        1 if result.state is SearchState.BROAD else 0,
        result.packages,
    )


def _header(count: int, threshold: int) -> str:
    return f"# {count} candidate(s); threshold={threshold} packages"


def _report_skips(skipped: list[str]) -> None:
    for spdx_id in skipped:
        print(f"  no distinctive window for {spdx_id}; skipped", file=sys.stderr)


def report(results: list[Result], threshold: int) -> None:
    print(_header(len(results), threshold))
    ordered = sorted(results, key=_sort_key, reverse=True)
    over = pending = 0
    for result in ordered:
        if result.state is SearchState.PENDING:
            pending += 1
            continue
        broad = result.state is SearchState.BROAD
        hit_threshold = broad or result.packages >= threshold
        if hit_threshold:
            over += 1
        packages = "many" if broad else str(result.packages)
        flag = "*" if hit_threshold else " "
        print(
            f"{flag} {result.candidate.label:32} packages={packages:>6} "
            f"hits={result.hits:>6}  q={result.candidate.query!r}"
        )
    done = len(results) - pending
    print(f"\n# {over}/{done} queried at or above {threshold} packages (*)")
    if pending:
        print(f"# {pending} not yet queried (drop --offline-report to fetch them)")


def _selected_ids(
    ids: list[str], *, exceptions: bool, all_: bool, max_chars: int | None
) -> list[str]:
    """Positional ids, or the full candidate set for the mode when --all."""
    if ids:
        return list(ids)
    if not all_:
        return []
    return all_exception_ids() if exceptions else all_license_ids(max_chars)


def collect_candidates(
    ids: list[str], *, exceptions: bool, text_mode: bool, limit: int = 0
) -> tuple[list[Candidate], list[str]]:
    """(candidates, skipped-ids). Only text mode can skip; others skip nothing.

    ``limit`` (0 = no cap) trims the ids *before* building, so text mode does
    not compute a distinctive window for ids that would be dropped anyway.
    """
    if limit:
        ids = ids[:limit]
    if exceptions:
        return exception_candidates(ids), []
    if text_mode:
        return text_license_candidates(ids)
    return license_candidates(ids), []


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0])
    parser.add_argument("ids", nargs="*", help="SPDX licence/exception ids to rank")
    parser.add_argument(
        "--all", action="store_true", help="rank the full candidate set for the mode"
    )
    # --exceptions and --text-mode are alternate licence-search modes; argparse
    # rejects passing both rather than a hand-rolled check inside main().
    mode = parser.add_mutually_exclusive_group()
    mode.add_argument(
        "--exceptions", action="store_true", help="rank exception ids, not licences"
    )
    mode.add_argument(
        "--text-mode",
        action="store_true",
        help="query a distinctive licence-text window instead of the SPDX tag",
    )
    parser.add_argument(
        "--max-chars",
        type=int,
        default=None,
        help="with --all, drop licences whose licenseText is longer than this",
    )
    parser.add_argument(
        "--threshold", type=int, default=DEFAULT_THRESHOLD, help="package-count flag"
    )
    parser.add_argument(
        "--limit", type=int, default=0, help="cap the candidate count (0 = no cap)"
    )
    parser.add_argument(
        "--list", action="store_true", help="show queries; do not hit network"
    )
    parser.add_argument(
        "--offline-report",
        action="store_true",
        help="report on cached results only (no network, no key needed); "
        "candidates with no cache entry yet are counted as pending",
    )
    parser.add_argument("--refresh", action="store_true", help="ignore cache; re-query")
    parser.add_argument("--sleep", type=float, default=DEFAULT_SLEEP)
    parser.add_argument("--key-file", type=Path, default=None)
    parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE)
    return parser


def _print_query_list(candidates: list[Candidate], threshold: int) -> None:
    print(_header(len(candidates), threshold))
    for candidate in candidates:
        print(f"{candidate.label:32} q={candidate.query!r}")


def _search_for(args: argparse.Namespace) -> Search:
    """Build the client from parsed args and pick its cached/live search."""
    key = "" if args.offline_report else load_key(args.key_file)
    client = DcsClient(
        DcsConfig(
            key=key,
            user_agent=USER_AGENT,
            cache_dir=args.cache_dir,
            sleep=args.sleep,
            refresh=args.refresh,
            cache_prefix="rank",
        )
    )
    return client.search_cached if args.offline_report else client.search_live


def main() -> None:
    args = build_parser().parse_args()
    ids = _selected_ids(
        args.ids, exceptions=args.exceptions, all_=args.all, max_chars=args.max_chars
    )
    candidates, skipped = collect_candidates(
        ids, exceptions=args.exceptions, text_mode=args.text_mode, limit=args.limit
    )
    _report_skips(skipped)
    if not candidates:
        msg = "No candidates: pass ids, or --all (with --exceptions for those)."
        raise SystemExit(msg)
    if args.list:
        _print_query_list(candidates, args.threshold)
        return
    results = fetch_results(
        _search_for(args), candidates, progress=not args.offline_report
    )
    report(results, args.threshold)


if __name__ == "__main__":
    main()
