#!/usr/bin/env python3

# Generates the CP2K Dashboard html page
# Inspired by Iain's cp2k_page_update.sh
#
# author: Ole Schuett

from configparser import ConfigParser
from datetime import datetime, timedelta
from email.mime.text import MIMEText
from os import path
from pathlib import Path
from pprint import pformat
from subprocess import check_output
import argparse
import dataclasses
import gzip
import hashlib
import html
import itertools
import os
import pickle
import re
import requests
import smtplib
import sys
import traceback
from typing import Any, List, Dict, cast, Optional, ValuesView, NewType

try:
    from typing import Literal
except ImportError:
    from typing_extensions import Literal  # type: ignore

import matplotlib as mpl

mpl.use("Agg")  # change backend, to run without X11
import matplotlib.pyplot as plt
from matplotlib.ticker import AutoMinorLocator

# ======================================================================================
GitSha = NewType("GitSha", str)
ReportStatus = Literal["OK", "FAILED", "OUTDATED", "UNKNOWN"]


# ======================================================================================
class GitLog:
    def __init__(self) -> None:
        self.commits: List[Commit] = []
        cmd = ["git", "log", "--pretty=format:%H%n%ct%n%an%n%ae%n%s%n%b<--separator-->"]
        outbytes = check_output(cmd)
        output = outbytes.decode("utf-8", errors="replace")
        for entry in output.split("<--separator-->")[:-1]:
            lines = entry.strip().split("\n")
            commit = Commit(
                sha=GitSha(lines[0]),
                date=datetime.fromtimestamp(float(lines[1])),
                author_name=lines[2],
                author_email=lines[3],
                message=lines[4],
            )
            self.commits.append(commit)

        # git-log outputs entries from new to old.
        self.index_by_sha = {c.sha: i for i, c in enumerate(self.commits)}


# ======================================================================================
@dataclasses.dataclass
class Commit:
    sha: GitSha
    author_name: str
    author_email: str
    date: datetime
    message: str


# ======================================================================================
@dataclasses.dataclass
class PlotCurve:
    label: str
    x: List[float] = dataclasses.field(default_factory=list)
    y: List[float] = dataclasses.field(default_factory=list)
    yerr: List[float] = dataclasses.field(default_factory=list)


# ======================================================================================
class Plot:
    def __init__(self, name: str, title: str, ylabel: str, **_: Any):
        self.name = name
        self.title = title
        self.ylabel = ylabel


# ======================================================================================
class AggregatedPlot:
    def __init__(self, plot: Plot):
        self.name = plot.name
        self.title = plot.title
        self.ylabel = plot.ylabel
        self.curves_by_name: Dict[str, PlotCurve] = {}

    @property
    def curves(self) -> ValuesView[PlotCurve]:
        return self.curves_by_name.values()


# ======================================================================================
class PlotPoint:
    def __init__(
        self, plot: str, name: str, label: str, y: float, yerr: float, **_: Any
    ):
        self.plot_name = plot
        self.curve_name = name
        self.curve_label = label
        self.y = y
        self.yerr = yerr


# ======================================================================================
@dataclasses.dataclass
class Report:
    status: ReportStatus
    summary: str
    sha: Optional[GitSha] = None
    plots: List[Plot] = dataclasses.field(default_factory=list)
    plot_points: List[PlotPoint] = dataclasses.field(default_factory=list)
    archive_path: Optional[str] = None


# ======================================================================================
@dataclasses.dataclass
class Status:
    notified: bool = False
    last_ok: Optional[GitSha] = None


# ======================================================================================
def main() -> None:
    parser = argparse.ArgumentParser(description="Generates the CP2K Dashboard.")
    parser.add_argument("config_file", type=Path)
    parser.add_argument("status_file", type=Path)
    parser.add_argument("output_dir", type=Path)
    parser.add_argument("--send-emails", action="store_true")
    args = parser.parse_args()

    assert args.config_file.exists()
    config = ConfigParser()
    config.read(args.config_file)
    log = GitLog()  # Reads history from local git repo.

    gen_frontpage(config, log, args.status_file, args.output_dir, args.send_emails)
    gen_archive(config, log, args.output_dir)
    gen_url_list(config, args.output_dir)


# ======================================================================================
def gen_frontpage(
    config: ConfigParser, log: GitLog, status_fn: Path, outdir: Path, send_emails: bool
) -> None:
    status: Dict[str, Status] = {}
    if path.exists(status_fn):
        with open(status_fn, "rb") as f:
            status = pickle.load(f)

    output = html_header(title="CP2K Dashboard")
    output += '<div id="flex-container"><div>\n'
    output += html_gitbox(log)
    output += html_linkbox()
    output += "</div>\n"
    output += '<table border="1" cellspacing="3" cellpadding="5">\n'
    output += "<tr><th>Name</th><th>Host</th><th>Status</th>"
    output += "<th>Commit</th><th>Summary</th><th>Last OK</th></tr>\n\n"

    now = datetime.utcnow().replace(microsecond=0)

    for s in sorted(config.sections(), key=lambda s: config.getint(s, "sortkey")):
        print("Working on summary entry of: " + s)
        name = config.get(s, "name")
        host = config.get(s, "host")
        report_url = config.get(s, "report_url")
        do_notify = config.getboolean(s, "notify", fallback=True)
        timeout = config.getint(s, "timeout", fallback=24)

        # find latest commit that should have been tested by now
        threshold = now - timedelta(hours=timeout)
        threshold_age = next(i for i, c in enumerate(log.commits) if c.date < threshold)

        # get and parse report
        report_txt = retrieve_report(report_url)
        report = parse_report(report_txt, log)

        if s not in status:
            status[s] = Status()

        if report.status == "OK":
            status[s].last_ok = report.sha
            status[s].notified = False
        elif do_notify and not status[s].notified:
            send_notification(report, status[s].last_ok, log, name, s, send_emails)
            status[s].notified = True

        if report.sha:
            age = log.index_by_sha[report.sha]
            if age > threshold_age:
                report.status = "OUTDATED"
            elif report.status in ("OK", "FAILED"):
                # store only useful and fresh reports, prevents overwriting archive
                assert report_txt
                store_report(report, report_txt, s, outdir)

        up_to_date = report.sha == log.commits[0].sha  # report from latest commit?
        output += '<tr align="center">'
        output += f'<td align="left"><a href="archive/{s}/index.html">{name}</a></td>'
        output += f'<td align="left">{host}</td>'
        output += status_cell(report.status, report_url, up_to_date)

        # Commit
        output += commit_cell(report.sha, log)

        # Summary
        output += f'<td align="left">{report.summary}</td>'

        # Last OK
        if report.status != "OK":
            output += commit_cell(status[s].last_ok, log)
        else:
            output += "<td></td>"

        output += "</tr>\n\n"

    output += "</table>\n"
    output += '<div id="dummybox"></div></div>\n'  # complete flex-container
    output += html_footer()
    write_file(outdir / "index.html", output)
    with open(status_fn, "wb") as f:
        pickle.dump(status, f)


# ======================================================================================
def gen_archive(config: ConfigParser, log: GitLog, outdir: Path) -> None:
    for s in config.sections():
        print(f"Working on archive page of: {s}")
        name = config.get(s, "name")
        info_url = config.get(s, "info_url", fallback=None)
        archive_dir = outdir / f"archive/{s}"
        archive_files = archive_dir.glob("commit_*.txt.gz")

        # read cache
        reports_cache: Dict[Path, Report] = {}
        cache_fn = f"{archive_dir}/reports.cache"
        if path.exists(cache_fn):
            with open(cache_fn, "rb") as f:
                reports_cache = pickle.load(f)
            cache_age = path.getmtime(cache_fn)
            # remove outdated cache entries
            reports_cache = {
                k: v for k, v in reports_cache.items() if path.getmtime(k) < cache_age
            }

        # read all archived reports
        archive_reports: Dict[GitSha, Report] = {}
        for fn in archive_files:
            report: Report
            if fn in reports_cache:
                report = reports_cache[fn]
            else:
                with open(fn, "rb") as f:
                    content = gzip.decompress(f.read())
                report = parse_report(content.decode("utf-8", errors="replace"), log)
                report.archive_path = path.basename(fn)[:-3]
                reports_cache[fn] = report
            assert report.sha and report.sha not in archive_reports
            archive_reports[report.sha] = report

        # write cache
        if reports_cache:
            with open(cache_fn, "wb") as f:
                pickle.dump(reports_cache, f)

        # loop over all relevant commits
        all_url_rows = []
        all_html_rows = []
        max_age_full = max([-1] + [log.index_by_sha[sha] for sha in archive_reports])
        for commit in log.commits[: max_age_full + 1]:
            sha = commit.sha
            html_row = "<tr>"
            html_row += commit_cell(sha, log)
            if sha in archive_reports:
                report = archive_reports[sha]
                assert report.archive_path
                html_row += status_cell(report.status, report.archive_path)
                html_row += f'<td align="left">{report.summary}</td>'
                archive_base_url = "https://dashboard.cp2k.org/archive"
                url_row = f"{archive_base_url}/{s}/{report.archive_path}.gz\n"
            else:
                html_row += 2 * "<td></td>"
                url_row = ""
            html_row += f'<td align="left">{html.escape(commit.author_name)}</td>'
            html_row += f'<td align="left">{html.escape(commit.message)}</td>'
            html_row += "</tr>\n\n"
            all_html_rows.append(html_row)
            all_url_rows.append(url_row)

        # generate html pages
        for full_archive in (False, True):
            if full_archive:
                html_out_postfix = "index_full.html"
                urls_out_postfix = "list_full.txt"
                toggle_link = '<p>View <a href="index.html">recent archive</a></p>'
                max_age = max_age_full
            else:
                html_out_postfix = "index.html"
                urls_out_postfix = "list_recent.txt"
                toggle_link = '<p>View <a href="index_full.html">full archive</a></p>'
                max_age = 100

            # generate archive index
            output = html_header(title=name)
            output += '<p>Go back to <a href="../../index.html">main page</a></p>\n'
            if info_url:
                output += f'<p>Get <a href="{info_url}">more information</a></p>\n'
            output += gen_plots(archive_reports, log, archive_dir, full_archive)
            output += toggle_link
            output += '<table border="1" cellspacing="3" cellpadding="5">\n'
            output += "<tr><th>Commit</th><th>Status</th><th>Summary</th>"
            output += "<th>Author</th><th>Commit Message</th></tr>\n\n"
            output += "".join(all_html_rows[: max_age + 1])
            output += "</table>\n"
            output += toggle_link
            output += html_footer()
            write_file(archive_dir / html_out_postfix, output)

            url_list = "".join(all_url_rows[: max_age + 1])
            write_file(archive_dir / urls_out_postfix, url_list)


# ======================================================================================
def gen_url_list(config: ConfigParser, outdir: Path) -> None:
    print("Working on url lists.")
    for postfix in ("list_full.txt", "list_recent.txt"):
        url_list = ""
        for s in config.sections():
            fn = outdir / f"archive/{s}/{postfix}"
            if not path.exists(fn):
                continue
            with open(fn, encoding="utf8") as f:
                url_list += f.read()
        write_file(outdir / f"archive/{postfix}", url_list)


# ======================================================================================
def gen_plots(
    archive_reports: Dict[GitSha, Report], log: GitLog, outdir: Path, full_archive: bool
) -> str:
    ordered_shas = [c.sha for c in log.commits]
    ordered_reports = [archive_reports[s] for s in ordered_shas if s in archive_reports]

    # aggregate plot data
    aggregated_plots: Dict[str, AggregatedPlot] = {}
    for report in ordered_reports:
        for p in report.plots:
            if p.name not in aggregated_plots:
                aggregated_plots[p.name] = AggregatedPlot(p)
        for pp in report.plot_points:
            plot = aggregated_plots[pp.plot_name]
            cname = pp.curve_name
            if cname not in plot.curves_by_name:
                plot.curves_by_name[cname] = PlotCurve(label=pp.curve_label)
            assert report.sha
            age = log.index_by_sha[report.sha]
            plot.curves_by_name[cname].x.append(-age)
            plot.curves_by_name[cname].y.append(pp.y)
            plot.curves_by_name[cname].yerr.append(pp.yerr)

    max_age_full = max([-1] + [log.index_by_sha[sha] for sha in archive_reports])
    max_age = max_age_full if full_archive else 100

    output = ""
    for pname, plot in aggregated_plots.items():
        print(f"Working on plot: {pname}")
        fn = f"{pname}_full" if full_archive else pname
        make_plot_data(outdir / Path(f"{fn}.txt"), plot, max_age, log)
        make_plot_image(outdir / Path(f"{fn}.png"), plot, max_age, full_archive)
        output += f'<a href="{fn}.txt"><img src="{fn}.png" alt="{plot.title}"></a>\n'

    return output


# ======================================================================================
def make_plot_data(fn: Path, plot: AggregatedPlot, max_age: int, log: GitLog) -> None:
    output = f"# {plot.title}\n"
    headers = [f"{c.label:>6}" for c in plot.curves]
    output += "# " + "\t".join(["age", "commit"] + headers) + "\n"
    for age in range(max_age + 1):
        values = []
        for curve, header in zip(plot.curves, headers):
            if -age in curve.x:
                i = curve.x.index(-age)
                value = "{:.1f}".format(curve.y[i])
            else:
                value = "?"
            values.append(value.rjust(len(header)))
        if any([v.strip() != "?" for v in values]):
            index_colums = [f"{-age:5d}", log.commits[age].sha[:6]]
            output += "\t".join(index_colums + values) + "\n"

    write_file(fn, output)


# ======================================================================================
def make_plot_image(
    fn: Path, plot: AggregatedPlot, max_age: int, full_archive: bool
) -> None:
    # Setup figure.
    fig = plt.figure(figsize=(12, 4))
    fig.subplots_adjust(bottom=0.18, left=0.06, right=0.70)
    fig.suptitle(plot.title, fontsize=14, fontweight="bold", x=0.4)
    ax = fig.add_subplot(111)
    ax.set_xlabel("Commit Age")
    ax.set_ylabel(plot.ylabel)
    markers = itertools.cycle("os>^*")

    # Draw curves.
    for curve in plot.curves:
        if full_archive:
            ax.plot(curve.x, curve.y, label=curve.label, linewidth=2)  # less crowded
        else:
            ax.errorbar(
                curve.x,
                curve.y,
                yerr=curve.yerr,
                label=curve.label,
                marker=next(markers),
                linewidth=2,
                markersize=6,
            )
    ax.set_xlim(-max_age - 1, 0)
    ax.xaxis.set_minor_locator(AutoMinorLocator())

    # Draw legend.
    legend_style = dict(numpoints=1, fancybox=True, shadow=True, borderaxespad=0.0)
    ax.legend(bbox_to_anchor=(1.01, 1), loc="upper left", **legend_style)

    # Determine y-range such that all curves are visible while ignoring outlayers.
    visibles = []
    for curve in plot.curves:
        ys = [y for x, y in zip(curve.x, curve.y) if x >= -max_age]  # visible y-values
        visibles += [ys] if ys else []  # ignore completely invisible curves
    if not visibles:
        print("Warning: Found no visible plot curve.")
    else:
        ymin = min([min(ys) for ys in visibles])  # lowest point from lowest curve
        ymax = max([max(ys) for ys in visibles])  # highest point from highest curve
        if full_archive:
            ax.set_ylim(0.98 * ymin, 1.02 * ymax)
        else:
            ymax2 = max([min(ys) for ys in visibles])  # lowest point from highest curve
            ax.set_ylim(0.98 * ymin, min(1.02 * ymax, 1.3 * ymax2))  # ignore outlayers

    # Save figure to file.
    fig.savefig(fn)
    plt.close(fig)


# ======================================================================================
def send_notification(
    report: Report,
    last_ok: Optional[GitSha],
    log: GitLog,
    name: str,
    s: str,
    send_emails: bool,
) -> None:
    idx_end = log.index_by_sha[report.sha] if report.sha else 0
    if not last_ok:
        return  # we don't know when this started
    idx_last_ok = log.index_by_sha[last_ok]
    if idx_end == idx_last_ok:
        return  # probably a flapping tester
    emails_raw = set(c.author_email for c in log.commits[idx_end:idx_last_ok])
    emails = [e for e in emails_raw if "noreply" not in e]
    emails_str = ", ".join(emails)
    if not emails:
        return  # no author emails found
    if len(emails) > 3:
        print("Spam protection, found more than three authors: " + emails_str)
        return
    if not send_emails:
        print("Email sending disabled, would otherwise send to: " + emails_str)
        return

    print("Sending email to: " + emails_str)

    msg_txt = "Dear CP2K developer,\n\n"
    msg_txt += "the dashboard has detected a problem"
    msg_txt += " that one of your recent commits might have introduced.\n\n"
    msg_txt += f"   test name:      {name}\n"
    msg_txt += f"   report state:   {report.status}\n"
    msg_txt += f"   report summary: {report.summary}\n"
    msg_txt += f"   last OK commit: {last_ok[:7]}\n\n"
    msg_txt += "For more information visit:\n"
    msg_txt += f"   https://dashboard.cp2k.org/archive/{s}/index.html \n\n"
    msg_txt += "Sincerely,\n"
    msg_txt += "  your CP2K Dashboard ;-)\n"

    msg = MIMEText(msg_txt)
    msg["Subject"] = f"Problem with {name}"
    msg["From"] = "CP2K Dashboard <dashboard@cp2k.org>"
    msg["To"] = ", ".join(emails)

    smtp_conn = smtplib.SMTP("localhost")
    smtp_conn.sendmail(msg["From"], emails, msg.as_string())
    smtp_conn.quit()


# ======================================================================================
def html_header(title: str) -> str:
    output = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">\n'
    output += "<html><head>\n"
    output += '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n'
    output += '<meta http-equiv="refresh" content="200">\n'
    output += '<link rel="icon" type="image/x-icon" href="data:image/x-icon;base64,AAABAAEAEBAQAAAAAAAoAQAAFgAAACgAAAAQAAAAIAAAAAEABAAAAAAAgAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAD/AAmRCQAAb/8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAiIgAAAAAAACIiAAAAAAAAIiIAAAAAAAAAAAAAAAAAADMzAAAAAAAAMzMAAAAAAAAzMwAAAAAAAAAAAAAAAAAAEREAAAAAAAAREQAAAAAAABERAAAAAAAAAAAAAAD+fwAA/n8AAPw/AAD4HwAA+B8AAPgfAAD4HwAA+B8AAPgfAAD4HwAA+B8AAPgfAAD4HwAA+B8AAPgfAAD4HwAA">\n'
    output += '<style type="text/css">\n'
    output += ".ribbon {\n"
    output += "  overflow: hidden;\n"
    output += "  position: absolute;\n"
    output += "  right:0px;\n"
    output += "  top: 0px;\n"
    output += "  width: 200px;\n"
    output += "  height: 200px;\n"
    output += "}\n"
    output += ".ribbon a {\n"
    output += "  position: relative;\n"
    output += "  white-space: nowrap;\n"
    output += "  background-color: #a00;\n"
    output += "  border: 1px solid #faa;\n"
    output += "  color: #fff;\n"
    output += "  display: block;\n"
    output += "  font: bold 11pt sans-serif;\n"
    output += "  padding: 7px;\n"
    output += "  top: 35px;\n"
    output += "  right: 10px;\n"
    output += "  width: 300px;\n"
    output += "  text-align: center;\n"
    output += "  text-decoration: none;\n"
    output += "  transform: rotate(45deg);\n"
    output += "  box-shadow: 0 0 10px #888;\n"
    output += "}\n"
    output += "#flex-container {\n"
    output += "  display: -webkit-flex; /* Safari */\n"
    output += "  display: flex;\n"
    output += "  -webkit-flex-flow: row wrap-reverse; /* Safari */\n"
    output += "  flex-flow:         row wrap-reverse;\n"
    output += "  -webkit-justify-content: space-around; /* Safari */\n"
    output += "  justify-content:         space-around;\n"
    output += "  -webkit-align-items: flex-end; /* Safari */\n"
    output += "  align-items:         flex-end;\n"
    output += "}\n"
    output += ".sidebox {\n"
    output += "  width: 15em;\n"
    output += "  border-radius: 1em;\n"
    output += "  box-shadow: .2em .2em .7em 0 #777;\n"
    output += "  background: #f7f7f0;\n"
    output += "  padding: 1em;\n"
    output += "  margin: 40px 20px;\n"
    output += "}\n"
    output += ".sidebox h2 {\n"
    output += "  margin: 0 0 0.5em 0;\n"
    output += "}\n"
    output += ".sidebox p {\n"
    output += "  margin: 0.5em;\n"
    output += "}\n"
    output += "#dummybox {\n"
    output += "  width: 15em;\n"
    output += "}\n"
    output += "</style>\n"
    output += f"<title>{title}</title>\n"
    output += "</head><body>\n"
    output += '<div class="ribbon"><a href="https://github.com/cp2k/cp2k/tree/master/tools/dashboard#readme">Need Help?</a></div>\n'
    output += f"<center><h1>{title.upper()}</h1></center>\n"
    return output


# ======================================================================================
def html_linkbox() -> str:
    output = '<div class="sidebox">\n'
    output += "<h2>More...</h2>\n"
    output += '<a href="regtest_survey.html">Regtest Survey</a><br>\n'
    output += '<a href="https://www.cp2k.org/static/coverage/">Test Coverage</a><br>\n'
    output += '<a href="discontinued_tests.html">Discontinued Tests</a><br>\n'
    output += '<a href="https://www.cp2k.org/dev:compiler_support/">Supported compilers</a><br>\n'
    output += "</div>\n"
    return output


# ======================================================================================
def html_gitbox(log: GitLog) -> str:
    now = datetime.utcnow()
    output = '<div class="sidebox">\n'
    output += "<h2>Recent Commits</h2>\n"
    for commit in log.commits[0:10]:
        url = "https://github.com/cp2k/cp2k/commit/" + commit.sha
        autor_name = html.escape(commit.author_name)
        title = html.escape(commit.message)
        cm = commit.message if len(commit.message) < 28 else commit.message[:26] + "..."
        output += f'<p><a title="{title}" href="{url}">{html.escape(cm)}</a><br>\n'
        delta = now - commit.date
        age = delta.days * 24.0 + delta.seconds / 3600.0
        output += "<small>git:" + commit.sha[:7]
        output += f"<br>\n{autor_name} {age:.1f}h ago.</small></p>\n"
    output += "</div>\n"
    return output


# ======================================================================================
def html_footer() -> str:
    now = datetime.utcnow().replace(microsecond=0)
    output = f"<p><small>Page last updated: {now.isoformat()}</small></p>\n"
    output += "</body></html>"
    return output


# ======================================================================================
def write_file(fn: Path, content: str, gz: bool = False) -> None:
    d = fn.parent
    if not d.exists():
        os.makedirs(d)
        print(f"Created dir: {d}")
    if fn.exists():
        with open(fn, "rb") as f:
            old_bytes = gzip.decompress(f.read()) if gz else f.read()
        old_content = old_bytes.decode("utf-8", errors="replace")
        if old_content == content:
            print(f"File did not change: {fn}")
            return

    encoded_content = content.encode("utf-8")
    with open(fn, "wb") as f:
        f.write(gzip.compress(encoded_content) if gz else encoded_content)
    print(f"Wrote: {fn}")


# ======================================================================================
def status_cell(status: ReportStatus, report_url: str, up_to_date: bool = True) -> str:
    if status == "OK":
        bgcolor = "#00FF00" if (up_to_date) else "#8CE18C"
    elif status == "FAILED":
        bgcolor = "#FF0000" if (up_to_date) else "#E18C8C"
    else:
        bgcolor = "#d3d3d3"
    return f'<td bgcolor="{bgcolor}"><a href="{report_url}">{status}</a></td>'


# ======================================================================================
def commit_cell(sha: Optional[GitSha], log: GitLog) -> str:
    if sha is None:
        return "<td>N/A</td>"
    idx = log.index_by_sha[sha]
    commit = log.commits[idx]
    github_url = "https://github.com/cp2k/cp2k/commit/" + sha
    return f'<td align="left"><a href="{github_url}">{sha[:7]}</a> ({-idx})</td>'


# ======================================================================================
def retrieve_report(url: str) -> Optional[str]:
    try:
        # see if we have a cached entry
        h = hashlib.md5(url.encode("utf8")).hexdigest()
        etag_file = Path(f"/tmp/dashboard_retrieval_cache_{h}.etag")
        data_file = Path(f"/tmp/dashboard_retrieval_cache_{h}.data")
        etag = etag_file.read_text() if etag_file.exists() else ""

        # make conditional http request
        r = requests.get(url, headers={"If-None-Match": etag}, timeout=5)
        r.raise_for_status()
        if r.status_code == 304:  # Not Modified - cache hit
            return data_file.read_text()

        # check report size
        report_size = int(r.headers["Content-Length"])
        assert report_size < 3 * 1024 * 1024  # 3 MB

        # download full report
        report_text = r.text.replace("\r\n", "\n").replace("\r", "\n")

        # cache miss - store response
        if "ETag" in r.headers:
            data_file.write_text(report_text)
            etag_file.write_text(r.headers["ETag"])
        return report_text

    except Exception:
        traceback.print_exc()
        return None


# ======================================================================================
def store_report(report: Report, report_txt: str, section: str, outdir: Path) -> None:
    fn = outdir / f"archive/{section}/commit_{report.sha}.txt.gz"
    write_file(fn, report_txt, gz=True)


# ======================================================================================
def parse_report(report_txt: Optional[str], log: GitLog) -> Report:
    if report_txt is None:
        return Report(status="UNKNOWN", summary="Error while retrieving report.")
    try:
        m = re.search(r"^CommitSHA: (\w{40})$", report_txt, re.MULTILINE)
        sha = GitSha(m.group(1)) if m else None
        summary = re.findall(r"^Summary: (.+)$", report_txt, re.MULTILINE)[-1]
        status = re.findall(r"^Status: (.+)$", report_txt, re.MULTILINE)[-1]
        plot_matches = re.findall(r"^Plot: (.+)$", report_txt, re.MULTILINE)
        plots = [cast(Plot, eval(f"Plot({m})")) for m in plot_matches]
        point_matches = re.findall(r"^PlotPoint: (.+)$", report_txt, re.MULTILINE)
        points = [cast(PlotPoint, eval(f"PlotPoint({m})")) for m in point_matches]

        # Check that every plot has at least one PlotPoint
        for plot in plots:
            if not any([p.plot_name == plot.name for p in points]):
                return Report("FAILED", f"Plot {plot.name} has no PlotPoints.")

        # Check that CommitSHA belongs to the master branch.
        if sha not in log.index_by_sha:
            return Report("FAILED", "Unknown CommitSHA.")

        return Report(status, summary, sha=sha, plots=plots, plot_points=points)
    except Exception:
        traceback.print_exc()
        return Report("UNKNOWN", "Error while parsing report.")


# ======================================================================================
main()

# EOF
