groups:
- name: janitor.rules
  rules:
  - alert: JanitorRunnerDown
    expr: up{job="janitor.runner"} == 0
    for: 1h
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: Job {{ $labels.job }} instance {{ $labels.instance }} is down.
      summary: Janitor runner is down
  - alert: JanitorPublisherDown
    expr: up{job="janitor.publisher"} == 0
    for: 1h
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: Job {{ $labels.job }} instance {{ $labels.instance }} is down.
      summary: Janitor publisher is down
  - alert: JanitorSiteDown
    expr: up{job="janitor.site"} == 0
    for: 1h
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: Job {{ $labels.job }} instance {{ $labels.instance }} is down.
      summary: Janitor site is down
  - alert: JanitorDifferDown
    expr: up{job="janitor.differ"} == 0
    for: 1h
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: Job {{ $labels.job }} instance {{ $labels.instance }} is down.
      summary: Janitor differ is down
  - alert: JanitorArchiverDown
    expr: up{job="janitor.archiver"} == 0
    for: 1h
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: Job {{ $labels.job }} instance {{ $labels.instance }} is down.
      summary: Janitor archiver is down
  - alert: JanitorArchiverDown
    expr: up{job="janitor.archiver"} == 0
    for: 1h
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: Job {{ $labels.job }} instance {{ $labels.instance }} is down.
      summary: Janitor archiver is down
  - alert: JanitorScheduleNotRunning
    expr: ((time() - job_last_success_unixtime{exported_job="janitor.schedule"}) / 3600) > 48
    for: 1h
    annotations:
      summary: janitor.schedule not running
  - alert: JanitorIrcRelayDown
    expr: up{job="irc_relay"} == 0
    for: 1h
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: Job {{ $labels.job }} instance {{ $labels.instance }} is down.
      summary: Prometheus IRC relay is down
  - alert: JanitorIrcNotifyDown
    expr: up{job="irc_notify"} == 0
    for: 1h
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: Job {{ $labels.job }} instance {{ $labels.instance }} is down.
      summary: Janitor IRC notifier is down
  - alert: JanitorMastodonNotifyDown
    expr: up{job="mastodon_notify"} == 0
    for: 1h
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: Job {{ $labels.job }} instance {{ $labels.instance }} is down.
      summary: Janitor Mastodon notifier is down

  - alert: JanitorVcsStoreDown
    expr: up{job="janitor.vcs_store"} == 0
    for: 1h
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: Job {{ $labels.job }} instance {{ $labels.instance }} is down.
      summary: Janitor vcs store is down

  - alert: JanitorRunFinishFailing
    expr: increase(request_exceptions_total{job="janitor.runner",route="finish"}[5m]) > 0
    for: 0m
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: Finish requests to the runner {{ $labels.instance }} are failing.
      summary: Run finish requests are failing

  - alert: JanitorRunAssignFailing
    expr: increase(request_exceptions_total{job="janitor.runner",route="assign"}[5m]) > 0
    for: 0m
    labels:
      severity: critical
      project: debian-janitor
    annotations:
      description: "Assign requests to the runner {{ $labels.instance }} are failing."
      summary: Run assign requests are failing

#  - alert: JanitorHighRequestFailureRate
#    expr: increase(request_exceptions_total[30m]) > 5
#    for: 0m
#    labels:
#      severity: critical
#      project: debian-janitor
#    annotations:
#      description: "{{ $labels.route }} requests to {{ $labels.instance }} are failing."
#      summary: {{ $labels.route }} requests are failing
#
#  - alert: JanitorRequestFailures
#    expr: increase(request_exceptions_total[30m]) > 0
#    for: 0m
#    labels:
#      severity: warning
#      project: debian-janitor
#    annotations:
#      description: "{{ $labels.route }} requests to {{ $labels.instance }} are failing."
#      summary: {{ $labels.route }} requests are failing

  - alert: JobRestarting
    expr: avg without(instance)(changes(process_start_time_seconds[1h])) > 3
    for: 10m
    labels:
      severity: critical
      project: debian-janitor

  - alert: SiteDown
    expr: probe_http_status_code{job="blackbox_https"} != 200
    for: 5m
    labels:
      severity: critical
      project: debian-janitor

  - alert: WorkerCrashing
    expr: avg without(instance)(changes(run{result_code="worker-failure"}[10m])) > 5
    for: 10m
    labels:
      severity: critical
      project: debian-janitor

#  - alert: UploadsFailing
#    expr: increase(upload_failed_total) > 0
#    labels:
#      severity: warning
#      project: debian-janitor
#    annotations:
#      description: "Some uploads are failing {{ $labels.instance }}"
#      summary: "Some uploads are failing"

#  - alert: UploadSignFailing
#    expr: increase(debsign_failed_total) > 0
#    labels:
#      severity: warning
#      project: debian-janitor
#    annotations:
#      description: "Some debsign commands are failing {{ $labels.instance }}"
#      summary: "Some debsign commands are failing"
