/*
  This file is part of TALER
  Copyright (C) 2017-2024 Taler Systems SA

  TALER is free software; you can redistribute it and/or modify it under the
  terms of the GNU General Public License as published by the Free Software
  Foundation; either version 3, or (at your option) any later version.

  TALER is distributed in the hope that it will be useful, but WITHOUT ANY
  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  A PARTICULAR PURPOSE.  See the GNU General Public License for more details.

  You should have received a copy of the GNU General Public License along with
  TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
*/
/**
 * @file auditor/taler-helper-auditor-transfer.c
 * @brief audits that deposits past due date are
 *    aggregated and have a matching wire transfer
 * database.
 *
 * Three checks, each with its own cursor over the exchange's tables:
 *
 * 1) every batch deposit must eventually be aggregated (`total_amount_lag`),
 * 2) nothing may be aggregated before there is a deposit to justify it
 *    (`total_early_aggregation`),
 * 3) every aggregation must eventually turn into a wire transfer.
 *
 * The third is what keeps "the exchange paid the merchant" apart from "the
 * exchange decided what it owes and then sat on the money": an
 * `aggregation_tracking` row is not proof of payment, a `wire_out` row is.
 * What is still owed is split by the reason the exchange gave for not paying
 * -- `total_aml_hold` for an open KYC requirement, `total_small_aggregate` for
 * an aggregate that does not yet cover its wire fee, and `total_transfer_lag`
 * for a transfer the exchange never gave any reason for -- and each held
 * transfer is listed in `auditor_aml_holds`.
 *
 * @author Christian Grothoff
 */
#include "platform.h"
#include <gnunet/gnunet_util_lib.h>
#include <gnunet/gnunet_curl_lib.h>
#include "auditordb_lib.h"
#include "exchangedb_lib.h"
#include "taler/taler_json_lib.h"
#include "report-lib.h"
#include "taler/taler_dbevents.h"
#include "auditor-database/delete_aml_hold.h"
#include "auditor-database/delete_early_aggregation.h"
#include "auditor-database/delete_pending_deposit.h"
#include "auditor-database/event_listen.h"
#include "auditor-database/get_auditor_progress.h"
#include "auditor-database/get_balance.h"
#include "auditor-database/insert_aml_hold.h"
#include "auditor-database/insert_amount_arithmetic_inconsistency.h"
#include "auditor-database/insert_auditor_progress.h"
#include "auditor-database/insert_balance.h"
#include "auditor-database/insert_early_aggregation.h"
#include "auditor-database/insert_pending_deposit.h"
#include "auditor-database/insert_row_inconsistency.h"
#include "auditor-database/preflight.h"
#include "auditor-database/start.h"
#include "auditor-database/update_aml_hold.h"
#include "exchange-database/get_aggregation_deferral_by_wtid.h"
#include "exchange-database/get_aggregation_transient_by_wtid.h"
#include "exchange-database/get_open_legitimization_measure.h"
#include "exchange-database/get_pending_aggregation.h"
#include "exchange-database/preflight.h"
#include "exchange-database/rollback.h"
struct AggregationContext;
#define TALER_EXCHANGEDB_AGGREGATION_RESULT_CLOSURE struct AggregationContext
#include "exchange-database/iterate_aggregations_above_serial_id.h"
struct ImportMissingWireContext;
#define TALER_EXCHANGEDB_WIRE_MISSING_RESULT_CLOSURE struct \
        ImportMissingWireContext
#include "exchange-database/iterate_batch_deposits_missing_wire.h"
struct HoldContext;
#define TALER_EXCHANGEDB_AGGREGATION_WTID_RESULT_CLOSURE struct HoldContext
#include "exchange-database/iterate_aggregation_wtids_above_serial_id.h"
#define TALER_AUDITORDB_AML_HOLD_RESULT_CLOSURE struct HoldContext
#include "auditor-database/iterate_aml_holds.h"
#include "exchange-database/start_read_only.h"


/**
 * Run in test mode. Exit when idle instead of
 * going to sleep and waiting for more work.
 */
static int test_mode;

/**
 * Return value from main().
 */
static int global_ret;

/**
 * Last reserve_out / wire_out serial IDs seen.
 */
static TALER_ARL_DEF_PP (wire_batch_deposit_id);
static TALER_ARL_DEF_PP (wire_aggregation_id);

/**
 * Row of `aggregation_tracking' up to which we have looked for wire transfers
 * the exchange aggregated but did not execute.
 */
static TALER_ARL_DEF_PP (wire_hold_aggregation_id);

/**
 * Total amount which the exchange did not aggregate/transfer in time.
 */
static TALER_ARL_DEF_AB (total_amount_lag);

/**
 * Total amount which the exchange did aggregate/transfer too early.
 */
static TALER_ARL_DEF_AB (total_early_aggregation);

/**
 * Total amount the exchange aggregated but did not wire because it says a
 * legitimization requirement against the recipient is still open.  Money the
 * exchange is withholding for legal reasons.
 */
static TALER_ARL_DEF_AB (total_aml_hold);

/**
 * Total amount the exchange aggregated but did not wire because what it has
 * collected for the account so far does not cover the wire fee.  Money that is
 * waiting for the next deposit rather than for anyone's decision, and thus not
 * an irregularity at all -- it is kept apart so that it does not inflate the
 * two balances that are.
 */
static TALER_ARL_DEF_AB (total_small_aggregate);

/**
 * Total amount the exchange aggregated but did not wire and gave no reason
 * for.  Money the exchange simply has not paid out.
 */
static TALER_ARL_DEF_AB (total_transfer_lag);

/**
 * Should we run checks that only work for exchange-internal audits?
 */
static int internal_checks;

/**
 * Database event handler to wake us up again.
 */
static struct GNUNET_DB_EventHandler *eh;

/**
 * The auditors's configuration.
 */
static const struct GNUNET_CONFIGURATION_Handle *cfg;


/**
 * Task run on shutdown.
 *
 * @param cls NULL
 */
static void
do_shutdown (void *cls)
{
  (void) cls;
  if (NULL != eh)
  {
    TALER_AUDITORDB_event_listen_cancel (eh);
    eh = NULL;
  }
  TALER_ARL_done ();
  TALER_EXCHANGEDB_unload_accounts ();
  TALER_ARL_cfg = NULL;
}


/**
 * Closure for import_wire_missing_cb().
 */
struct ImportMissingWireContext
{
  /**
   * Set to maximum row ID encountered.
   */
  uint64_t max_batch_deposit_uuid;

  /**
   * Set to database errors in callback.
   */
  enum GNUNET_DB_QueryStatus err;
};


/**
 * Function called on deposits that need to be checked for their
 * wire transfer.
 *
 * @param wc closure, points to a `struct ImportMissingWireContext`
 * @param batch_deposit_serial_id serial of the entry in the batch deposits table
 * @param total_amount value of the missing deposits, including fee
 * @param wire_target_h_payto where should the funds be wired
 * @param deadline what was the earliest requested wire transfer deadline
 */
static void
import_wire_missing_cb (
  struct ImportMissingWireContext *wc,
  uint64_t batch_deposit_serial_id,
  const struct TALER_Amount *total_amount,
  const struct TALER_FullPaytoHashP *wire_target_h_payto,
  struct GNUNET_TIME_Timestamp deadline)
{
  enum GNUNET_DB_QueryStatus qs;

  if (wc->err < 0)
    return; /* already failed */
  GNUNET_assert (batch_deposit_serial_id >= wc->max_batch_deposit_uuid);
  wc->max_batch_deposit_uuid = batch_deposit_serial_id + 1;
  qs = TALER_AUDITORDB_delete_early_aggregation (
    TALER_ARL_adb,
    batch_deposit_serial_id);
  switch (qs)
  {
  case GNUNET_DB_STATUS_SOFT_ERROR:
    GNUNET_break (0);
    wc->err = qs;
    return;
  case GNUNET_DB_STATUS_HARD_ERROR:
    GNUNET_break (0);
    wc->err = qs;
    return;
  case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS:
    qs = TALER_AUDITORDB_insert_pending_deposit (
      TALER_ARL_adb,
      batch_deposit_serial_id,
      wire_target_h_payto,
      total_amount,
      deadline);
    if (0 > qs)
    {
      GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
      wc->err = qs;
      return;
    }
    if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs)
      TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_amount_lag),
                            &TALER_ARL_USE_AB (total_amount_lag),
                            total_amount);
    break;
  case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT:
    TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_early_aggregation),
                               &TALER_ARL_USE_AB (total_early_aggregation),
                               total_amount);
    break;
  default:
    GNUNET_assert (0);
  }
}


/**
 * Checks for wire transfers that should have happened.
 *
 * @return transaction status
 */
static enum GNUNET_DB_QueryStatus
check_for_required_transfers (void)
{
  enum GNUNET_DB_QueryStatus qs;
  struct ImportMissingWireContext wc = {
    .max_batch_deposit_uuid = TALER_ARL_USE_PP (wire_batch_deposit_id),
    .err = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT
  };

  qs = TALER_EXCHANGEDB_iterate_batch_deposits_missing_wire (
    TALER_ARL_edb,
    TALER_ARL_USE_PP (wire_batch_deposit_id),
    &import_wire_missing_cb,
    &wc);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    return qs;
  }
  if (0 > wc.err)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == wc.err);
    return wc.err;
  }
  TALER_ARL_USE_PP (wire_batch_deposit_id) = wc.max_batch_deposit_uuid;
  return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
}


/**
 * Closure for #clear_finished_transfer_cb().
 */
struct AggregationContext
{
  /**
   * Set to maximum row ID encountered.
   */
  uint64_t max_aggregation_serial;

  /**
   * Set to database errors in callback.
   */
  enum GNUNET_DB_QueryStatus err;
};


/**
 * Function called on aggregations that were done for
 * a (batch) deposit.
 *
 * @param ac closure
 * @param amount affected amount
 * @param tracking_serial_id where in the table are we
 * @param batch_deposit_serial_id which batch deposit was aggregated
 */
static void
clear_finished_transfer_cb (
  struct AggregationContext *ac,
  const struct TALER_Amount *amount,
  uint64_t tracking_serial_id,
  uint64_t batch_deposit_serial_id)
{
  enum GNUNET_DB_QueryStatus qs;

  if (0 > ac->err)
    return; /* already failed */
  GNUNET_assert (ac->max_aggregation_serial <= tracking_serial_id);
  ac->max_aggregation_serial = tracking_serial_id + 1;
  qs = TALER_AUDITORDB_delete_pending_deposit (
    TALER_ARL_adb,
    batch_deposit_serial_id);
  switch (qs)
  {
  case GNUNET_DB_STATUS_SOFT_ERROR:
    GNUNET_break (0);
    ac->err = qs;
    return;
  case GNUNET_DB_STATUS_HARD_ERROR:
    GNUNET_break (0);
    ac->err = qs;
    return;
  case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS:
    qs = TALER_AUDITORDB_insert_early_aggregation (
      TALER_ARL_adb,
      batch_deposit_serial_id,
      tracking_serial_id,
      amount);
    if (0 > qs)
    {
      GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
      ac->err = qs;
      return;
    }
    if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs)
      TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_early_aggregation),
                            &TALER_ARL_USE_AB (total_early_aggregation),
                            amount);
    break;
  case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT:
    TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_amount_lag),
                               &TALER_ARL_USE_AB (total_amount_lag),
                               amount);
    break;
  default:
    GNUNET_assert (0);
  }
}


/**
 * Checks that all wire transfers that should have happened
 * (based on deposits) have indeed happened.
 *
 * @return transaction status
 */
static enum GNUNET_DB_QueryStatus
check_for_completed_transfers (void)
{
  struct AggregationContext ac = {
    .max_aggregation_serial = TALER_ARL_USE_PP (wire_aggregation_id),
    .err = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT
  };
  enum GNUNET_DB_QueryStatus qs;

  qs = TALER_EXCHANGEDB_iterate_aggregations_above_serial_id (
    TALER_ARL_edb,
    TALER_ARL_USE_PP (wire_aggregation_id),
    &clear_finished_transfer_cb,
    &ac);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    return qs;
  }
  if (0 > ac.err)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == ac.err);
    return ac.err;
  }
  TALER_ARL_USE_PP (wire_aggregation_id) = ac.max_aggregation_serial;
  return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
}


/**
 * Closure for #note_held_transfer_cb() and #evaluate_hold_cb().
 */
struct HoldContext
{
  /**
   * Wire transfers already evaluated in this round, so that the three
   * `aggregation_tracking` rows of one transfer cost one evaluation, and so
   * that the pass over the holds already on file does not redo them.
   */
  struct GNUNET_CONTAINER_MultiHashMap *seen;

  /**
   * Sum of the holds the exchange attributes to an open KYC requirement.
   */
  struct TALER_Amount aml_hold;

  /**
   * Sum of the holds the exchange attributes to the aggregate being too small
   * to be worth its wire fee.
   */
  struct TALER_Amount small_aggregate;

  /**
   * Sum of the holds the exchange gave no reason for.
   */
  struct TALER_Amount transfer_lag;

  /**
   * Set to maximum row ID encountered.
   */
  uint64_t max_aggregation_serial;

  /**
   * Set to database errors in the callback.
   */
  enum GNUNET_DB_QueryStatus err;
};


/**
 * Report a row inconsistency.
 *
 * @param[in,out] hc our state, to flag database failures in
 * @param table name of the exchange table the bad row is in
 * @param rowid row that is bad, 0 if the finding is not about one row
 * @param diagnostic what is wrong with it
 */
static void
report_row (struct HoldContext *hc,
            const char *table,
            uint64_t rowid,
            const char *diagnostic)
{
  struct TALER_AUDITORDB_RowInconsistency ri = {
    .row_id = rowid,
    .row_table = (char *) table,
    .diagnostic = (char *) diagnostic
  };
  enum GNUNET_DB_QueryStatus qs;

  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
              "Row %llu of `%s' is bad: %s\n",
              (unsigned long long) rowid,
              table,
              diagnostic);
  qs = TALER_AUDITORDB_insert_row_inconsistency (TALER_ARL_adb,
                                                 &ri);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    hc->err = qs;
  }
}


/**
 * Work out whether the exchange still owes the wire transfer @a wtid, how much
 * it owes and what it says is keeping it from paying; then record the result
 * and count it towards the running totals.
 *
 * The classification is the exchange's own, read out of `aggregation_deferrals`
 * -- append-only, and therefore the one statement of its reasons that survives
 * replication to an external auditor.  Taking it at face value is deliberate:
 * whether a hold is *justified* is a question for the people reading
 * `/monitoring/aml-holds`, not for the auditor.  What the auditor does check is
 * that the claim is consistent with everything else it can see: the amount
 * against its own recomputation, a KYC claim against the legitimization tables,
 * and under `-i` the whole thing against the live `aggregation_transient`.  A
 * disagreement is reported rather than resolved in either side's favour.
 *
 * A transfer with no `aggregation_deferrals` row at all is a transfer the
 * exchange decided on, did not make, and never explained; that is
 * `total_transfer_lag`.
 *
 * @param[in,out] hc our state
 * @param wtid wire transfer to examine
 */
static void
evaluate_hold (struct HoldContext *hc,
               const struct TALER_WireTransferIdentifierRawP *wtid)
{
  struct TALER_FullPaytoHashP h_payto;
  struct TALER_FullPayto payto = { NULL };
  struct TALER_NormalizedPaytoHashP h_normalized_payto;
  struct TALER_Amount deposited;
  struct TALER_Amount refunded;
  struct TALER_Amount deposit_fee;
  struct TALER_Amount deductions;
  struct TALER_Amount amount;
  struct TALER_Amount claimed;
  struct GNUNET_TIME_Timestamp deferral_time;
  struct GNUNET_TIME_Absolute measure_start;
  enum TALER_EXCHANGEDB_DeferralReason reason = TALER_EXCHANGEDB_DR_NONE;
  uint64_t claimed_measure = 0;
  uint64_t measure = 0;
  bool have_claim;
  bool open_measure;
  enum GNUNET_DB_QueryStatus qs;

  qs = TALER_EXCHANGEDB_get_pending_aggregation (TALER_ARL_edb,
                                                 wtid,
                                                 &h_payto,
                                                 &payto,
                                                 &deposited,
                                                 &refunded,
                                                 &deposit_fee);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    hc->err = qs;
    return;
  }
  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
  {
    /* The `wire_out' row has appeared: the money left the exchange, so the
       hold ends and stops counting towards either total.  This is also the
       only place a hold is ever cleared -- there is no separate pass over
       `wire_out`, because a transfer that was executed simply stops being a
       pending aggregation. */
    qs = TALER_AUDITORDB_delete_aml_hold (TALER_ARL_adb,
                                          wtid);
    if (0 > qs)
    {
      GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
      hc->err = qs;
    }
    return;
  }

  /* What the exchange still owes, computed the way `exchange_do_aggregate'
     computes it, so that it is comparable with what the exchange itself
     recorded. */
  TALER_ARL_amount_add (&deductions,
                        &refunded,
                        &deposit_fee);
  if (TALER_ARL_SR_INVALID_NEGATIVE ==
      TALER_ARL_amount_subtract_neg (&amount,
                                     &deposited,
                                     &deductions))
  {
    char *diag;

    GNUNET_asprintf (&diag,
                     "refunds and deposit fees exceed the deposits aggregated"
                     " into wire transfer %s",
                     TALER_B2S (wtid));
    report_row (hc,
                "aggregation_tracking",
                0,
                diag);
    GNUNET_free (diag);
    GNUNET_assert (GNUNET_OK ==
                   TALER_amount_set_zero (TALER_ARL_currency,
                                          &amount));
  }

  /* What the exchange says about it.  Absent for a transfer it never
     explained, and for one deferred by a version of the exchange from before
     `aggregation_deferrals' existed. */
  qs = TALER_EXCHANGEDB_get_aggregation_deferral_by_wtid (TALER_ARL_edb,
                                                          wtid,
                                                          &claimed,
                                                          &reason,
                                                          &claimed_measure,
                                                          &deferral_time);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    hc->err = qs;
    GNUNET_free (payto.full_payto);
    return;
  }
  have_claim = (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs);
  if (! have_claim)
  {
    reason = TALER_EXCHANGEDB_DR_NONE;
    claimed_measure = 0;
  }

  /* `legitimization_measures' is reached through the account's *normalized*
     payto hash, and `wire_targets.h_normalized_payto' is not replicated, so
     the normalization has to happen here rather than in the query. */
  TALER_full_payto_normalize_and_hash (payto,
                                       &h_normalized_payto);
  qs = TALER_EXCHANGEDB_get_open_legitimization_measure (
    TALER_ARL_edb,
    &h_normalized_payto,
    &measure,
    &measure_start);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    hc->err = qs;
    GNUNET_free (payto.full_payto);
    return;
  }
  open_measure = (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs);
  if (! open_measure)
    measure = 0;
  switch (reason)
  {
  case TALER_EXCHANGEDB_DR_NONE:
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Exchange holds %s for %s and says nothing about why\n",
                TALER_amount2s (&amount),
                payto.full_payto);
    break;
  case TALER_EXCHANGEDB_DR_AMOUNT_TOO_SMALL:
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Exchange holds %s for %s, too little to cover the wire fee\n",
                TALER_amount2s (&amount),
                payto.full_payto);
    break;
  case TALER_EXCHANGEDB_DR_KYC:
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Exchange holds %s for %s awaiting KYC measure %llu\n",
                TALER_amount2s (&amount),
                payto.full_payto,
                (unsigned long long) claimed_measure);
    break;
  }

  if (have_claim &&
      (0 != TALER_amount_cmp (&claimed,
                              &amount)) )
  {
    struct TALER_AUDITORDB_AmountArithmeticInconsistency aai = {
      .problem_row_id = 0,
      .operation = (char *) "aggregation deferral",
      .exchange_amount = claimed,
      .auditor_amount = amount,
      /* claiming to hold less than it owes understates the exchange's
         liabilities, which is the direction that favours the exchange */
      .profitable = (0 > TALER_amount_cmp (&claimed,
                                           &amount))
    };

    qs = TALER_AUDITORDB_insert_amount_arithmetic_inconsistency (
      TALER_ARL_adb,
      &aai);
    if (0 > qs)
    {
      GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
      hc->err = qs;
    }
  }

  if ( (TALER_EXCHANGEDB_DR_KYC == reason) &&
       (! open_measure) )
  {
    char *diag;

    /* The one claim the auditor can check on its own: a legitimization
       requirement is a row in `legitimization_measures' that no
       `legitimization_processes' row carried through to a decision, and that
       is replicated.  Claiming KYC without one is how an exchange would dress
       up simply keeping the money. */
    GNUNET_asprintf (&diag,
                     "exchange withholds wire transfer %s for legitimization"
                     " measure %llu, which is not open",
                     TALER_B2S (wtid),
                     (unsigned long long) claimed_measure);
    report_row (hc,
                "aggregation_deferrals",
                claimed_measure,
                diag);
    GNUNET_free (diag);
  }

  if (internal_checks)
  {
    struct TALER_Amount transient;
    uint64_t transient_measure;

    /* `aggregation_transient' is the live version of what the deferral row
       claims: updated in place, deleted on payout, and unreplicable.  On an
       internal audit we can hold the two against each other, which is what
       catches a `aggregation_deferrals' row that was written once and then
       left behind by an aggregate that has moved on. */
    qs = TALER_EXCHANGEDB_get_aggregation_transient_by_wtid (
      TALER_ARL_edb,
      &h_payto,
      wtid,
      &transient,
      &transient_measure);
    if (0 > qs)
    {
      GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
      hc->err = qs;
      GNUNET_free (payto.full_payto);
      return;
    }
    if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
    {
      char *diag;

      /* The exchange marked the deposits as aggregated, did not transfer the
         money, and is not tracking that it owes it either. */
      GNUNET_asprintf (&diag,
                       "wire transfer %s was aggregated but is neither in"
                       " wire_out nor in aggregation_transient",
                       TALER_B2S (wtid));
      report_row (hc,
                  "aggregation_transient",
                  0,
                  diag);
      GNUNET_free (diag);
    }
    else
    {
      if (0 !=
          TALER_amount_cmp (&transient,
                            &amount))
      {
        struct TALER_AUDITORDB_AmountArithmeticInconsistency aai = {
          .problem_row_id = 0,
          .operation = (char *) "aggregation transient",
          .exchange_amount = transient,
          .auditor_amount = amount,
          /* claiming to hold less than it owes understates the exchange's
             liabilities, which is the direction that favours the exchange */
          .profitable = (0 > TALER_amount_cmp (&transient,
                                               &amount))
        };

        qs = TALER_AUDITORDB_insert_amount_arithmetic_inconsistency (
          TALER_ARL_adb,
          &aai);
        if (0 > qs)
        {
          GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
          hc->err = qs;
        }
      }
      if ( (TALER_EXCHANGEDB_DR_KYC == reason) !=
           (0 != transient_measure) )
      {
        char *diag;

        GNUNET_asprintf (&diag,
                         "exchange is tracking wire transfer %s as %s but its"
                         " last aggregation_deferrals row says %s",
                         TALER_B2S (wtid),
                         (0 != transient_measure)
                         ? "awaiting legitimization"
                         : "not awaiting legitimization",
                         (TALER_EXCHANGEDB_DR_KYC == reason)
                         ? "it is"
                         : "it is not");
        report_row (hc,
                    "aggregation_transient",
                    transient_measure,
                    diag);
        GNUNET_free (diag);
      }
    }
  }

  qs = TALER_AUDITORDB_insert_aml_hold (TALER_ARL_adb,
                                        wtid,
                                        &h_payto,
                                        payto,
                                        &amount,
                                        (uint32_t) reason,
                                        claimed_measure);
  GNUNET_free (payto.full_payto);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    hc->err = qs;
    return;
  }
  /* Insert-then-update, not "insert if new": the amount grows as further
     deposits join the transfer, and the reason the exchange gives for holding
     it can change long after the hold began. */
  qs = TALER_AUDITORDB_update_aml_hold (TALER_ARL_adb,
                                        wtid,
                                        &amount,
                                        (uint32_t) reason,
                                        claimed_measure);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    hc->err = qs;
    return;
  }
  {
    struct TALER_Amount *bucket;

    switch (reason)
    {
    case TALER_EXCHANGEDB_DR_AMOUNT_TOO_SMALL:
      bucket = &hc->small_aggregate;
      break;
    case TALER_EXCHANGEDB_DR_KYC:
      bucket = &hc->aml_hold;
      break;
    case TALER_EXCHANGEDB_DR_NONE:
    default:
      bucket = &hc->transfer_lag;
      break;
    }
    TALER_ARL_amount_add (bucket,
                          bucket,
                          &amount);
  }
}


/**
 * Function called on each row of the aggregation tracking table.
 *
 * @param hc closure
 * @param rowid row of the entry in the aggregation tracking table
 * @param wtid wire transfer the deposit was aggregated into
 * @param wire_target_h_payto account the wire transfer should go to
 * @param pending false if the exchange really did make the transfer
 * @return #GNUNET_OK to continue to iterate
 */
static enum GNUNET_GenericReturnValue
note_held_transfer_cb (
  struct HoldContext *hc,
  uint64_t rowid,
  const struct TALER_WireTransferIdentifierRawP *wtid,
  const struct TALER_FullPaytoHashP *wire_target_h_payto,
  bool pending)
{
  struct GNUNET_HashCode key;

  (void) wire_target_h_payto;
  if (0 > hc->err)
    return GNUNET_SYSERR; /* already failed */
  GNUNET_assert (hc->max_aggregation_serial <= rowid);
  hc->max_aggregation_serial = rowid + 1;
  if (! pending)
  {
    /* Aggregated and paid, the ordinary case.  It cannot have a hold on file
       either: a wtid is only ever wired once, so the rows we are walking here
       are the ones that created it, and a hold from an earlier run is caught
       by the pass over the holds instead. */
    return GNUNET_OK;
  }
  GNUNET_CRYPTO_hash (wtid,
                      sizeof (*wtid),
                      &key);
  if (GNUNET_OK !=
      GNUNET_CONTAINER_multihashmap_put (
        hc->seen,
        &key,
        hc->seen,
        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
    return GNUNET_OK; /* another row of the same transfer, already evaluated */
  evaluate_hold (hc,
                 wtid);
  if (0 > hc->err)
    return GNUNET_SYSERR;
  return GNUNET_OK;
}


/**
 * Function called on each hold we already had on file.
 *
 * @param hc closure
 * @param ah the hold
 * @return #GNUNET_OK to continue to iterate
 */
static enum GNUNET_GenericReturnValue
evaluate_hold_cb (struct HoldContext *hc,
                  const struct TALER_AUDITORDB_AmlHold *ah)
{
  struct GNUNET_HashCode key;

  if (0 > hc->err)
    return GNUNET_SYSERR; /* already failed */
  GNUNET_CRYPTO_hash (&ah->wtid,
                      sizeof (ah->wtid),
                      &key);
  if (GNUNET_YES ==
      GNUNET_CONTAINER_multihashmap_contains (hc->seen,
                                              &key))
    return GNUNET_OK; /* fresh rows arrived for it, already re-evaluated */
  evaluate_hold (hc,
                 &ah->wtid);
  if (0 > hc->err)
    return GNUNET_SYSERR;
  return GNUNET_OK;
}


/**
 * Checks which wire transfers the exchange aggregated but did not execute, and
 * what it says is keeping it from making them.
 *
 * Unlike the two checks above, the three balances this maintains are current
 * state and not a running tally: they are recomputed from the holds on file on
 * every run.  That is what keeps them convergent, because a hold changes in
 * three independent ways -- it grows as deposits are added to the transfer, it
 * is reclassified when the exchange states a different reason, and it ends when
 * the transfer is finally made -- and only the first of those is announced by
 * a new row that a cursor could walk.
 *
 * @return transaction status
 */
static enum GNUNET_DB_QueryStatus
check_for_held_transfers (void)
{
  struct HoldContext hc = {
    .max_aggregation_serial = TALER_ARL_USE_PP (wire_hold_aggregation_id),
    .err = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT
  };
  enum GNUNET_DB_QueryStatus qs;

  GNUNET_assert (GNUNET_OK ==
                 TALER_amount_set_zero (TALER_ARL_currency,
                                        &hc.aml_hold));
  GNUNET_assert (GNUNET_OK ==
                 TALER_amount_set_zero (TALER_ARL_currency,
                                        &hc.small_aggregate));
  GNUNET_assert (GNUNET_OK ==
                 TALER_amount_set_zero (TALER_ARL_currency,
                                        &hc.transfer_lag));
  hc.seen = GNUNET_CONTAINER_multihashmap_create (128,
                                                  GNUNET_NO);
  /* First the aggregations we have not seen before, then the holds we already
     knew about; the map keeps the two from doing each other's work. */
  qs = TALER_EXCHANGEDB_iterate_aggregation_wtids_above_serial_id (
    TALER_ARL_edb,
    TALER_ARL_USE_PP (wire_hold_aggregation_id),
    &note_held_transfer_cb,
    &hc);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    goto cleanup;
  }
  if (0 > hc.err)
  {
    qs = hc.err;
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    goto cleanup;
  }
  qs = TALER_AUDITORDB_iterate_aml_holds (TALER_ARL_adb,
                                          &evaluate_hold_cb,
                                          &hc);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    goto cleanup;
  }
  if (0 > hc.err)
  {
    qs = hc.err;
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    goto cleanup;
  }
  TALER_ARL_USE_PP (wire_hold_aggregation_id) = hc.max_aggregation_serial;
  TALER_ARL_USE_AB (total_aml_hold) = hc.aml_hold;
  TALER_ARL_USE_AB (total_small_aggregate) = hc.small_aggregate;
  TALER_ARL_USE_AB (total_transfer_lag) = hc.transfer_lag;
  /* One TALER_amount2s() per statement: it returns a single static buffer. */
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Exchange is holding %s awaiting KYC\n",
              TALER_amount2s (&hc.aml_hold));
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Exchange is holding %s in aggregates below the wire fee\n",
              TALER_amount2s (&hc.small_aggregate));
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Exchange is holding %s for no stated reason\n",
              TALER_amount2s (&hc.transfer_lag));
  qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
cleanup:
  GNUNET_CONTAINER_multihashmap_destroy (hc.seen);
  return qs;
}


/**
 * Start the database transactions and begin the audit.
 *
 * @return transaction status
 */
static enum GNUNET_DB_QueryStatus
begin_transaction (void)
{
  enum GNUNET_DB_QueryStatus qs;

  if (GNUNET_SYSERR ==
      TALER_EXCHANGEDB_preflight (TALER_ARL_edb))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to initialize exchange database connection.\n");
    return GNUNET_DB_STATUS_HARD_ERROR;
  }
  if (GNUNET_SYSERR ==
      TALER_AUDITORDB_preflight (TALER_ARL_adb))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to initialize auditor database session.\n");
    return GNUNET_DB_STATUS_HARD_ERROR;
  }
  if (GNUNET_OK !=
      TALER_AUDITORDB_start (TALER_ARL_adb,
                             "auditor-transfer"))
  {
    GNUNET_break (0);
    return GNUNET_DB_STATUS_HARD_ERROR;
  }
  if (GNUNET_OK !=
      TALER_EXCHANGEDB_start_read_only (TALER_ARL_edb,
                                        "transfer auditor"))
  {
    GNUNET_break (0);
    TALER_AUDITORDB_rollback (TALER_ARL_adb);
    return GNUNET_DB_STATUS_HARD_ERROR;
  }
  qs = TALER_AUDITORDB_get_auditor_progress (
    TALER_ARL_adb,
    TALER_ARL_GET_PP (wire_batch_deposit_id),
    TALER_ARL_GET_PP (wire_aggregation_id),
    TALER_ARL_GET_PP (wire_hold_aggregation_id),
    NULL);
  if (0 > qs)
    goto handle_db_error;

  qs = TALER_AUDITORDB_get_balance (
    TALER_ARL_adb,
    TALER_ARL_GET_AB (total_amount_lag),
    TALER_ARL_GET_AB (total_early_aggregation),
    TALER_ARL_GET_AB (total_aml_hold),
    TALER_ARL_GET_AB (total_small_aggregate),
    TALER_ARL_GET_AB (total_transfer_lag),
    NULL);
  if (0 > qs)
    goto handle_db_error;
  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                "First analysis of with transfer auditor, starting audit from scratch\n");
  }
  else
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Resuming transfer audit at %llu / %llu / %llu\n",
                (unsigned long long) TALER_ARL_USE_PP (wire_batch_deposit_id),
                (unsigned long long) TALER_ARL_USE_PP (wire_aggregation_id),
                (unsigned long long) TALER_ARL_USE_PP (
                  wire_hold_aggregation_id));
  }

  qs = check_for_required_transfers ();
  if (0 > qs)
    goto handle_db_error;
  qs = check_for_completed_transfers ();
  if (0 > qs)
    goto handle_db_error;
  qs = check_for_held_transfers ();
  if (0 > qs)
    goto handle_db_error;

  qs = TALER_AUDITORDB_insert_auditor_progress (
    TALER_ARL_adb,
    TALER_ARL_SET_PP (wire_batch_deposit_id),
    TALER_ARL_SET_PP (wire_aggregation_id),
    TALER_ARL_SET_PP (wire_hold_aggregation_id),
    NULL);
  if (0 > qs)
    goto handle_db_error;
  qs = TALER_AUDITORDB_insert_balance (
    TALER_ARL_adb,
    TALER_ARL_SET_AB (total_amount_lag),
    TALER_ARL_SET_AB (total_early_aggregation),
    TALER_ARL_SET_AB (total_aml_hold),
    TALER_ARL_SET_AB (total_small_aggregate),
    TALER_ARL_SET_AB (total_transfer_lag),
    NULL);
  if (0 > qs)
    goto handle_db_error;
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Concluded audit step at %llu/%llu/%llu\n",
              (unsigned long long) TALER_ARL_USE_PP (wire_aggregation_id),
              (unsigned long long) TALER_ARL_USE_PP (wire_batch_deposit_id),
              (unsigned long long) TALER_ARL_USE_PP (
                wire_hold_aggregation_id));
  TALER_EXCHANGEDB_rollback (TALER_ARL_edb);
  qs = TALER_AUDITORDB_commit (TALER_ARL_adb);
  if (0 > qs)
    goto handle_db_error;
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Transaction concluded!\n");
  return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
handle_db_error:
  TALER_AUDITORDB_rollback (TALER_ARL_adb);
  TALER_EXCHANGEDB_rollback (TALER_ARL_edb);
  GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
  return qs;
}


/**
 * Start auditor process.
 */
static void
start (void)
{
  enum GNUNET_DB_QueryStatus qs;

  for (unsigned int max_retries = 3; max_retries>0; max_retries--)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Trying again (%u attempts left)\n",
                max_retries);
    qs = begin_transaction ();
    if (GNUNET_DB_STATUS_SOFT_ERROR != qs)
      break;
  }
  if (0 > qs)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Audit failed\n");
    GNUNET_break (0);
    global_ret = EXIT_FAILURE;
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
}


/**
 * Function called on events received from Postgres.
 *
 * @param cls closure, NULL
 * @param extra additional event data provided
 * @param extra_size number of bytes in @a extra
 */
static void
db_notify (void *cls,
           const void *extra,
           size_t extra_size)
{
  (void) cls;
  (void) extra;
  (void) extra_size;

  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Received notification to wake transfer helper\n");
  start ();
}


/**
 * Main function that will be run.
 *
 * @param cls closure
 * @param args remaining command-line arguments
 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
 * @param c configuration
 */
static void
run (void *cls,
     char *const *args,
     const char *cfgfile,
     const struct GNUNET_CONFIGURATION_Handle *c)
{
  (void) cls;
  (void) args;
  (void) cfgfile;
  cfg = c;
  if (GNUNET_OK !=
      TALER_ARL_init (c))
  {
    global_ret = EXIT_FAILURE;
    return;
  }
  GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
                                 NULL);
  if (GNUNET_OK !=
      TALER_EXCHANGEDB_load_accounts (TALER_ARL_cfg,
                                      TALER_EXCHANGEDB_ALO_DEBIT
                                      | TALER_EXCHANGEDB_ALO_CREDIT
                                      | TALER_EXCHANGEDB_ALO_AUTHDATA))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "No bank accounts configured\n");
    global_ret = EXIT_NOTCONFIGURED;
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  if (0 == test_mode)
  {
    // FIXME-Optimization: use different event type in the future!
    struct GNUNET_DB_EventHeaderP es = {
      .size = htons (sizeof (es)),
      .type = htons (TALER_DBEVENT_EXCHANGE_AUDITOR_WAKE_HELPER_WIRE)
    };

    eh = TALER_AUDITORDB_event_listen (TALER_ARL_adb,
                                       &es,
                                       GNUNET_TIME_UNIT_FOREVER_REL,
                                       &db_notify,
                                       NULL);
    GNUNET_assert (NULL != eh);
  }
  start ();
}


/**
 * The main function of the wire auditing tool. Checks that
 * the exchange's records of wire transfers match that of
 * the wire gateway.
 *
 * @param argc number of arguments from the command line
 * @param argv command line arguments
 * @return 0 ok, 1 on error
 */
int
main (int argc,
      char *const *argv)
{
  const struct GNUNET_GETOPT_CommandLineOption options[] = {
    GNUNET_GETOPT_option_flag ('i',
                               "internal",
                               "perform checks only applicable for exchange-internal audits",
                               &internal_checks),
    GNUNET_GETOPT_option_flag ('t',
                               "test",
                               "run in test mode and exit when idle",
                               &test_mode),
    GNUNET_GETOPT_option_timetravel ('T',
                                     "timetravel"),
    GNUNET_GETOPT_OPTION_END
  };
  enum GNUNET_GenericReturnValue ret;

  ret = GNUNET_PROGRAM_run (
    TALER_AUDITOR_project_data (),
    argc,
    argv,
    "taler-helper-auditor-transfer",
    gettext_noop (
      "Audit exchange database for consistency of aggregations/transfers with respect to deposit deadlines"),
    options,
    &run,
    NULL);
  if (GNUNET_SYSERR == ret)
    return EXIT_INVALIDARGUMENT;
  if (GNUNET_NO == ret)
    return EXIT_SUCCESS;
  return global_ret;
}


/* end of taler-helper-auditor-transfer.c */
