#!/bin/bash
#
#  This file is part of TALER
#  Copyright (C) 2014-2022 Taler Systems SA
#
#  TALER is free software; you can redistribute it and/or modify it under the
#  terms of the GNU General Public License as published by the Free Software
#  Foundation; either version 3, or (at your option) any later version.
#
#  TALER is distributed in the hope that it will be useful, but WITHOUT ANY
#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
#  A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License along with
#  TALER; see the file COPYING.  If not, If not, see <http://www.gnu.org/license>
#
# Setup database which was generated from a exchange-wallet interaction
# with revocations and run the auditor against it.
#
# Check that the auditor report is as expected.
#
# shellcheck disable=SC2317
#
# Requires 'jq' tool and Postgres superuser rights!
set -eu
# set -x

# Set of numbers for all the testcases.
# When adding new tests, increase the last number:
ALL_TESTS=$(seq 0 4)

# $TESTS determines which tests we should run.
# This construction is used to make it easy to
# only run a subset of the tests. To only run a subset,
# pass the numbers of the tests to run as the FIRST
# argument to test-auditor.sh, i.e.:
#
# $ test-revocation.sh "1 3"
#
# to run tests 1 and 3 only.  By default, all tests are run.
#
TESTS=${1:-$ALL_TESTS}

export TALER_AUDITOR_TOKEN="secret-token:D4CST1Z6AHN3RT03M0T9NSTF2QGHTB5ZD2D3RYZB4HAWG8SX0JEFWBXCKXZHMB7Y3Z7KVFW0B3XPXD5BHCFP8EB0R6CNH2KAWDWVET0"
export TALER_AUDITOR_SALT="64S36D1N6RVKGC9J6CT3ADHQ70RK4CSM6MV3EE1H68SK8D9P6WW32CHK6GTKCDSR64S36D1N6RVKGC9J6CT3ADHQ70RK4CSM6MV3EE0"

# Global variable to run the auditor processes under valgrind
# VALGRIND=valgrind
VALGRIND=""
LOGLEVEL="INFO"

. setup.sh

# Cleanup to run whenever we exit
function cleanup()
{
    if [ ! -z "${EPID:-}" ]
    then
        echo -n "Stopping exchange $EPID..."
        kill -TERM "$EPID"
        wait "$EPID"
        echo " DONE"
        unset EPID
    fi
    stop_libeufin
}

# Cleanup to run whenever we exit
function exit_cleanup()
{
    echo "Running exit-cleanup"
    if [ ! -z "${POSTGRES_PATH:-}" ]
    then
        echo "Stopping Postgres at ${POSTGRES_PATH}"
        "${POSTGRES_PATH}/pg_ctl" \
            -D "$TMPDIR" \
            -l /dev/null \
            stop \
            &> /dev/null \
            || true
    fi
    cleanup
    for n in $(jobs -p)
    do
        kill "$n" 2> /dev/null || true
    done
    wait
    echo "DONE"
}

# Install cleanup handler (except for kill -9)
trap exit_cleanup EXIT


# Operations to run before the actual audit
function pre_audit () {
    # Launch bank
    echo -n "Launching bank "
    export CONF
    export MY_TMP_DIR
    launch_libeufin
    for n in $(seq 1 80)
    do
        echo -n "."
        sleep 0.1
        OK=1
        wget http://localhost:8082/ \
             -o /dev/null \
             -O /dev/null \
             >/dev/null && break
        OK=0
    done
    if [ 1 != "$OK" ]
    then
        exit_skip "Failed to launch libeufin-bank"
    fi
    echo " DONE"
    if [ "${1:-no}" = "aggregator" ]
    then
	    echo -n "Running exchange aggregator ... (config: $CONF)"
        taler-exchange-aggregator \
            -L "$LOGLEVEL" \
            -t \
            -c "$CONF" \
            -y \
            2> "${MY_TMP_DIR}/aggregator.log" \
            || exit_fail "FAIL"
        echo " DONE"
        echo -n "Running exchange closer ..."
        taler-exchange-closer \
            -L "$LOGLEVEL" \
            -t \
            -c "$CONF" \
            2> "${MY_TMP_DIR}/closer.log" \
            || exit_fail "FAIL"
        echo " DONE"
        echo -n "Running exchange transfer ..."
        taler-exchange-transfer \
            -L "$LOGLEVEL" \
            -t \
            -c "$CONF" \
            2> "${MY_TMP_DIR}/transfer.log" \
            || exit_fail "FAIL"
        echo " DONE"
    fi
}

# actual audit run
function audit_only () {
    # Run the auditor!
    echo -n "Running audit(s) ... (conf is $CONF)"

    # Restart so that first run is always fresh, and second one is incremental
    taler-auditor-dbinit \
        -r \
        -c "$CONF"
    # Each helper is run twice: the first run audits from scratch, the
    # second one exercises the incremental path.  Findings end up in the
    # auditor database and are inspected via the auditor's REST API.
    for helper in aggregation aml coins deposits reserves \
                  wire-credit wire-debit purses transfer
    do
        for pass in "" "-inc"
        do
            $VALGRIND "taler-helper-auditor-${helper}" \
                      -i \
                      -L "$LOGLEVEL" \
                      -c "$CONF" \
                      -t \
                      > "${MY_TMP_DIR}/test-audit-${helper}${pass}.out" \
                      2> "${MY_TMP_DIR}/test-audit-${helper}${pass}.err" \
                || exit_fail "${helper} audit${pass} failed (see ${MY_TMP_DIR}/test-audit-${helper}${pass}.*)"
            echo -n "."
        done
    done
    echo " DONE"
}


function stop_auditor_httpd() {
    if [ -n "${APID:-}" ]
    then
        echo -n "Stopping auditor $APID..."
        kill -TERM "$APID"
        wait "$APID" || true
        echo "DONE"
        unset APID
    fi
}


function run_auditor_httpd() {
    echo -n "Starting auditor..."
    $VALGRIND taler-auditor-httpd \
        -c "${CONF}" \
        -L "$LOGLEVEL" \
        2> "${MY_TMP_DIR}/auditor-httpd.err" &
    APID=$!
    # Wait for auditor service to be available
    for n in $(seq 1 50)
    do
        echo -n "."
        sleep 0.2
        wget "http://localhost:8083/config" \
             -o /dev/null \
             -O /dev/null \
             >/dev/null \
            || continue
        break
    done
    echo "... DONE."
}


function check_auditor_running() {
    ARUNSTATUS=$(curl -Is http://localhost:8083/config | head -1)
    if [ -n "${ARUNSTATUS:-}" ]
    then
        echo "Auditor running"
    else
        echo "Auditor not running, starting it"
        run_auditor_httpd
    fi
    unset ARUNSTATUS
}


function call_endpoint() {
    if [ -n "${2+x}" ]
    then
        curl -s \
             -H "Accept: application/json" \
             -H "Authorization: Bearer ${TALER_AUDITOR_TOKEN}" \
             -o "${MY_TMP_DIR}/${2}.json" \
             "localhost:8083/monitoring/${1}?limit=50&balance_key=${2}"
        echo "endpoint ${1} called (with balance_key)... "
    else
        curl -s \
             -H "Accept: application/json" \
             -H "Authorization: Bearer ${TALER_AUDITOR_TOKEN}" \
             -o "${MY_TMP_DIR}/${1}.json" \
             "localhost:8083/monitoring/${1}?limit=50"
        echo "endpoint ${1} called... "
    fi
}


function check_balance() {
    call_endpoint "balances" "$1"
    BAL=$(jq -r .balances[0].balance_value < "${MY_TMP_DIR}/${1}.json")
    if [ "$BAL" != "$2" ]
    then
        exit_fail "$3 (got $BAL, wanted $2)"
    fi
    echo "PASS"
}


function check_not_balance() {
    call_endpoint "balances" "$1"
    BAL=$(jq -r .balances[0].balance_value < "${MY_TMP_DIR}/${1}.json")
    if [ "$BAL" = "$2" ]
    then
        exit_fail "$3 (got $BAL, wanted NOT $2)"
    fi
    echo "PASS"
}


function check_report() {
    call_endpoint "$1"
    NAME=$(echo "$1" | tr '-' '_')
    # shellcheck disable=SC2086
    VAL=$(jq -r .\"${NAME}\"[0].\"$2\" < "${MY_TMP_DIR}/${1}.json")
    if [ "$VAL" != "$3" ]
    then
        exit_fail "$1::$2 (got $VAL, wanted $3)"
    fi
    echo "PASS"
}


function check_no_report() {
    call_endpoint "$1"
    NAME=$(echo "$1" | tr '-' '_')
    # shellcheck disable=SC2086
    jq -e .\"${NAME}\"[0] \
       < "${MY_TMP_DIR}/${1}.json" \
       > /dev/null \
       && exit_fail "Wanted empty report for $1, but got incidents"
    echo "PASS"
}


function check_report_neg() {
    call_endpoint "$1"
    NAME=$(echo "$1" | tr '-' '_')
    # shellcheck disable=SC2086
    VAL=$(jq -r .\"${NAME}\"[0].\"$2\" < "${MY_TMP_DIR}/${1}.json")
    if [ "$VAL" == "$3" ]
    then
        exit_fail "$1::$2 (got $VAL, wanted $3)"
    fi
    echo "PASS"
}


# Check that at least one entry of report $1 has field $2 set to $3.
function check_report_any() {
    call_endpoint "$1"
    NAME=$(echo "$1" | tr '-' '_')
    # shellcheck disable=SC2086
    jq -e --arg want "$3" "any(.\"${NAME}\"[]; .\"$2\" == \$want)" \
       < "${MY_TMP_DIR}/${1}.json" \
       > /dev/null \
        || exit_fail "$1::$2 (no entry with value $3)"
    echo "PASS"
}


# Run audit process on current database, including report
# generation.  Pass "aggregator" as $1 to run
# $ taler-exchange-aggregator
# before auditor (to trigger pending wire transfers).
function run_audit () {
    pre_audit "${1:-no}"
    audit_only
    cleanup
    echo " DONE"
}


# Do a full reload of the (original) database
function full_reload()
{
    echo -n "Doing full reload of the database... "
    stop_auditor_httpd
    dropdb -f "$DB" 2> /dev/null || true
    createdb -T template0 "$DB" \
        || exit_skip "could not create database $DB (at ${PGHOST:-})"
    # Import pre-generated database, -q(ietly) using single (-1) transaction
    psql -Aqt "$DB" \
         -q \
         -1 \
         -f "${BASEDB}.sql" \
         > /dev/null \
        || exit_skip "Failed to load database $DB from ${BASEDB}.sql"
    echo "DONE"
}


function test_0() {
    echo "===========0: normal run with aggregator==========="
    run_audit aggregator
    check_auditor_running

    echo "Checking output"
    # if an emergency was detected, that is a bug and we should fail
    echo -n "Test for emergencies... "
    check_no_report "emergency"
    echo -n "Test for emergencies by count... "
    check_no_report "emergency-by-count"
    echo -n "Test for deposit confirmation problems... "
    check_no_report "deposit-confirmation"
    echo -n "Test for wire inconsistencies... "
    check_no_report "denomination-key-validity-withdraw-inconsistency"
    check_no_report "wire-out-inconsistency"
    check_no_report "reserve-in-inconsistency"
    check_no_report "misattribution-in-inconsistency"
    check_no_report "row-inconsistency"
    check_no_report "row-minor-inconsistencies"
    check_no_report "wire-format-inconsistency"

    # Just to test the endpoint and for logging ...
    call_endpoint "balances"

    echo -n "Testing bad sig loss balance... "
    check_balance \
        "aggregation_total_bad_sig_loss" \
        "TESTKUDOS:0" \
        "Wrong total bad sig loss from aggregation"
    echo -n "Testing coin irregular loss balances... "
    check_balance \
        "coin_irregular_loss" \
        "TESTKUDOS:0" \
        "Wrong total bad sig loss from coins"
    echo -n "Testing reserves bad sig loss balances... "
    check_balance \
        "reserves_total_bad_sig_loss" \
        "TESTKUDOS:0" \
        "Wrong total bad sig loss from reserves"

    echo -n "Test for bad incoming delta plus... "
    check_balance \
        "total_bad_amount_in_plus" \
        "TESTKUDOS:0" \
        "Expected total wire in delta plus wrong"
    echo -n "Test for bad incoming delta minus... "
    check_balance \
        "total_bad_amount_in_minus" \
        "TESTKUDOS:0" \
        "Expected total wire in delta minus wrong"
    echo -n "Test for aggregation wire out delta plus... "
    check_balance \
        "aggregation_total_wire_out_delta_plus" \
        "TESTKUDOS:0" \
        "Expected total wire out delta plus wrong"
    echo -n "Test for aggregation wire out delta minus... "
    check_balance \
        "aggregation_total_wire_out_delta_minus" \
        "TESTKUDOS:0" \
        "Expected total wire out delta minus wrong"
    echo -n "Test for misattribution amounts... "
    check_balance \
        "total_misattribution_in" \
        "TESTKUDOS:0" \
        "Expected total misattribution in wrong"

    echo -n "Checking for unexpected aggregation delta plus differences... "
    check_balance \
        "aggregation_total_arithmetic_delta_plus" \
        "TESTKUDOS:0" \
        "Wrong arithmetic delta plus from aggregations"
    echo -n "Checking for unexpected aggregation delta minus differences... "
    check_balance \
        "aggregation_total_arithmetic_delta_minus" \
        "TESTKUDOS:0" \
        "Wrong arithmetic delta minus from aggregations"
    echo -n "Checking for unexpected coin delta plus differences... "
    check_balance \
        "coins_total_arithmetic_delta_plus" \
        "TESTKUDOS:0" \
        "Wrong arithmetic delta plus from coins"
    echo -n "Checking for unexpected coin delta minus differences... "
    check_balance \
        "coins_total_arithmetic_delta_minus" \
        "TESTKUDOS:0" \
        "Wrong arithmetic delta minus from coins"
    echo -n "Checking for unexpected reserves delta plus... "
    check_balance \
        "reserves_total_arithmetic_delta_plus" \
        "TESTKUDOS:0" \
        "Wrong arithmetic delta plus from reserves"
    echo -n "Checking for unexpected reserves delta minus... "
    check_balance \
        "reserves_total_arithmetic_delta_minus" \
        "TESTKUDOS:0" \
        "Wrong arithmetic delta minus from reserves"

    echo -n "Checking for unexpected arithmetic inconsistencies... "
    check_no_report "amount-arithmetic-inconsistency"

    # cannot easily undo aggregator, hence full reload
    full_reload
}


# Run without aggregator, hence auditor should detect wire
# transfer lag!
function test_1() {

    echo "===========1: normal run==========="
    run_audit
    check_auditor_running

    echo "Checking output"
    # if an emergency was detected, that is a bug and we should fail
    echo -n "Test for emergencies... "
    check_no_report "emergency"
    echo -n "Test for emergencies by count... "
    check_no_report "emergency-by-count"

    echo -n "Test for wire inconsistencies... "
    check_no_report "wire-out-inconsistency"
    check_no_report "reserve-in-inconsistency"
    check_no_report "misattribution-in-inconsistency"
    check_no_report "row-inconsistency"
    check_no_report "row-minor-inconsistencies"
    check_no_report "wire-format-inconsistency"

    echo -n "Test for bad incoming delta plus... "
    check_balance \
        "total_bad_amount_in_plus" \
        "TESTKUDOS:0" \
        "Expected total wire in delta plus wrong"
    echo -n "Test for bad incoming delta minus... "
    check_balance \
        "total_bad_amount_in_minus" \
        "TESTKUDOS:0" \
        "Expected total wire in delta minus wrong"
    echo -n "Test for misattribution amounts... "
    check_balance \
        "total_misattribution_in" \
        "TESTKUDOS:0" \
        "Expected total misattribution in wrong"

    # Database was unmodified, no need to undo
    echo "OK"
}



# Change recoup amount
function test_2() {

    echo "===========2: recoup amount inconsistency==========="
    echo "UPDATE exchange.recoup SET amount.val=5 WHERE recoup_uuid=1" | psql -Aqt "$DB"

    run_audit
    check_auditor_running

    # Reserve balance is now wrong
    echo -n "Testing reserve balance inconsistency detection... "
    check_report \
        "reserve-balance-summary-wrong-inconsistency" \
        "auditor_amount" "TESTKUDOS:3"
    check_report \
        "reserve-balance-summary-wrong-inconsistency" \
        "exchange_amount" "TESTKUDOS:0"
    # Coin spent exceeded coin's value
    echo -n "Testing coin arithmetic inconsistency detection... "
    check_report \
        "amount-arithmetic-inconsistency" \
        "auditor_amount" "TESTKUDOS:2"
    check_report \
        "amount-arithmetic-inconsistency" \
        "exchange_amount" "TESTKUDOS:5"
    echo "OK"

    # Undo database modification
    echo "UPDATE exchange.recoup SET amount.val=2 WHERE recoup_uuid=1" | psql -Aqt "$DB"
    full_reload
}


# Change recoup-refresh amount
function test_3() {

    echo "===========3: recoup-refresh amount inconsistency==========="
    echo "UPDATE exchange.recoup_refresh SET amount.val=5 WHERE recoup_refresh_uuid=1" | psql -Aqt "$DB"

    run_audit
    check_auditor_running

    echo -n "Testing inconsistency detection... "
    # Coin spent exceeded coin's value
    check_balance \
        "coins_total_arithmetic_delta_minus" \
        "TESTKUDOS:5" \
        "Arithmetic delta minus amount is wrong"
    check_balance \
        "coins_total_arithmetic_delta_plus" \
        "TESTKUDOS:0" \
        "Arithmetic delta plus amount is wrong"
    echo "OK"

    # Undo database modification
    echo "UPDATE exchange.recoup_refresh SET amount.val=0 WHERE recoup_refresh_uuid=1" | psql -Aqt "$DB"
    full_reload
}


# Void recoup-refresh entry by 'unrevoking' denomination
function test_4() {

    echo "===========4: invalid recoup==========="
    echo "DELETE FROM exchange.denomination_revocations;" | psql -Aqt "$DB"

    run_audit
    check_auditor_running

    echo -n "Testing inconsistency detection... "
    # Recoup of a denomination that was never revoked is not legitimate
    check_report_neg \
        "bad-sig-losses" \
        "loss" "TESTKUDOS:0"
    echo -n "Testing irregular loss balance... "
    check_not_balance \
        "coin_irregular_loss" \
        "TESTKUDOS:0" \
        "Total bad sig losses are wrong"
    echo -n "Testing row inconsistency table attribution... "
    check_report_any \
        "row-inconsistency" \
        "row_table" "recoup"
    echo "OK"

    # Undo database modification (can't easily undo DELETE, so full reload)
    full_reload
}




# *************** Main test loop starts here **************


# Run all the tests against the database given in $1.
# Sets $fail to 0 on success, non-zero on failure.
function check_with_database()
{
    BASEDB="$1"
    # Configuration file to use
    CONF="$1.conf"
    echo "Running test suite with database $BASEDB using configuration $CONF"

    MASTER_PRIV_FILE="${BASEDB}.mpriv"
    taler-exchange-config -f \
                          -c "${CONF}" \
                          -s exchange-offline \
                          -o MASTER_PRIV_FILE \
                          -V "${MASTER_PRIV_FILE}"
    MASTER_PUB=$(gnunet-ecc -p "$MASTER_PRIV_FILE")

    echo "MASTER PUB is ${MASTER_PUB} using file ${MASTER_PRIV_FILE}"

    # Load database
    full_reload
    # Run test suite
    fail=0
    for i in $TESTS
    do
        "test_$i"
        if [ 0 != "$fail" ]
        then
            break
        fi
    done
    # echo "Cleanup (disabled, leaving database $DB behind)"
    dropdb "$DB"
}

# If this script is not running as root, create
# the temporary directory structure for storage for postgres
# Will set PGHOST accordingly
function perform_dbinit() {
    # Available directly in path?
    INITDB_BIN=$(command -v initdb) || true
    if [[ -n "$INITDB_BIN" ]]; then
      echo "FOUND (in path) at $INITDB_BIN"
    else
      HAVE_INITDB=$(find /usr -name "initdb" | head -1 2> /dev/null | grep postgres) || exit_skip " MISSING"
      echo "FOUND at " "$(dirname "$HAVE_INITDB")"
      INITDB_BIN=$(echo "$HAVE_INITDB" | grep bin/initdb | grep postgres | sort -n | tail -n1)
    fi
    echo -n "Setting up Postgres DB"
    POSTGRES_PATH=$(dirname "$INITDB_BIN")
    TMPDIR="${MY_TMP_DIR}/postgres/"
    mkdir -p "$TMPDIR"
    echo -n "Setting up Postgres DB at $TMPDIR ..."
    "$INITDB_BIN" \
        --no-sync \
        --auth=trust \
        -D "${TMPDIR}" \
        > "${MY_TMP_DIR}/postgres-dbinit.log" \
        2> "${MY_TMP_DIR}/postgres-dbinit.err"
    echo " DONE"
    mkdir "${TMPDIR}/sockets"
    echo -n "Launching Postgres service at $POSTGRES_PATH"
    cat - >> "$TMPDIR/postgresql.conf" <<EOF
unix_socket_directories='${TMPDIR}/sockets'
fsync=off
max_wal_senders=0
synchronous_commit=off
wal_level=minimal
listen_addresses=''
EOF
    grep -v host \
         < "$TMPDIR/pg_hba.conf" \
         > "$TMPDIR/pg_hba.conf.new"
    mv "$TMPDIR/pg_hba.conf.new" "$TMPDIR/pg_hba.conf"
    "${POSTGRES_PATH}/pg_ctl" \
        -D "$TMPDIR" \
        -l /dev/null \
        start \
        > "${MY_TMP_DIR}/postgres-start.log" \
        2> "${MY_TMP_DIR}/postgres-start.err"
    echo " DONE"
    PGHOST="$TMPDIR/sockets"
    export PGHOST
}


# *************** Main logic starts here **************

# ####### Setup globals ######
# Postgres database to use
DB=revoke-basedb


# test required commands exist
echo "Testing for jq"
jq -h > /dev/null || exit_skip "jq required"
echo "Testing for faketime"
faketime -h > /dev/null \
    || exit_skip "faketime required"
echo "Testing for libeufin-bank"
libeufin-bank --help \
             >/dev/null \
             2> /dev/null \
             </dev/null \
    || exit_skip "libeufin-bank required"
echo "Testing for taler-wallet-cli"
taler-wallet-cli -h \
                 >/dev/null \
                 </dev/null \
                 2>/dev/null \
    || exit_skip "taler-wallet-cli required"

MY_TMP_DIR=$(mktemp -d /tmp/taler-auditor-basedbXXXXXX)
export MY_TMP_DIR
echo "Using $MY_TMP_DIR for logging and temporary data"

echo -n "Testing for Postgres "
[ $(id -u) == 0 ] || perform_dbinit


MYDIR="${MY_TMP_DIR}/basedb"
mkdir -p "${MYDIR}"

# Generating the reference database is by far the slowest part of this
# test.  Set REUSE_BASEDB_DIR to the directory printed at the end of a
# previous run to audit that database again instead of regenerating it.
if [ -z "${REUSE_BASEDB_DIR+x}" ]
then
    echo "Generating fresh database at $MYDIR"
    set +e
    faketime -f '-1 d' ./generate-revoke-basedb.sh "$MYDIR/$DB"
    GENSTATUS=$?
    set -e
    if [ 77 = "$GENSTATUS" ]
    then
        # Prerequisite missing (currently: recoup is not implemented by the
        # exchange, see FIXME_9828); report as skipped, not as a failure.
        echo "SKIPPING: could not generate the revocation database"
        exit 77
    fi
    if [ 0 != "$GENSTATUS" ]
    then
        echo "Generation failed"
        exit 1
    fi
    echo "To reuse this database in the future, use:"
    echo "export REUSE_BASEDB_DIR=$MY_TMP_DIR"
else
    echo "Reusing existing database from ${REUSE_BASEDB_DIR}"
    cp -r "${REUSE_BASEDB_DIR}/basedb"/* "${MYDIR}/"
fi

check_with_database "$MYDIR/$DB"
if [ "x$fail" != "x0" ]
then
    exit "$fail"
fi

if [ -z "${REUSE_BASEDB_DIR+x}" ]
then
    echo "Run 'export REUSE_BASEDB_DIR=${MY_TMP_DIR}' to re-run tests against the same database"
fi

exit 0
