#!/bin/bash
# This file is in the public domain.
#
# Script to generate the reference database for the KYC/AML auditor tests
# from a 'correct' interaction between exchange, wallet, merchant and bank
# with KYC enabled.
#
# Creates "$1.sql", "$1.conf" and "$1.mpriv".
#
# Unlike generate-auditor-basedb.sh the resulting database deliberately
# contains *both* KYC outcomes the auditor can observe.  Two merchant
# instances are paid the same amount, and both prove that they own their
# bank account with a KYCAUTH wire transfer, but only one of them then
# passes the KYC form:
#
#  - instance 'default'  (account DE474361) passes, so the funds it is
#    owed are aggregated and wired out normally;
#  - instance 'blocked'  (account DE61109451) does not, so the exchange
#    aggregates its coins but parks the payout in aggregation_transient
#    and never wires it anywhere.
#
# The second case is what makes this database worth auditing: the exchange
# is holding money it has not paid out, and test-kyc.sh pins down what the
# auditor does and does not see of that (see issues.txt, issue 9).
#
# Requires the wallet CLI to be installed and in the path.  Furthermore, the
# user running this script must be Postgres superuser and be allowed to
# create/drop databases.
#
set -eu

# The auditor tests inject faults into specific rows of the generated
# database, so the wallet must pick the same coins and make the same
# transactions on every run.  The coin selection introduced after 2024 is
# not deterministic in that way; 'legacy-2024' restores the old behaviour
# (see https://bugs.gnunet.org/view.php?id=11272).
export TALER_WALLET_COINSEL="legacy-2024"

. setup.sh

CONF="generate-kyc-basedb.conf"
# Parse command-line options
while getopts ':c:d:h' OPTION; do
    case "$OPTION" in
        c)
            CONF="$OPTARG"
            ;;
        d)
            BASEDB="$OPTARG"
            ;;
        h)
            echo 'Supported options:'
# shellcheck disable=SC2016
            echo '  -c $CONF     -- set configuration'
# shellcheck disable=SC2016
            echo '  -d $DB       -- set database name'
            ;;
        ?)
        exit_fail "Unrecognized command line option"
        ;;
    esac
done

# Where do we write the result?
if [ ! -v BASEDB ]
then
    exit_fail "-d option required"
fi

echo -n "Testing for curl ..."
curl --help >/dev/null </dev/null || exit_skip " MISSING"
echo " FOUND"
echo -n "Testing for jq ..."
jq -h >/dev/null </dev/null || exit_skip " MISSING"
echo " FOUND"

# The bank account of the merchant instance that passes KYC, and the one
# of the instance that stays blocked.
CLEARED_PAYTO="payto://iban/DE474361?receiver-name=Merchant43"
BLOCKED_PAYTO="payto://iban/DE61109451?receiver-name=Blocked44"

# reset database
echo -n "Reset 'auditor-basedb' database at ${PGHOST:-} ..."
dropdb --if-exists "auditor-basedb" > /dev/null 2> /dev/null || true
createdb "auditor-basedb" || exit_skip "Could not create database '$BASEDB' at ${PGHOST:-}"
echo " DONE"

# Launch exchange, merchant and bank.
setup -c "$CONF" \
      -abemw \
      -d "iban"

# obtain key configuration data
EXCHANGE_URL=$(taler-exchange-config -c "$CONF" -s EXCHANGE -o BASE_URL)
MERCHANT_PORT=$(taler-merchant-config -c "$CONF" -s MERCHANT -o PORT)
MERCHANT_URL="http://localhost:${MERCHANT_PORT}/"
BANK_PORT=$(taler-exchange-config -c "$CONF" -s BANK -o HTTP_PORT)
BANK_URL="http://localhost:${BANK_PORT}/"
WIRE_GATEWAY_URL=$(taler-exchange-config -c "${CONF}.edited" \
                                         -s exchange-accountcredentials-1 \
                                         -o WIRE_GATEWAY_URL)

echo -n "Checking setup worked ..."
wget \
    --tries=1 \
    --timeout=1 \
    "${EXCHANGE_URL}config" \
    -o /dev/null \
    -O /dev/null >/dev/null
echo "DONE"

export MERCHANT_URL

# Create merchant instance $1 with bank account $2, and a matching bank
# account at libeufin named $3.
function setup_instance() {
    local id="$1"
    local payto="$2"
    local user="$3"
    local base

    if [ "$id" = "admin" ]
    then
        base="${MERCHANT_URL}"
    else
        base="${MERCHANT_URL}instances/${id}/"
    fi

    echo -n "Setting up merchant instance ${id} ..."
    curl -s -o /dev/null \
         -H "Content-Type: application/json" \
         -X POST \
         -d "$(jq -n --arg id "$id" \
                  '{auth: {method: "external"},
                    id: $id,
                    name: $id,
                    address: {},
                    jurisdiction: {},
                    default_max_wire_fee: "TESTKUDOS:1",
                    default_max_deposit_fee: "TESTKUDOS:1",
                    default_wire_fee_amortization: 1,
                    default_wire_transfer_delay: {d_us: 0},
                    default_pay_delay: {d_us: 3600000000},
                    use_stefan: false}')" \
         "${MERCHANT_URL}management/instances"
    echo " DONE"

    echo -n "Setting up merchant account for ${id} ..."
    STATUS=$(curl -H "Content-Type: application/json" -X POST \
                  "${base}private/accounts" \
                  -d "$(jq -n --arg p "$payto" '{payto_uri: $p}')" \
                  -w "%{http_code}" -s -o /dev/null)
    if [ "$STATUS" != "200" ]
    then
        exit_fail "Expected 200 OK from ${base}private/accounts. Got: $STATUS"
    fi
    echo " DONE"

    echo -n "Setting up libeufin account ${user} ..."
    libeufin-bank create-account \
                  --config="${CONF}" \
                  --name="${user}" \
                  --username="${user}" \
                  --password="password" \
                  --payto_uri="${payto}" \
        > /dev/null
    echo " DONE"
}

# ----------------------------------------------------------------------
# With ENABLE_KYC the exchange refuses to say anything about an account it
# has never seen, and the merchant in turn refuses to create orders (451).
# A merchant proves that it owns its bank account by having a small KYCAUTH
# wire transfer made from it, quoting its own instance public key.  Those
# transfers are also the only KYC-specific wire credits the auditor knows
# about, so the reference database needs them anyway.
#
# Sets ACCESS_TOKEN to the KYC access token the exchange handed out for the
# account of instance $1.
# ----------------------------------------------------------------------

function kycauth_instance() {
    local id="$1"
    local base
    local filter
    local pub
    local payto
    local status
    local n

    if [ "$id" = "admin" ]
    then
        base="${MERCHANT_URL}"
    else
        base="${MERCHANT_URL}instances/${id}/"
    fi
    # An instance also carries the default accounts of the public demo
    # exchanges, so pick out the entry that describes *our* exchange.
    filter=".kyc_data[] | select(.exchange_url == \"${EXCHANGE_URL}\")"

    echo -n "Reading public key of instance ${id} ..."
    pub=$(curl -s "${base}private" | jq -r '.merchant_pub')
    if [ -z "$pub" ] || [ "$pub" = "null" ]
    then
        exit_fail "Could not determine public key of instance ${id}"
    fi
    echo " ${pub}"

    echo -n "Waiting for instance ${id} to talk to the exchange ..."
    payto=""
    for n in $(seq 1 100)
    do
        echo -n "."
        sleep 0.5
        KYC_JSON=$(curl -s "${base}private/kyc") || continue
        status=$(echo "$KYC_JSON" | jq -r "${filter} | .exchange_http_status // empty")
        # 0 means "exchange not reachable yet"
        if [ -z "$status" ] || [ "$status" = "0" ]
        then
            continue
        fi
        payto=$(echo "$KYC_JSON" | jq -r "${filter} | .payto_uri")
        break
    done
    if [ -z "$payto" ] || [ "$payto" = "null" ]
    then
        echo "$KYC_JSON"
        exit_fail "Instance ${id} never obtained a KYC status from the exchange"
    fi
    echo " DONE (status $status, account $payto)"

    echo -n "Making the KYCAUTH wire transfer for ${id} ..."
    status=$(curl -s -o "kycauth-${id}.json" -w "%{http_code}" \
                  -u "admin:secret-password" \
                  -H "Content-Type: application/json" \
                  -X POST \
                  -d "$(jq -n \
                           --arg pub "$pub" \
                           --arg debit "$payto" \
                           '{account_pub: $pub,
                             amount: "TESTKUDOS:0.1",
                             debit_account: $debit}')" \
                  "${WIRE_GATEWAY_URL}admin/add-kycauth")
    if [ "$status" != "200" ]
    then
        cat "kycauth-${id}.json"
        exit_fail "Expected 200 OK from admin/add-kycauth. Got: $status"
    fi
    echo " DONE"

    echo -n "Waiting for the exchange to accept the KYCAUTH of ${id} ..."
    for n in $(seq 1 100)
    do
        echo -n "."
        sleep 0.5
        KYC_JSON=$(curl -s "${base}private/kyc") || continue
        status=$(echo "$KYC_JSON" | jq -r "${filter} | .exchange_http_status // empty")
        if [ "$status" = "200" ]
        then
            break
        fi
    done
    if [ "$status" != "200" ]
    then
        echo "$KYC_JSON"
        exit_fail "Exchange never accepted the KYCAUTH transfer of ${id}"
    fi
    ACCESS_TOKEN=$(echo "$KYC_JSON" | jq -r "${filter} | .access_token // empty")
    if [ -z "$ACCESS_TOKEN" ]
    then
        echo "$KYC_JSON" | jq "${filter}"
        exit_fail "Instance ${id} did not receive a KYC access token"
    fi
    echo " DONE"
}

# Satisfy KYC measure M1 for the account behind access token $1 by
# submitting the form that KYC-CHECK-C1 asks for.
# taler-exchange-helper-measure-test-form then raises every limit to
# TESTKUDOS:1000, which un-blocks the account.
function submit_kyc_form() {
    local token="$1"
    local id
    local n

    # The measure is only instantiated when someone asks for it, so poll
    # until the exchange offers something to do.
    echo -n "Fetching KYC requirements ..."
    for n in $(seq 1 100)
    do
        echo -n "."
        curl -s -o kyc-info.json "${EXCHANGE_URL}kyc-info/${token}"
        id=$(jq -r '.requirements[]? | select(.id != null) | .id' < kyc-info.json \
                 | head -n1)
        if [ -n "$id" ]
        then
            break
        fi
        sleep 0.5
    done
    if [ -z "$id" ]
    then
        cat kyc-info.json
        exit_fail "Exchange did not offer a KYC requirement to fulfill"
    fi
    echo " DONE (requirement $id)"

    echo -n "Submitting the KYC form ..."
    STATUS=$(curl -s -o kyc-upload.json -w "%{http_code}" \
                  -H "Content-Type: application/json" \
                  -X POST \
                  -d '{"FULL_NAME":"Merchant Fourtythree",
                       "DATE_OF_BIRTH":"2000-01-01",
                       "FORM_ID":"full_name_and_birthdate"}' \
                  "${EXCHANGE_URL}kyc-upload/${id}")
    if [ "$STATUS" != "204" ]
    then
        cat kyc-upload.json
        exit_fail "Expected 204 No Content from kyc-upload. Got: $STATUS"
    fi
    echo " DONE"
}

# Run the aggregator once, logging to aggregator-$1.log.
#
# Note the deliberate absence of '-y' (--kyc-off): that option makes the
# aggregator wire funds out without consulting KYC at all, which is exactly
# what this reference database must not do.
#
# The '-T' travels two hours forward so that the deposits' wire deadlines
# have passed -- the wallet's integration-test order sets them a quarter of
# an hour out, and simply moving them into the past instead makes the
# aggregator spin forever on the KYC-blocked transient aggregation.  The
# price is that the wire_out rows come out stamped two hours ahead of when
# the bank actually moved the money, which the auditor would rightly report
# as a minor row inconsistency; the caller fixes the timestamps up again
# once the transfer has been made.
function run_aggregator() {
    echo -n "Running exchange aggregator ($1) ..."
    taler-exchange-aggregator \
        -T 7200000000 \
        -L INFO \
        -t \
        -c "${CONF}.edited" \
        2> "aggregator-$1.log" \
        || exit_fail "aggregator failed"
    echo " DONE"
}

# Pay TESTKUDOS:4 to the merchant instance $1, withdrawing TESTKUDOS:10
# first.  Uses its own wallet database so that coin selection does not
# depend on what an earlier instance already spent.
function pay_instance() {
    local id="$1"
    local base
    local db="wallet-${id}.wdb"
    local wlog="taler-wallet-cli-${id}.log"

    if [ "$id" = "admin" ]
    then
        base="${MERCHANT_URL}"
    else
        base="${MERCHANT_URL}instances/${id}/"
    fi

    rm -f "$db"
    echo -n "Running wallet against instance ${id} ..."
    taler-wallet-cli \
        --no-throttle \
        --wallet-db="$db" \
        api \
        --expect-success \
        'runIntegrationTest' \
        "$(jq -n '
        {
          amountToSpend: "TESTKUDOS:4",
          amountToWithdraw: "TESTKUDOS:10",
          corebankApiBaseUrl: $BANK_URL,
          exchangeBaseUrl: $EXCHANGE_URL,
          merchantBaseUrl: $MERCHANT_URL,
        }' \
        --arg MERCHANT_URL "$base" \
        --arg EXCHANGE_URL "$EXCHANGE_URL" \
        --arg BANK_URL "$BANK_URL"
      )" &> "$wlog" || {
        echo " FAILED! Last lines from $wlog:"
        tail "$wlog"
        exit 2
    }
    echo " DONE"
    taler-wallet-cli --wallet-db="$db" run-until-done &>> "$wlog"
}

# ----------------------------------------------------------------------
# Both instances prove that they own their bank account -- without that
# the exchange refuses their deposits outright -- and are then paid the
# same TESTKUDOS:4.  Both amounts exceed the AGGREGATE threshold, so the
# exchange accepts the coins but may not wire either of them out yet.
# ----------------------------------------------------------------------

setup_instance "admin" "$CLEARED_PAYTO" "Merchant43"
kycauth_instance "admin"
CLEARED_ACCESS_TOKEN="$ACCESS_TOKEN"

setup_instance "blocked" "$BLOCKED_PAYTO" "Blocked44"
kycauth_instance "blocked"
BLOCKED_ACCESS_TOKEN="$ACCESS_TOKEN"

pay_instance "admin"
pay_instance "blocked"

# The KYC rule only fires once the exchange actually tries to aggregate,
# so run the aggregator before there is a measure to satisfy.
run_aggregator "first"

echo -n "Checking that no wire transfer happened yet ..."
WIRE_OUT=$(psql -Aqt "auditor-basedb" \
                -c "SELECT COUNT(*) FROM exchange.wire_out;")
if [ "$WIRE_OUT" != "0" ]
then
    exit_fail "KYC should have blocked every aggregation, but got ${WIRE_OUT} wire_out rows"
fi
echo " DONE"

# ----------------------------------------------------------------------
# Only instance 'admin' passes the form.  After that its funds may be
# wired out; instance 'blocked' stays blocked, and the payout it is owed
# stays parked in aggregation_transient for good.
# ----------------------------------------------------------------------

submit_kyc_form "$CLEARED_ACCESS_TOKEN"

run_aggregator "second"

echo -n "Running exchange transfer ..."
taler-exchange-transfer \
    -L INFO \
    -t \
    -c "${CONF}.edited" \
    2> transfer.log \
    || exit_fail "transfer failed"
echo " DONE"


# Undo the side effect of the aggregator's time travel: the exchange
# recorded the wire transfer as having been executed two hours from now,
# while the bank stamped it with the real time.  The auditor compares the
# two and reports any disagreement of more than 15 minutes, so bring the
# exchange's timestamp back to reality.  wire_out.execution_date carries no
# signature; test-auditor.sh's test 17 edits the very same field.  Round to
# whole seconds: a GNUNET_TIME_Timestamp may not carry a sub-second part,
# and the helpers assert on that when they read the row back.
echo -n "Correcting the wire_out execution dates ..."
psql -Aqt "auditor-basedb" \
     -c "UPDATE exchange.wire_out SET execution_date=1000000*((SELECT transaction_date FROM libeufin_bank.bank_account_transactions WHERE direction='debit' ORDER BY bank_transaction_id DESC LIMIT 1)/1000000);" \
     > /dev/null
echo " DONE"

echo -n "Checking that exactly the cleared account was paid ..."
WIRE_OUT=$(psql -Aqt "auditor-basedb" \
                -c "SELECT COUNT(*) FROM exchange.wire_out;")
if [ "$WIRE_OUT" != "1" ]
then
    exit_fail "Expected exactly one wire transfer, got ${WIRE_OUT}"
fi
echo " DONE"

echo "KYC access token of the blocked account: ${BLOCKED_ACCESS_TOKEN}"

# Dump database
mkdir -p "$(dirname "$BASEDB")"

echo "Dumping database ${BASEDB}.sql"
pg_dump -O "auditor-basedb" | sed -e '/AS integer/d' > "${BASEDB}.sql"
cp "${CONF}.edited" "${BASEDB}.conf"
cp "$(taler-exchange-config -c "${CONF}.edited" -s exchange-offline -o MASTER_PRIV_FILE -f)" "${BASEDB}.mpriv"

# clean up
echo -n "Final clean up ..."
kill -TERM "$SETUP_PID"
wait
unset SETUP_PID
dropdb "auditor-basedb"
echo " DONE"

echo "====================================="
echo "Finished generation of ${BASEDB}.sql"
echo "====================================="

exit 0
