#!/bin/bash
set -euo pipefail

dir=`dirname $0`
updateoracle=false
suffix=.opt
removeoutfile=true
files=""
failed=""
whydata=$(bin/why3"$suffix" --print-datadir)
whylib=$(bin/why3"$suffix" --print-libdir)

while test $# != 0; do
  case "$1" in
    "--update-oracle")
      updateoracle=true
      shift 1
      ;;
    "--keep-out-files")
      removeoutfile=false
      shift 1
      ;;
    "--suffix")
      suffix="$2"
      shift 2
      ;;
    "-"*)
      printf "unknown option: %s\n" "$1"
      printf "usage: check-ce-bench [--update-oracle] [--keep-out-file] [--suffix s] <files>\n"
      printf "  <files> must be given without the '.mlw' suffix.\n"
      printf "  If <files> is empty, use all files from directory 'check-ce'.\n"
      printf "  The suffix <s> is appended on the Why3 executable name, e.g. '.opt'.\n"
      exit 2
      ;;
    *)
      files="$files $1"
      shift 1
  esac
done

if test "$files" = "" ; then
  files="$dir/check-ce/*.mlw"
fi

colorize() {
  if command -v pygmentize &> /dev/null; then
    pygmentize -ldiff
  else
    cat
  fi
}

remove_solver_details () {
  sed -e "s|$whydata|WHY3DATA|g" -e "s|$whylib|WHY3LIB|g" \
  | sed 's/ ([0-9.]\+\.[0-9]\+s, [0-9]\+ steps)\.$/\./' \
  | sed 's/Prover result is: \(Timeout\.\|Unknown (\(unknown\|incomplete\|unknown + \(incomplete\|interrupted\)\))\.\|Out of memory.\|Step limit exceeded\.\|Step limit exceeded ([^)]*)\.\)$/Prover result is: Unknown or time\/memory\/step limit./'
}


record_time () {
    elapsed=$(date +%s%N)
}

# print elapsed time
elapsed_time () {
    t=$(date +%s%N)
    d=$(expr $t - $elapsed)
#    printf "d=${d}\n"
    l=$(expr length $d)
#    printf "l=${l}\n"
    lmn=$(expr $l - 9 || true)
    if [ "$lmn" = "0" ] ; then
	d=0"$d"
	lmn=1
    fi
#    printf "lmn=${lmn}\n"
    s=$(expr substr "${d}" 1 ${lmn} || true)
#    printf "s=${s}\n"
    c=$(expr substr "${d}" ${lmn} 2 || true)
#    printf "c=${c}\n"
    elapsed="$s"."$c"
#    printf "${elapsed}\n"
}




# $1 = prover
# $2 = dir
# $3 = filename
# $4 = true for WP, false for SP
# $5 = steps
run () {
  file_path="$2/$3"
  debug_vc_sp=""
  if $4; then
    f="${file_path}_$1_WP"
    oracle_file="$2/oracles/$3_$1_WP.oracle"
    printf "${file_path},WP,$1... "
  else
    f="${file_path}_$1_SP"
    oracle_file="$2/oracles/$3_$1_SP.oracle"
    debug_vc_sp=",vc_sp"
    printf "${file_path},SP,$1... "
  fi
  steps=$5
  if [ "$3" = "strings" ] && [[ $1 == CVC* ]] ; then
    prover="$1,strings+counterexamples"
  else
    prover="$1,counterexamples"
  fi
  # one may add reduction_cont_size,stack_trace, in
  # the debug flags below to test cont_invariant in reduction engine
  # (note that stack trace may be more detailed with bytecode)
  record_time
  ($dir/../bin/why3"$suffix" prove -a split_vc -P $prover\
      --stepslimit=$steps --timelimit=10 \
      --rac-steplimit=$steps --rac-timelimit=10 \
      --check-ce --rac-prover="$1" --ce-log-verbosity=5 \
      --library="$2" "${file_path}.mlw" \
      --debug=check_ce:categorization${debug_vc_sp} \
    || true) 2>&1 \
    | remove_solver_details \
	  > "$f.out"
  elapsed_time
  str_out=$(cat "$f.out")
  if [ -e "$oracle_file" ]; then
    str_oracle=$(cat "$oracle_file")
  else
    str_oracle=""
  fi
  if [ "$str_oracle" = "$str_out" ] ; then
    echo "OK (in ${elapsed}s)"
  else
    if $updateoracle; then
      echo "Updating oracle"
      cp "$f.out" "${oracle_file}"
    else
      echo "FAILED"
      echo "diff is the following:"
	    (diff -u "$oracle_file" "$f.out" \
        || [ $? -eq 1 ])|colorize
      failed="$failed$f\n"
    fi
  fi
  if $removeoutfile; then
    rm "$f.out"
  fi
}



start=$(date +%s)
for file in $files; do
  filedir=`dirname $file`
  filebase=`basename $file .mlw`
  if [ "$filebase" = "maps_poly" -o "$filebase" = "maps_mono" ] ; then
    echo "Skipping tests on $filedir/$filebase.mlw";
  else
    printf "# Running provers on $filedir/$filebase.mlw\n";
    cvc4steps=150000
    cvc5steps=150000
    z3steps=3000000
    case "$filebase" in
      "640_no_loc_failure")
        z3steps=500000
        ;;
      "657")
        z3steps=500000
        ;;
      "668_projection")
        z3steps=500000
        ;;
      "bv32")
        z3steps=2500000
        ;;
      "bv32_mono")
        z3steps=100000
        ;;
      "bv32_toBig")
        z3steps=100000
        ;;
      "floats")
        z3steps=400000
        ;;
      "int_overflow")
        z3steps=400000
        ;;
      "loop_inv_real")
        z3steps=1000000
        ;;
      "maps_poly")
        cvc5steps=300000
        ;;
      "strings")
        z3steps=3000
        ;;
      "threshold")
        z3steps=500000
        ;;
      *)
        ;;
    esac
    run CVC4,1.8    $filedir $filebase true  $cvc4steps
    run CVC4,1.8    $filedir $filebase false $cvc4steps
    run CVC5,1.0.0  $filedir $filebase true  $cvc5steps
    run CVC5,1.0.0  $filedir $filebase false $cvc5steps
    run Z3,4.8.10   $filedir $filebase true  $z3steps
    run Z3,4.8.10   $filedir $filebase false $z3steps
  fi
done

# Check the reproduction of the experiments by Petiot (2018).
if $updateoracle; then
  updatearg="--updateoracle"
else
  updatearg=""
fi

# $1 = prover
# $2 = stepslimit
petiot () {
bench/check-ce/petiot2018/experiments.sh \
  --prover $1 --rac-prover $1 \
  --ce-prover $1,counterexamples --stepslimit $2 \
  $updatearg
if [ $? = 1 ]; then failed="${failed}Petiot experiments with $1.\n"; fi
}

petiot Z3,4.8.10  500000
petiot CVC4,1.8 10000
petiot CVC5,1.0.0 10000

end=$(date +%s)

runtime=$((end-start))
if [ "$failed" = "" ]; then
  echo "check-ce-bench: success (runtime = $runtime seconds)"
  exit 0
else
  printf "\ncheck-ce-bench: failed for the following files (runtime = $runtime seconds)\n$failed\n"
  exit 1
fi
