/* This file is part of the 'stringi' project.
 * Copyright (c) 2013-2024, Marek Gagolewski <https://www.gagolewski.com/>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */


#include "stri_stringi.h"
#include <unicode/ucsdet.h>
#include <unicode/locid.h>
#include <unicode/uloc.h>
#include <unicode/locid.h>
#include <unicode/ulocdata.h>
#include <unicode/uniset.h>
#include <map>
#include <vector>
#include <algorithm>
#include "stri_container_listraw.h"
#include "stri_container_logical.h"
#include "stri_ucnv.h"
using namespace std;


/** Check if a string may be valid 8-bit (including UTF-8) encoded
 *
 *  simple check whether all charcodes are nonzero
 *
 * @param str_cur_s character vector
 * @param str_cur_n number of bytes
 * @param get_confidence determine confidence value or do exact check
 *
 * @return confidence value in [0,1]
 *
 * @version 0.1-?? (Bartek Tartanus)
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-06)
 *          separate func
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-13)
 *          warnchars count added
 */
double stri__enc_check_8bit(const char* str_cur_s, R_len_t str_cur_n, bool get_confidence) {
    R_len_t warnchars = 0;
    for (R_len_t j=0; j < str_cur_n; ++j) {
        if (str_cur_s[j] == 0)
            return 0.0;
        if (get_confidence && (str_cur_s[j] <= 31 || str_cur_s[j] == 127)) {
            switch (str_cur_s[j]) {
            case 9:  // \t
            case 10: // \n
            case 13: // \r
            case 26: // ASCII SUBSTITUTE
                break; // ignore
            default:
                warnchars++;
            }
        }
    }
    return (get_confidence?(double)warnchars/double(str_cur_n):1.0);
}


/** Check if a string is valid ASCII
 *
 *  simple check whether charcodes are in [1..127]
 * by using U8_IS_SINGLE
 *
 * @param str_cur_s character vector
 * @param str_cur_n number of bytes
 * @param get_confidence determine confidence value or do exact check
 *
 * @return confidence value in [0,1]
 *
 * @version 0.1-?? (Bartek Tartanus)
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-06)
 *          separate func
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-13)
 *          warnchars count added
 */
double stri__enc_check_ascii(const char* str_cur_s, R_len_t str_cur_n, bool get_confidence) {
    R_len_t warnchars = 0;
    for (R_len_t j=0; j < str_cur_n; ++j) {
        if (!U8_IS_SINGLE(str_cur_s[j]) || str_cur_s[j] == 0) // i.e., 0 < c <= 127
            return 0.0;
        if (get_confidence && (str_cur_s[j] <= 31 || str_cur_s[j] == 127)) {
            switch (str_cur_s[j]) {
            case 9:  // \t
            case 10: // \n
            case 13: // \r
            case 26: // ASCII SUBSTITUTE
                break; // ignore
            default:
                warnchars++;
            }
        }
    }
    return (get_confidence?(double)(str_cur_n-warnchars)/double(str_cur_n):1.0);
}


/** Check if a string is valid UTF-8
 *
 * checks if a string is probably UTF-8-encoded;
 * simple check with U8_NEXT
 *
 *
 * @param str_cur_s character vector
 * @param str_cur_n number of bytes
 * @param get_confidence determine confidence value or do exact check
 *
 * @return confidence value in [0,1]
 *
 * @version 0.1-?? (Bartek Tartanus)
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-06)
 *          separate func
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-13)
 *          confidence calculation basing on ICU's i18n/csrutf8.cpp
 */
double stri__enc_check_utf8(const char* str_cur_s, R_len_t str_cur_n, bool get_confidence)
{
    if (!get_confidence) {
        UChar32 c;
        for (R_len_t j=0; j < str_cur_n; ) {
            if (str_cur_s[j] == 0)
                return 0.0; // definitely not valid UTF-8

            U8_NEXT(str_cur_s, j, str_cur_n, c);
            if (c < 0) // ICU utf8.h doc for U8_NEXT: c -> output UChar32 variable, set to <0 in case of an error
                return 0.0; // definitely not valid UTF-8
        }
        return 1.0;
    }
    else {
        // Based on ICU's i18n/csrutf8.cpp [with own mods]
        bool hasBOM = (str_cur_n >= 3 &&
                       (uint8_t)(str_cur_s[0]) == UTF8_BOM_BYTE1 &&
                       (uint8_t)(str_cur_s[1]) == UTF8_BOM_BYTE2 &&
                       (uint8_t)(str_cur_s[2]) == UTF8_BOM_BYTE3);
        R_len_t numValid = 0;   // counts only valid UTF-8 multibyte seqs
        R_len_t numInvalid = 0;

        // Scan for multi-byte sequences
        for (R_len_t i=0; i < str_cur_n; i += 1) {
            uint32_t b = str_cur_s[i];

            if ((b & 0x80) == 0) {
                continue;   // ASCII => OK
            }

            // Hi bit on char found.  Figure out how long the sequence should be
            R_len_t trailBytes = 0;
            if ((b & 0x0E0) == 0x0C0)
                trailBytes = 1;
            else if ((b & 0x0F0) == 0x0E0)
                trailBytes = 2;
            else if ((b & 0x0F8) == 0xF0)
                trailBytes = 3;
            else {
                numInvalid += 1;
                if (numInvalid > 5)
                    break; // that's enough => not UTF-8
                continue;
            }

            // Verify that we've got the right number of trail bytes in the sequence
            while (true) {
                i += 1;

                if (i >= str_cur_n)
                    break;

                b = str_cur_s[i];

                if ((b & 0xC0) != 0x080) {
                    numInvalid += 1;
                    break;
                }

                if (--trailBytes == 0) {
                    numValid += 1;
                    break;
                }
            }
        }

        // Cook up some sort of confidence score, based on presense of a BOM
        //    and the existence of valid and/or invalid multi-byte sequences.
        if (hasBOM && numInvalid == 0)
            return 1.0;
        else if (hasBOM && numValid > numInvalid*10)
            return 0.75;
        else if (numValid > 3 && numInvalid == 0)
            return 1.0;
        else if (numValid > 0 && numInvalid == 0)
            return 0.50; // too few multibyte UTF-8 seqs to be quite sure
        else if (numValid == 0 && numInvalid == 0)
            // Plain ASCII. => It's OK for UTF-8
            return 0.50;
        else if (numValid > numInvalid*10)
            // Probably corrupt utf-8 data.  Valid sequences aren't likely by chance.
            return 0.25;
        else
            return 0.0;
    }
}


/** Check if a string is valid UTF-16LE or UTF-16BE
 *
 * @param str_cur_s character vector
 * @param str_cur_n number of bytes
 * @param get_confidence determine confidence value or do exact check
 * @param le check for UTF-16LE?
 *
 * @return confidence value in [0,1]
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-09)
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-14)
 *          confidence calculation basing on ICU's i18n/csucode.cpp
 */
double stri__enc_check_utf16(const char* str_cur_s, R_len_t str_cur_n,
                             bool get_confidence, bool le)
{
    if (str_cur_n % 2 != 0)
        return 0.0;

    bool hasLE_BOM = STRI__ENC_HAS_BOM_UTF16LE(str_cur_s, str_cur_n);
    bool hasBE_BOM = STRI__ENC_HAS_BOM_UTF16BE(str_cur_s, str_cur_n);

    if ((!le && hasLE_BOM) || (le && hasBE_BOM))
        return 0.0;

    R_len_t warnchars = 0;

    for (R_len_t i=0; i<str_cur_n; i += 2) {
        uint16_t c = (le)?
                     STRI__GET_INT16_LE(str_cur_s, i):
                     STRI__GET_INT16_BE(str_cur_s, i);

        if (U16_IS_SINGLE(c)) {
            if (c == 0)
                return 0.0;
            else if (c >= 0x0530) // last cyrrilic supplement
                warnchars += 2;
            continue;
        }

        if (!U16_IS_SURROGATE_LEAD(c))
            return 0.0;

        i += 2;
        if (i >= str_cur_n)
            return 0.0;
        c = (le)?
            STRI__GET_INT16_LE(str_cur_s, i):
            STRI__GET_INT16_BE(str_cur_s, i);
        if (!U16_IS_SURROGATE_TRAIL(c))
            return 0.0;
    }

    return (get_confidence?(double)(str_cur_n-warnchars)/double(str_cur_n):1.0);
}


/** Check if a string is valid UTF-16BE
 *
 * @param str_cur_s character vector
 * @param str_cur_n number of bytes
 * @param get_confidence determine confidence value or to exact check
 *
 * @return confidence value in [0,1]
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-09)
 */
double stri__enc_check_utf16be(const char* str_cur_s, R_len_t str_cur_n, bool get_confidence)
{
    return stri__enc_check_utf16(str_cur_s, str_cur_n, get_confidence, false);
}


/** Check if a string is valid UTF-16LE
 *
 * @param str_cur_s character vector
 * @param str_cur_n number of bytes
 * @param get_confidence determine confidence value or do exact check
 *
 * @return confidence value in [0,1]
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-09)
 */
double stri__enc_check_utf16le(const char* str_cur_s, R_len_t str_cur_n, bool get_confidence)
{
    return stri__enc_check_utf16(str_cur_s, str_cur_n, get_confidence, true);
}


/** Check if a string is valid UTF-32LE or UTF-32BE
 *
 * @param str_cur_s character vector
 * @param str_cur_n number of bytes
 * @param get_confidence determine confidence value or do exact check
 * @param le check for UTF-32LE?
 *
 * @return confidence value in [0,1]
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-09)
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-13)
 *          confidence calculation basing on ICU's i18n/csucode.cpp
 */
double stri__enc_check_utf32(const char* str_cur_s, R_len_t str_cur_n,
                             bool get_confidence, bool le)
{
    if (str_cur_n % 4 != 0)
        return 0.0;

    bool hasLE_BOM = STRI__ENC_HAS_BOM_UTF32LE(str_cur_s, str_cur_n);
    bool hasBE_BOM = STRI__ENC_HAS_BOM_UTF32BE(str_cur_s, str_cur_n);

    if ((!le && hasLE_BOM) || (le && hasBE_BOM))
        return 0.0;

    R_len_t numValid = 0;
    R_len_t numInvalid = 0;

    for (R_len_t i=0; i<str_cur_n; i+=4) {
        int32_t ch = le?
                     (int32_t)STRI__GET_INT32_LE(str_cur_s, i):
                     (int32_t)STRI__GET_INT32_BE(str_cur_s, i);

        if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
            if (!get_confidence)
                return 0.0;
            else
                numInvalid++;
        }
        else
            numValid++;
    }

    if (!get_confidence)
        return 1.0;

    if ((hasLE_BOM || hasBE_BOM) && numInvalid==0)
        return 1.0;
    else if ((hasLE_BOM || hasBE_BOM) && numValid > numInvalid*10)
        return 0.80;
    else if (numValid > 3 && numInvalid == 0)
        return 1.0;
    else if (numValid > 0 && numInvalid == 0)
        return 0.80;
    else if (numValid > numInvalid*10)
        return 0.25; // Probably corruput UTF-32BE data. Valid sequences aren't likely by chance.
    else
        return 0.0;
}


/** Check if a string is valid UTF-32BE
 *
 * @param str_cur_s character vector
 * @param str_cur_n number of bytes
 * @param get_confidence determine confidence value or do exact check
 *
 * @return confidence value in [0,1]
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-13)
 */
double stri__enc_check_utf32be(const char* str_cur_s, R_len_t str_cur_n, bool get_confidence)
{
    return stri__enc_check_utf32(str_cur_s, str_cur_n, get_confidence, false);
}


/** Check if a string is valid UTF-32LE
 *
 * @param str_cur_s character vector
 * @param str_cur_n number of bytes
 * @param get_confidence determine confidence value or do exact check
 *
 * @return confidence value in [0,1]
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-13)
 */
double stri__enc_check_utf32le(const char* str_cur_s, R_len_t str_cur_n, bool get_confidence)
{
    return stri__enc_check_utf32(str_cur_s, str_cur_n, get_confidence, true);
}


/** Which string is in given encoding
 *
 *
 *  @param str character vector or raw vector or list of raw vectors
 *  @param type (single integer, internal)
 *  @return logical vector
 *
 * @version 0.1-?? (Bartek Tartanus)
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-08)
 *          use StriContainerListRaw
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-09)
 *          one function for is_*, do dispatch
 *
 * @version 0.3-1 (Marek Gagolewski, 2014-11-04)
 *    Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
 *
 * @version 0.6-1 (Marek Gagolewski, 2015-07-11)
 *    this is internal function now
 */
SEXP stri_enc_isenc(SEXP str, int _type)
{
    double (*isenc)(const char*, R_len_t, bool) = NULL;
    switch (_type) {
    case 1:
        isenc = stri__enc_check_ascii;
        break;
    case 2:
        isenc = stri__enc_check_utf8;
        break;
    case 3:
        isenc = stri__enc_check_utf16be;
        break;
    case 4:
        isenc = stri__enc_check_utf16le;
        break;
    case 5:
        isenc = stri__enc_check_utf32be;
        break;
    case 6:
        isenc = stri__enc_check_utf32le;
        break;
    default:
        Rf_error(MSG__INCORRECT_INTERNAL_ARG); // error() call allowed here
    }


    PROTECT(str = stri__prepare_arg_list_raw(str, "str"));

    STRI__ERROR_HANDLER_BEGIN(1)
    StriContainerListRaw str_cont(str);
    R_len_t str_length = str_cont.get_n();

    SEXP ret;
    STRI__PROTECT(ret = Rf_allocVector(LGLSXP, str_length));
    int* ret_tab = LOGICAL(ret); // may be faster than LOGICAL(ret)[i] all the time

    for (R_len_t i=0; i < str_length; ++i) {
        if (str_cont.isNA(i)) {
            ret_tab[i] = NA_LOGICAL;
            continue;
        }

        bool get_confidence = false; // TO BE DONE
        ret_tab[i] = isenc(str_cont.get(i).c_str(), str_cont.get(i).length(), get_confidence) != 0.0;
    }

    STRI__UNPROTECT_ALL
    return ret;

    STRI__ERROR_HANDLER_END({ /* no-op on error */ })
}


/** Which string is in ASCII
 *
 *  @param str character vector or raw vector or list of raw vectors
 *  @return logical vector
 *
 * @version 0.6-1 (Marek Gagolewski, 2015-07-11)
 *    call stri_enc_isenc
 */
SEXP stri_enc_isascii(SEXP str) {
    return stri_enc_isenc(str, 1);
}


/** Which string is in UTF8
 *
 *  @param str character vector or raw vector or list of raw vectors
 *  @return logical vector
 *
 * @version 0.6-1 (Marek Gagolewski, 2015-07-11)
 *    call stri_enc_isenc
 */
SEXP stri_enc_isutf8(SEXP str) {
    return stri_enc_isenc(str, 2);
}


/** Which string is in UTF-16BE
 *
 *  @param str character vector or raw vector or list of raw vectors
 *  @return logical vector
 *
 * @version 0.6-1 (Marek Gagolewski, 2015-07-11)
 *    call stri_enc_isenc
 */
SEXP stri_enc_isutf16be(SEXP str) {
    return stri_enc_isenc(str, 3);
}


/** Which string is in UTF16-LE
 *
 *  @param str character vector or raw vector or list of raw vectors
 *  @return logical vector
 *
 * @version 0.6-1 (Marek Gagolewski, 2015-07-11)
 *    call stri_enc_isenc
 */
SEXP stri_enc_isutf16le(SEXP str) {
    return stri_enc_isenc(str, 4);
}


/** Which string is in UTF-32BE
 *
 *  @param str character vector or raw vector or list of raw vectors
 *  @return logical vector
 *
 * @version 0.6-1 (Marek Gagolewski, 2015-07-11)
 *    call stri_enc_isenc
 */
SEXP stri_enc_isutf32be(SEXP str) {
    return stri_enc_isenc(str, 5);
}


/** Which string is in UTF32-LE
 *
 *  @param str character vector or raw vector or list of raw vectors
 *  @return logical vector
 *
 * @version 0.6-1 (Marek Gagolewski, 2015-07-11)
 *    call stri_enc_isenc
 */
SEXP stri_enc_isutf32le(SEXP str) {
    return stri_enc_isenc(str, 6);
}


/** Detect encoding and language
 *
 * @param str character vector
 * @param filter_angle_brackets logical vector
 *
 * @return list
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-03)
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-08)
 *          use StriContainerListRaw + BUGFIX
 *
 * @version 0.3-1 (Marek Gagolewski, 2014-11-04)
 *    Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
 */
SEXP stri_enc_detect(SEXP str, SEXP filter_angle_brackets)
{
    PROTECT(str = stri__prepare_arg_list_raw(str, "str"));
    PROTECT(filter_angle_brackets = stri__prepare_arg_logical(filter_angle_brackets, "filter_angle_brackets"));

    UCharsetDetector* ucsdet = NULL;


    STRI__ERROR_HANDLER_BEGIN(2)

    UErrorCode status = U_ZERO_ERROR;
    ucsdet = ucsdet_open(&status);
    STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})

    StriContainerListRaw str_cont(str);
    R_len_t str_n = str_cont.get_n();

    R_len_t vectorize_length = stri__recycling_rule(true, 2, str_n, LENGTH(filter_angle_brackets));
    str_cont.set_nrecycle(vectorize_length); // must be set after container creation

    SEXP ret, names, wrong;
    STRI__PROTECT(ret = Rf_allocVector(VECSXP, vectorize_length));

    STRI__PROTECT(names = Rf_allocVector(STRSXP, 3));
    SET_STRING_ELT(names, 0, Rf_mkChar("Encoding"));
    SET_STRING_ELT(names, 1, Rf_mkChar("Language"));
    SET_STRING_ELT(names, 2, Rf_mkChar("Confidence"));

    STRI__PROTECT(wrong = Rf_allocVector(VECSXP, 3));
    SET_VECTOR_ELT(wrong, 0, stri__vector_NA_strings(1));
    SET_VECTOR_ELT(wrong, 1, stri__vector_NA_strings(1));
    SET_VECTOR_ELT(wrong, 2, stri__vector_NA_integers(1));
    Rf_setAttrib(wrong, R_NamesSymbol, names);

    StriContainerLogical filter(filter_angle_brackets, vectorize_length);
    for (R_len_t i=0; i<vectorize_length; ++i) {
        if (str_cont.isNA(i) || filter.isNA(i)) {
            SET_VECTOR_ELT(ret, i, wrong);
            continue;
        }

        const char* str_cur_s = str_cont.get(i).c_str();
        R_len_t str_cur_n     = str_cont.get(i).length();

        status = U_ZERO_ERROR;
        ucsdet_setText(ucsdet, str_cur_s, str_cur_n, &status);
        STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
        ucsdet_enableInputFilter(ucsdet, filter.get(i));

        status = U_ZERO_ERROR;
        int matchesFound;
        const UCharsetMatch** match = ucsdet_detectAll(ucsdet, &matchesFound, &status);
        if (U_FAILURE(status) || !match || matchesFound <= 0) {
            SET_VECTOR_ELT(ret, i, wrong);
            continue;
        }


        SEXP val_enc, val_lang, val_conf;
        STRI__PROTECT(val_enc  = Rf_allocVector(STRSXP, matchesFound));
        STRI__PROTECT(val_lang = Rf_allocVector(STRSXP, matchesFound));
        STRI__PROTECT(val_conf = Rf_allocVector(REALSXP, matchesFound));

        for (R_len_t j=0; j<matchesFound; ++j) {
            status = U_ZERO_ERROR;
            const char* name = ucsdet_getName(match[j], &status);
            if (U_FAILURE(status) || !name)
                SET_STRING_ELT(val_enc, j, NA_STRING);
            else
                SET_STRING_ELT(val_enc, j, Rf_mkChar(name));

            status = U_ZERO_ERROR;
            int32_t conf = ucsdet_getConfidence(match[j], &status);
            if (U_FAILURE(status))
                REAL(val_conf)[j] = NA_REAL;
            else
                REAL(val_conf)[j] = (double)(conf)/100.0;

            status = U_ZERO_ERROR;
            const char* lang = ucsdet_getLanguage(match[j], &status);
            if (U_FAILURE(status) || !lang)
                SET_STRING_ELT(val_lang, j, NA_STRING);
            else
                SET_STRING_ELT(val_lang, j, Rf_mkChar(lang));
        }

        SEXP val;
        STRI__PROTECT(val = Rf_allocVector(VECSXP, 3));
        SET_VECTOR_ELT(val, 0, val_enc);
        SET_VECTOR_ELT(val, 1, val_lang);
        SET_VECTOR_ELT(val, 2, val_conf);
        Rf_setAttrib(val, R_NamesSymbol, names);
        SET_VECTOR_ELT(ret, i, val);
        STRI__UNPROTECT(4);
    }

    if (ucsdet) {
        ucsdet_close(ucsdet);
        ucsdet = NULL;
    }
    STRI__UNPROTECT_ALL
    return ret;

    STRI__ERROR_HANDLER_END(
    if (ucsdet) {
    ucsdet_close(ucsdet);
        ucsdet = NULL;
    })
}


// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------


/** locale-dependent 8-bit converter check  [DEPRECATED]
 *
 * help struct for stri_enc_detect2
 *
 * @version 0.1-??
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-18)
 *          be locale-dependent, use ICU ulocdata
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-11-13)
 *          allow only ASCII-supersets
 *
 * @version 0.2-1 (Marek Gagolewski, 2014-03-28)
 *          use StriUcnv
 */
struct Converter8bit {
    bool isNA;
    bool countChars[256];
    bool badChars[256];
    const char* name;
    const char* friendlyname;

    Converter8bit(const char* _name, const char* _friendlyname, const UnicodeSet* exset) {
        isNA = true;
        name = NULL;
        friendlyname = NULL;
        StriUcnv ucnv_obj(_name);
        if (!ucnv_obj.is8bit())
            return; // not an 8-bit converter

        //ucnv_obj.setCallBackSubstitute(); // restore default (no warn) callbacks
        UConverter* ucnv = ucnv_obj.getConverter(false);


        // Check which characters in given encoding
        // are not mapped to Unicode [badChars]
        char allChars[256+1]; // all bytes 0-255
        for (R_len_t i=0; i<256; ++i)
            allChars[i] = (char)i;
        allChars[256] = '\0';

        // reset tabs
        for (R_len_t i=0; i<256; ++i) {
            countChars[i] = false;
            badChars[i]   = false;
        }

        UnicodeSet curset;
        std::map<UChar32, uint8_t> curmap;
        const char* text_start = allChars+1;
        const char* text_end   = allChars+256;
        ucnv_reset(ucnv);
        for (R_len_t i=1; i<256; ++i) {
            UErrorCode status = U_ZERO_ERROR;
            UChar32 c = ucnv_getNextUChar(ucnv, &text_start, text_end, &status);
            if (U_FAILURE(status)) {
                return;
            }
            if (i >= 32 && i <= 127 && c != (UChar32)i) {
                // allow only ASCII supersets
                return;
            }

            if (c == UCHAR_REPLACEMENT || c < 0) {
                badChars[i] = true;
            }
            else {
                if (!u_isdefined(c) || u_isalpha(c))
                    badChars[i] = true;
                curset.add(c);
                curmap[c] = (uint8_t)i;
            }
        }


        if (!curset.containsAll(*exset)) {
            // not all characters are representable in given encoding
            return;
        }


        // now mark all characters form exset to be counted
        R_len_t exset_size = exset->size();
        for (R_len_t k=0; k<exset_size; ++k) {
            UChar32 c = exset->charAt(k);
            if (c >= 0) {
                uint8_t ind = curmap[c];
                countChars[ind] = true;
            }
        }

        isNA = false;
        this->name = _name;
        this->friendlyname = _friendlyname;
    }
};


// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------


/** Guesses text encoding; help struct for stri_enc_detect2  [DEPRECATED]
 *
 * @version 0.1-?? (Marek Gagolewski)
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-08-18)
 *          locale-dependent, use ulocdata
 *
 * @version 0.1-?? (Marek Gagolewski, 2013-11-13)
 *          allow qloc==NULL in 8bit check
 *
 * @version 0.5-1 (Marek Gagolewski, 2015-02-24)
 *          #146 warnings removed
 */
struct EncGuess {
    const char* name;
    const char* friendlyname;
    double confidence;

    EncGuess(const char* _name, const char* _friendlyname, double _confidence) {
        name = _name;
        friendlyname = _friendlyname;
        confidence = _confidence;
    }

    bool operator<(const EncGuess& e2) const {
        return (this->confidence > e2.confidence); // decreasing sort
    }

    static void do_utf32(vector<EncGuess>& guesses, const char* str_cur_s,
                         R_len_t str_cur_n)
    {
        /* check UTF-32LE, UTF-32BE or UTF-32+BOM */
        double isutf32le = stri__enc_check_utf32le(str_cur_s, str_cur_n, true);
        double isutf32be = stri__enc_check_utf32be(str_cur_s, str_cur_n, true);
        if (isutf32le >= 0.25 && isutf32be >= 0.25) {
            // no BOM, both valid
            // i think this will never happen
            guesses.push_back(EncGuess("UTF-32LE", "UTF-32LE", isutf32le));
            guesses.push_back(EncGuess("UTF-32BE", "UTF-32BE", isutf32be));
        }
        else if (isutf32le >= 0.25) {
            if (STRI__ENC_HAS_BOM_UTF32LE(str_cur_s, str_cur_n))
                guesses.push_back(EncGuess("UTF-32", "UTF-32", isutf32le)); // with BOM
            else
                guesses.push_back(EncGuess("UTF-32LE", "UTF-32LE", isutf32le));
        }
        else if (isutf32be >= 0.25) {
            if (STRI__ENC_HAS_BOM_UTF32BE(str_cur_s, str_cur_n))
                guesses.push_back(EncGuess("UTF-32", "UTF-32", isutf32be)); // with BOM
            else
                guesses.push_back(EncGuess("UTF-32BE", "UTF-32BE", isutf32be));
        }
    }

    static void do_utf16(vector<EncGuess>& guesses, const char* str_cur_s,
                         R_len_t str_cur_n)
    {
        /* check UTF-16LE, UTF-16BE or UTF-16+BOM */
        double isutf16le = stri__enc_check_utf16le(str_cur_s, str_cur_n, true);
        double isutf16be = stri__enc_check_utf16be(str_cur_s, str_cur_n, true);
        if (isutf16le >= 0.25 && isutf16be >= 0.25) {
            // no BOM, both valid
            // this may sometimes happen
            guesses.push_back(EncGuess("UTF-16LE", "UTF-16LE", isutf16le));
            guesses.push_back(EncGuess("UTF-16BE", "UTF-16BE", isutf16be));
        }
        else if (isutf16le >= 0.25) {
            if (STRI__ENC_HAS_BOM_UTF16LE(str_cur_s, str_cur_n))
                guesses.push_back(EncGuess("UTF-16", "UTF-16", isutf16le)); // with BOM
            else
                guesses.push_back(EncGuess("UTF-16LE", "UTF-16LE", isutf16le));
        }
        else if (isutf16be >= 0.25) {
            if (STRI__ENC_HAS_BOM_UTF16BE(str_cur_s, str_cur_n))
                guesses.push_back(EncGuess("UTF-16", "UTF-16", isutf16be)); // with BOM
            else
                guesses.push_back(EncGuess("UTF-16BE", "UTF-16BE", isutf16be));
        }
    }

    static void do_8bit(vector<EncGuess>& guesses, const char* str_cur_s,
                        R_len_t str_cur_n, const char* qloc)
    {
        double is8bit = stri__enc_check_8bit(str_cur_s, str_cur_n, false);
        if (is8bit != 0.0) {
            // may be an 8-bit encoding
            double isascii = stri__enc_check_ascii(str_cur_s, str_cur_n, true);
            if (isascii >= 0.25) // i.e., equal to 1.0 => nothing more to check
                guesses.push_back(EncGuess("US-ASCII", "US-ASCII", isascii));
            else {
                // not ascii
                double isutf8 = stri__enc_check_utf8(str_cur_s, str_cur_n, true);
                if (isutf8 >= 0.25)
                    guesses.push_back(EncGuess("UTF-8", "UTF-8", isutf8));
                if (isutf8 < 1.0 && qloc) {
                    do_8bit_locale(guesses, str_cur_s, str_cur_n, qloc);
                }
            }
        }
    }

    static void do_8bit_locale(vector<EncGuess>& guesses, const char* str_cur_s,
                               R_len_t str_cur_n, const char* qloc)
    {
        vector<Converter8bit> converters;
        if (!qloc) throw StriException(MSG__INTERNAL_ERROR); // just to be sure

        UErrorCode status = U_ZERO_ERROR;
        ULocaleData* uld = ulocdata_open(qloc, &status);
        STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})

        USet* exset_tmp = ulocdata_getExemplarSet(uld, NULL,
                          USET_ADD_CASE_MAPPINGS, ULOCDATA_ES_STANDARD, &status);
        STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
        UnicodeSet* exset = UnicodeSet::fromUSet(exset_tmp); // don't delete, just a pointer
        exset->removeAllStrings();


        R_len_t ucnv_count = (R_len_t)ucnv_countAvailable();
        for (R_len_t i=0; i<ucnv_count; ++i) { // for each converter
            Converter8bit conv(ucnv_getAvailableName(i), StriUcnv::getFriendlyName(ucnv_getAvailableName(i)), exset);
            if (!conv.isNA) converters.push_back(conv);
        }

        uset_close(exset_tmp);
        exset = NULL;
        ulocdata_close(uld);

        if (converters.size() <= 0)
            return;

        // count all bytes with codes >= 128 in str_cur_s
        R_len_t counts[256];
        R_len_t countsge128 = 0; // total count
        for (R_len_t k=0; k<256; ++k)
            counts[k] = 0; // reset tab
        for (R_len_t j=0; j<str_cur_n; ++j) {
            if ((uint8_t)(str_cur_s[j]) >= (uint8_t)128) {
                counts[(uint8_t)(str_cur_s[j])]++;
                countsge128++;
            }
        }
        // assert: countsge128 > 0 (otherwise ASCII, so this function hasn't been not called)

        std::vector<int> badCounts(converters.size(), 0); // filled with 0
        std::vector<int> desiredCounts(converters.size(),0);
        R_len_t maxDesiredCounts = 0;


        for (R_len_t j=0; j<(R_len_t)converters.size(); ++j) { // for each converter
            for (R_len_t k=128; k<256; ++k) { // for each character
                // 1. Count bytes that are BAD and NOT COUNTED in this encoding
                if (converters[j].badChars[k] && !converters[j].countChars[k]) {
                    badCounts[j] += (int)counts[k];
                }
                // 2. Count indicated characters
                if (converters[j].countChars[k]) {
                    desiredCounts[j] += (int)counts[k];
                }
            }
            if (desiredCounts[j] > maxDesiredCounts)
                maxDesiredCounts = desiredCounts[j];
        }

        // add guesses
        for (R_len_t j=0; j<(R_len_t)converters.size(); ++j) { // for each converter
            // some heuristic:
            double conf = min(1.0, max(0.0,
                                       (double)(countsge128-0.5*badCounts[j]-maxDesiredCounts+desiredCounts[j])/
                                       (double)(countsge128)));
            if (conf > 0.25)
                guesses.push_back(EncGuess(converters[j].name, converters[j].friendlyname, conf));
        }
    }
};


// -----------------------------------------------------------------------
// -----------------------------------------------------------------------


/** Detect encoding with initial guess  [DEPRECATED]
 *
 * @param str character or raw vector or a list of raw vectors
 * @param loc locale id
 *
 * @return list
 *
 * @version 0.1-?? (2013-08-15, Marek Gagolewski)
 *
 * @version 0.1-?? (2013-08-18, Marek Gagolewski)
 *          improved 8-bit confidence measurement,
 *          some code moved to structs, use locale & ICU locdata
 *
 * @version 0.1-?? (2013-11-13, Marek Gagolewski)
 *          added loc NA handling (no locale)
 *
 * @version 0.3-1 (Marek Gagolewski, 2014-11-04)
 *    Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
 */
SEXP stri_enc_detect2(SEXP str, SEXP loc)
{
    const char* qloc = /* this is R_alloc'ed */
        stri__prepare_arg_locale(loc, "locale");
    // raw vector, character vector, or list of raw vectors:
    PROTECT(str = stri__prepare_arg_list_raw(str, "str"));

    STRI__ERROR_HANDLER_BEGIN(1)

    StriContainerListRaw str_cont(str);
    R_len_t str_n = str_cont.get_n();

    SEXP ret, names, wrong;
    STRI__PROTECT(ret = Rf_allocVector(VECSXP, str_n));

    STRI__PROTECT(names = Rf_allocVector(STRSXP, 3));
    SET_STRING_ELT(names, 0, Rf_mkChar("Encoding"));
    SET_STRING_ELT(names, 1, Rf_mkChar("Language"));
    SET_STRING_ELT(names, 2, Rf_mkChar("Confidence"));

    STRI__PROTECT(wrong = Rf_allocVector(VECSXP, 3));
    SET_VECTOR_ELT(wrong, 0, stri__vector_NA_strings(1));
    SET_VECTOR_ELT(wrong, 1, stri__vector_NA_strings(1));
    SET_VECTOR_ELT(wrong, 2, stri__vector_NA_integers(1));
    Rf_setAttrib(wrong, R_NamesSymbol, names);

    for (R_len_t i=0; i<str_n; ++i) {
        if (str_cont.isNA(i)) {
            SET_VECTOR_ELT(ret, i, wrong);
            continue;
        }

        const char* str_cur_s = str_cont.get(i).c_str();
        R_len_t str_cur_n     = str_cont.get(i).length();
        if (str_cur_n <= 0) {
            SET_VECTOR_ELT(ret, i, wrong);
            continue;
        }

        vector<EncGuess> guesses;
        guesses.reserve(6);

        EncGuess::do_utf32(guesses, str_cur_s, str_cur_n);
        EncGuess::do_utf16(guesses, str_cur_s, str_cur_n);
        EncGuess::do_8bit(guesses, str_cur_s, str_cur_n, qloc);  // includes UTF-8

        R_len_t matchesFound = (R_len_t)guesses.size();
        if (matchesFound <= 0) {
            SET_VECTOR_ELT(ret, i, wrong);
            continue;
        }

        std::stable_sort(guesses.begin(), guesses.end());

        SEXP val_enc, val_lang, val_conf;
        STRI__PROTECT(val_enc  = Rf_allocVector(STRSXP, matchesFound));
        STRI__PROTECT(val_lang = Rf_allocVector(STRSXP, matchesFound));
        STRI__PROTECT(val_conf = Rf_allocVector(REALSXP, matchesFound));

        for (R_len_t j=0; j<matchesFound; ++j) {
            SET_STRING_ELT(val_enc, j, Rf_mkChar(guesses[j].friendlyname));
            REAL(val_conf)[j] = guesses[j].confidence;
            SET_STRING_ELT(val_lang, j, NA_STRING); // always no lang
        }

        SEXP val;
        STRI__PROTECT(val = Rf_allocVector(VECSXP, 3));
        SET_VECTOR_ELT(val, 0, val_enc);
        SET_VECTOR_ELT(val, 1, val_lang);
        SET_VECTOR_ELT(val, 2, val_conf);
        Rf_setAttrib(val, R_NamesSymbol, names);
        SET_VECTOR_ELT(ret, i, val);
        STRI__UNPROTECT(4);
    }

    STRI__UNPROTECT_ALL
    return ret;

    STRI__ERROR_HANDLER_END({ /* no-op on error */ })
}
