/***************************************************************************
 *   Copyright (C) 2004-2017 by Pere Constans
 *   constans@molspaces.com
 *   cb2Bib version 1.9.7. Licensed under the GNU GPL version 3.
 *   See the LICENSE file that comes with this distribution.
 ***************************************************************************/
#include "cb2bib_utilities.h"

#include <QRegularExpressionMatchIterator>

#ifdef C2B_USE_LZO
#include <lzo/lzoconf.h>
#include <lzo/lzo1x.h>
#endif

namespace c2bUtils
{

QString setCapitalization(const QString& str)
{
    QString cap_string(str);
    if (isUpperCaseString(str))
        cap_string = cap_string.toLower();
    bool do_upper(true);
    for (int i = 0; i < cap_string.length(); i++)
        if (cap_string.at(i).isLetter())
        {
            if (do_upper)
                cap_string[i] = cap_string.at(i).toUpper();
            do_upper = false;
        }
        else if (cap_string.at(i) == '.' || cap_string.at(i) == ':')
            do_upper = true;
    return (cap_string);
}

QString& simplifyString(QString& str)
{
    if (str.length() == 0)
        return str;
    const ushort space(32);
    ushort* const c0 = (ushort*)str.data();
    ushort* const cn = c0 + str.length();
    ushort* c = c0;
    ushort* o = c0;
    while (c < cn)
    {
        const ushort ch = *c;
        if ((ch > 32 && ch < 160) || !(ch == space || QChar(ch).isSpace()))
            *o++ = ch;
        else if (o > c0 && *(o - 1) != space)
            *o++ = space;
        ++c;
    }
    if (o > c0 && *(o - 1) == space)
        --o;
    str.truncate(int(o - c0));
    return str;
}

QString& fillString(QString& str, const QStringMatcher& pattern, const QChar& ch)
{
    if (str.length() == 0)
        return str;
    const int pl(pattern.pattern().length());
    const ushort uch(ch.unicode());
    ushort* const c0((ushort*)str.data());
    int p(0);
    while (p >= 0)
    {
        p = pattern.indexIn(str, p);
        if (p > -1)
        {
            ushort* c(c0 + p);
            const ushort* const cpl(c + pl);
            while (c < cpl)
                *c++ = uch;
            p += pl;
        }
    }
    return str;
}

QString& fillString(QString& str, const QString& pattern, const QChar& ch)
{
    return fillString(str, QStringMatcher(pattern, Qt::CaseSensitive), ch);
}

QString& fillString(QString& str, const QRegExp& pattern, const QChar& ch)
{
    if (str.length() == 0)
        return str;
    const ushort uch(ch.unicode());
    ushort* const c0((ushort*)str.data());
    int p(0);
    while (p >= 0)
    {
        p = pattern.indexIn(str, p);
        if (p > -1)
        {
            const int pl(pattern.matchedLength());
            ushort* c(c0 + p);
            const ushort* const cpl(c + pl);
            while (c < cpl)
                *c++ = uch;
            p += pl;
        }
    }
    return str;
}

QString& fillString(QString& str, const QRegularExpression& pattern, const QChar& ch)
{
    if (str.length() == 0)
        return str;
    const ushort uch(ch.unicode());
    ushort* const c0((ushort*)str.data());
    QRegularExpressionMatchIterator it(pattern.globalMatch(str));
    while (it.hasNext())
    {
        const QRegularExpressionMatch match(it.next());
        ushort* c(c0 + match.capturedStart());
        const ushort* const cpl(c + match.capturedLength());
        while (c < cpl)
            *c++ = uch;
    }
    return str;
}

static const unsigned short _cyrillic_to_ascii[] =
{
    // Code points 1024 to 1309
    // See http://en.wikipedia.org/wiki/ISO_9
    69, 69, 68, 71, 69, 90, 73, 73, 74, 76, 78, 67, 75, 73, 85, 68, 65,
    66, 86, 71, 68, 69, 90, 90, 73, 74, 75, 76, 77, 78, 79, 80, 82, 83,
    84, 85, 70, 72, 67, 67, 83, 83, 698, 89, 697, 69, 85, 65, 97, 98, 118,
    103, 100, 101, 122, 122, 105, 106, 107, 108, 109, 110, 111, 112, 114, 115, 116, 117,
    102, 104, 99, 99, 115, 115, 698, 121, 697, 101, 117, 97, 101, 101, 100, 103, 101,
    122, 105, 105, 106, 108, 110, 99, 107, 105, 117, 100, 1120, 1121, 69, 101, 1124, 1125,
    1126, 1127, 1128, 1129, 65, 97, 1132, 1133, 1134, 1135, 1136, 1137, 70, 102, 89, 121, 89,
    121, 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159,
    1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 71, 103, 71, 103, 71, 103, 90, 122, 1176,
    1177, 75, 107, 75, 107, 75, 107, 75, 107, 78, 110, 78, 110, 80, 112, 79, 111,
    83, 115, 84, 116, 85, 117, 85, 117, 72, 104, 67, 99, 67, 99, 67, 99, 72,
    104, 67, 99, 67, 99, 1216, 90, 122, 75, 107, 76, 108, 78, 110, 78, 110, 67,
    99, 1229, 1230, 1231, 65, 97, 65, 97, 1236, 1237, 69, 101, 65, 97, 65, 97, 90,
    122, 90, 122, 90, 122, 73, 105, 73, 105, 79, 111, 79, 111, 79, 111, 69, 101,
    85, 117, 85, 117, 85, 117, 67, 99, 1270, 1271, 89, 121, 1274, 1275, 1276, 1277, 1278,
    1279, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 78, 110, 1292, 1293, 84, 116,
    1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 1305, 81, 113, 87, 119
};

static inline QString& _to_ascii_transliterate(QString& str)
{
    // Strip diacritics, undo ligatures, transliterate
    if (str.length() == 0)
        return str;
    ushort* const c0 = (ushort*)str.data();
    ushort* const cn = c0 + str.length();
    ushort* c = c0 - 1;
    bool do_ligatures_198(false);
    bool do_ligatures_223(false);
    bool do_ligatures_230(false);
    bool do_ligatures_338(false);
    bool do_ligatures_339(false);
    while (c < cn)
    {
        ++c;
        if (*c < 128)
            continue;
        if (*c > 1023 && *c < 1310)
        {
            *c = _cyrillic_to_ascii[*c-1024];
            if (*c == 1236)
            {
                *c = 198;
                do_ligatures_198 = true;
            }
            if (*c == 1237)
            {
                *c = 230;
                do_ligatures_230 = true;
            }
            continue;
        }
        QChar qc(*c);
        if (!qc.isLetter())
            continue;
        switch (*c)
        {
        case 216:
            *c = QChar('O').unicode();
            break;
        case 248:
            *c = QChar('o').unicode();
            break;
        case 272:
            *c = QChar('D').unicode();
            break;
        case 273:
            *c = QChar('d').unicode();
            break;
        case 321:
            *c = QChar('L').unicode();
            break;
        case 322:
            *c = QChar('l').unicode();
            break;
        case 198:
            do_ligatures_198 = true;
            break;
        case 223:
            do_ligatures_223 = true;
            break;
        case 230:
            do_ligatures_230 = true;
            break;
        case 338:
            do_ligatures_338 = true;
            break;
        case 339:
            do_ligatures_339 = true;
            break;
        }
        if (qc.decompositionTag() == QChar::NoDecomposition)
            continue;
        qc = qc.decomposition().at(0);
        *c = qc.unicode();
        if (qc.decompositionTag() == QChar::NoDecomposition)
            continue;
        qc = qc.decomposition().at(0);
        *c = qc.unicode();
    }
    if (do_ligatures_198)
        str.replace(QChar(198), "AE", Qt::CaseSensitive);
    if (do_ligatures_223)
        str.replace(QChar(223), "ss", Qt::CaseSensitive);
    if (do_ligatures_230)
        str.replace(QChar(230), "ae", Qt::CaseSensitive);
    if (do_ligatures_338)
        str.replace(QChar(338), "OE", Qt::CaseSensitive);
    if (do_ligatures_339)
        str.replace(QChar(339), "oe", Qt::CaseSensitive);
    return str;
}

static inline QString& _to_ascii_keep_words(QString& str)
{
    // Do:
    // const QRegExp nonAsciiWords("[^A-Za-z0-9\\+\\- ]");
    // str.replace(nonAsciiWords, " ");
    // str = str.simplified();
    if (str.length() == 0)
        return str;
    const ushort dash(QChar('-').unicode());
    const ushort la(QChar('a').unicode());
    const ushort lz(QChar('z').unicode());
    const ushort n0(QChar('0').unicode());
    const ushort n9(QChar('9').unicode());
    const ushort plus(QChar('+').unicode());
    const ushort space(QChar(' ').unicode());
    const ushort ua(QChar('A').unicode());
    const ushort uz(QChar('Z').unicode());

    ushort* const c0 = (ushort*)str.data();
    ushort* const cn = c0 + str.length();
    ushort* c = c0;
    ushort* o = c0;

    while (c < cn)
    {
        const ushort ch = *c;
        if ((ch >= la && ch <= lz) || (ch >= ua && ch <= uz) || (ch >= n0 && ch <= n9) || ch == dash || ch == plus)
            *o++ = ch;
        else if (o > c0 && *(o - 1) != space)
            *o++ = space;
        ++c;
    }
    if (o > c0 && *(o - 1) == space)
        --o;
    str.truncate(int(o - c0));
    return str;
}

QString toAscii(const QString& str, const AsciiConversion type)
{
    QString ascii(str);
    if (type == FromBibTeX)
        cleanEquations(ascii);
    _to_ascii_transliterate(ascii);
    if (type == Collation)
    {
        for (int i = 0; i < ascii.length(); ++i)
            if (ascii.at(i).category() == QChar::Punctuation_Dash)
                ascii[i] = ' ';
        return ascii.toCaseFolded();
    }
    if (type == KeepWords || type == FromBibTeX)
        _to_ascii_keep_words(ascii);
    else if (type == Cleanup)
        ascii.remove(nonAsciiLetter);
    return ascii;
}

QString& stripDiacritics(QString& str)
{
    _to_ascii_transliterate(str);
    return str;
}

QString& c2bToBib(QString& str)
{
    // Escape common Extended Latin Characters
    str.replace(" &", " \\&");
    str.replace(QChar(183), "$\\cdot$");
    str.replace(QChar(192), "{\\`A}");
    str.replace(QChar(193), "{\\'A}");
    str.replace(QChar(194), "{\\^A}");
    str.replace(QChar(195), "{\\~A}");
    str.replace(QChar(196), "{\\\"A}");
    str.replace(QChar(197), "{\\AA{}}");
    str.replace(QChar(198), "{\\AE{}}");
    str.replace(QChar(199), "{\\c{C}}");
    str.replace(QChar(200), "{\\`E}");
    str.replace(QChar(201), "{\\'E}");
    str.replace(QChar(202), "{\\^E}");
    str.replace(QChar(203), "{\\\"E}");
    str.replace(QChar(204), "{\\`I}");
    str.replace(QChar(205), "{\\'I}");
    str.replace(QChar(206), "{\\^I}");
    str.replace(QChar(207), "{\\\"I}");
    str.replace(QChar(209), "{\\~N}");
    str.replace(QChar(210), "{\\`O}");
    str.replace(QChar(211), "{\\'O}");
    str.replace(QChar(212), "{\\^O}");
    str.replace(QChar(213), "{\\~O}");
    str.replace(QChar(214), "{\\\"O}");
    str.replace(QChar(216), "{\\O}");
    str.replace(QChar(217), "{\\`U}");
    str.replace(QChar(218), "{\\'U}");
    str.replace(QChar(219), "{\\^U}");
    str.replace(QChar(220), "{\\\"U}");
    str.replace(QChar(221), "{\\'Y}");
    str.replace(QChar(223), "{\\ss}");
    str.replace(QChar(224), "{\\`a}");
    str.replace(QChar(225), "{\\'a}");
    str.replace(QChar(226), "{\\^a}");
    str.replace(QChar(227), "{\\~a}");
    str.replace(QChar(228), "{\\\"a}");
    str.replace(QChar(229), "{\\aa{}}");
    str.replace(QChar(230), "{\\ae{}}");
    str.replace(QChar(231), "{\\c{c}}");
    str.replace(QChar(232), "{\\`e}");
    str.replace(QChar(233), "{\\'e}");
    str.replace(QChar(234), "{\\^e}");
    str.replace(QChar(235), "{\\\"e}");
    str.replace(QChar(236), "{\\`i}");
    str.replace(QChar(237), "{\\'i}");
    str.replace(QChar(238), "{\\^i}");
    str.replace(QChar(239), "{\\\"i}");
    str.replace(QChar(241), "{\\~n}");
    str.replace(QChar(242), "{\\`o}");
    str.replace(QChar(243), "{\\'o}");
    str.replace(QChar(244), "{\\^o}");
    str.replace(QChar(245), "{\\~o}");
    str.replace(QChar(246), "{\\\"o}");
    str.replace(QChar(248), "{\\o}");
    str.replace(QChar(249), "{\\`u}");
    str.replace(QChar(250), "{\\'u}");
    str.replace(QChar(251), "{\\^u}");
    str.replace(QChar(252), "{\\\"u}");
    str.replace(QChar(253), "{\\'y}");
    str.replace(QChar(255), "{\\\"y}");
    str.replace(QChar(263), "{\\'c}");
    str.replace(QChar(268), "{\\v{C}}");
    str.replace(QChar(269), "{\\v{c}}");
    str.replace(QChar(272), "{\\DJ}");
    str.replace(QChar(273), "{\\dj}");
    str.replace(QChar(321), "{\\L}");
    str.replace(QChar(322), "{\\l}");
    str.replace(QChar(323), "{\\'N}");
    str.replace(QChar(324), "{\\'n}");
    str.replace(QChar(338), "{\\OE}");
    str.replace(QChar(339), "{\\oe}");
    str.replace(QChar(352), "{\\v{S}}");
    str.replace(QChar(353), "{\\v{s}}");
    str.replace(QChar(376), "{\\\"Y");
    str.replace(QChar(381), "{\\v{Z}}");
    str.replace(QChar(382), "{\\v{z}}");
    // Escape common Greek and math
    str.replace(QChar(913), "$\\Alpha$"); // Some uppercases might require engrec package
    str.replace(QChar(914), "$\\Beta$");
    str.replace(QChar(915), "$\\Gamma$");
    str.replace(QChar(916), "$\\Delta$");
    str.replace(QChar(917), "$\\Epsilon$");
    str.replace(QChar(918), "$\\Zeta$");
    str.replace(QChar(919), "$\\Eta$");
    str.replace(QChar(920), "$\\Theta$");
    str.replace(QChar(921), "$\\Iota$");
    str.replace(QChar(922), "$\\Kappa$");
    str.replace(QChar(923), "$\\Lambda$");
    str.replace(QChar(924), "$\\Mu$");
    str.replace(QChar(925), "$\\Nu$");
    str.replace(QChar(926), "$\\Xi$");
    str.replace(QChar(927), "$\\Omicron$");
    str.replace(QChar(928), "$\\Pi$");
    str.replace(QChar(929), "$\\Rho$");
    str.replace(QChar(931), "$\\Sigma$");
    str.replace(QChar(932), "$\\Tau$");
    str.replace(QChar(933), "$\\Upsilon$");
    str.replace(QChar(934), "$\\Phi$");
    str.replace(QChar(935), "$\\Chi$");
    str.replace(QChar(936), "$\\Psi$");
    str.replace(QChar(937), "$\\Omega$");
    str.replace(QChar(945), "$\\alpha$");
    str.replace(QChar(946), "$\\beta$");
    str.replace(QChar(947), "$\\gamma$");
    str.replace(QChar(948), "$\\delta$");
    str.replace(QChar(949), "$\\varepsilon$");
    str.replace(QChar(950), "$\\zeta$");
    str.replace(QChar(951), "$\\eta$");
    str.replace(QChar(952), "$\\theta$");
    str.replace(QChar(953), "$\\iota$");
    str.replace(QChar(954), "$\\kappa$");
    str.replace(QChar(955), "$\\lambda$");
    str.replace(QChar(956), "$\\mu$");
    str.replace(QChar(957), "$\\nu$");
    str.replace(QChar(958), "$\\xi$");
    str.replace(QChar(959), "$\\omicron$");
    str.replace(QChar(960), "$\\pi$");
    str.replace(QChar(961), "$\\rho$");
    str.replace(QChar(962), "$\\varsigma$");
    str.replace(QChar(963), "$\\sigma$");
    str.replace(QChar(964), "$\\tau$");
    str.replace(QChar(965), "$\\upsilon$");
    str.replace(QChar(966), "$\\phi$");
    str.replace(QChar(967), "$\\chi$");
    str.replace(QChar(968), "$\\psi$");
    str.replace(QChar(969), "$\\omega$");
    str.replace(QChar(977), "$\\vartheta$");
    str.replace(QChar(981), "$\\varphi$");
    str.replace(QChar(982), "$\\varpi$");
    str.replace(QChar(989), "$\\digamma$");
    str.replace(QChar(1008), "$\\varkappa$");
    str.replace(QChar(1009), "$\\varrho$");
    str.replace(QChar(1013), "$\\epsilon$");
    str.replace(QChar(8706), "$\\partial$");
    str.replace(QChar(8722), '-');
    str.replace(QChar(8734), "$\\infty$");
    return str;
}

QString& bibToC2b(QString& str)
{
    // Escape TeX special characters to Unicode
    str.replace("\\&", "&");
    // From \LaTeX{} syntax to {\LaTeX} for the implemented subset
    str.replace(QRegExp("\\\\(.{1,2})\\{(.{0,2})\\}"), "{\\\\1\\2}");
    if (hasLatexDiacritic.indexIn(str) >= 0)
    {
        str.replace("{\\`A}", QChar(192));
        str.replace("{\\'A}", QChar(193));
        str.replace("{\\^A}", QChar(194));
        str.replace("{\\~A}", QChar(195));
        str.replace("{\\\"A}", QChar(196));
        str.replace("{{\\AA}}", QChar(197));
        str.replace("{{\\AE}}", QChar(198));
        str.replace("{\\AA}", QChar(197));    // {\\AA{}}
        str.replace("{\\AE}", QChar(198));    // {\\AE{}}
        str.replace("{{\\cC}}", QChar(199));  // {\\c{C}}
        str.replace("{\\cC}", QChar(199));    // {\\c{C}}
        str.replace("{\\`E}", QChar(200));
        str.replace("{\\'E}", QChar(201));
        str.replace("{\\^E}", QChar(202));
        str.replace("{\\\"E}", QChar(203));
        str.replace("{\\`I}", QChar(204));
        str.replace("{\\'I}", QChar(205));
        str.replace("{\\^I}", QChar(206));
        str.replace("{\\\"I}", QChar(207));
        str.replace("{\\~N}", QChar(209));
        str.replace("{\\`O}", QChar(210));
        str.replace("{\\'O}", QChar(211));
        str.replace("{\\^O}", QChar(212));
        str.replace("{\\~O}", QChar(213));
        str.replace("{\\\"O}", QChar(214));
        str.replace("{\\O}", QChar(216));
        str.replace("{\\`U}", QChar(217));
        str.replace("{\\'U}", QChar(218));
        str.replace("{\\^U}", QChar(219));
        str.replace("{\\\"U}", QChar(220));
        str.replace("{\\'Y}", QChar(221));
        str.replace("{\\ss}", QChar(223));
        str.replace("{\\`a}", QChar(224));
        str.replace("{\\'a}", QChar(225));
        str.replace("{\\^a}", QChar(226));
        str.replace("{\\~a}", QChar(227));
        str.replace("{\\\"a}", QChar(228));
        str.replace("{{\\aa}}", QChar(229));
        str.replace("{{\\ae}}", QChar(230));
        str.replace("{\\aa}", QChar(229));    // {\\aa{}}
        str.replace("{\\ae}", QChar(230));    // {\\ae{}}
        str.replace("{{\\cc}}", QChar(231));  // {\\c{c}}
        str.replace("{\\cc}", QChar(231));    // {\\c{c}}
        str.replace("{\\`e}", QChar(232));
        str.replace("{\\'e}", QChar(233));
        str.replace("{\\^e}", QChar(234));
        str.replace("{\\\"e}", QChar(235));
        str.replace("{\\`i}", QChar(236));
        str.replace("{\\'i}", QChar(237));
        str.replace("{\\^i}", QChar(238));
        str.replace("{\\\"i}", QChar(239));
        str.replace("{\\`\\i}", QChar(236));
        str.replace("{\\'\\i}", QChar(237));
        str.replace("{\\^\\i}", QChar(238));
        str.replace("{\\\"\\i}", QChar(239));
        str.replace("{\\~n}", QChar(241));
        str.replace("{\\`o}", QChar(242));
        str.replace("{\\'o}", QChar(243));
        str.replace("{\\^o}", QChar(244));
        str.replace("{\\~o}", QChar(245));
        str.replace("{\\\"o}", QChar(246));
        str.replace("{\\o}", QChar(248));
        str.replace("{\\`u}", QChar(249));
        str.replace("{\\'u}", QChar(250));
        str.replace("{\\^u}", QChar(251));
        str.replace("{\\\"u}", QChar(252));
        str.replace("{\\'y}", QChar(253));
        str.replace("{\\\"y}", QChar(255));
        str.replace("{\\'c}", QChar(263));
        str.replace("{{\\vC}}", QChar(268));  // {\\v{C}}
        str.replace("{\\vC}", QChar(268));
        str.replace("{{\\vc}}", QChar(269));  // {\\v{c}
        str.replace("{\\vc}", QChar(269));
        str.replace("{\\DJ}", QChar(272));
        str.replace("{\\dj}", QChar(273));
        str.replace("{\\L}", QChar(321));
        str.replace("{\\l}", QChar(322));
        str.replace("{\\'N}", QChar(323));
        str.replace("{\\'n}", QChar(324));
        str.replace("{\\OE}", QChar(338));
        str.replace("{\\oe}", QChar(339));
        str.replace("{{\\vS}}", QChar(352));  // {\\v{S}}
        str.replace("{\\vS}", QChar(352));
        str.replace("{{\\vs}}", QChar(353));  // {\\v{s}}
        str.replace("{\\vs}", QChar(353));
        str.replace("{\\\"Y}", QChar(376));
        str.replace("{{\\vZ}}", QChar(381));  // {\\v{Z}}
        str.replace("{\\vZ}", QChar(381));
        str.replace("{{\\vz}}", QChar(382));  // {\\v{Z}}
        str.replace("{\\vz}", QChar(382));
    }
    if (hasLatexSymbol.indexIn(str) >= 0)
    {
        str.replace("$\\cdot$", QChar(183));
        str.replace("$\\Alpha$", QChar(913));
        str.replace("$\\Beta$", QChar(914));
        str.replace("$\\Gamma$", QChar(915));
        str.replace("$\\Delta$", QChar(916));
        str.replace("$\\Epsilon$", QChar(917));
        str.replace("$\\Zeta$", QChar(918));
        str.replace("$\\Eta$", QChar(919));
        str.replace("$\\Theta$", QChar(920));
        str.replace("$\\Iota$", QChar(921));
        str.replace("$\\Kappa$", QChar(922));
        str.replace("$\\Lambda$", QChar(923));
        str.replace("$\\Mu$", QChar(924));
        str.replace("$\\Nu$", QChar(925));
        str.replace("$\\Xi$", QChar(926));
        str.replace("$\\Omicron$", QChar(927));
        str.replace("$\\Pi$", QChar(928));
        str.replace("$\\Rho$", QChar(929));
        str.replace("$\\Sigma$", QChar(931));
        str.replace("$\\Tau$", QChar(932));
        str.replace("$\\Upsilon$", QChar(933));
        str.replace("$\\Phi$", QChar(934));
        str.replace("$\\Chi$", QChar(935));
        str.replace("$\\Psi$", QChar(936));
        str.replace("$\\Omega$", QChar(937));
        str.replace("$\\alpha$", QChar(945));
        str.replace("$\\beta$", QChar(946));
        str.replace("$\\gamma$", QChar(947));
        str.replace("$\\delta$", QChar(948));
        str.replace("$\\varepsilon$", QChar(949));
        str.replace("$\\zeta$", QChar(950));
        str.replace("$\\eta$", QChar(951));
        str.replace("$\\theta$", QChar(952));
        str.replace("$\\iota$", QChar(953));
        str.replace("$\\kappa$", QChar(954));
        str.replace("$\\lambda$", QChar(955));
        str.replace("$\\mu$", QChar(956));
        str.replace("$\\nu$", QChar(957));
        str.replace("$\\xi$", QChar(958));
        str.replace("$\\omicron$", QChar(959));
        str.replace("$\\pi$", QChar(960));
        str.replace("$\\rho$", QChar(961));
        str.replace("$\\sigmaf$", QChar(962));
        str.replace("$\\varsigma$", QChar(962));   // Equal to \sigmaf
        str.replace("$\\sigma$", QChar(963));
        str.replace("$\\tau$", QChar(964));
        str.replace("$\\upsilon$", QChar(965));
        str.replace("$\\phi$", QChar(966));
        str.replace("$\\chi$", QChar(967));
        str.replace("$\\psi$", QChar(968));
        str.replace("$\\omega$", QChar(969));
        str.replace("$\\vartheta$", QChar(977));
        str.replace("$\\varphi$", QChar(981));
        str.replace("$\\varpi$", QChar(982));
        str.replace("$\\digamma$", QChar(989));
        str.replace("$\\varkappa$", QChar(1008));
        str.replace("$\\varrho$", QChar(1009));
        str.replace("$\\epsilon$", QChar(1013));
        str.replace("$\\partial$", QChar(8706));
        str.replace("$\\infty$", QChar(8734));
    }
    return str;
}

const QString fromUtf8(const QByteArray& ba)
{
// Based on Qt's QString::fromUtf8 function. Input ba must be an UTF-8
// encoded array produced by QString::toUtf8. Encoding correctness is
// assumed and checking omitted. It performs a 20% faster compared to
// Qt 4.5, and expected more compared to Qt 4.6.

    const char* b = ba.constData();
    const char* const bn = b + ba.length();
    QString output;
    output.resize(ba.length());
    ushort* o = (ushort*)output.unicode();
    uint c;
    while (b < bn)
    {
        c = uint(*b);
        if (c & 0x80)
        {
            if ((c & 0xe0) == 0xc0)
            {
                c &= 0x1f;
                c = (uint)((c << 6) | (*++b & 0x3f));
            }
            else if ((c & 0xf0) == 0xe0)
            {
                c &= 0x0f;
                c = (uint)((c << 6) | (b[1] & 0x3f));
                c = (uint)((c << 6) | (b[2] & 0x3f));
                b += 2;
            }
            else if ((c & 0xf8) == 0xf0)
            {
                c &= 0x07;
                c = (uint)((c << 6) | (b[1] & 0x3f));
                c = (uint)((c << 6) | (b[2] & 0x3f));
                c = (uint)((c << 6) | (b[3] & 0x3f));
                *o++ = QChar::highSurrogate(c);
                c = QChar::lowSurrogate(c);
                b += 3;
            }
            else
                c = QChar::ReplacementCharacter;
        }
        *o++ = ushort(c);
        ++b;
    }
    output.truncate(int(o - (ushort*)output.unicode()));
//    QString check = QString::fromUtf8(ba);
//    qDebug() << (check == output);
    return output;
}

#ifdef C2B_USE_LZO
const QByteArray lzo::compress(const QByteArray& data)
{
    const lzo_uint src_s(data.size());
    if (src_s == 0)
        return QByteArray();
    if (lzo_init() != LZO_E_OK)
    {
        warn("compress: lzo: internal error - initialization failed");
        return QByteArray();
    }
    const lzo_bytep src_p = reinterpret_cast<const lzo_bytep>(data.constData());

    QByteArray compressed;
    const lzo_uint dest_s = HEADER_LENGTH + (src_s + src_s / 16 + 64 + 3);
    compressed.resize((int)dest_s);
    lzo_bytep compressed_p = reinterpret_cast<lzo_bytep>(compressed.data());
    lzo_bytep dest_p = compressed_p + HEADER_LENGTH;
    lzo_uint enc_dest_s = dest_s - HEADER_LENGTH;

    QByteArray wrkmem;
    wrkmem.resize(LZO1X_999_MEM_COMPRESS);
    lzo_bytep wrkmem_p = reinterpret_cast<lzo_bytep>(wrkmem.data());

    if (lzo1x_999_compress(src_p, src_s, dest_p, &enc_dest_s, wrkmem_p) == LZO_E_OK)
    {
        compressed.resize((int)enc_dest_s + HEADER_LENGTH);
        QByteArray header(QByteArray::number((qulonglong)src_s));
        for (int i = 0; i < qMin(header.length(), HEADER_LENGTH); ++i)
            compressed[i] = header[i];
        for (int i = qMin(header.length(), HEADER_LENGTH); i < HEADER_LENGTH; ++i)
            compressed[i] = ' ';
        return compressed;
    }
    else
    {
        warn("compress: lzo: internal error - compression failed");
        return QByteArray();
    }
}

const QByteArray lzo::uncompress(const QByteArray& data)
{
    const lzo_uint data_s(data.size());
    if (data_s <= (unsigned int)HEADER_LENGTH)
        return QByteArray();
    if (lzo_init() != LZO_E_OK)
    {
        warn("uncompress: lzo: internal error - initialization failed");
        return QByteArray();
    }
    const lzo_bytep data_p = reinterpret_cast<const lzo_bytep>(data.constData());
    const lzo_bytep src_p = data_p + HEADER_LENGTH;
    const lzo_uint src_s = data_s - HEADER_LENGTH;

    QByteArray uncompressed;
    const ulong expected_dest_s = data.left(HEADER_LENGTH).trimmed().toULong();
    lzo_uint dest_s = qMax(expected_dest_s, lzo_uint(1));
    uncompressed.resize((int)dest_s);
    lzo_bytep dest_p = reinterpret_cast<lzo_bytep>(uncompressed.data());

    if (lzo1x_decompress(src_p, src_s, dest_p, &dest_s, NULL) == LZO_E_OK && (ulong)dest_s == expected_dest_s)
        return uncompressed;
    else
    {
        warn("uncompress: lzo: internal error - uncompression failed");
        return QByteArray();
    }
}
#endif

} // namespace c2bUtils
