/* $Id: e2_utf8.c 881 2008-05-12 22:54:30Z tpgww $

Copyright (C) 2004-2008 tooar <tooar@gmx.net>
Portion _maybe_ copyright (C) 2004-2005 Tarot Osuji

This file is part of emelFM2.
emelFM2 is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3, or (at your option)
any later version.

emelFM2 is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with emelFM2; see the file GPL. If not, contact the Free Software
Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

/**
@file src/utils/e2_utf8.c
@brief utf8 string utilities

This file contains utilitiy functions for utf8 string handling.
*/

#include "emelfm2.h"
#include <string.h>
#include "e2_utf8.h"
#include "e2_cl_option.h"


/**
@brief get position in string @a str

This function acts like g_utf8_offset_to_pointer() except that if it finds a
decomposable character it consumes the decomposition length from the given
offset.  So it's useful when the offset was calculated for the normalized
version of str, but we need a pointer to str itself

@param str string to scan
@param offset character offset within @a str

@return the resulting pointer
*/
static const gchar *_e2_utf8_pointer_from_offset_skipping_decomp
	(const gchar *str, gint offset)
{
	gsize decomp_len;
	gunichar *decomp;
	const gchar *p;

	p = str;
	while (offset > 0)
	{
		decomp = g_unicode_canonical_decomposition (g_utf8_get_char (p), &decomp_len);
		g_free (decomp);
		p = g_utf8_next_char (p);
		offset -= decomp_len;
	}
	return p;
}
/**
@brief find first occurrence of @a needle in @a haystack, regardless of case of @a haystack

@param haystack string to scan
@param needle string to find

@return pointer to @a needle in @a haystack, or NULL if not found
*/
const gchar *e2_utf8_strcasestr (const gchar *haystack, const gchar *needle)
{
	g_return_val_if_fail (haystack != NULL, NULL);
	g_return_val_if_fail (needle != NULL, NULL);

	gsize haystack_len, needle_len;

	needle_len = g_utf8_strlen (needle, -1);
	if (needle_len == 0)
		return haystack;

	gchar *p, *caseless_haystack;
	const gchar *retval = NULL;

	p = g_utf8_casefold (haystack, -1);
	caseless_haystack = g_utf8_normalize (p, -1, G_NORMALIZE_ALL);
	g_free (p);

	haystack_len = g_utf8_strlen (caseless_haystack, -1);
	if (haystack_len < needle_len)
		goto cleanup;

	p = caseless_haystack;
	needle_len = strlen (needle);
	gint i = 0;

	while (*p)
	{
		if ((strncmp (p, needle, needle_len) == 0))
		{
			retval = _e2_utf8_pointer_from_offset_skipping_decomp (haystack, i);
			goto cleanup;
		}

		p = g_utf8_next_char (p);
		i++;
	}

cleanup:
	g_free (caseless_haystack);

	return retval;
}
/**
@brief find last occurrence of @a needle in @a haystack, regardless of case of @a haystack

@param haystack string to scan
@param needle string to find

@return pointer to @a needle in @a haystack, or NULL if not found
*/
const gchar *e2_utf8_strrcasestr (const gchar *haystack, const gchar *needle)
{
	g_return_val_if_fail (haystack != NULL, NULL);
	g_return_val_if_fail (needle != NULL, NULL);

	gsize haystack_len, needle_len;

	needle_len = g_utf8_strlen (needle, -1);
	if (needle_len == 0)
		return haystack;

	gchar *p, *caseless_haystack;
	const gchar *retval = NULL;

	p = g_utf8_casefold (haystack, -1);
	caseless_haystack = g_utf8_normalize (p, -1, G_NORMALIZE_ALL);
	g_free (p);

	haystack_len = g_utf8_strlen (caseless_haystack, -1);
	if (haystack_len < needle_len)
		goto cleanup;

	gint i = haystack_len - needle_len;
	p = g_utf8_offset_to_pointer (caseless_haystack, i);
	needle_len = strlen (needle);

	while (p >= caseless_haystack)
	{
		if (strncmp (p, needle, needle_len) == 0)
		{
			retval = _e2_utf8_pointer_from_offset_skipping_decomp (haystack, i);
			goto cleanup;
		}

		p = g_utf8_prev_char (p);
		i--;
	}

cleanup:
	g_free (caseless_haystack);

	return retval;
}
/**
@brief compare utf8 strings @a s1 and @a s2, without reference to case

@param s1 string to compare
@param s2 other string to compare
@param n1 length of @a s1 (in bytes) or -1 if @a s1 is NULL-terminated
@param n2 length of @a s2 (in bytes) or -1 if @a s2 is NULL-terminated

@return TRUE if @a s1 matches @a s2, apart from any case-difference
*/
gboolean e2_utf8_caseless_match (const gchar *s1, const gchar *s2,
		gssize n1, gssize n2)
{
	gchar *casefold;
	gchar *normalized_s1;
	gchar *normalized_s2;
//	gint len_s1;
//	gint len_s2;
	gboolean ret;

	g_return_val_if_fail (s1 != NULL, FALSE);
	g_return_val_if_fail (s2 != NULL, FALSE);
	g_return_val_if_fail (n1 >= -1, FALSE);
	g_return_val_if_fail (n2 >= -1, FALSE);

	if (n1 == 0 && n2 == 0)
		return TRUE;
	else if (n1 == 0 || n2 == 0)
		return FALSE;

	casefold = g_utf8_casefold (s1, n1);
	normalized_s1 = g_utf8_normalize (casefold, -1, G_NORMALIZE_ALL);
	g_free (casefold);
//	len_s1 = strlen (normalized_s1);

	casefold = g_utf8_casefold (s2, n2);
	normalized_s2 = g_utf8_normalize (casefold, -1, G_NORMALIZE_ALL);
	g_free (casefold);
//	len_s2 = strlen (normalized_s2);

//	ret = (len_s1 == len_s2) ?
//		(strcmp (normalized_s1, normalized_s2) == 0) : FALSE;
	ret = (strcmp (normalized_s1, normalized_s2) == 0);

	g_free (normalized_s1);
	g_free (normalized_s2);

	return ret;
}
/**
@brief duplicate utf8 string, truncated after @a num characters if the string is longer than that
@param str the string to be duplicated
@param num maximum no. of characters in @a str to be processed
@return the duplicated string
*/
gchar *e2_utf8_ndup (const gchar *str, glong num)
{
	glong size = g_utf8_strlen (str, -1);
	if (num > size)
		num = size;
	gchar *end = g_utf8_offset_to_pointer (str, num);
	glong byte_size = end - str + 1;
	gchar *utf8 = g_malloc (byte_size);
	return g_utf8_strncpy (utf8, str, num);
}
/**
@brief escape all instances of character @a c in utf-8 string @a str
@param str the string to be processed
@param c the character to be escaped if found in @a str
@return newly-allocated string with any occurrence of @a c escaped with a '\'
*/
gchar *e2_utf8_escape (const gchar *str, gunichar c)
{
	glong clen = g_utf8_strlen (str, -1);
	glong i;
	gint count = 0;
	const gchar *tmp = str;
	for (i = 0; i < clen; i++)
	{
		if (g_utf8_get_char (tmp) == c)
			count++;
		tmp = g_utf8_next_char (tmp);
	}
	if (count > 0)
	{
		gint len = strlen (str);
		gint size = (len + count + 1) * sizeof (gchar);
		gchar *ret = g_malloc (size);
		count = 0;
		for (i = 0; i < clen; i++)
		{
			gchar *cur = g_utf8_offset_to_pointer (str, i);
			if (g_utf8_get_char (cur) == c)
				g_utf8_strncpy (g_utf8_offset_to_pointer (ret, count++), "\\", 1);
			g_utf8_strncpy (g_utf8_offset_to_pointer (ret, count++), cur, 1);
		}
		ret[size - 1] = '\0';
		return ret;
	} else
		return g_strdup (str);
}
/**
@brief unescape all instances of character @a c in utf-8 string @a str
@param str the string to be processed
@param c the character to be unescaped if found in @a str
@return newly-allocated string without any occurrence of '\'in front of @a c, or NULL after error
*/
gchar *e2_utf8_unescape (const gchar *str, gunichar c)
{
	glong count = 0;
	gint size = (strlen (str) + 1) * sizeof (gchar);
	gchar *ret = g_try_malloc (size);	//FIXME nandle error
	CHECKALLOCATEDWARN (ret, return NULL);
	*ret = '\0';	//in case str is empty
	const gchar *tmp = str;
	while (*tmp != '\0')
	{
		if ((g_utf8_get_char (tmp)) == '\\')
		{
			gchar *next = g_utf8_next_char (tmp);
			if ((next != NULL) && g_utf8_get_char (next) == c)
				tmp = next;
		}
		g_utf8_strncpy (g_utf8_offset_to_pointer (ret, count++), tmp, 1);
		tmp = g_utf8_next_char (tmp);
	}
/* not needed - each g_utf8_strncpy() appends a '\0'
	if (count < g_utf8_strlen (str, -1))
		g_utf8_strncpy (g_utf8_offset_to_pointer (ret, count), "\0" , 1);
	else
		ret[size - 1] = '\0';
*/
	printd (DEBUG, "ret: %s", ret);
	return ret;
}

//inline
gchar *e2_utf8_not_converted (const gchar *d)
{
	return (gchar *) d;
}
void e2_utf8_not_freed (gpointer dummy)
{}

#ifndef E2_FILES_UTF8ONLY
//with run-time checking, a pointer may be set to this func, so no inline
//inline
gchar *e2_utf8_filename_from_locale (const gchar *d)
{
	GError *error = NULL;
	gchar *ret = g_filename_to_utf8 (d, -1, NULL, NULL, &error);
	if (error == NULL)
		return ret;
	printd (WARN, "locale filename to UTF8 conversion failed: %s",
		error->message);
	g_error_free (error);
//	return e2_utf8_filename_from_locale_backup (d);
//}

//gchar *e2_utf8_filename_from_locale_backup (const gchar *d)
//{
	printd (WARN, "falling back to '%s'", e2_cl_options.fallback_encoding);
//	GError *
	error = NULL;
//	gchar *
	ret = g_convert_with_fallback (d, -1, "UTF-8", e2_cl_options.fallback_encoding,
			"?", NULL, NULL, &error);
	if (error != NULL)
	{
		printd (WARN, "fallback locale filename to UTF8 conversion failed: %s",
			error->message);
		g_error_free (error);
		ret = g_strdup(_("Unknown"));
	}
	return ret;
}

//with run-time checking, a pointer may be set to this func, so no inline
//inline
gchar *e2_utf8_filename_to_locale (const gchar *d)
{
	GError *error = NULL;
	gchar *ret = g_filename_from_utf8 (d, -1, NULL, NULL, &error);
	if (error == NULL)
		return ret;
	printd (WARN, "UTF8 filename to locale conversion failed: %s", error->message);
	g_error_free (error);
//	return e2_utf8_filename_to_locale_backup (d);
//}

//gchar *e2_utf8_filename_to_locale_backup (const gchar *d)
//{
	printd (WARN, "falling back to '%s'", e2_cl_options.fallback_encoding);
//	GError *
	error = NULL;
//	gchar *
	ret = g_convert_with_fallback (d, -1, e2_cl_options.fallback_encoding,
		"UTF-8", "?", NULL, NULL, &error);
	if (error != NULL)
	{
		printd (WARN, "fallback UTF8 filename to locale conversion failed: %s", error->message);
		g_error_free (error);
		ret = g_strdup(_("Unknown"));
	}
	return ret;
}
#endif // ndef E2_FILES_UTF8ONLY

gchar *e2_utf8_from_locale (const gchar *d)
{
	if (g_utf8_validate (d, -1, NULL))
		return g_strdup (d);
	gchar *ret = g_locale_to_utf8 (d, -1, NULL, NULL, NULL);
	if (ret != NULL)
		return ret;
	printd (WARN, "initial conversion of localised string %s to UTF8 failed", d);
	return e2_utf8_from_locale_fallback (d);
}

gchar *e2_utf8_from_locale_fallback (const gchar *d)
{
	printd (WARN, "falling back to '%s'", e2_cl_options.fallback_encoding);
	GError *error = NULL;
	gchar *ret = g_convert_with_fallback (d, -1, "UTF-8",
		e2_cl_options.fallback_encoding, "?", NULL, NULL, &error);
	if (error != NULL)
	{
		printd (WARN, "fallback locale string to UTF8 conversion failed: %s",
			error->message);
		g_error_free (error);
		ret = g_strdup(_("Unknown"));
	}
	return ret;
}

//#if 0
//UNUSED
//inline
gchar *e2_utf8_to_locale (const gchar *d)
{
	GError *error = NULL;
	gchar *ret = g_locale_from_utf8 (d, -1, NULL, NULL, &error);
	if (error == NULL)
		return ret;
	printd (WARN, "UTF8 string to locale conversion failed: %s", error->message);
	g_error_free (error);
	return e2_utf8_to_locale_fallback (d);
}

gchar *e2_utf8_to_locale_fallback (const gchar *d)
{
	printd (WARN, "falling back to '%s'", e2_cl_options.fallback_encoding);
	GError *error = NULL;
	gchar *ret = g_convert_with_fallback (d, -1, e2_cl_options.fallback_encoding,
		"UTF-8",	"?", NULL, NULL, &error);
	if (error != NULL)
	{
		printd (WARN, "fallback UTF8 string to locale conversion failed: %s", error->message);
		g_error_free (error);
		error = NULL;
		ret = g_strdup(_("Unknown"));
	}
	return ret;
}
//#endif //0

/*************************************
encoding-detection code adapted from leafpad 0.8.7 by Tarot Osuji
*/

/* A complete list of charsets officially supported in glibc 2.2.3 is:
ISO-8859-{1,2,3,5,6,7,8,9,13,15}, CP1251, UTF-8, EUC-{KR,JP,TW}, KOI8-{R,U},
GB2312, GB18030, GBK, BIG5, BIG5-HKSCS and TIS-620
(Romanian may be switching to ISO-8859-16)
*/

enum
{
	IANA = 0,
	OPENI18N,
	CODEPAGE,
	ENCODING_TYPECOUNT
};

enum
{
	LATIN1 = 0,
	LATIN2,
	LATIN3,
//	LATIN4,
	LATIN5,
	LATIN6,
	LATIN7,
	ARABIC,
	CYRILIC,
	CYRILIC_TJ,
	CYRILIC_UA,
	GREEK,
	HEBREW,
	CHINESE_CN,
	CHINESE_HK,
	CHINESE_TW,
	JAPANESE,
	KOREAN,
	VIETNAMESE,
	THAI,
	GEORGIAN,
	ENCODE_COUNT
};

//up to this many locales per encoding
#define MAX_LOCALES 10

const gchar *locale_table[ENCODE_COUNT][MAX_LOCALES] =
{
/* LATIN1 */	{NULL},
/* LATIN2 */	{"cs", "de", "hr", "hu", "ro", "pl", "sk", "sq", "sr", "uz"},	//this effectively sets MAX_LOCALES
/* LATIN3 */	{"eo", "gl", "mt", NULL},
// / * LATIN4 * /	{"et", "mi", NULL}, Estonian, Latvian, and Lithuanian essentially obsolete;
/* LATIN5 */	{"az", "tr", NULL},
/* LATIN6 */	{"et", "ga", "is", "kl", "mi", NULL},
/* LATIN7 */	{"da", "fi", "lt", "lv", "no", "pl", "se", "sl", "sv", NULL},
/* ARABIC */	{"ar", "fa", "ur", NULL},
/* CYRILIC */	{"be", "bg", "ky", "mk", "mn", "ru", "tt", NULL},
/* CYRILIC_TJ */{"tg", NULL},
/* CYRILIC_UA */{"uk", NULL},
/* GREEK */		{"el", NULL},	//modern
/* HEBREW */	{"he", "iw", "yi", NULL}, //modern
/* CHINESE_CN */{"zh_CN", NULL},
/* CHINESE_HK */{"zh_HK", NULL},
/* CHINESE_TW */{"zh_TW", NULL},
/* JAPANESE */	{"ja", NULL},
/* KOREAN */	{"ko", NULL},
/* VIETNAMESE */{"vi", NULL},
/* THAI */		{"th", NULL},
/* GEORGIAN */	{"ka", NULL}
};

/*******************************************
/ * LATIN8 * /	{ "ISO-8859-14",  , },	//a.k.a celtic adds missing gaelic and welsh (cy) to latin1 mainly Old Gaelic, also ok for Albanian, Breton, Catalan, Danish, Dutch, English, Finnish, French, Gaelic, German, Irish, Italian, Norwegian, Portuguese, Swedish and Welsh.
/ * LATIN9 * /	{ "ISO-8859-15",  , },	//latin1 with euro and some missing things
/ * LATIN10 * /	{ "ISO-8859-16",  , },	//for Albanian, Croatian, Hungarian, Polish, Romanian and Slovenian, but also French, German, Italian and Irish Gaelic (new orthography). It differs from the other
/ * ARABIC * /	{ "ISO-8859-6-E",  , },	//explicit bi-directional vartiant
/ * ARABIC * /	{ "ISO-8859-6-I",  , }, //implicit bi-directional variant
/ * HEBREW * /	{ "ISO-8859-8-E",  , }, //explicit bi-directional variant
/ * HEBREW * /	{ "ISO-8859-8-I",  , }, //implicit bi-directional variant
******************************************/

const gchar *encoding_table [ENCODE_COUNT][ENCODING_TYPECOUNT] =
{
				/*  IANA          OpenI18N       Codepage */
/* LATIN1 */	{ "ISO-8859-1",  "ISO-8859-15", "CP1252" },
/* LATIN2 */	{ "ISO-8859-2",  "ISO-8859-16", "CP1250" },
/* LATIN3 */	{ "ISO-8859-3",  NULL,          NULL },
// /* LATIN4 */	{ "ISO-8859-4",  "ISO-8859-13", "CP1257" },
/* LATIN5 */	{ "ISO-8859-9",  NULL,          "CP1254" },
/* LATIN6 */	{ "ISO-8859-10", NULL,          "CP1257" },
/* LATIN7 */	{ "ISO-8859-13", NULL,          "CP1257" },
/* ARABIC */	{ "ISO-8859-6",  NULL,          "CP1256" },
//ISO-8859-5 for e.g. bulgarian, belarusian, russian and macedonian, not ukrainian, "never really caught on"
/* CYRILIC */	{ "ISO-8859-5",  "KOI8-R",      "CP1251" },	//russian, macedonian
/* CYRILIC_TJ */{ "ISO-8859-5",  "KOI8-T",      "CP1251" }, //tajic
/* CYRILIC_UA */{ "ISO-8859-5",  "KOI8-U",      "CP1251" },	//ukrainian
/* GREEK */		{ "ISO-8859-7",  NULL,          "CP1253" },	//modern
/* HEBREW */	{ "ISO-8859-8",  NULL,          "CP1255" }, //modern
/* CHINESE_CN */{ "GB2312",      "GB18030",     "CP936" },
/* CHINESE_HK */{ "BIG5",        "BIG5-HKSCS",  "CP950" },
/* CHINESE_TW */{ "BIG5",        "EUC-TW",      "CP950" },
/* JAPANESE */	{ "ISO-2022-JP", "EUC-JP",      "CP932" },
/* KOREAN */	{ "ISO-2022-KR", "EUC-KR",      "CP949" },
/* VIETNAMESE */{ NULL,          "VISCII",      "CP1258" },
/* THAI */		{ "ISO-8859-11", "TIS-620",     "CP874" }, //ISO-8859-11 is unofficial
/* GEORGIAN */	{ NULL,          "GEORGIAN-PS", NULL }
};


static guint get_locale_index (void)
{
	static guint code = ENCODE_COUNT;

	if (code == ENCODE_COUNT)	//we only look this up once
	{
		const gchar *env = g_getenv ("LC_ALL");
		if (env == NULL)
			env = g_getenv ("LANG");
		if (env != NULL && strlen (env) >= 2)
		{
			guint i, j = 1;
			while (code == ENCODE_COUNT && j < ENCODE_COUNT)
			{
				for (i = 0; i < MAX_LOCALES; i++)
				{
					if (locale_table[j][i] == NULL)
						break;
					if (strncmp (env, locale_table[j][i], strlen(locale_table[j][i])) == 0)
					{
						code = j;
						break;
					}
				}
				j++;
			}
		}
		if (code == ENCODE_COUNT)
			code = LATIN1;	//default if we can't find anything supported
	}

	return code;
}

static const gchar *detect_charset_cyrillic (const guchar *text)
{
	guchar c;
	gboolean noniso = FALSE;
	guint32 xc = 0, xd = 0, xef = 0;
	const gchar *charset;

	while ((c = *text++) != '\0')
	{
		if (c >= 0x80 && c <= 0x9F)
			noniso = TRUE;
		else if (c >= 0xC0 && c <= 0xCF)
			xc++;
		else if (c >= 0xD0 && c <= 0xDF)
			xd++;
		else if (c >= 0xE0)
			xef++;
	}

	if (!noniso && ((xc + xef) < xd))
		charset = "ISO-8859-5";
	else if ((xc + xd) < xef)
		charset = "CP1251";
	else
		charset = encoding_table [get_locale_index()][OPENI18N];

	return charset;
}

static const gchar *detect_charset_chinese (const guchar *text)
{
	guchar c;

	const gchar *charset = NULL;

	while ((c = *text++) != '\0')
	{
		if (c >= 0x81 && c <= 0x87)
		{
			charset = "GB18030";
			break;
		}
		else if (c >= 0x88 && c <= 0xA0)
		{
			c = *text++;
			if ((c >= 0x30 && c <= 0x39) || (c >= 0x80 && c <= 0xA0))
			{
				charset = "GB18030";
				break;
			} //else GBK/Big5-HKSCS cannot determine
		}
		else if ((c >= 0xA1 && c <= 0xC6) || (c >= 0xC9 && c <= 0xF9))
		{
			c = *text++;
			if (c >= 0x40 && c <= 0x7E)
			{
				charset = "BIG5";
				break;
			}
			else if ((c >= 0x30 && c <= 0x39) || (c >= 0x80 && c <= 0xA0))
			{
				charset = "GB18030";
				break;
			}
		}
		else if (c >= 0xC7)
		{
			c = *text++;
			if ((c >= 0x30 && c <= 0x39) || (c >= 0x80 && c <= 0xA0))
			{
				charset = "GB18030";
				break;
			}
		}
	}

	if (charset == NULL)
		charset = encoding_table [get_locale_index()][IANA];

	return charset;
}

static const gchar *detect_charset_japanese (const guchar *text)
{
	guchar c;
	gchar *charset = NULL;

	while ((c = *text++) != '\0')
	{
		if (c >= 0x81 && c <= 0x9F)
		{
			if (c == 0x8E) /* SS2 */
			{
				c = *text++;
				if ((c >= 0x40 && c <= 0xA0) || (c >= 0xE0 && c <= 0xFC))
				{
					charset = "CP932";
					break;
				}
			}
			else if (c == 0x8F) /* SS3 */
			{
				c = *text++;
				if (c >= 0x40 && c <= 0xA0)
				{
					charset = "CP932";
					break;
				}
				else if (c >= 0xFD)
					break;
			}
			else
			{
				charset = "CP932";
				break;
			}
		}
		else if (c >= 0xA1 && c <= 0xDF)
		{
			c = *text++;
			if (c <= 0x9F)
			{
				charset = "CP932";
				break;
			}
			else if (c >= 0xFD)
				break;
		}
		else if (c >= 0xE0 && c <= 0xEF)
		{
			c = *text++;
			if (c >= 0x40 && c <= 0xA0)
			{
				charset = "CP932";
				break;
			}
			else if (c >= 0xFD)
				break;
		}
		else if (c >= 0xF0)
			break;
	}

	if (charset == NULL)
		charset = "EUC-JP";

	return charset;
}

static const gchar *detect_charset_korean (const guchar *text)
{
	guchar c;
	gboolean noneuc = FALSE;
	gboolean nonjohab = FALSE;
	gchar *charset = NULL;

	while ((c = *text++) != '\0')
	{
		if (c >= 0x81 && c < 0x84)
		{
			charset = "CP949";
			break;
		}
		else if (c >= 0x84 && c < 0xA1)
		{
			noneuc = TRUE;
			c = *text++;
			if ((c > 0x5A && c < 0x61) || (c > 0x7A && c < 0x81))
			{
				charset = "CP1361";
				break;
			}
			else if (c == 0x52 || c == 0x72 || c == 0x92 || (c > 0x9D && c < 0xA1)
				|| c == 0xB2 || (c > 0xBD && c < 0xC1) || c == 0xD2
				|| (c > 0xDD && c < 0xE1) || c == 0xF2 || c == 0xFE)
			{
				charset = "CP949";
				break;
			}
		}
		else if (c >= 0xA1 && c <= 0xC6)
		{
			c = *text++;
			if (c < 0xA1)
			{
				noneuc = TRUE;
				if ((c > 0x5A && c < 0x61) || (c > 0x7A && c < 0x81))
				{
					charset = "CP1361";
					break;
				}
				else if (c == 0x52 || c == 0x72 || c == 0x92 || (c > 0x9D && c < 0xA1))
				{
					charset = "CP949";
					break;
				}
				else if (c == 0xB2 || (c > 0xBD && c < 0xC1) || c == 0xD2
					|| (c > 0xDD && c < 0xE1) || c == 0xF2 || c == 0xFE)
					nonjohab = TRUE;
			}
		}
		else if (c > 0xC6 && c <= 0xD3)
		{
			c = *text++;
			if (c < 0xA1)
			{
				charset = "CP1361";
				break;
			}
		}
		else if (c > 0xD3 && c < 0xD8)
		{
			nonjohab = TRUE;
			c = *text++;
		}
		else if (c >= 0xD8)
		{
			c = *text++;
			if (c < 0xA1)
			{
				charset = "CP1361";
				break;
			}
		}
		if (noneuc && nonjohab)
		{
			charset = "CP949";
			break;
		}
	}

	if (charset == NULL)
	{
		if (noneuc)
			charset = "CP949";
		else
			charset = "EUC-KR";
	}

	return charset;
}

static gboolean detect_noniso (const guchar *text)
{
	guchar c;

	while ((c = *text++) != '\0')
	{
		if (c >= 0x80 && c <= 0x9F)
			return TRUE;
	}
	return FALSE;
}

/**
@brief detect any recognised byte-order-mark at start of @a text
@param text the string to be processed
Note
There's no distinction between flavours of UTF-16 and the corresponding
UCS-2's, or between UTF-32's and corresponding UCS-4's
No checking for file length, particularly in the case of UTF-32
@return encoding name if any is recognised, or NULL
*/
static const gchar *detect_charset_BOM (const guchar *text)
{
	guchar c;
	gint count;
	guchar array[3];

	count = 0;
	switch (*text++)
	{
		case '+':
			//2B 2F 76 and one of the following bytes: [ 3C | 3D | 3E | 3F ]
			array[0] = '/';
			array[1] = 'v';
			while (count < 2 && (c = *text++) == array[count]) {count++;}
			if (count == 2 && ((c = *text) >= '\\' && c <= '_'))
				return "UTF-7";
			break;
		case 0xEF:
			//EF BB BF
			array[0] = 0xBB;
			array[1] = 0xBF;
			while (count < 2 && (c = *text++) == array[count]) {count++;}
			if (count == 2)
				return "UTF-8";
			break;
		/*UCS-2, UCS-2, UCS-2BE, UCS-2LE (often conflated with UTF-16 but they're not the same)
		  have same BOM as corresponding UTF-16. Ditto for UCS-4's and UTF-32's	*/
		case 0xFE:
			//Big Endian FE FF
			if (*text == 0xFF)
				return "UTF-16BE"; //could actually be UCS-2BE, not quite the same
			break;
		case 0xFF:
			//Little Endian	FF FE 00 00
			//Little Endian FF FE
			array[0] = 0xFE;
			array[1] = 0;
			array[2] = 0;
			while (count < 3 && (c = *text++) == array[count]) {count++;}
			if (count == 3)
				return "UTF-32LE";
			if (count == 1)
				return "UTF-16LE"; //could actually be UCS-2LE, not quite the same
			break;
		case 0: //CHECKME if file is empty, next checks may be a buffer overflow ?
			//Big Endian 00 00 FE FF
			array[0] = 0;
			array[1] = 0xFE;
			array[2] = 0xFF;
			while (count < 3 && (c = *text++) == array[count]) {count++;}
			if (count == 3)
				return "UTF-32BE";
			break;
		case 0x0E:
			//0E FE FF
			array[0] = 0xFE;
			array[1] = 0xFF;
			while (count < 2 && (c = *text++) == array[count]) {count++;}
			if (count == 2)
				return "SCSU";
			break;
		case 0xDD:
			//DD 73 66 73
			array[0] = 's';
			array[1] = 'f';
			array[2] = 's';
			while (count < 3 && (c = *text++) == array[count]) {count++;}
			if (count == 3)
				return "UTF-EBCDIC";
			break;
		case 0xFB:
			//FB EE 28
			array[0] = 0xEE;
			array[1] = 0x28;
			while (count < 2 && (c = *text++) == array[count]) {count++;}
			if (count == 2)
				return "BOCU-1";
		default:
			break;
	}
	return NULL;
}

/**
@brief detect character encoding of 0-terminated string @a text
@a text may be validated as UTF-8 in spite of a nominally different coding
@a charset is set to "UNKNOWN" if nothing specific is recognised
@param text 0-terminated string to be processed
@param charset store for name of detected charset
@return TRUE if @a text is found to be valid UTF-8 (which includes plain ascii)
*/
gboolean e2_utf8_detect_charset (const guchar *text, const gchar **charset)
{
	const guchar *start;
	guchar c;
	gboolean isutf8;

	*charset = detect_charset_BOM (text);
	if (*charset != NULL)	//FIXME test for short file length if UTF-32*
		return (g_str_equal (*charset, "UTF-8"));

	start = text;
	isutf8 = FALSE;
	while ((c = *text++) != '\0')
	{
		if (c > 0x7F)
		{
			if ((c & 0xC0) == 0xC0)
				isutf8 = g_utf8_validate ((gchar *)--text, -1, NULL);
			break;
		}
		else if (c == '~')
		{
			c = *text++;
			if (c == '{')
			{
				c = *text++;	//maybe 1st GB2312 (chinese) char
				if (c != '\0')
				{
					c = *text++;	//maybe } or 2nd GB2312 (chinese) char
					if (c == '}' && *text == '~')
					{
						*charset = "HZ";
						break;
					}
					else
					{
						c = *text++;	//maybe } after 2nd GB2312 (chinese) char
						if (c == '}' && *text == '~' && *(text-2) > 0x7F)
						{
							*charset = "HZ";
							break;
						}
					}
				}
			}
		}
		else if (c == 0x1B) /* ESC */
		{
			c = *text++;
			if (c == '$')
			{
				c = *text++;
				switch (c)
				{
					case 'B': // JIS X 0208-1983
					case '@': // JIS X 0208-1978
						*charset = "ISO-2022-JP";
						break;
					case 'A': // GB2312-1980
						*charset = "ISO-2022-JP-2";
						break;
					case '(':
						c = *text++;
						switch (c)
						{
							case 'C': // KSC5601-1987
							case 'D': // JIS X 0212-1990
								*charset = "ISO-2022-JP-2";
						}
						break;
					case ')':
						c = *text++;
						if (c == 'C')
							*charset = "ISO-2022-KR"; // KSC5601-1987
					default:
						break;
				}
				if (*charset != NULL)
					break;
			}
		}
	}

	if (isutf8)
	{	//validation succeeded
		if (!g_get_charset (charset))	//use default encoding if it's utf8-compatible
			*charset = "UTF-8";
	}
	else	//no validation requested, or validation failed
		if (c == '\0')	//reached end of text without detection
	{	//it's almost certainly ASCII, assume the default encoding applies
		//FIXME except if that's incompatible with ascii
//		printd (DEBUG, "Specific UTF8 detection failed, reverted to DEFAULT CHARSET %s", *charset);
		e2_utils_get_charset (charset);
		isutf8 = TRUE;	//no conversion needed
	}
	else // !isutf8 && not scanned to end of buffer
		if (*charset == NULL)	//not detected ISO2022 or HZ
	{
		//next, assume that the file originates in the user's locale
		guint code = get_locale_index();
		switch (code)
		{
			case CYRILIC:
			case CYRILIC_UA:
			case CYRILIC_TJ:
				*charset = detect_charset_cyrillic(text); // fuzzy...
				break;
			case CHINESE_CN:
			case CHINESE_TW:
			case CHINESE_HK:
				*charset = detect_charset_chinese(text);
				break;
			case JAPANESE:
				*charset = detect_charset_japanese(text);
				break;
			case KOREAN:
				*charset = detect_charset_korean(text);
				break;
			case VIETNAMESE:
			case THAI:
			case GEORGIAN:
				*charset = encoding_table [code][OPENI18N];
				break;
			default:
				e2_utils_get_charset (charset);
				if (strcmp (*charset, "UTF-8") == 0)	//CHECKME this is no use as the text isn't compliant
				{
					if (detect_noniso (text))
						*charset = encoding_table [code][CODEPAGE];
					else
						*charset = encoding_table [code][OPENI18N];
				}
				if (*charset == NULL)
					*charset = encoding_table [code][IANA];
				break;
		}
	}

	if (*charset == NULL)
	{
		//all checks failed, send back something ...
//		e2_utils_get_charset (charset);
//		printd (DEBUG, "Detection failed 2, reverted to DEFAULT CHARSET %s", *charset);
		*charset = "UNKNOWN";	//no translation
	}

	return isutf8;
}
