/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2020 NKI/AVL, Netherlands Cancer Institute
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "revision.hpp"

#include <filesystem>
#include <fstream>
#include <functional>
#include <stack>
#include <unordered_set>

#include <cif++.hpp>
#include <cif++/gzio.hpp>
#include <mcfp/mcfp.hpp>

namespace fs = std::filesystem;

using unicode = char32_t;

namespace zeep
{

// inlines

/// \brief Append a single unicode character to an utf-8 string
inline void append(std::string &s, unicode uc)
{
	if (uc < 0x080)
		s += (static_cast<char>(uc));
	else if (uc < 0x0800)
	{
		char ch[2] = {
			static_cast<char>(0x0c0 | (uc >> 6)),
			static_cast<char>(0x080 | (uc & 0x3f))
		};
		s.append(ch, 2);
	}
	else if (uc < 0x00010000)
	{
		char ch[3] = {
			static_cast<char>(0x0e0 | (uc >> 12)),
			static_cast<char>(0x080 | ((uc >> 6) & 0x3f)),
			static_cast<char>(0x080 | (uc & 0x3f))
		};
		s.append(ch, 3);
	}
	else
	{
		char ch[4] = {
			static_cast<char>(0x0f0 | (uc >> 18)),
			static_cast<char>(0x080 | ((uc >> 12) & 0x3f)),
			static_cast<char>(0x080 | ((uc >> 6) & 0x3f)),
			static_cast<char>(0x080 | (uc & 0x3f))
		};
		s.append(ch, 4);
	}
}

/// \brief remove the last unicode character from an utf-8 string
inline unicode pop_last_char(std::string &s)
{
	unicode result = 0;

	if (not s.empty())
	{
		std::string::iterator ch = s.end() - 1;

		if ((*ch & 0x0080) == 0)
		{
			result = *ch;
			s.erase(ch);
		}
		else
		{
			int o = 0;

			do
			{
				result |= (*ch & 0x03F) << o;
				o += 6;
				--ch;
			} while (ch != s.begin() and (*ch & 0x0C0) == 0x080);

			switch (o)
			{
				case 6: result |= (*ch & 0x01F) << 6; break;
				case 12: result |= (*ch & 0x00F) << 12; break;
				case 18: result |= (*ch & 0x007) << 18; break;
			}

			s.erase(ch, s.end());
		}
	}

	return result;
}

// this code only works if the input is valid utf-8
/// \brief return the first unicode and the advanced pointer from a string
template <typename Iter>
std::tuple<unicode, Iter> get_first_char(Iter ptr)
{
	unicode result = static_cast<unsigned char>(*ptr);
	++ptr;

	if (result > 0x07f)
	{
		unsigned char ch[3];

		if ((result & 0x0E0) == 0x0C0)
		{
			ch[0] = static_cast<unsigned char>(*ptr);
			++ptr;
			result = ((result & 0x01F) << 6) | (ch[0] & 0x03F);
		}
		else if ((result & 0x0F0) == 0x0E0)
		{
			ch[0] = static_cast<unsigned char>(*ptr);
			++ptr;
			ch[1] = static_cast<unsigned char>(*ptr);
			++ptr;
			result = ((result & 0x00F) << 12) | ((ch[0] & 0x03F) << 6) | (ch[1] & 0x03F);
		}
		else if ((result & 0x0F8) == 0x0F0)
		{
			ch[0] = static_cast<unsigned char>(*ptr);
			++ptr;
			ch[1] = static_cast<unsigned char>(*ptr);
			++ptr;
			ch[2] = static_cast<unsigned char>(*ptr);
			++ptr;
			result = ((result & 0x007) << 18) | ((ch[0] & 0x03F) << 12) | ((ch[1] & 0x03F) << 6) | (ch[2] & 0x03F);
		}
	}

	return std::make_tuple(result, ptr);
}

// --------------------------------------------------------------------

inline std::string to_hex(uint32_t i)
{
	char s[sizeof(i) * 2 + 3];
	char *p = s + sizeof(s);
	*--p = 0;

	const char kHexChars[] = "0123456789abcdef";

	while (i)
	{
		*--p = kHexChars[i & 0x0F];
		i >>= 4;
	}

	*--p = 'x';
	*--p = '0';

	return p;
}
} // namespace zeep

// -----------------------------------------------------------------------

namespace cql
{

using unicode = uint32_t;
using cif::iequals;

class Statement;
typedef std::shared_ptr<Statement> StatementPtr;

// -----------------------------------------------------------------------

class Statement
{
  public:
	Statement(const Statement &) = delete;
	Statement &operator=(const Statement &) = delete;

	virtual ~Statement() {}

	virtual void Execute() = 0;

  protected:
	Statement() {}
};

// -----------------------------------------------------------------------

class StatementList : public Statement
{
  public:
	StatementList() {}

	void Add(StatementPtr stmt)
	{
		mStatements.emplace_back(stmt);
	}

	virtual void Execute()
	{
		for (auto stmt : mStatements)
			stmt->Execute();
	}

  private:
	std::vector<StatementPtr> mStatements;
};

// -----------------------------------------------------------------------

class SelectStatement : public Statement
{
  public:
	SelectStatement(cif::category &category, bool distinct, std::vector<std::string> &&items, cif::condition &&where)
		: mCategory(category)
		, mDistinct(distinct)
		, mItems(std::move(items))
		, mWhere(std::move(where))
	{
	}

	virtual void Execute()
	{
		std::vector<std::string> fields(mItems.size());
		std::unordered_set<std::string> seen;

		std::cout << cif::join(mItems, "\t") << '\n';

		for (auto r : mCategory.find(std::move(mWhere)))
		{
			transform(mItems.begin(), mItems.end(), fields.begin(),
				[r](auto item)
				{
					return r[item].template as<std::string>();
				});

			std::string line = cif::join(fields, "\t");
			bool seenLine = seen.count(line);

			if (not mDistinct or not seenLine)
				std::cout << line << '\n';

			if (mDistinct and not seenLine)
				seen.insert(line);
		}
	}

  private:
	cif::category &mCategory;
	bool mDistinct;
	std::vector<std::string> mItems;
	cif::condition mWhere;
};

// -----------------------------------------------------------------------

class DeleteStatement : public Statement
{
  public:
	DeleteStatement(cif::category &category, cif::condition &&where)
		: mCategory(category)
		, mWhere(std::move(where))
	{
	}

	virtual void Execute()
	{
		std::vector<cif::row_handle> remove;

		mWhere.prepare(mCategory);

		for (auto r : mCategory)
		{
			if (mWhere(r))
				remove.insert(remove.end(), r);
		}

		for (auto r : remove)
			mCategory.erase(r);

		std::cout << "Number of removed rows " << remove.size() << '\n';
	}

  private:
	cif::category &mCategory;
	cif::condition mWhere;
};

// -----------------------------------------------------------------------

class UpdateStatement : public Statement
{
  public:
	UpdateStatement(cif::category &category, std::vector<std::pair<std::string, std::string>> &&itemValuePairs, cif::condition &&where)
		: mCategory(category)
		, mItemValuePairs(std::move(itemValuePairs))
		, mWhere(std::move(where))
	{
	}

	virtual void Execute()
	{
		size_t updated = 0;

		mWhere.prepare(mCategory);

		for (auto r : mCategory)
		{
			if (mWhere(r))
			{
				for (auto iv : mItemValuePairs)
					r[iv.first] = iv.second;

				++updated;
			}
		}

		std::cout << "Number of updated rows: " << updated << '\n';
	}

  private:
	cif::category &mCategory;
	std::vector<std::pair<std::string, std::string>> mItemValuePairs;
	cif::condition mWhere;
};

// -----------------------------------------------------------------------

class Parser
{
  public:
	Parser(cif::datablock &db)
		: mDb(db)
	{
		if (mDb.empty())
			throw std::runtime_error("Empty datablock");
	}

	StatementPtr Parse(std::streambuf *is);

  private:
	enum class Token
	{
		EOLN,

		UNDEF,

		BRACE_OPEN,
		BRACE_CLOSE,

		DOT,
		COMMA,
		COLON,
		SEMICOLON,
		ASTERISK,

		EQ,
		LT,
		LE,
		GT,
		GE,
		NE,

		STRING,
		INTEGER,
		NUMBER,

		IDENT,

		SELECT,
		DISTINCT,
		FROM,
		UPDATE,
		SET,
		WHERE,
		AND,
		OR,
		NOT,
		INSERT,
		DELETE,
		INTO,
		VALUES,
		IS,
		NULL_
	};

	std::string Describe(Token token)
	{
		switch (token)
		{
			case Token::EOLN: return "<EOLN>";
			case Token::UNDEF: return "<UNDEFINED>";
			case Token::BRACE_OPEN: return "'('";
			case Token::BRACE_CLOSE: return "')'";

			case Token::DOT: return "'.'";
			case Token::COMMA: return "','";
			case Token::COLON: return "':'";
			case Token::SEMICOLON: return "';'";
			case Token::ASTERISK: return "'*'";

			case Token::EQ: return "'='";
			case Token::LT: return "'<'";
			case Token::LE: return "'<='";
			case Token::GT: return "'>'";
			case Token::GE: return "'>='";
			case Token::NE: return "'<>'";

			case Token::STRING: return "string";
			case Token::INTEGER: return "integer";
			case Token::NUMBER: return "number";
			case Token::IDENT: return "identifier";

			case Token::SELECT: return "SELECT";
			case Token::DISTINCT: return "DISTINCT";
			case Token::FROM: return "FROM";
			case Token::UPDATE: return "UPDATE";
			case Token::SET: return "SET";
			case Token::WHERE: return "WHERE";
			case Token::AND: return "AND";
			case Token::OR: return "OR";
			case Token::NOT: return "NOT";
			case Token::INSERT: return "INSERT";
			case Token::DELETE: return "DELETE";
			case Token::INTO: return "INTO";
			case Token::VALUES: return "VALUES";
			case Token::IS: return "IS";
			case Token::NULL_: return "NULL";

			default: assert(false); return "unknown token";
		}
	}

	uint8_t GetNextByte();
	unicode GetNextUnicode();
	unicode GetNextChar();
	Token GetNextToken();

	void Retract();
	void Match(Token token);

	// parser rules
	StatementPtr ParseStatement();
	StatementPtr ParseSelect();
	StatementPtr ParseDelete();
	StatementPtr ParseUpdate();
	std::vector<std::string> ParseItemList();

	cif::condition ParseWhereClause(cif::category &cat);
	cif::condition ParseNotWhereClause(cif::category &cat);

	cif::datablock &mDb;
	std::streambuf *mIs;
	Token mLookahead;
	std::stack<unicode> mBuffer;
	std::string mToken;
	double mTokenFloat;
	int64_t mTokenInteger;
};

// -----------------------------------------------------------------------

uint8_t Parser::GetNextByte()
{
	int result = mIs->sbumpc();

	if (result == std::streambuf::traits_type::eof())
		result = 0;

	return static_cast<uint8_t>(result);
}

unicode Parser::GetNextUnicode()
{
	unicode result = GetNextByte();

	if (result & 0x080)
	{
		uint8_t ch[3];

		if ((result & 0x0E0) == 0x0C0)
		{
			ch[0] = GetNextByte();
			if ((ch[0] & 0x0c0) != 0x080)
				throw std::runtime_error("Invalid utf-8");
			result = ((result & 0x01F) << 6) | (ch[0] & 0x03F);
		}
		else if ((result & 0x0F0) == 0x0E0)
		{
			ch[0] = GetNextByte();
			ch[1] = GetNextByte();
			if ((ch[0] & 0x0c0) != 0x080 or (ch[1] & 0x0c0) != 0x080)
				throw std::runtime_error("Invalid utf-8");
			result = ((result & 0x00F) << 12) | ((ch[0] & 0x03F) << 6) | (ch[1] & 0x03F);
		}
		else if ((result & 0x0F8) == 0x0F0)
		{
			ch[0] = GetNextByte();
			ch[1] = GetNextByte();
			ch[2] = GetNextByte();
			if ((ch[0] & 0x0c0) != 0x080 or (ch[1] & 0x0c0) != 0x080 or (ch[2] & 0x0c0) != 0x080)
				throw std::runtime_error("Invalid utf-8");
			result = ((result & 0x007) << 18) | ((ch[0] & 0x03F) << 12) | ((ch[1] & 0x03F) << 6) | (ch[2] & 0x03F);

			if (result > 0x10ffff)
				throw std::runtime_error("invalid utf-8 character (out of range)");
		}
	}

	return result;
}

unicode Parser::GetNextChar()
{
	unicode result = 0;

	if (not mBuffer.empty()) // if buffer is not empty we already did all the validity checks
	{
		result = mBuffer.top();
		mBuffer.pop();
	}
	else
	{
		result = GetNextUnicode();

		if (result >= 0x080)
		{
			if (result == 0x0ffff or result == 0x0fffe)
				throw std::runtime_error("character " + zeep::to_hex(result) + " is not allowed");

			// surrogate support
			else if (result >= 0x0D800 and result <= 0x0DBFF)
			{
				unicode uc2 = GetNextChar();
				if (uc2 >= 0x0DC00 and uc2 <= 0x0DFFF)
					result = (result - 0x0D800) * 0x400 + (uc2 - 0x0DC00) + 0x010000;
				else
					throw std::runtime_error("leading surrogate character without trailing surrogate character");
			}
			else if (result >= 0x0DC00 and result <= 0x0DFFF)
				throw std::runtime_error("trailing surrogate character without a leading surrogate");
		}
	}

	//	append(mToken, result);
	// somehow, append refuses to inline, so we have to do it ourselves
	if (result < 0x080)
		mToken += (static_cast<char>(result));
	else if (result < 0x0800)
	{
		char ch[2] = {
			static_cast<char>(0x0c0 | (result >> 6)),
			static_cast<char>(0x080 | (result & 0x3f))
		};
		mToken.append(ch, 2);
	}
	else if (result < 0x00010000)
	{
		char ch[3] = {
			static_cast<char>(0x0e0 | (result >> 12)),
			static_cast<char>(0x080 | ((result >> 6) & 0x3f)),
			static_cast<char>(0x080 | (result & 0x3f))
		};
		mToken.append(ch, 3);
	}
	else
	{
		char ch[4] = {
			static_cast<char>(0x0f0 | (result >> 18)),
			static_cast<char>(0x080 | ((result >> 12) & 0x3f)),
			static_cast<char>(0x080 | ((result >> 6) & 0x3f)),
			static_cast<char>(0x080 | (result & 0x3f))
		};
		mToken.append(ch, 4);
	}

	return result;
}

void Parser::Retract()
{
	assert(not mToken.empty());
	mBuffer.push(zeep::pop_last_char(mToken));
}

bool is_name_start_char(unicode uc)
{
	return (uc >= L'A' and uc <= L'Z') or
	       uc == L'_' or
	       (uc >= L'a' and uc <= L'z') or
	       (uc >= 0x0C0 and uc <= 0x0D6) or
	       (uc >= 0x0D8 and uc <= 0x0F6) or
	       (uc >= 0x0F8 and uc <= 0x02FF) or
	       (uc >= 0x0370 and uc <= 0x037D) or
	       (uc >= 0x037F and uc <= 0x01FFF) or
	       (uc >= 0x0200C and uc <= 0x0200D) or
	       (uc >= 0x02070 and uc <= 0x0218F) or
	       (uc >= 0x02C00 and uc <= 0x02FEF) or
	       (uc >= 0x03001 and uc <= 0x0D7FF) or
	       (uc >= 0x0F900 and uc <= 0x0FDCF) or
	       (uc >= 0x0FDF0 and uc <= 0x0FFFD) or
	       (uc >= 0x010000 and uc <= 0x0EFFFF);
}

bool is_name_char(unicode uc)
{
	return (uc >= '0' and uc <= '9') or
	       uc == 0x0B7 or
	       is_name_start_char(uc) or
	       (uc >= 0x00300 and uc <= 0x0036F) or
	       (uc >= 0x0203F and uc <= 0x02040);
}

Parser::Token Parser::GetNextToken()
{
	enum class State
	{
		Start,
		Negative,
		Zero,
		NegativeZero,
		Number,
		NumberFraction,
		NumberExpSign,
		NumberExpDigit1,
		NumberExpDigit2,
		Literal,
		String,
		Escape,
		EscapeHex1,
		EscapeHex2,
		EscapeHex3,
		EscapeHex4,

		Less,
		Greater
	} state = State::Start;

	Token token = Token::UNDEF;
	double fraction = 1.0, exponent = 1;
	bool negative = false, negativeExp = false;

	unicode hx = 0;

	mToken.clear();

	while (token == Token::UNDEF)
	{
		unicode ch = GetNextChar();

		switch (state)
		{
			case State::Start:
				switch (ch)
				{
					case 0:
						token = Token::EOLN;
						break;
					case '(':
						token = Token::BRACE_OPEN;
						break;
					case ')':
						token = Token::BRACE_CLOSE;
						break;
					// case '[':
					// 	token = Token::LeftBracket;
					// 	break;
					// case ']':
					// 	token = Token::RightBracket;
					// 	break;
					case '.':
						token = Token::DOT;
						break;
					case ',':
						token = Token::COMMA;
						break;
					case ':':
						token = Token::COLON;
						break;
					case ';':
						token = Token::SEMICOLON;
						break;
					case '*':
						token = Token::ASTERISK;
						break;
					case '=':
						token = Token::EQ;
						break;
					case '<':
						state = State::Less;
						break;
					case '>':
						state = State::Greater;
						break;
					case ' ':
					case '\n':
					case '\r':
					case '\t':
						mToken.clear();
						break;
					case '\'':
						mToken.pop_back();
						state = State::String;
						break;
					case '-':
						state = State::Negative;
						break;
					default:
						if (ch == '0')
						{
							state = State::Zero;
							mTokenInteger = 0;
						}
						else if (ch >= '1' and ch <= '9')
						{
							mTokenInteger = ch - '0';
							state = State::Number;
						}
						else if (is_name_start_char(ch))
							state = State::Literal;
						else
							throw std::runtime_error("invalid character (" + zeep::to_hex(ch) + "/'" + (isprint(ch) ? static_cast<char>(ch) : '.') + "') in command");
				}
				break;

			case State::Less:
				if (ch == '=')
					token = Token::LE;
				else if (ch == '>')
					token = Token::NE;
				else
				{
					Retract();
					token = Token::LT;
				}
				break;

			case State::Greater:
				if (ch == '=')
					token = Token::GE;
				else
				{
					Retract();
					token = Token::GT;
				}
				break;

			case State::Negative:
				if (ch == '0')
					state = State::NegativeZero;
				else if (ch >= '1' and ch <= '9')
				{
					state = State::Number;
					mTokenInteger = ch - '0';
					negative = true;
				}
				else
					throw std::runtime_error("invalid character '-' in command");
				break;

			case State::NegativeZero:
				if (ch >= '0' or ch <= '9')
					throw std::runtime_error("invalid number in command, should not start with zero");
				token = Token::NUMBER;
				break;

			case State::Zero:
				if (ch >= '0' or ch <= '9')
					throw std::runtime_error("invalid number in command, should not start with zero");
				token = Token::NUMBER;
				break;

			case State::Number:
				if (ch >= '0' and ch <= '9')
					mTokenInteger = 10 * mTokenInteger + (ch - '0');
				else if (ch == '.')
				{
					mTokenFloat = mTokenInteger;
					fraction = 0.1;
					state = State::NumberFraction;
				}
				else
				{
					Retract();
					token = Token::INTEGER;
				}
				break;

			case State::NumberFraction:
				if (ch >= '0' and ch <= '9')
				{
					mTokenFloat += fraction * (ch - '0');
					fraction /= 10;
				}
				else if (ch == 'e' or ch == 'E')
					state = State::NumberExpSign;
				else
				{
					Retract();
					token = Token::NUMBER;
				}
				break;

			case State::NumberExpSign:
				if (ch == '+')
					state = State::NumberExpDigit1;
				else if (ch == '-')
				{
					negativeExp = true;
					state = State::NumberExpDigit1;
				}
				else if (ch >= '0' and ch <= '9')
				{
					exponent = (ch - '0');
					state = State::NumberExpDigit2;
				}
				break;

			case State::NumberExpDigit1:
				if (ch >= '0' and ch <= '9')
				{
					exponent = (ch - '0');
					state = State::NumberExpDigit2;
				}
				else
					throw std::runtime_error("invalid floating point format in command");
				break;

			case State::NumberExpDigit2:
				if (ch >= '0' and ch <= '9')
					exponent = 10 * exponent + (ch - '0');
				else
				{
					Retract();
					mTokenFloat *= pow(10, (negativeExp ? -1 : 1) * exponent);
					if (negative)
						mTokenFloat = -mTokenFloat;
					token = Token::NUMBER;
				}
				break;

			case State::Literal:
				if (not is_name_char(ch))
				{
					Retract();

					if (iequals(mToken, "SELECT"))
						token = Token::SELECT;
					else if (iequals(mToken, "DISTINCT"))
						token = Token::DISTINCT;
					else if (iequals(mToken, "FROM"))
						token = Token::FROM;
					else if (iequals(mToken, "UPDATE"))
						token = Token::UPDATE;
					else if (iequals(mToken, "SET"))
						token = Token::SET;
					else if (iequals(mToken, "WHERE"))
						token = Token::WHERE;
					else if (iequals(mToken, "AND"))
						token = Token::AND;
					else if (iequals(mToken, "OR"))
						token = Token::OR;
					else if (iequals(mToken, "NOT"))
						token = Token::NOT;
					else if (iequals(mToken, "INSERT"))
						token = Token::INSERT;
					else if (iequals(mToken, "DELETE"))
						token = Token::DELETE;
					else if (iequals(mToken, "INTO"))
						token = Token::INTO;
					else if (iequals(mToken, "VALUES"))
						token = Token::VALUES;
					else if (iequals(mToken, "IS"))
						token = Token::IS;
					else if (iequals(mToken, "NULL"))
						token = Token::NULL_;
					else
						token = Token::IDENT;
				}
				break;

			case State::String:
				if (ch == '\'')
				{
					token = Token::STRING;
					mToken.pop_back();
				}
				else if (ch == 0)
					throw std::runtime_error("Invalid unterminated std::string in command");
				else if (ch == '\\')
				{
					state = State::Escape;
					mToken.pop_back();
				}
				break;

			case State::Escape:
				switch (ch)
				{
					case '\'':
					case '\\':
					case '/':
						break;

					case 'n': mToken.back() = '\n'; break;
					case 't': mToken.back() = '\t'; break;
					case 'r': mToken.back() = '\r'; break;
					case 'f': mToken.back() = '\f'; break;
					case 'b': mToken.back() = '\b'; break;

					case 'u':
						state = State::EscapeHex1;
						mToken.pop_back();
						break;

					default:
						throw std::runtime_error("Invalid escape sequence in command (\\" + std::string{ static_cast<char>(ch) } + ')');
				}
				if (state == State::Escape)
					state = State::String;
				break;

			case State::EscapeHex1:
				if (/*ch >= 0 and*/ ch <= '9')
					hx = ch - '0';
				else if (ch >= 'a' and ch <= 'f')
					hx = 10 + ch - 'a';
				else if (ch >= 'A' and ch <= 'F')
					hx = 10 + ch - 'A';
				else
					throw std::runtime_error("Invalid hex sequence in command");
				mToken.pop_back();
				state = State::EscapeHex2;
				break;

			case State::EscapeHex2:
				if (/*ch >= 0 and*/ ch <= '9')
					hx = 16 * hx + ch - '0';
				else if (ch >= 'a' and ch <= 'f')
					hx = 16 * hx + 10 + ch - 'a';
				else if (ch >= 'A' and ch <= 'F')
					hx = 16 * hx + 10 + ch - 'A';
				else
					throw std::runtime_error("Invalid hex sequence in command");
				mToken.pop_back();
				state = State::EscapeHex3;
				break;

			case State::EscapeHex3:
				if (/*ch >= 0 and*/ ch <= '9')
					hx = 16 * hx + ch - '0';
				else if (ch >= 'a' and ch <= 'f')
					hx = 16 * hx + 10 + ch - 'a';
				else if (ch >= 'A' and ch <= 'F')
					hx = 16 * hx + 10 + ch - 'A';
				else
					throw std::runtime_error("Invalid hex sequence in command");
				mToken.pop_back();
				state = State::EscapeHex4;
				break;

			case State::EscapeHex4:
				if (/*ch >= 0 and*/ ch <= '9')
					hx = 16 * hx + ch - '0';
				else if (ch >= 'a' and ch <= 'f')
					hx = 16 * hx + 10 + ch - 'a';
				else if (ch >= 'A' and ch <= 'F')
					hx = 16 * hx + 10 + ch - 'A';
				else
					throw std::runtime_error("Invalid hex sequence in command");
				mToken.pop_back();
				zeep::append(mToken, hx);
				state = State::String;
				break;
		}
	}

	return token;
}

void Parser::Match(Token expected)
{
	if (mLookahead != expected)
		throw std::runtime_error("Syntax error in command, expected " + Describe(expected) + " but found " + Describe(mLookahead) + " (" + mToken + ")");

	mLookahead = GetNextToken();
}

StatementPtr Parser::Parse(std::streambuf *is)
{
	mIs = is;

	mLookahead = GetNextToken();
	std::shared_ptr<StatementList> result(new StatementList());

	while (mLookahead != Token::EOLN)
	{
		auto stmt = ParseStatement();
		result->Add(stmt);
	}

	return result;
}

// -----------------------------------------------------------------------

StatementPtr Parser::ParseStatement()
{
	StatementPtr result;

	switch (mLookahead)
	{
		case Token::SELECT:
			Match(Token::SELECT);
			result = ParseSelect();
			break;

		case Token::DELETE:
			Match(Token::DELETE);
			result = ParseDelete();
			break;

		case Token::UPDATE:
			Match(Token::UPDATE);
			result = ParseUpdate();
			break;

		default:
			// force error
			Match(Token::SELECT);
	}

	Match(Token::SEMICOLON);

	return result;
}

// -----------------------------------------------------------------------

StatementPtr Parser::ParseSelect()
{
	bool distinct = false;
	if (mLookahead == Token::DISTINCT)
	{
		distinct = true;
		Match(Token::DISTINCT);
	}

	auto items = ParseItemList();

	Match(Token::FROM);

	std::string cat = mToken;
	Match(Token::IDENT);

	auto category = mDb.get(cat);
	if (category == nullptr)
		throw std::runtime_error("Category " + cat + " is not defined in this file");

	auto cv = category->get_cat_validator();
	if (cv != nullptr)
	{
		std::vector<std::string> nItems;

		for (auto item : items)
		{
			if (item == "*")
				transform(cv->m_item_validators.begin(), cv->m_item_validators.end(), back_inserter(nItems),
					[cat](auto iv)
					{ return iv.m_item_name; });
			else
				nItems.push_back(item);
		}

		swap(items, nItems);

		items.erase(remove_if(items.begin(), items.end(), [category](auto item)
						{ return not category->has_item(item); }),
			items.end());

		for (auto item : items)
		{
			auto iv = cv->get_validator_for_item(item);
			if (iv == nullptr)
				throw std::runtime_error("Item " + item + " is not defined in the PDBx dictionary for category " + cat);
		}
	}

	if (mLookahead == Token::WHERE)
	{
		Match(Token::WHERE);
		return StatementPtr{ new SelectStatement(*category, distinct, std::move(items), ParseNotWhereClause(*category)) };
	}
	else
		return StatementPtr{ new SelectStatement(*category, distinct, std::move(items), cif::all()) };
}

// -----------------------------------------------------------------------

StatementPtr Parser::ParseDelete()
{
	Match(Token::FROM);

	std::string cat = mToken;
	Match(Token::IDENT);

	auto category = mDb.get(cat);
	if (category == nullptr)
		throw std::runtime_error("Category " + cat + " is not defined in this file");

	if (mLookahead == Token::WHERE)
	{
		Match(Token::WHERE);
		return StatementPtr{ new DeleteStatement(*category, ParseNotWhereClause(*category)) };
	}
	else
		return StatementPtr{ new DeleteStatement(*category, cif::all()) };
}

// -----------------------------------------------------------------------

StatementPtr Parser::ParseUpdate()
{
	std::string cat = mToken;
	Match(Token::IDENT);

	auto category = mDb.get(cat);
	if (category == nullptr)
		throw std::runtime_error("Category " + cat + " is not defined in this file");

	auto cv = category->get_cat_validator();

	Match(Token::SET);

	std::vector<std::pair<std::string, std::string>> itemValuePairs;
	for (;;)
	{
		std::string item = mToken;
		Match(Token::IDENT);

		auto iv = cv ? cv->get_validator_for_item(item) : nullptr;
		if (cv and iv == nullptr)
			throw std::runtime_error("Invalid item '" + item + "' for category '" + cat + '\'');

		Match(Token::EQ);

		std::string value = mToken;
		switch (mLookahead)
		{
			case Token::INTEGER:
			case Token::NUMBER:
			case Token::STRING:
				Match(mLookahead);
				break;
			default:
				Match(Token::STRING);
		}

		if (iv)
			iv->operator()(value);

		itemValuePairs.emplace_back(item, value);

		if (mLookahead == Token::COMMA)
		{
			Match(Token::COMMA);
			continue;
		}

		break;
	}

	if (mLookahead == Token::WHERE)
	{
		Match(Token::WHERE);
		return StatementPtr{ new UpdateStatement(*category, std::move(itemValuePairs), ParseNotWhereClause(*category)) };
	}
	else
		return StatementPtr{ new UpdateStatement(*category, std::move(itemValuePairs), cif::all()) };
}

// -----------------------------------------------------------------------

std::vector<std::string> Parser::ParseItemList()
{
	std::vector<std::string> items;

	for (;;)
	{
		if (mLookahead == Token::ASTERISK)
		{
			Match(Token::ASTERISK);
			items.push_back("*");
		}
		else
		{
			items.push_back(mToken);
			Match(Token::IDENT);
		}

		if (mLookahead == Token::COMMA)
		{
			Match(Token::COMMA);
			continue;
		}

		break;
	}

	return items;
}

// -----------------------------------------------------------------------

cif::condition Parser::ParseNotWhereClause(cif::category &cat)
{
	cif::condition result;

	if (mLookahead == Token::NOT)
	{
		Match(Token::NOT);
		result = not ParseNotWhereClause(cat);
	}
	else if (mLookahead == Token::BRACE_OPEN)
	{
		Match(Token::BRACE_OPEN);
		result = ParseNotWhereClause(cat);
		Match(Token::BRACE_CLOSE);
	}
	else
	{
		result = ParseWhereClause(cat);

		for (;;)
		{
			if (mLookahead == Token::AND)
			{
				Match(Token::AND);
				result = std::move(result) and ParseNotWhereClause(cat);
				continue;
			}

			if (mLookahead == Token::OR)
			{
				Match(Token::OR);
				result = std::move(result) or ParseNotWhereClause(cat);
				continue;
			}

			break;
		}
	}
	return result;
}

// -----------------------------------------------------------------------

cif::condition Parser::ParseWhereClause(cif::category &cat)
{
	std::string item = mToken;
	Match(Token::IDENT);

	auto cv = cat.get_cat_validator();
	if (cv != nullptr and cv->get_validator_for_item(item) == nullptr)
	{
		throw std::runtime_error("Invalid item '" + item + "' for category '" + cat.name() + "' in where clause");
	}

	if (mLookahead == Token::IS)
	{
		Match(mLookahead);

		if (mLookahead == Token::NOT)
		{
			Match(mLookahead);
			Match(Token::NULL_);
			return cif::key(item) != cif::null;
		}
		else
		{
			Match(Token::NULL_);
			return cif::key(item) == cif::null;
		}
	}
	else
	{
		if (mLookahead < Token::EQ or mLookahead > Token::NE)
			Match(Token::EQ);

		auto oper = mLookahead;
		Match(mLookahead);

		cif::condition c;
		std::string value = mToken;

		switch (mLookahead)
		{
			case Token::INTEGER:
			case Token::NUMBER:
			case Token::STRING:
				Match(mLookahead);
				break;
			default:
				Match(Token::STRING);
		}

		switch (oper)
		{
			case Token::EQ: return cif::key(item) == value;
			case Token::LT: return cif::key(item) < value;
			case Token::LE: return cif::key(item) <= value;
			case Token::GT: return cif::key(item) > value;
			case Token::GE: return cif::key(item) >= value;
			case Token::NE: return cif::key(item) != value;
			default: throw std::logic_error("should never happen");
		}
	}
}

} // namespace cql

// -----------------------------------------------------------------------

int pr_main(int argc, char *argv[])
{
	auto &config = mcfp::config::instance();

	config.init("mmCQL [options] input [output]",
		mcfp::make_option("help,h", "Display help message"),
		mcfp::make_option("version", "Print version"),
		mcfp::make_option("verbose,V", "Verbose output, repeat to increase verbosity level"),

		mcfp::make_option("force", "Force writing of output file, even if it is the same as the input file"),

		mcfp::make_option<std::string>("script,f", "Read commands from script"),

		mcfp::make_option<std::string>("data-block,D", "Datablock to use, default is first"));

	config.parse(argc, argv);

	if (config.has("version"))
	{
		write_version_string(std::cout, config.has("verbose"));
		exit(0);
	}

	if (config.has("help") or config.operands().empty() or config.operands().size() > 2)
	{
		std::cerr << config << '\n';
		exit(config.has("help") ? 0 : 1);
	}

	cif::VERBOSE = config.count("verbose");

	if (config.operands().size() == 2 and config.operands().front() == config.operands().back() and not config.has("force"))
	{
		std::cerr << "Cowardly refusing to overwrite input file (specify --force to force overwriting)\n";
		exit(1);
	}

	cif::gzio::ifstream in(config.operands().front());
	if (not in.is_open())
		throw std::runtime_error("Could not open file " + config.operands().front());

	cif::file file{ in };

	cql::Parser parser(config.has("data-block") ? file[config.get<std::string>("data-block")] : file.front());

	if (config.has("script"))
	{
		std::ifstream cmdFile(config.get<std::string>("script"));
		if (not cmdFile.is_open())
			throw std::runtime_error("Failed to open command file " + config.get<std::string>("script"));

		auto stmt = parser.Parse(cmdFile.rdbuf());
		if (stmt)
			stmt->Execute();
	}
	else
	{
		std::string cmd;
		while (std::getline(std::cin, cmd))
		{
			try
			{
				std::istringstream is(cmd);

				auto stmt = parser.Parse(is.rdbuf());
				if (stmt)
					stmt->Execute();
			}
			catch (const std::exception &e)
			{
				std::cerr << e.what() << '\n';
			}
		}
	}

	if (config.operands().size() == 2)
	{
		cif::gzio::ofstream out(config.operands().back());
		if (not out.is_open())
			throw std::runtime_error("Could not open output file " + config.operands().back());

		file.save(out);
	}

	return 0;
}
