/*
 *  cache-mysql.c
 *  mod_musicindex
 *
 *  $Id: cache-mysql.c 1011 2012-08-07 20:23:39Z varenet $
 *
 *  Created by Thibaut VARENE on Tue Feb 22 2005.
 *  Copyright (c) 2005,2007,2009-2010,2012 Thibaut VARENE
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU Lesser General Public License version 2.1,
 *  as published by the Free Software Foundation.
 *
 */

/**
 * @file
 * MySQL cache management subsystem.
 *
 * This cache backend is expected to be impressively efficient. Almost no locking
 * is necessary, since MySQL has its own locking mechanism. Locking is only necessary
 * where things could go wrong if some data were to change between two requests;
 * typically if mysql_cache_remove_dir() happens between two requests.
 * We try to let MySQL do most of the computation, passing it or retrieving
 * end results for complex queries.
 *
 * This cache backend should be the primary choice for an evolved cache system
 * where it would be possible to search the cache directly (almost) without looking at
 * the original files, and where it would be possible to fetch directory
 * listings without looking at the original directory either.
 *
 * The SQL cache backend relies on the use of the following tables:
 * - table files:
 *	|id|pid|filetype|flags|track|posn|cvers|date|freq|length|bitrate|size|mtime|album|artist|title|genre|filename|
 * - table dirs:
 *	|id|timestamp|fullpath|
 * - table format:
 *	|formatid|
 *
 * The PID in the 'files' table will match the 'id' field of one (and only one)
 * entry in the 'dirs' table, to be able to link a cache entry with its
 * corresponding location on the filesystem.
 * The FILENAME field in the 'files' table will store the basename of the full
 * path to the original file, so that by concatenating the corresponding 'dirs'
 * entry with that field we can reconstruct the full path to the file.
 * The TIMESTAMP field in the 'dirs' table will store a recorded mtime from the
 * original directory, for cache up-to-date-ness checking.
 *
 * @todo clean error handler when data corruption happens.
 *
 * @author Thibaut Varene
 * @version $Revision: 1011 $
 * @date 2005,2007,2009-2010,2012
 */

#include "playlist.h"
#include "cache-mysql.h"

#include <libgen.h>	/* basename() */
#include <string.h>	/* strdup() */
#include <mysql.h>
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>	/* file handling */
#endif

#define TABLE_FILES	"musicindexfiles"	/**< table name for files */
#define TABLE_DIRS	"musicindexdirs"	/**< table name for directories */
#define TABLE_FORMAT	"musicindexformat"	/**< table name for cache global format */
#define TABLE_FORMAT_ID	1			/**< table format identifier. If this changes, drop the tables */
#define SQL_SMAX_UPD	64			/**< max length for user/pass/db */
#define SQL_SMAX_H	256			/**< max length for host */
#define SUBSTRINGIFY(x)	STRINGIFY(x)
#define STRINGIFY(x)	#x
#define CACHE_VERS	3
#define AINC_OVFLERR	1062			/**< AUTO_INCREMENT overflow error code: 1062 == duplicate entry */

/* http://dev.mysql.com/doc/mysql/en/c.html */

/** mysql connexion settings */
static struct {
	char user[SQL_SMAX_UPD];	/**< mysql db user */
	char pass[SQL_SMAX_UPD];	/**< mysql db user's pass */
	char db[SQL_SMAX_UPD];		/**< mysql db */
	char host[SQL_SMAX_H];		/**< mysql db's host */
	short ready;
} mysql_params;

/** Order of the fields for the query, to be in sync with mysql_cache_new_ent(), cvers first, mtime 2nd */
#define SQL_CDATAF "cvers,mtime,filetype,flags,track,posn," \
		"date,freq,length,bitrate,size," \
		"album,artist,title,genre"
/** Number of fields in CDATAF */
#define SQL_CDATAN	15

#define CA_OK		0
#define	CA_CREATE	1
#define	CA_STALE	2
#define CA_NOTREADY	3
#define	CA_FATAL	-1

/**
 * Makes a new #mu_ent from the contents of the provided mysql_row.
 *
 * @param pool Allocation pool
 * @param mysql_row The mysql row containing data
 * @return populated #mu_ent structure
 */
static inline mu_ent*
mysql_cache_new_ent(apr_pool_t *pool, MYSQL_ROW mysql_row)
{
	mu_ent *p = NEW_ENT(pool);
	if (unlikely(!p))
		return p;
	
	p->mtime = (unsigned)atol(mysql_row[1]);
	p->filetype = (signed char)atoi(mysql_row[2]);
	p->flags = (unsigned char)atoi(mysql_row[3]);
	p->track = (unsigned char)atoi(mysql_row[4]);
	p->posn = (unsigned char)atoi(mysql_row[5]);
	p->date = (unsigned short)atoi(mysql_row[6]);
	p->freq = (unsigned short)atoi(mysql_row[7]);
	p->length = (unsigned short)atoi(mysql_row[8]);
	p->bitrate = (unsigned)atol(mysql_row[9]);
	p->size = (unsigned)atol(mysql_row[10]);
	p->album = mysql_row[11] ? apr_pstrdup(pool, mysql_row[11]) : NULL;
	p->artist = mysql_row[12] ? apr_pstrdup(pool, mysql_row[12]) : NULL;
	p->title = apr_pstrdup(pool, mysql_row[13]);
	p->genre = mysql_row[14] ? apr_pstrdup(pool, mysql_row[14]) : NULL;
	
	return p;
}

/**
 * Initializes sql cache subsystem.
 *
 * The cache_setup variable is expected to contain a string formatted as:
 *	"user[:pass]@host/db"
 * All fields are mandatory except 'pass'. Size limitations apply, see
 * SQL_SMAX_UDP and SQL_SMAX_H.
 * We clean up any existing table if they don't look like what we expect.
 *
 * @todo add a table prefix, so that all tables with that prefix
 *	will be under module's control
 *
 * @param s Apache server_rec struct to handle log writings.
 * @param setup_string configuration string in which the path to the cache root can be found.
 *
 * @return 0 on succes, -1 otherwise.
 */
static int
mysql_cache_init(server_rec *s, const char *const setup_string)
{
	MYSQL		*mysql;
	MYSQL_RES	*mysql_res = NULL;
	MYSQL_ROW	mysql_row;
	unsigned short	len, create_dirs = 1, create_files = 1, create_format = 1;
	char		*temp;
	int		ret = -1;
	
	/* XXX TODO indexes */
	const char const drop_tformat[] = "DROP TABLE IF EXISTS `" TABLE_FORMAT "`";
	const char const create_tformat[] = "CREATE TABLE `" TABLE_FORMAT "` ("
			"`formatid` TINYINT UNSIGNED NOT NULL DEFAULT " SUBSTRINGIFY(TABLE_FORMAT_ID) ")";
	const char const init_tformat[] = "INSERT INTO `" TABLE_FORMAT "` () VALUES ()";

	const char const drop_tdirs[] = "DROP TABLE IF EXISTS `" TABLE_DIRS "`";
	const char const create_tdirs[] = "CREATE TABLE `" TABLE_DIRS "` ("
			"`id` SMALLINT UNSIGNED SERIAL DEFAULT VALUE,"	/* 65535 dirs should be plenty enough :P */
			"`timestamp` INT UNSIGNED,"
#if MYSQL_VERSION_ID > 50003	/* MySQL 5.0.3 and above support up to 65,535 wide CHAR columns */
			"`fullpath` VARCHAR(" SUBSTRINGIFY(MAX_PATHNAME) ") CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,"
			"INDEX `check` (`fullpath`))"
#else	/* earlier versions stop at 255. No point in creating a fulltext index, we're not MATCH()'ing strings */
			"`fullpath` TEXT CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,"
#endif
			"CHARACTER SET = utf8,"
			"COLLATE = utf8_bin";
	
	/* XXX hardcoded genre length - we have to use _ci collation for the fields that are used
	 * in the search FULLTEXT index, otherwise MATCH() is case sensitive. Use
	 * COLLATE utf8_bin for all case sensitive searches */
	const char const drop_tfiles[] = "DROP TABLE IF EXISTS `" TABLE_FILES "`";
	const char const create_tfiles[] = "CREATE TABLE `" TABLE_FILES "` ("
			"`id` MEDIUMINT UNSIGNED SERIAL DEFAULT VALUE,"	/* 16,777,215 files. Call me when you overflow! */
			"`pid` SMALLINT UNSIGNED NOT NULL REFERENCES `" TABLE_DIRS "`(`id`),"	/* XXX should eventually be a FOREIGN KEY() */
			"`filetype` TINYINT NOT NULL,"	/* XXX this should be an enum, but I can't figure out how to do it without hardcoding values */
			"`flags` TINYINT UNSIGNED NOT NULL,"
			"`track` TINYINT UNSIGNED NOT NULL,"
			"`posn` TINYINT UNSIGNED NOT NULL,"
			"`cvers` TINYINT UNSIGNED NOT NULL," /* XXX this should be useless since we trash the whole cache on version bumps... */
			"`date` SMALLINT UNSIGNED NOT NULL,"
			"`freq` SMALLINT UNSIGNED NOT NULL,"
			"`length` SMALLINT UNSIGNED NOT NULL,"
			"`bitrate` MEDIUMINT UNSIGNED NOT NULL,"
			"`size` INT UNSIGNED NOT NULL,"
			"`mtime` INT UNSIGNED NOT NULL,"
			"`genre` VARCHAR(" SUBSTRINGIFY(MAX_GENRE) ") CHARACTER SET utf8 COLLATE utf8_unicode_ci,"
			"`album` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci,"
			"`artist` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci,"
			"`title` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL,"
			"`filename` VARCHAR(" SUBSTRINGIFY(MAX_FNAME) ") CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL,"
			"INDEX `make_entry` (`pid`,`filename`),"
			"FULLTEXT INDEX `search` (`filename`,`title`,`artist`,`album`))"
			"CHARACTER SET = utf8,"
			"COLLATE = utf8_bin";
	
	/* from http://dev.mysql.com/doc/mysql/en/fulltext-search.html
	 For natural-language full-text searches, it is a requirement that the columns named
	 in the MATCH() function be the same columns included in some FULLTEXT index in
	 your table. For the preceding query, note that the columns named in the MATCH()
	 function (title and body) are the same as those named in the definition of the article
	 table's FULLTEXT index. If you wanted to search the title or body separately, you would
	 need to create FULLTEXT indexes for each column. */
	
	/* get the fields of interest:
	user[:pass]@host/db
	setup struct mysql_params accordingly */
	mysql_params.ready = 0;
	
	/* first, check whether we have a passwd */
	temp = strchr(setup_string, ':');
	
	if (temp) {
		/* copy the passwd */
		len = strchr(setup_string, '@') - (temp+1);
		if (len >= SQL_SMAX_UPD)
			goto err1;
		
		strncpy(mysql_params.pass, temp+1, len);
		mysql_params.pass[len] = '\0';

		/* store user field length */
		len = temp - (setup_string);
	}
	else {
		/* NULLify for safety */
		mysql_params.pass[0] = '\0';

		/* store user field length */
		len = strchr(setup_string, '@') - (setup_string);
	}
	
	/* copy the user field anyway */
	if (len >= SQL_SMAX_UPD)
		goto err1;
	
	strncpy(mysql_params.user, setup_string, len);
	mysql_params.user[len] = '\0';
	
	/* as well as the host field */
	temp = strchr(setup_string, '@')+1;
	len = strchr(temp, '/') - temp;
	
	if (len >= SQL_SMAX_H)
		goto err1;
	
	strncpy(mysql_params.host, temp, len);
	mysql_params.host[len] = '\0';

	/* be done with db field */
	temp = strchr(temp, '/')+1;
	len = strlen(temp);
	
	if (len >= SQL_SMAX_UPD)
		goto err1;
	
	strcpy(mysql_params.db, temp);
	
	mysql_params.ready = 1;
	
	/* perform sanity check for DB access
	mysqlconnect/selectdb... 
	create the tables if they are empty/non existent
	(do that ourselves so that we can drop them if we change the format */
	
	mysql = mysql_init(NULL);
	
	if (!mysql_real_connect(mysql, mysql_params.host, mysql_params.user,
			mysql_params.pass, mysql_params.db, 0, NULL, 0))	
		goto err2;
	
	if (mysql_set_character_set(mysql, "utf8"))
		goto err2;
	
	mysql_res = mysql_list_tables(mysql, NULL);	/* will be freed later */
	if (!mysql_res)
		goto err2;
	
	/* goto algorithm bullshit
	1. check we have >0 tables
		no -> break, (drop existing), create everything
		yes:
	2. check we have format table
		no -> break, drop existing, create everything
		yes:
	3. check we have good format
		no -> break, drop existing, create everything
		yes:
	4. check we have all tables
		no -> drop possibly out of sync tables, create missing
	*/

	/* 1. */
	if (mysql_num_rows(mysql_res) <= 0)
		goto create;
	
	/* 2. */
	while ((mysql_row = mysql_fetch_row(mysql_res))) {
		if (!strcmp(mysql_row[0], TABLE_FORMAT)) {
			create_format = 0;
			break;
		}
	}

	if (create_format)
		goto create;
	
	/* 3. */
	mysql_free_result(mysql_res);
	mysql_res = NULL;
	if (mysql_query(mysql, "SELECT formatid FROM `" TABLE_FORMAT "`"))
		goto err2;

	mysql_res = mysql_use_result(mysql);
	mysql_row = mysql_fetch_row(mysql_res);

	if (!mysql_row)
		goto err2;

	if (((unsigned)atoi(mysql_row[0]) != TABLE_FORMAT_ID)) {
		create_format = 1;		/* hard update format table */
		goto create;
	}
	
	/* 4. */
	mysql_free_result(mysql_res);		/* clear the previous result first and... */
	mysql_res = mysql_list_tables(mysql, NULL);	/* ...make sure we're 'rewinded' */
	if (!mysql_res)
		goto err2;

	while ((mysql_row = mysql_fetch_row(mysql_res))) {
		if (!strcmp(mysql_row[0], TABLE_FILES))
			create_files = 0;
		if (!strcmp(mysql_row[0], TABLE_DIRS))
			create_dirs = 0;
	}

	if (1 == create_dirs)	/* if dirs is missing, (re)create files no matter what */
		create_files = 1;	/* files depends on dirs for pid */

create:
	mysql_free_result(mysql_res);
	mysql_res = NULL;

	if (create_dirs) {
		if (mysql_query(mysql, drop_tdirs))
			goto err2;
		if (mysql_query(mysql, create_tdirs))
			goto err2;
	}
	if (create_files) {
		if (mysql_query(mysql, drop_tfiles))
			goto err2;
		if (mysql_query(mysql, create_tfiles))
			goto err2;
	}		
	if (create_format) {
		if (mysql_query(mysql, drop_tformat))
			goto err2;
		if (mysql_query(mysql, create_tformat))
			goto err2;
		if (mysql_query(mysql, init_tformat))
			goto err2;
	}	
		
	ret = 0;
	
	/* optimize table upon server startup. Helps keeping them fresh, and
	 * is harmless if they were just created */
	mysql_query(mysql, "OPTIMIZE TABLE `" TABLE_DIRS "`, `" TABLE_FILES "`");

err2:
	if (mysql_errno(mysql))
		mi_serror("An error occured: %s", mysql_error(mysql));
	mysql_free_result(mysql_res);
	mysql_close(mysql);
err1:
	if (!mysql_params.ready)
		mi_serror("Likely length overflow in configuration fields: "
			  "user: %zu/%u, pass: %zu/%u, db: %zu/%u, host: %zu/%u",
			  strlen(mysql_params.user), SQL_SMAX_UPD, strlen(mysql_params.pass),
			  SQL_SMAX_UPD, strlen(mysql_params.db), SQL_SMAX_UPD,
			  strlen(mysql_params.host), SQL_SMAX_H);
	return ret;

}

/**
 * Truncates files and dirs tables.
 *
 * Especially useful if AUTO_INCREMENT fields overflow.
 *
 * @warning Truncate operations are not transaction-safe, so an error occurs if the
 * session attempts one during an active transaction or while holding a table lock.
 *
 * @param r Apache request_rec struct to handle log writings
 * @param mysql mysql handler to communicate with the server
 */
static void
mysql_cache_trunc_tables(request_rec *r, MYSQL *mysql)
{
	if (mysql_query(mysql, "TRUNCATE TABLE `" TABLE_FILES "`"))
		goto exit;
	mysql_query(mysql, "TRUNCATE TABLE `" TABLE_DIRS "`");

	/* since we just did a major change, let's defragment too */
	mysql_query(mysql, "OPTIMIZE TABLE `" TABLE_DIRS "`, `" TABLE_FILES "`");
	
exit:
	if (mysql_errno(mysql))
		mi_rerror("An error occured: %s", mysql_error(mysql));
}

/**
 * Creates cache subdirectories.
 *
 * This subroutine takes care of creating all requested directories and
 * subdirectories if they don't already exist and if possible.
 *
 * @warning Asserts we're already connected to the db.
 *
 * @param r Apache request_rec struct to handle log writings and pool allocation.
 * @param dirpath A string representing a path to create.
 * @param timestamp the mtime for the directory to be cached.
 * @param mysql mysql handler to communicate with the server
 *
 * @return CA_OK on succes, CA_FATAL otherwise.
 */
static int
mysql_cache_make_dir(request_rec *r, const char *const dirpath,
			const unsigned long timestamp, MYSQL *mysql)
{
	char		*restrict sqlstr, *restrict query = NULL;
	MYSQL_RES	*mysql_res = NULL;
	int		ret = CA_FATAL;
	unsigned int	myerrno = 0;
		
	if (!(sqlstr = apr_palloc(r->pool, 2*strlen(dirpath)+1)))
		goto error;

	mysql_real_escape_string(mysql, sqlstr, dirpath, strlen(dirpath));
	
	/* ON DUPLICATE cannot deal with long paths (UNIQUE keys must be <1000 bytes),
	 * we need to do the check by hand:
	 * SELECT; if (num_rows > 0) UPDATE else INSERT
	 * Since this is no longer an atomic SQL operation, we have to lock.
	 * Using LOCK_TABLES as updates shouldn't happen frequently anyway.
	 * See http://dev.mysql.com/doc/refman/4.1/en/lock-tables.html */
	
	mysql_query(mysql, "LOCK TABLES " TABLE_DIRS " WRITE");
	
	query = apr_psprintf(r->pool, "SELECT `id` FROM `" TABLE_DIRS "` WHERE `fullpath`='%s'", sqlstr);

	if (unlikely(!query) || unlikely(mysql_query(mysql, query)))
		goto error;
	
	mysql_res = mysql_store_result(mysql);
	
	if (mysql_num_rows(mysql_res))
		query = apr_psprintf(r->pool, "UPDATE `" TABLE_DIRS "` SET `timestamp`='%lu' "
				     "WHERE `fullpath`='%s'", timestamp, sqlstr);	
	else
		query = apr_psprintf(r->pool, "INSERT INTO `" TABLE_DIRS "` (timestamp, fullpath) VALUES ('%lu','%s')",
				     timestamp, sqlstr);
			
	if (unlikely(!query) || unlikely(mysql_query(mysql, query)))
		goto error;
			
	ret = CA_OK;

error:
	myerrno = mysql_errno(mysql);
	if (unlikely(myerrno))
		query = apr_pstrdup(r->pool, (char *)mysql_error(mysql)); /* static buffer */
	
	mysql_query(mysql, "UNLOCK TABLES");
	
	mysql_free_result(mysql_res);
	
	if (unlikely(myerrno)) {
		if (AINC_OVFLERR == myerrno)
			mysql_cache_trunc_tables(r, mysql);
		else
			mi_rerror("An error occured: %s", query);
	}
	return ret;
}

/**
 * Removes cache subdirectories.
 *
 * This subroutine takes care of removing any given directory and
 * its content (recursively) if any, and if possible.
 *
 * @warning Asserts we're already connected to the db.
 *
 * @param r Apache request_rec struct to handle log writings and pool allocation.
 * @param curdir A string representing the absolute path of the corresponding
 *	parent directory on the "original" filesystem.
 * @param mysql mysql handler to communicate with the server.
 *
 * @return CA_OK on succes, CA_FATAL otherwise.
 */
static int
mysql_cache_remove_dir(request_rec *r, const char *const curdir, MYSQL *mysql)
{
	char	*restrict sqlstr, *restrict query = NULL;
	int	ret = CA_FATAL;
	
	if (!(sqlstr = apr_palloc(r->pool, 2*strlen(curdir)+1)))
		goto error;

	mysql_real_escape_string(mysql, sqlstr, curdir, strlen(curdir));

	query = apr_psprintf(r->pool, "DELETE FROM `" TABLE_FILES "` WHERE `pid` IN (SELECT id FROM "
			TABLE_DIRS " WHERE `fullpath` LIKE '%s%%')", sqlstr);
	
	mysql_query(mysql, "LOCK TABLES " TABLE_DIRS " WRITE, " TABLE_FILES " WRITE");
	
	if (mysql_query(mysql, query))
		goto error;
	
	/* If we don't trash the subtree we'll keep junk in the cache */
	query = apr_psprintf(r->pool, "DELETE FROM `" TABLE_DIRS "` WHERE `fullpath` LIKE '%s%%'", sqlstr);

	if (unlikely(!query) || unlikely(mysql_query(mysql, query)))
		goto error;

	ret = CA_OK;
	
error:
	if (unlikely(mysql_errno(mysql)))
		mi_rerror("An error occured: %s", mysql_error(mysql));
	mysql_query(mysql, "UNLOCK TABLES");
	return ret;
}

/**
 * Checks if a directory already exists in the cache.
 *
 * This function is called when the caller wants to know whether a given
 * directory (as found in the @a path) exists or not in the cache.
 * If the directory exists, the function returns successfully if the cache is
 * still valid, otherwise it checks whether the cache was not ready or stale
 * and returns accordingly.
 *
 * @warning Asserts we're already connected to the db.
 * @warning Asserts path won't contain trailing slashes.
 *
 * @param r Apache request_rec struct to handle log writings.
 * @param path A string containing the full directory path, without trailing '/'.
 * @param mysql mysql handler to communicate with the server.
 *
 * @return CA_OK if dir exists and is up to date,
 *	CA_CREATE if dir doesnt't exist,
 *	CA_STALE if dir exists but is stale data,
 *	CA_NOTREADY if dir exists but doesn't contain any data,
 *	CA_FATAL otherwise.
 */
static int
mysql_cache_check_dir(request_rec *r, const char *const path, MYSQL *mysql)
{
	MYSQL_RES	*mysql_res = NULL;
	MYSQL_ROW	mysql_row;
	int		ret = CA_FATAL;
	char		*restrict sqlstr = NULL, *restrict query = NULL;
	struct stat	dirstat;

	if (unlikely(!path))
		return ret;

	if (unlikely(!(sqlstr = malloc(2*strlen(path)+1))))
		goto error;
	
	mysql_real_escape_string(mysql, sqlstr, path, strlen(path));

	/* XXX MEM OPTIM: asprintf() */
	query = apr_psprintf(r->pool, "SELECT timestamp FROM `" TABLE_DIRS "` WHERE `fullpath`='%s'", sqlstr);
	free(sqlstr);

	if (unlikely(!query) || unlikely(mysql_query(mysql, query)))
		goto error;

	mysql_res = mysql_store_result(mysql);
	
	if ((mysql_num_rows(mysql_res) == 0))
		ret = CA_CREATE;
	else {
		/* Checking for cache sanity. */
		unsigned long	test;
		mysql_row = mysql_fetch_row(mysql_res);
		
		if (unlikely(!mysql_row))
			goto error;

		ret = CA_OK;
		
		stat(path, &dirstat);
				
		test = (unsigned)atol(mysql_row[0]);
		if (test < dirstat.st_mtime) {
			if (0 == test)
				ret = CA_NOTREADY;
			else
				ret = CA_STALE;
		}
	}

error:
	mysql_free_result(mysql_res);

	if (unlikely(mysql_errno(mysql)))
		mi_rerror("An error occured: %s", mysql_error(mysql));
	return ret;
}

/**
 * Fills in the information fields about a music file from the cache.
 *
 * This function reads the tags from the cache db
 * and fills in the struct mu_ent fields accordingly.
 *
 * It doesn't need to check for mods on the original, since this is handled at
 * the directory level (see #mysql_cache_check_dir() and #mysql_cache_dircontents()).
 *
 * @warning @b MUST close in file handler on success.
 *
 * @param r Apache request_rec struct to handle log writings.
 * @param pool Apache pool
 * @param in Not used (except for closing).
 * @param filename current filename.
 *
 * @return When possible, struct mu_ent correctly set up, file stream closed,
 * 		NULL otherwise.
 */
static mu_ent*
mysql_cache_make_entry(request_rec *r, apr_pool_t *pool, FILE *const in,
	const char *const filename)
{
	const mu_config *const conf = (mu_config *)ap_get_module_config(r->per_dir_config, &musicindex_module);
	MYSQL		*mysql = conf->cache_setup;
	MYSQL_RES	*mysql_res = NULL;
	MYSQL_ROW	mysql_row;
	mu_ent		*p = NULL;
	char		* restrict sqldirn = NULL, * restrict sqlbasen = NULL, * restrict query = NULL;
	char		*dirs = NULL, *bases = NULL, *dirn = NULL, *basen = NULL;
	struct stat	statbuf;

	if (unlikely((!mysql) || (!mysql_params.ready)))
		return p;
	
	if (unlikely(stat(filename, &statbuf)))
		return p;

	dirs = strdup(filename);
	bases = strdup(filename);
	if (unlikely(!dirs || !bases))
		goto exit;

	dirn = dirname(dirs);
	basen = basename(bases);
	
	sqldirn = malloc(2*strlen(dirn)+1);
	sqlbasen = malloc(2*strlen(basen)+1);
	if (unlikely(!sqldirn || !sqlbasen))
		goto exit;

	mysql_real_escape_string(mysql, sqldirn, dirn, strlen(dirn));
	mysql_real_escape_string(mysql, sqlbasen, basen, strlen(basen));

	/* XXX MEM OPTIM: asprintf() */
	query = apr_psprintf(pool, "SELECT " SQL_CDATAF " FROM `" TABLE_FILES
			     "` WHERE `pid`=(SELECT `id` from `" TABLE_DIRS "` WHERE `fullpath`='%s')"
			     "AND `filename`='%s' COLLATE utf8_bin",
			     sqldirn, sqlbasen);
	
	if (unlikely(!query) || unlikely(mysql_query(mysql, query)))
		goto error;

	mysql_res = mysql_store_result(mysql);
	
	/* Actually check for the file in the cache. */
	if ((mysql_num_rows(mysql_res) == 0))
		goto exit;	/* creation of the cache entry is handled separately */
	
	/* XXX we don't check whether there's more than 1 result. That *outta* be impossible */
	mysql_row = mysql_fetch_row(mysql_res);

	if (unlikely(!mysql_row))
		goto error;

	/* Check whether the cache is somehow invalid */
	if ((unlikely(((unsigned short)atoi(mysql_row[0]) < CACHE_VERS))) ||
	    (unlikely((unsigned)atol(mysql_row[1]) < statbuf.st_mtime))    )
		goto exit;

	p = mysql_cache_new_ent(pool, mysql_row);
	if (likely(p))
		fclose(in);	/* mandatory */

error:
	if (unlikely(mysql_errno(mysql)))
		mi_rerror("An error occured: %s", mysql_error(mysql));
exit:
	mysql_free_result(mysql_res);
	free(dirs), free(bases);
	free(sqldirn), free(sqlbasen);
	return p;
}

/**
 * Creates and writes cache information.
 *
 * This function creates a new cache entry for (using #filename), and
 * fills it with the data contained in the mu_ent p structure.
 *
 * @bug we don't check enough return values of apr_psprintf()/apr_pstrcat()...
 *
 * @param r Apache request_rec struct to handle log writings and pool allocation.
 * @param p A mu_ent struct containing the data to store.
 * @param filename current filename of the file to cache.
 */
static void
mysql_cache_write(request_rec *r, const mu_ent *const p, const char *const filename)
{
	const mu_config *const conf = (mu_config *)ap_get_module_config(r->per_dir_config, &musicindex_module);
	MYSQL		*mysql = conf->cache_setup;
	MYSQL_RES	*mysql_res = NULL;
	char		* restrict sqldirn = NULL, * restrict sqlbasen = NULL, * restrict query = NULL;
	char		* restrict sqlalb = NULL, * restrict sqlart = NULL, * restrict sqltit = NULL, * restrict sqlgen = NULL;
	char		*dirs = NULL, *bases = NULL, *dirn = NULL, *basen = NULL;
	unsigned int	myerrno = 0;
	
	if (unlikely((!mysql) || (!mysql_params.ready)))
		return;

	/* if we have a directory, it could be a child dir in a given directory listing
	 * we need to record it so that mysql_cache_dircontents() works properly, but
	 * we also need to differentiate from normally created dirs by making sure that
	 * mysql_cache_dircontents() will not try to return its content later on. We
	 * thus "invalidate" this cache entry by giving it a 0 timestamp */
	if (p->filetype < 0) {
		int check = mysql_cache_check_dir(r, filename, mysql);
		if (CA_CREATE == check)
			mysql_cache_make_dir(r, filename, 0, mysql);
		goto exit;
	}
	
	dirs = strdup(filename);
	bases = strdup(filename);
	if (unlikely(!dirs || !bases))
		goto exit;
	
	dirn = dirname(dirs);
	basen = basename(bases);
	
	sqldirn = malloc(2*strlen(dirn)+1);
	sqlbasen = malloc(2*strlen(basen)+1);
	if (unlikely(!sqldirn || !sqlbasen))
		goto exit;

	mysql_real_escape_string(mysql, sqldirn, dirn, strlen(dirn));
	mysql_real_escape_string(mysql, sqlbasen, basen, strlen(basen));
	
	/* Prepare the strings properly */
	if (p->album) {
		if (unlikely(!(sqlalb = malloc(2*strlen(p->album)+1))))
			goto exit;
		mysql_real_escape_string(mysql, sqlalb, p->album, strlen(p->album));
	}
	if (p->artist) {
		if (unlikely(!(sqlart = malloc(2*strlen(p->artist)+1))))
			goto exit;
		mysql_real_escape_string(mysql, sqlart, p->artist, strlen(p->artist));
	}
	if (p->genre) {
		if (unlikely(!(sqlgen = malloc(2*strlen(p->genre)+1))))
			goto exit;
		mysql_real_escape_string(mysql, sqlgen, p->genre, strlen(p->genre));
	}
	if (unlikely(!(sqltit = malloc(2*strlen(p->title)+1))))
		goto exit;
	mysql_real_escape_string(mysql, sqltit, p->title, strlen(p->title));

	/* We need to lock here because the rest of the logic relies on the results
	 * of this first SELECT */
	mysql_query(mysql, "LOCK TABLES " TABLE_FILES " WRITE, " TABLE_DIRS " READ");
	
	query = apr_psprintf(r->pool, "SELECT `id` FROM `" TABLE_DIRS "` WHERE `fullpath`='%s'", sqldirn);
	
	if (unlikely(!query) || unlikely(mysql_query(mysql, query)))
		goto error;
	
	mysql_res = mysql_store_result(mysql);
	
	/* if the cache directory doesn't exist that means either we haven't called cache_check_dir
	 first or it's been trashed since then. Have to get out of here. */
	if (unlikely(mysql_num_rows(mysql_res) == 0))
		goto error;
	
	mysql_free_result(mysql_res);
	mysql_res = NULL;
	
	query = apr_psprintf(r->pool, "SELECT `id` FROM `" TABLE_FILES "` WHERE `filename`='%s' COLLATE utf8_bin "
			     "AND pid=(SELECT `id` FROM `" TABLE_DIRS "` WHERE `fullpath`='%s')",
			     sqlbasen, sqldirn);

	if (unlikely(!query) || unlikely(mysql_query(mysql, query)))
		goto error;
	
	mysql_res = mysql_store_result(mysql);

	if (unlikely(mysql_num_rows(mysql_res))) {
		/* This update might happen if dircontents() found an out of date entry */
		/* XXX there shouldn't be more than 1 row */
		MYSQL_ROW	mysql_row;
		
		mysql_row = mysql_fetch_row(mysql_res);
		
		query = apr_psprintf(r->pool, "UPDATE `" TABLE_FILES
			"` SET cvers='%hi',filetype='%hu',flags='%hu',track='%hu',"
			"posn='%hu',date='%hu',freq='%hu',length='%hu',"
			"bitrate='%lu',size='%lu',mtime='%lu',title='%s'",
			CACHE_VERS, p->filetype, (p->flags & EF_FLAGSTOSAVE), p->track, p->posn, p->date,
			p->freq, p->length, p->bitrate, p->size, p->mtime, sqltit);
		
		query = apr_pstrcat(r->pool, query,	/* XXX no alloc error checking on psprintf() */
				sqlalb ? apr_psprintf(r->pool, ",album='%s'", sqlalb) : ",album=NULL",
				sqlart ? apr_psprintf(r->pool, ",artist='%s'", sqlart) : ",artist=NULL",
				sqlgen ? apr_psprintf(r->pool, ",genre='%s'", sqlgen) : ",genre=NULL",
				apr_psprintf(r->pool, " WHERE id='%lu'", (unsigned long)atol(mysql_row[0])), NULL);
	}
	else {
		query = apr_psprintf(r->pool, "INSERT INTO `" TABLE_FILES
			"` (`pid`,`cvers`,`filetype`,`flags`,`track`,`posn`,`date`,`freq`,"
			"`length`,`bitrate`,`size`,`mtime`,`title`,`filename`,`album`,`artist`,"
			"`genre`) VALUES "
			"((SELECT `id` FROM `" TABLE_DIRS "` WHERE `fullpath`='%s'),"
			"'%hi','%hu','%hu','%hu','%hu','%hu','%hu','%hu',"
			"'%lu','%lu','%lu','%s','%s',",
			sqldirn, CACHE_VERS, p->filetype, (p->flags & EF_FLAGSTOSAVE), p->track, p->posn, p->date,
			p->freq, p->length, p->bitrate, p->size, p->mtime, sqltit, sqlbasen);
		query = apr_pstrcat(r->pool, query,
				sqlalb ? apr_psprintf(r->pool, "'%s',", sqlalb) : "NULL,", /* XXX no alloc error checking on psprintf() */
				sqlart ? apr_psprintf(r->pool, "'%s',", sqlart) : "NULL,",
				sqlgen ? apr_psprintf(r->pool, "'%s'", sqlgen) : "NULL",
				")", NULL);
	}

	/* submit the query */
	if (likely(query))
		mysql_query(mysql, query);

error:
	myerrno = mysql_errno(mysql);
	if (unlikely(myerrno)) {
		query = apr_pstrdup(r->pool, (char *)mysql_error(mysql)); /* have to dup as mysql_error() sends a static buffer */
		mysql_cache_make_dir(r, dirn, 0, mysql); /* the query failed we must invalidate the directory. XXX no fail check */
	}
	
	mysql_query(mysql, "UNLOCK TABLES");
	
	if (unlikely(myerrno)) {
		if (AINC_OVFLERR == myerrno)
			mysql_cache_trunc_tables(r, mysql);
		else
			mi_rerror("An error occured: %s", query);
	}
exit:
	mysql_free_result(mysql_res);
	free(dirs), free(bases);
	free(sqldirn), free(sqlbasen);
	free(sqlalb), free(sqlart), free(sqlgen), free(sqltit);
	return;
}

/** hand crafted "dirent" applied to our mysql data structure */
typedef struct mysql_dir {
	char *d_name;
	const struct mysql_dir *next;
} mysql_dir;

/**
 * Fetch the content of a whole directory in the cache.
 *
 * This function creates a new list of entries from a cached directory.
 * 
 * It works as follows:
 *	- 1. select all immediate subdirs (if any) in that dir (using regexp on
 *	     TABLE_DIRS) and add them to the queue of files that need more processing.
 *	- 2. select all songs in that dir, new_ent() them if valid or a) trash the
 *	     cache directory if they're gone or b) requeue them if they've changed.
 *
 * @warning Asserts we're already connected to the db.
 * @warning Asserts filename doesn't contain trailing /.
 *
 * @param r Apache request_rec struct to handle log writings.
 * @param pool Pool allocation.
 * @param pack A pack to add new entries to.
 * @param filename current file path.
 * @param uri current server-side uri.
 * @param mysql mysql handler to talk to the server.
 * @param soptions Flags to use for created entries.
 *
 * @return NULL or a chained list of entries needing more processing from the core system
 *	(typically, directories if any will be added to that list for further processing/recursion).
 */
static mysql_dir
*mysql_cache_dircontents(request_rec *r, apr_pool_t *pool, mu_pack *const pack,
	const char *const filename, const char *const uri, MYSQL *mysql, unsigned long soptions)
{
	const mu_config *const conf = (mu_config *)ap_get_module_config(r->per_dir_config, &musicindex_module);
	MYSQL_RES	*mysql_res;
	MYSQL_ROW	mysql_row;
	mu_ent		*p = NULL;
	const mu_ent	*prev = pack->head;
	mysql_dir	*mhead = NULL, *mdir = NULL, *ret = NULL;
	char		*restrict sqlstr = NULL, *restrict query = NULL;
	char		*restrict basen = NULL, *restrict fullpath = NULL;
	unsigned long	nb = 0, size = 0;
	const size_t	fnlen = strlen(filename);
	
	/* 1. we need to treat *only* the immediate subdirs, as we can't assert that treating all
	subdirs is idempotent (eg: won't duplicate work with the rest of make_music_entry()).
	To the contrary, we would have duplicate entries if we sent all subdirs */

	sqlstr = malloc(2*fnlen+1);
	if (unlikely(!sqlstr))
		goto error;

	mysql_real_escape_string(mysql, sqlstr, filename, fnlen);

	/* this request should hopefully return only immediate subdirs,
	as it will only match entries not containing any '/' after the current
	path. Indeed, we always strip trailing '/' in mysql_cache_check_dir().
	XXX correct modifier for size_t is 'z' but seems (some versions of?)
	apache doesn't know about it. */
	/* XXX MEM OPTIM: asprintf() */
	query = apr_psprintf(pool, "SELECT fullpath FROM " TABLE_DIRS " WHERE "
				"`fullpath` LIKE '%s/%%' AND (LOCATE('/', fullpath, %lu)=0)",
						 sqlstr, strlen(sqlstr)+2);

	if (unlikely(!query) || unlikely(mysql_query(mysql, query)))
		goto error;

	mysql_res = mysql_use_result(mysql);
	
	while ((mysql_row = mysql_fetch_row(mysql_res))) {
		basen = apr_pstrdup(pool, mysql_row[0]+(fnlen+1));
		/* simulate basename by skipping the prefix path */
		mdir = (mysql_dir *)apr_pcalloc(pool, sizeof(mysql_dir));
		if (likely(basen && mdir)) {
			mdir->d_name = basen;
			mdir->next = mhead;
			mhead = mdir;
		}
	}

	mysql_free_result(mysql_res);
	mysql_res = NULL;
	/* 1. done */

	/* 2. */
	/* we're reusing the previous content of sqlstr */
	/* XXX MEM OPTIM: asprintf() */
	query = apr_psprintf(pool, "SELECT " SQL_CDATAF ",filename FROM `" TABLE_FILES
			"` WHERE `pid`="
			"(SELECT `id` from `" TABLE_DIRS "` WHERE `fullpath`='%s')",
			sqlstr);

	/* attention, MATCH has a builtin list of stopwords, see
	 * http://dev.mysql.com/doc/refman/4.1/en/fulltext-natural-language.html */
	if (unlikely(conf->search)) {
		char * restrict escsearch = apr_palloc(pool, 2*strlen(conf->search)+1);
			if (!escsearch)
				goto error;
		mysql_real_escape_string(mysql, escsearch, conf->search, strlen(conf->search));
		query = apr_pstrcat(pool, query, " AND MATCH(`filename`,`title`,`artist`,`album`) AGAINST ('",
							escsearch, "')", NULL);
	}
	
	if (unlikely(!query) || unlikely(mysql_query(mysql, query)))
		goto error;

	mysql_res = mysql_use_result(mysql);
	
	/* filename + '/' + '\0' */
	fullpath = malloc(fnlen + 2);
	if (!fullpath) {
		ret = NULL;
		goto error;
	}
	strncpy(fullpath, filename, fnlen);
	fullpath[fnlen] = '/';
	fullpath[fnlen+1] = '\0';
	
	while ((mysql_row = mysql_fetch_row(mysql_res))) {
		/* Check whether the cache is somehow invalid, if so, requeue the file */
		if (unlikely(((unsigned)atoi(mysql_row[0]) < CACHE_VERS))) {
			/* XXX MEM OPTIM: malloc here, free in readdir() */
			basen = apr_pstrdup(pool, mysql_row[SQL_CDATAN]);
			mdir = (mysql_dir *)apr_pcalloc(pool, sizeof(mysql_dir));
			if (likely(basen && mdir)) {
				mdir->d_name = basen;
				mdir->next = mhead;
				mhead = mdir;
			}
		}
		else {
			p = mysql_cache_new_ent(pool, mysql_row);
			if (likely(p)) {
				struct stat statbuf;
				fullpath = realloc(fullpath, fnlen + 1 + strlen(mysql_row[SQL_CDATAN]) + 1);
				if (unlikely(!fullpath)) {
					ret = NULL;
					goto error;
				}
				fullpath[fnlen+1] = '\0';	/* ignore previous entry */
				strncat(fullpath, mysql_row[SQL_CDATAN], strlen(mysql_row[SQL_CDATAN]));
				
				/* Here we check 1) that the file still exists (stat will fail otherwise) and
				 * 2) that it hasn't been modified since it went into the cache
				 * If 1) is not verified, we trash the directory cache and exit
				 * If 2) is not verified, we requeue the file */
				if (unlikely(stat(fullpath, &statbuf))) {
					mi_rdebug("Trashing cache for: %s", filename);
					mysql_free_result(mysql_res);
					mysql_cache_remove_dir(r, filename, mysql);
					ret = NULL;
					goto error;
				}
				if (unlikely(p->mtime < statbuf.st_mtime)) {
					mi_rdebug("Requeuing: %s", fullpath);
					basen = apr_pstrdup(pool, mysql_row[SQL_CDATAN]);
					mdir = (mysql_dir *)apr_pcalloc(pool, sizeof(mysql_dir));
					if (likely(basen && mdir)) {
						mdir->d_name = basen;
						mdir->next = mhead;
						mhead = mdir;
					}
					continue;
				}
				
				p->uri = apr_pstrcat(pool, uri, mysql_row[SQL_CDATAN], NULL);
				p->filename = p->file = p->uri;
				p->filename = (strrchr(p->filename, '/') + 1);
				P_FLAGS_OPTIONS(p, soptions);
				if ((soptions & MI_CUSTOM) == 0)
					p->file += strlen(r->parsed_uri.path);  /* offset the path before the filename relative to request current dir, except for custom playlists */				
				if (conf->options & MI_TARBALL)
					p->filename = apr_pstrdup(pool, fullpath);	/* tarball needs full path for access */
				p->flags |= EF_INCACHE;
				nb++;
				size += p->size;
				p->next = prev;
				prev = p;
			}
		}
	}

	mysql_free_result(mysql_res);
	/* 2. done */

	if (likely(p)) {
		pack->filenb += nb;
		pack->fsize += size;
		pack->head = p;
	}
	
	ret = (mysql_dir *)apr_pcalloc(pool, sizeof(mysql_dir));
	if (unlikely(!ret))
		goto error;
	ret->d_name = NULL;
	ret->next = mhead;

error:
	if (unlikely(mysql_errno(mysql)))
		mi_rerror("An error occured: %s", mysql_error(mysql));
	free(sqlstr), free(fullpath);
	return ret;
}

/**
 * A nifty wrapper
 *
 * Under the pretense of opening a given directory, actually check that this directory
 * exists in the cache. If not, create it, or update it if needed.
 * Cache up-to-dateness management is done here, as directory creation can happen
 * elsewhere in the code (in particular in mysql_cache_make_entry()).
 *
 * @warning Asserts @a filename won't contain trailing /.
 * @todo uses MI_QUICKPL, API should be fixed.
 *
 * @param r Apache request_rec struct to handle log writings and pool allocation.
 * @param pack A pack to add new entries to.
 * @param filename current file path.
 * @param uri current server-side uri.
 * @param soptions Flags to use for created entries.
 *
 * @return the chained list from mysql_cache_dircontents(), if any, NULL otherwise.
 */
static void
*mysql_cache_opendir(request_rec *r, mu_pack *const pack, const char * const filename,
			const char * const uri, unsigned long soptions)
{
	const mu_config *const conf = (mu_config *)ap_get_module_config(r->per_dir_config, &musicindex_module);
	MYSQL		*mysql = conf->cache_setup;
	int		val;
	mysql_dir	*mdir = NULL;
	void		*ret = NULL;
	struct stat	dirstat;

	if (unlikely((!mysql) || (!mysql_params.ready))) {
		mi_rdebug("FATAL: Entered with mysql=%x, mysql_params.ready=%d", mysql, mysql_params.ready);
		goto error;
	}

	val = mysql_cache_check_dir(r, filename, mysql);
	switch (val) {
		case CA_OK:
			break;
		case CA_STALE:
			mysql_cache_remove_dir(r, filename, mysql);
		case CA_CREATE:
		case CA_NOTREADY:
			if (likely(!(conf->options & MI_QUICKPL))) {
				/* XXX HACK. This is a gross workaround until the interface
				 * is better specified: we don't want to record a directory
				 * in the cache as valid when we're not going to record any
				 * content for it, which happens when MI_QUICKPL is on */
				stat(filename, &dirstat);
				mysql_cache_make_dir(r,	filename, dirstat.st_mtime, mysql);
			}
		case CA_FATAL:
		default:
			goto error;
	}
	
	mdir = mysql_cache_dircontents(r, r->pool, pack, filename, uri, mysql, soptions);
	
	ret = (void *)mdir;	/* NULL if mysql_cache_dircontents() failed, which is what we want */

error:
	return ret;
}

/**
 * Returns directory contents to be processed in make_music_entry() a la readdir().
 *
 * @param dir pointer to the head of the chainlist of elements
 * @return filename
 */
static const char
*mysql_cache_readdir(void *dir)
{
	mysql_dir	*mdir;
	const char	*ret = NULL;

	if (unlikely(!dir))
		return NULL;
	
	mdir = (mysql_dir *)dir;

	/* the following is a silly workaround the fact that dir is not void** */
	if (likely(mdir->next)) {
		ret = mdir->next->d_name;
		mdir->next = mdir->next->next;
	}
	
	return ret;
}

/**
 * Initializes the database connection.
 *
 * We really only want one database connection max per request.
 *
 * @param r Apache request_rec struct to handle log writings and pool allocation.
 */
static void
mysql_cache_prologue(request_rec *r)
{
	mu_config *const conf = (mu_config *)ap_get_module_config(r->per_dir_config, &musicindex_module);
	MYSQL	*mysql = NULL;

	if (unlikely(!mysql_params.ready))
		goto out;

	mysql = mysql_init(NULL);
	if (unlikely(!mysql))
		goto out;

	if (unlikely(!mysql_real_connect(mysql, mysql_params.host, mysql_params.user,
					 mysql_params.pass, mysql_params.db, 0, NULL, 0))) {
		mysql_close(mysql);
		goto out;
	}

	if (unlikely(mysql_set_character_set(mysql, "utf8"))) {
		mysql_close(mysql);
		goto out;
	}

	conf->cache_setup = mysql;
out:
	return;
}

/**
 * Closes the database connection.
 *
 * This is the counter part to the prologue(): we need to properly shutdown the
 * database connection that we established there.
 *
 * @param r Apache request_rec struct to handle log writings and pool allocation.
 */
static void
mysql_cache_epilogue(request_rec *r)
{
	mu_config *const conf = (mu_config *)ap_get_module_config(r->per_dir_config, &musicindex_module);
	MYSQL	*mysql = (MYSQL *)conf->cache_setup;
	
	if (unlikely(!mysql))
		goto out;

	mysql_close(mysql);
	conf->cache_setup = NULL;	
out:
	return;
}

static const cache_backend cache_backend_mysql = {
	.opendir =	mysql_cache_opendir,
	.readdir =	mysql_cache_readdir,
	.closedir =	NULL,
	.make_entry =	mysql_cache_make_entry,
	.write =	mysql_cache_write,
	.prologue =	mysql_cache_prologue,
	.epilogue =	mysql_cache_epilogue,
};

int cache_mysql_setup(cmd_parms *cmd, const char *const setup_string, mu_config *const conf)
{
	static const char biniou[] = "mysql://";
	int ret = 1;

	if (strncmp(biniou, setup_string, 8) == 0) {
		ret = mysql_cache_init(cmd->server, setup_string+8);
		if (ret)
			goto exit;
		
		conf->cache_setup = NULL;	/* this is prologue()'s baby */
		conf->cache = &cache_backend_mysql;
	}

exit:
	return ret;
}
