/*++++++++++++++++++++
  refdbdupdb.c: refdbd code to update main databases
  markus@mhoenicka.de 2006-09-06

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.
   
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
   
   You should have received a copy of the GNU General Public License
   along with this program; if not, see <http://www.gnu.org/licenses/>

   +++++++++++++++++++++++++*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <syslog.h>
#include <dbi/dbi.h>

#include "strfncs.h"
#include "linklist.h"
#include "connect.h"
#include "refdb.h"
#include "refdbd.h"
#include "refdbdupdb.h"
#include "dbfncs.h"

/* global variables */
extern int n_log_level;
extern char refdblib[];
extern char main_db[];
extern dbi_inst Inst;

/* prototypes of local functions */
static short int dispatch_update_main_db(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, short int current_version, short int target_version);
static short int update_mysql_main_db(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, short int current_version, short int target_version);
static int mysql_duplicate_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, Liliform* ptr_sentinel);
static int execute_sql_mysql(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, const char* sqlfile);
static int mysql_merge_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, Liliform* ptr_sentinel, short int target_version);
static int install_mysql_main_db(struct CLIENT_REQUEST* ptr_clrequest);
static short int update_pgsql_main_db(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, short int current_version, short int target_version);
static int pgsql_duplicate_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, Liliform* ptr_sentinel);
static int execute_sql_pgsql(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, const char* sqlfile, int is_create);
static int pgsql_merge_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, Liliform* ptr_sentinel, short int target_version);
static int install_pgsql_main_db(struct CLIENT_REQUEST* ptr_clrequest);
static int set_main_db_permissions_pgsql(dbi_conn conn);
static short int update_sqlite_main_db(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, short int current_version, short int target_version);
static int sqlite_duplicate_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn);
static int execute_sql_sqlite(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, const char* sqlfile);
static int sqlite_merge_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, short int target_version);
static int install_sqlite_main_db(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn);
static char* assemble_column_list(const char* create_statement, short int target_version);
static char* read_sql_from_file(const char* path);
static int read_sql_from_file_to_list(const char* path, Liliform* ptr_sentinel);


/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  update_main_db(): updates the main database to the current version

  int update_main_db returns 0 if ok, >0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  int n_updatedb if 1, update main database; if 0, just report version

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int update_main_db(struct CLIENT_REQUEST* ptr_clrequest, int n_updatedb) {
  dbi_conn conn; /* libdbi connection structure */
  dbi_result dbires; /* libdbi result structure */
  char sql_command[] = "SELECT meta_dbversion FROM "MAIN_META" WHERE meta_type='refdb'";
  char msg_buf[256] = "";
  short int n_main_dbversion;
  int retval = 0;

  LOG_PRINT(LOG_INFO, "check main database version");

  if ((conn = connect_to_db(ptr_clrequest, main_db, 0)) != NULL) {
    /* check version tag of main database */
    dbires = dbi_conn_query(conn, sql_command);
    if (!dbires || !dbi_result_next_row(dbires)) {
      if (!strcmp(ptr_clrequest->dbserver, "sqlite")
	  /* a missing sqlite main db will throw us here as the
	     connection will always succeed by creating an empty
	     database (as long as we have write permissions, that
	     is). Proceed by installing the main database */
	  || !strcmp(ptr_clrequest->dbserver, "sqlite3")) {
	return install_sqlite_main_db(ptr_clrequest, conn);
      }
      else {
	/* too old (there is no t_meta) or corrupt */
	LOG_PRINT(LOG_ERR, get_status_msg(203));
	return 1;
      }
    }
      
    n_main_dbversion = dbi_result_get_short(dbires, "meta_dbversion");
		
    if (dbi_conn_error(conn, NULL)) {
      /* corrupt */
      LOG_PRINT(LOG_ERR, get_status_msg(203));
      dbi_result_free(dbires);
      return 1;
    }

    dbi_result_free(dbires);

    sprintf(msg_buf, "current main database version: %d", n_main_dbversion);
    LOG_PRINT(LOG_INFO, msg_buf);

    /* we're done if we shall only check the connection. Else, see
       whether the main database is in need of an upgrade */
    if (n_updatedb) {
      /* this loop runs until we're at the current version, or until
	 an error occurs. The dispatch function may choose to upgrade
	 older databases in more than one step. */
      while (n_main_dbversion < MAIN_DB_VERSION) {
	retval = dispatch_update_main_db(ptr_clrequest, conn, n_main_dbversion, MAIN_DB_VERSION);

	if (!retval) { /* error */
	  dbi_conn_close(conn);
	  return 1;
	}
	else {
	  n_main_dbversion = retval;
	}
      }
    }
    dbi_conn_close(conn);
  }
  else {
    /* can't access main database. This may be due to incorrect
       permissions on the database server side, or the database is
       indeed missing */
    LOG_PRINT(LOG_WARNING, get_status_msg(202));

    if (!n_updatedb) {
      return 1;
    }

    /* attempt to create the database */
    if (!strcmp(ptr_clrequest->dbserver, "mysql")) {
      retval = install_mysql_main_db(ptr_clrequest);
    }
    else if (!strcmp(ptr_clrequest->dbserver, "pgsql")) {
      retval = install_pgsql_main_db(ptr_clrequest);
    }
    else {
      LOG_PRINT(LOG_ERR, "Can't access or create SQLite database");
      retval = 1;
    }

    return retval;
    
  }

  return 0;
}


/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  dispatch_update_main_db(): sets the upgrade strategy

  short int dispatch_update_main_db returns new database version if ok,
                          or 0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  short int n_current_version current database version

  short int n_target_version target version

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static short int dispatch_update_main_db(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, short int current_version, short int target_version) {
  short int retval = 0;

  if (target_version == 2) {
    LOG_PRINT(LOG_INFO, "updating main database to version 2");

    /* the upgrade strategies differ between the supported databases */
    if (!strcmp(ptr_clrequest->dbserver, "mysql")) {
      retval = update_mysql_main_db(ptr_clrequest, conn, 1, 2);
    }
    else if (!strcmp(ptr_clrequest->dbserver, "pgsql")) {
      retval = update_pgsql_main_db(ptr_clrequest, conn, 1, 2);
    }
    else if (!strcmp(ptr_clrequest->dbserver, "sqlite")
	     || !strcmp(ptr_clrequest->dbserver, "sqlite3")) {
      retval = update_sqlite_main_db(ptr_clrequest, conn, 1, 2);
    }

    return retval;
  }
  else if (target_version == 3) {
    LOG_PRINT(LOG_INFO, "updating main database to version 3");

    /* the upgrade strategies differ between the supported databases */
    if (!strcmp(ptr_clrequest->dbserver, "mysql")) {
      retval = update_mysql_main_db(ptr_clrequest, conn, 2, 3);
    }
    else if (!strcmp(ptr_clrequest->dbserver, "pgsql")) {
      retval = update_pgsql_main_db(ptr_clrequest, conn, 2, 3);
    }
    else if (!strcmp(ptr_clrequest->dbserver, "sqlite")
	     || !strcmp(ptr_clrequest->dbserver, "sqlite3")) {
      retval = update_sqlite_main_db(ptr_clrequest, conn, 2, 3);
    }

    return retval;
  }
  /* else if: add future cases here */
  else {
    /* upgrade not supported */
    return 0;
  }
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  update_mysql_main_db(): updates the main database (mysql)

  short int update_mysql_main_db returns new database version if ok,
                          or 0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  short int n_current_version current database version

  short int n_target_version target version

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static short int update_mysql_main_db(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, short int current_version, short int target_version) {
  char path[_POSIX_PATH_MAX+1] = "";
  Liliform sentinel;

  /* assemble path to sql script */
  snprintf(path, _POSIX_PATH_MAX, "%s/sql/refdb.%d.dump.mysql41", refdblib, target_version);

  /* initialize linked list sentinel */
  sentinel.ptr_next = NULL;
  sentinel.name[0] = '\0';
  sentinel.value = NULL;

  /* start a transaction */
  if (my_dbi_conn_begin(conn)) {
    return 0;
  }

  /* first replace existing tables with temporary copies */
  if (mysql_duplicate_tables(ptr_clrequest, conn, &sentinel)) {
    my_dbi_conn_rollback(conn);
    delete_all_liliform(&sentinel);
    return 0;
  }

  /* create new tables */
  if (execute_sql_mysql(ptr_clrequest, conn, path)) {
    my_dbi_conn_rollback(conn);
    return 0;
  }

  /* finally write the data back into the new tables */
  if (mysql_merge_tables(ptr_clrequest, conn, &sentinel, target_version)) {
    my_dbi_conn_rollback(conn);
    return 0;
  }

  my_dbi_conn_commit(conn);


  return target_version;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  mysql_duplicate_tables(): duplicates the tables into temporary ones

  int mysql_duplicate_tables returns 0 if ok, or >0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  Liliform* ptr_sentinel ptr to linked list for table names

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int mysql_duplicate_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, Liliform* ptr_sentinel) {
  char sql_command[256];
  const char* table_name;
  dbi_result dbires;
  dbi_result dbires1;

  /* get a list of tables */
  dbires = dbi_conn_get_table_list(conn, "refdb", NULL);

  if (!dbires) {
    return 1;
  }

  /* loop over all existing tables */
  while (dbi_result_next_row(dbires)) {
    table_name = my_dbi_result_get_string_idx(dbires, 1);
    if (!table_name) {
      dbi_result_free(dbires);
      return 1;
    }

    /* create a temporary copy of the table */
    sprintf(sql_command, "CREATE TEMPORARY TABLE temp%s SELECT * from %s", table_name, table_name);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      dbi_result_free(dbires);
      return 1;
    }
    dbi_result_free(dbires1);

    /* add the table name to the linked list */
    if (insert_liliform(ptr_sentinel, "dummy", (char*)table_name)) {
      dbi_result_free(dbires);
      return 1;
    }

    /* drop the existing table */
    sprintf(sql_command, "DROP TABLE %s", table_name);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      dbi_result_free(dbires);
      return 1;
    }
    dbi_result_free(dbires1);
  }

  dbi_result_free(dbires);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  execute_sql_mysql(): runs an SQL script for MySQL

  int execute_sql_mysql returns 0 if ok, or >0 if an error occurred
                         The return value is the exit code of the
			 mysql command line application, or 1
			 if there was insufficient memory to assemble
			 the command string

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  const char* sqlfile path of the SQL script to execute

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int execute_sql_mysql(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, const char* sqlfile) {
  dbi_result dbires;
  Liliform sentinel;
  Liliform* ptr_curr;

  /* initialize linked list sentinel */
  sentinel.ptr_next = NULL;
  sentinel.name[0] = '\0';
  sentinel.value = NULL;

  /* read the sql script into a linked list of individual commands */
  if (read_sql_from_file_to_list(sqlfile, &sentinel)) {
    return 1;
  }

  ptr_curr = &sentinel;

  /* loop over all commands */
  while ((ptr_curr = get_next_liliform(ptr_curr)) != NULL) {
    if (ptr_curr->value && *(ptr_curr->value)) {
      LOG_PRINT(LOG_DEBUG, ptr_curr->value);

      dbires = dbi_conn_query(conn, ptr_curr->value);
      
      if (!dbires) {
	return 1;
      }
      dbi_result_free(dbires);
    }
  }

  /* clean up the linked list */
  delete_all_liliform(&sentinel);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  mysql_merge_tables(): merges the temporary and new tables. It is assumed
                         that the new table contains all (and maybe more)
			 columns of the temporary table

  int mysql_merge_tables returns 0 if ok, or >0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  Liliform* ptr_sentinel ptr to linked list for table names

  short int target_version target version of the update procedure

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int mysql_merge_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, Liliform* ptr_sentinel, short int target_version) {
  char column_buffer[128];
  char* sql_command;
  char* new_sql_command;
  char* column_list = NULL;
  const char* table_name;
  size_t sql_cmd_len = 512;
  Liliform* ptr_curr;
  dbi_result dbires1;


  /* get some memory */
  if ((sql_command = malloc(sql_cmd_len)) == NULL) {
    return 1;
  }

  ptr_curr = ptr_sentinel;

  /* loop over all existing tables. The linked list contains the
     original table names, not the temporary table names */
  while ((ptr_curr = get_next_liliform(ptr_curr)) != NULL) {
    table_name = ptr_curr->value;
    if (!table_name) {
      free(sql_command);
      return 1;
    }

    /* retrieve the column names */
    sprintf(sql_command, "SHOW CREATE TABLE temp%s", table_name);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      const char* errmsg;
      dbi_conn_error(conn, &errmsg);
      LOG_PRINT(LOG_WARNING, "retrieve column names failed");
      LOG_PRINT(LOG_DEBUG, errmsg);
      free(sql_command);
      return 1;
    }

    /* assemble a command which inserts the existing data into the
       columns that were present in the original table */
    sprintf(sql_command, "INSERT IGNORE INTO %s (", table_name);

    /* this should run only once */
    while (dbi_result_next_row(dbires1)) {
      char* create_command;

      create_command = my_dbi_result_get_string_copy_idx(dbires1, 2);

      if (!create_command) {
	free(sql_command);
	return 1;
      }

      /* convert the create command into a comma-separated list of
	 column names */
      column_list = assemble_column_list(create_command, target_version);

      free(create_command);

      if (!column_list || (new_sql_command = mstrcat(sql_command, column_list, &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, get_status_msg(801));
	free(column_list);
	free(sql_command);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
    }

    dbi_result_free(dbires1);

    if (!column_list) {
      return 1;
    }

    if ((new_sql_command = mstrcat(sql_command, ") SELECT ", &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      free(sql_command);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    if ((new_sql_command = mstrcat(sql_command, column_list, &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      free(sql_command);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    free(column_list);

    sprintf(column_buffer, " FROM temp%s", table_name);

    if ((new_sql_command = mstrcat(sql_command, column_buffer, &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      free(sql_command);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      const char* errmsg;
      dbi_conn_error(conn, &errmsg);
      LOG_PRINT(LOG_WARNING, "INSERT into main database failed");
      LOG_PRINT(LOG_DEBUG, errmsg);
      free(sql_command);
      return 1;
    }
    dbi_result_free(dbires1);
  }

  free(sql_command);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  install_mysql_main_db(): installs the main database (mysql)

  int install_mysql_main_db returns 0 if ok,
                          or >0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int install_mysql_main_db(struct CLIENT_REQUEST* ptr_clrequest) {
  unsigned int server_version;
  char path[_POSIX_PATH_MAX+1] = "";
  char sql_command[128];
  dbi_conn conn;
  dbi_result dbires;

  /* get a connection to the MySQL system database */
  if ((conn = my_dbi_conn_new("mysql", ptr_clrequest->inst)) == NULL) {
    LOG_PRINT(LOG_ERR, "Cannot connect to database server. Installation of main database aborted");
    return 1;
  }

  LOG_PRINT(LOG_DEBUG, ptr_clrequest->server_ip);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->username);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->passwd);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->current_db);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->dbs_port_address);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->dbserver);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->db_path);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->db_encoding);

  dbi_conn_set_option(conn, "username", ptr_clrequest->username);
  dbi_conn_set_option(conn, "password", (ptr_clrequest->passwd && *(ptr_clrequest->passwd)) ? ptr_clrequest->passwd : "");
  dbi_conn_set_option(conn, "dbname", "mysql");
  dbi_conn_set_option(conn, "encoding", "auto");
  dbi_conn_set_option_numeric(conn, "port", atoi(ptr_clrequest->dbs_port_address));
  dbi_conn_set_option(conn, "host", ptr_clrequest->server_ip);
  dbi_conn_set_option_numeric(conn, "mysql_include_trailing_null", 1);


  if (dbi_conn_connect(conn) < 0) { /* -1 and -2 indicate errors */
    LOG_PRINT(LOG_ERR, "could not connect to the database server. Main database upgrade aborted");
    return 1;
  }

  /* get MySQL server version */
  server_version = dbi_conn_get_engine_version(conn);

  /* assemble path to sql script */
  if (server_version == 0 || server_version > 40100) {
    snprintf(path, _POSIX_PATH_MAX, "%s/sql/refdb.%d.dump.mysql41", refdblib, MAIN_DB_VERSION);
    LOG_PRINT(LOG_INFO, "using MySQL 4.1 and later dump file");
    snprintf(sql_command, 128, "CREATE DATABASE %s CHARACTER SET %s", main_db, dbi_driver_encoding_from_iana(dbi_conn_get_driver(conn), ptr_clrequest->db_encoding));
  }
  else {
    LOG_PRINT(LOG_INFO, "libdbi requires MySQL 4.1 or higher");
    return 1;
  }

  /* create the database */
  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);

  if (!dbires) {
    dbi_result_free(dbires);
    dbi_conn_close(conn);
    return 1;
  }
  dbi_result_free(dbires);
  
  /* close connection and reconnect to new database */
  dbi_conn_close(conn);

  if ((conn = my_dbi_conn_new("mysql", ptr_clrequest->inst)) == NULL) {
    LOG_PRINT(LOG_ERR, "Cannot connect to database server. Installation of main database aborted");
    return 1;
  }

  dbi_conn_set_option(conn, "username", ptr_clrequest->username);
  dbi_conn_set_option(conn, "password", (ptr_clrequest->passwd && *(ptr_clrequest->passwd)) ? ptr_clrequest->passwd : "");
  dbi_conn_set_option(conn, "dbname", main_db);
  dbi_conn_set_option(conn, "encoding", "auto");
  dbi_conn_set_option_numeric(conn, "port", atoi(ptr_clrequest->dbs_port_address));
  dbi_conn_set_option(conn, "host", ptr_clrequest->server_ip);
  dbi_conn_set_option_numeric(conn, "mysql_include_trailing_null", 1);


  if (dbi_conn_connect(conn) < 0) { /* -1 and -2 indicate errors */
    LOG_PRINT(LOG_ERR, "could not connect to the database server. Main database upgrade aborted");
    dbi_conn_close(conn);
    return 1;
  }

  /* start a transaction for the insertion of data */
  my_dbi_conn_begin(conn);

  /* create new tables */
  if (execute_sql_mysql(ptr_clrequest, conn, path)) {
    my_dbi_conn_rollback(conn);
    dbi_conn_close(conn);
    return 1;
  }

  my_dbi_conn_commit(conn);
  dbi_conn_close(conn);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  update_pgsql_main_db(): updates the main database (pgsql)

  short int update_pgsql_main_db returns new database version if ok,
                          or 0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  short int n_current_version current database version

  short int n_target_version target version

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static short int update_pgsql_main_db(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, short int current_version, short int target_version) {
  char path[_POSIX_PATH_MAX+1] = "";
  Liliform sentinel;

  /* assemble path to sql script */
  snprintf(path, _POSIX_PATH_MAX, "%s/sql/refdb.%d.dump.pgsql", refdblib, target_version);

  /* initialize linked list sentinel */
  sentinel.ptr_next = NULL;
  sentinel.name[0] = '\0';
  sentinel.value = NULL;

  /* start a transaction */
  if (my_dbi_conn_begin(conn)) {
    return 0;
  }

  /* first replace existing tables with temporary copies */
  if (pgsql_duplicate_tables(ptr_clrequest, conn, &sentinel)) {
    my_dbi_conn_rollback(conn);
    delete_all_liliform(&sentinel);
    return 0;
  }

  /* create new tables */
  if (execute_sql_pgsql(ptr_clrequest, conn, path, 0)) {
    my_dbi_conn_rollback(conn);
    return 0;
  }

  /* finally write the data back into the new tables ... */
  if (pgsql_merge_tables(ptr_clrequest, conn, &sentinel, target_version)) {
    my_dbi_conn_rollback(conn);
    return 0;
  }

  /* ... and update the permissions */
  if (set_main_db_permissions_pgsql(conn)) {
    my_dbi_conn_rollback(conn);
    dbi_conn_close(conn);
  }

  my_dbi_conn_commit(conn);


  return target_version;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  pgsql_duplicate_tables(): duplicates the tables into temporary ones

  int pgsql_duplicate_tables returns 0 if ok, or >0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  Liliform* ptr_sentinel ptr to linked list for table names

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int pgsql_duplicate_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, Liliform* ptr_sentinel) {
  char sql_command[256];
  const char* table_name;
  dbi_result dbires;
  dbi_result dbires1;

  /* get a list of tables */
  dbires = dbi_conn_get_table_list(conn, "refdb", NULL);

  if (!dbires) {
    return 1;
  }

  /* loop over all existing tables */
  while (dbi_result_next_row(dbires)) {
    table_name = my_dbi_result_get_string_idx(dbires, 1);
    if (!table_name) {
      dbi_result_free(dbires);
      return 1;
    }

    /* create a temporary copy of the table */
    sprintf(sql_command, "CREATE TEMPORARY TABLE temp%s AS SELECT * from %s", table_name, table_name);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      dbi_result_free(dbires);
      return 1;
    }
    dbi_result_free(dbires1);

    /* add the table name to the linked list */
    if (insert_liliform(ptr_sentinel, "dummy", (char*)table_name)) {
      dbi_result_free(dbires);
      return 1;
    }

    /* drop the existing table */
    sprintf(sql_command, "DROP TABLE %s", table_name);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      dbi_result_free(dbires);
      return 1;
    }
    dbi_result_free(dbires1);
  }

  dbi_result_free(dbires);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  execute_sql_pgsql(): runs an SQL script for PostgreSQL

  int execute_sql_pgsql returns 0 if ok, or >0 if an error occurred
                         The return value is the exit code of the
			 psql command line application, or 1
			 if there was insufficient memory to assemble
			 the command string

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  const char* sqlfile path of the SQL script to execute

  int is_create if 1, database is being created; if 0, database is being
                updated

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int execute_sql_pgsql(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, const char* sqlfile, int is_create) {
  dbi_result dbires;
  Liliform sentinel;
  Liliform* ptr_curr;

  /* initialize linked list sentinel */
  sentinel.ptr_next = NULL;
  sentinel.name[0] = '\0';
  sentinel.value = NULL;

  /* read the sql script into a linked list of individual commands */
  if (read_sql_from_file_to_list(sqlfile, &sentinel)) {
    return 1;
  }

  ptr_curr = &sentinel;

  /* loop over all commands */
  while ((ptr_curr = get_next_liliform(ptr_curr)) != NULL) {
    if (ptr_curr->value && *(ptr_curr->value)) {
      /* create group only if this is a first installation */
      if (is_create || !strstr(ptr_curr->value, "CREATE GROUP")) {
	LOG_PRINT(LOG_DEBUG, ptr_curr->value);

	dbires = dbi_conn_query(conn, ptr_curr->value);
      
	if (!dbires) {
	  return 1;
	}
	dbi_result_free(dbires);
      }
    }
  }

  /* clean up the linked list */
  delete_all_liliform(&sentinel);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  pgsql_merge_tables(): merges the temporary and new tables. It is assumed
                         that the new table contains all (and maybe more)
			 columns of the temporary table

  int pgsql_merge_tables returns 0 if ok, or >0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  Liliform* ptr_sentinel ptr to linked list for table names

  short int target_version target version of the update procedure

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int pgsql_merge_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, Liliform* ptr_sentinel, short int target_version) {
  char column_buffer[128];
  char* sql_command;
  char* column_list;
  char* new_sql_command;
  const char* table_name;
  size_t sql_cmd_len = 512;
  size_t column_list_len = 512;
  Liliform* ptr_curr;
  dbi_result dbires1;


  /* get some memory */
  if ((sql_command = malloc(sql_cmd_len)) == NULL) {
    return 1;
  }

  /* get some memory */
  if ((column_list = malloc(column_list_len)) == NULL) {
    free(sql_command);
    return 1;
  }


  ptr_curr = ptr_sentinel;

  /* loop over all existing tables. The linked list contains the
     original table names, not the temporary table names */
  while ((ptr_curr = get_next_liliform(ptr_curr)) != NULL) {
    table_name = ptr_curr->value;
    if (!table_name) {
      free(sql_command);
      free(column_list);
      return 1;
    }

    if (!strcmp(table_name, "t_meta")) {
      /* t_meta provides meta information about the database and
	 should therefore contain the new data only. The duplicate
	 check built into the code below will fail as the version info
	 and the timestamps will inevitably differ */
      continue;
    }

    /* retrieve the column names */
    *column_list = '\0';

    sprintf(sql_command, "SELECT pg_attribute.attname FROM pg_class,pg_attribute WHERE pg_class.relname='temp%s' and pg_class.oid=pg_attribute.attrelid", table_name);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      const char* errmsg;
      dbi_conn_error(conn, &errmsg);
      LOG_PRINT(LOG_WARNING, "retrieve column names failed");
      LOG_PRINT(LOG_DEBUG, errmsg);
      free(sql_command);
      free(column_list);
      return 1;
    }

    /* assemble a command which inserts the existing data into the
       columns that were present in the original table */
    sprintf(sql_command, "INSERT INTO %s (", table_name);

    while (dbi_result_next_row(dbires1)) {
      const char* column_name;

      column_name = my_dbi_result_get_string_idx(dbires1, 1);
      if (!column_name) {
	LOG_PRINT(LOG_WARNING, get_status_msg(801));
	free(sql_command);
	free(column_list);
	return 1;
      }

      if (!strcmp(column_name, "tableoid")
	  || !strcmp(column_name, "cmax")
	  || !strcmp(column_name, "xmax")
	  || !strcmp(column_name, "xmin")
	  || !strcmp(column_name, "cmin")
	  || !strcmp(column_name, "cmin")
	  || !strcmp(column_name, "oid")
	  || !strcmp(column_name, "ctid")) {
	/* these are maintained by PostgreSQL and none of our business */
	continue;
      }
	  
      /* skip columns no longer needed in version 3 */
      if (target_version == 3) {
	if (!strcmp(column_name, "misc1preceeding")
	    || !strcmp(column_name, "misc1following")
	    || !strcmp(column_name, "misc1style")
	    || !strcmp(column_name, "misc2preceeding")
	    || !strcmp(column_name, "misc2following")
	    || !strcmp(column_name, "misc2style")
	    || !strcmp(column_name, "misc3preceeding")
	    || !strcmp(column_name, "misc3following")
	    || !strcmp(column_name, "misc3style")) {
	  continue;
	}
      }

      if ((new_sql_command = mstrcat(column_list, (char*)column_name, &column_list_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, get_status_msg(801));
	free(sql_command);
	free(column_list);
	return 1;
      }
      else {
	column_list = new_sql_command;
      }

      if ((new_sql_command = mstrcat(column_list, ",", &column_list_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, get_status_msg(801));
	free(sql_command);
	free(column_list);
	return 1;
      }
      else {
	column_list = new_sql_command;
      }
    }

    dbi_result_free(dbires1);

    if ((new_sql_command = mstrcat(sql_command, column_list, &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      free(sql_command);
      free(column_list);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    if ((new_sql_command = mstrcat(sql_command, ") SELECT ", &sql_cmd_len, 1)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      free(sql_command);
      free(column_list);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    if ((new_sql_command = mstrcat(sql_command, column_list, &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      free(sql_command);
      free(column_list);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    sprintf(column_buffer, " FROM temp%s EXCEPT SELECT ", table_name);

    /* skip trailing comma */
    if ((new_sql_command = mstrcat(sql_command, column_buffer, &sql_cmd_len, 1)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      free(sql_command);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    if ((new_sql_command = mstrcat(sql_command, column_list, &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      free(sql_command);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    sprintf(column_buffer, " FROM %s", table_name);

    /* overwrite trailing comma */
    if ((new_sql_command = mstrcat(sql_command, column_buffer, &sql_cmd_len, 1)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      free(sql_command);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      const char* errmsg;
      dbi_conn_error(conn, &errmsg);
      LOG_PRINT(LOG_WARNING, "INSERT into main database failed");
      LOG_PRINT(LOG_DEBUG, errmsg);
      free(sql_command);
      return 1;
    }
    dbi_result_free(dbires1);
  }

  free(sql_command);
  free(column_list);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  install_pgsql_main_db(): installs the main database (pgsql)

  int install_pgsql_main_db returns 0 if ok,
                          or >0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int install_pgsql_main_db(struct CLIENT_REQUEST* ptr_clrequest) {
  char path[_POSIX_PATH_MAX+1] = "";
  char sql_command[128];
  dbi_conn conn;
  dbi_result dbires;

  /* assemble path to sql script */
  snprintf(path, _POSIX_PATH_MAX, "%s/sql/refdb.%d.dump.pgsql", refdblib, MAIN_DB_VERSION);

  /* get a connection to template1 */
  if ((conn = my_dbi_conn_new("pgsql", ptr_clrequest->inst)) == NULL) {
    LOG_PRINT(LOG_ERR, "Cannot connect to database server. Installation of main database aborted");
    return 1;
  }

  LOG_PRINT(LOG_DEBUG, ptr_clrequest->server_ip);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->username);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->passwd);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->current_db);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->dbs_port_address);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->dbserver);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->db_path);
  LOG_PRINT(LOG_DEBUG, ptr_clrequest->db_encoding);

  dbi_conn_set_option(conn, "username", ptr_clrequest->username);
  dbi_conn_set_option(conn, "password", (ptr_clrequest->passwd && *(ptr_clrequest->passwd)) ? ptr_clrequest->passwd : "");
  dbi_conn_set_option(conn, "dbname", "template1");
  dbi_conn_set_option(conn, "encoding", "auto");
  dbi_conn_set_option_numeric(conn, "port", atoi(ptr_clrequest->dbs_port_address));
  if (strcmp(ptr_clrequest->server_ip, "localhost")) {
    dbi_conn_set_option(conn, "host", ptr_clrequest->server_ip);
  }


  if (dbi_conn_connect(conn) < 0) { /* -1 and -2 indicate errors */
    LOG_PRINT(LOG_ERR, "could not connect to the database server. Main database upgrade aborted");
    return 1;
  }

  /* create the database */
  snprintf(sql_command, 128, "CREATE DATABASE %s WITH ENCODING=\'%s\'", main_db, dbi_driver_encoding_from_iana(dbi_conn_get_driver(conn), ptr_clrequest->db_encoding));
  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);

  if (!dbires) {
    LOG_PRINT(LOG_ERR, "Could not create main database");
    dbi_result_free(dbires);
    dbi_conn_close(conn);
    return 1;
  }
  dbi_result_free(dbires);

  /* close connection and reconnect to new database */
  dbi_conn_close(conn);

  if ((conn = my_dbi_conn_new("pgsql", ptr_clrequest->inst)) == NULL) {
    LOG_PRINT(LOG_ERR, "Cannot connect to database server. Installation of main database aborted");
    return 1;
  }

  dbi_conn_set_option(conn, "username", ptr_clrequest->username);
  dbi_conn_set_option(conn, "password", (ptr_clrequest->passwd && *(ptr_clrequest->passwd)) ? ptr_clrequest->passwd : "");
  dbi_conn_set_option(conn, "dbname", main_db);
  dbi_conn_set_option(conn, "encoding", "auto");
  dbi_conn_set_option_numeric(conn, "port", atoi(ptr_clrequest->dbs_port_address));
  if (strcmp(ptr_clrequest->server_ip, "localhost")) {
    dbi_conn_set_option(conn, "host", ptr_clrequest->server_ip);
  }

  if (dbi_conn_connect(conn) < 0) { /* -1 and -2 indicate errors */
    LOG_PRINT(LOG_ERR, "could not connect to the database server. Main database upgrade aborted");
    dbi_conn_close(conn);
    return 1;
  }

  /* start a transaction for the insertion of data */
  my_dbi_conn_begin(conn);

  /* create new tables */
  if (execute_sql_pgsql(ptr_clrequest, conn, path, 1)) {
    my_dbi_conn_rollback(conn);
    dbi_conn_close(conn);
    return 1;
  }

  if (set_main_db_permissions_pgsql(conn)) {
    my_dbi_conn_rollback(conn);
    dbi_conn_close(conn);
  }

  my_dbi_conn_commit(conn);
  dbi_conn_close(conn);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  set_main_db_permissions_pgsql(): sets the main database permissions (pgsql)

  int set_main_db_permissions_pgsql returns 0 if ok,
                                    or >0 if an error occurred

  dbi_conn conn database connection structure

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int set_main_db_permissions_pgsql(dbi_conn conn) {
  char sql_command[256];
  dbi_result dbires;
  dbi_result dbires1;

  /* check for existing group */
  snprintf(sql_command, 256, "SELECT groname FROM pg_group WHERE groname=\'%suser\'", main_db);
  
  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);

  if (!dbires) {
    LOG_PRINT(LOG_ERR, "Could not query main database user group");
    return 1;
  }

  if (!dbi_result_get_numrows(dbires)) {
    /* group already exists */
    snprintf(sql_command, 256, "CREATE GROUP %suser", main_db);
    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      LOG_PRINT(LOG_ERR, "Could not create main database user group");
      dbi_result_free(dbires);
      return 1;
    }
    dbi_result_free(dbires1);
  }

  dbi_result_free(dbires);
  

  snprintf(sql_command, 256, "GRANT SELECT ON CITSTYLE, POSITIONS, REFSTYLE, SEPARATORS, t_journal_words, %s TO GROUP %suser", MAIN_META, main_db);
  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);

  if (!dbires) {
    LOG_PRINT(LOG_ERR, "Could not grant permissions to main database user group");
    return 1;
  }

  dbi_result_free(dbires);
  
  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  update_sqlite_main_db(): updates the main database (sqlite)

  short int update_sqlite_main_db returns new database version if ok,
                          or 0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  short int n_current_version current database version

  short int n_target_version target version

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static short int update_sqlite_main_db(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, short int current_version, short int target_version) {
  char path[_POSIX_PATH_MAX+1] = "";

  /* assemble path to sql script */
  snprintf(path, _POSIX_PATH_MAX, "%s/sql/refdb.%d.dump.sqlite", refdblib, target_version);

  /* start a transaction */
  if (my_dbi_conn_begin(conn)) {
    return 0;
  }

  /* first replace existing tables with temporary copies */
  if (sqlite_duplicate_tables(ptr_clrequest, conn)) {
    my_dbi_conn_rollback(conn);
    return 0;
  }

  /* create new tables */
  if (execute_sql_sqlite(ptr_clrequest, conn, path)) {
    my_dbi_conn_rollback(conn);
    return 0;
  }

  if (sqlite_merge_tables(ptr_clrequest, conn, target_version)) {
    my_dbi_conn_rollback(conn);
    return 0;
  }

  my_dbi_conn_commit(conn);


  return target_version;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  sqlite_duplicate_tables(): duplicates the tables into temporary ones

  int sqlite_duplicate_tables returns 0 if ok, or >0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int sqlite_duplicate_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn) {
  char sql_command[256];
  const char* table_name;
  dbi_result dbires;
  dbi_result dbires1;

  dbires = dbi_conn_get_table_list(conn, "refdb", NULL);

  if (!dbires) {
    return 1;
  }

  /* loop over all existing tables */
  while (dbi_result_next_row(dbires)) {
    table_name = my_dbi_result_get_string_idx(dbires, 1);
    if (!table_name) {
      dbi_result_free(dbires);
      return 1;
    }

    /* create a temporary copy of the table */
    sprintf(sql_command, "CREATE TEMPORARY TABLE temp%s AS SELECT * from %s", table_name, table_name);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      dbi_result_free(dbires);
      return 1;
    }
    dbi_result_free(dbires1);

    /* drop the existing table */
    sprintf(sql_command, "DROP TABLE %s", table_name);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      dbi_result_free(dbires);
      return 1;
    }
    dbi_result_free(dbires1);
  }

  dbi_result_free(dbires);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  execute_sql_sqlite(): runs an SQL script for SQLite

  int execute_sql_sqlite returns 0 if ok, or >0 if an error occurred
                         The return value is the exit code of the
			 sqlite(3) command line application, or 1
			 if there was insufficient memory to assemble
			 the command string

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  const char* sqlfile path of the SQL script to execute

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int execute_sql_sqlite(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, const char* sqlfile) {
  char* sql_command;
  dbi_result dbires;

  if ((sql_command = read_sql_from_file(sqlfile)) == NULL) {
    return 1;
  }

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);

  free(sql_command);

  if (!dbires) {
    return 1;
  }

  dbi_result_free(dbires);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  sqlite_merge_tables(): merges the temporary and new tables. It is assumed
                         that the new table contains all (and maybe more)
			 columns of the temporary table

  int sqlite_merge_tables returns 0 if ok, or >0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  short int target_version target version of the update procedure

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int sqlite_merge_tables(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn, short int target_version) {
  char column_buffer[128];
  char* sql_command;
  char* new_sql_command;
  char* column_list = NULL;
  const char* table_name;
  size_t sql_cmd_len = 512;
  dbi_result dbires;
  dbi_result dbires1;

  if ((sql_command = malloc(sql_cmd_len)) == NULL) {
    LOG_PRINT(LOG_WARNING, get_status_msg(801));
    return 1;
  }

  sprintf(sql_command, "SELECT name FROM sqlite_temp_master WHERE type='table'");

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);

  if (!dbires) {
    return 1;
  }

  /* loop over all existing temporary tables */
  while (dbi_result_next_row(dbires)) {
    table_name = my_dbi_result_get_string(dbires, "name");
    if (!table_name) {
      dbi_result_free(dbires);
      return 1;
    }
    else if (!strcmp(table_name, "libdbi_tablenames")
	     || !strcmp(table_name, "tempt_meta")) {
      /* this is a libdbi temporary table used to list tables */
      continue;
    }

    /* retrieve the column names */
    sprintf(sql_command, "SELECT sql FROM sqlite_temp_master WHERE tbl_name='%s'", table_name);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      const char* errmsg;
      dbi_conn_error(conn, &errmsg);
      LOG_PRINT(LOG_WARNING, "retrieve column names failed");
      LOG_PRINT(LOG_DEBUG, errmsg);
      dbi_result_free(dbires);
      return 1;
    }

    sprintf(sql_command, "INSERT OR REPLACE INTO %s (", table_name+4);

    while (dbi_result_next_row(dbires1)) {
      char* create_command;

      create_command = my_dbi_result_get_string_copy(dbires1, "sql");

      if (!create_command) {
	dbi_result_free(dbires);
	return 1;
      }

      column_list = assemble_column_list(create_command, target_version);

      free(create_command);

      if (!column_list || (new_sql_command = mstrcat(sql_command, column_list, &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, get_status_msg(801));
	free(column_list);
	dbi_result_free(dbires);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
    }

    dbi_result_free(dbires1);

    if (!column_list) {
      return 1;
    }

    if ((new_sql_command = mstrcat(sql_command, ") SELECT ", &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      dbi_result_free(dbires);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    if ((new_sql_command = mstrcat(sql_command, column_list, &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      dbi_result_free(dbires);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    free(column_list);

    sprintf(column_buffer, " FROM %s", table_name);

    if ((new_sql_command = mstrcat(sql_command, column_buffer, &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, get_status_msg(801));
      dbi_result_free(dbires);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);

    if (!dbires1) {
      const char* errmsg;
      dbi_conn_error(conn, &errmsg);
      LOG_PRINT(LOG_WARNING, "INSERT into main database failed");
      LOG_PRINT(LOG_DEBUG, errmsg);
      dbi_result_free(dbires);
      return 1;
    }
    dbi_result_free(dbires1);
  }

  dbi_result_free(dbires);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  install_sqlite_main_db(): installs the main database (sqlite)

  int install_sqlite_main_db returns 0 if ok,
                          or >0 if an error occurred

  struct CLIENT_REQUEST* ptr_clrequest ptr to struct with request data

  dbi_conn conn connection structure

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int install_sqlite_main_db(struct CLIENT_REQUEST* ptr_clrequest, dbi_conn conn) {
  char path[_POSIX_PATH_MAX+1] = "";

  /* assemble path to sql script */
  snprintf(path, _POSIX_PATH_MAX, "%s/sql/refdb.%d.dump.sqlite", refdblib, MAIN_DB_VERSION);

  /* create new tables */
  if (execute_sql_sqlite(ptr_clrequest, conn, path)) {
    return 1;
  }

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  assemble_column_list(): assembles a list of column names
                         from a create table statement

  char assemble_column_list returns an allocated string containing
                         the comma-separated list of column names, or NULL
			 if an error occurs

  const char* create_statement ptr to buffer containing the create table
                         statement

  short int target_version target version of the upgrade procedure

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static char* assemble_column_list(const char* create_statement, short int target_version) {
  char* start;
  char* item;
  char* item_end;
  char* column_list;
  char* new_column_list;
  size_t column_list_len = 256;

  if (!create_statement || !*create_statement) {
    return NULL;
  }

/*   printf("create statement went to:%s<<\n", create_statement); */
/*   fflush(stdout); */

  if ((column_list = malloc(column_list_len)) == NULL) {
    return NULL;
  }
  *column_list = '\0';

  start = strchr(create_statement, (int)'(');

  if (!start) {
    return NULL;
  }

  item = strtok(start+1, ",");

  while (item) {
    char* new_item;

    new_item = stripwhite(item, 1, 1);
    item_end = strchr(new_item, (int)' ');
    
    if (item_end) {
      *item_end = '\0';

      new_item = strip_quote(new_item);

      /* skip primary key statements */
      if (!strcmp(new_item, "PRIMARY")) {
	continue;
      }

      /* skip columns no longer used in version 3 */
      if (target_version == 3) {
	if (!strcmp(new_item, "MISC1PRECEEDING")
	    || !strcmp(new_item, "MISC1FOLLOWING")
	    || !strcmp(new_item, "MISC1STYLE")
	    || !strcmp(new_item, "MISC2PRECEEDING")
	    || !strcmp(new_item, "MISC2FOLLOWING")
	    || !strcmp(new_item, "MISC2STYLE")
	    || !strcmp(new_item, "MISC3PRECEEDING")
	    || !strcmp(new_item, "MISC3FOLLOWING")
	    || !strcmp(new_item, "MISC3STYLE")) {
	  continue;
	}
      }

      if ((new_column_list = mstrcat(column_list, new_item, &column_list_len, 0)) == NULL) {
	return NULL;
      }
      else {
	column_list = new_column_list;
      }
      
      if ((new_column_list = mstrcat(column_list, ",", &column_list_len, 0)) == NULL) {
	return NULL;
      }
      else {
	column_list = new_column_list;
      }
    }
    item = strtok(NULL, ",");
  }

  /* remove trailing comma */
  if (strlen(column_list)) {
    column_list[strlen(column_list)-1] = '\0';
  }

  return column_list;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  read_sql_from_file(): reads a sql script from a file and turns it into
                        a query string without newlines

  char* read_sql_from_file returns the query string in an allocated buffer
                          or NULL if an error occurs

  const char* path ptr to buffer containing the path to the sql file

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static char* read_sql_from_file(const char* path) {
  char* sql_command;
  char* new_sql_command;
  char* linebuf;
  size_t sql_cmd_len = 65536;
  size_t linebuf_len = 65536;
  FILE *fp;

  if ((sql_command = malloc(sql_cmd_len)) == NULL) {
    return NULL;
  }
  *sql_command = '\0';

  if ((linebuf = malloc(linebuf_len)) == NULL) {
    free(sql_command);
    return NULL;
  }

  fp = fopen(path, "r");
  if (fp == NULL) {
    free(sql_command);
    free(linebuf);
    return NULL;
  }

  /* loop as long as we find lines */
  while (fgets(linebuf, linebuf_len, fp) != NULL) {
    /* skip empty lines and comments */
    if (*linebuf == '\n'
	|| (*linebuf == '-' && *(linebuf+1) == '-')) {
      continue;
    }

    /* strip whitespace including newline from the end */
    stripwhite(linebuf, 2, 1);

    if ((new_sql_command = mstrcat(sql_command, linebuf, &sql_cmd_len, 0)) == NULL) {
      free(linebuf);
      free(sql_command);
      return NULL;
    }
    else {
      sql_command = new_sql_command;
    }
  }

  fclose(fp);

  free(linebuf);
  return sql_command;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  read_sql_from_file_to_list(): reads a sql script from a file and 
                        inserts individual commands into a linked list

  char* read_sql_from_file_to_list returns 0 if ok or 1 if an error
                          occurred

  const char* path ptr to buffer containing the path to the sql file

  Liliform* ptr_sentinel ptr to the starting member of a linked list
                         that will receive the sql commands

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int read_sql_from_file_to_list(const char* path, Liliform* ptr_sentinel) {
  char* sql_command;
  char* new_sql_command;
  char* linebuf;
  size_t sql_cmd_len = 65536;
  size_t linebuf_len = 65536;
  FILE *fp;

  if ((sql_command = malloc(sql_cmd_len)) == NULL) {
    return 1;
  }
  *sql_command = '\0';

  if ((linebuf = malloc(linebuf_len)) == NULL) {
    free(sql_command);
    return 1;
  }

  fp = fopen(path, "r");
  if (fp == NULL) {
    free(sql_command);
    free(linebuf);
    return 1;
  }

  /* loop as long as we find lines */
  while (fgets(linebuf, linebuf_len, fp) != NULL) {
    /* skip empty lines and comments */
    if (*linebuf == '\n'
	|| (*linebuf == '-' && *(linebuf+1) == '-')) {
      continue;
    }

    if (strchr(linebuf, (int)';')) {
      /* this is either an oneliner or the last line of a multiline
	 command. Append to existing command string and insert into
	 linked list */
      /* strip whitespace excluding newline from the end */
      stripwhite(linebuf, 2, 0);
      if ((new_sql_command = mstrcat(sql_command, linebuf, &sql_cmd_len, 0)) == NULL) {
	free(sql_command);
	free(linebuf);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }

      if (append_liliform(ptr_sentinel, "dummy", sql_command)) {
	free(linebuf);
	return 1;
      }

      *sql_command = '\0';
    }
    else if (*linebuf) { /* don't bother if there's nothing to read */
      /* this is a part of a multiline command */
      /* strip whitespace including newline from the end */
      stripwhite(linebuf, 2, 1);

      if ((new_sql_command = mstrcat(sql_command, linebuf, &sql_cmd_len, 0)) == NULL) {
	free(sql_command);
	free(linebuf);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
    }
  }

  fclose(fp);

  free(sql_command);
  free(linebuf);
  return 0;
}

