/*+++++++++++++++++++++++++++++
  risdb.c: functions to write RIS datasets to a database
  markus@mhoenicka.de 3-26-00
  $Id: risdb.c,v 1.44.2.13 2005/11/03 20:40:19 mhoenicka Exp $

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.
   
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
   
   You should have received a copy of the GNU General Public License
   along with this program; if not, see <http://www.gnu.org/licenses/>

   ++++++++++++++++++++++++++*/


/* ToDo: think about consolidating int variables that are used as switches */


#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <syslog.h> /* priority levels of log messages */
#include <iconv.h>
#include <errno.h>
#include <sys/types.h> /* for getpid() */
#include <unistd.h>    /* for getpid() */

#include <dbi/dbi.h>

#include "linklist.h"
#include "strfncs.h"
#include "refdb.h"
#include "refdbd.h"
#include "risdb.h"
#include "tokenize.h"
#include "readris.h"
#include "connect.h"
#include "dbfncs.h"
#include "authorinfo.h"
#include "risdata.h"

extern int n_log_level; /* level up to which messages are logged */
extern int nongeek_offset;
extern char keep_pnames[];
extern char upper_citekey[];
extern char main_db[];

/* forward declarations of local functions */
static char* dotify_journal(char* journal, dbi_conn conn, dbi_conn conn_refdb);
static unsigned long long find_journal(int level, struct PERIODICAL_INFO* ptr_perinfo, dbi_conn conn, int replace_ref);
static char* strip_oddchars(char* string);
static int update_user_field(const char* fieldvalue, unsigned long long n_refdb_id, unsigned long long n_user_id, dbi_conn conn, dbi_driver driver, const char* query_stub, const char* errmsg);
static int check_filelink(const char* url);



/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  dotify_journal(): adds periods after abbreviated names in JO/JA
                    The modified string finally ends up in the same
                    location as the passed string. This is ok as the
                    modification in no case increases the length of
                    the string more than 1 char (it will actually
                    decrease the length of the string if it contains
                    trailing or leading or consecutive whitespace).
                    Each token will be compared with a wordlist in
                    refdb. If a match is found, a space is used as
                    separator. If no match is found, the word is
                    considered an abbreviation and gets a dot
                    appended.

  static char* dotify_journal returns pointer to the modified string. Returns
                    NULL if the argument is NULL or an empty string
                    or if we're out of memory.

  char* journal pointer to the string containing the abbreviated
                    journal. IMPORTANT: The array holding the
                    abbreviated journal must be allocated to have
                    one char more than the length of the string incl
                    the terminating \0 (i.e. strlen(JA) + 2).

  dbi_conn conn pointer to a data structure for a previously
                    established database connection. The caller is
                    responsible to establish this connection and
                    to take it down again after this function
                    returns.

  dbi_conn conn_refdb database connection to refdb, used by some
                    drivers

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static char* dotify_journal(char* journal, dbi_conn conn, dbi_conn conn_refdb) {
  char *dest;
  char *start_token;
  char *end_string;
  char *mytoken;
  char *local_journal;
  char *sql_command;
  const char *drivername;
  size_t token_len;
  dbi_result dbires = NULL;

  /* take a shortcut home if no journal is specified */
  if (journal == NULL || !*journal) {
    return NULL;
  }

  dest = malloc(strlen(journal)+2);
  if (dest == NULL) {
    return NULL;
  }

  sql_command = malloc(356); /* 100 sql command + 255 token + 1 \0 */
  if (sql_command == NULL) {
    free(dest);
    return NULL;
  }

/*   printf("in: %s", journal); */
  dest[0] = '\0'; /* terminate string */

  drivername = dbi_driver_get_name(dbi_conn_get_driver(conn));

  local_journal = stripwhite(journal, 0, 0);

  end_string = local_journal+strlen(local_journal);

  start_token = nstrtok(local_journal, &token_len, ".- \t");

  while (start_token != NULL) {
    start_token[token_len] = '\0';
    strcat(dest, start_token);

    if (start_token[strlen(start_token)-1] != '.') {
      /* check whether the token is likely to be an abbreviation */
      mytoken = strdup(start_token);
      if (!mytoken) {
	free(dest);
	return NULL;
      }
      strup(mytoken); /* convert token to uppercase */

      if (dbi_conn_quote_string(conn, &mytoken) == 0) {
	free(dest);
	free(mytoken);
	return NULL;
      }

      if (token_len < 255) {
	if (!strcmp(my_dbi_conn_get_cap(conn, "multiple_db"), "t")) {
	  sprintf(sql_command, "SELECT name FROM %s.t_journal_words WHERE name=%s", main_db, mytoken);
	}
	else {
	  sprintf(sql_command, "SELECT name FROM t_journal_words WHERE name=%s", mytoken);
	}
	
	free(mytoken);

	LOG_PRINT(LOG_DEBUG, sql_command);

	if (!strcmp(my_dbi_conn_get_cap(conn, "multiple_db"), "t")) {
	  dbires = dbi_conn_query(conn, sql_command);
	}
	else {
	  dbires = dbi_conn_query(conn_refdb, sql_command);
	}

	/* the following test will also fail if an unknown driver is used */
	if (!dbires) {
	  LOG_PRINT(LOG_WARNING, "error in t_journal_words query");
	}
	else {
	  if (dbi_result_get_numrows(dbires) > 0 || start_token[token_len-1] == '.') {
	    strcat(dest, " ");
	  }
	  else {
	    strcat(dest, ".");
	  }
	}
      }
      else { /* such a long token is not allowed as it exceeds the maximum length of the JO/JA field. We just make sure to avoid a segfault */
	strcat(dest, " ");
	free(mytoken);
      }

      if (dbires) {
	dbi_result_free(dbires);
      }
    }
    /* else: period is already in place, nothing to do */

    start_token += token_len+1;
    if (start_token >= end_string) {
      start_token = NULL;
    }
    else {
      start_token = nstrtok(start_token, &token_len, ".- \t");
    }
  }
  stripwhite(dest, 2, 0); /* strip trailing space, if any */
/*   printf(" out: %s\n", dest); */
  strcpy(journal, dest);
  free(dest);
  free(sql_command);
  return journal;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  process_ris_set(): adds a given RIS dataset to the database

  int process_ris_set returns 0 if failed, 1 if successfully added,
                              2 if successfully replaced
			      3 if skipped 

  char* set string holding a RIS dataset

  dbi_conn conn pointer to a data structure for a previously
                    established database session. The caller is
                    responsible to establish this connection and
                    to take it down again after this function
                    returns.

  dbi_conn conn_refdb optional ptr to a database session for refdb
                    (used by some dbi drivers)

  int replace_ref if ADDREF_UPDATE, ref will be updated according to ID field
                  if ADDREF_ADD, ref will be added, ignoring an ID field
		  if ADDREF_UPDATE_PERSONAL, only the personal settings
                     (N1, AV, RP) will be updated according to the ID field
                  if ADDREF_CHECK, ref will be added to temporary tables
                     and checked

  char *set_owner string denoting the user who adds/owns this reference

  int fd file descriptor of socket

  int n_keep_id if 1, any existing reference ID will be saved in U5

  Lilid* ptr_id_sentinel ptr to linked list. This fn will add the
                         ID of the current dataset to this list if
			 adding the set was successful

  unsigned long long set_count number of current dataset

  char** ptr_msg_pool pointer to string receiving status messages

  size_t* ptr_msg_pool_len ptr to length of *ptr_msg_pool

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/

int process_ris_set(char* set, dbi_conn conn, dbi_conn conn_refdb, int replace_ref, char* set_owner, struct CLIENT_REQUEST* ptr_clrequest, int n_keep_id, Lilid* ptr_id_sentinel, unsigned long long set_count, char** ptr_msg_pool, size_t* ptr_msg_pool_len) {
  char* token = NULL; /* one line of the original RIS file - starts with tag */
  char* sql_command = NULL; /* buffer to assemble query */
  char *buffer; /* buffer for assembling strings */
  char *real_citekey; /* copy of citekey, for preprocessing the contents */
  char date_buffer[11] = ""; /* buffer to hold the date information of PY*/
  char otherinfo_buffer[256] = ""; /* buffer to hold the other information of PY */
  char first_author[256] = ""; /* buffer to hold first author of a ref */
  char type[7] = "GEN"; /* the reference type */ 
  char prefix[] = TEMP_TABLE_NAME_PREFIX;
  const char *drivername = NULL; /* name of the libdbi driver */
  const char *citekey = NULL; /* new citation key */
  char* new_msgpool; /* used to reallocate *msg_pool */
  size_t sql_cmd_len; /* currently allocated length of sql_command */
/*   int error; */ /* error switch */
  unsigned long long n_refdb_id = 0; /* ID of the reference */
  unsigned long long n_periodical_id = 0; /* periodical_id of a search result */
  unsigned long long n_user_id = 0; /* user_id of a search result */
  int result;
  int string_len; /* length of a temporary string */
  size_t token_len; /* length of a token */
  int part_xauthor_pos = 0; /* the position of an author in the author list */
  int publication_xauthor_pos = 0; /* the position of an author in the author list */
  int series_xauthor_pos = 0; /* the position of an author in the author list */
  int rp_type; /* reprint type 0=IN FILE, 1=NOT IN FILE, 2=ON REQUEST */
  int year_type; /* type of year info 0=PY, 1=Y1 */
  int create_new = 1; /* if 1, we create a new dataset; if 0, we update */
  int have_rp = 0; /* if remains 0, dataset did not contain a RP field */
  int have_py = 0; /* if remains 0, dataset did not contain a PY field */
  int n_have_citekey = 0; /* if 1, dataset contains citekey */
  int retval = 0;
  int nis_dummy = 0; /* will be 1 if dataset is a dummy */
  short int year = 0; /* the year */
  short int year2 = 0; /* the secondary year */
  dbi_driver driver;
  dbi_result dbires;
  struct lilimem sentinel;
  struct PERIODICAL_INFO perinfo; /* periodical name synonyms */
  struct AUTHOR_INFO* ptr_ainfo;
  struct RISDATA* ptr_risdata;
  struct ADDRESULT addresult;

  sentinel.ptr_mem = NULL;
  sentinel.ptr_next = NULL;
  sentinel.varname[0] = '\0';

  perinfo.full = NULL;
  perinfo.abbrev = NULL;
  perinfo.custabbrev1 = NULL;
  perinfo.custabbrev2 = NULL;


  driver = dbi_conn_get_driver(conn);
  drivername = dbi_driver_get_name(driver);

  /* fix prefix */
  if (replace_ref != ADDREF_CHECK) {
    *prefix = '\0';
  }

  /* get a buffer to assemble queries */
  sql_cmd_len = 4096; /* enough for inserting all 4 journal columns */
  sql_command = malloc(sql_cmd_len); /* big enough for replace_ref search */
  if (sql_command == NULL || insert_lilimem(&sentinel, (void**)&sql_command, "sql_command")) {
    if ((new_msgpool = mstrcat(*ptr_msg_pool, "801\n", ptr_msg_pool_len, 0)) == NULL) {
      return 0;
    }
    else {
      *ptr_msg_pool = new_msgpool;
    }
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    return 0;
  }

  /* another buffer to assemble strings */
  buffer = malloc(512);
  if (buffer == NULL || insert_lilimem(&sentinel, (void**)&buffer, "buffer")) {
    if ((new_msgpool = mstrcat(*ptr_msg_pool, "801\n", ptr_msg_pool_len, 0)) == NULL) {
      return 0;
    }
    else {
      *ptr_msg_pool = new_msgpool;
    }
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    return 0;
  }

  /* another buffer for the citation key */
  real_citekey = malloc(256);
  if (real_citekey == NULL || insert_lilimem(&sentinel, (void**)&real_citekey, "real_citekey")) {
    if ((new_msgpool = mstrcat(*ptr_msg_pool, "801\n", ptr_msg_pool_len, 0)) == NULL) {
      return 0;
    }
    else {
      *ptr_msg_pool = new_msgpool;
    }
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    return 0;
  }
  *real_citekey = '\0';

  ptr_ainfo = new_authorinfo();
  if (ptr_ainfo == NULL || insert_lilimem(&sentinel, (void**)&ptr_ainfo, "ptr_ainfo")) {
    sprintf(sql_command, "801:"ULLSPEC"\n", (unsigned long long)set_count);
    if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
      return 0;
    }
    else {
      *ptr_msg_pool = new_msgpool;
    }
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    delete_all_lilimem(&sentinel);
    return 0;
  }

  ptr_risdata = new_risdata();
  if (ptr_risdata == NULL) {
    sprintf(sql_command, "801:"ULLSPEC"\n", (unsigned long long)set_count);
    if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
      return 0;
    }
    else {
      *ptr_msg_pool = new_msgpool;
    }
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    delete_all_lilimem(&sentinel);
    return 0;
  }

  /* If the db server supports it, start a transaction. We want one
     transaction per reference */
  if (my_dbi_conn_begin(conn)) {
    sprintf(sql_command, "227:"ULLSPEC"\n", (unsigned long long)set_count);
    if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
      return 0;
    }
    else {
      *ptr_msg_pool = new_msgpool;
    }
    LOG_PRINT(LOG_ERR, get_status_msg(227));
    delete_all_lilimem(&sentinel);
    free_risdata(ptr_risdata);
    return 0;
  }
  
  /* lock the tables we'll write to to prevent concurrent writes
     from different clients */
  if (my_dbi_conn_lock(conn, replace_ref)) {
    sprintf(sql_command, "228:"ULLSPEC"\n", (unsigned long long)set_count);
    if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
      return 0;
    }
    else {
      *ptr_msg_pool = new_msgpool;
    }
    LOG_PRINT(LOG_ERR, get_status_msg(228));
    delete_all_lilimem(&sentinel);
    free_risdata(ptr_risdata);
    return 0;
  }

  /* sanity check: scan dataset for a TY string */
  token = strstr(set, "TY  - ");
  if (token == NULL) {
    sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)set_count);
    if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
      return 0;
    }
    else {
      *ptr_msg_pool = new_msgpool;
    }
    LOG_PRINT(LOG_ERR, get_status_msg(233));
    delete_all_lilimem(&sentinel);
    free_risdata(ptr_risdata);
    return 0;
  }

  /* scan dataset for ID */
  token = strstr(set, "ID  - ");
  if (token != NULL) { /* if dataset has an ID string */
    string_len = strcspn(token, "\n");
    if (string_len > 261) { /* largest reasonable citekey, 255+6 for tag */
      sprintf(sql_command, "410:"ULLSPEC"\n", (unsigned long long)set_count);
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      LOG_PRINT(LOG_ERR, get_status_msg(410));
      retval = 1;
      goto cleanup;
    }

    /* two cases: either a numeric ID or an alphanumeric citekey */
    strncpy(buffer, token+6, string_len-6);
    buffer[string_len-6] = '\0';
    if (buffer[strlen(buffer)-1] == '\r') { /* cgi data have \r\n */
      buffer[strlen(buffer)-1] = '\0';
    }

    if (is_number(buffer)) { /* ID field contains numeric ID */
      if (string_len > 26) { /* largest 8byte-integer, 20+6 for tag */
	sprintf(sql_command, "410:"ULLSPEC"\n", (unsigned long long)set_count);
	if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	  return 0;
	}
	else {
	  *ptr_msg_pool = new_msgpool;
	}

	LOG_PRINT(LOG_ERR, get_status_msg(410));
	retval = 1;
	goto cleanup;
      }
      sprintf(sql_command, "SELECT refdb_id, refdb_citekey FROM t_%srefdb WHERE refdb_id=%s", prefix, buffer);
    }
    else { /* ID field contains citekey */
      char *new_buffer;
      n_have_citekey = 1;
      if ((new_buffer = preprocess_citekey_copy(buffer, 255)) == NULL) {
	return 0;
      }
      else {
	free(buffer);
	buffer = new_buffer;
      }
      sprintf(sql_command, "SELECT refdb_id, refdb_citekey FROM t_%srefdb WHERE refdb_citekey=\'%s\'", prefix, buffer);
    }
    
    /* search for existing entry */
    LOG_PRINT(LOG_DEBUG, sql_command);
    
    dbires = dbi_conn_query(conn, sql_command);
    if (!dbires) {
      sprintf(sql_command, "410:"ULLSPEC"\n", (unsigned long long)set_count);
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }

      LOG_PRINT(LOG_ERR, get_status_msg(410));
      retval = 1;
      goto cleanup;
    }
      
    /* we have to consider these cases:
       - addref: entry with same ID or citekey doesn't exist-> add
       - addref: entry with same ID/citekey exists-> error
       - updateref: entry with same ID/citekey doesn't exist-> add
       - updateref: entry with same ID/citekey exists-> update */
    
    if (dbi_result_next_row(dbires)) { /* requested ID exists */
      if (replace_ref != ADDREF_ADD && replace_ref != ADDREF_CHECK) {
	create_new = 0;
	n_refdb_id = my_dbi_result_get_idval(dbires, "refdb_id");
	set_risdata_field(ptr_risdata, "citekey", my_dbi_result_get_string(dbires, "refdb_citekey"), conn);
      }
      else { /* we're supposed to add */
	if (n_have_citekey) {/* we can't add lest we overwrite existing data */
 	  LOG_PRINT(LOG_INFO, "refused to overwrite existing dataset");
/* 	  sprintf(sql_command, "citekey %s already exists, please try again\n", buffer); */
	  sprintf(sql_command, "407:"ULLSPEC":%s\n", (unsigned long long)(set_count + nongeek_offset), buffer);
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  retval = 2;
	  my_dbi_conn_rollback(conn);
	  goto cleanup;
	}
	else { /* ignore numerical ID and assign a new one */
	  LOG_PRINT(LOG_INFO, "ignore numerical ID");
/* 	  sprintf(sql_command, "ID %s ignored, I have assigned a new one\n", buffer); */
	  sprintf(sql_command, "409:"ULLSPEC"%s\n", (unsigned long long)(set_count + nongeek_offset), buffer);
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	}
      }
    }
    else { /* requested ID could not be found */
      if (replace_ref == ADDREF_UPDATE) {
	create_new = 1;
      }
      else if (replace_ref == ADDREF_UPDATE_PERSONAL) {
	dbi_result_free(dbires);
	LOG_PRINT(LOG_WARNING, "ID not found");
	if (is_number(buffer)) { /* ID field contains numeric ID */
	  sprintf(sql_command, "410:"ULLSPEC":%s\n", (unsigned long long)(set_count + nongeek_offset), buffer);
	}
	else {
	  sprintf(sql_command, "411:"ULLSPEC":%s\n", (unsigned long long)(set_count + nongeek_offset), buffer);
	}
	if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	  return 0;
	}
	else {
	  *ptr_msg_pool = new_msgpool;
	}
	retval = 1;
	goto cleanup;
      }
      else if (!n_have_citekey) { /* if add, ignore numerical ID and assign a new one */
	LOG_PRINT(LOG_INFO, "ignore numerical ID");
	sprintf(sql_command, "409:"ULLSPEC"%s\n", (unsigned long long)(set_count + nongeek_offset), buffer);
	if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	  return 0;
	}
	else {
	  *ptr_msg_pool = new_msgpool;
	}
      }
    }
    dbi_result_free(dbires);
  }
  else { /* no ID string */
    if (replace_ref == ADDREF_UPDATE) {
      create_new = 1; /* if no ID string, simply add the dataset */
    }
    else if (replace_ref == ADDREF_UPDATE_PERSONAL){
      LOG_PRINT(LOG_WARNING, "ID missing");
      sprintf(sql_command, "412:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      retval = 1;
      goto cleanup;
    }
    /* else: add will assign new ID anyway */
  }

  /* if we're replacing, first remove the existing entry as far as necessary */
  if (!create_new && replace_ref != ADDREF_UPDATE_PERSONAL) {
/*     sprintf(sql_command, "try to replace reference "ULLSPEC"\n", (unsigned long long)n_refdb_id); */
/*     tiwrite(ptr_clrequest->fd, sql_command, TERM_NO); */

    /* search orphans in t_keyword */
/*      printf("orphans in t_keyword\n"); */
    result = remove_keyword_entries(n_refdb_id, conn, 0);
    if (result) {
      sprintf(sql_command, "229:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      retval = 1;
      goto cleanup;
    }

    /* search orphans in t_author */
    result = remove_author_entries(n_refdb_id, conn);
    if (result) {
      sprintf(sql_command, "230:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      retval = 1;
      goto cleanup;
    }

    /* search orphans in t_xuser */
    result = remove_xuser_entries(n_refdb_id, set_owner, conn);
    if (result != 0 && result != 4) {
      sprintf(sql_command, "253:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      retval = 1;
      goto cleanup;
    }

    /* search orphans in t_ulink */
    result = remove_ulink_entries(n_refdb_id, /*set_owner*/NULL, conn, 0 /* reference */);
    if (result != 0 && result != 4) {
      sprintf(sql_command, "253:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      retval = 1;
      goto cleanup;
    }

    /* search orphans in t_periodical */
    sprintf(sql_command, "SELECT refdb_periodical_id FROM t_%srefdb WHERE refdb_id="ULLSPEC, prefix, (unsigned long long)n_refdb_id);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires = dbi_conn_query(conn, sql_command);
    if (!dbires) {
      sprintf(sql_command, "231:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      retval = 1;
      goto cleanup;
    }

    while (dbi_result_next_row(dbires)) {
      n_periodical_id = my_dbi_result_get_idval(dbires, "refdb_periodical_id");
      if (n_periodical_id != 0 && my_dbi_conn_error_flag(conn) == 0) {
	if (remove_periodical_entries(n_periodical_id, conn)) {
	  sprintf(sql_command, "231:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  retval = 1;
	  goto cleanup;
	}
      }
    } /* end while */
    dbi_result_free(dbires);

    /* reset all values in t_refdb, keep citekey */
    sprintf(sql_command, "UPDATE t_%srefdb SET refdb_type=NULL, refdb_pubyear=NULL, refdb_secyear=NULL, refdb_startpage=NULL, refdb_endpage=NULL, refdb_abstract=NULL, refdb_title=NULL, refdb_volume=NULL, refdb_issue=NULL, refdb_booktitle=NULL, refdb_city=NULL, refdb_publisher=NULL, refdb_title_series=NULL, refdb_address=NULL, refdb_issn=NULL, refdb_pyother_info=NULL, refdb_secother_info=NULL, refdb_periodical_id=NULL, refdb_user1=NULL, refdb_user2=NULL, refdb_user3=NULL, refdb_user4=NULL, refdb_user5=NULL, refdb_typeofwork=NULL, refdb_area=NULL, refdb_ostype=NULL, refdb_degree=NULL, refdb_runningtime=NULL, refdb_classcodeintl=NULL, refdb_classcodeus=NULL, refdb_senderemail=NULL, refdb_recipientemail=NULL, refdb_mediatype=NULL, refdb_numvolumes=NULL, refdb_edition=NULL, refdb_computer=NULL, refdb_conferencelocation=NULL, refdb_registrynum=NULL, refdb_classification=NULL, refdb_section=NULL, refdb_pamphletnum=NULL, refdb_chapternum=NULL WHERE refdb_id="ULLSPEC, prefix, (unsigned long long)n_refdb_id);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires = dbi_conn_query(conn, sql_command);
    if (!dbires) {
      sprintf(sql_command, "232:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      LOG_PRINT(LOG_WARNING, get_status_msg(232));
      retval = 1;
      goto cleanup;
    }
    dbi_result_free(dbires);
  }

  /* start inserting the new data */
  if (create_new) {
    /* if my_hostname is too long, the db will chop off the string
       which is ok */
    sprintf(sql_command, "INSERT INTO t_%srefdb (refdb_type,refdb_citekey) VALUES (\'DUMMY\',\'DUMMY%s%d\')", prefix, ptr_clrequest->my_hostname, getpid());
    /* insert a new empty dataset into the main table to start with */
    LOG_PRINT(LOG_DEBUG, sql_command);
    dbires = dbi_conn_query(conn, sql_command);
    if (!dbires) {
      sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      LOG_PRINT(LOG_WARNING, get_status_msg(233));
      retval = 1;
      goto cleanup;
    }

    /* retrieve refdb_id of newly created dataset */
    if (!strcmp(my_dbi_conn_get_cap(conn, "named_seq"), "f")) {
      n_refdb_id = dbi_conn_sequence_last(conn, NULL);
    }
    else {
      sprintf(sql_command, "t_%srefdb_refdb_id_seq", prefix);
      n_refdb_id = dbi_conn_sequence_last(conn, sql_command);
    }

/*     sprintf(sql_command, "try to add set as reference "ULLSPEC"\n", (unsigned long long)n_refdb_id); */
/*     tiwrite(ptr_clrequest->fd, sql_command, TERM_NO); */

    dbi_result_free(dbires);
  }

  /* save the refdb ID in our risdata struct */
  ptr_risdata->n_id = n_refdb_id;

  /* create user and xuser entries */
  result = insert_user(set_owner, n_refdb_id, &n_user_id, conn, drivername, replace_ref, 0);

  if (result) {
    sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
    if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
      return 0;
    }
    else {
      *ptr_msg_pool = new_msgpool;
    }
    LOG_PRINT(LOG_WARNING, get_status_msg(233));
    retval = 1;
    goto cleanup;
  }
  /* else: all fine */


  /* cut reference into pieces */
  token = ris_strtok(set, &token_len, "\n");

  while (token != NULL) {
    if (token[token_len-1] == '\r') { /* cgi data have \r\n */
      token[token_len-1] = '\0';
    }
    token[token_len] = '\0';

    /* ignore empty tag lines */
    if (!token[6]) {
      token = ris_strtok(token + token_len + 1, &token_len, "\n");
      continue;
    }
/*     printf("token went to %s<<\n", token); */
    /* ------------------------------------------------------------ */
    if (strncmp("RP  - ", token, 6) == 0) {
      /* reprint status */
      if ((rp_type = ris_rpdate(date_buffer, &token[6])) != -1) {
	have_rp++;
	result = update_reprint(rp_type, n_refdb_id, n_user_id, conn, date_buffer, replace_ref);
	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
      }
    }

    /* ------------------------------------------------------------ */
    else if (strncmp("AB  - ", token, 6) == 0 || strncmp("N1  - ", token, 6) == 0) {
      /* notes */
      result = update_notes(&token[6], n_refdb_id, n_user_id, conn, driver, replace_ref);

      if (result) {
	sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	  return 0;
	}
	else {
	    *ptr_msg_pool = new_msgpool;
	}
	LOG_PRINT(LOG_WARNING, get_status_msg(233));
	retval = 1;
	goto cleanup;
      }
      /* else: all fine */
    }

    /* ------------------------------------------------------------ */
    else if (strncmp("AV  - ", token, 6) == 0) {
      /* availability */
      result = update_avail(&token[6], n_refdb_id, n_user_id, conn, driver, replace_ref);

      if (result) {
	sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	  return 0;
	}
	else {
	    *ptr_msg_pool = new_msgpool;
	}
	LOG_PRINT(LOG_WARNING, get_status_msg(233));
	retval = 1;
	goto cleanup;
      }
      /* else: all fine */
    }

    /* ------------------------------------------------------------ */
    else if (strncmp("L1  - ", token, 6) == 0 || strncmp("L2  - ", token, 6) == 0 || strncmp("L3  - ", token, 6) == 0 || strncmp("L4  - ", token, 6) == 0) {
      int n_type = 0;
      
      if (!strncmp("L1  - ", token, 6)) {
	n_type = 1;
      }
      else if (!strncmp("L2  - ", token, 6)) {
	n_type = 2;
      }
      else if (!strncmp("L3  - ", token, 6)) {
	n_type = 3;
      }
      else if (!strncmp("L4  - ", token, 6)) {
	n_type = 4;
      }

      result = insert_ulink(&token[6], n_type, n_refdb_id, conn, driver, drivername, n_user_id, 0 /* reference */, replace_ref);
      /* 	result = set_risdata_field(ptr_risdata, tofield, &token[6], conn); */
      if (result == 1) {
	sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	  return 0;
	}
	else {
	  *ptr_msg_pool = new_msgpool;
	}
	LOG_PRINT(LOG_WARNING, get_status_msg(233));
	retval = 1;
	goto cleanup;
      }
      /* else: all fine */
    }

    else if (replace_ref != ADDREF_UPDATE_PERSONAL) {
      /* ------------------------------------------------------------ */
      if (strncmp("TY  - ", token, 6) == 0) {
	/* reference type */
	if (!strcmp(&token[6], "DUMMY")) {
	  nis_dummy = 1;
	}
	else if (!strstr(REFTYPES, &token[6])) {
	  sprintf(sql_command, "426:%s->GEN\n", &token[6]);
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }

	  /* change type to default */
	  strcpy(type, "GEN");
	}
	else {
	  nis_dummy = 0;

	  /* save the type for later checks */
	  strncpy(type, &token[6], 6);
	  type[6] = '\0';
	}

	result = set_risdata_field(ptr_risdata, "type", type, conn);
	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("AU  - ", token, 6) == 0 || strncmp("A1  - ", token, 6) == 0 || strncmp("ED  - ", token, 6) == 0 || strncmp("A2  - ", token, 6) == 0 || strncmp("A3  - ", token, 6) == 0) {
	int author_type = 0;
	struct AUTHORTOKENS atokens;
	Lilistring* ptr_middle;

	/* reset the structure */
	reset_authorinfo(ptr_ainfo);

	/* author/editor/author series */
	strncpy(ptr_ainfo->name, &token[6], 255);
	(ptr_ainfo->name)[255] = '\0';
	  
	/* tokenize author string */
	if (!tokenize_author(&token[6], &atokens)) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}

	if (*atokens.first) {
	  strncpy(ptr_ainfo->firstname, atokens.first, 255);
	  (ptr_ainfo->firstname)[255] = '\0';
	}

	if (atokens.ptr_middlelist) {
	  ptr_middle = atokens.ptr_middlelist;
	  while ((ptr_middle = get_next_lilistring(ptr_middle)) != NULL) {
	    strncat(ptr_ainfo->middlename, ptr_middle->token, 255-strlen(ptr_ainfo->middlename));
	    strncat(ptr_ainfo->middlename, " ", 255-strlen(ptr_ainfo->middlename));
	  }
	  stripwhite(ptr_ainfo->middlename, 2, 0);
	}

	if (*atokens.sur) {
	  strncpy(ptr_ainfo->lastname, atokens.sur, 255);
	  (ptr_ainfo->lastname)[255] = '\0';
	}

	if (*atokens.lineage) {
	  strncpy(ptr_ainfo->suffix, atokens.lineage, 255);
	  (ptr_ainfo->suffix)[255] = '\0';
	}

	/* if we have name parts, assemble the full string again
	   to normalize it */
	if (*atokens.first || atokens.ptr_middlelist
	    || *atokens.sur || *atokens.lineage) {
	  char* temp_author;
	  int n_error = 0;

	  temp_author = assemble_full_author(ptr_ainfo, &n_error);

	  if (temp_author) {
	    /* truncate author string */
	    strncpy(ptr_ainfo->name, temp_author, 255);
	    (ptr_ainfo->name)[255] = '\0';
	    free(temp_author);
	  }
	}

	clean_authortokens(&atokens);

	/* try to untangle the RIS braindeadness related to the orthogonal concepts of author type and author role */
	if (strncmp("AU  - ", token, 6) == 0 || strncmp("A1  - ", token, 6) == 0) {
	  /* keep a copy of the first author for the citation key */
	  if (!*first_author) {
	    strncpy(first_author, &token[6], 255);
	    first_author[255] = '\0';
	  }

	  strcpy(ptr_ainfo->role, "author");

	  /* two options: author of part and author of publication */
	  if (has_part_data(type)) {
	    author_type = 1; /* part author */
	    result = insert_author(ptr_ainfo, author_type, part_xauthor_pos, n_refdb_id, conn, driver, drivername, replace_ref);
	  }
	  else {
	    author_type = 2; /* publication author */
	    result = insert_author(ptr_ainfo, author_type, publication_xauthor_pos, n_refdb_id, conn, driver, drivername, replace_ref);
	  }
	}
	else if (strncmp("A2  - ", token, 6) == 0 || strncmp("ED  - ", token, 6) == 0) {
	  /* this is usually an editor */
	  strcpy(ptr_ainfo->role, "editor");
	  author_type = 2;
	  result = insert_author(ptr_ainfo, author_type, publication_xauthor_pos, n_refdb_id, conn, driver, drivername, replace_ref);
	}
	else if (strncmp("A3  - ", token, 6) == 0) {
	  /* this is always an editor */
	  strcpy(ptr_ainfo->role, "editor");
	  author_type = 3;
	  result = insert_author(ptr_ainfo, 3 /*series editor*/, series_xauthor_pos, n_refdb_id, conn, driver, drivername, replace_ref);
	}

	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}

	/* increment author counter */
	if (author_type == 1) {
	  part_xauthor_pos++;
	}
	else if (author_type == 2) {
	  publication_xauthor_pos++;
	}
	else {
	  series_xauthor_pos++;
	}
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("PY  - ", token, 6) == 0 || strncmp("Y1  - ", token, 6) == 0 || strncmp("Y2  - ", token, 6) == 0) {
	/* use separate year variables for primary and secondary year */
	if (strncmp("Y2  - ", token, 6) != 0) {
	  have_py++;
	  year_type = 0;

	  /* publication year */
	  year = risdate(otherinfo_buffer, &token[6]);

	  result = set_risdata_dateinfo(ptr_risdata, year_type, year, otherinfo_buffer, conn);
	}
	else {
	  year_type = 1;

	  /* publication year */
	  year2 = risdate(otherinfo_buffer, &token[6]);

	  result = set_risdata_dateinfo(ptr_risdata, year_type, year2, otherinfo_buffer, conn);
	}

	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("SP  - ", token, 6) == 0) {
	/* start page number */
	result = set_risdata_field(ptr_risdata, "startpage", &token[6], conn);
	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("EP  - ", token, 6) == 0) {
	/* end page number */
	result = set_risdata_field(ptr_risdata, "endpage", &token[6], conn);
	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("KW  - ", token, 6) == 0) {
	/* keyword */
	result = insert_keyword(&token[6], n_refdb_id, conn, driver, drivername, 0, replace_ref);

	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }


      /* ------------------------------------------------------------ */
      else if (strncmp("TI  - ", token, 6) == 0 || strncmp("T1  - ", token, 6) == 0 || strncmp("CT  - ", token, 6) == 0) {

	if (token[token_len-1] == '.') { /* trim trailing period */
	  token[token_len-1] = '\0';
	}

	if (has_part_data(type)) {
	  /* part title */
	  result = set_risdata_field(ptr_risdata, "title", &token[6], conn);
	}
	else {
	  /* publication title */
	  result = set_risdata_field(ptr_risdata, "booktitle", &token[6], conn);
	}

	if (result == 1) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("BT  - ", token, 6) == 0 || strncmp("T2  - ", token, 6) == 0) {
	/* book title */
	result = set_risdata_field(ptr_risdata, "booktitle", &token[6], conn);
	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("T3  - ", token, 6) == 0) {
	/* title series, in COMP and PAT: doi */
	if (!strcmp(type, "COMP")
	    || !strcmp(type, "PAT")) {
	  result = insert_ulink(&token[6], 5 /* doi */, n_refdb_id, conn, driver, drivername, n_user_id, 0 /* reference */, replace_ref);
	}
	else {
	  result = set_risdata_field(ptr_risdata, "title_series", &token[6], conn);
	}

	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("JF  - ", token, 6) == 0) {
	/* journal full name */
	perinfo.full = strdup(&token[6]);
	if (perinfo.full == NULL || insert_lilimem(&sentinel, (void**)&(perinfo.full), "journal_full")) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}

	/* escape any characters that the database server cannot digest */
	if (dbi_conn_quote_string(conn, &(perinfo.full)) == 0) {
	  sprintf(sql_command, "801:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_CRIT, get_status_msg(801));
	  retval = 1;
	  goto cleanup;
	}
      }
      else if (strncmp("J1  - ", token, 6) == 0) {
	/* journal custom abbreviation 1 */
	perinfo.custabbrev1 = strdup(&token[6]);
	if (perinfo.custabbrev1 == NULL || insert_lilimem(&sentinel, (void**)&(perinfo.custabbrev1), "journal_custabbrev1")) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}

	if (dbi_conn_quote_string(conn, &(perinfo.custabbrev1)) == 0) {
	  sprintf(sql_command, "801:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_CRIT, get_status_msg(801));
	  retval = 1;
	  goto cleanup;
	}
      }
      else if (strncmp("J2  - ", token, 6) == 0) {
	/* journal custom abbreviation 2 */
	perinfo.custabbrev2 = strdup(&token[6]);
	if (perinfo.custabbrev2 == NULL || insert_lilimem(&sentinel, (void**)&(perinfo.custabbrev2), "journal_custabbrev2")) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}

	if (dbi_conn_quote_string(conn, &(perinfo.custabbrev2)) == 0) {
	  sprintf(sql_command, "801:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_CRIT, get_status_msg(801));
	  retval = 1;
	  goto cleanup;
	}
      }
      else if (strncmp("JO  - ", token, 6) == 0 || strncmp("JA  - ", token, 6) == 0) {
	/* journal abbreviation */
	perinfo.abbrev = malloc(strlen(&token[6]) + 2); /* Allocate one char extra to leave space for an additional period when the string is dotified */
	if (perinfo.abbrev == NULL || insert_lilimem(&sentinel, (void**)&(perinfo.abbrev), "journal_abbrev")) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	strcpy(perinfo.abbrev, &token[6]);

	/* add periods, use the connection to conn_refdb if necessary */
	dotify_journal(perinfo.abbrev, conn, conn_refdb);

	if (dbi_conn_quote_string(conn, &(perinfo.abbrev)) == 0) {
	  sprintf(sql_command, "801:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_CRIT, get_status_msg(801));
	  retval = 1;
	  goto cleanup;
	}
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("VL  - ", token, 6) == 0) {
	/* volume number */
	/* in BOOK  and CHAP entries this is the edition */
	if (!strcmp(type, "BOOK")
	    || !strcmp(type, "CHAP")
	    || !strcmp(type, "ART")
	    || !strcmp(type, "ADVS")
	    || !strcmp(type, "MUSIC")
	    || !strcmp(type, "NEWS")) {
	  result = set_risdata_field(ptr_risdata, "edition", &token[6], conn);
	}
	else {
	  result = set_risdata_field(ptr_risdata, "volume", &token[6], conn);
	}

	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("CY  - ", token, 6) == 0) {
	/* city of publication */
	result = set_risdata_field(ptr_risdata, "city", &token[6], conn);
	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("PB  - ", token, 6) == 0) {
	/* publisher */
	result = set_risdata_field(ptr_risdata, "publisher", &token[6], conn);
	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("CP  - ", token, 6) == 0
	       || strncmp("IS  - ", token, 6) == 0
	       || strncmp("ET  - ", token, 6) == 0) {
	/* chapter number or issue. ET goes here too although this is
	   not part of the RIS spec as of RefMan 8.01. However,
	   Endnote seems to use it for edition or something */
	/* MPCT, SOUND, and VIDEO use this field for doi */
	/* to add more confusion, BOOK uses IS to store VL. We also move CHAP's IS (which here means chapter number) to  */
	if (!strncmp("IS  - ", token, 6)
	    && (!strcmp(type, "MPCT")
		|| !strcmp(type, "SOUND")
		|| !strcmp(type, "VIDEO"))) {
	  result = insert_ulink(&token[6], 5 /* doi */, n_refdb_id, conn, driver, drivername, n_user_id, 0 /* reference */, replace_ref);
	}
	else if (!strcmp(type, "BOOK")
		 || !strcmp(type, "ADVS")
		 || !strcmp(type, "MUSIC")
		 || !strcmp(type, "NEWS")) {
	  result = set_risdata_field(ptr_risdata, "volume", &token[6], conn);
	}
	else if (!strcmp(type, "CHAP")) {
	  result = set_risdata_field(ptr_risdata, "chapternum", &token[6], conn);
	}
	else {
	  result = set_risdata_field(ptr_risdata, "issue", &token[6], conn);
	}

	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

    /* ------------------------------------------------------------ */
      else if (strncmp("N2  - ", token, 6) == 0) {
	/* abstract */
	result = set_risdata_field(ptr_risdata, "abstract", &token[6], conn);
	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("SN  - ", token, 6) == 0) {
	/* ISBN/ISSN */
	result = set_risdata_field(ptr_risdata, "issn", &token[6], conn);
	if (result == 1) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("AD  - ", token, 6) == 0) {
	/* address */
	result = set_risdata_field(ptr_risdata, "address", &token[6], conn);
	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("U1  - ", token, 6) == 0
	       || strncmp("U2  - ", token, 6) == 0
	       || strncmp("U3  - ", token, 6) == 0
	       || strncmp("U4  - ", token, 6) == 0
	       || strncmp("U5  - ", token, 6) == 0
	       || strncmp("M1  - ", token, 6) == 0
	       || strncmp("M2  - ", token, 6) == 0
	       || strncmp("M3  - ", token, 6) == 0) {
	char tofield[11];

	if (!strncmp("U1  - ", token, 6)) {
	  strcpy(tofield, "user1");
	}
	else if (!strncmp("U2  - ", token, 6)) {
	  strcpy(tofield, "user2");
	}
	else if (!strncmp("U3  - ", token, 6)) {
	  strcpy(tofield, "user3");
	}
	else if (!strncmp("U4  - ", token, 6)) {
	  strcpy(tofield, "user4");
	}
	else if (!strncmp("U5  - ", token, 6)) {
	  strcpy(tofield, "user5");
	}
	else if (!strncmp("M1  - ", token, 6)) {
	  strcpy(tofield, "misc1");
	}
	else if (!strncmp("M2  - ", token, 6)) {
	  if (!strcmp(type, "ADVS")) {
	    strcpy(tofield, "doi");
	  }
	  else {
	    strcpy(tofield, "misc2");
	  }
	}
	else if (!strncmp("M3  - ", token, 6)) {
	  if (strcmp(type, "ADVS")
	      && strcmp(type, "COMP")
	      && strcmp(type, "MPCT")
	      && strcmp(type, "PAT")
	      && strcmp(type, "SOUND")
	      && strcmp(type, "VIDEO")) {
	    strcpy(tofield, "doi");
	  }
	  else if (!strcmp(type, "ADVS")
		   || !strcmp(type, "COMP")
		   || !strcmp(type, "MPCT")
		   || !strcmp(type, "SOUND")
		   || !strcmp(type, "VIDEO")) {
	    strcpy(tofield, "mediatype");
	  }
	  else if (!strcmp(type, "PAT")) {
	    strcpy(tofield, "typeofwork");
	  }
	}

	if (!strcmp(tofield, "doi")) {
	  result = insert_ulink(&token[6], 5 /* doi */, n_refdb_id, conn, driver, drivername, n_user_id, 0 /* reference */, replace_ref);
	}
	else {
	  result = set_risdata_field(ptr_risdata, tofield, &token[6], conn);
	}

	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("UR  - ", token, 6) == 0) {
	/* URL */
	result = insert_ulink(&token[6], 0 /* url */, n_refdb_id, conn, driver, drivername, n_user_id, 0 /* reference */, replace_ref);

	if (result) {
	  sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	  if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	    return 0;
	  }
	  else {
	    *ptr_msg_pool = new_msgpool;
	  }
	  LOG_PRINT(LOG_WARNING, get_status_msg(233));
	  retval = 1;
	  goto cleanup;
	}
	/* else: all fine */
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("ID  - ", token, 6) == 0) {

	/* use ID value as citekey if non-numeric */
	if (n_have_citekey) {
	  char *new_citekey;

	  strncpy(real_citekey, &token[6], 255);
	  real_citekey[255] = '\0'; /* terminate string just in case */

	  if ((new_citekey = preprocess_citekey_copy(real_citekey, 255)) == NULL) {
	    return 0;
	  }
	  else {
	    real_citekey = new_citekey;
	  }

	  result = set_risdata_field(ptr_risdata, "citekey", real_citekey, conn);
	  if (result) {
	    sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
	    if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	      return 0;
	    }
	    else {
	      *ptr_msg_pool = new_msgpool;
	    }
	    LOG_PRINT(LOG_WARNING, get_status_msg(233));
	    retval = 1;
	    goto cleanup;
	  }
	  /* else: all fine */
	}
      }

      /* ------------------------------------------------------------ */
      else if (strncmp("ER  - ", token, 6) != 0) {
	/* reuse sql_command to assemble log message */
	sprintf(sql_command, "unknown token:%s<<%d:%d:%d:%d:%d:%d<<\n", token, (int)token[0], (int)token[1], (int)token[2], (int)token[3], (int)token[4], (int)token[5]);
	LOG_PRINT(LOG_WARNING, sql_command);
      }
    } /* end if not update personal */
    token = ris_strtok(token + token_len + 1, &token_len, "\n");
  }

  /* ------------------------------------------------------------ */
  /* insert journal information */

  result = insert_periodical(&perinfo, n_refdb_id, replace_ref, nis_dummy, conn, drivername);

  if (result) {
    sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
    if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
      return 0;
    }
    else {
      *ptr_msg_pool = new_msgpool;
    }
    LOG_PRINT(LOG_WARNING, get_status_msg(233));
    retval = 1;
    goto cleanup;
  }
  /* else: all fine */


  /* ------------------------------------------------------------ */
  /* insert default RP if none was specified */
  if (replace_ref != ADDREF_UPDATE_PERSONAL && !have_rp && !nis_dummy) {
    /* set a default reprint status if none was specified */
    result = update_reprint(1 /* not in file */, n_refdb_id, n_user_id, conn, NULL, replace_ref);
    if (result) {
      sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      LOG_PRINT(LOG_WARNING, get_status_msg(233));
      retval = 1;
      goto cleanup;
    }
  }

  /* ------------------------------------------------------------ */
  /* insert a default pubyear if none was specified */
  if (replace_ref != ADDREF_UPDATE_PERSONAL && !have_py && !nis_dummy) {
    /* set a default publication year if none was specified */
    result = set_risdata_dateinfo(ptr_risdata, 0 /* primary */, 0 /* default */, NULL, conn);

    if (result) {
      sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      LOG_PRINT(LOG_WARNING, get_status_msg(233));
      retval = 1;
      goto cleanup;
    }
    /* else: all fine */
  }
  
  /* ------------------------------------------------------------ */
  /* insert a default citekey if none was specified */
  if (!n_have_citekey && create_new) {
    citekey = get_unique_citekey(conn, first_author, year, 0 /* refs */, replace_ref);

    if (citekey && set_risdata_field(ptr_risdata, "citekey", citekey, conn) != 0) {
      if (citekey) {
	free((char*)citekey);
      }
      sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      LOG_PRINT(LOG_WARNING, get_status_msg(233));
      retval = 1;
      goto cleanup;
    }
    strcpy(real_citekey, citekey);
    free((char*)citekey);
  }

  if (insert_lilid(ptr_id_sentinel, n_refdb_id)) {
    LOG_PRINT(LOG_WARNING, get_status_msg(801));
  }
  
  if (*real_citekey) { /* still empty if we update the reference */
    sprintf(sql_command, "406:"ULLSPEC":%s:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset), real_citekey, n_refdb_id);
    if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
      return 0;
    }
    else {
      *ptr_msg_pool = new_msgpool;
    }
    LOG_PRINT(LOG_WARNING, sql_command);
  }

  if (replace_ref != ADDREF_UPDATE_PERSONAL) {
    /* now write the collected t_refdb data to the database */
    result = commit_risdata_fields(ptr_risdata, conn, replace_ref);
    if (result) {
      sprintf(sql_command, "233:"ULLSPEC"\n", (unsigned long long)(set_count + nongeek_offset));
      if ((new_msgpool = mstrcat(*ptr_msg_pool, sql_command, ptr_msg_pool_len, 0)) == NULL) {
	return 0;
      }
      else {
	*ptr_msg_pool = new_msgpool;
      }
      LOG_PRINT(LOG_WARNING, get_status_msg(233));
      retval = 1;
    }
    /* else: all fine */
  }
    
 cleanup:
  if (retval == 1) {
    if (strcmp(my_dbi_conn_get_cap(conn, "transaction"), "t")) {
      /* we have to delete the junk reference manually */
      if (n_refdb_id) { /* if no id, no dataset was added anyway */
	delete_ref_by_id(n_refdb_id, conn, ptr_clrequest, &addresult);
      }
    }
    else {
      my_dbi_conn_rollback(conn);
    }
    my_dbi_conn_unlock(conn);
    delete_all_lilimem(&sentinel);
    free_risdata(ptr_risdata);
    return 0;
  }
  else {
    my_dbi_conn_commit(conn);
  }
  
  my_dbi_conn_unlock(conn);
  delete_all_lilimem(&sentinel);
  free_risdata(ptr_risdata);

  if (retval == 2) {
    return 3;
  }
  else if (create_new == 1) {
    return 1; /* successfully added */
  }
  else {
    return 2; /* successfully updated */
  }
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  get_unique_citekey(): creates a unique citekey for a given reference
                        or note

  const char* get_unique_citekey returns a citekey string in
                  an allocated buffer that the calling function has
		  to free or NULL if some error occurs

  dbi_conn conn ptr to current database connection structure

  const char* first_author ptr to string with first author

  int year the publication year. If 0, don't use year info

  int type 0 = references 1 = notes

  int replace_ref indicates the context of the operation
                  ADDREF_ADD = add data
		  ADDREF_UPDATE = replace data
		  ADDREF_UPDATE_PERSONAL = update personal data
		  ADDREF_CHECK = add to temporary tables

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
const char* get_unique_citekey(dbi_conn conn, const char* first_author, int year, int type, int replace_ref) {
  char *citekey;
  char *new_citekey;
  char *sep;
  const char *my_author;
  char anonymous[] = "Anonymous";
  char yearstring[5] = ""; /* add no year info if no year is provided */
  char citekey_base[256] = "";
  char sql_command[1024] = "";
  char suffix[12] = "";
  char prefix[] = TEMP_TABLE_NAME_PREFIX;
  int n_authorlen;
  int n_unique = 0;
  dbi_result dbires;

  if ((citekey = malloc(CITEKEY_BASE_LENGTH+26)) == NULL) {
    LOG_PRINT(LOG_ERR, get_status_msg(801));
    return NULL;
  }

  /* fix insert */
  if (replace_ref != ADDREF_CHECK) {
    *prefix = '\0';
  }

  if (!first_author || !*first_author) {
    my_author = anonymous;
  }
  else {
    my_author = first_author;
  }

  sep = strchr(my_author, (int)',');
  if (sep) { /* first name ends at comma */
    n_authorlen = (sep-my_author > CITEKEY_BASE_LENGTH) ? CITEKEY_BASE_LENGTH : sep-my_author;
  }
  else { /* no comma, so use full name */
    n_authorlen = strlen(my_author);
    n_authorlen = (n_authorlen > CITEKEY_BASE_LENGTH) ? CITEKEY_BASE_LENGTH : n_authorlen;
  }

  strncpy(citekey_base, my_author, n_authorlen); /* leave space for year and suffix */
  citekey_base[n_authorlen] = '\0'; /* terminate just in case */

  /* remove any unwanted characters and uppercase string */
  /* leave room for year and up to 15 aabb characters */
  if ((new_citekey = preprocess_citekey_copy(citekey_base, CITEKEY_BASE_LENGTH)) == NULL) {
    return NULL;
  }

  
  if (year > 0 && year < 10000) {
    sprintf(yearstring, "%hd", year);
  }

  strcat(new_citekey, yearstring);

  /* suffix is empty string when the loop starts */
  while (!n_unique) {
    strcpy(citekey, new_citekey);
    strcat(citekey, suffix);

    /* now check whether the citekey already exists */
    if (!type) {
      sprintf(sql_command, "SELECT refdb_id FROM t_%srefdb WHERE refdb_citekey=\'%s\'", prefix, citekey);
    }
    else {
      sprintf(sql_command, "SELECT note_id FROM t_note WHERE note_key=\'%s\'", citekey);
    }

    LOG_PRINT(LOG_DEBUG, sql_command);
    
    dbires = dbi_conn_query(conn, sql_command);
    if (!dbires) {
      LOG_PRINT(LOG_WARNING, "Search ID failed");
      free(new_citekey);
      free(citekey);
      return NULL;
    }
      
    if (dbi_result_next_row(dbires) == 0) { /* citekey doesn't exist */
      n_unique++;
    }
    else {
      /* increment suffix */
      if (increment_suffix(suffix, 11, (*upper_citekey == 't') ? 1:0)) {
	free(citekey);
	free(new_citekey);
	return NULL;
      }
    }
  } /* end while */

  free(new_citekey);
  return citekey;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  find_journal(): matches strings against entries in t_periodical to
                  find existing entries

  static int find_journal returns the ID of a t_periodical set if a match is
                  found; returns 0 if no match is found or if an
                  error occurs

  int level indicates in which field to search (0 = journal_name, ...
                  3 = journal_custabbrev)

  struct PERIODICAL_INFO* ptr_perinfo ptr to struct containing ptrs
                  to properly quoted strings of the journal name synonyms
		  names must be < 256 chars

  dbi_conn conn ptr to current database connection structure

  int replace_ref indicates the context of the operation
                  ADDREF_ADD = add data
		  ADDREF_UPDATE = replace data
		  ADDREF_UPDATE_PERSONAL = update personal data
		  ADDREF_CHECK = add to temporary tables

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static unsigned long long find_journal(int level, struct PERIODICAL_INFO* ptr_perinfo, dbi_conn conn, int replace_ref) {
  char sql_command[512];
  char prefix[] = TEMP_TABLE_NAME_PREFIX;
  unsigned long long periodical_id; /* the ID of a match in t_periodical */
  dbi_result dbires;
  
  /* fix insert */
  if (replace_ref != ADDREF_CHECK) {
    *prefix = '\0';
  }

  /* search for existing entry in descending order of importance */
  if (ptr_perinfo->full != NULL && level == 0) {
    sprintf(sql_command, "SELECT periodical_id FROM t_%speriodical WHERE periodical_name=", prefix);
    strcat(sql_command, ptr_perinfo->full);
  }
  else if (ptr_perinfo->abbrev != NULL && level == 1) {
    sprintf(sql_command, "SELECT periodical_id FROM t_%speriodical WHERE periodical_abbrev=", prefix);
    strcat(sql_command, ptr_perinfo->abbrev);
  }
  else if (ptr_perinfo->custabbrev1 != NULL && level == 2) {
    sprintf(sql_command, "SELECT periodical_id FROM t_%speriodical WHERE periodical_custabbrev1=", prefix);
    strcat(sql_command, ptr_perinfo->custabbrev1);
  }
  else if (ptr_perinfo->custabbrev2 != NULL && level == 3) { /* journal_custabbrev2 != NULL */
    sprintf(sql_command, "SELECT periodical_id FROM t_%speriodical WHERE periodical_custabbrev2=", prefix);
    strcat(sql_command, ptr_perinfo->custabbrev2);
  }
  else {
    return 0;
  }

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "connect failed");
    return 0;
  }

  if (!dbi_result_next_row(dbires)) {
    dbi_result_free(dbires);
    return 0;
  }
  else {
    periodical_id = my_dbi_result_get_idval(dbires, "periodical_id");
    dbi_result_free(dbires);
    return periodical_id;
  }
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  insert_periodical(): inserts periodical info into t_periodical and t_refdb

  int insert_periodical returns 0 if ok, >0 if error
                         error codes: 1 = out of memory
                                      2 = insert JX failed
				      3 = driver not supported
				      4 = select from t_periodical failed
				      5 = read from t_periodical failed
				      6 = update JF failed
				      7 = update JO failed
				      8 = update J1 failed
				      9 = update J2 failed
				     10 = update periodical failed 

  struct PERIODICAL_INFO* ptr_perinfo

  unsigned long long id id of the row to update

  int replace_ref indicates the context of the operation
                  ADDREF_ADD = add data
		  ADDREF_UPDATE = replace data
		  ADDREF_UPDATE_PERSONAL = update personal data
		  ADDREF_CHECK = add to temporary tables

  int nis_dummy if 1, dataset is a dummy

  dbi_conn conn the database connection

  const char* drivername name of database driver

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int insert_periodical(struct PERIODICAL_INFO* ptr_perinfo, unsigned long long n_refdb_id, int replace_ref, int nis_dummy, dbi_conn conn, const char* drivername) {
  int n_jformats_avail = 0;
  int i;
  int is_first = 0;
  unsigned long long n_periodical_id;
  char* sql_command;
  char prefix[] = TEMP_TABLE_NAME_PREFIX;
  size_t sql_cmd_len = 2048;
  dbi_result dbires;
  dbi_result dbires1;

  if (!(sql_command = malloc(sql_cmd_len))) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    return 1;
  }

  /* fix insert */
  if (replace_ref != ADDREF_CHECK) {
    *prefix = '\0';
  }

  /* check which information is available. We do this rather here than in
     the main loop because a malformed RIS file (e.g. with two JO entries)
     would lead to erroneous counts */
  n_jformats_avail += (ptr_perinfo->full != NULL) ? 1 : 0;
  n_jformats_avail += (ptr_perinfo->abbrev != NULL) ? 1 : 0;
  n_jformats_avail += (ptr_perinfo->custabbrev1 != NULL) ? 1 : 0;
  n_jformats_avail += (ptr_perinfo->custabbrev2 != NULL) ? 1 : 0;

  if (replace_ref != ADDREF_UPDATE_PERSONAL
      && n_jformats_avail > 0
      && !nis_dummy) {
    /* set the periodical information from what we intermediately gathered */

    i = 0;
    n_periodical_id = 0L;

    /* loop over all available journal items and try to find a match in
       t_periodical */
    while (i < 4 && n_periodical_id == 0L) {
      n_periodical_id = find_journal(i, ptr_perinfo, conn, replace_ref);
      i++;
    }

    /* if no match was found, we insert a new entry into t_periodical */
    if (n_periodical_id == 0L) {
      sprintf(sql_command, "INSERT INTO t_%speriodical (", prefix);
      is_first = 1;

      if (ptr_perinfo->full != NULL) {
	strcat(sql_command, "periodical_name");
	is_first = 0; /* use colon for next item */
      }

      if (ptr_perinfo->abbrev != NULL) {
	if (is_first) {
	  strcat(sql_command, "periodical_abbrev");
	  is_first = 0; /* use colon for next item */
	}
	else {
	  strcat(sql_command, ", periodical_abbrev");
	}
      }

      if (ptr_perinfo->custabbrev1 != NULL) {
	if (is_first) {
	  strcat(sql_command, "periodical_custabbrev1");
	  is_first = 0; /* use colon for next item */
	}
	else {
	  strcat(sql_command, ", periodical_custabbrev1");
	}
      }

      if (ptr_perinfo->custabbrev2 != NULL) {
	if (is_first) {
	  strcat(sql_command, "periodical_custabbrev2");
	}
	else {
	  strcat(sql_command, ", periodical_custabbrev2");
	}
      }

      strcat(sql_command, ") VALUES (");

      is_first = 1;

      if (ptr_perinfo->full != NULL) {
	strcat(sql_command, ptr_perinfo->full);
	is_first = 0; /* use colon for next item */
      }

      if (ptr_perinfo->abbrev != NULL) {
	if (is_first) {
	  strcat(sql_command, ptr_perinfo->abbrev);
	  is_first = 0; /* use colon for next item */
	}
	else {
	  strcat(sql_command, ",");
	  strcat(sql_command, ptr_perinfo->abbrev);
	}
      }

      if (ptr_perinfo->custabbrev1 != NULL) {
	if (is_first) {
	  strcat(sql_command, ptr_perinfo->custabbrev1);
	  is_first = 0; /* use colon for next item */
	}
	else {
	  strcat(sql_command, ",");
	  strcat(sql_command, ptr_perinfo->custabbrev1);
	}
      }

      if (ptr_perinfo->custabbrev2 != NULL) {
	if (is_first) {
	  strcat(sql_command, ptr_perinfo->custabbrev2);
	}
	else {
	  strcat(sql_command, ",");
	  strcat(sql_command, ptr_perinfo->custabbrev2);
	}
      }

      strcat(sql_command, ")");

      LOG_PRINT(LOG_DEBUG, sql_command);

      dbires = dbi_conn_query(conn, sql_command);
      if (!dbires) {
	LOG_PRINT(LOG_WARNING, "insert JX failed");
	free(sql_command);
	return 2;
      }

      if (!strcmp(my_dbi_conn_get_cap(conn, "named_seq"), "f")) {
	n_periodical_id = dbi_conn_sequence_last(conn, NULL);
      }
      else {
	sprintf(sql_command, "t_%speriodical_periodical_id_seq", prefix);
	n_periodical_id = dbi_conn_sequence_last(conn, sql_command);
      }

      dbi_result_free(dbires);
    }
    else {
      /* we have a matching entry in t_periodical. See whether we can fill
	 in any missing information */
      sprintf(sql_command, "SELECT periodical_name, periodical_abbrev, periodical_custabbrev1, periodical_custabbrev2 FROM t_%speriodical WHERE periodical_id="ULLSPEC, prefix, (unsigned long long)n_periodical_id);

      LOG_PRINT(LOG_DEBUG, sql_command);

      dbires = dbi_conn_query(conn, sql_command);
      if (!dbires) {
	LOG_PRINT(LOG_WARNING, "select from t_periodical failed");
	free(sql_command);
	return 4;
      }

      if (!dbi_result_next_row(dbires)) {
	dbi_result_free(dbires);
	LOG_PRINT(LOG_WARNING, "read from t_periodical failed");
	free(sql_command);
	return 5;
      }
      else {
	if (/* dbi_result_get_string(dbires, "periodical_name") != NULL */
/* 	    &&  */ptr_perinfo->full != NULL) {
	  sprintf(sql_command, "UPDATE t_%speriodical SET periodical_name=%s WHERE periodical_id = "ULLSPEC, prefix, ptr_perinfo->full, (unsigned long long)n_periodical_id);

	  LOG_PRINT(LOG_DEBUG, sql_command);

	  dbires1 = dbi_conn_query(conn, sql_command);
	  if (!dbires1) {
	    dbi_result_free(dbires);
	    LOG_PRINT(LOG_WARNING, "update JF failed");
	    free(sql_command);
	    return 6;
	  }
	  dbi_result_free(dbires1);
	}
	if (/* dbi_result_get_string(dbires, "periodical_abbrev") &&  */ptr_perinfo->abbrev != NULL) {
	  sprintf(sql_command, "UPDATE t_%speriodical SET periodical_abbrev=%s WHERE periodical_id = "ULLSPEC, prefix, ptr_perinfo->abbrev, (unsigned long long)n_periodical_id);

	  LOG_PRINT(LOG_DEBUG, sql_command);

	  dbires1 = dbi_conn_query(conn, sql_command);
	  if (!dbires1) {
	    dbi_result_free(dbires);
	    LOG_PRINT(LOG_WARNING, "update JO failed");
	    free(sql_command);
	    return 7;
	  }
	  dbi_result_free(dbires1);
	}
	if (/* dbi_result_get_string(dbires, "periodical_custabbrev1") != NULL */
/* 	    &&  */ptr_perinfo->custabbrev1 != NULL) {
	  sprintf(sql_command, "UPDATE t_%speriodical SET periodical_custabbrev1=%s WHERE periodical_id = "ULLSPEC, prefix, ptr_perinfo->custabbrev1, (unsigned long long)n_periodical_id);

	  LOG_PRINT(LOG_DEBUG, sql_command);

	  dbires1 = dbi_conn_query(conn, sql_command);
	  if (!dbires1) {
	    dbi_result_free(dbires);
	    LOG_PRINT(LOG_WARNING, "update J1 failed");
	    free(sql_command);
	    return 8;
	  }
	  dbi_result_free(dbires1);
	}
	if (/* dbi_result_get_string(dbires, "periodical_custabbrev2") != NULL */
/* 	    &&  */ptr_perinfo->custabbrev2 != NULL) {
	  sprintf(sql_command, "UPDATE t_%speriodical SET periodical_custabbrev2=%s WHERE periodical_id = "ULLSPEC, prefix, ptr_perinfo->custabbrev2, (unsigned long long)n_periodical_id);

	  LOG_PRINT(LOG_DEBUG, sql_command);

	  dbires1 = dbi_conn_query(conn, sql_command);
	  if (!dbires1) {
	    dbi_result_free(dbires);
	    LOG_PRINT(LOG_WARNING, "update J2 failed");
	    free(sql_command);
	    return 9;
	  }
	  dbi_result_free(dbires1);
	}
	dbi_result_free(dbires);
      }
    }

    /* finally create entry in t_refdb table */
    sprintf(sql_command, "UPDATE t_%srefdb SET refdb_periodical_id="ULLSPEC" WHERE refdb_id = "ULLSPEC, prefix, (unsigned long long)n_periodical_id, (unsigned long long)n_refdb_id);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires = dbi_conn_query(conn, sql_command);

    if (!dbires) {
      LOG_PRINT(LOG_WARNING, "update periodical failed");
      free(sql_command);
      return 10;
    }
    dbi_result_free(dbires);
  }

  free(sql_command);
  return 0;
}


/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  insert_user(): checks for a user in the database and creates an
                 entry if the user doesn't exist yet

  int insert_user returns 0 if ok, >0 if error
                  error codes: 1 = search user failed
		               2 = insert t_user failed
			       3 = insert t_xuser failed
                               4 = username too long
			       5 = other
			       6 = query t_xuser failed
			       7 = pick reference failed
			       8 = out of memory
			       
  const char* set_owner name of user who owns the data

  unsigned long long n_refdb_id  reference ID

  unsigned long long* ptr_n_user_id  user ID (value will be updated)

  dbi_conn conn the database connection

  const char* drivername name of database driver

  int replace_ref indicates the context of the operation
                  ADDREF_ADD = add data
		  ADDREF_UPDATE = replace data
		  ADDREF_UPDATE_PERSONAL = update personal data
		  ADDREF_CHECK = add to temporary tables

  int mode 0 = reference entry 1 = note entry

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int insert_user(const char* set_owner, unsigned long long n_refdb_id, unsigned long long* ptr_n_user_id, dbi_conn conn, const char* drivername, int replace_ref, int mode) {
  char sql_command[256];
  char prefix[] = TEMP_TABLE_NAME_PREFIX;
  int nhave_xuser = 1;
  dbi_result dbires;
  dbi_result dbires1;

  if (!set_owner || !*set_owner) {
    /* nothing to do */
    return 0;
  }
  else if (strlen(set_owner) > DBUSER_LENGTH) {
    LOG_PRINT(LOG_WARNING, "username too long");
    return 4;
  }

  /* fix insert */
  if (replace_ref != ADDREF_CHECK) {
    *prefix = '\0';
  }

  /* create user and xuser entries */
  /* search for existing entry */
  sprintf(sql_command, "SELECT user_id FROM t_%suser WHERE user_name=\'%s\'", prefix, set_owner);

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "search user failed");
    return 1;
  }

  if (!dbi_result_next_row(dbires) || (*ptr_n_user_id = my_dbi_result_get_idval(dbires, "user_id")) == 0) { /* requested user not found */
    sprintf(sql_command, "INSERT INTO t_%suser (user_name) VALUES (\'%s\')", prefix, set_owner);
      
    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);
    if (!dbires1) {
      dbi_result_free(dbires);
      LOG_PRINT(LOG_WARNING, "insert user failed");
      return 2;
    }

    /* retrieve user_id of newly created user entry */
    if (!strcmp(my_dbi_conn_get_cap(conn, "named_seq"), "f")) {
      *ptr_n_user_id = dbi_conn_sequence_last(conn, NULL);
    }
    else {
      sprintf(sql_command, "t_%suser_user_id_seq", prefix);
      *ptr_n_user_id = dbi_conn_sequence_last(conn, sql_command);
    }
    dbi_result_free(dbires1);
  }

  if (!mode
      && (replace_ref == ADDREF_UPDATE
	  || replace_ref == ADDREF_UPDATE_PERSONAL)) {
    sprintf(sql_command, "SELECT user_id FROM t_xuser WHERE user_id="ULLSPEC" AND refdb_id="ULLSPEC, (unsigned long long)(*ptr_n_user_id), (unsigned long long)n_refdb_id);
    LOG_PRINT(LOG_DEBUG, sql_command);
    
    dbires1 = dbi_conn_query(conn, sql_command);
    if (!dbires1) {
      dbi_result_free(dbires);
      LOG_PRINT(LOG_WARNING, "query t_xuser failed");
      return 6;
    }
    if (!dbi_result_next_row(dbires1)) { /* requested entry missing */
      nhave_xuser = 0;
    }
    dbi_result_free(dbires1);
  }

  if (!mode && (replace_ref == ADDREF_ADD
		|| replace_ref == ADDREF_CHECK
		|| !nhave_xuser)) {
    int result;
    struct ADDRESULT* ptr_dummy_addresult = new_addresult(512);

    if (!ptr_dummy_addresult) {
      return 8;
    }

    /* create entry in xuser table */
    sprintf(sql_command, "INSERT INTO t_%sxuser (user_id, refdb_id) VALUES ("ULLSPEC","ULLSPEC")", prefix, (unsigned long long)(*ptr_n_user_id), (unsigned long long)n_refdb_id);
    
    LOG_PRINT(LOG_DEBUG, sql_command);
    
    dbires1 = dbi_conn_query(conn, sql_command);
    if (!dbires1) {
      free(ptr_dummy_addresult);
      dbi_result_free(dbires);
      LOG_PRINT(LOG_WARNING, "insert into t_xuser failed");
      return 3;
    }
    dbi_result_free(dbires1);

    if (replace_ref != ADDREF_CHECK) {
      /* add the ref to the default personal reference list */
      result = pick_one_reference(conn, n_refdb_id, *ptr_n_user_id, "" /* use default list */, set_owner, ptr_dummy_addresult);

      /* todo: should the message in ptr_dummy_addresult->msg be passed
	 to the calling function? */
      if (result) {
	free_addresult(ptr_dummy_addresult);
	dbi_result_free(dbires);
	return 7;
      }
    }
    free_addresult(ptr_dummy_addresult);
  }

  dbi_result_free(dbires);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  update_reprint(): updates the reprint info in the t_xuser table

  int update_reprint returns 0 if ok, 1 if error
			       
  int rp_type type of reprint 0=in file, 1=not in file, 2=on request

  unsigned long long n_refdb_id  reference ID

  unsigned long long n_user_id user id

  dbi_conn conn the database connection

  const char* date_buffer ptr to buffer holding date for on request type

  int replace_ref indicates the context of the operation
                  ADDREF_ADD = add data
		  ADDREF_UPDATE = replace data
		  ADDREF_UPDATE_PERSONAL = update personal data
		  ADDREF_CHECK = add to temporary tables

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int update_reprint(int rp_type, unsigned long long n_refdb_id, unsigned long long n_user_id, dbi_conn conn, const char* date_buffer, int replace_ref) {
  char sql_command[512];
  char buffer[128];
  char prefix[] = TEMP_TABLE_NAME_PREFIX;
  dbi_result dbires;

  /* fix insert */
  if (replace_ref != ADDREF_CHECK) {
    *prefix = '\0';
  }

  sprintf(sql_command, "UPDATE t_%sxuser SET xuser_reprint = \'", prefix);
  if (rp_type == 0) {
    strcat(sql_command, "IN FILE");
  }
  else if (rp_type == 1) {
    strcat(sql_command, "NOT IN FILE");
  }
  else {
    strcat(sql_command, "ON REQUEST");
  }
  sprintf(buffer, "\' WHERE refdb_id="ULLSPEC" AND user_id="ULLSPEC, (unsigned long long)n_refdb_id, (unsigned long long)n_user_id);
  strcat(sql_command, buffer);

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "update RP failed");
    return 1;
  }
  dbi_result_free(dbires);

  if (rp_type == 2 && date_buffer && *date_buffer) { /* on request */
    sprintf(sql_command, "UPDATE t_%sxuser SET xuser_date = \'", prefix);
    strcat(sql_command, date_buffer);
    sprintf(buffer, "\' WHERE refdb_id="ULLSPEC" AND user_id="ULLSPEC, (unsigned long long)n_refdb_id, (unsigned long long)n_user_id);
    strcat(sql_command, buffer);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires = dbi_conn_query(conn, sql_command);
    if (!dbires) {
      LOG_PRINT(LOG_WARNING, "update RP failed");
      return 1;
    }
    dbi_result_free(dbires);
  }
  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  update_notes(): updates the notes info in the t_xuser table

  int update_notes returns 0 if ok, 1 if malloc error, 2 if db error
			       
  const char* notes the notes string

  unsigned long long n_refdb_id  reference ID

  unsigned long long n_user_id user id

  dbi_conn conn the database connection

  dbi_driver driver ptr to database driver

  int replace_ref indicates the context of the operation
                  ADDREF_ADD = add data
		  ADDREF_UPDATE = replace data
		  ADDREF_UPDATE_PERSONAL = update personal data
		  ADDREF_CHECK = add to temporary tables

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int update_notes(const char* notes, unsigned long long n_refdb_id, unsigned long long n_user_id, dbi_conn conn, dbi_driver driver, int replace_ref) {
  char prefix[] = TEMP_TABLE_NAME_PREFIX;
  char sql_command[64];

  /* fix insert */
  if (replace_ref != ADDREF_CHECK) {
    *prefix = '\0';
  }

  sprintf(sql_command, "UPDATE t_%sxuser SET xuser_notes=", prefix);

  return update_user_field(notes, n_refdb_id, n_user_id, conn, driver, sql_command, "update AB failed");
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  update_avail(): updates the availability info in the t_xuser table

  int update_avail returns 0 if ok, 1 if malloc error, 2 if db error
			       
  const char* avail the availability string

  unsigned long long n_refdb_id  reference ID

  unsigned long long n_user_id user id

  dbi_conn conn the database connection

  dbi_driver driver ptr to database driver

  int replace_ref indicates the context of the operation
                  ADDREF_ADD = add data
		  ADDREF_UPDATE = replace data
		  ADDREF_UPDATE_PERSONAL = update personal data
		  ADDREF_CHECK = add to temporary tables

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int update_avail(const char* avail, unsigned long long n_refdb_id, unsigned long long n_user_id, dbi_conn conn, dbi_driver driver, int replace_ref) {
  char prefix[] = TEMP_TABLE_NAME_PREFIX;
  char sql_command[64];

  /* fix insert */
  if (replace_ref != ADDREF_CHECK) {
    *prefix = '\0';
  }

  sprintf(sql_command, "UPDATE t_%sxuser SET xuser_avail=", prefix);

  return update_user_field(avail, n_refdb_id, n_user_id, conn, driver, sql_command, "update AV failed");
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  update_user_field(): updates info in the t_xuser table

  static int update_user_field returns 0 if ok, 1 if malloc error, 2 if db error
			       
  const char* fieldvalue the value string

  unsigned long long n_refdb_id  reference ID

  unsigned long long n_user_id user id

  dbi_conn conn the database connection

  dbi_driver driver ptr to database driver

  const char* query_stub beginning of query string

  const char* errmsg error message for database error

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int update_user_field(const char* fieldvalue, unsigned long long n_refdb_id, unsigned long long n_user_id, dbi_conn conn, dbi_driver driver, const char* query_stub, const char* errmsg) {
  char* sql_command;
  char* new_sql_command;
  char* escape_buffer;
  char buffer[128];
  size_t sql_cmd_len = 4096;
  dbi_result dbires;

  if (!fieldvalue || !*fieldvalue) {
    /* nothing to do */
    return 0;
  }

  if (!(sql_command = malloc(sql_cmd_len))) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    return 1;
  }

  strcpy(sql_command, query_stub);

  escape_buffer = strdup(fieldvalue);

  if (!escape_buffer) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    free(sql_command);
    return 1;
  }

  /* escape any characters that the database server cannot digest */
  if (dbi_conn_quote_string(conn, &escape_buffer) == 0) {
    LOG_PRINT(LOG_WARNING, "out of memory");
    return 1;
  }

  /* use mstrcat as notes length is not limited */
  if ((new_sql_command = mstrcat(sql_command, escape_buffer, &sql_cmd_len, 0)) == NULL) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    free(escape_buffer);
    free(sql_command);
    return 1;
  }
  else {
    sql_command = new_sql_command;
  }

  free(escape_buffer);

  sprintf(buffer, " WHERE refdb_id="ULLSPEC" AND user_id="ULLSPEC, (unsigned long long)n_refdb_id, (unsigned long long)n_user_id);
  if ((new_sql_command = mstrcat(sql_command, buffer, &sql_cmd_len, 0)) == NULL) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    return 1;
  }
  else {
    sql_command = new_sql_command;
  }

  LOG_PRINT(LOG_DEBUG, sql_command);

  /* now send query */
  dbires = dbi_conn_query(conn, sql_command);
  free(sql_command);

  if (!dbires) {
    LOG_PRINT(LOG_WARNING, errmsg);
    return 2;
  }
  dbi_result_free(dbires);
  
  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  insert_author(): inserts author info into t_author and t_xauthor tables

  int insert_author returns 0 if ok, >0 if error
                    error codes: 1 = malloc failed
		                 2 = query AU failed
				 3 = insert AU failed
				 4 = driver not supported
				 5 = insert AU x failed
			       
  struct AUTHOR_INFO* ptr_ainfo ptr to structure with the name parts

  int author_type 1 = primary, 2 = secondary, 3 = tertiary

  int xauthor_pos position of author in the authorlist

  unsigned long long n_refdb_id  reference ID

  dbi_conn conn the database connection

  dbi_driver driver ptr to database driver

  const char* drivername name of database driver

  int replace_ref indicates the context of the operation
                  ADDREF_ADD = add data
		  ADDREF_UPDATE = replace data
		  ADDREF_UPDATE_PERSONAL = update personal data
		  ADDREF_CHECK = add to temporary tables

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int insert_author(struct AUTHOR_INFO* ptr_ainfo, int author_type, int xauthor_pos, unsigned long long n_refdb_id, dbi_conn conn, dbi_driver driver, const char* drivername, int replace_ref) {
  char* sql_command;
  char* new_sql_command;
  char cropped_author[256];
  char buffer[128];
  char prefix[] = TEMP_TABLE_NAME_PREFIX;
  char* escape_buffer;
  size_t sql_cmd_len = 512;
  unsigned long long n_author_id;
  dbi_result dbires;
  dbi_result dbires1;

  /* todo: continue here */
  if (ptr_ainfo && *(ptr_ainfo->name)) {
    /* make sure author doesn't exceed length limit */
    strncpy(cropped_author, ptr_ainfo->name, 255);
    cropped_author[255] = '\0';
  }
  else {
    /* nothing to do */
    return 0;
  }

  if (!(sql_command = malloc(sql_cmd_len))) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    return 1;
  }
    
  /* fix insert */
  if (replace_ref != ADDREF_CHECK) {
    *prefix = '\0';
  }

  /* first search for existing entry */
  sprintf(sql_command, "SELECT author_id FROM t_%sauthor WHERE author_name=", prefix);
  escape_buffer = strdup(cropped_author);
  if (escape_buffer == NULL) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    free(sql_command);
    return 1;
  }

  /* escape any characters that the database server cannot digest */
  if (dbi_conn_quote_string(conn, &escape_buffer) == 0) {
    LOG_PRINT(LOG_WARNING, "out of memory");
    free(sql_command);
    free(escape_buffer);
    return 1;
  }

  if ((new_sql_command = mstrcat(sql_command, escape_buffer, &sql_cmd_len, 0)) == NULL) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    free(sql_command);
    free(escape_buffer);
    return 1;
  }
  else {
    sql_command = new_sql_command;
  }

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "query AU failed");
    free(sql_command);
    free(escape_buffer);
    return 2;
  }

  if (dbi_result_next_row(dbires) == 0) {
    char* another_escape_buffer;
    /* next create entry in author table, if necessary */
    sprintf(sql_command, "INSERT INTO t_%sauthor (author_name, author_lastname, author_firstname, author_middlename, author_suffix) VALUES (", prefix);
    if ((new_sql_command = mstrcat(sql_command, escape_buffer, &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, "malloc failed");
      free(sql_command);
      free(escape_buffer);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    if ((new_sql_command = mstrcat(sql_command, ",", &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, "malloc failed");
      free(sql_command);
      free(escape_buffer);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    /* lastname ---------------------------------------------------- */
    if (*(ptr_ainfo->lastname)) {
      another_escape_buffer = strdup(ptr_ainfo->lastname);
      if (another_escape_buffer == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	return 1;
      }

      /* escape any characters that the database server cannot digest */
      if (dbi_conn_quote_string(conn, &another_escape_buffer) == 0) {
	LOG_PRINT(LOG_WARNING, "out of memory");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }

      if ((new_sql_command = mstrcat(sql_command, another_escape_buffer, &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
      free(another_escape_buffer);
    }
    else {
      if ((new_sql_command = mstrcat(sql_command, "NULL", &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
    }

    if ((new_sql_command = mstrcat(sql_command, ",", &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, "malloc failed");
      free(sql_command);
      free(escape_buffer);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }


    /* firstname ---------------------------------------------------- */
    if (*(ptr_ainfo->firstname)) {
      another_escape_buffer = strdup(ptr_ainfo->firstname);
      if (another_escape_buffer == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	return 1;
      }

      /* escape any characters that the database server cannot digest */
      if (dbi_conn_quote_string(conn, &another_escape_buffer) == 0) {
	LOG_PRINT(LOG_WARNING, "out of memory");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }

      if ((new_sql_command = mstrcat(sql_command, another_escape_buffer, &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
      free(another_escape_buffer);
    }
    else {
      if ((new_sql_command = mstrcat(sql_command, "NULL", &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
    }

    if ((new_sql_command = mstrcat(sql_command, ",", &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, "malloc failed");
      free(sql_command);
      free(escape_buffer);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }


    /* middlename ---------------------------------------------------- */
    if (*(ptr_ainfo->middlename)) {
      another_escape_buffer = strdup(ptr_ainfo->middlename);
      if (another_escape_buffer == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	return 1;
      }

      /* escape any characters that the database server cannot digest */
      if (dbi_conn_quote_string(conn, &another_escape_buffer) == 0) {
	LOG_PRINT(LOG_WARNING, "out of memory");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }

      if ((new_sql_command = mstrcat(sql_command, another_escape_buffer, &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
      free(another_escape_buffer);
    }
    else {
      if ((new_sql_command = mstrcat(sql_command, "NULL", &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
    }

    if ((new_sql_command = mstrcat(sql_command, ",", &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, "malloc failed");
      free(sql_command);
      free(escape_buffer);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }


    /* suffix ---------------------------------------------------- */
    if (*(ptr_ainfo->suffix)) {
      another_escape_buffer = strdup(ptr_ainfo->suffix);
      if (another_escape_buffer == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	return 1;
      }

      /* escape any characters that the database server cannot digest */
      if (dbi_conn_quote_string(conn, &another_escape_buffer) == 0) {
	LOG_PRINT(LOG_WARNING, "out of memory");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }

      if ((new_sql_command = mstrcat(sql_command, another_escape_buffer, &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
      free(another_escape_buffer);
    }
    else {
      if ((new_sql_command = mstrcat(sql_command, "NULL", &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(sql_command);
	free(escape_buffer);
	free(another_escape_buffer);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
    }

    if ((new_sql_command = mstrcat(sql_command, ")", &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, "malloc failed");
      free(sql_command);
      free(escape_buffer);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);
    if (!dbires1) {
      LOG_PRINT(LOG_WARNING, "insert AU failed");
      free(sql_command);
      free(escape_buffer);
      return 3;
    }
    dbi_result_free(dbires1);

    if (!strcmp(my_dbi_conn_get_cap(conn, "named_seq"), "f")) {
      n_author_id = dbi_conn_sequence_last(conn, NULL);
    }
    else {
      sprintf(sql_command, "t_%sauthor_author_id_seq", prefix);
      n_author_id = dbi_conn_sequence_last(conn, sql_command);
    }
  }
  else {
    n_author_id = my_dbi_result_get_idval(dbires, "author_id");
  }
  dbi_result_free(dbires);
  free(escape_buffer);

  /* finally create entry in author xref table */
  sprintf(sql_command, "INSERT INTO t_%sxauthor (author_id, refdb_id, xauthor_type, xauthor_position, xauthor_role) VALUES (", prefix);

  escape_buffer = strdup(ptr_ainfo->role);
  if (escape_buffer == NULL) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    free(sql_command);
    return 1;
  }

  /* escape any characters that the database server cannot digest */
  if (dbi_conn_quote_string(conn, &escape_buffer) == 0) {
    LOG_PRINT(LOG_WARNING, "out of memory");
    free(sql_command);
    free(escape_buffer);
    return 1;
  }

  sprintf(buffer, ULLSPEC", "ULLSPEC", %s, %d, %s)", (unsigned long long)n_author_id, (unsigned long long)n_refdb_id, get_author_type_string(driver, author_type), xauthor_pos, escape_buffer);

  strcat(sql_command, buffer);
  free(escape_buffer);

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "insert AU x failed");
    free(sql_command);
    return 5;
  }
  dbi_result_free(dbires);
  free(sql_command);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  update_dateinfo(): updates pubyear and otherinfo fields

  int update dateinfo returns 0 if ok, >0 if error
                    error codes: 1 = malloc failed
		                 2 = update YX failed
				 3 = update YX (other) failed
			       
  int year_type 0 = primary, 1 = secondary

  int year the year

  const char* otherinfo string containing otherinfo

  unsigned long long n_refdb_id  reference ID

  dbi_conn conn the database connection

  dbi_driver driver ptr to database driver

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int update_dateinfo(int date_type, int year, const char* otherinfo, unsigned long long n_refdb_id, dbi_conn conn, dbi_driver driver) {
  char cropped_otherinfo[256];
  char* sql_command;
  char* escape_buffer;
  size_t sql_cmd_len = 512;
  dbi_result dbires;

  if (otherinfo && *otherinfo) {
    strncpy(cropped_otherinfo, otherinfo, 255);
    cropped_otherinfo[255] = '\0';
  }
  else {
    cropped_otherinfo[0] = '\0';
  }

  if (!(sql_command = malloc(sql_cmd_len))) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    return 1;
  }

  if (!date_type) {
    sprintf(sql_command, "UPDATE t_refdb SET refdb_pubyear=%hd WHERE refdb_id="ULLSPEC, year, (unsigned long long)n_refdb_id);
  }
  else {
    sprintf(sql_command, "UPDATE t_refdb SET refdb_secyear=%hd WHERE refdb_id="ULLSPEC, year, (unsigned long long)n_refdb_id);
  }

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "update YX failed");
    free(sql_command);
    return 2;
  }

  dbi_result_free(dbires);

  if (!*cropped_otherinfo) {
    /* we're done */
    free(sql_command);
    return 0;
  }

  escape_buffer = strdup(cropped_otherinfo);
  if (escape_buffer == NULL) {
    free(sql_command);
    LOG_PRINT(LOG_WARNING, "malloc failed");
    return 1;
  }

  /* escape any characters that the database server cannot digest */
  if (dbi_conn_quote_string(conn, &escape_buffer) == 0) {
    LOG_PRINT(LOG_WARNING, "out of memory");
    free(sql_command);
    free(escape_buffer);
    return 1;
  }

  if (!date_type) {
    sprintf(sql_command, "UPDATE t_refdb SET refdb_pyother_info=%s WHERE refdb_id="ULLSPEC, escape_buffer, (unsigned long long)n_refdb_id);
  }
  else {
    sprintf(sql_command, "UPDATE t_refdb SET refdb_secother_info=%s WHERE refdb_id="ULLSPEC, escape_buffer, (unsigned long long)n_refdb_id);
  }

  free(escape_buffer);

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    free(sql_command);
    LOG_PRINT(LOG_WARNING, "update YX (other) failed");
    return 3;
  }
  dbi_result_free(dbires);
  free(sql_command);

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  insert_keyword(): inserts keyword info into t_keyword and t_xkeyword

  int insert_keyword returns 0 if ok, >0 if error
                    error codes: 1 = malloc failed
		                 2 = query KW failed
				 3 = insert KW failed
				 4 = driver not supported
				 5 = insert KW x failed
			       
  const char* keyword string containing keyword

  unsigned long long n_xref_id  reference or note ID

  dbi_conn conn the database connection

  dbi_driver driver ptr to database driver

  const char* drivername name of database driver

  int mode 0 = reference 1 = note

  int replace_ref indicates the context of the operation
                  ADDREF_ADD = add data
		  ADDREF_UPDATE = replace data
		  ADDREF_UPDATE_PERSONAL = update personal data
		  ADDREF_CHECK = add to temporary tables

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int insert_keyword(const char* keyword, unsigned long long n_xref_id, dbi_conn conn, dbi_driver driver, const char* drivername, int mode, int replace_ref) {
  char* sql_command;
  char* new_sql_command;
  char* escape_buffer;
  char cropped_keyword[256];
  char prefix[] = TEMP_TABLE_NAME_PREFIX;
  size_t sql_cmd_len = 512;
  unsigned long long n_keyword_id;
  dbi_result dbires;
  dbi_result dbires1;

  if (!keyword || !*keyword) {
    /* we're done */
    return 0;
  }

  if (!(sql_command = malloc(sql_cmd_len))) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    return 1;
  }

  /* fix insert */
  if (replace_ref != ADDREF_CHECK) {
    *prefix = '\0';
  }

  strncpy(cropped_keyword, keyword, 255);
  cropped_keyword[255] = '\0';
    
  /* first search for existing entry */
  sprintf(sql_command, "SELECT keyword_id FROM t_%skeyword WHERE keyword_name=", prefix);
  escape_buffer = strdup(keyword);
  if (escape_buffer == NULL) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    free(sql_command);
    return 1;
  }

  /* escape any characters that the database server cannot digest */
  if (dbi_conn_quote_string(conn, &escape_buffer) == 0) {
    LOG_PRINT(LOG_WARNING, "out of memory");
    return 1;
  }

  if ((new_sql_command = mstrcat(sql_command, escape_buffer, &sql_cmd_len, 0)) == NULL) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    free(escape_buffer);
    free(sql_command);
    return 1;
  }
  else {
    sql_command = new_sql_command;
  }

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "query KW failed");
    return 2;
  }

  if (dbi_result_next_row(dbires) == 0) {
    /* next create entry in keyword table, if necessary */
    sprintf(sql_command, "INSERT INTO t_%skeyword (keyword_name) VALUES (", prefix);
    if ((new_sql_command = mstrcat(sql_command, escape_buffer, &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, "malloc failed");
      free(escape_buffer);
      free(sql_command);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    if ((new_sql_command = mstrcat(sql_command, ")", &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, "malloc failed");
      free(escape_buffer);
      free(sql_command);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }
    
    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);
    if (!dbires1) {
      LOG_PRINT(LOG_WARNING, "insert KW failed");
      free(escape_buffer);
      free(sql_command);
      return 3;
    }
    dbi_result_free(dbires1);

    if (!strcmp(my_dbi_conn_get_cap(conn, "named_seq"), "f")) {
      n_keyword_id = dbi_conn_sequence_last(conn, NULL);
    }
    else {
      sprintf(sql_command, "t_%skeyword_keyword_id_seq", prefix);
      n_keyword_id = dbi_conn_sequence_last(conn, sql_command);
    }
  }
  else {
    n_keyword_id = my_dbi_result_get_idval(dbires, "keyword_id");
  }
  dbi_result_free(dbires);
  free(escape_buffer);

  /* finally create entry in keyword xref table */
  sprintf(sql_command, "INSERT INTO t_%sxkeyword (keyword_id, xref_id, xkeyword_type) VALUES ("ULLSPEC", "ULLSPEC", \'%s\')", prefix, (unsigned long long)n_keyword_id, (unsigned long long)n_xref_id, (mode) ? "NOTE":"REFERENCE");

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);
  free(sql_command);
  
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "insert KW x failed");
    return 5;
  }
  dbi_result_free(dbires);
  
  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  insert_ulink(): inserts url info into t_link and t_xlink

  int insert_ulink returns 0 if ok, >0 if error
                    error codes: 1 = malloc failed
		                 2 = query UR failed
				 3 = insert UR failed
				 4 = driver not supported
				 5 = insert UR x failed
			       
  const char* ulink string containing keyword

  int type 0 = URL, 1 = PDF, 2 = FULLTEXT 3 = RELATED 4 = IMAGE 5 = DOI

  unsigned long long n_xref_id  reference or note ID

  dbi_conn conn the database connection

  dbi_driver driver ptr to database driver

  const char* drivername name of database driver

  unsigned long long n_user_id ID of reference owner. This is used only
                  for L1-L4 links and only if these use the protocol
                  file://. These links are assumed to point to a local
                  repository owned by the user. Only these are supposed
		  to be updated if replace_ref is ADDREF_UPDATE_PERSONAL

  int mode 0 = reference 1 = note

  int replace_ref indicates the context of the operation
                  ADDREF_ADD = add data
		  ADDREF_UPDATE = replace data
		  ADDREF_UPDATE_PERSONAL = update personal data
		  ADDREF_CHECK = add to temporary tables

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int insert_ulink(const char* ulink, int type, unsigned long long n_xref_id, dbi_conn conn, dbi_driver driver, const char* drivername, unsigned long long n_user_id, int mode, int replace_ref) {
  char* sql_command;
  char* new_sql_command;
  char* escape_buffer;
/*   char cropped_ulink[256]; */
  char type_string[9];
  char prefix[] = TEMP_TABLE_NAME_PREFIX;
  size_t sql_cmd_len = 512;
  int is_filelink = 0;
  unsigned long long n_ulink_id;
  dbi_result dbires;
  dbi_result dbires1;

  if (!ulink || !*ulink) {
    /* we're done */
    return 0;
  }

  if (!(sql_command = malloc(sql_cmd_len))) {
    LOG_PRINT(LOG_WARNING, "malloc failed");
    return 1;
  }

  /* fix insert */
  if (replace_ref != ADDREF_CHECK) {
    *prefix = '\0';
  }

/*   strncpy(cropped_ulink, ulink, 255); */
/*   cropped_ulink[255] = '\0'; */
    
  if (!type) {
    strcpy(type_string, "URL");
  }
  else if (type == 1) {
    strcpy(type_string, "PDF");
    is_filelink = check_filelink(ulink);
  }
  else if (type == 2) {
    strcpy(type_string, "FULLTEXT");
    is_filelink = check_filelink(ulink);
  }
  else if (type == 3) {
    strcpy(type_string, "RELATED");
    is_filelink = check_filelink(ulink);
  }
  else if (type == 4) {
    strcpy(type_string, "IMAGE");
    is_filelink = check_filelink(ulink);
  }
  else if (type == 5) {
    strcpy(type_string, "DOI");
  }

  if ((type > 0 && type < 5) || replace_ref != ADDREF_UPDATE_PERSONAL) {
    /* first search for existing entry */
    sprintf(sql_command, "SELECT link_id FROM t_%slink WHERE link_url=", prefix);
    escape_buffer = strdup(ulink);
    if (escape_buffer == NULL) {
      LOG_PRINT(LOG_WARNING, "malloc failed");
      free(sql_command);
      return 1;
    }

    /* escape any characters that the database server cannot digest */
    if (dbi_conn_quote_string(conn, &escape_buffer) == 0) {
      LOG_PRINT(LOG_WARNING, "out of memory");
      return 1;
    }

    if ((new_sql_command = mstrcat(sql_command, escape_buffer, &sql_cmd_len, 0)) == NULL) {
      LOG_PRINT(LOG_WARNING, "malloc failed");
      free(escape_buffer);
      free(sql_command);
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires = dbi_conn_query(conn, sql_command);
    if (!dbires) {
      const char* errmsg;
      dbi_conn_error(conn, &errmsg);
      LOG_PRINT(LOG_WARNING, "query UR failed");
      LOG_PRINT(LOG_DEBUG, errmsg);
      return 2;
    }
    
    if (dbi_result_next_row(dbires) == 0) {
      /* next create entry in link table, if necessary */
      sprintf(sql_command, "INSERT INTO t_%slink (link_url) VALUES (", prefix);
      if ((new_sql_command = mstrcat(sql_command, escape_buffer, &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(escape_buffer);
	free(sql_command);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }

      if ((new_sql_command = mstrcat(sql_command, ")", &sql_cmd_len, 0)) == NULL) {
	LOG_PRINT(LOG_WARNING, "malloc failed");
	free(escape_buffer);
	free(sql_command);
	return 1;
      }
      else {
	sql_command = new_sql_command;
      }
      
      LOG_PRINT(LOG_DEBUG, sql_command);
      
      dbires1 = dbi_conn_query(conn, sql_command);
      if (!dbires1) {
	LOG_PRINT(LOG_WARNING, "insert UR failed");
	free(escape_buffer);
	free(sql_command);
	return 3;
      }
      dbi_result_free(dbires1);
      
      if (!strcmp(my_dbi_conn_get_cap(conn, "named_seq"), "f")) {
	n_ulink_id = dbi_conn_sequence_last(conn, NULL);
      }
      else {
	sprintf(sql_command, "t_%slink_link_id_seq", prefix);
	n_ulink_id = dbi_conn_sequence_last(conn, sql_command);
      }
    }
    else {
      n_ulink_id = my_dbi_result_get_idval(dbires, "link_id");
    }
    dbi_result_free(dbires);
    free(escape_buffer);

    /* finally create entry in ulink xref table */
/*       printf("is_filelink went to %d; n_user_id was "ULLSPEC"\n", is_filelink, n_user_id); */
    if (is_filelink && n_user_id) {
      sprintf(sql_command, "INSERT INTO t_%sxlink (link_id, xref_id, user_id, xlink_type, xlink_source) VALUES ("ULLSPEC", "ULLSPEC", "ULLSPEC", \'%s\', \'%s\')", prefix, (unsigned long long)n_ulink_id, (unsigned long long)n_xref_id, (unsigned long long)n_user_id, type_string, (mode) ? "NOTE":"REFERENCE");
    }
    else {
      sprintf(sql_command, "INSERT INTO t_%sxlink (link_id, xref_id, xlink_type, xlink_source) VALUES ("ULLSPEC", "ULLSPEC", \'%s\', \'%s\')", prefix, (unsigned long long)n_ulink_id, (unsigned long long)n_xref_id, type_string, (mode) ? "NOTE":"REFERENCE");
    }
    
    LOG_PRINT(LOG_DEBUG, sql_command);
    
    dbires = dbi_conn_query(conn, sql_command);
    free(sql_command);
  
    if (!dbires) {
      LOG_PRINT(LOG_WARNING, "insert UR x failed");
      return 5;
    }
    dbi_result_free(dbires);
  } /* end if either L1-L4 or not update personal */

  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  check_filelink(): checks whether an URL uses the file:// protocol

  int check_filelink returns 1 if the URL starts with file://
                             0 if not

  const char* url ptr to URL

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static int check_filelink(const char* url) {
  if (!url || !*url) {
    return 0;
  }

  if (!strncmp(url, "file://", 7)) {
    return 1;
  }
  else {
    return 0;
  }
}


/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  remove_keyword_entries(): removes keyword entries associated with
                            a particular dataset

  int remove_keyword_entries returns 0 if ok, > 0 if error
                             error codes: 1 = select from t_xkeyword failed
			                  2 = delete from t_keyword failed
			                  3 = delete from t_xkeyword failed
					  4 = delete from t_xnote failed

  unsigned long long ref_id  reference ID

  dbi_conn conn the database connection

  int mode 0 = reference entry 1 = note entry

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int remove_keyword_entries(unsigned long long ref_id, dbi_conn conn, int mode) {
  char sql_command[256];
  unsigned long long n_keyword_id; /* keyword_id of a search result */
  unsigned long long n_xkeyword_id; /* xkeyword_id of a search result */
  unsigned long long numrows;
  dbi_result dbires;
  dbi_result dbires1;
  dbi_result dbires2;

  sprintf(sql_command, "SELECT keyword_id, xkeyword_id FROM t_xkeyword WHERE xkeyword_type=\'%s\' AND xref_id="ULLSPEC, (mode) ? "NOTE":"REFERENCE", (unsigned long long)ref_id);
  LOG_PRINT(LOG_DEBUG, sql_command);
  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "select from t_xkeyword failed");
    return 1;
  }

  while (dbi_result_next_row(dbires)) {
    n_keyword_id = my_dbi_result_get_idval(dbires, "keyword_id");
    n_xkeyword_id = my_dbi_result_get_idval(dbires, "xkeyword_id");

    sprintf(sql_command, "SELECT xkeyword_id FROM t_xkeyword WHERE keyword_id="ULLSPEC, (unsigned long long)n_keyword_id);
    LOG_PRINT(LOG_DEBUG, sql_command);
    dbires1 = dbi_conn_query(conn, sql_command);
    if (!dbires1) {
      dbi_result_free(dbires);
      LOG_PRINT(LOG_WARNING, "select from t_xkeyword failed");
      return 1;
    }
    
    numrows = dbi_result_get_numrows(dbires1);

    if (numrows == 1) { /* if no other reference uses this keyword */
      int result;

      /* delete entry in keyword table */
      sprintf(sql_command, "DELETE FROM t_keyword WHERE keyword_id="ULLSPEC, (unsigned long long)n_keyword_id);
      LOG_PRINT(LOG_DEBUG, sql_command);
      dbires2 = dbi_conn_query(conn, sql_command);
      if (!dbires2) {
	dbi_result_free(dbires);
	dbi_result_free(dbires1);
	LOG_PRINT(LOG_WARNING, "delete from t_keyword failed");
	return 2;
      }
      dbi_result_free(dbires2);

      /* remove orphans in t_xnote */
      result = remove_xnote_entries(0 /* any note */, n_keyword_id, conn, 2 /*keyword*/);

      if (result != 0 && result != 4) {
	dbi_result_free(dbires1);
	dbi_result_free(dbires);
	return 4;
      }
    }
      
    /* delete entry in xkeyword table */
    sprintf(sql_command, "DELETE FROM t_xkeyword WHERE xkeyword_id="ULLSPEC, (unsigned long long)n_xkeyword_id);
    LOG_PRINT(LOG_DEBUG, sql_command);
    dbires2 = dbi_conn_query(conn, sql_command);
    if (!dbires2) {
      dbi_result_free(dbires);
      dbi_result_free(dbires1);
      LOG_PRINT(LOG_WARNING, "delete from t_xkeyword failed");
      return 3;
    }
    dbi_result_free(dbires2);
    dbi_result_free(dbires1);
  }

  dbi_result_free(dbires);
  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  remove_ulink_entries(): removes ulink entries associated with
                            a particular dataset

  int remove_ulink_entries returns 0 if ok, > 0 if error
                             error codes: 1 = select from t_xlink failed
			                  2 = delete from t_link failed
			                  3 = delete from t_xlink failed

  unsigned long long ref_id  reference ID

  const char* set_owner name of dataset owner (may be NULL to remove all)

  dbi_conn conn the database connection

  int mode 0 = reference entry 1 = note entry

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int remove_ulink_entries(unsigned long long ref_id, const char* set_owner, dbi_conn conn, int mode) {
  char sql_command[256];
  unsigned long long n_ulink_id; /* ulink_id of a search result */
  unsigned long long n_xulink_id; /* xulink_id of a search result */
  unsigned long long numrows;
  dbi_result dbires;
  dbi_result dbires1;
  dbi_result dbires2;

  if (set_owner && *set_owner) {
    sprintf(sql_command, "SELECT link_id, xlink_id FROM t_xlink INNER JOIN t_user ON t_xlink.user_id=t_user.user_id WHERE xlink_source=\'%s\' AND xref_id="ULLSPEC" AND t_user.user_name='%s'", (mode) ? "NOTE":"REFERENCE", (unsigned long long)ref_id, set_owner);
  }
  else {
    sprintf(sql_command, "SELECT link_id, xlink_id FROM t_xlink WHERE xlink_source=\'%s\' AND xref_id="ULLSPEC, (mode) ? "NOTE":"REFERENCE", (unsigned long long)ref_id);
  }

  LOG_PRINT(LOG_DEBUG, sql_command);
  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "select from t_xlink failed");
    return 1;
  }

  while (dbi_result_next_row(dbires)) {
    n_ulink_id = my_dbi_result_get_idval(dbires, "link_id");
    n_xulink_id = my_dbi_result_get_idval(dbires, "xlink_id");

    sprintf(sql_command, "SELECT xlink_id FROM t_xlink WHERE link_id="ULLSPEC, (unsigned long long)n_ulink_id);
    LOG_PRINT(LOG_DEBUG, sql_command);
    dbires1 = dbi_conn_query(conn, sql_command);
    if (!dbires1) {
      dbi_result_free(dbires);
      LOG_PRINT(LOG_WARNING, "select from t_xlink failed");
      return 1;
    }
    
    numrows = dbi_result_get_numrows(dbires1);

    if (numrows == 1) { /* if no other reference uses this link */
      /* delete entry in link table */
      sprintf(sql_command, "DELETE FROM t_link WHERE link_id="ULLSPEC, (unsigned long long)n_ulink_id);
      LOG_PRINT(LOG_DEBUG, sql_command);
      dbires2 = dbi_conn_query(conn, sql_command);
      if (!dbires2) {
	dbi_result_free(dbires);
	dbi_result_free(dbires1);
	LOG_PRINT(LOG_WARNING, "delete from t_link failed");
	return 2;
      }
      dbi_result_free(dbires2);

      /* remove orphans in t_xnote */
/*       result = remove_xnote_entries(n_keyword_id, conn, 2 /\*keyword*\/); */

/*       if (result != 0 && result != 4) { */
/* 	dbi_result_free(dbires1); */
/* 	dbi_result_free(dbires); */
/* 	return 4; */
/*       } */
    }
      
    /* delete entry in xlink table */
    sprintf(sql_command, "DELETE FROM t_xlink WHERE xlink_id="ULLSPEC, (unsigned long long)n_xulink_id);
    LOG_PRINT(LOG_DEBUG, sql_command);
    dbires2 = dbi_conn_query(conn, sql_command);
    if (!dbires2) {
      dbi_result_free(dbires);
      dbi_result_free(dbires1);
      LOG_PRINT(LOG_WARNING, "delete from t_xlink failed");
      return 3;
    }
    dbi_result_free(dbires2);
    dbi_result_free(dbires1);
  } /* end while */

  dbi_result_free(dbires);
  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  remove_author_entries(): removes author entries associated with
                            a particular dataset

  int remove_author_entries returns 0 if ok, > 0 if error
                            error codes: 1 = select from t_xauthor failed
			                 2 = delete from t_author failed
					 3 = delete from t_xauthor failed
					 4 = remove from t_xnote failed

  unsigned long long ref_id reference id

  dbi_conn conn the database connection

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int remove_author_entries(unsigned long long ref_id, dbi_conn conn) {
  char sql_command[256];
  unsigned long long n_author_id; /* author_id of a search result */
  unsigned long long n_xauthor_id; /* xauthor_id of a search result */
  unsigned long long numrows;
  dbi_result dbires;
  dbi_result dbires1;
  dbi_result dbires2;

  sprintf(sql_command, "SELECT author_id, xauthor_id FROM t_xauthor WHERE refdb_id="ULLSPEC, (unsigned long long)ref_id);
  LOG_PRINT(LOG_DEBUG, sql_command);
  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "select from t_xauthor failed");
    return 1;
  }

  while (dbi_result_next_row(dbires)) {
    n_author_id = my_dbi_result_get_idval(dbires, "author_id");
    n_xauthor_id = my_dbi_result_get_idval(dbires, "xauthor_id");

    sprintf(sql_command, "SELECT xauthor_id FROM t_xauthor WHERE author_id="ULLSPEC, (unsigned long long)n_author_id);
    LOG_PRINT(LOG_DEBUG, sql_command);
    dbires1 = dbi_conn_query(conn, sql_command);
    if (!dbires1) {
      dbi_result_free(dbires);
      LOG_PRINT(LOG_WARNING, "select from t_xauthor failed");
      return 1;
    }

    numrows = dbi_result_get_numrows(dbires1);

    if (numrows == 1) { /* if no other reference uses this author */
      int result;

      /* delete entry in author table */
      sprintf(sql_command, "DELETE FROM t_author WHERE author_id="ULLSPEC, (unsigned long long)n_author_id);
      LOG_PRINT(LOG_DEBUG, sql_command);
      dbires2 = dbi_conn_query(conn, sql_command);
      if (!dbires2) {
	dbi_result_free(dbires1);
	dbi_result_free(dbires);
	LOG_PRINT(LOG_WARNING, "delete from t_author failed");
	return 2;
      }
      dbi_result_free(dbires2);

      /* remove orphans in t_xnote */
      result = remove_xnote_entries(0 /* any note */, n_author_id, conn, 1 /*author*/);

      if (result != 0 && result != 4) {
	dbi_result_free(dbires1);
	dbi_result_free(dbires);
	return 4;
      }
    }

    /* delete entry in xauthor table */
    sprintf(sql_command, "DELETE FROM t_xauthor WHERE xauthor_id="ULLSPEC, (unsigned long long)n_xauthor_id);
    LOG_PRINT(LOG_DEBUG, sql_command);
    dbires2 = dbi_conn_query(conn, sql_command);
    if (!dbires2) {
      dbi_result_free(dbires1);
      dbi_result_free(dbires);
      LOG_PRINT(LOG_WARNING, "delete from t_xauthor failed");
      return 3;
    }
    dbi_result_free(dbires2);
    dbi_result_free(dbires1);
  }


  dbi_result_free(dbires);
  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  remove_user_entries(): removes the user entry associated with
                            a particular dataset if it is an orphan

  int remove_user_entries returns 0 if ok, > 0 if error
                            error codes: 1 = select from t_user failed
			                 2 = select from t_note failed
					 3 = delete from t_user failed
					 4 = not an orphan

  unsigned long long user_id id of the user to be deleted

  dbi_conn conn the database connection

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int remove_user_entries(unsigned long long user_id, dbi_conn conn) {
  char sql_command[256];
  unsigned long long numrows;

  dbi_result dbires;

  /* select all xuser and note entries of the given user; if there are
     none, the user is an orphan and can safely be deleted */
  sprintf(sql_command, "SELECT xuser_id FROM t_xuser WHERE user_id="ULLSPEC, (unsigned long long)user_id);

  LOG_PRINT(LOG_DEBUG, sql_command);
  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "select from t_xuser failed");
    return 1;
  }

  numrows = dbi_result_get_numrows(dbires);
  
  dbi_result_free(dbires);

  sprintf(sql_command, "SELECT note_id FROM t_note WHERE note_user_id="ULLSPEC, (unsigned long long)user_id);

  LOG_PRINT(LOG_DEBUG, sql_command);
  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "select from t_note failed");
    return 2;
  }

  numrows += dbi_result_get_numrows(dbires);

  dbi_result_free(dbires);

  if (!numrows) {
    /* no other note or reference belong to this user */
    /* delete entry in t_user table */
    sprintf(sql_command, "DELETE FROM t_user WHERE user_id="ULLSPEC, (unsigned long long)user_id);
    LOG_PRINT(LOG_DEBUG, sql_command);
    dbires = dbi_conn_query(conn, sql_command);
    if (!dbires) {
      return 3;
    }
    dbi_result_free(dbires);
    return 0;
  }

  return 4;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  remove_xuser_entries(): removes the xuser entry associated with
                            a particular dataset and a particular user

  int remove_xuser_entries returns 0 if ok, > 0 if error
                            error codes: 1 = select from t_xuser failed
					 3 = delete from t_xuser failed
					 4 = no entry found

  unsigned long long ref_id reference id

  char* set_owner name of the user who owns the dataset. If NULL, all
                  entries regarding the reference ID will be removed

  dbi_conn conn the database connection

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int remove_xuser_entries(unsigned long long ref_id, char* set_owner, dbi_conn conn) {
  char sql_command[256];

  unsigned long long n_xuser_id = 0; /* xid of set_owner regarding current ref */
  dbi_result dbires;
  dbi_result dbires1;

  if (set_owner && *set_owner) {
    sprintf(sql_command, "SELECT xuser_id FROM t_xuser INNER JOIN t_user ON t_xuser.user_id=t_user.user_id WHERE t_user.user_name=\'%s\' AND t_xuser.refdb_id="ULLSPEC, set_owner, (unsigned long long)ref_id);
  }
  else {
    sprintf(sql_command, "SELECT xuser_id FROM t_xuser, t_user WHERE t_xuser.refdb_id="ULLSPEC, (unsigned long long)ref_id);
  }

  LOG_PRINT(LOG_DEBUG, sql_command);
  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "select from t_xuser failed");
    return 1;
  }

  while (dbi_result_next_row(dbires)) {
    n_xuser_id = my_dbi_result_get_idval(dbires, "xuser_id");

    /* delete entry in xuser table */
    sprintf(sql_command, "DELETE FROM t_xuser WHERE xuser_id="ULLSPEC, (unsigned long long)n_xuser_id);
    LOG_PRINT(LOG_DEBUG, sql_command);
    dbires1 = dbi_conn_query(conn, sql_command);
    if (!dbires1) {
      dbi_result_free(dbires);
      LOG_PRINT(LOG_WARNING, "delete from t_xnote failed");
      return 3;
    }
    dbi_result_free(dbires1);
  }

  dbi_result_free(dbires);

  if (n_xuser_id) {
    return 0;
  }
  else {
    return 4;
  }
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  remove_xnote_entries(): removes xnote entries associated with
                            a particular dataset

  int remove_xnote_entries returns 0 if ok, > 0 if error
                            error codes: 1 = select from t_xnote failed
					 3 = delete from t_xnote failed
					 4 = no entry found

  unsigned long long note_id note id, or 0 if all notes should be considered

  unsigned long long xref_id link target id, or 0 if all targets should be considered

  dbi_conn conn the database connection

  int mode 0 = reference entry 1 = author entry 2 = keyword entry
           3 = periodical entry 4 = all entries

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int remove_xnote_entries(unsigned long long note_id, unsigned long long xref_id, dbi_conn conn, int mode) {
  char sql_command[256];
  char mode_string[4][11] = {
    "REFERENCE",
    "AUTHOR",
    "KEYWORD",
    "PERIODICAL"};
  unsigned long long n_xnote_id = 0; /* xnote_id of a search result */
  dbi_result dbires;
  dbi_result dbires1;


  if (mode == 4) {
    sprintf(sql_command, "SELECT xnote_id FROM t_xnote WHERE note_id="ULLSPEC, (unsigned long long)xref_id);
  }
  else if (note_id) {
    sprintf(sql_command, "SELECT xnote_id FROM t_xnote WHERE note_id="ULLSPEC" AND xnote_type=\'%s\' AND xref_id="ULLSPEC, (unsigned long long)note_id, mode_string[mode], (unsigned long long)xref_id);
  }
  else {
    sprintf(sql_command, "SELECT xnote_id FROM t_xnote WHERE xnote_type=\'%s\' AND xref_id="ULLSPEC, mode_string[mode], (unsigned long long)xref_id);
  }

  LOG_PRINT(LOG_DEBUG, sql_command);
  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "select from t_xnote failed");
    return 1;
  }

  while (dbi_result_next_row(dbires)) {
    n_xnote_id = my_dbi_result_get_idval(dbires, "xnote_id");

    /* delete entry in xnote table */
    sprintf(sql_command, "DELETE FROM t_xnote WHERE xnote_id="ULLSPEC, (unsigned long long)n_xnote_id);
    LOG_PRINT(LOG_DEBUG, sql_command);
    dbires1 = dbi_conn_query(conn, sql_command);
    if (!dbires1) {
      dbi_result_free(dbires);
      LOG_PRINT(LOG_WARNING, "delete from t_xnote failed");
      return 3;
    }
    dbi_result_free(dbires1);
  }

  dbi_result_free(dbires);
  
  if (n_xnote_id) {
    return 0;
  }
  else {
    return 4;
  }
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  remove_periodical_entries(): removes periodical entries associated with
                            a particular dataset

  int remove_periodical_entries returns 0 if ok, nonzero if error

  unsigned long long n_periodical_id ID value of the periodical

  dbi_conn conn the database connection

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int remove_periodical_entries(unsigned long long n_periodical_id, dbi_conn conn) {
  char sql_command[256];
  unsigned long long numrows;
  dbi_result dbires;
  dbi_result dbires1;

  /* check whether we should keep the t_periodical entries */
  if (*keep_pnames == 't') {
    return 0; /* nothing to do, we're done */
  }

  sprintf(sql_command, "SELECT refdb_id FROM t_refdb WHERE refdb_periodical_id="ULLSPEC, (unsigned long long)n_periodical_id);

  LOG_PRINT(LOG_DEBUG, sql_command);

  dbires = dbi_conn_query(conn, sql_command);
  if (!dbires) {
    LOG_PRINT(LOG_WARNING, "select from t_refdb failed");
    dbi_result_free(dbires);
    return 1;
  }

  numrows = dbi_result_get_numrows(dbires);

  if (numrows <= 1) { /* if no other reference uses this periodical */
    int result;

    /* delete entry in periodical table */
    sprintf(sql_command, "DELETE FROM t_periodical WHERE periodical_id="ULLSPEC, (unsigned long long)n_periodical_id);

    LOG_PRINT(LOG_DEBUG, sql_command);

    dbires1 = dbi_conn_query(conn, sql_command);
    if (!dbires1) {
      LOG_PRINT(LOG_WARNING, "delete from t_periodical failed");
      dbi_result_free(dbires);
      return 1;
    }
    dbi_result_free(dbires1);

    /* remove orphans in t_xnote */
    result = remove_xnote_entries(0 /* any note */, n_periodical_id, conn, 3 /*periodical*/);

    if (result != 0 && result != 4) {
      dbi_result_free(dbires);
      return 4;
    }
  }
  dbi_result_free(dbires);
  return 0;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  strip_oddchars(): removes unwanted characters from strings

  char* strip_oddchars returns ptr to the input string

  char* string pointer to string to be modified

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
static char* strip_oddchars(char* string) {
  /* The following character sequences are no good candidates for
     citation keys. Nevertheless, they may show up from author names
     imported from bibtex bibliographies. We also don't want spaces or
     quotation marks and such */
  /*
 Accents that may be immediately followed by a letter
( before creating key strip the backslash and the next char)

 1.   grave         \`o
 2.   acute         \'o
 3.   circumflex    \^o
 4.   umlaut        \"o
 5.   tilde         \~o
 6.   macron        \=o
 7.   dot           \.o


 Accents requering an intervening space
( before creating key strip the backslash and 2 next chars)

 8.  cedilla       \c o
 9.  underdot      \d o
10.  underbar      \b o
11.  hacek         \v o
12.  breve         \u o
13.  tie           \t oo
14.  Hun           \H o

 Foreign letters
( before creating key strip the backslash only)

15.               \AE  
16.               \ae
17.               \OE  
18.               \oe
19.               \AA
20.               \aa
21.               \O
22.               \o
23.               \L
24.               \l
25.               \ss
26.               \i
27.               \j

The list is taken from "A Gentle Introduction to TeX" by Michael Doob
(thanks Alexander)
*/
  char *thechar;

  thechar = string;

  while (*thechar) {
    if (!strncmp(thechar, "\\`", 2) ||
	!strncmp(thechar, "\\'", 2) ||
	!strncmp(thechar, "\\^", 2) ||
	!strncmp(thechar, "\\\"", 2) ||
	!strncmp(thechar, "\\~", 2) ||
	!strncmp(thechar, "\\=", 2) ||
	!strncmp(thechar, "\\.", 2)) {
      /* remove backslash and following character */
      remove_substring(thechar, 2);
    }
    else if (!strncmp(thechar, "\\c ", 3) ||
	     !strncmp(thechar, "\\d ", 3) ||
	     !strncmp(thechar, "\\b ", 3) ||
	     !strncmp(thechar, "\\v ", 3) ||
	     !strncmp(thechar, "\\u ", 3) ||
	     !strncmp(thechar, "\\t ", 3) ||
	     !strncmp(thechar, "\\H ", 3)) {
      /* remove backslash and two following characters */
      remove_substring(thechar, 3);
    }
    else if (!strncmp(thechar, "\\AE", 3) ||
	     !strncmp(thechar, "\\ae", 3) ||
	     !strncmp(thechar, "\\OE", 3) ||
	     !strncmp(thechar, "\\oe", 3) ||
	     !strncmp(thechar, "\\AA", 3) ||
	     !strncmp(thechar, "\\aa", 3) ||
	     !strncmp(thechar, "\\O", 2) ||
	     !strncmp(thechar, "\\o", 2) ||
	     !strncmp(thechar, "\\L", 2) ||
	     !strncmp(thechar, "\\l", 2) ||
	     !strncmp(thechar, "\\ss", 3) ||
	     !strncmp(thechar, "\\i", 2) ||
	     !strncmp(thechar, "\\j", 2)) {
      /* remove backslash only */
      remove_substring(thechar, 1);
    }
    else if ((*thechar == '\'') ||
	     (*thechar == '\"') ||
	     (*thechar == '&') ||
	     (*thechar == '<') ||
	     (*thechar == '>') ||
	     (*thechar == ':') ||
	     (*thechar == '(') ||
	     (*thechar == ')') ||
	     (*thechar == '?') ||
	     (*thechar == ' ')) {
      /* remove offending character */
      remove_substring(thechar, 1);
    }
    else if ((unsigned char)(*thechar) > 127) {
      /* remove offending character */
      remove_substring(thechar, 1);
    }
    else if (!strncmp(thechar, "-ID", 3)) {
      /* replace dash with an underscore */
      *thechar = '_';
    }
    else {
      thechar++;
    }
  }

  return string;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  preprocess_citekey_copy(): removes unwanted characters from a string and
                        returns a cleaned version in an allocated buffer
			The calling function is responsible for freeing
			the buffer again

  char* preprocess_citekey returns ptr to an allocated buffer containing
                        the cleaned string

  const char* string pointer to string to be analyzed

  size_t n_maxlen maximum length, sans \0, of the returned string
                        If 0, the string length is not limited

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
char* preprocess_citekey_copy(const char* string, size_t n_maxlen) {
  iconv_t conv_descriptor = NULL;
  size_t inlength;
  size_t outlength;
  size_t orig_outlength;
  char* my_outbuffer = NULL; /* this ptr will be modified by iconv() */
  char* my_outbuffer_start = NULL; /* records initial state of outbuffer */
  const char* my_instring = NULL; /* this ptr will be modified by iconv() */
  
  /* try to transliterate odd characters instead of discarding them */
  /* we may have to obtain the input character encoding, assume UTF-8 for now */
  conv_descriptor = iconv_open("ASCII//TRANSLIT", "UTF-8");
  if (conv_descriptor == (iconv_t)(-1)) {
    my_outbuffer_start = (char*)string;
    LOG_PRINT(LOG_WARNING, get_status_msg(701));
    goto postprocess;
  }

  inlength = strlen(string)+1;
  outlength = 6*inlength;
  orig_outlength = outlength;
  
  if ((my_outbuffer = malloc(outlength)) == NULL) {
    LOG_PRINT(LOG_WARNING, get_status_msg(801));
    iconv_close(conv_descriptor);
    return 0;
  }

  my_outbuffer_start = my_outbuffer;

  my_instring = (const char*)string;

/*   printf("calling iconv here with string length %d, outlength %d\n", inlength, outlength); */
  if (iconv(conv_descriptor, &my_instring, &inlength, &my_outbuffer, &outlength) == (size_t)(-1)) {
    if (errno == EILSEQ) {
      LOG_PRINT(LOG_WARNING, "iconv: invalid input character sequence");
    }
    else if (errno == E2BIG) {
      LOG_PRINT(LOG_WARNING, "iconv: output buffer too small");
    }
    else if (errno == EINVAL) {
      LOG_PRINT(LOG_WARNING, "iconv: incomplete input character");
    }
    my_outbuffer_start = (char*)string;
    LOG_PRINT(LOG_WARNING, get_status_msg(701));
    goto postprocess;
  }
/*   else { */
/*     printf("iconv returned success\n"); */
/*   } */

  postprocess:
  if (conv_descriptor) {
    iconv_close(conv_descriptor);
  }

/*   printf("outbuffer went to: %s<<\n", my_outbuffer_start); */
  strip_oddchars(my_outbuffer_start);
  if (*upper_citekey == 't') {
    strup(my_outbuffer_start);
  }

  /* truncate string if requested */
  if (n_maxlen && strlen(my_outbuffer_start) >= n_maxlen) {
    my_outbuffer_start[n_maxlen] = '\0';
  }

  return my_outbuffer_start;
}

