/*++++++++++++++++++++
  refdbdgetref.c: refdb getref-related functions
  markus@mhoenicka.de 2006-08-22

  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; either version 2 of the License, or
  (at your option) any later version.
   
  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
   
  You should have received a copy of the GNU General Public License
  along with this program; if not, see <http://www.gnu.org/licenses/>

  +++++++++++++++++++++++++*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <syslog.h> /* priority level definitions */
#include <errno.h>
#include <iconv.h>
#include <dbi/dbi.h>

#include "linklist.h"
#include "refdb.h"
#include "backend.h"
#include "refdbd.h" /* depends on backend.h */
#include "strfncs.h"
#include "tokenize.h"
#include "dbfncs.h"
#include "connect.h"
#include "refdbdgetref.h"
#include "writeris.h"

/* some globals */
extern int n_log_level; /* numeric version of log_level */
extern char cs_term[];

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  assemble_author_subselect(): assembles a string that replaces an author
                                subselect

  char* assemble_author_subselect returns an allocated string if ok, or NULL
                              if an error occurred. The calling function
			      is responsible for freeing the allocated
			      memory

  char* author_string string containing the author query fragment

  const char* operator string containing the query operator

  dbi_conn conn database connection structure

  dbi_driver driver the database driver

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
char* assemble_author_subselect(char* author_string, const char* operator, int author_type, dbi_conn conn, dbi_driver driver) {
  char author_type_start[64];
  char author_type_end[4];
  int counter;
  size_t len = 0;
  size_t sql_command_len = 4096;
  size_t stringbuf_len = 4096;
  char bitsandpieces[512];
  char* sql_command;
  char* new_sql_command;
  char* token;
  char* quoted_token;
  char *stringbuf;
  char *new_stringbuf;

  sql_command = malloc(sql_command_len); 

  if (sql_command == NULL) {
    return NULL;
  }
  sql_command[0] = '\0'; /* start with an empty string */

  stringbuf = malloc(stringbuf_len); 

  if (stringbuf == NULL) {
    free(sql_command);
    return NULL;
  }
  stringbuf[0] = '\0'; /* start with an empty string */


  token = nstrtok(author_string, &len, " ");
  if (token != NULL) {
	      
    /* prepare bits and pieces for AND/OR queries*/
    if (author_type == 4) { /* any type of author */
      *author_type_start = '\0';
      strcpy(author_type_end, ") ");
    }
    else { /* particular type of author */
      sprintf(author_type_start, "(t_xauthor.xauthor_type=%s AND ", get_author_type_string(driver, author_type));
      strcpy(author_type_end, ")) ");
    }

    /* plain query with one author */
    if (*token != '&' && *token != '|') {
      if (author_type == 4) { /* any type of author */
	sprintf(sql_command, "SELECT DISTINCT t_xauthor.refdb_id FROM t_xauthor INNER JOIN t_author ON t_author.author_id=t_xauthor.author_id WHERE t_author.author_name %s ", (*operator == '=' || (*operator == '!' && *(operator+1) == '=')) ? "=" : my_dbi_conn_get_cap(conn, "rlike"));
      }
      else { /* particular type of author */
	sprintf(sql_command, "SELECT DISTINCT t_xauthor.refdb_id FROM t_xauthor INNER JOIN t_author ON t_author.author_id=t_xauthor.author_id WHERE t_xauthor.xauthor_type=%s AND t_author.author_name %s ", get_author_type_string(driver, author_type), (*operator == '=' || (*operator == '!' && *(operator+1) == '=')) ? "=" : my_dbi_conn_get_cap(conn, "rlike"));
      }
      quoted_token = mstrdup(token);
      if (!quoted_token) {
	free(sql_command);
	free(stringbuf);
	return NULL;
      }
      if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	free(sql_command);
	free(stringbuf);
	return NULL;
      }

      if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
	free(quoted_token);
	free(sql_command);
	free(stringbuf);
	return NULL;
      }
      else {
	sql_command = new_sql_command;
      }
      free(quoted_token);
    }

    /* OR query with several authors */
    else if (*token == '|') {
      sprintf(sql_command, "SELECT DISTINCT t_xauthor.refdb_id FROM t_xauthor INNER JOIN t_author ON t_author.author_id=t_xauthor.author_id WHERE %s (", author_type_start);

      while (token != NULL) {
	token = nstrtok(token+len, &len, " ");
	if (token != NULL) {
	  if (*operator == '=' || (*operator == '!' && *(operator+1) == '=')) {
	    strcpy(bitsandpieces, "t_author.author_name = ");
	  }
	  else {
	    sprintf(bitsandpieces, "t_author.author_name %s ", my_dbi_conn_get_cap(conn, "rlike"));
	  }

	  if ((new_sql_command = mstrcat(sql_command, bitsandpieces, &sql_command_len, 0)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  if ((new_stringbuf = mstrncpy(stringbuf, token, len, &stringbuf_len)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    stringbuf = new_stringbuf;
	  }
	  stringbuf[len] = '\0';
	  quoted_token = mstrdup(stringbuf);
	  if (!quoted_token) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
	    free(quoted_token);
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  free(quoted_token);
	  if ((new_sql_command = mstrcat(sql_command, " OR ", &sql_command_len, 0)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	}
	else {
	  if ((new_sql_command = mstrcat(sql_command, author_type_end, &sql_command_len, 4)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	}
      } /* end while */
    }

    /* AND query with several authors */
    else if (*token == '&') {
      counter = 0;
      sprintf(sql_command, "SELECT DISTINCT t_xauthor.refdb_id FROM t_xauthor INNER JOIN t_author ON t_author.author_id=t_xauthor.author_id WHERE %s (", author_type_start);

      while (token != NULL) {
	token = nstrtok(token+len, &len, " ");
	if (token != NULL) {
	  if (*operator == '=' || (*operator == '!' && *(operator+1) == '=')) {
	    strcpy(bitsandpieces, "t_author.author_name = ");
	  }
	  else {
	    sprintf(bitsandpieces, "t_author.author_name %s ", my_dbi_conn_get_cap(conn, "rlike"));
	  }

	  if ((new_sql_command = mstrcat(sql_command, bitsandpieces, &sql_command_len, 0)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  if ((new_stringbuf = mstrncpy(stringbuf, token, len, &stringbuf_len)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    stringbuf = new_stringbuf;
	  }
	  stringbuf[len] = '\0';
	  quoted_token = mstrdup(stringbuf);
	  if (!quoted_token) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	    free(quoted_token);
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
	    free(quoted_token);
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  free(quoted_token);
	  if ((new_sql_command = mstrcat(sql_command, " OR ", &sql_command_len, 0)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  counter++;
	}
	else {
	  if ((new_sql_command = mstrcat(sql_command, author_type_end, &sql_command_len, 4)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	}
      }
      if ((new_sql_command = mstrcat(sql_command, "GROUP BY t_xauthor.refdb_id HAVING COUNT(*)=", &sql_command_len, 0)) == NULL) {
	free(sql_command);
	free(stringbuf);
	return NULL;
      }
      else {
	sql_command = new_sql_command;
      }
      sprintf(author_string, "%d", counter);
      if ((new_sql_command = mstrcat(sql_command, author_string, &sql_command_len, 0)) == NULL) {
	free(sql_command);
	free(stringbuf);
	return NULL;
      }
      else {
	sql_command = new_sql_command;
      }
    }
  }

  free(stringbuf);

  return sql_command;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  assemble_keyword_subselect(): assembles a string that replaces a keyword
                                subselect

  char* assemble_keyword_subselect returns an allocated string if ok, or NULL
                              if an error occurred. The calling function
			      is responsible for freeing the allocated
			      memory

  char* keyword_string string containing the keyword query fragment

  const char* operator string containing the query operator

  dbi_conn conn database connection structure

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
char* assemble_keyword_subselect(char* keyword_string, const char* operator, dbi_conn conn) {
  int counter;
  size_t len = 0;
  size_t sql_command_len = 4096;
  size_t stringbuf_len = 4096;
  char bitsandpieces[512];
  char* sql_command;
  char* new_sql_command;
  char* token;
  char* quoted_token;
  char *stringbuf;
  char *new_stringbuf;

  sql_command = malloc(sql_command_len); 

  if (sql_command == NULL) {
    return NULL;
  }
  sql_command[0] = '\0'; /* start with an empty string */

  stringbuf = malloc(stringbuf_len); 

  if (stringbuf == NULL) {
    free(sql_command);
    return NULL;
  }
  stringbuf[0] = '\0'; /* start with an empty string */

  token = nstrtok(keyword_string, &len, " ");
  if (token != NULL) {
    if (*token != '&' && *token != '|') {
      /* simple keyword query */
      snprintf(sql_command, sql_command_len, "SELECT DISTINCT t_xkeyword.xref_id FROM t_xkeyword INNER JOIN t_keyword ON t_keyword.keyword_id=t_xkeyword.keyword_id WHERE t_xkeyword.xkeyword_type=\'REFERENCE\' AND t_keyword.keyword_name %s ", (*operator == '=' || (*operator == '!' && *(operator+1) == '=')) ? "=" : my_dbi_conn_get_cap(conn, "rlike"));
      quoted_token = mstrdup(token);
      if (!quoted_token) {
	free(sql_command);
	free(stringbuf);
	return NULL;
      }
      if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	free(sql_command);
	free(stringbuf);
	return NULL;
      }
      if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
	free(sql_command);
	free(stringbuf);
	return NULL;
      }
      else {
	sql_command = new_sql_command;
      }
      free(quoted_token);
    }
    else if (*token == '|') {
      /* several keywords OR'ed */
      strcpy(sql_command, "SELECT DISTINCT t_xkeyword.xref_id FROM t_xkeyword INNER JOIN t_keyword ON t_keyword.keyword_id=t_xkeyword.keyword_id WHERE t_xkeyword.xkeyword_type=\'REFERENCE\' AND (");
      while (token != NULL) {
	token = nstrtok(token+len, &len, " ");
	if (token != NULL) {
	  if (*operator == '=' || (*operator == '!' && *(operator+1) == '=')) {
	    strcpy(bitsandpieces, "t_keyword.keyword_name = ");
	  }
	  else {
	    sprintf(bitsandpieces, "t_keyword.keyword_name %s ", my_dbi_conn_get_cap(conn, "rlike"));
	  }

	  if ((new_sql_command = mstrcat(sql_command, bitsandpieces, &sql_command_len, 0)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  if ((new_stringbuf = mstrncpy(stringbuf, token, len, &stringbuf_len)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    stringbuf = new_stringbuf;
	  }
	  stringbuf[len] = '\0';
	  quoted_token = mstrdup(stringbuf);
	  if (!quoted_token) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	    free(quoted_token);
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
	    free(quoted_token);
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  free(quoted_token);

	  if ((new_sql_command = mstrcat(sql_command, " OR ", &sql_command_len, 0)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	}
	else {
	  if ((new_sql_command = mstrcat(sql_command, ") ", &sql_command_len, 4)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	}
      }
    }
    else if (*token == '&') {
      /* several keywords AND'ed */
      counter = 0;
      strcpy(sql_command, "SELECT DISTINCT t_xkeyword.xref_id FROM t_xkeyword INNER JOIN t_keyword ON t_keyword.keyword_id=t_xkeyword.keyword_id WHERE t_xkeyword.xkeyword_type=\'REFERENCE\' AND (");
      while (token != NULL) {
	token = nstrtok(token+len, &len, " ");
	if (token != NULL) {
	  if (*operator == '=' || (*operator == '!' && *(operator+1) == '=')) {
	    strcpy(bitsandpieces, "t_keyword.keyword_name = ");
	  }
	  else {
	    sprintf(bitsandpieces, "t_keyword.keyword_name %s ", my_dbi_conn_get_cap(conn, "rlike"));
	  }

	  if ((new_sql_command = mstrcat(sql_command, bitsandpieces, &sql_command_len, 0)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  if ((new_stringbuf = mstrncpy(stringbuf, token, len, &stringbuf_len)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    stringbuf = new_stringbuf;
	  }
	  stringbuf[len] = '\0';
	  quoted_token = mstrdup(stringbuf);
	  if (!quoted_token) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	    free(quoted_token);
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
	    free(quoted_token);
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  free(quoted_token);

	  if ((new_sql_command = mstrcat(sql_command, " OR ", &sql_command_len, 0)) == NULL) {
	    free(quoted_token);
	    free(sql_command);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  counter++;
	}
	else {
	  if ((new_sql_command = mstrcat(sql_command, ") ", &sql_command_len, 4)) == NULL) {
	    free(quoted_token);
	    free(sql_command);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	}
      }
      if ((new_sql_command = mstrcat(sql_command, "GROUP BY t_xkeyword.xref_id HAVING COUNT(*)=", &sql_command_len, 0)) == NULL) {
	free(quoted_token);
	free(sql_command);
	return NULL;
      }
      else {
	sql_command = new_sql_command;
      }
      sprintf(keyword_string, "%d", counter);
      if ((new_sql_command = mstrcat(sql_command, keyword_string, &sql_command_len, 0)) == NULL) {
	free(quoted_token);
	free(sql_command);
	return NULL;
      }
      else {
	sql_command = new_sql_command;
      }
    }
  }

  free(stringbuf);

  return sql_command;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  assemble_link_subselect(): assembles a string that replaces a link
                                subselect

  char* assemble_link_subselect returns an allocated string if ok, or NULL
                              if an error occurred. The calling function
			      is responsible for freeing the allocated
			      memory

  char* link_string string containing the link query fragment

  const char* operator string containing the query operator

  const char* link_type string containing the type (URL, PDF...)

  dbi_conn conn database connection structure

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
char* assemble_link_subselect(char* link_string, const char* operator, const char* link_type, dbi_conn conn) {
  int counter;
  size_t len = 0;
  size_t sql_command_len = 4096;
  size_t stringbuf_len = 4096;
  char bitsandpieces[512];
  char* sql_command;
  char* new_sql_command;
  char* token;
  char* quoted_token;
  char *stringbuf;
  char *new_stringbuf;

  sql_command = malloc(sql_command_len); 

  if (sql_command == NULL) {
    return NULL;
  }
  sql_command[0] = '\0'; /* start with an empty string */

  stringbuf = malloc(stringbuf_len); 

  if (stringbuf == NULL) {
    free(sql_command);
    return NULL;
  }
  stringbuf[0] = '\0'; /* start with an empty string */

  token = nstrtok(link_string, &len, " ");
  if (token != NULL) {
    if (*token != '&' && *token != '|') {
      /* simple link query */
      snprintf(sql_command, sql_command_len, "SELECT DISTINCT t_xlink.xref_id FROM t_xlink INNER JOIN t_link ON t_link.link_id=t_xlink.link_id WHERE t_xlink.xlink_source=\'REFERENCE\' AND t_xlink.xlink_type=\'%s\' AND t_link.link_url %s ", link_type, (*operator == '=' || (*operator == '!' && *(operator+1) == '=')) ? "=" : my_dbi_conn_get_cap(conn, "rlike"));
      quoted_token = mstrdup(token);
      if (!quoted_token) {
	free(sql_command);
	free(stringbuf);
	return NULL;
      }
      if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	free(sql_command);
	free(stringbuf);
	return NULL;
      }
      if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
	free(sql_command);
	free(stringbuf);
	return NULL;
      }
      else {
	sql_command = new_sql_command;
      }
      free(quoted_token);
    }
    else if (*token == '|') {
      /* several links OR'ed */
      sprintf(sql_command, "SELECT DISTINCT t_xlink.xref_id FROM t_xlink INNER JOIN t_link ON t_link.link_id=t_xlink.link_id WHERE t_xlink.xlink_source=\'REFERENCE\' AND t_xlink.xlink_type=\'%s\' AND (", link_type);
      while (token != NULL) {
	token = nstrtok(token+len, &len, " ");
	if (token != NULL) {
	  if (*operator == '=' || (*operator == '!' && *(operator+1) == '=')) {
	    strcpy(bitsandpieces, "t_link.link_url = ");
	  }
	  else {
	    sprintf(bitsandpieces, "t_link.link_url %s ", my_dbi_conn_get_cap(conn, "rlike"));
	  }

	  if ((new_sql_command = mstrcat(sql_command, bitsandpieces, &sql_command_len, 0)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  if ((new_stringbuf = mstrncpy(stringbuf, token, len, &stringbuf_len)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    stringbuf = new_stringbuf;
	  }
	  stringbuf[len] = '\0';
	  quoted_token = mstrdup(stringbuf);
	  if (!quoted_token) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	    free(quoted_token);
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
	    free(quoted_token);
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  free(quoted_token);

	  if ((new_sql_command = mstrcat(sql_command, " OR ", &sql_command_len, 0)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	}
	else {
	  if ((new_sql_command = mstrcat(sql_command, ") ", &sql_command_len, 4)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	}
      }
    }
    else if (*token == '&') {
      /* several links AND'ed */
      counter = 0;
      sprintf(sql_command, "SELECT DISTINCT t_xlink.xref_id FROM t_xlink INNER JOIN t_link ON t_link.link_id=t_xlink.link_id WHERE t_xlink.xlink_source=\'REFERENCE\' AND t_xlink.xlink_type=\'%s\' AND (", link_type);
      while (token != NULL) {
	token = nstrtok(token+len, &len, " ");
	if (token != NULL) {
	  if (*operator == '=' || (*operator == '!' && *(operator+1) == '=')) {
	    strcpy(bitsandpieces, "t_link.link_url = ");
	  }
	  else {
	    sprintf(bitsandpieces, "t_link.link_url %s ", my_dbi_conn_get_cap(conn, "rlike"));
	  }

	  if ((new_sql_command = mstrcat(sql_command, bitsandpieces, &sql_command_len, 0)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  if ((new_stringbuf = mstrncpy(stringbuf, token, len, &stringbuf_len)) == NULL) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    stringbuf = new_stringbuf;
	  }
	  stringbuf[len] = '\0';
	  quoted_token = mstrdup(stringbuf);
	  if (!quoted_token) {
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	    free(quoted_token);
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
	    free(quoted_token);
	    free(sql_command);
	    free(stringbuf);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  free(quoted_token);

	  if ((new_sql_command = mstrcat(sql_command, " OR ", &sql_command_len, 0)) == NULL) {
	    free(quoted_token);
	    free(sql_command);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	  counter++;
	}
	else {
	  if ((new_sql_command = mstrcat(sql_command, ") ", &sql_command_len, 4)) == NULL) {
	    free(quoted_token);
	    free(sql_command);
	    return NULL;
	  }
	  else {
	    sql_command = new_sql_command;
	  }
	}
      }
      if ((new_sql_command = mstrcat(sql_command, "GROUP BY t_xlink.xref_id HAVING COUNT(*)=", &sql_command_len, 0)) == NULL) {
	free(quoted_token);
	free(sql_command);
	return NULL;
      }
      else {
	sql_command = new_sql_command;
      }
      sprintf(link_string, "%d", counter);
      if ((new_sql_command = mstrcat(sql_command, link_string, &sql_command_len, 0)) == NULL) {
	free(quoted_token);
	free(sql_command);
	return NULL;
      }
      else {
	sql_command = new_sql_command;
      }
    }
  }

  free(stringbuf);

  return sql_command;
}

/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  getref(): implements the client command getref

  int getref returns 0 = no error, 1 = out of memory or db open error, 2 = subselect failed

  struct CLIENT_REQUEST* ptr_clrequest ptr to structure with client info

  struct bibinfo *ptr_biblio_info ptr to structure holding bibliography style

  int ref_format the output format 0 = simple screen rendering,
        1 = RIS format, 2 = DocBook format, 3 = BibTeX format,
        4 = HTML 10 = RISX 11 = XHTML

  int n_privatelist if set to 1, limit search to user's private list
                    if set to 0, search all

  struct ADDRESULT* ptr_addresult ptr to result counter structure

  int send_data if set to 1, sends back matching datasets (getref)
                if set to 0, sends back number of matching datasets (scanref)

  int frequency if 1, provide frequency information (not necessarily supported
                by all backends), otherwise don't

  ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int getref(struct CLIENT_REQUEST* ptr_clrequest, struct bibinfo *ptr_biblio_info, int ref_format, int n_privatelist, struct ADDRESULT* ptr_addresult, int send_data, int frequency) {
  dbi_conn conn;
  dbi_result dbires = NULL;
  dbi_driver driver;
  char **inargv; /* tokens of the client request */
  int inargc; /* number of tokens of the client request */
  int inargcmax; /* maximum number of tokens of the client request */
  int n_isalpha;
  int n_havesome;
  int numbyte; /* number of bytes written */
  int author_type; /* signals type of author query, if any */
  int render_res;
  int cs_status;
  int retval = 0;
  unsigned long long n_id;
  unsigned long long nref_counter = 0;
  unsigned long long numrows = 0;
  size_t buffer_len; /* these are the allocated lengths of the buffers */
  size_t sql_command_len;
  size_t sql_command1_len;
  size_t stringbuf_len;
  size_t result_len = 0;
  size_t n_bufsize;
  char *sql_command; /* these are ptrs to the buffers and temporary ptrs */
  char *new_sql_command;
  char *sql_command1;
  char *new_sql_command1;
  char *buffer; 
  char *new_buffer;
  char *stringbuf;
  char *token;
  char *newtoken;
  char *quoted_token;
  char *eostring;
  char *query_string;
  const char* db_encoding;
  char operator[5];
  char the_id[32];
  char bitsandpieces[512];
  char limitstring[128] = "";
  struct SQLTOKEN sqltoken;
  struct lilimem sentinel;
  struct renderinfo rendinfo;
  iconv_t conv_descriptor;

  sentinel.ptr_mem = NULL;
  sentinel.ptr_next = NULL;
  sentinel.varname[0] = '\0';

  /* fill in invariant elements of structure */
  rendinfo.ptr_biblio_info = ptr_biblio_info;
  rendinfo.ref_format = ref_format;
  rendinfo.nuse_citestyle = 0;
  rendinfo.dbname = NULL;
  /* todo: remove the redundance */
  rendinfo.database = ptr_clrequest->current_db;
  rendinfo.username = ptr_clrequest->username;
  rendinfo.pdfroot = ptr_clrequest->pdfroot;
  rendinfo.cgi_url = ptr_clrequest->cgi_url;
  rendinfo.ptr_clrequest = ptr_clrequest;
  rendinfo.javascript = 0;
  rendinfo.frequency = frequency;

  /* get buffer to analyze the request */
  inargc = 0;
  inargcmax = 10;
  inargv = malloc((size_t)inargcmax*sizeof(char*));
  if (inargv == NULL || insert_lilimem(&sentinel, (void**)&inargv, NULL)) {
    send_status(ptr_clrequest->fd, 801, TERM_NO);
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    return 1;
  }

  /* get buffer to assemble the SQL queries */
  sql_command_len = 4096; 
  sql_command = malloc(sql_command_len); 

  if (sql_command == NULL || insert_lilimem(&sentinel, (void**)&sql_command, NULL)) {
    delete_all_lilimem(&sentinel);
    send_status(ptr_clrequest->fd, 801, TERM_NO);
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    return 1;
  }
  sql_command[0] = '\0'; /* start with an empty string */

  /* get another buffer to assemble the SQL queries */
  sql_command1_len = 4096;
  sql_command1 = malloc(sql_command1_len); 

  if (sql_command1 == NULL || insert_lilimem(&sentinel, (void**)&sql_command1, NULL)) {
    delete_all_lilimem(&sentinel);
    send_status(ptr_clrequest->fd, 801, TERM_NO);
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    return 1;
  }
  sql_command1[0] = '\0'; /* start with an empty string */

  /* get a buffer to hold tokens */
  buffer_len = 4096;
  buffer = malloc(buffer_len); 

  if (buffer == NULL || insert_lilimem(&sentinel, (void**)&buffer, NULL)) {
    delete_all_lilimem(&sentinel);
    send_status(ptr_clrequest->fd, 801, TERM_NO);
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    return 1;
  }
  buffer[0] = '\0'; /* start with an empty string */

  /* get a buffer to hold tokens */
  stringbuf_len = 4096;
  stringbuf = malloc(stringbuf_len); 

  if (stringbuf == NULL || insert_lilimem(&sentinel, (void**)&stringbuf, NULL)) {
    delete_all_lilimem(&sentinel);
    send_status(ptr_clrequest->fd, 801, TERM_NO);
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    return 1;
  }
  buffer[0] = '\0'; /* start with an empty string */

  /* get length of the input string */
  n_bufsize = atoi(ptr_clrequest->argument);

  /* refuse queries if buffer length is unreasonable */
  if (n_bufsize <= TERM_LEN) {
    LOG_PRINT(LOG_WARNING, get_status_msg(301));
    send_status(ptr_clrequest->fd, 301, TERM_NO);
    return 1;
  }

  /* try to allocate the amount the client requested */
  query_string = malloc(n_bufsize);
  if (query_string == NULL || insert_lilimem(&sentinel, (void**)&query_string, NULL)) {
    send_status(ptr_clrequest->fd, 801, TERM_NO);
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    return 1;
  }

  /* send acknowledgement to client */
  send_status(ptr_clrequest->fd, 0, TERM_NO);

  if ((cs_status = read_status(ptr_clrequest->fd)) != 0) {
    LOG_PRINT(LOG_INFO, get_status_msg(112));
    delete_all_lilimem(&sentinel);
    return 1;
  }

  /* read id list from client */
  numbyte = tread(ptr_clrequest->fd, query_string, n_bufsize);
  if (numbyte == -1) {
    LOG_PRINT(LOG_INFO, get_status_msg(109));
    delete_all_lilimem(&sentinel);
    return 1;
  }

/*   printf("query: %s<<\n", query_string); */
  eostring = query_string + strlen(query_string);

  /* connect to the database */
  if ((conn = connect_to_db(ptr_clrequest, NULL, 0)) == NULL) {
    send_status(ptr_clrequest->fd, 204, TERM_NO);
    LOG_PRINT(LOG_WARNING, get_status_msg(204));
    delete_all_lilimem(&sentinel);
    return 1;
  }

  driver = dbi_conn_get_driver(conn);

  /* get the database encoding */
  db_encoding = dbi_conn_get_encoding(conn);

  /* if we need to convert, create a conversion descriptor for iconv() */
  if (db_encoding && *(ptr_biblio_info->encoding) && strcmp(db_encoding, ptr_biblio_info->encoding)) {
    char to_encoding[64];

    if (!strcmp(ptr_biblio_info->encoding, "US-ASCII")) {
      strcpy(to_encoding, "ASCII//TRANSLIT");
    }
    else {
      snprintf(to_encoding, 64, "%s//TRANSLIT", ptr_biblio_info->encoding);
    }

    conv_descriptor = iconv_open(to_encoding, !strcmp(db_encoding, "US-ASCII") ? "ASCII" : db_encoding);
    if (conv_descriptor == (iconv_t)(-1)) {
      LOG_PRINT(LOG_WARNING, "cannot set conversion descriptor (database/output):");
      LOG_PRINT(LOG_DEBUG, db_encoding);
      LOG_PRINT(LOG_DEBUG, ptr_biblio_info->encoding);
      send_status(ptr_clrequest->fd, 701, TERM_NO);
      LOG_PRINT(LOG_WARNING, get_status_msg(701));
      delete_all_lilimem(&sentinel);
      return 1;
    }
    else {
      LOG_PRINT(LOG_DEBUG, "database encoding is:");
      LOG_PRINT(LOG_DEBUG, db_encoding);
    }
  }
  else {
    conv_descriptor = NULL;
    LOG_PRINT(LOG_DEBUG, "no character encoding conversion required");
  }

  LOG_PRINT(LOG_DEBUG, "output encoding is:");
  if (*(ptr_biblio_info->encoding)) {
    LOG_PRINT(LOG_DEBUG, ptr_biblio_info->encoding);
  }
  else {
    LOG_PRINT(LOG_DEBUG, db_encoding);
  }


  token = query_string;
  newtoken = token;

  /* general strategy: we extract the tokens one after the other and insert
     brackets, logical operators, and those requests which map to simple
     t_refdb queries directly into the future search string. For those 
     queries that require a relational lookup a surrogate subselect query
     is formulated, as mySQL does not (yet) support true subselects.
     The results of these subselects are lists of refdb_id values which
     are translated into query fragments and get inserted into the future
     search string. The final search string is sent to the db server
     and provides us with the final list of the t_refdb fields we're
     interested in. This list is finally used to extract the additional
     data from the t_author, t_keyword, and t_user fields that we need to
     send back the result. */

  /* loop as long as we find more tokens */
  while (newtoken != NULL) {
    token = sql_tokenize(newtoken, &sqltoken);
    newtoken = sqltoken.next_token;

    if (token != NULL) {
      /* extract the token and save to a temporary string */
      if (sqltoken.length > buffer_len) {
	new_buffer = (char*)realloc(buffer, (size_t)sqltoken.length);
	if (new_buffer == NULL) { /* out of memory */
	  retval = 1;
	  break;
	}
	else {
	  buffer = new_buffer;
	  buffer_len = sqltoken.length;
	}
      }
      strncpy(buffer, token, sqltoken.length);
      buffer[sqltoken.length] ='\0';
/*       printf("token went to:%s<<\n", buffer); */
      if (sqltoken.type != 4) { /* we recycle these tokens */
	if ((new_sql_command1 = mstrcat(sql_command1, buffer, &sql_command1_len, 0)) == NULL) {
	  retval = 1;
	  break;
	}
	else {
	  sql_command1 = new_sql_command1;
	}
      }
      else {
	/*----------------------------------------------------------------*/
	/* author */
	if (strncmp(buffer, ":AU:", 4) == 0 || strncmp(buffer, ":A1:", 4) == 0) {
	  author_type = 1;
	}
	else if (strncmp(buffer, ":A2:", 4) == 0 || strncmp(buffer, ":ED:", 4) == 0) {
	  author_type = 2;
	}
	else if (strncmp(buffer, ":A3:", 4) == 0) {
	  author_type = 3;
	}
	else if (strncmp(buffer, ":AX:", 4) == 0) {
	  author_type = 4;
	}
	else {
	  author_type = 0;
	}

	if (author_type != 0) { /* author query */
	  char* subselect = NULL;

	  strncpy(operator, &token[4], sqltoken.length-4); /* save the comparison operator */
	  operator[sqltoken.length-4] = '\0'; /* terminate string */

	  /* assemble a surrogate subselect query */
	  token = sql_tokenize(newtoken, &sqltoken);
	  newtoken = sqltoken.next_token;
	  if (token != NULL) {

	    strncpy(buffer, token, sqltoken.length);
	    buffer[sqltoken.length] ='\0'; /* terminate string */
/* 	    printf("author_token went to:%s<<\n", buffer); */
	    if ((subselect = assemble_author_subselect(buffer, operator, author_type, conn, driver)) == NULL) {
	      retval = 1;
	      break;
	    }
	  }

	  LOG_PRINT(LOG_DEBUG, subselect);

	  /* dbires will contain all refdb_ids of those entries that 
	     match the author requirements in the pseudo-subselect */
	  dbires = dbi_conn_query(conn, subselect);
	  if (!dbires) {
	    free(subselect);
	    retval = 2;
	    break;
	  }

	  free(subselect);

	  if (*operator == '!') {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_id NOT IN (", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_id IN (", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }

	  n_havesome = 0;
	  while (dbi_result_next_row(dbires) != 0) {
	    n_havesome = 1;
	    n_id = my_dbi_result_get_idval_idx(dbires, 1); /* 1-base index */
	    sprintf(the_id, ULLSPEC, (unsigned long long)n_id);
	    if ((new_sql_command1 = mstrcat(sql_command1, the_id, &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	    if ((new_sql_command1 = mstrcat(sql_command1, ",", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }

	  if (!n_havesome) { /* use a dummy value that does not exist in the database */
	    if ((new_sql_command1 = mstrcat(sql_command1, "-1,", &sql_command1_len, 0)) == NULL) { /* the comma will be removed further down */
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }	    
	  dbi_result_free(dbires);
	  if (retval) {
	    break;
	  }
	  if ((new_sql_command1 = mstrcat(sql_command1, ")", &sql_command1_len, 1)) == NULL) {
	    retval = 1;
	    break;
	  }
	  else {
	    sql_command1 = new_sql_command1;
	  }
	}

	/*----------------------------------------------------------------*/
	/* keyword */
	else if (strncmp(token, ":KW:", 4) == 0) { /* keyword query */
	  char* subselect = NULL;

	  strncpy(operator, &token[4], sqltoken.length-4); /* save the comparison operator */
	  operator[sqltoken.length-4] = '\0'; /* terminate string */

	  /* assemble a surrogate subselect query */
	  token = sql_tokenize(newtoken, &sqltoken);
	  newtoken = sqltoken.next_token;
	  if (token != NULL) {
	    strncpy(buffer, token, sqltoken.length);
	    buffer[sqltoken.length] ='\0'; /* terminate string */

	    if ((subselect = assemble_keyword_subselect(buffer, operator, conn)) == NULL) {
	      retval = 1;
	      break;
	    }
	  }

	  LOG_PRINT(LOG_DEBUG, subselect);

	  /* dbires will contain all refdb_ids that match the above
	     keyword requirements in the pseudo-subselect */
	  dbires = dbi_conn_query(conn, subselect);
	  if (!dbires) {
	    free(subselect);
	    retval = 2;
	    break;
	  }

	  free(subselect);

	  if (*operator == '!') {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_id NOT IN (", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_id IN (", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }

	  n_havesome = 0;
	  while (dbi_result_next_row(dbires) != 0) {
	    n_havesome = 1;
	    n_id = my_dbi_result_get_idval_idx(dbires, 1);
	    sprintf(the_id, ULLSPEC, (unsigned long long)n_id);
	    if ((new_sql_command1 = mstrcat(sql_command1, the_id, &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	    if ((new_sql_command1 = mstrcat(sql_command1, ",", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  if (!n_havesome) { /* use a dummy value that does not exist in the database */
	    if ((new_sql_command1 = mstrcat(sql_command1, "-1,", &sql_command1_len, 0)) == NULL) { /* the comma will be removed further down */
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }	    

	  dbi_result_free(dbires);
	  if (retval) {
	    break;
	  }
	  if ((new_sql_command1 = mstrcat(sql_command1, ")", &sql_command1_len, 1)) == NULL) {
	    retval = 1;
	    break;
	  }
	  else {
	    sql_command1 = new_sql_command1;
	  }
	}

	/*----------------------------------------------------------------*/
	/* URL, L1-L4 */
	else if (strncmp(token, ":UR:", 4) == 0
		 || strncmp(token, ":L1:", 4) == 0
		 || strncmp(token, ":L2:", 4) == 0
		 || strncmp(token, ":L3:", 4) == 0
		 || strncmp(token, ":L4:", 4) == 0
		 || strncmp(token, ":DO:", 4) == 0) { /* link query */
	  char* subselect = NULL;
	  char link_type[9];

	  strncpy(operator, &token[4], sqltoken.length-4); /* save the comparison operator */
	  operator[sqltoken.length-4] = '\0'; /* terminate string */

	  if (token[2] == 'R') {
	    strcpy(link_type, "URL");
	  }
	  else if (token[2] == '1') {
	    strcpy(link_type, "PDF");
	  }
	  else if (token[2] == '2') {
	    strcpy(link_type, "FULLTEXT");
	  }
	  else if (token[2] == '3') {
	    strcpy(link_type, "RELATED");
	  }
	  else if (token[2] == '4') {
	    strcpy(link_type, "IMAGE");
	  }
	  else if (token[2] == 'O') {
	    strcpy(link_type, "DOI");
	  }

	  /* assemble a surrogate subselect query */
	  token = sql_tokenize(newtoken, &sqltoken);
	  newtoken = sqltoken.next_token;
	  if (token != NULL) {
	    strncpy(buffer, token, sqltoken.length);
	    buffer[sqltoken.length] ='\0'; /* terminate string */

	    if ((subselect = assemble_link_subselect(buffer, operator, link_type, conn)) == NULL) {
	      retval = 1;
	      break;
	    }
	  }

	  LOG_PRINT(LOG_DEBUG, subselect);

	  /* dbires will contain all refdb_ids that match the above
	     keyword requirements in the pseudo-subselect */
	  dbires = dbi_conn_query(conn, subselect);
	  if (!dbires) {
	    free(subselect);
	    retval = 2;
	    break;
	  }

	  free(subselect);

	  if (*operator == '!') {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_id NOT IN (", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_id IN (", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }

	  n_havesome = 0;
	  while (dbi_result_next_row(dbires) != 0) {
	    n_havesome = 1;
	    n_id = my_dbi_result_get_idval_idx(dbires, 1);
	    sprintf(the_id, ULLSPEC, (unsigned long long)n_id);
	    if ((new_sql_command1 = mstrcat(sql_command1, the_id, &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	    if ((new_sql_command1 = mstrcat(sql_command1, ",", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  if (!n_havesome) { /* use a dummy value that does not exist in the database */
	    if ((new_sql_command1 = mstrcat(sql_command1, "-1,", &sql_command1_len, 0)) == NULL) { /* the comma will be removed further down */
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }	    

	  dbi_result_free(dbires);
	  if (retval) {
	    break;
	  }
	  if ((new_sql_command1 = mstrcat(sql_command1, ")", &sql_command1_len, 1)) == NULL) {
	    retval = 1;
	    break;
	  }
	  else {
	    sql_command1 = new_sql_command1;
	  }
	}

	/*----------------------------------------------------------------*/
	/* journal */
	else if (strncmp(token, ":JO:", 4) == 0
		 || strncmp(token, ":JF:", 4) == 0
		 || strncmp(token, ":J1:", 4) == 0
		 || strncmp(token, ":J2:", 4) == 0
		 || strncmp(token, ":JX:", 4) == 0) { /* periodical query */
	  char my_token[5]; /* backup of token */
	  char my_bool[4];
	  strncpy(my_token, token, 4);
	  my_token[4] = '\0';
	  strncpy(operator, &token[4], sqltoken.length-4); /* save the comparison operator */
	  operator[sqltoken.length-4] = '\0'; /* terminate string */

	  /* translate operator to regexp operator */
	  if (*operator == '=') {
	    strcpy(bitsandpieces, " = ");
	    strcpy(my_bool, "OR");
	  }
	  else if (*operator == '~') {
	    sprintf(bitsandpieces, " %s ", my_dbi_conn_get_cap(conn, "rlike"));
	    strcpy(my_bool, "OR");
	  }
	  else if (*(operator+1) == '=') { 
	    strcpy(bitsandpieces, " != ");
	    strcpy(my_bool, "AND");
	  }
	  else { /* treat all other operators as non-equal regexp*/
	    sprintf(bitsandpieces, " %s ", my_dbi_conn_get_cap(conn, "not_rlike"));
	    strcpy(my_bool, "AND");
	  }
	    
	  token = sql_tokenize(newtoken, &sqltoken);
	  newtoken = sqltoken.next_token;
	  if (token != NULL) {
	    if ((new_buffer = mstrncpy(buffer, token, sqltoken.length, &buffer_len)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      buffer = new_buffer;
	    }
	    buffer[sqltoken.length] ='\0';
	    quoted_token = mstrdup(buffer);
	    if (!quoted_token) {
	      retval = 1;
	      break;
	    }
	    if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	      retval = 1;
	      break;
	    }
	  }
	  else {
	    quoted_token = mstrdup("");
	  }

	  /* assemble a surrogate subselect query */
	  if (strncmp(my_token, ":JX:", 4) == 0) {
	    char jxbuffer[2048];
	    snprintf(jxbuffer, 2048, "SELECT t_refdb.refdb_periodical_id FROM t_refdb INNER JOIN t_periodical ON t_periodical.periodical_id=t_refdb.refdb_periodical_id WHERE (t_periodical.periodical_name%s%s %s t_periodical.periodical_abbrev%s%s %s t_periodical.periodical_custabbrev1%s%s %s t_periodical.periodical_custabbrev2%s%s)", bitsandpieces, quoted_token, my_bool, bitsandpieces, quoted_token, my_bool, bitsandpieces, quoted_token, my_bool, bitsandpieces, quoted_token);
	    free(quoted_token);
	    *sql_command = '\0';
	    if ((new_sql_command = mstrcat(sql_command, jxbuffer, &sql_command_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command = new_sql_command;
	    }
	  }
	  else {
	    if (strncmp(my_token, ":JF:", 4) == 0) {
	      strcpy(sql_command, "SELECT t_refdb.refdb_periodical_id FROM t_refdb INNER JOIN t_periodical ON t_periodical.periodical_id=t_refdb.refdb_periodical_id WHERE t_periodical.periodical_name");
	    }
	    else if (strncmp(my_token, ":JO:", 4) == 0) {
	      strcpy(sql_command, "SELECT t_refdb.refdb_periodical_id FROM t_refdb INNER JOIN t_periodical ON t_periodical.periodical_id=t_refdb.refdb_periodical_id WHERE t_periodical.periodical_abbrev");
	    }
	    else if (strncmp(my_token, ":J1:", 4) == 0) {
	      strcpy(sql_command, "SELECT t_refdb.refdb_periodical_id FROM t_refdb INNER JOIN t_periodical ON t_periodical.periodical_id=t_refdb.refdb_periodical_id WHERE t_periodical.periodical_custabbrev1");
	    }
	    else if (strncmp(my_token, ":J2:", 4) == 0){
	      strcpy(sql_command, "SELECT t_refdb.refdb_periodical_id FROM t_refdb INNER JOIN t_periodical ON t_periodical.periodical_id=t_refdb.refdb_periodical_id WHERE t_periodical.periodical_custabbrev2");
	    }

	    if ((new_sql_command = mstrcat(sql_command, bitsandpieces, &sql_command_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command = new_sql_command;
	    }
	    
	    if (quoted_token != NULL) {
	      if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
		retval = 1;
		free(quoted_token);
		break;
	      }
	      else {
		sql_command = new_sql_command;
	      }
	      free(quoted_token);
	    }
	  } /* end if JX */

	  LOG_PRINT(LOG_DEBUG, sql_command);
	  dbires = dbi_conn_query(conn, sql_command);
	  if (!dbires) {
	    retval = 2;
	    break;
	  }

	  if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_periodical_id IN (", &sql_command1_len, 0)) == NULL) {
	    retval = 1;
	    break;
	  }
	  else {
	    sql_command1 = new_sql_command1;
	  }

	  if (dbi_result_get_numrows(dbires)) {
	    while (dbi_result_next_row(dbires) != 0) {
	      n_id = my_dbi_result_get_idval_idx(dbires, 1);
	      sprintf(the_id, ULLSPEC, (unsigned long long)n_id);
	      if ((new_sql_command1 = mstrcat(sql_command1, the_id, &sql_command1_len, 0)) == NULL) {
		retval = 1;
		dbi_result_free(dbires);
		break;
	      }
	      else {
		sql_command1 = new_sql_command1;
	      }
	      if ((new_sql_command1 = mstrcat(sql_command1, ",", &sql_command1_len, 0)) == NULL) {
		retval = 1;
		dbi_result_free(dbires);
		break;
	      }
	      else {
		sql_command1 = new_sql_command1;
	      }
	    }
	    /* remove trailing comma */
	    sql_command1[strlen(sql_command1)-1] = '\0';
	  }
	  else { /* use a dummy value that does not exist in the database */
	    if ((new_sql_command1 = mstrcat(sql_command1, "-1", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }	    
	  
	  if ((new_sql_command1 = mstrcat(sql_command1, ")", &sql_command1_len, 0)) == NULL) {
	    retval = 1;
	    dbi_result_free(dbires);
	    break;
	  }
	  else {
	    sql_command1 = new_sql_command1;
	  }
	  dbi_result_free(dbires);
	}

	/*----------------------------------------------------------------*/
	/* search in all titles */
	else if (strncmp(token, ":TA:", 4) == 0 /* deprecated */
		 || strncmp(token, ":TX:", 4) == 0) { /* all titles query */
	  char table[3][27] = {"t_refdb.refdb_title",
			    "t_refdb.refdb_booktitle",
			    "t_refdb.refdb_title_series"};
	  char the_op[6];
	  int i;
	  int n_isfirst = 1;

	  /* todo: fixme */
	  strncpy(operator, &token[4], sqltoken.length-4); /* save the comparison operator */
	  operator[sqltoken.length-4] = '\0'; /* terminate string */

	  if (*operator == '=') {
	    strcpy(bitsandpieces, " = ");
	    strcpy(the_op, " OR ");
	  }
	  else if (*operator == '~') {
	    sprintf(bitsandpieces, " %s ", my_dbi_conn_get_cap(conn, "rlike"));
	    strcpy(the_op, " OR ");
	  }
	  else if (*operator == '!' && *(operator+1) == '=') {
	    sprintf(bitsandpieces, " != ");
	    strcpy(the_op, " AND ");
	  }
	  else { /* treat all other operators as non-equal */
	    sprintf(bitsandpieces, " %s ", my_dbi_conn_get_cap(conn, "not_rlike"));
	    strcpy(the_op, " AND ");
	  }
	  
	  token = sql_tokenize(newtoken, &sqltoken);
	  newtoken = sqltoken.next_token;
	  if (token != NULL) {
/*  	    printf("%s:%d\n", token, error); */
/*  	    printf("%s\n", newtoken); */
	    if ((new_buffer = mstrncpy(buffer, token, sqltoken.length, &buffer_len)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      buffer = new_buffer;
	    }
	    buffer[sqltoken.length] ='\0';
	    quoted_token = mstrdup(buffer);
	    if (!quoted_token) {
	      retval = 1;
	      break;
	    }
	    if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	      retval = 1;
	      break;
	    }

	    for (i = 0; i<3; i++) {
	      if (!n_isfirst) {
		if ((new_sql_command1 = mstrcat(sql_command1, the_op, &sql_command1_len, 0)) == NULL) {
		  retval = 1;
		  break;
		}
		else {
		  sql_command1 = new_sql_command1;
		}
	      }
	      else {
		n_isfirst--;
		if ((new_sql_command1 = mstrcat(sql_command1, "(", &sql_command1_len, 0)) == NULL) {
		  retval = 1;
		  break;
		}
		else {
		  sql_command1 = new_sql_command1;
		}
	      }

	      if ((new_sql_command1 = mstrcat(sql_command1, table[i], &sql_command1_len, 0)) == NULL) {
		retval = 1;
		break;
	      }
	      else {
		sql_command1 = new_sql_command1;
	      }
	  
	      if ((new_sql_command1 = mstrcat(sql_command1, bitsandpieces, &sql_command1_len, 0)) == NULL) {
		retval = 1;
		break;
	      }
	      else {
		sql_command1 = new_sql_command1;
	      }
	      
	      if ((new_sql_command1 = mstrcat(sql_command1, quoted_token, &sql_command1_len, 0)) == NULL) {
		retval = 1;
		free(quoted_token);
		break;
	      }
	      else {
		sql_command1 = new_sql_command1;
	      }
	    } /* end for */

	    if ((new_sql_command1 = mstrcat(sql_command1, ")", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }

	    free(quoted_token);
	  }
	}

	/*----------------------------------------------------------------*/
	/* extended notes - note ID */
	else if (strncmp(token, ":NID:", 5) == 0) { /* extended note ID query */
	  strncpy(operator, &token[5], sqltoken.length-5); /* save the comparison operator */
	  operator[sqltoken.length-5] = '\0'; /* terminate string */

	  /* assemble a surrogate subselect query */
	  quoted_token = mstrdup(ptr_clrequest->username);
	  if (!quoted_token) {
	    retval = 1;
	    break;
	  }
	  if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	    retval = 1;
	    break;
	  }

	  sprintf(sql_command, "SELECT t_refdb.refdb_id FROM t_refdb INNER JOIN t_xnote ON t_refdb.refdb_id=t_xnote.xref_id INNER JOIN t_note ON t_note.note_id=t_xnote.note_id INNER JOIN t_user ON t_note.note_user_id=t_user.user_id WHERE t_user.user_name=%s AND t_xnote.xnote_type=\'REFERENCE\' AND t_note.note_id", quoted_token);

	  free(quoted_token);

	  /* append operator */
	  if (*operator == '~') {
	    strcpy(bitsandpieces, "=");
	  }
	  else if (*operator == '!' && *(operator+1) == '~') {
	    strcpy(bitsandpieces, "!=");
	  }
	  else {
	    strcpy(bitsandpieces, operator);
	  }

	  if ((new_sql_command = mstrcat(sql_command, bitsandpieces, &sql_command_len, 0)) == NULL) {
	    retval = 1;
	    break;
	  }
	  else {
	    sql_command = new_sql_command;
	  }

	  token = sql_tokenize(newtoken, &sqltoken);
	  newtoken = sqltoken.next_token;
	  if (token != NULL) {
	    if ((new_buffer = mstrncpy(buffer, token, sqltoken.length, &buffer_len)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      buffer = new_buffer;
	    }
	    buffer[sqltoken.length] ='\0';

	    if ((new_sql_command = mstrcat(sql_command, buffer, &sql_command_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command = new_sql_command;
	    }
	  }
	  LOG_PRINT(LOG_DEBUG, sql_command);

	  dbires = dbi_conn_query(conn, sql_command);
	  if (!dbires) {
	    retval = 2;
	    break;
	  }

	  if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_id IN (", &sql_command1_len, 0)) == NULL) {
	    retval = 1;
	    break;
	  }
	  else {
	    sql_command1 = new_sql_command1;
	  }

	  if (dbi_result_get_numrows(dbires)) {
	    while (dbi_result_next_row(dbires) != 0) {
	      n_id = my_dbi_result_get_idval_idx(dbires, 1);
	      sprintf(the_id, ULLSPEC, (unsigned long long)n_id);
	      if ((new_sql_command1 = mstrcat(sql_command1, the_id, &sql_command1_len, 0)) == NULL) {
		retval = 1;
		dbi_result_free(dbires);
		break;
	      }
	      else {
		sql_command1 = new_sql_command1;
	      }
	      if ((new_sql_command1 = mstrcat(sql_command1, ",", &sql_command1_len, 0)) == NULL) {
		retval = 1;
		dbi_result_free(dbires);
		break;
	      }
	      else {
		sql_command1 = new_sql_command1;
	      }
	    }
	    /* remove trailing comma */
	    sql_command1[strlen(sql_command1)-1] = '\0';
	  }
	  else { /* use a dummy value that does not exist in the database */
	    if ((new_sql_command1 = mstrcat(sql_command1, "-1", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }	    
	  
	  if ((new_sql_command1 = mstrcat(sql_command1, ")", &sql_command1_len, 0)) == NULL) {
	    retval = 1;
	    dbi_result_free(dbires);
	    break;
	  }
	  else {
	    sql_command1 = new_sql_command1;
	  }
	  dbi_result_free(dbires);
	}

	/*----------------------------------------------------------------*/
	/* extended notes - note citekey */
	else if (strncmp(token, ":NCK:", 5) == 0) { /* extended note citekey query */
	  strncpy(operator, &token[5], sqltoken.length-5); /* save the comparison operator */
	  operator[sqltoken.length-5] = '\0'; /* terminate string */

	  /* assemble a surrogate subselect query */
	  quoted_token = mstrdup(ptr_clrequest->username);
	  if (!quoted_token) {
	    retval = 1;
	    break;
	  }
	  if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	    retval = 1;
	    break;
	  }

	  sprintf(sql_command, "SELECT t_refdb.refdb_id FROM t_refdb INNER JOIN t_xnote ON t_refdb.refdb_id=t_xnote.xref_id INNER JOIN t_note ON t_note.note_id=t_xnote.note_id INNER JOIN t_user ON t_note.note_user_id=t_user.user_id WHERE (t_user.user_name=%s OR (t_user.user_name!=%s AND t_note.note_share!='0')) AND t_xnote.xnote_type=\'REFERENCE\' AND t_note.note_key", quoted_token, quoted_token);

	  free(quoted_token);

	  /* translate operator to regexp operator */
	  if (*operator == '=') {
	    strcpy(bitsandpieces, " = ");
	  }
	  else if (*operator == '~') {
	    sprintf(bitsandpieces, " %s ", my_dbi_conn_get_cap(conn, "rlike"));
	  }
	  else if (*operator == '!' && *(operator+1) == '=') {
	    strcpy(bitsandpieces, " != ");
	  }
	  else { /* treat all other operators as non-equal */
	    sprintf(bitsandpieces, " %s ", my_dbi_conn_get_cap(conn, "not_rlike"));
	  }

	  if ((new_sql_command = mstrcat(sql_command, bitsandpieces, &sql_command_len, 0)) == NULL) {
	    retval = 1;
	    break;
	  }
	  else {
	    sql_command = new_sql_command;
	  }

	  token = sql_tokenize(newtoken, &sqltoken);
	  newtoken = sqltoken.next_token;
	  if (token != NULL) {
	    if ((new_buffer = mstrncpy(buffer, token, sqltoken.length, &buffer_len)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      buffer = new_buffer;
	    }
	    buffer[sqltoken.length] ='\0';
	    quoted_token = mstrdup(buffer);
	    if (!quoted_token) {
	      retval = 1;
	      break;
	    }
	    if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	      retval = 1;
	      break;
	    }
	    if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
	      retval = 1;
	      free(quoted_token);
	      break;
	    }
	    else {
	      sql_command = new_sql_command;
	    }
	    free(quoted_token);
	  }
	  LOG_PRINT(LOG_DEBUG, sql_command);

	  dbires = dbi_conn_query(conn, sql_command);
	  if (!dbires) {
	    retval = 2;
	    break;
	  }

	  if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_id IN (", &sql_command1_len, 0)) == NULL) {
	    retval = 1;
	    break;
	  }
	  else {
	    sql_command1 = new_sql_command1;
	  }

	  if (dbi_result_get_numrows(dbires)) {
	    while (dbi_result_next_row(dbires) != 0) {
	      n_id = my_dbi_result_get_idval_idx(dbires, 1);
	      sprintf(the_id, ULLSPEC, (unsigned long long)n_id);
	      if ((new_sql_command1 = mstrcat(sql_command1, the_id, &sql_command1_len, 0)) == NULL) {
		retval = 1;
		dbi_result_free(dbires);
		break;
	      }
	      else {
		sql_command1 = new_sql_command1;
	      }
	      if ((new_sql_command1 = mstrcat(sql_command1, ",", &sql_command1_len, 0)) == NULL) {
		retval = 1;
		dbi_result_free(dbires);
		break;
	      }
	      else {
		sql_command1 = new_sql_command1;
	      }
	    }
	    /* remove trailing comma */
	    sql_command1[strlen(sql_command1)-1] = '\0';
	  }
	  else { /* use a dummy value that does not exist in the database */
	    if ((new_sql_command1 = mstrcat(sql_command1, "-1", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }	    
	  
	  if ((new_sql_command1 = mstrcat(sql_command1, ")", &sql_command1_len, 0)) == NULL) {
	    retval = 1;
	    dbi_result_free(dbires);
	    break;
	  }
	  else {
	    sql_command1 = new_sql_command1;
	  }
	  dbi_result_free(dbires);
	}

	/*----------------------------------------------------------------*/
	/* Personal info: reprint, notes, availability */
	else if (strncmp(token, ":RP:", 4) == 0 || /* reprint query */
		 strncmp(token, ":N1:", 4) == 0 || /* notes query */
		 strncmp(token, ":AV:", 4) == 0) { /* availability query */
	  strncpy(operator, &token[4], sqltoken.length-4); /* save the comparison operator */
	  operator[sqltoken.length-4] = '\0'; /* terminate string */

	  /* assemble a surrogate subselect query */
	  if (*operator == '=') {
	    if (strncmp(token, ":RP:", 4) == 0) {
	      strcpy(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_reprint = ");
	    }
	    else if (strncmp(token, ":N1:", 4) == 0) {
	      strcpy(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_notes = ");
	    }
	    else {
	      strcpy(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_avail = ");
	    }
	  }
	  else if (*operator == '~') {
	    if (strncmp(token, ":RP:", 4) == 0) {
	      sprintf(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_reprint %s ", my_dbi_conn_get_cap(conn, "rlike"));
	    }
	    else if (strncmp(token, ":N1:", 4) == 0) {
	      sprintf(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_notes %s ", my_dbi_conn_get_cap(conn, "rlike"));
	    }
	    else {
	      sprintf(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_avail %s ", my_dbi_conn_get_cap(conn, "rlike"));
	    }
	  }
	  else if (*operator == '!' && *(operator+1) == '=') {
	    if (strncmp(token, ":RP:", 4) == 0) {
	      strcpy(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_reprint != ");
	    }
	    else if (strncmp(token, ":N1:", 4) == 0) {
	      strcpy(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_notes != ");
	    }
	    else {
	      strcpy(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_avail != ");
	    }
	  }
	  else {
	    if (strncmp(token, ":RP:", 4) == 0) {
	      sprintf(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_reprint %s ", my_dbi_conn_get_cap(conn, "not_rlike"));
	    }
	    else if (strncmp(token, ":N1:", 4) == 0) {
	      sprintf(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_notes %s ", my_dbi_conn_get_cap(conn, "not_rlike"));
	    }
	    else {
	      sprintf(sql_command, "SELECT t_xuser.refdb_id FROM t_refdb INNER JOIN t_xuser ON t_refdb.refdb_id=t_xuser.refdb_id WHERE t_xuser.xuser_avail %s ", my_dbi_conn_get_cap(conn, "not_rlike"));
	    }
	  }

	  token = sql_tokenize(newtoken, &sqltoken);
	  newtoken = sqltoken.next_token;
	  if (token != NULL) {
	    if ((new_buffer = mstrncpy(buffer, token, sqltoken.length, &buffer_len)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      buffer = new_buffer;
	    }
	    buffer[sqltoken.length] ='\0';
	    quoted_token = mstrdup(buffer);
	    if (!quoted_token) {
	      retval = 1;
	      break;
	    }
	    if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	      retval = 1;
	      break;
	    }
	    if ((new_sql_command = mstrcat(sql_command, quoted_token, &sql_command_len, 0)) == NULL) {
	      retval = 1;
	      free(quoted_token);
	      break;
	    }
	    else {
	      sql_command = new_sql_command;
	    }
	    free(quoted_token);
	  }
	  LOG_PRINT(LOG_DEBUG, sql_command);

	  /* dbires will contain a list of all refdb_ids that match
	     the above publication subquery */
	  dbires = dbi_conn_query(conn, sql_command);
	  if (!dbires) {
	    retval = 2;
	    break;
	  }

	  if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_id IN (", &sql_command1_len, 0)) == NULL) {
	    retval = 1;
	    break;
	  }
	  else {
	    sql_command1 = new_sql_command1;
	  }

	  n_havesome = 0;
	  while (dbi_result_next_row(dbires) != 0) {
	    n_havesome = 1;
	    n_id = my_dbi_result_get_idval_idx(dbires, 1);
	    sprintf(the_id, ULLSPEC, (unsigned long long)n_id);
	    if ((new_sql_command1 = mstrcat(sql_command1, the_id, &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	    if ((new_sql_command1 = mstrcat(sql_command1, ",", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  if (!n_havesome) { /* use a dummy value that does not exist in the database */
	    if ((new_sql_command1 = mstrcat(sql_command1, "-1,", &sql_command1_len, 0)) == NULL) { /* trailing comma will be overwritten below */
	      retval = 1;
	      dbi_result_free(dbires);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }	    

	  dbi_result_free(dbires);
	  if (retval) {
	    break;
	  }
	  if ((new_sql_command1 = mstrcat(sql_command1, ")", &sql_command1_len, 1)) == NULL) { /* overwrites trailing comma */
	    retval = 1;
	    break;
	  }
	  else {
	    sql_command1 = new_sql_command1;
	  }
	}

	/*----------------------------------------------------------------*/
	/* simple queries */
	else { /* all simple t_refdb queries */
	  strncpy(operator, &token[4], sqltoken.length-4); /* save the comparison operator */
	  operator[sqltoken.length-4] = '\0'; /* terminate string */
	  
	  /* initialize alphanumeric indicator */
	  n_isalpha = 1;

	  if (strncmp(token, ":TY:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_type", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":PY:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_pubyear", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	      n_isalpha = 0;
	    }
	  }
	  else if (strncmp(token, ":Y2:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_secyear", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	      n_isalpha = 0;
	    }
	  }
	  else if (strncmp(token, ":ID:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_id", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	      n_isalpha = 0;
	    }
	  }
	  else if (strncmp(token, ":CK:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_citekey", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":TI:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_title", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":T2:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_booktitle", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":T3:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_title_series", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":SN:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_issn", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":N2:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_abstract", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":SP:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_startpage", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":EP:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_endpage", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":VL:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_volume", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":IS:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_issue", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }

	  else if (strncmp(token, ":AD:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_address", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":CY:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_city", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":PB:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_publisher", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":U1:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_user1", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":U2:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_user2", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":U3:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_user3", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":U4:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_user4", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":U5:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_user5", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }

	  /*
	    TO typeofwork
	    AR area
	    OS ostype
	    DG degree
	    RT runningtime
	    CI classcodeintl
	    CU classcodeus
	    SE senderemail
	    RE recipientemail
	    MT mediatype
	    NV numvolumes
	    EI edition
	    CO computer
	    CF conferencelocation
	    RN registrynum
	    CL classification
	    SC section
	    PN pamphletnum
	    CN chapternum
	  */

	  else if (strncmp(token, ":TO:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_typeofwork", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":AR:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_area", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":OS:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_ostype", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":DG:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_degree", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":RT:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_runningtime", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":CI:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_classcodeintl", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":CU:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_classcodeus", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":SE:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_senderemail", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":RE:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_recipientemail", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":MT:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_mediatype", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":NV:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_numvolumes", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":EI:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_edition", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":CO:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_computer", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":CF:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_conferencelocation", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":RN:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_registrynum", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":CL:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_classification", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":SC:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_section", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":PN:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_pamphletnum", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else if (strncmp(token, ":CN:", 4) == 0) {
	    if ((new_sql_command1 = mstrcat(sql_command1, "t_refdb.refdb_chapternum", &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }

	  /* we have to treat two types of columns: numerical columns
	     can reuse the operator directly. Alphanumeric columns need
	     some intelligent translation into a regexp-based query */
	  if (n_isalpha) {
	    if (*operator == '=') {
 	      strcpy(bitsandpieces, " = ");
	    }
	    else if (*operator == '~') {
	      sprintf(bitsandpieces, " %s ", my_dbi_conn_get_cap(conn, "rlike"));
	    }
	    else if (*operator == '!' && *(operator+1) == '=') {
 	      sprintf(bitsandpieces, " != ");
	    }
	    else { /* treat all other operators as non-equal */
	      sprintf(bitsandpieces, " %s ", my_dbi_conn_get_cap(conn, "not_rlike"));
	    }

	    if ((new_sql_command1 = mstrcat(sql_command1, bitsandpieces, &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  else {
	    if ((new_sql_command1 = mstrcat(sql_command1, operator, &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	  }
	  token = sql_tokenize(newtoken, &sqltoken);
	  newtoken = sqltoken.next_token;
	  if (token != NULL) {
/*  	    printf("%s:%d\n", token, retval); */
/*  	    printf("%s\n", newtoken); */
	    if ((new_buffer = mstrncpy(buffer, token, sqltoken.length, &buffer_len)) == NULL) {
	      retval = 1;
	      break;
	    }
	    else {
	      buffer = new_buffer;
	    }
	    buffer[sqltoken.length] ='\0';
	    quoted_token = mstrdup(buffer);
	    if (!quoted_token) {
	      retval = 1;
	      break;
	    }
	    if (dbi_conn_quote_string(conn, &quoted_token) == 0) {
	      retval = 1;
	      break;
	    }
	    if ((new_sql_command1 = mstrcat(sql_command1, quoted_token, &sql_command1_len, 0)) == NULL) {
	      retval = 1;
	      free(quoted_token);
	      break;
	    }
	    else {
	      sql_command1 = new_sql_command1;
	    }
	    free(quoted_token);
	  }
	}
      } /* if (sqltoken.type != 4) / else */
    } /* if (token != NULL) */
  } /* while (newtoken != NULL) */

  if (retval) {
    if (retval == 1) {
      send_status(ptr_clrequest->fd, 801, TERM_NO);
      LOG_PRINT(LOG_CRIT, get_status_msg(801));
    }
    else if (retval == 2) {
      send_status(ptr_clrequest->fd, 234, TERM_NO);
      LOG_PRINT(LOG_WARNING, get_status_msg(234));
    }
/*      printf("retval detected\n"); */
    dbi_conn_close(conn);
    delete_all_lilimem(&sentinel);
    if (conv_descriptor) {
      iconv_close(conv_descriptor);
    }
    return 1;
  }

  /* now assemble the SQL query string proper */
/*    printf("assembling query\n"); */
  /* 0  refdb_id             ID  - (if numeric)
     1  refdb_type           TY  - 
     2  refdb_pubyear        PY  - (partial)
     3  refdb_startpage      SP  - 
     4  refdb_endpage        EP  - 
     5  refdb_abstract       N2  - 
     6  refdb_title          TI  - 
     7  refdb_volume         VL  - 
     8  refdb_issue          CP  - 
     9  refdb_booktitle      BT  - 
    10  refdb_city           CY  - 
    11  refdb_publisher      PB  - 
    12  refdb_title_series   T3  - 
    13  refdb_address        AD  - 
    14  refdb_issn           SN  - 
    15  refdb_periodical_id  JO  - (indirect)
    16  refdb_pyother_info   PY  - (partial)
    17  refdb_secyear        Y2  - (partial)
    18  refdb_secother_info  Y2  - (partial)
    19  refdb_user1          U1  - 
    20  refdb_user2          U2  - 
    21  refdb_user3          U3  - 
    22  refdb_user4          U4  - 
    23  refdb_user5          U5  - 
    24  refdb_typeofwork     ..  - 
    25  refdb_area           ..  - 
    26  refdb_ostype         ..  - 
    27  refdb_degree         ..  - 
    28  refdb_runningtime    ..  - 
    29  refdb_classcodeintl  ..  - 
    30  refdb_classcodeus    ..  - 
    31  refdb_senderemail    ..  - 
    32  refdb_recipientemail ..  - 
    33  refdb_mediatype      ..  - 
    34  refdb_numvolumes     ..  - 
    35  refdb_edition        ..  - 
    36  refdb_computer       ..  - 
    37  refdb_conferencelocation   ..  - 
    38  refdb_registrynum    ..  - 
    39  refdb_classification ..  - 
    40  refdb_section        ..  - 
    41  refdb_pamphletnum    ..  - 
    42  refdb_chapternum     ..  - 
    43  refdb_citekey        ID  - (if non-numeric)
*/
  if (*(ptr_clrequest->listname)) {
    /* have to query t_note and t_xnote as well */
    strcpy(sql_command, "SELECT DISTINCT t_refdb.refdb_id, t_refdb.refdb_type, t_refdb.refdb_pubyear, t_refdb.refdb_startpage, t_refdb.refdb_endpage, t_refdb.refdb_abstract, t_refdb.refdb_title, t_refdb.refdb_volume, t_refdb.refdb_issue, t_refdb.refdb_booktitle, t_refdb.refdb_city, t_refdb.refdb_publisher, t_refdb.refdb_title_series, t_refdb.refdb_address, t_refdb.refdb_issn, t_refdb.refdb_periodical_id, t_refdb.refdb_pyother_info, t_refdb.refdb_secyear, t_refdb.refdb_secother_info, t_refdb.refdb_user1, t_refdb.refdb_user2, t_refdb.refdb_user3, t_refdb.refdb_user4, t_refdb.refdb_user5, t_refdb.refdb_typeofwork, t_refdb.refdb_area, t_refdb.refdb_ostype, t_refdb.refdb_degree, t_refdb.refdb_runningtime, t_refdb.refdb_classcodeintl, t_refdb.refdb_classcodeus, t_refdb.refdb_senderemail, t_refdb.refdb_recipientemail, t_refdb.refdb_mediatype, t_refdb.refdb_numvolumes, t_refdb.refdb_edition, t_refdb.refdb_computer, t_refdb.refdb_conferencelocation, t_refdb.refdb_registrynum, t_refdb.refdb_classification, t_refdb.refdb_section, t_refdb.refdb_pamphletnum, t_refdb.refdb_chapternum, t_refdb.refdb_citekey FROM t_refdb INNER JOIN t_xnote ON t_refdb.refdb_id=t_xnote.xref_id INNER JOIN t_note ON t_xnote.note_id=t_note.note_id WHERE ");
  }
  else {
    /* have to query only t_refdb */
    strcpy(sql_command, "SELECT DISTINCT t_refdb.refdb_id, t_refdb.refdb_type, t_refdb.refdb_pubyear, t_refdb.refdb_startpage, t_refdb.refdb_endpage, t_refdb.refdb_abstract, t_refdb.refdb_title, t_refdb.refdb_volume, t_refdb.refdb_issue, t_refdb.refdb_booktitle, t_refdb.refdb_city, t_refdb.refdb_publisher, t_refdb.refdb_title_series, t_refdb.refdb_address, t_refdb.refdb_issn, t_refdb.refdb_periodical_id, t_refdb.refdb_pyother_info, t_refdb.refdb_secyear, t_refdb.refdb_secother_info, t_refdb.refdb_user1, t_refdb.refdb_user2, t_refdb.refdb_user3, t_refdb.refdb_user4, t_refdb.refdb_user5, t_refdb.refdb_typeofwork, t_refdb.refdb_area, t_refdb.refdb_ostype, t_refdb.refdb_degree, t_refdb.refdb_runningtime, t_refdb.refdb_classcodeintl, t_refdb.refdb_classcodeus, t_refdb.refdb_senderemail, t_refdb.refdb_recipientemail, t_refdb.refdb_mediatype, t_refdb.refdb_numvolumes, t_refdb.refdb_edition, t_refdb.refdb_computer, t_refdb.refdb_conferencelocation, t_refdb.refdb_registrynum, t_refdb.refdb_classification, t_refdb.refdb_section, t_refdb.refdb_pamphletnum, t_refdb.refdb_chapternum, t_refdb.refdb_citekey FROM t_refdb WHERE ");
  }

  if (ref_format != 1 || !strstr(ptr_biblio_info->format_string, "NOHOLES")) {
    if ((new_sql_command = mstrcat(sql_command, "refdb_type!='DUMMY' AND ", &sql_command_len, 0)) == NULL) {
      send_status(ptr_clrequest->fd, 801, TERM_NO);
      LOG_PRINT(LOG_CRIT, get_status_msg(801));
      dbi_conn_close(conn);
      delete_all_lilimem(&sentinel);
      if (conv_descriptor) {
	iconv_close(conv_descriptor);
      }
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }
  }

  if ((new_sql_command = mstrcat(sql_command, sql_command1, &sql_command_len, 0)) == NULL) {
    send_status(ptr_clrequest->fd, 801, TERM_NO);
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    dbi_conn_close(conn);
    delete_all_lilimem(&sentinel);
    if (conv_descriptor) {
      iconv_close(conv_descriptor);
    }
    return 1;
  }
  else {
    sql_command = new_sql_command;
  }

  /* see whether we need a personal reference list instead of the whole list */
  if (*(ptr_clrequest->listname)) {
    char *quoted_listname;

    quoted_listname = strdup(ptr_clrequest->listname);

    if (!quoted_listname || !dbi_conn_quote_string(conn, &quoted_listname)) {
      send_status(ptr_clrequest->fd, 801, TERM_NO);
      LOG_PRINT(LOG_CRIT, get_status_msg(801));
      dbi_result_free(dbires);
      dbi_conn_close(conn);
      delete_all_lilimem(&sentinel);
      if (conv_descriptor) {
	iconv_close(conv_descriptor);
      }
      return 1;
    }
      
    sprintf(bitsandpieces, " AND t_xnote.xnote_type='REFERENCE' AND t_note.note_key=%s", quoted_listname);
    if ((new_sql_command = mstrcat(sql_command, bitsandpieces, &sql_command_len, 0)) == NULL) {
      send_status(ptr_clrequest->fd, 801, TERM_NO);
      LOG_PRINT(LOG_CRIT, get_status_msg(801));
      dbi_result_free(dbires);
      dbi_conn_close(conn);
      delete_all_lilimem(&sentinel);
      if (conv_descriptor) {
	iconv_close(conv_descriptor);
      }
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }
  }

  /* sort the output */
  if (strncmp(ptr_biblio_info->sort_string, "PY", 2) == 0) {
    if (retval || ((new_sql_command = mstrcat(sql_command, " ORDER BY t_refdb.refdb_pubyear", &sql_command_len, 0)) == NULL)) {
      send_status(ptr_clrequest->fd, 801, TERM_NO);
      LOG_PRINT(LOG_CRIT, get_status_msg(801));
      dbi_conn_close(conn);
      delete_all_lilimem(&sentinel);
      if (conv_descriptor) {
	iconv_close(conv_descriptor);
      }
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }
  }
  else {
    if (retval || ((new_sql_command = mstrcat(sql_command, " ORDER BY t_refdb.refdb_id", &sql_command_len, 0)) == NULL)) {
      send_status(ptr_clrequest->fd, 801, TERM_NO);
      LOG_PRINT(LOG_CRIT, get_status_msg(801));
      dbi_conn_close(conn);
      delete_all_lilimem(&sentinel);
      if (conv_descriptor) {
	iconv_close(conv_descriptor);
      }
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }
  }

  /* see whether the query should be limited to a range */
  if (*(ptr_clrequest->limit)) {
    char *colon;
    
    colon = strchr(ptr_clrequest->limit, (int)':');
      
    if (!colon) {
      snprintf(limitstring, 128, " LIMIT %s ", ptr_clrequest->limit);
    }
    else {
      *colon = '\0';
      snprintf(limitstring, 128, " LIMIT %s OFFSET %s ", ptr_clrequest->limit, colon+1);
    }

    if ((new_sql_command = mstrcat(sql_command, limitstring, &sql_command_len, 0)) == NULL) {
      send_status(ptr_clrequest->fd, 801, TERM_NO);
      LOG_PRINT(LOG_CRIT, get_status_msg(801));
      dbi_conn_close(conn);
      delete_all_lilimem(&sentinel);
      if (conv_descriptor) {
	iconv_close(conv_descriptor);
      }
      return 1;
    }
    else {
      sql_command = new_sql_command;
    }
  }

  LOG_PRINT(LOG_DEBUG, sql_command);

  /* actually run the query */
  rendinfo.dbires = dbi_conn_query(conn, sql_command);
  if (!rendinfo.dbires) {
    send_status(ptr_clrequest->fd, 234, TERM_NO);
    LOG_PRINT(LOG_WARNING, get_status_msg(234));
    dbi_conn_close(conn);
    delete_all_lilimem(&sentinel);
    if (conv_descriptor) {
      iconv_close(conv_descriptor);
    }
    return 1;
  }

  numrows = dbi_result_get_numrows(rendinfo.dbires);

  if (!send_data) { /* countref */
    ptr_addresult->success = numrows;
    dbi_result_free(rendinfo.dbires);
    dbi_conn_close(conn);
    delete_all_lilimem(&sentinel);
    send_status(ptr_clrequest->fd, 402, TERM_YES);
    
    return retval;
  }
  /* else: getref */


  /* create a "header" if necessary */
  /* prepare_render resets sql_command */

  /* fill in variable elements of structure */
  rendinfo.ptr_ref = &sql_command;
  rendinfo.ptr_ref_len = &sql_command_len;
  if (!*(rendinfo.ptr_biblio_info->encoding)) {
    rendinfo.ptr_biblio_info->encoding = dbi_conn_get_encoding(conn);
  }

  /* real output starts here */
  if (prepare_render(&rendinfo)) {
    send_status(ptr_clrequest->fd, 801, TERM_NO);
    LOG_PRINT(LOG_CRIT, get_status_msg(801));
    dbi_result_free(rendinfo.dbires);
    dbi_conn_close(conn);
    delete_all_lilimem(&sentinel);
    if (conv_descriptor) {
      iconv_close(conv_descriptor);
    }
    return 1;
  }

  if (numrows) {
    int chunksize; /* number of refs grouped together before sending */

    /* we use a larger chunksize for queries intended to retrieve ID
       lists (done by -t ris -s ID) as the returned documents are
       tiny, and a smaller one for other queries as we might
       run into swapping */
    if (!strcmp(rendinfo.ptr_biblio_info->format_string, "ID")
	&& ref_format == REFRIS) {
      chunksize = 100;
    }
    else {
      chunksize = 10;
    }

    /* fetch all articles we're interested in */
    while (dbi_result_next_row(rendinfo.dbires) != 0) {
      nref_counter++;
      /* fill in variable elements of structure */
      rendinfo.ptr_ref = &sql_command;
      rendinfo.ptr_ref_len = &sql_command_len;
      rendinfo.nref_counter = nref_counter;
      
      /* todo: count query errors and such as failed datasets, bail out only
	 after irrecoverable errors */
      if ((render_res = render_ref(&rendinfo)) != 0) {
	if (render_res == 801) {
	  retval = 1;
	  send_status(ptr_clrequest->fd, render_res, TERM_NO);
	  dbi_result_free(rendinfo.dbires);
	  dbi_conn_close(conn);
	  delete_all_lilimem(&sentinel);
	  if (conv_descriptor) {
	    iconv_close(conv_descriptor);
	  }
	  return 0;
	}
	else {
	  ptr_addresult->failure++;
	}
      }

      /* run a character encoding conversion if required */
      if (conv_descriptor && *sql_command) {
	size_t inlength;
	size_t outlength;
	char* my_sql_command = NULL; /* this ptr will be modified by iconv() */
	char* my_sql_command_start = NULL; /* records initial state of my_elvalue */
	const char* my_instring = NULL; /* this ptr will be modified by iconv() */
	/* strlen should be ok here as this can't be a multibyte encoding */
	inlength = strlen(sql_command)/*  + 1 */;
	/* with the encodings supported by our database engines, the converted
	   string can't be longer than six times the input string */
	/* todo: is this assumption correct? */
	outlength = 6*inlength;
	
	if ((my_sql_command = (char*)calloc(outlength, sizeof(char))) == NULL) {
	  retval = 1;
	  send_status(ptr_clrequest->fd, 801, TERM_NO);
	  LOG_PRINT(LOG_CRIT, get_status_msg(801));
	  dbi_result_free(rendinfo.dbires);
	  dbi_conn_close(conn);
	  delete_all_lilimem(&sentinel);
	  if (conv_descriptor) {
	    iconv_close(conv_descriptor);
	  }
	  return 0;
	}

	/* keep start of the converted string */
	my_sql_command_start = my_sql_command;

	/* variable will be modified by iconv, so don't use original */
	my_instring = (const char*)sql_command;

	/* now actually do the conversion */
	if (iconv(conv_descriptor, &my_instring, &inlength, &my_sql_command, &outlength) == (size_t)(-1)) {
	  if (errno == EILSEQ) {
	    LOG_PRINT(LOG_WARNING, "iconv: invalid input character sequence");
	  }
	  else if (errno == E2BIG) {
	    LOG_PRINT(LOG_WARNING, "iconv: output buffer too small");
	  }
	  else if (errno == EINVAL) {
	    LOG_PRINT(LOG_WARNING, "iconv: incomplete input character");
	  }
	  
	  retval = 1;
	  send_status(ptr_clrequest->fd, 702, TERM_NO);
	  dbi_result_free(rendinfo.dbires);
	  dbi_conn_close(conn);
	  delete_all_lilimem(&sentinel);
	  if (conv_descriptor) {
	    iconv_close(conv_descriptor);
	  }
	  return 0;
	}
	/* else: conversion went ok. We free the original string and replace
	   it with the converted copy */
	if (sql_command) {
	  free(sql_command);
	}
	sql_command = my_sql_command_start;
	sql_command_len = outlength;
	result_len = (size_t)(my_sql_command - my_sql_command_start);
      }
      else { /* no conversion required */
	result_len = strlen(sql_command);
      }

      ptr_addresult->success++;

      if (nref_counter % chunksize == 0) {
	/* send ok status, then the terminated result string */
	send_status(ptr_clrequest->fd, 404, TERM_NO);
	iwrite(ptr_clrequest->fd, sql_command, result_len); 
	iwrite(ptr_clrequest->fd, cs_term, TERM_LEN);
      
	/* reset buffer string */
	sql_command[0] = '\0';
      
	/* read client response */
	cs_status = read_status(ptr_clrequest->fd);

	if (cs_status) {
	  break;
	}
      }
    } /* end while */

    /* send remaining data */
    if (nref_counter % chunksize != 0) {
      /* send ok status, then the terminated result string */
      send_status(ptr_clrequest->fd, 404, TERM_NO);
      iwrite(ptr_clrequest->fd, sql_command, result_len); 
      iwrite(ptr_clrequest->fd, cs_term, TERM_LEN);
      
      /* reset buffer string */
      sql_command[0] = '\0';
      
      /* read client response */
      cs_status = read_status(ptr_clrequest->fd);
    }
  }
  else {
    /* no datasets, return an empty body */
      send_status(ptr_clrequest->fd, 402, TERM_NO);
      iwrite(ptr_clrequest->fd, cs_term, TERM_LEN);
  }

  if (conv_descriptor) {
    iconv_close(conv_descriptor);
  }

  if (ptr_addresult->success) {
    /* create a "footer" if necessary */
    /* fill in variable elements of structure */
/*     rendinfo.ptr_ref = &sql_command; */
/*     rendinfo.ptr_ref_len = &sql_command_len; */

    if ((render_res = finish_render(&rendinfo)) != 0) {
      retval = 1;
      send_status(ptr_clrequest->fd, 801, TERM_NO);
      dbi_result_free(rendinfo.dbires);
      dbi_conn_close(conn);
      delete_all_lilimem(&sentinel);
      return 0;
    }
    send_status(ptr_clrequest->fd, 402, TERM_NO);
    tiwrite(ptr_clrequest->fd, sql_command, TERM_YES); 
  }

  dbi_result_free(rendinfo.dbires);
  dbi_conn_close(conn);
  delete_all_lilimem(&sentinel);

  return retval;
}

