/**
 *    WHAM - high-throughput sequence aligner
 *    Copyright (C) 2011  WHAM Group, University of Wisconsin
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

/*	$Id: aligner.cpp 166 2012-11-26 20:28:17Z yinan $ */

#include <memory.h>
#include <string.h>
#include <sys/time.h>
#include <pthread.h>
#include "aligner.h"
#include "error.h"
#include "model.h"
#include "writer.h"
#include "bitread.h"
#include "util.h"
#include "perfcounters.h"
#include "rdtsc.h"

#define NO_PARTITION -1

Aligner::Aligner() {
  memset(this, 0, sizeof(Aligner));
}

Aligner::Aligner(char * path) {
  loadHead(path);

  /* set the load path */
  indexpath = path;
}

void Aligner::printInfo() {
  elog(DEBUG1, "  ---- Index Info ----\n");
  elog(DEBUG1, "  read length: %d\n", length);
  elog(DEBUG1, "  num of errors: %d\n", nError);
  elog(DEBUG1, "  num of fragments: %d\n", nPartition);
}

/*
 *	Aligner::computeNumIndex()
 *	Given the number of errors and the number of partitions, 
 *	compute the number of required hash tables, according to the 
 *	formula C(nPartition-1, nMismatch).
 */

int Aligner::computeNumIndex() {
  int i;
  int64 x = 1;

  for (i = nPartition - nError; i <= nPartition - 1; i++)
    x *= i;

  for (i = 1; i <= nError; i++)
    x /= i;

  return (int) x;
}

/*
 *	Aligner::computeNumLookup()
 *	Given the number of errors and the number of partitions, 
 *	compute the number of lookups for each alignment, according 
 *	to the formula C(nPartition, nMismatch).
 */

int Aligner::computeNumLookup() {
  int i;
  int64 x = 1;

  for (i = nPartition - nError + 1; i <= nPartition; i++)
    x *= i;

  for (i = 1; i <= nError; i++)
    x /= i;

  return (int) x;
}

/*
 *	Aligner::initLookupArray()
 *	compute the number of hash tables and lookups, and initialize 
 *	the lookupIndex and lookupOffset arrays.
 *	LookupIndex stores the partitioning group (hash table) ID for 
 *	each parititioning.
 *	LookupOffset stores the offset of the first indexed fragments 
 *	in each partitioning.
 */
int Aligner::initLookupArray() {
  int iid = 0, i, j, m, k, l, i4, i5;
  unsigned int x;
  unsigned int * indexBits;


  /*	allocate the array for lookup infomations */
  lookupIndex = new int[nLookup];
  if (lookupIndex == NULL
    )
    return ERR_MEM;

  lookupOffset = new int[nLookup];
  if (lookupOffset == NULL
    )
    return ERR_MEM;

  indexBits = new unsigned int[nHashTable];
  if (indexBits == NULL
    )
    return ERR_MEM;

  /*
   *	we initialize lookupOffset and lookupIndex based on bitwise
   *	techniques. A partitioning is represented by a binary string.
   *	An indexed fragment is represented by 1, whereas a non-indexed
   *	fragment is represented by 0. For example, 1010 represents four
   *	fragments. The first and third fragments are indexed fragments.
   */
  switch (nError) {
  case 1:
    /*
     *	enumerate the non-indexed fragment position. Binary
     *	representations for all partitionings are stored in
     *	indexBits.
     */
    for (i = 0, iid = 0; i < (nPartition - 1); i++) {
      indexBits[iid] = (0xffffffff >> (32 - nPartition)) & ~(1 << i);
      iid++;
    }
    for (i = 0, iid = 0; i < (nPartition); i++) {
      /*
       *	generate a binary representation for a partitioning
       *	group.
       */
      x = (0xffffffff >> (32 - nPartition)) & ~(1 << i);

      /*
       *	find the partitions that matches x by sliding the
       *	partitions.
       */
      for (k = 0; k < nHashTable; k++) {
        for (l = 0; l < nPartition; l++) {
          if (indexBits[k] == (x << l)) {
            /*	update element values */
            lookupIndex[iid] = k;
            lookupOffset[iid] = l * lenPartition;
            break;
          }
        }
      }
      iid++;
    }

    break;

  case 2:
    /*
     *	enumerate the non-indexed fragment positions. Binary
     *	representations for all partitionings are stored in
     *	indexBits.
     */
    for (i = 0, iid = 0; i < (nPartition - 2); i++) {
      for (j = i; j < (nPartition - 2); j++) {
        indexBits[iid] = (0xffffffff >> (32 - nPartition)) & ~(1 << i)
            & ~(1 << (j + 1));
        iid++;
      }
    }
    for (i = 0, iid = 0; i < (nPartition - 1); i++) {
      for (j = i; j < (nPartition - 1); j++) {
        /*
         *	generate a binary representation for a partitioning
         *	group.
         */
        x = (0xffffffff >> (32 - nPartition)) & ~(1 << i) & ~(1 << (j + 1));

        /*
         *	find the partitions that matches x by sliding the
         *	partitions.
         */
        for (k = 0; k < nHashTable; k++) {
          for (l = 0; l < nPartition; l++) {
            if (indexBits[k] == (x << l)) {
              /*	update element values */
              lookupIndex[iid] = k;
              lookupOffset[iid] = l * lenPartition;
              break;
            }
          }
        }
        iid++;
      }
    }
    break;

  case 3:
    /*
     *	enumerate the non-indexed fragment positions. Binary
     *	representations for all partitionings are stored in
     *	indexBits.
     */
    for (i = 0, iid = 0; i < (nPartition - 3); i++) {
      for (j = i; j < (nPartition - 3); j++) {
        for (m = j; m < (nPartition - 3); m++) {
          indexBits[iid] = (0xffffffff >> (32 - nPartition)) & ~(1 << i)
              & ~(1 << (j + 1)) & ~(1 << (m + 2));
          iid++;
        }
      }
    }
    for (i = 0, iid = 0; i < (nPartition - 2); i++) {
      for (j = i; j < (nPartition - 2); j++) {
        for (m = j; m < (nPartition - 2); m++) {
          /*
           *	generate a binary representation for a partitioning
           *	group.
           */
          x = (0xffffffff >> (32 - nPartition)) & ~(1 << i) & ~(1 << (j + 1))
              & ~(1 << (m + 2));

          /*
           *	find the partitions that matches x by sliding the
           *	partitions.
           */
          for (k = 0; k < nHashTable; k++) {
            for (l = 0; l < nPartition; l++) {
              if (indexBits[k] == (x << l)) {
                /*	update element values */
                lookupIndex[iid] = k;
                lookupOffset[iid] = l * lenPartition;
                break;
              }
            }
          }
          iid++;
        }
      }
    }
    break;

  case 4:
    /*
     *	enumerate the non-indexed fragment positions. Binary
     *	representations for all partitionings are stored in
     *	indexBits.
     */
    for (i = 0, iid = 0; i < (nPartition - 4); i++) {
      for (j = i; j < (nPartition - 4); j++) {
        for (m = j; m < (nPartition - 4); m++) {
          for (i4 = m; i4 < (nPartition - 4); i4++) {
            indexBits[iid] = (0xffffffff >> (32 - nPartition)) & ~(1 << i)
                & ~(1 << (j + 1)) & ~(1 << (m + 2)) & ~(1 << (i4 + 3));
            iid++;
          }
        }
      }
    }
    for (i = 0, iid = 0; i < (nPartition - 3); i++) {
      for (j = i; j < (nPartition - 3); j++) {
        for (m = j; m < (nPartition - 3); m++) {
          for (i4 = m; i4 < (nPartition - 3); i4++) {
            /*
             *	generate a binary representation for a partitioning
             *	group.
             */
            x = (0xffffffff >> (32 - nPartition)) & ~(1 << i) & ~(1 << (j + 1))
                & ~(1 << (m + 2)) & ~(1 << (i4 + 3));

            /*
             *	find the partitions that matches x by sliding the
             *	partitions.
             */
            for (k = 0; k < nHashTable; k++) {
              for (l = 0; l < nPartition; l++) {
                if (indexBits[k] == (x << l)) {
                  /*	update element values */
                  lookupIndex[iid] = k;
                  lookupOffset[iid] = l * lenPartition;
                  break;
                }
              }
            }
            iid++;
          }
        }
      }
    }
    break;

  case 5:
    /*
     *	enumerate the non-indexed fragment positions. Binary
     *	representations for all partitionings are stored in
     *	indexBits.
     */
    for (i = 0, iid = 0; i < (nPartition - 5); i++) {
      for (j = i; j < (nPartition - 5); j++) {
        for (m = j; m < (nPartition - 5); m++) {
          for (i4 = m; i4 < (nPartition - 5); i4++) {
            for (i5 = i4; i5 < (nPartition - 5); i5++) {
              indexBits[iid] = (0xffffffff >> (32 - nPartition)) & ~(1 << i)
                  & ~(1 << (j + 1)) & ~(1 << (m + 2)) & ~(1 << (i4 + 3))
                  & ~(1 << (i5 + 4));
              iid++;
            }
          }
        }
      }
    }
    for (i = 0, iid = 0; i < (nPartition - 4); i++) {
      for (j = i; j < (nPartition - 4); j++) {
        for (m = j; m < (nPartition - 4); m++) {
          for (i4 = m; i4 < (nPartition - 4); i4++) {
            for (i5 = i4; i5 < (nPartition - 4); i5++) {
              /*
               *	generate a binary representation for a partitioning
               *	group.
               */
              x = (0xffffffff >> (32 - nPartition)) & ~(1 << i)
                  & ~(1 << (j + 1)) & ~(1 << (m + 2)) & ~(1 << (i4 + 3))
                  & ~(1 << (i5 + 4));

              /*
               *	find the partitions that matches x by sliding the
               *	partitions.
               */
              for (k = 0; k < nHashTable; k++) {
                for (l = 0; l < nPartition; l++) {
                  if (indexBits[k] == (x << l)) {
                    /*	update element values */
                    lookupIndex[iid] = k;
                    lookupOffset[iid] = l * lenPartition;
                    break;
                  }
                }
              }
              iid++;
            }
          }
        }
      }
    }
    break;
  }

  /* initialize infos for embed table lookups */
  int keySpan;
  int keyPartitions[10];
  memset(keyPartitions, 0, sizeof(int) * 10);
  for (i = 0; i < nHashTable; i++) {
    for (j = 0; j < nPartition; j++) {
      if ((indexBits[i] >> j) & 0x1 == 1) {
        keySpan = nPartition - j;
        break;
      }
    }
    for (j = nPartition - 1, k = 0; j >= 0; j--) {
      if ((indexBits[i] >> j) & 0x1 == 1) {
        keyPartitions[k++] = j * lenPartition;
      }
    }
    hashTables[i].setLookupInfo(keySpan, keyPartitions);
  }

  delete[] indexBits;

  return SUCCESS;
}

/*
 *	Aligner::init()
 *	initialize the aligner structure.
 */
int Aligner::init(CompactSequence * seq, int len, int numPartition,
    unsigned int numBucket, int numError, int numInsert, int numDelete,
    int repeat, bool embed) {

  if (numError > 5) {
    printf("nError = %d is not supported in this version.\n", numError);
    return ERR_PARA;
  }

  if (numPartition > 0 && numPartition <= numError) {
    printf(
        "The number of partitions should be larger than the number of mismatches.\n");
    return ERR_PARA;
  }

  /*	initialize the genome sequence */
  sequence = seq;
  numEntry = sequence->getNum();

  /*	choose number of partition based on a cost model */
  if (numPartition == 0) {
    numPartition = AlignerModel::estimateNumPartition(numEntry, len, numError,
        false);
    elog(INFO, "optimizer chooses %d fragments\n", numPartition);
  }

  /*	check the index size */

  if (!AlignerModel::isFitMemory(numEntry, len, numError, numPartition)) {
    int nPartition;
    nPartition = AlignerModel::estimateNumPartition(numEntry, len, numError,
        true);
    if (nPartition == 0) {
      elog(
          ERROR,
          "insufficient memory to build any WHAM index. Increase memory size or use smaller reference sequence.\n");
      return ERR_MEM;
    }
  }

  /*	update the parameters */
  length = len;
  nPartition = numPartition;
  nError = numError;
  nInsert = numInsert;
  nDelete = numDelete;
  maxRepeat = repeat;
  embedHashTable = embed;
  nMaxError = nError;
  maxQual = MAX_INT;
  words = NUM_LONGWORD(length * BITS_PER_BASE);

  /*	initialize partitioning info	*/
  lenKey = length * BITS_PER_BASE;
  lenPartition = (length - nInsert) / nPartition * BITS_PER_BASE;
  lenPartitions = lenPartition * nPartition;
  lenRest = lenKey - lenPartitions;

  /*  initialize head mask */
  BitRead::genHeadMask(headMask, lenPartitions);

  /*
   *  compute the number of hashtables, and the number of
   *  lookups
   */
  nHashTable = computeNumIndex();
  nLookup = computeNumLookup();

  hashTables = new HashTable[nHashTable];
  if (hashTables == NULL
    )
    return ERR_MEM;

  for (int i = 0; i < nHashTable; i++) {
    hashTables[i].init(seq, len, numBucket, nError, nInsert, nDelete,
        nPartition, maxRepeat, embed, i);
  }

  /*  initialize lookup info  */
  int ret = initLookupArray();
  if (ret != SUCCESS)
    return ret;

  return SUCCESS;
}

/*
 *	Aligner::valid
 *	this function is used to check if the aligner is compatible 
 *	with the specificied parameters 
 */
int Aligner::valid(int len, int numPartition, int numError, int numInsert,
    int numDelete) {
  if (len != length)
    return ERR_PARA;

  if (numPartition != nPartition)
    return ERR_PARA;

  if (numError != nError)
    return ERR_PARA;

  if (numInsert != nInsert)
    return ERR_PARA;

  if (numDelete != nDelete)
    return ERR_PARA;

  return SUCCESS;
}

/*
 *	Alinger::preProcessHashTables()
 *	This function is used to collect infomation about the sequence for 
 *	building the hash table. If indexID is not specified (indexID == 
 *	ALIGNER_ALL_INDEX), we process on all indexes. Otherwise, we only 
 *	process the specified index.
 *
 *	To save the space, the collectted infomation is stored in the hash 
 *	buckets that is allocated in this function. In  particular, we scan 
 *	the genome sequence to:
 *	1) count the number of entries in each hash bucket. The number is 
 *	stored in corresponding bucket.
 *	2) identify empty hash buckets. The values of these buckets are set 
 *	to be HASH_EMPTY.
 *	3) identify collision in hash buckets. If more than one entry is hashed
 *	into a bucket, the MSB of the bucket is set to be 1.
 */

int Aligner::preProcessHashTables() {
  unsigned int i;
  int i1, i2, i3, i4, i5;
  unsigned int num;
  int ret, pid;
  int64 tspace[96];
  int64 * key = &tspace[8], *subkey1 = &tspace[24], *subkey2 = &tspace[40],
      *subkey3 = &tspace[56], *subkey4 = &tspace[72], *subkey5 = &tspace[88];
  int64 * seq;

  /*	clear all bits in key space */
  memset(tspace, 0, sizeof(int64) * 16 * nError);

  /*
   *	allocate hash buckets and bitmaps for specified
   *	hash tables. The hash buckets are temporarily used
   *	to store statistics information.
   */
  for (i = 0; i < nHashTable; i++) {
    ret = hashTables[i].preProcessInit();
    if (ret != SUCCESS
      )
      return ret;
  }

  num = numEntry - length + nError * lenPartition / BITS_PER_BASE;
  seq = sequence->getSequence();

  ProgressBar bar(num, PROGRESS_BAR_WIDTH);

  for (i = 0; i <= num; i++) {
    /*	update the progress bar.*/
    bar.update(i);

    /*
     *	extract a portion starting from k-th character in the
     *	sequence, store it to a right-aligned integer array.
     *	The offset is plused by lenRest to discard the leftmost
     *	lenRest bits that are not used for lookup. The portion
     *	has lenPartitions bits.
     */
//		sequence->get(key, i * BITS_PER_BASE + lenRest, lenPartitions);
    BitRead::extract(seq, key, i * BITS_PER_BASE_LL + lenRest, lenPartitions);

    switch (nError) {
    case 0:
      /*	exact match. No partitioning. */
      hashTables[0].preProcessInsert(key);
      break;
    case 1:
      /*
       *	generate all partitioning on the query sequence by
       *	enumerating the unindexed fragment.
       */
      pid = 0;
      for (i1 = 0; i1 < lenPartition * (nPartition - 1); i1 += lenPartition) {
        /*
         *	extract the indexed segments by removing the intervals
         *	from the sequence. The interval is a segment starting
         *	from i1 with the length of lenPartition.
         */
        BitRead::removeInterval(key, subkey1, i1, lenPartition);

        /*	insert the segment into specified hash table.	*/
        hashTables[pid].preProcessInsert(subkey1);
        pid++;
      }
      break;

    case 2:
      /*
       *	generate all partitioning on the query sequence by
       *	enumerating two unindexed fragments.
       */
      pid = 0;
      for (i1 = 0; i1 < lenPartition * (nPartition - 2); i1 += lenPartition) {
        /*
         *	extract the intermediate sequence by removing the first intervals
         *	from the sequence. The first interval is a segment starting
         *	from i1 with the length of lenPartition.
         */
        BitRead::removeInterval(key, subkey1, i1, lenPartition);
        for (i2 = i1; i2 < lenPartition * (nPartition - 2);
            i2 += lenPartition) {
          /*
           *	extract the indexed fragments by removing the second intervals
           *	from the intermediate sequence. The second interval is a segment
           *	starting from i2 with the length of lenPartition.
           */
          BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);

          /*	insert the segment into specified hash table.	*/
          hashTables[pid].preProcessInsert(subkey2);

          pid++;
        }
      }
      break;

    case 3:
      /*
       *	generate all partitioning on the query sequence by
       *	enumerating three unindexed fragments.
       */
      pid = 0;
      for (i1 = 0; i1 < lenPartition * (nPartition - 3); i1 += lenPartition) {
        /*
         *	extract the intermediate sequence by removing the first intervals
         *	from the sequence. The first interval is a segment starting
         *	from i1 with the length of lenPartition.
         */
        BitRead::removeInterval(key, subkey1, i1, lenPartition);
        for (i2 = i1; i2 < lenPartition * (nPartition - 3);
            i2 += lenPartition) {
          /*
           *	extract the intermediate sequence by removing the second intervals
           *	from the intermediate sequence. The second interval is a segment
           *	starting from i2 with the length of lenPartition.
           */
          BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);
          for (i3 = i2; i3 < lenPartition * (nPartition - 3);
              i3 += lenPartition) {
            /*
             *	extract the indexed fragments by removing the third intervals
             *	from the intermediate sequence. The second interval is a segment
             *	starting from i3 with the length of lenPartition.
             */
            BitRead::removeInterval(subkey2, subkey3, i3, lenPartition);

            /*	insert the segment into specified hash table.	*/
            hashTables[pid].preProcessInsert(subkey3);

            pid++;
          }
        }
      }
      break;

    case 4:
      pid = 0;
      for (i1 = 0; i1 < lenPartition * (nPartition - 4); i1 += lenPartition) {
        BitRead::removeInterval(key, subkey1, i1, lenPartition);
        for (i2 = i1; i2 < lenPartition * (nPartition - 4);
            i2 += lenPartition) {
          BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);
          for (i3 = i2; i3 < lenPartition * (nPartition - 4);
              i3 += lenPartition) {
            BitRead::removeInterval(subkey2, subkey3, i3, lenPartition);
            for (i4 = i3; i4 < lenPartition * (nPartition - 4); i4 +=
                lenPartition) {
              BitRead::removeInterval(subkey3, subkey4, i4, lenPartition);

              /*	insert the segment into specified hash table.	*/
              hashTables[pid].preProcessInsert(subkey4);

              pid++;
            }
          }
        }
      }
      break;

    case 5:
      pid = 0;
      for (i1 = 0; i1 < lenPartition * (nPartition - 5); i1 += lenPartition) {
        BitRead::removeInterval(key, subkey1, i1, lenPartition);
        for (i2 = i1; i2 < lenPartition * (nPartition - 5);
            i2 += lenPartition) {
          BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);
          for (i3 = i2; i3 < lenPartition * (nPartition - 5);
              i3 += lenPartition) {
            BitRead::removeInterval(subkey2, subkey3, i3, lenPartition);
            for (i4 = i3; i4 < lenPartition * (nPartition - 5); i4 +=
                lenPartition) {
              BitRead::removeInterval(subkey3, subkey4, i4, lenPartition);
              for (i5 = i4; i5 < lenPartition * (nPartition - 5); i5 +=
                  lenPartition) {
                BitRead::removeInterval(subkey4, subkey5, i5, lenPartition);

                /*	insert the segment into specified hash table.	*/
                hashTables[pid].preProcessInsert(subkey5);

                pid++;
              }
            }
          }
        }
      }
      break;
    }
  }

  /*
   *	collect information and store the results in specificed
   *	hash buckets.
   */
  for (i = 0; i < nHashTable; i++) {
    ret = hashTables[i].preProcessEnd();
    if (ret != SUCCESS
      )
      return ret;
  }

  return SUCCESS;
}

int Aligner::build(char * path) {
  int ret;
  unsigned int i, j;
  Timer timer;
  double t;

  timer.start();

  ret = buildHashTables(path);
  if (ret != SUCCESS
    )
    return ret;

  t = timer.stop();
  elog(INFO, "building time: %.6f sec\n", t);

//	ret = removeHashTables();
//	if (ret != SUCCESS)
//		return ret;

  /* check the test result */
  /*	elog(INFO, "testing WHAM indexes...");
   ret = check(1000);
   if (ret != SUCCESS)
   {
   elog(ERROR, "failed\n");
   return ret;
   }
   elog(INFO, "pass\n");
   */

  /*
   * write aligner head. The index has been tested.
   * The saved index is always correct.
   */
  if (path != NULL)
  {
    ret = saveHead(path);
    if (ret != SUCCESS
      )
      return ret;
  }

  return SUCCESS;
}

/*
 *	Alinger::buildHashTables()
 *	build the hash table. If indexID is not specified (indexID == 
 *	ALIGNER_ALL_INDEX), we perform build all indexes. Otherwise, 
 *	we only build the specified index.
 *	(1) invoke preProcess to collect statistics informations.
 *	(2) allocate the overflow array according to the number of 
 *	collision entries.
 *	(3) scan the genome sequence to insert entries into specified 
 *	index(es).
 */
int Aligner::buildHashTables(char * path) {
  unsigned int i, i1, i2, i3, i4, i5;
  unsigned int num;
  int ret, pid;
  int64 tspace[96];
  int64 * key = &tspace[8], *subkey1 = &tspace[24], *subkey2 = &tspace[40],
      *subkey3 = &tspace[56], *subkey4 = &tspace[72], *subkey5 = &tspace[88];
  int64 * seq;

  /*	clear all bits in key space */
  memset(tspace, 0, sizeof(int64) * 16 * nError);

  elog(INFO, "preprocessing...\n");

  /*
   *	invoke preProcess to get the statistics info, and allocate
   *	hash buckets and bitmaps.
   */
  ret = preProcessHashTables();
  if (ret != SUCCESS
    )
    return ret;

  elog(INFO, "building...\n");

  /*
   *	allocate overflow pool for specified hash tables.
   */
  for (i = 0; i < nHashTable; i++) {
    ret = hashTables[i].buildInit();
    if (ret != SUCCESS
      )
      return ret;
  }

  num = numEntry - length + nError * lenPartition / BITS_PER_BASE;
  seq = sequence->getSequence();

  ProgressBar bar(num, PROGRESS_BAR_WIDTH);

  for (i = num; i >= 0; i--) {
    /*	update the progress bar.*/
    bar.update(num - i);

    /*
     *	extract a portion starting from k-th character in the
     *	sequence, store it to a right-aligned integer array.
     *	The offset is plused by lenRest to discard the leftmost
     *	lenRest bits that are not used for lookup. The portion
     *	has lenPartitions bits.
     */
    BitRead::extract(seq, key, i * BITS_PER_BASE_LL + lenRest, lenPartitions);

    switch (nError) {
    case 0:
      /*	exact match. No partitioning. */
      hashTables[0].insert(key, i);
      break;

    case 1:
      /*
       *	generate all partitioning on the query sequence by
       *	enumerating the unindexed fragment.
       */
      pid = 0;
      for (i1 = 0; i1 < lenPartition * (nPartition - 1); i1 += lenPartition) {
        /*
         *	extract the indexed segments by removing the intervals
         *	from the sequence. The interval is a segment starting
         *	from i1 with the length of lenPartition.
         */
        BitRead::removeInterval(key, subkey1, i1, lenPartition);

        /*	insert the segment into specified hash table.	*/
        hashTables[pid].insert(subkey1, i);
        pid++;
      }
      break;

    case 2:
      /*
       *	generate all partitioning on the query sequence by
       *	enumerating two unindexed fragments.
       */
      pid = 0;
      for (i1 = 0; i1 < lenPartition * (nPartition - 2); i1 += lenPartition) {
        /*
         *	extract the intermediate sequence by removing the first intervals
         *	from the sequence. The first interval is a segment starting
         *	from i1 with the length of lenPartition.
         */
        BitRead::removeInterval(key, subkey1, i1, lenPartition);
        for (i2 = i1; i2 < lenPartition * (nPartition - 2);
            i2 += lenPartition) {
          /*
           *	extract the indexed fragments by removing the second intervals
           *	from the intermediate sequence. The second interval is a segment
           *	starting from i2 with the length of lenPartition.
           */
          BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);

          /*	insert the segment into specified hash table.	*/
          hashTables[pid].insert(subkey2, i);

          pid++;
        }
      }
      break;

    case 3:
      /*
       *	generate all partitioning on the query sequence by
       *	enumerating three unindexed fragments.
       */
      pid = 0;
      for (i1 = 0; i1 < lenPartition * (nPartition - 3); i1 += lenPartition) {
        /*
         *	extract the intermediate sequence by removing the first intervals
         *	from the sequence. The first interval is a segment starting
         *	from i1 with the length of lenPartition.
         */
        BitRead::removeInterval(key, subkey1, i1, lenPartition);
        for (i2 = i1; i2 < lenPartition * (nPartition - 3);
            i2 += lenPartition) {
          /*
           *	extract the intermediate sequence by removing the second intervals
           *	from the intermediate sequence. The second interval is a segment
           *	starting from i2 with the length of lenPartition.
           */
          BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);
          for (i3 = i2; i3 < lenPartition * (nPartition - 3);
              i3 += lenPartition) {
            /*
             *	extract the indexed fragments by removing the third intervals
             *	from the intermediate sequence. The second interval is a segment
             *	starting from i3 with the length of lenPartition.
             */
            BitRead::removeInterval(subkey2, subkey3, i3, lenPartition);

            /*	insert the segment into specified hash table.	*/
            hashTables[pid].insert(subkey3, i);

            pid++;
          }
        }
      }
      break;
    case 4:
      pid = 0;
      for (i1 = 0; i1 < lenPartition * (nPartition - 4); i1 += lenPartition) {
        BitRead::removeInterval(key, subkey1, i1, lenPartition);
        for (i2 = i1; i2 < lenPartition * (nPartition - 4);
            i2 += lenPartition) {
          BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);
          for (i3 = i2; i3 < lenPartition * (nPartition - 4);
              i3 += lenPartition) {
            BitRead::removeInterval(subkey2, subkey3, i3, lenPartition);
            for (i4 = i3; i4 < lenPartition * (nPartition - 4); i4 +=
                lenPartition) {
              BitRead::removeInterval(subkey3, subkey4, i4, lenPartition);

              /*	insert the segment into specified hash table.	*/
              hashTables[pid].insert(subkey4, i);

              pid++;
            }
          }
        }
      }
      break;
    case 5:
      pid = 0;
      for (i1 = 0; i1 < lenPartition * (nPartition - 5); i1 += lenPartition) {
        BitRead::removeInterval(key, subkey1, i1, lenPartition);
        for (i2 = i1; i2 < lenPartition * (nPartition - 5);
            i2 += lenPartition) {
          BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);
          for (i3 = i2; i3 < lenPartition * (nPartition - 5);
              i3 += lenPartition) {
            BitRead::removeInterval(subkey2, subkey3, i3, lenPartition);
            for (i4 = i3; i4 < lenPartition * (nPartition - 5); i4 +=
                lenPartition) {
              BitRead::removeInterval(subkey3, subkey4, i4, lenPartition);
              for (i5 = i4; i5 < lenPartition * (nPartition - 5); i5 +=
                  lenPartition) {
                BitRead::removeInterval(subkey4, subkey5, i5, lenPartition);

                /*	insert the segment into specified hash table.	*/
                hashTables[pid].insert(subkey5, i);

                pid++;
              }
            }
          }
        }
      }
      break;
    }
    if (i == 0)
      break;
  }

  if (embedHashTable) {
    elog(INFO, "building embeded tables...\n");
    for (i = 0; i < nHashTable; i++) {
      int ret = hashTables[i].buildEmbedTable();
      if (ret != SUCCESS)
        return ret;
    }
  }

  /*
   elog(INFO, "sorting repeats...\n");
   for (i = 0; i < nHashTable; i++)
   {
   ret = hashTables[i].sortList();
   if (ret != SUCCESS)
   {
   elog(ERROR, "failed to remove repeats in hash table %d.\n", i);
   return ret;
   }
   }
   */

  elog(INFO, "saving...\n");
  if (path != NULL)
  {
    for (i = 0; i < nHashTable; i++) {
      ret = hashTables[i].save(path);
      if (ret != SUCCESS)
      {
        elog(ERROR, "failed to save hash table %d.\n", i);
        return ret;
      }
    }
  }

  return SUCCESS;
}

void Aligner::sortList() {
  int ret;

  elog(INFO, "sorting repeats...\n");
  for (int i = 0; i < nHashTable; i++) {
    ret = hashTables[i].sortList();
    if (ret != SUCCESS)
    {
      elog(ERROR, "failed to remove repeats in hash table %d.\n", i);
      return;
    }
  }
}

unsigned int Aligner::alignFirst(int64 * orgkey, char * quals, strand s,
    int rid, HitSet * hits, bool noGap) {
  uint32 offset;
  int64 tspace[96];
  int64 * key = &tspace[8], *subkey1 = &tspace[24], *subkey2 = &tspace[40],
      *subkey3 = &tspace[56], *subkey4 = &tspace[72], *subkey5 = &tspace[88];
  int nHit = 0;
  int ret;

  /*	clear all bits in key space */
  memset(tspace, 0, sizeof(int64) * 16 * nError);

  /*
   *	The leftmost $lenPartitions$ bits are used to lookups on hash
   *	tables. The bits are extracted and stored in $key$..
   */
  BitRead::removeHead(orgkey, key, headMask);

  switch (nError) {
  case 0:
    ret = hashTables[0].lookup(orgkey, key, 0, quals, s, rid, hits, noGap);
    break;
  case 1:
    BitRead::removeInterval(key, subkey1, 0, lenPartition);

    ret = hashTables[lookupIndex[0]].lookup(orgkey, subkey1, lookupOffset[0],
        quals, s, rid, hits, noGap);
    break;
  case 2:
    BitRead::removeInterval(key, subkey1, 0, lenPartition);
    BitRead::removeInterval(subkey1, subkey2, 0, lenPartition);

    ret = hashTables[lookupIndex[0]].lookup(orgkey, subkey2, lookupOffset[0],
        quals, s, rid, hits, noGap);
    break;
  case 3:
    BitRead::removeInterval(key, subkey1, 0, lenPartition);
    BitRead::removeInterval(subkey1, subkey2, 0, lenPartition);
    BitRead::removeInterval(subkey2, subkey3, 0, lenPartition);

    ret = hashTables[lookupIndex[0]].lookup(orgkey, subkey3, lookupOffset[0],
        quals, s, rid, hits, noGap);
    break;
  case 4:
    BitRead::removeInterval(key, subkey1, 0, lenPartition);
    BitRead::removeInterval(subkey1, subkey2, 0, lenPartition);
    BitRead::removeInterval(subkey2, subkey3, 0, lenPartition);
    BitRead::removeInterval(subkey3, subkey4, 0, lenPartition);

    ret = hashTables[lookupIndex[0]].lookup(orgkey, subkey4, lookupOffset[0],
        quals, s, rid, hits, noGap);
    break;
  case 5:
    BitRead::removeInterval(key, subkey1, 0, lenPartition);
    BitRead::removeInterval(subkey1, subkey2, 0, lenPartition);
    BitRead::removeInterval(subkey2, subkey3, 0, lenPartition);
    BitRead::removeInterval(subkey3, subkey4, 0, lenPartition);
    BitRead::removeInterval(subkey4, subkey5, 0, lenPartition);

    ret = hashTables[lookupIndex[0]].lookup(orgkey, subkey5, lookupOffset[0],
        quals, s, rid, hits, noGap);
    break;
  }

  if (ret == MSG_HITSETFULL)
  {
    HASH_DEBUG(printf("\n")); HASH_DEBUG(getchar());
    return ret;
  }

  return SUCCESS;
}

/*
 *	Aligner::align
 *	This function is used to perform an alignment on a query sequence. 
 *	The query sequence is storen in orgkey. If indexID is not specified
 *	(indexID == ALIGNER_ALL_INDEX), we perform lookups on all indexes. 
 *	Otherwise, we only search the specified index. The successful 
 *	alingments are appended to the file.
 */

unsigned int Aligner::align(int64 * orgkey, char * quals, strand s, int rid,
    HitSet * hits, bool skipFirst, bool noGap) {
  int i1, i2, i3, i4, i5;
  int pid;
  uint32 offset;
  int64 tspace[96];
  int64 * key = &tspace[8], *subkey1 = &tspace[24], *subkey2 = &tspace[40],
      *subkey3 = &tspace[56], *subkey4 = &tspace[72], *subkey5 = &tspace[88];
  int nHit = 0;
  int ret;
//	HitPositionList list;

  /*	clear all bits in key space */
  memset(tspace, 0, sizeof(int64) * 16 * nError);

  /*
   *	The leftmost $lenPartitions$ bits are used to lookups on hash
   *	tables. The bits are extracted and stored in $key$..
   */
  BitRead::removeHead(orgkey, key, headMask);

  nHit = 0;
  offset = HASH_NOT_FOUND;
  switch (nError) {
  case 0:
    /*	exact match. No partitioning. */
    ret = hashTables[0].lookup(orgkey, key, 0, quals, s, rid, hits, noGap);
    if (ret == MSG_HITSETFULL
      )
      return ret;
    break;

  case 1:
    /*
     *	generate all partitioning on the query sequence by
     *	enumerating the unindexed fragment.
     */
    pid = 0;
    for (i1 = 0; i1 < lenPartition * (nPartition); i1 += lenPartition) {
      if (skipFirst && pid == 0) {
        pid++;
        continue;
      }

      /*
       *	extract the indexed segments by removing the intervals
       *	from the sequence. The interval is a segment starting
       *	from i1 with the length of lenPartition.
       */
      BitRead::removeInterval(key, subkey1, i1, lenPartition);

      /*
       *	search specified hash table to find potential matched
       *	segment based on the partitioning.
       */
      ret = hashTables[lookupIndex[pid]].lookup(orgkey, subkey1,
          lookupOffset[pid], quals, s, rid, hits, noGap);
      if (ret == MSG_HITSETFULL
        )
        return ret;
      pid++;
    }
    break;

  case 2:
    /*
     *	generate all partitioning on the query sequence by
     *	enumerating two unindexed fragments.
     */
    pid = 0;
    for (i1 = 0; i1 < lenPartition * (nPartition - 1); i1 += lenPartition) {
      /*
       *	extract the intermediate sequence by removing the first intervals
       *	from the sequence. The first interval is a segment starting
       *	from i1 with the length of lenPartition.
       */
      BitRead::removeInterval(key, subkey1, i1, lenPartition);
      for (i2 = i1; i2 < lenPartition * (nPartition - 1); i2 += lenPartition) {
        if (skipFirst && pid == 0) {
          pid++;
          continue;
        }

        /*
         *	extract the indexed fragments by removing the second intervals
         *	from the intermediate sequence. The second interval is a segment
         *	starting from i2 with the length of lenPartition.
         */
        BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);

        /*
         *	search specified hash table to find potential matched
         *	segment based on the partitioning.
         */
        ret = hashTables[lookupIndex[pid]].lookup(orgkey, subkey2,
            lookupOffset[pid], quals, s, rid, hits, noGap);
        if (ret == MSG_HITSETFULL)
        {
          HASH_DEBUG(printf("\n")); HASH_DEBUG(getchar());
          return ret;
        }

        pid++;
      }
    }
    break;

  case 3:
    /*
     *	generate all partitioning on the query sequence by
     *	enumerating three unindexed fragments.
     */
    pid = 0;
    for (i1 = 0; i1 < lenPartition * (nPartition - 2); i1 += lenPartition) {
      /*
       *	extract the intermediate sequence by removing the first intervals
       *	from the sequence. The first interval is a segment starting
       *	from i1 with the length of lenPartition.
       */
      BitRead::removeInterval(key, subkey1, i1, lenPartition);
      for (i2 = i1; i2 < lenPartition * (nPartition - 2); i2 += lenPartition) {
        /*
         *	extract the intermediate sequence by removing the second intervals
         *	from the intermediate sequence. The second interval is a segment
         *	starting from i2 with the length of lenPartition.
         */
        BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);
        for (i3 = i2; i3 < lenPartition * (nPartition - 2);
            i3 += lenPartition) {
          if (skipFirst && pid == 0) {
            pid++;
            continue;
          }

          /*
           *	extract the indexed fragments by removing the third intervals
           *	from the intermediate sequence. The second interval is a segment
           *	starting from i3 with the length of lenPartition.
           */
          BitRead::removeInterval(subkey2, subkey3, i3, lenPartition);

          /*
           *	search specified hash table to find potential matched
           *	segment based on the partitioning.
           */
          ret = hashTables[lookupIndex[pid]].lookup(orgkey, subkey3,
              lookupOffset[pid], quals, s, rid, hits, noGap);
          if (ret == MSG_HITSETFULL
            )
            return ret;

          pid++;
        }
      }
    }
    break;

  case 4:
    /*
     *	generate all partitioning on the query sequence by
     *	enumerating three unindexed fragments.
     */
    pid = 0;
    for (i1 = 0; i1 < lenPartition * (nPartition - 3); i1 += lenPartition) {
      /*
       *	extract the intermediate sequence by removing the first intervals
       *	from the sequence. The first interval is a segment starting
       *	from i1 with the length of lenPartition.
       */
      BitRead::removeInterval(key, subkey1, i1, lenPartition);
      for (i2 = i1; i2 < lenPartition * (nPartition - 3); i2 += lenPartition) {
        /*
         *	extract the intermediate sequence by removing the second intervals
         *	from the intermediate sequence. The second interval is a segment
         *	starting from i2 with the length of lenPartition.
         */
        BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);
        for (i3 = i2; i3 < lenPartition * (nPartition - 3);
            i3 += lenPartition) { /*
             *	extract the intermediate sequence by removing the second intervals
             *	from the intermediate sequence. The second interval is a segment
             *	starting from i2 with the length of lenPartition.
             */
          BitRead::removeInterval(subkey2, subkey3, i3, lenPartition);
          for (i4 = i3; i4 < lenPartition * (nPartition - 3);
              i4 += lenPartition) {
            if (skipFirst && pid == 0) {
              pid++;
              continue;
            }

            /*
             *	extract the indexed fragments by removing the third intervals
             *	from the intermediate sequence. The second interval is a segment
             *	starting from i3 with the length of lenPartition.
             */
            BitRead::removeInterval(subkey3, subkey4, i4, lenPartition);

            /*
             *	search specified hash table to find potential matched
             *	segment based on the partitioning.
             */
            ret = hashTables[lookupIndex[pid]].lookup(orgkey, subkey4,
                lookupOffset[pid], quals, s, rid, hits, noGap);
            if (ret == MSG_HITSETFULL
              )
              return ret;

            pid++;
          }
        }
      }
    }
    break;

  case 5:
    /*
     *	generate all partitioning on the query sequence by
     *	enumerating three unindexed fragments.
     */
    pid = 0;
    for (i1 = 0; i1 < lenPartition * (nPartition - 4); i1 += lenPartition) {
      /*
       *	extract the intermediate sequence by removing the first intervals
       *	from the sequence. The first interval is a segment starting
       *	from i1 with the length of lenPartition.
       */
      BitRead::removeInterval(key, subkey1, i1, lenPartition);
      for (i2 = i1; i2 < lenPartition * (nPartition - 4); i2 += lenPartition) {
        /*
         *	extract the intermediate sequence by removing the second intervals
         *	from the intermediate sequence. The second interval is a segment
         *	starting from i2 with the length of lenPartition.
         */
        BitRead::removeInterval(subkey1, subkey2, i2, lenPartition);
        for (i3 = i2; i3 < lenPartition * (nPartition - 4);
            i3 += lenPartition) {
          /*
           *	extract the intermediate sequence by removing the second intervals
           *	from the intermediate sequence. The second interval is a segment
           *	starting from i2 with the length of lenPartition.
           */
          BitRead::removeInterval(subkey2, subkey3, i3, lenPartition);
          for (i4 = i3; i4 < lenPartition * (nPartition - 4);
              i4 += lenPartition) {
            /*
             *	extract the intermediate sequence by removing the second intervals
             *	from the intermediate sequence. The second interval is a segment
             *	starting from i2 with the length of lenPartition.
             */
            BitRead::removeInterval(subkey3, subkey4, i4, lenPartition);
            for (i5 = i4; i5 < lenPartition * (nPartition - 4); i5 +=
                lenPartition) {
              if (skipFirst && pid == 0) {
                pid++;
                continue;
              }

              /*
               *	extract the indexed fragments by removing the third intervals
               *	from the intermediate sequence. The second interval is a segment
               *	starting from i3 with the length of lenPartition.
               */
              BitRead::removeInterval(subkey4, subkey5, i5, lenPartition);

              /*
               *	search specified hash table to find potential matched
               *	segment based on the partitioning.
               */
              ret = hashTables[lookupIndex[pid]].lookup(orgkey, subkey5,
                  lookupOffset[pid], quals, s, rid, hits, noGap);
              if (ret == MSG_HITSETFULL
                )
                return ret;

              pid++;
            }
          }
        }
      }
    }
    break;
  }

  HASH_DEBUG(printf("\n")); HASH_DEBUG(getchar());

  /*
   ret = hashTables[0].lookupHitList(
   orgkey, subkey2, lookupOffset[pid], quals, s, rid, hits, &list);
   if (ret == MSG_HITSETFULL)
   return ret;
   */
  return SUCCESS;

}

/*
 *	Alinger::check
 *	This function is used to examine the errors in the alinger. 
 *	We randomly choose some positions from the compact sequence. 
 *	Beginning from these positions, we extract subsequences and 
 *	manually modify some chracters. The modified subsequences are 
 *	searched in the aligner and check if we can find those choose 
 *	positions.
 */
int Aligner::check(int num) {
  unsigned int i, j, sid, pos = 0, word;
  int64 k1[16], k2[16], code;
  int64 * key = &k1[8], *key2 = &k2[8];
  int64 * seqVector;
  HitSet * hits;
  int ret;

  memset(k1, 0, 16 * sizeof(int64));
  memset(k2, 0, 16 * sizeof(int64));

  hits = new HitSet(0, 0, 0, false, false);
  hits->init(sequence, length);

  seqVector = sequence->getSequence();

  for (i = 0; i < num; i++) {
    /* choose a random position in the compact sequence */
    sid = RAND() % (numEntry - length + 1);

    /* get the subsequence beginning from the position */
    BitRead::extract(seqVector, key, sid * BITS_PER_BASE_LL
        , length * BITS_PER_BASE);

    BitRead::copy(key, key2);

    /*
     *	modify nError characters in the subsequence. Note that
     *	the modified character may be the same as the original
     *	character. Thus, we generate a sequence that can be
     *	aligned to the compact sequence within numError errors.
     */
    for (j = 0; j < nError; j++) {
      pos = RAND() % length;
      word = 3 - (pos * BITS_PER_BASE / BITS_PER_LONGWORD);

      /* align the position to the boundary of characters */
      pos = (pos * BITS_PER_BASE) % BITS_PER_LONGWORD;
      if (pos + BITS_PER_BASE > BITS_PER_LONGWORD
        )
        continue;

      /* generate a random character */
      code = RAND() % 4;

      /* modify the character */
      key2[word] = (key2[word] & ~((int64) 7 << pos)) | (code << pos);
    }

    /* align the modified subsequence */
    hits->reset();
    align(key2, NULL, FORWARD, i, hits);
    if (hits->getNumHits() <= 0) {
      return ERR_CHECK;
    }
  }

  delete hits;

  return SUCCESS;
}

AlignRes Aligner::align(AlignInfo * info, char * filename) {
  AlignRes ret;
  if (info->reader2 == NULL
    )
    ret = alignSingleEnd(info, filename);
  else
    ret = alignPairEnd(info, filename);

  printStat();

  return ret;
}

/*
 *	Aligner::align
 *	This function is used to align all query sequences in the short 
 *	read structure in batch.
 */
AlignRes Aligner::alignSingleEnd(AlignInfo * info, char * filename) {
  unsigned int i;
  FILE * file = NULL, *alfile = NULL, *unfile = NULL;
  int64 * query;
  char * quals = NULL;
  int strand;
  HitSet * hits;
  bool isQual = false;
  float ratio;
  Writer * writer = NULL;
  AlignRes res;

  ShortRead * read = info->reader1;
  int maxHit = info->maxHit, maxMatch = info->maxMatch, maxQual = info->maxQual;
  bool sorted = info->sorted, strata = info->strata;

  isQual = sorted || (maxQual <= 255);

  hits = new HitSet(maxHit, maxMatch, maxQual, sorted, false);
  hits->init(sequence, length);

  if (filename[0] != '\0') {
    if (info->outputFormat == MODE_NORMAL
      )
      writer = new SimpleWriter(sequence, read, filename);
    else if (info->outputFormat == MODE_SAM
      )
      writer = new SamWriter(sequence, read, filename);
  }

  //for (i = 0; i < nHashTable; i++)
  //	hashTables[i].resetStat();

  if (writer != NULL
    )
    writer->writeHead();

  res.nRead = read->getNumReads();
  res.nValidRead = 0;
  res.nValidAlignment = 0;

  for (i = 0; i < nHashTable; i++) {
    elog(DEBUG1, "Max Scan: %d\n", hashTables[i].getMaxScan());
  }

  ProgressBar bar(res.nRead - 1, PROGRESS_BAR_WIDTH);

  for (i = 0; i < res.nRead; i++) {
    /*	update the progress bar.*/
    if (info->showBar)
      bar.update(i);

    hits->reset();

    /* forward case */
    if (read->isForward()) {
      HASH_DEBUG(printf("forward (first fragment):\n"));
      query = read->getRead(i, FORWARD);
      if (isQual)
        quals = read->getQual(i);
      alignFirst(query, quals, FORWARD, i, hits, true);
    }

    /* backward case */
    if (read->isBackward() && !hits->isFull()) {
      HASH_DEBUG(printf("backward (first fragment):\n"));
      query = read->getRead(i, BACKWARD);
      if (isQual)
        quals = read->getQual(i);
      alignFirst(query, quals, BACKWARD, i, hits, true);
    }

    /* forward case */
    if (read->isForward() && !hits->isFull()) {
      HASH_DEBUG(printf("forward:\n"));
      query = read->getRead(i, FORWARD);
      if (isQual)
        quals = read->getQual(i);
      align(query, quals, FORWARD, i, hits, true, true);
    }

    /* backward case */
    if (read->isBackward() && !hits->isFull()) {
      HASH_DEBUG(printf("backward:\n"));
      query = read->getRead(i, BACKWARD);
      if (isQual)
        quals = read->getQual(i);
      align(query, quals, BACKWARD, i, hits, true, true);
    }

    if (info->maxGap > 0) {
      if (read->isForward() && !hits->isFull()) {
        HASH_DEBUG(printf("forward:\n"));
        query = read->getRead(i, FORWARD);
        if (isQual)
          quals = read->getQual(i);
        align(query, quals, FORWARD, i, hits, false, false);
      }

      if (read->isBackward() && !hits->isFull()) {
        HASH_DEBUG(printf("backward:\n"));
        query = read->getRead(i, BACKWARD);
        if (isQual)
          quals = read->getQual(i);
        align(query, quals, BACKWARD, i, hits, false, false);
      }
    }

    hits->verifyQual();

    if (hits->getNumHits() > 0) {
      res.nValidRead++;
      res.nValidAlignment += hits->getNumHits();
      read->printAlign(i);
    } else {
      read->printUnalign(i);
    }

    //write may chagne the nubmer of hits in the set
    if (writer != NULL)
    {
      writer->writeAlignment(i, hits);
    }
  }

  //for (i = 0; i < nHashTable; i++)
  //	printf("%u probes, %u empty, %u collision, %u seq probe.\n", hashTables[i].getStatProbe(),
  //	hashTables[i].getStatEmpty(), hashTables[i].getStatCollision(), hashTables[i].getStatSeqProbe());

  delete writer;
  delete hits;

  return res;
}

AlignRes Aligner::alignPairEnd(AlignInfo * info, char * filename) {
  unsigned int i;
  bool success;
  unsigned int nSuccessRead = 0;
  FILE * file = NULL;
  int64 * query;
  char * quals = NULL;
  HitPairSet * hitpair;
  HitSet * set1, *set2;
  bool isQual = false;
  Writer * writer = NULL;
  char mateFilename[256];
  AlignRes res;

  ShortRead * read1 = info->reader1, *read2 = info->reader2;
  unsigned int minins = info->minins, maxins = info->maxins;
  int maxHit = info->maxHit, maxMatch = info->maxMatch, maxQual = info->maxQual;
  bool sorted = info->sorted, strata = info->strata;

  isQual = sorted || (maxQual <= 255);

  set1 = new HitSet(info->maxMate, 0, maxQual, false, false);
  set2 = new HitSet(info->maxMate, 0, maxQual, false, false);
  hitpair = new HitPairSet(maxHit, maxMatch, maxQual, sorted, false);
  set1->init(sequence, length);
  set2->init(sequence, length);
  hitpair->init(sequence, length);

  if (filename[0] != '\0') {
    if (info->outputFormat == MODE_NORMAL
      )
      writer = new SimplePairWriter(sequence, read1, read2, filename);
    else if (info->outputFormat == MODE_SAM
      )
      writer = new SamPairWriter(sequence, read1, read2, filename);
  }

  if (writer != NULL
    )
    writer->writeHead();

  //for (i = 0; i < nHashTable; i++)
  //	hashTables[i].resetStat();

  res.nRead = read1->getNumReads();
  res.nValidRead = 0;
  res.nValidAlignment = 0;

  ProgressBar bar(res.nRead - 1, PROGRESS_BAR_WIDTH);

  for (i = 0; i < res.nRead; i++) {
    /*	update the progress bar.*/
    if (info->showBar)
      bar.update(i);

    set1->reset();

    /* forward case for mate1 */
    if (read1->isForward()) {
      query = read1->getRead(i, FORWARD);
      if (isQual)
        quals = read1->getQual(i);
      align(query, quals, FORWARD, i, set1);
    }

    /* backward case for mate1*/
    if (read1->isBackward()) {
      query = read1->getRead(i, BACKWARD);
      if (isQual)
        quals = read1->getQual(i);
      align(query, quals, BACKWARD, i, set1);
    }

    set2->reset();

    /* forward case for mate2*/
    if (read2->isForward()) {
      query = read2->getRead(i, FORWARD);
      if (isQual)
        quals = read2->getQual(i);
      align(query, quals, FORWARD, i, set2);
    }

    /* backward case for mate2*/
    if (read2->isBackward()) {
      query = read2->getRead(i, BACKWARD);
      if (isQual)
        quals = read2->getQual(i);
      align(query, quals, BACKWARD, i, set2);
    }

    hitpair->build(set1, set2, minins, maxins, info->pairStrand,
        info->mateMatch);

    hitpair->verifyQual();

    if (hitpair->isProperMatch() && hitpair->getNumHits() > 0) {
      res.nValidRead++;
      res.nValidAlignment += hitpair->getNumHits() / 2;
      read1->printAlign(i);
      read2->printAlign(i);
    } else {
      read1->printUnalign(i);
      read2->printUnalign(i);
    }

    if (writer != NULL)
    {
      writer->writeAlignment(i, hitpair);
    }
  }

  //for (i = 0; i < nHashTable; i++)
  //	printf("%u probes, %u empty, %u collision, %u seq probe.\n", hashTables[i].getStatProbe(),
  //	hashTables[i].getStatEmpty(), hashTables[i].getStatCollision(), hashTables[i].getStatSeqProbe());

  delete set1;
  delete set2;
  delete hitpair;
  delete writer;

  return res;
}

AlignRes Aligner::merge(AlignInfo * info, int step, char * filename) {
  if (info->reader2 == NULL
    )
    return mergeSingleEnd(info, step, filename);
  else
    return mergePairEnd(info, step, filename);
}

/*
 * Aligner::mergeSingleEnd()
 * This function is used to merge the hits in various files into a single
 * hit file for single-end alignments. The input hit file is in RAW format,
 * and is generated by search a partition of indexes. The output hit file
 * is in a format specified by the users.
 */
AlignRes Aligner::mergeSingleEnd(AlignInfo * info, int step, char * filename) {
  int i, ret;
  Hit * hits;
  HitSet * set;
  int num;
  char fname[MAX_LENGTH_PATH];
  int curid;
  Writer * writer = NULL;
  RawReader * readers;
  AlignRes res;

  if (info->outputFormat == MODE_NORMAL
    )
    writer = new SimpleWriter(sequence, info->reader1, filename);
  else if (info->outputFormat == MODE_SAM
    )
    writer = new SamWriter(sequence, info->reader1, filename);

  if (writer != NULL
    )
    writer->writeHead();

  set = new HitSet(info->maxHit, info->maxMatch, info->maxQual, info->sorted,
      false);
  set->init(sequence, length);

  num = (nHashTable + step - 1) / step;
  hits = new Hit[num];

  res.nRead = info->reader1->getNumReads();
  res.nValidRead = 0;
  res.nValidAlignment = 0;

  readers = new RawReader[num];
  for (i = 0; i < num; i++) {
    sprintf(fname, "%s.p%d", filename, i * step);

    if (readers[i].init(fname, res.nRead) == false) {
      elog(ERROR, "ERROR: open raw align files.\n");
      return res;
    }

    readers[i].next(&hits[i]);
  }

  /*
   * the merge algorithms is effcient when most reads
   * have valid alignments
   */
  for (curid = 0; curid < res.nRead; curid++) {
    /* empty the hit set */
    set->reset();

    for (i = 0; i < num; i++) {
      while (hits[i].id == curid) {
        set->add(hits[i].query, hits[i].reference, hits[i].pos, hits[i].strand,
            &hits[i].error, hits[i].qual, hits[i].id);

        readers[i].next(&hits[i]);
      }
    }

    set->verifyQual();

    if (set->getNumHits() > 0) {
      res.nValidRead++;
      res.nValidAlignment += set->getNumHits();
      info->reader1->printAlign(curid);
    } else
      info->reader1->printUnalign(curid);

    if (writer != NULL)
    {
      writer->writeAlignment(curid, set);
    }
  }

  delete writer;
  delete set;
  delete[] hits;
  delete[] readers;

  /* remove all intermediate hit files */
  char command[MAX_LENGTH_PATH];
  for (i = 0; i < num; i++) {
    sprintf(command, "rm -f %s.p%d", filename, i * step);
    system(command);
  }

  return res;
}

/*
 * Aligner::mergePairEnd()
 * This function is used to merge the hits in various files into a single
 * hit file for paired-end alignments. The input hit file is in RAW format,
 * and is generated by search a partition of indexes. The output hit file
 * is in a format specified by the users.
 */
AlignRes Aligner::mergePairEnd(AlignInfo * info, int step, char * filename) {
  int i, ret;
  Hit * hits1, *hits2;
  HitSet * set1, *set2;
  HitPairSet * hitpair;
  int num;
  char fname[MAX_LENGTH_PATH];
  int curid;
  Writer * writer = NULL;
  RawReader * readers1, *readers2;
  AlignRes res;

  if (info->outputFormat == MODE_NORMAL
    )
    writer = new SimplePairWriter(sequence, info->reader1, info->reader2,
        filename);
  else if (info->outputFormat == MODE_SAM
    )
    writer = new SamPairWriter(sequence, info->reader1, info->reader2,
        filename);

  if (writer != NULL
    )
    writer->writeHead();

  set1 = new HitSet(0, 0, 0, false, false);
  set2 = new HitSet(0, 0, 0, false, false);
  hitpair = new HitPairSet(info->maxHit, info->maxMatch, info->maxQual,
      info->sorted, false);
  set1->init(sequence, length);
  set2->init(sequence, length);
  hitpair->init(sequence, length);

  num = (nHashTable + step - 1) / step;
  hits1 = new Hit[num];
  hits2 = new Hit[num];

  res.nRead = info->reader1->getNumReads();
  res.nValidRead = 0;
  res.nValidAlignment = 0;

  readers1 = new RawReader[num];
  readers2 = new RawReader[num];
  for (i = 0; i < num; i++) {
    /* initialize the reader for mate1 */
    sprintf(fname, "%s.m1.p%d", filename, i * step);

    if (readers1[i].init(fname, res.nRead) == false) {
      elog(ERROR, "ERROR: open raw align files.\n");
      return res;
    }

    readers1[i].next(&hits1[i]);

    /* initialize the reader for mate2 */
    sprintf(fname, "%s.m2.p%d", filename, i * step);

    if (readers2[i].init(fname, res.nRead) == false) {
      elog(ERROR, "ERROR: open raw align files.\n");
      return res;
    }

    readers2[i].next(&hits2[i]);
  }

  /*
   * the merge algorithms is effcient when most reads
   * have valid alignments
   */
  for (curid = 0; curid < res.nRead; curid++) {
    /* merge hits for mate 1 */
    /* empty the hit set */
    set1->reset();

    for (i = 0; i < num; i++) {
      while (hits1[i].id == curid) {
        set1->add(hits1[i].query, hits1[i].reference, hits1[i].pos,
            hits1[i].strand, &hits1[i].error, hits1[i].qual, hits1[i].id);

        readers1[i].next(&hits1[i]);
      }
    }

    /* merge hits for mate 2 */
    /* empty the hit set */
    set2->reset();

    for (i = 0; i < num; i++) {
      while (hits2[i].id == curid) {
        set2->add(hits2[i].query, hits2[i].reference, hits2[i].pos,
            hits2[i].strand, &hits2[i].error, hits2[i].qual, hits2[i].id);
        readers2[i].next(&hits2[i]);
      }
    }

    hitpair->build(set1, set2, info->minins, info->maxins, info->pairStrand,
        info->mateMatch);

    hitpair->verifyQual();

    if (hitpair->isProperMatch() && hitpair->getNumHits() > 0) {
      res.nValidRead++;
      res.nValidAlignment += hitpair->getNumHits() / 2;
      info->reader1->printAlign(curid);
      info->reader2->printAlign(curid);
    } else {
      info->reader1->printUnalign(curid);
      info->reader2->printUnalign(curid);
    }

    if (writer != NULL)
    {
      writer->writeAlignment(curid, hitpair);
    }
  }

  delete writer;
  delete set1;
  delete set2;
  delete hitpair;
  delete[] hits1;
  delete[] hits2;
  delete[] readers1;
  delete[] readers2;

  /* remove all intermediate hit files */
  char command[MAX_LENGTH_PATH];
  for (i = 0; i < num; i++) {
    sprintf(command, "rm -f %s.m1.p%d", filename, i * step);
    system(command);
    sprintf(command, "rm -f %s.m2.p%d", filename, i * step);
    system(command);
  }

  return res;
}

void Aligner::printStat() {
#ifdef DEBUG_STAT
  for (int i = 0; i < nHashTable; i++)
  {
    printf("Hash table %d: \n", i);
    hashTables[i].printStat();
  }
#endif
}

/*
 *	Aligner::save
 *	This function is used to save the in-memory index on disk. If indexID
 *	is not specified (indexID == ALIGNER_ALL_INDEX), we save all hash 
 *	tables. Otherwise, we only save the specified index. The Aligner 
 *	structure is stored in file head.whm in the specified data path. Hash 
 *	tables are separately stored in h$ID$.whm.
 */

int Aligner::save(char * path, int indexID) {
  int ret;
  char fname[MAX_LENGTH_PATH];
  FILE * file;

  if (strlen(path) > 240)
    return ERR_PARA;

  if (indexID != ALIGNER_ALL_INDEX && indexID >= nHashTable)
    return ERR_PARA;

  sprintf(fname, "%s.head.whm", path);
  file = fopen(fname, "wb");
  if (file == NULL)
  {
    elog(DEBUG1, "ERROR: open data file to write aligner structure.\n");
    return ERR_PARA;
  }

  ret = fwrite(this, sizeof(Aligner), 1, file);
  if (ret != 1) {
    elog(DEBUG1, "ERROR: write aligner structure.\n");
    return ERR_FILE;
  }

  ret = fwrite(lookupIndex, sizeof(int), nLookup, file);
  if (ret != nLookup) {
    elog(DEBUG1, "ERROR: write aligner structure.\n");
    return ERR_FILE;
  }

  ret = fwrite(lookupOffset, sizeof(int), nLookup, file);
  if (ret != nLookup) {
    elog(DEBUG1, "ERROR: write aligner structure.\n");
    return ERR_FILE;
  }

  ret = fflush(file);
  if (ret != 0) {
    elog(DEBUG1, "ERROR: flush the aligner structure.\n");
    return ERR_FILE;
  }

  ret = fclose(file);
  if (ret != 0) {
    elog(DEBUG1, "ERROR: close data file for aligner structure.\n");
    return ERR_FILE;
  }

  if (indexID == ALIGNER_ALL_INDEX)
  {
    for (int i = 0; i < nHashTable; i++) {
      ret = hashTables[i].save(path);
      if (ret != SUCCESS)
      {
        elog(DEBUG1, "ERROR: write hash table %d.\n", i);
        return ret;
      }
    }
  } else {
    ret = hashTables[indexID].save(path);
    if (ret != SUCCESS)
    {
      elog(DEBUG1, "ERROR: write hash table %d.\n", indexID);
      return ret;
    }
  }

  return SUCCESS;
}

/*
 *	Aligner::load
 *	This function is used to load the on-disk index into memory. If indexID
 *	is not specified (indexID == ALIGNER_ALL_INDEX), we load all hash 
 *	tables. Otherwise, we only load the specified index. The Aligner 
 *	structure is loaded from the file head.whm in the specified data path. Hash 
 *	tables are separately loaded from files h$ID$.whm.
 */

int Aligner::load(char * path, int indexID) {
  int ret;
  char fname[256];
  FILE * file;

  if (strlen(path) > 240)
    return ERR_PARA;

  if (indexID != ALIGNER_ALL_INDEX && indexID >= nHashTable)
    return ERR_PARA;

  sprintf(fname, "%s.head.whm", path);
  file = fopen(fname, "rb");
  if (file == NULL)
  {
    elog(DEBUG1, "ERROR: head file does not exist.\n");
    return ERR_PARA;
  }

  ret = fread(this, sizeof(Aligner), 1, file);
  if (ret != 1) {
    elog(DEBUG1, "ERROR: read head data file.\n");
    return ERR_FILE;
  }

  lookupIndex = new int[nLookup];
  if (lookupIndex == NULL
    )
    return ERR_MEM;

  ret = fread(lookupIndex, sizeof(int), nLookup, file);
  if (ret != nLookup) {
    elog(DEBUG1, "ERROR: read head data file.\n");
    return ERR_FILE;
  }

  lookupOffset = new int[nLookup];
  if (lookupIndex == NULL
    )
    return ERR_MEM;

  ret = fread(lookupOffset, sizeof(int), nLookup, file);
  if (ret != nLookup) {
    elog(DEBUG1, "ERROR: read head data file.\n");
    return ERR_FILE;
  }

  ret = fclose(file);
  if (ret != 0) {
    elog(DEBUG1, "ERROR: close the head data file.\n");
    return ERR_FILE;
  }

  /* load the compact sequence */
  sequence = new CompactSequence();
  ret = sequence->load(path);
  if (ret != SUCCESS)
  {
    elog(ERROR, "failed to load reference sequences.\n");
    return ret;
  }

  /* verify the sequence matches with the aligner */
  ret = sequence->valid(length, nError);
  if (ret != SUCCESS)
  {
    elog(ERROR, "ERROR: unmismatched sequence/index.\n");
    return ERR_INDEX;
  }

  /* load hash table one by one */
  hashTables = new HashTable[nHashTable];
  if (hashTables == NULL
    )
    return ERR_MEM;

  if (indexID == ALIGNER_ALL_INDEX)
  {
    for (int i = 0; i < nHashTable; i++) {
      ret = hashTables[i].load(path, i, sequence);
      if (ret != SUCCESS)
      {
        elog(DEBUG1, "ERROR: read hash table %d data file.\n", i);
        return ret;
      }
    }
  } else {
    ret = hashTables[indexID].load(path, indexID, sequence);
    if (ret != SUCCESS)
    {
      elog(DEBUG1, "ERROR: read hash table %d data file.\n", indexID);
      return ret;
    }
  }

  return SUCCESS;
}

int Aligner::saveHead(char * path) {
  int ret;
  char fname[256];
  FILE * file;

  if (strlen(path) > 240)
    return ERR_PARA;

  sprintf(fname, "%s.head.whm", path);
  file = fopen(fname, "wb");
  if (file == NULL)
  {
    elog(DEBUG1, "ERROR: open data file to write aligner structure.\n");
    return ERR_PARA;
  }

  ret = fwrite(this, sizeof(Aligner), 1, file);
  if (ret != 1) {
    elog(DEBUG1, "ERROR: write aligner structure.\n");
    return ERR_FILE;
  }

  ret = fwrite(lookupIndex, sizeof(int), nLookup, file);
  if (ret != nLookup) {
    elog(DEBUG1, "ERROR: write aligner structure.\n");
    return ERR_FILE;
  }

  ret = fwrite(lookupOffset, sizeof(int), nLookup, file);
  if (ret != nLookup) {
    elog(DEBUG1, "ERROR: write aligner structure.\n");
    return ERR_FILE;
  }

  ret = fflush(file);
  if (ret != 0) {
    elog(DEBUG1, "ERROR: flush the aligner structure.\n");
    return ERR_FILE;
  }

  ret = fclose(file);
  if (ret != 0) {
    elog(DEBUG1, "ERROR: close data file for aligner structure.\n");
    return ERR_FILE;
  }

  return SUCCESS;
}

int Aligner::saveIndex(char * path, int indexID) {
  int ret;

  if (strlen(path) > 240)
    return ERR_PARA;

  if (indexID < 0 || indexID >= nHashTable)
    return ERR_PARA;

  ret = hashTables[indexID].save(path);
  if (ret != SUCCESS)
  {
    elog(DEBUG1, "ERROR: write hash table %d.\n", indexID);
    return ret;
  }

  return SUCCESS;
}

int Aligner::loadHead(char * path) {
  int ret;
  char fname[256];
  FILE * file;

  if (strlen(path) > 240)
    return ERR_PARA;

  sprintf(fname, "%s.head.whm", path);
  file = fopen(fname, "rb");
  if (file == NULL)
  {
    elog(DEBUG1, "ERROR: head file does not exist.\n");
    return ERR_PARA;
  }

  ret = fread(this, sizeof(Aligner), 1, file);
  if (ret != 1) {
    elog(DEBUG1, "ERROR: read head data file.\n");
    return ERR_FILE;
  }

//	sequence = seq;

  lookupIndex = new int[nLookup];
  if (lookupIndex == NULL
    )
    return ERR_MEM;

  ret = fread(lookupIndex, sizeof(int), nLookup, file);
  if (ret != nLookup) {
    elog(DEBUG1, "ERROR: read head data file.\n");
    return ERR_FILE;
  }

  lookupOffset = new int[nLookup];
  if (lookupIndex == NULL
    )
    return ERR_MEM;

  ret = fread(lookupOffset, sizeof(int), nLookup, file);
  if (ret != nLookup) {
    elog(DEBUG1, "ERROR: read head data file.\n");
    return ERR_FILE;
  }

  ret = fclose(file);
  if (ret != 0) {
    elog(DEBUG1, "ERROR: close the head data file.\n");
    return ERR_FILE;
  }

  hashTables = new HashTable[nHashTable];
  if (hashTables == NULL
    )
    return ERR_MEM;

  /* load the compact sequence */
  sequence = new CompactSequence();
  ret = sequence->load(path);
  if (ret != SUCCESS)
  {
    elog(ERROR, "failed to load reference sequences.\n");
    return ret;
  }

  /* verify the sequence matches with the aligner */
  ret = sequence->valid(length, nError);
  if (ret != SUCCESS)
  {
    elog(ERROR, "ERROR: unmismatched sequence/index.\n");
    return ERR_INDEX;
  }

  return SUCCESS;
}

int Aligner::loadHashtables(char * path) {
  int i;
  int ret;

  if (strlen(path) > 240)
    return ERR_PARA;

  for (i = 0; i < nHashTable; i++) {
    ret = hashTables[i].load(path, i, sequence);
    if (ret != SUCCESS)
    {
      elog(ERROR, "ERROR: read hash table %d data file.\n", i);
      return ret;
    }

    hashTables[i].setErrorModel(nMaxError, nInsert, maxQual);
  }

  return SUCCESS;
}

/*
 *	Aligner::remove
 *	free the space of specified index
 */
int Aligner::removeHashTables() {
  int i;
  int ret;

  for (i = 0; i < nHashTable; i++) {
    ret = hashTables[i].remove();
    if (ret != SUCCESS)
    {
      return ret;
    }
  }

  return SUCCESS;
}
