/**
 *    WHAM - high-throughput sequence aligner
 *    Copyright (C) 2011  WHAM Group, University of Wisconsin
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

/*	$Id: hash.cpp 167 2012-11-26 20:33:46Z yinan $ */

#include <stdlib.h>
#include <memory.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <math.h>
#include "hash.h"
#include "bitread.h"
#include "error.h"
#include "pair.h"
#include "edit_distance.h"
#include "util.h"
#include "rdtsc.h"
#include "interval.h"

//#define NUM_BUCKET	1350000011
#define NUM_BUCKET		1500000001
//#define NUM_BUCKET	2000000011
#define HASH_OVERFLOW_LIST_SCAN_BOUND 64

#define BITWISE_ALIGNMENT

//#define DEBUG_PRINT_LIST

unsigned long long timePairAlign = 0;
unsigned long long numPairAlign = 0;
unsigned long long numPairAlignFilter = 0;
unsigned long long numPairAlignDBA = 0;
unsigned long long statHashLookupEntry = 0;
unsigned long long statHashLookup = 0;

int64 lookuptspace[96];
int64 lookuptspace2[16];

typedef struct EntryCounter {
  int64 key[6];
  uint32 num;
  uint32 offset;
} EntryCounter;

int compareEntryCounter(const void * a, const void * b) {
  EntryCounter * p1 = (EntryCounter *) a;
  EntryCounter * p2 = (EntryCounter *) b;
  return p1->num - p2->num;
}

HashTable::HashTable() {
  memset(this, 0, sizeof(HashTable));
  memset(lookuptspace, 0, sizeof(int64) * 96);
  memset(lookuptspace2, 0, sizeof(int64) * 16);
}

HashTable::~HashTable() {
  delete[] buckets;
  delete[] overflowPool;
}

/*
 *	HashTable::init()
 *	initialize the private variables
 */
void HashTable::init(CompactSequence * seq, int len, unsigned int nBucket,
    int numError, int numInsert, int numDelete, int numPartition, int maxRepeat,
    bool embed, int index) {
  sequence = seq;

  indexID = index;
  length = len;
  lenSeq = length * BITS_PER_BASE;

  nMismatch = numError;
  nInsert = numInsert;
  nDelete = numDelete;
  nMaxError = nMismatch;
  nPartition = numPartition;
//	nLookup = numLookup;
  nMaxGap = 0;
  maxQual = MAX_INT;
  this->maxRepeat = maxRepeat;

  embedTables = NULL;

  lenPartition = length / nPartition * BITS_PER_BASE;
  lenKey = length / nPartition * (nPartition - nMismatch) * BITS_PER_BASE;
  lenRest = length * BITS_PER_BASE - lenPartition * nPartition;

  BitRead::genHeadMask(headMask, lenPartition * nPartition);
  BitRead::genHeadMask(embedHeadMask,
      (lenSeq - lenKey - lenRest) / BITS_PER_BASE / (nMismatch + 1) * (nMismatch + 1) * BITS_PER_BASE);

  if (nBucket == 0) {
    double nEntry, nSpace;

    nEntry = (double) seq->getNum();
    nSpace = pow(8.0, length / nPartition * (nPartition - nMismatch));
    numBucket = nEntry < nSpace ? (int) nEntry : (int) nSpace;
  } else
    numBucket = nBucket;

  numBucket = nextPrime(numBucket);
  numEmpty = numBucket;

  /*
   *	if the sequence size is greater than 2^31, we have to
   *	use a normal hash table, otherwise, we use a compressed
   *	hash table to speedup the searches.
   */
  if (seq->getNum() < COMPRESS_TABLE_SIZE)
    compressedTable = true;
  else
    compressedTable = false;

  bUseEmbedTables = embed;
//	compressedTable = false;

  resetStat();
}

/*
 *	HashTable::preProcessInit()
 *	Allocate and initialize the hash bucket array and bitmap arrays 
 *	for collision bits and empty bits.
 */
int HashTable::preProcessInit() {
  /*	allocate hash buckets.	*/
  buckets = (unsigned int *) malloc((int64) numBucket * sizeof(unsigned int));
  if (buckets == NULL
  )
    return ERR_MEM;

  /*
   *	allocate bitmap arrays to identify empty buckets
   *	and collision buckets.
   */
  emptyBits = (unsigned char *) malloc((int64) numBucket / BITS_PER_BYTE + 1);
  if (emptyBits == NULL
  )
    return ERR_MEM;

  collisionBits = (unsigned char *) malloc(
      (int64) numBucket / BITS_PER_BYTE + 1);
  if (collisionBits == NULL
  )
    return ERR_MEM;

  /* Initialization */
  memset(buckets, 0, numBucket * sizeof(unsigned int));
  memset(emptyBits, 0, numBucket / BITS_PER_BYTE + 1);
  memset(collisionBits, 0, numBucket / BITS_PER_BYTE + 1);

  /* embed table bis */
  numEmbedTables = 0;
  embedBits = (unsigned char *) malloc((int64) numBucket / BITS_PER_BYTE + 1);
  if (embedBits == NULL
  )
    return ERR_MEM;
  memset(embedBits, 0, numBucket / BITS_PER_BYTE + 1);

  return SUCCESS;
}

/*
 *	HashTable::preProcessEnd()
 *	apply the empty bits and collision bits to the hash buckets. 
 *	For empty buckets, the bucket values are set to be HASH_EMPTY. 
 *	For the buckets with collisions, the most significant bits in 
 *	the buckets are set to be 1.
 */
int HashTable::preProcessEnd() {
  uint32 i;
  uint32 tmp, sum = 0;
  uint32 collision;
  const double ln2 = log(2);

  if (buckets == NULL)
  {
    elog(DEBUG1, "ERROR: unallocated bucket array in hash table.\n");
    return ERR_PARA;
  }

  if (emptyBits == NULL || collisionBits == NULL)
  {
    elog(DEBUG1, "ERROR: unallocated bitmap in hash table.\n");
    return ERR_PARA;
  }

  if (numOverflowEntry < 0)
    return ERR_PARA;

  /* allicate overflow pool and bitmaps */
  overflowPool = (unsigned int *) malloc(
      (int64) numOverflowEntry * sizeof(unsigned int));
  if (overflowPool == NULL
  )
    return ERR_MEM;

  memset(overflowPool, 0, numOverflowEntry * sizeof(unsigned int));

  if (!compressedTable) {
    overflowBits = (unsigned char *) malloc(
        (int64) numOverflowEntry / BITS_PER_BYTE + 1);
    if (overflowBits == NULL
    )
      return ERR_MEM;

    memset(overflowBits, 0,
        (numOverflowEntry / BITS_PER_BYTE + 1) * sizeof(char));
  }

  for (i = 0; i < numBucket; i++) {
    tmp = buckets[i];
    /* update histogram */
    int h = 0;
    if (tmp > 0)
      h = (int) ceil(log(tmp) / ln2);
    if (h >= nHistogram)
      histogram[nHistogram - 1]++;
    else
      histogram[h]++;
  }

  maxScan = 64;
  if (useEmbedTables()) {
    embedShreshold = 64;

    setScanThreshold(0.001);
    maxScan = embedShreshold;

    numEmbedTables = 0;
    for (int i = nHistogram - 1; i >= 0; i--) {
      if ((0x1 << i) == embedShreshold)
        break;
      numEmbedTables += histogram[i];
    }
    embedTableSizes = (int *) malloc(sizeof(int) * numEmbedTables);
    embedTableBucketIds = (uint32 *) malloc(sizeof(uint32) * numEmbedTables);
  }

  /*
   *	scan the hash buckets to appy the empty bits and
   *	collision bits.
   */
  int embedId = 0;
  for (i = 0; i < numBucket; i++) {
    /*
     *	The current value of the bucket is the number of collision
     *	entries in each bucket. We accumulate this value to compute
     *	the position of the last entries in the overflow array for each bucket,
     *	and store the position into the bucket. For the non-collision
     *	buckets, the values will be updated to the position of segment
     *	in the function insert.
     */
    tmp = buckets[i];
    sum += tmp;
    buckets[i] = sum;
    if (buckets[i] > 0) {
      if (compressedTable) {
        HASH_SET_END(overflowPool[buckets[i] - 1]);
      } else {
        BITMAP_SET(overflowBits[(buckets[i] - 1) / BITS_PER_BYTE],
            (buckets[i] - 1) % BITS_PER_BYTE);
      }
    }

    if (useEmbedTables() && tmp > embedShreshold) {
      /* set embed bit */
      BITMAP_SET(embedBits[i/BITS_PER_BYTE], i % BITS_PER_BYTE);
      assert(embedId < numEmbedTables);
      embedTableSizes[embedId] = tmp;
      embedTableBucketIds[embedId] = i;
      embedId++;
    }

    /*	set the values for empty buckets */
    if (!BITMAP_IS(emptyBits[i / BITS_PER_BYTE], i % BITS_PER_BYTE))
      buckets[i] = HASH_EMPTY;
    else {
      if (compressedTable) {
        /* apply the collision bit to the most significant bit of the bucket */
        collision =
            BITMAP_IS(collisionBits[i / BITS_PER_BYTE], i % BITS_PER_BYTE);
        buckets[i] |= HASH_COLLISION_MASK(collision);
      }
    }
  }

  /*	free the bitmap arrays */
  free(emptyBits);
  emptyBits = NULL;

  if (compressedTable) {
    free(collisionBits);
    collisionBits = NULL;
  }

  elog(
      DEBUG1,
      "  numBucket|   numEmpty|  Collision|   numEntry| Col Rat| Emp Rat|Avg List|Avg Miss\n");
  elog(
      DEBUG1,
      "%11u %11u %11u %11u %8.2f %8.2f %8.2f %8.2f\n",
      numBucket,
      numEmpty,
      numCollision,
      numEntry,
      (double) (numCollision) / (numBucket - numEmpty),
      (double) (numEmpty) / (numBucket),
      (double) (numOverflowEntry) / numCollision,
      (double) (numOverflowEntry + numBucket - numEmpty - numCollision)
          / (numBucket - numEmpty));

  return SUCCESS;
}

void HashTable::setScanThreshold(double r) {
  int64 sum = 0;
  int64 total = numBucket;
  int64 top = total - total * r;
//	uint32 top = total - total / 1000;
  elog(DEBUG1, "Scan threshold: %f\n", r);
  elog(DEBUG1, "Hash List Length Histogram:\n");
  elog(DEBUG1, "Empty: %d\n", numEmpty);
  for (int i = 0; i < nHistogram - 1; i++) {
    sum += histogram[i];
    if (sum <= top)
      maxScan = 0x1 << (i + 1);
    elog(DEBUG1, "List length <= %d: %d (%.2f%%)\n",
        (0x1 << i), histogram[i], histogram[i] * 100.0 / total);
      }
  sum += histogram[nHistogram - 1];
  if (sum <= top)
    maxScan = 0x1 << nHistogram;
  elog(DEBUG1, "List length for the rest: %d (%.2f%%)\n",
      histogram[nHistogram - 1], histogram[nHistogram - 1] * 100.0 / total);
  elog(DEBUG1, "Choose maximum of embed threshold: %d\n", maxScan);
}

/*
 *	HashTable::preProcessInsert()
 *	update the statistics infos for the hash tables. In particular, 
 *	we update the empty bitmap array and collision bitmap array, 
 *	and update the bucket value to be the number of collision entries 
 *	hashed into the bucket.
 */
void HashTable::preProcessInsert(int64 * key) {
  uint32 bucketID;

  numEntry++;

  /*	compute the hash value */
  HASH_FUNCTION(key, numBucket, words, bucketID);

  if (!BITMAP_IS(emptyBits[bucketID/BITS_PER_BYTE], bucketID % BITS_PER_BYTE)) {
    /* set the empty bit*/
    BITMAP_SET(emptyBits[bucketID/BITS_PER_BYTE], bucketID % BITS_PER_BYTE);
    numEmpty--;
  } else {
    if (!BITMAP_IS(collisionBits[bucketID/BITS_PER_BYTE], bucketID % BITS_PER_BYTE)) {
      /* The two collsision entries will be added into the overflow array */
      BITMAP_SET(collisionBits[bucketID/BITS_PER_BYTE],
          bucketID % BITS_PER_BYTE);
      numCollision++;
      numOverflowEntry += 2;
      buckets[bucketID] += 2;
    } else {
      /* The collsision entry will be added into the overflow array */
      numOverflowEntry++;
      buckets[bucketID]++;
    }
  }
}

/*
 *	HashTable::buildInit()
 *	allocate and initialize the overflow pool.
 */
int HashTable::buildInit() {
  return SUCCESS;
}

/*
 *	HashTable::insert()
 *	insert an segment(entry) into the hash table. If the collision
 *	bit is 0, the position of the segment is directly stored in the 
 *	hash bucket. Otherwise, the position of the segment is stored in
 *	the end of the overflow list of the bucket.	
 */
void HashTable::insert(int64 * key, unsigned int offset) {
  uint32 curOverflowEntry;
  uint32 bucketId;
  uint32 seqOffset;
  bool collision;
  uint32 counter = 0;
  int64 * seqVector;
  int64 tspace[16];
  int64 * target = &tspace[8];
  bool isBloomFilter;
  uint32 * bloomFilter;
  uint32 bloomFilterNum;

  seqVector = sequence->getSequence();

  /*	compute the hash value */
  HASH_FUNCTION(key, numBucket, words, bucketId);

  if (compressedTable)
    collision = HASH_IS_COLLISION(buckets[bucketId]);
  else
    collision =
        BITMAP_IS(collisionBits[bucketId / BITS_PER_BYTE], bucketId % BITS_PER_BYTE);

  if (!collision) {
    /*	store the position into the bucket */
    if (compressedTable)
      buckets[bucketId] = HASH_GET_OFFSET(offset);
    else
      buckets[bucketId] = offset;
  } else {
    buckets[bucketId]--;

    /* get the overflow list position */
    if (compressedTable)
      curOverflowEntry = HASH_GET_OFFSET(buckets[bucketId]);
    else
      curOverflowEntry = buckets[bucketId];
    /*
     isBloomFilter = BLOOM_FILTER_EMPLOY(overflowPool[curOverflowEntry]);
     if (isBloomFilter) {
     bloomFilterNum = BLOOM_FILTER_GET_NUM(overflowPool[curOverflowEntry]);
     bloomFilter = &overflowPool[curOverflowEntry + 1];
     curOverflowEntry += 1 + BLOOM_FILTER_BYTES(bloomFilterNum);
     }
     */

    /*
     *	append the position of the new segment to the end of
     *	overflow list.
     */
    if (compressedTable)
      overflowPool[curOverflowEntry] =
          HASH_SET_OFFSET(overflowPool[curOverflowEntry], offset);
    else
      overflowPool[curOverflowEntry] = offset;
  }
}

int HashTable::buildEmbedTable() {
  int64 ttspace[80];
  int64 * query = &ttspace[8], *rest = &ttspace[24], *rest1 = &ttspace[40],
      *rest2 = &ttspace[56], *noheadkey = &ttspace[72];
  int64 * seqVector = sequence->getSequence();
  uint32 seqOffset, entryOffset, startOffset;

  memset(ttspace, 0, sizeof(int64) * 80);

  /* create space for embed tables */
  numEmbedTablesPerList = (nPartition - widthKeySpan + 1) * (nMismatch + 1);
  numLongLists = numEmbedTables;
  numEmbedTables *= numEmbedTablesPerList;

  elog(DEBUG1, "Number of long lists: %d\n", numLongLists);
  elog(DEBUG1, "Number of embed tables per list: %d\n", numEmbedTablesPerList);
  elog(DEBUG1, "Size: %lu bytes\n", sizeof(EmbedHashTable));
  elog(DEBUG1, "Size: %llu\n", (int64) numEmbedTables * sizeof(EmbedHashTable));

  embedTables = new EmbedHashTable[numEmbedTables];
  if (embedTables == NULL)
    return ERR_MEM;
  for (int i = 0; i < numEmbedTables; i++) {
    int size = embedTableSizes[i / numEmbedTablesPerList];
    embedTables[i].init(sequence, length, size, nMismatch, nPartition);
    int ret = embedTables[i].preProcessInit();
    if (ret != SUCCESS)
      return ret;
  }

  for (int l = 0; l < numLongLists; l++) {
    unsigned int bucketId = embedTableBucketIds[l];

    if (compressedTable)
      entryOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      entryOffset = buckets[bucketId];

    startOffset = entryOffset;

    /* scan the overflow list */
    while (entryOffset < numOverflowEntry) {
      /*	get the position of potential matched portion */
      if (compressedTable)
        seqOffset = HASH_GET_OFFSET(overflowPool[entryOffset]);
      else
        seqOffset = overflowPool[entryOffset];

      if (seqOffset != HASH_EMPTY)
      {
        for (int i = 0; i < nPartition - widthKeySpan + 1; i++) {
          BitRead::extract(seqVector, query,
              seqOffset * BITS_PER_BASE_LL - i * lenPartition, lenSeq);
          preinsertEmbedTableEntry(query, l, i);
        }
      }

      if (compressedTable) {
        if (HASH_IS_END(overflowPool[entryOffset]))
          break;
      } else {
        if (BITMAP_IS(overflowBits[entryOffset / BITS_PER_BYTE], entryOffset % BITS_PER_BYTE))
          break;
      }

      entryOffset++;
    }
  }

  for (int i = 0; i < numEmbedTables; i++) {
    int ret = embedTables[i].preProcessEnd();
    if (ret != SUCCESS)
      return ret;
  }

  for (int l = 0; l < numLongLists; l++) {
    unsigned int bucketId = embedTableBucketIds[l];

    if (compressedTable)
      entryOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      entryOffset = buckets[bucketId];

    startOffset = entryOffset;

    /* scan the overflow list */
    while (entryOffset < numOverflowEntry) {
      /*	get the position of potential matched portion */
      if (compressedTable)
        seqOffset = HASH_GET_OFFSET(overflowPool[entryOffset]);
      else
        seqOffset = overflowPool[entryOffset];

      if (seqOffset != HASH_EMPTY)
      {
        for (int i = 0; i < nPartition - widthKeySpan + 1; i++) {
          BitRead::extract(seqVector, query,
              seqOffset * BITS_PER_BASE_LL - i * lenPartition, lenSeq);
          insertEmbedTableEntry(query, seqOffset, l, i);
        }
      }

      if (compressedTable) {
        if (HASH_IS_END(overflowPool[entryOffset]))
          break;
      } else {
        if (BITMAP_IS(overflowBits[entryOffset / BITS_PER_BYTE], entryOffset % BITS_PER_BYTE))
          break;
      }
      entryOffset++;
    }
    //update the bucket value
    buckets[bucketId] = l;
  }

  return SUCCESS;
}

void HashTable::preinsertEmbedTableEntry(int64 * query, int embedId, int keyId) {
  int64 tspace[112];
  int64 * noheadkey = &tspace[8], *rest = &tspace[24];
  int64 * subkeys[6] = {query, &tspace[40], &tspace[56], &tspace[72], &tspace[88], &tspace[104]};
  int64 * subkey1 = &tspace[8], *subkey2 = &tspace[56], *subkey3 = &tspace[72],
      *subkey4 = &tspace[88], *subkey5 = &tspace[104];

  memset(tspace, 0, sizeof(int64) * (nMismatch + 2) * 16);

//  BitRead::removeHead(query, noheadkey, headMask);
  for (int i = 0; i < nPartition - nMismatch; i++) {
    assert(i < 6);
    assert(keyPartitions[i] - keyId * lenPartition >= 0);
    BitRead::removeInterval(subkeys[i], subkeys[i+1], keyPartitions[i] - keyId * lenPartition, lenPartition);
  }
//  BitRead::removeInterval(query, noheadkey,
//      lenPartition * (nPartition - 1 - keyId), lenPartition);
  BitRead::removeHead(subkeys[nPartition - nMismatch], rest, embedHeadMask);

  int newNumPartition = nMismatch + 1;
  int newLenPartition = (lenSeq - lenRest - lenKey) / BITS_PER_BASE / newNumPartition * BITS_PER_BASE;
  int pid = 0;
  switch(nMismatch) {
  case 0:
    /*  insert the segment into specified hash table. */
    embedTables[embedId * numEmbedTablesPerList + keyId * newNumPartition + pid].preProcessInsert(
        rest);
    break;
  case 1:
    for (int i1 = 0; i1 < newLenPartition * (newNumPartition); i1 += newLenPartition) {
      BitRead::removeInterval(rest, subkey1, i1, newLenPartition);
      /*  insert the segment into specified hash table. */
      embedTables[embedId * numEmbedTablesPerList + keyId * newNumPartition + pid].preProcessInsert(
          subkey1);
      pid++;
    }
    break;
  case 2:
    for (int i1 = 0; i1 < newLenPartition * (newNumPartition - 1); i1 +=
        newLenPartition) {
      BitRead::removeInterval(rest, subkey1, i1, newLenPartition);
      for (int i2 = i1; i2 < newLenPartition * (newNumPartition - 1); i2 +=
          newLenPartition) {
        BitRead::removeInterval(subkey1, subkey2, i2, newLenPartition);

        /*  insert the segment into specified hash table. */
        embedTables[embedId * numEmbedTablesPerList + keyId * newNumPartition + pid].preProcessInsert(
            subkey2);
        pid++;
      }
    }
    break;
  case 3:
    for (int i1 = 0; i1 < newLenPartition * (newNumPartition - 2); i1 +=
        newLenPartition) {
      BitRead::removeInterval(rest, subkey1, i1, newLenPartition);
      for (int i2 = i1; i2 < newLenPartition * (newNumPartition - 2); i2 +=
          newLenPartition) {
        BitRead::removeInterval(subkey1, subkey2, i2, newLenPartition);
        for (int i3 = i2; i3 < newLenPartition * (newNumPartition - 2); i3 +=
            newLenPartition) {
          BitRead::removeInterval(subkey2, subkey3, i3, newLenPartition);
          /*  insert the segment into specified hash table. */
          embedTables[embedId * numEmbedTablesPerList + keyId * newNumPartition + pid].preProcessInsert(
              subkey3);
          pid++;
        }
      }
    }
    break;
  default:
    assert(0);
  }
}

void HashTable::insertEmbedTableEntry(int64 * query, uint32 seqOffset, int embedId, int keyId) {
  int64 tspace[112];
  int64 * noheadkey = &tspace[8], *rest = &tspace[24];
  int64 * subkeys[6] = {query, &tspace[40], &tspace[56], &tspace[72], &tspace[88], &tspace[104]};
  int64 * subkey1 = &tspace[8], *subkey2 = &tspace[56], *subkey3 = &tspace[72],
      *subkey4 = &tspace[88], *subkey5 = &tspace[104];

  memset(tspace, 0, sizeof(int64) * (nMismatch + 2) * 16);

//  BitRead::removeHead(query, noheadkey, headMask);
  for (int i = 0; i < nPartition - nMismatch; i++) {
    assert(i < 6);
    assert(keyPartitions[i] - keyId * lenPartition >= 0);
    BitRead::removeInterval(subkeys[i], subkeys[i+1], keyPartitions[i] - keyId * lenPartition, lenPartition);
  }
//  BitRead::removeInterval(query, noheadkey,
//      lenPartition * (nPartition - 1 - keyId), lenPartition);
  BitRead::removeHead(subkeys[nPartition - nMismatch], rest, embedHeadMask);

  memset(tspace + 40, 0, sizeof(int64) * (nPartition - nMismatch) * 16);

  int newNumPartition = nMismatch + 1;
  int newLenPartition = (lenSeq - lenRest - lenKey) / BITS_PER_BASE / newNumPartition * BITS_PER_BASE;
  int pid = 0;
  switch(nMismatch) {
  case 0:
    /*  insert the segment into specified hash table. */
    embedTables[embedId * numEmbedTablesPerList + keyId * newNumPartition + pid].insert(
        rest, seqOffset);
    break;
  case 1:
    for (int i1 = 0; i1 < newLenPartition * (newNumPartition); i1 += newLenPartition) {
      BitRead::removeInterval(rest, subkey1, i1, newLenPartition);
      /*  insert the segment into specified hash table. */
      embedTables[embedId * numEmbedTablesPerList + keyId * newNumPartition + pid].insert(
          subkey1, seqOffset);
      pid++;
    }
    break;
  case 2:
    for (int i1 = 0; i1 < newLenPartition * (newNumPartition - 1); i1 +=
        newLenPartition) {
      BitRead::removeInterval(rest, subkey1, i1, newLenPartition);
      for (int i2 = i1; i2 < newLenPartition * (newNumPartition - 1); i2 +=
          newLenPartition) {
        BitRead::removeInterval(subkey1, subkey2, i2, newLenPartition);

        /*  insert the segment into specified hash table. */
        embedTables[embedId * numEmbedTablesPerList + keyId * newNumPartition + pid].insert(
            subkey2, seqOffset);
        pid++;
      }
    }
    break;
  case 3:
    for (int i1 = 0; i1 < newLenPartition * (newNumPartition - 2); i1 +=
        newLenPartition) {
      BitRead::removeInterval(rest, subkey1, i1, newLenPartition);
      for (int i2 = i1; i2 < newLenPartition * (newNumPartition - 2); i2 +=
          newLenPartition) {
        BitRead::removeInterval(subkey1, subkey2, i2, newLenPartition);
        for (int i3 = i2; i3 < newLenPartition * (newNumPartition - 2); i3 +=
            newLenPartition) {
          BitRead::removeInterval(subkey2, subkey3, i3, newLenPartition);
          /*  insert the segment into specified hash table. */
          embedTables[embedId * numEmbedTablesPerList + keyId * newNumPartition + pid].insert(
              subkey3, seqOffset);
          pid++;
        }
      }
    }
    break;
  default:
    assert(0);
  }
}

/*
 *	HashTable::lookup()
 *	search the segment(key) on the hash table, find the potential 
 *	matched portions in the genome sequence. Call function 
 *	pairAligner::pairAlign to perform pairwise alignment between 
 *	query sequence and the potential matched portions.
 */
unsigned int HashTable::lookup(int64 * orgkey, int64 * key, int keyOffset,
    char * quals, strand s, int rid, HitSet * hits, bool noGap) {
  int num;
  uint32 bucketId;
  uint32 seqOffset, entryOffset, startOffset;
  int64 tspace1[16], tspace2[16];
  int64 * diff = &tspace1[8], *target = &tspace2[8];
  int64 * seqVector;
  bool collision;
  uint32 sid, soffset;
  int ret;
  ErrorVector error;
  int nScanEntry = 0;
  int rett = SUCCESS;
  int maxGap = 0;

  if (!noGap)
    maxGap = nMaxGap;

#ifdef DEBUG_STAT
  statProbe++;
#endif

  seqVector = sequence->getSequence();

  /*	compute the hash value */
  HASH_FUNCTION(key, numBucket, words, bucketId);

  if (compressedTable)
    collision = HASH_IS_COLLISION(buckets[bucketId]);
  else
    collision =
        BITMAP_IS(collisionBits[bucketId / BITS_PER_BYTE], bucketId % BITS_PER_BYTE);

  //if we don't use embed hash tables
  if (useEmbedTables()) {
    bool embed =
        BITMAP_IS(embedBits[bucketId / BITS_PER_BYTE], bucketId % BITS_PER_BYTE);
    if (embed == true) {
      lookupEmbedTable(orgkey, bucketId, keyOffset, quals, s, rid, hits, noGap);
  //		printf("embed list\n");
      return rett;
    }
  }

  statHashLookup++;

  if (!collision) {
    /*	get the position of potential matched portion */
    if (compressedTable)
      seqOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      seqOffset = buckets[bucketId];

    if (seqOffset != HASH_EMPTY)
    {
      statHashLookupEntry++;
      HASH_DEBUG(printf(" %u", seqOffset - keyOffset / BITS_PER_BASE));

#ifdef DEBUG_HASH_PRINT
      BitRead::extract(seqVector, target,
          seqOffset * BITS_PER_BASE_LL + lenRest,
          lenKey);
      if (BitRead::compare(target, key))
      printf("*");
#endif
      /*	get the potential matched portion */
      BitRead::extract(seqVector, target,
          seqOffset * BITS_PER_BASE_LL - maxGap * BITS_PER_BASE - keyOffset,
          lenSeq + maxGap * 2 * BITS_PER_BASE);

      nScanEntry++;

      /**
       *	perform the pairwise alignment under the constraint
       *	on the number of errors.
       */
      error = PairAligner::pairAlign(orgkey, target, length, nMaxError, maxGap);

      if (error.num <= nMaxError) {
        seqOffset = seqOffset - maxGap + error.offset
            - keyOffset / BITS_PER_BASE;

        if (maxGap != 0) {
          /**
           * 	if supports indel, re-extract the matched portion
           * 	with proper offset and length
           **/
          BitRead::extract(seqVector, target, seqOffset * BITS_PER_BASE_LL,
          error.len * BITS_PER_BASE);
        }

        ret = hits->add(orgkey, target, seqOffset, s, &error, error.qual, rid);
        if (ret == MSG_HITSETFULL)
        {
          HASH_DEBUG(printf(" HIT"));
          rett = ret;
        }
      }
    }
//		else
//			stat_empty++;
  } else {
//		stat_collision++;
    /* get the position of the overflow list */
    if (compressedTable)
      entryOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      entryOffset = buckets[bucketId];

    startOffset = entryOffset;

    /* scan the overflow list */
    while (entryOffset < numOverflowEntry) {
      /*	get the position of potential matched portion */
      if (compressedTable)
        seqOffset = HASH_GET_OFFSET(overflowPool[entryOffset]);
      else
        seqOffset = overflowPool[entryOffset];

      if (seqOffset != HASH_EMPTY)
      {
        statHashLookupEntry++;
        HASH_DEBUG(printf(" %u", seqOffset - keyOffset / BITS_PER_BASE));
#ifdef DEBUG_HASH_PRINT
        BitRead::extract(seqVector, target,
            seqOffset * BITS_PER_BASE_LL + lenRest,
            lenKey);
        if (BitRead::compare(target, key))
        printf("*");
#endif

        /*	get the potential matched portion */
        BitRead::extract(seqVector, target,
            seqOffset * BITS_PER_BASE_LL - maxGap * BITS_PER_BASE - keyOffset,
            lenSeq + maxGap * 2 * BITS_PER_BASE);

        nScanEntry++;

        /**
         *	perform the pairwise alignment under the constraint
         *	on the number of errors.
         */
        error = PairAligner::pairAlign(orgkey, target, length, nMaxError,
            maxGap);

        if (error.num <= nMaxError) {
          seqOffset = seqOffset - maxGap + error.offset
              - keyOffset / BITS_PER_BASE;

          if (maxGap != 0) {
            /**
             * 	if supports indel, re-extract the matched portion
             * 	with proper offset and length
             **/
            BitRead::extract(seqVector, target, seqOffset * BITS_PER_BASE_LL,
            error.len * BITS_PER_BASE);
          }

          ret = hits->add(orgkey, target, seqOffset, s, &error, error.qual,
              rid);
          if (ret == MSG_HITSETFULL)
          {
            HASH_DEBUG(printf(" HIT"));
            rett = ret;
            break;
          }
        }
      }

//			if (entryOffset > startOffset + HASH_OVERFLOW_LIST_SCAN_BOUND)
      if (entryOffset > startOffset + maxScan * 2)
        break;

      if (compressedTable) {
        if (HASH_IS_END(overflowPool[entryOffset]))
          break;
      } else {
        if (BITMAP_IS(overflowBits[entryOffset / BITS_PER_BYTE], entryOffset % BITS_PER_BYTE))
          break;
      }

      entryOffset++;
    }
  }

  HASH_DEBUG(printf("\n"));
  /*	if (nScanEntry >= 100)
   {
   if (hits->getNumHits() > 0)
   printf("Pos: %u, Error: %d\n", seqOffset, error.num);
   statSeqProbe += nScanEntry > 10000? 10000: nScanEntry;
   statEmpty += printOverflowList(bucketId, keyOffset, orgkey);
   getchar();
   }
   */

#ifdef DEBUG_STAT
//	statSeqProbe += nScanEntry;
  int i;
  for (i = 0; i <= STAT_DISTRIBUTION_NUM - 1; i++)
  {
    if (nScanEntry <= statDistributionRange[i])
    {
      statDistribution[i]++;
      break;
    }
  }
  if (i >= STAT_DISTRIBUTION_NUM - 1)
  statDistribution[STAT_DISTRIBUTION_NUM - 1]++;
#endif

  return rett;
}

unsigned int HashTable::lookupEmbedTable(int64 * orgkey, uint32 bucketId,
    int keyOffset, char * quals, strand s, int rid, HitSet * hits, bool noGap) {
  int64 tspace[112];
  int64 * noheadkey = &tspace[8], *rest = &tspace[24];
  int64 * subkeys[6] = {orgkey, &tspace[40], &tspace[56], &tspace[72], &tspace[88], &tspace[104]};
  int64 * subkey1 = &tspace[8], *subkey2 = &tspace[56], *subkey3 = &tspace[72],
      *subkey4 = &tspace[88], *subkey5 = &tspace[104];

  assert(useEmbedTables() == true);

  memset(tspace, 0, sizeof(int64) * (nMismatch + 2) * 16);
  int embedId = buckets[bucketId];

  int offset = keyOffset / lenPartition;
  int64 * seqVector = sequence->getSequence();

  //  BitRead::removeHead(query, noheadkey, headMask);
  for (int i = 0; i < nPartition - nMismatch; i++) {
    assert(i < 6);
    assert(keyPartitions[i] - keyOffset >= 0);
    BitRead::removeInterval(subkeys[i], subkeys[i+1], keyPartitions[i] - keyOffset, lenPartition);
  }
//  BitRead::removeInterval(query, noheadkey,
//      lenPartition * (nPartition - 1 - keyId), lenPartition);
  BitRead::removeHead(subkeys[nPartition - nMismatch], rest, embedHeadMask);

  int newNumPartition = nMismatch + 1;
  int newLenPartition = (lenSeq - lenRest - lenKey) / BITS_PER_BASE / newNumPartition * BITS_PER_BASE;
  int pid = 0, ret;
  switch (nMismatch) {
  case 0:
    ret = embedTables[embedId * numEmbedTablesPerList + offset * newNumPartition
        + pid].lookup(orgkey, rest, keyOffset, quals, s, rid, hits, noGap);
    if (ret == MSG_HITSETFULL)
      return ret;
    break;
  case 1:
    for (int i1 = 0; i1 < newLenPartition * (newNumPartition);
        i1 += newLenPartition) {
      BitRead::removeInterval(rest, subkey1, i1, newLenPartition);
      /*	insert the segment into specified hash table.	*/
      ret = embedTables[embedId * numEmbedTablesPerList + offset * newNumPartition
          + pid].lookup(orgkey, subkey1, keyOffset, quals, s, rid, hits, noGap);
      if (ret == MSG_HITSETFULL)
        return ret;
      pid++;
    }
    break;
  case 2:
    for (int i1 = 0; i1 < newLenPartition * (newNumPartition - 1); i1 +=
        newLenPartition) {
      BitRead::removeInterval(rest, subkey1, i1, newLenPartition);
      for (int i2 = i1; i2 < newLenPartition * (newNumPartition - 1); i2 +=
          newLenPartition) {
        BitRead::removeInterval(subkey1, subkey2, i2, newLenPartition);

        /*	insert the segment into specified hash table.	*/
        ret = embedTables[embedId * numEmbedTablesPerList + offset * newNumPartition
            + pid].lookup(orgkey, subkey2, keyOffset, quals, s, rid, hits,
            noGap);
        if (ret == MSG_HITSETFULL)
          return ret;
        pid++;
      }
    }
    break;
  case 3:
    for (int i1 = 0; i1 < newLenPartition * (newNumPartition - 2); i1 +=
        newLenPartition) {
      BitRead::removeInterval(rest, subkey1, i1, newLenPartition);
      for (int i2 = i1; i2 < newLenPartition * (newNumPartition - 2); i2 +=
          newLenPartition) {
        BitRead::removeInterval(subkey1, subkey2, i2, newLenPartition);
        for (int i3 = i2; i3 < newLenPartition * (newNumPartition - 2); i3 +=
            newLenPartition) {
          BitRead::removeInterval(subkey2, subkey3, i3, newLenPartition);
          /*	insert the segment into specified hash table.	*/
          ret = embedTables[embedId * numEmbedTablesPerList
              + offset * newNumPartition + pid].lookup(orgkey, subkey3, keyOffset,
              quals, s, rid, hits, noGap);
          if (ret == MSG_HITSETFULL)
            return ret;
          pid++;
        }
      }
    }
    break;
  default:
    assert(0);
  }
}

bool HashTable::lookup(int64 * key, uint32 offset) {
  int num;
  uint32 bucketId;
  uint32 seqOffset, entryOffset;
  int64 tspace1[16], tspace2[16];
  int64 * diff = &tspace1[8], *target = &tspace2[8];
  int64 * seqVector;
  bool collision;
  uint32 sid, soffset;

  seqVector = sequence->getSequence();

  /*      compute the hash value */
  HASH_FUNCTION(key, numBucket, words, bucketId);

  if (compressedTable)
    collision = HASH_IS_COLLISION(buckets[bucketId]);
  else
    collision =
        BITMAP_IS(collisionBits[bucketId / BITS_PER_BYTE], bucketId % BITS_PER_BYTE);

  if (!collision) {
    /*      get the position of potential matched portion */
    if (compressedTable)
      seqOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      seqOffset = buckets[bucketId];

    if (seqOffset != HASH_EMPTY && seqOffset == offset)
      return true;
  } else {
    /* get the position of the overflow list */
    if (compressedTable)
      entryOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      entryOffset = buckets[bucketId];

    /* scan the overflow list */
    while (entryOffset < numOverflowEntry) {

      /*      get the position of potential matched portion */
      if (compressedTable)
        seqOffset = HASH_GET_OFFSET(overflowPool[entryOffset]);
      else
        seqOffset = overflowPool[entryOffset];

      if (seqOffset != HASH_EMPTY && seqOffset == offset)
        return true;
#ifdef DEBUG_PRINT_LIST
      printf("%u ", HASH_GET_OFFSET(overflowPool[entryOffset]));
#endif

      if (compressedTable) {
        if (HASH_IS_END(overflowPool[entryOffset]))
          break;
      } else {
        if (BITMAP_IS(overflowBits[entryOffset / BITS_PER_BYTE], entryOffset % BITS_PER_BYTE))
          break;
      }

      entryOffset++;
    }
#ifdef DEBUG_PRINT_LIST
    printf("\n");
#endif
  }

  return false;
}

typedef struct SeqHit {
  char str[128];
  uint32 pos;
} SeqHit;

int compareSeqHit(const void * a, const void * b) {
  SeqHit * p1 = (SeqHit *) a;
  SeqHit * p2 = (SeqHit *) b;

  return strcmp(p1->str, p2->str);
}

uint32 HashTable::printOverflowList(uint32 bucketId, uint32 keyOffset,
    int64 * key) {
  uint32 seqOffset, entryOffset, startOffset;
  int64 tspace1[16], tspace2[16];
  int64 * diff = &tspace1[8], *target = &tspace2[8];
  int64 * seqVector;
  bool collision;
  SeqHit set[10000];
  int nhit = 0;
  int64 statEntry = 0;
  int64 statSameEntry = 0;

  char keystr[128];
  CompactSequence::decompose(keystr, length, key);
  printf("\nkey        %s\n", keystr);

  seqVector = sequence->getSequence();

  if (compressedTable)
    collision = HASH_IS_COLLISION(buckets[bucketId]);
  else
    collision =
        BITMAP_IS(collisionBits[bucketId / BITS_PER_BYTE], bucketId % BITS_PER_BYTE);

  if (!collision) {
    /*	get the position of potential matched portion */
    if (compressedTable)
      seqOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      seqOffset = buckets[bucketId];

    printf("%10u ", seqOffset);

    if (seqOffset != HASH_EMPTY) {

      /*	get the potential matched portion */
      BitRead::extract(seqVector, target,
          seqOffset * BITS_PER_BASE_LL - keyOffset, lenSeq);

      CompactSequence::decompose(set[0].str, length, target);
      printf("%s", set[0].str);
    }
    printf("\n");
  } else {
    /* get the position of the overflow list */
    if (compressedTable)
      entryOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      entryOffset = buckets[bucketId];

    printf("\n           ");
    for (int i = 0; i < keyOffset / BITS_PER_BASE; i++)
      printf("-");
    for (int i = 0; i < lenKey / BITS_PER_BASE; i++)
      printf("+");
    for (int i = 0; i < (lenSeq - keyOffset - lenKey) / BITS_PER_BASE; i++)
      printf("-");
    printf("\n");

    startOffset = entryOffset;

    /* scan the overflow list */
    while (entryOffset < numOverflowEntry) {

      /*	get the position of potential matched portion */
      if (compressedTable)
        seqOffset = HASH_GET_OFFSET(overflowPool[entryOffset]);
      else
        seqOffset = overflowPool[entryOffset];

//			printf("%10u ", seqOffset);

      if (seqOffset != HASH_EMPTY)
      {
        if (nhit < 10000) {
          /*	get the potential matched portion */
          BitRead::extract(seqVector, target,
              seqOffset * BITS_PER_BASE_LL - keyOffset, lenSeq);

          CompactSequence::decompose(set[nhit].str, length, target);
          set[nhit].pos = seqOffset;

          nhit++;
//				printf("%s", str);
        }
      }
//			printf("\n");

      if (entryOffset > startOffset + HASH_OVERFLOW_LIST_SCAN_BOUND
      )
        break;

      if (compressedTable) {
        if (HASH_IS_END(overflowPool[entryOffset]))
          break;
      } else {
        if (BITMAP_IS(overflowBits[entryOffset / BITS_PER_BYTE], entryOffset % BITS_PER_BYTE))
          break;
      }

      entryOffset++;
    }

    qsort(set, nhit, sizeof(SeqHit), compareSeqHit);

    statEntry += nhit;
    for (int i = 0; i < nhit; i++)
      if (i > 0 && strcmp(set[i].str, set[i - 1].str) == 0)
        statSameEntry++;

    for (int i = 0; i < nhit; i++) {
      printf("%10u %s", set[i].pos, set[i].str);
      if (i > 0 && strcmp(set[i].str, set[i - 1].str) != 0) {
        int ndiff = 0;
        for (int j = 0; j < strlen(set[i].str); j++)
          if (set[i].str[j] != set[i - 1].str[j])
            ndiff++;
        printf(" %d", ndiff);
      }
      printf("\n");
      int diff = 0;
      for (int j = 0; j < strlen(set[i].str); j++)
        if (set[i].str[j] != keystr[j]) {
          printf("*");
          diff++;
        } else
          printf(" ");
      printf(" %2d\n", diff);
    }

  }

//	printf("\n%llu %llu %.2f\n", statSameEntry, statEntry, (double)statSameEntry / statEntry);
  return statSameEntry;
}

/*
 * longOverflowList()
 * check whether the overflow list of the specified bucket is 
 * longer than maxlen entries.
 */
bool HashTable::longOverflowList(uint32 bucketId, uint32 maxlen) {
  bool collision;
  uint32 entryOffset;
  uint32 lenlist;

  if (buckets[bucketId] == HASH_EMPTY
  )
    return false;

  if (compressedTable)
    collision = HASH_IS_COLLISION(buckets[bucketId]);
  else
    collision =
        BITMAP_IS(collisionBits[bucketId / BITS_PER_BYTE], bucketId % BITS_PER_BYTE);

  if (!collision) {
    lenlist = 1;
  } else {
    if (compressedTable)
      entryOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      entryOffset = buckets[bucketId];

    /* scan the overflow list */
    lenlist = 0;
    while (entryOffset < numOverflowEntry) {
      lenlist++;

      if (compressedTable) {
        if (HASH_IS_END(overflowPool[entryOffset]) || lenlist >= maxlen)
          break;
      } else {
        if (BITMAP_IS(overflowBits[entryOffset / BITS_PER_BYTE], entryOffset % BITS_PER_BYTE)
            || lenlist >= maxlen)
          break;
      }
    }
  }

  if (lenlist >= maxlen)
    return true;
  else
    return false;
}

int HashTable::sortList() {
  uint32 bucketId;
  uint32 entryOffset, seqOffset, offset;
  bool collision;
  uint32 i, numKey = 0;
  int64 tspace1[16];
  int64 * target = &tspace1[8];
  int64 * seqVector;
  EntryCounter * counters;
  uint32 * space;
  uint32 numEntryCounter = 1000000;

  counters = new EntryCounter[numEntryCounter];
  space = new uint32[1000000];

  int64 statLongList = 0;
  int64 statLongListEntry = 0;

  seqVector = sequence->getSequence();

  ProgressBar bar(numBucket - 1, PROGRESS_BAR_WIDTH);

  for (bucketId = 0; bucketId < numBucket; bucketId++) {

    /* update the progress bar */
    bar.update(bucketId);

    /* quickpath -- empty bucket */
    if (buckets[bucketId] == HASH_EMPTY
    )
      continue;

    if (compressedTable)
      collision = HASH_IS_COLLISION(buckets[bucketId]);
    else
      collision =
          BITMAP_IS(collisionBits[bucketId / BITS_PER_BYTE], bucketId % BITS_PER_BYTE);

    if (!collision)
      continue;

    numKey = 0;

    /* get the position of the overflow list */
    if (compressedTable)
      entryOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      entryOffset = buckets[bucketId];

    offset = entryOffset;

    /* scan the overflow list, get the number of each key */
    while (entryOffset < numOverflowEntry) {

      /* get the position of potential matched portion */
      if (compressedTable)
        seqOffset = HASH_GET_OFFSET(overflowPool[entryOffset]);
      else
        seqOffset = overflowPool[entryOffset];

      if (seqOffset != HASH_EMPTY)
      {

        BitRead::extract(seqVector, target,
            seqOffset * BITS_PER_BASE_LL + lenRest, lenKey);

        for (i = 0; i < numKey; i++) {
          if (BitRead::compare(counters[i].key, target)) {
            counters[i].num++;
            break;
          }
        }

        if (i >= numKey) {
          if (numKey >= numEntryCounter) {
            elog(WARNING, "WARNING: increase number of entry counters [%u].\n",
                numEntryCounter * 10);
            EntryCounter * tmp = counters;
            counters = new EntryCounter[numEntryCounter * 10];memcpy
            (counters, tmp, sizeof(EntryCounter) * numEntryCounter);
            numEntryCounter *= 10;
            delete[] tmp;
          }
          BitRead::copy(target, counters[numKey].key);
          counters[numKey].num = 1;
          numKey++;
        }
      }

      if (compressedTable) {
        if (HASH_IS_END(overflowPool[entryOffset]))
          break;
      } else {
        if (BITMAP_IS(overflowBits[entryOffset / BITS_PER_BYTE], entryOffset % BITS_PER_BYTE))
          break;
      }

      entryOffset++;
    }

    if (entryOffset == numOverflowEntry)
      entryOffset--;

    qsort(counters, numKey, sizeof(EntryCounter), compareEntryCounter);

    int nhit = 0;
    for (i = 0; i < numKey; i++) {
      counters[i].offset = nhit;
      nhit += counters[i].num;
    }

    uint32 * pool;
    if (nhit < 1000000)
      pool = space;
    else
      pool = new uint32[nhit];

    if (nhit > 500) {
      statLongList++;
      statLongListEntry += nhit;
    }
#ifdef ABCD
    entryOffset = offset;

    /* scan the overflow list again, remove all repeat keys */
    while (entryOffset < numOverflowEntry)
    {

      /* get the position of potential matched portion */
      if (compressedTable)
      seqOffset = HASH_GET_OFFSET(overflowPool[entryOffset]);
      else
      seqOffset = overflowPool[entryOffset];

      if (seqOffset != HASH_EMPTY)
      {
        /*	get the potential matched portion */
        BitRead::extract(seqVector, target, seqOffset * BITS_PER_BASE_LL + lenRest, lenKey);

        for (i = 0; i < numKey; i++)
        {
          if (BitRead::compare(counters[i].key, target))
          break;
        }

#ifdef DEBUG
        assert(i < numKey);
#endif
        pool[counters[i].offset] = seqOffset;
        counters[i].offset++;
      }

      if (compressedTable)
      {
        if (HASH_IS_END(overflowPool[entryOffset]))
        break;
      }
      else
      {
        if (BITMAP_IS(overflowBits[entryOffset / BITS_PER_BYTE], entryOffset % BITS_PER_BYTE))
        break;
      }

      entryOffset++;
    }

    memcpy(&overflowPool[offset], pool, sizeof(uint32) * nhit);

    if (compressedTable)
    HASH_SET_END(overflowPool[offset + nhit - 1]);
    else
    BITMAP_SET(overflowBits[(offset + nhit - 1)/BITS_PER_BYTE], (offset + nhit - 1) % BITS_PER_BYTE);

    if (pool != space)
    delete [] pool;
#endif
  }

  elog(INFO, "\nNumber of long list: %llu\n", statLongList);
  elog(INFO, "Number of long list entry: %llu\n", statLongListEntry);

  delete[] counters;
  delete[] space;

  return SUCCESS;
}

/*
 * removeRepeat()
 * This function is used to remove all entries that appear 
 * more than num times.
 */
int HashTable::removeRepeat(uint32 num) {
  uint32 bucketId;
  uint32 entryOffset, seqOffset, offset;
  uint32 i, numKey = 0;
  int64 tspace1[16];
  int64 * target = &tspace1[8];
  int64 * seqVector;
  EntryCounter * counters;
  uint32 numCounter;

  seqVector = sequence->getSequence();

  if (num < 2)
    return SUCCESS;

  numCounter = 128;
  counters = new EntryCounter[numCounter];

  ProgressBar bar(numBucket - 1, PROGRESS_BAR_WIDTH);

  for (bucketId = 0; bucketId < numBucket; bucketId++) {
    /* update the progress bar */
    bar.update(bucketId);

    /* quickpath -- empty bucket */
    if (buckets[bucketId] == HASH_EMPTY
    )
      continue;

    /* if the list contain less than num entries, continue */
    if (!longOverflowList(bucketId, num))
      continue;

    numKey = 0;

    /* get the position of the overflow list */
    if (compressedTable)
      entryOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      entryOffset = buckets[bucketId];

    offset = entryOffset;

    /* scan the overflow list, get the number of each key */
    while (entryOffset < numOverflowEntry) {

      /* get the position of potential matched portion */
      if (compressedTable)
        seqOffset = HASH_GET_OFFSET(overflowPool[entryOffset]);
      else
        seqOffset = overflowPool[entryOffset];

      if (seqOffset != HASH_EMPTY)
      {

        BitRead::extract(seqVector, target, seqOffset * BITS_PER_BASE_LL
        , lenKey);

        for (i = 0; i < numKey; i++) {
          if (BitRead::compare(counters[i].key, target)) {
            counters[i].num++;
            break;
          }
        }

        if (i >= numKey) {
          if (numKey >= numCounter) {
            EntryCounter * tmp;

            tmp = new EntryCounter[numCounter * 2];memcpy
            (tmp, counters, numCounter * sizeof(EntryCounter));
            delete[] counters;
            counters = tmp;
            numCounter = numCounter * 2;
          }
          BitRead::copy(target, counters[numKey].key);
          counters[numKey].num = 1;
          numKey++;
        }
      }

      if (compressedTable) {
        if (HASH_IS_END(overflowPool[entryOffset]))
          break;
      } else {
        if (BITMAP_IS(overflowBits[entryOffset / BITS_PER_BYTE], entryOffset % BITS_PER_BYTE))
          break;
      }

      entryOffset++;
    }

    /* quickpath -- all entries are removed */
    for (i = 0; i < numKey; i++)
      if (counters[i].num < num)
        break;
    if (i >= numKey) {
      buckets[bucketId] = HASH_EMPTY;
      if (!compressedTable)
        BITMAP_CLEAR(collisionBits[bucketId/BITS_PER_BYTE],
            bucketId % BITS_PER_BYTE);

      continue;
    }

    entryOffset = offset;

    /* scan the overflow list again, remove all repeat keys */
    while (entryOffset < numOverflowEntry) {

      /* get the position of potential matched portion */
      if (compressedTable)
        seqOffset = HASH_GET_OFFSET(overflowPool[entryOffset]);
      else
        seqOffset = overflowPool[entryOffset];

      if (seqOffset != HASH_EMPTY)
      {
        /*	get the potential matched portion */
        BitRead::extract(seqVector, target, seqOffset * BITS_PER_BASE_LL
        , lenKey);

        for (i = 0; i < numKey; i++) {
          if (BitRead::compare(counters[i].key, target))
            break;
        }

#ifdef DEBUG
        assert(i < numKey);
#endif
        /* move the entry forward */
        if (counters[i].num < num) {
          overflowPool[offset] = overflowPool[entryOffset];
          offset++;
        }
      }

      if (compressedTable) {
        if (HASH_IS_END(overflowPool[entryOffset])) {
          HASH_SET_END(overflowPool[offset - 1]);
          break;
        }
      } else {
        if (BITMAP_IS(overflowBits[entryOffset / BITS_PER_BYTE], entryOffset % BITS_PER_BYTE)) {
          BITMAP_SET(overflowBits[(offset - 1)/BITS_PER_BYTE],
              (offset - 1) % BITS_PER_BYTE);
          break;
        }
      }

      entryOffset++;
    }

  }

  delete[] counters;

  return SUCCESS;
}

void HashTable::checkRepeat(uint32 num) {
  uint32 bucketId;
  uint32 entryOffset, seqOffset, offset;
  uint32 i, numKey = 0;
  int64 tspace1[16];
  int64 * target = &tspace1[8];
  int64 * seqVector;
  EntryCounter * counters;
  uint32 numCounter;

  seqVector = sequence->getSequence();

  if (num < 2)
    return;

  numCounter = 128;
  counters = new EntryCounter[numCounter];

  for (bucketId = 0; bucketId < numBucket; bucketId++) {
    restart: if (buckets[bucketId] == HASH_EMPTY
    )
      continue;

    if (!longOverflowList(bucketId, num))
      continue;

    numKey = 0;

    /* get the position of the overflow list */
    if (compressedTable)
      entryOffset = HASH_GET_OFFSET(buckets[bucketId]);
    else
      entryOffset = buckets[bucketId];

    offset = entryOffset;

    /* scan the overflow list, get the number of each key */
    while (entryOffset < numOverflowEntry) {

      /* get the position of potential matched portion */
      if (compressedTable)
        seqOffset = HASH_GET_OFFSET(overflowPool[entryOffset]);
      else
        seqOffset = overflowPool[entryOffset];

      if (seqOffset != HASH_EMPTY)
      {

        BitRead::extract(seqVector, target, seqOffset * BITS_PER_BASE_LL
        , lenKey);

        for (i = 0; i < numKey; i++) {
          if (BitRead::compare(counters[i].key, target)) {
            counters[i].num++;
            break;
          }
        }

        if (i >= numKey) {
          if (numKey >= numCounter) {
            EntryCounter * tmp;

            tmp = new EntryCounter[numCounter * 2];memcpy
            (tmp, counters, numCounter * sizeof(EntryCounter));
            delete[] counters;
            counters = tmp;
            numCounter = numCounter * 2;
          }
          BitRead::copy(target, counters[numKey].key);
          counters[numKey].num = 1;
          numKey++;
        }
      }

      if (compressedTable) {
        if (HASH_IS_END(overflowPool[entryOffset]))
          break;
      } else {
        if (BITMAP_IS(overflowBits[entryOffset / BITS_PER_BYTE], entryOffset % BITS_PER_BYTE))
          break;
      }

      entryOffset++;
    }

    for (i = 0; i < numKey; i++) {
      if (counters[i].num >= num) {
        printf("%u: (%llu, %llu, %llu, %llu), %u", bucketId, counters[i].key[0],
            counters[i].key[1], counters[i].key[2], counters[i].key[3],
            counters[i].num);
        //		goto restart;
      }
    }
  }

  delete[] counters;
}
/*
 *	HashTable::save()
 *	This function is used to save the in-memory hash table on 
 *	disk.
 */
int HashTable::save(char * path) {
  size_t ret;
  char fname[256];
  FILE * file;

  if (strlen(path) > 240)
    return ERR_PARA;

  sprintf(fname, "%s.i%d.whm", path, indexID);
  file = fopen(fname, "wb");
  if (file == NULL
  )
    return ERR_PARA;

  ret = fwrite(this, sizeof(HashTable), 1, file);
  if (ret != 1) {
    return ERR_FILE;
  }

  ret = fwrite(buckets, sizeof(uint32), numBucket, file);
  if (ret != numBucket) {
    return ERR_FILE;
  }

  ret = fwrite(overflowPool, sizeof(uint32), numOverflowEntry, file);
  if (ret != numOverflowEntry) {
    return ERR_FILE;
  }

  if (!compressedTable) {
    ret = fwrite(collisionBits, sizeof(unsigned char),
        numBucket / BITS_PER_BYTE + 1, file);
    if (ret != numBucket / BITS_PER_BYTE + 1) {
      return ERR_FILE;
    }

    ret = fwrite(overflowBits, sizeof(unsigned char),
        numOverflowEntry / BITS_PER_BYTE + 1, file);
    if (ret != numOverflowEntry / BITS_PER_BYTE + 1) {
      return ERR_FILE;
    }
  }

  ret = fwrite(embedBits, sizeof(unsigned char), numBucket / BITS_PER_BYTE + 1,
      file);
  if (ret != numBucket / BITS_PER_BYTE + 1) {
    return ERR_FILE;
  }

  if (useEmbedTables()) {
    for (int i = 0; i < numEmbedTables; i++)
      embedTables[i].save(file);
  }

  ret = fflush(file);
  if (ret != 0) {
    return ERR_FILE;
  }

  ret = fclose(file);
  if (ret != 0) {
    return ERR_FILE;
  }

  return SUCCESS;
}

/*
 *	HashTable::load()
 *	This function is used to load the on-disk copy of hash table 
 *	into memory.
 */
int HashTable::load(char * path, int index, CompactSequence * seq) {
  size_t ret;
  char fname[256];
  FILE * file;

  if (strlen(path) > 240)
    return ERR_PARA;

  sprintf(fname, "%s.i%d.whm", path, index);
  file = fopen(fname, "rb");
  if (file == NULL
  )
    return ERR_PARA;

  ret = fread(this, sizeof(HashTable), 1, file);
  if (ret != 1) {
    elog(ERROR, "failed to load hash table head.\n");
    return ERR_FILE;
  }

  sequence = seq;

  /* we can use smaller block to reduct the memory consumption */
  buckets = (uint32 *) malloc((int64) numBucket * sizeof(uint32));
  ret = fread(buckets, sizeof(uint32), numBucket, file);
  if (ret != numBucket) {
    elog(ERROR, "failed to load buckets.\n");
    return ERR_FILE;
  }

  overflowPool = (uint32 *) malloc((int64) numOverflowEntry * sizeof(uint32));
  ret = fread(overflowPool, sizeof(uint32), numOverflowEntry, file);
  if (ret != numOverflowEntry) {
    elog(ERROR, "failed to load overflow array.\n");
    return ERR_FILE;
  }

  if (!compressedTable) {
    collisionBits = (unsigned char *) malloc(
        (int64) (numBucket / BITS_PER_BYTE + 1) * sizeof(unsigned char));
    ret = fread(collisionBits, sizeof(unsigned char),
        numBucket / BITS_PER_BYTE + 1, file);
    if (ret != numBucket / BITS_PER_BYTE + 1) {
      elog(ERROR, "failed to load collision bits.\n");
      return ERR_FILE;
    }

    overflowBits = (unsigned char *) malloc(
        (int64) (numOverflowEntry / BITS_PER_BYTE + 1) * sizeof(unsigned char));
    ret = fread(overflowBits, sizeof(unsigned char),
        numOverflowEntry / BITS_PER_BYTE + 1, file);
    if (ret != numOverflowEntry / BITS_PER_BYTE + 1) {
      elog(ERROR, "failed to load overflow bits. %d %d\n", ret,
          numOverflowEntry / BITS_PER_BYTE + 1);
      return ERR_FILE;
    }
  }

  if (useEmbedTables()) {
    embedBits = (unsigned char *) malloc(
        (int64) (numBucket / BITS_PER_BYTE + 1) * sizeof(unsigned char));
    if (embedBits == NULL)
      return ERR_MEM;
    ret = fread(embedBits, sizeof(unsigned char), numBucket / BITS_PER_BYTE + 1,
        file);
    if (ret != numBucket / BITS_PER_BYTE + 1) {
      elog(ERROR, "failed to load collision bits.\n");
      return ERR_FILE;
    }

    embedTables = (EmbedHashTable *) malloc(
        (int64) numEmbedTables * sizeof(EmbedHashTable));
    if (embedTables == NULL)
      return ERR_MEM;
    for (int i = 0; i < numEmbedTables; i++) {
      int ret = embedTables[i].load(file, sequence);
      if ( ret != SUCCESS)
        return ret;
    }
    elog(DEBUG1, "load %d embed hash tables\n", numEmbedTables);
  }

  ret = fclose(file);
  if (ret != 0) {
    return ERR_FILE;
  }

  resetStat();

  return SUCCESS;
}

/*
 *	HashTable::remove
 *	free the space occupied by the hash index.
 */
int HashTable::remove() {
  if (buckets) {
    free(buckets);
    buckets = NULL;
  }

  if (overflowPool) {
    free(overflowPool);
    overflowPool = NULL;
  }

  if (emptyBits) {
    free(emptyBits);
    emptyBits = NULL;
  }

  if (collisionBits) {
    free(collisionBits);
    collisionBits = NULL;
  }

  return SUCCESS;
}

/*
 * nextPrime()
 * return the least prime number that is greater 
 * than the input number.
 */
unsigned int HashTable::nextPrime(unsigned int num) {
  unsigned int i, j, x;

  num = num / 2 * 2 + 1;
  for (i = num; i < num + 1000; i += 2) {
    x = (unsigned int) sqrtl(i);
    for (j = 3; j < x; j += 2) {
      if (i % j == 0)
        break;
    }

    if (j >= x)
      return i;
  }

  return i;
}

