/***************************************************************************
                           csearchindex.cpp  -  description
                             -------------------
    begin                : Mon May 14 2003
    copyright            : (C) 2003-2004 by Mathias Küster
    email                : mathen@users.berlios.de
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/

#include "csearchindex.h"

#ifndef WIN32
#include <unistd.h>
#include <stdlib.h>
#endif

#include <string.h>

#include "core/cbytearray.h"
#include "cconfig.h"
#include "core/cdir.h"
#include "core/cfile.h"
#include "dclib.h"
#include "cfilemanager.h"
#include "core/cbase32.h"

#include "hash/compat.h"
#include "hash/MerkleTree.h"

#include "cfilehasher.h"

/* "9 levels of leaves" = max of 512 hashes, which is 12KiB, which I doubled */
#define MAX_LEAVES_SIZE 2*512*dcpp::TigerTree::BYTES

/** */
CSearchIndex::CSearchIndex()
{
	m_pBaseArray       = new CByteArray();
	m_pFileBaseArray   = new CByteArray();
	m_pPathBaseArray   = new CByteArray();
	
	m_pHashBaseArray = new CByteArray();
	m_pHashFileBaseArray = new CByteArray();
	m_pHashPathBaseArray = new CByteArray();
	m_pHashIndex = new CByteArray();
	
	m_pCaseFoldedBase = new CByteArray();
	m_pCaseFoldedData = new CByteArray();
	
	m_pNewHashLeaves = 0;
	hashleavessize = 0; // this will get set when m_pNewHashLeaves is created
	
	/* during list refresh, these are created and updated
	 * and replace the old byte arrays when refresh stage is finished
	 */
	m_pUpdatingFileBaseArray     = 0;
	m_pUpdatingPathBaseArray     = 0;
	m_pUpdatingBaseArray         = 0;
	
	m_pUpdatingHashBaseArray     = 0;
	m_pUpdatingHashFileBaseArray = 0;
	m_pUpdatingHashPathBaseArray = 0;
	m_pUpdatingHashIndex         = 0;

	m_pUpdatingCaseFoldedBase    = 0;
	m_pUpdatingCaseFoldedData    = 0;

	m_bLoadedOK = LoadIndex();
}

/** */
CSearchIndex::~CSearchIndex()
{
	delete m_pBaseArray;
	m_pBaseArray = 0;
	delete m_pFileBaseArray;
	m_pFileBaseArray = 0;
	delete m_pPathBaseArray;
	m_pPathBaseArray = 0;
	
	delete m_pHashBaseArray;
	m_pHashBaseArray = 0;
	delete m_pHashFileBaseArray;
	m_pHashFileBaseArray = 0;
	delete m_pHashPathBaseArray;
	m_pHashPathBaseArray = 0;
	delete m_pHashIndex;
	m_pHashIndex = 0;
	
	delete m_pCaseFoldedBase;
	m_pCaseFoldedBase = 0;
	delete m_pCaseFoldedData;
	m_pCaseFoldedData = 0;
	
	if ( m_pNewHashLeaves )
	{
		DPRINTF("~CSearchIndex: warning new hash leaves not saved!\n");
		SaveNewHashLeaves();
	}
	
	delete m_pNewHashLeaves;
	m_pNewHashLeaves = 0;
	
	delete m_pUpdatingFileBaseArray;
	m_pUpdatingFileBaseArray = 0;
	delete m_pUpdatingPathBaseArray;
	m_pUpdatingPathBaseArray = 0;
	delete m_pUpdatingBaseArray;
	m_pUpdatingBaseArray = 0;
	
	delete m_pUpdatingHashBaseArray;
	m_pUpdatingHashBaseArray = 0;
	delete m_pUpdatingHashFileBaseArray;
	m_pUpdatingHashFileBaseArray = 0;
	delete m_pUpdatingHashPathBaseArray;
	m_pUpdatingHashPathBaseArray = 0;
	delete m_pUpdatingHashIndex;
	m_pUpdatingHashIndex = 0;
	
	delete m_pUpdatingCaseFoldedBase;
	m_pUpdatingCaseFoldedBase = 0;
	delete m_pUpdatingCaseFoldedData;
	m_pUpdatingCaseFoldedData = 0;
}

/** */
void CSearchIndex::PrepareUpdate()
{
	// files list
	delete m_pUpdatingFileBaseArray;
	delete m_pUpdatingPathBaseArray;
	delete m_pUpdatingBaseArray;
	
	m_pUpdatingFileBaseArray = new CByteArray(0);
	m_pUpdatingPathBaseArray = new CByteArray(0);
	m_pUpdatingBaseArray     = new CByteArray(0);
	
	// search lists of case folded path+filenames
	delete m_pUpdatingCaseFoldedBase;
	delete m_pUpdatingCaseFoldedData;
	
	m_pUpdatingCaseFoldedBase = new CByteArray(0);
	m_pUpdatingCaseFoldedData = new CByteArray(0);
	
	// hash lists
	delete m_pUpdatingHashBaseArray;
	delete m_pUpdatingHashFileBaseArray;
	delete m_pUpdatingHashPathBaseArray;
	delete m_pUpdatingHashIndex;
	
	m_pUpdatingHashBaseArray = new CByteArray(0);
	m_pUpdatingHashFileBaseArray = new CByteArray(0);
	m_pUpdatingHashPathBaseArray = new CByteArray(0);
	m_pUpdatingHashIndex = new CByteArray(0);
	
	m_pUpdatingHashBaseArray->Append( m_pHashBaseArray->Data(), m_pHashBaseArray->Size() );
	m_pUpdatingHashFileBaseArray->Append( m_pHashFileBaseArray->Data(), m_pHashFileBaseArray->Size() );
	m_pUpdatingHashPathBaseArray->Append( m_pHashPathBaseArray->Data(), m_pHashPathBaseArray->Size() );
	m_pUpdatingHashIndex->Append( m_pHashIndex->Data(), m_pHashIndex->Size() );
}

/** */
void CSearchIndex::FinishUpdate()
{
	CByteArray * old1  = m_pFileBaseArray;
	CByteArray * old2  = m_pPathBaseArray;
	CByteArray * old3  = m_pBaseArray;
	CByteArray * old4  = m_pCaseFoldedBase;
	CByteArray * old5  = m_pCaseFoldedData;
	CByteArray * old7  = m_pHashBaseArray;
	CByteArray * old8  = m_pHashFileBaseArray;
	CByteArray * old9  = m_pHashPathBaseArray;
	CByteArray * old10 = m_pHashIndex;
	
	// swap to new lists / append new data - this section needs to be as fast as possible
	// essentially all operations need to be done simultaneously
	// it used to have some data appending, but now
	// it's just 7 pointer swaps, so it should be OK
	m_pFileBaseArray     = m_pUpdatingFileBaseArray;
	m_pPathBaseArray     = m_pUpdatingPathBaseArray;
	m_pBaseArray         = m_pUpdatingBaseArray;
	m_pCaseFoldedBase    = m_pUpdatingCaseFoldedBase;
	m_pCaseFoldedData    = m_pUpdatingCaseFoldedData;
	m_pHashFileBaseArray = m_pUpdatingHashFileBaseArray;
	m_pHashPathBaseArray = m_pUpdatingHashPathBaseArray;
	m_pHashIndex         = m_pUpdatingHashIndex;
	m_pHashBaseArray     = m_pUpdatingHashBaseArray;
	// end of speed critical section
	
	m_pUpdatingFileBaseArray     = 0;
	m_pUpdatingPathBaseArray     = 0;
	m_pUpdatingBaseArray         = 0;
	m_pUpdatingCaseFoldedBase    = 0;
	m_pUpdatingCaseFoldedData    = 0;
	m_pUpdatingHashBaseArray     = 0;
	m_pUpdatingHashFileBaseArray = 0;
	m_pUpdatingHashPathBaseArray = 0;
	m_pUpdatingHashIndex         = 0;

	delete old1;
	delete old2;
	delete old3;
	delete old4;
	delete old5;
	delete old7;
	delete old8;
	delete old9;
	delete old10;
}

/** */
void CSearchIndex::ResetHashIndex()
{
	m_pHashBaseArray->SetSize(0);
	m_pHashFileBaseArray->SetSize(0);
	m_pHashPathBaseArray->SetSize(0);
	m_pHashIndex->SetSize(0);
	
	if ( m_pNewHashLeaves == 0 )
	{
		m_pNewHashLeaves = new CByteArray();
	}
	m_pNewHashLeaves->SetSize(0);
	m_pNewHashLeaves->SaveToFile(CConfig::Instance()->GetConfigPath()+"hashleaves.bin");
	delete m_pNewHashLeaves;
	m_pNewHashLeaves = 0;
}

/** */
std::set<unsigned long> * CSearchIndex::SearchHash( unsigned char * hash )
{
	unsigned long hbi,hi,bi;
	std::set<unsigned long> * results = 0;

	hi = 0;
	
	while ( FindHash( hash, &hi ) )
	{
		if ( HashBaseIndexFromHashIndex( hi, &hbi ) )
		{
			if ( BaseIndexFromHashBaseIndex( hbi, &bi ) )
			{
				if ( !results )
				{
					results = new std::set<unsigned long>;
				}
				results->insert(bi);
			}
		}
		hi+=dcpp::TigerTree::BYTES;
	}

	return results;
}

/** */
bool CSearchIndex::GetFileBaseObject( unsigned long index, struct filebaseobject * fbo )
{
	bool res = false;

	if ( (index*sizeof(struct filebaseobject)) < m_pBaseArray->Size() )
	{
		memcpy( fbo, m_pBaseArray->Data()+(index*sizeof(struct filebaseobject)), sizeof(struct filebaseobject) );

		res = true;
	}

	return res;
}

/** */
bool CSearchIndex::GetFileBaseObjectDuringUpdate( unsigned long index, struct filebaseobject * fbo )
{
	bool res = false;

	if ( (index*sizeof(struct filebaseobject)) < m_pUpdatingBaseArray->Size() )
	{
		memcpy( fbo, m_pUpdatingBaseArray->Data()+(index*sizeof(struct filebaseobject)), sizeof(struct filebaseobject) );

		res = true;
	}

	return res;
}

/** */
bool CSearchIndex::GetFileBaseObject( unsigned long index, struct filebaseobject * fbo, CString & filename )
{
	bool res = false;

	if ( GetFileBaseObject(index,fbo) )
	{
		if ( m_pPathBaseArray->Size() > fbo->m_nPathIndex )
		{
			filename = (char*)m_pPathBaseArray->Data()+fbo->m_nPathIndex;

			if ( filename.NotEmpty() )
			{
				filename += DIRSEPARATOR;
			}
		}

		if ( m_pFileBaseArray->Size() > fbo->m_nFileIndex )
		{
			filename += (char*)m_pFileBaseArray->Data()+fbo->m_nFileIndex;
		}

		res = true;
	}

	return res;
}

/** */
bool CSearchIndex::GetFileBaseObjectDuringUpdate( unsigned long index, struct filebaseobject * fbo, CString & filename )
{
	bool res = false;

	if ( GetFileBaseObjectDuringUpdate(index,fbo) )
	{
		if ( m_pUpdatingPathBaseArray->Size() > fbo->m_nPathIndex )
		{
			filename = (char*)m_pUpdatingPathBaseArray->Data()+fbo->m_nPathIndex;

			if ( filename.NotEmpty() )
			{
				filename += DIRSEPARATOR;
			}
		}

		if ( m_pUpdatingFileBaseArray->Size() > fbo->m_nFileIndex )
		{
			filename += (char*)m_pUpdatingFileBaseArray->Data()+fbo->m_nFileIndex;
		}

		res = true;
	}

	return res;
}

/** */
bool CSearchIndex::GetCaseFoldedName( unsigned long index, struct filebaseobject * fbo, CString & name )
{
	unsigned long bapos = index*sizeof(struct filebaseobject);
	unsigned long cfpos = index*sizeof(unsigned long);
	
	if ( (bapos < m_pBaseArray->Size()) && (cfpos < m_pCaseFoldedBase->Size()) )
	{
		memcpy( fbo, m_pBaseArray->Data()+bapos, sizeof(struct filebaseobject) );
		
		unsigned long cfi;
		
		memcpy( &cfi, m_pCaseFoldedBase->Data()+cfpos, sizeof(unsigned long) );
		
		if ( cfi < m_pCaseFoldedData->Size() )
		{
			name = (char*) m_pCaseFoldedData->Data()+cfi;
			return true;
		}
	}
	
	return false;
}

/** */
bool CSearchIndex::LoadIndex()
{
	bool err = false;
	CDir d;
	ulonglong filesize;

	// first try to load hash index
	if ( m_pHashBaseArray->LoadFromFile(CConfig::Instance()->GetConfigPath()+"hashbase.bin") == false )
	{
		err = true;
	}
	if ( err == false )
	{
		if ( m_pHashFileBaseArray->LoadFromFile(CConfig::Instance()->GetConfigPath()+"hashfilebase.bin") == false )
		{
			err = true;
		}
	}
	if ( err == false )
	{
		if ( m_pHashPathBaseArray->LoadFromFile(CConfig::Instance()->GetConfigPath()+"hashpathbase.bin") == false )
		{
			err = true;
		}
	}
	if ( err == false )
	{
		if ( m_pHashIndex->LoadFromFile(CConfig::Instance()->GetConfigPath()+"hashindex.bin") == false )
		{
			err = true;
		}
	}
	
	if ( err == true )
	{
		ResetHashIndex();
		
		err = false;
	}
	
	// sanity check
	filesize = d.getFileSize(CConfig::Instance()->GetConfigPath()+"database.bin",false);

	if ( (filesize%sizeof(struct filebaseobject)) != 0 )
	{
		err = true;
	}

	// load the filelist
	if ( err == false )
	{
		if ( m_pBaseArray->LoadFromFile(CConfig::Instance()->GetConfigPath()+"database.bin") == false )
		{
			err = true;
		}
	}

	if ( err == false )
	{
		if ( m_pFileBaseArray->LoadFromFile(CConfig::Instance()->GetConfigPath()+"filebase.bin") == false )
		{
			err = true;
		}
	}

	if ( err == false )
	{
		if ( m_pPathBaseArray->LoadFromFile(CConfig::Instance()->GetConfigPath()+"pathbase.bin") == false )
		{
			err = true;
		}
	}

	if ( err )
	{
		m_pBaseArray->SetSize(0);
		m_pFileBaseArray->SetSize(0);
		m_pPathBaseArray->SetSize(0);
	}
	
	if ( !err )
	{
		filesize = d.getFileSize(CConfig::Instance()->GetConfigPath()+"casefoldedbase.bin");
		if ( filesize%sizeof(unsigned long) != 0 )
		{
			err = true;
		}
		
		if ( !err )
		{
			err = !(m_pCaseFoldedBase->LoadFromFile(CConfig::Instance()->GetConfigPath()+"casefoldedbase.bin"));
		}
		
		if ( !err )
		{
			err = !(m_pCaseFoldedData->LoadFromFile(CConfig::Instance()->GetConfigPath()+"casefoldeddata.bin"));
		}
	}

	return !err;
}

/** */
void CSearchIndex::SaveIndex()
{
	/*
	 * Hash leaves not being available does not matter as much and will not crash because
	 * the file seek will just fail.
	 */
	
	bool ok = m_pHashIndex->SaveToFile(CConfig::Instance()->GetConfigPath()+"hashindex.bin.new");
	ok = ok && m_pHashFileBaseArray->SaveToFile(CConfig::Instance()->GetConfigPath()+"hashfilebase.bin.new");
	ok = ok && m_pHashPathBaseArray->SaveToFile(CConfig::Instance()->GetConfigPath()+"hashpathbase.bin.new");
	ok = ok && m_pHashBaseArray->SaveToFile(CConfig::Instance()->GetConfigPath()+"hashbase.bin.new");
	ok = ok && m_pFileBaseArray->SaveToFile(CConfig::Instance()->GetConfigPath()+"filebase.bin.new");
	ok = ok && m_pPathBaseArray->SaveToFile(CConfig::Instance()->GetConfigPath()+"pathbase.bin.new");
	ok = ok && m_pBaseArray->SaveToFile(CConfig::Instance()->GetConfigPath()+"database.bin.new");
	ok = ok && m_pCaseFoldedData->SaveToFile(CConfig::Instance()->GetConfigPath()+"casefoldeddata.bin.new");
	ok = ok && m_pCaseFoldedBase->SaveToFile(CConfig::Instance()->GetConfigPath()+"casefoldedbase.bin.new");
	
	if ( ok )
	{
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"hashindex.bin" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"hashfilebase.bin" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"hashpathbase.bin" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"hashbase.bin" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"filebase.bin" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"pathbase.bin" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"database.bin" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"casefoldeddata.bin" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"casefoldedbase.bin" );
		
		CFile::Rename( CConfig::Instance()->GetConfigPath()+"hashindex.bin.new", CConfig::Instance()->GetConfigPath()+"hashindex.bin" );
		CFile::Rename( CConfig::Instance()->GetConfigPath()+"hashfilebase.bin.new", CConfig::Instance()->GetConfigPath()+"hashfilebase.bin" );
		CFile::Rename( CConfig::Instance()->GetConfigPath()+"hashpathbase.bin.new", CConfig::Instance()->GetConfigPath()+"hashpathbase.bin" );
		CFile::Rename( CConfig::Instance()->GetConfigPath()+"hashbase.bin.new", CConfig::Instance()->GetConfigPath()+"hashbase.bin" );
		CFile::Rename( CConfig::Instance()->GetConfigPath()+"filebase.bin.new", CConfig::Instance()->GetConfigPath()+"filebase.bin" );
		CFile::Rename( CConfig::Instance()->GetConfigPath()+"pathbase.bin.new", CConfig::Instance()->GetConfigPath()+"pathbase.bin" );
		CFile::Rename( CConfig::Instance()->GetConfigPath()+"database.bin.new", CConfig::Instance()->GetConfigPath()+"database.bin" );
		CFile::Rename( CConfig::Instance()->GetConfigPath()+"casefoldeddata.bin.new", CConfig::Instance()->GetConfigPath()+"casefoldeddata.bin" );
		CFile::Rename( CConfig::Instance()->GetConfigPath()+"casefoldedbase.bin.new", CConfig::Instance()->GetConfigPath()+"casefoldedbase.bin" );
		
		if ( m_pNewHashLeaves )
		{
			SaveNewHashLeaves();
		}
	}
	else
	{
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"hashindex.bin.new" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"hashfilebase.bin.new" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"hashpathbase.bin.new" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"hashbase.bin.new" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"filebase.bin.new" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"pathbase.bin.new" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"database.bin.new" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"casefoldeddata.bin.new" );
		CFile::UnLink( CConfig::Instance()->GetConfigPath()+"casefoldedbase.bin.new" );
		
		delete m_pNewHashLeaves;
		m_pNewHashLeaves = 0;
	}
}

/** */
unsigned long CSearchIndex::IndexCount()
{
	unsigned long i = 0;

	if ( m_pBaseArray )
	{
		i = m_pBaseArray->Size()/sizeof(struct filebaseobject);
	}

	return i;
}

/** */
unsigned long CSearchIndex::IndexCountDuringUpdate()
{
	unsigned long i = 0;

	if ( m_pUpdatingBaseArray )
	{
		i = m_pUpdatingBaseArray->Size()/sizeof(struct filebaseobject);
	}

	return i;
}

/** */
unsigned long CSearchIndex::AddIndex( CFileInfo *fileinfo, CString path, eFileTypes filetype )
{
	struct filebaseobject fbo;
	unsigned long i;

	fbo.m_eFileType  = filetype;
	fbo.m_nFileIndex = m_pUpdatingFileBaseArray->Size();
	fbo.m_nPathIndex = m_pUpdatingPathBaseArray->Size();
	fbo.m_nHashIndex = (unsigned long)-1;
	fbo.m_nSize      = fileinfo->size;
	fbo.m_tModTime   = fileinfo->st_m_time;

	i = m_pUpdatingBaseArray->Size()/sizeof(struct filebaseobject);

	m_pUpdatingBaseArray->Append( (const char*)&fbo, sizeof(struct filebaseobject) );
	m_pUpdatingFileBaseArray->Append( fileinfo->name.Data(), fileinfo->name.Length()+1 );
	m_pUpdatingPathBaseArray->Append( path.Data(), path.Length()+1 );
	
	return i;
}

/** Only used during update */
void CSearchIndex::AddSearchIndex( const CString & data )
{
	unsigned long pos = m_pUpdatingCaseFoldedData->Size();
	
	m_pUpdatingCaseFoldedBase->Append( (const char*)&pos, sizeof(unsigned long) );
	m_pUpdatingCaseFoldedData->Append( data.Data(), data.Length() + 1 );
}

/** UpdateIndex is only used during update */
void CSearchIndex::UpdateIndex( unsigned long index, struct filebaseobject * fbo )
{
	if ( (index*sizeof(struct filebaseobject)) < m_pUpdatingBaseArray->Size() )
	{
		memcpy( m_pUpdatingBaseArray->Data()+(index*sizeof(struct filebaseobject)), fbo, sizeof(struct filebaseobject) );
	}
}

/** */
bool CSearchIndex::FindHash( unsigned char * hash, unsigned long * hi )
{
	unsigned long i;
	
	for(i=*hi;i<m_pHashIndex->Size();i+=dcpp::TigerTree::BYTES)
		if ( memcmp(hash,m_pHashIndex->Data()+i,dcpp::TigerTree::BYTES) == 0 )
		{
			*hi = i;
			return true;
		}
		
	return false;
}

/** */
bool CSearchIndex::FindHashDuringUpdate( unsigned char * hash, unsigned long * hi )
{
	unsigned long i;
	
	for(i=*hi;i<m_pUpdatingHashIndex->Size();i+=dcpp::TigerTree::BYTES)
		if ( memcmp(hash,m_pUpdatingHashIndex->Data()+i,dcpp::TigerTree::BYTES) == 0 )
		{
			*hi = i;
			return true;
		}
		
	return false;
}

/** */
CString CSearchIndex::GetHash( unsigned long hbi )
{
	CString s;
	CByteArray src;
	struct hashbaseobject * hbo;

	if ( hbi < m_pHashBaseArray->Size() )
	{
		hbo = (struct hashbaseobject *)(m_pHashBaseArray->Data()+hbi);
		src.Append(m_pHashIndex->Data()+hbo->m_nHashIndex,dcpp::TigerTree::BYTES);
		CBase32::Encode( &s, &src );
	}
	
	return s;
}

/** */
bool CSearchIndex::HashBaseIndexFromHashIndex( unsigned long hi, unsigned long * hbi )
{
	unsigned long i;
	struct hashbaseobject * hbo;

	for(i=0;i<m_pHashBaseArray->Size();i+=sizeof(struct hashbaseobject))
	{
		hbo = (struct hashbaseobject *)(m_pHashBaseArray->Data()+i);
		if ( hbo->m_nHashIndex == hi )
		{
			*hbi = i;
			return true;
		}
	}
			
	return false;
}

/** */
bool CSearchIndex::HashBaseIndexFromHashIndexDuringUpdate( unsigned long hi, unsigned long * hbi )
{
	unsigned long i;
	struct hashbaseobject * hbo;

	for(i=0;i<m_pUpdatingHashBaseArray->Size();i+=sizeof(struct hashbaseobject))
	{
		hbo = (struct hashbaseobject *)(m_pUpdatingHashBaseArray->Data()+i);
		if ( hbo->m_nHashIndex == hi )
		{
			*hbi = i;
			return true;
		}
	}
			
	return false;
}

/** */
bool CSearchIndex::BaseIndexFromHashBaseIndex( unsigned long hbi, unsigned long * bi )
{
	unsigned long i;
	struct filebaseobject * fbo;
	
	for(i=0;i<m_pBaseArray->Size();i+=sizeof(struct filebaseobject))
	{
		fbo = (struct filebaseobject*)(m_pBaseArray->Data()+i);
		
		if ( fbo->m_nHashIndex == hbi )
		{
			*bi = i/sizeof(struct filebaseobject);
			return true;
		}
	}
	
	return false;
}

/** */
bool CSearchIndex::FindHashBaseIndex( struct filebaseobject * fbo, unsigned long * hbi )
{
	unsigned long i;
	struct hashbaseobject * hbo;

	for(i=0;i<m_pHashBaseArray->Size();i+=sizeof(struct hashbaseobject))
	{
		hbo = (struct hashbaseobject *)(m_pHashBaseArray->Data()+i);
		
		if ( Compare(fbo,hbo) )
		{
			*hbi = i;
			return true;
		}
	}
	
	return false;
}

/** */
bool CSearchIndex::FindHashBaseIndexDuringUpdate( struct filebaseobject * fbo, unsigned long * hbi )
{
	unsigned long i;
	struct hashbaseobject * hbo;

	for(i=0;i<m_pUpdatingHashBaseArray->Size();i+=sizeof(struct hashbaseobject))
	{
		hbo = (struct hashbaseobject *)(m_pUpdatingHashBaseArray->Data()+i);
		
		if ( CompareDuringUpdate(fbo,hbo) )
		{
			*hbi = i;
			return true;
		}
	}
	
	return false;
}

/** */
bool CSearchIndex::Compare( struct filebaseobject * fbo, struct hashbaseobject * hbo )
{
	if ( fbo->m_nSize != hbo->m_nSize )
		return false;
	if ( fbo->m_tModTime != hbo->m_tModTime )
		return false;

	CString s1,s2;
	
	s1 = (char*)m_pFileBaseArray->Data()+fbo->m_nFileIndex;
	s2 = (char*)m_pHashFileBaseArray->Data()+hbo->m_nFileIndex;
	
	if ( s1 != s2 )
		return false;
	
	s1 = (char*)m_pPathBaseArray->Data()+fbo->m_nPathIndex;
	s2 = (char*)m_pHashPathBaseArray->Data()+hbo->m_nPathIndex;
	
	if ( s1 != s2 )
		return false;

	return true;
}

/** */
bool CSearchIndex::CompareDuringUpdate( struct filebaseobject * fbo, struct hashbaseobject * hbo )
{
	if ( fbo->m_nSize != hbo->m_nSize )
		return false;
	if ( fbo->m_tModTime != hbo->m_tModTime )
		return false;

	CString s1,s2;
	
	s1 = (char*)m_pUpdatingFileBaseArray->Data()+fbo->m_nFileIndex;
	s2 = (char*)m_pUpdatingHashFileBaseArray->Data()+hbo->m_nFileIndex;
	
	if ( s1 != s2 )
		return false;
	
	s1 = (char*)m_pUpdatingPathBaseArray->Data()+fbo->m_nPathIndex;
	s2 = (char*)m_pUpdatingHashPathBaseArray->Data()+hbo->m_nPathIndex;
	
	if ( s1 != s2 )
		return false;

	return true;
}

/** */
void CSearchIndex::AddHashIndex( unsigned long filebaseindex, unsigned char * hash, unsigned char * leaves, unsigned long lsize )
{
	struct filebaseobject fbo;
	struct hashbaseobject hbo;
	char * c;
	unsigned long hi,hbi;
	ulonglong storedlsize;
	
	if ( !GetFileBaseObjectDuringUpdate(filebaseindex,&fbo) )
	{
		return;
	}
	
	hi = 0;
	if ( FindHashDuringUpdate(hash,&hi) )
	{
		if ( HashBaseIndexFromHashIndexDuringUpdate(hi,&hbi) )
		{
			if ( CompareDuringUpdate(&fbo,(struct hashbaseobject *)(m_pUpdatingHashBaseArray->Data()+hbi)) )
			{
				printf("hash found\n");
				fbo.m_nHashIndex = hbi;
				UpdateIndex(filebaseindex,&fbo);
				return;
			}
		}
	}
	
	hbo.m_nSize = fbo.m_nSize;
	hbo.m_tModTime = fbo.m_tModTime;
	hbo.m_nFileIndex = m_pUpdatingHashFileBaseArray->Size();
	hbo.m_nPathIndex = m_pUpdatingHashPathBaseArray->Size();
	hbo.m_nHashIndex = m_pUpdatingHashIndex->Size();
	
	if ( (lsize == 0) || (leaves == 0) )
	{
		hbo.m_nHashLeavesIndex = (unsigned long)-1;
	}
	else
	{
		if ( m_pNewHashLeaves == 0 )
		{
			m_pNewHashLeaves = new CByteArray();
			hashleavessize = CDir().getFileSize(CConfig::Instance()->GetConfigPath()+"hashleaves.bin",false);
		}
		
		storedlsize = lsize;
		hbo.m_nHashLeavesIndex = hashleavessize + m_pNewHashLeaves->Size();
		m_pNewHashLeaves->Append((unsigned char*)&storedlsize,sizeof(storedlsize));
		m_pNewHashLeaves->Append(leaves,lsize);
	}
	
	fbo.m_nHashIndex = m_pUpdatingHashBaseArray->Size();

	m_pUpdatingHashBaseArray->Append((const char*)&hbo, sizeof(struct hashbaseobject));
	c = (char*)m_pUpdatingFileBaseArray->Data()+fbo.m_nFileIndex;
	m_pUpdatingHashFileBaseArray->Append(c,strlen(c)+1);
	c = (char*)m_pUpdatingPathBaseArray->Data()+fbo.m_nPathIndex;
	m_pUpdatingHashPathBaseArray->Append(c,strlen(c)+1);
	m_pUpdatingHashIndex->Append( hash, dcpp::TigerTree::BYTES );

	UpdateIndex(filebaseindex,&fbo);
}

/** */
CString CSearchIndex::GetFileName( unsigned long i )
{
	CString s;
	struct filebaseobject * fbo;

	if ( (i*sizeof(struct filebaseobject)) < m_pBaseArray->Size() )
	{
		fbo = (struct filebaseobject *) (m_pBaseArray->Data()+(i*sizeof(struct filebaseobject)));

		s = (char*) (m_pFileBaseArray->Data()+fbo->m_nFileIndex);
	}

	return s;
}

/** */
CString CSearchIndex::GetFileNameDuringUpdate( unsigned long i )
{
	CString s;
	struct filebaseobject * fbo;

	if ( (i*sizeof(struct filebaseobject)) < m_pUpdatingBaseArray->Size() )
	{
		fbo = (struct filebaseobject *) (m_pUpdatingBaseArray->Data()+(i*sizeof(struct filebaseobject)));

		s = (char*) (m_pUpdatingFileBaseArray->Data()+fbo->m_nFileIndex);
	}

	return s;
}

/** */
CByteArray * CSearchIndex::GetHashLeaves( CString tth )
{
	CByteArray * ba = 0;
	unsigned long hi = 0, hbi = 0;
	
	struct hashbaseobject * hbo = 0;
	CByteArray dst;
	
	if (CBase32::Decode(&dst, &tth) != dcpp::TigerTree::BYTES)
	{
		DPRINTF("GetHashLeaves: Decoded TTH length != dcpp::TigerTree::BYTES\n");
		return 0;
	}
	
	while ( FindHash(dst.Data(), &hi) )
	{
		//DPRINTF("GetHashLeaves: HashIndex=%llu\n", hi);
		
		if ( HashBaseIndexFromHashIndex( hi, &hbi ) )
		{
			//DPRINTF("GetHashLeaves: HashBaseIndex=%llu\n", hbi);
			hbo = (struct hashbaseobject *)(m_pHashBaseArray->Data()+hbi);
			
			if (hbo->m_nHashLeavesIndex == (unsigned long)-1)
			{
				DPRINTF("GetHashLeaves: No Leaves available.\n");
				return 0;
			}
			
			//DPRINTF("GetHashLeaves: HashLeavesIndex=%llu\n", hli);
			
			/* New code for on disk hash leaves */
			CFile leaffile;
			
			if ( leaffile.Open(CConfig::Instance()->GetConfigPath()+"hashleaves.bin", IO_RAW | IO_READONLY) == false )
			{
				printf("GetHashLeaves: cannot open hashleaves.bin\n");
				return 0;
			}
			
			ba = new CByteArray();
			
			if ( ReadLeaves( &leaffile, hbo->m_nHashLeavesIndex, ba ) == false )
			{
				printf("GetHashLeaves: hli=%lu tth=%s\n", hbo->m_nHashLeavesIndex, tth.Data());
				printf("GetHashLeaves: failed to read hash leaves, try database validation\n");
				delete ba;
				ba = 0;
			}
			
			leaffile.Close();
			
			return ba;
		}
		
		/* wrong one, move on to next hash */
		hi += dcpp::TigerTree::BYTES;
	}

	//printf("FindHash for %s failed\n", tth.Data());
	return 0;
}

/** */
bool CSearchIndex::SaveNewHashLeaves()
{
	bool res;
	
	if (m_pNewHashLeaves != 0)
	{
		
		CFile file;
		res = file.Open( CConfig::Instance()->GetConfigPath()+"hashleaves.bin",
			   IO_RAW | IO_WRITEONLY | IO_APPEND | IO_CREAT,
			   MO_IRUSR | MO_IWUSR
		);
		
		if ( res == false )
		{
			printf("CSearchIndex::SaveNewHashLeaves: open hashleaves.bin failed\n");
		}
		else
		{
			long wrote = file.Write( (const char*) m_pNewHashLeaves->Data(), m_pNewHashLeaves->Size() );
			
			if ( wrote == m_pNewHashLeaves->Size() )
			{
				res = true;
			}
			else
			{
				printf("CSearchIndex::SaveNewHashLeaves: only wrote %ld out of %ld bytes\n",wrote,m_pNewHashLeaves->Size());
				if ( wrote > 0 )
				{
					printf("CSearchIndex::SaveNewHashLeaves: hashleaves.bin has been corrupted, do /rebuild\n");
				}
				res = false;
			}
			
			file.Close();
			
			/* If data was written to hashleaves.bin, even if only some, delete m_pNewHashLeaves */
			if ( wrote > 0 )
			{
				delete m_pNewHashLeaves;
				m_pNewHashLeaves = 0;
			}
		}
	}
	else
	{
		//DPRINTF("CSearchIndex::SaveNewHashLeaves: nothing to save\n");
		res = false;
	}
	
	return res;
}

/**
 * Since the files database does not contain entries for files no
 * longer shared, all that is needed is for each entry in files
 * database, find entry in hash database and copy the data
 * to the new hash database.
 *
 * There is also the hash index in the files database so we
 * also need to make a new array of filebaseobjects but
 * the number of entries in it will not change.
 */
long CSearchIndex::RebuildLists()
{
	CFile leaffile;
	if ( leaffile.Open( CConfig::Instance()->GetConfigPath()+"hashleaves.bin", IO_RAW | IO_READONLY ) == false )
	{
		printf("CSearchIndex::RebuildLists: cannot open hashleaves.bin\n");
		return 0;
	}
	
	CByteArray * pNewBaseArray         = new CByteArray();
	
	CByteArray * pNewHashBaseArray     = new CByteArray();
	CByteArray * pNewHashFileBaseArray = new CByteArray();
	CByteArray * pNewHashPathBaseArray = new CByteArray();
	CByteArray * pNewHashIndex         = new CByteArray();
	CByteArray * pNewHashLeaves        = new CByteArray();
	
	if ( dclibVerbose() > 0 )
	{
		printf("##### Before Rebuild #####\n");
		PrintDatabaseStats();
		printf("##########################\n");
	}
	
	struct filebaseobject fbo;
	struct hashbaseobject hbo;
	CString hashfilename, hashfilepath;
	
	/* FIXME shouldn't have been a ulonglong the max size is 12KiB */
	ulonglong lsize;
	CByteArray leaves;
	CByteArray root(dcpp::TigerTree::BYTES);
	
	unsigned long errors = 0;
	
	for ( unsigned long fbi = 0; fbi < m_pBaseArray->Size(); fbi += sizeof(struct filebaseobject) )
	{
		if ( CFileManager::Instance()->Stopped() )
		{
			DPRINTF("CSearchIndex::RebuildLists: interrupted\n");
			
			delete pNewBaseArray;
			
			delete pNewHashBaseArray;
			delete pNewHashFileBaseArray;
			delete pNewHashPathBaseArray;
			delete pNewHashIndex;
			delete pNewHashLeaves;
			
			leaffile.Close();
			
			return 0;
		}
		
		memcpy( &fbo, m_pBaseArray->Data()+fbi, sizeof(struct filebaseobject) );
		
		if ( fbo.m_nHashIndex != (unsigned long)-1 )
		{
			if ( fbo.m_nHashIndex < m_pHashBaseArray->Size() )
			{
				memcpy( &hbo, m_pHashBaseArray->Data()+fbo.m_nHashIndex, sizeof(struct hashbaseobject) );
				
				if ( hbo.m_nPathIndex < m_pHashPathBaseArray->Size() )
				{
					hashfilepath = (const char*) m_pHashPathBaseArray->Data()+hbo.m_nPathIndex;
					
					hbo.m_nPathIndex = pNewHashPathBaseArray->Size();
					pNewHashPathBaseArray->Append( hashfilepath.Data(), hashfilepath.Length()+1 );
				}
				else
				{
					++errors;
					hbo.m_nPathIndex = (unsigned long)-1;
				}
				
				if ( hbo.m_nFileIndex < m_pHashFileBaseArray->Size() )
				{
					hashfilename = (const char*) m_pHashFileBaseArray->Data()+hbo.m_nFileIndex;
					
					hbo.m_nFileIndex = pNewHashFileBaseArray->Size();
					pNewHashFileBaseArray->Append( hashfilename.Data(), hashfilename.Length()+1 );
				}
				else
				{
					++errors;
					hbo.m_nFileIndex = (unsigned long)-1;
				}
				
				if ( hbo.m_nHashIndex != (unsigned long)-1 )
				{
					if ( hbo.m_nHashIndex + dcpp::TigerTree::BYTES <= m_pHashIndex->Size() )
					{
						memcpy( root.Data(), m_pHashIndex->Data()+hbo.m_nHashIndex, dcpp::TigerTree::BYTES );
						
						hbo.m_nHashIndex = pNewHashIndex->Size();
						fbo.m_nHashIndex = pNewHashIndex->Size();
						
						pNewHashIndex->Append( root.Data(), dcpp::TigerTree::BYTES );
					}
					else
					{
						++errors;
						hbo.m_nHashIndex = (unsigned long)-1;
					}
				}
				
				if ( hbo.m_nHashLeavesIndex != (unsigned long)-1 )
				{
					if ( ReadLeaves( &leaffile, hbo.m_nHashLeavesIndex, &leaves ) )
					{
						hbo.m_nHashLeavesIndex = pNewHashLeaves->Size();
						
						lsize = leaves.Size();
						pNewHashLeaves->Append( (const unsigned char*) &lsize, sizeof(lsize) );
						pNewHashLeaves->Append( leaves.Data(), leaves.Size() );
					}
					else
					{
						++errors;
						hbo.m_nHashLeavesIndex = (unsigned long)-1;
					}
				}
				
				fbo.m_nHashIndex = pNewHashBaseArray->Size();
				pNewHashBaseArray->Append( (const unsigned char*) &hbo, sizeof(struct hashbaseobject) );
			}
			else
			{
				++errors;
				fbo.m_nHashIndex = (unsigned long)-1;
			}
		}
		
		pNewBaseArray->Append( (const unsigned char*) &fbo, sizeof(struct filebaseobject) );
		
	}
	
	leaffile.Close();
	
	/* swap the byte arrays */
	CByteArray * tmp1 = m_pBaseArray;
	CByteArray * tmp2 = m_pHashBaseArray;
	CByteArray * tmp3 = m_pHashFileBaseArray;
	CByteArray * tmp4 = m_pHashPathBaseArray;
	CByteArray * tmp5 = m_pHashIndex;
	
	m_pBaseArray         = pNewBaseArray;
	m_pHashBaseArray     = pNewHashBaseArray;
	m_pHashFileBaseArray = pNewHashFileBaseArray;
	m_pHashPathBaseArray = pNewHashPathBaseArray;
	m_pHashIndex         = pNewHashIndex;
	
	delete tmp1;
	delete tmp2;
	delete tmp3;
	delete tmp4;
	delete tmp5;
	
	/* write new hash leaves file */
	pNewHashLeaves->SaveToFile( CConfig::Instance()->GetConfigPath()+"hashleaves.bin" );
	delete pNewHashLeaves;
	
	if ( dclibVerbose() > 0 )
	{
		printf("##### After Rebuild #####\n");
		PrintDatabaseStats();
		printf("#########################\n");
	}
	
	DPRINTF("CSearchIndex::RebuildLists: %ld errors detected\n",errors);
	return errors;
}

/** */
void CSearchIndex::PrintDatabaseStats()
{
	printf("BaseArray size=%lu itemsize=%lu items=%lu\n", m_pBaseArray->Size(), (unsigned long) sizeof(struct filebaseobject), m_pBaseArray->Size()/sizeof(struct filebaseobject) );
	printf("FileBaseArray size=%lu\n", m_pFileBaseArray->Size());
	printf("PathBaseArray size=%lu\n", m_pPathBaseArray->Size());
	
	printf("CaseFoldedBase size=%lu itemsize=%lu items=%lu\n", m_pCaseFoldedBase->Size(), (unsigned long) sizeof(unsigned long), m_pCaseFoldedBase->Size()/sizeof(unsigned long) );
	printf("CaseFoldedData size=%lu\n", m_pCaseFoldedData->Size());
	
	printf("HashBaseArray size=%lu itemsize=%lu items=%lu\n", m_pHashBaseArray->Size(), (unsigned long) sizeof(struct hashbaseobject), m_pHashBaseArray->Size()/sizeof(struct hashbaseobject));
	printf("HashFileBaseArray size=%lu\n", m_pHashFileBaseArray->Size());
	printf("HashPathBaseArray size=%lu\n", m_pHashPathBaseArray->Size());
	printf("HashIndex size=%lu itemsize=%lu items=%lu\n", m_pHashIndex->Size(), (unsigned long) dcpp::TigerTree::BYTES, m_pHashIndex->Size()/dcpp::TigerTree::BYTES);
	
	printf("HashLeaves size=%llu (on disk)\n", CDir().getFileSize(CConfig::Instance()->GetConfigPath()+"hashleaves.bin",false));
}

/** */
long CSearchIndex::ValidateHashLeaves()
{
	long removed = 0;
	bool valid;
	
	CFile leaffile;
	if ( leaffile.Open( CConfig::Instance()->GetConfigPath()+"hashleaves.bin", IO_RAW | IO_READONLY ) == false )
	{
		printf("CSearchIndex::ValidateHashLeaves: cannot open hashleaves.bin\n");
		return removed;
	}
	
	CByteArray * pNewHashBaseArray = new CByteArray();
	
	struct hashbaseobject hbo;
	CByteArray root(dcpp::TigerTree::BYTES);
	CByteArray leaves;
	
	for ( unsigned long hbi = 0; hbi < m_pHashBaseArray->Size(); hbi += sizeof(struct hashbaseobject) )
	{
		if ( CFileManager::Instance()->Stopped() )
		{
			delete pNewHashBaseArray;
			leaffile.Close();
			DPRINTF("CSearchIndex::ValidateHashLeaves: interrupted\n");
			return 0;
		}
		
		valid = false;
		
		memcpy( &hbo, m_pHashBaseArray->Data()+hbi, sizeof(struct hashbaseobject) );
		
		/* check pointer to hash root */
		if ( hbo.m_nHashIndex + dcpp::TigerTree::BYTES <= m_pHashIndex->Size() )
		{
			/* get hash root */
			memcpy( root.Data(), m_pHashIndex->Data()+hbo.m_nHashIndex, dcpp::TigerTree::BYTES );
			
			/* get hash leaves */
			if ( ReadLeaves( &leaffile, hbo.m_nHashLeavesIndex, &leaves ) )
			{
				/* got root and leaves, validate them */
				valid = CFileHasher::ValidateHashLeaves( &root, &leaves, hbo.m_nSize );
			}
		}
		
		if ( valid )
		{
			pNewHashBaseArray->Append( (unsigned char*) &hbo, sizeof(struct hashbaseobject) );
		}
		else
		{
			++removed;
		}
	}
	
	leaffile.Close();
	
	if ( removed > 0 )
	{
		CByteArray * tmp = m_pHashBaseArray;
		m_pHashBaseArray = pNewHashBaseArray;
		delete tmp;
	}
	else
	{
		delete pNewHashBaseArray;
	}
	
	DPRINTF("CSearchIndex::ValidateHashLeaves: removed %ld invalid entries\n",removed);
	return removed;
}

/** */
bool CSearchIndex::ReadLeaves( CFile * file, unsigned long hli, CByteArray * dest )
{
	if ( file->Seek( hli, SEEK_SET ) )
	{
		/* FIXME shouldn't have been a ulonglong the max size is 12KiB */
		ulonglong lsize;
		
		if ( file->Read( (char*) &lsize, sizeof(lsize) ) == sizeof(lsize) )
		{
			if ( lsize < MAX_LEAVES_SIZE )
			{
				dest->SetSize( (unsigned long) lsize );
				if ( file->Read( (char*) dest->Data(), (long) lsize ) == (long) lsize )
				{
					return true;
				}
			}
		}
	}
	
	return false;
}

