/*
 * Copyright (C) 2019-2021 Matthieu Gautier <mgautier@kymeria.fr>
 * Copyright (C) 2021 Maneesh P M <manu.pm55@gmail.com>
 * Copyright (C) 2021 Veloman Yunkan
 * Copyright (C) 2009 Tommi Maekitalo
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
 * NON-INFRINGEMENT.  See the GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 */

#include <zim/writer/creator.h>

#include "config.h"

#include "creatordata.h"
#include "cluster.h"
#include "debug.h"
#include "workers.h"
#include "clusterWorker.h"
#include <zim/blob.h>
#include <zim/error.h>
#include <zim/writer/contentProvider.h>
#include <zim/tools.h>
#include "../endian_tools.h"
#include <algorithm>
#include <fstream>
#include "../md5.h"
#include "../constants.h"
#include "counterHandler.h"
#include "writer/_dirent.h"

#if defined(ENABLE_XAPIAN)
# include "xapianHandler.h"
# include "xapianIndexer.h"
#endif

#ifdef _WIN32
# include <io.h>
# include <fcntl.h>
#else
# include <unistd.h>
# define _write(fd, addr, size) if(::write((fd), (addr), (size)) != (ssize_t)(size)) \
{throw std::runtime_error("Error writing");}
#endif

#include <sys/stat.h>
#include <stdio.h>
#include <fcntl.h>
#include <limits>
#include <stdexcept>
#include <sstream>
#include <ctime>
#include <deque>
#include "log.h"
#include "../fs.h"
#include "../tools.h"

log_define("zim.writer.creator")

#define INFO(e) \
    do { \
        log_info(e); \
        std::cout << e << std::endl; \
    } while(false)

#define TINFO(e) \
    if (m_verbose) { \
        double seconds = difftime(time(NULL), data->start_time); \
        std::cout << "T:" << (int)(seconds) \
                  << "; " << e << std::endl; \
    }

#define TPROGRESS() \
    if (m_verbose ) { \
        double seconds = difftime(time(NULL),data->start_time);  \
        std::cout << "T:" << (int)seconds \
                  << "; A:" << data->dirents.size() \
                  << "; RA:" << data->nbRedirectItems \
                  << "; CA:" << data->nbCompItems \
                  << "; UA:" << data->nbUnCompItems \
                  << "; C:" << data->nbClusters \
                  << "; CC:" << data->nbCompClusters \
                  << "; UC:" << data->nbUnCompClusters \
                  << "; WC:" << data->taskList.size() \
                  << std::endl; \
    }


#define CLUSTER_BASE_OFFSET 2048

namespace zim
{

namespace writer
{

namespace
{

void reportInvalidRedirect(const Dirent& dirent)
{
  const Dirent& targetDirent = *dirent.getRedirectTargetDirent();
  INFO("Removing invalid redirection "
      << NsAsChar(dirent.getNamespace()) << '/' << dirent.getPath()
      << " redirecting to (missing) "
      << NsAsChar(targetDirent.getNamespace()) << '/' << targetDirent.getPath()
  );
}

InvalidEntry direntConflictError(const Dirent& existingDirent, const Dirent& newDirent)
{
  Formatter fmt;
  fmt << "Impossible to add " << NsAsChar(newDirent.getNamespace()) << "/" << newDirent.getPath() << "\n"
      << "  dirent's title to add is : " << newDirent.getTitle() << "\n"
      << "  existing dirent's title is : " << existingDirent.getTitle() << "\n";
  return InvalidEntry(fmt);
}

entry_index_type markRedirectChain(Dirent* d, entry_index_type i)
{
  ASSERT(d->isRemoved(), ==, false);

  const entry_index_type startingIndex = i;
  while ( !d->isItem() ) {
    d->setIdx(entry_index_t(i++));
    d = d->getRedirectTargetDirent();
    if ( d == nullptr || d->isRemoved() || d->getIdx().v >= startingIndex ) {
      // hit a dead end or detected being caught in a loop
      return startingIndex;
    }
  }

  return i;
}

void markBlindRedirectChainForRemoval(Dirent* d)
{
  while ( !d->isRemoved() ) {
    Dirent* const t = d->getRedirectTargetDirent();
    d->markRemoved();
    if ( t == nullptr )
      break;

    INFO("Redirection "
        << NsAsChar(d->getNamespace()) << '/' << d->getPath()
        << " -> "
        << NsAsChar(t->getNamespace()) << '/' << t->getPath()
        << " belongs to a blind chain or loop. Removing..."
    );
    d = t;
  }
}

void setFrontArticle(Dirent& dirent, const Hints& hints)
{
  const auto it = hints.find(FRONT_ARTICLE);
  if ( it != hints.end() && it->second != 0 )
    dirent.markFrontArticle();
}

bool getCompressionHint(const Hints& hints)
{
  const auto it = hints.find(COMPRESS);
  return it != hints.end() && it->second != 0;
}

class TitleListingProvider : public ContentProvider {
  public:
    explicit TitleListingProvider(const CreatorData::UrlSortedDirents& dirents) {
      for ( Dirent* const d : dirents ) {
        if ( d->isFrontArticle() ) {
          m_dirents.push_back(d);
        }
      }

      std::sort(m_dirents.begin(), m_dirents.end(), compareTitle);
      m_it = m_dirents.begin();
    }

    zim::size_type getSize() const override {
        return m_dirents.size() * sizeof(zim::entry_index_type);
    }

    zim::Blob feed() override {
      if (m_it == m_dirents.end()) {
        return zim::Blob(nullptr, 0);
      }
      zim::toLittleEndian((*m_it)->getIdx().v, buffer);
      m_it++;
      return zim::Blob(buffer, sizeof(zim::entry_index_type));
    }

  private:
    typedef std::deque<const Dirent*> DirentPtrs;
    DirentPtrs m_dirents;
    char buffer[sizeof(zim::entry_index_type)];
    DirentPtrs::const_iterator m_it;
};

} // unnamed namespace

Creator::Creator()
  : m_clusterSize(DEFAULT_CLUSTER_SIZE)
{}

Creator::~Creator() = default;

Creator& Creator::configVerbose(bool verbose)
{
  m_verbose = verbose;
  return *this;
}

Creator& Creator::configCompression(Compression compression)
{
  m_compression = compression;
  return *this;
}

Creator& Creator::configClusterSize(zim::size_type targetSize)
{
  m_clusterSize = targetSize;
  return *this;
}

Creator& Creator::configIndexing(bool indexing, const std::string& language)
{
  m_withIndex = indexing;
  m_indexingLanguage = language;
  return *this;
}

Creator& Creator::configNbWorkers(unsigned nbWorkers)
{
  m_nbWorkers = nbWorkers;
  return *this;
}

void Creator::startZimCreation(const std::string& filepath)
{
  data = std::unique_ptr<CreatorData>(
    new CreatorData(filepath, m_verbose, m_withIndex, m_indexingLanguage, m_compression, m_clusterSize)
  );

  for(unsigned i=0; i<m_nbWorkers; i++)
  {
    std::thread thread(taskRunner, this->data.get());
    data->workerThreads.push_back(std::move(thread));
  }

  data->writerThread = std::thread(clusterWriter, this->data.get());
}

void Creator::addItem(std::shared_ptr<Item> item)
{
  checkError();
  const auto path = item->getPath();
  auto mimetype = item->getMimeType();
  if (mimetype.empty()) {
    std::cerr << "WARNING: mimetype missing for " << path << std::endl;
    mimetype = "application/octet-stream";
  }
  const auto mimeTypeIdx = data->getMimeTypeIdx(mimetype);
  const Hints& itemHints = item->getAmendedHints();
  const bool compressContent = getCompressionHint(itemHints);

  Dirent direntToAdd(NS::C, path, item->getTitle(), mimeTypeIdx);
  data->ensureDirentCanBeAdded(direntToAdd);
  setFrontArticle(direntToAdd, itemHints);
  data->addItemData(direntToAdd, item->getContentProvider(), compressContent);
  data->handle(direntToAdd, item);
  data->addOrUpdate(std::move(direntToAdd));

  if (data->dirents.size()%1000 == 0) {
    TPROGRESS();
  }
}

void Creator::addMetadata(const std::string& name, const std::string& content, const std::string& mimetype)
{
  checkError();
  auto provider = std::unique_ptr<ContentProvider>(new StringProvider(content));
  addMetadata(name, std::move(provider), mimetype);
}

void Creator::addMetadata(const std::string& name, std::unique_ptr<ContentProvider> provider, const std::string& mimetype)
{
  checkError();
  auto compressContent = isCompressibleMimetype(mimetype);
  const auto mimeTypeIdx = data->getMimeTypeIdx(mimetype);
  Dirent direntToAdd(NS::M, name, "", mimeTypeIdx);
  data->ensureDirentCanBeAdded(direntToAdd);
  data->addItemData(direntToAdd, std::move(provider), compressContent);
  data->handle(direntToAdd);
  data->addOrUpdate(std::move(direntToAdd));
}

void Creator::addIllustration(const IllustrationInfo& ii, const std::string& content)
{
  checkError();
  auto provider = std::unique_ptr<ContentProvider>(new StringProvider(content));
  addIllustration(ii, std::move(provider));
}

void Creator::addIllustration(const IllustrationInfo& ii, std::unique_ptr<ContentProvider> provider)
{
  checkError();
  addMetadata(ii.asMetadataItemName(), std::move(provider), "image/png");
}

void Creator::addIllustration(unsigned int size, const std::string& content)
{
  addIllustration(IllustrationInfo{size, size, 1, {}}, content);
}

void Creator::addIllustration(unsigned int size, std::unique_ptr<ContentProvider> provider)
{
  addIllustration(IllustrationInfo{size, size, 1, {}}, std::move(provider));
}

void Creator::addRedirection(const std::string& path, const std::string& title, const std::string& targetPath, const Hints& hints)
{
  checkError();
  const auto targetDirent = data->addOrUpdate(Dirent(NS::C, targetPath));
  const auto dirent = data->addOrUpdate(Dirent(NS::C, path, title, targetDirent));
  setFrontArticle(*dirent, hints);

  if (data->dirents.size()%1000 == 0){
    TPROGRESS();
  }

  data->handle(*dirent);
}

void Creator::addAlias(const std::string& path, const std::string& title, const std::string& targetPath, const Hints& hints)
{
  checkError();
  const auto existing_dirent_it = data->findDirent(NS::C, targetPath);

  if (existing_dirent_it == data->dirents.end()) {
    Formatter fmt;
    fmt << "Impossible to alias C/" << targetPath << " as C/" << path << std::endl;
    fmt << "C/" << targetPath << " doesn't exist." << std::endl;
    throw InvalidEntry(fmt);
  }

  Dirent direntToAdd(path, title, **existing_dirent_it);
  data->ensureDirentCanBeAdded(direntToAdd);
  setFrontArticle(direntToAdd, hints);
  data->handle(direntToAdd);
  data->addOrUpdate(std::move(direntToAdd));
}

void Creator::finishZimCreation()
{
  checkError();

  data->createDirent(NS::X, "listing/titleOrdered/v1", "application/octet-stream+zimlisting", "");

  // Create a redirection for the mainPage.
  // We need to keep the created dirent to set the fileheader.
  // Dirent doesn't have to be deleted.
  if (!m_mainPath.empty()) {
    const auto mainPageDirentIt = data->findDirent(NS::C, m_mainPath);
    if ( mainPageDirentIt != data->dirents.end() ) {
      data->mainPageDirent = data->addOrUpdate(Dirent(NS::W, "mainPage", "", *mainPageDirentIt));
      data->handle(*data->mainPageDirent);
    }
  }

  TPROGRESS();

  // We need to create all dirents before resolving redirects and setting entry
  // indexes.
  for(auto& handler:data->m_direntHandlers) {
    // This silently create all the needed dirents.
    handler->getDirents();
  }

  // Now we have all the dirents (but not the data), we must correctly set/fix
  // the dirents before we ask data to the handlers
  TINFO("ResolveRedirectIndexes");
  data->detectDanglingRedirects();
  data->removeLoopsAndBlindChainsOfRedirects();
  data->dropRemovedRedirects();

  data->indexTitles();

  TINFO("Set entry indices");
  data->setEntryIndexes();

  TINFO("Resolve mimetype");
  data->resolveMimeTypes();

  // We can now stop the direntHandlers, and get their content
  for(auto& handler:data->m_direntHandlers) {
    handler->stop();
    const auto& dirents = handler->getDirents();
    if (dirents.empty()) {
      continue;
    }
    auto providers = handler->getContentProviders();
    ASSERT(dirents.size(), ==, providers.size());
    auto provider_it = providers.begin();
    for(auto& dirent:dirents) {
      // As we use a "handler level" isCompressible, all content of the same
      // handler must have the same compression.
      data->addItemData(*dirent, std::move(*provider_it), handler->isCompressible());
      provider_it++;
    }
  }

  data->addTitleListingData();

  // All the data has been added, we can now close all clusters
  if (data->compCluster->count())
    data->closeCluster(true);

  if (data->uncompCluster->count())
    data->closeCluster(false);

  TINFO("Waiting for workers");
  // wait all cluster compression has been done
  ClusterTask::waitNoMoreTask(data.get());

  data->quitAllThreads();
  checkError();

  // Delete all handler (they will clean there own data)
  data->m_direntHandlers.clear();

  TINFO(data->dirents.size() << " title index created");
  TINFO(data->clustersList.size() << " clusters created");

  TINFO("write zimfile :");
  writeLastParts();
  ::close(data->out_fd);
  data->out_fd = -1;

  TINFO("rename tmpfile to final one.");
  DEFAULTFS::rename(data->tmpFileName, data->zimName);
  data->tmpFileName.clear();

  TINFO("finish");
}

void Creator::fillHeader(Fileheader* header) const
{
  header->setMainPage(
    data->mainPageDirent
    ? entry_index_type(data->mainPageDirent->getIdx())
    : std::numeric_limits<entry_index_type>::max());
  header->setLayoutPage(std::numeric_limits<entry_index_type>::max());

  header->setUuid( m_uuid );
  header->setArticleCount( data->dirents.size() );

  header->setMimeListPos( Fileheader::size );

  header->setTitleIdxPos(offset_type(-1));
  header->setClusterCount( data->clustersList.size() );
}

void Creator::writeLastParts() const
{
  Fileheader header;
  fillHeader(&header);

  int out_fd = data->out_fd;

  lseek(out_fd, header.getMimeListPos(), SEEK_SET);
  TINFO(" write mimetype list");
  for(auto& mimeType: data->mimeTypesList)
  {
    _write(out_fd, mimeType.c_str(), mimeType.size()+1);
  }

  _write(out_fd, "", 1);

  ASSERT(lseek(out_fd, 0, SEEK_CUR), <, CLUSTER_BASE_OFFSET);

  { // writing dirents

  // TODO: Memory usage by direntOffsets can be reduced by replacing
  // TODO: the offset values with dirent size values and reconstructing
  // TODO: the offsets as a running sum starting from the offset of the
  // TODO: first dirent
  std::deque<offset_t> direntOffsets;

  TINFO(" write directory entries");
  lseek(out_fd, 0, SEEK_END);
  for (Dirent* dirent: data->dirents)
  {
    direntOffsets.push_back(offset_t(lseek(out_fd, 0, SEEK_CUR)));
    dirent->write(out_fd);
  }

  TINFO(" write path ptr list");
  header.setPathPtrPos(lseek(out_fd, 0, SEEK_CUR));
  for (auto offset : direntOffsets)
  {
    char tmp_buff[sizeof(offset_type)];
    toLittleEndian(offset, tmp_buff);
    _write(out_fd, tmp_buff, sizeof(offset_type));
  }

  } // writing dirents

  TINFO(" write cluster offset list");
  header.setClusterPtrPos(lseek(out_fd, 0, SEEK_CUR));
  for (auto cluster : data->clustersList)
  {
    char tmp_buff[sizeof(offset_type)];
    toLittleEndian(cluster->getOffset(), tmp_buff);
    _write(out_fd, tmp_buff, sizeof(offset_type));
  }

  header.setChecksumPos(lseek(out_fd, 0, SEEK_CUR));

  TINFO(" write header");
  lseek(out_fd, 0, SEEK_SET);
  header.write(out_fd);

  TINFO(" write checksum");
  struct zim_MD5_CTX md5ctx;
  unsigned char batch_read[1024+1];
  lseek(out_fd, 0, SEEK_SET);
  zim_MD5Init(&md5ctx);
  while (true) {
     auto r = read(out_fd, batch_read, 1024);
     if (r == -1) {
       throw std::runtime_error(std::strerror(errno));
     }
     if (r == 0)
       break;
     batch_read[r] = 0;
     zim_MD5Update(&md5ctx, batch_read, r);
  }
  unsigned char digest[16];
  zim_MD5Final(digest, &md5ctx);
  _write(out_fd, reinterpret_cast<const char*>(digest), 16);
}

void Creator::checkError()
{
  if (data->m_errored) {
    throw CreatorStateError();
  }
  std::lock_guard<std::mutex> l(data->m_exceptionLock);
  if (data->m_exceptionSlot) {
    std::cerr << "ERROR Detected" << std::endl;
    data->m_errored = true;
    throw AsyncError(data->m_exceptionSlot);
  }
}

CreatorData::CreatorData(const std::string& fname,
                               bool verbose,
                               bool withIndex,
                               std::string language,
                               Compression c,
                               size_t clusterSize)
  : mainPageDirent(nullptr),
    m_errored(false),
    compression(c),
    zimName(fname),
    tmpFileName(fname + ".tmp"),
    clusterSize(clusterSize),
    withIndex(withIndex),
    indexingLanguage(language),
    verbose(verbose),
    nbRedirectItems(0),
    nbCompItems(0),
    nbUnCompItems(0),
    nbClusters(0),
    nbCompClusters(0),
    nbUnCompClusters(0),
    start_time(time(NULL))
{
#ifdef _WIN32
  int flag = _O_RDWR | _O_CREAT | _O_TRUNC | _O_BINARY;
  int mode =  _S_IREAD | _S_IWRITE;
#else
  int flag = O_RDWR | O_CREAT | O_TRUNC;
  mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
#endif
  out_fd = open(tmpFileName.c_str(), flag, mode);
  if (out_fd == -1){
    throw std::runtime_error(std::strerror(errno));
  }
  if(lseek(out_fd, CLUSTER_BASE_OFFSET, SEEK_SET) != CLUSTER_BASE_OFFSET) {
    close(out_fd);
    throw std::runtime_error(std::strerror(errno));
  }

  // We keep both a "compressed cluster" and an "uncompressed cluster"
  // because we don't know which one will fill up first.  We also need
  // to track the dirents currently in each, so we can fix up the
  // cluster index if the other one ends up written first.
  compCluster = new Cluster(compression);
  uncompCluster = new Cluster(Compression::None);

#if defined(ENABLE_XAPIAN)
  if ( withIndex ) {
    m_direntHandlers.push_back(std::make_shared<XapianHandler>(this));
  }
#endif

  m_direntHandlers.push_back(std::make_shared<CounterHandler>(this));

  for(auto& handler:m_direntHandlers) {
    handler->start();
  }
}

CreatorData::~CreatorData()
{
  quitAllThreads();
  if (compCluster)
    delete compCluster;
  if (uncompCluster)
    delete uncompCluster;
  for(auto& cluster: clustersList) {
    delete cluster;
  }
  if ( out_fd != - 1 ) {
    ::close(out_fd);
  }
  if ( ! tmpFileName.empty() ) {
    DEFAULTFS::removeFile(tmpFileName);
  }
}

void CreatorData::addError(const std::exception_ptr exception)
{
  std::lock_guard<std::mutex> l(m_exceptionLock);
  if (!m_exceptionSlot) {
    m_exceptionSlot = exception;
  }
}

bool CreatorData::isErrored() const
{
  if (m_errored) {
    return true;
  }
  std::lock_guard<std::mutex> l(m_exceptionLock);
  if (m_exceptionSlot) {
    return true;
  }
  return false;
}

void CreatorData::quitAllThreads() {
  // Quit all workerThreads
  for (auto i=0U; i< workerThreads.size(); i++) {
    taskList.pushToQueue(nullptr);
  }
  for(auto& thread: workerThreads) {
    thread.join();
  }
  workerThreads.clear();

  // Wait for writerThread to finish.
  if (writerThread.joinable()) {
    clusterToWrite.pushToQueue(nullptr);
    writerThread.join();
  }
}

CreatorData::DirentIterator CreatorData::findDirent(NS ns, const std::string& path)
{
  Dirent tmpDirent(ns, path);
  return dirents.find(&tmpDirent);
}

// Returns the dirent with the namespace and path of that passed as the
// argument. Throws an exception if overwriting the returned dirent with the
// query one would mean that there has been a dirent conflict.
Dirent* CreatorData::getMatchingDirent(Dirent& d)
{
  const auto it = dirents.find(&d);
  if ( it != dirents.end() ) {
    Dirent* const existingDirent = *it;
    if ( !d.isPlaceholder() ) {
      if ( !existingDirent->isPlaceholder() ) {
        throw direntConflictError(*existingDirent, d);
      }
    }
    return existingDirent;
  }
  return nullptr;
}

void CreatorData::ensureDirentCanBeAdded(Dirent& d)
{
  getMatchingDirent(d);
}

Dirent* CreatorData::addOrUpdate(Dirent&& d)
{
  if ( Dirent* const existingDirent = getMatchingDirent(d) ) {
    if ( !d.isPlaceholder() ) {
      existingDirent->~Dirent();
      new(existingDirent) Dirent(std::move(d));
    }
    return existingDirent;
  }

  return add(std::move(d));
}

CreatorData::DirentIterator CreatorData::removeDirent(DirentIterator it)
{
  Dirent* const dirent = *it;
  dirent->markRemoved();
  if (dirent == mainPageDirent) {
    mainPageDirent = nullptr;
  }
  return dirents.erase(it);
}

void CreatorData::removeDirent(Dirent* dirent)
{
  const auto it = dirents.find(dirent);
  ASSERT(it != dirents.end(), ==, true);
  removeDirent(it);
}

Dirent* CreatorData::add(Dirent&& d)
{
  auto dirent = pool.add(std::move(d));
  auto ret = dirents.insert(dirent);
  if (!ret.second) {
    throw direntConflictError(**ret.first, *dirent);
  };

  if (dirent->isRedirect()) {
    nbRedirectItems++;
  }
  return dirent;
}

void CreatorData::addItemData(Dirent& dirent, std::unique_ptr<ContentProvider> provider, bool compressContent)
{
  // Add blob data to compressed or uncompressed cluster.
  auto itemSize = provider->getSize();
  if (itemSize > 0)
  {
    isEmpty = false;
  }

  auto cluster = compressContent ? compCluster : uncompCluster;

  // If cluster will be too large, write it to dis, and open a new
  // one for the content.
  if ( cluster->count()
    && cluster->size().v+itemSize >= clusterSize
     )
  {
    log_info("cluster with " << cluster->count() << " items, " <<
             cluster->size() << " bytes; current title \"" <<
             dirent.getTitle() << '\"');
    cluster = closeCluster(compressContent);
  }

  dirent.setCluster(cluster);
  cluster->addContent(std::move(provider));

  if (compressContent) {
    nbCompItems++;
  } else {
    nbUnCompItems++;
  }
}

Dirent* CreatorData::createDirent(NS ns, const std::string& path, const std::string& mimetype, const std::string& title)
{
  return add(Dirent(ns, path, title, getMimeTypeIdx(mimetype)));
}

Cluster* CreatorData::closeCluster(bool compressed)
{
  Cluster *cluster;
  nbClusters++;
  if (compressed )
  {
    cluster = compCluster;
    nbCompClusters++;
  } else {
    cluster = uncompCluster;
    nbUnCompClusters++;
  }
  cluster->setClusterIndex(cluster_index_t(clustersList.size()));
  clustersList.push_back(cluster);
  taskList.pushToQueue(std::make_shared<ClusterTask>(cluster));
  clusterToWrite.pushToQueue(cluster);

  if (compressed)
  {
    cluster = compCluster = new Cluster(compression);
  } else {
    cluster = uncompCluster = new Cluster(Compression::None);
  }
  return cluster;
}

void CreatorData::setEntryIndexes()
{
  INFO("Set entry indices");
  entry_index_t idx(0);
  for (auto& dirent: dirents) {
    dirent->setIdx(idx);
    idx += 1;
  }
}

void CreatorData::detectDanglingRedirects()
{
  INFO("Detect dangling redirects");
  for (Dirent* const dirent : dirents)
  {
    if ( dirent->isUnresolvedRedirect() ) {
      reportInvalidRedirect(*dirent);
      dirent->getRedirectTargetDirent()->markRemoved();
      dirent->markRemoved();
    }
  }
}

// XXX: This procedure uses the 'idx' field of Dirent and is therefore
// XXX: implemented under the assumption that Dirent::setIdx() has not yet been
// XXX: called
void CreatorData::removeLoopsAndBlindChainsOfRedirects()
{
  INFO("Detect loops and/or blind chains of redirects");
  entry_index_type index = 1;
  for (Dirent* const d : dirents)
  {
    if ( !d->isRemoved() && d->isRedirect() && d->getIdx().v == 0 ) {
      const entry_index_type newIndex = markRedirectChain(d, index);
      if ( newIndex == index ) {
        markBlindRedirectChainForRemoval(d);
      }
      index = newIndex;
    }
  }

  // Restore/reset Dirent::idx to 0
  for (Dirent* const d : dirents)
  {
    d->setIdx(entry_index_t(0));
  }
}

void CreatorData::dropRemovedRedirects()
{
  for (auto it = dirents.begin(); it != dirents.end(); )
  {
    if ( (*it)->isRemoved() ) {
      it = removeDirent(it);
    } else  {
      ++it;
    }
  }
}

void CreatorData::resolveMimeTypes()
{
  std::vector<std::string> oldMImeList;
  std::vector<uint16_t> mapping;

  for (auto& rmimeType: rmimeTypesMap)
  {
    oldMImeList.push_back(rmimeType.second);
    mimeTypesList.push_back(rmimeType.second);
  }

  mapping.resize(oldMImeList.size());
  std::sort(mimeTypesList.begin(), mimeTypesList.end());

  for (unsigned i=0; i<oldMImeList.size(); ++i)
  {
    for (unsigned j=0; j<mimeTypesList.size(); ++j)
    {
      if (oldMImeList[i] == mimeTypesList[j])
        mapping[i] = static_cast<uint16_t>(j);
    }
  }

  for (auto& dirent: dirents)
  {
    if (dirent->isItem())
      dirent->setMimeType(mapping[dirent->getMimeType()]);
  }
}

uint16_t CreatorData::getMimeTypeIdx(const std::string& mimeType)
{
  auto it = mimeTypesMap.find(mimeType);
  if (it == mimeTypesMap.end())
  {
    if (nextMimeIdx >= std::numeric_limits<uint16_t>::max())
      throw CreatorError("too many distinct mime types");
    mimeTypesMap[mimeType] = nextMimeIdx;
    rmimeTypesMap[nextMimeIdx] = mimeType;
    return nextMimeIdx++;
  }

  return it->second;
}

const std::string& CreatorData::getMimeType(uint16_t mimeTypeIdx) const
{
  auto it = rmimeTypesMap.find(mimeTypeIdx);
  if (it == rmimeTypesMap.end())
    throw CreatorError("mime type index not found");
  return it->second;
}

void CreatorData::addTitleListingData()
{
  Dirent* const d = *findDirent(NS::X, "listing/titleOrdered/v1");
  auto listingProvider = std::make_unique<TitleListingProvider>(this->dirents);
  addItemData(*d, std::move(listingProvider), false);
}

#if defined(ENABLE_XAPIAN)
namespace
{

void indexTitle(XapianIndexer& indexer, const Dirent& dirent)
{
  const auto title = dirent.getTitle();
  if ( !title.empty() ) {
    const auto path = dirent.getPath();
    if (dirent.isRedirect()) {
      const auto redirectPath = dirent.getRedirectPath();
      indexer.indexTitle(path, title, redirectPath);
    } else {
      indexer.indexTitle(path, title);
    }
  }
}

} // unnamed namespace
#endif // defined(ENABLE_XAPIAN)

void CreatorData::indexTitles()
{
  INFO("Index titles");
#if defined(ENABLE_XAPIAN)
  const std::string tmpFilePath = zimName + "_title.idx";
  XapianIndexer xi(tmpFilePath, indexingLanguage, IndexingMode::TITLE, true);
  xi.indexingPrelude();
  for ( Dirent* const d : dirents ) {
    if ( d->isFrontArticle() ) {
      indexTitle(xi, *d);
    }
  }
  xi.indexingPostlude();

  if (!xi.is_empty()) {
    Dirent* const d = createDirent(NS::X, "title/xapian",
                                   "application/octet-stream+xapian", "");
    auto fileDataProvider = std::make_unique<FileProvider>(tmpFilePath);
    addItemData(*d, std::move(fileDataProvider), false);
  }
#endif // defined(ENABLE_XAPIAN)
}

} // namespace writer

} // namespace zim
