/* BEGIN software license
 *
 * msXpertSuite - mass spectrometry software suite
 * -----------------------------------------------
 * Copyright(C) 2009,...,2018 Filippo Rusconi
 *
 * http://www.msxpertsuite.org
 *
 * This file is part of the msXpertSuite project.
 *
 * The msXpertSuite project is the successor of the massXpert project. This
 * project now includes various independent modules:
 *
 * - massXpert, model polymer chemistries and simulate mass spectrometric data;
 * - mineXpert, a powerful TIC chromatogram/mass spectrum viewer/miner;
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * END software license
 */


////////////////////////////// Qt includes
#include <QStringList>
#include <QFile>
#include <QRegularExpression>
#include <QDebug>
#include <QVector>


////////////////////////////// local includes
#include <minexpert/nongui/MassSpecDataFileLoaderBrukerXy.hpp>
#include <libmass/DataPoint.hpp>

#include <minexpert/nongui/globals.hpp>


namespace msXpSmineXpert
{

MassSpecDataFileLoaderBrukerXy::MassSpecDataFileLoaderBrukerXy(
  const QString &fileName)
  : MassSpecDataFileLoader{fileName}
{
}


MassSpecDataFileLoaderBrukerXy::~MassSpecDataFileLoaderBrukerXy()
{
}


QString
MassSpecDataFileLoaderBrukerXy::formatDescription() const
{
  QString description = QString(
    "The brukerXy file format is available from the Bruker software.\n"
    "It is typically along these lines:\n"
    "0.0191333,+,ESI,ms1,-,line,500.0000-2000.0000,8882,494.6429 148,....\n"
    "0.03585,+,ESI,ms1,-,line,500.0000-2000.0000,9025,494.6433 113,494.7604 "
    "66....\n"
    "This is the format:\n"
    "<retentionTime>,<ionizationMode>,<IonSourceType>,<msLevel>,-,\n"
    "<dataType>,<mzStart-mzEnd>,<numberPoints>\n"
    "and then a long list of <mz i> pairs delimited with ',' up to the end of "
    "line.\n"
    "This file format may hold data for any number of mass spectra.\n");

  return description;
}


QString
MassSpecDataFileLoaderBrukerXy::craftSpectrumHeaderRegExpString()
{
  // Line format:
  // "0.0191333,+,ESI,ms1,-,line,500.0000-2000.0000,8882,494.6429 148,....\n"

  // A number of values are double precision real numbers (with decimal dot or
  // not).

  QString mzRegExpString("\\d*\\.?\\d+");
  QString rtRegExpString            = mzRegExpString;
  QString iRegExpString             = mzRegExpString;
  QString ionPosNegModeRegExpString = QString("[+-]");
  QString ionSourceTypeRegExpString = QString("[A-Z]+");
  QString msLevelRegExpString       = QString("ms\\d+");
  QString dataFormatRegExpString    = QString("[a-z]+");
  QString numPointsRegExpString     = QString("\\d+");

  QString regExpString = QString("(%1),(%2),(%3),(%4),-,(%5),(%6)-(%7),(%8)")
                           .arg(rtRegExpString)
                           .arg(ionPosNegModeRegExpString)
                           .arg(ionSourceTypeRegExpString)
                           .arg(msLevelRegExpString)
                           .arg(dataFormatRegExpString)
                           .arg(mzRegExpString)
                           .arg(iRegExpString)
                           .arg(numPointsRegExpString);

  // qDebug() << __FILE__ << __LINE__
  //<< "regExpString:" << regExpString;

  return regExpString;
}


QString
MassSpecDataFileLoaderBrukerXy::craftSpectrumLineRegExpString()
{
  // A number of values are double precision real numbers (with decimal dot or
  // not).
  QString headerRegExpString = craftSpectrumHeaderRegExpString();

  QString realDataRegExpString("[\\d\\s\\.,]+");

  QString lineRegExpString =
    QString("%1,(%2)").arg(headerRegExpString).arg(realDataRegExpString);

  return lineRegExpString;
}


void
MassSpecDataFileLoaderBrukerXy::cancelOperation()
{
  m_isOperationCancelled = true;

  qDebug() << __FILE__ << __LINE__ << "MzmlFileParser::cancelOperation";
}


int
MassSpecDataFileLoaderBrukerXy::readSpectrumCount()
{
  // The brukerXy file as produced by Bruker CompassXport has no header.
  // It has the following format:
  // "0.0191333,+,ESI,ms1,-,line,500.0000-2000.0000,8882,494.6429 148,....\n"
  // "0.03585,+,ESI,ms1,-,line,500.0000-2000.0000,9025,494.6433
  // 113,494.7604 66....\n"

  // Reinit the spectrum count.
  m_spectrumCount = 0;

  if(m_fileName.isEmpty())
    {
      qDebug() << __FILE__ << __LINE__ << "fileName is empty";

      return -1;
    }

  QFile file(m_fileName);

  if(!file.open(QFile::ReadOnly | QFile::Text))
    {
      qDebug() << __FILE__ << __LINE__ << "Failed to open file" << m_fileName;

      return -1;
    }

  QString headerRegExpString = craftSpectrumHeaderRegExpString();
  QRegularExpression headerRegExp(headerRegExpString);
  QRegularExpressionMatch match;

  QString errorMsg;

  while(!file.atEnd())
    {
      QString line = file.readLine();

      if(line.isEmpty())
        continue;

      match = headerRegExp.match(line);

      if(match.hasMatch())
        ++m_spectrumCount;
      else
        return false;
    }

  file.close();

  return m_spectrumCount;
}


bool
MassSpecDataFileLoaderBrukerXy::canLoadData(QString fileName)
{
  int LIMIT_NUMBER_LINE_TO_READ = 50;

  QFile file(fileName);

  if(!file.open(QFile::ReadOnly | QFile::Text))
    {
      qDebug() << __FILE__ << __LINE__ << "Failed to open file" << fileName;

      return -1;
    }

  QString headerRegExpString = craftSpectrumHeaderRegExpString();
  QRegularExpression headerRegExp(headerRegExpString);
  QRegularExpressionMatch match;

  int count = 0;

  while(!file.atEnd() && count < LIMIT_NUMBER_LINE_TO_READ)
    {
      QString line = file.readLine();

      match = headerRegExp.match(line);

      if(match.hasMatch())
        ++count;
      else
        return false;
    }

  return true;
}


msXpSlibmass::MassSpectrum *
MassSpecDataFileLoaderBrukerXy::parseSpectrum(QString line)
{
  if(line.isEmpty())
    return 0;

  // The line is in the format:
  // "0.0191333,+,ESI,ms1,-,line,500.0000-2000.0000,8882,494.6429 148,....\n"
  //
  // but what we actually get in the 'line' parameter is the last sequence
  // of mz<space>i,mz<space>i,.... with a new line at the end.

  // Remove that new line.
  line.remove("\n");

  msXpSlibmass::MassSpectrum *massSpectrum = new msXpSlibmass::MassSpectrum;

  QStringList mzIstringList = line.split(',');

  // qDebug() << __FILE__ << __LINE__
  //<< "Split line into " << mzIstringList.size() << "mz,i pairs";

  QRegularExpression realDataRegExp("(\\d*\\.?\\d+)\\s+(-?\\d*\\.?\\d*e?\\d*)");
  QRegularExpressionMatch match;

  bool ok = false;

  for(int iter = 0; iter < mzIstringList.size(); ++iter)
    {

      // qDebug() << __FILE__ << __LINE__
      //<< "Current mz,i textual pair:" << mzIstringList.at(iter);

      match = realDataRegExp.match(mzIstringList.at(iter));

      if(match.hasMatch())
        {
          double mz = match.captured(1).toDouble(&ok);

          if(!ok)
            {
              qDebug() << __FILE__ << __LINE__ << "Failed to convert mz"
                       << match.captured(1) << "to double.";

              delete massSpectrum;
              return nullptr;
            }

          double i = match.captured(2).toDouble(&ok);

          if(!ok)
            {
              qDebug() << __FILE__ << __LINE__ << "Failed to convert i"
                       << match.captured(2) << "to double.";

              delete massSpectrum;
              return nullptr;
            }

          // At this point we know that we have proper mz and i values. Let's
          // process these by creating a mass peak that we can append to the
          // spectrum that we allocated at the beginning of this function.

          // qDebug() << __FILE__ << __LINE__
          //<< "Appending MassPeak:" << mz << "-" << i;

          massSpectrum->append(new msXpSlibmass::DataPoint(mz, i));
        }
    }

  // qDebug() << __FILE__ << __LINE__
  //<< "Returning mass spectrum of " << massSpectrum->size() << "points";

  return massSpectrum;
}


int
MassSpecDataFileLoaderBrukerXy::loadData(MassSpecDataSet *massSpecDataSet)
{

  if(massSpecDataSet == nullptr)
    qFatal("Fatal error at %s@%d. Program aborted.", __FILE__, __LINE__);

  QFile file(m_fileName);

  if(!file.open(QFile::ReadOnly | QFile::Text))
    {
      qDebug() << __FILE__ << __LINE__ << "Failed to open file" << m_fileName;

      return -1;
    }

  QString lineRegExpString = craftSpectrumLineRegExpString();
  QRegularExpression lineRegExp(lineRegExpString);
  QRegularExpressionMatch match;

  double rt;
  int numberPoints;
  QString spectralData;
  int spectraRead = 0;

  bool ok = false;

  while(!file.atEnd())
    {
      if(m_isOperationCancelled)
        break;

      QString line = file.readLine();
      // qDebug() << __FILE__ << __LINE__
      //<< "line is: " << line;

      // The parsing of the full line happens in two steps. First validate the
      // whole line and extract the retention time and the number of points in
      // the spectrum. Then parse the spectral data.

      match = lineRegExp.match(line);

      if(!match.hasMatch())
        {

          qDebug() << __FILE__ << __LINE__
                   << "line does not match the spectrum line regexp.";

          continue;
        }

      rt = match.captured(1).toDouble(&ok);
      if(!ok)
        {
          qDebug() << __FILE__ << __LINE__
                   << "Failed to convert retention time:" << match.captured(1)
                   << "to double.";

          return -1;
        }
      // qDebug() << __FILE__ << __LINE__
      //<< "rt:"
      //<< match.captured(1);

      numberPoints = match.captured(8).toInt(&ok);
      if(!ok)
        {
          qDebug() << __FILE__ << __LINE__
                   << "Failed to convert numberPoints:" << match.captured(8)
                   << "to int.";

          return -1;
        }
      // qDebug() << __FILE__ << __LINE__
      //<< "number of points in spectrum:"
      //<< match.captured(8);

      spectralData = match.captured(9);

      // qDebug() << __FILE__ << __LINE__
      //<< "matched the spectral data with string:"
      //<< match.captured(9);

      msXpSlibmass::MassSpectrum *newSpectrum = parseSpectrum(spectralData);

      if(newSpectrum == nullptr)
        qFatal(
          "Fatal error at %s@%d. Failed to parse spectrum. Program "
          "aborted.",
          __FILE__,
          __LINE__);

      if(newSpectrum->size() != numberPoints)
        {
          qDebug() << __FILE__ << __LINE__
                   << "Failed to parse the proper number of points in spectrum"
                   << "for retention time" << rt << "--" << newSpectrum->size()
                   << "is different than" << numberPoints;

          qFatal(
            "Fatal error at %s@%d. Failed to parse spectrum. "
            "Program aborted.",
            __FILE__,
            __LINE__);

          return -1;
        }


      // Do not forget to set that retention time to the spectrum
      // itself !

      newSpectrum->rrt() = rt;

      if(massSpecDataSet->isStreamed())
        {
          // We are reading in streamed mode. This is the chance we
          // have to compute the TIC for the spectrum:

          double newTic = newSpectrum->valSum();

          // Had already another spectrum have the same rt ? If so
          // another TIC was already computed.

          double oldTic = massSpecDataSet->m_ticChromMap.value(rt, qSNaN());

          if(!qIsNaN(oldTic))
            {

              // A rt by the same value alreay existed, update the
              // tic value and insert it, that will erase the old
              // value, because the map is NOT a multimap.

              newTic += oldTic;
            }

          // And now store the key/value pair for later to use to
          // plot the TIC chromatogram.

          massSpecDataSet->m_ticChromMap.insert(rt, newTic);

          // Now delete the mass spectrum, that we won't use anymore.
          delete newSpectrum;
        }
      else
        {
          massSpecDataSet->appendMassSpectrum(newSpectrum);
          massSpecDataSet->insertRtHash(rt, newSpectrum);
        }

      // At this point we can increment the number of spectra that were
      // effectively read.
      ++spectraRead;

      QString msg = QString("Parsed spectrum number %1\n").arg(spectraRead);

      emit updateFeedbackSignal(
        msg, spectraRead, true /* setVisible */, LogType::LOG_TO_BOTH);
    }
  // End of
  // while(!file.atEnd())

  // We may be here because the operation was cancelled. Of course, in that
  // case we have not read all the spectra, but it is not an error, the user
  // knows he has cancelled the operation.

  if(!m_isOperationCancelled)
    {
      if(spectraRead != m_spectrumCount)
        {
          qDebug() << __FILE__ << __LINE__
                   << "The number of parsed spectra does "
                      "not match the previous analysis "
                      "of the file.";


          qFatal("Fatal error at %s@%d. Failed to parse file. Program aborted.",
                 __FILE__,
                 __LINE__);
        }

      // Sanity check:
      if(!massSpecDataSet->isStreamed())
        {
          if(massSpecDataSet->m_massSpectra.size() != spectraRead)
            {
              qDebug() << __FILE__ << __LINE__
                       << "The number of parsed spectra does "
                          "not match the previous analysis "
                          "of the file.";


              qFatal(
                "Fatal error at %s@%d. Failed to parse file. Program "
                "aborted.",
                __FILE__,
                __LINE__);
            }
        }
    }

  file.close();

  return spectraRead;
}


int
MassSpecDataFileLoaderBrukerXy::streamedIntegration(
  const History &history,
  IntegrationType integrationType,
  QVector<double> *keyVector,
  QVector<double> *valVector)
{

  // We need to look into the m_fileName file and extract from the file all
  // the spectra that match the requirements specified by the arguments.

  if(!history.isValid())
    {
      qDebug() << __FILE__ << __LINE__
               << "The History is not valid. Returning.";

      return -1;
    }

  if(integrationType != IntegrationType::ANY_TO_MZ)
    {
      qDebug() << __FILE__ << __LINE__
               << "No valid integration type is "
                  "defined for the data to be "
                  "produced. Returning.";
      return -1;
    }

  // Now we can call a more specialized function:
  if(integrationType == IntegrationType::ANY_TO_MZ)
    return streamedMzIntegration(history, keyVector, valVector);
  else
    qFatal("Fatal error at %s@%d. Program aborted.", __FILE__, __LINE__);

  // Will never be reached
  return -1;
}


int
MassSpecDataFileLoaderBrukerXy::streamedMzIntegration(
  const History &history,
  QVector<double> *keyVector,
  QVector<double> *valVector)
{
  if(keyVector == nullptr || valVector == nullptr)
    qFatal("Fatal error at %s@%d. Program aborted.", __FILE__, __LINE__);

  if(!history.isValid())
    {
      qDebug() << __FILE__ << __LINE__
               << "The History is not valid. Returning.";

      return -1;
    }

  // We are asked to compute a mass spectrum on the basis of what is in
  // the history parameter. Let's craft a SQL query string, perform the
  // query and then compute the TIC chromatogram, of which time is the key
  // and count is the value, returned in the keyVector and valVector
  // parameters.

  // At this point, we need to make sure we have all the rt/dt
  // integrations so that we can take them into account while doing the
  // SELECT operations for the database (m_db).

  // RT integration.
  // ===============
  bool rtIntegration = false;
  double rtStart     = -1;
  double rtEnd       = -1;
  rtIntegration      = history.innermostRtRange(&rtStart, &rtEnd);

  if(!rtIntegration)
    qFatal(
      "Fatal error at %s@%d. Cannot integration over RT with no"
      "integration specifications overs RT. Program aborted.",
      __FILE__,
      __LINE__);

  qDebug() << __FILE__ << __LINE__
           << "The Mz integration is asked between RT range [" << rtStart << "-"
           << rtEnd << "]";

  // We need to iterate in the file and interrogate the various
  //##PAGE= T= 1.158000 lines to extract only the spectra matching the
  // integration rt range.

  // Before going on with the actual results parsing, let's setup the
  // feedback routine. Note the min = 0 and max = 0 values because we cannot
  // determine the count of spectra to load. Simply provide a moving
  // feedback.

  QString msgSkeleton = "Computing streamed m/z integration";

  emit updateFeedbackSignal(
    msgSkeleton, 0, true /*setVisible */, LogType::LOG_TO_BOTH);

  QFile file(m_fileName);

  if(!file.open(QFile::ReadOnly | QFile::Text))
    {
      qDebug() << __FILE__ << __LINE__ << "Failed to open file" << m_fileName;

      return -1;
    }

  // Then, each spectrum has a header like this and then the data:

  //##PAGE= T= 1.158000
  //##NPOINTS= 74528
  //##DATA TABLE= (XY..XY), PEAKS
  // 694.499324,35.000000;694.516080,44.000000;... goes on and on in a single
  // line of mz,i pairs.

  // Craft the proper regular expression for the mass data:

  QRegularExpression endOfLine("^\\s+$");
  QRegularExpression pageTimeLine("^##PAGE= T= (\\d*\\.?\\d*)\\s*$");
  QRegularExpression numberPointsLine("^##NPOINTS= (\\d+)\\s*$");
  QRegularExpression dataTablePeaksLine(
    "^##DATA TABLE= \\(XY..XY\\), PEAKS\\s*$");
  QRegularExpression realDataLine(
    "(\\d*\\.?\\d*)([\\s,]*)(-?\\d*\\.?\\d*e?\\d*);");

  QRegularExpressionMatch match;

  msXpSlibmass::MassSpectrum combinedMassSpectrum;

  int spectraRead  = 0;
  double rt        = 0.0;
  int numberPoints = 0;

  bool dataTablePeaksLineWasLast = false;

  bool ok = false;

  while(!file.atEnd())
    {
      if(m_isOperationCancelled)
        break;

      QString line = file.readLine();
      // qDebug() << __FILE__ << __LINE__
      //<< "line is: " << line;

      if(line.isEmpty())
        {
          dataTablePeaksLineWasLast = false;

          // qDebug() << __FILE__ << __LINE__
          //<< "line is empty.";

          continue;
        }

      // If the line has only a newline character, go on.
      match = endOfLine.match(line);
      if(match.hasMatch())
        {
          dataTablePeaksLineWasLast = false;

          // qDebug() << __FILE__ << __LINE__
          //<< "matched the end of line.";

          continue;
        }

      match = pageTimeLine.match(line);
      if(match.hasMatch())
        {
          dataTablePeaksLineWasLast = false;

          // Get the retention time of the current spectrum.
          rt = match.captured(1).toDouble(&ok);
          if(!ok)
            {
              qDebug() << __FILE__ << __LINE__ << "Failed to convert"
                       << match.captured(1) << "to double";

              return -1;
            }

          // This is the place where we should check that the data are
          // actually correct for the rt integration range.

          if(rt < rtStart)
            continue;
          if(rt > rtEnd)
            break;
        }

      // At this point we know we are in the target retention time interval.
      // Go on with the parsing.

      match = numberPointsLine.match(line);
      if(match.hasMatch())
        {
          dataTablePeaksLineWasLast = false;

          // Get the number of points of the current spectrum.
          numberPoints = match.captured(1).toInt(&ok);
          if(!ok)
            {
              qDebug() << __FILE__ << __LINE__ << "Failed to convert"
                       << match.captured(1) << "to int";

              return -1;
            }

          // qDebug() << __FILE__ << __LINE__ << "numberPoints:" <<
          // numberPoints;
        }

      match = dataTablePeaksLine.match(line);

      if(match.hasMatch())
        {
          dataTablePeaksLineWasLast = true;

          // qDebug() << __FILE__ << __LINE__
          //<< "matched the DATA TABLE PEAKS line.";

          continue;
        }

      if(line.startsWith("##"))
        {
          dataTablePeaksLineWasLast = false;

          continue;
        }

      // If previous line was the data table peaks line, then this one should
      // be the data. Or there is an error.

      // Format of the numerical string:
      // 694.642704,106.000000;694.759649,92.000000;\n

      match = realDataLine.match(line);

      if(!match.hasMatch())
        {
          if(dataTablePeaksLineWasLast)
            {
              qDebug() << __FILE__ << __LINE__ << "File " << m_fileName
                       << "is not a dx file";

              return false;
            }
        }
      else // was: if(!match.hasMatch())
        {
          if(!dataTablePeaksLineWasLast)
            {
              qDebug() << __FILE__ << __LINE__ << "File " << m_fileName
                       << "is not a dx file";

              return false;
            }

          // qDebug() << __FILE__ << __LINE__
          //<< "matched the realData with strings:"
          //<< match.captured(1) << "and" << match.captured(3);

          msXpSlibmass::MassSpectrum *newSpectrum = parseSpectrum(line);

          if(newSpectrum == nullptr)
            qFatal(
              "Fatal error at %s@%d. Failed to parse spectrum. Program "
              "aborted.",
              __FILE__,
              __LINE__);

          if(newSpectrum->size() != numberPoints)
            {
              qDebug()
                << __FILE__ << __LINE__
                << "Failed to parse the proper number of points in spectrum"
                << "for retention time" << rt << "--" << newSpectrum->size()
                << "is different than" << numberPoints;

              qFatal(
                "Fatal error at %s@%d. Failed to parse spectrum. "
                "Program aborted.",
                __FILE__,
                __LINE__);

              return -1;
            }


          // Do not forget to set that retention time to the spectrum
          // itself !

          newSpectrum->rrt() = rt;

          // We are reading in streamed mode. We need to combine each new
          // spectrum to the previous ones.

          // qDebug() << __FILE__ << __LINE__ << "integrating for rt:" << rt;

          combinedMassSpectrum.combine(*newSpectrum);

          // At this point we can increment the number of spectra that were
          // effectively read.
          ++spectraRead;

          QString msg = QString(" -- spectrum number %1\n").arg(spectraRead);

          emit updateFeedbackSignal(msgSkeleton + msg,
                                    -1,
                                    true /* setVisible */,
                                    LogType::LOG_TO_STATUS_BAR);

          // qDebug() << __FILE__ << __LINE__ << msg;
        }
      // End of
      // else // was: if(!match.hasMatch())
    }
  // End of
  // while(!file.atEnd())

  *keyVector = QVector<double>::fromList(combinedMassSpectrum.keyList());
  *valVector = QVector<double>::fromList(combinedMassSpectrum.valList());

  file.close();

  // qDebug() << __FILE__ << __LINE__ << "Mz streamed integration has parsed "
  //<< spectraRead << "spectra.";

  return spectraRead;
}

} // namespace msXpSmineXpert
