/* 
 * 3DapReader.cpp - File parser for 3DAP systems 
 * Copyright (C) 2018, D Haley
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */


#include "3DapReader.h"
#include <vector>
#include <iostream>
#include <cstdlib>
#include <fstream>

#include <gsl/gsl_matrix.h>
#include <gsl/gsl_fit.h>
#include <gsl/gsl_multifit.h>



#include "common/basics.h"

using std::vector;
using std::endl;
using std::string;


//TODO: We may have to deal with 2 or 3 delay line configurations	
//3Dap uses hexanode detector??
const unsigned int NUM_XY=2;
		

//Charge on an electron (coulombs -- aka A.s)
const float CHARGE_ELECTRON= 1.602176e-19;
//Atomic mass units per kilo
const float AMU_PER_KG =  6.0221409e26;

const char *OPS_ENUM_ERRSTRINGS[] =
{
	"",
       	"\"C\" line (exp. parameters) not in expected format",
	"\"I\"	line (Detector parameters) not in expected format",
	"\"-\" line (Time-of-flights) not in expected format",
	"\"V\" line (voltage data) not in expected format",
	"Error interpreting \"V\" line (voltage data) data",
	"Missing beta value from \"V\" line - needs to be in voltage, or in system header (\"C\" line",
	"Incorrect number of \"V\" line (voltage data) data entries",
	"Unknown linetype in data file",
	"\"P\" line (detector channels) not in expected format",
	"Unable to interpret channels line",
	"Incorrect number of events in \"S\" (hit position) line",
	"Unable to interpret event data on \"S\" (hit position) line",
	"Unable to parse \"S\" (hit position) line event count (eg 8 in S8 ...)",
	"\"S\" (hit position) line data not preceded by TOF data, as it should have been",
	"Duplicate system data/experiment setup (\"C\") entry found -- there can only be one",
	"Duplicate detector (\"I\") data entry found -- there can only be one",
	"Trailing \"-\" line found -- should have be followed by a \"S\" line, but wasn't.",
	"Duplicate\"-\" line found -- should have be followed by a \"S\" line, but wasn't.",
	"Unable to open file",
	"unable to read file, after opening",
	"Abort requested"
};
//Single 3DAP hit event
struct SINGLE_HIT
{
	float x,y;
	float tof;
	float massToCharge;
};

//Voltage data structure -- these updates occur during the experiment
struct VOLTAGE_DATA
{
	size_t nextHitGroupOffset;
	float voltage;
	float pulseVolt;
	float beta; //For some reason, OPS files seem to have duplicate beta data
};

//experimental setup data
struct THREEDAP_DATA
{
	float flightPath; //in mm
	float alpha; //Pulse coupling coefficient
	float beta; 
	float tZero; // in ns
	float detectorRadius;// in mm	
	unsigned int detectorChannels;
};

//Hits obtained on same pulse
struct THREEDAP_EXPERIMENT
{
	THREEDAP_DATA params;	
	vector<vector<SINGLE_HIT> > eventData;
	vector<VOLTAGE_DATA> voltageData;
	vector<unsigned long long> eventPulseNumber;
};

float evaluate_polyBase(float x, float y, 
	unsigned int baseNum, unsigned int order);

//Polynomial fit via GSL
unsigned int polyFitData(const vector<float> &obsX, const vector<float> &obsY,
	       		const vector<float> &obsZ, vector<float> &coeff, unsigned int polyOrder);

//Function to read POSAP "OPS" files, which contain data
//on voltage, positioned hits and time of flights
//as well as some instrument parameters
unsigned int readOps(const char *file, 
		THREEDAP_EXPERIMENT &data, 
			unsigned int &badLine, unsigned int &progress, 
			ATOMIC_BOOL &wantAbort, bool strictMode=false)
{

	COMPILE_ASSERT(THREEDEP_ARRAYSIZE(OPS_ENUM_ERRSTRINGS) == OPSREADER_ENUM_END);

	const char COMMENT_MARKER='*';

	badLine=0;

	size_t totalFilesize;
	if(!getFilesize(file,totalFilesize))
		return OPSREADER_OPEN_ERR;

	std::ifstream f(file);

	if(!f)
		return OPSREADER_OPEN_ERR;


	enum
	{
		OPS_LINE_DASH,
		OPS_LINE_OTHER_OR_NONE
	};

	bool haveSystemParams=false;
	bool haveDetectorSize=false;

	unsigned int lastLine=OPS_LINE_OTHER_OR_NONE;	

	vector<string> strVec;
	while(!f.eof())
	{
		std::string strLine;
		getline(f,strLine);
		badLine++;


		if(f.eof())
			break;

		if(!f.good())
			return OPSREADER_READ_ERR;

		//If user wants to abort, please do so
		if(wantAbort)
			return OPSREADER_ABORT_ERR;

		//Remove any ws delimiters
		strLine=stripWhite(strLine);

		//Is a blank or comment? Don't process line
		if(!strLine.size() || strLine[0] == COMMENT_MARKER)
			continue;

		//Check to see what type of line this is
		strVec.clear();

		splitStrsRef(strLine.c_str(),"\t ",strVec);
		stripZeroEntries(strVec);
		switch(strLine[0])
		{
			//"C" line contains the system parameters
			// flight path, pulse coupling coefficients (alpha, beta -- see Sebastien et al) and time zero offset.
			// "New method for the calibration of three-dimensional atom-probe mass"
			// Review of Scientific Instruments, 2001
			case 'C':
			{
				if(haveSystemParams)
					return OPSREADER_FORMAT_DUPLICATE_SYSDATA;

				haveSystemParams = true;

				//Should have something eg
				// C 123 1.01 0.8 -123 
				if(strVec.size() != 5)
					return OPSREADER_FORMAT_CLINE_ERR;

				if(stream_cast(	data.params.flightPath,strVec[1]))
					return OPSREADER_FORMAT_CLINE_ERR;

				if(stream_cast(	data.params.alpha,strVec[2]))
					return OPSREADER_FORMAT_CLINE_ERR;
				
				if(stream_cast(	data.params.beta,strVec[3]))
					return OPSREADER_FORMAT_CLINE_ERR;

				if(stream_cast(	data.params.tZero,strVec[4]))
					return OPSREADER_FORMAT_CLINE_ERR;

				lastLine=OPS_LINE_OTHER_OR_NONE;
				break;
			}
			//"I" Line contains detector radius
			case 'I':
			{
				if(haveDetectorSize)
					return OPSREADER_FORMAT_DUPLICATE_DETECTORSIZE;

				haveDetectorSize=true;

				//Should have I or IR
				//I - No reflectron
				//IR - reflectron
				//and a floating pt number
				if(strVec.size() != 2)
					return OPSREADER_FORMAT_DETECTORLINE_ERR;

				if(stream_cast(data.params.detectorRadius,strVec[1]))
					return OPSREADER_FORMAT_DETECTORLINE_ERR;
				lastLine=OPS_LINE_OTHER_OR_NONE;

				break;
			}
			//"S" line contains the hit TOFs, and hit frequency
			case 'S':
			{
				//OK, so this is the actual data -- each "S"" line should be
				//preceded by a "-" line.
				if(lastLine !=OPS_LINE_DASH)
				{
					if(strictMode)
						return OPSREADER_FORMAT_SLINE_PREFIX_ERR;
					else
					{
						//FIXME: better warning mechanism
						//cerr << "WARNING : S line not preceded by \"-\" line; skipping" << endl;
						continue; //oK, this is kinda a confusing situation, lets just get to the next line, which we hope is an S.
					}
				}



				//If we have less positions than TOFs, then something is wrong 
				//-- we have to discard the previous "-" line
				// could have a wierd line that makes no sense (for eg, I found this):
				//
				//-151 13
				//S0
				// I assume that this means that 
				if((strVec.size()-1) /(NUM_XY+1)< data.eventData.back().size())
				{
					ASSERT(data.eventData.size())
					//We have to discard the previous hit sequence
					data.eventData.pop_back();
					lastLine=OPS_LINE_OTHER_OR_NONE;
					break;
				}


				//OK, so the event numbers are coded like
				//"S8", where S indicates it is an event
				//line, and the next numbers indicate the
				//number of observed events
				strVec[0]= strVec[0].substr(1);

				unsigned int numEvents;
				if(stream_cast(numEvents,strVec[0]))
				{
					if(strictMode)
						return OPSREADER_FORMAT_SLINE_FORMAT_ERR;
					else
					{
						//FIXME : better warn mechanism
						//cerr << "Warning : skipping S-line  at file line:" << badLine << " unable to interpret event count" << endl;
						data.eventData.pop_back();
						lastLine=OPS_LINE_OTHER_OR_NONE;
						break;
					}
				}


				if(numEvents != (strVec.size()-1)/(NUM_XY+1))
					return OPSREADER_FORMAT_SLINE_EVENTCOUNT_ERR;

				
				vector<SINGLE_HIT> &curHits=data.eventData.back();
			
				//The TOF must be preset by a "-" line, as checked above.
				//The contents of this "S" line must have at least enough elements
				//to cover the TOFs. The other hits are assigned a zero TOF
				if(curHits.size()> numEvents)
				{
					if(strictMode)
						return OPSREADER_FORMAT_SLINE_EVENTCOUNT_ERR;
					else
					{
						//FIXME: Better warning mechanism
						//cerr << "Warning : skipping S-line  at file line:" << badLine << " unable to interpret event count" << endl;
						data.eventData.pop_back();
						lastLine=OPS_LINE_OTHER_OR_NONE;
						break;
					}
				}


				SINGLE_HIT h;
				h.tof=0;
				curHits.resize(numEvents,h);
				vector<float> tofs;

				//Retrieve the TOF values, so we can redirect them
				//as the TOF values above's correspondence map
				//is stored at the end of the hit sequence, one 
				//entry per hit
				//zero means untimed, otherwise TOF
				tofs.resize(curHits.size());
				for(unsigned int ui=0;ui<curHits.size();ui++)
					tofs[ui]=curHits[ui].tof;


				//Read up until the TOF timing map
				bool everythingOK=true;
				unsigned int timingIndex=curHits.size()*2+1;
				for(unsigned int ui=1;ui<timingIndex;ui+=2)
				{
					//Push the current hit into a vector 
					unsigned int offset;
					offset=ui/2;

					if(stream_cast(curHits[offset].x,strVec[ui]))
					{
						if(strictMode)
							return OPSREADER_FORMAT_SLINE_EVENTDATA_ERR;
						else
						{
							everythingOK=false;
							break;
						}

					}
					
					if(stream_cast(curHits[offset].y,strVec[ui+1]))
					{
						if(strictMode)
							return OPSREADER_FORMAT_SLINE_EVENTDATA_ERR;
						else
						{
							everythingOK=false;
							break;
						}
					}
				}

				if(!everythingOK)
				{
					//FIXME: Better warning mechanism
					//cerr << "WARNING : problem interpeting s line at file line :" << badLine << "Skipping." << endl;
					data.eventData.pop_back();
					lastLine=OPS_LINE_OTHER_OR_NONE;
					break;
				}	

				vector<unsigned int> timeMap;
				//Read the TOF timing map
				for(unsigned int ui=timingIndex; ui<strVec.size();ui++)
				{
					unsigned int mapEntry;
					if(stream_cast(mapEntry,strVec[ui]))
					{
						if(strictMode)
							return OPSREADER_FORMAT_SLINE_EVENTDATA_ERR;
						else
						{
							everythingOK=false;
							break;

						}
					}

					timeMap.push_back(mapEntry);
				}

				if(!everythingOK)
				{
					//FIXME: Better warn method
					//cerr << "WARNING : problem interpreting timing data in s line at file line :" << badLine << "Skipping." << endl;
					data.eventData.pop_back();
					lastLine=OPS_LINE_OTHER_OR_NONE;
					break;
				}	
				ASSERT(timeMap.size() == curHits.size());

				//use the TOF timing map to assign TOF values
				//to observed TOF entries from the "-" line
				for(unsigned int ui=0;ui<curHits.size();ui++)
				{
					if(timeMap[ui])
						curHits[ui].tof=tofs[timeMap[ui]-1];
					else
						curHits[ui].tof=0;
				}
				
				lastLine=OPS_LINE_OTHER_OR_NONE;
				break;
			}
			//"-" line contains the detector hit positions
			case '-':
			{
				if(lastLine==OPS_LINE_DASH)
					return OPSREADER_FORMAT_DOUBLEDASH;

				if(strVec.size() < 2)
					return OPSREADER_FORMAT_LINE_DASH_ERR;


				size_t pulseDelta;
				if(stream_cast(pulseDelta,strVec[0].substr(1)))
					return OPSREADER_FORMAT_LINE_DASH_ERR;

				//First bit of string is the number of pulses
				//preceding this event
				if(data.eventPulseNumber.empty())
					data.eventPulseNumber.push_back(pulseDelta+1);
				else
				{
					data.eventPulseNumber.push_back(
						data.eventPulseNumber.back()+pulseDelta+1);
				}

				vector<float> tofs;
				for(unsigned int ui=1; ui< strVec.size(); ui++)
				{
					float timeOfFlight;
					if(stream_cast(timeOfFlight,strVec[ui]))
						return OPSREADER_FORMAT_LINE_DASH_ERR;

					tofs.push_back(timeOfFlight);
				}

				vector<SINGLE_HIT> s;

				s.resize(tofs.size());
				for(unsigned int ui=0;ui<tofs.size();ui++)
					s[ui].tof =tofs[ui]; 
				data.eventData.push_back(s);

				lastLine=OPS_LINE_DASH;
				break;
			}
			//\"V\" voltage data
			case 'V':
			{

				//V line may have
				if(! (strVec.size() == 4 || strVec.size() == 3) )
					return OPSREADER_FORMAT_LINE_VOLTAGE_DATACOUNT_ERR;

				VOLTAGE_DATA vDat;
				if(stream_cast(vDat.voltage,strVec[1]))
					return OPSREADER_FORMAT_LINE_VOLTAGE_DATA_ERR;

				float pulseV;
				if(stream_cast(pulseV,strVec[2]))
					return OPSREADER_FORMAT_LINE_VOLTAGE_DATA_ERR;
				vDat.pulseVolt=pulseV;

				if(strVec.size() ==4)
				{
					if(stream_cast(vDat.beta,strVec[3]))
						return OPSREADER_FORMAT_LINE_VOLTAGE_DATA_ERR;
				}
				else // 3 items
				{
					//If we have not seen the system parameter's beta, then we cannot compute a good effective voltage
					if(!haveSystemParams)
						return OPSREADER_FORMAT_LINE_VOLTAGE_NOBETA;
					vDat.beta = data.params.beta;
				}

				vDat.nextHitGroupOffset=data.eventData.size();
				data.voltageData.push_back(vDat);
				lastLine=OPS_LINE_OTHER_OR_NONE;
				break;
			}
			case 'P':
			{
				if(strVec.size() != 2)
					return OPSREADER_FORMAT_CHANNELS_ERR;
				
				if(stream_cast(data.params.detectorChannels,strVec[1]))
					return OPSREADER_CHANNELS_DATA_ERR;
				break;
			}
			case 'T':
			{
				//This is a timing,  temperature and pressure line.
				// ignore.
				break;
			}
			default:
				return OPSREADER_FORMAT_LINETYPE_ERR;
		}

		progress=(float)f.tellg()/(float)totalFilesize*100.0f;
	}

	if(lastLine==OPS_LINE_DASH)
	{
		//Damn, we have a problem. We saw a "-" line,
		//but failed to find the next "S" line, which should have been there.
		//Something is wrong.
		if(strictMode)
			return OPSREADER_FORMAT_TRAILING_DASH_ERR;
		
		data.eventData.pop_back();
		
	}

	return 0;
}

void computeMToC(THREEDAP_EXPERIMENT &experiment)
{

	const VOLTAGE_DATA *v,*vNext;
	v=vNext=nullptr;

	unsigned int voltageOffset=0;
	for(size_t ui=0;ui<experiment.eventData.size();ui++)
	{

		vector<float> curMTocs;

		curMTocs.clear();
		//Ensure that we are pointing to the *previous* voltage
		// that we have seen
		if (!v || (vNext && ui >=vNext->nextHitGroupOffset+1))
		{
			ASSERT(voltageOffset <=experiment.voltageData.size());
			v=&(experiment.voltageData[voltageOffset]);
			if(voltageOffset <experiment.voltageData.size() )
			{
				vNext=&(experiment.voltageData[voltageOffset+1]);
				voltageOffset++;
			}
			else
				vNext=nullptr;
		}

		for(size_t uj=0;uj<experiment.eventData[ui].size();uj++)
		{

			//if the corresponding TOF value is zero, then it is not a valid hit
			if(experiment.eventData[ui][uj].tof 
					< std::numeric_limits<float>::epsilon())
				continue;
			
			//effective voltage
			float rat;
			//Inverse Velocity
			rat=(experiment.eventData[ui][uj].tof - experiment.params.tZero)/(experiment.params.flightPath);

			//Corrected voltage (kV)
			//FIXME: Where are these magic numbers coming from?
			float tmpVoltage;
			tmpVoltage=0.983*(v->voltage+105);
			float tmpPulse;

			tmpPulse=1.07*(v->pulseVolt-85);
			tmpVoltage=(tmpVoltage+tmpPulse*v->beta)*experiment.params.alpha/1.0e3;

			//Compute m/c information
			experiment.eventData[ui][uj].massToCharge=0.1929796*tmpVoltage*rat*rat;
		}

	}
}

//Reconstruct onto an imaginary cylinder
// - this may not be ideal as 3dap systems use unmasked, square detectors.
void cylinderReconstruct(const THREEDAP_EXPERIMENT &experiment,
				float ionVolume, float scaleRadius,
				std::vector<IonHit> &ions)
{

	//Detector radius in
	const float dz = ionVolume/(scaleRadius*scaleRadius);

	ASSERT(ions.empty());

	float zOff=0;
	ions.reserve(experiment.eventData.size());
	for(unsigned int ui=0;ui<experiment.eventData.size();ui++)
	{
		for(unsigned int uj=0;uj<experiment.eventData[ui].size();uj++)
		{
			//Check that the tof that corresponds to this position
			//is valid
		
			//Convert to fractional	
			float unitVx=experiment.eventData[ui][uj].x;
			float unitVy=experiment.eventData[ui][uj].y;
			unitVx/=(float)experiment.params.detectorChannels;
			unitVy/=(float)experiment.params.detectorChannels;

			unitVx-=0.5;
			unitVy-=0.5;

			IonHit h;
			h.setPos(Point3D(unitVx*scaleRadius,
					 unitVy*scaleRadius,zOff));

			
			h.setMassToCharge(experiment.eventData[ui][uj].massToCharge);

			zOff+=dz;
			ions.push_back(h);
		}
	}
}


unsigned int reconstruct3DapData(const char *file, const THREEDAP_RECON_PARAMS &params,unsigned int &progress, ATOMIC_BOOL &wantAbort, std::vector<IonHit> &ionData)
{
	unsigned int errCode;
	unsigned int badLine;
	THREEDAP_EXPERIMENT experimentData;
	errCode=readOps(file,experimentData,badLine,progress,wantAbort);
	if(errCode)
		return errCode;

	computeMToC(experimentData); 

	//TODO: Would be sensible to range the data first, then
	// perform reconstruction later. This however requires a more
	// complex UI	
	//RangeData(...)

	cylinderReconstruct(experimentData,params.ionVolume/params.detectionEfficiency,params.radiusCylinder, ionData); 
	
	
	return 0;	
}

