/*
** (c) 1996-2000 The Regents of the University of California (through
** E.O. Lawrence Berkeley National Laboratory), subject to approval by
** the U.S. Department of Energy.  Your use of this software is under
** license -- the license agreement is attached and included in the
** directory as license.txt or you may contact Berkeley Lab's Technology
** Transfer Department at TTD@lbl.gov.  NOTICE OF U.S. GOVERNMENT RIGHTS.
** The Software was developed under funding from the U.S. Government
** which consequently retains certain rights as follows: the
** U.S. Government has been granted for itself and others acting on its
** behalf a paid-up, nonexclusive, irrevocable, worldwide license in the
** Software to reproduce, prepare derivative works, and perform publicly
** and display publicly.  Beginning five (5) years after the date
** permission to assert copyright is obtained from the U.S. Department of
** Energy, and subject to any subsequent five (5) year renewals, the
** U.S. Government is granted for itself and others acting on its behalf
** a paid-up, nonexclusive, irrevocable, worldwide license in the
** Software to reproduce, prepare derivative works, distribute copies to
** the public, perform publicly and display publicly, and to permit
** others to do so.
*/

//
// $Id: AmrDeriveStochTD.cpp,v 1.27 2002/08/16 18:59:11 lijewski Exp $
//
// This is the time-dependent version that reads input from PlotFiles.
//
#include <winstd.H>

#include <algorithm>
#include <new>
#include <iomanip>
#include <iostream>
#include <fstream>
#include <cmath>
#include <cstdlib>
#include <cstring>
#include <cstdio>
#include <vector>

#ifndef WIN32
#include <unistd.h>
#endif

#include "REAL.H"
#include "Box.H"
#include "FArrayBox.H"
#include "ParmParse.H"
#include "ParallelDescriptor.H"
#include "DataServices.H"
#include "Utility.H"
#include "VisMF.H"
#include "Derived.H"
#include "Tuple.H"
#include "Profiler.H"
#include "Thread.H"
#include "WorkQueue.H"
#include "ChemKinDriver.H"
#include "Geometry.H"
#include "xtra_F.H"
//
// This MUST be defined if don't have pubsetbuf() in I/O Streams Library.
//
#ifdef BL_USE_SETBUF
#define pubsetbuf setbuf
#endif
//
// ParmParse'd arguments ...
//
static bool build_prob_dist      = false;
static bool calc_lambda_max      = false;
static bool do_not_write_trajs   = false;
static bool do_not_write_events  = false;

static bool write_intermediate_plotfiles = false;

static int seed = 12345;
static int init_spec;
static int n_particles;

D_TERM(static Real init_x = .0000125;,
       static Real init_y = .0000125;,
       static Real init_z = .0000125;);

static Real dt           = .00001;
static Real dt_spec      = -1;
static Real stop_time    = 0;
static Real strt_time    = 0;

static Real conc_epsilon = 1.0e-10;
static Real chem_limiter = 0.1;

static std::string TrajFile;
static std::string EventFile;
static std::string EdgeDataFile;

static std::string chemInFile;
static std::string chemOutFile;
static std::string thermInFile;
static std::string tranInFile;

static std::string TraceElement;

static std::string InitialSpecies;

static Array<Real>        TimeSteps;
static Array<std::string> PlotFiles;
//
// Stuff we set from the AmrData object after reading PlotFile.
//
static int                  is_rz;
static int                  finest_level;
static Array<int>           ref_ratio;
static Array<Real>          prob_lo;
static Array<Real>          prob_hi;
static Array<Box>           prob_domain;
static Array<Geometry>      geom;
static Array< Array<Real> > dx;
static Array< Array<Real> > dxinv;

D_TERM(static PArray<MultiFab> A_mf(PArrayManage);,
       static PArray<MultiFab> C_mf(PArrayManage);,
       static PArray<MultiFab> E_mf(PArrayManage););

D_TERM(static PArray<MultiFab> B_mf(PArrayManage);,
       static PArray<MultiFab> D_mf(PArrayManage);,
       static PArray<MultiFab> F_mf(PArrayManage););

static PArray<MultiFab> U_mf(PArrayManage);
static PArray<MultiFab> Rf_mf(PArrayManage);
static PArray<MultiFab> Rr_mf(PArrayManage);
static PArray<MultiFab> Conc_mf(PArrayManage);
static PArray<MultiFab> Prob_mf(PArrayManage);
//
// Other global data.
//
static Array<int>           pedges;
static std::vector<int>     edgedata;
static Array< Array<Real> > AccumProb;
//
// Particles on our trajectory (includes time) ...
//
typedef Tuple<Real,BL_SPACEDIM+1> PartPos;
//
// Position in space.
//
typedef Tuple<Real,BL_SPACEDIM> X;

static
void
ScanArguments ()
{
    ParmParse pp;

    pp.query("dt", dt);

    BL_ASSERT(dt != 0);

    pp.query("dt_spec", dt_spec);

    pp.query("strt_time", strt_time);
    pp.query("stop_time", stop_time);

    BL_ASSERT(stop_time > 0);
    BL_ASSERT(strt_time >= 0);
    BL_ASSERT(stop_time > strt_time);

    BL_ASSERT(dt_spec != 0);

    pp.query("seed", seed);

    pp.query("edgedatafile", EdgeDataFile);
    if (EdgeDataFile.empty())
        BoxLib::Abort("You must specify `edgedatafile'");

    pp.query("event_file", EventFile);
    if (EventFile.empty())
        BoxLib::Abort("You must specify `event_file'");

    pp.query("traj_file", TrajFile);
    if (TrajFile.empty())
        BoxLib::Abort("You must specify `traj_file'");

    if (pp.contains("plot_files"))
    {
        PlotFiles.resize(pp.countval("plot_files"));

        for (int i = 0; i < PlotFiles.size(); i++)
            pp.get("plot_files", PlotFiles[i], i);
        //
        // Calculate TimeSteps.
        //
        TimeSteps.resize(PlotFiles.size());

        for (int i = 0; i < TimeSteps.size()-1; i++)
        {
            DataServices::SetBatchMode();
            FileType fileType(NEWPLT);
            DataServices dataServices(PlotFiles[i+1], fileType);

            if (!dataServices.AmrDataOk())
                //
                // This calls ParallelDescriptor::EndParallel() and exit()
                //
                DataServices::Dispatch(DataServices::ExitRequest, NULL);
    
            TimeSteps[i] = dataServices.AmrDataRef().Time();
        }

        TimeSteps[TimeSteps.size()-1] = stop_time;

        for (int i = TimeSteps.size()-1; i > 0; i--)
            BL_ASSERT(TimeSteps[i-1] < TimeSteps[i]);
    }

    BL_ASSERT(PlotFiles.size() > 0);
    BL_ASSERT(TimeSteps.size() > 0);

    BL_ASSERT(strt_time < TimeSteps[0]);

    BL_ASSERT(TimeSteps.size() == PlotFiles.size());

    pp.query("chem_infile", chemInFile);
    if (chemInFile.empty())
        BoxLib::Abort("You must specify `chem_infile'");

    pp.query("chem_outfile", chemOutFile);
    if (chemOutFile.empty())
        BoxLib::Abort("You must specify `chem_outfile'");

    pp.query("therm_infile", thermInFile);
    if (thermInFile.empty())
        BoxLib::Abort("You must specify `therm_infile'");

    pp.query("tran_infile", tranInFile);
    if (tranInFile.empty())
        BoxLib::Abort("You must specify `tran_infile'");

    pp.query("trace_element", TraceElement);
    if (TraceElement.empty())
        BoxLib::Abort("You must specify `trace_element'");
    //
    // Append `.seed=N' to EventFile & TrajFile
    //
    char buf[12];

    TrajFile += ".seed="; EventFile += ".seed=";

    sprintf(buf, "%d", seed);

    TrajFile += buf; EventFile += buf;

    pp.query("calc_lambda_max", calc_lambda_max);

    pp.query("do_not_write_events", do_not_write_events);

    pp.query("do_not_write_trajs", do_not_write_trajs);

    pp.query("write_intermediate_plotfiles", write_intermediate_plotfiles);

    pp.query("conc_epsilon", conc_epsilon);

    pp.query("chem_limiter", chem_limiter);

    BL_ASSERT(conc_epsilon > 0);
    BL_ASSERT(chem_limiter > 0);

    D_TERM(pp.query("init_x", init_x);,
           pp.query("init_y", init_y);,
           pp.query("init_z", init_z););

    pp.query("initial_species", InitialSpecies);

    pp.query("n_particles", n_particles);

    BL_ASSERT(n_particles > 0);

    pp.query("build_prob_dist", build_prob_dist);

    if (ParallelDescriptor::IOProcessor())
    {
        std::cout << "dt = " << dt << '\n';

        std::cout << "dt_spec = " << dt_spec << '\n';

        std::cout << "strt_time = " << strt_time << '\n';

        std::cout << "stop_time = " << stop_time << '\n';

        std::cout << "seed = " << seed << '\n';

        std::cout << "edgedatafile = " << EdgeDataFile << '\n';

        std::cout << "plot_files =";
        for (int i = 0; i < PlotFiles.size(); i++)
            std::cout << ' ' << PlotFiles[i];
        std::cout << std::endl;

        std::cout << "time_steps =";
        for (int i = 0; i < TimeSteps.size(); i++)
            std::cout << ' ' << TimeSteps[i];
        std::cout << std::endl;

        std::cout << "traj_file = "  << TrajFile  << '\n';
        std::cout << "event_file = " << EventFile << '\n';

        std::cout << "chem_infile = "  << chemInFile  << '\n';
        std::cout << "chem_outfile = " << chemOutFile << '\n';
        std::cout << "therm_infile = " << thermInFile << '\n';
        std::cout << "tran_infile = "  << tranInFile  << '\n';

        std::cout << "trace_element = "  << TraceElement  << '\n';

        if (calc_lambda_max)
            std::cout << "calc_lambda_max = true" << '\n';

        if (do_not_write_events)
            std::cout << "do_not_write_events = true" << '\n';

        if (do_not_write_trajs)
            std::cout << "do_not_write_trajs = true" << '\n';

        if (write_intermediate_plotfiles)
            std::cout << "write_intermediate_plotfiles = true" << '\n';

        std::cout << "Using conc_epsilon = " << conc_epsilon << '\n';

        std::cout << "Using chem_limiter = " << chem_limiter << '\n';

        D_TERM(std::cout << "init_x = " << init_x << '\n';,
               std::cout << "init_y = " << init_y << '\n';,
               std::cout << "init_z = " << init_z << '\n';);

        std::cout << "initial_species = " << InitialSpecies << '\n';

        std::cout << "n_particles = " << n_particles << '\n';

        std::cout << "build_prob_dist = " << build_prob_dist << '\n';
    }
}

static
void
PrintUsage (char* progName)
{
    if (ParallelDescriptor::IOProcessor())
        std::cout << "Usage: " << progName << " inputfile" << '\n';
    exit(1);
}

static
Real
FFF (Real s)
{
    const Real R1 = .007;
    const Real R2 = .014;

    const Real A = (R2*R2-R1*R1)/(4*std::log(R1/R2));
    const Real B = (R1*R1*log(R2)-R2*R2*log(R1))/(4*log(R1/R2));

    Real result = 8 * A * std::log(s);

    s *= s;

    result += (s + 8*B -4*A);

    result *= s/16;

    return result;
}

static Real LookupTable[1001];

static
void
BuildLookupTable ()
{
    const Real R1    = .007;
    const Real R2    = .014;
    const Real Dx    = .001*(R2-R1);
    const Real FFFR1 = FFF(R1);
    const Real FFFR2 = FFF(R2);

    for (int i = 0; i < 1001; i++)
    {
        Real s = R1+i*Dx;

        LookupTable[i] = (FFF(s)-FFFR1)/(FFFR2-FFFR1);
    }
}

static
Real
RandomXPos (BoxLib::mt19937& rand)
{
    const Real Rn  = rand.d1_value();
    const Real Rf = .006;

#if 1
    //
    // Carbon stuff.
    //
    return Rf * sqrt(1-sqrt(1-Rn));
#else
    //
    // Nitrogen stuff.
    //
    const Real Ratio1 =  220.0/3380.0;
    const Real Ratio2 = 3160.0/3380.0;
    const Real R1     = .007;
    const Real R2     = .014;
    const Real Dx     = .001*(R2-R1);

    if (Rn < Ratio1)
    {
        return Rf * sqrt(1-sqrt(1-Rn/Ratio1));
    }
    else
    {
        Real x = (Rn-Ratio1)/Ratio2;

        int i = 0;

        for ( ; i < 1000; i++)
            if (x < LookupTable[i])
                break;
        
        return R1+(i-1+(x-LookupTable[i-1])/(LookupTable[i]-LookupTable[i-1]))*Dx;
    }
#endif
}

static
void
ReadEdgeData ()
{
    BL_ASSERT(!EdgeDataFile.empty());

    if (ParallelDescriptor::IOProcessor())
        std::cout << "Reading edgefile ... " << std::endl;

    std::ifstream ifs;

    ifs.open(EdgeDataFile.c_str(), std::ios::in);

    if (!ifs.good()) BoxLib::FileOpenFailed(EdgeDataFile);

    int N, cnt, tmp;

    ifs >> N;

    BL_ASSERT(N > 0);

    pedges.resize(N);

    if (ParallelDescriptor::IOProcessor())
        std::cout << "N = " << N << '\n';

    for (int i = 0; i < N; i++)
    {
        ifs >> cnt;

        BL_ASSERT(cnt > 0);

        pedges[i] = edgedata.size();

        edgedata.push_back(cnt);

        for (int j = 0; j < 4*cnt; j++)
        {
            ifs >> tmp;

            edgedata.push_back(tmp);
        }
    }

    if (!ifs.good()) BoxLib::Abort("Failure reading EdgeData file");

    if (ParallelDescriptor::IOProcessor())
    {
        std::cout << "EdgeData on startup:\n";

        for (int i = 0; i < pedges.size(); i++)
        {
            int cnt = edgedata[pedges[i]];

            std::cout << cnt << ' ';

            for (int j = 1; j <= 4*cnt; j++)
                std::cout << edgedata[pedges[i]+j] << ' ';

            std::cout << '\n';
        }
    }
}

static
void
TwiddleConc (PArray<MultiFab>& conc)
{
    std::cout << "Twiddling concentration ... " << std::endl;
    //
    // First normalize it.
    //
    for (int ispec = 0; ispec < conc[0].nComp(); ispec++)
    {
        Real c_floor = 0;

        for (int lev = 0; lev < conc.size(); lev++)
        {
            for (MFIter mfi(conc[lev]); mfi.isValid(); ++mfi)
            {
                const Box& vb = mfi.validbox();

                c_floor = std::max(c_floor,conc[lev][mfi].max(vb,ispec)*conc_epsilon);
            }
        }

        if (ParallelDescriptor::IOProcessor())
        {
            std::cout << "c_floor(" << ispec << "): " << c_floor << std::endl;
        }

        for (int lev = 0; lev < conc.size(); lev++)
        {
            for (MFIter mfi(conc[lev]); mfi.isValid(); ++mfi)
            {
                const long N = conc[lev][mfi].box().numPts();

                Real* data = conc[lev][mfi].dataPtr(ispec);

                for (long i = 0; i < N; i++)
                {
                    data[i] = std::max(data[i],c_floor);
                }       
            }
        }
    }

    std::cout << "Done twiddling concentration. " << std::endl;
}

static
void
MassageConc ()
{
    std::cout << "Massaging concentration ... " << std::endl;
    //
    // We've already been Twiddle()d.
    // We invert it so we can multiply by it instead of divide.
    //
    for (int lev = 0; lev < Conc_mf.size(); lev++)
    {
        for (MFIter mfi(Conc_mf[lev]); mfi.isValid(); ++mfi)
        {
            Conc_mf[lev][mfi].invert(1.0);
        }
    }
    //
    // We now set those cells covered by fine grid to zero.
    // This simplified some of our algorithms.
    //
    const int Ncomp = Conc_mf[0].nComp();

    for (int l = 0; l <= finest_level; l++)
    {
        for (MFIter mfi(Conc_mf[l]); mfi.isValid(); ++mfi)
        {
            if (l < finest_level)
            {
                const BoxArray& f_ba = Conc_mf[l+1].boxArray();

                for (int j = 0; j < f_ba.size(); j++)
                {
                    Box c_box = BoxLib::coarsen(f_ba[j],ref_ratio[l]);

                    if (c_box.intersects(Conc_mf[l][mfi].box()))
                    {
                        Box isect = c_box & Conc_mf[l][mfi].box();

                        Conc_mf[l][mfi].setVal(0,isect,0,Ncomp);
                    }
                }
            }
        }
    }

    std::cout << "Done massaging concentration. " << std::endl;
}

static
void
Twiddle_DiffCoeffs ()
{
    //
    // B = A + B; C = A + B + C; D = A + B + C + D; ...
    //
    for (int l = 0; l <= finest_level; l++)
    {
        for (MFIter mfi(A_mf[l]); mfi.isValid(); ++mfi)
        {
            B_mf[l][mfi] += A_mf[l][mfi];
            C_mf[l][mfi] += B_mf[l][mfi];
            D_mf[l][mfi] += C_mf[l][mfi];
#if BL_SPACEDIM==3
            E_mf[l][mfi] += D_mf[l][mfi];
            F_mf[l][mfi] += E_mf[l][mfi];
#endif
        }
    }
}

static
void
ClonePlotfile (AmrData&                  amrData,
               PArray<MultiFab>&         mfout,
               std::string&              oFile,
               const Array<std::string>& names)
{
    //
    // Sink up grow cells under valid region.
    //
    for (int i = 0; i < mfout.size(); i++)
        mfout[i].FillBoundary();

    if (ParallelDescriptor::IOProcessor())
        if (!BoxLib::UtilCreateDirectory(oFile,0755))
            BoxLib::CreateDirectoryFailed(oFile);
    //
    // Force other processors to wait till directory is built.
    //
    ParallelDescriptor::Barrier();
    
    std::string oFileHeader(oFile);
    oFileHeader += "/Header";
    
    VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
    
    std::ofstream os;
    
    os.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
    
    if (ParallelDescriptor::IOProcessor())
        std::cout << "Opening file = " << oFileHeader << '\n';
    
    os.open(oFileHeader.c_str(), std::ios::out|std::ios::binary);
    
    if (os.fail()) BoxLib::FileOpenFailed(oFileHeader);
    //
    // Start writing plotfile.
    //
    os << amrData.PlotFileVersion() << '\n';
    int n_var = mfout[0].nComp();
    os << n_var << '\n';
    for (int n = 0; n < n_var; n++) os << names[n] << '\n';
    os << BL_SPACEDIM << '\n';
    os << amrData.Time() << '\n';
    const int finestLevel = mfout.size() - 1;
    os << finestLevel << '\n';
    for (int i = 0; i < BL_SPACEDIM; i++) os << amrData.ProbLo()[i] << ' ';
    os << '\n';
    for (int i = 0; i < BL_SPACEDIM; i++) os << amrData.ProbHi()[i] << ' ';
    os << '\n';
    for (int i = 0; i < finestLevel; i++) os << amrData.RefRatio()[i] << ' ';
    os << '\n';
    for (int i = 0; i <= finestLevel; i++) os << amrData.ProbDomain()[i] << ' ';
    os << '\n';
    for (int i = 0; i <= finestLevel; i++) os << 0 << ' ';
    os << '\n';
    for (int i = 0; i <= finestLevel; i++)
    {
        for (int k = 0; k < BL_SPACEDIM; k++)
            os << amrData.DxLevel()[i][k] << ' ';
        os << '\n';
    }
    os << amrData.CoordSys() << '\n';
    os << "0\n"; // The bndry data width.
    //
    // Write out level by level.
    //
    for (int iLevel = 0; iLevel <= finestLevel; ++iLevel)
    {
        //
        // Write state data.
        //
        int nGrids = amrData.boxArray(iLevel).size();
        char buf[64];
        sprintf(buf, "Level_%d", iLevel);
        
        if (ParallelDescriptor::IOProcessor())
        {
            os << iLevel << ' ' << nGrids << ' ' << amrData.Time() << '\n';
            os << 0 << '\n';
            
            for (int i = 0; i < nGrids; ++i)
            {
                for (int n = 0; n < BL_SPACEDIM; n++)
                {
                    os << amrData.GridLocLo()[iLevel][i][n]
                       << ' '
                       << amrData.GridLocHi()[iLevel][i][n]
                       << '\n';
                }
            }
            //
            // Build the directory to hold the MultiFabs at this level.
            //
            std::string Level(oFile);
            Level += '/';
            Level += buf;
            
            if (!BoxLib::UtilCreateDirectory(Level, 0755))
                BoxLib::CreateDirectoryFailed(Level);
        }
        //
        // Force other processors to wait till directory is built.
        //
        ParallelDescriptor::Barrier();
        //
        // Now build the full relative pathname of the MultiFab.
        //
        static const std::string MultiFabBaseName("/MultiFab");
        
        std::string PathName(oFile);
        PathName += '/';
        PathName += buf;
        PathName += MultiFabBaseName;
        
        if (ParallelDescriptor::IOProcessor())
        {
            //
            // The full name relative to the Header file.
            //
            std::string RelativePathName(buf);
            RelativePathName += MultiFabBaseName;
            os << RelativePathName << '\n';
        }
        //
        // Write out ghost cells as is ...
        //
        VisMF::Write(mfout[iLevel], PathName, VisMF::OneFilePerCPU, false);
    }
    
    os.close();
}

static
std::vector<std::string>
Tokenize (const std::string& str,
          const std::string& sep)
{
    std::vector<char*> ptr;
    //
    // Make copy of line that we can modify.
    //
    char* line = new char[str.size()+1];

    (void) strcpy(line, str.c_str());

    char* token = 0;

    if (!((token = strtok(line, sep.c_str())) == 0))
    {
        ptr.push_back(token);
        while (!((token = strtok(0, sep.c_str())) == 0))
            ptr.push_back(token);
    }

    vector<std::string> tokens(ptr.size());

    for (int i = 1; i < ptr.size(); i++)
    {
        char* p = ptr[i];

        while (strchr(sep.c_str(),*(p-1)) != 0)
            *--p = 0;
    }

    for (int i = 0; i < ptr.size(); i++)
        tokens[i] = ptr[i];

    delete line;

    return tokens;
}

static
std::string
GetFileRoot (const std::string& infile)
{
    std::vector<std::string> tokens = Tokenize(infile,std::string("/"));

    return tokens[tokens.size()-1];
}

static
void
WriteOutPlotFiles (const std::string&        infile,
                   ChemKinDriver&            ckd,
                   AmrData&                  amrData,
                   int                       Nspec,
                   int                       Nreac,
                   const Array<std::string>& chemnames,
                   vector<std::string>&      trSp,
                   vector<int>&              trRxn)
{
    if (!write_intermediate_plotfiles) return;

    std::string nfile_conc(GetFileRoot(infile)+"_conc");
    Array<std::string> nnames_conc(Nspec);
    for (int j = 0; j < Nspec; ++j)
        nnames_conc[j] = "C(" + chemnames[ckd.index(trSp[j])] + ")";

    std::cout << "Writing Concentration to " << nfile_conc << std::endl;
    ClonePlotfile(amrData,Conc_mf,nfile_conc,nnames_conc);

    std::string nfile_rf(GetFileRoot(infile)+"_rf");
    std::string nfile_rr(GetFileRoot(infile)+"_rr");

    Array<std::string> nnames_rf(Nreac);
    for (int j = 0; j < Nreac; ++j)
        nnames_rf[j] = ckd.reactionString(trRxn[j]);

    std::cout << "Writing Forward Rates to " << nfile_rf << std::endl;
    ClonePlotfile(amrData,Rf_mf,nfile_rf,nnames_rf);

    std::cout << "Writing Backward Rates to " << nfile_rr << std::endl;
    ClonePlotfile(amrData,Rr_mf,nfile_rr,nnames_rf);

    const char* VelName[3] = { "x_velocity", "y_velocity", "z_velocity" };
    std::string nfile_u(GetFileRoot(infile)+"_u");
    Array<std::string> nnames_u(BL_SPACEDIM);
    for (int j = 0; j < BL_SPACEDIM; ++j)
        nnames_u[j] = VelName[j];

    std::cout << "Writing Velocity to " << nfile_u << std::endl;
    ClonePlotfile(amrData,U_mf,nfile_u,nnames_u);

    std::string nfile_a(GetFileRoot(infile)+"_a");
    Array<std::string> nnames_a(Nspec);
    for (int j = 0; j < Nspec; ++j)
        nnames_a[j] = "a_" + chemnames[ckd.index(trSp[j])];

    std::cout << "Writing A to " << nfile_a << std::endl;
    ClonePlotfile(amrData,A_mf,nfile_a,nnames_a);

    std::string nfile_b(GetFileRoot(infile)+"_b");
    Array<std::string> nnames_b(Nspec);
    for (int j = 0; j < Nspec; ++j)
        nnames_b[j] = "b_" + chemnames[ckd.index(trSp[j])];

    std::cout << "Writing B to " << nfile_b << std::endl;
    ClonePlotfile(amrData,B_mf,nfile_b,nnames_b);

    std::string nfile_c(GetFileRoot(infile)+"_c");
    Array<std::string> nnames_c(Nspec);
    for (int j = 0; j < Nspec; ++j)
        nnames_c[j] = "c_" + chemnames[ckd.index(trSp[j])];

    std::cout << "Writing C to " << nfile_c << std::endl;
    ClonePlotfile(amrData,C_mf,nfile_c,nnames_c);

    std::string nfile_d(GetFileRoot(infile)+"_d");
    Array<std::string> nnames_d(Nspec);
    for (int j = 0; j < Nspec; ++j)
        nnames_d[j] = "d_" + chemnames[ckd.index(trSp[j])];

    std::cout << "Writing D to " << nfile_d << std::endl;
    ClonePlotfile(amrData,D_mf,nfile_d,nnames_d);

#if BL_SPACEDIM==3
    std::string nfile_e(GetFileRoot(infile)+"_e");
    Array<std::string> nnames_e(Nspec);
    for (int j = 0; j < Nspec; ++j)
        nnames_e[j] = "e_" + chemnames[ckd.index(trSp[j])];

    std::cout << "Writing E to " << nfile_e << std::endl;
    ClonePlotfile(amrData,E_mf,nfile_e,nnames_e);

    std::string nfile_f(GetFileRoot(infile)+"_f");
    Array<std::string> nnames_f(Nspec);
    for (int j = 0; j < Nspec; ++j)
        nnames_f[j] = "f_" + chemnames[ckd.index(trSp[j])];

    std::cout << "Writing F to " << nfile_f << std::endl;
    ClonePlotfile(amrData,E_mf,nfile_f,nnames_f);
#endif
}

static
void
ReadPlotFile (ChemKinDriver&     ckd,
              const std::string& file)
{
    const int MyProc = ParallelDescriptor::MyProc();

    DataServices::SetBatchMode();
    FileType fileType(NEWPLT);
    DataServices dataServices(file, fileType);

    if (!dataServices.AmrDataOk())
        //
        // This calls ParallelDescriptor::EndParallel() and exit()
        //
        DataServices::Dispatch(DataServices::ExitRequest, NULL);
    
    AmrData&   amrData         = dataServices.AmrDataRef();
    const int  Nlev            = amrData.FinestLevel() + 1;
    const bool use_all_species = false;
    const Real Patm            = 1.0;
    //
    // Set some globals ...
    //
    is_rz        = amrData.CoordSys() == CoordSys::RZ;
    prob_lo      = amrData.ProbLo();
    prob_hi      = amrData.ProbHi();
    ref_ratio    = amrData.RefRatio();
    prob_domain  = amrData.ProbDomain();
    finest_level = amrData.FinestLevel();

    dx.resize(finest_level+1);
    dxinv.resize(finest_level+1);
    geom.resize(finest_level+1);

    for (int l = 0; l <= finest_level; l++)
    {
        dx[l]    = amrData.DxLevel()[l];
        dxinv[l] = amrData.DxLevel()[l];

        for (int i = 0; i < BL_SPACEDIM; i++)
            dxinv[l][i] = 1.0/dx[l][i];

        RealBox rb(prob_lo.dataPtr(),prob_hi.dataPtr());

        geom[l].define(prob_domain[l], &rb, amrData.CoordSys());
    }
    //
    // Reclaim any old memory ...
    //
    D_TERM(A_mf.clear();, C_mf.clear();, E_mf.clear(););
    D_TERM(B_mf.clear();, D_mf.clear();, F_mf.clear(););

    U_mf.clear();
    Rf_mf.clear();
    Rr_mf.clear();
    Conc_mf.clear();
    //
    // Size our PArrays ...
    //
    D_TERM(A_mf.resize(Nlev);, C_mf.resize(Nlev);, E_mf.resize(Nlev););
    D_TERM(B_mf.resize(Nlev);, D_mf.resize(Nlev);, F_mf.resize(Nlev););

    U_mf.resize(Nlev);
    Rf_mf.resize(Nlev);
    Rr_mf.resize(Nlev);
    Conc_mf.resize(Nlev);

    const Array<std::string> plotnames = amrData.PlotVarNames();
    const Array<std::string> chemnames = ckd.speciesNames();

    int idT = -1, idX = -1, idU = -1, idR = -1;

    for (int i = 0; i < plotnames.size(); ++i)
    {
        if (plotnames[i] == "temp")                     idT = i;
        if (plotnames[i] == "X(" + chemnames[0] + ")" ) idX = i;
        if (plotnames[i] == "x_velocity")               idU = i;
        if (plotnames[i] == "density")                  idR = i;
    }

    BL_ASSERT(idT >= 0);
    BL_ASSERT(idX >= 0);
    BL_ASSERT(idU >= 0);
    BL_ASSERT(idR >= 0);

    vector<std::string> trSp;
    vector<int>         trRx;

    for (int i = 0; i < chemnames.size(); i++)
        if (use_all_species || ckd.numberOfElementXinSpeciesY(TraceElement,chemnames[i]) > 0)
            trSp.push_back(chemnames[i]);

    for (int i = 0; i < ckd.numReactions(); i++)
    {
        if (use_all_species)
        {
            trRx.push_back(i);
        }
        else
        {
            std::map<std::string,int> net;

            Array< std::pair<std::string,int> > coef = ckd.specCoeffsInReactions(i);

            for (int j = 0; j < coef.size(); j++)
            {
                const int          co = coef[j].second;
                const std::string& sp = coef[j].first;

                if (std::count(trSp.begin(),trSp.end(),sp) > 0)
                {
                    if (net.find(sp) == net.end())
                    {
                        net[sp] = co;
                    }
                    else
                    {
                        net[sp] += co;
                        if (net[sp] == 0) net.erase(sp);
                    }
                }
            }

            if (!net.empty()) trRx.push_back(i);
        }
    }
    //
    // Got to set init_spec.
    //
    init_spec = -1;

    for (int i = 0; i < trSp.size(); i++)
        if (chemnames[ckd.index(trSp[i])] == InitialSpecies)
            init_spec = i;

    BL_ASSERT(init_spec != -1);

    std::cout << "init_spec = " << init_spec << std::endl;

    const int Nspec = trSp.size();
    const int Nreac = trRx.size();

    std::cout << "Nspec = " << Nspec << std::endl;
    std::cout << "Nreac = " << Nreac << std::endl;

    for (int i = 0; i < trSp.size(); i++)
        std::cout << i << ": " << chemnames[ckd.index(trSp[i])] << std::endl;

    Array<int> rxnIDs(Nreac);

    for (int i = 0; i < Nreac; i++)
        rxnIDs[i] = trRx[i];

    FArrayBox        tmpfab;
    PArray<MultiFab> temp(Nlev,PArrayManage);
    PArray<MultiFab> mole(Nlev,PArrayManage);

    for (int lev = 0; lev < Nlev; ++lev)
    {
        temp.set(lev,new MultiFab(amrData.boxArray(lev),1,1));
        mole.set(lev,new MultiFab(amrData.boxArray(lev),ckd.numSpecies(),1));

        std::cout << "Reading mole & temp data at lev=" << lev << " ... " << std::flush;

        temp[lev].copy(amrData.GetGrids(lev,idT),0,0,1);

        amrData.FlushGrids(idT);

        for (int j = 0; j < ckd.numSpecies(); ++j)
        {
            mole[lev].copy(amrData.GetGrids(lev,idX+j),0,j,1);

            amrData.FlushGrids(idX+j);
        }

        for (MFIter mfi(mole[lev]); mfi.isValid(); ++mfi)
        {
            const Box& box = mfi.validbox();

            int ncompM = mole[lev].nComp();
            int ncompT = temp[lev].nComp();

            FORT_PUSHVTOG(box.loVect(),box.hiVect(),
                          box.loVect(),
                          box.hiVect(),
                          temp[lev][mfi].dataPtr(),
                          ARLIM(temp[lev][mfi].loVect()),
                          ARLIM(temp[lev][mfi].hiVect()),&ncompT);

            FORT_PUSHVTOG(box.loVect(),box.hiVect(),
                          box.loVect(),
                          box.hiVect(),
                          mole[lev][mfi].dataPtr(),
                          ARLIM(mole[lev][mfi].loVect()),
                          ARLIM(mole[lev][mfi].hiVect()),&ncompM);
        }

        temp[lev].FillBoundary();
        mole[lev].FillBoundary();

        geom[lev].FillPeriodicBoundary(temp[lev]);
        geom[lev].FillPeriodicBoundary(mole[lev]);

        for (MFIter mfi(mole[lev]); mfi.isValid(); ++mfi)
        {
            FArrayBox& x = mole[lev][mfi];
            FArrayBox& t = temp[lev][mfi];

            Box bx = t.box() & amrData.ProbDomain()[lev];

            ckd.moleFracToMassFrac(x,x,bx,0,0);
            ckd.normalizeMassFrac(x,x,"N2",bx,0,0);
            ckd.massFracToMolarConc(x,x,t,Patm,bx,0,0,0);
            //
            // mole now contains concentration ...
            //
        }

        std::cout << "done" << std::endl;
    }
    //
    // Got to twiddle the concentration.
    // Then save the twiddle'd concentration into Conc_mf.
    // Then convert the twiddle'd concentration back to mole fractions.
    //
    TwiddleConc(mole);

    for (int lev = 0; lev < mole.size(); lev++)
    {
        mole[lev].FillBoundary();
        geom[lev].FillPeriodicBoundary(mole[lev]);
    }

    for (int lev = 0; lev < Nlev; ++lev)
    {
        std::cout << "Building Conc lev=" << lev << " ... " << std::flush;

        Conc_mf.set(lev,new MultiFab(amrData.boxArray(lev),Nspec,0));

        for (MFIter mfi(mole[lev]); mfi.isValid(); ++mfi)
        {
            FArrayBox& c = Conc_mf[lev][mfi];
            FArrayBox& x = mole[lev][mfi];

            for (int i = 0; i < Nspec; i++)
                c.copy(x,ckd.index(trSp[i]),i,1);

            ckd.molarConcToMoleFrac(x,x,x.box(),0,0);
            //
            // mole now contains twiddle'd mole fractions.
            //
        }

        std::cout << "done" << std::endl;
    }

    for (int lev = 0; lev < Nlev; ++lev)
    {
        std::cout << "Building Rf & Rr lev=" << lev << " ... " << std::flush;

        Rf_mf.set(lev,new MultiFab(amrData.boxArray(lev),Nreac,0));
        Rr_mf.set(lev,new MultiFab(amrData.boxArray(lev),Nreac,0));

        for (MFIter mfi(mole[lev]); mfi.isValid(); ++mfi)
        {
            FArrayBox& f = Rf_mf[lev][mfi];
            FArrayBox& r = Rr_mf[lev][mfi];
            FArrayBox& t = temp[lev][mfi];
            FArrayBox& x = mole[lev][mfi];

            Box bx = t.box() & amrData.ProbDomain()[lev];

            ckd.fwdRevReacRatesGivenXTP(f,r,rxnIDs,x,t,Patm,bx,0,0,0,0);
            ckd.moleFracToMassFrac(x,x,bx,0,0);
            ckd.normalizeMassFrac(x,x,"N2",bx,0,0);
            //
            // mole now contains mass fractions!!!
            //
        }

        std::cout << "done" << std::endl;
    }
    //
    // Now do velocity ...
    //
    for (int lev = 0; lev < Nlev; ++lev)
    {
        U_mf.set(lev,new MultiFab(amrData.boxArray(lev),BL_SPACEDIM,0));

        std::cout << "Reading velocity data at lev=" << lev << " ... " << std::flush;

        for (int j = 0; j < BL_SPACEDIM; ++j)
        {
            U_mf[lev].copy(amrData.GetGrids(lev,idU+j),0,j,1);
            amrData.FlushGrids(idU+j);
        }

        std::cout << "done" << std::endl;
    }
    //
    // Now do a, b, c & d ...
    //
    for (int lev = 0; lev < Nlev; ++lev)
    {
        FArrayBox rho, rhoDr;

        D_TERM(A_mf.set(lev,new MultiFab(amrData.boxArray(lev),Nspec,0));,
               C_mf.set(lev,new MultiFab(amrData.boxArray(lev),Nspec,0));,
               E_mf.set(lev,new MultiFab(amrData.boxArray(lev),Nspec,0)););

        D_TERM(B_mf.set(lev,new MultiFab(amrData.boxArray(lev),Nspec,0));,
               D_mf.set(lev,new MultiFab(amrData.boxArray(lev),Nspec,0));,
               F_mf.set(lev,new MultiFab(amrData.boxArray(lev),Nspec,0)););

        std::cout << "Building Diffusion Coefficients at lev=" << lev << " ... " << std::flush;
        //
        // We have temp and mass fractions over grow cells within ProbDomain.
        //
        for (MFIter mfi(mole[lev]); mfi.isValid(); ++mfi)
        {
            Box bx = mole[lev][mfi].box() & amrData.ProbDomain()[lev];

            rho.resize(bx,1);
            rhoDr.resize(bx,ckd.numSpecies());

            rho.setVal(0);
            rhoDr.setVal(0);

            amrData.FillVar(&rho,bx,lev,plotnames[idR],MyProc);

            ckd.getMixAveragedRhoDiff(rhoDr,mole[lev][mfi],temp[lev][mfi],Patm,bx,0,0,0);

            IntVect se = bx.smallEnd();
            IntVect be = bx.bigEnd();

            D_TERM(const Real dx = amrData.DxLevel()[lev][0];,
                   const Real dy = amrData.DxLevel()[lev][1];,
                   const Real dz = amrData.DxLevel()[lev][2];);

#if BL_SPACEDIM==2
            if (is_rz)
            {
                FArrayBox getR;

                getR.resize(bx,1);
                getR.setVal(0);
                //
                // Set rhoDr to rhodiff*r
                //
                const Real dr = amrData.DxLevel()[lev][0];

                for (int i = se[0]; i <= be[0]; i++)
                {
                    IntVect sbse = IntVect(i,se[1]);
                    IntVect sbbe = IntVect(i,be[1]);
                    getR.setVal((i+0.5)*dr,Box(sbse,sbbe),0,1);
                }

                for (int i = 0; i < ckd.numSpecies(); i++)
                    rhoDr.mult(getR,0,i,1);
                //
                // Set rho to rho*r*dr*dr*2
                //
                for (int i = se[0]; i <= be[0]; i++)
                {
                    const IntVect sbse = IntVect(i,se[1]);
                    const IntVect sbbe = IntVect(i,be[1]);
                    getR.setVal((i+0.5)*dr*dr*dr*2,Box(sbse,sbbe),0,1);
                }

                rho.mult(getR);
            }
#endif
            tmpfab.resize(bx,1);

            for (int i = 0; i < Nspec; i++)
            {
                tmpfab.copy(rhoDr,ckd.index(trSp[i]),0,1);

                D_TERM(FArrayBox& afab = A_mf[lev][mfi];,
                       FArrayBox& cfab = C_mf[lev][mfi];,
                       FArrayBox& efab = E_mf[lev][mfi];);

                D_TERM(FArrayBox& bfab = B_mf[lev][mfi];,
                       FArrayBox& dfab = D_mf[lev][mfi];,
                       FArrayBox& ffab = F_mf[lev][mfi];);

                afab.copy(tmpfab,0,i,1);
                tmpfab.shift(0,1);
                afab.plus(tmpfab,0,i,1);
                tmpfab.shift(0,-1);
                afab.divide(rho,0,i,1);

                bfab.copy(tmpfab,0,i,1);
                tmpfab.shift(0,-1);
                bfab.plus(tmpfab,0,i,1);
                tmpfab.shift(0,1);
                bfab.divide(rho,0,i,1);

                cfab.copy(tmpfab,0,i,1);
                tmpfab.shift(1,1);
                cfab.plus(tmpfab,0,i,1);
                tmpfab.shift(1,-1);
                cfab.divide(rho,0,i,1);

                dfab.copy(tmpfab,0,i,1);
                tmpfab.shift(1,-1);
                dfab.plus(tmpfab,0,i,1);
                tmpfab.shift(1,1);
                dfab.divide(rho,0,i,1);

#if BL_SPACEDIM==3
                efab.copy(tmpfab,0,i,1);
                tmpfab.shift(2,1);
                efab.plus(tmpfab,0,i,1);
                tmpfab.shift(2,-1);
                efab.divide(rho,0,i,1);

                ffab.copy(tmpfab,0,i,1);
                tmpfab.shift(2,-1);
                ffab.plus(tmpfab,0,i,1);
                tmpfab.shift(2,1);
                ffab.divide(rho,0,i,1);
#endif
                if (!is_rz)
                {
                    D_TERM(afab.divide(dx*dx*2,i,1);,
                           cfab.divide(dy*dy*2,i,1);,
                           efab.divide(dz*dz*2,i,1););

                    D_TERM(bfab.divide(dx*dx*2,i,1);,
                           dfab.divide(dy*dy*2,i,1);,
                           ffab.divide(dz*dz*2,i,1););
                }
            }
        }

        amrData.FlushGrids(idR);

        const Box& domain = amrData.ProbDomain()[lev];
        //
        // Zero out diffusion coefficients on physical
        // boundary, unless we're periodic in that direction.
        //
        if (!geom[lev].isPeriodic(0))
        {
            A_mf[lev].setVal(0,domain & Box(domain).shift(0,-domain.length(0)+1),0,Nspec,0);
            B_mf[lev].setVal(0,domain & Box(domain).shift(0, domain.length(0)-1),0,Nspec,0);
        }

        if (!geom[lev].isPeriodic(1))
        {
            C_mf[lev].setVal(0,domain & Box(domain).shift(1,-domain.length(1)+1),0,Nspec,0);
            D_mf[lev].setVal(0,domain & Box(domain).shift(1, domain.length(1)-1),0,Nspec,0);
        }

#if BL_SPACEDIM==3
        if (!geom[lev].isPeriodic(2))
        {
            E_mf[lev].setVal(0,domain & Box(domain).shift(2,-domain.length(2)+1),0,Nspec,0);
            F_mf[lev].setVal(0,domain & Box(domain).shift(2, domain.length(2)-1),0,Nspec,0);
        }
#endif

        std::cout << "done" << std::endl;
    }

    WriteOutPlotFiles(file,ckd,amrData,Nspec,Nreac,chemnames,trSp,trRx);

    MassageConc();

    Twiddle_DiffCoeffs();
    //
    // Some sanity checks ...
    //
    BL_ASSERT(A_mf.size() == B_mf.size());
    BL_ASSERT(A_mf.size() == C_mf.size());
    BL_ASSERT(A_mf.size() == D_mf.size());
    BL_ASSERT(A_mf.size() == U_mf.size());
    BL_ASSERT(A_mf.size() == Rf_mf.size());
    BL_ASSERT(A_mf.size() == Rr_mf.size());
    BL_ASSERT(A_mf.size() == Conc_mf.size());

    BL_ASSERT(A_mf[0].boxArray() == B_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == C_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == D_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == U_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == Rf_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == Rr_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == Conc_mf[0].boxArray());
    //
    // Print out some stuff ...
    //
    if (ParallelDescriptor::IOProcessor())
    {
        std::cout << "finest_level = " << finest_level << '\n';

        for (int l = 0; l <= finest_level; l++)
        {
            std::cout << "dx[" << l << "]: ";

            for (int i = 0; i < BL_SPACEDIM; i++)
                std::cout << dx[l][i] << ' ';
            std::cout << '\n';

            std::cout << "prob_domain[" << l << "]: " << prob_domain[l] << '\n';
        }

        std::cout << "ref_ratio: ";
        for (int l = 0; l < finest_level; l++)
            std::cout << ref_ratio[l] << ' ';
        std::cout << '\n';
    }
}

static
void
WriteEvent (int           part,
            X&            xpos,
            int           spec,
            int           reac,
            Real          time,
            std::ostream& eofs)
{
    if (do_not_write_events) return;

//    int oprec = eofs.precision(20);

    eofs << part << ' ';

    for (int i = 0; i < BL_SPACEDIM; i++)
    {
        eofs << xpos[i] << ' ';
    }

//    eofs.precision(oprec);

    eofs << spec << ' ' << reac << ' ' << time << '\n';

    if (!eofs.good()) BoxLib::Abort("Problem with the event output file");
}

static
void
WriteTrajectory (int                         particleNo,
                 const std::vector<PartPos>& trajectory,
                 std::ofstream&              traj_ofs)

{
    if (do_not_write_trajs) return;
    //
    // Only write out every N ...
    //
    const int N = 25;

    traj_ofs << particleNo << ":\n";

    for (int i = 0; i < trajectory.size(); i++)
    {
        if (i % N == 0)
        {
            traj_ofs << trajectory[i][0] << ' '
                     << trajectory[i][1] << ' '
                     << trajectory[i][2] << '\n';
        }
    }

    if (!traj_ofs.good()) BoxLib::Abort("Problem with the trajectory file");
}

static
void
UpdateTrajectory (const X&              x,
                  Real                  t,
                  std::vector<PartPos>& traj)
{
    if (do_not_write_trajs) return;

    PartPos p;

    D_TERM(p[0] = x[0];, p[1] = x[1];, p[2] = x[2];)

    p[BL_SPACEDIM] = t;

    traj.push_back(p);
}

static
void
MaxLambda ()
{
    if (ParallelDescriptor::IOProcessor())
        std::cout << "Calculating Max Lambda ..." << std::endl;

    const int nspec = pedges.size();

    BL_ASSERT(nspec == Conc_mf[0].nComp());
    BL_ASSERT(finest_level+1 == Conc_mf.size());
    //
    // Now do the appropriate computation over levels using C_tmp.
    //
    // This relies on the Concentration being zero'd under fine grids.
    //
    for (int spec = 0; spec < nspec; spec++)
    {
        Real lmax = 0;

        const int start = pedges[spec];
        const int nedge = edgedata[start];

        for (int l = 0; l <= finest_level; l++)
        {
            const MultiFab& Rf   = Rf_mf[l];
            const MultiFab& Rr   = Rr_mf[l];
            const MultiFab& Conc = Conc_mf[l];

            for (MFIter mfi(Conc); mfi.isValid(); ++mfi)
            {
                const Box&       bx   = Conc[mfi].box();
                const FArrayBox& rf   = Rf[mfi];
                const FArrayBox& rr   = Rr[mfi];
                const FArrayBox& conc = Conc[mfi];

                for (IntVect p = bx.smallEnd(); p <= bx.bigEnd(); bx.next(p))
                {
                    Real lambda = 0;

                    for (int ie = 0; ie < nedge; ie++)
                    {
                        const int  rxnid  = edgedata[start+ie*4+1];
                        const int  factor = edgedata[start+ie*4+2];
                        const int  nu     = edgedata[start+ie*4+4];
                        const Real X      = nu*conc(p,spec)/factor;

                        if (factor > 0)
                        {
                            lambda += X*rf(p,rxnid);
                        }
                        else
                        {
                            lambda -= X*rr(p,rxnid);
                        }
                    }

                    lmax = std::max(lmax,lambda);
                }
            }
        }

        if (ParallelDescriptor::IOProcessor())
        {
            std::cout  << "spec = " << spec << ", lmax = " << lmax << std::endl;
        }
    }

    exit(0);
}

static
void
MaxDtLambda (int            spec,
             Real&          dt,
             int            lev,
             int            idx,
             const IntVect& iv)
{
    Real             lambda = 0;
    const int        start  = pedges[spec];
    const int        nedge  = edgedata[start];
    const FArrayBox& rf     = Rf_mf[lev][idx];
    const FArrayBox& rr     = Rr_mf[lev][idx];
    const FArrayBox& conc   = Conc_mf[lev][idx];

    for (int ie = 0; ie < nedge; ie++)
    {
        const int rxnid  = edgedata[start+ie*4+1];
        const int factor = edgedata[start+ie*4+2];
        const int nu     = edgedata[start+ie*4+4];

        const Real X = nu*conc(iv,spec)/factor;

        if (factor > 0)
        {
            lambda += X*rf(iv,rxnid);
        }
        else
        {
            lambda -= X*rr(iv,rxnid);
        }
    }
    
    if (lambda > 0) dt = std::min(dt,chem_limiter/lambda);
}

static
void
Chemistry (int&             spec,
           int&             rxn,
           Real             dt,
           int              lev,
           int              idx,
           const IntVect&   iv,
           BoxLib::mt19937& rand)
{
    const Real       rn     = rand.d1_value();
    Real             lambda = 0;
    const int        start  = pedges[spec];
    const int        nedge  = edgedata[start];
    const FArrayBox& rf     = Rf_mf[lev][idx];
    const FArrayBox& rr     = Rr_mf[lev][idx];
    const FArrayBox& conc   = Conc_mf[lev][idx];

    for (int ie = 0; ie < nedge; ie++)
    {
        const int  rxnid  = edgedata[start+ie*4+1];
        const int  factor = edgedata[start+ie*4+2];
        const int  tospec = edgedata[start+ie*4+3];
        const int  nu     = edgedata[start+ie*4+4];

        const Real X = nu*conc(iv,spec)*dt/factor;

        if (factor > 0)
        {
            lambda += X*rf(iv,rxnid);
        }
        else
        {
            lambda -= X*rr(iv,rxnid);
        }

        if (rn < lambda)
        {
            rxn  = rxnid;
            spec = tospec;

            break;
        }
    }
}

static
IntVect
Index (const X& x,
       int      lev)
{
    IntVect iv;

    D_TERM(iv[0]=int((x[0]-prob_lo[0])*dxinv[lev][0]);,
           iv[1]=int((x[1]-prob_lo[1])*dxinv[lev][1]);,
           iv[2]=int((x[2]-prob_lo[2])*dxinv[lev][2]););

    iv += prob_domain[lev].smallEnd();

    return iv;
}

static
bool
Where (const X& x,
       int&     lev,
       int&     idx,
       IntVect& iv)
{
    for (lev = finest_level; lev >= 0; lev--)
    {
        iv = Index(x,lev);

        const BoxArray& ba = A_mf[lev].boxArray();

        for (idx = 0; idx < ba.size(); idx++)
            if (ba[idx].contains(iv))
                return true;
    }

    return false;
}

static
void
GetInitialPosition (int&                  spec,
                    int                   rxn,
                    Real                  t,
                    std::vector<PartPos>& traj,
                    int                   i,
                    X&                    x,
                    IntVect&              iv,
                    int&                  lev,
                    int&                  idx,
                    std::ostream&         eofs,
                    BoxLib::mt19937&      rand)
{
//    D_TERM(x[0]=(build_prob_dist?0:RandomXPos(rand));,x[1]=init_y;,x[2]=init_z;)

//    x[0]=.0115;
//    x[1]=.01 + .0075 * rand.d1_value();

    D_TERM(x[0]=init_x;, x[1]=init_y;, x[2]=init_z;);

    if (build_prob_dist)
    {
        const Real rn = rand.d1_value();
        //
        // Find level and index in Prob_mf containing appropriate probability.
        //
        int l, i;
        //
        // Find first AccumProb[lev][idx] combo >= rn ...
        //
        for (l = 0; l <= finest_level; l++)
            for (i = 0; i < AccumProb[l].size(); i++)
                if (rn <= AccumProb[l][i])
                    goto gotit;
    gotit:

        BL_ASSERT(l >= 0 && l <= finest_level);
        BL_ASSERT(i >= 0 && i < AccumProb[l].size());

        const FArrayBox& Prob = Prob_mf[l][i];
        const int*       p_lo = Prob.box().loVect();
        const int*       p_hi = Prob.box().hiVect();

        FORT_SELECTPOS(Prob.dataPtr(),
                       ARLIM(p_lo), ARLIM(p_hi),
                       &rn,
                       D_DECL(&dx[l][0],&dx[l][1],&dx[l][2]),
                       D_DECL(&x[0],&x[1],&x[2]));
    }

    UpdateTrajectory(x,t,traj);

    WriteEvent(i,x,spec,rxn,t,eofs);
    //
    // Now that we have x set lev, idx and iv ...
    //
    bool found = Where(x,lev,idx,iv);

    BL_ASSERT(found == true);

    if (build_prob_dist)
    {
        Real dt_tmp = 1.0e20;

        MaxDtLambda(spec,dt_tmp,lev,idx,iv);

        dt_tmp *= 10;

        const int old_spec = spec;

        Chemistry(spec,rxn,dt_tmp,lev,idx,iv,rand);

        if (spec != old_spec) WriteEvent(i,x,spec,rxn,t,eofs);
    }
}

static
void
BuildProbDist ()
{
    Prob_mf.resize(finest_level+1);

    AccumProb.resize(finest_level+1);

    for (int l = 0; l <= finest_level; l++)
    {
        Prob_mf.set(l, new MultiFab(A_mf[l].boxArray(),1,0));

        AccumProb[l].resize(Prob_mf[l].size(),0);

        Prob_mf[l].setVal(0);
    }

    const int elen  = edgedata.size();
    const int nspec = pedges.size();
    const int nreac = Rf_mf[0].nComp();

    for (int l = 0; l <= finest_level; l++)
    {
        for (MFIter mfi(Prob_mf[l]); mfi.isValid(); ++mfi)
        {
            const int* a_lo = Rf_mf[l][mfi].box().loVect();
            const int* a_hi = Rf_mf[l][mfi].box().hiVect();
            const int* p_lo = Prob_mf[l][mfi].box().loVect();
            const int* p_hi = Prob_mf[l][mfi].box().hiVect();

            FORT_PROBFAB(Rf_mf[l][mfi].dataPtr(),
                         ARLIM(a_lo), ARLIM(a_hi),
                         Rr_mf[l][mfi].dataPtr(),
                         Prob_mf[l][mfi].dataPtr(),
                         ARLIM(p_lo), ARLIM(p_hi),
                         &nspec, &nreac, &init_spec,
                         (int*)&edgedata[0], &elen, pedges.dataPtr(),
                         &is_rz,
                         D_DECL(&dx[l][0],&dx[l][1],&dx[l][2]));

            if (l < finest_level)
            {
                const BoxArray& f_ba = Prob_mf[l+1].boxArray();

                for (int j = 0; j < f_ba.size(); j++)
                {
                    Box c_box = BoxLib::coarsen(f_ba[j],ref_ratio[l]);

                    if (c_box.intersects(Prob_mf[l][mfi].box()))
                    {
                        Box isect = c_box & Prob_mf[l][mfi].box();

                        Prob_mf[l][mfi].setVal(0,isect,0);
                    }
                }
            }
        }
    }

    Real totreact = 0;

    for (int l = 0; l <= finest_level; l++)
        for (MFIter mfi(Prob_mf[l]); mfi.isValid(); ++mfi)
            totreact += Prob_mf[l][mfi].sum(0);

    Real cumprob = 0;

    for (int l = 0; l <= finest_level; l++)
    {
        for (MFIter mfi(Prob_mf[l]); mfi.isValid(); ++mfi)
        {
            const int* p_lo = Prob_mf[l][mfi].box().loVect();
            const int* p_hi = Prob_mf[l][mfi].box().hiVect();

            FORT_ACCUMPROB(Prob_mf[l][mfi].dataPtr(),
                           ARLIM(p_lo),
                           ARLIM(p_hi),
                           &totreact,
                           &cumprob);
        }
    }

    if (ParallelDescriptor::IOProcessor())
    {
        std::cout << "cumulative probability: " << cumprob << std::endl;
    }

    totreact = 0;

    for (int l = 0; l <= finest_level; l++)
    {
        for (MFIter mfi(Prob_mf[l]); mfi.isValid(); ++mfi)
        {
            totreact += Prob_mf[l][mfi].sum(0);

            AccumProb[l][mfi.index()] = totreact;
        }
    }
    //
    // Force sum to 1 just to be safe !!!
    //
    AccumProb[finest_level][AccumProb[finest_level].size()-1] = 1;
}

static
void
MaxOverBox (int        spec,
            int        lev,
            const Box& bx,
            Real&      dmax)
{
    const MultiFab& mf = (BL_SPACEDIM == 2) ? D_mf[lev] : F_mf[lev];

    for (MFIter mfi(mf); mfi.isValid(); ++mfi)
    {
        const FArrayBox& fab = mf[mfi];
        const Box        ibx = bx & fab.box();

        if (ibx.ok())
        {
            for (IntVect p = ibx.smallEnd(); p <= ibx.bigEnd(); ibx.next(p))
            {
                dmax = std::max(dmax,fab(p,spec));
            }
        }
    }
}

static
void
MaxDtDiffusion (int            spec,
                Real&          dt,
                const IntVect& iv,
                int            lev)
{
    Box bx(iv,iv);

    Real dmax = 0;

    MaxOverBox(spec,
               lev,
               BoxLib::grow(bx,2),
               dmax);

    if (lev < finest_level)
    {
        MaxOverBox(spec,
                   lev+1,
                   BoxLib::grow(BoxLib::refine(bx,ref_ratio[lev]),2), 
                   dmax);
    }

    if (dmax > 0) dt = std::min(dt,0.1/dmax);
}

static
void
UpdateCellPosition (X&             x,
                    int&           lev,
                    int&           idx,
                    IntVect&       iv,
                    const IntVect& iv_tmp,
                    bool&          calc_diff_dt,
                    bool&          out_of_bounds)
{
    if (!prob_domain[lev].contains(iv_tmp))
    {
        out_of_bounds = true;
    }
    else
    {
        calc_diff_dt = true;

        const FArrayBox& c = Conc_mf[lev][idx];

        if (c.box().contains(iv_tmp) && c(iv_tmp,0) > 0)
        {
            //
            // Recall that concentration is nonzero except under
            // fine grid.  If we got here we must still be in the
            // same fab.  That it to say, lev and idx are OK, just
            // set iv.
            //
            iv = iv_tmp;
        }
        else
        {
            bool found = Where(x,lev,idx,iv);

            BL_ASSERT(found == true);
        }
    }
}

static
void
TwiddlePosition (X&  x,
                 int lev)
{
    if (is_rz)
    {
        if (x[0] < 0)
        {
            x[0] = -x[0];
        }
    }
    else if (geom[lev].isAnyPeriodic())
    {
        for (int i = 0; i < BL_SPACEDIM; i++)
        {
            if (geom[lev].isPeriodic(i) && x[i] < prob_lo[i])
            {
                x[i] = prob_hi[i] - (prob_lo[i] - x[i]);
            }
            if (geom[lev].isPeriodic(i) && x[i] > prob_hi[i])
            {
                x[i] = prob_lo[i] + (x[i] - prob_hi[i]);
            }
        }
    }
}

static
void
AdvanceParticles (int              tid,
                  int              beg,
                  int              cnt,
                  Real             dt,
                  Real             tstrt,
                  Real             tstop,
                  Array<X>&        particles,
                  Array<int>&      species,
                  BoxLib::mt19937& rand,
                  std::ofstream&   eofs,
                  std::ofstream&   tofs)
{
    const int  NoReac = -1;
    const Real dtsave = dt;

    for (int i = beg; i < cnt+beg; i++)
    {
        X       x             = particles[i];
        Real    t             = tstrt;
        int     rxn           = NoReac;
        int     spec          = species[i];
        bool    reacted       = dt_spec < 0 ? true : false;
        bool    calc_diff_dt  = true;
        bool    out_of_bounds = false;
        Real    dt_diff_save  = dtsave;

        int     lev;  // What level in the AmrData hierarchy?
        int     idx;  // What index into BoxArray at that level?
        IntVect iv;   // What cell position in that Box?
        IntVect iv_tmp;

        std::vector<PartPos> traj;

        if (t == strt_time)
        {
            //
            // This sets x, iv, lev and idx (among possibly others) ...
            //
            GetInitialPosition(spec,rxn,t,traj,i,x,iv,lev,idx,eofs,rand);
        }
        else if (!Where(x,lev,idx,iv))
        {
            //
            // This particles is already done.
            // Don't output any more stuff to the Event/Traj files.
            //
            goto skip;
        }

        while (t < tstop)
        {
            dt = dt_diff_save;
            //
            // Get maximum dt for diffusion.
            //
            if (calc_diff_dt)
            {
                dt = dtsave;
                MaxDtDiffusion(spec,dt,iv,lev);
                dt_diff_save = dt;
                calc_diff_dt = false;
            }

            dt = std::min(dt,tstop-t);
            //
            // Advect x ...
            //
            D_TERM(x[0]+=U_mf[lev][idx](iv,0)*dt;,
                   x[1]+=U_mf[lev][idx](iv,1)*dt;,
                   x[2]+=U_mf[lev][idx](iv,2)*dt;);
            //
            // Do appropriate boundary manipulations.
            //
            TwiddlePosition(x,lev);

            iv_tmp = Index(x,lev);

            if (!(iv == iv_tmp))
            {
                UpdateCellPosition(x,lev,idx,iv,iv_tmp,calc_diff_dt,out_of_bounds);

                if (out_of_bounds) goto done;
            }
            //
            // Diffuse x ...
            //
            const Real rn = rand.d1_value();

            if      (rn < A_mf[lev][idx](iv,spec)*dt) x[0] -= dx[lev][0];
            else if (rn < B_mf[lev][idx](iv,spec)*dt) x[0] += dx[lev][0];
            else if (rn < C_mf[lev][idx](iv,spec)*dt) x[1] -= dx[lev][1];
            else if (rn < D_mf[lev][idx](iv,spec)*dt) x[1] += dx[lev][1];
#if BL_SPACEDIM==3
            else if (rn < E_mf[lev][idx](iv,spec)*dt) x[2] -= dx[lev][2];
            else if (rn < F_mf[lev][idx](iv,spec)*dt) x[2] += dx[lev][2];
#endif
            t += dt;
            //
            // Do appropriate boundary manipulations.
            //
            TwiddlePosition(x,lev);

            iv_tmp = Index(x,lev);

            if (!(iv == iv_tmp))
            {
                UpdateCellPosition(x,lev,idx,iv,iv_tmp,calc_diff_dt,out_of_bounds);

                if (out_of_bounds) goto done;

                UpdateTrajectory(x,t,traj);
            }
            //
            // The particle does not move from here on down.
            //
            Real t_chem       = 0;
            Real dt_chem      = dt;
            bool calc_chem_dt = true;

            do
            {
                if (calc_chem_dt)
                {
                    dt_chem      = dt;
                    calc_chem_dt = false;
                    MaxDtLambda(spec,dt_chem,lev,idx,iv);
                }

                dt_chem = std::min(dt_chem,dt-t_chem);

                if (!reacted) dt_chem = dt_spec;

                const int old_spec = spec;

                Chemistry(spec,rxn,dt_chem,lev,idx,iv,rand);

                if (spec != old_spec)
                {
                    reacted = calc_chem_dt = true;
                    WriteEvent(i,x,spec,rxn,t,eofs);
                }

                t_chem += dt_chem;
            }
            while (t_chem < dt);
        }

    done:

        if (out_of_bounds || t >= stop_time) WriteEvent(i,x,spec,NoReac,t,eofs);

        UpdateTrajectory(x,t,traj);

        if (spec != init_spec) WriteTrajectory(i,traj,tofs);

    skip:

        species[i]   = spec;
        particles[i] = x;
    }
}

class task_advance
    :
    public WorkQueue::task
{
public:

    task_advance (int              tid,
                  int              beg,
                  int              cnt,
                  Real             dt,
                  Real             t_strt,
                  Real             t_stop,
                  Array<X>&        particles,
                  Array<int>&      species,
                  BoxLib::mt19937& rn,
                  std::ofstream&   eofs,
                  std::ofstream&   tofs)
        :
        m_tid(tid),
        m_beg(beg),
        m_cnt(cnt),
        m_dt(dt),
        m_tstrt(t_strt),
        m_tstop(t_stop),
        m_particles(particles),
        m_species(species),
        m_rn(rn),
        m_eofs(eofs),
        m_tofs(tofs)
    {}

    virtual void run ();

private:

    int              m_tid;
    int              m_beg;
    int              m_cnt;
    Real             m_dt;
    Real             m_tstrt;
    Real             m_tstop;
    Array<X>&        m_particles;
    Array<int>&      m_species;
    BoxLib::mt19937& m_rn;
    std::ofstream&   m_eofs;
    std::ofstream&   m_tofs;
};

void
task_advance::run ()
{
    AdvanceParticles(m_tid,
                     m_beg,
                     m_cnt,
                     m_dt,
                     m_tstrt,
                     m_tstop,
                     m_particles,
                     m_species,
                     m_rn,
                     m_eofs,
                     m_tofs);
}
       
int
main (int   argc,
      char* argv[])
{
    BoxLib::Initialize(argc,argv);

    if (argc < 2) PrintUsage(argv[0]);

    const Real run_strt = ParallelDescriptor::second();

    const int NProcs = ParallelDescriptor::NProcs();
    const int MyProc = ParallelDescriptor::MyProc();
    const int IOProc = ParallelDescriptor::IOProcessorNumber();

    ScanArguments();

    ChemKinDriver ckd(chemInFile,chemOutFile,thermInFile,tranInFile);

    Real plt_time = 0; // Time spent reading plot files.

    Array<X> particles(n_particles);

    Array<int> species(n_particles);

    int nthreads = BoxLib::theWorkQueue().max_threads();

    if (nthreads == 0) nthreads = 1;

    PArray<BoxLib::mt19937> rn(nthreads,PArrayManage);

    for (int tid = 0; tid < nthreads; tid++)
        rn.set(tid,new BoxLib::mt19937(seed+tid*101));

    PArray<std::ofstream> eofs(nthreads,PArrayManage);
    PArray<std::ofstream> tofs(nthreads,PArrayManage);

    for (int tid = 0; tid < nthreads; tid++)
    {
        eofs.set(tid,new std::ofstream);
        tofs.set(tid,new std::ofstream);
        //
        // Append node & thread id to file names ...
        //
        std::string EFile = EventFile;
        std::string TFile = TrajFile;

        char buf[12];

        sprintf(buf, ".%02d.%02d", ParallelDescriptor::MyProc(), tid);

        TFile += buf; EFile += buf;

        if (!do_not_write_events)
        {
            eofs[tid].open(EFile.c_str(), std::ios::trunc|std::ios::out);

            if (!eofs[tid].good()) BoxLib::FileOpenFailed(EFile);
        }

        if (!do_not_write_trajs)
        {
            tofs[tid].open (TFile.c_str(),std::ios::trunc|std::ios::out);

            if (!tofs[tid].good()) BoxLib::FileOpenFailed(TFile);
        }
    }

    const int ParticlesPerNode   = n_particles / NProcs;
    const int ParticlesPerThread = ParticlesPerNode / nthreads;

    for (int n = 0; n < PlotFiles.size(); n++)
    {
        Real plt_strt = ParallelDescriptor::second();

        ReadPlotFile(ckd,PlotFiles[n]);

        plt_time += (ParallelDescriptor::second()-plt_strt);

        if (n == 0)
        {
            ReadEdgeData();

            BL_ASSERT(pedges.size() == Conc_mf[0].nComp());
            BL_ASSERT(pedges.size() > 0 && edgedata.size() > 0);
            //
            // init_spec has been set ...
            //
            for (int i = 0; i < species.size(); i++)
                species[i] = init_spec;

            if (calc_lambda_max) MaxLambda();

            if (build_prob_dist) BuildProbDist();

            BuildLookupTable();
        }

        for (int tid = 0; tid < nthreads; tid++)
        {
            int cnt = ParticlesPerThread;
            int beg = ParticlesPerNode * MyProc + cnt * tid;

            if (tid == nthreads - 1)
            {
                //
                // Last thread per node gets remaining per node particles.
                //
                cnt += ParticlesPerNode % nthreads;

                if (NProcs > 1 && MyProc == NProcs - 1)
                    //
                    // Last thread in last node gets all remaining particles.
                    //
                    cnt += n_particles % NProcs;
            }

            std::cout << "Thread: "
                      << tid
                      << ", node: "
                      << MyProc
                      << ", particles: ["
                      << beg
                      << ','
                      << beg+cnt
                      << ")\n";

            const Real tstrt = (n == 0 ? strt_time : TimeSteps[n-1]);
            const Real tstop = TimeSteps[n];

            BoxLib::theWorkQueue().add(new task_advance(tid,
                                                        beg,
                                                        cnt,
                                                        dt,
                                                        tstrt,
                                                        tstop,
                                                        particles,
                                                        species,
                                                        rn[tid],
                                                        eofs[tid],
                                                        tofs[tid]));
        }

        BoxLib::theWorkQueue().wait();
        //
        // Flush the output files after each interval.
        // This'll make it a tad nicer when tail'ing the output files.
        //
        for (int tid = 0; tid < nthreads; tid++)
        {
            if (!do_not_write_trajs)  tofs[tid].flush();
            if (!do_not_write_events) eofs[tid].flush();
        }
    }

    Real run_stop = ParallelDescriptor::second() - run_strt;

    ParallelDescriptor::ReduceRealMax(run_stop, IOProc);

    if (ParallelDescriptor::IOProcessor())
    {
        std::cout << "\n**Run time = "
                  << run_stop
                  << "\n**Avg Runtime/Particle: "
                  << (run_stop-plt_time)/n_particles
                  << std::endl;
    }

    BoxLib::Finalize();
}
