/*
** (c) 1996-2000 The Regents of the University of California (through
** E.O. Lawrence Berkeley National Laboratory), subject to approval by
** the U.S. Department of Energy.  Your use of this software is under
** license -- the license agreement is attached and included in the
** directory as license.txt or you may contact Berkeley Lab's Technology
** Transfer Department at TTD@lbl.gov.  NOTICE OF U.S. GOVERNMENT RIGHTS.
** The Software was developed under funding from the U.S. Government
** which consequently retains certain rights as follows: the
** U.S. Government has been granted for itself and others acting on its
** behalf a paid-up, nonexclusive, irrevocable, worldwide license in the
** Software to reproduce, prepare derivative works, and perform publicly
** and display publicly.  Beginning five (5) years after the date
** permission to assert copyright is obtained from the U.S. Department of
** Energy, and subject to any subsequent five (5) year renewals, the
** U.S. Government is granted for itself and others acting on its behalf
** a paid-up, nonexclusive, irrevocable, worldwide license in the
** Software to reproduce, prepare derivative works, distribute copies to
** the public, perform publicly and display publicly, and to permit
** others to do so.
*/

//
// $Id: AmrDeriveStoch.cpp,v 1.70 2002/05/16 16:22:23 lijewski Exp $
//
// This is the version that reads input from PlotFiles.
//

#include <algorithm>
#include <new>
#include <iomanip>
#include <iostream>
#include <fstream>
#include <cmath>
#include <cstdlib>
#include <cstring>
#include <cstdio>
#include <vector>

#include <unistd.h>

#include "REAL.H"
#include "Box.H"
#include "FArrayBox.H"
#include "ParmParse.H"
#include "ParallelDescriptor.H"
#include "DataServices.H"
#include "Utility.H"
#include "VisMF.H"
#include "Derived.H"
#include "Tuple.H"
#include "Profiler.H"
#include "Thread.H"
#include "WorkQueue.H"
//
// ParmParse'd arguments ...
//
static bool build_prob_dist      = false;
static bool calc_lambda_max      = false;
static bool do_not_write_trajs   = false;
static bool do_not_write_events  = false;
static bool do_not_write_resfile = false;

static int seed             = 12345;
static int res_spec         = 0;
static int init_spec        = 6;
static int part_start       = 0;
static int n_particles      = 0;
static int restime_interval = 250;

static Real init_y       = .0000125;

#if BL_SPACEDIM==3
static Real init_z       = .0000125;
#endif

static Real dt           = .00001;
static Real dt_spec      = -1;
static Real stop_time    = 0;

static Real conc_epsilon = 1.0e-10;
static Real chem_limiter = 0.1;

static std::string ResFile;
static std::string TrajFile;
static std::string EventFile;
static std::string EdgeDataFile;
//
// This is the name of the base PlotFile.
//
// We expect to find PlotFiles of the form:
//
//   PlotFile_a
//   PlotFile_b
//   PlotFile_c
//   PlotFile_d
//   PlotFile_u
//   PlotFile_rf
//   PlotFile_rr
//   PlotFile_conc
//
static std::string PlotFile;
//
// Stuff we set from the AmrData object after reading PlotFile.
//
static int                  finest_level;
static Array<int>           ref_ratio;
static Array<Real>          prob_lo;
static Array<Real>          prob_hi;
static Array<Box>           prob_domain;
static Array< Array<Real> > dx;
static Array< Array<Real> > dxinv;

static PArray<MultiFab> A_mf(PArrayManage);
static PArray<MultiFab> B_mf(PArrayManage);
static PArray<MultiFab> C_mf(PArrayManage);
static PArray<MultiFab> D_mf(PArrayManage);
static PArray<MultiFab> U_mf(PArrayManage);
static PArray<MultiFab> Rf_mf(PArrayManage);
static PArray<MultiFab> Rr_mf(PArrayManage);
static PArray<MultiFab> Conc_mf(PArrayManage);
static PArray<MultiFab> Prob_mf(PArrayManage);
//
// Other global data.
//
static Array<int>           pedges;
static std::vector<int>     edgedata;
static Array< Array<Real> > AccumProb;
//
// Particles on our trajectory ...
//
typedef Tuple<Real,BL_SPACEDIM+1> PartPos;

static
void
ScanArguments (ParmParse& pp)
{
    pp.query("dt", dt);

    BL_ASSERT(dt != 0);

    pp.query("dt_spec", dt_spec);

    BL_ASSERT(dt_spec != 0);

    pp.query("stop_time", stop_time);

    BL_ASSERT(stop_time > 0);

    pp.query("seed", seed);

    pp.query("restime_interval", restime_interval);

    pp.query("edgedatafile", EdgeDataFile);
    if (EdgeDataFile.empty())
        BoxLib::Abort("You must specify `edgedatafile'");

    pp.query("event_file", EventFile);
    if (EventFile.empty())
        BoxLib::Abort("You must specify `event_file'");

    pp.query("res_file", ResFile);
    if (ResFile.empty())
        BoxLib::Abort("You must specify `res_file'");

    pp.query("traj_file", TrajFile);
    if (TrajFile.empty())
        BoxLib::Abort("You must specify `traj_file'");

    pp.query("plot_file", PlotFile);
    if (PlotFile.empty())
        BoxLib::Abort("You must specify `plot_file'");
    //
    // Append `.seed=N' to EventFile & TrajFile & ResFile.
    //
    char buf[12];

    ResFile   += ".seed="; TrajFile  += ".seed="; EventFile += ".seed=";

    sprintf(buf, "%d", seed);

    ResFile += buf; TrajFile += buf; EventFile += buf;

    pp.query("calc_lambda_max", calc_lambda_max);

    pp.query("do_not_write_events", do_not_write_events);

    pp.query("do_not_write_trajs", do_not_write_trajs);

    pp.query("do_not_write_resfile", do_not_write_resfile);

    pp.query("conc_epsilon", conc_epsilon);

    pp.query("chem_limiter", chem_limiter);

    BL_ASSERT(conc_epsilon > 0);
    BL_ASSERT(chem_limiter > 0);

    pp.query("init_y", init_y);

#if BL_SPACEDIM==3
    pp.query("init_z", init_z);
#endif

    pp.query("init_spec", init_spec);

    pp.query("res_spec", res_spec);

    pp.query("n_particles", n_particles);

    BL_ASSERT(n_particles > 0);

    pp.query("part_start", part_start);

    BL_ASSERT(part_start >= 0);

    pp.query("build_prob_dist", build_prob_dist);

    if (ParallelDescriptor::IOProcessor())
    {
        std::cout << "dt = " << dt << '\n';

        std::cout << "dt_spec = " << dt_spec << '\n';

        std::cout << "stop_time = " << stop_time << '\n';

        std::cout << "seed = " << seed << '\n';

        std::cout << "restime_interval = " << restime_interval << '\n';

        std::cout << "edgedatafile = " << EdgeDataFile << '\n';

        std::cout << "res_file = "   << ResFile   << '\n';
        std::cout << "plot_file = "  << PlotFile  << '\n';
        std::cout << "traj_file = "  << TrajFile  << '\n';
        std::cout << "event_file = " << EventFile << '\n';

        if (calc_lambda_max)
            std::cout << "calc_lambda_max = true" << '\n';

        if (do_not_write_events)
            std::cout << "do_not_write_events = true" << '\n';

        if (do_not_write_trajs)
            std::cout << "do_not_write_trajs = true" << '\n';

        if (do_not_write_resfile)
            std::cout << "do_not_write_resfile = true" << '\n';

        std::cout << "Using conc_epsilon = " << conc_epsilon << '\n';

        std::cout << "Using chem_limiter = " << chem_limiter << '\n';

        std::cout << "init_y = " << init_y << '\n';

#if BL_SPACEDIM==3
        std::cout << "init_z = " << init_z << '\n';
#endif
        std::cout << "init_spec = " << init_spec << '\n';

        std::cout << "res_spec = " << res_spec << '\n';

        std::cout << "n_particles = " << n_particles << '\n';

        std::cout << "part_start = " << part_start << '\n';

        std::cout << "build_prob_dist = " << build_prob_dist << '\n';
    }
}

static
void
PrintUsage (char* progName)
{
    if (ParallelDescriptor::IOProcessor())
        std::cout << "Usage: " << progName << "inputfile" << '\n';
    exit(1);
}

static
Real
FFF (Real s)
{
    const Real R1 = .007;
    const Real R2 = .014;

    const Real A = (R2*R2-R1*R1)/(4*std::log(R1/R2));
    const Real B = (R1*R1*log(R2)-R2*R2*log(R1))/(4*log(R1/R2));

    Real result = 8 * A * std::log(s);

    s *= s;

    result += (s + 8*B -4*A);

    result *= s/16;

    return result;
}

static Real LookupTable[1001];

static
void
BuildLookupTable ()
{
    const Real R1    = .007;
    const Real R2    = .014;
    const Real Dx    = .001*(R2-R1);
    const Real FFFR1 = FFF(R1);
    const Real FFFR2 = FFF(R2);

    for (int i = 0; i < 1001; i++)
    {
        Real s = R1+i*Dx;

        LookupTable[i] = (FFF(s)-FFFR1)/(FFFR2-FFFR1);
    }
}

static
Real
RandomXPos (BoxLib::mt19937& rand)
{
    const Real Rn  = rand.d1_value();
    const Real Rf = .006;

#if 0
    //
    // Carbon stuff.
    //
    return Rf * sqrt(1-sqrt(1-Rn));
#else
    //
    // Nitrogen stuff.
    //
    const Real Ratio1 =  220.0/3380.0;
    const Real Ratio2 = 3160.0/3380.0;
    const Real R1     = .007;
    const Real R2     = .014;
    const Real Dx     = .001*(R2-R1);

    if (Rn < Ratio1)
    {
        return Rf * sqrt(1-sqrt(1-Rn/Ratio1));
    }
    else
    {
        Real x = (Rn-Ratio1)/Ratio2;

        int i = 0;

        for ( ; i < 1000; i++)
            if (x < LookupTable[i])
                break;
        
        return R1+(i-1+(x-LookupTable[i-1])/(LookupTable[i]-LookupTable[i-1]))*Dx;
    }
#endif
}

static
void
ReadEdgeData ()
{
    BL_ASSERT(!EdgeDataFile.empty());

    if (ParallelDescriptor::IOProcessor())
        std::cout << "Reading edgefile ... " << std::endl;

    std::ifstream ifs;

    ifs.open(EdgeDataFile.c_str(), std::ios::in);

    if (!ifs.good())
        BoxLib::FileOpenFailed(EdgeDataFile);

    int N, cnt, tmp;

    ifs >> N;

    BL_ASSERT(N > 0);

    pedges.resize(N);

    if (ParallelDescriptor::IOProcessor())
        std::cout << "N = " << N << '\n';

    for (int i = 0; i < N; i++)
    {
        ifs >> cnt;

        BL_ASSERT(cnt > 0);

        pedges[i] = edgedata.size();

        edgedata.push_back(cnt);

        for (int j = 0; j < 4*cnt; j++)
        {
            ifs >> tmp;

            edgedata.push_back(tmp);
        }
    }

    if (!ifs.good())
        BoxLib::Abort("Failure reading EdgeData file");

    if (ParallelDescriptor::IOProcessor())
    {
        std::cout << "EdgeData on startup:\n";

        for (int i = 0; i < pedges.size(); i++)
        {
            int cnt = edgedata[pedges[i]];

            std::cout << cnt << ' ';

            for (int j = 1; j <= 4*cnt; j++)
                std::cout << edgedata[pedges[i]+j] << ' ';

            std::cout << '\n';
        }
    }
}

static
void
GetMFsFromPlotFile (const std::string& file,
                    PArray<MultiFab>&  pamf)
{
    std::cout << "Opening PlotFile: " << file << std::endl;

    DataServices::SetBatchMode();
    FileType fileType(NEWPLT);
    DataServices dataServices(file, fileType);

    if (!dataServices.AmrDataOk())
        //
        // This calls ParallelDescriptor::EndParallel() and exit()
        //
        DataServices::Dispatch(DataServices::ExitRequest, NULL);
    
    AmrData& amrData = dataServices.AmrDataRef();

    static bool first = true;

    if (first)
    {
        first = false;
        //
        // Set some other globals ...
        //
        prob_lo      = amrData.ProbLo();
        prob_hi      = amrData.ProbHi();
        ref_ratio    = amrData.RefRatio();
        prob_domain  = amrData.ProbDomain();
        finest_level = amrData.FinestLevel();

        dx.resize(finest_level+1);
        dxinv.resize(finest_level+1);

        for (int l = 0; l <= finest_level; l++)
        {
            dx[l]    = amrData.DxLevel()[l];
            dxinv[l] = amrData.DxLevel()[l];

            for (int i = 0; i < BL_SPACEDIM; i++)
                dxinv[l][i] = 1.0/dx[l][i];
        }
    }

    const int Nlev  = amrData.FinestLevel() + 1;
    const int Ncomp = amrData.NComp();

    pamf.resize(Nlev);

    for (int lev = 0; lev < Nlev; lev++)
    {
        pamf.set(lev,new MultiFab);

        std::cout << "Reading MultiFab data at lev=" << lev << " ... " << std::flush;

        std::string name = file;

        name += '/';

        char buf[64];
        sprintf(buf, "Level_%d", lev);

        name += buf;

        name += "/MultiFab";

        VisMF::Read(pamf[lev],name);

        std::cout << "done" << std::endl;
    }
}

static
void
ReadPlotFiles ()
{
    const int N = 8;

    const char* suffix[N] =
    {
        "_a", "_b", "_c", "_d", "_u", "_rf", "_rr", "_conc"
    };

    PArray<MultiFab>* pamf[N] =
    {
        &A_mf, &B_mf, &C_mf, &D_mf, &U_mf, &Rf_mf, &Rr_mf, &Conc_mf
    };

    std::string files[N];

    for (int i = 0; i < N; i++)
    {
        files[i] = PlotFile + suffix[i];
    }

    for (int i = 0; i < N; i++)
    {
        GetMFsFromPlotFile(files[i],*pamf[i]);
    }
    //
    // Some sanity checks ...
    //
    BL_ASSERT(A_mf.size() == B_mf.size());
    BL_ASSERT(A_mf.size() == C_mf.size());
    BL_ASSERT(A_mf.size() == D_mf.size());
    BL_ASSERT(A_mf.size() == U_mf.size());
    BL_ASSERT(A_mf.size() == Rf_mf.size());
    BL_ASSERT(A_mf.size() == Rr_mf.size());
    BL_ASSERT(A_mf.size() == Conc_mf.size());

    BL_ASSERT(A_mf[0].boxArray() == B_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == C_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == D_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == U_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == Rf_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == Rr_mf[0].boxArray());
    BL_ASSERT(A_mf[0].boxArray() == Conc_mf[0].boxArray());
    //
    // Print out some stuff ...
    //
    if (ParallelDescriptor::IOProcessor())
    {
        std::cout << "finest_level = " << finest_level << '\n';

        for (int l = 0; l <= finest_level; l++)
        {
            std::cout << "dx[" << l << "]: ";

            for (int i = 0; i < BL_SPACEDIM; i++)
                std::cout << dx[l][i] << ' ';
            std::cout << '\n';

            std::cout << "prob_domain[" << l << "]: " << prob_domain[l] << '\n';
        }

        std::cout << "ref_ratio: ";
        for (int l = 0; l < finest_level; l++)
            std::cout << ref_ratio[l] << ' ';
        std::cout << '\n';
    }
}

static
void
Twiddle_ABCD ()
{
    //
    // B = A + B; C = A + B + C; D = A + B + C + D
    //
    for (int l = 0; l <= finest_level; l++)
    {
        MultiFab& A = A_mf[l];
        MultiFab& B = B_mf[l];
        MultiFab& C = C_mf[l];
        MultiFab& D = D_mf[l];

        for (MFIter mfi(A); mfi.isValid(); ++mfi)
        {
            B[mfi] += A[mfi];
            C[mfi] += B[mfi];
            D[mfi] += C[mfi];
        }
    }
}

static
void
TwiddleConcentration ()
{
    //
    // The concentration is already normalized.
    // We invert it so we can multiply by it instead of divide.
    //
    for (int lev = 0; lev < Conc_mf.size(); lev++)
    {
        for (MFIter mfi(Conc_mf[lev]); mfi.isValid(); ++mfi)
        {
            Conc_mf[lev][mfi].invert(1.0);
        }
    }
    //
    // We now set those cells covered by fine grid to zero.
    // This simplified some of our algorithms.
    //
    const int Ncomp = Conc_mf[0].nComp();

    for (int l = 0; l <= finest_level; l++)
    {
        for (MFIter mfi(Conc_mf[l]); mfi.isValid(); ++mfi)
        {
            if (l < finest_level)
            {
                const BoxArray& f_ba = Conc_mf[l+1].boxArray();

                for (int j = 0; j < f_ba.size(); j++)
                {
                    Box c_box = BoxLib::coarsen(f_ba[j],ref_ratio[l]);

                    if (c_box.intersects(Conc_mf[l][mfi].box()))
                    {
                        Box isect = c_box & Conc_mf[l][mfi].box();

                        Conc_mf[l][mfi].setVal(0,isect,0,Ncomp);
                    }
                }
            }
        }
    }
}

static
void
WriteEvent (int           part,
            Array<Real>&  xpos,
            int           spec,
            int           reac,
            Real          time,
            std::ostream& event_ofs)
{
    BL_PROFILE("WriteEvent()");

    if (do_not_write_events) return;

    event_ofs << part << ' ';

    for (int i = 0; i < BL_SPACEDIM; i++)
    {
        event_ofs << xpos[i] << ' ';
    }

    event_ofs << spec << ' ' << reac << ' ' << time << '\n';

    if (!event_ofs.good()) BoxLib::Abort("Problem with the event output file");
}

static
void
WriteResTime (const std::string&      theResFile,
              const PArray<MultiFab>& ResTime)

{
    if (do_not_write_resfile) return;
    //
    // TODO -- implement this !!!
    //
#if 0
    std::ofstream res_ofs;

    res_ofs.open(theResFile.c_str(),std::ios::out|std::ios::trunc);

    if (!res_ofs.good()) BoxLib::FileOpenFailed(ResFile);

    ResTime.writeOn(res_ofs);
#endif
}

static
void
WriteTrajectory (int                         particleNo,
                 const std::vector<PartPos>& trajectory,
                 std::ofstream&              traj_ofs)

{
    BL_PROFILE("WriteTrajectory()");

    if (do_not_write_trajs) return;
    //
    // Only write out every N ...
    //
    const int N = 25;

    traj_ofs << particleNo << ":\n";

    for (int i = 0; i < trajectory.size(); i++)
    {
        if (i % N == 0)
        {
            traj_ofs << trajectory[i][0] << ' '
                     << trajectory[i][1] << ' '
                     << trajectory[i][2] << '\n';
        }
    }

    if (!traj_ofs.good()) BoxLib::Abort("Problem with the trajectory file");
}

inline
void
UpdateTrajectory (const Array<Real>&    x,
                  Real                  t,
                  std::vector<PartPos>& traj)
{
    if (do_not_write_trajs) return;

    PartPos p;

    D_TERM(p[0] = x[0];, p[1] = x[1];, p[2] = x[2];)

    p[BL_SPACEDIM] = t;

    traj.push_back(p);
}

static
void
MaxLambda ()
{
    if (ParallelDescriptor::IOProcessor())
        std::cout << "Calculating Max Lambda ..." << std::endl;

    const int nspec = pedges.size();

    BL_ASSERT(nspec == Conc_mf[0].nComp());
    BL_ASSERT(finest_level+1 == Conc_mf.size());
    //
    // Now do the appropriate computation over levels using C_tmp.
    //
    // This relies on the Concentration being zero'd under fine grids.
    //
    for (int spec = 0; spec < nspec; spec++)
    {
        Real lmax = 0;

        const int start = pedges[spec];
        const int nedge = edgedata[start];

        for (int l = 0; l <= finest_level; l++)
        {
            const MultiFab& Rf   = Rf_mf[l];
            const MultiFab& Rr   = Rr_mf[l];
            const MultiFab& Conc = Conc_mf[l];

            for (MFIter mfi(Conc); mfi.isValid(); ++mfi)
            {
                const Box& bx = Conc[mfi].box();

                const FArrayBox& rf   = Rf[mfi];
                const FArrayBox& rr   = Rr[mfi];
                const FArrayBox& conc = Conc[mfi];

                for (IntVect p = bx.smallEnd(); p <= bx.bigEnd(); bx.next(p))
                {
                    Real lambda = 0;

                    for (int ie = 0; ie < nedge; ie++)
                    {
                        const int  rxnid  = edgedata[start+ie*4+1];
                        const int  factor = edgedata[start+ie*4+2];
                        const int  nu     = edgedata[start+ie*4+4];
                        const Real X      = nu*conc(p,spec)/factor;

                        if (factor > 0)
                        {
                            lambda += X*rf(p,rxnid);
                        }
                        else
                        {
                            lambda -= X*rr(p,rxnid);
                        }
                    }

                    lmax = std::max(lmax,lambda);
                }
            }
        }

        if (ParallelDescriptor::IOProcessor())
        {
            std::cout  << "spec = " << spec << ", lmax = " << lmax << std::endl;
        }
    }

    exit(0);
}

static
void
MaxDtLambda (int            spec,
             Real&          dt,
             int            lev,
             int            idx,
             const IntVect& iv)
{
    BL_PROFILE("MaxDtLambda()");

    Real lambda = 0;

    const int start = pedges[spec];
    const int nedge = edgedata[start];

    const FArrayBox& rf   = Rf_mf[lev][idx];
    const FArrayBox& rr   = Rr_mf[lev][idx];
    const FArrayBox& conc = Conc_mf[lev][idx];

    for (int ie = 0; ie < nedge; ie++)
    {
        const int rxnid  = edgedata[start+ie*4+1];
        const int factor = edgedata[start+ie*4+2];
        const int nu     = edgedata[start+ie*4+4];

        const Real X = nu*conc(iv,spec)/factor;

        if (factor > 0)
        {
            lambda += X*rf(iv,rxnid);
        }
        else
        {
            lambda -= X*rr(iv,rxnid);
        }
    }
    
    if (lambda > 0) dt = std::min(dt,chem_limiter/lambda);
}

static
void
Chemistry (int&             spec,
           int&             rxn,
           Real             dt,
           int              lev,
           int              idx,
           const IntVect&   iv,
           BoxLib::mt19937& rand)
{
    BL_PROFILE("Chemistry()");

    const Real rn = rand.d1_value();

    Real lambda = 0;

    const int start = pedges[spec];
    const int nedge = edgedata[start];

    const FArrayBox& rf   = Rf_mf[lev][idx];
    const FArrayBox& rr   = Rr_mf[lev][idx];
    const FArrayBox& conc = Conc_mf[lev][idx];

    for (int ie = 0; ie < nedge; ie++)
    {
        const int  rxnid  = edgedata[start+ie*4+1];
        const int  factor = edgedata[start+ie*4+2];
        const int  tospec = edgedata[start+ie*4+3];
        const int  nu     = edgedata[start+ie*4+4];

        const Real X = nu*conc(iv,spec)*dt/factor;

        if (factor > 0)
        {
            lambda += X*rf(iv,rxnid);
        }
        else
        {
            lambda -= X*rr(iv,rxnid);
        }

        if (rn < lambda)
        {
            rxn  = rxnid;
            spec = tospec;

            break;
        }
    }
}

inline
IntVect
Index (const Array<Real>& x,
       int                lev)
{
    BL_PROFILE("Index()");

    IntVect iv;

    const IntVect& lo = prob_domain[lev].smallEnd();
        
    D_TERM(iv[0]=int((x[0]-lo[0])*dxinv[lev][0]);,
           iv[1]=int((x[1]-lo[1])*dxinv[lev][1]);,
           iv[2]=int((x[2]-lo[2])*dxinv[lev][2]););

    iv += lo;

    return iv;
}

static
void
Where (const Array<Real>& x,
       int&               lev,
       int&               idx,
       IntVect&           iv)
{
    BL_PROFILE("Where()");

    for (lev = finest_level; lev >= 0; lev--)
    {
        iv = Index(x,lev);

        const BoxArray& ba = A_mf[lev].boxArray();

        for (idx = 0; idx < ba.size(); idx++)
            if (ba[idx].contains(iv))
                return;
    }

    if (ParallelDescriptor::IOProcessor())
    {
        std::cout << "Couldn't find position: ( ";
        for (int i = 0; i < BL_SPACEDIM; i++)
            std::cout << x[i] << ' ';
        std::cout << ')' << std::endl;
    }

    BoxLib::Abort("Where() failed");
}

static
void
GetInitialPosition (int&                  spec,
                    int                   rxn,
                    Real                  t,
                    std::vector<PartPos>& traj,
                    int                   i,
                    Array<Real>&          x,
                    IntVect&              iv,
                    int&                  lev,
                    int&                  idx,
                    std::ostream&         event_ofs,
                    BoxLib::mt19937&      rand)
{
    D_TERM(x[0]=(build_prob_dist?0:RandomXPos(rand));,x[1]=init_y;,x[2]=init_z;)

    if (build_prob_dist)
    {
        const Real rn = rand.d1_value();
        //
        // Find level and index in Prob_mf containing appropriate probability.
        //
        int l, i;
        //
        // Find first AccumProb[lev][idx] combo >= rn ...
        //
        for (l = 0; l <= finest_level; l++)
            for (i = 0; i < AccumProb[l].size(); i++)
                if (rn <= AccumProb[l][i])
                    goto gotit;
    gotit:

        BL_ASSERT(l >= 0 && l <= finest_level);
        BL_ASSERT(i >= 0 && i < AccumProb[l].size());

        const FArrayBox& Prob = Prob_mf[l][i];
        const int*       p_lo = Prob.box().loVect();
        const int*       p_hi = Prob.box().hiVect();

        FORT_SELECTPOS(Prob.dataPtr(),
                       ARLIM(p_lo), ARLIM(p_hi),
                       &rn,
                       D_DECL(&dx[l][0],&dx[l][1],&dx[l][2]),
                       D_DECL(&x[0],&x[1],&x[2]));
    }

    UpdateTrajectory(x,t,traj);

    WriteEvent(i,x,spec,rxn,t,event_ofs);
    //
    // Now that we have x set lev, idx and iv ...
    //
    Where(x,lev,idx,iv);

    if (build_prob_dist)
    {
        Real dt_tmp = 1.0e20;

        MaxDtLambda(spec,dt_tmp,lev,idx,iv);

        dt_tmp *= 10;

        const int old_spec = spec;

        Chemistry(spec,rxn,dt_tmp,lev,idx,iv,rand);

        if (spec != old_spec) WriteEvent(i,x,spec,rxn,t,event_ofs);
    }
}

static
void
BuildProbDist ()
{
    Prob_mf.resize(finest_level+1);

    AccumProb.resize(finest_level+1);

    for (int l = 0; l <= finest_level; l++)
    {
        Prob_mf.set(l, new MultiFab(A_mf[l].boxArray(),1,0));

        AccumProb[l].resize(Prob_mf[l].size(),0);

        Prob_mf[l].setVal(0);
    }

    const int elen  = edgedata.size();
    const int nspec = pedges.size();
    const int nreac = Rf_mf[0].nComp();

    for (int l = 0; l <= finest_level; l++)
    {
        for (MFIter mfi(Prob_mf[l]); mfi.isValid(); ++mfi)
        {
            const int* a_lo = Rf_mf[l][mfi].box().loVect();
            const int* a_hi = Rf_mf[l][mfi].box().hiVect();
            const int* p_lo = Prob_mf[l][mfi].box().loVect();
            const int* p_hi = Prob_mf[l][mfi].box().hiVect();

            FORT_PROBFAB(Rf_mf[l][mfi].dataPtr(),
                         ARLIM(a_lo), ARLIM(a_hi),
                         Rr_mf[l][mfi].dataPtr(),
                         Prob_mf[l][mfi].dataPtr(),
                         ARLIM(p_lo), ARLIM(p_hi),
                         &nspec, &nreac, &init_spec,
                         (int*)&edgedata[0], &elen, pedges.dataPtr(),
                         D_DECL(&dx[l][0],&dx[l][1],&dx[l][2]));

            if (l < finest_level)
            {
                const BoxArray& f_ba = Prob_mf[l+1].boxArray();

                for (int j = 0; j < f_ba.size(); j++)
                {
                    Box c_box = BoxLib::coarsen(f_ba[j],ref_ratio[l]);

                    if (c_box.intersects(Prob_mf[l][mfi].box()))
                    {
                        Box isect = c_box & Prob_mf[l][mfi].box();

                        Prob_mf[l][mfi].setVal(0,isect,0);
                    }
                }
            }
        }
    }

    Real totreact = 0;

    for (int l = 0; l <= finest_level; l++)
        for (MFIter mfi(Prob_mf[l]); mfi.isValid(); ++mfi)
            totreact += Prob_mf[l][mfi].sum(0);

    Real cumprob = 0;

    for (int l = 0; l <= finest_level; l++)
    {
        for (MFIter mfi(Prob_mf[l]); mfi.isValid(); ++mfi)
        {
            const int* p_lo = Prob_mf[l][mfi].box().loVect();
            const int* p_hi = Prob_mf[l][mfi].box().hiVect();

            FORT_ACCUMPROB(Prob_mf[l][mfi].dataPtr(),
                           ARLIM(p_lo),
                           ARLIM(p_hi),
                           &totreact,
                           &cumprob);
        }
    }

    if (ParallelDescriptor::IOProcessor())
    {
        std::cout << "cumulative probability: " << cumprob << std::endl;
    }

    totreact = 0;

    for (int l = 0; l <= finest_level; l++)
    {
        for (MFIter mfi(Prob_mf[l]); mfi.isValid(); ++mfi)
        {
            totreact += Prob_mf[l][mfi].sum(0);

            AccumProb[l][mfi.index()] = totreact;
        }
    }
    //
    // Force sum to 1 just to be safe !!!
    //
    AccumProb[finest_level][AccumProb[finest_level].size()-1] = 1;
}

static
void
MaxOverBox (int        spec,
            int        lev,
            const Box& bx,
            Real&      dmax)
{
    for (MFIter mfi(D_mf[lev]); mfi.isValid(); ++mfi)
    {
        const FArrayBox& D = D_mf[lev][mfi];

        const Box isect = bx & D.box();

        if (isect.ok())
        {
            IntVect p = isect.smallEnd();

            for ( ; p <= isect.bigEnd(); isect.next(p))
            {
                dmax = std::max(dmax,D(p,spec));
            }
        }
    }
}

static
void
MaxDtDiffusion (int            spec,
                Real&          dt,
                const IntVect& iv,
                int            lev)
{
    BL_PROFILE("MaxDtDiffusion()");

    Box bx(D_DECL(iv,iv,iv));

    Real dmax = 0;

    MaxOverBox(spec,
               lev,
               BoxLib::grow(bx,2),
               dmax);

    if (lev < finest_level)
    {
        MaxOverBox(spec,
                   lev+1,
                   BoxLib::grow(BoxLib::refine(bx,ref_ratio[lev]),2), 
                   dmax);
    }

    if (dmax > 0) dt = std::min(dt,0.1/dmax);
}

static
void
AdvanceParticles (int  tid,
                  int  beg,
                  int  cnt,
                  Real dt)
{
    BL_PROFILE("AdvanceParticles()");

    const int  NoReac  = -1;
    const Real dt_save = dt;
    //
    // Append node & thread id to file names ...
    //
    std::string theResFile   = ResFile;
    std::string theTrajFile  = TrajFile;
    std::string theEventFile = EventFile;

    char buf[12];
    
    sprintf(buf, ".%02d.%02d", ParallelDescriptor::MyProc(), tid);

    theResFile += buf; theTrajFile += buf; theEventFile += buf;

    std::ofstream traj_ofs, event_ofs;

    if (!do_not_write_events)
    {
        event_ofs.open(theEventFile.c_str(),std::ios::out|std::ios::trunc);

        if (!event_ofs.good())
            BoxLib::FileOpenFailed(EventFile);

#ifndef __GNUC__
        event_ofs << std::scientific; 
#endif
    }

    if (!do_not_write_trajs)
    {
        traj_ofs.open (theTrajFile.c_str(), std::ios::out|std::ios::trunc);

        if (!traj_ofs.good())
            BoxLib::FileOpenFailed(TrajFile);

#ifndef __GNUC__
        traj_ofs  << std::scientific;
#endif
    }

    int                  lev;  // What level in the AmrData hierarchy?
    int                  idx;  // What index into BoxArray at that level?
    IntVect              iv;   // What cell position in that Box?
    IntVect              iv_tmp;
    Real                 t;
    Array<Real>          x(BL_SPACEDIM);
    std::vector<PartPos> traj;
    BoxLib::mt19937      rand(seed+beg);
    int                  counter = 0;
    PArray<MultiFab>     ResTime(PArrayManage);

    if (!do_not_write_resfile)
    {
        ResTime.resize(finest_level+1);

        for (int l = 0; l <= finest_level; l++)
        {
            ResTime.set(l, new MultiFab(A_mf[l].boxArray(),1,0));

            ResTime[l].setVal(0);
        }
    }

    for (int i = beg; i < cnt+beg; i++)
    {
        int  rxn          = NoReac;
        int  spec         = init_spec;
        bool reacted      = dt_spec < 0 ? true : false;
        bool calc_diff_dt = true;
        Real dt_diff_save = dt_save;
        //
        // We always begin from zero.
        //
        t = 0;
        //
        // This sets x, iv, lev and idx (among possibly others) ...
        //
        GetInitialPosition(spec,rxn,t,traj,i,x,iv,lev,idx,event_ofs,rand);

        while (t < stop_time)
        {
            dt = dt_diff_save;
            //
            // Get maximum dt for diffusion.
            //
            if (calc_diff_dt)
            {
                dt = dt_save;

                MaxDtDiffusion(spec,dt,iv,lev);

                dt_diff_save = dt;

                calc_diff_dt = false;
            }
            //
            // Advect x ...
            //
            D_TERM(x[0]+=U_mf[lev][idx](iv,0)*dt;,
                   x[1]+=U_mf[lev][idx](iv,1)*dt;,
                   x[2]+=U_mf[lev][idx](iv,2)*dt;);

            if (x[0] < 0) x[0] = -x[0];

            iv_tmp = Index(x,lev);

            if (!(iv == iv_tmp))
            {
                //
                // We're in a different cell.
                //
                if (!prob_domain[lev].contains(iv_tmp)) goto done;

                calc_diff_dt = true;

                const FArrayBox& c = Conc_mf[lev][idx];

                if (c.box().contains(iv_tmp) && c(iv_tmp,0) > 0)
                {
                    //
                    // Recall that concentration is nonzero except under
                    // fine grid.  If we got here we must still be in the
                    // same fab.  That it to say, lev and idx are OK, just
                    // set iv.
                    //
                    iv = iv_tmp;
                }
                else
                {
                    Where(x,lev,idx,iv);
                }
            }
            //
            // Diffuse x ...
            //
            const Real rn = rand.d1_value();

            if (rn < A_mf[lev][idx](iv,spec)*dt)
            {
                x[0] -= dx[lev][0];
            }
            else if (rn < B_mf[lev][idx](iv,spec)*dt)
            {
                x[0] += dx[lev][0];
            }
            else if (rn < C_mf[lev][idx](iv,spec)*dt)
            {
                x[1] -= dx[lev][1];
            }
            else if (rn < D_mf[lev][idx](iv,spec)*dt)
            {
                x[1] += dx[lev][1];
            }

            t += dt;

            iv_tmp = Index(x,lev);

            if (!(iv == iv_tmp))
            {
                //
                // We're in a different cell.
                //
                if (!prob_domain[lev].contains(iv_tmp)) goto done;

                calc_diff_dt = true;

                const FArrayBox& c = Conc_mf[lev][idx];

                if (c.box().contains(iv_tmp) && c(iv_tmp,0) > 0)
                {
                    //
                    // Recall that concentration is nonzero except under
                    // fine grid.  If we got here we must still be in the
                    // same fab.  That it to say, lev and idx are OK, just
                    // set iv.
                    //
                    iv = iv_tmp;
                }
                else
                {
                    Where(x,lev,idx,iv);
                }

                UpdateTrajectory(x,t,traj);
            }
            //
            // The particle does not move from here on down.
            //
            Real tchem        = 0;
            Real dt_chem      = dt;
            bool calc_chem_dt = true;

            do
            {
                if (calc_chem_dt)
                {
                    dt_chem      = dt;
                    calc_chem_dt = false;

                    MaxDtLambda(spec,dt_chem,lev,idx,iv);
                }

                dt_chem = std::min(dt_chem,dt-tchem);

                if (!reacted) dt_chem = dt_spec;

                const int old_spec = spec;

                if (spec == res_spec && !do_not_write_resfile)
                {
                    ResTime[lev][idx](iv) += 0.9*dt_chem;
                }

                Chemistry(spec,rxn,dt_chem,lev,idx,iv,rand);

                if (spec == res_spec && !do_not_write_resfile)
                {
                    ResTime[lev][idx](iv) += 0.1*dt_chem;
                }

                if (spec != old_spec)
                {
                    reacted = calc_chem_dt = true;
                    WriteEvent(i,x,spec,rxn,t,event_ofs);
                }

                tchem += dt_chem;
            }
            while (tchem < dt);
        }

    done:

        if (++counter % restime_interval == 0 || i == cnt+beg-1)
        {
            WriteResTime(theResFile,ResTime);
        }

        WriteEvent(i,x,spec,NoReac,t,event_ofs);

        UpdateTrajectory(x,t,traj);

        if (spec != init_spec) WriteTrajectory(i,traj,traj_ofs);

        traj.clear();
    }
}

class task_advance
    :
    public WorkQueue::task
{
public:

    task_advance (int tid, int beg, int cnt, Real dt)
        :
        m_tid(tid),
        m_beg(beg),
        m_cnt(cnt),
        m_dt(dt)
    {}
    virtual void run ();

private:

    int  m_tid;
    int  m_beg;
    int  m_cnt;
    Real m_dt;
};

void
task_advance::run ()
{
    BL_PROFILE(BL_PROFILE_THIS_NAME() + "::run()");

    AdvanceParticles(m_tid,m_beg,m_cnt,dt);
}
       
int
main (int   argc,
      char* argv[])
{
    BoxLib::Initialize(argc,argv);

    const int NProcs = ParallelDescriptor::NProcs();
    const int MyProc = ParallelDescriptor::MyProc();
    const int IOProc = ParallelDescriptor::IOProcessorNumber();

    if (argc < 2) PrintUsage(argv[0]);

    ParmParse pp;

    ScanArguments(pp);

    ReadPlotFiles();

    TwiddleConcentration();

    Twiddle_ABCD();

    ReadEdgeData();
    //
    // Don't measure the time to read plotfiles ...
    //
    const Real run_strt = ParallelDescriptor::second();

    BL_ASSERT(pedges.size() > 0 && edgedata.size() > 0);

    BL_ASSERT(pedges.size() == Conc_mf[0].nComp());

    if (calc_lambda_max) MaxLambda();

    if (build_prob_dist) BuildProbDist();

    BuildLookupTable();

    int nthreads = BoxLib::theWorkQueue().max_threads();

    if (nthreads == 0) nthreads = 1;

    const int ParticlesPerNode   = n_particles / NProcs;
    const int ParticlesPerThread = ParticlesPerNode / nthreads;

    for (int tid = 0; tid < nthreads; tid++)
    {
        int cnt = ParticlesPerThread;

        int beg = part_start + ParticlesPerNode * MyProc + cnt * tid;

        if (tid == nthreads - 1)
        {
            //
            // Last thread per node gets remaining per node particles.
            //
            cnt += ParticlesPerNode % nthreads;

            if (NProcs > 1 && MyProc == NProcs - 1)
                //
                // Last thread in last node gets all remaining particles.
                //
                cnt += n_particles % NProcs;
        }

        std::cout << "Thread: "
                  << tid
                  << ", node: "
                  << MyProc
                  << ", particles: ["
                  << beg
                  << ','
                  << beg+cnt
                  << ")\n";

        BoxLib::theWorkQueue().add(new task_advance(tid,beg,cnt,dt));
    }

    BoxLib::theWorkQueue().wait();

    Real run_stop = ParallelDescriptor::second() - run_strt;

    ParallelDescriptor::ReduceRealMax(run_stop, IOProc);

    if (ParallelDescriptor::IOProcessor())
    {
        std::cout << "\n**Run time = "
                  << run_stop
                  << "\n**Avg Runtime/Particle: "
                  << run_stop/n_particles
                  << std::endl;
    }

    BoxLib::Finalize();
}
