/*
** (c) 1996-2000 The Regents of the University of California (through
** E.O. Lawrence Berkeley National Laboratory), subject to approval by
** the U.S. Department of Energy.  Your use of this software is under
** license -- the license agreement is attached and included in the
** directory as license.txt or you may contact Berkeley Lab's Technology
** Transfer Department at TTD@lbl.gov.  NOTICE OF U.S. GOVERNMENT RIGHTS.
** The Software was developed under funding from the U.S. Government
** which consequently retains certain rights as follows: the
** U.S. Government has been granted for itself and others acting on its
** behalf a paid-up, nonexclusive, irrevocable, worldwide license in the
** Software to reproduce, prepare derivative works, and perform publicly
** and display publicly.  Beginning five (5) years after the date
** permission to assert copyright is obtained from the U.S. Department of
** Energy, and subject to any subsequent five (5) year renewals, the
** U.S. Government is granted for itself and others acting on its behalf
** a paid-up, nonexclusive, irrevocable, worldwide license in the
** Software to reproduce, prepare derivative works, distribute copies to
** the public, perform publicly and display publicly, and to permit
** others to do so.
*/

#include <winstd.H>

#include <algorithm>
#include <string>
#include <iostream>
#include <iomanip>
#include <fstream>
#include <stdiostream.h>

#include <Utility.H>
#include <REAL.H>
#include <IntVect.H>
#include <Box.H>
#include <Grid.H>
#include <globals.H>
#include <Array.H>
#include <GRID_F.H>
#include <ParmParse.H>
#include <VisMF.H>
#include <BCTypes.H>

//
// This MUST be defined if don't have pubsetbuf() in I/O Streams Library.
//
#ifdef BL_USE_SETBUF
#define pubsetbuf setbuf
#endif

#if (BL_SPACEDIM == 2)
#define N_STATE 4
#elif (BL_SPACEDIM == 3)
#define N_STATE 5
#endif

int Grid::slope_order = 4;
int Grid::init_iter   = 2;
Real Grid::fixed_dt   = -1.0;

int Grid::verbose     = 0;
int Grid::writeTurb   = 0;

// ************************************************************************
// ** constructor ** 
// ************************************************************************

Grid::Grid(BoxArray Grids, Geometry& Geom, std::string probin_file_read):
    grids(Grids), geom(Geom)
{
   probin_file = probin_file_read;

   numscal = N_STATE - BL_SPACEDIM;

   parseInputs();

   dx = geom.CellSize();
  
   is_rz = CoordSys::IsRZ();

   geom.GetVolume(vol,grids,1);
   for (int n = 0; n < BL_SPACEDIM; n++)
     geom.GetFaceArea(area[n],grids,n,0);

   vol.FillBoundary(0,1);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(vol,0,1,true);
}

// ************************************************************************
// ** parseInputs ** 
// ************************************************************************

void
Grid::parseInputs()
{
    ParmParse pp("grid");

    is_conserv.resize(numscal);
    int in_inputs = pp.countval("is_conserv");
    if (in_inputs != numscal) {
      if (ParallelDescriptor::IOProcessor())
        std::cout << "WRONG NUMBER OF INPUTS FOR grid.is_conserv " << std::endl;
      exit(0);
    }
     
    pp.getarr("is_conserv",is_conserv,0,numscal);

    for (int i = 0; i < numscal; i++) {
      if (is_conserv[i] != 0 && is_conserv[i] != 1) {
        if (ParallelDescriptor::IOProcessor())
          std::cout << "each value of is_conserv must be 0 or 1 " << std::endl;
        exit(0);
      }
    }

    visc_coef = 0.0;
    pp.query("visc_coef",visc_coef);

    diff_coef.resize(numscal);
    in_inputs = pp.countval("diff_coef");
    if (in_inputs != numscal) {
      if (ParallelDescriptor::IOProcessor())
        std::cout << "WRONG NUMBER OF INPUTS FOR grid.diff_coef " << std::endl;
      exit(0);
    }

    pp.getarr("diff_coef",diff_coef,0,numscal);

    if (diff_coef[0] > 0.0) {
      if (ParallelDescriptor::IOProcessor())
        std::cout << "DENSITY SHOULD NOT DIFFUSE " << std::endl;
      exit(0);
    }

    pp.query("slope_order",slope_order);

    if (slope_order != 0 && slope_order != 2 && slope_order != 4) {
      if (ParallelDescriptor::IOProcessor())
        std::cout << "slope_order must be 0 or 2 or 4" << std::endl;
      exit(0);
    }

    pp.query("init_iter",init_iter);
    pp.query("restart",restart_file);

    pp.query("fixed_dt",fixed_dt);

    pp.query("verbose",verbose);

    pp.query("writeTurb",writeTurb);
}

// ************************************************************************
// ** init ** 
// ************************************************************************

// decide whether to initialize from beginning or from restart
void
Grid::init(int& nstep, Real& time, Real& dt)
{
    int probin_file_length = probin_file.length();

    Array<int> probin_file_name(probin_file_length);

    for (int i = 0; i < probin_file_length; i++)
        probin_file_name[i] = probin_file[i];

    FORT_PROBINIT(probin_file_name.dataPtr(),
                  &probin_file_length);

    init_data_structures();

    if (restart_file != "") {

      restart(restart_file,nstep,time,dt);

    } else {

      initialInit(nstep,time,dt);

    }
}

// ************************************************************************
// ** init_data_structures
// ************************************************************************

void 
Grid::init_data_structures()
{
   int ngrow       = 1;
   int ngrow_state = 3;

   state.define(grids,N_STATE,ngrow_state,Fab_allocate);
   staten.define(grids,N_STATE,ngrow_state,Fab_allocate);
   slopex.define(grids,N_STATE,ngrow,Fab_allocate);
   slopey.define(grids,N_STATE,ngrow,Fab_allocate);
#if (BL_SPACEDIM == 3)
   slopez.define(grids,N_STATE,ngrow,Fab_allocate);
#endif
   rhonph.define(grids,      1,ngrow,Fab_allocate);

   visc.define(grids,   BL_SPACEDIM,ngrow,Fab_allocate);
   visc.setVal(0.0);

   diff.define(grids,numscal,ngrow,Fab_allocate);
   diff.setVal(0.0);

   BoxArray nodal_grids(grids);
   nodal_grids.surroundingNodes();
   pressure.define(nodal_grids,1,0,Fab_allocate);
   pressure.setVal(0.0);

   gradp.define(grids,BL_SPACEDIM,ngrow,Fab_allocate);
   gradp.setVal(0.0);

   force.define(grids,BL_SPACEDIM,ngrow,Fab_allocate);
   force.setVal(0.0);

   scalforce.define(grids,numscal,ngrow,Fab_allocate);
   scalforce.setVal(0.0);

   //
   //  These are the edge-based advection velocities and fluxes.
   //
   BoxArray edgex_grids(grids);
   edgex_grids.surroundingNodes(0);
   uadv.define(edgex_grids,1,0,Fab_allocate);
   edgex.define(edgex_grids,N_STATE,0,Fab_allocate);

   BoxArray edgey_grids(grids);
   edgey_grids.surroundingNodes(1);
   vadv.define(edgey_grids,1,0,Fab_allocate);
   edgey.define(edgey_grids,N_STATE,0,Fab_allocate);

#if (BL_SPACEDIM == 3)
   BoxArray edgez_grids(grids);
   edgez_grids.surroundingNodes(2);
   wadv.define(edgez_grids,1,0,Fab_allocate);
   edgez.define(edgez_grids,N_STATE,0,Fab_allocate);
#endif

   //
   //  These are the edge velocities used to construct transverse derivatives.
   //
   utrans.define(edgex_grids,1,1,Fab_allocate);
   utrans.setVal(0.);

   vtrans.define(edgey_grids,1,1,Fab_allocate);
   vtrans.setVal(0.);

#if (BL_SPACEDIM == 3)
   wtrans.define(edgez_grids,1,1,Fab_allocate);
   wtrans.setVal(0.);
#endif

   proj = new hg_projector(grids,geom,area,vol);

   macproj = new mac_projector(grids,geom,area,vol);

   diffuse_op = new diffuser(grids,geom,area,vol);
}

// ************************************************************************
// ** initialInit ** 
// ************************************************************************

void 
Grid::initialInit(int& nstep, Real& time, Real& dt) 
{
   nstep = 0;
   time = 0.0;
   dt = 0.0;

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     FORT_INITDATA(state[mfi].dataPtr(),ARLIM(lo),ARLIM(hi),
                   dx,&time,&numscal);
   }

   state.FillBoundary(Xvel,BL_SPACEDIM+numscal);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(state,Xvel,BL_SPACEDIM+numscal,true);

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();
     FORT_SETVELBC(state[mfi].dataPtr(Xvel), 
                   ARLIM(lo),ARLIM(hi),bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                   &is_rz,
#endif
                   &visc_coef,dx,&time);

     const int* slo = state[mfi].loVect();
     const int* shi = state[mfi].hiVect();

     for (int n = 0; n < numscal; n++) {
       FORT_SETSCALBC(state[mfi].dataPtr(Density+n),ARLIM(lo),ARLIM(hi),
                      ARLIM(slo),ARLIM(shi),
                      bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                      &is_rz,
#endif
                      &n,dx,&time);
     }
   }

   MultiFab rhotemp(grids,1,1);
   rhotemp.setVal(1.0);

   BoxArray nodal_grids(grids);
   nodal_grids.surroundingNodes();
   MultiFab divusrc(nodal_grids,1,0);

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     FORT_MKDIVUNOD(divusrc[mfi].dataPtr(),state[mfi].dataPtr(Xvel),
                    state[mfi].dataPtr(Density),
                    &time,dx,ARLIM(lo),ARLIM(hi));
   }

   proj->project(&state,&pressure,&rhotemp,&divusrc,time,dt);

   pressure.setVal(0.0);

   state.FillBoundary(Xvel,BL_SPACEDIM);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(state,Xvel,BL_SPACEDIM,true);

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     FORT_SETVELBC(state[mfi].dataPtr(Xvel),
                   ARLIM(lo),ARLIM(hi),bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                   &is_rz,
#endif
                   &visc_coef,dx,&time);
   }

   staten.setVal(0.);
   staten.copy(state,0,0,N_STATE);
}

// ************************************************************************
// ** restart ** 
// ************************************************************************

void
Grid::restart(std::string& filename, int& nstep, Real& time, Real& dt)
{
   Real dRestartTime0 = ParallelDescriptor::second();

   if (verbose && ParallelDescriptor::IOProcessor())
       std::cout << "Restarting calculation from file: " << filename << std::endl;
   //
   // Open the checkpoint header file for reading.
   //
   std::string File = filename;

   File += '/';
   File += "Header";

   VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);

   std::ifstream is;

   is.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());

   is.open(File.c_str(), std::ios::in);

   if (!is.good())
       BoxLib::FileOpenFailed(File);

   // Read the checkpoint version.
   std::string first_line;
   std::getline(is,first_line);

   // Read the number of space dimensions.
   int     spdim;
   is >> spdim;
   BL_ASSERT(BL_SPACEDIM == spdim);

   // Read time, geom, dt, nstep
   is >> time;
   is >> geom;
   is >> dt;
   is >> nstep;

   // Read the state data.
   {
   std::string mf_name;
   is >> mf_name;
   std::string FullPathName = filename;
   if (!filename.empty() && filename[filename.length()-1] != '/')
       FullPathName += '/';
   FullPathName += mf_name;
   MultiFab* new_data = new MultiFab;
   VisMF::Read(*new_data, FullPathName);
   MultiFab::Copy(state,*new_data,0,0,N_STATE,state.nGrow());
   delete new_data;
   }

   // Read the pressure data.
   {
   std::string mf_name;
   is >> mf_name;
   std::string FullPathName = filename;
   if (!filename.empty() && filename[filename.length()-1] != '/')
       FullPathName += '/';
   FullPathName += mf_name;
   MultiFab* new_data = new MultiFab;
   VisMF::Read(*new_data, FullPathName);
   MultiFab::Copy(pressure,*new_data,0,0,1,0);
   delete new_data;
   }

   state.FillBoundary(Xvel,BL_SPACEDIM+numscal);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(state,Xvel,N_STATE,true);

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     FORT_SETVELBC(state[mfi].dataPtr(Xvel),
                   ARLIM(lo),ARLIM(hi),bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                   &is_rz,
#endif
                   &visc_coef,dx,&time);

     const int* slo = state[mfi].loVect();
     const int* shi = state[mfi].hiVect();

     for (int n = 0; n < numscal; n++) {
       FORT_SETSCALBC(state[mfi].dataPtr(Density+n),ARLIM(lo),ARLIM(hi),
                      ARLIM(slo),ARLIM(shi),
                      bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                      &is_rz,
#endif
                      &n,dx,&time);
     }
   }

    //
    // Initialize local stats indicating this is a restart.
    //
    const int IOProc = ParallelDescriptor::IOProcessorNumber();

    Real dRestartTime1 = ParallelDescriptor::second();
    Real dRestartTime  = dRestartTime1 - dRestartTime0;

    ParallelDescriptor::ReduceRealMax(dRestartTime,IOProc);

    if (ParallelDescriptor::IOProcessor())
        std::cout << "Restart time = " << dRestartTime << " seconds." << std::endl;
}

// ************************************************************************
// ** destructor ** 
// ************************************************************************

Grid::~Grid() {
   delete proj;
   delete macproj;
   delete diffuse_op;
}

// ************************************************************************
// ** writeCheckPoint ** 
// ************************************************************************

void 
Grid::writeCheckPoint(int nstep, Real time, Real dt, 
		      std::string& check_file_root) 
{
  // construct file name
   std::string chkfile = check_file_root;
   char buf[sizeof(int)+1];
   sprintf (buf,"%04d",nstep);
   chkfile += buf;

   const char* chkfile_sep_str ="/";

   if (chkfile.length() == 0) {
     BoxLib::Error("invalid check_file_root");
   }

   if (strchr(chkfile.c_str(), *chkfile_sep_str) == 0) {
     // No slashes in the path, so no need to create directory.
   } else {
     // Make copy of the directory pathname so we can write to it.
     char* path = new char[chkfile.length() + 1];
     (void) strcpy(path, chkfile.c_str());

     char* slash = strrchr(path, *chkfile_sep_str);
     int fileNameLength = strlen(slash);
     int entireNameLength = strlen(path);
     int dirNameLength = entireNameLength - fileNameLength;

     char* chkdir= new char[dirNameLength+1];
     strncpy(chkdir,path,dirNameLength);
     chkdir[dirNameLength] = path[entireNameLength];
     
     // create the directory
     if (!BoxLib::UtilCreateDirectory(chkdir,0755)){
       BoxLib::CreateDirectoryFailed(chkdir);
     }
     delete [] chkdir;
     delete [] path;
   }

      // open stream
   std::ofstream os(chkfile.c_str(),std::ios::out);
   os.precision(15);

   os << nstep << '\n';
   os << time << '\n';
   os << dt << '\n';
   os << geom << '\n';

   for (MFIter statemfi(state); statemfi.isValid(); ++statemfi)
   {
     state[statemfi].writeOn(os);
     pressure[statemfi].writeOn(os);
   }
}

void
Grid::printMaxMin()
{
   Real umin = state.min(Xvel,0);
   Real umax = state.max(Xvel,0);
   if (ParallelDescriptor::IOProcessor()) 
     std::cout << std::endl << "UMAX/UMIN " << umax << " " << umin << std::endl;

   Real vmin = state.min(Yvel,0);
   Real vmax = state.max(Yvel,0);
   if (ParallelDescriptor::IOProcessor()) 
     std::cout << "VMAX/VMIN " << vmax << " " << vmin << std::endl;

#if (BL_SPACEDIM == 3)
   Real wmin = state.min(Zvel,0);
   Real wmax = state.max(Zvel,0);
   if (ParallelDescriptor::IOProcessor()) 
     std::cout << "WMAX/WMIN " << wmax << " " << wmin << std::endl;
#endif

   Real rmin = state.min(Density,0);
   Real rmax = state.max(Density,0);
   if (ParallelDescriptor::IOProcessor()) 
     std::cout << "RHOMAX/RHOMIN " << rmax << " " << rmin << std::endl;

   if (ParallelDescriptor::IOProcessor()) 
     std::cout << std::endl;
}

// ************************************************************************
// ** initialIter ** 
// ************************************************************************

void 
Grid::initialIter(Real time, Real dt) 
{
   for (int iter = 0; iter < init_iter; iter++) {
     timeStep(time,dt);
   }

}

// ************************************************************************
// ** advance ** 
// ************************************************************************

void 
Grid::advance(Real time, Real dt) 
{
   timeStep(time,dt);

   state.copy(staten,0,0,N_STATE);
   staten.setVal(0.);
}

// ************************************************************************
// ** timeStep ** 
// ************************************************************************

void 
Grid::timeStep(Real time, Real dt) 
{
   state.FillBoundary(Xvel,BL_SPACEDIM+1);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(state,Xvel,BL_SPACEDIM+1,true);

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     FORT_SETVELBC(state[mfi].dataPtr(Xvel),
                   ARLIM(lo),ARLIM(hi),bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                   &is_rz,
#endif
                   &visc_coef,dx,&time);
   }

   getSlopes(0,BL_SPACEDIM);

   getGradP();
   getForce(time);
   getViscTerm();

   printMaxMin();
#if BL_SPACEDIM==3
   getKinetic(time);
#endif

   makeAdvVels(time,dt);

   state.FillBoundary(Density,numscal);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(state,Density,numscal,true);

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     const int* slo = state[mfi].loVect();
     const int* shi = state[mfi].hiVect();

     for (int n = 0; n < numscal; n++) {
       FORT_SETSCALBC(state[mfi].dataPtr(Density+n),ARLIM(lo),ARLIM(hi),
                      ARLIM(slo),ARLIM(shi),
                      bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                      &is_rz,
#endif
                      &n,dx,&time);
     }
   }

   getSlopes(Density,numscal);

   updateScalars(time,dt);

   getRhoHalf(time,dt);

   updateVelocity(time,dt);
}

// ************************************************************************
// ** getSlopes ** 
// ************************************************************************

void 
Grid::getSlopes(int first_comp, int num_comp)
{
   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     int lenx = grids[i].length()[0]+4;
     Real *scratch = new Real[4*lenx];

#if (BL_SPACEDIM == 2)
     FORT_SLOPEX(state[mfi].dataPtr(first_comp),
                 slopex[mfi].dataPtr(first_comp),
                 scratch,ARLIM(lo),ARLIM(hi),&first_comp,&num_comp,
                 bc[i].dataPtr(), &is_rz, &slope_order);
#elif (BL_SPACEDIM == 3)
     FORT_SLOPEX(state[mfi].dataPtr(first_comp),
                 slopex[mfi].dataPtr(first_comp),
                 scratch,ARLIM(lo),ARLIM(hi),&num_comp,
                 bc[i].dataPtr(), &slope_order);
#endif
     delete scratch;
   }
   slopex.FillBoundary(first_comp,num_comp);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(slopex,first_comp,num_comp,true);

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     int leny = grids[i].length()[1]+4;
     Real *scratch = new Real[4*leny];

     FORT_SLOPEY(state[mfi].dataPtr(first_comp),
                 slopey[mfi].dataPtr(first_comp),
                 scratch,ARLIM(lo),ARLIM(hi),&num_comp,
                 bc[i].dataPtr(), &slope_order);

     delete scratch;
   }
   slopey.FillBoundary(first_comp,num_comp);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(slopey,first_comp,num_comp,true);

#if (BL_SPACEDIM == 3)
   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     int lenz = grids[i].length()[2]+4;
     Real *scratch = new Real[4*lenz];

     FORT_SLOPEZ(state[mfi].dataPtr(first_comp),
                 slopez[mfi].dataPtr(first_comp),
                 scratch,ARLIM(lo),ARLIM(hi),&num_comp,
                 bc[i].dataPtr(),&slope_order);

     delete scratch;
   }
   slopez.FillBoundary(first_comp,num_comp);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(slopez,first_comp,num_comp,true);
#endif

}

// ************************************************************************
// ** getViscTerm ** 
// ************************************************************************

void 
Grid::getViscTerm()
{
  if (visc_coef > 0.) {
   for (int n = 0; n < BL_SPACEDIM; n++) {
     for (MFIter mfi(state); mfi.isValid(); ++mfi)
     {
       int i = mfi.index();
       const int* lo = grids[i].loVect();
       const int* hi = grids[i].hiVect();

#if (BL_SPACEDIM == 2)
       FORT_LAPLAC(state[mfi].dataPtr(n), visc[mfi].dataPtr(n),
                   area[0][mfi].dataPtr(), area[1][mfi].dataPtr(),
                   vol[mfi].dataPtr(), ARLIM(lo),ARLIM(hi),
                   dx,&visc_coef,&is_rz,bc[i].dataPtr(),&n);
#elif (BL_SPACEDIM == 3)
       FORT_LAPLAC(state[mfi].dataPtr(n), visc[mfi].dataPtr(n),
                   area[0][mfi].dataPtr(), area[1][mfi].dataPtr(),
                   area[2][mfi].dataPtr(), vol[mfi].dataPtr(),
                   ARLIM(lo),ARLIM(hi),dx,&visc_coef,bc[i].dataPtr());
#endif

     }
    }
    visc.FillBoundary(0,BL_SPACEDIM);
    if (geom.isAnyPeriodic())
      geom.FillPeriodicBoundary(visc,0,BL_SPACEDIM,true);
  }
}

// ************************************************************************
// ** getDiffTerm ** 
// ************************************************************************

void 
Grid::getDiffTerm()
{
  for (int n = 0; n < numscal; n++) {
    Real dc = diff_coef[n];
    if (dc > 0.) {
      int icomp = Density + n;
      for (MFIter mfi(state); mfi.isValid(); ++mfi)
      {
         int i = mfi.index();
         const int* lo = grids[i].loVect();
         const int* hi = grids[i].hiVect();

#if (BL_SPACEDIM == 2)
         FORT_LAPLAC(state[mfi].dataPtr(icomp),diff[mfi].dataPtr(n),
                     area[0][mfi].dataPtr(),area[1][mfi].dataPtr(),
                     vol[mfi].dataPtr(), ARLIM(lo),ARLIM(hi),
                     dx,&dc,&is_rz,bc[i].dataPtr(),&icomp);
#elif (BL_SPACEDIM == 3)
         FORT_LAPLAC(state[mfi].dataPtr(icomp),diff[mfi].dataPtr(n),
                     area[0][mfi].dataPtr(), area[1][mfi].dataPtr(),
                     area[2][mfi].dataPtr(), vol[mfi].dataPtr(),
                     ARLIM(lo),ARLIM(hi),dx,&dc,bc[i].dataPtr());
#endif
      }
    }
  }
  diff.FillBoundary(0,numscal);
  if (geom.isAnyPeriodic())
      geom.FillPeriodicBoundary(diff,0,numscal,true);
}

// ************************************************************************
// ** getGradP ** 
// ************************************************************************

void 
Grid::getGradP()
{
   proj->gradient(&gradp,&pressure);
}

// ************************************************************************
//  compute kinetic energy
// ************************************************************************

void 
Grid::getKinetic(Real current_time)
{
  Real xmean = 0.;
  Real ymean = 0.;
  Real zmean = 0.;
  Real tix = 0.;
  Real tiy = 0.;
  Real tiz = 0.;
  for (MFIter mfi(state); mfi.isValid(); ++mfi)
  {
    int i = mfi.index();
    const int* lo = grids[i].loVect();
    const int* hi = grids[i].hiVect();
    const FArrayBox& fab = state[mfi];

    FORT_KINE(fab.dataPtr(Xvel),ARLIM(fab.loVect()),ARLIM(fab.hiVect()),
              D_DECL(&xmean,&ymean,&zmean),D_DECL(&tix,&tiy,&tiz),lo,hi);
    
  }

  ParallelDescriptor::ReduceRealSum(xmean);
  ParallelDescriptor::ReduceRealSum(ymean);
  ParallelDescriptor::ReduceRealSum(tix);
  ParallelDescriptor::ReduceRealSum(tiy);

#if BL_SPACEDIM==3
  ParallelDescriptor::ReduceRealSum(zmean);
  ParallelDescriptor::ReduceRealSum(tiz);
#endif

  Real factor = 1./float(grids.numPts());

  Real kine = 0.5*(D_TERM(tix, + tiy, + tiz));
  for (int i=0; i<BL_SPACEDIM; ++i)
      kine *= dx[i];

  xmean = xmean*factor;
  ymean = ymean*factor;
  tix = sqrt(tix*factor - xmean*xmean);
  tiy = sqrt(tiy*factor - ymean*ymean);

#if BL_SPACEDIM==3
  zmean = zmean*factor;
  tiz = sqrt(tiz*factor - zmean*zmean);
#endif

  Real ke = 0.5*(D_TERM(tix*tix, + tiy*tiy, + tiz*tiz));

  if (ParallelDescriptor::IOProcessor())
  {
      cout << "  tke = " << current_time << " " << ke << endl;
      cout << "  mean x velocity = " << current_time << " " << xmean << endl;
      cout << "  mean y velocity = " << current_time << " " << ymean << endl;
#if BL_SPACEDIM==3
      cout << "  mean z velocity = " << current_time << " " << zmean << endl;
#endif
      cout << "  turbulent intensity = " << current_time
           D_TERM(<< "  " << tix, << "  "  << tiy, << "  "  << tiz) << endl;

      std::cout << "Kinetic energy " << current_time << " " << kine << std::endl;
  }

  if (writeTurb)
  {
      const Box& box = geom.Domain();
      FArrayBox vfab(box,BL_SPACEDIM);
      state.copy(vfab,Xvel,0,BL_SPACEDIM);
      
      if (ParallelDescriptor::IOProcessor())
      {
          cout << "...writing turbulence file..." << endl;
          
          FORT_WRITEVEL(vfab.dataPtr(),ARLIM(vfab.loVect()),ARLIM(vfab.hiVect()),
                        box.loVect(), box.hiVect(), geom.ProbLength());

          cout << "done." << endl;
      }
  }
}


// ************************************************************************
// ** getForce ** 
// ************************************************************************

void 
Grid::getForce(Real current_time)
{
  for (MFIter mfi(state); mfi.isValid(); ++mfi)
  {
    int i = mfi.index();
    const int* lo = grids[i].loVect();
    const int* hi = grids[i].hiVect();

    FORT_MKFORCE(force[mfi].dataPtr(), state[mfi].dataPtr(Xvel),
                 state[mfi].dataPtr(Density),
                 &gravity,&visc_coef,&current_time,dx,ARLIM(lo),ARLIM(hi));
  }
}

// ************************************************************************
// ** getScalForce ** 
// ************************************************************************

void 
Grid::getScalForce(Real current_time)
{
  for (MFIter mfi(state); mfi.isValid(); ++mfi)
  {
    int i = mfi.index();
    const int* lo = grids[i].loVect();
    const int* hi = grids[i].hiVect();

    FORT_MKSCALFORCE(scalforce[mfi].dataPtr(), state[mfi].dataPtr(Density),
                     &current_time,dx,ARLIM(lo),ARLIM(hi),&numscal);
  }
}

// ************************************************************************
// ** makeAdvVel ** 
// ************************************************************************

void 
Grid::makeAdvVels(Real time, Real dt)
{
  Real halftime = time + .5*dt;

// Reset the bcs if inflow profile (for the case of time-dep inflow)

   bool is_inflow = false;
   for (int i = 0; i < grids.size(); i++) 
     for (int n = 0; n < BL_SPACEDIM; n++)
       if (bc[i][2*n] == INLET || bc[i][2*n+1] == INLET)
         is_inflow = true;
   
   if (is_inflow)
   {
     state.FillBoundary(Xvel,BL_SPACEDIM);
     if (geom.isAnyPeriodic())
       geom.FillPeriodicBoundary(state,Xvel,BL_SPACEDIM,true);

     for (MFIter mfi(state); mfi.isValid(); ++mfi)
     {
        int i = mfi.index();
        const int* lo = grids[i].loVect();
        const int* hi = grids[i].hiVect();
 
        FORT_SETVELBC(state[mfi].dataPtr(Xvel),
                      ARLIM(lo),ARLIM(hi),bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                      &is_rz,
#endif
                      &visc_coef,dx,&halftime);
     }
   }

   MultiFab total_forces(grids,BL_SPACEDIM,1,Fab_allocate);

   int velpred = 1;
   int nstart = 1;
   int nend = BL_SPACEDIM;

// This creates the edge velocities used in constructing the
//   transverse derivatives.
   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

//   Define the forcing term as F - Gp/rho + lapu/rho
     total_forces[mfi].copy(gradp[mfi],0,0,BL_SPACEDIM);
     total_forces[mfi].mult(-1.0);
     total_forces[mfi].plus(visc[mfi],0,0,BL_SPACEDIM);
     for (int n = 0; n < BL_SPACEDIM; n++)
       total_forces[mfi].divide(state[mfi],Density,n,1);
     total_forces[mfi].plus(force[mfi],0,0,BL_SPACEDIM);

     FORT_MKUTRANS(utrans[mfi].dataPtr(), vtrans[mfi].dataPtr(),
#if (BL_SPACEDIM == 3)
                   wtrans[mfi].dataPtr(),
#endif
                   state[mfi].dataPtr(Xvel),
                   slopex[mfi].dataPtr(Xvel),slopey[mfi].dataPtr(Xvel),
#if (BL_SPACEDIM == 3)
                   slopez[mfi].dataPtr(Xvel),
#endif
                   total_forces[mfi].dataPtr(),
                   dx,dt,ARLIM(lo),ARLIM(hi),
                   bc[i].dataPtr());
   }

   utrans.FillBoundary(0,1);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(utrans,0,1,true);
   vtrans.FillBoundary(0,1);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(vtrans,0,1,true);
#if (BL_SPACEDIM == 3)
   wtrans.FillBoundary(0,1);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(wtrans,0,1,true);
#endif

// This extrapolates u to the x-edges and v to the y-edges only.
   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

//   Define the forcing term as F - Gp/rho + lapu/rho
     total_forces[mfi].copy(gradp[mfi],0,0,BL_SPACEDIM);
     total_forces[mfi].mult(-1.0);
     total_forces[mfi].plus(visc[mfi],0,0,BL_SPACEDIM);
     for (int n = 0; n < BL_SPACEDIM; n++)
       total_forces[mfi].divide(state[mfi],Density,n,1);
     total_forces[mfi].plus(force[mfi],0,0,BL_SPACEDIM);

     int lenx = grids[i].length()[0]+3;
     int leny = grids[i].length()[1]+3;

     Real *stlft = new Real[2*lenx];
     Real *strgt = stlft + lenx;
     Real *stbot = new Real[2*leny];
     Real *sttop = stbot + leny;

#if (BL_SPACEDIM == 2)
     FORT_MKFLUX(state[mfi].dataPtr(), 
                 edgex[mfi].dataPtr(), edgey[mfi].dataPtr(),
                 slopex[mfi].dataPtr(), slopey[mfi].dataPtr(),
                 uadv[mfi].dataPtr(), vadv[mfi].dataPtr(), 
                 utrans[mfi].dataPtr(), vtrans[mfi].dataPtr(),
                 total_forces[mfi].dataPtr(),
  	         stlft,strgt,stbot,sttop,
  	         ARLIM(lo),ARLIM(hi),
                 dx,&dt,&visc_coef,
                 &is_rz,bc[i].dataPtr(),&velpred,&nstart,&nend);

#elif (BL_SPACEDIM == 3)

     int lenz = grids[i].length()[2]+3;
     Real *stdwn = new Real[2*lenz];
     Real *stup  = stdwn + lenz;

     FORT_MKFLUX(state[mfi].dataPtr(), 
                 edgex[mfi].dataPtr(),  edgey[mfi].dataPtr(),
                 edgez[mfi].dataPtr(), 
                 slopex[mfi].dataPtr(), slopey[mfi].dataPtr(), 
                 slopez[mfi].dataPtr(), 
                 uadv[mfi].dataPtr(), vadv[mfi].dataPtr(), 
                 wadv[mfi].dataPtr(), 
                 utrans[mfi].dataPtr(), vtrans[mfi].dataPtr(),
                 wtrans[mfi].dataPtr(), total_forces[mfi].dataPtr(),
  	         stlft,strgt,stbot,sttop,stdwn,stup,
  	         ARLIM(lo),ARLIM(hi),
                 dx,&dt,&visc_coef,
                 bc[i].dataPtr(),&velpred,&nstart,&nend);
     delete stdwn;
#endif

     delete stlft;
     delete stbot;
   }

   state.FillBoundary(Density,1);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(state,Density,1,true);

// Create the cell-based source S for the divergence constraint Div(U) = S
   int ngrow = 0;
   MultiFab divusrc(grids,1,ngrow);
   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     FORT_MKDIVUCC(divusrc[mfi].dataPtr(), state[mfi].dataPtr(Xvel),
                   state[mfi].dataPtr(Density),&halftime,dx,
                   ARLIM(lo),ARLIM(hi));
   }

// We set the bc's on density here for use in the MAC projection.
   int n = 0;

   state.FillBoundary(Density,1);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(state,Density,1,true);

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
      int i = mfi.index();
      const int* lo = grids[i].loVect();
      const int* hi = grids[i].hiVect();

      const int* slo = state[mfi].loVect();
      const int* shi = state[mfi].hiVect();
  
      FORT_SETSCALBC(state[mfi].dataPtr(Density),ARLIM(lo),ARLIM(hi),
                     ARLIM(slo),ARLIM(shi),
                     bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                     &is_rz,
#endif
                     &n,dx,&halftime);
   }

// This enforces the div(U) = S condition on the edge-based half-time 
//  normal velocities.  
   if ((verbose==1) && ParallelDescriptor::IOProcessor()) 
      std::cout << "MAC Projecting the advection velocity... " << std::endl;
#if (BL_SPACEDIM == 2)
   macproj->project(&uadv,&vadv,&state,&divusrc);
#elif (BL_SPACEDIM == 3)
   macproj->project(&uadv,&vadv,&wadv,&state,&divusrc);
#endif
}

// ************************************************************************
// ** updateScalars ** 
// ************************************************************************

void 
Grid::updateScalars(Real time, Real dt)
{

   if ((verbose==1) && ParallelDescriptor::IOProcessor()) 
      std::cout << "Updating the scalars ... " << std::endl;

   Real newtime  = time+dt;
   Real halftime = time+0.5*dt;

   bool is_inflow = false;
   for (int i = 0; i < grids.size(); i++) 
     for (int n = 0; n < BL_SPACEDIM; n++)
       if (bc[i][2*n] == INLET || bc[i][2*n+1] == INLET)
         is_inflow = true;

   state.FillBoundary(Density,numscal);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(state,Density,numscal,true);

   if (is_inflow) 
   {
     for (MFIter mfi(state); mfi.isValid(); ++mfi)
     {
         int i = mfi.index();
         const int* lo = grids[i].loVect();
         const int* hi = grids[i].hiVect();

         const int* slo = state[mfi].loVect();
         const int* shi = state[mfi].hiVect();
  
         for (int n = 0; n < numscal; n++) 
           FORT_SETSCALBC(state[mfi].dataPtr(Density+n),ARLIM(lo),ARLIM(hi),
                          ARLIM(slo),ARLIM(shi),
                          bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                          &is_rz,
#endif
                          &n,dx,&halftime);
     }
   }

   getDiffTerm();
   getScalForce(time);

   MultiFab total_forces(grids,numscal,1,Fab_allocate);

   int velpred = 0;
   int nstart = Density+1;
   int nend   = Density+numscal;

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     total_forces[mfi].copy(scalforce[mfi],0,0,numscal);
     total_forces[mfi].plus(diff[mfi],0,0,numscal);

     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     int lenx = grids[i].length()[0]+3;
     int leny = grids[i].length()[1]+3;
     Real *stlft = new Real[2*lenx];
     Real *strgt = stlft + lenx;
     Real *stbot = new Real[2*leny];
     Real *sttop = stbot + leny;

#if (BL_SPACEDIM == 2)
     FORT_MKFLUX(state[mfi].dataPtr(), edgex[mfi].dataPtr(), edgey[mfi].dataPtr() ,  
                 slopex[mfi].dataPtr(), slopey[mfi].dataPtr(),
                 uadv[mfi].dataPtr(), vadv[mfi].dataPtr(), 
                 utrans[mfi].dataPtr(), vtrans[mfi].dataPtr(),
		 total_forces[mfi].dataPtr(),
                 stlft,strgt,stbot,sttop,
   	         ARLIM(lo),ARLIM(hi),
                 dx,&dt,&visc_coef,&is_rz,
                 bc[i].dataPtr(),&velpred,&nstart,&nend);
#elif (BL_SPACEDIM == 3)
     int lenz = grids[i].length()[2]+3;
     Real *stdwn = new Real[2*lenz];
     Real *stup  = stdwn + lenz;

     FORT_MKFLUX(state[mfi].dataPtr(), edgex[mfi].dataPtr(), edgey[mfi].dataPtr() ,  
                 edgez[mfi].dataPtr(),
                 slopex[mfi].dataPtr(), slopey[mfi].dataPtr(),
                 slopez[mfi].dataPtr(),
                 uadv[mfi].dataPtr(), vadv[mfi].dataPtr(), 
                 wadv[mfi].dataPtr(), 
                 utrans[mfi].dataPtr(), vtrans[mfi].dataPtr(),
		 wtrans[mfi].dataPtr(), total_forces[mfi].dataPtr(),
                 stlft,strgt,stbot,sttop,stdwn,stup,
   	         ARLIM(lo),ARLIM(hi),
                 dx,&dt,&visc_coef,
                 bc[i].dataPtr(),&velpred,&nstart,&nend);
     delete stdwn;
#endif

     delete stlft;
     delete stbot;
   }

   getScalForce(halftime);

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     FORT_SCALUPD(state[mfi].dataPtr(Density), staten[mfi].dataPtr(Density),
                  edgex[mfi].dataPtr(Density), edgey[mfi].dataPtr(Density),
#if (BL_SPACEDIM == 3)
                  edgez[mfi].dataPtr(Density), 
#endif
                  uadv[mfi].dataPtr(), vadv[mfi].dataPtr(), 
#if (BL_SPACEDIM == 3)
                  wadv[mfi].dataPtr(), 
#endif
                  diff[mfi].dataPtr(), scalforce[mfi].dataPtr(),
	          area[0][mfi].dataPtr(), area[1][mfi].dataPtr(), 
#if (BL_SPACEDIM == 3)
	          area[2][mfi].dataPtr(),
#endif
	          vol[mfi].dataPtr(),
                  ARLIM(lo),ARLIM(hi),
                  dx,&dt,is_conserv.dataPtr(),&numscal); 
   }

   staten.FillBoundary(Density,numscal);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(staten,Density,numscal,true);

   for (MFIter mfi(staten); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     const int* slo = staten[mfi].loVect();
     const int* shi = staten[mfi].hiVect();

     for (int n = 0; n < numscal; n++) {
       FORT_SETSCALBC(staten[mfi].dataPtr(Density+n),ARLIM(lo),ARLIM(hi),
                      ARLIM(slo),ARLIM(shi),
                      bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                      &is_rz,
#endif
                      &n,dx,&newtime);
     }
   }

   for (int n = 0; n < numscal; n++) {
     if (diff_coef[n] > 0.0) {
       Real mu = .5*diff_coef[n]*dt;
       if ((verbose==1) && ParallelDescriptor::IOProcessor()) 
       {
          if (n == 0) std::cout << "Diffusing the 1st scalar..." << std::endl;
          if (n == 1) std::cout << "Diffusing the 2nd scalar..." << std::endl;
          if (n == 2) std::cout << "Diffusing the 3rd scalar..." << std::endl;
          if (n > 2) 
            std::cout << "Diffusing the " << n+1 <<"th scalar..." << std::endl;
       }
       diffuse_op->solveScal(&staten,mu,Density+n);
     }
   }

   staten.FillBoundary(Density,numscal);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(staten,Density,numscal,true);

   for (MFIter mfi(staten); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     const int* slo = staten[mfi].loVect();
     const int* shi = staten[mfi].hiVect();

     for (int n = 0; n < numscal; n++) {
       FORT_SETSCALBC(staten[mfi].dataPtr(Density+n),ARLIM(lo),ARLIM(hi),
                      ARLIM(slo),ARLIM(shi),
                      bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                      &is_rz,
#endif
                      &n,dx,&newtime);
     }
   }
}

// ************************************************************************
// ** getRhoHalf ** 
// ************************************************************************

void 
Grid::getRhoHalf(Real time, Real dt)
{
   rhonph.copy(state,Density,0,1);
  
   for (MFIter mfi(rhonph); mfi.isValid(); ++mfi)
   {
      rhonph[mfi].plus(staten[mfi],grids[mfi.index()],Density,0,1);
      rhonph[mfi].mult(0.5,grids[mfi.index()]);
   }

   Real halftime = time + 0.5*dt;
   int n = 0;

   rhonph.FillBoundary();
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(rhonph,0,1,true);

   for (MFIter mfi(rhonph); mfi.isValid(); ++mfi)
   {
       int i = mfi.index();
       const int* lo = grids[i].loVect();
       const int* hi = grids[i].hiVect();

       const int* slo = rhonph[mfi].loVect();
       const int* shi = rhonph[mfi].hiVect();

       FORT_SETSCALBC(rhonph[mfi].dataPtr(),ARLIM(lo),ARLIM(hi),
                      ARLIM(slo),ARLIM(shi),
                      bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                      &is_rz,
#endif
                      &n,dx,&halftime);
   }
}

// ************************************************************************
// ** updateVelocity ** 
// ************************************************************************

void 
Grid::updateVelocity(Real time, Real dt)
{

  if ((verbose==1) && ParallelDescriptor::IOProcessor()) 
     std::cout << "Updating the velocity ... " << std::endl;

   MultiFab total_forces(grids,BL_SPACEDIM,1,Fab_allocate);

   int velpred = 0;
   int nstart = 1;
   int nend = BL_SPACEDIM;

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     int lenx = grids[i].length()[0]+3;
     int leny = grids[i].length()[1]+3;

     Real *stlft = new Real[2*lenx];
     Real *strgt = stlft + lenx;
     Real *stbot = new Real[2*leny];
     Real *sttop = stbot + leny;

//   Define the forcing term as F - Gp/rho + lapu/rho
     total_forces[mfi].copy(gradp[mfi],0,0,BL_SPACEDIM);
     total_forces[mfi].mult(-1.0);
     total_forces[mfi].plus(visc[mfi],0,0,BL_SPACEDIM);
     for (int n = 0; n < BL_SPACEDIM; n++)
       total_forces[mfi].divide(state[mfi],Density,n,1);
     total_forces[mfi].plus(force[mfi],0,0,BL_SPACEDIM);

#if (BL_SPACEDIM == 2)
     FORT_MKFLUX(state[mfi].dataPtr(), edgex[mfi].dataPtr(),  edgey[mfi].dataPtr(),
                 slopex[mfi].dataPtr(), slopey[mfi].dataPtr(),
                 uadv[mfi].dataPtr(), vadv[mfi].dataPtr(), 
                 utrans[mfi].dataPtr(), vtrans[mfi].dataPtr(),
                 total_forces[mfi].dataPtr(),
  	         stlft,strgt,stbot,sttop,
  	         ARLIM(lo),ARLIM(hi),
                 dx,&dt,&visc_coef,
                 &is_rz,bc[i].dataPtr(),&velpred,&nstart,&nend);

#elif (BL_SPACEDIM == 3)

     int lenz = grids[i].length()[2]+3;
     Real *stdwn = new Real[2*lenz];
     Real *stup  = stdwn + lenz;

     FORT_MKFLUX(state[mfi].dataPtr(), edgex[mfi].dataPtr(),  edgey[mfi].dataPtr(),
                 edgez[mfi].dataPtr(), 
                 slopex[mfi].dataPtr(), slopey[mfi].dataPtr(),
                 slopez[mfi].dataPtr(), 
                 uadv[mfi].dataPtr(), vadv[mfi].dataPtr(), 
                 wadv[mfi].dataPtr(), 
                 utrans[mfi].dataPtr(), vtrans[mfi].dataPtr(),
                 wtrans[mfi].dataPtr(), total_forces[mfi].dataPtr(),
  	         stlft,strgt,stbot,sttop, stdwn,stup,
  	         ARLIM(lo),ARLIM(hi),
                 dx,&dt,&visc_coef,
                 bc[i].dataPtr(),&velpred,&nstart,&nend);
     delete stdwn;
#endif

     delete stlft;
     delete stbot;
   }

   Real halftime = time + 0.5*dt;
   getForce(halftime);

   for (MFIter mfi(state); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     FORT_VELUPD(state[mfi].dataPtr(Xvel), staten[mfi].dataPtr(Xvel),
                 visc[mfi].dataPtr(),gradp[mfi].dataPtr(),
                 rhonph[mfi].dataPtr(),
    	         uadv[mfi].dataPtr(),vadv[mfi].dataPtr(),
#if (BL_SPACEDIM == 3)
    	         wadv[mfi].dataPtr(),
#endif
                 edgex[mfi].dataPtr(),edgey[mfi].dataPtr(),
#if (BL_SPACEDIM == 3)
    	         edgez[mfi].dataPtr(),
#endif
                 force[mfi].dataPtr(),&dt,ARLIM(lo),ARLIM(hi),dx);
   }

   Real  newtime = time+dt;

   staten.FillBoundary(Xvel,BL_SPACEDIM);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(staten,Xvel,BL_SPACEDIM,true);

   for (MFIter mfi(staten); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     FORT_SETVELBC(staten[mfi].dataPtr(Xvel),
                   ARLIM(lo),ARLIM(hi),bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                   &is_rz,
#endif
                   &visc_coef,dx,&newtime);

   }

   if (visc_coef > 0.0) {
     Real mu = .5*visc_coef*dt;
     diffuse_op->solveVel(&staten,&rhonph,mu);

     staten.FillBoundary(Xvel,BL_SPACEDIM);
     if (geom.isAnyPeriodic())
       geom.FillPeriodicBoundary(staten,Xvel,BL_SPACEDIM,true);

     for (MFIter mfi(staten); mfi.isValid(); ++mfi)
     {
       int i = mfi.index();
       const int* lo = grids[i].loVect();
       const int* hi = grids[i].hiVect();

       FORT_SETVELBC(staten[mfi].dataPtr(Xvel),
                     ARLIM(lo),ARLIM(hi),bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                     &is_rz,
#endif
                     &visc_coef,dx,&newtime);
     }
   }

// Create the node-based source S for the divergence constraint Div(U) = S
   BoxArray nodal_grids(grids);
   nodal_grids.surroundingNodes();
   MultiFab divusrc(nodal_grids,1,0);

   for (MFIter mfi(staten); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     FORT_MKDIVUNOD(divusrc[mfi].dataPtr(),
                    staten[mfi].dataPtr(Xvel), 
                    staten[mfi].dataPtr(Density),
                    &newtime,dx,ARLIM(lo),ARLIM(hi));
     divusrc[mfi].mult(1./dt);
     staten[mfi].divide(dt,0,BL_SPACEDIM);
   }

   proj->project(&staten,&pressure,&rhonph,&divusrc,time,dt);

   staten.FillBoundary(Xvel,BL_SPACEDIM);
   if (geom.isAnyPeriodic())
     geom.FillPeriodicBoundary(staten,Xvel,BL_SPACEDIM,true);

   for (MFIter mfi(staten); mfi.isValid(); ++mfi)
   {
     int i = mfi.index();
     const int* lo = grids[i].loVect();
     const int* hi = grids[i].hiVect();

     staten[mfi].mult(dt,0,BL_SPACEDIM);

     FORT_SETVELBC(staten[mfi].dataPtr(Xvel),
                   ARLIM(lo),ARLIM(hi),bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                   &is_rz,
#endif
                   &visc_coef,dx,&newtime);
   }
}

// ************************************************************************
// ** estimateDt ** 
// ************************************************************************

Real 
Grid::estimateDt(Real& dt, Real cfl) 
{
  getGradP();
  
  Real cfl_temp = cfl;
  if (fixed_dt > 0) cfl_temp = 0.9;

  Real dt_old   = dt;
  Real dt_local = 1.0e+20;
       dt       = 1.0e+20;
  
  for (MFIter mfi(state); mfi.isValid(); ++mfi)
  {
      int i = mfi.index();
      const int* lo = grids[i].loVect();
      const int* hi = grids[i].hiVect();

      FORT_CMPDT(state[mfi].dataPtr(Xvel), 
                 state[mfi].dataPtr(Density),
                 gradp[mfi].dataPtr(Xvel),
                 force[mfi].dataPtr(),
                 dx,&dt_old,&dt_local,&cfl_temp,
    	         ARLIM(lo),ARLIM(hi)); 

      dt = std::min(dt,dt_local);
  }
  ParallelDescriptor::ReduceRealMin(dt);
  
  if (fixed_dt > 0) {
    if (fixed_dt > dt && ParallelDescriptor::IOProcessor()) {
      std::cout << "WARNING: fixed_dt may be too big" << std::endl;
      std::cout << "fixed_dt = " << fixed_dt 
           << " with CFL =0.9 computed_dt = " << dt << std::endl;
    }
    dt = fixed_dt;
  }
  if ((verbose==1) && ParallelDescriptor::IOProcessor()) 
     std::cout << "Computing dt to be " << dt << std::endl;
  return dt;
}

// ************************************************************************
// ** DeriveFunc ** 
// ************************************************************************

// ------------  Define assoc between var name and derive func
typedef void (*DeriveFunc) (Real* state, Real* derval, 
#if (BL_SPACEDIM == 2)
			    const int& der_lo_1, const int& der_lo_2,
			    const int& der_hi_1, const int& der_hi_2,
			    const int& lo_1, const int& lo_2,
			    const int& hi_1, const int& hi_2,
#elif (BL_SPACEDIM == 3)
			    const int& der_lo_1, const int& der_lo_2,
			    const int& der_lo_3, 
			    const int& der_hi_1, const int& der_hi_2,
			    const int& der_hi_3,
			    const int& lo_1, const int& lo_2,
			    const int& lo_3,
			    const int& hi_1, const int& hi_2,
			    const int& hi_3,
#endif
                            const Real* dx);

// ************************************************************************
// ** DeriveRec ** 
// ************************************************************************

class DeriveRec
{
public:
    DeriveRec(std::string name, DeriveFunc f)
        : varname(name), func(f) {};
    ~DeriveRec() {};
    std::string       varname;
    DeriveFunc    func;
};

DeriveRec vor_rec("vort", FORT_DERVORT);

static DeriveRec deriveList[] = {
    vor_rec,
};

int numDerive = sizeof(deriveList)/sizeof(DeriveRec);

// ************************************************************************
// ** deriveData ** 
// ************************************************************************

void Grid::deriveData(MultiFab& derval, const std::string& name, Real time)
{
     // is it a state variable
   int nGrow = 0;
   if (name == "x_velocity") {
       MultiFab::Copy(derval,state,Xvel,0,1,nGrow);
       return;
   } else if (name == "y_velocity") {
       MultiFab::Copy(derval,state,Yvel,0,1,nGrow);
       return;
   } else if (name == "density") {
       MultiFab::Copy(derval,state,Density,0,1,nGrow);
       return;
   } else if (name == "tracer") {
       MultiFab::Copy(derval,state,Tracer,0,1,nGrow);
       return;
//  IF YOU WISH TO ADD A NEW SCALAR VARIABLE, UNCOMMENT THESE LINES
// } else if (name == "new_scalar") {
//     MultiFab::Copy(derval,state,NewScal,0,1,nGrow);
//     return;
   } else if (name == "pressure") {
     for (MFIter pmfi(pressure); pmfi.isValid(); ++pmfi)
     {
       const int i = pmfi.index();
       const int* lo = grids[i].loVect();
       const int* hi = grids[i].hiVect();

       FORT_DERAVGP(pressure[pmfi].dataPtr(), derval[pmfi].dataPtr(),
                    ARLIM(lo), ARLIM(hi));
     }
       return;
   } else if (name == "px") {
       MultiFab::Copy(derval,gradp,Xvel,0,1,nGrow);
       return;
   } else if (name == "py") {
       MultiFab::Copy(derval,gradp,Yvel,0,1,nGrow);
       return;
   }

   //
   // If we got here, check list of known derived quantities.
   //
   for (int n = 0; n < numDerive; n++) 
   {
       if (name == deriveList[n].varname) {

         state.FillBoundary(Xvel,BL_SPACEDIM);
         if (geom.isAnyPeriodic())
           geom.FillPeriodicBoundary(state,Xvel,BL_SPACEDIM,true);
	 
         for (MFIter mfi(state); mfi.isValid(); ++mfi)
         {
           int i = mfi.index();
           const int* lo = grids[i].loVect();
           const int* hi = grids[i].hiVect();
	   FORT_SET_CELL_VELBC(state[mfi].dataPtr(Xvel),
                               ARLIM(lo),ARLIM(hi),
	 		       bc[i].dataPtr(),&is_rz,&visc_coef,dx,&time);
         }
	 
         for (MFIter mfi(state); mfi.isValid(); ++mfi)
         {
           int i = mfi.index();
           const int* lo = grids[i].loVect();
           const int* hi = grids[i].hiVect();
           const int* derlo = derval[mfi].loVect();
           const int* derhi = derval[mfi].hiVect();
	   deriveList[n].func(state[mfi].dataPtr(Xvel),derval[mfi].dataPtr(),
			      ARLIM(derlo),ARLIM(derhi),
			      ARLIM(lo),ARLIM(hi),dx);
         }

         state.FillBoundary(Xvel,BL_SPACEDIM);
         if (geom.isAnyPeriodic())
           geom.FillPeriodicBoundary(state,Xvel,BL_SPACEDIM,true);
	 
         for (MFIter mfi(state); mfi.isValid(); ++mfi)
         {
           int i = mfi.index();
           const int* lo = grids[i].loVect();
           const int* hi = grids[i].hiVect();
	   FORT_SETVELBC(state[mfi].dataPtr(Xvel),
                         ARLIM(lo),ARLIM(hi),
	 	         bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                         &is_rz,
#endif
                         &visc_coef,dx,&time);
         }
	 return;
       }
   }

      // if we got here, dont know how to derive this
    std::cerr << "Dont know how to derive " << name << std::endl;
    abort();
}

// ************************************************************************
// ** writePlotFile ** 
// ************************************************************************

void
Grid::writePlotFile(MultiFab& plotMF, Real time)
{
  // Copy all of the data into the MultiFab.
  int nGrow = 0;
  int num_in_temp = std::max(BL_SPACEDIM,NumDerive());
  MultiFab temp(grids,num_in_temp,0);

  int counter = 0;

  // Writing the state variables (velocity, density, tracer)
  MultiFab::Copy(plotMF,state,0,0,N_STATE,nGrow);
  counter += N_STATE;

  // Average the pressure from nodes to cell centers.
  for (MFIter pmfi(pressure); pmfi.isValid(); ++pmfi)
  {
    const int i = pmfi.index();
    const int* lo = grids[i].loVect();
    const int* hi = grids[i].hiVect();

    FORT_DERAVGP(pressure[pmfi].dataPtr(), temp[pmfi].dataPtr(),
                 ARLIM(lo), ARLIM(hi));
  }

  MultiFab::Copy(plotMF,temp,0,counter,1,nGrow);
  counter++;

  // Writing the pressure gradient
  proj->gradient(&temp,&pressure);
  MultiFab::Copy(plotMF,temp,0,counter,BL_SPACEDIM,nGrow);
  counter += BL_SPACEDIM;

  ParallelDescriptor::Barrier();

  state.FillBoundary(Xvel,BL_SPACEDIM);

  if (geom.isAnyPeriodic())
    geom.FillPeriodicBoundary(state,Xvel,BL_SPACEDIM,true);
  
  // Writing the vorticity
  for (MFIter mfi(state); mfi.isValid(); ++mfi)
  {
    const int i = mfi.index();
    const int* lo = grids[i].loVect();
    const int* hi = grids[i].hiVect();

    FORT_SET_CELL_VELBC(state[mfi].dataPtr(Xvel), 
                        ARLIM(lo),ARLIM(hi),bc[i].dataPtr(),
  		        &is_rz,&visc_coef,dx,&time);
  }

  state.FillBoundary(Xvel,BL_SPACEDIM);
  if (geom.isAnyPeriodic())
    geom.FillPeriodicBoundary(state,Xvel,BL_SPACEDIM,true);

  for (MFIter mfi(state); mfi.isValid(); ++mfi)
  {
    const int i = mfi.index();
    const int* lo = grids[i].loVect();
    const int* hi = grids[i].hiVect();

    for(int n = 0; n < NumDerive(); n++) {
      deriveList[n].func(state[mfi].dataPtr(),temp[mfi].dataPtr(n),
  		         ARLIM(lo),ARLIM(hi),ARLIM(lo),ARLIM(hi),dx);
    }
  }
  MultiFab::Copy(plotMF,temp,0,counter,numDerive,nGrow);
  counter += numDerive;

  for (MFIter mfi(state); mfi.isValid(); ++mfi)
  {
    const int i = mfi.index();
    const int* lo = grids[i].loVect();
    const int* hi = grids[i].hiVect();
 
    FORT_SETVELBC(state[mfi].dataPtr(Xvel),
                  ARLIM(lo),ARLIM(hi),bc[i].dataPtr(),
#if (BL_SPACEDIM == 2)
                  &is_rz,
#endif
                  &visc_coef,dx,&time);
  }
}

// ************************************************************************
// ** NumState ** 
// ************************************************************************

int 
Grid::NumState() { 
  int n = N_STATE;
  return n;
}

// ************************************************************************
// ** NumDerive ** 
// ************************************************************************

int 
Grid::NumDerive() { 
  int numDerive = sizeof(deriveList)/sizeof(DeriveRec);
  return numDerive;
}

void
Grid::checkPoint (const std::string& dir,
                  std::ostream&       os,
                  VisMF::How     how)
{
    int ndesc = NumState() + NumDerive();
    //
    // Build directory to hold the MultiFabs in the StateData at this level.
    // The directory is relative the the directory containing the Header file.
    //
    char buf[64];
    sprintf(buf, "Level_%d", 0);
    std::string Level = buf;
    //
    // Now for the full pathname of that directory.
    //
    std::string FullPath = dir;
    if (!FullPath.empty() && FullPath[FullPath.length()-1] != '/')
    {
        FullPath += '/';
    }
    FullPath += Level;
    //
    // Only the I/O processor makes the directory if it doesn't already exist.
    //
    if (ParallelDescriptor::IOProcessor())
        if (!BoxLib::UtilCreateDirectory(FullPath, 0755))
            BoxLib::CreateDirectoryFailed(FullPath);
    //
    // Force other processors to wait till directory is built.
    //
    ParallelDescriptor::Barrier();

    // Output data contained in "staten."
    {
    std::string PathNameInHeader = Level;
    sprintf(buf, "/State");
    PathNameInHeader += buf;
    std::string FullPathName = FullPath;
    FullPathName += buf;
    std::string mf_name_new = PathNameInHeader;
    if (ParallelDescriptor::IOProcessor())
      os << mf_name_new << '\n';
    std::string mf_fullpath_new = FullPathName;
    VisMF::Write(state,mf_fullpath_new,how);
    }

    // Output data contained in "pressure"
    {
    std::string PathNameInHeader = Level;
    sprintf(buf, "/Pressure");
    PathNameInHeader += buf;
    std::string FullPathName = FullPath;
    FullPathName += buf;
    std::string mf_name_new = PathNameInHeader;
    if (ParallelDescriptor::IOProcessor())
      os << mf_name_new << '\n';
    std::string mf_fullpath_new = FullPathName;
    VisMF::Write(pressure,mf_fullpath_new,how);
    }
}
