/*
** (c) 1996-2000 The Regents of the University of California (through
** E.O. Lawrence Berkeley National Laboratory), subject to approval by
** the U.S. Department of Energy.  Your use of this software is under
** license -- the license agreement is attached and included in the
** directory as license.txt or you may contact Berkeley Lab's Technology
** Transfer Department at TTD@lbl.gov.  NOTICE OF U.S. GOVERNMENT RIGHTS.
** The Software was developed under funding from the U.S. Government
** which consequently retains certain rights as follows: the
** U.S. Government has been granted for itself and others acting on its
** behalf a paid-up, nonexclusive, irrevocable, worldwide license in the
** Software to reproduce, prepare derivative works, and perform publicly
** and display publicly.  Beginning five (5) years after the date
** permission to assert copyright is obtained from the U.S. Department of
** Energy, and subject to any subsequent five (5) year renewals, the
** U.S. Government is granted for itself and others acting on its behalf
** a paid-up, nonexclusive, irrevocable, worldwide license in the
** Software to reproduce, prepare derivative works, distribute copies to
** the public, perform publicly and display publicly, and to permit
** others to do so.
*/

//
// $Id: BLWorkQueue.cpp,v 1.12 2001/11/16 18:05:16 car Exp $
//

#include <winstd.H>
#include <cstdlib>
#include <cstdio>
#include <memory>
#include <queue>

#include <ParallelDescriptor.H>
#include <ParmParse.H>
#include <Thread.H>
#include <Profiler.H>
#include <WorkQueue.H>

namespace
{
Mutex print_mutex;
WorkQueue* bl_wrkq = 0;
int verbose = 0;
int stacksize = 0;
}

namespace BoxLib
{

WorkQueue&
theWorkQueue()
{
    return *bl_wrkq;
}
}

void
WorkQueue::Initialize ()
{
    ParmParse pp("workqueue");

    int maxthreads = 0;

    pp.query("maxthreads", maxthreads);
    pp.query("verbose", verbose);
    pp.query("stacksize", stacksize);

    if (verbose && ParallelDescriptor::IOProcessor())
    {
        std::cout << "workqueue.maxthreads = " << maxthreads << std::endl;
        std::cout << "workqueue.stacksize = " << stacksize << std::endl;
    }

    bl_wrkq = new WorkQueue(maxthreads);
}

void
WorkQueue::Finalize ()
{
    delete bl_wrkq;
}

#define DPRINTF(arg)							\
do									\
{									\
    if ( verbose > 2 )							\
    {									\
	Lock<Mutex> lock(print_mutex);					\
	std::cout << "tid(" << Thread::getID() << "): "			\
		  << arg << std::endl;					\
    }									\
}									\
while (false)

WorkQueue::WorkQueue(int maxthreads_)
    : quit(false), eof(false), maxthreads(maxthreads_), numthreads(0), idlethreads(0), tasks(0)
{
    if ( maxthreads_ >= Thread::max_threads() )
    {
	BoxLib::Error("maxthreads_ in workqueue exceeds system limit");
    }
    if ( maxthreads_ < 0 )
    {
	BoxLib::Error("maxthreads_ must be >= 0");
    }
}

int
WorkQueue::max_threads() const
{
    return maxthreads;
}

int
WorkQueue::num_threads() const
{
    Lock<ConditionVariable> lock(cv);
    return numthreads;
}

WorkQueue::task::~task()
{
}

extern "C"
void*
WorkQueue_server(void* arg)
{
    BL_PROFILE("WorkQueue_server()");
    WorkQueue* wq = static_cast<WorkQueue*>(arg);
    return wq->server();
}

void*
WorkQueue::server()
{
    DPRINTF("A worker is starting");
    Lock<ConditionVariable> lock(cv);
    DPRINTF("Worker locked 0");
    for (;;)
    {
	if ( tasks == 0 && eof )
	{
	    gate.open(); gate.release();
	    eof = false;
	}
	DPRINTF("Worker waiting for work");
	while ( wrkq.empty() && !quit )
	{
	    idlethreads++;
	    cv.wait();
	    idlethreads--;
	}
	DPRINTF("Work queue: wrkq.empty()(" << wrkq.empty()<< "), "
		<< "quit(" << quit << "), "
		<< "eof("  << eof << "), "
		<< "tasks(" << tasks << ")");
	if ( !wrkq.empty() )
	{
	    std::auto_ptr<task> we(wrkq.front());
	    wrkq.pop();
	    if ( we.get() )
	    {
		eof = false;
		DPRINTF("Worker calling engine");
		cv.unlock();
		we->run();
		cv.lock();
		DPRINTF("Worker returning engine");
	    }
	    else
	    {
		DPRINTF("EOF reached");
		eof = true;
	    }
	    tasks--;
        }
	if ( wrkq.empty() && quit )
	{
	    DPRINTF("Worker shutting down");
	    if ( --numthreads == 0 )
	    {
		cv.broadcast();	// FIXME same predicate!
	    }
	    break;
        }
    }
    DPRINTF("Worker exiting");
    return 0;
}

void
WorkQueue::drain()
{
    Lock<ConditionVariable> lock(cv);
    if ( numthreads > 0 )
    {
	quit = true;
	if ( idlethreads > 0 )
	{
	    cv.broadcast();
        }
	while ( numthreads > 0 )
	{
	    cv.wait();		// FIXME??? using same cv for two predicates
        }
    }
}

WorkQueue::~WorkQueue()
{
    drain();
}

void
WorkQueue::add(task* item)
{
    std::auto_ptr<task> tsk(item);
    Lock<ConditionVariable> lock(cv);
    if ( maxthreads == 0 )
    {
	DPRINTF("maxthreads ==0 task");
	if ( item )
	{
	    item->run();
	}
	return;
    }
    wrkq.push(tsk.release());
    tasks++;
    if ( idlethreads > 0 )
    {
	DPRINTF("Signaling idle worker");
	cv.signal();
    }
    else if ( numthreads < maxthreads )
    {
	DPRINTF("Creating new worker");
	FunctionThread ft(WorkQueue_server,this,FunctionThread::Detached,stacksize);
	numthreads++;
    }
}

void
WorkQueue::wait()
{
    add( 0 );
    if ( maxthreads )
    {
	gate.wait();
    }
    gate.close();
    DPRINTF("wait: finished...");
}

