/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2010 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */
#include "mpi.h" 
#include "stdio.h"
#include "stdlib.h"
#include "mpitest.h"

/* 
 * Tests for lock contention, including special cases within the MPICH code 
 * (any MPI implementation should pass these tests; in the MPICH case, our
 * coverage analysis showed that the lockcontention.c test was not covering
 * all cases, and in fact, this test revealed a bug in the code).
 *
 * In all of these tests, each process writes (or accesses) the values
 * rank + i*size_of_world for NELM times.
 *
 * This test strives to avoid operations not strictly permitted by MPI RMA,
 * for example, it doesn't target the same locations with multiple put/get
 * calls in the same access epoch.
 */

#define NELM 200
#define NBLOCK 10
#define MAX_ERRS_REPORT 10

/* 
 *  Each process writes data into the rmabuf on the process with target rank
 *  trank.  The final result in rmabuf are the consecutive integers starting
 *  from 0.  Each process, however, does not write a consecutive block.  
 *  Instead, they write these locations:
 *
 *  for i=0,...,NELM-1
 *     for j=0,...,NBLOCK-1
 *         j + NBLOCK * (rank + i * wsize)
 *  
 * The value written is the location.
 *
 * In many cases, multiple RMA operations are needed.  Where these must not
 * overlap, the above pattern is replicated at NBLOCK*NELM*wsize.
 * (NBLOCK is either 1 or NBLOCK in the code below, depending on use) 
 */

static int toterrs = 0;

int testValues( int, int, int, int *, const char * );

int main(int argc, char *argv[]) 
{ 
    int rank, wsize, i, j, cnt;
    int *rmabuf, *localbuf, *localbuf2, *vals;
    MPI_Win win;
    int trank = 0;
    int windowsize;

    MTest_Init(&argc,&argv); 
    MPI_Comm_size(MPI_COMM_WORLD,&wsize); 
    MPI_Comm_rank(MPI_COMM_WORLD,&rank); 

    if (wsize < 2) {
        fprintf(stderr, "Run this program with at least 2 processes\n");
        MPI_Abort(MPI_COMM_WORLD,1);
    }

    windowsize = (2*NBLOCK + 2) * NELM * wsize;
    rmabuf     = (int *)malloc( windowsize * sizeof(int) );
    localbuf   = (int *)malloc( NELM * sizeof(int) );
    localbuf2  = (int *)malloc( NELM * NBLOCK * sizeof(int) );
    vals       = (int *)malloc( NELM*sizeof(int) );

    /* 
     * Initialize the buffers
     */
    for (i=0; i<NELM; i++) {
      localbuf[i] = rank + i*wsize;
    }
    cnt = 0;
    for (i=0; i<NELM; i++) {
	for (j=0; j<NBLOCK; j++) {
	    localbuf2[cnt++] = j + NBLOCK * (rank + i*wsize);
	}
    }
    for (i=0; i<windowsize; i++) {
      rmabuf[i] = -1;
    }

    /* Create the window */
    MPI_Win_create(rmabuf, windowsize*sizeof(int), sizeof(int), MPI_INFO_NULL, 
		   MPI_COMM_WORLD, &win); 

    /* Multiple puts, with contention at trank */
    MPI_Barrier( MPI_COMM_WORLD );
    for (i=0; i<NELM; i++) {
	MPI_Win_lock( MPI_LOCK_EXCLUSIVE, trank, 0, win );
	MPI_Put( &localbuf[i], 1, MPI_INT, trank, 
		 rank + i*wsize, 1, MPI_INT, win );
	MPI_Put( &localbuf[i], 1, MPI_INT, trank, 
		 rank + (i+NELM)*wsize, 1, MPI_INT, win );
	MPI_Win_unlock( trank, win );
    }
    MPI_Barrier( MPI_COMM_WORLD );
    if (rank == trank) {
	MPI_Win_lock( MPI_LOCK_EXCLUSIVE, trank, 0, win );
	toterrs += testValues( 1, NELM, wsize, rmabuf, "Multiple puts (1)" );
	toterrs += testValues( 1, NELM, wsize, rmabuf + wsize*NELM,
			       "Multiple puts (2)" );
	MPI_Win_unlock( trank, win );
    }
    MPI_Barrier( MPI_COMM_WORLD );
    /* Reinit the rmabuf */
    for (i=0; i<windowsize; i++) {
      rmabuf[i] = -1;
    }
    MPI_Barrier( MPI_COMM_WORLD );

    /* Single put with contention */
    trank = 0;
    for (i=0; i<NELM; i++) {
	MPI_Win_lock( MPI_LOCK_EXCLUSIVE, trank, 0, win );
	MPI_Put( &localbuf[i], 1, MPI_INT, trank, rank + i*wsize, 1, MPI_INT, 
		 win );
	MPI_Win_unlock( trank, win );
    }
    MPI_Barrier( MPI_COMM_WORLD );
    if (rank == trank) {
	MPI_Win_lock( MPI_LOCK_EXCLUSIVE, trank, 0, win );
	toterrs += testValues( 1, NELM, wsize, rmabuf, "Single put" );
	MPI_Win_unlock( trank, win );
    }

    /* Reinit the rmabuf */
    for (i=0; i<windowsize; i++) {
	rmabuf[i] = -1;
    }
    /* Longer puts with contention at trank */
    MPI_Barrier( MPI_COMM_WORLD );
    for (i=0; i<NELM; i++) {
	MPI_Win_lock( MPI_LOCK_EXCLUSIVE, trank, 0, win );
	if (rank != trank) {
	    MPI_Put( &localbuf2[i*NBLOCK], NBLOCK, MPI_INT, trank, 
		     NELM * wsize + NBLOCK*(rank+i*wsize), NBLOCK, 
		     MPI_INT, win );
	    MPI_Put( &localbuf2[i*NBLOCK], NBLOCK, MPI_INT, trank, 
		     NELM * wsize + NBLOCK*(rank+(i+NELM)*wsize), NBLOCK, 
		     MPI_INT, win );
	}
	MPI_Put( &localbuf[i], 1, MPI_INT, trank, rank+i*wsize, 1, MPI_INT, 
		 win );
	MPI_Win_unlock( trank, win );
    }
    MPI_Barrier( MPI_COMM_WORLD );
    if (rank == trank) {
	/* For simplicity in testing, set the values that rank==trank
	   would have set. */
	for (i=0; i<NELM; i++) {
	    for (j=0; j<NBLOCK; j++) {
		rmabuf[NELM*wsize + NBLOCK*(trank+i*wsize) + j] = 
		    j + NBLOCK*(trank +i*wsize);
		rmabuf[NELM*wsize + NBLOCK*(trank+(i+NELM)*wsize) + j] = 
		    j + NBLOCK*(trank + i*wsize);
	    }
	}
	MPI_Win_lock( MPI_LOCK_EXCLUSIVE, trank, 0, win );
	toterrs += testValues( 1, NELM, wsize, rmabuf, "Long puts (1)" );
	toterrs += testValues( NBLOCK, NELM, wsize, rmabuf + NELM * wsize,
			       "Long puts(2)" );
	toterrs += testValues( NBLOCK, NELM, wsize, 
			       rmabuf + NELM * wsize * (1 + NBLOCK),
			       "Long puts(3)" );
	MPI_Win_unlock( trank, win );
    }
    
    /* Reinit the rmabuf */
    for (i=0; i<windowsize; i++) {
	rmabuf[i] = -1;
    }
    for (i=0; i< NELM; i++) 
	vals[i] = -2;
    
    /* Put mixed with Get */
    MPI_Barrier( MPI_COMM_WORLD );
    for (i=0; i<NELM; i++) {
	MPI_Win_lock( MPI_LOCK_EXCLUSIVE, trank, 0, win );
	if (rank != trank) {
	    MPI_Put( &localbuf2[i], NBLOCK, MPI_INT, trank, 
		     NELM*wsize + NBLOCK*(rank + i*wsize), NBLOCK, MPI_INT, 
		     win );
	    MPI_Put( &localbuf[i], 1, MPI_INT, trank, 
		     rank + i*wsize, 1, MPI_INT, win );
	}
	else {
	    MPI_Get( &vals[i], 1, MPI_INT, trank, i, 1, MPI_INT, win );
	}
	MPI_Win_unlock( trank, win );
    }
    MPI_Barrier( MPI_COMM_WORLD );
    if (rank == trank) {
	/* Just test the Get */
	for (i=0; i<wsize; i++) {
	    if (i == trank) {
		if (vals[i] != -1) {
		    toterrs++;
		    if (toterrs < MAX_ERRS_REPORT) {
			printf( "put/get: vals[%d] = %d, expected -1\n",
				i, vals[i] );
		    }
		}
	    }
	    else if (vals[i] != i && vals[i] != -1) {
		toterrs++;
		if (toterrs < MAX_ERRS_REPORT) {
		    printf( "put/get: vals[%d] = %d, expected -1 or %d\n",
			    i, vals[i], i );
		}
	    }
	}
    }

    /* Contention only with get */
    for (i=0; i<windowsize; i++) {
	rmabuf[i] = -i;
    }
    for (i=0; i<NELM; i++)
	vals[i] = -2;

    MPI_Barrier( MPI_COMM_WORLD );
    for (i=0; i<NELM; i++) {
	MPI_Win_lock( MPI_LOCK_EXCLUSIVE, trank, 0, win );
	MPI_Get( &vals[i], 1, MPI_INT, trank, i, 1, MPI_INT, win );
	MPI_Win_unlock( trank, win );
    }
    MPI_Barrier( MPI_COMM_WORLD );
    if (rank == trank) {
	for (i=0; i<NELM; i++) {
	    if (vals[i] != -i) {
		toterrs++;
		if (toterrs < MAX_ERRS_REPORT) {
		    printf( "single get: vals[%d] = %d, expected %d\n",
			    i, vals[i], -i );
		}
	    }
	}
    }

    /* Contention with accumulate */
    MPI_Barrier( MPI_COMM_WORLD );
    for (i=0; i<NELM*wsize; i++) {
	rmabuf[i] = 0;
    }
    MPI_Barrier( MPI_COMM_WORLD );
    for (i=0; i<NELM; i++) {
	MPI_Win_lock( MPI_LOCK_EXCLUSIVE, trank, 0, win );
	MPI_Accumulate( &localbuf[i], 1, MPI_INT, trank, rank+i*wsize, 
			1, MPI_INT, MPI_SUM, win );
	MPI_Accumulate( &localbuf[i], 1, MPI_INT, trank, rank+i*wsize, 
			1, MPI_INT, MPI_SUM, win );
	MPI_Win_unlock( trank, win );
    }
    MPI_Barrier( MPI_COMM_WORLD );
    if (rank == trank) {
	MPI_Win_lock( MPI_LOCK_EXCLUSIVE, trank, 0, win );
	for (i=0; i<NELM * wsize; i++) {
	    if (rmabuf[i] != 2*i) {
		toterrs++;
		if (toterrs < MAX_ERRS_REPORT) {
		    printf( "2 accumulate: rmabuf[%d] = %d, expected %d\n",
			    i, rmabuf[i], 2*i );
		}
	    }
	}
	MPI_Win_unlock( trank, win );
    }

    MPI_Win_free( &win );

    free( rmabuf );
    free( localbuf );
    free( localbuf2 );
    free( vals );
   
    MTest_Finalize(toterrs);
    MPI_Finalize(); 
    return 0; 
} 

/* Test the values in the rmabuf against the expected values.  Return the 
   number of errors */
int testValues( int nb, int nelm, int wsize, int *rmabuf, const char *msg )
{
    int i, errs = 0;
		
    for (i=0; i<nb * nelm * wsize; i++) {
	if (rmabuf[i] != i) {
	    if (toterrs + errs < MAX_ERRS_REPORT) {
		printf( "%s:rmabuf[%d] = %d expected %d\n", 
			msg, i, rmabuf[i], i );
	    }
	    errs++;
	}
    }

    return errs;
}
