/*
    monte_carlo_marginal

    Copyright (C) 2013 - 2015 Douglas L. Theobald

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    -/_|:|_|_\-
*/
/******************************************************************************
 *
 *  File:           monte_carlo_marginal.c
 *
 *  Function:
 *
 *  Author(s):      Douglas L. Theobald
 *
 *  Copyright:      Copyright (c) 2013 Douglas L. Theobald
 *                  All Rights Reserved.
 *
 *  Source:         Started anew.
 *
 *  Notes:
 *
 *  Change History:
 *          2011_04_15_nnn    Started source
 *
 *****************************************************************************/
/*
gcc -O3 -ffast-math -Wall -std=c99 -pedantic -mtune=native -o monte_carlo_marginal \
-lgsl -lgslcblas -lpthread -lm monte_carlo_marginal.c; \
sudo cp monte_carlo_marginal /usr/local/bin/
*/

// ./monte_carlo_marginal -d20 -n 10000   -i 1000000 -l 1e-10 -ep -s10
// ./monte_carlo_marginal -d2  -n 10000   -i 1000000 -l 1e-10 -gp
// ./monte_carlo_marginal -d10 -n 100     -i 100000  -l 10    -egp
// ./monte_carlo_marginal -d1  -n 1000000 -i 100000  -l 1 -e -s1


#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#ifdef   __linux__
  #include <getopt.h>
#endif
#include <ctype.h>
#include <math.h>
#include <float.h>
#include <pthread.h>
#include <assert.h>
#include <sys/resource.h>
#include <time.h>
#include <gsl/gsl_rng.h>
#include <gsl/gsl_randist.h>
//#include <gsl/gsl_integration.h>
#include <gsl/gsl_math.h>
#include <gsl/gsl_eigen.h>
#include <gsl/gsl_linalg.h>
#include <gsl/gsl_sf_psi.h>
#include <gsl/gsl_version.h>
#include <gsl/gsl_sort_double.h>
#include <gsl/gsl_sf_gamma.h>

#define VERSION "0.7"
#define LN2PI  ( M_LN2 + M_LNPI )
#define DIGAMMA2 ( gsl_sf_psi_int(2) )
#define DIGAMMA3 ( gsl_sf_psi_int(3) )
#define DIGAMMA4 ( gsl_sf_psi_int(4) )


double             burnin = 0.0;
int                iters = 1000000;
double             lambda_0 = 1.0;
unsigned long int  seed = 0;
int                expo_model = 0;
int                gauss_model = 0;
int                write_files = 0;
int                thrdnum = 1;
int                parallel = 0;
int                entropy_calc = 0;
int                klentk = 1;  /* K-L entropy calc k value (kth smallest dist) */
int                randmeth = 1;

int                dim = 1;  /* number of params */
int                hdim = 1; /* # hierarchical params */
int                ndata = 100;
int                nd;

/* data globals */
double           **data = NULL; /* data */
double            *y = NULL;    /* data averages */
double            *x2 = NULL;
double             yt, yt2, x2t;

/* parameter globals */
double            *pave = NULL; /* parameter averages */
double            *h = NULL;    /* hyperparameter */
double            *lnpost = NULL;
double            *lnlike = NULL;
double            *lnprior = NULL;
double            *lnscore = NULL;
double           **x = NULL; /* posterior sample */
double           **cov = NULL;


typedef struct
{
    gsl_rng        *r2;

    double          lambda_0;
    double          avelike;
    double          amlik;
    double          dic;
    double          bicm_20;
    double          bicm_dlt;
    double          marg_lnnorm;
    double          varlnlike;
    double          avelnprior;
    double          avelnlike;
    double          avelnpost;
    double          avelnscore;
    double          varlnpost;
    double          avelnprlk2;
    double          avelnprlk;
    double          explnlike;
    double          pi_explnlike;
    double          pi_expexplnlike;
    double          pi_explnlike2;
    double          explnprior;
    double          explnpost;
    double          pi_explnpost;
    double          entlnpost;
    double          Nentropy;
    double          lapmet;
    double          edge_entropy;
    double          expmet;
    double          hme;
    double          DIC;
    double          marglik;
    double          maxpost;
    double          maxprior;
    double          maxlike;
    double          lndet;
    double          lnfish;
    double          kl;

    double          pi_avelnlike;
    double          pi_avelnpost;
    double          pi_avenoprior;
    double          pi_varlnpost;
    double          pi_entlnpost;
    double          pi_dic;

    double        **cov;
    double         *pave;
    double        **x;

    int             dim, iters, ndata, hdim;
} MonteCarloDat;


void
Version(void)
{
    printf("\n  MONTE_CARLO_MARGINAL version %s compiled on %s %s\n  by user %s with machine \"%s\" \n",
           VERSION, __DATE__, __TIME__, getenv("USER"), getenv("HOST"));
    printf("  Compiled with GSL version %s\n\n", GSL_VERSION);
    fflush(NULL);
}


void
MonteCarloDatInit(MonteCarloDat *mcdat)
{
    mcdat->r2 = NULL;
    mcdat->lambda_0        = INFINITY;
    mcdat->avelike         = INFINITY;
    mcdat->amlik           = INFINITY;
    mcdat->dic             = INFINITY;
    mcdat->bicm_20         = INFINITY;
    mcdat->bicm_dlt        = INFINITY;
    mcdat->marg_lnnorm     = INFINITY;
    mcdat->varlnlike       = INFINITY;
    mcdat->avelnprior      = INFINITY;
    mcdat->avelnlike       = INFINITY;
    mcdat->avelnpost       = INFINITY;
    mcdat->avelnscore      = INFINITY;
    mcdat->varlnpost       = INFINITY;
    mcdat->avelnprlk2      = INFINITY;
    mcdat->avelnprlk       = INFINITY;
    mcdat->explnlike       = INFINITY;
    mcdat->pi_explnlike    = INFINITY;
    mcdat->pi_expexplnlike = INFINITY;
    mcdat->pi_explnlike2   = INFINITY;
    mcdat->explnprior      = INFINITY;
    mcdat->explnpost       = INFINITY;
    mcdat->pi_explnpost    = INFINITY;
    mcdat->entlnpost       = INFINITY;
    mcdat->Nentropy        = INFINITY;
    mcdat->lapmet          = INFINITY;
    mcdat->edge_entropy    = INFINITY;
    mcdat->expmet          = INFINITY;
    mcdat->hme             = INFINITY;
    mcdat->DIC             = INFINITY;
    mcdat->marglik         = INFINITY;
    mcdat->maxpost         = INFINITY;
    mcdat->maxprior        = INFINITY;
    mcdat->maxlike         = INFINITY;
    mcdat->lndet           = INFINITY;
    mcdat->lnfish          = INFINITY;
    mcdat->kl              = INFINITY;

    mcdat->pi_avelnlike    = INFINITY;
    mcdat->pi_avelnpost    = INFINITY;
    mcdat->pi_avenoprior   = INFINITY;
    mcdat->pi_varlnpost    = INFINITY;
    mcdat->pi_entlnpost    = INFINITY;
    mcdat->pi_dic          = INFINITY;

    mcdat->cov = NULL;
    mcdat->pave = NULL;
    mcdat->x = NULL;

    mcdat->dim = mcdat->iters = mcdat->ndata = mcdat->hdim = 0;
}


void
Usage(void);


double
average(const double *data, const int dim)
{
    double          m = 0.0;
    int             i = dim;

    while(i-- > 0)
        m += *data++;

    return(m / (double) dim);
}


double
variance(const double *data, const int dim, const double mean)
{
    double          v = 0.0, tmpv;
    int             i = dim;

    while(i-- > 0)
    {
        tmpv = *data++ - mean;
        v += (tmpv * tmpv);
    }

    return(v / dim);
}


void
VecPrint(double *vec, const int size)
{
    int             i;

    for (i = 0; i < size; ++i)
        printf(" %4d [ % 14.8e ]\n", i, vec[i]);

    printf("\n");

    fflush(NULL);
}


void
MatPrintLowerDiag(double **matrix, const int size)
{
    int             i, j;

    printf("\n\n");
    for (i = 0; i < size; ++i)
    {
        printf("%-2d: [", i);
        for (j = 0; j <= i; ++j)
            printf(" % 14.6f", matrix[i][j]);
        printf(" ]\n");
    }

    printf("     ");
    for (i = 0; i < size; ++i)
        printf(" % 14d", i);
    printf("\n");

    fflush(NULL);
}


void
MatPrint(double **matrix, const int size)
{
    int             i, j;

    printf("\n\n");
    for (i = 0; i < size; ++i)
    {
        printf("%-2d: [", i);
        for (j = 0; j < size; ++j)
            printf(" % 14.6f", matrix[i][j]);
        printf(" ]\n");
    }

    printf("     ");
    for (i = 0; i < size; ++i)
        printf(" % 14d", i);
    printf("\n");

    fflush(NULL);
}


void
MatDestroy(double ***matrix_ptr)
{
    double       **matrix = *matrix_ptr;

    if (matrix != NULL)
    {
        if (matrix[0] != NULL)
        {
            free(matrix[0]);
            matrix[0] = NULL;
        }

        free(matrix);
        *matrix_ptr = NULL;
    }
}


double
**MatAlloc(const int rows, const int cols)
{
    int            i;
    double       **matrix = NULL;
    double        *matspace = NULL;

    matspace = (double *) calloc((rows * cols), sizeof(double));
    if (matspace == NULL)
    {
        perror("\n ERROR");
        printf("\n ERROR: Failure to allocate matrix space in MatAlloc(): (%d x %d)\n", rows, cols);
        exit(EXIT_FAILURE);
    }

    /* allocate room for the pointers to the rows */
    matrix = (double **) malloc(rows * sizeof(double *));
    if (matrix == NULL)
    {
        perror("\n ERROR");
        printf("\n ERROR: Failure to allocate room for row pointers in MatAlloc(): (%d)\n", rows);
        exit(EXIT_FAILURE);
    }

    /*  now 'point' the pointers */
    for (i = 0; i < rows; i++)
        matrix[i] = matspace + (i * cols);

    return(matrix);
}


/*
Calculate eigenvalues of a square, symmetric, real matrix, using GSL.
Eigenvalues are returned in descending order, largest first.
Pointer *eval must be allocated.
Input matrix **cov is NOT perturbed.
*/
void
EigenvalsGSL(double **cov, const int dim, double *eval)
{
    double        *cov_cpy = NULL;

    cov_cpy = malloc(dim * dim * sizeof(double));
    for (int i = 0; i < dim; ++i)
        memcpy((cov_cpy + i*dim), cov[i], dim * sizeof(double));
    gsl_matrix_view m = gsl_matrix_view_array(cov_cpy, dim, dim);
    gsl_vector_view evalv = gsl_vector_view_array(eval, dim);
    gsl_eigen_symm_workspace *w = gsl_eigen_symm_alloc(dim);

    gsl_eigen_symm(&m.matrix, &evalv.vector, w);

    gsl_eigen_symm_free(w);
    free(cov_cpy);
}


/* This one destroys half of the input matrix **cov */
void
EigenvalsGSLDest(double **cov, const int dim, double *eval)
{
    gsl_matrix_view m = gsl_matrix_view_array(cov[0], dim, dim);
    gsl_vector_view evalv = gsl_vector_view_array(eval, dim);
    gsl_eigen_symm_workspace *w = gsl_eigen_symm_alloc(dim);
    gsl_eigen_symm(&m.matrix, &evalv.vector, w);
    gsl_eigen_symm_free(w);
}


void
CholeskyGSLDest(double **mat, const int dim)
{
    gsl_matrix_view m = gsl_matrix_view_array(mat[0], dim, dim);
    gsl_linalg_cholesky_decomp(&m.matrix);
}


double
RandScale(double variance, int randmeth, double b)
{
    double          scale;

    switch(randmeth)
    {
        case 1:
        case 'n': /* normal */
            scale = sqrt(variance);
            break;

        case 2:
        case 'l': /* logistic */
            scale = sqrt(3.0 * variance) / M_PI;
            break;

        case 3:
        case 'L': /* Laplacian */
            scale = sqrt(variance / 2.0);
            break;

        case 4:
        case 'C': /* Cauchy */
            scale = 1;
            break;

        case 5:
        case 'g': /* gamma */
            scale = sqrt(variance / b);
            break;

        case 12:
        case 'e': /* exponential */
            scale = sqrt(variance);
            break;

        default:
            scale = sqrt(variance);
    }

    return(scale);
}


static void
RandFillVec(double *vec, int len, int randmeth, const gsl_rng *r2)
{
    int             j;
    double a = 1;
    double b = 4;
    double u;
    double alpha;
    double beta;
    double mu = 0.0;

    switch (randmeth)
    {
        case 1:
        case 'n': /* normal */
            for (j = 0; j < len; ++j)
                vec[j] = gsl_ran_gaussian(r2, 1.0);
            break;

        case 2:
        case 'l': /* logistic */
            for (j = 0; j < len; ++j)
                vec[j] = gsl_ran_logistic(r2, 1.0);
            break;

        case 3:
        case 'L': /* Laplacian */
            for (j = 0; j < len; ++j)
                vec[j] = gsl_ran_laplace(r2, 1.0);
            break;

        case 4:
        case 'C': /* Cauchy */
            for (j = 0; j < len; ++j)
                vec[j] = gsl_ran_cauchy(r2, 1.0);
            break;

        case 5:
        case 'g': /* gamma */
            alpha = 3.0;
            beta = 1.0;

            printf("\nentropy:%g %g %g\n", alpha, beta, alpha - log(beta) + lgamma(alpha) + (1-alpha)*gsl_sf_psi(alpha));

            for (j = 0; j < len; ++j)
                vec[j] = gsl_ran_gamma(r2, alpha, beta);
            break;

        case 12:
        case 'e': /* exponential */
            for (j = 0; j < len; ++j)
                vec[j] = gsl_ran_exponential(r2, 1.0);
            break;

        case 13:
        case 'b': /* beta */
            for (j = 0; j < len; ++j)
                vec[j] = gsl_ran_beta(r2, a,b);

            printf("\nentropy:%g %g %g\n", a, b, gsl_sf_lnbeta(a,b)
                   -(a-1.0)*gsl_sf_psi(a)
                   -(b-1.0)*gsl_sf_psi(b)
                   +(a+b-2.0)*gsl_sf_psi(a+b));
            break;

        case 14:
        case 'E': /* extreme value */

            beta = 0.1;

            for (j = 0; j < len; ++j)
            {
                u = gsl_rng_uniform(r2);
                vec[j] = mu - beta * log(-log(u));
            }

            printf("\nentropy:%g %g %g\n", mu, beta, log(beta)+M_EULER+1.0);
            break;

        default:
            printf("\n  ERROR888: Bad random param -r '%c' \n",
                   (char) randmeth);
            Usage();
            exit(EXIT_FAILURE);
    }
}


/*
The analyitical entropy of the sampled distribution is easy to calculate:
The total entropy is the sum of the raw uncorrelated entropy and
1/2 log(det(C)), where C is the covariance matrix used to correlate and
scale the raw variates.
*/
void
RandVec(double **vec, const int len, const int iters, int randmeth, const gsl_rng *r2)
{
    int            i, j, k;
    double       **covmat = MatAlloc(len, len);
    double       **cormat = MatAlloc(len, len);
    double       **tmpmat = MatAlloc(len, len);
    double        *diag = malloc(len * sizeof(double));
    double        *eval = malloc(len * sizeof(double));
    double       **tmpvec = MatAlloc(len, iters);
    double         lndet;

    for (i = 0; i < len; ++i)
        RandFillVec(tmpvec[i], iters, randmeth, r2);

//     for (i = 0; i < iters; ++i)
//         for (j = 0; j < len; ++j)
//             vec[j][i] = tmpvec[j][i];

    if (1)
    {
        for (i = 0; i < len; ++i)
            for (j = 0; j < i; ++j)
                tmpmat[i][j] = gsl_ran_flat(r2, -1.0, 1.0);

        for (i = 0; i < len; ++i)
            tmpmat[i][i] = gsl_ran_flat(r2, 0.0, 1.0);

        MatPrintLowerDiag(tmpmat, len);

        for (i = 0; i < len; ++i)
            for (j = 0; j < len; ++j)
                for (k = 0; k < len; ++k)
                    cormat[i][k] += tmpmat[i][j] * tmpmat[k][j];

        printf("\n\"correlation matrix\":");
        MatPrintLowerDiag(cormat, len);

    //    PrintCovMatGnuPlot((const double **) covmat, len, mystrcat(cdsA->algo->rootname, "_cor.mat"));

        for (i = 0; i < len; ++i)
            diag[i] = gsl_ran_gamma(r2, 2.0, 10.0);

        for (i = 0; i < len; ++i)
            for (j = 0; j < len; ++j)
                covmat[i][j] = cormat[i][j] * sqrt(diag[i] * diag[j]);

        for (i = 0; i < len; ++i)
            covmat[i][i] += 1.0;

        printf("\ncovariance matrix:");
        MatPrintLowerDiag(covmat, len);

        for (i = 0; i < len; ++i)
            diag[i] = covmat[i][i];

        printf("\nvariances:\n");

        for (i = 0; i < len; ++i)
            printf("%-3d %f\n", i, diag[i]);

        for (i = 0; i < len; ++i)
            for (j = 0; j < len; ++j)
                cormat[i][j] = covmat[i][j] / sqrt(diag[i] * diag[j]);

        printf("\ntrue correlation matrix:");
        MatPrintLowerDiag(cormat, len);

        EigenvalsGSL(covmat, len, eval);

        printf("\neigenvalues:\n");

        for (i = 0; i < len; ++i)
            printf("%-3d %f\n", i, eval[i]);

        lndet = 0.0;
        for(i = 0; i < len; ++i)
            lndet += log(eval[i]);

        printf("logdet: %f\n", lndet);
        printf("half logdet: %f\n", 0.5*lndet);

        double entropy = 0.5 * len * log(2.0 * M_PI * M_E) + 0.5 * lndet;
        printf("\nentropy:    %14.3f", entropy);

        CholeskyGSLDest(covmat, len);
        printf("\nCholesky lower diagonal matrix:");
        MatPrintLowerDiag(covmat, len);

        fflush(NULL);

        for (i = 0; i < iters; ++i)
            for (j = 0; j < len; ++j)
                for (k = 0; k <= j; ++k) /* because covmat is lower diagonal, upper should be all zeros */
                    vec[j][i] += covmat[j][k] * tmpvec[k][i];
    }

//     for (i = 0; i < iters; ++i)
//     {
//         printf("UNIFORM %4d", i);
//         for (j = 0; j < len; ++j)
//             printf(" %14.10f", erf(vec[j][i]/sqrt(2.0)));
//         printf("\n");
//     }
//     fflush(NULL);

    MatDestroy(&tmpvec);
    MatDestroy(&tmpmat);
    MatDestroy(&covmat);
    MatDestroy(&cormat);
    free(diag);
    free(eval);
}


/* Euclidean norm, L_2 */
inline double
Norm2(double **x, const int i, const int j, double dim)
{
    int         k;
    double      dist, tmpx;

    dist = 0.0;
    for (k = 0; k < dim; ++k)
    {
        tmpx = x[k][i] - x[k][j];
        dist += tmpx * tmpx;
    }

    return(dist);
}


/* Max norm, L_inf */
inline double
NormMax(double **x, const int i, const int j, double dim)
{
    int         k;
    double      dist, max;

    max = DBL_MIN;
    for (k = 0; k < dim; ++k)
    {
        dist = fabs(x[k][i] - x[k][j]);
        if (max < dist)
            max = dist;
    }

    return(max);
}


/* Manhattan norm, L_1 */
inline double
NormManhattan(double **x, const int i, const int j, double dim)
{
    int         k;
    double      dist;

    dist = 0.0;
    for (k = 0; k < dim; ++k)
        dist += fabs(x[k][i] - x[k][j]);

    return(dist);
}


/* Naive brute force search */
double
*FindLogDists(double **x, const int dim, const int n, const int k)
{
    double     *dists = NULL;
    double     *tmpdist = NULL;
    size_t     *smalls = NULL;
    int         i, j;

    dists = malloc(n * sizeof(double));
    tmpdist = malloc(n * sizeof(double));
    smalls = malloc(k * sizeof(size_t));

    for (i = 0; i < n; ++i)
    {
        for (j = 0; j < n; ++j)
        {
            //tmpdist[j] = Norm2(x, i, j, dim); // Euclidian norm L_2
            //tmpdist[j] = NormMax(x, i, j, dim); // Max norm
            tmpdist[j] = NormManhattan(x, i, j, dim); // Manhattan norm L_1
        }

        tmpdist[i] = DBL_MAX; // distance to itself = 0, so kill this one

        //gsl_sort_smallest(smalls, k, tmpdist, 1, n);
        gsl_sort_smallest_index(smalls, k, tmpdist, 1, n);

        //dists[i] = 0.5*log(smalls[k-1]); // Euclidian norm L_2
        dists[i] = log(tmpdist[smalls[k-1]]); // Max norm or Manhattan
    }

    free(smalls);
    free(tmpdist);

    return(dists);
}


/* Kozachenko-Leonenko non-parametric entropy estimate */
/* We can use any norm -- Euclidean and max-norm are popular choices,
   but I like Manhattan because its easy and fast */
double
CalcKLentropy(double **x, const int dim, const int n, const int k)
{
    double         *dists = NULL;
    double          entropy, volume, term1;

    dists = FindLogDists(x, dim, n, k);

    /* http://en.wikipedia.org/wiki/Volume_of_an_n-ball#Balls_in_Lp_norms */
    //volume = 0.5*dim*M_LNPI - lgamma(0.5*dim + 1.0); // Euclidian norm L_2
    //volume = dim * log(2); // Max norm L_inf
    volume = dim * log(2) - lgamma(dim + 1.0); // Manhattan norm L_1

    term1 = dim * average(dists, n);

    entropy = term1 + volume + gsl_sf_psi_int(n) - gsl_sf_psi_int(k);

    printf("\nvolume: %g\n", volume);
    printf("term1:  %g\n", term1);
    printf("lnN+g:  %g\n", log(n) + M_EULER);
    printf("lnN+g:  %g\n", gsl_sf_psi_int(n) + M_EULER);
    printf("lnN+g:  %g\n", gsl_sf_psi_int(n) + DIGAMMA2); // for k=2

    return(entropy);
}


/*
Calculate harmonic mean estimator, which should not be used, but we determine it for fun
and to see how bad it actually is.
As boni, we get the log arithmetic mean likelihood and log geometric mean likelihood.
*/
double
CalcHarmonicMean(MonteCarloDat *mcdat, const double *ll, const int len)
{
    double         blik, mlik, hmlik, amlik, diff, ediff, liksi, harm_mean, var, tmp;
    int            i;

    /* first center the log-likelihoods, as the likelihoods are probably too small to represent. */
    blik = 0.0;
    for (i = 0; i < len; ++i)
        blik += ll[i];

    blik /= len;

    mlik = hmlik = amlik = 0.0;
    for (i = 0; i < len; ++i)
    {
        liksi = ll[i];
        diff = liksi - blik;
        ediff = exp(diff);

        if (isfinite(ediff))
        {
            mlik  += ediff;
            hmlik += 1.0 / ediff;
        }

        amlik += liksi;
    }

    amlik /= len;

    var = 0.0;
    for (i = 0; i < len; ++i)
    {
        tmp = ll[i] - amlik;
        var += tmp*tmp;
    }

    var /= len;

    harm_mean = blik - log(hmlik) + log(len);

    mcdat->avelike = log(mlik / len) + blik;
    mcdat->amlik = amlik;
    mcdat->hme = harm_mean;
    mcdat->dic = amlik - var;
    mcdat->marg_lnnorm = amlik - 0.5 * var;
    mcdat->bicm_20 = amlik - var * (log(ndata) - 1.0);
    mcdat->bicm_dlt = amlik - var * log(ndata) + var * (ndata-1.0)/ndata;
    mcdat->varlnlike = var;

    return(harm_mean);
}


/*
Calculate the bias in the entropy estimate due to deviation from Normality.
Based on on Edgeworth expansion of a PDF in terms of its cumulants (moments).
The bias term is substracted from the usual multivariate Gaussian
entropy:

0.5 * d * log(2.0 * M_PI * M_E) + 0.5 * lndet

where lndet is the log of the determinant of the d*d covariance matrix.

Multivariate third order corrections (using the skewness) come from Van Hulle 2005:

See:

Marc M. Van Hulle (2005)
"Multivariate Edgeworth-based entropy estimation."
2005 IEEE Workshop on Machine Learning for Signal Processing,
Conference Proceedings
28-28 Sept. 2005
pp 311 - 316

or

Marc M. Van Hulle (2005)
"Edgeworth Approximation of Multivariate Differential Entropy"
Neural Computation 17, 1903–1910

See equation 2.2.

The fourth order corrections (kurtosis terms) are univariate only;
they don't account for cross-kurtosis between dimensions.
Fourth order corrections are from Comon 1994:

Comon, P. (1994)
"Independent component analysis, a new concept?"
Signal processing 36, 287–314.

Amari 1996 also has similar 4th order corrections, but they seem to be
wrong:

Amari, S.-I., Cichocki, A. and Yang, H. H. (1996)
"A new learning algorithm for blind signal separation."
Advances in neural information processing systems 8
Eds. D. Touretzky, M. Mozer, and M. Hasselmo.
MIT Press, Cambridge.
757–763 (1996).
*/
double
CalcEdgeworthVanHulleEntropy(double **vec, int dim, int len)
{
    int            i, j, k, m;
    double        *ave = NULL;
    double        *std = NULL;
    double        *eval = NULL;
    double       **dif = MatAlloc(dim,len);
    double         term1, term2, term3;
    double         term4, term5, term6;
    double         t3, t4;
    double         kappa_iii, kappa_iij, kappa_ijk;
    double         kappa_iiii;
    double         entropy, bias, lnscale, lndet, sum, var;
    double       **cor = MatAlloc(dim,dim);
    double       **cov = MatAlloc(dim,dim);
    double         invlen = 1.0/(len-1);

    ave  = malloc(dim * sizeof(double));
    std  = malloc(dim * sizeof(double));
    eval = malloc(dim * sizeof(double));

    printf("\nCalculating Edgeworth entropy approximation ...\n");
    fflush(NULL);

    /* First, normalize data vector to 0 mean, unit 1 variance */
    for (i = 0; i < dim; ++i)
        ave[i] = average(vec[i], len);

    //VecPrint(ave, dim);

    for (i = 0; i < dim; ++i)
        for (j = 0; j < len; ++j)
            dif[i][j] = vec[i][j] - ave[i];

    for (i = 0; i < dim; ++i)
    {
        var = 0.0;
        for (j = 0; j < len; ++j)
            var += dif[i][j] * dif[i][j];

        std[i] = sqrt(var * invlen);
    }

    //VecPrint(std, dim);

    /* Save the determinant of the scale transformation */
    lnscale = 0.0;
    for (i = 0; i < dim; ++i)
        lnscale += log(std[i]);

    /* rescale centered data */
    for (i = 0; i < dim; ++i)
        std[i] = 1.0 / std[i];

    for (i = 0; i < dim; ++i)
        for (j = 0; j < len; ++j)
            dif[i][j] *= std[i];

    /* Calculate the covariance matrix of transformed data (= correlation matrix) */
    for (i = 0; i < dim; ++i)
    {
        for (j = 0; j <= i; ++j)
        {
            sum = 0.0;
            for (k = 0; k < len; ++k)
                sum += dif[i][k] * dif[j][k];

            cor[i][j] = cor[j][i] = sum * invlen;
        }
    }

//     printf ("\n\nEdgeworth correlation matrix:");
//     MatPrintLowerDiag(cor, dim);
//
//     for (i = 0; i < dim; ++i)
//         for (j = 0; j < dim; ++j)
//             cov[i][j] = cor[i][j] / (std[i] * std[j]);
//
//     printf ("\n\nEdgeworth covariance matrix:");
//     MatPrintLowerDiag(cov, dim);

    EigenvalsGSL(cor, dim, eval);

    VecPrint(eval, dim);

    lndet = 0.0;
    for (i = 0; i < dim; i++)
    {
        if (isgreater(eval[i], DBL_EPSILON))
        {
            lndet += log(eval[i]);
        }
        else
        {
            printf("\n WARNING: excluding eigenvalue %d from determinant calculation", i);
            printf("\n WARNING: eigenvalue[%d] = %g < %g", i, eval[i], FLT_EPSILON);
        }
    }

    term1 = 0.0;
    term4 = 0.0;
    term5 = 0.0;
    term6 = 0.0;
    for (i = 0; i < dim; ++i)
    {
        kappa_iii = 0.0;
        kappa_iiii = 0.0;
        for (j = 0; j < len; ++j)
        {
            t3 = dif[i][j] * dif[i][j] * dif[i][j];
            kappa_iii += t3; /* skewness */
            kappa_iiii += t3 * dif[i][j]; /* kurtosis */
        }

        kappa_iii *= invlen;
        kappa_iiii *= invlen;
        kappa_iiii -= 3.0;

        t3 = kappa_iii * kappa_iii;
        t4 = kappa_iiii * kappa_iiii;
        term1 += t3;
        term4 += t4;
        term5 += t3*t3;
        term6 += t3 * kappa_iiii;
    }

    term2 = 0.0;
    for (i = 0; i < dim; ++i)
    {
        for (j = 0; j < dim; ++j)
        {
            if (i != j)
            {
                kappa_iij = 0.0;
                for (k = 0; k < len; ++k)
                    kappa_iij += dif[i][k] * dif[i][k] * dif[j][k];

                kappa_iij *= invlen;

                term2 += kappa_iij * kappa_iij;
            }
        }
    }

    term3 = 0.0;
    for (i = 0; i < dim; ++i)
    {
        for (j = 0; j < i; ++j)
        {
            for (k = 0; k < j; ++k)
            {
                kappa_ijk = 0.0;
                for (m = 0; m < len; ++m)
                    kappa_ijk += dif[i][m] * dif[j][m] * dif[k][m];

                kappa_ijk *= invlen;

                term3 += kappa_ijk * kappa_ijk;
            }
        }
    }

    /* There are d \kappa_{i,i,i} terms, 2 {d \choose 2} \kappa_{i,i,j} terms,
       and {d \choose 3} \kappa_{i,j,k} terms.
       gsl_sf_choose (unsigned int n, unsigned int m) */

    /* The following is based on Comon, P. (1994) Signal processing 36, 287–314.
       See eqn 3.4 (Theorem 14).
       The similar equations (7 & 8) in Amari, Cichocki, and Yang (1996) seem to be wrong.  */

    bias = (term1 + 3.0 * term2 + term3 / 6.0) / 12.0 + term4/48.0 + 7.0*term5/48.0 - term6/8.0;

    printf("\nEdgeworth term1: %g", term1/ 12.0);
    printf("\nEdgeworth term2: %g", 3.0*term2/ 12.0);
    printf("\nEdgeworth term3: %g", term3/(6.0*12.0));
    printf("\nEdgeworth term4: %g", +term4/48.0);
    printf("\nEdgeworth term5: %g", +7.0*term5/48.0);
    printf("\nEdgeworth term6: %g\n", - term6/8.0);

    printf("\n%-25s% 16.4f", "ln(det):", lndet);

    entropy = 0.5 * dim * log(2.0 * M_PI * M_E) + 0.5 * lndet;

    printf("\n%-25s% 16.4f", "white entropy:", entropy);
    printf("\n%-25s% 16.4f", "bias:", bias);
    printf("\n%-25s% 16.4f", "ln(scale):", lnscale);
    printf("\n%-25s% 16.4f", "Naive N-entropy:", entropy + lnscale);

    //entropy = entropy - bias + lnscale;

    printf("\n%-25s% 16.4f", "Edgeworth entropy (3O):", entropy - term1/12.0 + lnscale);
    printf("\n%-25s% 16.4f", "Edgeworth entropy (4O):", entropy - bias + lnscale);
    printf("\n\n");

    entropy = entropy - bias + lnscale;

//     /* From eqns (7 & 8) in Amari, Cichocki, and Yang (1996).
//        Seems to be wrong.  */
//     term1 = 0.0;
//     term4 = 0.0;
//     term5 = 0.0;
//     term6 = 0.0;
//     for (i = 0; i < dim; ++i)
//     {
//         kappa_iii = 0.0;
//         kappa_iiii = 0.0;
//         for (j = 0; j < len; ++j)
//         {
//             t3 = dif[i][j] * dif[i][j] * dif[i][j];
//             kappa_iii += t3; /* skewness */
//             kappa_iiii += t3 * dif[i][j]; /* kurtosis */
//         }
//
//         kappa_iii *= invlen;
//         kappa_iiii *= invlen;
//         kappa_iiii -= 3.0;
//
//         t3 = kappa_iii * kappa_iii;
//         t4 = kappa_iiii * kappa_iiii;
//         term1 += t3;
//         term4 += t4;
//         term5 += t4 * kappa_iiii; // k_4^3;
//         term6 += t3 * kappa_iiii; // k_3^2 k_4
//     }
//
//     bias = (term1 + 3.0 * term2 + term3 / 6.0) / 12.0 + term4/48.0 - term5/16.0 - 5.0*term6/8.0;
//
//     printf("\nEdgeworth term1: %g", term1/ 12.0);
//     printf("\nEdgeworth term2: %g", 3.0*term2/ 12.0);
//     printf("\nEdgeworth term3: %g", term3/(6.0*12.0));
//     printf("\nEdgeworth term4: %g", +term4/48.0);
//     printf("\nEdgeworth term5: %g", -term5/16.0);
//     printf("\nEdgeworth term6: %g\n", - 5.0*term6/8.0);
//
//     printf("\nln(det):           %14.3f", lndet);
//
//     entropy = 0.5 * dim * log(2.0 * M_PI * M_E) + 0.5 * lndet;
//
//     printf("\nwhite entropy:     %14.3f", entropy);
//     printf("\nbias:              %14.3f", bias);
//     printf("\nln(scale):         %14.3f", lnscale);
//
//     printf("\nNaive N-entropy:   %14.3f", entropy + lnscale);
//
//     //entropy = entropy - bias + lnscale;
//
//     printf("\nEdgeworth entropy: %14.3f", entropy - term1/12.0 + lnscale);
//     printf("\nEdgeworth entropy: %14.3f", entropy - bias + lnscale);
//     printf("\n\n");

    MatDestroy(&dif);
    MatDestroy(&cor);
    MatDestroy(&cov);
    free(eval);
    free(std);
    free(ave);

    return(entropy);
}


void
CalcPAve(double *pave, const double **p, const int dim, const int iters)
{
    int            i, j;

    for (i = 0; i < dim; ++i)
        pave[i] = 0.0;

    for (i = 0; i < iters; ++i)
        for (j = 0; j < dim; ++j)
            pave[j] += p[j][i];

    for (i = 0; i < dim; ++i)
        pave[i] /= iters;
}


void
CalcPCov(double **cov, const double *pave, const double **p, const int dim, const int iters)
{
    int            i, j, k;
    double         tmpi, tmpj, sum;
    double         inviters = 1.0/(iters-1.0);

    //printf("\n%4d %g", iters, inviters);

    for (i = 0; i < dim; ++i)
    {
        for (j = 0; j <= i; ++j)
        {
            sum = 0.0;
            for (k = 0; k < iters; ++k)
            {
                tmpi = p[i][k] - pave[i];
                tmpj = p[j][k] - pave[j];
                sum += tmpi * tmpj;
                //printf("\n%4d %4d %4d: %16.8f %16.8f", i, j, k, tmpi, tmpj);
            }

            cov[i][j] = cov[j][i] = sum * inviters;
            //printf("\n%4d %4d: %16.8f %16.8f %16.8f", i, j, cov[i][j], sum*inviters, sum);
        }
    }
}


/*
The Laplace-Metropolis estimator for calculating the marginal likelihood
from metropolis samples from the posterior distribution.

Steven M. Lewis, Adrian E. Raftery (1997)
"Estimating Bayes Factors via Posterior Stimulation with the Laplace-Metropolis Estimator."
Journal of the American Statistical Association, 92(438):648-655

Using equation 4, esp. see pp 649-650, first method to estimate \theta*.

IME, this is extremely accurate (using Gold Standard as a reference).
*/
double
CalcLaplaceMet(MonteCarloDat *mcdat)
{
    int            d, i, j;
    int            maxind;
    double         lndet, lapmet, lnh, lnfish;
    double         maxpost, maxprior, maxlike;
    double        *eval = NULL, **cov = NULL, *pave = NULL;

    printf("Calculating Laplace approximation ...\n");
    fflush(NULL);

    cov = mcdat->cov;
    pave = mcdat->pave;
    d = mcdat->dim;
    iters = mcdat->iters;

    CalcPAve(pave, (const double **) x, d, iters);
    CalcPCov(cov, pave, (const double **) x, d, iters);

    for (i = 0; i < d; ++i)
        printf("\nave p[%3d]:% 16.4f (+/- %16.4f)", i, pave[i], sqrt(cov[i][i]));

    printf ("\n\nParameter covariance matrix (estimate of minus inverse Hessian):");
    MatPrint(cov, d);

    ////////////////////////////////////////////////////////////////////////////////////////////////
    eval = calloc(d, sizeof(double));

    EigenvalsGSL(cov, d, eval);

    lndet = lnfish = 0.0;
    for (i = 0; i < d; i++)
    {
        if (isgreater(eval[i], DBL_EPSILON))
        {
            lndet += log(eval[i]);
            lnfish -= log(ndata * eval[i]);
        }
        else
        {
            printf("\n WARNING: excluding eigenvalue %d from determinant calculation", i);
            printf("\n WARNING: eigenvalue[%d] = %g < %g", i, eval[i], FLT_EPSILON);
        }
    }

    printf("\nln(FI):   %14.3f", lnfish);
    printf("\nln(det):  %14.3f", lndet);
    printf("\n-d ln(n): %14.3f", -d * log(ndata));
    printf("\ndet:      %g\n", exp(lndet));
    for (i = 0; i < d; i++)
        printf ("\neigenvalue[%d] = %g", i, eval[i]);
    printf ("\n");
    fflush(NULL);

    free(eval);

    ////////////////////////////////////////////////////////////////////////////////////////////////
    for (i = 0; i < d; ++i)
    {
        for (j = 0; j <= i; ++j)
        {
            if (cov[i][j] == 0.0)
                cov[i][j] = cov[j][i] = 0.0;
            else
                cov[i][j] = cov[j][i] = cov[i][j] / sqrt(cov[i][i] * cov[j][j]);
        }
    }

//     for (i = 0; i < d; ++i)
//         cov[i][i] = 1.0;

    printf ("\nParameter correlation matrix:");
    MatPrintLowerDiag(cov, d);

    ////////////////////////////////////////////////////////////////////////////////////////////////
    /* Find the parameters with the maximum posterior prob */
    maxpost = -DBL_MAX;
    maxind = 0;
    for (i = 0; i < iters; ++i)
    {
        //printf("\nlnpost[%6d]: %g %g", i, lnpost[i], maxpost);
        lnh = lnprior[i] + lnlike[i];
        if (maxpost < lnh)
        {
            maxpost = lnh;
            maxind = i;
        }
    }

    maxprior = lnprior[maxind];
    maxlike = lnlike[maxind];

//     for (i = 0; i < d; ++i)
//         printf("\nmax logPost p[%d]:   % 16.4f", i, x[i][maxind]);
//
//     printf("\n%.4f", x[0][maxind]);
//     for (i = 1; i < d; ++i)
//         printf(":%.4f", x[i][maxind]);
//     printf("\n");

    lapmet = maxpost + 0.5 * lndet + 0.5 * d * log(2.0 * M_PI);

    mcdat->maxpost = maxpost;
    mcdat->maxprior = maxprior;
    mcdat->maxlike = maxlike;
    mcdat->lndet = lndet;
    mcdat->lapmet = lapmet;
    mcdat->lnfish = lnfish;

    printf("Laplace approximation done ...\n");
    fflush(NULL);

    return(lapmet);
}


double
CalcLaplaceMetHier(MonteCarloDat *mcdat)
{
    int            d, i, j, k;
    int            maxind, hlen;
    double         lndet, lapmet, lnh, lnfish, sum, tmpi, tmpj;
    double         maxpost, maxprior, maxlike;
    double        *eval = NULL, **cov = NULL, *pave = NULL;

    printf("Calculating Laplace approximation ...\n");
    fflush(NULL);

    hdim = 1;

    d = mcdat->dim;
    iters = mcdat->iters;

    double inviters = 1.0/iters;

    hlen = d + hdim;

    cov = MatAlloc(hlen, hlen);
    pave = calloc(hlen,  sizeof(double));

    CalcPAve(pave, (const double **) x, d, iters);
    CalcPCov(cov, pave, (const double **) x, d, iters);

    pave[d] = 0.0;
    for (i = 0; i < iters; ++i)
    {
        pave[d] += h[i];
    }
    pave[d] *= inviters;

    VecPrint(pave,  hlen);

    for (j = 0; j < d; ++j)
    {
        sum = 0.0;
        for (k = 0; k < iters; ++k)
        {
            tmpi = h[k] - pave[d];
            tmpj = x[j][k] - pave[j];
            sum += tmpi * tmpj;
            //printf("\n%4d %4d %4d: %16.8f %16.8f", i, j, k, tmpi, tmpj);
        }

        cov[d][j] = cov[j][d] = sum * inviters;
        //printf("\n%4d %4d: %16.8f %16.8f %16.8f", i, j, cov[i][j], sum*inviters, sum);
    }

    sum = 0.0;
    for (k = 0; k < iters; ++k)
    {
         tmpi = h[k] - pave[d];
         sum += tmpi*tmpi;
    }

    cov[d][d] = sum * inviters;

    for (i = 0; i < hlen; ++i)
        printf("\nave p[%3d]:% 16.4f (+/- %16.4f)", i, pave[i], sqrt(cov[i][i]));

    printf ("\n\nParameter covariance matrix (estimate of minus inverse Hessian):");
    MatPrint(cov, hlen);

    ////////////////////////////////////////////////////////////////////////////////////////////////
    eval = calloc(hlen, sizeof(double));

    EigenvalsGSL(cov, hlen, eval);

    lndet = lnfish = 0.0;
    for (i = 0; i < hlen; i++)
    {
        if (isgreater(eval[i], DBL_EPSILON))
        {
            lndet += log(eval[i]);
            lnfish -= log(ndata * eval[i]);
        }
        else
        {
            printf("\n WARNING: excluding eigenvalue %d from determinant calculation", i);
            printf("\n WARNING: eigenvalue[%d] = %g < %g", i, eval[i], FLT_EPSILON);
        }
    }

    printf("\nln(FI):   %14.3f", lnfish);
    printf("\nln(det):  %14.3f", lndet);
    printf("\n-d ln(n): %14.3f", -d * log(ndata) - hdim * log(d));
    printf("\ndet:      %g\n", exp(lndet));
    for (i = 0; i < hlen; i++)
        printf ("\neigenvalue[%d] = %g", i, eval[i]);
    printf ("\n");
    fflush(NULL);

    free(eval);

    ////////////////////////////////////////////////////////////////////////////////////////////////
    for (i = 0; i < hlen; ++i)
    {
        for (j = 0; j <= i; ++j)
        {
            if (cov[i][j] == 0.0)
                cov[i][j] = cov[j][i] = 0.0;
            else
                cov[i][j] = cov[j][i] = cov[i][j] / sqrt(cov[i][i] * cov[j][j]);
        }
    }

//     for (i = 0; i < hdim; ++i)
//         cov[i][i] = 1.0;

    printf ("\nParameter correlation matrix:");
    MatPrintLowerDiag(cov, hlen);

    ////////////////////////////////////////////////////////////////////////////////////////////////
    /* Find the parameters with the maximum posterior prob */
    maxpost = -DBL_MAX;
    maxind = 0;
    for (i = 0; i < iters; ++i)
    {
        //printf("\nlnpost[%6d]: %g %g", i, lnpost[i], maxpost);
        lnh = lnprior[i] + lnlike[i];
        if (maxpost < lnh)
        {
            maxpost = lnh;
            maxind = i;
        }
    }

    maxprior = lnprior[maxind];
    maxlike = lnlike[maxind];

//     for (i = 0; i < d; ++i)
//         printf("\nmax logPost p[%d]:   % 16.4f", i, x[i][maxind]);
//
//     printf("\n%.4f", x[0][maxind]);
//     for (i = 1; i < d; ++i)
//         printf(":%.4f", x[i][maxind]);
//     printf("\n");

    lapmet = maxpost + 0.5 * lndet + 0.5 * hlen * log(2.0 * M_PI);

    mcdat->maxpost = maxpost;
    mcdat->maxprior = maxprior;
    mcdat->maxlike = maxlike;
    mcdat->lndet = lndet;
    mcdat->lapmet = lapmet;
    mcdat->lnfish = lnfish;

    MatDestroy(&cov);
    free(pave);

    printf("Laplace approximation done ...\n");
    fflush(NULL);

    return(lapmet);
}


double
CalcLaplaceMetUni(MonteCarloDat *mcdat)
{
    int            i;
    int            maxind;
    double         lndet, lapmet, lnh, lnfish;
    double         maxpost, maxprior, maxlike;
    double         ave, var;

    //d = mcdat->dim;
    iters = mcdat->iters;

    printf("Calculating Laplace approximation ...\n");
    fflush(NULL);

    ave = average(x[0], iters);
    var = variance(x[0], iters, ave);

    lndet = log(var);
    lnfish = -log(nd * var);

    ////////////////////////////////////////////////////////////////////////////////////////////////
    printf("\nln(FI):   %14.3f", lnfish);
    printf("\nln(det):  %14.3f", lndet);
    printf("\n-d ln(n): %14.3f", -log(nd));
    printf("\ndet:      %g\n", exp(lndet));
    fflush(NULL);

    ////////////////////////////////////////////////////////////////////////////////////////////////
    /* Find the parameters with the maximum posterior prob */
    maxpost = -DBL_MAX;
    maxind = 0;
    for (i = 0; i < iters; ++i)
    {
        //printf("\nlnpost[%6d]: %g %g", i, lnpost[i], maxpost);
        lnh = lnprior[i] + lnlike[i];
        if (maxpost < lnh)
        {
            maxpost = lnh;
            maxind = i;
        }
    }

    maxprior = lnprior[maxind];
    maxlike = lnlike[maxind];

    lapmet = maxpost + 0.5 * lndet + 0.5 * log(2.0 * M_PI);

    mcdat->maxpost = maxpost;
    mcdat->maxprior = maxprior;
    mcdat->maxlike = maxlike;
    mcdat->lndet = lndet;
    mcdat->lapmet = lapmet;
    mcdat->lnfish = lnfish;

    printf("Laplace approximation done ...\n");
    fflush(NULL);

    return(lapmet);
}


/*
The simple Gaussian model described on page 203 of

Lartillot N, Philippe H. (2006)
"Computing Bayes factors using thermodynamic integration."
Syst Biol. 55(2):195-207.

The real data is all zeros (a "data-free" likelihood).

It appears that they have forgotten the normalization constants.
*/
void
SimGaussLP(const double nu, const gsl_rng *r2)
{
    int             i, j;
    double          sigma = sqrt(nu / (1.0 + nu));
    double          sqrtnu = sqrt(nu), xij;
    double          tmp;
    double         avelnlike = 0.0, avelnprior = 0.0, varlnpost = 0.0;
    double         avelnprlk2 = 0.0, avelnpost = 0.0;

    avelnprior = avelnlike = avelnprlk2 = 0.0;

    for (i = 0; i < iters; ++i)
    {
        lnprior[i] = lnpost[i] = lnlike[i] = 0.0;
        for (j = 0; j < dim; ++j)
        {
            xij = gsl_ran_gaussian(r2, sigma);
            lnprior[i] += log(gsl_ran_gaussian_pdf(xij, sqrtnu));
            lnlike[i]  += log(gsl_ran_gaussian_pdf(xij, 1.0));
            lnpost[i]  += log(gsl_ran_gaussian_pdf(xij, sigma));
            x[j][i] = xij;
        }
        //printf("\nlnlike[%6d]: %g", i, lnlike[i]);
        avelnprior += lnprior[i];
        avelnlike  += lnlike[i];
        avelnprlk2 += (lnprior[i] + lnlike[i]) * (lnprior[i] + lnlike[i]);
    }

    avelnprior /= iters;
    avelnlike  /= iters;
    avelnpost = avelnprior + avelnlike;
    avelnprlk2 /= iters;

    varlnpost = 0.0;
    for (i = 0; i < iters; ++i)
    {
        tmp = lnprior[i] + lnlike[i] - avelnpost;
        varlnpost += tmp * tmp;
    }

    varlnpost /= iters;

    printf("\n%-25s% 16.4f", "varlnpost:", varlnpost);
    printf("\n%-25s% 16.4f", "avelnlike:", avelnlike);
    printf("\n%-25s% 16.4f", "avelnlike/n:", avelnlike/ndata);
    printf("\n%-25s% 16.4f", "avelnprior:", avelnprior);
    printf("\n%-25s% 16.4f", "avelnpost:", avelnpost);
    printf("\n%-25s% 16.4f", "avelnprlk2:", avelnprlk2);
    printf("\n%-25s% 16.4f", "entropy ln post:", 0.5 * log(2.0 * M_PI * varlnpost * M_E));
    printf("\n\n");
}


double
normal_lnpdf(const double x, const double mean, const double var)
{
    double          p;

    p = (-0.5 * log(2.0 * M_PI * var)) - ((x - mean)*(x - mean) / (2.0 * var));

    return (p);
}


typedef struct
{
    double        **x;
    int             idim, len, start, end;
} SimData;


static void
*sim_gauss_pth(void *simdata_ptr)
{
    SimData       *simdata = (SimData *) simdata_ptr;
    int            i;
    const int      idim = (const int) simdata->idim;
    double       **x = simdata->x;
    double         tmpmu;

    const gsl_rng_type    *T = NULL;
    gsl_rng               *r2 = NULL;
    unsigned long int      seed;

    /* Every thread gets its own rng generator -- otherwise, we get data race junk in valgrind */
    T = gsl_rng_ranlxs2;
    r2 = gsl_rng_alloc(T);
    seed = time(NULL) + (unsigned long int) pthread_self() + getpid() + clock();
    //printf("\nseed[%d]:%ld %ld\n", pthread_self(), seed, time(NULL));
    gsl_rng_set(r2, seed);

    //tmpmu = gsl_ran_gaussian(r2, 10.0);
    //printf("\nmu[%d]: %g", i, tmpmu);
    tmpmu = 1.0;
    for (i = 0; i < simdata->len; ++i)
    {
        x[idim][i] = gsl_ran_gaussian_ziggurat(r2, tmpmu);
        //x[idim][i] = gsl_ran_exponential(r2, tmpmu);
        //printf("\n%5d %5d % 16.6f", idim, i, x[idim][i]);
        //data[i][j] = 0.0;
    }

    printf("SimGauss thread %3d DONE\n", idim);
    fflush(NULL);

    gsl_rng_free(r2);
    r2 = NULL;

    pthread_exit((void *) 0);
}


void
SimGaussPth(double **data, SimData **simdata, pthread_t *callThd,
            pthread_attr_t *attr, const int thrdnum)
{
    const int      len = ndata;
    int            i, rc = 0;

    for (i = 0; i < thrdnum ; ++i)
    {
        simdata[i]->x = data;
        simdata[i]->idim = i;
        simdata[i]->len = len;

        rc = pthread_create(&callThd[i], attr, sim_gauss_pth, (void *) simdata[i]);

        if (rc)
        {
            printf("ERROR811: return code from pthread_create() %d is %d\n", i, rc);
            exit(EXIT_FAILURE);
        }
    }

    for (i = 0; i < thrdnum; ++i)
    {
        rc = pthread_join(callThd[i], (void **) NULL);

        if (rc)
        {
            printf("ERROR812: return code from pthread_join() %d is %d\n", i, rc);
            exit(EXIT_FAILURE);
        }
    }

    return;
}


static void
*sim_expo_pth(void *simdata_ptr)
{
    SimData       *simdata = (SimData *) simdata_ptr;
    int            i;
    const int      idim = (const int) simdata->idim;
    double       **x = simdata->x;
    double         tmpmu;

    const gsl_rng_type    *T = NULL;
    gsl_rng               *r2 = NULL;
    unsigned long int      seed;

    /* Every thread gets its own rng generator -- otherwise, we get data race junk in valgrind */
    T = gsl_rng_ranlxs2;
    r2 = gsl_rng_alloc(T);;
    seed = time(NULL) + (unsigned long int) pthread_self() + getpid() + clock();
    //printf("\nseed[%d]:%ld %ld\n", pthread_self(), seed, time(NULL));
    gsl_rng_set(r2, seed);

    //tmpmu = gsl_ran_gaussian(r2, 10.0);
    //printf("\nmu[%d]: %g", i, tmpmu);
    tmpmu = 1.0;
    for (i = 0; i < simdata->len; ++i)
    {
        //x[idim][i] = gsl_ran_gaussian_ziggurat(r2, tmpmu);
        x[idim][i] = gsl_ran_exponential(r2, tmpmu);
        //printf("\n%5d %5d % 16.6f", idim, i, x[idim][i]);
        //data[i][j] = 0.0;
    }

    printf("SimGauss thread %3d DONE\n", idim);
    fflush(NULL);

    gsl_rng_free(r2);
    r2 = NULL;

    pthread_exit((void *) 0);
}


void
SimExpoPth(double **data, SimData **simdata, pthread_t *callThd,
           pthread_attr_t *attr, const int thrdnum)
{
    const int      len = ndata;
    int            i, rc = 0;

    for (i = 0; i < thrdnum ; ++i)
    {
        simdata[i]->x = data;
        simdata[i]->idim = i;
        simdata[i]->len = len;

        rc = pthread_create(&callThd[i], attr, sim_expo_pth, (void *) simdata[i]);

        if (rc)
        {
            printf("ERROR811: return code from pthread_create() %d is %d\n", i, rc);
            exit(EXIT_FAILURE);
        }
    }

    for (i = 0; i < thrdnum; ++i)
    {
        rc = pthread_join(callThd[i], (void **) NULL);

        if (rc)
        {
            printf("ERROR812: return code from pthread_join() %d is %d\n", i, rc);
            exit(EXIT_FAILURE);
        }
    }

    return;
}


void
SimGauss(const gsl_rng *r2)
{
    int             i, j;
    double          tmpmu, tmpsigma;

    /* First, generate artificial data */
    /* precision/sigma/variance of likelihood is 1.0 */
    /* real mu = 0 */
    printf("Simulating Gaussian data ...\n");
    fflush(NULL);
    for (i = 0; i < dim; ++i)
    {
        tmpmu = gsl_ran_gaussian(r2, 0.5);
        //printf("\nmu[%d]: %g", i, tmpmu);
        //tmpmu = 1.0;
        tmpsigma = 1.0;
        for (j = 0; j < ndata; ++j)
        {
            data[i][j] = gsl_ran_gaussian_ziggurat(r2, tmpsigma) + tmpmu;
            //data[i][j] = 0.0;
        }
    }

    printf("\n\n");
}


void
SimExpo(const gsl_rng *r2)
{
    int             i, j;
    double          tmpmu;

    /* First, generate artificial data */
    /* scale of likelihood is 1.0 */
    printf("Simulating exponential data ...\n");
    fflush(NULL);
    for (i = 0; i < dim; ++i)
    {
        tmpmu = 1.0;
        //tmpmu = (1.0 / gsl_ran_exponential(r2, 10000));
        printf("\nmu[%d]: %g", i, tmpmu);
        for (j = 0; j < ndata; ++j)
        {
            data[i][j] = gsl_ran_exponential(r2, tmpmu);
            //printf("\n%g", data[i][j]);
            //data[i][j] = 0.0;
        }
    }

    printf("\n\n");
}


void
CalcCumulants(void)
{
    int             i, j;

    printf("Calculate data cumulants ...\n");
    fflush(NULL);

    for (j = 0; j < dim; ++j)
    {
        y[j] = 0.0;
        for (i = 0; i < ndata; ++i)
            y[j] += data[j][i];
    }

    for (j = 0; j < dim; ++j)
    {
        x2[j] = 0.0;
        for (i = 0; i < ndata; ++i)
            x2[j] += data[j][i]*data[j][i];
    }

    yt = 0.0;
    for (j = 0; j < dim; ++j)
        yt += y[j];

    yt2 = 0.0;
    for (j = 0; j < dim; ++j)
        yt2 += y[j]*y[j];

    x2t = 0.0;
    for (j = 0; j < dim; ++j)
        x2t += x2[j];
}


void
WriteChain(char *fname, double **chain, const int n, const int d)
{
    FILE          *fp = fopen(fname ,"w");
    int            i, j;

    for (i = 0; i < n; ++i)
    {
        for (j = 0; j < d; ++j)
            fprintf(fp, "%-18.8f ", chain[j][i]);

        fprintf(fp, "\n");
    }

    fprintf(fp, "\n\n");
    fclose(fp);
    fflush(NULL);
}


void
PrintRefMargEstimates(MonteCarloDat *mcdat)
{
    double pi_explnlike = mcdat->pi_explnlike;
    double pi_explnpost = mcdat->pi_explnpost;
    double entlnpost    = mcdat->pi_entlnpost;
    double varlnpost    = mcdat->pi_varlnpost;

    printf("\n%-25s% 16d", "dim:", mcdat->dim);
    printf("\n%-25s% 16d", "ndata:", mcdat->ndata);
    printf("\n%-25s% 16d", "samples:", iters);
    printf("\n");

    printf("\n%-25s% 16.4f", "pi_avelnlike:", mcdat->pi_avelnlike);
    if (isfinite(pi_explnlike))
        printf("\n%-25s% 16.4f", "pi_explnlike:", pi_explnlike); // reference prior
    if (isfinite(mcdat->pi_expexplnlike))
        printf("\n%-25s% 16.4f", "exppi_explnlike:", mcdat->pi_expexplnlike); // reference prior
    printf("\n%-25s% 16.4f", "pi_avelnpost:", mcdat->pi_avelnpost);
    if (isfinite(pi_explnpost))
        printf("\n%-25s% 16.4f", "pi_explnpost:", pi_explnpost); // reference prior
    printf("\n%-25s% 16.4f", "pi_avelnlike/n:", mcdat->pi_avelnlike/mcdat->ndata);
    printf("\n");
    if (isfinite(entlnpost))
        printf("\n%-25s% 16.4f", "N-pi_entlnpost:", entlnpost); // reference prior
    printf("\n%-25s% 16.4f", "posterior pi-entropy:", -mcdat->pi_avelnpost);
    printf("\n%-25s% 16.4f", "pi lnlik + entropy:", mcdat->pi_avenoprior);
    printf("\n%-25s% 16.4f", "pi_varlnpost:", varlnpost);
    printf("\n%-25s% 16.4f", "pi_DIC:", mcdat->pi_dic);
    printf("\n\n");
    fflush(NULL);
}


void
PrintMargEstimates(MonteCarloDat *mcdat)
{
    double lapmet        = mcdat->lapmet;
    //double edge_entropy  = mcdat->edge_entropy;
    double expmet        = mcdat->expmet;
    double avelnprior    = mcdat->avelnprior;
    double avelnlike     = mcdat->avelnlike;
    double explnlike     = mcdat->explnlike;
    double varlnpost     = mcdat->varlnpost;
    double explnprior    = mcdat->explnprior;
    double avelnprlk     = mcdat->avelnprlk;
    double avelnprlk2    = mcdat->avelnprlk2;
    double avelnpost     = mcdat->avelnpost;
    double avelnscore    = mcdat->avelnscore;
    double explnpost     = mcdat->explnpost;
    //double entlnpost     = mcdat->entlnpost;
    //double Nentropy      = mcdat->Nentropy;
    double hme           = mcdat->hme;
    double marglik       = mcdat->marglik;
    double maxpost       = mcdat->maxpost;
    double maxprior      = mcdat->maxprior;
    double maxlike       = mcdat->maxlike;
    double varlnlike     = mcdat->varlnlike;
    double amlik         = mcdat->amlik;
    double avelike       = mcdat->avelike;
    double marg_lnnorm   = mcdat->marg_lnnorm;
    //double dic           = mcdat->dic;
    double bicm_20       = mcdat->bicm_20;
    double bicm_dlt      = mcdat->bicm_dlt;


    printf("\n%-25s% 16d", "dim:", mcdat->dim);
    printf("\n%-25s% 16d", "ndata:", mcdat->ndata);
    printf("\n%-25s% 16d", "samples:", iters);
    printf("\n");

    //printf("\n%-25s% 16.4f", "ln arithmetic mean:", avelike);
    printf("\n%-25s% 16.4f", "ln geometric mean:", amlik);
    printf("\n%-25s% 16.4f", "Max ln post - ln p(D):", maxpost);
    printf("\n%-25s% 16.4f", "Max ln prior:", maxprior);
    printf("\n%-25s% 16.4f", "Max ln likelihood:", maxlike);
    printf("\n");
    //printf("\n%-24s% 16.4f", "log c:", -0.5 * log(2.0 * M_PI * varlnpost * M_E) + avelnlike + avelnprior); // this one is probably meaningless
    printf("\n%-25s% 16.4f", "avelnprlk:", avelnprlk);
    printf("\n%-25s% 16.4f", "sqrt avelnprlk2:", sqrt(avelnprlk2));
    //printf("\n%-25s% 16.4f", "entropy ln post:", entlnpost);
    printf("\n%-25s% 16.4f", "posterior entropy:", - avelnpost);
    printf("\n%-25s% 16.4f", "exp lnlik + entropy:", avelnlike - avelnpost);
    //printf("\n%-25s% 16.4f", "DIC:", dic);
    printf("\n%-25s% 16.4f", "exact exp ml:", avelnlike + avelnprior - avelnpost);
    printf("\n");
    printf("\n%-25s% 16.4f", "varlnlike:", varlnlike);
    printf("\n%-25s% 16.4f", "varlnpost:", varlnpost);
    printf("\n%-25s% 16.4f", "avelnlike:", avelnlike);
    if (isfinite(explnlike))
        printf("\n%-25s% 16.4f", "explnlike:", explnlike);
    printf("\n%-25s% 16.4f", "avelnprior:", avelnprior);
    if (isfinite(explnprior))
        printf("\n%-25s% 16.4f", "explnprior:", explnprior);
    printf("\n%-25s% 16.4f", "avelnpost:", avelnpost);
    if (isfinite(explnpost))
        printf("\n%-25s% 16.4f", "explnpost:", explnpost);

    printf("\n%-25s% 16.4f", "avelnlike/n:", avelnlike/mcdat->ndata);
    printf("\n%-25s% 16.4f", "ave Kulback-Leibler:", avelnpost - avelnprior);
    if (isfinite(explnprior) && isfinite(explnpost))
        printf("\n%-25s% 16.4f", "exp Kulback-Leibler:", explnpost - explnprior);

    if (isfinite(avelnscore))
    {
        printf("\n%-25s% 16.4f", "avelnscore:", avelnscore);
        double gammentropy = varlnpost + lgamma(varlnpost) + (1.0-varlnpost)*gsl_sf_psi(varlnpost);
        printf("\n%-25s% 16.4f", "entropy ln post:", gammentropy);
        printf("\n%-25s% 16.4f", "Score entropy:", gammentropy - avelnscore);
    }
    printf("\n");
    //printf("\n%-25s% 16.4f", "Posterior N-entropy:", Nentropy);
    //printf("\n%-25s% 16.4f", "Edgeworth entropy:", edge_entropy);
    printf("\n%-25s% 16.4f", "Post Expected Deviance:", avelnlike);
    printf("\n%-25s% 16.4f", "Harmonic Mean estimate:", hme);
    printf("\n%-25s% 16.4f", "Posterior Bayes marginal:", avelike);
    printf("\n%-25s% 16.4f", "ln normal estimate:", marg_lnnorm);
    printf("\n%-25s% 16.4f", "DIC:", avelnlike - varlnpost);
    printf("\n%-25s% 16.4f", "BPIC:", avelnlike - 2.0*varlnpost);
    printf("\n%-25s% 16.4f", "BICM_20 estimate:", bicm_20);
    printf("\n%-25s% 16.4f", "BICM_DLT estimate:", bicm_dlt);
    printf("\n%-25s% 16.4f", "DLT-Metropolis:", avelnprior + avelnlike - varlnpost * (log(ndata) - log(2.0 * M_PI * M_E)));
    printf("\n%-25s% 16.4f", "DLT-Metropolis no prior:",  avelnlike - varlnpost * (log(ndata) - log(2.0 * M_PI * M_E)));
    printf("\n%-25s% 16.4f", "Laplace-Metropolis:", lapmet);
    printf("\n%-25s% 16.4f", "Expected-Metropolis:", expmet);
    printf("\n%-25s% 16.4f", "Analytical marginal lik:", marglik);
    printf("\n\n");
    fflush(NULL);
}


/* mu = 0  */
/* lambda = precision of prior mu */
void
MCGauss(MonteCarloDat *mcdat)
{
    const double    lambda_0 = mcdat->lambda_0;
    const gsl_rng  *r2 = mcdat->r2;
    int             i, j;
    double          postvar = 1.0 / (ndata + lambda_0);
    double          postsigma = sqrt(postvar);
    double          tmp, factor, musimj;
    const double    ln2pi2 = 0.5*log(2.0*M_PI);
    const double    ln2pi = log(2.0*M_PI);
    double          musim2, diffsum;
    double          avelnlike = 0.0, avelnprior = 0.0, varlnpost = 0.0;
    double          avelnprlk2 = 0.0, avelnpost = 0.0, avelnprlk = 0.0;

    /* Now sample posterior of mu with MC */
    printf("Monte Carlo sampling ...\n");
    fflush(NULL);
    avelnprior = avelnlike = avelnpost = avelnprlk2 = 0.0;
    factor = 1.0 / (lambda_0 + ndata);

    memset(lnprior, 0, iters * sizeof(double));
    memset(lnpost,  0, iters * sizeof(double));
    memset(lnlike,  0, iters * sizeof(double));

    for (i = 0; i < iters; ++i)
    {
        for (j = 0; j < dim; ++j)
        {
            x[j][i] = musimj = gsl_ran_gaussian_ziggurat(r2, postsigma) + factor * y[j];
            //x[j][i] = gsl_ran_exponential(r2, 1.0);
            //x[j][i] = gsl_ran_gamma(r2, 5.0, 1.0);
            //x[j][i] = gsl_ran_weibull(r2, 1.0, 2.0);
            lnlike[i] += -ndata * ln2pi2
                         -0.5 *(x2[j] - 2.0*musimj*y[j] + ndata*musimj*musimj);
        }

        musim2 = 0.0;
        for (j = 0; j < dim; ++j)
            musim2 += x[j][i]*x[j][i];

        diffsum = 0.0;
        for (j = 0; j < dim; ++j)
        {
            tmp = y[j] * factor - x[j][i];
            diffsum += tmp*tmp;
        }

        lnprior[i] = 0.5*(-dim*ln2pi + dim*log(lambda_0) - lambda_0 * musim2);
        lnpost[i]  = 0.5*(-dim*ln2pi + dim*log(lambda_0 + ndata) - (lambda_0 + ndata)*diffsum);

        //printf("\n%-d% 16.4f % 16.4f % 16.4f", i, lnprior[i], lnlike[i], lnpost[i]);

        avelnprior += lnprior[i];
        avelnlike  += lnlike[i];
        avelnpost  += lnpost[i];
        avelnprlk2 += (lnprior[i] + lnlike[i]) * (lnprior[i] + lnlike[i]);
    }

    avelnprior /= iters;
    avelnlike  /= iters;
    avelnpost  /= iters;
    avelnprlk = avelnprior + avelnlike;
    avelnprlk2 /= iters;

    varlnpost = 0.0;
    for (i = 0; i < iters; ++i)
    {
        tmp = lnprior[i] + lnlike[i] - avelnprlk;
        varlnpost += tmp * tmp;
    }

    varlnpost /= iters;

    //double entlnpost = 0.5 * log(2.0 * M_PI * varlnpost * M_E);

    double explnprior = -0.5*(+ dim * log(2.0 * M_PI)
                              - dim* log(lambda_0)
                              + dim*lambda_0/(lambda_0+ndata)
                              + lambda_0*yt2/((lambda_0+ndata)*(lambda_0+ndata)));
    double explnlike = -0.5 * (+ dim*ndata * log(2.0 * M_PI)
                               + x2t
                               - (2.0*lambda_0+ndata)*yt2/((ndata+lambda_0)*(ndata+lambda_0))
                               + dim*ndata/(lambda_0 + ndata));
    double explnpost = -0.5 * dim * (log(2.0*M_PI) - log(ndata+lambda_0) + 1.0);
    //    marglik = - 0.5 * dim * log(2.0 * M_PI) - 0.5 * dim * log(1.0 + nu); // SimGaussLP
    double marglik = -0.5*(ndata*dim)* log(2.0*M_PI)
                     +0.5*dim*log(lambda_0/(lambda_0+ndata))
                     -0.5*x2t
                     +0.5*yt2/(lambda_0+ndata);


    mcdat->avelnprior = avelnprior;
    mcdat->explnprior = explnprior;
    mcdat->avelnlike  = avelnlike;
    mcdat->explnlike  = explnlike;
    mcdat->avelnpost  = avelnpost;
    mcdat->explnpost  = explnpost;
    mcdat->avelnprlk  = avelnprlk;
    mcdat->avelnprlk2 = avelnprlk2;
    mcdat->varlnpost  = varlnpost;
    mcdat->marglik    = marglik;

    mcdat->dim = dim;
    mcdat->ndata = ndata;

    printf("Monte Carlo done ...\n");
    fflush(NULL);

    if (write_files == 1)
        WriteChain("mc_gauss.txt", x, iters, dim);
}


void
MCGaussRef(MonteCarloDat *mcdat)
{
    const gsl_rng  *r2 = mcdat->r2;
    double          pi_avelnlike = 0.0;
    double          pi_avelnpost = 0.0;
    double          pi_varlnpost = 0.0;
    double          lnposti, lnlikei;
    double          inv_ndata = 1.0/ndata;
    double          pi_sigma = sqrt(inv_ndata);
    double          yj_ndata;
    double          delta;
    int             i, j;
    double          musimj;
    const double    ln2pi = log(2.0*M_PI);

    /* MC with reference priors, calculate E(lnpost) and E(lnlike) */
    printf("Reference posterior Monte Carlo sampling ...\n");
    fflush(NULL);

    for (i = 0; i < iters; ++i)
    {
        lnposti = lnlikei = 0.0;
        for (j = 0; j < dim; ++j)
        {
            yj_ndata = y[j]*inv_ndata;
            x[j][i] = musimj = gsl_ran_gaussian(r2, pi_sigma) + yj_ndata;
            lnlikei += -0.5 * ndata * ln2pi
                       -0.5 *(x2[j] - 2.0*musimj*y[j] + ndata*musimj*musimj);
            lnposti += normal_lnpdf(musimj, yj_ndata, inv_ndata);
        }

        /* running mean and variance */
        delta = lnposti - pi_avelnpost;
        pi_avelnpost += delta/(i+1);
        pi_varlnpost += delta*(lnposti - pi_avelnpost);

        pi_avelnlike += lnlikei;
    }

    double pi_explnpost;
    pi_explnpost = -0.5 * dim * log(2.0*M_PI*M_E/ndata);
    double pi_explnlike;
    pi_explnlike = -0.5 * (dim * ndata * log(2.0 * M_PI) + x2t - yt2/ndata + dim);

    mcdat->pi_explnlike  = pi_explnlike;
    mcdat->pi_explnpost  = pi_explnpost;

    mcdat->pi_avelnlike   = pi_avelnlike/iters;
    mcdat->pi_avelnpost   = pi_avelnpost;
    mcdat->pi_avenoprior  = pi_avelnlike/iters - pi_avelnpost;
    mcdat->pi_varlnpost   = pi_varlnpost/iters;
    mcdat->pi_dic         = pi_avelnlike/iters - pi_varlnpost/iters;

    mcdat->dim = dim;
    mcdat->ndata = ndata;

    printf("Monte Carlo done ...\n");
    fflush(NULL);

    if (write_files == 1)
        WriteChain("mc_gauss_ref.txt", x, iters, dim);
}


void
GibbsGaussHierarch(MonteCarloDat *mcdat)
{
    int             i, j;
    const double    phi_0 = mcdat->lambda_0;
    const gsl_rng  *r2 = mcdat->r2;
    double          postvar = 1.0 / (ndata + 1.0);
    double          postsigma = sqrt(postvar);
    double          tmp, musimj;
    const double    ln2pi2 = 0.5*log(2.0*M_PI);
    const double    ln2pi = log(2.0*M_PI);
    double          mu0sim, musum;
    double          postphi = phi_0/(dim*phi_0 + 1);
    double          phisigma = sqrt(postphi);
    double          avelnlike = 0.0, avelnprior = 0.0, varlnpost = 0.0;
    double          avelnprlk2 = 0.0, avelnpost = 0.0, avelnprlk = 0.0;

    /* Now sample posterior of mu  and mu_0 with Gibbs */
    printf("Gibbs sampling ...\n");
    fflush(NULL);

    avelnprior = avelnlike = avelnpost = avelnprlk2 = 0.0;

    memset(lnprior, 0, iters * sizeof(double));
    memset(lnpost,  0, iters * sizeof(double));
    memset(lnlike,  0, iters * sizeof(double));
    memset(h,       0, iters * sizeof(double));

    musum = 0.0;
    for (i = 0; i < iters; ++i)
    {
        h[i] = mu0sim = gsl_ran_gaussian_ziggurat(r2, phisigma) + postphi*musum;

        for (j = 0; j < dim; ++j)
        {
            x[j][i] = musimj = gsl_ran_gaussian_ziggurat(r2, postsigma) + postvar * (y[j] + mu0sim);

            lnlike[i]  += - ndata * ln2pi2
                          - 0.5 *(x2[j] - 2.0*musimj*y[j] + ndata*musimj*musimj);
            lnprior[i] += - 0.5*ln2pi - 0.5*(musimj - mu0sim)*(musimj - mu0sim);
            lnpost[i]  += - 0.5 * (ndata+1.0) * musimj * musimj
                          + mu0sim*musimj
                          + musimj*y[j]
                          - 0.5* y[j]*y[j]/(ndata+1.0);
        }

        lnprior[i] += - 0.5*ln2pi - 0.5*log(phi_0) - 0.5 * mu0sim * mu0sim / phi_0;
        lnpost[i]  += - 0.5*(dim+1)*ln2pi
                      - 0.5*log(phi_0)
                      + 0.5*log(ndata*dim*phi_0+ndata+1.0)
                      + 0.5*(dim-1.0)*log(ndata+1.0)
                      - 0.5*mu0sim*mu0sim*(dim*phi_0+1.0)/phi_0
                      - 0.5*phi_0 * yt * yt / ((ndata+1.0)*(ndata*dim*phi_0+ndata+1.0));

        //printf("\n%-d% 16.4f % 16.4f % 16.4f", i, lnprior[i], lnlike[i], lnpost[i]);

        avelnprior += lnprior[i];
        avelnlike  += lnlike[i];
        avelnpost  += lnpost[i];
        avelnprlk2 += (lnprior[i] + lnlike[i]) * (lnprior[i] + lnlike[i]);

        musum = 0.0;
        for (j = 0; j < dim; ++j)
            musum += x[j][i];
    }

    avelnprior /= iters;
    avelnlike  /= iters;
    avelnpost  /= iters;
    avelnprlk = avelnprior + avelnlike;
    avelnprlk2 /= iters;

    varlnpost = 0.0;
    for (i = 0; i < iters; ++i)
    {
        tmp = lnprior[i] + lnlike[i] - avelnprlk;
        varlnpost += tmp * tmp;
    }

    varlnpost /= iters;

    //double entlnpost = 0.5 * log(2.0 * M_PI * varlnpost * M_E);

    double marglik = -0.5*(ndata*dim)* log(2.0*M_PI)
                     -0.5*log(ndata*dim*phi_0 + ndata + 1.0)
                     -0.5*(dim-1.0)*log(ndata + 1.0)
                     -0.5*x2t
                     +0.5*yt2/(ndata+1.0)
                     +0.5*phi_0 * yt * yt / ((ndata+1.0)*(ndata*dim*phi_0+ndata+1.0));

    mcdat->avelnprior = avelnprior;
    //mcdat->explnprior = explnprior;
    mcdat->avelnlike  = avelnlike;
    //mcdat->explnlike  = explnlike;
    mcdat->avelnpost  = avelnpost;
    //mcdat->explnpost  = explnpost;
    mcdat->avelnprlk  = avelnprlk;
    mcdat->avelnprlk2 = avelnprlk2;
    mcdat->varlnpost  = varlnpost;
    mcdat->marglik    = marglik;
    //mcdat->entlnpost  = entlnpost;

    mcdat->dim = dim;
    mcdat->ndata = ndata;

    printf("Gibbs done ...\n");
    fflush(NULL);

    if (write_files == 1)
    {
        WriteChain("mc_gauss_hierarch.txt", x, iters, dim);
        WriteChain("mc_gauss_hierarch_mu0.txt", &h, iters, 1);
        fflush(NULL);
    }
}


void
MCGaussUni(MonteCarloDat *mcdat)
{
    const double    lambda_0 = mcdat->lambda_0;
    const gsl_rng  *r2 = mcdat->r2;
    int             i;
    double          postvar = 1.0 / (nd + lambda_0);
    double          postsigma = sqrt(postvar);
    double          tmpmu, tmp, musimj;
    double          priorvar = 1.0 / lambda_0;
    const double    ln2pi = log(2.0*M_PI);
    double          avelnlike = 0.0, avelnprior = 0.0, varlnpost = 0.0;
    double          avelnprlk2 = 0.0, avelnpost = 0.0, avelnprlk = 0.0;
    double          avelnscore = 0.0;

    /* Now simulate posterior of mu with Monte Carlo */
    printf("Monte Carlo sampling Gaussian uniparameter ...\n");
    fflush(NULL);
    avelnprior = avelnlike = avelnpost = avelnprlk2 = 0.0;
    tmpmu = yt/(lambda_0 + nd);

    memset(lnprior, 0, iters * sizeof(double));
    memset(lnpost,  0, iters * sizeof(double));
    memset(lnlike,  0, iters * sizeof(double));

    for (i = 0; i < iters; ++i)
    {
        musimj = gsl_ran_gaussian(r2, postsigma) + tmpmu;
        x[0][i] = musimj;
        lnprior[i] = normal_lnpdf(musimj, 0.0, priorvar);
        lnlike[i] = -0.5 * nd * ln2pi
                    -0.5 *(x2t - 2.0*musimj*yt + nd*musimj*musimj);
        lnpost[i] = normal_lnpdf(musimj, tmpmu, postvar);
        lnscore[i] = log(0.5*fabs(-(lambda_0 + nd)*musimj + yt));

        //printf("\npi ll lp %-d% 16.4f % 16.4f % 16.4f", i, lnprior[i], lnlike[i], lnpost[i]);

        avelnprior += lnprior[i];
        avelnlike  += lnlike[i];
        avelnpost  += lnpost[i];
        avelnprlk2 += (lnprior[i] + lnlike[i]) * (lnprior[i] + lnlike[i]);
        avelnscore += lnscore[i];
    }

    avelnprior /= iters;
    avelnlike  /= iters;
    avelnpost  /= iters;
    avelnprlk = avelnprior + avelnlike;
    avelnprlk2 /= iters;
    avelnscore /= iters;

    varlnpost = 0.0;
    for (i = 0; i < iters; ++i)
    {
        tmp = lnprior[i] + lnlike[i] - avelnprlk;
        varlnpost += tmp * tmp;
    }

    varlnpost /= iters;

    double explnprior = -0.5*(log(2.0 * M_PI) - log(lambda_0) + lambda_0 * (lambda_0 + nd + yt*yt)/((lambda_0+nd)*(lambda_0+nd)));
    double entlnpost = 0.5 * log(2.0 * M_PI * varlnpost * M_E);
    double explnlike = -0.5 * (nd * log(2.0 * M_PI) + x2t - (2.0*lambda_0+nd)*yt*yt/((nd+lambda_0)*(nd+lambda_0)) + nd/(lambda_0+nd));
    double explnpost = -0.5 * (log(2.0*M_PI) - log(nd+lambda_0) + 1.0);
    double marglik = + 0.5*(- x2t + yt*yt/(lambda_0+nd)
                     - nd*log(2.0*M_PI)
                     + log(lambda_0/(lambda_0 + nd)));

    mcdat->avelnprior = avelnprior;
    mcdat->explnprior = explnprior;
    mcdat->avelnlike  = avelnlike;
    mcdat->explnlike  = explnlike;
    mcdat->avelnpost  = avelnpost;
    mcdat->explnpost  = explnpost;
    mcdat->avelnprlk  = avelnprlk;
    mcdat->avelnprlk2 = avelnprlk2;
    mcdat->varlnpost  = varlnpost;
    mcdat->entlnpost  = entlnpost;
    mcdat->avelnscore = avelnscore;
    mcdat->marglik    = marglik;

    mcdat->dim = 1;
    mcdat->ndata = ndata*dim;

    printf("Monte Carlo done ...\n");
    fflush(NULL);

    if (write_files == 1)
        WriteChain("mc_gauss_uni.txt", x, iters, 1);
}


void
MCGaussUniRef(MonteCarloDat *mcdat)
{
    const gsl_rng  *r2 = mcdat->r2;
    int             i;
    double          pi_avelnlike = 0.0;
    double          pi_avelnpost = 0.0;
    double          pi_varlnpost = 0.0;
    double          lnposti, lnlikei;
    double          inv_nd = 1.0/nd;
    double          yt_nd, delta, musimj;
    double          ln2pi = log(2.0 * M_PI);

    /* Monte Carlo with reference priors, calculate exp(lnpost) and exp(lnlike) */
    printf("Reference posterior Monte Carlo sampling Gaussian uniparameter ...\n");
    fflush(NULL);

    for (i = 0; i < iters; ++i)
    {
        yt_nd = yt*inv_nd;
        x[0][i] = musimj = gsl_ran_gaussian(r2, sqrt(inv_nd)) + yt_nd;
        lnlikei = -0.5 * nd * ln2pi
                  -0.5 *(x2t - 2.0*musimj*yt + nd*musimj*musimj);
        lnposti = normal_lnpdf(musimj, yt_nd, inv_nd);

        /* running mean and variance */
        delta = lnposti - pi_avelnpost;
        pi_avelnpost += delta/(i+1);
        pi_varlnpost += delta*(lnposti - pi_avelnpost);

        pi_avelnlike += lnlikei;
    }

    double pi_explnlike = -0.5 * (nd * log(2.0 * M_PI) + x2t - (yt*yt/nd) + 1.0);
    double pi_explnpost = -0.5 * (log(2.0*M_PI) - log(nd) + 1.0);

    mcdat->pi_explnlike  = pi_explnlike;
    mcdat->pi_explnpost  = pi_explnpost;

    mcdat->pi_avelnlike  = pi_avelnlike/iters;
    mcdat->pi_avelnpost  = pi_avelnpost;
    mcdat->pi_avenoprior = pi_avelnlike/iters - pi_avelnpost;
    mcdat->pi_varlnpost  = pi_varlnpost/iters;
    mcdat->pi_dic        = pi_avelnlike/iters - pi_varlnpost/iters;

    mcdat->dim = 1;
    mcdat->ndata = ndata*dim;

    printf("Monte Carlo done ...\n");
    fflush(NULL);

    if (write_files == 1)
        WriteChain("mc_gauss_uni_ref.txt", x, iters, 1);
}


void
MCGaussPrecUni(MonteCarloDat *mcdat)
{
    const double    lambda_0 = mcdat->lambda_0;
    const gsl_rng  *r2 = mcdat->r2;
    int             i;
    double          tmp, atmp, btmp, binvtmp, musimj;
    double          ln2pi = log(2.0 * M_PI);
    double          avelnlike = 0.0, avelnprior = 0.0, varlnpost = 0.0;
    double          avelnprlk2 = 0.0, avelnpost = 0.0, avelnprlk = 0.0;

    /* prior is gamma; we assume alpha=1, so that prior is exponential with beta=lambda_0 */
    /* Now simulate posterior of mu with Monte Carlo */
    printf("Monte Carlo sampling Gaussian precision uniparameter ...\n");
    fflush(NULL);
    avelnprior = avelnlike = avelnpost = avelnprlk2 = 0.0;
    atmp = 1.0+0.5*nd;
    btmp = lambda_0 + 0.5*x2t;
    binvtmp = 1.0/btmp;

    memset(lnprior, 0, iters * sizeof(double));
    memset(lnpost,  0, iters * sizeof(double));
    memset(lnlike,  0, iters * sizeof(double));

    for (i = 0; i < iters; ++i)
    {
        musimj = gsl_ran_gamma(r2, atmp, binvtmp);
        x[0][i] = musimj;
        lnprior[i] = log(lambda_0) - musimj * lambda_0;
        lnlike[i] = -0.5*nd*ln2pi + 0.5*nd * log(musimj) - 0.5*musimj*x2t;
        lnpost[i] = atmp * log(btmp) + (atmp-1.0)*log(musimj)-btmp*musimj - lgamma(atmp);

        //printf("\n%-d% 16.4f % 16.4f % 16.4f", i, lnprior[i], lnlike[i], lnpost[i]);

        avelnprior += lnprior[i];
        avelnlike  += lnlike[i];
        avelnpost  += lnpost[i];
        avelnprlk2 += (lnprior[i] + lnlike[i]) * (lnprior[i] + lnlike[i]);
    }

    avelnprior /= iters;
    avelnlike  /= iters;
    avelnpost  /= iters;
    avelnprlk = avelnprior + avelnlike;
    avelnprlk2 /= iters;

    varlnpost = 0.0;
    for (i = 0; i < iters; ++i)
    {
        tmp = lnprior[i] + lnlike[i] - avelnprlk;
        varlnpost += tmp * tmp;
    }

    varlnpost /= iters;

    //double entlnpost = 0.5 * log(2.0 * M_PI * varlnpost * M_E);

    double Eloglam = gsl_sf_psi((nd+2.0)/2.0) - log(lambda_0+0.5*x2t);
    double Elam = (nd+2.0)/(2.0 * lambda_0+x2t);

    double explnlike = - 0.5*nd * log(2.0*M_PI)
                       + 0.5*nd*Eloglam
                       - 0.5*x2t * Elam;
    double explnpost = - (1.0+0.5*nd)
                       - lgamma(1.0+0.5*nd)
                       + log(lambda_0 + 0.5*x2t)
                       + 0.5*nd*gsl_sf_psi(1.0+0.5*nd);
    double explnprior = + log(lambda_0) - lambda_0 * (nd+2.0)/(2.0*lambda_0 + x2t);
    double marglik = - 0.5*nd*log(2.0*M_PI)
                     + log(lambda_0)
                     - atmp*log(btmp)
                     + lgamma(atmp);

    mcdat->avelnprior = avelnprior;
    mcdat->explnprior = explnprior;
    mcdat->avelnlike  = avelnlike;
    mcdat->explnlike  = explnlike;
    mcdat->avelnpost  = avelnpost;
    mcdat->explnpost  = explnpost;
    mcdat->avelnprlk  = avelnprlk;
    mcdat->avelnprlk2 = avelnprlk2;
    mcdat->varlnpost  = varlnpost;
    mcdat->marglik    = marglik;
    //mcdat->entlnpost = entlnpost;

    mcdat->dim = 1;
    mcdat->ndata = ndata*dim;

    printf("Monte Carlo done ...\n");
    fflush(NULL);

    if (write_files == 1)
        WriteChain("mc_gauss_prec_uni.txt", x, iters, 1);
}


/* not really reference, as alpha=1, beta=0 */
void
MCGaussPrecUniRef(MonteCarloDat *mcdat)
{
    const gsl_rng  *r2 = mcdat->r2;
    int             i;
    double          pi_avelnlike = 0.0;
    double          pi_avelnpost = 0.0;
    double          pi_varlnpost = 0.0;
    double          lnposti, lnlikei;
    double          delta, atmp, btmp, binvtmp, musimj;
    double          ln2pi = log(2.0 * M_PI);

    /* Monte Carlo with reference priors, calculate exp(lnpost) and exp(lnlike) */
    printf("Reference posterior Monte Carlo sampling Gaussian precision uniparameter ...\n");
    fflush(NULL);

    atmp = 0.5*nd;
    btmp = 0.5*x2t;
    binvtmp = 1.0/btmp;

    for (i = 0; i < iters; ++i)
    {
        musimj = gsl_ran_gamma(r2, atmp, binvtmp);
        lnlikei = -0.5*nd*ln2pi + 0.5*nd * log(musimj) - 0.5*musimj*x2t;
        lnposti = atmp * log(btmp) + (atmp-1.0)*log(musimj)-btmp*musimj - lgamma(atmp);

        /* running mean and variance */
        delta = lnposti - pi_avelnpost;
        pi_avelnpost += delta/(i+1);
        pi_varlnpost += delta*(lnposti - pi_avelnpost);

        pi_avelnlike += lnlikei;
    }

    double Eloglam = gsl_sf_psi((nd+2.0)/2.0) - log(0.5*x2t);
    double Elam = (nd+2.0)/x2t;

    double pi_explnlike = - 0.5*nd * log(2.0*M_PI)
                          + 0.5*nd*Eloglam
                          - 0.5*x2t * Elam; // analytical exact, posterior expected ln like (alpha=1)
    double pi_explnpost = - (1.0+0.5*nd)
                          - lgamma(1.0+0.5*nd)
                          + log(0.5*x2t)
                          + 0.5*nd*gsl_sf_psi(1.0+0.5*nd); // analytical exact, posterior expected ln like (alpha=1)

    Eloglam = gsl_sf_psi((nd+2.0)/2.0) - log(0.5*x2t);

    double pi_explnlike2 = 0.5*nd * (Eloglam - log(2.0*M_PI) - 1.0) - 1.0; // analytical exact, posterior expected ln like, reference prior (alpha=beta=0)

    mcdat->pi_explnlike  = pi_explnlike;
    mcdat->pi_explnlike2 = pi_explnlike2;

    mcdat->pi_avelnlike  = pi_avelnlike/iters;
    mcdat->pi_avelnpost  = pi_avelnpost;
    mcdat->pi_explnpost  = pi_explnpost;
    mcdat->pi_avenoprior = pi_avelnlike/iters - pi_avelnpost;
    mcdat->pi_varlnpost  = pi_varlnpost/iters;
    mcdat->pi_dic        = pi_avelnlike/iters - pi_varlnpost/iters;

    mcdat->dim = 1;
    mcdat->ndata = ndata*dim;

    printf("Monte Carlo done ...\n");
    fflush(NULL);

    if (write_files == 1)
        WriteChain("mc_gauss_prec_uni_ref.txt", x, iters, 1);
}


void
MCExpo(MonteCarloDat *mcdat)
{
    const double    alpha_0 = mcdat->lambda_0;
    const gsl_rng  *r2 = mcdat->r2;
    int             i, j;
    double          tmp, musimj;
    const double    beta_0 = alpha_0;
    double          avelnlike = 0.0, avelnprior = 0.0, varlnpost = 0.0;
    double          avelnprlk2 = 0.0, avelnpost = 0.0, avelnprlk = 0.0;

    /* Now simulate posterior of mu with Monte Carlo */
    printf("Monte Carlo sampling ...\n");
    fflush(NULL);

    memset(lnprior, 0, iters * sizeof(double));
    memset(lnpost,  0, iters * sizeof(double));
    memset(lnlike,  0, iters * sizeof(double));

    avelnprior = avelnlike = avelnpost = avelnprlk2 = 0.0;
    for (i = 0; i < iters; ++i)
    {
        for (j = 0; j < dim; ++j)
        {
            musimj = gsl_ran_gamma(r2, alpha_0 + ndata, 1.0/(beta_0 + y[j]));
            x[j][i] = musimj;
            lnprior[i] += alpha_0 * log(beta_0) - lgamma(alpha_0)
                          + (alpha_0-1.0)*log(musimj) - beta_0*musimj;
            lnlike[i] += ndata*log(musimj) - musimj*y[j];
            lnpost[i] += + (alpha_0 + ndata)*log(beta_0+y[j])
                         - lgamma(alpha_0+ndata)
                         + (alpha_0+ndata-1.0)*log(musimj)
                         - musimj*(beta_0+y[j]);
        }

        //printf("\n%-d% 16.4f % 16.4f % 16.4f", i, lnprior[i], lnlike[i], lnpost[i]);

        avelnprior += lnprior[i];
        avelnlike  += lnlike[i];
        avelnpost  += lnpost[i];
        avelnprlk2 += (lnprior[i] + lnlike[i]) * (lnprior[i] + lnlike[i]);
    }

    avelnprior /= iters;
    avelnlike  /= iters;
    avelnpost  /= iters;
    avelnprlk = avelnprior + avelnlike;
    avelnprlk2 /= iters;

    varlnpost = 0.0;
    for (i = 0; i < iters; ++i)
    {
        tmp = lnprior[i] + lnlike[i] - avelnprlk;
        varlnpost += tmp * tmp;
    }

    varlnpost /= iters;

    //double entlnpost = 0.5 * log(2.0 * M_PI * varlnpost * M_E);

    double lbyt= 0.0;
    for (j = 0; j < dim; ++j)
        lbyt += log(beta_0+y[j]);

    double ibyt= 0.0;
    for (j = 0; j < dim; ++j)
        ibyt += 1.0/(beta_0+y[j]);

    double tbyt= 0.0;
    for (j = 0; j < dim; ++j)
        tbyt += y[j]/(beta_0+y[j]);

    double explnprior = dim * alpha_0 * log(beta_0)
                        - dim * lgamma(alpha_0)
                        + (alpha_0-1.0) * (dim*gsl_sf_psi(alpha_0 + ndata) - lbyt)
                        - (alpha_0+ndata)*beta_0*ibyt;
    double explnlike = dim*ndata*gsl_sf_psi(alpha_0 + ndata)
                       - ndata*lbyt
                       - (alpha_0+ndata)*tbyt;
    double explnpost = lbyt - dim*lgamma(alpha_0+ndata)
                       + dim*(alpha_0+ndata-1.0)*gsl_sf_psi(alpha_0 + ndata)
                       - dim*(alpha_0 + ndata);
    double marglik = + dim *(lgamma(lambda_0+ndata)
                     - lgamma(lambda_0)
                     + lambda_0*log(lambda_0))
                     - (lambda_0+ndata)*lbyt;

    mcdat->avelnprior = avelnprior;
    mcdat->explnprior = explnprior;
    mcdat->avelnlike  = avelnlike;
    mcdat->explnlike  = explnlike;
    mcdat->avelnpost  = avelnpost;
    mcdat->explnpost  = explnpost;
    mcdat->avelnprlk  = avelnprlk;
    mcdat->avelnprlk2 = avelnprlk2;
    mcdat->varlnpost  = varlnpost;
    mcdat->marglik    = marglik;
    //mcdat->entlnpost  = entlnpost;

    mcdat->dim = dim;
    mcdat->ndata = ndata;

    printf("Monte Carlo done ...\n");
    fflush(NULL);

    if (write_files == 1)
        WriteChain("mc_expo.txt", x, iters, dim);
}


void
MCExpoRef(MonteCarloDat *mcdat)
{
    const gsl_rng  *r2 = mcdat->r2;
    int             i, j;
    double          pi_avelnlike = 0.0;
    double          pi_avelnpost = 0.0;
    double          pi_varlnpost = 0.0;
    double          musimj, lnposti, lnlikei, delta;

    /* Monte Carlo with reference priors, calculate exp(lnpost) and exp(lnlike) */
    printf("Reference posterior Monte Carlo sampling ...\n");
    fflush(NULL);

    for (i = 0; i < iters; ++i)
    {
        lnposti = lnlikei = 0.0;
        for (j = 0; j < dim; ++j)
        {
            musimj = gsl_ran_gamma(r2, ndata, 1.0/y[j]);
            lnlikei += ndata*log(musimj) - musimj*y[j];
            lnposti += ndata*log(y[j]) - lgamma(ndata)
                       +(ndata-1.0)*log(musimj)
                       - musimj*y[j];
        }

        /* running mean and variance */
        delta = lnposti - pi_avelnpost;
        pi_avelnpost += delta/(i+1);
        pi_varlnpost += delta*(lnposti - pi_avelnpost);

        pi_avelnlike += lnlikei;
    }

    double sumlnt = 0.0;
    for (j = 0; j < dim; ++j)
        sumlnt += log(y[j]);

    double ibyt= 0.0;
    for (j = 0; j < dim; ++j)
        ibyt += 1.0/y[j];

    double pi_explnlike = nd * gsl_sf_psi(ndata) - ndata*sumlnt - nd;
    double pi_explnpost = sumlnt - dim*lgamma(ndata)
                          + dim*(ndata-1.0)*gsl_sf_psi(ndata)
                          - dim*(ndata);

    mcdat->pi_avelnlike  = pi_avelnlike/iters;
    mcdat->pi_explnlike  = pi_explnlike;
    mcdat->pi_avelnpost  = pi_avelnpost;
    mcdat->pi_explnpost  = pi_explnpost;
    mcdat->pi_avenoprior = pi_avelnlike/iters - pi_avelnpost;
    mcdat->pi_varlnpost  = pi_varlnpost/iters;
    mcdat->pi_dic        = pi_avelnlike/iters - pi_varlnpost/iters;

    mcdat->dim = dim;
    mcdat->ndata = ndata;

    printf("Monte Carlo done ...\n");
    fflush(NULL);

    if (write_files == 1)
        WriteChain("mc_expo_ref.txt", x, iters, dim);
}


void
MCExpoUni(MonteCarloDat *mcdat)
{
    const double    alpha_0 = mcdat->lambda_0;
    const gsl_rng  *r2 = mcdat->r2;
    int             i;
    double          tmp, musimj;
    const double    beta_0 = alpha_0;
    double          avelnlike = 0.0, avelnprior = 0.0, varlnpost = 0.0;
    double          avelnprlk2 = 0.0, avelnpost = 0.0, avelnprlk = 0.0;
    double          avelnscore = 0.0;

    /* Now simulate posterior of mu with Monte Carlo */
    printf("Monte Carlo sampling ...\n");
    fflush(NULL);

    avelnprior = avelnlike = avelnpost = avelnprlk2 = 0.0;

    memset(lnprior, 0, iters * sizeof(double));
    memset(lnpost,  0, iters * sizeof(double));
    memset(lnlike,  0, iters * sizeof(double));

    for (i = 0; i < iters; ++i)
    {
        musimj = gsl_ran_gamma(r2, alpha_0 + nd, 1.0/(beta_0 + yt));
        x[0][i] = musimj;
        lnprior[i] = + alpha_0 * log(beta_0)
                     - lgamma(alpha_0)
                     + (alpha_0-1.0)*log(musimj)
                     - beta_0*musimj;
        lnlike[i] = nd*log(musimj) - musimj*yt;
        lnpost[i] = + (alpha_0 + nd)*log(beta_0+yt)
                    - lgamma(alpha_0+nd)
                    + (alpha_0+nd-1.0)*log(musimj)
                    - musimj*(beta_0+yt);
        lnscore[i] = log(0.5*fabs( nd/musimj - yt + (alpha_0 - 1.0)/musimj - beta_0 ));

        //printf("\n%-d% 16.4f % 16.4f % 16.4f", i, lnprior[i], lnlike[i], lnpost[i]);

        avelnprior += lnprior[i];
        avelnlike  += lnlike[i];
        avelnpost  += lnpost[i];
        avelnprlk2 += (lnprior[i] + lnlike[i]) * (lnprior[i] + lnlike[i]);
        avelnscore += lnscore[i];
    }

    avelnprior /= iters;
    avelnlike  /= iters;
    avelnpost  /= iters;
    avelnprlk = avelnprior + avelnlike;
    avelnprlk2 /= iters;
    avelnscore /= iters;

    varlnpost = 0.0;
    for (i = 0; i < iters; ++i)
    {
        tmp = lnprior[i] + lnlike[i] - avelnprlk;
        varlnpost += tmp * tmp;
    }

    varlnpost /= iters;

    double explnprior = alpha_0 * log(beta_0) - lgamma(alpha_0)
                        + (alpha_0-1.0) * (gsl_sf_psi(alpha_0 + nd) - log(beta_0+yt))
                        - (alpha_0+nd)*beta_0/(beta_0+yt);
    double explnlike = nd*(gsl_sf_psi(alpha_0 + nd) - log(beta_0+yt))
                        - (alpha_0+nd) *  yt /(beta_0+yt);
    double explnpost = log(beta_0+yt) - lgamma(alpha_0+nd)
                       + (alpha_0+nd-1.0)*gsl_sf_psi(alpha_0 + nd)
                       -alpha_0 - nd;
    double marglik = 0.0;
    for (i = 0; i < dim; ++i)
        marglik += log(lambda_0+y[i]);

    marglik = + lgamma(lambda_0+nd)
              - lgamma(lambda_0)
              + lambda_0*log(lambda_0)
              - (lambda_0+nd)*log(lambda_0+yt);

    //double entlnpost = 0.5 * log(2.0 * M_PI * varlnpost * M_E);

    mcdat->avelnprior = avelnprior;
    mcdat->explnprior = explnprior;
    mcdat->avelnlike  = avelnlike;
    mcdat->explnlike  = explnlike;
    mcdat->avelnpost  = avelnpost;
    mcdat->explnpost  = explnpost;
    mcdat->avelnprlk  = avelnprlk;
    mcdat->avelnprlk2 = avelnprlk2;
    mcdat->varlnpost  = varlnpost;
    mcdat->marglik    = marglik;
    //mcdat->entlnpost  = entlnpost;
    mcdat->avelnscore = avelnscore;

    mcdat->dim = 1;
    mcdat->ndata = ndata*dim;

    printf("Monte Carlo done ...\n");
    fflush(NULL);

    if (write_files == 1)
        WriteChain("mc_expo_uni.txt", x, iters, 1);
}


void
MCExpoUniRef(MonteCarloDat *mcdat)
{
    const gsl_rng  *r2 = mcdat->r2;
    int             i;
    double          pi_avelnlike = 0.0;
    double          pi_avelnpost = 0.0;
    double          pi_varlnpost = 0.0;
    double          musimj, lnposti, lnlikei, delta;

    /* Monte Carlo with reference priors, calculate exp(lnpost) and exp(lnlike) */
    printf("Reference posterior Monte Carlo sampling ...\n");
    fflush(NULL);

    for (i = 0; i < iters; ++i)
    {
        musimj = gsl_ran_gamma(r2, nd, 1.0/yt);
        lnlikei = nd*log(musimj) - musimj*yt;
        lnposti = nd*log(yt) - lgamma(nd)
                  + (nd-1.0)*log(musimj)
                  - musimj*yt;

        /* running mean and variance */
        delta = lnposti - pi_avelnpost;
        pi_avelnpost += delta/(i+1);
        pi_varlnpost += delta*(lnposti - pi_avelnpost);

        pi_avelnlike += lnlikei;
    }

    double pi_explnlike = nd * (gsl_sf_psi(nd) - log(yt) - 1.0);
    //double pi_expexplnlike = nd * (gsl_sf_psi(nd) - log(nd) - 1.0);
    double pi_explnpost = log(yt) - lgamma(nd) + gsl_sf_psi(nd)*(nd-1.0) - nd;
    double pi_entlnpost = 0.5 * log(2.0 * M_PI * pi_varlnpost * M_E);

    //mcdat->pi_expexplnlike = pi_expexplnlike;
    mcdat->pi_explnlike = pi_explnlike;
    mcdat->pi_explnpost = pi_explnpost;

    mcdat->pi_avelnlike   = pi_avelnlike/iters;
    mcdat->pi_avelnpost   = pi_avelnpost;
    mcdat->pi_avenoprior  = pi_avelnlike/iters - pi_avelnpost;
    mcdat->pi_varlnpost   = pi_varlnpost/iters;
    mcdat->pi_dic         = pi_avelnlike/iters - pi_varlnpost/iters;
    mcdat->pi_entlnpost   = pi_entlnpost;

    mcdat->dim = 1;
    mcdat->ndata = ndata*dim;

    //printf("\n exp exp lnlike: %g\n\n", nd * gsl_sf_psi(ndata) - nd*log(ndata) - nd);

    printf("Monte Carlo done ...\n");
    fflush(NULL);

    if (write_files == 1)
        WriteChain("mc_expo_uni_ref.txt", x, iters, 1);
}


void
Usage(void)
{
    printf("\n                           <  BEGIN MARG  > \n");
    printf("I===-===-===-===-===-===-===-===-===-===-===-===-===-===-===-===-===-===-===-==I\n");
    printf("  Usage: \n");
    printf("    marg [options] <data file> \n\n");
    printf("    -b  burnin (as a fraction) \n");
    printf("    -d  # of dimensions in models \n");
    printf("    -e  exponential models \n");
    printf("    -f  write samples to file \n");
    printf("    -g  Gaussian models \n");
    printf("    -H  calculate Edgworth entropy of simulation \n");
    printf("    -i  # of samples or sampling iterations \n");
    printf("    -k  kth smallest distance for K-L entropy calc [1] \n");
    printf("    -l  lambda, prior precision \n");
    printf("    -n  # of data points per dimension \n");
    printf("    -p  parallel simulation \n");
    printf("    -r  pdf for generating random variates \n");
    printf("    -s  seed for random number generators \n");
    printf("    -v  version and info \n");
    printf("I===-===-===-===-===-===-===-===-===-===-===-===-===-===-===-===-===-===-===-==I\n");
    printf("                            <  END MARG  > \n\n\n");
//     printf("                            %s<  END THESEUS %s  >%s \n\n\n",
//            tc_RED, VERSION, tc_NC);
    fflush(NULL);
}


void
GetOpts(int argc, char *argv[])
{
    int            option;

    /* get the options */
    while ((option = getopt(argc, argv, "b:d:efgHi:k:l:m:n:pr:s:t:v")) != -1)
    {
        switch (option)
        {
/*
            case 'P':
                sscanf(optarg, "%lf:%lf:%lf:%lf:%lf:%lf:%lf:%lf:%lf:%lf:%lf:%lf",
                       &prior[0], &prior[1], &prior[2], &prior[3],
                       &prior[4], &prior[5], &prior[6], &prior[7],
                       &prior[8], &prior[9], &prior[10], &prior[11]);
                for (i = 0; i < dim; ++i)
                    prior[i] *= 0.5;
                break;
*/
            case 'b':
                burnin = (double) strtod(optarg, NULL);

                if (burnin > 0.0 && burnin < 1.0)
                    burnin = 1.0 - burnin;
                else
                    burnin = 0.5;
                break;

            case 'd':
                dim = (int) strtol(optarg, NULL, 10);
                break;

            case 'e':
                expo_model = 1;
                break;

            case 'f':
                write_files = 1;
                break;

            case 'g':
                gauss_model = 1;
                break;

            case 'H':
                entropy_calc = 1;
                break;

            case 'i':
                iters = (int) strtol(optarg, NULL, 10);
                break;

            case 'k':
                klentk = (int) strtol(optarg, NULL, 10);
                break;

            case 'l':
                lambda_0 = (double) strtod(optarg, NULL);
                break;

            case 'n':
                ndata = (double) strtod(optarg, NULL);
                break;

            case 'p':
                parallel = 1;
                break;

            case 'r':
                randmeth = (int) strtol(optarg, NULL, 10);
                break;

            case 's':
                seed = (int) strtol(optarg, NULL, 10);
                break;

            case 't':
                thrdnum = (int) strtol(optarg, NULL, 10);
                break;

            case 'v':
                Version();
                exit(EXIT_FAILURE);
                break;

            default:
                perror("\n\n  ERROR");
                fprintf(stderr, "\nBad option '-%c' \n", optopt);
                Usage();
                exit(EXIT_FAILURE);
                break;
        }
    }
}


int
main(int argc, char *argv[])
{
    int            i;

    const gsl_rng_type     *T = NULL;
    gsl_rng                *r2 = NULL;

    if (argc == 1)
    {
        Usage();
        exit(EXIT_FAILURE);
    }

    GetOpts(argc, argv);

    //narguments = argc - optind; /* number of nonoption args */
    argv += optind; /* now argv is set with first arg = argv[0] */

    thrdnum = dim;

    SimData       **simdata = malloc(thrdnum * sizeof(SimData *));
    pthread_t      *callThd = malloc(thrdnum * sizeof(pthread_t));
    pthread_attr_t  attr;

    pthread_attr_init(&attr);
/*     pthread_mutexattr_t mattr; */
/*     pthread_mutexattr_init(&mattr); */
/*     pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_ERRORCHECK); */
/*     pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_NORMAL); */
/*     pthread_attr_getstacksize (&attr, &stacksize); */
/*     printf("\nDefault stack size = %d", (int) stacksize); */
    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
    pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);

    for (i = 0; i < thrdnum; ++i)
        simdata[i] = malloc(sizeof(SimData));

    nd = ndata * dim;

    gsl_rng_env_setup();
    if (seed == 0)
    {
        unsigned long int  clockn, timen, pidn;

        pidn = getpid();
        timen = time(NULL);
        clockn = clock();
        gsl_rng_default_seed = pidn + clockn + timen;

        printf("\npid:%lu  time:%lu clock:%lu\nseed:%lu\n\n", pidn, timen, clockn, gsl_rng_default_seed);
    }
    else
    {
        gsl_rng_default_seed = seed;
    }

    T = gsl_rng_ranlxd2;
    r2 = gsl_rng_alloc(T);
    //gsl_rng_set (r2, 1);

    cov = MatAlloc(dim, dim);
    lnpost  = calloc(iters, sizeof(double));
    lnlike  = calloc(iters, sizeof(double));
    lnprior = calloc(iters, sizeof(double));
    lnscore = calloc(iters, sizeof(double));
    h = calloc(iters, sizeof(double));
    pave = calloc(dim, sizeof(double));
    x = MatAlloc(dim, iters);
    data = calloc(dim, sizeof(double *));
    for (i = 0; i < dim; ++i)
        data[i] = calloc(ndata, sizeof(double));
    y = calloc(dim, sizeof(double));
    x2 = calloc(dim, sizeof(double));


    /************************************************************************************/
    if (entropy_calc == 1)
    {
        double         entropy;

        RandVec(x, dim, iters, randmeth, r2);

        entropy = CalcEdgeworthVanHulleEntropy(x, dim, iters);

        printf("\n-d ln(n):   %14.3f", -dim * log(ndata));
        printf("\nentropy:    %14.3f", entropy);
        printf ("\n\n");

        fflush(NULL);

        entropy = CalcKLentropy(x, dim, iters, klentk);

        printf("\nentropy:    %14.3f", entropy);
        printf ("\n\n");

        fflush(NULL);

        exit(EXIT_SUCCESS);
    }

    /************************************************************************************/
    if (parallel == 1)
    {
        SimGaussPth(data, simdata, callThd, &attr, thrdnum);
        //SimExpoPth(data, simdata, callThd, &attr, thrdnum);
    }
    else
    {
        SimGauss(r2);
        //SimExpo(r2);
    }

    CalcCumulants();


    /************************************************************************************/
    if (gauss_model == 1)
    {
        printf("\n************************************************************************************");
        printf("\nHierarchical gaussian model:\n");

        MonteCarloDat         *mcdat = NULL;
        mcdat = malloc(sizeof (MonteCarloDat));
        double      phi_0 = 1.0 / lambda_0;

        MonteCarloDatInit(mcdat);

        mcdat->r2 = r2;
        mcdat->iters = iters;
        mcdat->lambda_0 = phi_0;
        mcdat->cov = cov;
        mcdat->pave = pave;

        GibbsGaussHierarch(mcdat);

        mcdat->lapmet = CalcLaplaceMetHier(mcdat);
        mcdat->edge_entropy = CalcEdgeworthVanHulleEntropy(x, dim, iters);
        mcdat->expmet = mcdat->avelnprior + mcdat->avelnlike + mcdat->edge_entropy;
        mcdat->hme = CalcHarmonicMean(mcdat, lnlike, iters);

        PrintMargEstimates(mcdat);

        free(mcdat);
    }


    /************************************************************************************/
    /* gaussian model */
    if (gauss_model == 1)
    {
        printf("\n************************************************************************************");
        printf("\nGaussian model:\n");
        fflush(NULL);

        MonteCarloDat         *mcdat = NULL;
        mcdat = malloc(sizeof (MonteCarloDat));

        MonteCarloDatInit(mcdat);

        mcdat->r2 = r2;
        mcdat->iters = iters;
        mcdat->lambda_0 = lambda_0;
        mcdat->cov = cov;
        mcdat->pave = pave;

        MCGauss(mcdat);

        mcdat->lapmet = CalcLaplaceMet(mcdat);
        mcdat->edge_entropy = CalcEdgeworthVanHulleEntropy(x, dim, iters);
//          printf("\nKozachenko ##############************** %g *****************##############################\n",
//                 CalcKLentropy(x, dim, iters, klentk));
//         fflush(NULL);
        mcdat->expmet = mcdat->avelnprior + mcdat->avelnlike + mcdat->edge_entropy;
        mcdat->hme = CalcHarmonicMean(mcdat, lnlike, iters);

        PrintMargEstimates(mcdat);

        free(mcdat);
    }


    /************************************************************************************/
    if (gauss_model == 1)
    {
        printf("\n************************************************************************************");
        printf("\nGaussian reference model:\n");

        MonteCarloDat         *mcdat = NULL;
        mcdat = malloc(sizeof (MonteCarloDat));

        MonteCarloDatInit(mcdat);

        mcdat->r2 = r2;
        mcdat->iters = iters;

        MCGaussRef(mcdat);

        PrintRefMargEstimates(mcdat);

        printf("\nlogLmax: %f\n", -0.5*nd*log(2.0*M_PI) - 0.5*x2t + 0.5*yt2/ndata);

        free(mcdat);
    }


    /************************************************************************************/
    /* gaussian one-param model, unknown mu location parameter */
    if (gauss_model == 1)
    {
        printf("\n************************************************************************************");
        printf("\nGaussian one-parameter model:\n");

        MonteCarloDat         *mcdat = NULL;
        mcdat = malloc(sizeof (MonteCarloDat));

        MonteCarloDatInit(mcdat);

        mcdat->r2 = r2;
        mcdat->iters = iters;
        mcdat->lambda_0 = lambda_0;
        mcdat->cov = cov;
        mcdat->pave = pave;

        MCGaussUni(mcdat);

        mcdat->lapmet = CalcLaplaceMetUni(mcdat);
        mcdat->edge_entropy = CalcEdgeworthVanHulleEntropy(x, 1, iters);
        mcdat->expmet = mcdat->avelnprior + mcdat->avelnlike + mcdat->edge_entropy;
        mcdat->hme = CalcHarmonicMean(mcdat, lnlike, iters);

        PrintMargEstimates(mcdat);

        printf("\nlogLmax: %f\n", -0.5*nd*log(2.0*M_PI) - 0.5*x2t + 0.5*yt*yt/nd);
        printf("\ndiff: %f\n", -0.5*(yt*yt/nd - yt2/ndata + dim - 1.0));

        free(mcdat);
    }


    /************************************************************************************/
    if (gauss_model == 1)
    {
        printf("\n************************************************************************************");
        printf("\nGaussian reference one-parameter model:\n");

        MonteCarloDat         *mcdat = NULL;
        mcdat = malloc(sizeof (MonteCarloDat));

        MonteCarloDatInit(mcdat);

        mcdat->r2 = r2;
        mcdat->iters = iters;

        MCGaussUniRef(mcdat);

        PrintRefMargEstimates(mcdat);
        free(mcdat);
    }


    /************************************************************************************/
    /* gaussian one-param model, known mu=0, unknown lambda precision parameter */
    if (gauss_model == 1)
    {
        printf("\n************************************************************************************");
        printf("\nGaussian one-parameter precision model:\n");

        MonteCarloDat         *mcdat = NULL;
        mcdat = malloc(sizeof (MonteCarloDat));

        MonteCarloDatInit(mcdat);

        mcdat->r2 = r2;
        mcdat->iters = iters;
        mcdat->lambda_0 = lambda_0;
        mcdat->cov = cov;
        mcdat->pave = pave;

        MCGaussPrecUni(mcdat);

        mcdat->lapmet = CalcLaplaceMetUni(mcdat);
        mcdat->edge_entropy = CalcEdgeworthVanHulleEntropy(x, 1, iters);
        mcdat->expmet = mcdat->avelnprior + mcdat->avelnlike + mcdat->edge_entropy;
        mcdat->hme = CalcHarmonicMean(mcdat, lnlike, iters);

        PrintMargEstimates(mcdat);

        free(mcdat);
    }


    /************************************************************************************/
    if (gauss_model == 1)
    {
        printf("\n************************************************************************************");
        printf("\nGaussian reference one-parameter precision model:\n");

        MonteCarloDat         *mcdat = NULL;
        mcdat = malloc(sizeof (MonteCarloDat));

        MonteCarloDatInit(mcdat);

        mcdat->r2 = r2;
        mcdat->iters = iters;

        MCGaussPrecUniRef(mcdat);

        PrintRefMargEstimates(mcdat);
        free(mcdat);
    }


    /************************************************************************************/
    /* Exponential model */
    if (expo_model == 1)
    {
        printf("\n************************************************************************************");
        printf("\nExponential model:\n");

        MonteCarloDat         *mcdat = NULL;
        mcdat = malloc(sizeof (MonteCarloDat));

        MonteCarloDatInit(mcdat);

        mcdat->r2 = r2;
        mcdat->iters = iters;
        mcdat->lambda_0 = lambda_0;
        mcdat->cov = cov;
        mcdat->pave = pave;

        MCExpo(mcdat);

        mcdat->lapmet = CalcLaplaceMet(mcdat);
        mcdat->edge_entropy = CalcEdgeworthVanHulleEntropy(x, dim, iters);
        mcdat->expmet = mcdat->avelnprior + mcdat->avelnlike + mcdat->edge_entropy;
        mcdat->hme = CalcHarmonicMean(mcdat, lnlike, iters);

        PrintMargEstimates(mcdat);

        free(mcdat);
    }

    /************************************************************************************/
    if (expo_model == 1)
    {
        printf("\n************************************************************************************");
        printf("\nExponential reference model:\n");

        MonteCarloDat         *mcdat = NULL;
        mcdat = malloc(sizeof (MonteCarloDat));

        MonteCarloDatInit(mcdat);

        mcdat->r2 = r2;
        mcdat->iters = iters;

        MCExpoRef(mcdat);

        PrintRefMargEstimates(mcdat);
        free(mcdat);
    }


    /************************************************************************************/
    /* Exponential one-param model */
    if (expo_model == 1)
    {
        printf("\n************************************************************************************");
        printf("\nExponential one-parameter model:\n");

        MonteCarloDat         *mcdat = NULL;
        mcdat = malloc(sizeof (MonteCarloDat));

        MonteCarloDatInit(mcdat);

        mcdat->r2 = r2;
        mcdat->iters = iters;
        mcdat->lambda_0 = lambda_0;
        mcdat->cov = cov;
        mcdat->pave = pave;

        MCExpoUni(mcdat);

        mcdat->lapmet = CalcLaplaceMetUni(mcdat);
        mcdat->edge_entropy = CalcEdgeworthVanHulleEntropy(x, 1, iters);
        mcdat->expmet = mcdat->avelnprior + mcdat->avelnlike + mcdat->edge_entropy;
        mcdat->hme = CalcHarmonicMean(mcdat, lnlike, iters);

        PrintMargEstimates(mcdat);

        free(mcdat);
    }


    /************************************************************************************/
    if (expo_model == 1)
    {
        printf("\n************************************************************************************");
        printf("\nExponential reference one-parameter model:\n");

        MonteCarloDat         *mcdat = NULL;
        mcdat = malloc(sizeof (MonteCarloDat));

        MonteCarloDatInit(mcdat);

        mcdat->r2 = r2;
        mcdat->iters = iters;

        MCExpoUniRef(mcdat);

        PrintRefMargEstimates(mcdat);
        free(mcdat);
    }


    /************************************************************************************/
//    printf("\nwriting files ...\n\n");
    printf("\n");
    fflush(NULL);

//     fp = fopen("lnL.txt" ,"w");
//
//     for (i = 0; i < iters; ++i)
//         fprintf(fp, "%-12.3f\n", lnlike[i]);
//
//     fprintf(fp, "\n\n");
//     fclose(fp);

    /************************************************************************************/
    for (i = 0; i < dim; ++i)
        free(data[i]);
    free(data);
    MatDestroy(&x);
    free(lnscore);
    free(lnpost);
    free(lnprior);
    free(h);
    free(pave);
    free(y);
    free(x2);
    MatDestroy(&cov);

    pthread_attr_destroy(&attr);
    for (i = 0; i < thrdnum; ++i)
        free(simdata[i]);
    free(simdata);
    free(callThd);

    gsl_rng_free(r2);
    r2 = NULL;

    exit(EXIT_SUCCESS);
}


