/*
    Theseus - maximum likelihood superpositioning of macromolecular structures

    Copyright (C) 2004-2015 Douglas L. Theobald

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the:

    Free Software Foundation, Inc.,
    59 Temple Place, Suite 330,
    Boston, MA  02111-1307  USA

    -/_|:|_|_\-
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <limits.h>
#include <float.h>
#include <unistd.h>
#include <ctype.h>
#include "DLTmath.h"
#include "gamma_dist.h"
#include "normal_dist.h"
#include "invgauss_dist.h"
#include "statistics.h"


double
average(const double *data, const int dim)
{
    double          m = 0.0;
    int             i = dim;

    while(i-- > 0)
        m += *data++;

    return(m / (double) dim);
}


double
variance(const double *data, const int dim, const double mean)
{
    double          v = 0.0, tmpv;
    int             i = dim;

    while(i-- > 0)
    {
        tmpv = *data++ - mean;
        v += (tmpv * tmpv);
    }

    return(v / dim);
}


/* calculate the average standard error of a vector of variances
   see eqn 54 from http://mathworld.wolfram.com/NormalDistribution.html
   and http://mathworld.wolfram.com/SampleVarianceDistribution.html
   This calculation is parametric, and assumes a normal distribution.
   It is based on the fact that the variance of a variance is given by
   Var(var) =  2 * var^2 * (N-1) / N^2, where N is the sample size.
   Here N (sdim) is in the denom since this is the standard error of the mean,
   i.e. we are interested in the variance of the estimate, not the
   distribution.*/
double
VarVariance(const double *data, const int ddim, const int sdim)
{
    double          v = 0.0, tmpv;
    int             i = ddim;

    while(i-- > 0)
    {
        tmpv = *data++;
        v += (tmpv * tmpv);
    }

    return(2.0 * v * (sdim - 1) / (sdim * sdim * sdim * ddim));
    /* return(2.0 * v / (ddim * sdim * sdim)); */
}


int
moments(const double *data, const int dim, /* data array and length */
        double *ave, double *median,       /* mean and median */
        double *adev, double *mdev,        /* ave dev from mean and median */
        double *sdev, double *var,         /* std deviation, variance */
        double *skew, double *kurt,        /* skewness and kurtosis */
        double *hrange, double *lrange)    /* range of data, high and low */
{
    int             i;
    double          verror, sum, temp;
    double         *x;
    double          ddim = (double) dim;

    if (dim <= 1)
    {
        fprintf(stderr, "\nERROR: #data points in \'moments\' is less than 2 \n");
        return(0);
    }

    x = malloc((dim + 1) * sizeof(double));
    memcpy(x, data, dim * sizeof(double));
    x[dim] = DBL_MAX; /* required sentinel for quicksort */
    /* quicksort(x, dim); */
    qsort(x, dim, sizeof(double), dblcmp);

    if (dim % 2 == 0)
        *median = (x[dim / 2] + x[(dim - 1) / 2]) / 2.0;
    else
        *median = x[(dim - 1) / 2];

    *hrange = x[dim - 1];
    *lrange = x[0];

    sum = 0.0;
    for (i = 0; i < dim; ++i)
        sum += x[i];

    *ave = sum / ddim;

    (*adev) = (*mdev) = (*var) = (*skew) = (*kurt) = verror = 0.0;
    for (i = 0; i < dim; ++i)
    {
        sum = x[i] - (*ave);
        *adev += fabs(sum);
        *mdev += fabs(x[i] - (*median));
        verror += sum;
        temp = sum * sum;
        *var += temp;
        temp *= sum;
        *skew += temp;
        temp *= sum;
        *kurt += temp;
    }

    free(x);

    *adev /= ddim;
    *mdev /= ddim;
    *var = (*var - (verror * verror / ddim)) / (ddim - 1.0);
    *sdev = sqrt(*var);

    if (*var)
    {
        *skew /= (ddim * (*var) * (*sdev));
        *kurt = (*kurt) / (ddim * (*var) * (*var)) - 3.0;
        return(1);
    }
    else
        return(0);
}


/* chi_sqr_adapt() calculates the reduced chi^2 value for a fit
   of data to a distribution, and the log likelihood for the given
   parameters.

   You must specify the distribution's pdf, its ln(pdf), and a
   function for integrating the pdf between two values. For
   compatibility, all functions here take two parameters for the
   distribution, usu. a location and a shape parameter. If your
   dist has only one parameter, make a second dummy parameter and
   pass it a NULL value. The following function assumes that the
   distribution is approximately correct, and assigns equal numbers
   of data points to the bins (using the limits of that specific
   binned data for the integration). Chi^2 tests only work well
   when there are > 6 expected data points per bin. Thus, if the
   dist does not describe the data well, this might not be true,
   but in that case you should get a large chi^2 value anyway.

   double  *data            array of your data
   int      num             number of data points - must be >= 25
   int      nbins           number of bins for chi^2 fit
                            if you pass 0, the default is sqrt(num)
   double  *logL            pointer to log likelihood value
   double   ave             first distribution parameter, usu. location
   double   lambda          second dist param, usu. scale
   double  *dist_pdf        function pointer, calcualtes pdf for dist
   double  *dist_lnpdf      function pointer, calculates log of pdf for dist
   double  *dist_int        function pointer, integrates pdf between x and y
                            the function integrate_romberg() can be used to
                            integrate a pdf if the cdf or sdf are unknown
*/
double
chi_sqr_adapt(const double *data, const int num, int nbins, double *logL,
              const double ave, const double lambda,
              double (*dist_pdf)(double x, double param1, double param2),
              double (*dist_lnpdf)(double x, double param1, double param2),
              double (*dist_int)(double x, double y, double param1, double param2))
{
    int             i, j, bincount;
    double          chisq, temp, expect;
    double         *bins, *array;
/* *logL = dist_logL(dist_lnpdf, ave, lambda, data, num); */
/* if (1) return(0); */
    temp = expect = chisq = 0;

    if (nbins == 0)
    {
/*         if (num < 25) */
/*         { */
/*             fprintf(stderr, "\n\n ERROR: not enough data points in chi_sqr_adapt \n"); */
/*             exit(EXIT_FAILURE); */
/*         } */

        nbins = floor(sqrt((double) num));
    }

    bins  = calloc(nbins, sizeof(double));
    array = calloc(num,   sizeof(double));

    for (j = 0; j < num; ++j)
        array[j] = data[j];

    qsort(array, num, sizeof(double), dblcmp);

    chisq = 0.0;
    for (i = 0; i < nbins - 1; ++i)
    {
        expect = dist_int(array[i * nbins], array[(i+1) * nbins], ave, lambda);

        if (expect == 0.0)
            continue;
 
        expect *= (double) num;

        bincount = nbins;
        temp = bincount - expect;
        temp = mysquare(temp) / expect;
        chisq += temp;
/*         printf("\n%-7d range [%10.3f to %10.3f (%10.3f)] %-8.3f %-8.3f %-8.3f %-14.5f", */
/*                 i+1, array[i * nbins], array[(i+1) * nbins], array[(i+1) * nbins] - array[i * nbins],  */
/*                 (double) bincount, expect, (double) num * dist_int(array[i * nbins], array[(i+1) * nbins], ave, lambda), temp); */
/*         fflush(NULL); */
    }

    expect = dist_int(array[i * nbins], array[num - 1], ave, lambda);

    if (expect != 0.0)
    {
        expect *= (double) num;
    
        bincount = num - (nbins * (nbins - 1));
        temp = bincount - expect;
        temp = mysquare(temp) / expect;
        chisq += temp;
/*         printf("\n%-7d range [%10.3f to %10.3f] %-8.3f %-8.3f %-8.3f %-14.5f", */
/*                 i+1, array[i * nbins], array[num - 1], */
/*                 (double) bincount, expect, (double) num * dist_int(array[i * nbins], array[num - 1], ave, lambda), temp); */
        fflush(NULL);
    }

    *logL = dist_logL(dist_lnpdf, ave, lambda, data, num);

    free(bins);
    free(array);
/* printf("\n%f", chisq/nbins); */
    return (chisq / (double) nbins);
}


double
chi_sqr_adapt_mix(const double *data, const int num, int nbins, double *logL,
                  const double *p1, const double *p2, const double *mixp, const int nmix,
                  double (*dist_pdf)(double x, double param1, double param2),
                  double (*dist_lnpdf)(double x, double param1, double param2),
                  double (*dist_int)(double x, double y, double param1, double param2))
{
    int             i, j, bincount;
    double          chisq, temp, expect;
    double         *bins, *array;

    temp = expect = chisq = 0;

    if (nbins == 0)
        nbins = floor(sqrt((double) num));

    bins  = calloc(nbins, sizeof(double));
    array = calloc(num,   sizeof(double));

    for (j = 0; j < num; ++j)
        array[j] = data[j];

    qsort(array, num, sizeof(double), dblcmp);

    chisq = 0.0;
    for (i = 0; i < nbins - 1; ++i)
    {
        expect = 0.0;
        for (j = 0; j < nmix; ++j)
            expect += mixp[j] * dist_int(array[i * nbins], array[(i+1) * nbins], p1[j], p2[j]);

        if (expect == 0.0)
            continue;
 
        expect *= (double) num;

        bincount = nbins;
        temp = bincount - expect;
        temp = mysquare(temp) / expect;
        chisq += temp;
/*         printf("\n%-7d range [%10.3f to %10.3f (%10.3f)] %-8.3f %-8.3f %-8.3f %-14.5f", */
/*                 i+1, array[i * nbins], array[(i+1) * nbins], array[(i+1) * nbins] - array[i * nbins],  */
/*                 (double) bincount, expect, (double) num * dist_int(array[i * nbins], array[(i+1) * nbins], ave, lambda), temp); */
/*         fflush(NULL); */
    }

    expect = 0.0;
    for (j = 0; j < nmix; ++j)
        expect += mixp[j] * dist_int(array[i * nbins], array[num - 1], p1[j], p2[j]);

    if (expect != 0.0)
    {
        expect *= (double) num;
    
        bincount = num - (nbins * (nbins - 1));
        temp = bincount - expect;
        temp = mysquare(temp) / expect;
        chisq += temp;
/*         printf("\n%-7d range [%10.3f to %10.3f] %-8.3f %-8.3f %-8.3f %-14.5f", */
/*                 i+1, array[i * nbins], array[num - 1], */
/*                 (double) bincount, expect, (double) num * dist_int(array[i * nbins], array[num - 1], ave, lambda), temp); */
        fflush(NULL);
    }

    /* *logL = dist_logL(dist_lnpdf, ave, lambda, data, num); */

    free(bins);
    free(array);
/* printf("\n%f", chisq/nbins); */
    return (chisq / (double) nbins);
}


/* Calculates the log likelihood for a general distribution given
   (1) the data array, (2) a function that calculates the log of the pdf
   of your distribution, and (3) specified parameters.
   See chi_sqr_adapt() for more detail.
*/
double
dist_logL(double (*dist_lnpdf)(const double x, const double param1, const double param2),
          double param1, double param2, const double *data, const int num)
{
    int             i, outliers;
    double          logL, temp;

    logL = 0.0;
    outliers = 0;
    for (i = 0; i < num; ++i)
    {
        temp = dist_lnpdf(data[i], param1, param2);
/*         printf("\n%3d % e", i, temp); */
/*         fflush(NULL); */

        if (temp == DBL_MAX || temp == -DBL_MAX)
        {
            ++outliers;
            continue;
        }
        else
        {
            logL += temp;
            /* logL2 += temp * temp; */
        }
    }

    /* SCREAMF(sqrt((logL2/(double) num) - mysquare(logL/(double)num))); */

    if (outliers > 0)
    {
/*         for (i = 0; i < num; ++i) */
/*             printf("%3d % e\n", i, data[i]); */

        printf(" --vv logL calculated without %d outliers \n", outliers);
        fflush(NULL);
    }

    return(logL);
}


double
chi_sqr_hist(double *data, double *freq, int num, double *logL, double ave, double lambda,
             double (*dist_pdf)(double x, double param1, double param2),
             double (*dist_lnpdf)(double x, double param1, double param2),
             double (*dist_int)(double x, double y, double param1, double param2))
{
    int             i;
    double          chisq, temp, expect, norm;

    temp = expect = chisq = 0;

    norm = 0.0;
    for (i = 0; i < num; ++i)
        norm += freq[i];

SCREAMF(norm);

    chisq = 0.0;
    for (i = 0; i < num; ++i)
    {
        expect = dist_pdf(data[i], ave, lambda);
        expect = dist_int(data[i]-0.015811/2, data[i]+0.015811/2, ave, lambda);
        expect *= norm;

        if (expect == 0.0)
            continue;

        temp = freq[i] - expect;
        temp = mysquare(temp)/*  / expect */;
        chisq += temp;
        printf("%-7d range [%11.3f] %-8.3f %-8.3f %-14.5f\n",
                i+1, data[i], freq[i], expect, temp);
        fflush(NULL);
    }

    *logL = dist_logL(dist_lnpdf, ave, lambda, data, num);

    return (chisq / (double) num);
}


double
chi_sqr_one(double *data, int num, int nbins, double *prob, double ave, double lambda,
            double (*dist)(double x, double param1, double param2))
{
    int             i, j;
    double          chisq, temp, binrange, expect;
/*     double          x; */
    double         *bins, *array;
/*     FILE           *igfile = NULL; */

    if (nbins == 0)
    {
        nbins = num / 10;

        if (nbins > 100)
            nbins = 100;

        if (nbins < 5)
            nbins = 5;
    }

    bins  = (double *) calloc(nbins, sizeof(double));
    array = (double *) calloc(num,   sizeof(double));

    for (j = 0; j < num; ++j)
        array[j] = data[j];

    qsort(array, num, sizeof(double), dblcmp);

/*     for (i = 0; i < num; ++i) */
/*         printf("\n%-6d %-12.6 ", i, array[i]); */

    binrange = (array[num-1] - array[0]) / (double) nbins;

SCREAMF(binrange);

    j = 0;
    while (array[j] <= binrange + array[0])
        ++j;

SCREAMD(j);

    for (i = 1; i < nbins; ++i)
    {
        if (j == num)
            break;

        bins[i] = 0;
        while (array[j] <= (double)(i+1) * binrange + array[0])
        {
            ++bins[i];
            ++j;

            if (j == num)
                break;
        }
    }

SCREAMD(nbins);
SCREAMD(num);

    chisq = 0.0;
    for (j = 0; j < nbins; ++j)
    {
        expect = integrate_romberg(dist, ave, lambda, (double)j * binrange + array[0], (double)(j+1) * binrange + array[0]);
        /* printf("\n--> %-8d %-14.5 ", j, expect); */

        if (expect == 0.0)
            continue;
 
        expect *= (double) num;
        temp = bins[j] - expect;
        temp = mysquare(temp) / expect;
        chisq += temp;
        printf("%-8d %-14.5f %-14.5f %-14.5f\n", j+1, bins[j], expect, temp);
    }
    fflush(NULL);

    *prob = 0.0;
    for (j = 0; j < nbins; ++j)
        *prob += bins[j];

SCREAMD((int) *prob);
SCREAMD((int) num);

/*     igfile = fopen("invgamma_dist.txt", "w"); */
/*  */
/*     fprintf(igfile, "num = %-4d, binrange = %8.3e, ave = %8.3e, lambda = %8.3e \n", num, binrange, ave, lambda); */
/*  */
/*     for (i = 2; i <= nbins; ++i) */
/*     { */
/*         x = invgauss_pdf(binrange * (double)i, ave, lambda); */
/*         fprintf(igfile, "%-8.3 %-16.5e \n", (double) i * binrange, x); */
/*     } */
/*  */
/*     fclose(igfile); */
    free(bins);
    free(array);

    return (chisq / (double) nbins);
}


int
dblcmp(const void *dbl1, const void *dbl2)
{
    double *first  = (double *) dbl1;
    double *second = (double *) dbl2;

    if (*first < *second)
        return (-1);
    else if (*first == *second)
        return (0);
    else /* if (*first > *second) */
        return (1);
}


int
dblcmp_rev(const void *dbl1, const void *dbl2)
{
    double *first  = (double *) dbl1;
    double *second = (double *) dbl2;

    if (*first > *second)
        return (-1);
    else if (*first == *second)
        return (0);
    else /* if (*first < *second) */
        return (1);
}


