/*
    The WebDruid - a web server log analysis program

    Copyright (C) 2003-2004  Fabien Chevalier (fabien@juliana-multimedia.com)

    Original webalizer copyright:
    Copyright (C) 1997-2001  Bradford L. Barrett (brad@mrunix.net)

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version, and provided that the above
    copyright and permission notice is included with all distributed
    copies of this or derived software.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA

    This software uses the gd graphics library, which is copyright by
    Quest Protein Database Center, Cold Spring Harbor Labs.  Please
    see the documentation supplied with the library for additional
    information and license terms, or visit www.boutell.com/gd/ for the
    most recent version of the library and supporting documentation.
*/

/*********************************************/
/* STANDARD INCLUDES                         */
/*********************************************/

#include <time.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>                           /* normal stuff             */
#include <ctype.h>
#include <sys/utsname.h>
#include <sys/times.h>

/* ensure getopt */
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif

/* ensure sys/types */
#ifndef _SYS_TYPES_H
#include <sys/types.h>
#endif

/* some systems need this */
#ifdef HAVE_MATH_H
#include <math.h>
#endif

/* SunOS 4.x Fix */
#ifndef CLK_TCK
#define CLK_TCK _SC_CLK_TCK
#endif

#include "webdruid.h"                        /* main header              */
#include "lang.h"
#include "linklist.h"
#include "hashtab.h"
#include "utils.h"
#include "sengine.h"

/* internal function prototypes */

HNODEPTR  new_hnode(char *);                  /* new host node            */
UNODEPTR  new_unode(char *);                  /* new url node             */
RNODEPTR  new_rnode(char *);                  /* new referrer node        */
ANODEPTR  new_anode(char *);                  /* new user agent node      */
INODEPTR  new_inode(char *);                  /* new ident node           */
#ifdef USE_DNS
DNODEPTR  new_dnode(char *);                  /* new DNS node             */
#endif  /* USE_DNS */
OPNODEPTR new_opnode(char *, char *, char     /* new open path node       */
            *, char *, char *);
PNODEPTR  new_pnode(LISTPTR, char *, int);    /* new path node            */

void      update_entry(char *);               /* update entry/exit        */
void      update_exit(char *);                /* page totals              */

/* local data */
                                              /* hash tables for:         */
HNODEPTR  sm_htab[MAXHASH];                   /* sites (monthly)          */
HNODEPTR  sd_htab[MAXHASH];                   /* sites (daily)            */
UNODEPTR  um_htab[MAXHASH];                   /* urls                     */
RNODEPTR  rm_htab[MAXHASH];                   /* referrers and agents...  */
ANODEPTR  am_htab[MAXHASH];                   /* user agents              */
INODEPTR  im_htab[MAXHASH];                   /* ident table (username)   */
#ifdef USE_DNS
DNODEPTR  host_table[MAXHASH];                /* DNS hash table           */
#endif  /* USE_DNS */
OPNODEPTR op_htab[MAXHASH];                   /* currently opened path    */
PNODEPTR  gp_htab[MAXHASH];                   /* all path (not sorted)    */
PNODEPTR  fm_htab[MAXHASH];                   /* users flow (monthly)     */

/*********************************************/
/* NEW_HNODE - create host node              */
/*********************************************/

HNODEPTR new_hnode(char *str)
{
   HNODEPTR newptr;
   char     *sptr;

   if (strlen(str) >= MAXHOST)
   {
      if (verbose)
      {
         fprintf(stderr,"[new_hnode] %s (%d)",_("Warning: String exceeds storage size"),(int)strlen(str));
         if (debug_mode)
            fprintf(stderr,":\n--> %s",str);
         fprintf(stderr,"\n");
      }
      str[MAXHOST-1]=0;
   }

   if ( (sptr=malloc(strlen(str)+1))==NULL ) return (HNODEPTR)NULL;
   strcpy(sptr,str);

   if (( newptr = malloc(sizeof(struct hnode))) != NULL)
   {
      newptr->string    =sptr;
      newptr->visit     =0;
      newptr->tstamp    =0;
      newptr->lasturl   =blank_str;
   }
   else free(sptr);
   return newptr;
}

/*********************************************/
/* PUT_HNODE - insert/update host node       */
/*********************************************/

int put_hnode( char     *str,   /* Hostname  */
               int       type,  /* obj type  */
               u_long    count, /* hit count */
               u_long    file,  /* File flag */
               double    xfer,  /* xfer size */
               u_long   *ctr,   /* counter   */
               u_long    visit, /* visits    */
               u_long    tstamp,/* timestamp */
               char     *lasturl, /* lasturl */
               HNODEPTR *htab)  /* ptr>next  */
{
   HNODEPTR cptr,nptr;
   u_long hvalue = hash(str) & (MAXHASH - 1);

   /* check if hashed */
   if ( (cptr = htab[hvalue]) == NULL)
   {
      /* not hashed */
      if ( (nptr=new_hnode(str)) != NULL)
      {
         nptr->flag  = type;
         nptr->count = count;
         nptr->files = file;
         nptr->xfer  = xfer;
         nptr->next  = NULL;
         htab[hvalue] = nptr;
         if (type!=OBJ_GRP) (*ctr)++;

         if (visit)
         {
            nptr->visit=(visit-1);
            nptr->lasturl=find_url(lasturl);
            nptr->tstamp=tstamp;
            return 0;
         }
         else
         {
            if (ispage(log_rec.url))
            {
               if (htab==sm_htab) update_entry(log_rec.url);
               nptr->lasturl=find_url(log_rec.url);
               nptr->tstamp=tstamp;
               nptr->visit=1;
            }
         }
      }
   }
   else
   {
      /* hashed */
      while (cptr != NULL)
      {
         if (strcmp(cptr->string,str)==0)
         {
            if ((type==cptr->flag)||((type!=OBJ_GRP)&&(cptr->flag!=OBJ_GRP)))
            {
               /* found... bump counter */
               cptr->count+=count;
               cptr->files+=file;
               cptr->xfer +=xfer;

               if (ispage(log_rec.url))
               {
                  if ((tstamp-cptr->tstamp)>=visit_timeout)
                  {
                     cptr->visit++;
                     if (htab==sm_htab)
                     {
                        update_exit(cptr->lasturl);
                        update_entry(log_rec.url);
                     }
                  }
                  cptr->lasturl=find_url(log_rec.url);
                  cptr->tstamp=tstamp;
               }
               return 0;
            }
         }
         cptr = cptr->next;
      }
      /* not found... */
      if ( (nptr = new_hnode(str)) != NULL)
      {
         nptr->flag  = type;
         nptr->count = count;
         nptr->files = file;
         nptr->xfer  = xfer;
         nptr->next  = htab[hvalue];
         htab[hvalue]=nptr;
         if (type!=OBJ_GRP) (*ctr)++;

         if (visit)
         {
            nptr->visit = (visit-1);
            nptr->lasturl=find_url(lasturl);
            nptr->tstamp= tstamp;
            return 0;
         }
         else
         {
            if (ispage(log_rec.url))
            {
               if (htab==sm_htab) update_entry(log_rec.url);
               nptr->lasturl=find_url(log_rec.url);
               nptr->tstamp= tstamp;
               nptr->visit=1;
            }
         }
      }
   }

   if (nptr!=NULL)
   {
      /* set object type */
      if (type==OBJ_GRP) nptr->flag=OBJ_GRP;            /* is it a grouping? */
      else
      {
         /* check if it's a hidden object */
         if ((hide_sites)||(isinlist(hidden_sites,nptr->string)!=NULL))
           nptr->flag=OBJ_HIDE;
      }
   }
   return nptr==NULL;
}

/*********************************************/
/* DEL_HLIST - delete host hash table        */
/*********************************************/

void	del_hlist(HNODEPTR *htab)
{
   /* free memory used by hash table */
   HNODEPTR aptr,temp;
   int i;

   for (i=0;i<MAXHASH;i++)
   {
      if (htab[i] != NULL)
      {
         aptr = htab[i];
         while (aptr != NULL)
         {
            temp = aptr->next;
            free (aptr->string);    /* free hostname string space */
            free (aptr);            /* free hostname structure    */
            aptr = temp;
         }
         htab[i]=NULL;
      }
   }
}

/*********************************************/
/* NEW_UNODE - URL node creation             */
/*********************************************/

UNODEPTR new_unode(char *str)
{
   UNODEPTR newptr;
   char     *sptr;

   if (strlen(str) >= MAXURLH)
   {
      if (verbose)
      {
         fprintf(stderr,"[new_unode] %s (%d)",_("Warning: String exceeds storage size"),(int)strlen(str));
         if (debug_mode)
            fprintf(stderr,":\n--> %s",str);
         fprintf(stderr,"\n");
      }
      str[MAXURLH-1]=0;
   }

   if ( (sptr=malloc(strlen(str)+1))==NULL) return (UNODEPTR)NULL;
   strcpy(sptr,str);

   if (( newptr = malloc(sizeof(struct unode))) != NULL)
   {
      newptr->string=sptr;
      newptr->count = 0;
      newptr->flag  = OBJ_REG;
      /* This one is unused for now.
         TODO : check why there is a 'files' member in the unode structure,
         and maybe delete it. We have to think twice, as that will break
         incremental file compatibility */
      newptr->files = 0;
   }
   else free(sptr);
   return newptr;
}

/*********************************************/
/* PUT_UNODE - insert/update URL node        */
/*********************************************/

int put_unode(char *str, int type, u_long count, double xfer,
              u_long *ctr, u_long entry, u_long exit, UNODEPTR *htab)
{
   UNODEPTR cptr,nptr;
   u_long hvalue = hash(str) & (MAXHASH - 1);

   if (str[0]=='-') return 0;

   /* check if hashed */
   if ( (cptr = htab[hvalue]) == NULL)
   {
      /* not hashed */
      if ( (nptr=new_unode(str)) != NULL)
      {
         nptr->flag = type;
         nptr->count= count;
         nptr->xfer = xfer;
         nptr->next = NULL;
         nptr->entry= entry;
         nptr->exit = exit;
         htab[hvalue] = nptr;
         if (type!=OBJ_GRP) (*ctr)++;
      }
   }
   else
   {
      /* hashed */
      while (cptr != NULL)
      {
         if (strcmp(cptr->string,str)==0)
         {
            if ((type==cptr->flag)||((type!=OBJ_GRP)&&(cptr->flag!=OBJ_GRP)))
            {
               /* found... bump counter */
               cptr->count+=count;
               cptr->xfer += xfer;
               return 0;
            }
         }
         cptr = cptr->next;
      }
      /* not found... */
      if ( (nptr = new_unode(str)) != NULL)
      {
         nptr->flag = type;
         nptr->count= count;
         nptr->xfer = xfer;
         nptr->next = htab[hvalue];
         nptr->entry= entry;
         nptr->exit = exit;
         htab[hvalue]=nptr;
         if (type!=OBJ_GRP) (*ctr)++;
      }
   }
   if (nptr!=NULL)
   {
      if (type==OBJ_GRP) nptr->flag=OBJ_GRP;
      else if (isinlist(hidden_urls,nptr->string)!=NULL)
                         nptr->flag=OBJ_HIDE;
   }
   return nptr==NULL;
}

/*********************************************/
/* DEL_ULIST - delete URL hash table         */
/*********************************************/

void	del_ulist(UNODEPTR *htab)
{
   /* free memory used by hash table */
   UNODEPTR aptr,temp;
   int i;

   for (i=0;i<MAXHASH;i++)
   {
      if (htab[i] != NULL)
      {
         aptr = htab[i];
         while (aptr != NULL)
         {
            temp = aptr->next;
            free (aptr->string);  /* free up URL string memory */
            free (aptr);          /* free up URL struct node   */
            aptr = temp;
         }
         htab[i]=NULL;
      }
   }
}

/*********************************************/
/* NEW_RNODE - Referrer node creation        */
/*********************************************/

RNODEPTR new_rnode(char *str)
{
   RNODEPTR newptr;
   char     *sptr;

   if (strlen(str) >= MAXREFH)
   {
      if (verbose)
      {
         fprintf(stderr,"[new_rnode] %s (%d)",_("Warning: String exceeds storage size"),(int)strlen(str));
         if (debug_mode)
            fprintf(stderr,":\n--> %s",str);
         fprintf(stderr,"\n");
      }
      str[MAXREFH-1]=0;
   }

   if ( (sptr=malloc(strlen(str)+1))==NULL ) return (RNODEPTR)NULL;
   strcpy(sptr,str);

   if (( newptr = malloc(sizeof(struct rnode))) != NULL)
   {
      newptr->string= sptr;
      newptr->count = 1;
      newptr->flag  = OBJ_REG;
   }
   else free(sptr);
   return newptr;
}

/*********************************************/
/* PUT_RNODE - insert/update referrer node   */
/*********************************************/

int put_rnode(char *str, int type, u_long count, u_long *ctr, RNODEPTR *htab)
{
   RNODEPTR cptr,nptr;
   u_long hvalue = hash(str) & (MAXHASH - 1);

   if (str[0]=='-') strcpy(str,"- (Direct Request)");

   /* check if hashed */
   if ( (cptr = htab[hvalue]) == NULL)
   {
      /* not hashed */
      if ( (nptr=new_rnode(str)) != NULL)
      {
         nptr->flag  = type;
         nptr->count = count;
         nptr->next  = NULL;
         htab[hvalue] = nptr;
         if (type!=OBJ_GRP) (*ctr)++;
      }
   }
   else
   {
      /* hashed */
      while (cptr != NULL)
      {
         if (strcmp(cptr->string,str)==0)
         {
            if ((type==cptr->flag)||((type!=OBJ_GRP)&&(cptr->flag!=OBJ_GRP)))
            {
               /* found... bump counter */
               cptr->count+=count;
               return 0;
            }
         }
         cptr = cptr->next;
      }
      /* not found... */
      if ( (nptr = new_rnode(str)) != NULL)
      {
         nptr->flag  = type;
         nptr->count = count;
         nptr->next  = htab[hvalue];
         htab[hvalue]=nptr;
         if (type!=OBJ_GRP) (*ctr)++;
      }
   }
   if (nptr!=NULL)
   {
      if (type==OBJ_GRP) nptr->flag=OBJ_GRP;
      else if (isinlist(hidden_refs,nptr->string)!=NULL)
                         nptr->flag=OBJ_HIDE;
   }
   return nptr==NULL;
}

/*********************************************/
/* DEL_RLIST - delete referrer hash table    */
/*********************************************/

void	del_rlist(RNODEPTR *htab)
{
   /* free memory used by hash table */
   RNODEPTR aptr,temp;
   int i;

   for (i=0;i<MAXHASH;i++)
   {
      if (htab[i] != NULL)
      {
         aptr = htab[i];
         while (aptr != NULL)
         {
            temp = aptr->next;
            free (aptr->string);
            free (aptr);
            aptr = temp;
         }
         htab[i]=NULL;
      }
   }
}

/*********************************************/
/* NEW_ANODE - User Agent node creation      */
/*********************************************/

ANODEPTR new_anode(char *str)
{
   ANODEPTR newptr;
   char     *sptr;

   if (strlen(str) >= MAXAGENT)
   {
      if (verbose)
      {
         fprintf(stderr,"[new_anode] %s (%d)",_("Warning: String exceeds storage size"),(int)strlen(str));
         if (debug_mode)
            fprintf(stderr,":\n--> %s",str);
         fprintf(stderr,"\n");
      }
      str[MAXAGENT-1]=0;
   }

   if ( (sptr=malloc(strlen(str)+1))==NULL ) return (ANODEPTR)NULL;
   strcpy(sptr,str);

   if (( newptr = malloc(sizeof(struct anode))) != NULL)
   {
      newptr->string= sptr;
      newptr->count = 1;
      newptr->flag  = OBJ_REG;
   }
   else free(sptr);
   return newptr;
}

/*********************************************/
/* PUT_ANODE - insert/update user agent node */
/*********************************************/

int put_anode(char *str, int type, u_long count, u_long *ctr, ANODEPTR *htab)
{
   ANODEPTR cptr,nptr;
   u_long hvalue = hash(str) & (MAXHASH - 1);

   if (str[0]=='-') return 0;     /* skip bad user agents */

   /* check if hashed */
   if ( (cptr = htab[hvalue]) == NULL)
   {
      /* not hashed */
      if ( (nptr=new_anode(str)) != NULL)
      {
         nptr->flag = type;
         nptr->count= count;
         nptr->next = NULL;
         htab[hvalue] = nptr;
         if (type!=OBJ_GRP) (*ctr)++;
      }
   }
   else
   {
      /* hashed */
      while (cptr != NULL)
      {
         if (strcmp(cptr->string,str)==0)
         {
            if ((type==cptr->flag)||((type!=OBJ_GRP)&&(cptr->flag!=OBJ_GRP)))
            {
               /* found... bump counter */
               cptr->count+=count;
               return 0;
            }
         }
         cptr = cptr->next;
      }
      /* not found... */
      if ( (nptr = new_anode(str)) != NULL)
      {
         nptr->flag  = type;
         nptr->count = count;
         nptr->next  = htab[hvalue];
         htab[hvalue]=nptr;
         if (type!=OBJ_GRP) (*ctr)++;
      }
   }
   if (type==OBJ_GRP) nptr->flag=OBJ_GRP;
   else if (isinlist(hidden_agents,nptr->string)!=NULL)
                      nptr->flag=OBJ_HIDE;
   return nptr==NULL;
}

/*********************************************/
/* DEL_ALIST - delete user agent hash table  */
/*********************************************/

void	del_alist(ANODEPTR *htab)
{
   /* free memory used by hash table */
   ANODEPTR aptr,temp;
   int i;

   for (i=0;i<MAXHASH;i++)
   {
      if (htab[i] != NULL)
      {
         aptr = htab[i];
         while (aptr != NULL)
         {
            temp = aptr->next;
            free (aptr->string);
            free (aptr);
            aptr = temp;
         }
         htab[i]=NULL;
      }
   }
}

/*********************************************/
/* NEW_INODE - create ident (username) node  */
/*********************************************/

INODEPTR new_inode(char *str)
{
   INODEPTR newptr;
   char     *sptr;

   if (strlen(str) >= MAXIDENT)
   {
      if (verbose)
      {
         fprintf(stderr,"[new_inode] %s (%d)",_("Warning: String exceeds storage size"),(int)strlen(str));
         if (debug_mode)
            fprintf(stderr,":\n--> %s",str);
         fprintf(stderr,"\n");
      }
      str[MAXIDENT-1]=0;
   }

   if ( (sptr=malloc(strlen(str)+1))==NULL ) return (INODEPTR)NULL;
   strcpy(sptr,str);

   if (( newptr = malloc(sizeof(struct inode))) != NULL)
   {
      newptr->string    =sptr;
      newptr->visit     =1;
      newptr->tstamp    =0;
   }
   else free(sptr);
   return newptr;
}

/*********************************************/
/* PUT_INODE - insert/update ident node      */
/*********************************************/

int put_inode( char     *str,   /* ident str */
               int       type,  /* obj type  */
               u_long    count, /* hit count */
               u_long    file,  /* File flag */
               double    xfer,  /* xfer size */
               u_long   *ctr,   /* counter   */
               u_long    visit, /* visits    */
               u_long    tstamp,/* timestamp */
               INODEPTR *htab)  /* hashtable */
{
   INODEPTR cptr,nptr;
   u_long hvalue = hash(str) & (MAXHASH - 1);

   if ((str[0]=='-') || (str[0]==0)) return 0;  /* skip if no username */

   /* check if hashed */
   if ( (cptr = htab[hvalue]) == NULL)
   {
      /* not hashed */
      if ( (nptr=new_inode(str)) != NULL)
      {
         nptr->flag  = type;
         nptr->count = count;
         nptr->files = file;
         nptr->xfer  = xfer;
         nptr->next  = NULL;
         htab[hvalue] = nptr;
         if (type!=OBJ_GRP) (*ctr)++;

         if (visit)
         {
            nptr->visit=(visit-1);
            nptr->tstamp=tstamp;
            return 0;
         }
         else
         {
            if (ispage(log_rec.url)) nptr->tstamp=tstamp;
         }
      }
   }
   else
   {
      /* hashed */
      while (cptr != NULL)
      {
         if (strcmp(cptr->string,str)==0)
         {
            if ((type==cptr->flag)||((type!=OBJ_GRP)&&(cptr->flag!=OBJ_GRP)))
            {
               /* found... bump counter */
               cptr->count+=count;
               cptr->files+=file;
               cptr->xfer +=xfer;

               if (ispage(log_rec.url))
               {
                  if ((tstamp-cptr->tstamp)>=visit_timeout)
                     cptr->visit++;
                  cptr->tstamp=tstamp;
               }
               return 0;
            }
         }
         cptr = cptr->next;
      }
      /* not found... */
      if ( (nptr = new_inode(str)) != NULL)
      {
         nptr->flag  = type;
         nptr->count = count;
         nptr->files = file;
         nptr->xfer  = xfer;
         nptr->next  = htab[hvalue];
         htab[hvalue]=nptr;
         if (type!=OBJ_GRP) (*ctr)++;

         if (visit)
         {
            nptr->visit = (visit-1);
            nptr->tstamp= tstamp;
            return 0;
         }
         else
         {
            if (ispage(log_rec.url)) nptr->tstamp= tstamp;
         }
      }
   }

   if (nptr!=NULL)
   {
      /* set object type */
      if (type==OBJ_GRP) nptr->flag=OBJ_GRP;            /* is it a grouping? */
      else
      {
         /* check if it's a hidden object */
         if (isinlist(hidden_users,nptr->string)!=NULL)
           nptr->flag=OBJ_HIDE;
      }
   }
   return nptr==NULL;
}

/*********************************************/
/* DEL_ILIST - delete ident hash table       */
/*********************************************/

void	del_ilist(INODEPTR *htab)
{
   /* free memory used by hash table */
   INODEPTR aptr,temp;
   int i;

   for (i=0;i<MAXHASH;i++)
   {
      if (htab[i] != NULL)
      {
         aptr = htab[i];
         while (aptr != NULL)
         {
            temp = aptr->next;
            free (aptr->string);    /* free ident string space */
            free (aptr);            /* free ident structure    */
            aptr = temp;
         }
         htab[i]=NULL;
      }
   }
}

#ifdef USE_DNS   /* only add these for DNS   */

/*********************************************/
/* NEW_DNODE - DNS resolver node creation    */
/*********************************************/

DNODEPTR new_dnode(char *str)
{
   DNODEPTR newptr;
   char     *sptr;

   if (strlen(str) >= MAXHOST)
   {
      if (verbose)
      {
         fprintf(stderr,"[new_dnode] %s (%d)",_("Warning: String exceeds storage size"),(int)strlen(str));
         if (debug_mode)
            fprintf(stderr,":\n--> %s",str);
         fprintf(stderr,"\n");
      }
      str[MAXHOST-1]=0;
   }

   if ( (sptr=malloc(strlen(str)+1))==NULL ) return (DNODEPTR)NULL;
   strcpy(sptr,str);

   if (( newptr = malloc(sizeof(struct dnode))) != NULL)
   {
      newptr->string= sptr;
   }
   else free(sptr);
   return newptr;
}

/*********************************************/
/* PUT_DNODE - insert/update dns host node   */
/*********************************************/

#ifdef USE_IPV6
int put_dnode(char *str, struct sockaddr_storage *addr, DNODEPTR *htab)
#else
int put_dnode(char *str, struct in_addr *addr, DNODEPTR *htab)
#endif
{
   DNODEPTR cptr,nptr;
   u_long hvalue = hash(str) & (MAXHASH - 1);

   if (str[0]==0 || str[0]==' ') return 0;     /* skip bad hostnames */

   /* check if hashed */
   if ( (cptr = htab[hvalue]) == NULL)
   {
      /* not hashed */
      if ( (nptr=new_dnode(str)) != NULL)
      {
         #ifdef USE_IPV6
         if (addr) memcpy(&nptr->addr, addr, sizeof(struct sockaddr_storage));
            else   memset(&nptr->addr, 0, sizeof(struct sockaddr_storage));
         #else
         if (addr) memcpy(&nptr->addr, addr, sizeof(struct in_addr));
            else   memset(&nptr->addr, 0, sizeof(struct in_addr));
         #endif
         nptr->next = NULL;
         htab[hvalue] = nptr;
      }
   }
   else
   {
      /* hashed */
      while (cptr != NULL)
      {
         if (strcmp(cptr->string,str)==0) return 0;
         cptr = cptr->next;
      }
      /* not found... */
      if ( (nptr = new_dnode(str)) != NULL)
      {
         #ifdef USE_IPV6
         if (addr) memcpy(&nptr->addr, addr, sizeof(struct sockaddr_storage));
            else   memset(&nptr->addr, 0, sizeof(struct sockaddr_storage));
         #else
         if (addr) memcpy(&nptr->addr, addr, sizeof(struct in_addr));
            else   memset(&nptr->addr, 0, sizeof(struct in_addr));
         #endif
         nptr->next  = htab[hvalue];
         htab[hvalue]=nptr;
      }
   }
   return nptr==NULL;
}

/*********************************************/
/* DEL_DLIST - delete dns hash table         */
/*********************************************/

void	del_dlist(DNODEPTR *htab)
{
   /* free memory used by hash table */
   DNODEPTR dptr,temp;
   int i;

   for (i=0;i<MAXHASH;i++)
   {
      if (htab[i] != NULL)
      {
         dptr = htab[i];
         while (dptr != NULL)
         {
            temp = dptr->next;
            free (dptr->string);
            free (dptr);
            dptr = temp;
         }
         htab[i]=NULL;
      }
   }
}

#endif /* USE_DNS */

/*********************************************/
/* NEW_OPNODE - create open path node        */
/*********************************************/

/* No deep copies. Fields are pointers to
   respective node items
*/

OPNODEPTR new_opnode(char *hostname,
                     char *search_string,
                     char *user,
                     char *referrer,
                     char *url
                     )
{
   OPNODEPTR newptr;

   if (( newptr = malloc(sizeof(struct opnode))) != NULL)
   {
      LISTPTR path;

      newptr->hostname       = find_hostname(hostname);
      newptr->search_string  = ""; /* unused for now */
      newptr->user           = find_user(user);
      newptr->referrer       = find_referrer(referrer);
      newptr->tstamp         = 0;
      newptr->hash           = progressive_hash(url, 0);

      path = new_list(find_url(url), 0); /* no deep copy */

      if(path == NULL)
      {
         free(newptr);
         newptr = 0;
      }
      else
        newptr->path         = path;
   }
   return newptr;
}

/*********************************************/
/* ADD_OPITEM - add url to open path         */
/*********************************************/

/*
   returns:
      0 on success
      1 on failure
*/

int    add_opitem(char *hostname,
                  char *search_string,
                  char *user,
                  char *referrer,
                  char *url,
                  u_long tstamp,
                  OPNODEPTR * htab)
{
   OPNODEPTR cptr, nptr;
   u_long hvalue = hash(hostname) & (MAXHASH - 1);

   if( (cptr = htab[hvalue]) == NULL )
   {
      /* path not opened, open it now! */
      if( (nptr = new_opnode(hostname, search_string,
                            user, referrer, url)) != NULL )
      {
         nptr->next     = NULL;
         nptr->tstamp   = tstamp;
         htab[hvalue]  = nptr;
      }
      else
      {
         if(verbose)
            fprintf(stderr,"[add_opitem] %s \n",
               _("Warning: out of memory - could not create open path"));
         return 1;
      }
   }
   else
   {
      /* hashed */
      while (cptr != NULL)
      {
         if(strcmp(hostname, cptr->hostname) == 0)
         {
            /* We found our opened path! */
            if((tstamp - cptr->tstamp) >= visit_timeout)
            {
               /* we consider the visit is too old, let's begin a new path */
               if (flush_opnode(hvalue, hostname))
               {
                  if (verbose)
                     /* Error flushing node, skipping .... */
                     fprintf(stderr,"%s %s %s...\n",
                           _("Error flushing path, skipping"), cptr->hostname, (char*)cptr->path->item);
                  return 1;
               }
               if( (nptr = new_opnode(hostname, search_string,
                                    user, referrer, url)) != NULL )
               {
                  nptr->next     = htab[hvalue];
                  nptr->tstamp   = tstamp;
                  htab[hvalue]  = nptr;
               }
               else
               {
                  if(verbose)
                     fprintf(stderr,"[add_opitem] %s \n",
                        _("Warning: out of memory - could not create open path"));
                  return 1;
               }
            }
            else
            {
               /*
                  We add the url to the path, the path is growing!
               */

               /* get the last url of the list */
               LISTPTR plptr = 0, lptr = cptr->path;
               while(lptr != NULL)
               {
                  plptr = lptr;
                  lptr = lptr->next;
               }

               /* only add our url if it differs from the last url from the list */
               if(strcmp((char*)plptr->item, url) != 0) {
                  if(add_list(&plptr, find_url(url), 0) != 0)  /* no deep copy */
                  {
                     if(verbose)
                        fprintf(stderr,"[add_opitem] %s \n",
                           _("Warning: out of memory - could not add url to open path"));
                     return 1;
                  }
                  else
                     cptr->hash = progressive_hash(url, cptr->hash);
               }
               else {
                  static OPNODEPTR temp;
                  temp = cptr;
               }
            }
            return 0;
         }
         cptr = cptr->next;
      }
      /* not found... */
      if ( (nptr = new_opnode(hostname, search_string,
                            user, referrer, url)) != NULL )
      {
         nptr->tstamp    = tstamp;
         nptr->next      = htab[hvalue];
         htab[hvalue] = nptr;
      }
   }
   return nptr==NULL;
}

/*********************************************/
/* FLUSH_OPNODE                              */
/*********************************************/

/* Takes path out of the open path htab,
   and puts it into the global path htab,
   and cuts it into small pieces to put in
   the flow htab
   Does nothing if index'ed value is null
   If hostname null, flushes the first path
   found at position 'index' */

int flush_opnode(u_long index, char *hostname)
{
   /* find path to remove */
   OPNODEPTR ptr = op_htab[index], pptr = 0;

   if(hostname) /* find in list */
   {
      while(ptr != NULL)
      {
         if(strcmp(ptr->hostname, hostname) == 0)
            break;
         else
         {
            pptr = ptr;
            ptr = ptr->next;
         }
      }
   }

   if(ptr != NULL)
   {
      /* we don't keep path of length < 1 */
      if(ptr->path->next != NULL)
      {
         LISTPTR plptr, lptr;
         LISTPTR flow;
         u_long flowhash;

         /* truncate hash */
         u_long hash = ptr->hash % MAXHASH;

         /* cuts path down into small pieces and inserts it in fm_htab */
         plptr = ptr->path;
         lptr = plptr->next;
         while(lptr != NULL)
         {
            /* compute hash - it is based only on the source*/
            flowhash = progressive_hash((char *) plptr->item, 0);

            /* create flow item */
            flow = new_list(plptr->item, 0);
            add_list(&flow, lptr->item, 0);

            put_pnode(flow, NULL, 1, flowhash % MAXHASH, fm_htab);

            plptr = lptr;
            lptr = lptr->next;
         }

         /* put path in global hash table -- this will free the path
            if it is duplicated */
         put_pnode(ptr->path, NULL, 1, hash, gp_htab);
      }
      else
        del_list(&ptr->path, 0); /* no deep destroy */

      /* remove node from open path htab*/
      if(pptr != NULL)
         pptr->next = ptr->next;
      else
         op_htab[index] = ptr->next;

      /* free node */
      free(ptr);
      return 0;
   }
   else
      return 1;
}

/*********************************************/
/* FLUSH_OPHTAB                              */
/*********************************************/

/* flushes old (i.e. older than visit_timeout)
   nodes from the open path hash table

   rec_tstamp: last record timestamp. With this
               we can day if an opened path is old enough
               to be flushed.
               If = 0, force flush.
 */

void flush_ophtab(u_long rec_tstamp)
{
   /* close paths of open path hash table based on last given timestamp */
   int i;
   for(i=0; i<MAXHASH; i++) {
      OPNODEPTR op = op_htab[i];
      OPNODEPTR nextop;
      while(op != NULL)
      {
         nextop = op->next; /* we need to keep next item here because
                              the op structure is freed in case of
                              a flush_opnode call */
         if((rec_tstamp == 0) || ((rec_tstamp - op->tstamp) >= visit_timeout))
         {
            /* if too old we flush */
            if (flush_opnode(i, op->hostname))
            {
               if (verbose)
                  /* Error flushing open path, skipping .... */
                  fprintf(stderr,"%s %s %s...\n",
                     _("Error flushing path, skipping"), op->hostname, (char*)op->path->item);
            }
         }
         op = nextop;
      }
   }
}

/*********************************************/
/* DEL_OPLIST - delete opened path htabs     */
/*********************************************/

void del_oplist(OPNODEPTR *htab, int delpath)
{
   /* free memory used by hash table */
   OPNODEPTR aptr,temp;
   int i;

   for (i=0;i<MAXHASH;i++)
   {
      if (htab[i] != NULL)
      {
         aptr = htab[i];
         while (aptr != NULL)
         {
            temp = aptr->next;

            if(delpath != 0)
               /* we free the list - but not the pointed elements */
               del_list(&aptr->path, 0);

            free (aptr);                /* free path structure    */
            aptr = temp;
         }
         htab[i]=NULL;
      }
   }
}


/*********************************************/
/* NEW_PNODE - creates generic path node     */
/*********************************************/

PNODEPTR new_pnode(LISTPTR path, char *key, int count)
{
   PNODEPTR newptr;

   if ( (newptr = malloc(sizeof(struct pnode))) != NULL)
   {
      newptr->key   = key;
      newptr->path  = path;
      newptr->count = count;
   }
   return newptr;
}

/*********************************************/
/* PUT_PNODE - adds path node into given htab*/
/*********************************************/

/* key will be the (possibly null) string
   value attached to path node

   count is what will be used to initialize the
   path if it is not already in the htab
*/

int put_pnode(LISTPTR path, char *key, int count, u_long hashvalue, PNODEPTR *htab)
{
   PNODEPTR cptr,nptr;

   /* check if hashed */
   if ( (cptr = htab[hashvalue]) == NULL)
   {
      /* not hashed */
      if ( (nptr=new_pnode(path, key, count)) != NULL)
      {
         nptr->next = NULL;
         htab[hashvalue] = nptr;
      }
      else
      {
         if(verbose)
            fprintf(stderr,"[put_pnode] %s \n",
             _("Warning: out of memory - could not put path to path list"));
      }
   }
   else
   {
      /* hashed */
      while (cptr != NULL)
      {
         /* try to find if this path has already been stored */
         if((key == cptr->key) ||
               (strcmp(cptr->key, key) == 0))
         {
            /*key is equal, now check path */
            LISTPTR lptr1 = cptr->path;
            LISTPTR lptr2 = path;

            while( (lptr1 != 0) && (lptr2 != 0))
            {
               if(strcmp((char *)lptr1->item, (char *)lptr2->item) != 0)
                 break;
               else
               {
                  lptr1 = lptr1->next;
                  lptr2 = lptr2->next;
               }
            }
            if(lptr1 == lptr2)
            {
               /* both null... path match!*/
               cptr->count += count;
               /* don't forget to delete path! (thank you Valgrind) */
               del_list(&path, 0);
               return 0;
            }
         }
         cptr = cptr->next;
      }
      /* not found... */
      if ( (nptr = new_pnode(path, key, count)) != NULL)
      {
         nptr->next  = htab[hashvalue];
         htab[hashvalue] = nptr;
      }
      else {
         if(verbose)
            fprintf(stderr,"[put_pnode] %s \n",
             _("Warning: out of memory - could not put path to path list"));
      }

   }
   return nptr==NULL;
}

/*********************************************/
/* DEL_PLIST - delete path hash tables       */
/*********************************************/

/* This is a generic function meant to free
   path related hash tables.
   It's ok as long as:
   - items pointed by pnode
     fields are not to be freed by the
     del_plist function (expect path
     if delpath != 0)
   - first item of the xxpnode structs is the
     pointer on the next item
   - second item is a pointer on the path
*/

void	del_plist(PNODEPTR *htab, int delpath)
{
   /* free memory used by hash table */
   PNODEPTR aptr,temp;
   int i;

   for (i=0;i<MAXHASH;i++)
   {
      if (htab[i] != NULL)
      {
         aptr = htab[i];
         while (aptr != NULL)
         {
            temp = aptr->next;

            if(delpath != 0)
               /* we free the list - but not the pointed elements */
               del_list(&aptr->path, 0);

            free (aptr);                /* free path structure    */
            aptr = temp;
         }
         htab[i]=NULL;
      }
   }
}

/*********************************************/
/* FIND_URL - Find URL in hash table         */
/*********************************************/

char *find_url(char *str)
{
   UNODEPTR ptr = find_url_node(str);

   if ( ptr != NULL )
      return ptr->string;
   else
      return blank_str;   /* shouldn't get here */
}

/*********************************************/
/* FIND_URL_NODE - Find URL node in htab     */
/*********************************************/

UNODEPTR find_url_node(char *str)
{
   UNODEPTR cptr;

   if ( (cptr=um_htab[hash(str) & (MAXHASH - 1)]) != NULL)
   {
      while (cptr != NULL)
      {
         if (strcmp(cptr->string,str)==0)
            return cptr;
         cptr = cptr->next;
      }
   }
   return NULL;   /* shouldn't get here */
}

/*********************************************/
/* FIND_HOSTNAME - Find hname in hash table  */
/*********************************************/

char *find_hostname(char *str)
{
   HNODEPTR cptr;

   if ( (cptr=sm_htab[hash(str) & (MAXHASH - 1)]) != NULL)
   {
      while (cptr != NULL)
      {
         if (strcmp(cptr->string,str)==0)
            return cptr->string;
         cptr = cptr->next;
      }
   }
   return blank_str;   /* shouldn't get here */
}

/*********************************************/
/* FIND_USER                                 */
/* Find user string in hash table            */
/*********************************************/

char *find_user(char *str)
{
   ANODEPTR cptr;

   if ( (cptr=am_htab[hash(str) & (MAXHASH - 1)]) != NULL)
   {
      while (cptr != NULL)
      {
         if (strcmp(cptr->string,str)==0)
            return cptr->string;
         cptr = cptr->next;
      }
   }
   return blank_str;   /* shouldn't get here */
}

/*********************************************/
/* FIND_REFERRER                             */
/* Find referrer string in hash table        */
/*********************************************/

char *find_referrer(char *str)
{
   RNODEPTR cptr;

   if ( (cptr=rm_htab[hash(str) & (MAXHASH - 1)]) != NULL)
   {
      while (cptr != NULL)
      {
         if (strcmp(cptr->string,str)==0)
            return cptr->string;
         cptr = cptr->next;
      }
   }
   return blank_str;   /* shouldn't get here */
}

/*********************************************/
/* UPDATE_ENTRY - update entry page total    */
/*********************************************/

void update_entry(char *str)
{
   UNODEPTR uptr;

   if (str==NULL) return;
   if ( (uptr = um_htab[hash(str) & (MAXHASH - 1)]) == NULL) return;
   else
   {
      while (uptr != NULL)
      {
         if (strcmp(uptr->string,str)==0)
         {
            if (uptr->flag!=OBJ_GRP)
            {
               uptr->entry++;
               return;
            }
         }
         uptr=uptr->next;
      }
   }
}

/*********************************************/
/* UPDATE_EXIT  - update exit page total     */
/*********************************************/

void update_exit(char *str)
{
   UNODEPTR uptr;

   if (str==NULL) return;
   if ( (uptr = um_htab[hash(str) & (MAXHASH - 1)]) == NULL) return;
   else
   {
      while (uptr != NULL)
      {
         if (strcmp(uptr->string,str)==0)
         {
            if (uptr->flag!=OBJ_GRP)
            {
               uptr->exit++;
               return;
            }
         }
         uptr=uptr->next;
      }
   }
}

/*********************************************/
/* MONTH_UPDATE_EXIT  - eom exit page update */
/*********************************************/

void month_update_exit(u_long tstamp)
{
   HNODEPTR nptr;
   int i;

   for (i=0;i<MAXHASH;i++)
   {
      nptr=sm_htab[i];
      while (nptr!=NULL)
      {
         if (nptr->flag!=OBJ_GRP)
         {
            if ((tstamp-nptr->tstamp)>=visit_timeout)
               update_exit(nptr->lasturl);
         }
         nptr=nptr->next;
      }
   }
}

/*********************************************/
/* TOT_VISIT - calculate total visits        */
/*********************************************/

u_long tot_visit(HNODEPTR *list)
{
   HNODEPTR   hptr;
   u_long     tot=0;
   int        i;

   for (i=0;i<MAXHASH;i++)
   {
      hptr=list[i];
      while (hptr!=NULL)
      {
         if (hptr->flag!=OBJ_GRP) tot+=hptr->visit;
         hptr=hptr->next;
      }
   }
   return tot;
}
