/*
   Copyright (c) 2002-7, Andrew McNab, University of Manchester
   All rights reserved.

   Redistribution and use in source and binary forms, with or
   without modification, are permitted provided that the following
   conditions are met:

     o Redistributions of source code must retain the above
       copyright notice, this list of conditions and the following
       disclaimer. 
     o Redistributions in binary form must reproduce the above
       copyright notice, this list of conditions and the following
       disclaimer in the documentation and/or other materials
       provided with the distribution. 

   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
   CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
   INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
   MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
   BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
   ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   POSSIBILITY OF SUCH DAMAGE.
*/

/*---------------------------------------------------------------*
 * For more about GridSite: http://www.gridsite.org/             *
 *---------------------------------------------------------------*/

#ifndef VERSION
#define VERSION "0.0.0"
#endif

#define _GNU_SOURCE

#include <pwd.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <getopt.h>
#include <string.h>
#include <dirent.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/time.h>
#include <sys/select.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/types.h>
#include <curl/curl.h>
#include <errno.h>
#include <netdb.h>
#include <ctype.h>

#include "gridsite.h"

#define HTCP_GET	1
#define HTCP_PUT	2
#define HTCP_DELETE	3
#define HTCP_LIST	4
#define HTCP_LONGLIST	5
#define HTCP_MKDIR	6
#define HTCP_MOVE	7
#define HTCP_PING	8
#define HTCP_FIND	9
#define HTCP_RMTCP	10

#define HTCP_SITECAST_GROUPS 32

#define HTCP_HOST_CONF       "/etc/htcp.conf"
#define HTCP_USER_CONF       ".htcp.conf"

struct grst_stream_data { char *source;
                          char *destination;
                          int   ishttps;
                          int   method;
                          FILE *fp;
                          char *cert;
                          char *key;
                          char *capath;
                          char *useragent;
                          char *errorbuf;
                          int   noverify;
                          int   anonymous;
                          int   gridhttp;
                          int   verbose;	
                          int   timeout;
                          char *groups;
                          int   sitecast;
                          char *domain;    } ;
                          
struct grst_index_blob { char   *text;
                         size_t  used;
                         size_t  allocated; } ;

struct grst_dir_list { char   *filename;
                       size_t  length;
                       int     length_set;
                       time_t  modified;
                       int     modified_set; } ; 

struct grst_header_data { int    retcode;                         
                          char  *location;
                          char  *gridhttppasscode;
                          size_t length;
                          int    length_set;
                          time_t modified;                           
                          int    modified_set;
                          struct grst_stream_data *common_data; } ;

struct grst_sitecast_group { struct addrinfo *ai;
                             int timewait; int ttl; };

size_t headers_callback(void *ptr, size_t size, size_t nmemb, void *p)
/* Find the values of the return code, Content-Length, Last-Modified
   and Location headers */
{
  float f;
  char  *s, *q;
  size_t realsize;
  struct tm modified_tm;
  struct grst_header_data *header_data;
   
  header_data = (struct grst_header_data *) p;
  realsize = size * nmemb;
  s = malloc(realsize + 1);
  memcpy(s, ptr, realsize);
  s[realsize] = '\0';

  if      (sscanf(s, "Content-Length: %zd", &(header_data->length)) == 1)
            header_data->length_set = 1;
  else if (sscanf(s, "HTTP/%f %d ", &f, &(header_data->retcode)) == 2) ;
  else if (strncmp(s, "Location: ", 10) == 0) 
      {
        header_data->location = strdup(&s[10]);
        
        for (q=header_data->location; *q != '\0'; ++q)
         if ((*q == '\r') || (*q == '\n')) *q = '\0';
         
        if (header_data->common_data->verbose > 0)
             fprintf(stderr, "Received Location: %s\n", header_data->location);
      }
  else if (strncmp(s, "Set-Cookie: GRIDHTTP_PASSCODE=", 29) == 0) 
      {
        header_data->gridhttppasscode = strdup(&s[12]);
        q = index(header_data->gridhttppasscode, ';');
        if (q != NULL) *q = '\0';       

        if (header_data->common_data->verbose > 0)
             fprintf(stderr, "Received GridHTTP Auth Cookie: %s\n", 
                             header_data->gridhttppasscode);
      }
  else if (strncmp(s, "Last-Modified: ", 15) == 0)
      {
        /* follow RFC 2616: first try RFC 822 (kosher), then RFC 850 and 
           asctime() formats too. Must be GMT whatever the format. */

        if (strptime(&s[15], "%a, %d %b %Y %T GMT", &modified_tm) != NULL)
          {
            header_data->modified = mktime(&modified_tm);
            header_data->modified_set = 1;
          }
        else if (strptime(&s[15], "%a, %d-%b-%y %T GMT", &modified_tm) != NULL)
          {
            header_data->modified = mktime(&modified_tm);
            header_data->modified_set = 1;
          }
        else if (strptime(&s[15], "%a %b %d %T %Y", &modified_tm) != NULL)
          {
            header_data->modified = mktime(&modified_tm);
            header_data->modified_set = 1;
          }
      }
    
  free(s);
  return realsize;
}

int set_std_opts(CURL *easyhandle, struct grst_stream_data *common_data)
{
  struct stat statbuf;

  curl_easy_setopt(easyhandle, CURLOPT_FOLLOWLOCATION, 0);

  if ((common_data->cert != NULL) && (common_data->key != NULL))
    {
       curl_easy_setopt(easyhandle, CURLOPT_SSLENGINE,   NULL);
       curl_easy_setopt(easyhandle, CURLOPT_SSLCERTTYPE, "PEM");
       curl_easy_setopt(easyhandle, CURLOPT_SSLCERT,     common_data->cert);
       curl_easy_setopt(easyhandle, CURLOPT_SSLKEY,      common_data->key);
    }
  else
    {
       curl_easy_setopt(easyhandle, CURLOPT_SSLENGINE,   "RSA");
       curl_easy_setopt(easyhandle, CURLOPT_SSLCERTTYPE, "ENG");
    }

  if (common_data->capath != NULL)
    {
#if (LIBCURL_VERSION_NUM >= 0x070908)
       if ((stat(common_data->capath, &statbuf) == 0) &&
           S_ISDIR(statbuf.st_mode))
            curl_easy_setopt(easyhandle, CURLOPT_CAPATH, common_data->capath);
       else 
#endif       
            curl_easy_setopt(easyhandle, CURLOPT_CAINFO, common_data->capath);
    }

  if (common_data->noverify)
    {
      curl_easy_setopt(easyhandle, CURLOPT_SSL_VERIFYPEER, 0);
      curl_easy_setopt(easyhandle, CURLOPT_SSL_VERIFYHOST, 0);
    }      
  else 
    {
      curl_easy_setopt(easyhandle, CURLOPT_SSL_VERIFYPEER, 2);
      curl_easy_setopt(easyhandle, CURLOPT_SSL_VERIFYHOST, 2);
    }
  
  return 1;
}

int do_rmtcp(char *sources[], char *destination,
	     struct grst_stream_data *common_data)
{
  CURL *easyhandle;
  char        *p, *thisdestination;
  int          isrc, anyerror = 0, thiserror, isdirdest;
  struct       grst_header_data header_data;
  struct curl_slist *gh_header_slist=NULL, *nogh_header_slist=NULL;
  char remoteserver[255];

  easyhandle = curl_easy_init();
  if( !easyhandle )
    {
      fprintf(stderr, "Cannot initialize CURL handle while preparing to copy file.\n");
      exit(-1);
    }

  common_data->gridhttp = 1; // for debug purpose
  if (common_data->gridhttp)
    {               
      asprintf(&p, "Upgrade: GridHTTP/1.0");
      gh_header_slist = curl_slist_append(gh_header_slist, p);
      free(p);
      
      nogh_header_slist = curl_slist_append(nogh_header_slist, "Upgrade:");
    }
  
  //  common_data->verbose = 1;   //for debug purpose
  curl_easy_setopt(easyhandle, CURLOPT_USERAGENT, common_data->useragent);
  if (common_data->verbose > 1)
                   curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1);

  curl_easy_setopt(easyhandle, CURLOPT_HEADERFUNCTION, headers_callback);
  curl_easy_setopt(easyhandle, CURLOPT_WRITEHEADER,   &header_data);

  set_std_opts(easyhandle, common_data);

  curl_easy_setopt(easyhandle, CURLOPT_ERRORBUFFER, common_data->errorbuf);

  if (destination[strlen(destination) - 1] != '/') 
    {
      isdirdest = 0;
      thisdestination = destination;
    }
  else isdirdest = 1;

  for (isrc=0; sources[isrc] != NULL; ++isrc)
     {
       if (isdirdest)
         {
           p = rindex(sources[isrc], '/');
           if (p == NULL) p = sources[isrc];
           else           p++;

           asprintf(&thisdestination, "%s%s", destination, p);
         }
 
       if( strncmp(sources[isrc], "https://", 8) == 0 ){
	 if (common_data->verbose > 0)
	   fprintf(stderr, "%s -> %s\n", sources[isrc], thisdestination);

	 curl_easy_setopt(easyhandle, CURLOPT_URL,       sources[isrc]);
           
	 if ((common_data->gridhttp) &&
	     (strncmp(sources[isrc], "https://", 8) == 0))
	   {
             if (common_data->verbose > 0)
	       fprintf(stderr, "Add  Upgrade: GridHTTP/1.0\n");
             curl_easy_setopt(easyhandle,CURLOPT_HTTPHEADER,gh_header_slist);
	   }
	 else 
	   curl_easy_setopt(easyhandle,CURLOPT_HTTPHEADER,nogh_header_slist);

	 header_data.retcode  = 0;
	 header_data.location = NULL;
	 header_data.gridhttppasscode = NULL;
	 header_data.common_data = common_data;
	 thiserror = curl_easy_perform(easyhandle);

       }

	 asprintf(&p, "Destination: %s", thisdestination);
	 nogh_header_slist=NULL;
	 nogh_header_slist = curl_slist_append(nogh_header_slist,p);
	 //       fprintf(stdout, "complete destination file: %s\n", p);
	 free(p);

       // send request to destination server, 
       // to ask it to download file from source server
       strcpy( remoteserver, destination);
       while( (p=strrchr(remoteserver, '/')) !=NULL)
	 {
	   if( *(p-1) == '/' )break;
	   else *p = '\0';
	 }

       common_data->source = sources[isrc];
       common_data->destination = remoteserver;
       set_std_opts(easyhandle, common_data);
       // send copy request to copy server (destination)
       asprintf(&p, "COPY %s", sources[isrc]);
       curl_easy_setopt(easyhandle, CURLOPT_CUSTOMREQUEST, p);//"COPY");//gh_header_slist);
       curl_easy_setopt(easyhandle, CURLOPT_URL, remoteserver);
       curl_easy_setopt(easyhandle, CURLOPT_COOKIE, header_data.gridhttppasscode);

       curl_easy_setopt(easyhandle, CURLOPT_USERAGENT, common_data->useragent);
       curl_easy_setopt(easyhandle, CURLOPT_HTTPHEADER, nogh_header_slist);

       curl_easy_setopt(easyhandle, CURLOPT_ERRORBUFFER, common_data->errorbuf);
       thiserror = curl_easy_perform(easyhandle);
       free(p);
     }

  curl_easy_cleanup(easyhandle);
     
  return anyerror;

}

int do_copies(char *sources[], char *destination,
              struct grst_stream_data *common_data)
{
  char        *p, *thisdestination;
  int          isrc, anyerror = 0, thiserror, isdirdest;
  CURL        *easyhandle;
  struct stat  statbuf;
  struct       grst_header_data header_data;
  struct curl_slist *gh_header_slist = NULL, *nogh_header_slist = NULL;
  
  easyhandle = curl_easy_init();
  
  if (common_data->gridhttp)
    {               
      asprintf(&p, "Upgrade: GridHTTP/1.0");
      gh_header_slist = curl_slist_append(gh_header_slist, p);
      free(p);
      
      nogh_header_slist = curl_slist_append(nogh_header_slist, "Upgrade:");
    }
  
  curl_easy_setopt(easyhandle, CURLOPT_USERAGENT, common_data->useragent);
  if (common_data->verbose > 1)
                   curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1);

  curl_easy_setopt(easyhandle, CURLOPT_HEADERFUNCTION, headers_callback);
  curl_easy_setopt(easyhandle, CURLOPT_WRITEHEADER,   &header_data);

  set_std_opts(easyhandle, common_data);

  curl_easy_setopt(easyhandle, CURLOPT_ERRORBUFFER, common_data->errorbuf);

  if (destination[strlen(destination) - 1] != '/') 
    {
      isdirdest = 0;
      thisdestination = destination;
    }
  else isdirdest = 1;

  for (isrc=0; sources[isrc] != NULL; ++isrc)
     {
       if (isdirdest)
         {
           p = rindex(sources[isrc], '/');
           if (p == NULL) p = sources[isrc];
           else           p++;

           asprintf(&thisdestination, "%s%s", destination, p);
         }
 
       if (common_data->verbose > 0)
            fprintf(stderr, "Copy %s -> %s\n", sources[isrc], thisdestination);

       if (common_data->method == HTCP_GET)
         {
           common_data->fp = fopen(thisdestination, "w");
           if (common_data->fp == NULL)
             {
               fprintf(stderr,"... failed to open destination source file %s\n",
                               thisdestination);
               anyerror = 99;
               if (isdirdest) free(thisdestination);
               continue;
             }

           curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, common_data->fp);
           curl_easy_setopt(easyhandle, CURLOPT_URL,       sources[isrc]);
           
           if ((common_data->gridhttp) &&
               (strncmp(sources[isrc], "https://", 8) == 0))
             {
               if (common_data->verbose > 0)
                 fprintf(stderr, "Add  Upgrade: GridHTTP/1.0\n");
                 
               curl_easy_setopt(easyhandle,CURLOPT_HTTPHEADER,gh_header_slist);
             }
           else 
             curl_easy_setopt(easyhandle,CURLOPT_HTTPHEADER,nogh_header_slist);
         }
       else if (common_data->method == HTCP_PUT)
         {
           if (stat(sources[isrc], &statbuf) != 0)
             {
               fprintf(stderr, "... source file %s not found\n", sources[isrc]);
               anyerror = 99;
               if (isdirdest) free(thisdestination);
               continue;
             }
           
           common_data->fp = fopen(sources[isrc], "r");
           if (common_data->fp == NULL)
             {
               fprintf(stderr, "... failed to open source file %s\n",
                               sources[isrc]);
               anyerror = 99;
               if (isdirdest) free(thisdestination);
               continue;
             }

           curl_easy_setopt(easyhandle, CURLOPT_READDATA,   common_data->fp);
           curl_easy_setopt(easyhandle, CURLOPT_URL,        thisdestination);
           curl_easy_setopt(easyhandle, CURLOPT_INFILESIZE, statbuf.st_size);
           curl_easy_setopt(easyhandle, CURLOPT_UPLOAD,   1);

           if ((common_data->gridhttp) &&
               (strncmp(thisdestination, "https://", 8) == 0))
               curl_easy_setopt(easyhandle,CURLOPT_HTTPHEADER,gh_header_slist);
           else 
             curl_easy_setopt(easyhandle,CURLOPT_HTTPHEADER,nogh_header_slist);
         }

       header_data.retcode  = 0;
       header_data.location = NULL;
       header_data.gridhttppasscode = NULL;
       header_data.common_data = common_data;
       thiserror = curl_easy_perform(easyhandle);
       
       fclose(common_data->fp);

       if ((common_data->gridhttp) &&
           (thiserror == 0) &&
           (header_data.retcode == 302) &&
           (header_data.location != NULL) &&
           (strncmp(header_data.location, "http://", 7) == 0) &&
           (header_data.gridhttppasscode != NULL))
         {
           if (common_data->verbose > 0)
             fprintf(stderr, "... Found (%d)\nGridHTTP redirect to %s\n",
                     header_data.retcode, header_data.location);

           /* try again with new URL and all the previous CURL options */

           if (common_data->method == HTCP_GET)
             {
               common_data->fp = fopen(thisdestination, "w");
               if (common_data->fp == NULL)
                 {
                   fprintf(stderr, "... failed to open destination source "
                                   "file %s\n", thisdestination);
                   anyerror = 99;
                   if (isdirdest) free(thisdestination);
                   continue;
                 }
             }
           else if (common_data->method == HTCP_PUT)
             {
               common_data->fp = fopen(sources[isrc], "r");
               if (common_data->fp == NULL)
                 {
                   fprintf(stderr, "... failed to open source file %s\n",
                               sources[isrc]);
                   anyerror = 99;
                   if (isdirdest) free(thisdestination);
                   continue;
                 }
             }

           header_data.retcode  = 0;           
           curl_easy_setopt(easyhandle, CURLOPT_URL, header_data.location);
           curl_easy_setopt(easyhandle, CURLOPT_HTTPHEADER, nogh_header_slist);
           curl_easy_setopt(easyhandle, CURLOPT_COOKIE, 
                                                  header_data.gridhttppasscode);
           thiserror = curl_easy_perform(easyhandle);

           fclose(common_data->fp);
         }

       if ((thiserror != 0) ||
           (header_data.retcode >= 300))
         {
           fprintf(stderr, "... curl error: %s (%d), HTTP error: %d\n",
                   common_data->errorbuf, thiserror, header_data.retcode);
                   
           if (thiserror != 0) anyerror = thiserror;
           else                anyerror = header_data.retcode;
         }
       else if (common_data->verbose > 0) 
                  fprintf(stderr, "... OK (%d)\n", header_data.retcode);
        
       if (isdirdest) free(thisdestination);
     }

  curl_easy_cleanup(easyhandle);
     
  return anyerror;
}

int do_deletes(char *sources[], struct grst_stream_data *common_data)
{
  int    isrc, anyerror = 0, thiserror;
  CURL  *easyhandle;
  struct grst_header_data header_data;
  
  header_data.common_data = common_data;

  easyhandle = curl_easy_init();
  
  curl_easy_setopt(easyhandle, CURLOPT_USERAGENT, common_data->useragent);
  if (common_data->verbose > 1)
                   curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1);

  curl_easy_setopt(easyhandle, CURLOPT_HEADERFUNCTION, headers_callback);
  curl_easy_setopt(easyhandle, CURLOPT_WRITEHEADER,   &header_data);

  curl_easy_setopt(easyhandle, CURLOPT_ERRORBUFFER,   common_data->errorbuf);
  curl_easy_setopt(easyhandle, CURLOPT_CUSTOMREQUEST, "DELETE");
  curl_easy_setopt(easyhandle, CURLOPT_NOBODY,        1);

  set_std_opts(easyhandle, common_data);

  for (isrc=0; sources[isrc] != NULL; ++isrc)
     {
       if (common_data->verbose > 0)
            fprintf(stderr, "Deleting %s\n", sources[isrc]);

       curl_easy_setopt(easyhandle, CURLOPT_URL, sources[isrc]);

       header_data.retcode = 0;
       thiserror = curl_easy_perform(easyhandle);
       
       if ((thiserror != 0) ||
           (header_data.retcode >= 300))
         {
           fprintf(stderr, "... curl error: %s (%d), HTTP error: %d\n",
                   common_data->errorbuf, thiserror, header_data.retcode);
           
           if (thiserror != 0) anyerror = thiserror;
           else                anyerror = header_data.retcode;
         }
       else if (common_data->verbose > 0) 
                     fprintf(stderr, "... OK (%d)\n", header_data.retcode);
     }

  curl_easy_cleanup(easyhandle);
     
  return anyerror;
}

int do_move(char *source, char *destination, 
            struct grst_stream_data *common_data)
{
  int    anyerror = 0, thiserror;
  char  *destination_header;
  CURL  *easyhandle;
  struct grst_header_data header_data;
  struct curl_slist *header_slist = NULL;
  
  easyhandle = curl_easy_init();
  
  header_data.common_data = common_data;

  easyhandle = curl_easy_init();
  
  asprintf(&destination_header, "Destination: %s", destination);
  header_slist = curl_slist_append(header_slist, destination_header);
  curl_easy_setopt(easyhandle, CURLOPT_HTTPHEADER, header_slist);

  curl_easy_setopt(easyhandle, CURLOPT_USERAGENT, common_data->useragent);
  if (common_data->verbose > 1)
                   curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1);

  curl_easy_setopt(easyhandle, CURLOPT_HEADERFUNCTION, headers_callback);
  curl_easy_setopt(easyhandle, CURLOPT_WRITEHEADER,   &header_data);

  curl_easy_setopt(easyhandle, CURLOPT_ERRORBUFFER,   common_data->errorbuf);
  curl_easy_setopt(easyhandle, CURLOPT_CUSTOMREQUEST, "MOVE");
  curl_easy_setopt(easyhandle, CURLOPT_NOBODY,        1);

  set_std_opts(easyhandle, common_data);

  if (common_data->verbose > 0)
            fprintf(stderr, "Moving %s to %s\n", source, destination);

  curl_easy_setopt(easyhandle, CURLOPT_URL, source);

  header_data.retcode = 0;
  thiserror = curl_easy_perform(easyhandle);
       
  if ((thiserror != 0) ||
           (header_data.retcode >= 300))
         {
           fprintf(stderr, "... curl error: %s (%d), HTTP error: %d\n",
                   common_data->errorbuf, thiserror, header_data.retcode);
           
           if (thiserror != 0) anyerror = thiserror;
           else                anyerror = header_data.retcode;
         }
  else if (common_data->verbose > 0) 
                     fprintf(stderr, "... OK (%d)\n", header_data.retcode);

  curl_easy_cleanup(easyhandle);
     
  return anyerror;
}

int do_mkdirs(char *sources[], struct grst_stream_data *common_data)
{
  int    isrc, anyerror = 0, thiserror;
  CURL  *easyhandle;
  struct grst_header_data header_data;
  
  header_data.common_data = common_data;

  easyhandle = curl_easy_init();
  
  curl_easy_setopt(easyhandle, CURLOPT_USERAGENT, common_data->useragent);
  if (common_data->verbose > 1)
                   curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1);

  curl_easy_setopt(easyhandle, CURLOPT_HEADERFUNCTION, headers_callback);
  curl_easy_setopt(easyhandle, CURLOPT_WRITEHEADER,   &header_data);

  curl_easy_setopt(easyhandle, CURLOPT_ERRORBUFFER,   common_data->errorbuf);
  curl_easy_setopt(easyhandle, CURLOPT_CUSTOMREQUEST, "PUT");
  curl_easy_setopt(easyhandle, CURLOPT_NOBODY,        1);

  set_std_opts(easyhandle, common_data);

  for (isrc=0; sources[isrc] != NULL; ++isrc)
     {
       if (common_data->verbose > 0)
            fprintf(stderr, "Make directory %s\n", sources[isrc]);

       curl_easy_setopt(easyhandle, CURLOPT_URL, sources[isrc]);

       header_data.retcode = 0;
       thiserror = curl_easy_perform(easyhandle);
       
       if ((thiserror != 0) ||
           (header_data.retcode >= 300))
         {
           fprintf(stderr, "... curl error: %s (%d), HTTP error: %d\n",
                   common_data->errorbuf, thiserror, header_data.retcode);
           
           if (thiserror != 0) anyerror = thiserror;
           else                anyerror = header_data.retcode;
         }
       else if (common_data->verbose > 0)  
                        fprintf(stderr, "... OK (%d)\n", header_data.retcode);
     }

  curl_easy_cleanup(easyhandle);
     
  return anyerror;
}

static int
parse_groups(const char *grps, struct grst_sitecast_group *sitecast_groups, int group_num, int *res_num)
{
  struct addrinfo *ai;
  struct addrinfo hints;
  char *groups, *p, *port, *rest, def_port[8];
  int igroup, ret;

  igroup = -1;
  groups = strdup(grps);
  if (groups == NULL) {
	perror("Error parsing multicast groups");
	return ENOMEM;
  }

  memset(&hints, 0, sizeof(hints));
  hints.ai_socktype = SOCK_DGRAM;
  hints.ai_flags = AI_ADDRCONFIG | AI_NUMERICSERV | AI_NUMERICHOST;

  snprintf(def_port, sizeof(def_port), "%u", GRST_HTCP_PORT);

  p = strtok(groups, " ,");
  while (p && igroup+1 < group_num) {
	/* still IPv4 only: */
	port = strchr(p, ':');
	if (port == NULL) {
	    fprintf(stderr, "Failed to parse multicast group, port missing\n");
	    return CURLE_FAILED_INIT;
	}
	*port++ = '\0';
	
	rest = strchr(port, ':');
	if (rest)
	   *rest++ = '\0';
	
	ret = getaddrinfo(p, (port) ? port : def_port, &hints, &ai);
	if (ret) {
	    goto end_loop;
	}
	sitecast_groups[igroup+1].ai = ai;

	sitecast_groups[igroup+1].timewait = 1;
	sitecast_groups[igroup+1].ttl      = 1;
	if (rest) {
	    ret = sscanf(rest, "%d:%d", 
			&(sitecast_groups[igroup+1].ttl),
			&(sitecast_groups[igroup+1].timewait));
	    if (ret < 1) {
		fprintf(stderr, "Failed to parse multicast group "
			 "parameter %s\n", p);
	        return CURLE_FAILED_INIT;
	    }
	}
	
	igroup++;
end_loop:
	p = strtok(NULL, " ,");
  }
  free(groups);

  if (igroup == -1)
    {
      fprintf(stderr, "Failed to parse multicast group parameter %s\n", p);
      return CURLE_FAILED_INIT;
    }
  *res_num = igroup;
  return 0;
}

int do_ping(struct grst_stream_data *common_data_ptr)
{
  int request_length, response_length, i, ret, s, igroup, max_fd;
  struct sockaddr from;
  socklen_t fromlen;
#define MAXBUF 8192  
  char *request, response[MAXBUF];
  GRSThtcpMessage msg;
  struct timeval start_timeval, wait_timeval, response_timeval;
  struct grst_sitecast_group sitecast_groups[HTCP_SITECAST_GROUPS];
  fd_set readsckts, open_sckts;
  struct addrinfo *a;
  char host[INET6_ADDRSTRLEN];
  char serv[8];

  /* parse common_data_ptr->groups */ 
  if (common_data_ptr->groups == NULL)
    {
      fprintf(stderr, "No multicast groups given\n");
      return CURLE_FAILED_INIT;
    }

  ret = parse_groups(common_data_ptr->groups, sitecast_groups, HTCP_SITECAST_GROUPS, &igroup);
  if (ret)
	return ret;

  /* loop through multicast groups and send off the NOP pings */
  GRSThtcpNOPrequestMake(&request, &request_length,
			 (int) (start_timeval.tv_usec + i));
  gettimeofday(&start_timeval, NULL);
  FD_ZERO(&open_sckts);
  max_fd = -1;
  for (i=0; i <= igroup; ++i) {
	a = sitecast_groups[i].ai;
	for (a = sitecast_groups[i].ai; a != NULL; a = a->ai_next) {
	    s = socket(a->ai_family, a->ai_socktype, a->ai_protocol);
	    if (s < 0)
		continue;

	    ret = sendto(s, request, request_length, 0, a->ai_addr, a->ai_addrlen);
	    if (ret < 0) {
		close(s);
		continue;
	    }

	    if (common_data_ptr->verbose > 0) {
		getnameinfo(a->ai_addr, a->ai_addrlen, host, sizeof(host), NULL, 0, NI_NUMERICHOST);
		fprintf(stderr, "UDP/HTCP NOP ping to %s\n", host);
	    }

	    
	    FD_SET(s, &open_sckts);
	    if (s > max_fd)
		max_fd = s;
	}
  }
  free(request);

  /* reusing wait_timeval is a Linux-specific feature of select() */
  wait_timeval.tv_sec = common_data_ptr->timeout 
                                 ? common_data_ptr->timeout : 60;
  wait_timeval.tv_usec = 0;

  while ((wait_timeval.tv_sec > 0) || (wait_timeval.tv_usec > 0))
       {
	 readsckts = open_sckts;
 
         ret = select(max_fd + 1, &readsckts, NULL, NULL, &wait_timeval);
         gettimeofday(&response_timeval, NULL);

         if (ret > 0)
           {
	     for (s = 0; s <= max_fd; s++) {
		if (FD_ISSET(s, &readsckts))
		   break;
	     }
	     if (s > max_fd)
		break;

	     fromlen = sizeof(from);
             response_length = recvfrom(s, response, MAXBUF,
                                        0, &from, &fromlen);
  
	     getnameinfo(&from, fromlen,
			 host, sizeof(host),
			 serv, sizeof(serv), NI_NUMERICHOST);
             if (common_data_ptr->verbose > 0) {
              fprintf(stderr, "UDP mesg from %s:%s\n",
                              host, serv);
	     }

             if ((GRSThtcpMessageParse(&msg, response, response_length) 
                                                      == GRST_RET_OK) &&
                 (msg.opcode == GRSThtcpNOPop) && (msg.rr == 1) && 
                 (msg.trans_id >= (int) start_timeval.tv_usec) &&
                 (msg.trans_id <= (int) (start_timeval.tv_usec + igroup)))
               {
                 printf("%s:%s %.3fms\n", 
                          host, serv,
                          (((long) 1000000 * response_timeval.tv_sec) +
                           ((long) response_timeval.tv_usec) -
                           ((long) 1000000 * start_timeval.tv_sec) -
                           ((long) start_timeval.tv_usec)) / 1000.0);
               }
           }
       }
   for (s = 0; s <=max_fd; s++)
	if (FD_ISSET(s, &open_sckts))
		close(s);

   return GRST_RET_OK;
}

int do_finds(char *sources[], 
             struct grst_stream_data *common_data_ptr, int num)
{
  int          isrc;

  int request_length, response_length, i, ret, s, igroup, max_fd;
  struct sockaddr from;
  socklen_t fromlen;
#define MAXBUF 8192  
  char *request, response[MAXBUF];
  GRSThtcpMessage msg;
  struct timeval start_timeval, wait_timeval;
  struct grst_sitecast_group sitecast_groups[HTCP_SITECAST_GROUPS];
  fd_set readsckts, open_sckts;
  struct addrinfo *a;
  char host[INET6_ADDRSTRLEN];
  char serv[8];

  /* parse common_data_ptr->groups */ 
  if (common_data_ptr->groups == NULL)
    {
      fprintf(stderr, "No multicast groups given\n");
      return CURLE_FAILED_INIT;
    }

  ret = parse_groups(common_data_ptr->groups, sitecast_groups, HTCP_SITECAST_GROUPS, &igroup);
  if (ret)
        return ret;

  /* loop through multicast groups since we need to take each 
     ones timewait into account */

  gettimeofday(&start_timeval, NULL);

  for (i=0; i <= igroup; ++i)
     {
       
       FD_ZERO(&open_sckts);
       max_fd = -1;
       for (a = sitecast_groups[i].ai; a != NULL; a = a->ai_next) {
		s = socket(a->ai_family, a->ai_socktype, a->ai_protocol);
		if (s < 0)
			continue;

		if (common_data_ptr->verbose) {
			getnameinfo(a->ai_addr, a->ai_addrlen,
		    		host, sizeof(host),
		    		serv, sizeof(serv), NI_NUMERICHOST);
        		fprintf(stderr, "Querying multicast group %s:%s:%d:%d\n",
				host, serv, sitecast_groups[i].ttl,
                		sitecast_groups[i].timewait);
		}

	       /* send off queries, one for each source file */

	       for (isrc=0; sources[isrc] != NULL; ++isrc)
		  {
		    GRSThtcpTSTrequestMake(&request, &request_length, 
					   (int) (start_timeval.tv_usec + isrc),
					   "GET", sources[isrc], "");

		    sendto(s, request, request_length, 0, 
			       a->ai_addr, a->ai_addrlen);

		    free(request);
		  }
		FD_SET(s, &open_sckts);
		if (s > max_fd)
			max_fd = s;
	}
          
       	/* reusing wait_timeval is a Linux-specific feature of select() */
       	wait_timeval.tv_usec = 0;
      	wait_timeval.tv_sec  = sitecast_groups[i].timewait;

       	while ((wait_timeval.tv_sec > 0) || (wait_timeval.tv_usec > 0))
       {
	      readsckts = open_sckts;

              ret = select(max_fd + 1, &readsckts, NULL, NULL, &wait_timeval);

              if (ret > 0)
                {
		  for (s = 0; s <= max_fd; s++) {
			if (FD_ISSET(s, &readsckts))
				break;
		  }
		  if (s > max_fd)
			break;

		  fromlen = sizeof(from);
                  response_length = recvfrom(s, response, MAXBUF,
                                             0, &from, &fromlen);
  
                  if ((GRSThtcpMessageParse(&msg, response, response_length) 
                                                      == GRST_RET_OK) &&
                      (msg.opcode == GRSThtcpTSTop) && (msg.rr == 1) && 
                      (msg.trans_id >= (int) start_timeval.tv_usec) &&
                      (msg.trans_id < (int) (start_timeval.tv_usec + num)) &&
                      (msg.resp_hdrs != NULL) &&
                      (GRSThtcpCountstrLen(msg.resp_hdrs) > 12))
                    {                      
                      if (num > 1) printf("%s -> %.*s\n",
                          sources[msg.trans_id - (int) start_timeval.tv_usec],
                          GRSThtcpCountstrLen(msg.resp_hdrs) - 12, 
                          &(msg.resp_hdrs->text[10]));
                      else printf("%.*s\n",
                          GRSThtcpCountstrLen(msg.resp_hdrs) - 12, 
                          &(msg.resp_hdrs->text[10]));
                    }
                }
            }

     }
   for (s = 0; s <=max_fd; s++)
	if (FD_ISSET(s, &open_sckts))
		close(s);

   return GRST_RET_OK;
}

int translate_sitecast_url(char **source_ptr,
                           struct grst_stream_data *common_data_ptr)
{
  int request_length, response_length, i, ret, s, igroup, max_fd;
  struct sockaddr from;
  socklen_t fromlen;
#define MAXBUF 8192  
  char *request, response[MAXBUF];
  GRSThtcpMessage msg;
  struct timeval start_timeval, wait_timeval;
  struct grst_sitecast_group sitecast_groups[HTCP_SITECAST_GROUPS];
  fd_set readsckts, open_sckts;
  struct addrinfo *a;
  char host[INET6_ADDRSTRLEN];
  char serv[8];

  /* parse common_data_ptr->groups */ 

  if (common_data_ptr->groups == NULL)
    {
      fprintf(stderr, "No multicast groups given\n");
      return CURLE_FAILED_INIT;
    }

  ret = parse_groups(common_data_ptr->groups, sitecast_groups, HTCP_SITECAST_GROUPS, &igroup);
  if (ret)
	return ret;

  /* loop through multicast groups since we need to take each 
     ones timewait into account */

  gettimeofday(&start_timeval, NULL);

  GRSThtcpTSTrequestMake(&request, &request_length, 
                         (int) (start_timeval.tv_usec),
                         "GET", *source_ptr, "");

  for (i=0; i <= igroup; ++i)
     {
       FD_ZERO(&open_sckts);
       max_fd = -1;
       for (a = sitecast_groups[i].ai; a != NULL; a = a->ai_next) {
		s = socket(a->ai_family, a->ai_socktype, a->ai_protocol);
		if (s < 0)
			continue;
		if (common_data_ptr->verbose) {
		 getnameinfo(a->ai_addr, a->ai_addrlen,
			host, sizeof(host),
			serv, sizeof(serv), NI_NUMERICHOST);
		 fprintf(stderr, "Querying multicast group %s:%s:%d:%d\n",
			 host, serv, sitecast_groups[i].ttl,
			 sitecast_groups[i].timewait);
		}

       		/* send off queries, one for each source file */
       		sendto(s, request, request_length, 0, 
                       a->ai_addr, a->ai_addrlen);

       		FD_SET(s, &open_sckts);
       		if (s > max_fd)
			max_fd = s;
       }
       free(request);
          
       /* reusing wait_timeval is a Linux-specific feature of select() */
       wait_timeval.tv_usec = 0;
       wait_timeval.tv_sec  = sitecast_groups[i].timewait;

       while ((wait_timeval.tv_sec > 0) || (wait_timeval.tv_usec > 0))
            {
              readsckts = open_sckts;
  
              ret = select(max_fd + 1, &readsckts, NULL, NULL, &wait_timeval);

              if (ret > 0)
                {
		  for (s = 0; s <= max_fd; s++) {
			if (FD_ISSET(s, &readsckts))
				break;
		  }
		  if (s > max_fd)
			break;

		  fromlen = sizeof(from);
                  response_length = recvfrom(s, response, MAXBUF,
                                             0, &from, &fromlen);
  
                  if ((GRSThtcpMessageParse(&msg, response, response_length) 
                                                      == GRST_RET_OK) &&
                      (msg.opcode == GRSThtcpTSTop) && (msg.rr == 1) && 
                      (msg.trans_id == (int) start_timeval.tv_usec) &&
                      (msg.resp_hdrs != NULL) &&
                      (GRSThtcpCountstrLen(msg.resp_hdrs) > 12))
                    { 
                      /* found one */ 

                      if (common_data_ptr->verbose > 0)
                        fprintf(stderr, "Sitecast %s -> %.*s\n",
                                *source_ptr, 
                                GRSThtcpCountstrLen(msg.resp_hdrs) - 12,
                                &(msg.resp_hdrs->text[10]));

                      free(*source_ptr);
                      
                      asprintf(source_ptr, "%.*s",
                          GRSThtcpCountstrLen(msg.resp_hdrs) - 12, 
                          &(msg.resp_hdrs->text[10]));
                          
                      return GRST_RET_OK;
                    }
                }
            }

     }
  for (s = 0; s <=max_fd; s++)
	if (FD_ISSET(s, &open_sckts))
		close(s);
     
  return GRST_RET_OK;
}

size_t rawindex_callback(void *ptr, size_t size, size_t nmemb, void *data)
{
  if ( ((struct grst_index_blob *) data)->used + size * nmemb >=
                             ((struct grst_index_blob *) data)->allocated )
    {
      ((struct grst_index_blob *) data)->allocated = 
        ((struct grst_index_blob *) data)->used + size * nmemb + 4096;

      ((struct grst_index_blob *) data)->text = 
         realloc( ((struct grst_index_blob *) data)->text,
                  ((struct grst_index_blob *) data)->allocated );
    }
    
  memcpy( &( ((struct grst_index_blob *) 
                 data)->text[((struct grst_index_blob *) data)->used] ),
          ptr, size * nmemb);
          
  ((struct grst_index_blob *) data)->used += size * nmemb;
  
  return size * nmemb;
}

char *canonicalise(char *link, char *source)
{
  int   i, j, srclen;
  char *s;

  srclen = strlen(source);

  if ((strncmp(link, "https://", 8) == 0) ||
      (strncmp(link, "http://", 7) == 0))
    {
      if (strncmp(link, source, srclen) != 0) return NULL; /* other site */
      
      if (link[srclen] == '\0') return NULL; /* we dont self-link! */
      
      for (i=0; link[srclen + i] != '\0'; ++i)
        if (link[srclen + i] == '/')
          { 
            if (link[srclen + i + 1] != '\0') return NULL; /* no subdirs */
            else return strdup(&link[srclen]); /* resolves to this dir */
          }
    }
  else if (link[0] != '/') /* relative link - need to check for subsubdirs */
    {
      for (i=0; link[i] != '\0'; ++i) 
        if ((link[i] == '/') && (link[i+1] != '\0')) return NULL;

      s = strdup(link);
      
      for (i=0; s[i] != '\0'; ++i) 
       if (s[i] == '#')
         {
           s[i] = '\0';
           break;
         }

      return s;
    }

  /* absolute link on this server, starting / */

  for (i=8; source[i] != '\0'; ++i) if (source[i] == '/') break;
       
  if (strncmp(link, &source[i], srclen - i) != 0) return NULL;

  for (j = srclen - i; link[j] != '\0'; ++j) 
        if ((link[j] == '/') && (link[j+1] != '\0')) return NULL;
        
  s = strdup(&link[srclen - i]);
      
  for (i=0; s[i] != '\0'; ++i) 
       if (s[i] == '#')
         {
           s[i] = '\0';
           break;
         }

  if (s[0] == '\0') /* on second thoughts... */
    {
      free(s);
      return NULL;
    }
         
  return s;      
}

int grst_dir_list_cmp(const void *a, const void *b)
{
  return strcmp( ((struct grst_dir_list *) a)->filename, 
                 ((struct grst_dir_list *) b)->filename);
}

struct grst_dir_list *index_to_dir_list(char *text, char *source)
{
  int   taglevel = 0, wordnew = 1, i, namestart, used = 0, 
        allocated = 256;
  char *p, *s;
  struct grst_dir_list *list;
  
  list = (struct grst_dir_list *)
              malloc(allocated * sizeof(struct grst_dir_list));
              
  list[0].filename     = NULL;
  list[0].length       = 0;
  list[0].length_set   = 0;
  list[0].modified     = 0;
  list[0].modified_set = 0;
    
  for (p=text; *p != '\0'; ++p)
     {
       if (*p == '<') 
         {
           ++taglevel;
           
           if ((taglevel == 1) && (list[used].filename != NULL))
             {
               ++used;
               if (used >= allocated) 
                 {
                   allocated += 256;
                   list = (struct grst_dir_list *)
                           realloc((void *) list,
                                   allocated * sizeof(struct grst_dir_list));
                 }
                 
               list[used].filename     = NULL;
               list[used].length       = 0;
               list[used].length_set   = 0;
               list[used].modified     = 0;
               list[used].modified_set = 0;
             }

           wordnew = 1;
           continue;
         }

       if (*p == '>') 
         {
           --taglevel;
           wordnew = 1;
           continue;
         }
         
       if (isspace(*p))
         {
           wordnew = 1;
           continue;
         }

       if ((wordnew) && (taglevel == 1))
         {        
           if (((*p == 'h') || (*p == 'H')) && 
               (strncasecmp(p, "href=", 5) == 0))
             {
               if (p[5] == '"') { namestart = 6;
                                  for (i=namestart; (p[i] != '\0') &&
                                                    (p[i] != '"' ) &&
                                                    (p[i] != '\n') &&
                                                    (p[i] != '\t') &&
                                                    (p[i] != '>' ) ; ++i) ; }
               else { namestart = 5;
                      for (i=namestart; (p[i] != '\0') &&
                                        (p[i] != '"' ) &&
                                        (p[i] != ' ' ) &&
                                        (p[i] != '\n') &&
                                        (p[i] != '\t') &&
                                        (p[i] != ')' ) &&
                                        (p[i] != '>' ) ; ++i) ; }
               if (i > namestart) 
                 {
                   s = malloc(1 + i - namestart);
                   memcpy(s, &p[namestart], i - namestart);
                   s[i - namestart] = '\0';

                   list[used].filename = canonicalise(s, source);
                   free(s);
                 }
                 
               p = &p[i-1]; /* -1 since continue results in ++i */
               continue;
             }

           if (((*p == 'c') || (*p == 'C')) && 
               (strncasecmp(p, "content-length=", 15) == 0))
             {
               list[used].length     = 0;
               list[used].length_set = 1;
               
               if (p[15] == '"') list[used].length = atoi(&p[16]);
               else              list[used].length = atoi(&p[15]);

               p = &p[15];
               continue;
             }

           if (((*p == 'l') || (*p == 'L')) && 
               (strncasecmp(p, "last-modified=", 14) == 0))
             {
               list[used].modified     = 0;
               list[used].modified_set = 1;
               
               if (p[14] == '"') list[used].modified = atoi(&p[15]);
               else              list[used].modified = atoi(&p[14]);

               p = &p[14];
               continue;
             }
         }
         
       wordnew = 0;
     }  

  qsort((void *) list, used, sizeof(struct grst_dir_list), grst_dir_list_cmp);

  return list;  
}

int do_listings(char *sources[], struct grst_stream_data *common_data,
                int islonglist)
{
  int          isrc, anyerror = 0, thiserror, i, isdir, ilast, loclen, srclen;
  CURL        *easyhandle;
  const char  *months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", 
                            "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
  char        *s, *source_url;
  struct       grst_index_blob  rawindex;
  struct       grst_dir_list   *list;
  struct       grst_header_data header_data;
  struct       tm               modified_tm;
  time_t                        now;

  time(&now);

  header_data.common_data = common_data;

  easyhandle = curl_easy_init();
  
  curl_easy_setopt(easyhandle, CURLOPT_USERAGENT, common_data->useragent);
  if (common_data->verbose > 1)
                   curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1);

  curl_easy_setopt(easyhandle, CURLOPT_WRITEHEADER,   &header_data);
  curl_easy_setopt(easyhandle, CURLOPT_HEADERFUNCTION, headers_callback);

  curl_easy_setopt(easyhandle, CURLOPT_ERRORBUFFER, common_data->errorbuf);

  set_std_opts(easyhandle, common_data);

  for (isrc=0; sources[isrc] != NULL; ++isrc)
     {
       source_url = sources[isrc];
     
       if (common_data->verbose > 0)
            fprintf(stderr, "Listing %s\n", source_url);

       if (sources[1] != NULL) printf("\n%s:\n", source_url);

       curl_easy_setopt(easyhandle, CURLOPT_URL, source_url);

       if (source_url[strlen(source_url)-1] == '/')
         {
           isdir = 1;
           curl_easy_setopt(easyhandle,CURLOPT_WRITEFUNCTION,rawindex_callback);
           curl_easy_setopt(easyhandle,CURLOPT_WRITEDATA,(void *) &rawindex);
           curl_easy_setopt(easyhandle,CURLOPT_HTTPGET,1);
           curl_easy_setopt(easyhandle,CURLOPT_NOBODY,0);
           rawindex.text      = NULL;
           rawindex.used      = 0;
           rawindex.allocated = 0;
         }
       else
         {
           isdir = 0;
           curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, NULL);
           curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, NULL);
           curl_easy_setopt(easyhandle, CURLOPT_NOBODY, 1);
         }

       header_data.gridhttppasscode = NULL;
       header_data.length_set   = 0;
       header_data.modified_set = 0;
       header_data.retcode      = 0;
       thiserror = curl_easy_perform(easyhandle);

       if ((thiserror == 0) && 
           (header_data.retcode == 301) &&
           (header_data.location != NULL) &&
           ((loclen = strlen(header_data.location))
            == ((srclen = strlen(source_url)) + 1)) &&
           (strncmp(header_data.location, source_url, srclen) == 0) &&
           (header_data.location[loclen-1] == '/'))
         {
           if (common_data->verbose > 0) 
                fprintf(stderr, "... redirect to %s\n", header_data.location);

           source_url = strdup(header_data.location);
           isdir = 1;
           curl_easy_setopt(easyhandle,CURLOPT_WRITEFUNCTION,rawindex_callback);
           curl_easy_setopt(easyhandle,CURLOPT_WRITEDATA,(void *) &rawindex);
           curl_easy_setopt(easyhandle,CURLOPT_HTTPGET,1);
           curl_easy_setopt(easyhandle,CURLOPT_NOBODY,0);
           curl_easy_setopt(easyhandle,CURLOPT_URL, source_url);
           rawindex.text      = NULL;
           rawindex.used      = 0;
           rawindex.allocated = 0;
         
           header_data.gridhttppasscode = NULL;
           header_data.length_set   = 0;
           header_data.modified_set = 0;
           header_data.retcode      = 0;
           header_data.location     = NULL;

           thiserror = curl_easy_perform(easyhandle);
         }
         
       if ((thiserror != 0) ||
           (header_data.retcode >= 300))
         {
           fprintf(stderr, "... curl error: %s (%d), HTTP error: %d\n",
                   common_data->errorbuf, thiserror, header_data.retcode);
           
           if (thiserror != 0) anyerror = thiserror;
           else                anyerror = header_data.retcode;
         }
       else if (isdir)
         {
           if (common_data->verbose > 0) 
                  fprintf(stderr, "... OK (%d)\n", header_data.retcode);
           
           rawindex.text[rawindex.used] = '\0';

           list  = index_to_dir_list(rawindex.text, source_url);
           ilast = -1;

           for (i=0; list[i].filename != NULL; ++i)
              {
                if (list[i].filename[0] == '.') continue;
                
                if (strncmp(list[i].filename, "mailto:", 7) == 0) continue;
                
                if ((ilast >= 0) && 
                    (strcmp(list[i].filename, list[ilast].filename) == 0))
                                                                 continue;
                ilast=i;

                if (islonglist)
                  {
                    if (!list[i].length_set || !list[i].modified_set)
                      {
                        curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, 
                                                                        NULL);
                        curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, NULL);
                        curl_easy_setopt(easyhandle, CURLOPT_NOBODY, 1);
                        
                        asprintf(&s, "%s%s", source_url, list[i].filename);
                        curl_easy_setopt(easyhandle, CURLOPT_URL, s);

                        header_data.gridhttppasscode = NULL;
                        header_data.length_set   = 0;
                        header_data.modified_set = 0;
                        header_data.retcode = 0;
                        thiserror = curl_easy_perform(easyhandle);                        
                        free(s);
                        
                        if ((thiserror == 0) && 
                            (header_data.retcode >= 200) &&
                            (header_data.retcode <= 299))
                          {
                            if (header_data.length_set)
                              {
                                list[i].length_set = 1;
                                list[i].length     = header_data.length;
                              }
                          
                            if (header_data.modified_set)
                              {
                                list[i].modified_set = 1;
                                list[i].modified     = header_data.modified;
                              }
                          }
                      }

                    if (list[i].length_set) printf("%10zd ", list[i].length);
                    else fputs("         ? ", stdout);
                    
                    if (list[i].modified_set)
                      {
                        localtime_r(&(list[i].modified), &modified_tm);

                        if (list[i].modified < now - 15552000)
                             printf("%s %2d  %4d ", 
                               months[modified_tm.tm_mon],
                               modified_tm.tm_mday, 
                               modified_tm.tm_year + 1900);
                        else printf("%s %2d %02d:%02d ",
                               months[modified_tm.tm_mon],
                               modified_tm.tm_mday, 
                               modified_tm.tm_hour,
                               modified_tm.tm_min);
                      }
                    else fputs("  ?  ?     ? ", stdout);
                  }

                puts(list[i].filename);                  
              }
         }
       else
         {
           if (islonglist)
             {
               printf("%10zd ", header_data.length);
                    
               localtime_r(&(header_data.modified), &modified_tm);

               if (header_data.modified < now - 15552000)
                         printf("%s %2d  %4d ", 
                             months[modified_tm.tm_mon],
                             modified_tm.tm_mday, 
                             modified_tm.tm_year + 1900);
               else printf("%s %2d %02d:%02d ",
                             months[modified_tm.tm_mon],
                             modified_tm.tm_mday, 
                             modified_tm.tm_hour,
                             modified_tm.tm_min);
             }

           puts(source_url);
         }
     }

  curl_easy_cleanup(easyhandle);
     
  return anyerror;
}

#if (LIBCURL_VERSION_NUM < 0x070908)
char *make_tmp_ca_roots(char *dir)
/* libcurl before 7.9.8 doesnt support CURLOPT_CAPATH and the directory,
   so we make a temporary file with the concatenated CA root certs: that
   is, all the files in that directory which end in .0 */
{
  int    ofd, ifd, c;
  size_t size;
  char   tmp_ca_roots[] = "/tmp/.ca-roots-XXXXXX", buffer[4096], *s;
  DIR   *rootsDIR;
  struct dirent *root_ent;
  
  if ((rootsDIR = opendir(dir)) == NULL) return NULL;
  
  if ((ofd = mkstemp(tmp_ca_roots)) == -1)
    {
      closedir(rootsDIR);
      return NULL;
    }

  while ((root_ent = readdir(rootsDIR)) != NULL)
       {
         if ((root_ent->d_name[0] != '.') &&
             (strlen(root_ent->d_name) > 2) && 
             (strncmp(&(root_ent->d_name[strlen(root_ent->d_name)-2]),
                                                        ".0", 2) == 0))
           {
             asprintf(&s, "%s/%s", dir, root_ent->d_name);           
             ifd = open(s, O_RDONLY);             
             free(s);
             
             if (ifd != -1)
               {
                 while ((size = read(ifd, buffer, sizeof(buffer))) > 0)
                                                 write(ofd, buffer, size);             
                 close(ifd);
               }
           }
       }

  closedir(rootsDIR);
  
  if (close(ofd) == 0) return strdup(tmp_ca_roots);

  unlink(tmp_ca_roots); /* try to clean up */
  
  return NULL;
}
#endif

void printsyntax(char *argv0)
{
  char *p;
  
  p = rindex(argv0, '/');
  if (p != NULL) ++p;
  else           p = argv0;

  fprintf(stderr, "%s [options]  Source-URL[s]  [Destination URL]\n"
  "%s is one of a set of clients to fetch files or directory listings\n"
"from remote servers using HTTP or HTTPS, or to put or delete files or\n"
"directories onto remote servers using HTTPS. htcp is similar to scp(1)\n"
"but uses HTTP/HTTPS rather than ssh as its transfer protocol.\n"
"See the htcp(1) or http://www.gridsite.org/ for details.\n"
"(Version: %s)\n", p, p, VERSION);
}

struct option long_options[] = {	{"verbose",		0, 0, 'v'},
                			{"cert",		1, 0, 0},
			                {"key",			1, 0, 0},
             				{"capath",		1, 0, 0},
                			{"delete",		0, 0, 0},
					{"list",		0, 0, 0},
                			{"long-list",		0, 0, 0},
                			{"mkdir",		0, 0, 0},
                			{"no-verify",		0, 0, 0},
                			{"anon",		0, 0, 0},
                			{"grid-http",		0, 0, 0},
                			{"move",		0, 0, 0},
                			{"ping",		0, 0, 0},
                			{"groups",		1, 0, 0},
                			{"timeout",		1, 0, 0},
                			{"sitecast",		0, 0, 0},
                			{"domain",		1, 0, 0},
                			{"find",                0, 0, 0},
					{"rmtcp",		0, 0, 0},
                			{"conf",                1, 0, 0},
                			{0, 0, 0, 0}  };

int update_common_data(struct grst_stream_data *, int, char *);

void parse_conf(struct grst_stream_data *common_data_ptr, char *conf_file)
{
  int   option_index;
  char  line[1001], *p;
  FILE *fp;
  
  fp = fopen(conf_file, "r");
  if (fp == NULL)
    {
      if (common_data_ptr->verbose)
        fprintf(stderr, "Failed to open configuration file %s\n", conf_file);
      return;
    }
    
  if (common_data_ptr->verbose)
      fprintf(stderr, "Opened configuration file %s\n", conf_file);

  while (fgets(line, sizeof(line), fp) != NULL)
       {
         if ((p = index(line, '\n')) != NULL) *p = '\0';
       
         for (option_index=0; 
              long_options[option_index].name != NULL; ++option_index)
            {
              if (long_options[option_index].has_arg &&
                  (strncmp(line, long_options[option_index].name, 
                          strlen(long_options[option_index].name)) == 0) &&
                  (line[strlen(long_options[option_index].name)] == '='))
                {
                  update_common_data(common_data_ptr, option_index,
                   strdup(&line[strlen(long_options[option_index].name) + 1]));
                  break;
                }              

              if (!long_options[option_index].has_arg &&
                  (strcmp(line, long_options[option_index].name) == 0))
                {
                  update_common_data(common_data_ptr, option_index, "");
                  break;
                }              
            }
       }  
    
  fclose(fp);
}

int update_common_data(struct grst_stream_data *common_data_ptr,
                        int option_index, char *optarg)
{
  if      (option_index == 1) common_data_ptr->cert       = optarg;
  else if (option_index == 2) common_data_ptr->key        = optarg; 
  else if (option_index == 3) common_data_ptr->capath     = optarg;
  else if (option_index == 4) common_data_ptr->method     = HTCP_DELETE;
  else if (option_index == 5) common_data_ptr->method     = HTCP_LIST;
  else if (option_index == 6) common_data_ptr->method     = HTCP_LONGLIST;
  else if (option_index == 7) common_data_ptr->method     = HTCP_MKDIR;
  else if (option_index == 8) common_data_ptr->noverify   = 1;
  else if (option_index == 9) common_data_ptr->anonymous  = 1;
  else if (option_index ==10) common_data_ptr->gridhttp   = 1;
  else if (option_index ==11) common_data_ptr->method     = HTCP_MOVE;
  else if (option_index ==12) common_data_ptr->method     = HTCP_PING;
  else if (option_index ==13) common_data_ptr->groups     = optarg;
  else if (option_index ==14) common_data_ptr->timeout    = atoi(optarg);
  else if (option_index ==15) common_data_ptr->sitecast   = 1;
  else if (option_index ==16) { common_data_ptr->sitecast = 1;
                                common_data_ptr->domain   = optarg; }
  else if (option_index ==17) common_data_ptr->method     = HTCP_FIND;
  else if (option_index ==18) { printf("OK\n");common_data_ptr->method	  = HTCP_RMTCP;}
  /* option_index == 19 is used by the --conf command line-only option */
  else return GRST_RET_FAILED;
  
  return GRST_RET_OK;
}

int main(int argc, char *argv[])
{
  char **sources, *destination = NULL, *executable, *p, *htcp_conf;
  int    c, i, option_index, anyerror;
  struct stat statbuf;
  struct grst_stream_data common_data;
  struct passwd *userpasswd;

#if (LIBCURL_VERSION_NUM < 0x070908)
  char *tmp_ca_roots = NULL;
#endif

  if (argc == 1) 
    {
      printsyntax(argv[0]);
      return 0;
    }
 
  common_data.cert      = NULL;
  common_data.key       = NULL;
  common_data.capath    = NULL;
  common_data.method    = 0;
  common_data.errorbuf  = malloc(CURL_ERROR_SIZE);
  asprintf(&(common_data.useragent),
                          "htcp/%s (http://www.gridsite.org/)", VERSION);
  common_data.verbose   = 0;
  common_data.noverify  = 0;
  common_data.anonymous = 0;
  common_data.gridhttp  = 0;
  
  common_data.groups    = NULL;
  common_data.timeout   = 0;
  common_data.sitecast  = 0;
  common_data.domain    = NULL;

  if ((argc > 1) && ((strcmp(argv[1], "--verbose") == 0) || 
                     (strcmp(argv[1], "-v") == 0))) common_data.verbose = 1;
    
  /* examine any configuration files */
    
  parse_conf(&common_data, HTCP_HOST_CONF);  
  
  userpasswd = getpwuid(geteuid());
  asprintf(&htcp_conf, "%s/%s", userpasswd->pw_dir, HTCP_USER_CONF);
  parse_conf(&common_data, htcp_conf);
  free(htcp_conf);

  htcp_conf = getenv("HTCP_CONF");
  if (htcp_conf != NULL) parse_conf(&common_data, htcp_conf);

  common_data.verbose = 0;
    
  while (1)
       {
         option_index = 0;

         c = getopt_long(argc, argv, "v", long_options, &option_index);

         if      (c == -1) break;
         else if (c == 0)
           {
             if (option_index == 19) parse_conf(&common_data, optarg);
             else update_common_data(&common_data, option_index, optarg);
           }
         else if (c == 'v') ++(common_data.verbose);
       }

  if (common_data.verbose > 0) 
    {
      p = rindex(argv[0], '/');
      if (p != NULL) ++p;
      else           p = argv[0];
      fprintf(stderr, "%s version %s\n", p, VERSION);
    }

  if (common_data.anonymous) /* prevent any use of user certs */
    {
      common_data.cert = NULL;
      common_data.key  = NULL;
    }  
  else if ((common_data.cert == NULL) && (common_data.key != NULL)) 
           common_data.cert = common_data.key;
  else if ((common_data.cert != NULL) && (common_data.key == NULL))
           common_data.key = common_data.cert;
  else if ((common_data.cert == NULL) && (common_data.key == NULL))
    {
      common_data.cert = getenv("X509_USER_PROXY");
      if (common_data.cert != NULL) common_data.key = common_data.cert;
      else
        {
          asprintf(&(common_data.cert), "/tmp/x509up_u%d", geteuid());
          
          /* one fine day, we will check the proxy file for expiry too ... */
          
          if (stat(common_data.cert, &statbuf) == 0)
                     common_data.key = common_data.cert;                     
          else
            {
              common_data.cert = getenv("X509_USER_CERT");
              common_data.key  = getenv("X509_USER_KEY");
              
              if ((common_data.cert == NULL) &&
                  (userpasswd != NULL) &&
                  (userpasswd->pw_dir != NULL))
                asprintf(&(common_data.cert), "%s/.globus/usercert.pem", 
                                                    userpasswd->pw_dir);
              
              if ((common_data.key == NULL) &&
                  (userpasswd != NULL) &&
                  (userpasswd->pw_dir != NULL))
                asprintf(&(common_data.key), "%s/.globus/userkey.pem", 
                                                    userpasswd->pw_dir);              
            }            
        }    
    }

  if (common_data.capath == NULL) common_data.capath = getenv("X509_CERT_DIR");

  if (common_data.capath == NULL) 
                        common_data.capath = "/etc/grid-security/certificates";

#if (LIBCURL_VERSION_NUM < 0x070908)
  /* libcurl before 7.9.8 doesnt support CURLOPT_CAPATH and the directory */

  if ((common_data.capath != NULL) && 
      (stat(common_data.capath, &statbuf) == 0) && S_ISDIR(statbuf.st_mode))
    {
      tmp_ca_roots = make_tmp_ca_roots(common_data.capath);
      common_data.capath = tmp_ca_roots;
    }
#endif

  executable = rindex(argv[0], '/');
  if (executable != NULL) executable++;
  else                    executable = argv[0];
  
  if (common_data.method == 0) /* command-line options override exec name */
    {
      if      (strcmp(executable,"htls")==0) common_data.method=HTCP_LIST;
      else if (strcmp(executable,"htll")==0) common_data.method=HTCP_LONGLIST;
      else if (strcmp(executable,"htrm")==0) common_data.method=HTCP_DELETE;
      else if (strcmp(executable,"htmkdir")==0) common_data.method=HTCP_MKDIR;
      else if (strcmp(executable,"htmv")==0) common_data.method=HTCP_MOVE;
      else if (strcmp(executable,"htping")==0) common_data.method=HTCP_PING;
      else if (strcmp(executable,"htfind")==0) common_data.method=HTCP_FIND;
      else if (strcmp(executable,"htrmtcp")==0) common_data.method=HTCP_RMTCP;
    }
    
  if (common_data.method == HTCP_PING)
    {
      if (common_data.groups != NULL) return do_ping(&common_data);

      fprintf(stderr, "Must specify at least one multicast group\n\n"); 
      printsyntax(argv[0]);      
      return CURLE_FAILED_INIT;      
    }

  if ((common_data.method == HTCP_DELETE) || 
      (common_data.method == HTCP_LIST)   ||
      (common_data.method == HTCP_FIND)   ||
      (common_data.method == HTCP_MKDIR)  ||
      (common_data.method == HTCP_LONGLIST))
    {
      if (optind >= argc)
          {
            fprintf(stderr, "Must give at least 1 non-option argument\n\n"); 
            printsyntax(argv[0]);
            return CURLE_URL_MALFORMAT;
          }

      sources = (char **) malloc(sizeof(char *) * (1 + argc - optind));  
      for (i=0; i < argc - optind; ++i) 
         {
           sources[i] = argv[optind + i];
           
           if ((common_data.method == HTCP_MKDIR) &&
               (sources[i][strlen(sources[i])-1] != '/'))
             {
               fprintf(stderr, "Argument \"%s\" is not a "
                       "directory URL (no trailing /)\n\n", sources[i]);
               printsyntax(argv[0]);
               return CURLE_URL_MALFORMAT;
             }
         }

      sources[i]  = NULL;  

      if (common_data.method == HTCP_DELETE) 
                            anyerror = do_deletes(sources, &common_data);
      else if (common_data.method == HTCP_MKDIR) 
                            anyerror = do_mkdirs(sources, &common_data);
      else if (common_data.method == HTCP_FIND) 
                   anyerror = do_finds(sources, &common_data, argc - optind);
      else if (common_data.method == HTCP_LONGLIST) 
                            anyerror = do_listings(sources, &common_data, 1);
      else anyerror = do_listings(sources, &common_data, 0);

      if (anyerror > 99) anyerror = CURLE_HTTP_RETURNED_ERROR;

      return anyerror;
    }

  if (common_data.method == HTCP_MOVE)
    {
      if (optind >= argc - 1)
        {
          fputs("Must give exactly 2 non-option arguments\n\n", stderr);
          printsyntax(argv[0]);
          return CURLE_URL_MALFORMAT;
        }
      
      anyerror = do_move(argv[optind], argv[optind + 1], &common_data);

      if (anyerror > 99) anyerror = CURLE_HTTP_RETURNED_ERROR;

      return anyerror;
    }

  if (optind >= argc - 1) 
    {
      fputs("Must give at least 2 non-option arguments\n\n", stderr);
      printsyntax(argv[0]);
      return CURLE_URL_MALFORMAT;
    }
    
  sources = (char **) malloc(sizeof(char *) * (argc - optind));
  
  for (i=0; i < (argc - optind - 1); ++i) 
     {
       if (strncmp(argv[optind + i], "file:", 5) == 0)
            sources[i] = strdup(&argv[optind + i][5]);
       else sources[i] = strdup(argv[optind + i]);
       
       if (sources[i][0] == '\0') 
         {
           fprintf(stderr, "Source argument %d is empty\n\n", i + 1);
           printsyntax(argv[0]);
           return CURLE_URL_MALFORMAT;
         }
     }
  
  sources[i]  = NULL;  

  if (strncmp(argv[optind+i], "file:", 5) == 0)
    {
      if ((argv[optind+i][strlen(argv[optind+i]) - 1] != '/') &&
          (stat(&argv[optind + i][5], &statbuf) == 0) &&
          S_ISDIR(statbuf.st_mode))
                         asprintf(&destination, "%s/", &argv[optind + i][5]);
      else destination = strdup(&argv[optind + i][5]);
    }
  else if ((strncmp(argv[optind+i], "http://",  7) != 0) &&
           (strncmp(argv[optind+i], "https://", 8) != 0)) 
    {
      if ((argv[optind+i][strlen(argv[optind+i]) - 1] != '/') &&
          (stat(argv[optind+i], &statbuf) == 0) &&
          S_ISDIR(statbuf.st_mode))
                         asprintf(&destination, "%s/", argv[optind+i]);
      else destination = strdup(argv[optind+i]);
    }
  else destination = strdup(argv[optind+i]);
  
  if (destination[0] == '\0')
    {
      fputs("Destination argument is empty\n\n", stderr);
      printsyntax(argv[0]);
      return CURLE_URL_MALFORMAT;
    }

  if ((argc - optind > 2) && (destination[strlen(destination)-1] != '/'))
    {
      fputs("For multiple sources, destination "
            "must be a directory (end in /)\n\n", stderr);
      printsyntax(argv[0]);
      return CURLE_URL_MALFORMAT;
    }
  
  // remote file copy  
  if ( common_data.method == HTCP_RMTCP )
    {
      anyerror = do_rmtcp(sources, destination, &common_data);
      fprintf(stdout, "The file has been moved!\n");
      //      printsyntax(argv[0]);
      return CURLE_URL_MALFORMAT;
    }

  if ((strncmp(destination, "http://",  7) == 0) ||
      (strncmp(destination, "https://", 8) == 0)) 
       common_data.method = HTCP_PUT;
  else common_data.method = HTCP_GET;
  
  for (i=0; sources[i] != NULL; ++i)
         {
           if ((common_data.method == HTCP_PUT) && 
               ((strncmp(sources[i], "http://",  7) == 0) ||
                (strncmp(sources[i], "https://", 8) == 0)))
             {
               fputs("Cannot have both source and destination remote\n\n",stderr);
               printsyntax(argv[0]);
               return CURLE_URL_MALFORMAT;
             }
         
           if (common_data.method == HTCP_GET)
             {
               if ((strncmp(sources[i], "http://",  7) != 0) &&
                   (strncmp(sources[i], "https://", 8) != 0))
                 {
                   fputs("Cannot have both source and "
                     "destination local (for now)\n\n",stderr);
                   printsyntax(argv[0]);
                   return CURLE_URL_MALFORMAT;
                 }

               if ((common_data.sitecast) && 
                   ((common_data.domain == NULL) ||

                    ((strncmp(sources[i], "http://", 7) == 0) &&
                     (strncmp(&sources[i][7], common_data.domain, 
                                 strlen(common_data.domain)) == 0) &&
                     ((sources[i][7+strlen(common_data.domain)] == ':') ||
                      (sources[i][7+strlen(common_data.domain)] == '/'))) ||

                    ((strncmp(sources[i], "https://", 8) == 0) &&
                     (strncmp(&sources[i][8], common_data.domain, 
                                 strlen(common_data.domain)) == 0) &&
                     ((sources[i][8+strlen(common_data.domain)] == ':') ||
                      (sources[i][8+strlen(common_data.domain)] == '/')))))
                 {
                   translate_sitecast_url(&sources[i], &common_data);
                 }
             }
         }
         
  anyerror = do_copies(sources, destination, &common_data);
  if (anyerror > 99) anyerror = CURLE_HTTP_RETURNED_ERROR;
  
  return anyerror;
}
