#! /bin/perl --    # -*-Perl-*-
#
# outflow-stat - display outgoing feed statistics (#articles)
#                this works for innfeed and nntpsend/innxmit only
#
# History: This program is based on some ideas (and source code lines) of
#          Felix Kugler, SWITCH, inflow-package and could be part of it.
#
# 961029 V1.0   released
# 970109 V1.1   change ls to readdir for sitelist
#               fix strip path statement
# 970113 V1.2   aligned some paths, renumbered version (FK)
# 970128 V1.2.1	added config of innfeeds checkpointing (GW)
# 970220 V1.2.2	enriched outfile naming (FK)
# 970224 V1.2.3	innfeed.0.10 support added (FK)
# 970305 V1.2.4	read sitelist from conffile instead lockfiles (GW)
# 970310 V1.2.5	minor bug fixes concerning innfeed.conf parsing  
# 970705 V1.2.6 multiple innfeed support (FK)
# 971012 V1.3.0 nntpsend support, several minor modifications (FK)
# 971016 V1.3.1 improved handling of html-includes (FK)
# 971027 V1.3.2 added opt. external config file (FK)
# 971029 V1.4   added approximate line count (GW)
#               interactive calculation for one site
# 971109 V1.4.2 minor cosmetical modifications
# 971112 V1.5.0 version number aligned
# 980117 V1.5.3 show index of innfeed/nntpsend if multiple instances exist (FK)
# 980312 V1.5.4 added status file (backlog evolution) + innfeed fix for implicit ip-name (MR)
# 980406 V1.6.0 incorporated URLs to neighbour servers with monitoring
# 980514 V2.0.0 version number aligned
# 980525 V2.0.2 minor fix in html-header/footer handling
# 980609 V2.0.3 added warning if ext config file could not be found

$Copy="(c) 1996 Gerhard.Winkler\@univie.ac.at";

$RELDATE = "Tue Jun  9 09:55:47 MET DST 1998";
$RELEASE = "V2.0.3";

## local settings ----------------------------------------------------
#
$INFLOWCONF = "/home/news/config/inflow.conf"; # ext. local configs (optional)
#
## the following settings may be modified here or alternativly using
#  an external config file; the latter method makes updates more comfortable...
#
# innfeed stuff
$IFSPOOLDIR   = "/var/spool/news/out.going";
@INNFEEDCONF  = ( '/usr/local/news/inn/innfeed.conf',
		 '/usr/local/news/inn/innfeed.conf-1',
		 '/usr/local/news/inn/innfeed.conf-2',
		 '/usr/local/news/inn/innfeed.conf-3');
$OLDCONFSTYLE = 0;       # pre-0.10 innfeed uses old format for innfeed.conf
$CHKPT_FL     = 0;       # some innfeed versions use .checkpoint files
                         # some use .input for storing checkpoint info (>0.9.3)
#
$STATUSFILE="/home/news/inflow/outflow.status";

#
# approximate line count stuff
# please check this value; it is site dependant
$MAGIC = 65;           # approximate line length in spool file
$THRES_MAGIC = 6500000;  # do exact line count if filesize is smaller than
                         # threshold else do approximate line count
#
# nntpsend stuff
$NSSPOOLDIR   = "/newsspool/out.going";
@NNTPSENDCONF = ( '/newslib/etc/nntpsend.ctl',);
@NSPOSTFIXES  = ( '', '!n', '.work', '.nntp' );
#
# naming stuff
%ABBREV     = ( 'innfeed', 'innfeed',
		'nntpsend', 'nntpsend',
	       );
$OUTFILE      = "/usr/local/etc/httpd/htdocs/stats/news/outflow";  # generic outfile name
$WANTHOSTNAME = 0;       # add hostname to outfile name ( -host) ?
$GENERICTITLE = "Status of outgoing feeds";
#
# external programs & config files
@ADDTOPATH = ( '/usr/local/bin', '/newslib/bin' );
#
$DATE         = "date";
$WC           = "wc";
$GREP         = "grep";
$UNAME        = "uname";
$RESOLVCONF   = "/etc/resolv.conf";
#
# where to get this software
$INFLOWURL    = "ftp://ftp.switch.ch/info_service/netnews/wg/tools/inflow.tar.gz";
#
# URLs to other News servers
#
$URLLOOKUP = "/home/news/config/olm-lookup"; 
#
# local HTML stuff: you can put custom header & footer information into 
#                   local files; optional...
#
$HTMLHEADER = "/misc/inflow/config/header.html";
$HTMLFOOTER = "/misc/inflow/config/footer.html";
# 
# end local settings ------------------------------------------------

# initialisation ----------------------------------------------------
#
require "getopts.pl";

($path,$0) = ($0 =~ /^(.*)\/([^\/]+)$/);                # strip path...
umask 022;                                # file mask

&Getopts('adf:hi:wF');

&modify_config;
&update_PATH;
&gethostandfqdn;
&get_urllookup;

$Usage="$0    -  $Copy

Release $RELEASE  of $RELDATE

Process information from innfeed spoolfiles in $IFSPOOLDIR
and nntpsend batches in $NSSPOOLDIR. Show 
how many articles are currently waiting be sent to the remote site.
Output available either in Text or HTML format.

Usage: 	$0 [-adhiwF] [-f<configfile>]  

Parameters:

   -a:            approximate line count for batches >$THRES_MAGIC byte,
		  taking an average line lenght of $MAGIC byte as basis. 
   -d:            turn on verbosity; for debugging only
   -f<configfile>:load external configuration file 
                  default: $inflowconf
   -h:            This help.
   -i <sitename>: calculate this site only
   -w:            WWW support: show results in HTML table format
   -F:            write to file (defined in script header)

external config file: $inflowconfinfo

Optional HTML-stuff:

header file: $HTMLHEADER
footer file: $HTMLFOOTER

If several instances of innfeed/nntpsend are running, $0 shows 
to which one a feed belongs, if the config files honor one of the
following conventions (explained with an example):

  innfeed:      innfeed1.conf, innfeed-1.conf innfeed.conf1 innfeed.conf-1
  nntpsend.ctl: accordingly...

\n";

$date = `$DATE`;
if ($WANTHOSTNAME) { $OUTFILE .= "-$hostname"; }
$titleline = "$GENERICTITLE on $fqdn";

if ($opt_h) { print "$Usage"; exit 0; }

if ($opt_F) {
    if ($opt_w) { $OUTFILE .= ".html"; }
    open(OUT,">$OUTFILE.tmp") || die "can't open $OUTFILE\n";
} else {
    open(OUT,">-");
}

#  check for custom header & footer files
if ($opt_w) {
    if (open(IN,$HTMLHEADER)) {  
	while (<IN>) {
	    $line = &interpret;
	    $htmlheader .= $line;
	}
	close(IN); 
    } else {
	$htmlheader = "<html><head>
          <!--Content-type: text/html\nRefresh: 60-->\n
          <title>$titleline</title></head><body><h2>$titleline</h2>\n";
    }
    if (open(IN,$HTMLFOOTER)) { 
	while (<IN>) {
	    $line = &interpret; 
	    $htmlfooter .= $line; 
	}
	close(IN);
    } else {
	$htmlfooter = "</body></html>\n";
    }
}

&readconf;
&onesite if ($opt_i);
&readstatus;
&calculate;
&writestatus;
&initpage;
&writedata;
&closepage;

sub readstatus {
$now=time;
if ( -f $STATUSFILE) { 
	open(S, $STATUSFILE);
	$then=<S>;
	while(<S>) {
		($thesite, $thecount)=split;
		$backlog{$thesite}=$thecount;
	}
	close(S);
	$deltat=$now-$then;
} else {
	$deltat=0;
}
}

sub writestatus {
open(S, "> $STATUSFILE.tmp");
print S "$now\n";
for $site  (keys(%thecount)) {
	print S "$site $thecount{$site}\n";
}
close(S);
rename $STATUSFILE, $STATUSFILE.".old";
rename $STATUSFILE.".tmp", $STATUSFILE;
}

# calculate
# -----------------------------------------------------------------------------
# main calculation part
# we go through the sitelist once for every feedtype supported
# and sort the results at the end...
sub calculate {
    # innfeed
    if (chdir $IFSPOOLDIR) {
	foreach $site (sort keys %sitelist) {
            $appr = 0;
	    next if ($feedprog{$site} ne 'innfeed');
            chdir $backlogdir{$site};
	    $file = $site . ".lock";
	    unless (-e $file) {
		print "no lock file found: $site\n" if ($opt_d);
# is there a particular problem calculating backlog if innfeed
# is not running ? MR
#		&printsite("no data");
#		next;
	    }
	    print "check site: $site\n" if ($opt_d);
	    
	    $scount = $icount = $ocount = $offset = 0;
	    $file = $site;
	    if (-e $file) {
		$scount = &countfile($file);
		print "found articles in $file: $scount\n" if ($opt_d);
	    } else {
		print "no $file\n" if ($opt_d);
	    }
	    
	    $file = $site . ".input";
	    if (-e $file) {
		$icount = &countfile($file);
		print "found articles in $file: $icount\n" if ($opt_d);
		if ($CHKPT_FL) { $check = $site . ".checkpoint"; }
		else { $check = $site . ".input"; }
		$offset = 0;
		if (open(CFL,"<$check")) {
		    $offset = <CFL>;
		    chop($offset);
		    close(CFL);
		}
		print "found offset: $offset\n" if ($opt_d);
		@status = stat($file);
		$size = @status[7];
		$icount = $icount - ($icount * $offset / $size);
		$icount = int($icount);
		print "found articles in $file minus offset: $icount\n" if ($opt_d);
	    } else {
		print "no $file\n" if ($opt_d);
	    }
	    
	    $file = $site . ".output";
	    if (-e $file) {
		$ocount = &countfile($file);
		print "found articles in $file: $ocount\n" if ($opt_d);
	    } else {
		print "no $file\n" if ($opt_d);
	    }
	    
	    $count = $scount + $icount + $ocount;
	    &printsite($count);
	}
    }

    # nntpsend
    if (chdir $NSSPOOLDIR) {
	foreach $site (sort keys %sitelist) {
            $appr = 0;
	    next if ($feedprog{$site} ne 'nntpsend');
	    warn "check site $site\n" if $opt_d;
	    #
	    $count = $c = 0;
	    #
	    foreach $postfix (@NSPOSTFIXES) {
		$file = $site . $postfix;
		if (-e $file) {
		    $c = &countfile($file);
		    warn "nntpsend batch $file contains $c articles\n" if $opt_d;
		    $count += $c;
		} else {
		    warn "no file $file found in $NSSPOOLDIR\n" if $opt_d; 
		}
	    }
	    #
	    &printsite($count);
	}
    }
}


# countfile
# -----------------------------------------------------------------------------
# count number of articles (lines) in file
# maybe there will be some size (byte) of articles in it
sub countfile {
   local($f) = @_;
   local($ret,$c,$s);
   if ($opt_a) {
      @status = stat($f);
      $s = @status[7];
      if ($s > $THRES_MAGIC) {
         print "using approximate line count for $f\n" if ($opt_d);
         $appr = 1;
         $c = $s / $MAGIC;
         $c = int($c);
      }
      else {
         $ret = `$WC -l $f`;
         ($c) = ($ret =~ /^\s*(\d+)/);
      }
   }
   else {
      $ret = `$WC -l $f`;
      ($c) = ($ret =~ /^\s*(\d+)/);
   }
   return($c);
}


# readconf
# -----------------------------------------------------------------------------
# read innfeed/nntpsend configuration files and extract site names
# support for multiple conf files
# innfeed part requires version 0.10 and newer
sub readconf {
    # innfeed
    foreach $conffile (@INNFEEDCONF) {
	unless (open(F,"<$conffile")) {
	    warn "can\'t open innfeed config file $conffile\n" if $opt_d;
	    next;
	}
	warn "loading conf file: $conffile\n" if $opt_d;
	$conffilefound = 1;
	if ($conffile =~ /innfeed-?(\d+)\.conf$|innfeed\.conf-?(\d+)$/) {
	    $hasnumberedconfigs = 1;
	    $feednumber = $1;
	    warn "this is the config for innfeed $feednumber\n" if $opt_d; 
	}
	while(<F>) {
	    next if (/^\#/);
	    next if (/^$/);
	    $backlogdir=$1 if (/^\s*backlog-directory:\s+(\S+)/);

	    if (/^\s*peer\s+([^{}\s]+)/) { 
		$peer = $1; 
		$sitelist{$peer}++;
		$backlogdir{$peer}=$backlogdir;
		$feedprog{$peer} = 'innfeed';
		$feedid{$peer} = $feednumber;
		$conflist{$peer} = $peer;
	    }
	    if (/ip-name:\s+(\S+)/ && $peer) {
		$ipname = $1;
		$conflist{$peer} = $ipname;
		print "conflist{$peer} = $ipname\n" if $opt_d;
		$peer = "";
	    }
	}
	close(F);
    }

    # nntpsend
    foreach $conffile (@NNTPSENDCONF) {
	unless (open(F,"<$conffile")) {
	    warn "can\'t open nntpsend config file $conffile\n" if $opt_d;
	    next;
	}
	$conffilefound = 1;
	if ($conffile =~ /nntpsend-?(\d+)\.conf$|nntpsend\.conf-?(\d+)$/) {
	    $hasnumberedconfigs = 1;
	    $feednumber = $1;
	    warn "this is the config for nntpsend $feednumber\n" if $opt_d;
	}
	while(<F>) {
	    next if (/^\#/);
	    next if (/^$/);
	    next if (/^default:/);
	    ($peer,$ipname) = split(/:/);
	    $sitelist{$peer}++;
	    $conflist{$peer} = $ipname;
	    $feedprog{$peer} = 'nntpsend';
	    $feedid{$peer} = $nntpsendnumber;
	}
    }
    die "couldn\'t find any config files - aborting\n" unless $conffilefound;
    if ($opt_d) {
       warn "\nfound following sites:\n";
       foreach $peer (sort keys %conflist) {
	   warn "$peer: $conflist{$peer}, $feedprog{$peer}:$feedid{$peer}\n";
       }
       warn "\n\n";
    }
}



# onesite
# -----------------------------------------------------------------------------
# calculate for one site

sub onesite {
   local($i,$s);
   $s = $opt_i;

   print "\ncalculate onesite: $s\n" if ($opt_d);
   foreach $i (keys %sitelist) {
      if ($i ne $s) { delete $sitelist{$i}; }
   }
   if ((keys %sitelist) == 0) {
      print "site $s not found in config --> abort\n";
      exit;
   }

}


# printsite
# -----------------------------------------------------------------------------
# assemble result line for one site, do NOT print yet !!

sub printsite {
   local($c) = @_;

   $thecount{$site}=$c;
   $theconflist{$site}=$conflist{$site};
   $thefeedprog{$site}=$ABBREV{$feedprog{$site}};
   $thefeedid{$site}=$feedid{$site};

   if ($deltat>300) {
	   $backlog=int(3600*($c-$backlog{$site})/$deltat);
   } else {
	   $backlog="&nbsp;";	
   }
   $backlog="=" if ($backlog==0);
   $backlog="+".$backlog if ($backlog>0);

   if ($opt_w) {
      $c = "~ " . $c if ($appr);
      if ($hasnumberedconfigs && $feedid{$site} eq "") { # html cosmetics
	  $feedid{$site} = "&nbsp;"; 
      }
      $siteentry = $conflist{$site};  # set default
      if ($use_links) {               # add URL for html-output if available
	  if ($srvrname{$conflist{$site}}) {
	      $siteentry = "<A HREF=\"$moni_url{$srvrname{$conflist{$site}}}\">$conflist{$site}</A>";
	      warn "URL ($conflist{$site}): $moni_url{$srvrname{$conflist{$site}}}\n" if $opt_d;
	  } else {
	      ($truncated) = ($conflist{$site} =~ /^[^.]+\.([^.]+\..*)$/);
	      if ($truncated ne "" && $srvrname{$truncated}) {
		  warn "URL (truncated: $truncated): $moni_url{$srvrname{$truncated}}\n" if $opt_d;
		  $siteentry = "<A HREF=\"$moni_url{$srvrname{$truncated}}\">$conflist{$site}</A>";
	      } else { warn "no URL available for $conflist{$site}...\n" if $opt_d; }
	  }
      }

      $result{$site} = "<TR>
                 <TD> $site </TD>
                 <TD> $siteentry </TD>
                 <TD align=left> $ABBREV{$feedprog{$site}} </TD>
                 <TD align=right> $feedid{$site} </TD>
                 <TD align=right> $c </TD>
                 <TD align=right> $backlog </TD>
                 </TR>\n";
   } else {
      $c = "~ " . $c if ($appr);
      $result{$site} = sprintf("%15s %35s %8s %2s %10s\n",
		       $site,$conflist{$site},$ABBREV{$feedprog{$site}},
		       $feedid{$site},$c);
   }
}


# initpage
# -----------------------------------------------------------------------------
# Print Page Headers 
#
sub initpage {
    local($feedtypecaption) = $hasnumberedconfigs ? "Type + idx" : "Type";
    if ($opt_w) { 
	print OUT "$htmlheader
                   <p><b>calculated at:</b> $date</p>\n";
        print OUT "<center><TABLE BORDER>
                   <TR><TH align=left>Feedname</TH>
                   <TH align=left>Sitename</TH>
                   <TH colspan=2 align=center>$feedtypecaption</TH>
                   <TH>Articles<br>to transfer</TH>
                   <TH>Evolution<br>art/h</TH>
                   </TR>\n";
    } else {
        print OUT "$titleline\n\n";
        print OUT "calculated at: $date\n";
	printf(OUT "%15s %35s %11s %10s\n\n",
	       "Feedname","Sitename",$feedtypecaption,"Articles to transfer");
    }
}


# writedata
# -----------------------------------------------------------------------------
# sort and write results
#
sub writedata {
    foreach $site (sort by_count keys (%result)) {
	print OUT "$result{$site}\n";
    }
}

sub by_count 
{
        $thecount{$b} <=> $thecount{$a};
}


# closepage
# -----------------------------------------------------------------------------
# close page, write footers...
#
sub closepage {
    if ($opt_w) {
        print OUT "</table></center><P><P>
<FONT SIZE=\"-1\">Table created by $0 $RELEASE.
<A HREF=\"$INFLOWURL\">inflow</A> is a project of the
<A HREF=\"http://www.switch.ch/netnews/wg/netnews-wg.html\">
RIPE netnews-wg</A>.</FONT><P><P>
$htmlfooter\n";
    } else { 
	print OUT "\n\n"; 
    }
}
close(O);
if ($opt_F) {
	rename $OUTFILE, $OUTFILE.".old";
	rename $OUTFILE.".tmp", $OUTFILE;
}


# gethostandfqdn
#---------------------------------------------------------------------- 
# construct fully qualified domain name...
#
sub gethostandfqdn {
    chop($str=`$UNAME -n`);
    if ($str =~ /\./) {             # str is fqdn
	$fqdn = $str;
	($hostname) = ($str =~ /^([^.]+)\./);
    } else {                        # str is simple hostname
	$hostname = $str;
	$str = `$GREP domain $RESOLVCONF`;
	$str =~ /domain\s*(\S+)$/;
	$fqdn = $hostname . "." . $1;
    }
}


# get_urllookup
#----------------------------------------------------------------------
# read URL info from config file
#
# file format:
# <name> <srvrname> <operator_code> <monitoring-URL>
sub get_urllookup {
    if ($URLLOOKUP eq "") {
	warn "no lookup table specified\n";
	$use_links = 0;
	return;
    }
    unless (open(LOOKUP,"$URLLOOKUP")) {
        warn "unable to open lookup table $URLLOOKUP\n";
        $use_links = 0;
        return;
    }
    $use_links = 1;
    while (<LOOKUP>) {
	chop;
	next if (/^\#|^$/);
	($name,$srvr,$code,$url) = split;
	$srvrname{$name} = $srvr;
	$operatedby{$srvr} = $code;
	$moni_url{$srvr} = $url;
	warn "get_urllookup: name=$name  srvr=$srvr  url=$moni_url{$srvr}\n" if $opt_d;
    }
}


# update_PATH
# -----------------------------------------------------------------------------
# enhance PATH to support scripts run from cron
#
sub update_PATH {
    warn "OLD PATH=$ENV{'PATH'}\n" if $opt_d;
    @ENVPATH = split(/:/,$ENV{'PATH'});
    push(@ENVPATH,@ADDTOPATH);
    
    $ENV{'PATH'}='';
    foreach $p (@ENVPATH) {
	next if $envpathseen{$p}; # skip duplicates
	$envpathseen{$p} = 1;
	$ENV{'PATH'} .= $p . ":";
    }
    chop($ENV{'PATH'});           # cut last ":"
    warn "NEW PATH=$ENV{'PATH'}\n" if $opt_d;
}


# interpret
# -----------------------------------------------------------------------------
# interpret variable read from file; variables have to be separated by
# white space
#
sub interpret {
    local($line) = ""; 
    @words = split;
    foreach $w (@words) {
	if ($w =~ /^\$/) { eval "\$w=$w"; }
	$line .= "$w ";
    }
    return "$line\n";
}


# modify_config
# -----------------------------------------------------------------------------
#
sub modify_config {
    local($p,$n,$incpath);
    $inflowconf = $opt_f ? $opt_f : $INFLOWCONF;
    if ($inflowconf =~ /^(.*)\/([^\/]+)$/) {
	$p = $1; 
	$n = $2;
	push(@INC,$p);
    } else { 
	$n = $inflowconf; 
    }
    foreach (@INC) { $incpath .= "    $_\n"; }
    if (-e $inflowconf) {
	warn "loading local configs from $inflowconf...\n" if $opt_d;
	require $n;
	$inflowconfinfo = "$inflowconf $CFRELEASE\n";
	$inflowconfinfo .= " " x 16 . "from: $CFRELDATE";
    } else {
	if ($opt_f) {
	    die "abort - couldn't find or open $n on INC=\n$incpath\n";
	} 
	warn "didn't find any local configs $n on INC=\n$incpath\n" if $opt_d;
	$inflowconfinfo = "- none -";
    }
}
