#!/usr/local/bin/perl
#
# $Id: digest.pl,v 1.2 94/02/10 00:15:50 bert Exp $

require 'ctime.pl';

# configuration

$DEBUG = 1;
# $DUMP = 1;

$path =     '/var/local/www/logs/test';     # logs directory
@servers =  ('plexus', 'diswww');      # server names
@methods =  ('get', 'post');           # server names
@logtypes = ('host', 'path', 'proto'); # log types

# helper...

for (@methods) { $metOK{$_} = 1; }
for (@servers) { $srvOK{$_} = 1; }

# run stuff

print "running...\n" if $DEBUG;

&eat_daily_syslog( "$path/syslog.0" );

print "yum, good syslog.\n" if $DEBUG;

umask(002);			# no, *not* 664 =)

if ($DUMP) {
    for $serv (@servers) {
	for $req (@methods) {
	    for $log (@logtypes) {
		&dump( "${req}_${serv}'${log}" );
	    }
	}
    }
}

for $serv (@servers) {
    for $req (@methods) {
	for $log (@logtypes) {
	    &update_stats( "${req}_${serv}'${log}",
                                               "$path/${req}.${serv}.${log}" );
	}
	&update_list( "${req}_${serv}'daily", "$path/${req}.${serv}.daily" );
	print "done with ${req}.${serv}.daily\n" if $DEBUG;
    }
}

print "Ha.\n" if $DEBUG;

do generate_report;

### subroutines

sub eat_daily_syslog {
# Parse the incoming syslog into meaningful(?) information.

  local(*STATS, $name, $found, $mon, $day, $time, $shost, $who, @req);

  open(STATS, @_[0]);

  LINE:
  while (<STATS>) {
      ($mon, $day, $time, $shost, $who, @req) = split(/\s+/);

# Who's talking?

      ($srv, $pid) = ($who =~ /^(.*)\[(.*)\]/);
      undef $found;
      for $name (@servers) {
	  ($found = 1) if ($srv eq $name);;
      }
      unless ($found) {
	  push(@WSERV, $_);
	  next LINE;
      }

# Decide what the servers are trying to say to us.

      # error logs
      if ($req[0] =~ /^ERROR/) { push (@ERRORS, $_); next LINE; }
      # server restarts
      if ($req[0] =~ /^----[^-]/) { push (@RESTARTS, $_); next LINE; }
      # requests...
      undef $found;
      $req[1] =~ tr/A-Z/a-z/;
      if ($metOK{$req[1]}) {
	  $req[0] =~ tr/A-Z/a-z/;

	  eval <<"EndOfInc";	     # use $get_plexus'host, etc.
	  \$$req[1]_${srv}'host{\$req[0]}++;
	  \$$req[1]_${srv}'path{\$req[2]}++;
          \$$req[1]_${srv}'proto{\$req[3]}++;
	  push(\@$req[1]_${srv}'dlist, "\$mon \$day")
                      unless defined(\$$req[1]_${srv}'daily{"\$mon \$day"});
	  \$$req[1]_${srv}'daily{"\$mon \$day"}++;
EndOfInc
      } else {
          push (@WEIRD, $_);
      }
  }
}

sub generate_report {
# format and output errors, etc.

    return unless (@FATAL || @ERRORS || @WEIRD || @WSERV || @RESTARTS);

    print "\nReport of daily run of $0 -- ", &ctime(time);

    if (@FATAL) {
	print "\nFatal error(s) in $0 (DATA MAY BE CORRUPT OR LOST)\n   ----------\n";
	print @FATAL, "   ----------\n";
    }
    if (@ERRORS) {
	print "\nServer errors encountered:\n   ----------\n";
	print @ERRORS, "   ----------\n";
    }
    if (@WEIRD) {
	print "\nUnclassified syslogs:\n   ----------\n";
	print @WEIRD, "   ----------\n";
    }
    if (@WSERV) {
	print "\nSyslogs from unexpected sources:\n   ----------\n";
	print @WSERV, "   ----------\n";
    }
    if (@RESTARTS) {
	print "\nServer restarts:\n   ----------\n";
	print @RESTARTS, "   ----------\n";
    }
}

sub dump {
# generate a dump

    eval <<"EndOfCode";
    local(\$a,\$b);
    printf "---%-20s---------------\n", "@_[0]";
    while ((\$a,\$b)=each \%@_[0]) { printf " %6d %s\n",\$b,\$a; }
    printf "---%-20s---------------\n", "-"x20;
EndOfCode

    push(@FATAL, "dump failed for @_[0]:\n" . $@) if $@;
}

sub update_stats {
# update the sorted data in file @_[1] from variable @_[0] (which is changed)

    local(*STATS, $foo, $cnt, $key, $ign, $repc);
    local(@ignore, @replace) = ();

    eval <<"EndOfCode";
    open (STATS, \@_[1]) || die "can't open file \@_[1] in pass 1";
    while (<STATS>) {
	chop;
	(\$foo,\$cnt,\$key) = split(/[ \t\n]+/, \$_, 3);

	if (defined(\$@_[0]{\$key})) {
	    \$@_[0]{\$key} += \$cnt;
	    push( \@ignore, \$key );
	}
    }

    \@replace = sort {-(\$@_[0]{\$a}<=>\$@_[0]{\$b})} keys \%@_[0];

    print "middle of @_[1]\n" if $DEBUG;

    open (STATS, \@_[1]) || die "can't open file \@_[1] in pass 2";
    open (NEW, "> \@_[1].new") || die "can't open file \@_[1].new for writing";
    \$ign = shift(\@ignore);
    \$repc = \$@_[0]{\$replace[0]};
    while (<STATS>) {
	chop;
	(\$foo,\$cnt,\$key) = split(/\\s+/, \$_, 3);

	while ( \@replace && (\$cnt <= \$repc) ) {
	    printf NEW " %6d %s\n",\$repc,shift(\@replace);      # PRINT FORMAT
	    \$repc = \@replace ? \$@_[0]{\$replace[0]} : undef;
	}
	if ( \$key eq \$ign ) { \$ign = shift(\@ignore); }
	else { printf NEW " %6d %s\n",\$cnt,\$key; }             # PRINT FORMAT
    }
    for (\@replace) { printf NEW " %6d %s\n",\$@_[0]{\$_},\$_; } # PRINT FORMAT
    close(STATS);
    close(NEW);

    rename( \@_[1], "\@_[1].old" ) || die "can't rename \@_[1] to .old";
    rename( "\@_[1].new", \@_[1] ) || die "can't rename \@_[1] from .new";
EndOfCode

    push(@FATAL, "update_stats failed for @_[0]:\n" . $@) if $@;
    print "done with @_[1]\n" if $DEBUG;
}

sub update_list {
# update the list in file @_[1] from variable @_[0] (which is changed)

    local(*STATS, $foo, $cnt, $key, $ign, $repc);

    eval <<"EndOfCode";
    open (STATS, \@_[1]) || die "can't open file \@_[1]";
    open (NEW, "> \@_[1].new") || die "can't open file \@_[1].new for writing";
    while (<STATS>) {
	chop;
	(\$foo,\$cnt,\$key) = split(/\\s+/, \$_, 3);

	printf NEW " %6d %s\n",\$cnt+\$@_[0]{\$key},$key;	# PRINT FORMAT
	delete \$@_[0]{\$key};
    }
    while ((\$key,\$cnt) = each \%@_[0]) {
	printf NEW " %6d %s\n",\$cnt,\$key;			# PRINT FORMAT
    }
    close(STATS);
    close(NEW);

    rename( \@_[1], "\@_[1].old" ) || die "can't rename \@_[1] to .old";
    rename( "\@_[1].new", \@_[1] ) || die "can't rename \@_[1] from .new";
EndOfCode

    push(@FATAL, "update_list failed for @_[0]:\n" . $@) if $@;
    print "done with @_[1]\n" if $DEBUG;
}