#!/afs/athena/contrib/perl/p

require 'chat2.pl';
print("Good morning!\n");
open(DIN, ">/afs/sipb/project/www/index/dindex");
open(KIN, ">/afs/sipb/project/www/index/kindex");
$min = 4;
$num = 2;
select(DIN);$|=1;select(KIN);$|=1;select(STDOUT);
while(<>){
    print "\n--->".$_;
    $n++;
    chop;
    $url=$_;
    ($garbage, $garbage, $sithost, @path)= split('/', $_);
    $path = join('/', @path);
    ($site, $port) = split(':', $sithost);

    $handle =&chat'open_port($site, $port) || do {print("Failed following $site:$port\n");};
    print("Opened connection to $site:$port $path\n");
    &chat'print("GET /$path\n");
    $body='';
    $body = &listen(5);
    
    $body =~ /<title>(.+)<\/title>/i;
    $title = $1;

    print(DIN "$n$url$title$site\n");
    print("Writing record $n with url $url and title $title\n");
    print("Keywords for record $n: ");
    print(KIN "$n;");
    $body=~s/<[^>]+>//g;
    $body=~tr/A-Z/a-z/;

    @words = split(/[ \t\n]/, $body);
    %INST = ();
    foreach $word (@words)
    {
	$INST{$word}++;
    }
				# 
    foreach (sort byvalue keys %INST) # 
    {
	$times = $INST{$_};
	if($times >= $num)
	{			# 
	    if(length >= $min)
	    {			# 
		print(KIN "$_ ") if /[a-zA-Z]/;
		print("$_ ") if /[a-zA-Z]/;
	    }			# 
	}				# 
    }
	print(KIN "\n");	# 
	print("\n");
}
		
close(DIN);close(KIN);		# 
sub byvalue { $INST{$b} <=> $INST{$a}; }

sub listen {
        local($secs) = @_;
        local($return,$tmp) = "";
        while (length($tmp = &chat'expect($secs, '(.|\n)+', '$&'))) {
                print $tmp if $trace;
                $return .= $tmp;
        }
        $return;
}