#!/afs/athena/contrib/perl/p

require 'chat2.pl';
$SIG{'INT'}='report';
$SIG{'HUP'}='report';
$SIG{'QUIT'}='report';
$href=shift;
$commonwords = ' the to of and a is or at in on are can about you if please also comments information for document link that by here an ';
$port = 80;

($siteport, @path) = split('/', $href);
($site, $port)=split(':', $siteport);
for (@path){
#    print "Path member $pn--$_--\n";
    if($_ eq '..'){
#	print("Removing...($pn)\n");
	splice(@path, $pn-1, 3);
	$pn -=2;
    }
    if($_ eq '.'){
	splice(@path, $pn, 1);
	$pn -=1;
    }
    $pn++;
}
$path = join('/', @path);
$port = 80 if !$port;

print("Going to $site on port $port, getting $path\n");
$handle =&chat'open_port($site, $port) || do {print("Failed following $site:$port\n");};
&chat'print("GET /$path\n");
$doc=&listen(10);
&chat'close($handle);

$doc =~ s/<[^>]+>//g;

#print("********\n$doc\n***********\n");

@lines=split(/\n+/, $doc);

for $_ (@lines){
s/[\.\,\?\!\'\(\)\*\-\`\{\}\\]/ /g;
@words=split;
        foreach $word (@words)
                {
		    $tot++;
		    $word =~ tr/A-Z/a-z/;
#               if(!defined($INST{$word})){print(".");}
                $INST{$word}++;
                }
}
$num = $tot/150;
$num = $num <2 ? 2:$num;

foreach (sort byvalue keys %INST)
{
    $times = $INST{$_};
    if($times >= $num)
    {
	if(length >= $min)
	{
	    if($commonwords !~/ $_ /i){
		print"$_ ($times)\n";
	    }
	}
    }
    
}
sub byvalue { $INST{$b} <=> $INST{$a}; }
sub listen {
        local($secs) = @_;
        local($return,$tmp) = "";
        while (length($tmp = &chat'expect($secs, '(.|\n)+', '$&'))) {
                print $tmp if $trace;
                $return .= $tmp;
                (return $return) if (length($return) > 100000);
        }
        $return;
}