#!/afs/athena/contrib/perl/p

$/ = '';
$|=1;
$commonwords = ' the to of and a is or at in on are can about you if please also that by here an from this your do with have be work when will as use it how but could my thanks for would does has they any hi am get which some like i was ';


$*=1;
while(<>){
    $csa = $keywords = '';
    if($_ =~ /Question:\n([\w\W]+)Machine info:/){
    $question = $1;		# 
    @lines = split("\n", $question);

    $qnum = substr($_, 2, index($_, "\]")-2);
    $_ =~ /___________________________________________________________\n([\W\w]+)--\[\d+\]--/;
    $answer = $1;

sub make_keywords {
    @lines = split("\n", $question);
    for $_ (@lines){
	        s/\W/ /g;
        @words=split;
        foreach $word (@words)
        {
            $tot++;
            $word =~ tr/A-Z/a-z/;
#               if(!defined($INST{$word})){print(".");}
            $INST{$word}++;
        }
    }

    $num = $tot/200;

    foreach (sort byvalue keys %INST)
    {                           #
        $times = $INST{$_};             #
        if($times >= $num)              #
        {                               #
            if(length >= $min)  #
            {
                if($commonwords !~/ $_ /i){
                    $keywords .= $_.' ';
                    print("$_\n") unless $keyw{$_};
                    $keyw{$_}=1;
                }                       #
            }                   #
        }

    }
}

    while($answer =~ /^([ \t]+(\d+[\.*][ \t]+(.+)))$/g){
	$sa = $3;
	print("Getting name of $sa\n");
	next if($sa =~/^[A-Z ]+ Answers$/);
	$name_of_sa = `grep \"$sa\" /afs/sipb/user/mkgray/olc/index`;
	print("---> $name_of_sa\n");
	next if($name_of_sa eq '');
	($garbage, $garbage, $thisisit, @garbage)=split(':', $name_of_sa);
	print("Got it! $thisisit ($name_of_sa)\n");
	(@it) = split('/', $thisisit);
	$thisisit = pop(@it);
	$filename = `grep $thisisit /afs/sipb/user/mkgray/olc/index2`;
	$keywords='';%INST=();$tot=0;
	&make_keywords;
	$KWS{$filename} .= $keywords." ";
	print("Adding ($keywords) to $sa\n");
    }
#    if($csa ne ''){print("----------\nQuestion $qnum:\n$question\nAnswer:\n$csa\n");}
#    print("Q--> $question\n$answer\n");
    print ("$qnum\n"); # if ($qnum%10 == 0);
}
}


for $file (keys %KWS){
    print("$file $KWS{$file}\n");
}
sub byvalue { $INST{$b} <=> $INST{$a}; }