#!/afs/athena/contrib/perl/p

require 'chat2.pl';
$SIG{'INT'}='report';
$SIG{'HUP'}='report';
$SIG{'QUIT'}='report';
$commonwords = ' the to of and a is or at in on are can about you if please also comments information for document link that by here an from heres this ';

open(LOG, "/mit/mkgray/Public/ht/all.net");
print("Loading all.net\n");
while(<LOG>){
    chop;
    $things{$_}=1;
}
open(LOG2, "/mit/mkgray/Public/ht/all.sites");
print("Loading all.sites\n");
while(<LOG2>){
    chop;
    $sites{$_}=1;
}

open(LOG, ">>/mit/mkgray/Public/ht/all.net");
open(IDX, ">>/mit/mkgray/Public/ht/all.idx");
open(LOG2, ">>/mit/mkgray/Public/ht/all.sites");
open(KEY, ">>/mit/mkgray/Public/ht/all.key");
select(LOG);$|=1;
select(LOG2);$|=1;
select(IDX);$|=1;
select(KEY);$|=1;
select(STDOUT);

$n = '1';
    &traverse('www.ncsa.uiuc.edu', '/demoweb/demo.html', 80);
&report();
sub report{
print("\nFound the following things:\n");
#for (keys %things){
#    print $n++.') ';
#    print;
#    print "\n";
#}
exit(0);
}
sub traverse {
    local($mysite, $mypath, $myport) = @_;
    local($href, @hrefs, $keywords, $port, $path, $site);
    chop($mysite) if(substr($mysite, length($mysite)-1) eq '.');
    $keywords = '';

    #cleanup and check if I've been here
    $mysite=~s/sipb/www/;
    print ";";
    return if defined($things{"http://$mysite:$myport$mypath"});
    return if defined($started{"http://$mysite:$myport$mypath"});
    $started{"http://$mysite:$myport$mypath"}=1;
 


    #deal with boredom
   $bpath = substr($mypath, 0, ($mypath =~ /\?/) ? index($mypath, '?'):rindex($mypath, '/'));
    if($bpath eq $boring){
	$bored++;
	if($bored >10){
	    $bored = 0;
	    print("I'm bored of this.\n");
	    print(LOG "I'm bored...\n");
	    $getmeout=1;
	}
    }
    else{
	$boring = $bpath;
	$bored = 0;
	$getmeout=0;
    }
    return if $getmeout;




    #  Personalized things to skip
    return if $mypath=~/^\/discuss\?/;
    return if $mysite eq 'info.cern.ch' && $myport == 8004;
#    return if $mypath=~/humor/;
    return if $mypath=~/news:/;
    return if $mysite eq 'LIBROT1.LIB.UNC.EDU';
    return if ($mysite eq 'info.cern.ch.' && $myport == 8002);
#    return if ($mysite =~/info.cern.ch/ && $mypath =~/hypertext/ && $mypath !~/Servers\.html/);
    return if ($mysite eq 'www.vuw.ac.nz' && $mypath =~/entries/);
    return if ($mysite eq 'geb.phys.washington.edu' && length($mypath) > 20);

    if(!$myport){
	$myport=80;
    }

    #Getting hypertext
    print("Traversing $mysite:$myport$mypath\n");
    $handle =&chat'open_port($mysite, $myport) || do {print("Failed following $mysite:$myport\n");return;};
    &chat'print("GET $mypath\n");
    $doc=&listen(10);
    &chat'close($handle);


     
    return if $doc=~/Directory Index of/;
    return if $doc=~/404 Not Found/;
    return if $doc=~/unable to access document/i;

#    $things{"http://$mysite:$myport$mypath"} =1;

    print(LOG2 "$mysite:$myport\n") unless $sites{"$mysite:$myport"};
    $sites{"$mysite:$myport"}=1;


    while($doc=~/href=\"([^\n\"]+)\">/gi){
	$href = $1;
#	print("Found href on $mypath of $href\n");

	if(substr($href, 0, 4) ne 'http' && $href !~m,//,){ # 
	    $myspath = substr($mypath, 0, rindex($mypath, '/'));
	    if(substr($href, 0 ,1) eq '/'){
		$href =  "http://$mysite:$myport$href";
	    }
	    else{
		$href = "http://$mysite:$myport$myspath/".$href;
	    }
	}

#	else{print "Begins with http: $href\n";}
	if(!defined($things{$href})){
	    push(@hrefs, $href);

	}
    }

    $doc =~ tr/A-Z/a-z/;
    $titlewords='';
    $doc =~ s/\<title\>(.+)\<\/title\>/THIS WAS THE TITLE/;
    $titlewords = $1;
    print("-----Title of document is $titlewords\n");
    @twords = split(' ', $titlewords);
    for $w (@twords){
	$w =~s/\W//g;
	if($commonwords !~/ $w /){
	    print("Tadding $w as a keyword to current document ($mypath)\n");
	    $keywords .= $w." ";
	    print("===========v\n$w\n============^\n") unless $keyw{$w};
	    $keyw{$w}=1;
	}
    }
    
    $doc =~ s/<[^>]+>//g;
    $doc =~ s/[^\w ]//g;
    @lines=split(/\n+/, $doc);

    for $_ (@lines){
	s/[\.\,\?\!\'\(\)\*\-\`\{\}\\]/ /g;
	@words=split;
        foreach $word (@words)
	{
	    $tot++;
	    $word =~ tr/A-Z/a-z/;
#               if(!defined($INST{$word})){print(".");}
	    $INST{$word}++;
	}
    }

    $num = $tot/150;
    $num = $num < 2 ? 2:$num;
    $min = 2;			# 

foreach (sort byvalue keys %INST)
{				# 
    $times = $INST{$_};		# 
    if($times >= $num)		# 
    {				# 
	if(length >= $min)	# 
	{
	    if($commonwords !~/ $_ /i){
		print("Adding $_ as a keyword to current doc ($mypath)\n");
		$keywords .= $_.' ';
		print("===========v\n$_\n============^\n") unless $keyw{$_};
		$keyw{$_}=1;
	    }			# 
	}			# 
    }
				# 
}				# 
sub byvalue { $INST{$b} <=> $INST{$a}; }




				# 
return if defined($things{"http://$mysite:$myport$mypath"});
for $href (@hrefs){
#	print("Trying $href\n");
	($proto, $garbage, $siteport, @path) = split('/', $href);
	next if ($proto ne 'http:'); # 
	($site, $port)=split(':', $siteport);
#	print("Site $site:$port\n");
	$pn = 0;		# 
	for (@path){		# 
#	    print "Path member $pn--$_--\n";
	    if($_ eq '..'){	# 
#		print("Removing...($pn)\n");
		splice(@path, $pn-1, 3);
		$pn -=2;	# 
	    }			# 
	    if($_ eq '.'){	# 
		splice(@path, $pn, 1);
		$pn -=1;	# 
	    }			# 
	    $pn++;		# 
	}			# 

	$path = join('/', @path);
#	print("Path: $path\n");
	&traverse($site, '/'.$path, $port);
    }
$things{"http://$mysite:$myport$mypath"} =1;
print("*****Adding $titlewords~http://$mysite:$myport$mypath~$keywords~\n");
print(LOG "http://$mysite:$myport$mypath\n");
print(IDX "$titlewords~http://$mysite:$myport$mypath~$keywords~\n");
}      			# 

sub listen {
        local($secs) = @_;
        local($return,$tmp) = "";
        while (length($tmp = &chat'expect($secs, '(.|\n)+', '$&'))) {
                print $tmp if $trace;
                $return .= $tmp;
		(return $return) if (length($return) > 100000);
        }
        $return;
}