#!/afs/athena/contrib/perl/perl

require '/mit/mkgray/perl/chat2.pl';

# Set some filenames
$sites = "sites";
$finished = "finished";
$diskqueue = "dq";
$tmpqueue = "tmpq";
$logfile = "log";



open(FIN, ">>$finished");
open(SITES, ">>$sites");
open(LOG, ">>$logfile");
select(SITES);$|=1;
select(FIN);$|=1;
select(LOG);$|=1;
select(STDOUT);


$MyURL = $ARGV[0];
$MyURL =~ m#http://([^:/]+):?(\d*)/(.*)#;
$MySite = $1;  $MyPort = $2;  $MyPath = $3;

&wwwfollow($MySite, $MyPath, $MyPort, print("Testing!!!\n"));


#
#  The important routine
#
sub wwwfollow {
    print("Yeah, i made it into the subroutine!\n");
    local($mysite, $mypath, $myport) = (@_);
#    local($href, @hrefs, $port, $path, $site);

    print("Doing traversal of $mysite, $mypath on $myport\n");

#    chop($mysite) if(substr($mysite, length($mysite)-1) eq '.');

# Check if a queue cleanup is needed at this point
    if($#queue < 2 || $#addtodq > 150){
	print("Doing queue cleanup due to $#queue left on the queue and $#addtodq to go on\n");
	&queuecleanup();
    }


#
#   Check to see if I've already visited this URL or if it is already
#   queued
#
    return if &fg("http://$mysite:$myport$mypath", $diskqueue);
    return if &fg("http://$mysite:$myport$mypath", $logfile);
    return if grep(m^http://$mysite:$myport$mypath^, @addtodq);

#
#   Fix port number to default
#
    if(!$myport){ $myport=80; }


#
#  Make connection and get the document
#
    print("Traversing $mysite:$myport$mypath (q: $#queue, dq: $#addtodq)\n");
    $handle =&chat'open_port($mysite, $myport) || do {$! = "Malformed Link";return 0;};
    &chat'print("GET $mypath WWWWanderer\n\n");
    $doc=&listen(30);
    &chat'close($handle);

#  Initialize statistics counters
    $links = 0;
    $title = '';
    
#
#  Primitive parse HTML
#
    while($doc=~/href=\"([^\n\"]+)\">/gi){
	$href = $1;
	if(substr($href, 0, 4) ne 'http' && $href !~m,//,){ # 
	    $myspath = substr($mypath, 0, rindex($mypath, '/'));
	    if(substr($href, 0 ,1) eq '/'){
		$href =  "http://$mysite:$myport$href";
	    }
	    else{
		$href = "http://$mysite:$myport$myspath/".$href;
	    }
	}
	
#	else{print "Begins with http: $href\n";}

	if(!&visited($href)){
	    push(@addtodq, $href);
#	    print("Added $href to dq\n");
	}
	$links++;
    }
#  Fetch Title?
    if($doc=~/<title>([^<]*)<\/title>/){
	$title = $1;
    }


    print(LOG "http://$mysite:$myport$mypath $links <<$title>>\n");
    &queuecleanup if ($#queue < 1);

    
    while($href = shift(@queue)){
	print("Trying $href\n");
	($site, $path, $port) = &parseurl($href);
	print("trying... ($site, $path, $port)\n");
	if($site eq $MySite){
	    if(!&wwwfollow($site, '/'.$path, $port, print("hello\n"))){
		print("$!: $href in document http://$mysite$myport$mypath\n");
	    }
	}
    }
    print(FIN "$site:$port/$path\n");
    &queuecleanup;

    ($site, $path, $port) = &parseurl(shift(@queue));
    &wwwfollow($site, '/'.$path, $port);
    
}

sub visited {
    return 1 if grep(/$href/, @queue);
    return 1 if grep(/$href/, @addtodq);
    return 1 if &fg($href, $diskqueue);
    return 1 if &fg($href, $logfile);
    
    return 0;
}


sub queuecleanup {

    print("--------------Doing Queue Cleanup-------\n");
    open(DQ, ">>$diskqueue");
    for $qitem (@addtodq){
	if(&fg($qitem, $diskqueue)){
	    print("Hey, why am I adding $qitem again???????????\n");
	}
	print(DQ $qitem."\n") unless $qitem eq '';
#	print("Added $qitem to disk\n");
    }
    @addtodq=();


    open(DQ, "$diskqueue");
    for $i (1..(12-$#queue)) {
	chop($item = <DQ>);
	push(@queue, $item) unless $item eq '';
#	print("added $item to queue\n");
    }
    open(TMP, ">>$tmpqueue");
    while(<DQ>) {
	print TMP;
    }
    close(TMP);
    close(DQ);
    unlink($diskqueue);
    `mv $tmpqueue $diskqueue`;
    print("-----Done----\n");
}


sub report {
    print("Ack!\n");
    exit(0);
}

sub listen {
    local($secs) = @_;
    local($return,$tmp) = "";
    while (length($tmp = &chat'expect($secs, '(.|\n)+', '$&'))) {
        print $tmp if $trace;
        $return .= $tmp;
        (return $return) if (length($return) > 100000);
    }
    $return;
}

sub fg {
    local($expr, $file);

    open(FILE, $file);
    while(<FILE>){
	split;
	if ($_[0] eq $expr){
	    return 1;
	}
    }
    return 0;
}

sub parseurl {
    local($url) = @_;

    ($proto, $garbage, $siteport, @path) = split('/', $url);
    next if ($proto ne 'http:');
	($site, $port)=split(':', $siteport);
    $pn = 0;
    for (@path){
	if($_ eq '..'){
	    splice(@path, $pn-1, 3);
	    $pn -=2;
	    }
	if($_ eq '.'){
	    splice(@path, $pn, 1);
	    $pn -=1;
	}
	$pn++;
    }
    $path = join('/', @path);
    return($site, $path, $port);
}