#!/usr/bin/perl

unshift(@INC, "/netgen/www/comprehensive/quick250");
require 'should.pl';            # Should and auxilliary routines
require 'log.pl';               # Logging routines & badlink
require 'analyze.pl';           # Analyze document.  get_* & is_real...
require 'queue.pl';             # Qeueuing routines
require 'url.pl';               # get_url and auxilliary stuff
require '../src/util.pl';

@mylist = ();
$myn =$addn= 0; $| =1;
mkdir("/usr/tmp/quick.$$", 0777);
open(ALL, "/tmp/allservers");
open(LST, ">>/usr/tmp/quick.$$/$addn");
while(<ALL>){
    $thislist++;
    if($thislist >250){
	print("Writing list $addn\n");
	close(LST); $addn++;
	open(LST, ">>/usr/tmp/quick.$$/$addn");
	$thislist=0;
    }
    print LST $_;
}
    
#open(URL, ">/usr/tmp/quick.$$/0");
#print URL $ARGV[0]."\n";
#close(URL);

print("Initing logger\n");
&logger'init;
print("Generic Init\n");
&Init;

while($addn < 110000){
    if($#mylist >= 0){
	$url = shift(@mylist);
	print("checking memory for $url\n");
#	next if (`grep -F -x $url /usr/tmp/quick.$$/memory` ne "");
	print("Is it a hash?\n");
	next if $url =~ /#/;
	$addn++;
	print("Getting $url\n");
	($hdrs, $doc) = &get_url($url);

        if($hdrs =~ /Server: (.+)/){
		$servertype=$1;
$servertype=~s/\r//g;
        }
	else{
		$servertype="__Unknown__";
	}
	print("Server at [$url] is of type \"$servertype\"\n");

#&process_doc($url,$hdrs, $doc) if $addn==1;
}
else{
	&load_list($myn++);
}

}

sub load_list {
    local($file) = @_;
    print "Loading $file\n";
    if(-e "/usr/tmp/quick.$$/$file"){
	open(FILE, "/usr/tmp/quick.$$/$file");
	while(<FILE>){
	    chop;
	    print("Putting $_ on list\n");
	    push(@mylist, $_);
	}
	print("Done.\n");
    }
    else{
	print "Waiting...\n";
	sleep(5);
	&load_list($file);
    }
}

sub process_doc {
    local($url, $hdrs, $doc) = @_;

    print("Processing $url\n");
    ($site, $path, $port) = &url'ParseURL($url);
    return if $beenhere{$site};
    $beenhere{$site}++;
    if($hdrs =~ /Server: (.+)/){
       $server=$1;
       $server=~s/[\n\r]//g;
       print("Server at site $site ($url) is running \"$server\"\n");
    }


}
