#!/usr/local/bin/perl

$version = "2.01";
$reportfile = "/usr/local/archives/public/html/users/deepak/LogReports/Stud/report.html";
$keywordsfile = "/usr/local/archives/public/html/users/deepak/LogReports/Stud/keywords.html";
$logfile = "/infomall/logs/logs-ro/main/referer_log";
$UseOldReport = 1;
$excludefrom = "(http://www.npac.syr.edu)";
$excludeto = "(\.gif|\.jpg|\.jpeg|\.xbm|http://www.npac.syr.edu)";
$StripWWW = 0;
$MinHits = 5;

##############
# PROCESSING #
##############

print "Processing Referer Log\n";

$DefaultPageName = "(index\.[^.]*|welcome\.[^.]*)";

if ($reportfile =~ /.*\/([^\/]+)/) {
	$reporturl = $1;
}

if ($keywordsfile =~ /.*\/([^\/]+)/) {
	$keywordsurl = $1;
}

if ($UseOldReport) {
	if (-e "$reportfile") {
		open (OLDLOG,"$reportfile") || die "  Error opening file: $reportfile\n";
		while (<OLDLOG>) {
			chop;
			if (m#^<P><DT><STRONG>(.*)</STRONG></DT>#o) {
				$target = &Simplify($1);
				next;
			}
			next if (! $target);
			if (m#^<DD><A HREF=\"([^"]*)\">.*</A> \(([\d,]+) reference#o) {
				$refurl = $1;
				$count = $2;
				$count =~ s/,//g;
				$url = &Simplify($refurl);
				if ($TargetCounter{"$target $url"}) { $TargetCounter{"$target $url"} += $count; }
				else { $TargetCounter{"$target $url"} = $count; }
			}
		}
		close (OLDLOG);
	}
}

&Get_Date;

open (REFSLOG,"$logfile") || die "  Error opening file: $logfile\n";

while (<REFSLOG>) {
	chop;
	s/\s+/ /go;
	if (($Dom,$rfc,$user,$Time,$Request,$Status,$Bytes,$referer,$Agent) =
	  /^(\S+) (\S+) (\S+) \[(.+)\] \"(.+)\" (\S+) (\S+) \"(.*)\" \"(.*)\"/o) {
		($method,$target,$protocol)=split(/\s/,$Request,3);
		if (length($referer) < 5) { $referer = ""; }
	} else {
		($referer,$target)=split(/ -> /,$_,2);
		if (length($referer) < 5) { $referer = ""; }
	}
	next if (! ($referer && $target));
	$target=~ s/\%7[eE]/~/o;
	$target=~ s#//#/#go;
	$target=~ s#^ *$#/#o;
	$target=~ s/#.+$//o;
	$target=~ s/\?.*$//o;
	$target=~ s/${DefaultPageName}$//oi;
	$target = &Simplify($target);
	$referer=~ s/\%7[eE]/~/o;
	$referer=~ s#^(http://[^/]+):80/#$1/#o;
	$refquery = "";
	if ($referer =~ m#^.+//([\w|\.|-]+)/.*\?(.+)#o) {
		$refdomain = $1;
		$refquery = $2;
	}
	$referer=~ s/#.+$//o;
	$referer=~ s/\?.*$//o;
	$referer=~ s/${DefaultPageName}$//oi;
	$referer = &Simplify($referer);
	unless (($excludefrom && ($referer =~ m#$excludefrom#oi))
	  || ($excludeto && ($target =~ m#$excludeto#oi))) {
		$keyform=$target.' '.$referer;
		$keyform=~ s#\&#\&amp\;#og;
		$keyform=~ s#<#\&lt\;#og;
		$keyform=~ s#>#\&gt\;#og;
		$keyform=~ s#"#\&quot\;#og;
		$referer=~ s#\&#\&amp\;#og;
		$referer=~ s#<#\&lt\;#og;
		$referer=~ s#>#\&gt\;#og;
		$referer=~ s#"#\&quot\;#og;
		$TargetCounter{$keyform}++;
	}
	else {
		$referer = "";
	}
	if ($referer && $refquery) {
		$refquery =~ tr/A-Z/a-z/;
		$refquery =~ s/\%.{2}/ /gi;
		$refquery =~ s/\+/ /g;
		$refquery =~ s/\-/ /g;
		$refquery =~ s/\s+/ /g;
		if (($refdomain =~ /altavista/i) && ($refquery =~ /\&q=\s*([^&]+)/i)) {
			$altavista{$1}++;
		}
		elsif ($refdomain =~ /netfind/i) {
			if (($refquery =~ /search=\s*([^&]+)/i)
			  || ($refquery =~ /\&s=\s*([^&]+)/i)) {
				$netfind{$1}++;
			}
		}
		elsif ($refdomain =~ /excite/i) {
			if (($refquery =~ /search=\s*([^&]+)/i)
			  || ($refquery =~ /\&s=\s*([^&]+)/i)) {
				$excite{$1}++;
			}
		}
		elsif (($refdomain =~ /hotbot/i) && ($refquery =~ /mt=\s*([^&]+)/i)) {
			$hotbot{$1}++;
		}
		elsif ($refdomain =~ /infoseek/i) {
			$iseek = "";
			if ($refquery =~ /qt=\s*([^&]+)/i) { $iseek .= $1; }
			if ($refquery =~ /oq=\s*([^&]+)/i) { $iseek .= " ".$1; }
			$iseek =~ s/^\s+//g;
			$iseek =~ s/\s+/ /g;
			if ($iseek) { $infoseek{$iseek}++; }
		}
		elsif (($refdomain =~ /lycos/i) && ($refquery =~ /query=\s*([^&]+)/i)) {
			$lycos{$1}++;
		}
		elsif (($refdomain =~ /snap/i) && ($refquery =~ /keyword=\s*([^&]+)/i)) {
			$snap{$1}++;
		}
		elsif ($refdomain =~ /webcrawler/i) {
			if (($refquery =~ /search=\s*([^&]+)/i)
			  || ($refquery =~ /searchtext=\s*([^&]+)/i)
			  || ($refquery =~ /text=\s*([^&]+)/i)) {
				$webcrawler{$1}++;
			}
		}
		elsif (($refdomain =~ /yahoo/i) && ($refquery =~ /p=\s*([^&]+)/i)) {
			$yahoo{$1}++;
		}
		elsif (($refquery =~ /general=\s*([^&]+)/i)
		  || ($refquery =~ /keyword=\s*([^&]+)/i)
		  || ($refquery =~ /keywords=\s*([^&]+)/i)
		  || ($refquery =~ /query=\s*([^&]+)/i)
		  || ($refquery =~ /search=\s*([^&]+)/i)
		  || ($refquery =~ /searchstring=\s*([^&]+)/i)
		  || ($refquery =~ /searchtext=\s*([^&]+)/i)
		  || ($refquery =~ /term=\s*([^&]+)/i)
		  || ($refquery =~ /terms=\s*([^&]+)/i)
		  || ($refquery =~ /text=\s*([^&]+)/i)) {
			$othersearch{$1}++;
		}
	}
}
close (REFSLOG);

foreach $key (keys (%TargetCounter)) {
	($target,$referer)=split(/ /,$key,2);
	if ($TargetCounter{"$target/ $referer"} && $TargetCounter{"$target $referer"}) {
		$TargetCounter{$key}=-1;
	}
}

##########
# OUTPUT #
##########

open (REPORT,">$reportfile") || die "  Error opening file: $reportfile\n";

print REPORT "<HTML><HEAD><TITLE>";
print REPORT "Referring URL Statistics for the Infomall Web Server</TITLE></HEAD>\n";
print REPORT "<BODY BGCOLOR=\"#99ccff\" TEXT=\"#000000\">\n";
print REPORT "<H1 ALIGN=CENTER>Referring URL Statistics</H1>\n";
print REPORT "<P>This report logs the URLs reported by browsers as ";
print REPORT "the &quot;referers&quot; directing them to the various ";
print REPORT "listed pages. (Please be aware that this information ";
print REPORT "is far from perfect. Many browsers do not provide any ";
print REPORT "information on the referring page; even those that do ";
print REPORT "can at times provide false or misleading data. And the ";
print REPORT "fact that a page is listed as the referer to a given ";
print REPORT "page does <EM>not</EM> necessarily mean that it ";
print REPORT "actually contains a link to that page.)\n";
if ($MinHits > 1) {
	print REPORT "<P>Only referring URLs appearing at least ";
	print REPORT "<STRONG>$MinHits</STRONG> times ";
	print REPORT "in at least one of the log files analyzed ";
	print REPORT "are listed in the report below.\n";
}
print REPORT "<P>The report was last updated on ";
print REPORT "<STRONG>$date</STRONG> for the Infomall Web Server</P>\n";

if ($keywordsfile) {
	print REPORT "<P ALIGN=CENTER>[ <STRONG><A HREF=\"";
	print REPORT "$keywordsurl\">Keywords Report</A>";
	print REPORT "</STRONG> ]</P>\n";
}
print REPORT "<HR>\n";

print REPORT "<H2 ALIGN=CENTER>Web Pages and Referring URLs</H2>\n";
print REPORT "<DL>\n";
$LastWebPage = '';
foreach $key (sort bytargetthenhits keys(%TargetCounter)) {
	unless ($TargetCounter{$key} < $MinHits) {
		($target,$referer) = split(/ /,$key,2);
		if ("$target" ne "$LastWebPage") {
			print REPORT "<P><DT><STRONG>$target</STRONG></DT>\n";
		}
		print REPORT "<DD><A HREF=\"$referer\">$referer</A> ";
		print REPORT "(",&commas($TargetCounter{$key})," reference";
		if ($TargetCounter{$key} > 1) { print REPORT "s"; }
		print REPORT ")</DD>\n";
		$LastWebPage = $target;
	}
}

print REPORT "</P></DL><HR>\n";
if ($keywordsfile) {
	print REPORT "<P ALIGN=CENTER>[ <STRONG><A HREF=\"";
	print REPORT "$keywordsurl\">Keywords Report for Infomall Web Server</A>";
	print REPORT "</STRONG> ]</P>\n";
}
print REPORT "</BODY></HTML>\n";

close (REPORT);

if ($keywordsfile) {
	if (-e "$keywordsfile") {
		open (OLDLOG,"$keywordsfile") || die "  Error opening file: $keywordsfile\n";
		while (<OLDLOG>) {
			chop;
			if (m#<P><STRONG>(.*)</STRONG>#o) {
				$SearchEngine = $1;
			}
			if (m#\s*([\d,]+)     (.*)$#o ) {
				$count = $1;
				$phrase = $2;
				$count =~ s/,//g;
				if ($SearchEngine eq "AltaVista:") {
					if ($altavista{$phrase}) { $altavista{$phrase} += $count; }
					else { $altavista{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "AOL Netfind:") {
					if ($netfind{$phrase}) { $netfind{$phrase} += $count; }
					else { $netfind{$phrase} = $count; }
				}
				elsif ($SearchEngine =~ /Excite/) {
					if ($excite{$phrase}) { $excite{$phrase} += $count; }
					else { $excite{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "HotBot:") {
					if ($hotbot{$phrase}) { $hotbot{$phrase} += $count; }
					else { $hotbot{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "Infoseek:") {
					if ($infoseek{$phrase}) { $infoseek{$phrase} += $count; }
					else { $infoseek{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "Lycos:") {
					if ($lycos{$phrase}) { $lycos{$phrase} += $count; }
					else { $lycos{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "Snap:") {
					if ($snap{$phrase}) { $snap{$phrase} += $count; }
					else { $snap{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "WebCrawler:") {
					if ($webcrawler{$phrase}) { $webcrawler{$phrase} += $count; }
					else { $webcrawler{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "Yahoo:") {
					if ($yahoo{$phrase}) { $yahoo{$phrase} += $count; }
					else { $yahoo{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "Other Search Engines:") {
					if ($othersearch{$phrase}) { $othersearch{$phrase} += $count; }
					else { $othersearch{$phrase} = $count; }
				}
			}
		}
		close (OLDLOG);
	}
	open (REPORT,">$keywordsfile") || die "  Error opening file: $keywordsfile\n";
	print REPORT "<HTML><HEAD><TITLE>";
	print REPORT "Referring Keyword Statistics for Infomall Web Server</TITLE></HEAD>\n";
	print REPORT "<BODY BGCOLOR=\"#99ccff\" TEXT=\"#000000\">\n";
	print REPORT "<H1 ALIGN=CENTER>Referring Keyword Statistics for Infomall Web Server</H1>\n";
	print REPORT "<P>This report logs the keywords used by visitors to find ";
	print REPORT "this site in the various Internet search engines and directories. ";
	print REPORT "The major search engines are each listed individually. ";
	print REPORT "(Note that not all search engines provide search keywords ";
	print REPORT "in their URLs, and so some are not listed here.)\n";
	print REPORT "<P>The report was last updated on ";
	print REPORT "<STRONG>$date</STRONG>.</P>\n";

	print REPORT "<P ALIGN=CENTER>[ <STRONG><A HREF=\"";
	print REPORT "$reporturl\">Referring URLs Report</A>";
	print REPORT "</STRONG> ]</P>\n";
	print REPORT "<HR>\n";

	print REPORT "<H2 ALIGN=CENTER>Referring Keywords</H2>\n";
	if (%altavista) {
		print REPORT "<P><STRONG>AltaVista:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort AltaVistaByHits keys(%altavista)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($altavista{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%netfind) {
		print REPORT "<P><STRONG>AOL Netfind:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort NetfindByHits keys(%netfind)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($netfind{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%excite) {
		print REPORT "<P><STRONG>Excite:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort ExciteByHits keys(%excite)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($excite{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%hotbot) {
		print REPORT "<P><STRONG>HotBot:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort HotBotByHits keys(%hotbot)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($hotbot{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%infoseek) {
		print REPORT "<P><STRONG>Infoseek:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort InfoseekByHits keys(%infoseek)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($infoseek{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%lycos) {
		print REPORT "<P><STRONG>Lycos:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort LycosByHits keys(%lycos)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($lycos{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%snap) {
		print REPORT "<P><STRONG>Snap:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort SnapByHits keys(%snap)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($snap{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%webcrawler) {
		print REPORT "<P><STRONG>WebCrawler:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort WebCrawlerByHits keys(%webcrawler)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($webcrawler{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%yahoo) {
		print REPORT "<P><STRONG>Yahoo:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort YahooByHits keys(%yahoo)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($yahoo{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%othersearch) {
		print REPORT "<P><STRONG>Other Search Engines:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort OtherByHits keys(%othersearch)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($othersearch{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	print REPORT "<HR>\n";
	print REPORT "<P ALIGN=CENTER>[ <STRONG><A HREF=\"";
	print REPORT "$reporturl\">Referring URLs Report</A>";
	print REPORT "</STRONG> ]</P>\n";
	print REPORT "</BODY></HTML>\n";
	close (REPORT);
}

print "Complete\n";
#exit;

###############
# SUBROUTINES #
###############

sub Get_Date {
	@months =
	  (January,February,March,April,May,June,
	  July,August,September,October,November,December);
	$time = time;
	($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
	  localtime($time);
	$year += 1900;
	$date = "$mday $months[$mon] $year";
}

sub Simplify {
	$url = $_[0];
	$url =~ s/ //g;
	unless ($StripWWW) {
		return $url;
	}
	if ($url =~ m#(^.+)//([\w|\.|-]+)/(.+)#o) {
		$url_prefix = $1;
		$url_domain = $2;
		$url_path = $3;
		$url_domain = "\L$url_domain";
		$url_domain =~ s/^www\.//i;
		return $url_prefix."//".$url_domain."/".$url_path;
	}
	elsif ($url =~ m#(^.+)//([\w|\.|-]+)#o) {
		$url_prefix = $1;
		$url_domain = $2;
		$url_domain = "\L$url_domain";
		$url_domain =~ s/^www\.//i;
		return $url_prefix."//".$url_domain;
	}
	else {
		return $url;
	}
}

sub bytargetthenhits {
	($targeta)=($a=~m#^(.+)\s#o);
	($targetb)=($b=~m#^(.+)\s#o);
	$inequality=($targeta cmp $targetb);
	if ($inequality) {
		$inequality;
	} else {
		$TargetCounter{$b}<=>$TargetCounter{$a};
	}
}

sub AltaVistaByHits {
	$ahits=$altavista{$a};
	$bhits=$altavista{$b};
	$bhits<=>$ahits;
}

sub NetfindByHits {
	$ahits=$netfind{$a};
	$bhits=$netfind{$b};
	$bhits<=>$ahits;
}

sub ExciteByHits {
	$ahits=$excite{$a};
	$bhits=$excite{$b};
	$bhits<=>$ahits;
}

sub HotBotByHits {
	$ahits=$hotbot{$a};
	$bhits=$hotbot{$b};
	$bhits<=>$ahits;
}

sub InfoseekByHits {
	$ahits=$infoseek{$a};
	$bhits=$infoseek{$b};
	$bhits<=>$ahits;
}

sub LycosByHits {
	$ahits=$lycos{$a};
	$bhits=$lycos{$b};
	$bhits<=>$ahits;
}

sub SnapByHits {
	$ahits=$snap{$a};
	$bhits=$snap{$b};
	$bhits<=>$ahits;
}

sub WebCrawlerByHits {
	$ahits=$webcrawler{$a};
	$bhits=$webcrawler{$b};
	$bhits<=>$ahits;
}

sub YahooByHits {
	$ahits=$yahoo{$a};
	$bhits=$yahoo{$b};
	$bhits<=>$ahits;
}

sub OtherByHits {
	$ahits=$othersearch{$a};
	$bhits=$othersearch{$b};
	$bhits<=>$ahits;
}

sub commas {
	local($_)=@_;
	1 while s/(.*\d)(\d\d\d)/$1,$2/;
	$_;
}


#########
# SETUP #
#########
$reportsfile = "/usr/local/archives/public/html/users/deepak/LogReports/Stud/agents.html";
$logsfile = "/infomall/logs/logs-ro/main/agent_log";
$UseOldReports = 1;
$Ignores = "(\.gif|\.jpg|\.jpeg)";

##############
# PROCESSING #
##############

print "Processing Agent Log\n";

if ($UseOldReports) {
	if (-e "$reportsfile") {
		open (OLDLOG,"$reportsfile") || die "  Error opening file: $reportsfile\n";
		while (<OLDLOG>) {
			chop;
			if (m#\s*([\d,]+)                 (.*)$#o ) {
				$count = $1;
				$agent = $2;
				$count =~ s/,//g;
				if ($agentcounter{$agent}) {
					$agentcounter{$agent} += $count;
				}
				else {
					$agentcounter{$agent} = $count;
				}
				$refscounter += $count;
			}
		}
		close (OLDLOG);
	}
}

&Get_Date;

open (AGENTLOG,"$logsfile") || die "  Error opening file: $logsfile\n";
while(<AGENTLOG>) {
	chop;
	s/\s+/ /go;
	if (($Dom,$rfc,$user,$Time,$Request,$Status,$Bytes,$referer,$Agent) =
	  /^(\S+) (\S+) (\S+) \[(.+)\] \"(.+)\" (\S+) (\S+) \"(.*)\" \"(.*)\"/o) {
		if (length($Agent) < 5) { $Agent = ""; }
	} else {
		$Agent=$_;
		if (length($Agent) < 5) { $Agent = ""; }
	}
	next unless ($Agent);
	unless ($Ignores && ($Request=~m#$Ignores#oi)) {
		$refscounter++;
		$Agent =~ s/\&/\&amp\;/go;
		$Agent =~ s/"/\&quot\;/go;
		$Agent =~ s/</\&lt\;/go;
		$Agent =~ s/>/\&gt\;/go;
		($longagent,$other) = split(/ [ \n]/,$Agent,2);
		$agentcounter{$longagent}++;
	}
}
close (AGENTLOG);

foreach $agent (keys %agentcounter) {
	&Identify_Agent;
	&Identify_Platform;
	unless ($agentreport{$longagent}) {
		$agentreport{$longagent} = 0;
	}
	unless ($platformreport{$longplatform}) {
		$platformreport{$longplatform} = 0;
	}
	unless ($combinedreport{"$shortagent ($shortplatform)"}) {
		$combinedreport{"$shortagent ($shortplatform)"} = 0;
	}
	$agentreport{$longagent} += $agentcounter{$agent};
	$platformreport{$longplatform} += $agentcounter{$agent};
	$combinedreport{"$shortagent ($shortplatform)"} += $agentcounter{$agent};
}

##########
# OUTPUT #
##########

open (REPORT,">$reportsfile") || die "  Error opening file: $reportsfile\n";

print REPORT "<HTML><HEAD><TITLE>";
print REPORT "Web Browser Agent/Platform Statistics for Infomall Web Server</TITLE></HEAD>\n";
print REPORT "<BODY BGCOLOR=\"#99ccff\" TEXT=\"#000000\">\n";
print REPORT "<H1 ALIGN=CENTER>";
print REPORT "Web Browser Agent/Platform Statistics for InfoMall Web Server</H1>\n";
print REPORT "<P>These reports list the agents (browsers) and ";
print REPORT "platforms (operating systems) utilized by visitors to ";
print REPORT "these pages. (Browsers which &quot;spoof&quot; other ";
print REPORT "browsers -- such as MSIE or newer versions of AOL's ";
print REPORT "browser claiming to be Netscape, or WebTV claiming to ";
print REPORT "be MSIE claiming to be Netscape -- are identified as ";
print REPORT "what they really are, rather than as what they claim to ";
print REPORT "be.)\n";
print REPORT "<P>The first report details the agents utilized; the ";
print REPORT "second, the platforms. The third report combines the ";
print REPORT "data from the first two. The fourth report is a ";
print REPORT "complete and essentially unprocessed listing ";
print REPORT "of the raw data from the agent log.\n";
print REPORT "<P>These reports were last updated on <STRONG>$date</STRONG>. ";
print REPORT "A total of <STRONG>",&commas($refscounter),"</STRONG> ";
print REPORT "&quot;hits&quot; have thus far been recorded and ";
print REPORT "analyzed.</P>\n";

print REPORT "<H2 ALIGN=CENTER>Summary List<BR>(by Agent)</H2>\n";
print REPORT "<P><PRE>        Hits     Percent     Agent\n\n";
foreach $key (sort AgentByHits keys(%agentreport)) {
	$percentage=(100*$agentreport{$key}/$refscounter)+0.0051;
	if ($percentage < 10) {
		print REPORT "  ";
		printf REPORT "%10s",&commas($agentreport{$key});
		$percentage=~s/(....).*/$1/;
		print REPORT "       ${percentage}%     ";
		print REPORT $key,"\n";
	}
	else {
		print REPORT "  ";
		printf REPORT "%10s",&commas($agentreport{$key});
		$percentage=~s/(.....).*/$1/;
		print REPORT "      ${percentage}%     ";
		print REPORT $key,"\n";
	}
}
print REPORT "</PRE></P><HR>\n";

print REPORT "<H2 ALIGN=CENTER>Summary List<BR>(by Platform)</H2>\n";
print REPORT "<P><PRE>        Hits     Percent     Platform\n\n";
foreach $key (sort PlatformByHits keys(%platformreport)) {
	$percentage=(100*$platformreport{$key}/$refscounter)+0.0051;
	if ($percentage < 10) {
		print REPORT "  ";
		printf REPORT "%10s",&commas($platformreport{$key});
		$percentage=~s/(....).*/$1/;
		print REPORT "       ${percentage}%     ";
		print REPORT $key,"\n";
	}
	else {
		print REPORT "  ";
		printf REPORT "%10s",&commas($platformreport{$key});
		$percentage=~s/(.....).*/$1/;
		print REPORT "      ${percentage}%     ";
		print REPORT $key,"\n";
	}
}
print REPORT "</PRE></P><HR>\n";

print REPORT "<H2 ALIGN=CENTER>Summary List<BR>(by Agent and Platform)</H2>\n";
print REPORT "<P><PRE>        Hits     Percent     Agent (Platform)\n\n";
foreach $key (sort CombinedByHits keys(%combinedreport)) {
	$percentage=(100*$combinedreport{$key}/$refscounter)+0.0051;
	if ($percentage < 10) {
		print REPORT "  ";
		printf REPORT "%10s",&commas($combinedreport{$key});
		$percentage=~s/(....).*/$1/;
		print REPORT "       ${percentage}%     ";
		if ($key =~ "WebTV") { print REPORT "WebTV\n"; }
		else { print REPORT $key,"\n"; }
	}
	else {
		print REPORT "  ";
		printf REPORT "%10s",&commas($combinedreport{$key});
		$percentage=~s/(.....).*/$1/;
		print REPORT "      ${percentage}%     ";
		if ($key =~ "WebTV") { print REPORT "WebTV\n"; }
		else { print REPORT $key,"\n"; }
	}
}

print REPORT "</PRE></P><HR></BODY></HTML>\n";

close (REPORT);

print "Complete\n";
exit;

###############
# SUBROUTINES #
###############

sub Get_Date {
	@months =
	  (January,February,March,April,May,June,
	  July,August,September,October,November,December);
	$time = time;
	($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
	  localtime($time);
	$year += 1900;
	$date = "$mday $months[$mon] $year";
}

sub Identify_Agent {
	if ($agent =~ m#Mozilla/(\d)#oi) {
		if ($agent =~ m#compatible#oi) {
			if ($agent =~ m#WebTV#oi) {
				$longagent = "WebTV";
				$shortagent = "WebTV";
			}
			elsif ($agent =~ m#MSIE (\d)#oi) {
				$longagent = "MSIE v$1.X";
				$shortagent = "MSIE";
			}
			elsif ($agent =~ m#MSIE#oi) {
				$longagent = "MSIE";
				$shortagent = "MSIE";
			}
			elsif ($agent =~ m#AOL (\d)#oi) {
				$longagent = "AOL's Browser v$1.X";
				$shortagent = "AOL's Browser";
			}
			elsif ($agent =~ m#AOL-IWENG (\d)#oi) {
				$longagent = "AOL's Browser v$1.X";
				$shortagent = "AOL's Browser";
			}
			elsif ($agent =~ m#Opera/(\d)#oi) {
				$longagent = "Opera v$1.X";
				$shortagent = "Opera";
			}
			else {
				$longagent = "Other Agents";
				$shortagent = "Other Agents";
			}
		}
		else {
			$longagent = "Netscape v$1.X";
			$shortagent = "Netscape";
		}
	}
	elsif ($agent =~ m#Mozilla#oi) {
		$longagent = "Netscape";
		$shortagent = "Netscape";
	}
	elsif ($agent =~ m#Microsoft Internet Explorer/(\d)#oi) {
		$longagent = "MSIE v$1.X";
		$shortagent = "MSIE";
	}
	elsif ($agent =~ m#IWENG/(\d)#oi) {
		$longagent = "AOL's Browser v$1.X";
		$shortagent = "AOL's Browser";
	}
	elsif ($agent =~ m#aolbrowser/(\d)#oi) {
		$longagent = "AOL's Browser v$1.X";
		$shortagent = "AOL's Browser";
	}
	elsif ($agent =~ m#Lynx#oi) {
		$longagent = "Lynx";
		$shortagent = "Lynx";
	}
	elsif ($agent =~ m#QuarterDeck#oi) {
		$longagent = "QuarterDeck Mosaic";
		$shortagent = "QuarterDeck Mosaic";
	}
	elsif ($agent =~ m#SPRY#oi) {
		$longagent = "Compuserve's SPRY Mosaic";
		$shortagent = "Compuserve's SPRY Mosaic";
	}
	elsif ($agent =~ m#Enhanced_Mosaic#oi) {
		$longagent = "NCSA Mosaic (Enhanced)";
		$shortagent = "NCSA Mosaic (Enhanced)";
	}
	elsif ($agent =~ m#Mosaic#oi) {
		$longagent = "NCSA Mosaic";
		$shortagent = "NCSA Mosaic";
	}
	elsif ($agent =~ m#Crescent#oi) {
		$longagent = "Crescent Internet ToolPak";
		$shortagent = "Crescent Internet ToolPak";
	}
	elsif ($agent =~ m#IBM WebExplorer#oi) {
		$longagent = "IBM WebExplorer";
		$shortagent = "IBM WebExplorer";
	}
	elsif ($agent =~ m#PRODIGY#oi) {
		$longagent = "Prodigy's Browser";
		$shortagent = "Prodigy's Browser";
	}
	elsif ($agent =~ m#Teleport Pro#oi) {
		$longagent = "Teleport Pro";
		$shortagent = "Teleport Pro";
	}
	elsif ($agent =~ m#WebVCR#oi) {
		$longagent = "WebVCR";
		$shortagent = "WebVCR";
	}
	elsif ($agent =~ m#analyzer#oi
	  || $agent =~ m#bot#oi || $agent =~ m#check#oi
	  || $agent =~ m#crawl#oi || $agent =~ m#InfoSeek#oi
	  || $agent =~ m#roamer#oi || $agent =~ m#rover#oi
	  || $agent =~ m#Scooter#oi || $agent =~ m#siphon#oi
	  || $agent =~ m#Slurp#oi || $agent =~ m#spider#oi
	  || $agent =~ m#sweep#oi || $agent =~ m#URL#oi
	  || $agent =~ m#validator#oi || $agent =~ m#walker#oi) {
		$longagent = "Spiders/Robots";
		$shortagent = "Spiders/Robots";
	}
	else {
		$longagent = "Other Agents";
		$shortagent = "Other Agents";
	}
}

sub Identify_Platform {
	if ($agent =~ m#win#oi) {
		$shortplatform = "Windows";
	}
	if ($agent =~ m#win95#oi) {
		$longplatform = "Windows 95";
	}
	elsif ($agent =~ m#winNT#oi) {
		$longplatform = "Windows NT";
	}
	elsif ($agent =~ m#win16#oi) {
		$longplatform = "Windows 3.1/95 (16-bit)";
	}
	elsif ($agent =~ m#win32#oi) {
		$longplatform = "Windows 95/NT (32-bit)";
	}
	elsif ($agent =~ m#winweb#oi) {
		$longplatform = "Windows 3.1/95 (16-bit)";
	}
	elsif ($agent =~ m#Windows 95#oi) {
		$longplatform = "Windows 95";
	}
	elsif ($agent =~ m#Windows NT#oi) {
		$longplatform = "Windows NT";
	}
	elsif ($agent =~ m#Windows 3.1#oi) {
		$longplatform = "Windows 3.1";
	}
	elsif ($agent =~ m#Windows#oi) {
		if ($agent =~ m#32bit#oi) {
			$longplatform = "Windows 95/NT (32-bit)";
		}
		else {
			$longplatform = "Windows 3.1/95 (16-bit)";
		}
	}
	elsif ($agent =~ m#Window#oi) {
		$shortplatform = "X Windows";
		$longplatform = "X Windows";
	}
	elsif ($agent =~ m#Mac#oi) {
		$shortplatform = "Macintosh";
		if ($agent =~ m#PPC#oi) {
			$longplatform = "Macintosh (PowerPC)";
		}
		elsif ($agent =~ m#PowerPC#oi) {
			$longplatform = "Macintosh (PowerPC)";
		}
		else {
			$longplatform = "Macintosh (68K)";
		}
	}
	elsif ($agent =~ m#Amiga#oi) {
		$shortplatform = "Amiga";
		$longplatform = "Amiga";
	}
	elsif ($agent =~ m#OS/2#oi) {
		$shortplatform = "OS/2";
		$longplatform = "OS/2";
	}
	elsif ($agent =~ m#X11#oi) {
		$shortplatform = "UNIX";
		if ($agent =~ m#HP-UX#oi) {
			$longplatform = "UNIX (HP-UX)";
		}
		elsif ($agent =~ m#Linux#oi) {
			$longplatform = "UNIX (Linux)";
		}
		elsif ($agent =~ m#SunOS#oi) {
			$longplatform = "UNIX (SunOS)";
		}
		elsif ($agent =~ m#FreeBSD#oi) {
			$longplatform = "UNIX (FreeBSD)";
		}
		elsif ($agent =~ m#NetBSD#oi) {
			$longplatform = "UNIX (NetBSD)";
		}
		elsif ($agent =~ m#BSD#oi) {
			$longplatform = "UNIX (BSD)";
		}
		elsif ($agent =~ m#AIX#oi) {
			$longplatform = "UNIX (AIX)";
		}
		elsif ($agent =~ m#OSF1#oi) {
			$longplatform = "UNIX (OSF1)";
		}
		elsif ($agent =~ m#IRIX#oi) {
			$longplatform = "UNIX (IRIX)";
		}
		else {
			$longplatform = "UNIX (Other/Unspecified)";
		}
	}
	elsif ($agent =~ m#IWENG#oi) {
		$shortplatform = "Windows";
		$longplatform = "Windows 3.1/95 (16-bit)";
	}
	elsif ($agent =~ m#Lynx#oi) {
		$shortplatform = "UNIX";
		$longplatform = "UNIX (Other/Unspecified)";
	}
	elsif ($agent =~ m#WebTV#oi) {
		$shortplatform = "WebTV";
		$longplatform = "WebTV";
	}
	elsif ($agent =~ m#WebVCR#oi) {
		$shortplatform = "WebVCR";
		$longplatform = "WebVCR";
	}
	else {
		$shortplatform = "Unknown Platform";
		$longplatform = "Unknown Platform";
	}
}

sub commas {
	local($_)=@_;
	1 while s/(.*\d)(\d\d\d)/$1,$2/;
	$_;
}

sub AgentByHits {
	$ahits=$agentreport{$a};
	$bhits=$agentreport{$b};
	$bhits<=>$ahits;
}

sub PlatformByHits {
	$ahits=$platformreport{$a};
	$bhits=$platformreport{$b};
	$bhits<=>$ahits;
}

sub CombinedByHits {
	$ahits=$combinedreport{$a};
	$bhits=$combinedreport{$b};
	$bhits<=>$ahits;
}


