#!/usr/bin/perl

$version = "2.01";
$reportfile = "/illustra/httpd/htdocs/report.html";
$keywordsfile = "/illustra/httpd/htdocs/keywords.html";
$logfile = "/project/C067/logs/tmp/referer_log";
$UseOldReport = 1;
$excludefrom = "(http://www.npac.syr.edu)";
$excludeto = "(\.gif|\.jpg|\.jpeg|\.xbm|http://www.npac.syr.edu)";
$StripWWW = 0;
$MinHits = 5;

##############
# PROCESSING #
##############

print "Processing Referer Log\n";

$DefaultPageName = "(index\.[^.]*|welcome\.[^.]*)";

if ($reportfile =~ /.*\/([^\/]+)/) {
	$reporturl = $1;
}

if ($keywordsfile =~ /.*\/([^\/]+)/) {
	$keywordsurl = $1;
}

if ($UseOldReport) {
	if (-e "$reportfile") {
		open (OLDLOG,"$reportfile") || die "  Error opening file: $reportfile\n";
		while (<OLDLOG>) {
			chop;
			if (m#^<P><DT><STRONG>(.*)</STRONG></DT>#o) {
				$target = &Simplify($1);
				next;
			}
			next if (! $target);
			if (m#^<DD><A HREF=\"([^"]*)\">.*</A> \(([\d,]+) reference#o) {
				$refurl = $1;
				$count = $2;
				$count =~ s/,//g;
				$url = &Simplify($refurl);
				if ($TargetCounter{"$target $url"}) { $TargetCounter{"$target $url"} += $count; }
				else { $TargetCounter{"$target $url"} = $count; }
			}
		}
		close (OLDLOG);
	}
}

&Get_Date;

open (REFSLOG,"$logfile") || die "  Error opening file: $logfile\n";

while (<REFSLOG>) {
	chop;
	s/\s+/ /go;
	if (($Dom,$rfc,$user,$Time,$Request,$Status,$Bytes,$referer,$Agent) =
	  /^(\S+) (\S+) (\S+) \[(.+)\] \"(.+)\" (\S+) (\S+) \"(.*)\" \"(.*)\"/o) {
		($method,$target,$protocol)=split(/\s/,$Request,3);
		if (length($referer) < 5) { $referer = ""; }
	} else {
		($referer,$target)=split(/ -> /,$_,2);
		if (length($referer) < 5) { $referer = ""; }
	}
	next if (! ($referer && $target));
	$target=~ s/\%7[eE]/~/o;
	$target=~ s#//#/#go;
	$target=~ s#^ *$#/#o;
	$target=~ s/#.+$//o;
	$target=~ s/\?.*$//o;
	$target=~ s/${DefaultPageName}$//oi;
	$target = &Simplify($target);
	$referer=~ s/\%7[eE]/~/o;
	$referer=~ s#^(http://[^/]+):80/#$1/#o;
	$refquery = "";
	if ($referer =~ m#^.+//([\w|\.|-]+)/.*\?(.+)#o) {
		$refdomain = $1;
		$refquery = $2;
	}
	$referer=~ s/#.+$//o;
	$referer=~ s/\?.*$//o;
	$referer=~ s/${DefaultPageName}$//oi;
	$referer = &Simplify($referer);
	unless (($excludefrom && ($referer =~ m#$excludefrom#oi))
	  || ($excludeto && ($target =~ m#$excludeto#oi))) {
		$keyform=$target.' '.$referer;
		$keyform=~ s#\&#\&amp\;#og;
		$keyform=~ s#<#\&lt\;#og;
		$keyform=~ s#>#\&gt\;#og;
		$keyform=~ s#"#\&quot\;#og;
		$referer=~ s#\&#\&amp\;#og;
		$referer=~ s#<#\&lt\;#og;
		$referer=~ s#>#\&gt\;#og;
		$referer=~ s#"#\&quot\;#og;
		$TargetCounter{$keyform}++;
	}
	else {
		$referer = "";
	}
	if ($referer && $refquery) {
		$refquery =~ tr/A-Z/a-z/;
		$refquery =~ s/\%.{2}/ /gi;
		$refquery =~ s/\+/ /g;
		$refquery =~ s/\-/ /g;
		$refquery =~ s/\s+/ /g;
		if (($refdomain =~ /altavista/i) && ($refquery =~ /\&q=\s*([^&]+)/i)) {
			$altavista{$1}++;
		}
		elsif ($refdomain =~ /netfind/i) {
			if (($refquery =~ /search=\s*([^&]+)/i)
			  || ($refquery =~ /\&s=\s*([^&]+)/i)) {
				$netfind{$1}++;
			}
		}
		elsif ($refdomain =~ /excite/i) {
			if (($refquery =~ /search=\s*([^&]+)/i)
			  || ($refquery =~ /\&s=\s*([^&]+)/i)) {
				$excite{$1}++;
			}
		}
		elsif (($refdomain =~ /hotbot/i) && ($refquery =~ /mt=\s*([^&]+)/i)) {
			$hotbot{$1}++;
		}
		elsif ($refdomain =~ /infoseek/i) {
			$iseek = "";
			if ($refquery =~ /qt=\s*([^&]+)/i) { $iseek .= $1; }
			if ($refquery =~ /oq=\s*([^&]+)/i) { $iseek .= " ".$1; }
			$iseek =~ s/^\s+//g;
			$iseek =~ s/\s+/ /g;
			if ($iseek) { $infoseek{$iseek}++; }
		}
		elsif (($refdomain =~ /lycos/i) && ($refquery =~ /query=\s*([^&]+)/i)) {
			$lycos{$1}++;
		}
		elsif (($refdomain =~ /snap/i) && ($refquery =~ /keyword=\s*([^&]+)/i)) {
			$snap{$1}++;
		}
		elsif ($refdomain =~ /webcrawler/i) {
			if (($refquery =~ /search=\s*([^&]+)/i)
			  || ($refquery =~ /searchtext=\s*([^&]+)/i)
			  || ($refquery =~ /text=\s*([^&]+)/i)) {
				$webcrawler{$1}++;
			}
		}
		elsif (($refdomain =~ /yahoo/i) && ($refquery =~ /p=\s*([^&]+)/i)) {
			$yahoo{$1}++;
		}
		elsif (($refquery =~ /general=\s*([^&]+)/i)
		  || ($refquery =~ /keyword=\s*([^&]+)/i)
		  || ($refquery =~ /keywords=\s*([^&]+)/i)
		  || ($refquery =~ /query=\s*([^&]+)/i)
		  || ($refquery =~ /search=\s*([^&]+)/i)
		  || ($refquery =~ /searchstring=\s*([^&]+)/i)
		  || ($refquery =~ /searchtext=\s*([^&]+)/i)
		  || ($refquery =~ /term=\s*([^&]+)/i)
		  || ($refquery =~ /terms=\s*([^&]+)/i)
		  || ($refquery =~ /text=\s*([^&]+)/i)) {
			$othersearch{$1}++;
		}
	}
}
close (REFSLOG);

foreach $key (keys (%TargetCounter)) {
	($target,$referer)=split(/ /,$key,2);
	if ($TargetCounter{"$target/ $referer"} && $TargetCounter{"$target $referer"}) {
		$TargetCounter{$key}=-1;
	}
}

##########
# OUTPUT #
##########

open (REPORT,">$reportfile") || die "  Error opening file: $reportfile\n";

print REPORT "<HTML><HEAD><TITLE>";
print REPORT "Referring URL Statistics</TITLE></HEAD>\n";
print REPORT "<BODY BGCOLOR=\"#99ccff\" TEXT=\"#000000\">\n";
print REPORT "<H1 ALIGN=CENTER>Referring URL Statistics</H1>\n";
print REPORT "<P>This report logs the URLs reported by browsers as ";
print REPORT "the &quot;referers&quot; directing them to the various ";
print REPORT "listed pages. (Please be aware that this information ";
print REPORT "is far from perfect. Many browsers do not provide any ";
print REPORT "information on the referring page; even those that do ";
print REPORT "can at times provide false or misleading data. And the ";
print REPORT "fact that a page is listed as the referer to a given ";
print REPORT "page does <EM>not</EM> necessarily mean that it ";
print REPORT "actually contains a link to that page.)\n";
if ($MinHits > 1) {
	print REPORT "<P>Only referring URLs appearing at least ";
	print REPORT "<STRONG>$MinHits</STRONG> times ";
	print REPORT "in at least one of the log files analyzed ";
	print REPORT "are listed in the report below.\n";
}
print REPORT "<P>The report was last updated on ";
print REPORT "<STRONG>$date</STRONG>.</P>\n";

if ($keywordsfile) {
	print REPORT "<P ALIGN=CENTER>[ <STRONG><A HREF=\"";
	print REPORT "$keywordsurl\">Keywords Report</A>";
	print REPORT "</STRONG> ]</P>\n";
}
print REPORT "<HR>\n";

print REPORT "<H2 ALIGN=CENTER>Web Pages and Referring URLs</H2>\n";
print REPORT "<DL>\n";
$LastWebPage = '';
foreach $key (sort bytargetthenhits keys(%TargetCounter)) {
	unless ($TargetCounter{$key} < $MinHits) {
		($target,$referer) = split(/ /,$key,2);
		if ("$target" ne "$LastWebPage") {
			print REPORT "<P><DT><STRONG>$target</STRONG></DT>\n";
		}
		print REPORT "<DD><A HREF=\"$referer\">$referer</A> ";
		print REPORT "(",&commas($TargetCounter{$key})," reference";
		if ($TargetCounter{$key} > 1) { print REPORT "s"; }
		print REPORT ")</DD>\n";
		$LastWebPage = $target;
	}
}

print REPORT "</P></DL><HR>\n";
if ($keywordsfile) {
	print REPORT "<P ALIGN=CENTER>[ <STRONG><A HREF=\"";
	print REPORT "$keywordsurl\">Keywords Report</A>";
	print REPORT "</STRONG> ]</P>\n";
}
print REPORT "</BODY></HTML>\n";

close (REPORT);

if ($keywordsfile) {
	if (-e "$keywordsfile") {
		open (OLDLOG,"$keywordsfile") || die "  Error opening file: $keywordsfile\n";
		while (<OLDLOG>) {
			chop;
			if (m#<P><STRONG>(.*)</STRONG>#o) {
				$SearchEngine = $1;
			}
			if (m#\s*([\d,]+)     (.*)$#o ) {
				$count = $1;
				$phrase = $2;
				$count =~ s/,//g;
				if ($SearchEngine eq "AltaVista:") {
					if ($altavista{$phrase}) { $altavista{$phrase} += $count; }
					else { $altavista{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "AOL Netfind:") {
					if ($netfind{$phrase}) { $netfind{$phrase} += $count; }
					else { $netfind{$phrase} = $count; }
				}
				elsif ($SearchEngine =~ /Excite/) {
					if ($excite{$phrase}) { $excite{$phrase} += $count; }
					else { $excite{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "HotBot:") {
					if ($hotbot{$phrase}) { $hotbot{$phrase} += $count; }
					else { $hotbot{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "Infoseek:") {
					if ($infoseek{$phrase}) { $infoseek{$phrase} += $count; }
					else { $infoseek{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "Lycos:") {
					if ($lycos{$phrase}) { $lycos{$phrase} += $count; }
					else { $lycos{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "Snap:") {
					if ($snap{$phrase}) { $snap{$phrase} += $count; }
					else { $snap{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "WebCrawler:") {
					if ($webcrawler{$phrase}) { $webcrawler{$phrase} += $count; }
					else { $webcrawler{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "Yahoo:") {
					if ($yahoo{$phrase}) { $yahoo{$phrase} += $count; }
					else { $yahoo{$phrase} = $count; }
				}
				elsif ($SearchEngine eq "Other Search Engines:") {
					if ($othersearch{$phrase}) { $othersearch{$phrase} += $count; }
					else { $othersearch{$phrase} = $count; }
				}
			}
		}
		close (OLDLOG);
	}
	open (REPORT,">$keywordsfile") || die "  Error opening file: $keywordsfile\n";
	print REPORT "<HTML><HEAD><TITLE>";
	print REPORT "Referring Keyword Statistics</TITLE></HEAD>\n";
	print REPORT "<BODY BGCOLOR=\"#99ccff\" TEXT=\"#000000\">\n";
	print REPORT "<H1 ALIGN=CENTER>Referring Keyword Statistics</H1>\n";
	print REPORT "<P>This report logs the keywords used by visitors to find ";
	print REPORT "this site in the various Internet search engines and directories. ";
	print REPORT "The major search engines are each listed individually. ";
	print REPORT "(Note that not all search engines provide search keywords ";
	print REPORT "in their URLs, and so some are not listed here.)\n";
	print REPORT "<P>The report was last updated on ";
	print REPORT "<STRONG>$date</STRONG>.</P>\n";

	print REPORT "<P ALIGN=CENTER>[ <STRONG><A HREF=\"";
	print REPORT "$reporturl\">Referring URLs Report</A>";
	print REPORT "</STRONG> ]</P>\n";
	print REPORT "<HR>\n";

	print REPORT "<H2 ALIGN=CENTER>Referring Keywords</H2>\n";
	if (%altavista) {
		print REPORT "<P><STRONG>AltaVista:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort AltaVistaByHits keys(%altavista)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($altavista{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%netfind) {
		print REPORT "<P><STRONG>AOL Netfind:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort NetfindByHits keys(%netfind)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($netfind{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%excite) {
		print REPORT "<P><STRONG>Excite:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort ExciteByHits keys(%excite)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($excite{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%hotbot) {
		print REPORT "<P><STRONG>HotBot:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort HotBotByHits keys(%hotbot)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($hotbot{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%infoseek) {
		print REPORT "<P><STRONG>Infoseek:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort InfoseekByHits keys(%infoseek)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($infoseek{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%lycos) {
		print REPORT "<P><STRONG>Lycos:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort LycosByHits keys(%lycos)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($lycos{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%snap) {
		print REPORT "<P><STRONG>Snap:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort SnapByHits keys(%snap)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($snap{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%webcrawler) {
		print REPORT "<P><STRONG>WebCrawler:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort WebCrawlerByHits keys(%webcrawler)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($webcrawler{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%yahoo) {
		print REPORT "<P><STRONG>Yahoo:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort YahooByHits keys(%yahoo)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($yahoo{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	if (%othersearch) {
		print REPORT "<P><STRONG>Other Search Engines:</STRONG>\n";
		print REPORT "<P><PRE>";
		foreach $phrase (sort OtherByHits keys(%othersearch)) {
			print REPORT "  ";
			printf REPORT "%10s",&commas($othersearch{$phrase});
			print REPORT "     $phrase\n";
		}
		print REPORT "</PRE></P>\n";
	}
	print REPORT "<HR>\n";
	print REPORT "<P ALIGN=CENTER>[ <STRONG><A HREF=\"";
	print REPORT "$reporturl\">Referring URLs Report</A>";
	print REPORT "</STRONG> ]</P>\n";
	print REPORT "</BODY></HTML>\n";
	close (REPORT);
}

print "Complete\n";
exit;

###############
# SUBROUTINES #
###############

sub Get_Date {
	@months =
	  (January,February,March,April,May,June,
	  July,August,September,October,November,December);
	$time = time;
	($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
	  localtime($time);
	$year += 1900;
	$date = "$mday $months[$mon] $year";
}

sub Simplify {
	$url = $_[0];
	$url =~ s/ //g;
	unless ($StripWWW) {
		return $url;
	}
	if ($url =~ m#(^.+)//([\w|\.|-]+)/(.+)#o) {
		$url_prefix = $1;
		$url_domain = $2;
		$url_path = $3;
		$url_domain = "\L$url_domain";
		$url_domain =~ s/^www\.//i;
		return $url_prefix."//".$url_domain."/".$url_path;
	}
	elsif ($url =~ m#(^.+)//([\w|\.|-]+)#o) {
		$url_prefix = $1;
		$url_domain = $2;
		$url_domain = "\L$url_domain";
		$url_domain =~ s/^www\.//i;
		return $url_prefix."//".$url_domain;
	}
	else {
		return $url;
	}
}

sub bytargetthenhits {
	($targeta)=($a=~m#^(.+)\s#o);
	($targetb)=($b=~m#^(.+)\s#o);
	$inequality=($targeta cmp $targetb);
	if ($inequality) {
		$inequality;
	} else {
		$TargetCounter{$b}<=>$TargetCounter{$a};
	}
}

sub AltaVistaByHits {
	$ahits=$altavista{$a};
	$bhits=$altavista{$b};
	$bhits<=>$ahits;
}

sub NetfindByHits {
	$ahits=$netfind{$a};
	$bhits=$netfind{$b};
	$bhits<=>$ahits;
}

sub ExciteByHits {
	$ahits=$excite{$a};
	$bhits=$excite{$b};
	$bhits<=>$ahits;
}

sub HotBotByHits {
	$ahits=$hotbot{$a};
	$bhits=$hotbot{$b};
	$bhits<=>$ahits;
}

sub InfoseekByHits {
	$ahits=$infoseek{$a};
	$bhits=$infoseek{$b};
	$bhits<=>$ahits;
}

sub LycosByHits {
	$ahits=$lycos{$a};
	$bhits=$lycos{$b};
	$bhits<=>$ahits;
}

sub SnapByHits {
	$ahits=$snap{$a};
	$bhits=$snap{$b};
	$bhits<=>$ahits;
}

sub WebCrawlerByHits {
	$ahits=$webcrawler{$a};
	$bhits=$webcrawler{$b};
	$bhits<=>$ahits;
}

sub YahooByHits {
	$ahits=$yahoo{$a};
	$bhits=$yahoo{$b};
	$bhits<=>$ahits;
}

sub OtherByHits {
	$ahits=$othersearch{$a};
	$bhits=$othersearch{$b};
	$bhits<=>$ahits;
}

sub commas {
	local($_)=@_;
	1 while s/(.*\d)(\d\d\d)/$1,$2/;
	$_;
}


