#topitall.pl 
# Copyright John Belshaw Jan 2002
# This source code is distributed under the artistic license
# see the file docs/artistic.html for details
# 
# Revison 1.1 Jan 3 2002
# Code review by Rob Murgatroyd [RJM]
# tidied comment print statements
# added daemon fork
# Fixed prune file names and prune.cfg file
#
# Version 1.2 Jan 4 2002
# added netstat -i parameters
# 
# Version 2.0 Jan 12 2002 Uses RRD
# J Belshaw
# simplified so all measured parameters are in the hash Current
# removed prune code
# Added daemon pid information
#
# Version 2.2 Jan 13 2002 
# J. Belshaw
# Corrected bug in plotting title
# and removed use DBFile
# Changed RRD parameters such that measuremnet made every 60 seconds
# stored for 1 day at 1min resolution
# 1 month at 15min resolution
# 1 year at 6hr resolution 
# added plot -hour option

use Fcntl;
use POSIX;
use lib qw( /usr/local/rrdtool-1.0.42/lib/perl ../lib/perl );
use RRDs;

$Na=0;
# Do my own Getops processing
foreach $Arg(@ARGV) {
  if ($Arg =~/^-/) {
   $Arg = substr $Arg,1;
   if ((!defined $ARGV[($Na+1)]) || $ARGV[($Na+1)]=~/^-/) {
#      print "$Na !defined $ARGV[($Na+1)]) | $ARGV[($Na+1)] =~/^-/\n";
      $Arga{$Arg}=0; }
   else {
   $Arga{$Arg}=$ARGV[($Na+1)];
   }
  }
$Na++;
}

if (defined $Arga{"debug"}) { $DEBUG=1; }

print "N Args $Na\n" if ($DEBUG);

$V=0;
if (defined $Arga{"v"}) {
  if ($Arga{'v'}==0) { $Arga{'v'}=1; }
  $V=$Arga{"v"}; }

print "Some verbose messages \n" if ($V==1);
print "More verbose messages \n" if ($V==2);
print "All verbose messages \n"  if ($V>=3);


############################################ 
# daemon
############################################
if (defined $Arga{"daemon"}) {
#Platform Check
$OS=`uname `;
chomp $OS;
print  "OS :$OS:\n" if ($V>=1);
if ($OS eq "Linux") {
 @Vml=('Nrun','Nsleep','Nswapped','MemSwap','MemFree','MemBuff','MemCache','SwapIn','SwapOut','IOin','IOout','Interrupts','ContextSwitches','CPUuser','CPUsystem','CPUidle'); 
}
elsif ($OS eq "SunOS") {
 @Vml=('Nrun','Nsleep','Nswapped','MemSwap','MemFree','PageReclaims','MinorFaults','SwapIn','SwapOut','SwapFreed','MemShortfall','ScanRate','disk0','disk1','disk2','disk3','Interrupts','SystemCalls','ContextSwitches','CPUuser','CPUsystem','CPUidle');
}
else {
 @Vmstat=`vmstat`;
 @Vml=split(' ',$Vmstat[1]);
}; 
@Meas=(@Vml,'NProc','Users','LoadAv','Uptime','Mem');
$NMeas=scalar @Meas;
$NVml=scalar @Vml;
print "N: $NVml $NMeas @Meas:\n" if ($V>=2);

open(PROC,"< tiaProcess.cfg") || die("can't open Configuration file: tiaProcess.cfg $!");
while (<PROC>){
  if (!/^#/&!/^\s*$/) { #Lines beginning with # or all whitespace
    print "tiaProcess.cfg Line $.\n" if ($V>=3);
    ($Ptmp)=split;
    $Processes{$Ptmp}=$Ptmp; 
    print "Reading Process match : $Processes{$Ptmp}\n" if ($V>=1);
  }
}
close PROC;

if ($V>=1) {
  foreach $Val (keys %Processes) {
    print "Monitoring Processes matching $Val\n"; 
  }
}


#Read alert.cfg
$Nr=0;
open(PROC,"< tiaAlert.cfg")|| 
     die("can't open Configuration file: tiaAlert.cfg $!");
while (<PROC>){
  if (!/^#/&!/^\s*$/) {
    print "tiaAlert.cfg Line $.\n"  if ($V>=3);
    chomp;
   ($Alert[$Nr],$Category[$Nr],$NMatch[$Nr],$AlertMail[$Nr])=split;
   ($crap,$AlertComment[$Nr])=split '#';
   if ($Alert[$Nr] eq "System") {
     $Match[$Nr]= "(\$Current{\"$Category[$Nr]\"}$NMatch[$Nr])";
    $Nr++ ;
   }
   elsif ($Alert[$Nr] eq "Disk") {
     $Match[$Nr]= "(\$Current{\"$Category[$Nr]\"}$NMatch[$Nr])";
     $Nr++ ; 
   }
   elsif ($Alert[$Nr] eq "Net") {
     $Match[$Nr]= "(\$Current{\"$Category[$Nr]\"}$NMatch[$Nr])";
    $Nr++ ; }
   else {
     if (defined($Processes{$Alert[$Nr]})) { # is a valid process
      $Match[$Nr]= "\$Current{\"$Alert[$Nr].$Category[$Nr]\"}$NMatch[$Nr]";
       $Nr++;
     } 
     else {
     print "$Alert[$Nr] in alerts.cfg not found in top.cfg\n" if ($V>=1); 
     }
    }
  } 
}
close  PROC;

if ($DEBUG) {
  foreach $M (@Match) {
    print  "$M\n";
  }
}


#List of netstat -i parameters
if ($OS eq "Linux") {
 @Netsi=(
   'RXok',
   'RXerr',
   'RXdrp',
   'RXovr',
   'TXok',
   'TXerr',
   'TXdrp',
   'TXovr');
 $Nioffset=3;
}
else {
 @Netsi=(
   'RXok',
   'RXerr',
   'TXok',
   'TXerr',
   'Collisons',
   'Queue');
 $Nioffset=4;
}
 
 
 

print "Reading netstat -in\n" if ($V>=2) ;
# Read  netstat -i and discard first 2 lines
@LastNetstati=`netstat -in`;
chomp @LastNetstati;
if ($OS eq "linux") { shift @LastNetstati; }
shift @LastNetstati; 

($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =gmtime(time);
print "topitall.pl started at : $yday $hour $min $sec  : $isdst\n" if ($V>=1);

###################################### 
# daemon loop
######################################
# Fork a daemon and detach from terminal if run with -daemon and no other options. Parent exits.
# RJM & JCB
# Not sure if this is platform dependent
#if ( $OS eq "Linux" ) {
#
if ((defined $Arga{"daemon"})&& ($Na==1)) {
  my $pid = fork;
  if ($pid) {
    print STDERR "[$pid] Topitall running as daemon\n"; 
    exit;
  }
  die "Couldn't fork: $!" unless defined($pid);
  POSIX::setsid() or die "Could not detach from terminal (setsid())";
}

$t=time;
####################################
#Loop until $Finished  ie. forever
####################################
while (!$Finished) {
 ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =gmtime(time);
# give 30 secs to do my calculations 16,31,46,61
#  $Nsecs=((int($min/15)*15 +16)-$min) *60 -$sec-30;
$Nsecs=55;
#  print "Averaging  vmstat for $Nsecs seconds, Minute now $min:$sec\n" if ($V>=1); 

#set Time stamp for quick Mode
# Fudge current day + compress 24 hrs into 1 hour

  if (defined($Arga{'q'})) {
    $Nsecs=5; 
    $t=$t+60;
    print "Quick mode, Averages for 10 seconds, with butchered time stamps\n" if ($V>=1) ;
  }
  else {
    $t=time;
  }

  $crap=gmtime $t;
  print "$crap\n" if ($V>=2);



  @Vmstat= `vmstat $Nsecs 2`;
# Fourth Line
  $Vm=$Vmstat[3];
  chomp $Vm;
  print "Averaging $Nsecs : $Vm : Minute now $min:$sec\n" if ($V>=1); 

  (@Vmstat)=split(' ',$Vm); 
  for ($i=0;$i < $NVml;$i++) {
    $Param=$Vml[$i];
    $Current{$Param} = $Vmstat[$i];
    print "$i $Param $Current{$Param}\n" if ($V>=1); 
  }


  $Uptime= `uptime`;
  if ($Uptime =~ /min,/) {
    ( $Current{'Uptime'}, $Current{'Users'}, $Current{'LoadAv'}) = (split ' ', $Uptime)[2,4,10]; 
    $Current{'Uptime'}=$Current{'Uptime'}/1440 ;
  }
  elsif ($Uptime =~ /day/){
    ($Upday,$Uphour,$Current{'Users'},$Current{'LoadAv'})=(split ' ',$Uptime)[2,4,5,11];
    ($hour,$min)=split(':',$Uphour); 
    chop $min;
    $Current{'Uptime'}= $Upday + $hour/24 + $min/1440  ;
    print "Uptime : $Upday:$min:$hour $Current{'Uptime'}\n" if  ($DEBUG); 
  } 
  else {
    print "Uptime $Uptime\n" if ($V>=1);
   ($Uphour,$Current{'Users'},$Current{'LoadAv'})=(split ' ',$Uptime)[2,3,9]; 
   ($hour,$min)=split(':',$Uphour); 
   chop $min;
   $Current{'Uptime'}= $hour/24 + $min/1440 ;  
  }
  print "Uptime : $hour:$min $Current{'Uptime'} $Current{'LoadAv'}   $Current{'Users'}\n" if  ($V>=1); 

############### 
# End of uptime
###############
# Write plot.cfg file as we go along
# to determine the order of the parameters
# when they are eventually plotted

  $Hostname=`uname -n`;
  chomp $Hostname;
  if ($V>=2) {print "File tia/plot.cfg"; };
  open (PFH,"> tia/plot.cfg");
  print PFH "Host $Hostname\nMeas ";


#Initialise Proc Counts
  $Current{"Mem"}=0.0;
  $Current{"NProc"}= 0 ;
  for $M (keys %Processes) {
    $Current{"$M.N"}=0;
    $Current{"$M.Cpu"}=0.0;
    $Current{"$M.Mem"}=0.0;
    print PFH "$M.N $M.Cpu $M.Mem ";
  }

  @PS=`ps -e -o pcpu,pmem,comm`;
# Tot up the cpu, mem and number of processes 
  foreach $Ps (@PS) {
    $Current{"NProc"}++;
    chomp $Ps;
    print "-$Ps-\n" if ($DEBUG); 
    ($Cpu,$Mem,$Command)=split(' ',$Ps);
    $Current{"Mem"}=$Current{"Mem"}+$Mem;
    for $M (keys %Processes) {
      print "Match ($Command =~ /$M/)\n" if ($V>=3); 
      if ($Command =~ /$M/) {
#retrieve the current values per process
        print "($Command =~ /$M/)########## Succeeded\n" if ($V>=3); 
        $Current{"$M.N"}++;
        $Current{"$M.Cpu"}=$Current{"$M.Cpu"}+$Cpu;
        $Current{"$M.Mem"}=$Current{"$M.Mem"}+$Mem;
# Process name match
      }
    }
  }

######################## 
# End of Process
########################
# Write list of parameters to plot.cfg file
#
  foreach $Var (@Meas) {
    print PFH "$Var "; 
  }


# Read  netstat -i calculate difference and write to files;
  if ($V>=1) { print "Reading netstat -i, again, to calculate elapsed counters\n";};
  @Netstati=`netstat -in`;
  chomp @Netstati;
  shift @Netstati if ($OS eq "Linux");
  shift @Netstati; 

  $NRec=0;
  foreach $Line (@Netstati) {
    print "Netstati $Line\n" if ($DEBUG) ;
    @Nnow=split(' ',$Line);
    @Nlast=split(' ',$LastNetstati[$NRec]);
#  Iterate over the netstat -i variables 
    for  ($i=$Nioffset;$i<(scalar @Netsi + $Nioffset);$i++) {
      $Net{$Netsi[$i-$Nioffset]}=$Nnow[$i]-$Nlast[$i];
# For RRD
      $M="$Nnow[0].$Netsi[$i-$Nioffset]";
      $Current{$M}=$Net{$Netsi[$i-$Nioffset]};
#
      print "$Netsi[$i-$Nioffset] Subtract $Nnow[$i] - $Nlast[$i] = $Net{$Netsi[$i-$Nioffset]}\n" if ($V>=3);
      print PFH "$M ";
    }
    $NRec++;
  }
  @LastNetstati=@Netstati;

###############################################################
# RRD 
###############################################################

# Create list of all measured parameters
#
#@AllMeas=(keys %Processes, @Meas, @Netsi);
  @AllMeas=(keys %Current);
  $NMeas=@AllMeas;
  print "Number Measured : $NMeas\n" if ($V>=2);



#Now read DF table and print it out
  print "Reading filesystem capacities with df -k\n" if ($V>=1); 
  @DF= `df -k -l`;
  chomp @DF;
#get rid of first line RJM
  shift @DF;
  for $NL(@DF) {
    @Line=split(' ',$NL);
    print "$Line[0]\n" if  ($DEBUG) ;
    if ($Line[0] =~ /dev/) {
      ($Current{$Line[0]},$FSmount{$Line[0]})=(split(' ',$NL))[4,5];
      print "FS $Line[0] $FSmount{$Line[0]}\n" if  ($V>=1) ;
     chop $Current{$Line[0]}; #get rid of 
     $crap = $Line[0];
     $crap =~ s|\/|_|g;
     print PFH "$crap ";
    }
  }
  print PFH "\n";
  foreach $Var (keys %FSmount) {
    print PFH "$Var $FSmount{$Var}\n";
  }
  close (PFH);

############################# 
# End of Disks
#############################

# Check to see if rrd file exists and create it if necessary
#
#$t=time;
  foreach $Val (@AllMeas) {
    $Fname="$Val.rrd";
    $Fname =~ s|\/|_|g;
    $Fname="tia/$Fname";
    print "$Fname " if ($V>=3);
    if (!stat $Fname)  {
      print "Creating RRD for $Val \n";
      RRDs::create ($Fname, "--start",$t-10, "--step",60,
              "DS:Val:GAUGE:120:U:U",
              "RRA:AVERAGE:0.5:1:1440",
              "RRA:AVERAGE:0.5:15:2688",
              "RRA:AVERAGE:0.5:360:1464");
      $ERROR = RRDs::error;
    die "$0: unable to create `$rrd': $ERROR\n" if $ERROR;
    }
# Update the rrd file
    print "$Val $Current{$Val}\n" if ($V>=2);
    RRDs::update $Fname, "$t:$Current{$Val}";
#    if ((defined $Arga{'q'}) & ($ERROR = RRDs::error)) {
    if ($ERROR = RRDs::error) {
      print STDERR "$0: unable to update `$rrd': $ERROR\n";
    }
  }


# Now we have saved everything process the Alerts
#
  print "Now we have saved everything, process the Alerts\n" if  ($V>=1) ;
  print "My hostname is $Hostname\n" if  ($V>=2) ;

  for ($i=0;$i<scalar @Alert;$i++) {
    print "$i $Alert[$i] $Match[$i] $AlertMail[$i]\n" if  ($DEBUG);
    if (eval $Match[$i]) {
# Alert matched, now log to syslog, write a mail and send it
      print "############# Alert $Alert[$i] $Category[$i] $NMatch[$i]\n" if  ($V>=1);
      $Lstring = "/usr/bin/logger -p local6.info -t Topitall \"Alert $Alert[$i] $Category[$i] $NMatch[$i]\"";
      system ($Lstring);
      $Fname="mail$i.sh";
      open (OFH,"> $Fname");
      print OFH "/usr/lib/sendmail $AlertMail[$i] <<EOF\n";
      print OFH "Subject: $Hostname $Alert[$i] : $Category[$i] $NMatch[$i]\n";
      print OFH "\n$AlertComment[$i]\n\nAutomatic Mail from Topitall\n";
      print OFH ".\nEOF\n";
      close (OFH);
      system ("sh $Fname");
      system ("rm $Fname");
    }
  }
}
###################################
# End of daemon
###################################
}

#########################################
# plot
#########################################

elsif (defined $Arga{"plot"}){

$ConfigFile='tia/plot.cfg';
if (defined $Arga{'config'}) {
  if (stat "$Arga{'config'}")  {
    $ConfigFile="$Arga{'config'}"; 
  }
  else  {
    $ConfigFile='tia/plot.cfg';
  }
}
print "plot\n" if ($V>=1); 
open(CFG,"< $ConfigFile") || die("can't open Configuration file:  $ConfigFile $!");
while (<CFG>) {
chomp;
  ($P1,$P2)=split;
  if (/^Host/) {$Hostname=$P2;}
  if (/^Meas/) {@Meas=split;}
#  shift @Meas;
  if (/^\/dev/) { 
    print "Disk $P1 $P2\n" if ($V>=2);
    $FSmount{$P1}=$P2;
  }
}

$NMeas=scalar @Meas;
print "$NMeas :" if ($V>=1);
for ($i=0;$i<$NMeas;$i++) {
  print "$Meas[$i] " if ($V>=1);
}
  

$yday=(sprintf "%d",$DayNow) + 0;
  print "Day of Year $yday\n" if ($V>=1);

# Allow to specify start times in future
if (defined $Arga{'future'}) {
  $Multiplier=-1; 
  }
  else {
  $Multiplier=1;
  }

$t=time+120;
#$t=time+86400*7;

# Make default a day
$TP="day";
  if ($Arga{'day'}==0) {
    $tfirst=$t-86400;
    $tlast=$t; }
  else {
    $tlast=$t-$Arga{'day'}*86400*$Multiplier;
    $tfirst=$tlast - 86400 ; 
  }
 
if (defined $Arga{'hour'}) { 
  $TP="hour";
  if ($Arga{'hour'}==0) {
    $tlast=$t;
    $tfirst=$t-7200; }
  else {
    $tlast=$t-3600*$Arga{'hour'}*$Multiplier;
    $tfirst=$tlast-7200 ; 
  }
}

if (defined $Arga{'week'}) { 
  $TP="week";
  if ($Arga{'week'}==0) {
    $tlast=$t;
    $tfirst=$t-86400*7; }
  else {
    $tlast=$t-7*86400*$Arga{'week'}*$Multiplier;
    $tfirst=$tlast-86400*7 ; 
  }
}

if (defined $Arga{'month'}) { 
  $TP="month";
  if ($Arga{'month'}==0) {
    $tlast=$t;
    $tfirst=$t-86400*28;     
  }
  else {
     $tlast=$t-28*86400*$Arga{'month'}*$Multiplier;
    $tfirst=$tlast-86400*28;
  }
}

if (defined $Arga{"year"}) { 
  print "year requested \n";
  $TP="year";
  $tlast=$t;
  $tfirst=$t-366*86400;
}

print  "Period :$tfirst - $tlast $TP\n" if ($V>=1);


#write html file
$Filename="html/$Hostname.$TP.html";
print "File : $Filename\n" if ($V>=1);
open (GFH,"> $Filename");
print GFH "\n
<!doctype html public>\n
<html>\n
<head>\n
   <meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\">\n
</head>\n
<body>\n
<h1>\n
Topitall : $Hostname </h1>\n
<h2>\n";


# Now the graphs
for ($i=1;$i<$NMeas;$i++) {
  print  "$Meas[$i] File $TP.$Meas[$i].gif\n" if ($V>=1);
#  print GFH "<img SRC=\"$TP.$Meas[$i].gif\" height=200 width=640>\n";
  print GFH "<img SRC=\"$TP.$Meas[$i].gif\" >\n";
}

print GFH "</body>\n</html>\n";
close (GFH);

# Now create the gifs
# 
$Ntp=$Arga{$TP};
if ($Ntp==0) {$Ntp="latest";}

for ($i=1;$i<$NMeas;$i++) {
  $Val=$Meas[$i];
  $Title=$Val;
  if ($Val =~ /dev/) { 
      $Val =~ s|_|/|g;
      $Title = "$FSmount{$Val} [$Val]";
  }
  print "Title $Val            $TP [$Ntp]\n" if ($V>=2);
  $Title = "$Title            $TP [$Ntp]";
  

  $Val =~ s|\/|_|g;
  $OFname= "html/$TP.$Val.gif";
  $IFname="tia/$Val.rrd";
  print "$IFname $OFname" if ($V>=2); 

  RRDs::graph "$OFname",
    "--title", "$Title",
    "--start", "$tfirst",
    "--end", "$tlast",
    "--imgformat","GIF",
   "--width=640",
   "COMMENT:Topitall / J Belshaw                  ",
#    "--height=200",
    "DEF:a=$IFname:Val:AVERAGE",
    "LINE2:a#00b674:$Val\\c";
  if ($ERROR = RRDs::error) {
  print STDERR "ERROR: $ERROR\n";
  }
}
}

################################### Usage
else { 
print <<EOF
topitall -daemon [-v [1-3]] [-q]
run the data collection agent 
-h		This screen
-v [1-3]	Be verbose 
-q		Run in quick mode, 1 min every ten seconds
 
topitall -plot [-day] [days ago] [-week] [weeks ago] [-month] [months ago] [-hour] [hours ago] [-year] [-future] 
[-config configFile]

To specify the time the argument is number of days, weeks, or months in 
the past.  However for testing with the -q option the time runs forwards 
and so you use the -future option to create a plot counting the time forwards.

Creates gif and html files from the data in the topitall html directory :
hostname.day.html
hostname.week.html
hostname.month.html
hostname.year.html
hostname.hour.html

EOF
}



