#!/usr/pkg/bin/perl -w
#
# ufdb_top_urls.pl  -  retrieve most used URLs from ufdbguardd.log
#
# WARNING: retrieval of top urls is slow since large log files are processed.
#          It may take 50 seconds to process 400 MB on a common server.
#
# $Id: ufdb_top_urls.pl.in,v 1.2 2016/10/12 18:34:57 root Exp root $

use Getopt::Long;

my $dummy;
my $debug = 0;
my $need_help;
my $logfilename;

my $min_count = 2;
my $report_size = 20;

my $n_blocked = 0;
my $n_passed = 0;

my $domain_only = 0;
my $process_passed = 1;
my $process_blocked = 0;

my %urls;


sub parse_logfile ($)
{
   my $fn = shift;
   my $ldate;
   my $ltime;
   my $laction;
   my $luser;
   my $lcategory;
   my $lurl;

   print "logfile $fn\n"  if ($debug);

   open IN, "< $fn"  or die "cannot open file \"$fn\": $!";
   while (<IN>)
   {
      chomp;
      @terms = split;
      $laction = $terms[3];
      if (defined($laction))
      {
	 $n_blocked++  if $laction eq 'BLOCK';
	 $n_passed++   if $laction eq 'PASS';
	 if (($laction eq 'PASS'  && $process_passed)  ||
	     ($laction eq 'BLOCK' && $process_blocked))
	 {
	    $lurl = $terms[8];
	    $lurl =~ s,^http://,,;
	    if ($domain_only)
	    {
	       $lurl =~ s,/.*,,;
	    }
	    else
	    {
	       $lurl =~ s,[&\?].*,....,;
	    }
	    if ($debug > 1) 
	    {
	       $ldate = $terms[0];
	       $ltime = $terms[1];
	       $luser = $terms[4];
	       $luser = 'anonymous'  if ($luser eq '-');
	       $lcategory = $terms[7];
	       printf "%s %s  %-14s  %-14s  %-5s  %s\n", $ldate, $ltime, $luser, $lcategory, $laction, $lurl;
	    }

	    $urls{ $lurl } ++;
	 }
      }
   }
   close IN;
}


sub print_topurls ()
{
   my $nlines;

   printf "%d URLs: %d blocked, %d passed.\n", $n_blocked+$n_passed, $n_blocked, $n_passed;
   if ($process_passed && $process_blocked)
   {
      printf "The top list includes blocked and passed URLs.\n";
   }
   elsif ($process_passed) 
   {
      printf "The top list includes only passed URLs.\n";
   }
   elsif ($process_blocked) 
   {
      printf "The top list includes only blocked URLs.\n";
   }

   printf "sorting...\n"  if $debug;

   $nlines = 0;
   foreach $key (sort { $urls {$b} <=> $urls {$a}} keys %urls )
   {
      my $n = $urls{$key};
      last  if $n < $min_count;
      printf "%7d  %s\n", $n, $key;
      $nlines++;
      last  if $nlines >= $report_size;
   }
}


$dummy = GetOptions(
		'help|?'         => \$need_help,
		"debug"          => \$debug,
		"report-size=i"  => \$report_size,
		"domain-only"    => \$domain_only,
		"count-passed!"  => \$process_passed,
		"count-blocked!" => \$process_blocked );

if ($process_passed == 0  &&  $process_blocked == 0)
{
   print "error: no-count-passed and no-count-blocked options imply that not one URL will be counted.\n";
   exit 2;
}

if ($need_help  ||  !defined( $ARGV[0] ))
{
   print "usage: ufdb_top_urls.pl [-[no-]count-passed] [-[no-]count-blocked] [-report-size=N] [-domain-only] [-debug] <logfiles>\n";
   print "defaults: count-passed, no-count-blocked, report-size=20\n";
   exit 1;
}

foreach $logfilename (@ARGV)
{
   parse_logfile $logfilename;
}
print_topurls;

exit 0;
