#!/usr/bin/perl -w
#
##############################################################################
#
# File: user_agent_stats.pl
#
# Usage: cat apache.log | grep somepattern | ./user_agent_stats.pl -p somepattern_agents
#
# Purpose: To parse Apache logs and print stats of user-agent strings in
#          HTTP requests.
#
# Author: Michael Rash <mbr@cipherdyne.org>
#
# License: GPL v2
#
##############################################################################
#

use Date::Calc qw/Decode_Month Date_to_Time/;
use Getopt::Long 'GetOptions';
use strict;

my $file_prefix = '';
my $num_agents_to_plot = 10;  ### default
my $tot_agents = 0;

my %date_cache = ();
my %user_agents = ();
my %user_agents_per_day = ();
my %top_agents = ();
my @top_agents = ();
my $min_time = 0;
my $max_time = 0;
my $help = 0;

my %agent_abbreviations = (
    'Googlebot'          => qr/googlebot/,
    'YahooSlurp'         => qr/\sSlurp/,
    'FeedFetcher-Google' => qr/feedfetcher/,
    'Safari'             => qr/Safari/,
    'wget'               => qr/Wget\//,
    'ichiro'             => qr/ichiro/,
    'TwengaBot'          => qr/TwengaBot/,
    'Bingbot'            => qr/bingbot/i,
    'Yandex'             => qr/Yandex/i,
    'ScoutJet'           => qr/ScoutJet/,
    'Fwknop'             => qr/Fwknop/,
    'Spinn3r'            => qr/Spinn3r/,
    'Akregator'          => qr/Akregator/,
    'Tiny Tiny'          => qr/Tiny\sTiny/,
    'FeedDemon'          => qr/FeedDemon/,
    'Java'               => qr/Java/,
    'curl'               => qr/curl/,
    'Mediapartners-Google' => qr/Mediapartners\-Google/,
    'FeedBurner'         => qr/FeedBurner/,
    'Opera'              => qr/Opera/,
    'PostRank'           => qr/PostRank/,
    'YahooPipes'         => qr/Yahoo\sPipes/,
    'Lynx'               => qr/Lynx/,
);

die "[*] See '$0 -h' for usage information" unless (GetOptions(
    'prefix=s' => \$file_prefix,
    'number=i' => \$num_agents_to_plot,
    'help'     => \$help
));

&usage() if $help;

die "[*] Must specify file prefix with -p <name>" unless $file_prefix;

print "[+] Parsing Apache log data...\n";
while (<STDIN>) {
    ### cipherdyne.org-Aug-2011.gz:66.249.68.147 - - [31/Jul/2011:07:39:13 -0600] \
    ### "GET /git/fwsnort.git/objects/d7/ HTTP/1.1" 200 638 "-" "Mozilla/5.0 \
    ### (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
    if (m|.*\s\"(.*)\"$|) {
        my $agent = $1;
        $agent =~ s|\sfeed\-id=\d+||;  ### subtract out google feed ID's

        if (m|\s\-\s\-\s\[(\d{2})/(\w+)/(\d{4})\:\d{2}|) {
            my $day  = $1;
            my $mon  = $2;
            my $year = $3;

            $day = '01' if $day eq '00';
            ### print "$day $mon $year " . Date_to_Time($year, Decode_Month($mon), $day, 1, 1, 1) . "\n";
            my $time = '';
            if (defined $date_cache{"$day$mon$year"}) {
                $time = $date_cache{"$day$mon$year"};
            } else {
                eval {
                    $time = Date_to_Time($year, Decode_Month($mon), $day, 1, 1, 1);
                };
                if ($@) {
                    die "tried to convert: $year $mon $day    FROM: $_";
                }
                $date_cache{"$day$mon$year"} = $time;
            }
            if ($min_time == 0 and $max_time == 0) {
                $min_time = $max_time = $time;
            }
            $min_time = $time if $time < $min_time;
            $max_time = $time if $time > $max_time;

            my $short_agent = '';
            my $matched = 0;
            for my $agent_abbrev (keys %agent_abbreviations) {
                if ($agent =~ $agent_abbreviations{$agent_abbrev}) {
                    $short_agent = $agent_abbrev;
                    $matched = 1;
                    last;
                }
            }
            unless ($matched) {
                if ($agent =~/\(compatible\;\s*(\S+)/) {
                    $short_agent = $1;
                } else {
                    $short_agent = $agent;
                }
            }
            $short_agent =~ s|\(.*\)||;
            $short_agent =~ s|\S?http\://\S+\s*\)?||;  ### strip out links
            $short_agent =~ s|Mozilla\S+||i;
            $short_agent =~ s|/\S+||g;
            $short_agent =~ s|\s*||g;

            if ($short_agent) {
                $user_agents_per_day{$time}{$short_agent}++;
                $top_agents{$short_agent}++;
            }
        }

        $user_agents{$agent}++;
        $tot_agents++;
    }
}

print "[+] Total agents: " . (keys %user_agents) .
    " (abbreviated to: " . (keys %top_agents) . " agents)\n";

### get the top agents into an array
my $ctr = 0;

for my $agent (sort {$top_agents{$b} <=> $top_agents{$a}} keys %top_agents) {
    push @top_agents, $agent;
    $ctr++;
    last if $ctr >= $num_agents_to_plot;
}

### print out agent stats
open F, "> ${file_prefix}.agents"
    or die "[*] Could not open ${file_prefix}.agents: $!";
for my $agent (sort {$user_agents{$b} <=> $user_agents{$a}} keys %user_agents) {
    printf F "%6d (%.2f%%): %s\n", $user_agents{$agent},
        (100 * $user_agents{$agent} / $tot_agents), $agent;
}
close F;

### write gnuplot data file
open F, "> ${file_prefix}.dat"
    or die "[*] Could not open ${file_prefix}.dat: $!";
for my $time (sort keys %user_agents_per_day) {
    my $str = $time;
    for my $agent (@top_agents) {
        if (defined $user_agents_per_day{$time}{$agent}) {
            $str .= " $user_agents_per_day{$time}{$agent}";
        } else {
            $str .= " 0";
        }
    }
    print F $str, "\n";
}
close F;

### write out gnuplot file
open F, "> ${file_prefix}.gnu"
    or die "[*] Could not open ${file_prefix}.gnu: $!";
print F <<_GNUPLOT_;
set title "Top user agents ($file_prefix)"
set terminal gif nocrop enhanced
set output "$file_prefix.gif"
set xdata time
set timefmt x "%s"
set format x "%b"
set xlabel "time"
# set log xy
set key left
set xrange ["$min_time":"$max_time"]
set grid
_GNUPLOT_

print F qq|plot '$file_prefix.dat' using |;

my $i = 2;
for my $agent (@top_agents) {
    if ($i == 2) {
        print F qq|1:2 with lines title '$agent'|;
    } else {
        print F qq|    '' using 1:$i with lines title '$agent'|;
    }
    $i++;
    if ($i == $#top_agents+3) {
        print F "\n";
    } else {
        print F qq|, \x5c\n|;
    }
}
close F;

print "[+] Executing gnuplot...\n";
system "gnuplot ${file_prefix}.gnu";

print
    "    Plot file:  $file_prefix.gif\n",
    "    Agent stats: $file_prefix.agents\n";

exit 0;

sub usage() {
    print <<_HELP_;
Usage: $0 -p <name> [-n]

Options:

    -p, --prefix <name>       - File prefix that the .gnu, .dat, and .agent
                                files will be named.
    -n, --number <num>        - Number of user agents to plot - default is 10.
    -h, --help                - Display usage info and exit.
_HELP_
    exit 0;
}
