apache-logparser/apache-logparser/logparser.py

#!/bin/env python

#    Simple Apache log parser
#    Copyright (C) 2020  Pekka Helenius
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <https://www.gnu.org/licenses/>.

################################################################

# TODO prev_host: instead of comparing to previous entry, check if such IP has been seen in XXX seconds
# store IP values for temporary list for XXX seconds, and check list values

import argparse
import os
import re
import subprocess
import sys
from datetime import datetime
from apachelogs import LogParser

out_fields_list = ['log_file_name', 'http_status', 'remote_host', 'country', 'city', 'time', 'time_diff', 'user_agent', 'http_request']
out_timeformat  = "%d-%m-%Y %H:%M:%S"
dayformat       = "%d-%m-%Y"
ot              = '"' + re.sub(r'%', '%%', out_timeformat) + '"'
geotool         = "geoiplookup"
geodb           = "/usr/share/GeoIP/"

# Log format as defined in Apache/HTTPD configuration file (LogFormat directive)
in_log_syntax   = "%h %u %t %T \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" \"%{cache-status}e\" %I %O"

argparser = argparse.ArgumentParser()

argparser.add_argument('-d',  '--dir',            help = 'Apache log file directory.', nargs = '?', dest = 'log_dir', required = True)
argparser.add_argument('-f',  '--files',          help = 'Apache log files. Regular expressions supported.', nargs = '+', dest = 'log_file', required = True)
argparser.add_argument('-s',  '--logsyntax',      help = 'Apache log files syntax, defined as "LogFormat" directive in Apache configuration.', nargs = '?', dest = 'log_syntax')
argparser.add_argument('-c',  '--statuscodes',    help = 'Print only these status codes. Regular expressions supported.', nargs = '+', dest = 'status_code')
argparser.add_argument('-cf', '--countryfilter',  help = 'Include only these countries. Negative match (exclude): "\!Country"', nargs = '+', dest = 'country')
argparser.add_argument('-ot', '--outtimeformat',  help = 'Output time format.\nDefault: ' + ot, nargs = '?', dest = 'out_timeformat')
argparser.add_argument('-of', '--outfields',      help = 'Output fields.\nDefault: ' + ', '.join(out_fields_list), nargs = '+', dest = 'out_field')
argparser.add_argument('-ng', '--nogeo',          help = 'Skip country check with external "geoiplookup" tool.', action='store_true', dest = 'no_geo')
argparser.add_argument('-gd', '--geodir',         help = 'Database file directory for "geoiplookup" tool.\nDefault: ' + geodb, nargs = '?', dest = 'geodb')
argparser.add_argument('-dl', '--daylower',       help = 'Do not check log entries older than this day.\nDay syntax: 31-12-2020', nargs = '?', dest = 'day_lower')
argparser.add_argument('-du', '--dayupper',       help = 'Do not check log entries newer than this day.\nDay syntax: 31-12-2020', nargs = '?', dest = 'day_upper')
argparser.add_argument('-sb', '--sortby',         help = 'Sort by an output field.', nargs = '?', dest = 'sortby_field')
argparser.add_argument('-sbr', '--sortbyreverse', help = 'Sort by an output field, reverse order.', nargs = '?', dest = 'sortby_field_reverse')
argparser.add_argument('-st', '--stats',          help = 'Show short statistics at the end.', action='store_true', dest = 'show_count')
argparser.add_argument('-np', '--noprogress',     help = 'Do not show progress information.', action='store_true', dest = 'no_progress')
args = argparser.parse_args()

if args.status_code is None:
    status_filter = False
    skip_line_1   = False
else:
    status_filter = True
    skip_line_1   = True
    status_codes  = args.status_code

    http_valid_codes    = [
        '100',
        '101',
        '102',
        '103',
        '200',
        '201',
        '202',
        '203',
        '204',
        '205',
        '206',
        '207',
        '208',
        '226',
        '300',
        '301',
        '302',
        '303',
        '304',
        '305',
        '306',
        '307',
        '308',
        '400',
        '401',
        '402',
        '403',
        '404',
        '405',
        '406',
        '407',
        '408',
        '409',
        '410',
        '411',
        '412',
        '413',
        '414',
        '415',
        '416',
        '417',
        '418',
        '421',
        '422',
        '423',
        '424',
        '425',
        '426',
        '428',
        '429',
        '431',
        '451',
        '500',
        '501',
        '502',
        '503',
        '504',
        '505',
        '506',
        '507',
        '508',
        '510',
        '511',
        '218'
    ]

    code_statuses = []
    for status_input in status_codes:
        init_status     = False
        status_append   = status_input
        status_appended = False

        for status_valid in http_valid_codes:

            if re.search(status_input, status_valid):
                status_append = status_valid
                init_status     = True
                status_appended = True
                code_statuses.append((status_append, init_status))
            else:
                 init_status  = False
        if not status_appended:
            code_statuses.append((status_append, init_status))

    error_msg = ""
    for vl in code_statuses:
        status, init_status = vl

        if not init_status:
            error_msg += "Invalid status code '" + status + "' supplied\n"

    if error_msg != "":
        raise Exception("\n" + error_msg)

if args.country is None:
    country_filter = False
    skip_line_2    = False
else:
    country_filter = True
    countries_filter_list = args.country
    skip_line_2    = True

if args.out_timeformat is not None:
    out_timeformat = args.out_timeformat

if args.out_field is not None:
    out_fields_list = args.out_field

if args.day_lower is not None:
    day_lower = datetime.strptime(args.day_lower, dayformat)
else:
    day_lower = None
if args.day_upper is not None:
    day_upper = datetime.strptime(args.day_upper, dayformat)
else:
    day_upper = None

if args.log_syntax is None:
    log_syntax = in_log_syntax
else:
    log_syntax = args.log_syntax

log_dir     = args.log_dir
files       = args.log_file
no_progress = args.no_progress
files_tmp   = []
parser      = LogParser(log_syntax)

for file_regex in files:
    for file in os.listdir(log_dir):
        fullpath = log_dir + file
        if os.path.isfile(fullpath):
            if re.search(file_regex, file):
                files_tmp.append(file)

    files_tmp.sort()
    files = files_tmp

def fileCheck(file, flag, env=None):
    if env is None:
        filepath = file
    else:
        for path in os.environ[env].split(os.pathsep):
            filepath = os.path.join(path, file)
            if os.path.isfile(filepath):
                break

    if os.access(filepath, eval(flag)):
        return True

    return False

# TODO Really exclude, when no additional args are passed to either of both
if args.sortby_field is not None and args.sortby_field_reverse is not None:
    raise Exception("Use either normal or reverse sorting.")

sortby_field = None
if args.sortby_field is not None:
    sortby_field  = args.sortby_field
    reverse_order = False
elif args.sortby_field_reverse is not None:
    sortby_field  = args.sortby_field_reverse
    reverse_order = True

i            = 0
country_seen = False
prev_host    = ""
host_country = ""
host_city    = ""
log_entries  = []

for file in files:
    if not no_progress:
        print("Processing file: " + file)

    with open(log_dir + file, 'r') as f:

        for line in f:

            if not no_progress:
                print("Processing log entry: " + str(i), end = "\r")

            if i != 0 and not (skip_line_1 or skip_line_2):
                prev_host      = entry_remote_host
                prev_host_time = entry_time

            entry              = parser.parse(line)
            entry_time         = entry.request_time.replace(tzinfo=None)

            # TODO Handle situations where date_upper & date_lower are equal
            if day_upper is not None and day_lower is not None:
                if day_lower > day_upper:
                    raise Exception("Earlier day can't be later than later day")

            if day_upper is not None:
                if day_upper > datetime.now():
                    raise Exception("Day can't be in the future")

            if day_lower is not None:
                if day_lower > datetime.now():
                    raise Exception("Day can't be in the future")

            if day_lower is not None:
                if entry_time <= day_lower: continue

            if day_upper is not None:
                if entry_time >= day_upper: continue

            entry_remote_host  = entry.remote_host
            entry_http_status  = entry.final_status
            entry_user_agent   = entry.headers_in["User-Agent"]

            # In case where request has newline or other similar chars. Tell Python interpreter to escape them
            entry_http_request = str(entry.request_line).encode('unicode_escape').decode()

            if status_filter:
                for status in code_statuses:
                    num, num_ok = status
                    status = int(num)
                    if status != entry_http_status:
                        skip_line_1 = True
                    else:
                        skip_line_1 = False
                        break

            if not args.no_geo and fileCheck(geotool, "os.X_OK", "PATH") and fileCheck(geodb, "os.R_OK"):
                if prev_host == entry.remote_host:
                    country_seen = True
                else:
                    country_seen = False

                if not country_seen:
                    host_country_main = subprocess.check_output([geotool,'-d',geodb,entry_remote_host]).rstrip().decode()
                    host_country_main = host_country_main.split('\n')
                    host_country = re.sub(r"^.*, (.*)", r'\1', host_country_main[0])

                    if re.search("Address not found", host_country):
                        host_country = "Unknown"

                    else:
                        if len(host_country_main) > 1:
                            host_city = host_country_main[1].split(', ')[4]
                            if re.search("N/A", host_city):
                                host_city = "Unknown: " + host_country_main[1].split(', ')[6] + ', ' + host_country_main[1].split(', ')[7]

                if country_filter:
                    for country in countries_filter_list:
                        if country[1] == "!":
                            country = country[2:]
                            if country.lower() == host_country.lower():
                                skip_line_2 = True
                                break
                            else:
                                skip_line_2 = False

                        elif country.lower() != host_country.lower():
                            skip_line_2 = True
                        else:
                            skip_line_2 = False
                            break

            else:
                skip_line_2 = False

            if skip_line_1 or skip_line_2:
                i += 1
                continue

            time_diff = str("NEW_CONN")
            if prev_host == entry_remote_host:
                time_diff = ( entry_time - prev_host_time ).total_seconds()
                if time_diff > 0:
                    time_diff = "+" + str(time_diff)
            if i == 0:
                time_diff = float(0.0)

            # TODO: Optimize stri generation logic, avoid generating multiple times since it's really not necessary
            out_fields = [
                ('log_file_name', file,               '{:s}'  ),
                ('http_status',   entry_http_status,  '{:3s}' ),
                ('remote_host',   entry_remote_host,  '{:15s}'),
                ('country',       host_country,       '{:20s}'),
                ('city',          host_city,          '{:15s}'),
                ('time',          entry_time,         '{:8s}' ),
                ('time_diff',     time_diff,          '{:8s}' ),
                ('user_agent',    entry_user_agent,   '{:s}'  ),
                ('http_request',  entry_http_request, '{:s}'  )
            ]

            stri = ""
            printargs = []
            t = 0
            while t <= len(out_fields_list) - 1:

                for out_field in out_fields:
                    entry, data, striformat = out_field

                    if args.no_geo and (entry == "country" or entry == "city"):
                        continue

                    if out_fields_list[t] == entry:
                        stri += "\t" + striformat
                        printargs.append(data)
                        break
                t += 1

            log_entries.append(printargs)

            i += 1

if sortby_field is not None:
    sort_field_found = False
    d = 0
    for field in out_fields_list:
        if field == sortby_field:
            sort_field_index = d
            sort_field_found = True
            break
        d += 1

    if sort_field_found:
        log_entries.sort(key = lambda log_entries: log_entries[sort_field_index], reverse=reverse_order)

if not no_progress:
    print("\n")

for entry in log_entries:
    c = 0
    entry_tmp = []
    while c <= len(entry) - 1:
        entry_tmp.append(str(entry[c]))
        c += 1
    print(stri.format(*entry_tmp).lstrip())

if args.show_count:
    print(("\n" +
        "Processed files:       {:s}\n" +
        "Processed log entries: {:d}\n" +
        "Matched log entries:   {:d}\n").format(
                ', '.join(files),
                i,
                len(log_entries)
            )
        )