168da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs#!/usr/bin/env python 268da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 368da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs# Copyright 2017 The Chromium OS Authors. All rights reserved. 468da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs# Use of this source code is governed by a BSD-style license that can be 568da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs# found in the LICENSE file. 668da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 768da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs"""A script to parse apache error logs 868da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 968da574876290eeaace08ab38ed1b63ac596b962Paul HobbsThe script gets the contents of the log file through stdin, and emits a counter 1068da574876290eeaace08ab38ed1b63ac596b962Paul Hobbsmetric for the beginning of each error message it recognizes. 1168da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs""" 1268da574876290eeaace08ab38ed1b63ac596b962Paul Hobbsfrom __future__ import print_function 1368da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 1468da574876290eeaace08ab38ed1b63ac596b962Paul Hobbsimport argparse 1568da574876290eeaace08ab38ed1b63ac596b962Paul Hobbsimport re 1668da574876290eeaace08ab38ed1b63ac596b962Paul Hobbsimport sys 1768da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 1868da574876290eeaace08ab38ed1b63ac596b962Paul Hobbsimport common 1968da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 2068da574876290eeaace08ab38ed1b63ac596b962Paul Hobbsfrom chromite.lib import metrics 2168da574876290eeaace08ab38ed1b63ac596b962Paul Hobbsfrom chromite.lib import ts_mon_config 225bbdda437a6cf7eb80808e806b2e9ce86700f164Prathmesh Prabhu# infra_libs comes from chromite's third_party modules. 235bbdda437a6cf7eb80808e806b2e9ce86700f164Prathmesh Prabhufrom infra_libs import ts_mon 2468da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 255d122a264d9cfeb1ba2fa6eae4a49f1a09d486f3Paul Hobbsfrom autotest_lib.site_utils.stats import log_daemon_common 2668da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 2768da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 2868da574876290eeaace08ab38ed1b63ac596b962Paul HobbsLOOP_INTERVAL = 60 2968da574876290eeaace08ab38ed1b63ac596b962Paul HobbsERROR_LOG_METRIC = '/chromeos/autotest/apache/error_log' 305c2ced33986dd7682e2f136e20ee3958b90739caPaul HobbsERROR_LOG_LINE_METRIC = '/chromeos/autotest/apache/error_log_line' 312d2556f3a15e26ee700291c72066170435c8d6d7Paul HobbsSEGFAULT_METRIC = '/chromeos/autotest/apache/segfault_count' 322d2556f3a15e26ee700291c72066170435c8d6d7Paul HobbsSTART_METRIC = '/chromeos/autotest/apache/start_count' 332d2556f3a15e26ee700291c72066170435c8d6d7Paul HobbsSTOP_METRIC = '/chromeos/autotest/apache/stop_count' 342d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 3568da574876290eeaace08ab38ed1b63ac596b962Paul HobbsERROR_LOG_MATCHER = re.compile( 3668da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs r'^\[[^]]+\] ' # The timestamp. We don't need this. 379eff288abe3f64e9e96139a4e89d46841dacf14fPrathmesh Prabhu r'\[(mpm_event|core)?:(?P<log_level>\S+)\] ' 3868da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs r'\[pid \d+[^]]+\] ' # The PID, possibly followed by a task id. 3968da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs # There may be other sections, such as [remote <ip>] 4068da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs r'(?P<sections>\[[^]]+\] )*' 4168da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs r'\S' # first character after pid must be non-space; otherwise it is 4268da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs # indented, meaning it is just a continuation of a previous message. 4368da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs r'(?P<mod_wsgi>od_wsgi)?' # Note: the 'm' of mod_wsgi was already matched. 442d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs r'(?P<rest>.*)' 4568da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs) 4668da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 472d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbsdef EmitSegfault(_m): 482d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs """Emits a Counter metric for segfaults. 492d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 502d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs @param _m: A regex match object 512d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs """ 522d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs metrics.Counter( 532d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs SEGFAULT_METRIC, 542d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs description='A metric counting segfaults in apache', 552d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs field_spec=None, 562d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs ).increment() 572d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 582d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 592d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbsdef EmitStart(_m): 602d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs """Emits a Counter metric for apache service starts. 612d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 622d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs @param _m: A regex match object 632d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs """ 642d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 652d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs metrics.Counter( 662d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs START_METRIC, 672d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs description="A metric counting Apache service starts.", 682d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs field_spec=None, 692d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs ).increment() 702d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 712d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 722d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbsdef EmitStop(_m, graceful): 732d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs """Emits a Counter metric for apache service stops 742d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 752d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs @param _m: A regex match object 762d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs @param graceful: Whether apache was stopped gracefully. 772d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs """ 782d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs metrics.Counter( 792d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs STOP_METRIC, 802d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs description="A metric counting Apache service stops.", 812d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs field_spec=[ts_mon.BooleanField('graceful')] 822d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs ).increment(fields={ 832d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 'graceful': graceful 842d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs }) 852d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 862d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 872d2556f3a15e26ee700291c72066170435c8d6d7Paul HobbsMESSAGE_PATTERNS = { 882d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs r'Segmentation fault': EmitSegfault, 892d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs r'configured -- resuming normal operations': EmitStart, 902d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs r'caught SIGTERM, shutting down': lambda m: EmitStop(m, graceful=True), 912d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs # TODO(phobbs) add log message for when Apache dies ungracefully 922d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs} 932d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 942d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 952d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbsdef EmitErrorLog(m): 962d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs """Emits a Counter metric for error log messages. 9768da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 982d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs @param m: A regex match object 992d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs """ 1002d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs log_level = m.group('log_level') or '' 1012d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs # It might be interesting to see whether the error/warning was emitted 1022d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs # from python at the mod_wsgi process or not. 1032d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs mod_wsgi_present = bool(m.group('mod_wsgi')) 10468da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 1052d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs metrics.Counter(ERROR_LOG_METRIC).increment(fields={ 1062d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 'log_level': log_level, 1072d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs 'mod_wsgi': mod_wsgi_present}) 10868da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 1092d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs rest = m.group('rest') 1102d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs for pattern, handler in MESSAGE_PATTERNS.iteritems(): 1112d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs if pattern in rest: 1122d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs handler(m) 11368da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 11468da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 1152d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbsdef EmitErrorLogLine(_m): 1162d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs """Emits a Counter metric for each error log line. 1175c2ced33986dd7682e2f136e20ee3958b90739caPaul Hobbs 1182d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs @param _m: A regex match object. 1192d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs """ 1202d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs metrics.Counter( 1212d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs ERROR_LOG_LINE_METRIC, 1222d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs description="A count of lines emitted to the apache error log.", 1232d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs field_spec=None, 1242d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs ).increment() 1255c2ced33986dd7682e2f136e20ee3958b90739caPaul Hobbs 1265c2ced33986dd7682e2f136e20ee3958b90739caPaul Hobbs 12768da574876290eeaace08ab38ed1b63ac596b962Paul HobbsMATCHERS = [ 1282d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs (ERROR_LOG_MATCHER, EmitErrorLog), 1292d2556f3a15e26ee700291c72066170435c8d6d7Paul Hobbs (re.compile(r'.*'), EmitErrorLogLine), 13068da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs] 13168da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 13268da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 1333358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbsdef ParseArgs(): 1343358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs """Parses the command line arguments.""" 13568da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs p = argparse.ArgumentParser( 13668da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs description='Parses apache logs and emits metrics to Monarch') 13768da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs p.add_argument('--output-logfile') 1383358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs p.add_argument('--debug-metrics-file', 1393358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs help='Output metrics to the given file instead of sending ' 1403358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs 'them to production.') 1413358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs return p.parse_args() 1423358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs 1433358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs 1443358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbsdef Main(): 1453358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs """Sets up logging and runs matchers against stdin""" 1463358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs args = ParseArgs() 1475d122a264d9cfeb1ba2fa6eae4a49f1a09d486f3Paul Hobbs log_daemon_common.SetupLogging(args) 14868da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 1493358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs # Set up metrics sending and go. 1503358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs ts_mon_args = {} 1513358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs if args.debug_metrics_file: 1523358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs ts_mon_args['debug_file'] = args.debug_metrics_file 1533358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs 1543358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs with ts_mon_config.SetupTsMonGlobalState('apache_error_log_metrics', 1553358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs **ts_mon_args): 1565d122a264d9cfeb1ba2fa6eae4a49f1a09d486f3Paul Hobbs log_daemon_common.RunMatchers(sys.stdin, MATCHERS) 1573358c52046874b3929b5ff1df723e1ddd8d60082Paul Hobbs metrics.Flush() 15868da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 15968da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs 16068da574876290eeaace08ab38ed1b63ac596b962Paul Hobbsif __name__ == '__main__': 16168da574876290eeaace08ab38ed1b63ac596b962Paul Hobbs Main() 162