1#!/usr/bin/env python
2
3""" This runs netstat on a local or remote server. It calculates some simple
4statistical information on the number of external inet connections. It groups
5by IP address. This can be used to detect if one IP address is taking up an
6excessive number of connections. It can also send an email alert if a given IP
7address exceeds a threshold between runs of the script. This script can be used
8as a drop-in Munin plugin or it can be used stand-alone from cron. I used this
9on a busy web server that would sometimes get hit with denial of service
10attacks. This made it easy to see if a script was opening many multiple
11connections. A typical browser would open fewer than 10 connections at once. A
12script might open over 100 simultaneous connections.
13
14./topip.py [-s server_hostname] [-u username] [-p password] {-a from_addr,to_addr} {-n N} {-v} {--ipv6}
15
16    -s : hostname of the remote server to login to.
17    -u : username to user for login.
18    -p : password to user for login.
19    -n : print stddev for the the number of the top 'N' ipaddresses.
20    -v : verbose - print stats and list of top ipaddresses.
21    -a : send alert if stddev goes over 20.
22    -l : to log message to /var/log/topip.log
23    --ipv6 : this parses netstat output that includes ipv6 format.
24        Note that this actually only works with ipv4 addresses, but for versions of
25        netstat that print in ipv6 format.
26    --stdev=N : Where N is an integer. This sets the trigger point for alerts and logs.
27        Default is to trigger if max value is above 5 standard deviations.
28
29Example:
30
31    This will print stats for the top IP addresses connected to the given host:
32
33        ./topip.py -s www.example.com -u mylogin -p mypassword -n 10 -v
34
35    This will send an alert email if the maxip goes over the stddev trigger value and
36    the the current top ip is the same as the last top ip (/tmp/topip.last):
37
38        ./topip.py -s www.example.com -u mylogin -p mypassword -n 10 -v -a alert@example.com,user@example.com
39
40    This will print the connection stats for the localhost in Munin format:
41
42        ./topip.py
43
44Noah Spurrier
45
46$Id: topip.py 489 2007-11-28 23:40:34Z noah $
47"""
48
49import pexpect, pxssh # See http://pexpect.sourceforge.net/
50import os, sys, time, re, getopt, pickle, getpass, smtplib
51import traceback
52from pprint import pprint
53
54TOPIP_LOG_FILE = '/var/log/topip.log'
55TOPIP_LAST_RUN_STATS = '/var/run/topip.last'
56
57def exit_with_usage():
58
59    print globals()['__doc__']
60    os._exit(1)
61
62def stats(r):
63
64    """This returns a dict of the median, average, standard deviation, min and max of the given sequence.
65
66    >>> from topip import stats
67    >>> print stats([5,6,8,9])
68    {'med': 8, 'max': 9, 'avg': 7.0, 'stddev': 1.5811388300841898, 'min': 5}
69    >>> print stats([1000,1006,1008,1014])
70    {'med': 1008, 'max': 1014, 'avg': 1007.0, 'stddev': 5.0, 'min': 1000}
71    >>> print stats([1,3,4,5,18,16,4,3,3,5,13])
72    {'med': 4, 'max': 18, 'avg': 6.8181818181818183, 'stddev': 5.6216817577237475, 'min': 1}
73    >>> print stats([1,3,4,5,18,16,4,3,3,5,13,14,5,6,7,8,7,6,6,7,5,6,4,14,7])
74    {'med': 6, 'max': 18, 'avg': 7.0800000000000001, 'stddev': 4.3259218670706474, 'min': 1}
75    """
76
77    total = sum(r)
78    avg = float(total)/float(len(r))
79    sdsq = sum([(i-avg)**2 for i in r])
80    s = list(r)
81    s.sort()
82    return dict(zip(['med', 'avg', 'stddev', 'min', 'max'] , (s[len(s)//2], avg, (sdsq/len(r))**.5, min(r), max(r))))
83
84def send_alert (message, subject, addr_from, addr_to, smtp_server='localhost'):
85
86    """This sends an email alert.
87    """
88
89    message = 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n' % (addr_from, addr_to, subject) + message
90    server = smtplib.SMTP(smtp_server)
91    server.sendmail(addr_from, addr_to, message)
92    server.quit()
93
94def main():
95
96    ######################################################################
97    ## Parse the options, arguments, etc.
98    ######################################################################
99    try:
100        optlist, args = getopt.getopt(sys.argv[1:], 'h?valqs:u:p:n:', ['help','h','?','ipv6','stddev='])
101    except Exception, e:
102        print str(e)
103        exit_with_usage()
104    options = dict(optlist)
105
106    munin_flag = False
107    if len(args) > 0:
108        if args[0] == 'config':
109            print 'graph_title Netstat Connections per IP'
110            print 'graph_vlabel Socket connections per IP'
111            print 'connections_max.label max'
112            print 'connections_max.info Maximum number of connections per IP'
113            print 'connections_avg.label avg'
114            print 'connections_avg.info Average number of connections per IP'
115            print 'connections_stddev.label stddev'
116            print 'connections_stddev.info Standard deviation'
117            return 0
118        elif args[0] != '':
119            print args, len(args)
120            return 0
121            exit_with_usage()
122    if [elem for elem in options if elem in ['-h','--h','-?','--?','--help']]:
123        print 'Help:'
124        exit_with_usage()
125    if '-s' in options:
126        hostname = options['-s']
127    else:
128        # if host was not specified then assume localhost munin plugin.
129        munin_flag = True
130        hostname = 'localhost'
131    # If localhost then don't ask for username/password.
132    if hostname != 'localhost' and hostname != '127.0.0.1':
133        if '-u' in options:
134            username = options['-u']
135        else:
136            username = raw_input('username: ')
137        if '-p' in options:
138            password = options['-p']
139        else:
140            password = getpass.getpass('password: ')
141    else:
142        use_localhost = True
143
144    if '-l' in options:
145        log_flag = True
146    else:
147        log_flag = False
148    if '-n' in options:
149        average_n = int(options['-n'])
150    else:
151        average_n = None
152    if '-v' in options:
153        verbose = True
154    else:
155        verbose = False
156    if '-a' in options:
157        alert_flag = True
158        (alert_addr_from, alert_addr_to) = tuple(options['-a'].split(','))
159    else:
160        alert_flag = False
161    if '--ipv6' in options:
162        ipv6_flag = True
163    else:
164        ipv6_flag = False
165    if '--stddev' in options:
166        stddev_trigger = float(options['--stddev'])
167    else:
168        stddev_trigger = 5
169
170    if ipv6_flag:
171        netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+::ffff:(\S+):(\S+)\s+.*?\r'
172    else:
173        netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(?:::ffff:)*(\S+):(\S+)\s+.*?\r'
174        #netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+):(\S+)\s+.*?\r'
175
176    # run netstat (either locally or via SSH).
177    if use_localhost:
178        p = pexpect.spawn('netstat -n -t')
179        PROMPT = pexpect.TIMEOUT
180    else:
181        p = pxssh.pxssh()
182        p.login(hostname, username, password)
183        p.sendline('netstat -n -t')
184        PROMPT = p.PROMPT
185
186    # loop through each matching netstat_pattern and put the ip address in the list.
187    ip_list = {}
188    try:
189        while 1:
190            i = p.expect([PROMPT, netstat_pattern])
191            if i == 0:
192                break
193            k = p.match.groups()[4]
194            if k in ip_list:
195                ip_list[k] = ip_list[k] + 1
196            else:
197                ip_list[k] = 1
198    except:
199        pass
200
201    # remove a few common, uninteresting addresses from the dictionary.
202    ip_list = dict([ (key,value) for key,value in ip_list.items() if '192.168.' not in key])
203    ip_list = dict([ (key,value) for key,value in ip_list.items() if '127.0.0.1' not in key])
204
205    # sort dict by value (count)
206    #ip_list = sorted(ip_list.iteritems(),lambda x,y:cmp(x[1], y[1]),reverse=True)
207    ip_list = ip_list.items()
208    if len(ip_list) < 1:
209        if verbose: print 'Warning: no networks connections worth looking at.'
210        return 0
211    ip_list.sort(lambda x,y:cmp(y[1],x[1]))
212
213    # generate some stats for the ip addresses found.
214    if average_n <= 1:
215        average_n = None
216    s = stats(zip(*ip_list[0:average_n])[1]) # The * unary operator treats the list elements as arguments
217    s['maxip'] = ip_list[0]
218
219    # print munin-style or verbose results for the stats.
220    if munin_flag:
221        print 'connections_max.value', s['max']
222        print 'connections_avg.value', s['avg']
223        print 'connections_stddev.value', s['stddev']
224        return 0
225    if verbose:
226        pprint (s)
227        print
228        pprint (ip_list[0:average_n])
229
230    # load the stats from the last run.
231    try:
232        last_stats = pickle.load(file(TOPIP_LAST_RUN_STATS))
233    except:
234        last_stats = {'maxip':None}
235
236    if s['maxip'][1] > (s['stddev'] * stddev_trigger) and s['maxip']==last_stats['maxip']:
237        if verbose: print 'The maxip has been above trigger for two consecutive samples.'
238        if alert_flag:
239            if verbose: print 'SENDING ALERT EMAIL'
240            send_alert(str(s), 'ALERT on %s' % hostname, alert_addr_from, alert_addr_to)
241        if log_flag:
242            if verbose: print 'LOGGING THIS EVENT'
243            fout = file(TOPIP_LOG_FILE,'a')
244            #dts = time.strftime('%Y:%m:%d:%H:%M:%S', time.localtime())
245            dts = time.asctime()
246            fout.write ('%s - %d connections from %s\n' % (dts,s['maxip'][1],str(s['maxip'][0])))
247            fout.close()
248
249    # save state to TOPIP_LAST_RUN_STATS
250    try:
251        pickle.dump(s, file(TOPIP_LAST_RUN_STATS,'w'))
252        os.chmod (TOPIP_LAST_RUN_STATS, 0664)
253    except:
254        pass
255    # p.logout()
256
257if __name__ == '__main__':
258    try:
259        main()
260        sys.exit(0)
261    except SystemExit, e:
262        raise e
263    except Exception, e:
264        print str(e)
265        traceback.print_exc()
266        os._exit(1)
267
268