1#!/usr/bin/env python
2
3##
4## chewie.py
5## chews browser http log.  draws graph of connections
6## Be sure there is only one pageload in the log.
7##
8## you'll want to
9##   sudo apt-get install python-matplotlib
10## before running this
11##
12
13import sys, pylab
14
15# can't just use a dict, because there can be dups
16class Queue:
17    def __init__(self):
18        self.queue = []
19
20    def add(self, url, time):
21        self.queue.append([url, time])
22
23    def get(self, url):
24        for x in range(len(self.queue)):
25            rec = self.queue[x]
26            if rec[0] == url:
27                del self.queue[x]
28                return rec[1]
29
30## pull out request lag -- queue to start to done
31def lag():
32
33    font = {'color': '#909090', 'fontsize': 6}
34    extractMe = {
35        'RequestQueue.queueRequest': "Q",
36        'Connection.openHttpConnection()': "O",
37        'Request.sendRequest()': "S",
38        'Request.requestSent()': "T",
39        'processRequests()': 'R',
40        'Request.readResponse():': "D",         # done
41        'clearPipe()': 'U',	                    # unqueue
42        'Request.readResponse()': 'B',          # read data block
43        'Request.readResponseStatus():': 'HR',  # read http response line
44        'hdr': 'H',                             # http header
45        }
46    keys = extractMe.keys()
47
48    f = open(sys.argv[1], "r")
49
50    t0 = None
51
52    # thread, queued, opened, send, sent, reading, read, uri, server, y
53    # 0       1       2       3     4     5        6     7    8       9
54    vals = []
55
56    queued = Queue()
57    opened = {"http0": None,
58              "http1": None,
59              "http2": None,
60              "http3": None,
61              "http4": None,
62              "http5": None}
63    active = {"http0": [],
64              "http1": [],
65              "http2": [],
66              "http3": [],
67              "http4": [],
68              "http5": []}
69    connectionCount = 0
70    byteCount = 0
71    killed = [[], []]
72
73    while (True):
74        line = f.readline()
75        if len(line) == 0: break
76
77        splitup = line.split()
78
79        # http only
80        if splitup[0] != "V/http": continue
81
82        x = splitup[3:]
83
84        # filter to named lines
85        if x[2] not in keys: continue
86        x[2] = extractMe[x[2]]
87
88        # normalize time
89        if t0 == None: t0 = int(x[0])
90        x[0] = int(x[0]) - t0
91
92        thread, action = x[1], x[2]
93        if action == "Q":
94            time, url = x[0], x[3]
95            queued.add(url, time)
96        elif action == "O":
97            # save opened time and server for this thread, so we can stuff it in l8r
98            time, thread, host = x[0], x[1], x[4]
99            opened[thread] = [time, host, connectionCount]
100            connectionCount += 1
101        elif action == "S":
102            time, thread, url = x[0], x[1], x[3]
103            opentime, host, connection = opened[thread]
104            qtime = queued.get(url)
105            record = [thread, qtime, opentime, time, None, None, None, url, host, connection]
106            active[thread].append(record)
107        elif action == "T":
108            time, thread = x[0], x[1]
109            record = active[thread][-1]
110            record[4] = time
111        elif action == "R":
112            print x
113            if x[3] in ["sleep", "no", "wait"]: continue
114            time, thread, = x[0], x[1]
115            record = active[thread][0]
116            record[5] = time
117        elif action == 'U':
118            thread = x[1]
119            record = active[thread][0]
120            killed[0].append(record[9])
121            killed[1].append(x[0])
122            queued.add(record[7], record[1])
123            del active[thread][0]
124        elif action == "D":
125            time, thread = x[0], x[1]
126            record = active[thread][0]
127            record[6] = time
128            vals.append(record)
129            del active[thread][0]
130            print record
131            # print record[3] / 1000, record[6] / 1000, record[7]
132        elif action == "B":
133            byteCount += int(x[3])
134        elif action == "HR":
135            byteCount += int(x[2])
136
137    f.close()
138
139    rng = range(connectionCount)
140
141    opened = []
142    drawn = [False for x in rng]
143    for val in vals:
144        y= val[9]
145        if not drawn[y]:
146            drawn[y] = True
147            opened.append(val[2])
148            pylab.text(0, y - 0.25, "%s %s %s" % (val[9], val[0][4], val[8]), font)
149
150    # define limits
151    # pylab.plot([vals[-1][6]], rng)
152
153    print opened, rng
154    pylab.plot(opened, rng, 'ro')
155    pylab.plot(killed[1], killed[0], 'rx')
156
157    for val in vals:
158        thread, queued, opened, send, sent, reading, read, uri, server, y = val
159        # send arrow
160        arrow = pylab.Arrow(send, y, sent - send, 0)
161        arrow.set_facecolor("g")
162        ax = pylab.gca()
163        ax.add_patch(arrow)
164        # read arrow
165        arrow = pylab.Arrow(reading, y, read - reading, 0)
166        arrow.set_facecolor("r")
167        ax = pylab.gca()
168        ax.add_patch(arrow)
169
170    caption = \
171            "\nrequests: %s\n" % len(vals) + \
172            "byteCount: %s\n" % byteCount + \
173            "data rate: %s\n" % (1000 * byteCount / vals[-1][6])+ \
174            "connections: %s\n" % connectionCount
175
176    pylab.figtext(0.82, 0.30, caption, bbox=dict(facecolor='lightgrey', alpha=0.5))
177
178    # print lines, [[x, x] for x in range(len(vals))]
179    # pylab.plot(lines, [[x, x] for x in range(len(vals))], 'r-')
180
181    pylab.grid()
182    pylab.show()
183
184if __name__ == '__main__': lag()
185