1#! /usr/bin/env python
2#
3# btt_plot.py: Generate matplotlib plots for BTT generate data files
4#
5#  (C) Copyright 2009 Hewlett-Packard Development Company, L.P.
6#
7#  This program is free software; you can redistribute it and/or modify
8#  it under the terms of the GNU General Public License as published by
9#  the Free Software Foundation; either version 2 of the License, or
10#  (at your option) any later version.
11#
12#  This program is distributed in the hope that it will be useful,
13#  but WITHOUT ANY WARRANTY; without even the implied warranty of
14#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15#  GNU General Public License for more details.
16#
17#  You should have received a copy of the GNU General Public License
18#  along with this program; if not, write to the Free Software
19#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20#
21
22"""
23btt_plot.py: Generate matplotlib plots for BTT generated data files
24
25Files handled:
26  AQD	- Average Queue Depth		Running average of queue depths
27
28  BNOS	- Block numbers accessed	Markers for each block
29
30  Q2D	- Queue to Issue latencies	Running averages
31  D2C	- Issue to Complete latencies	Running averages
32  Q2C	- Queue to Complete latencies	Running averages
33
34Usage:
35  btt_plot_aqd.py	equivalent to: btt_plot.py -t aqd	<type>=aqd
36  btt_plot_bnos.py	equivalent to: btt_plot.py -t bnos	<type>=bnos
37  btt_plot_q2d.py	equivalent to: btt_plot.py -t q2d	<type>=q2d
38  btt_plot_d2c.py	equivalent to: btt_plot.py -t d2c	<type>=d2c
39  btt_plot_q2c.py	equivalent to: btt_plot.py -t q2c	<type>=q2c
40
41Arguments:
42  [ -A          | --generate-all   ] Default: False
43  [ -L          | --no-legend      ] Default: Legend table produced
44  [ -o <file>   | --output=<file>  ] Default: <type>.png
45  [ -T <string> | --title=<string> ] Default: Based upon <type>
46  [ -v          | --verbose        ] Default: False
47  <data-files...>
48
49  The -A (--generate-all) argument is different: when this is specified,
50  an attempt is made to generate default plots for all 5 types (aqd, bnos,
51  q2d, d2c and q2c). It will find files with the appropriate suffix for
52  each type ('aqd.dat' for example). If such files are found, a plot for
53  that type will be made. The output file name will be the default for
54  each type. The -L (--no-legend) option will be obeyed for all plots,
55  but the -o (--output) and -T (--title) options will be ignored.
56"""
57
58__author__ = 'Alan D. Brunelle <alan.brunelle@hp.com>'
59
60#------------------------------------------------------------------------------
61
62import matplotlib
63matplotlib.use('Agg')
64import getopt, glob, os, sys
65import matplotlib.pyplot as plt
66
67plot_size	= [10.9, 8.4]	# inches...
68
69add_legend	= True
70generate_all	= False
71output_file	= None
72title_str	= None
73type		= None
74verbose		= False
75
76types		= [ 'aqd', 'q2d', 'd2c', 'q2c', 'bnos' ]
77progs		= [ 'btt_plot_%s.py' % t for t in types ]
78
79get_base 	= lambda file: file[file.find('_')+1:file.rfind('_')]
80
81#------------------------------------------------------------------------------
82def fatal(msg):
83	"""Generate fatal error message and exit"""
84
85	print >>sys.stderr, 'FATAL: %s' % msg
86	sys.exit(1)
87
88#----------------------------------------------------------------------
89def get_data(files):
90	"""Retrieve data from files provided.
91
92	Returns a database containing:
93		'min_x', 'max_x' 	- Minimum and maximum X values found
94		'min_y', 'max_y' 	- Minimum and maximum Y values found
95		'x', 'y'		- X & Y value arrays
96		'ax', 'ay'		- Running average over X & Y --
97					  if > 10 values provided...
98	"""
99	#--------------------------------------------------------------
100	def check(mn, mx, v):
101		"""Returns new min, max, and float value for those passed in"""
102
103		v = float(v)
104		if mn == None or v < mn: mn = v
105		if mx == None or v > mx: mx = v
106		return mn, mx, v
107
108	#--------------------------------------------------------------
109	def avg(xs, ys):
110		"""Computes running average for Xs and Ys"""
111
112		#------------------------------------------------------
113		def _avg(vals):
114			"""Computes average for array of values passed"""
115
116			total = 0.0
117			for val in vals:
118				total += val
119			return total / len(vals)
120
121		#------------------------------------------------------
122		if len(xs) < 1000:
123			return xs, ys
124
125		axs = [xs[0]]
126		ays = [ys[0]]
127		_xs = [xs[0]]
128		_ys = [ys[0]]
129
130		x_range = (xs[-1] - xs[0]) / 100
131		for idx in range(1, len(ys)):
132			if (xs[idx] - _xs[0]) > x_range:
133				axs.append(_avg(_xs))
134				ays.append(_avg(_ys))
135				del _xs, _ys
136
137				_xs = [xs[idx]]
138				_ys = [ys[idx]]
139			else:
140				_xs.append(xs[idx])
141				_ys.append(ys[idx])
142
143		if len(_xs) > 1:
144			axs.append(_avg(_xs))
145			ays.append(_avg(_ys))
146
147		return axs, ays
148
149	#--------------------------------------------------------------
150	global verbose
151
152	db = {}
153	min_x = max_x = min_y = max_y = None
154	for file in files:
155		if not os.path.exists(file):
156			fatal('%s not found' % file)
157		elif verbose:
158			print 'Processing %s' % file
159
160		xs = []
161		ys = []
162		for line in open(file, 'r'):
163			f = line.rstrip().split(None)
164			if line.find('#') == 0 or len(f) < 2:
165				continue
166			(min_x, max_x, x) = check(min_x, max_x, f[0])
167			(min_y, max_y, y) = check(min_y, max_y, f[1])
168			xs.append(x)
169			ys.append(y)
170
171		db[file] = {'x':xs, 'y':ys}
172		if len(xs) > 10:
173			db[file]['ax'], db[file]['ay'] = avg(xs, ys)
174		else:
175			db[file]['ax'] = db[file]['ay'] = None
176
177	db['min_x'] = min_x
178	db['max_x'] = max_x
179	db['min_y'] = min_y
180	db['max_y'] = max_y
181	return db
182
183#----------------------------------------------------------------------
184def parse_args(args):
185	"""Parse command line arguments.
186
187	Returns list of (data) files that need to be processed -- /unless/
188	the -A (--generate-all) option is passed, in which case superfluous
189	data files are ignored...
190	"""
191
192	global add_legend, output_file, title_str, type, verbose
193	global generate_all
194
195	prog = args[0][args[0].rfind('/')+1:]
196	if prog == 'btt_plot.py':
197		pass
198	elif not prog in progs:
199		fatal('%s not a valid command name' % prog)
200	else:
201		type = prog[prog.rfind('_')+1:prog.rfind('.py')]
202
203	s_opts = 'ALo:t:T:v'
204	l_opts = [ 'generate-all', 'type', 'no-legend', 'output', 'title',
205		   'verbose' ]
206
207	try:
208		(opts, args) = getopt.getopt(args[1:], s_opts, l_opts)
209	except getopt.error, msg:
210		print >>sys.stderr, msg
211		fatal(__doc__)
212
213	for (o, a) in opts:
214		if o in ('-A', '--generate-all'):
215			generate_all = True
216		elif o in ('-L', '--no-legend'):
217			add_legend = False
218		elif o in ('-o', '--output'):
219			output_file = a
220		elif o in ('-t', '--type'):
221			if not a in types:
222				fatal('Type %s not supported' % a)
223			type = a
224		elif o in ('-T', '--title'):
225			title_str = a
226		elif o in ('-v', '--verbose'):
227			verbose = True
228
229	if type == None and not generate_all:
230		fatal('Need type of data files to process - (-t <type>)')
231
232	return args
233
234#------------------------------------------------------------------------------
235def gen_title(fig, type, title_str):
236	"""Sets the title for the figure based upon the type /or/ user title"""
237
238	if title_str != None:
239		pass
240	elif type == 'aqd':
241		title_str = 'Average Queue Depth'
242	elif type == 'bnos':
243		title_str = 'Block Numbers Accessed'
244	elif type == 'q2d':
245		title_str = 'Queue (Q) To Issue (D) Average Latencies'
246	elif type == 'd2c':
247		title_str = 'Issue (D) To Complete (C) Average Latencies'
248	elif type == 'q2c':
249		title_str = 'Queue (Q) To Complete (C) Average Latencies'
250
251	title = fig.text(.5, .95, title_str, horizontalalignment='center')
252	title.set_fontsize('large')
253
254#------------------------------------------------------------------------------
255def gen_labels(db, ax, type):
256	"""Generate X & Y 'axis'"""
257
258	#----------------------------------------------------------------------
259	def gen_ylabel(ax, type):
260		"""Set the Y axis label based upon the type"""
261
262		if type == 'aqd':
263			str = 'Number of Requests Queued'
264		elif type == 'bnos':
265			str = 'Block Number'
266		else:
267			str = 'Seconds'
268		ax.set_ylabel(str)
269
270	#----------------------------------------------------------------------
271	xdelta = 0.1 * (db['max_x'] - db['min_x'])
272	ydelta = 0.1 * (db['max_y'] - db['min_y'])
273
274	ax.set_xlim(db['min_x'] - xdelta, db['max_x'] + xdelta)
275	ax.set_ylim(db['min_y'] - ydelta, db['max_y'] + ydelta)
276	ax.set_xlabel('Runtime (seconds)')
277	ax.grid(True)
278	gen_ylabel(ax, type)
279
280#------------------------------------------------------------------------------
281def generate_output(type, db):
282	"""Generate the output plot based upon the type and database"""
283
284	#----------------------------------------------------------------------
285	def color(idx, style):
286		"""Returns a color/symbol type based upon the index passed."""
287
288                colors = [ 'b', 'g', 'r', 'c', 'm', 'y', 'k' ]
289		l_styles = [ '-', ':', '--', '-.' ]
290		m_styles = [ 'o', '+', '.', ',', 's', 'v', 'x', '<', '>' ]
291
292		color = colors[idx % len(colors)]
293		if style == 'line':
294			style = l_styles[(idx / len(l_styles)) % len(l_styles)]
295		elif style == 'marker':
296			style = m_styles[(idx / len(m_styles)) % len(m_styles)]
297
298		return '%s%s' % (color, style)
299
300	#----------------------------------------------------------------------
301	def gen_legends(a, legends):
302		leg = ax.legend(legends, 'best', shadow=True)
303		frame = leg.get_frame()
304		frame.set_facecolor('0.80')
305		for t in leg.get_texts():
306			t.set_fontsize('xx-small')
307
308	#----------------------------------------------------------------------
309	global add_legend, output_file, title_str, verbose
310
311	if output_file != None:
312		ofile = output_file
313	else:
314		ofile = '%s.png' % type
315
316	if verbose:
317		print 'Generating plot into %s' % ofile
318
319	fig = plt.figure(figsize=plot_size)
320	ax = fig.add_subplot(111)
321
322	gen_title(fig, type, title_str)
323	gen_labels(db, ax, type)
324
325	idx = 0
326	if add_legend:
327		legends = []
328	else:
329		legends = None
330
331	keys = []
332	for file in db.iterkeys():
333		if not file in ['min_x', 'max_x', 'min_y', 'max_y']:
334			keys.append(file)
335
336	keys.sort()
337	for file in keys:
338		dat = db[file]
339		if type == 'bnos':
340			ax.plot(dat['x'], dat['y'], color(idx, 'marker'),
341				markersize=1)
342		elif dat['ax'] == None:
343			continue	# Don't add legend
344		else:
345			ax.plot(dat['ax'], dat['ay'], color(idx, 'line'),
346				linewidth=1.0)
347		if add_legend:
348			legends.append(get_base(file))
349		idx += 1
350
351	if add_legend and len(legends) > 0:
352		gen_legends(ax, legends)
353	plt.savefig(ofile)
354
355#------------------------------------------------------------------------------
356def get_files(type):
357	"""Returns the list of files for the -A option based upon type"""
358
359	if type == 'bnos':
360		files = []
361		for fn in glob.glob('*c.dat'):
362			for t in [ 'q2q', 'd2d', 'q2c', 'd2c' ]:
363				if fn.find(t) >= 0:
364					break
365			else:
366				files.append(fn)
367	else:
368		files = glob.glob('*%s.dat' % type)
369	return files
370
371#------------------------------------------------------------------------------
372if __name__ == '__main__':
373	files = parse_args(sys.argv)
374
375	if generate_all:
376		output_file = title_str = type = None
377		for t in types:
378			files = get_files(t)
379			if len(files) == 0:
380				continue
381			elif t != 'bnos':
382				generate_output(t, get_data(files))
383				continue
384
385			for file in files:
386				base = get_base(file)
387				title_str = 'Block Numbers Accessed: %s' % base
388				output_file = 'bnos_%s.png' % base
389				generate_output(t, get_data([file]))
390	elif len(files) < 1:
391		fatal('Need data files to process')
392	else:
393		generate_output(type, get_data(files))
394	sys.exit(0)
395