1# tko/nightly.py code shared by various tko/*.cgi graphing scripts 2 3import cgi, cgitb 4import os, sys 5import common 6from autotest_lib.tko import db, plotgraph, perf 7from autotest_lib.client.common_lib import kernel_versions 8 9 10def add_kernel_jobs(label_pattern): 11 cmd = "select job_idx from tko_jobs where label like '%s'" % label_pattern 12 nrows = perf.db_cur.execute(cmd) 13 return [row[0] for row in perf.db_cur.fetchall()] 14 15 16def is_filtered_platform(platform, platforms_filter): 17 if not platforms_filter: 18 return True 19 for p in platforms_filter: 20 if platform.startswith(p): 21 return True 22 return False 23 24 25def get_test_attributes(testrunx): 26 cmd = ( "select attribute, value from tko_test_attributes" 27 " where test_idx = %d" % testrunx ) 28 nrows = perf.db_cur.execute(cmd) 29 return dict(perf.db_cur.fetchall()) 30 31 32def get_antag(testrunx): 33 attrs = get_test_attributes(testrunx) 34 return attrs.get('antag', None) 35 36 37def matching_test_attributes(attrs, required_test_attributes): 38 if not required_test_attributes: 39 return True 40 matches = [attrs[key] == required_test_attributes[key] 41 for key in attrs if key in required_test_attributes] 42 return min(matches+[True]) # True if all jointly-existing keys matched 43 44 45def collect_testruns(jobs, test, test_attributes, 46 platforms_filter, by_hosts, no_antag): 47 # get test_runs run #s for 1 test on 1 kernel and some platforms 48 # TODO: Is jobs list short enough to use directly in 1 sql cmd? 49 # TODO: add filtering on test series? 50 runs = {} # platform --> list of test runs 51 for jobx in jobs: 52 cmd = ( "select test_idx, machine_idx from tko_tests" 53 " where job_idx = %s and test = %s" ) 54 args = [jobx, test] 55 nrows = perf.db_cur.execute(cmd, args) 56 for testrunx, machx in perf.db_cur.fetchall(): 57 platform, host = perf.machine_idx_to_platform_host(machx) 58 if by_hosts: 59 platform += '.'+host 60 if ( is_filtered_platform(platform, platforms_filter) and 61 matching_test_attributes(get_test_attributes(testrunx), 62 test_attributes) and 63 (not no_antag or get_antag(testrunx) == '') ): 64 runs.setdefault(platform, []).append(testrunx) 65 return runs 66 67 68def all_tested_platforms(test_runs): 69 # extract list of all tested platforms from test_runs table 70 platforms = set() 71 for kernel in test_runs: 72 platforms.update(set(test_runs[kernel].keys())) 73 return sorted(platforms) 74 75 76def divide_twoway_testruns(test_runs, platform): 77 # partition all twoway runs based on name of antagonist progs 78 twoway_runs = {} 79 antagonists = set() 80 for kernel in test_runs: 81 runs = {} 82 for testrunx in test_runs[kernel].get(platform, []): 83 antag = get_antag(testrunx) 84 if antag is not None: 85 runs.setdefault(antag, []).append(testrunx) 86 antagonists.add(antag) 87 twoway_runs[kernel] = runs 88 return twoway_runs, sorted(antagonists) 89 90 91def collect_raw_scores(runs, metric): 92 # get unscaled scores of test runs for 1 test on certain jobs 93 # arrange them by platform type 94 platform_scores = {} # platform --> list of perf scores 95 for platform in runs: 96 vals = perf.get_metric_at_point(runs[platform], metric) 97 if vals: 98 platform_scores[platform] = vals 99 return platform_scores 100 101 102def collect_scaled_scores(metric, test_runs, regressed_platforms, relative): 103 # get scores of test runs for 1 test on some kernels and platforms 104 # optionally make relative to oldest (?) kernel on that platform 105 # arrange by plotline (ie platform) for gnuplot 106 plot_data = {} # platform --> (kernel --> list of perf scores) 107 baseline = {} 108 for kernel in sorted(test_runs.keys()): 109 for platform in test_runs[kernel]: 110 if not (regressed_platforms is None or 111 platform in regressed_platforms): 112 continue # delete results for uninteresting platforms 113 vals = perf.get_metric_at_point(test_runs[kernel][platform], 114 metric) 115 if vals: 116 if relative: 117 if platform not in baseline: 118 baseline[platform], std = plotgraph.avg_dev(vals) 119 vals = [v/baseline[platform] for v in vals] 120 pdp = plot_data.setdefault(platform, {}) 121 pdp.setdefault(kernel, []).extend(vals) 122 return plot_data 123 124 125def collect_twoway_scores(metric, antagonists, twoway_runs, relative): 126 alone = '' 127 plot_data = {} 128 for kernel in twoway_runs: 129 for test2 in antagonists: 130 runs = twoway_runs[kernel].get(test2, []) 131 vals = perf.get_metric_at_point(runs, metric) 132 plot_data.setdefault(test2, {}) 133 if vals: 134 plot_data[test2][kernel] = vals 135 if relative: 136 vals = plot_data[alone].get(kernel, []) 137 if vals: 138 baseline = perf.average(vals) 139 for test2 in antagonists: 140 vals = plot_data[test2].get(kernel, []) 141 vals = [val/baseline for val in vals] 142 if vals: 143 plot_data[test2][kernel] = vals 144 else: 145 for test2 in antagonists: 146 if kernel in plot_data[test2]: 147 del plot_data[test2][kernel] 148 return plot_data 149 150 151def find_regressions(kernels, test_runs, metric): 152 # A test is regressed on some platform if its latest results are 153 # definitely lower than on the reference kernel. 154 # Runs for the latest kernel may be underway and incomplete. 155 # In that case, selectively use next-latest kernel. 156 # TODO: the next-latest method hurts if latest run is not sorted last, 157 # or if there are several dev threads 158 ref = kernels[0] 159 latest = kernels[-1] 160 prev = kernels[-2:][0] 161 scores = {} # kernel --> (platform --> list of perf scores) 162 for k in [ref, prev, latest]: 163 if k in test_runs: 164 scores[k] = collect_raw_scores(test_runs[k], metric) 165 regressed_platforms = [] 166 for platform in scores[ref]: 167 if latest in scores and platform in scores[latest]: 168 k = latest 169 elif prev in scores and platform in scores[prev]: 170 k = prev 171 else: # perhaps due to decay of test machines 172 k = ref # no regression info avail 173 ref_avg, ref_std = plotgraph.avg_dev(scores[ref][platform]) 174 avg, std = plotgraph.avg_dev(scores[ k ][platform]) 175 if avg+std < ref_avg-ref_std: 176 regressed_platforms.append(platform) 177 return sorted(regressed_platforms) 178 179 180def get_testrun_context(testrun): 181 cmd = ( 'select tko_jobs.label, tko_jobs.tag, tko_tests.subdir,' 182 ' tko_tests.started_time' 183 ' from tko_jobs, tko_tests' 184 ' where tko_jobs.job_idx = tko_tests.job_idx' 185 ' and tko_tests.test_idx = %d' % testrun ) 186 nrows = perf.db_cur.execute(cmd) 187 assert nrows == 1 188 row = perf.db_cur.fetchone() 189 row = [row[0], row[1], row[2], row[3].strftime('%m/%d/%y %H:%M')] 190 return row 191 192 193def html_top(): 194 print "Content-Type: text/html\n\n<html><body>" 195 196 197def abs_rel_link(myurl, passthru): 198 # link redraws current page with opposite absolute/relative choice 199 mod_passthru = passthru[:] 200 if 'absolute' in passthru: 201 mod_passthru.remove('absolute') 202 opposite = 'relative' 203 else: 204 mod_passthru.append('absolute') 205 opposite = 'absolute' 206 url = '%s?%s' % (myurl, '&'.join(mod_passthru)) 207 return "<a href='%s'> %s </a>" % (url, opposite) 208 209 210def table_1_metric_all_kernels(plot_data, columns, column_argname, 211 kernels, kernel_dates, 212 myurl, filtered_passthru): 213 # generate html table of graph's numbers 214 # for 1 benchmark metric over all kernels (rows), 215 # over various platforms or various antagonists etc (cols). 216 ref_thresholds = {} 217 print "<table border=1 cellpadding=3 cellspacing=0>" 218 print "<tr> <td><b> Kernel </b></td>", 219 for label in columns: 220 if not label and column_argname == 'antag': 221 label = 'no antag' 222 print "<td><b>", label.replace('_', '<br>_'), "</b></td>" 223 print "</tr>" 224 for kernel in kernels: 225 print "<tr> <td><b>", kernel, "</b>", 226 if kernel in kernel_dates: 227 print "<br><small>", kernel_dates[kernel], "</small>" 228 print "</td>" 229 for col in columns: 230 print "<td", 231 vals = plot_data[col].get(kernel, []) 232 if not vals: 233 print "> ?", 234 else: 235 (avg, std_dev) = plotgraph.avg_dev(vals) 236 if col not in ref_thresholds: 237 ref_thresholds[col] = avg - std_dev 238 if avg+std_dev < ref_thresholds[col]: 239 print "bgcolor=pink", 240 print "> ", 241 args = filtered_passthru[:] 242 perf.append_cgi_args(args, 243 {column_argname:col, 'kernel':kernel}) 244 print "<a href='%s?%s&runs&attrs'>" % (myurl, 245 '&'.join(args)) 246 print "<b>%.4g</b>" % avg, "</a><br>", 247 print " <small> %dr </small>" % len(vals), 248 print " <small> %.3g </small>" % std_dev, 249 print "</td>" 250 print "</tr>\n" 251 print "</table>" 252 print "<p> <b>Bold value:</b> Average of this metric, then <br>" 253 print "number of good test runs, then standard deviation of those runs" 254 print "<br> Pink if regressed from reference kernel" 255 256 257def table_all_metrics_1_platform(test_runs, platform, relative): 258 # TODO: show std dev in cells 259 # can't mark regressions, since some metrics improve downwards 260 kernels = perf.sort_kernels(test_runs.keys()) 261 scores = {} 262 attrs = set() 263 for kernel in kernels: 264 testruns = test_runs[kernel].get(platform, []) 265 if testruns: 266 d = perf.collect_all_metrics_scores(testruns) 267 scores[kernel] = d 268 attrs.update(set(d.keys())) 269 else: 270 print "No runs completed on", kernel, "<br>" 271 attrs = sorted(list(attrs))[:100] 272 273 print "<table border=1 cellpadding=4 cellspacing=0>" 274 print "<tr><td> Metric </td>" 275 for kernel in kernels: 276 kernel = kernel.replace("_", "_<br>") 277 print "<td>", kernel, "</td>" 278 print "</tr>" 279 for attr in attrs: 280 print "<tr>" 281 print "<td>", attr, "</td>" 282 baseline = None 283 for kernel in kernels: 284 print "<td>", 285 if kernel in scores and attr in scores[kernel]: 286 (avg, dev) = plotgraph.avg_dev(scores[kernel][attr]) 287 if baseline and relative: 288 percent = (avg/baseline - 1)*100 289 print "%+.1f%%" % percent, 290 else: 291 baseline = avg 292 print "%.4g" % avg, 293 else: 294 print "?" 295 print "</td>" 296 print "</tr>" 297 print "</table>" 298 299 300def table_variants_all_tests(plot_data, columns, colkeys, benchmarks, 301 myurl, filtered_passthru): 302 # generate html table of graph's numbers 303 # for primary metric over all benchmarks (rows), 304 # on one platform and one kernel, 305 # over various combos of test run attribute constraints (cols). 306 ref_thresholds = {} 307 print "<table border=1 cellpadding=3 cellspacing=0>" 308 print "<tr> <td><b> Benchmark </b></td>", 309 for col in columns: 310 print "<td><b>", colkeys[col].replace(',', ',<br>'), "</b></td>" 311 print "</tr>" 312 for benchmark in benchmarks: 313 print "<tr> <td><b>", benchmark, "</b></td>" 314 for col in columns: 315 print "<td>", 316 vals = plot_data[col].get(benchmark, []) 317 if not vals: 318 print "?", 319 else: 320 (avg, std_dev) = plotgraph.avg_dev(vals) 321 args = filtered_passthru[:] 322 perf.append_cgi_args(args, {'test':benchmark}) 323 for keyval in colkeys[col].split(','): 324 key, val = keyval.split('=', 1) 325 perf.append_cgi_args(args, {key:val}) 326 print "<a href='%s?%s&runs&attrs'>" % (myurl, 327 '&'.join(args)) 328 print "<b>%.4g</b>" % avg, "</a><br>", 329 print " <small> %dr </small>" % len(vals), 330 print " <small> %.3g </small>" % std_dev, 331 print "</td>" 332 print "</tr>\n" 333 print "</table>" 334 print "<p> <b>Bold value:</b> Average of this metric, then <br>" 335 print "number of good test runs, then standard deviation of those runs" 336 337 338def table_testrun_details(runs, metric, tko_server, show_attrs): 339 print "<table border=1 cellpadding=4 cellspacing=0>" 340 print "<tr><td> %s metric </td>" % metric 341 print "<td> Job label </td> <td> Job tag </td> <td> Run results </td>" 342 print "<td> Started_time </td>" 343 if show_attrs: 344 print "<td> Test attributes </td>" 345 print "</tr>\n" 346 347 for testrunx in runs: 348 print "<tr> <td>", 349 vals = perf.get_metric_at_point([testrunx], metric) 350 for v in vals: 351 print "%.4g " % v, 352 print "</td>" 353 row = get_testrun_context(testrunx) 354 row[2] = ( "<a href='//%s/results/%s/%s/results/keyval'> %s </a>" 355 % (tko_server, row[1], row[2], row[2]) ) 356 for v in row: 357 print "<td> %s </td>" % v 358 if show_attrs: 359 attrs = get_test_attributes(testrunx) 360 print "<td>", 361 for attr in sorted(attrs.keys()): 362 if attr == "sysinfo-cmdline": continue 363 if attr[:4] == "svs-": continue 364 val = attrs[attr] 365 if len(val) > 40: 366 val = val[:40-3] + "..." 367 print "%s=%s " % (attr, val) 368 print "</td>" 369 print "</tr>\n" 370 print "</table>" 371 372 373def overview_thumb(test, metric, myurl, passthru): 374 pass_ = passthru + ['test=%s' % test] 375 if metric: 376 pass_ += ['metric=%s' % metric] 377 pass_ = '&'.join(pass_) 378 print "<a href='%s?%s&table'>" % (myurl, pass_) 379 print " <img src='%s?%s&size=450,500'> </a>" % (myurl, pass_) 380 # embedded graphs fit 3 across on 1400x1050 laptop 381 382 383def graph_1_test(title, metric, plot_data, line_argname, lines, 384 kernel_legend, relative, size, dark=False): 385 # generate graph image for one benchmark, showing avg and 386 # std dev of one metric, over various kernels (X columns), 387 # over various platforms or antagonists etc (graphed lines) 388 xlegend = kernel_legend 389 ylegend = metric.capitalize() 390 if relative: 391 ylegend += ', Relative' 392 ymin = 0.8 393 else: 394 ymin = None 395 if len(lines) > 1: 396 keytitle = line_argname.capitalize() + ':' 397 else: 398 keytitle = '' 399 graph = plotgraph.gnuplot(title, xlegend, ylegend, size=size, 400 xsort=perf.sort_kernels, keytitle=keytitle) 401 for line in lines: 402 label = line 403 if not label and line_argname == 'antag': 404 label = 'no antag' 405 graph.add_dataset(label, plot_data[line]) 406 graph.plot(cgi_header=True, ymin=ymin, dark=dark) 407 408 409def graph_variants_all_tests(title, plot_data, linekeys, size, dark): 410 # generate graph image showing all benchmarks 411 # on one platform and one kernel, 412 # over various combos of test run attribute constraints (lines). 413 xlegend = "Benchmark" 414 ylegend = "Relative Perf" 415 graph = plotgraph.gnuplot(title, xlegend, ylegend, size=size) 416 for i in linekeys: 417 graph.add_dataset(linekeys[i], plot_data[i]) 418 graph.plot(cgi_header=True, dark=dark, ymin=0.8) 419 420 421class generate_views(object): 422 423 424 def __init__(self, kernel_legend, benchmarks, test_group, 425 site_benchmark_metrics, tko_server, 426 jobs_selector, no_antag): 427 self.kernel_legend = kernel_legend 428 self.benchmarks = benchmarks 429 self.test_group = test_group 430 self.tko_server = tko_server 431 self.jobs_selector = jobs_selector 432 self.no_antag = no_antag 433 434 cgitb.enable() 435 test, antagonists = self.parse_most_cgi_args() 436 437 perf.init(tko_server=tko_server) 438 for b in site_benchmark_metrics: 439 perf.add_benchmark_main_metric(b, site_benchmark_metrics[b]) 440 441 self.test_runs = {} # kernel --> (platform --> list of test runs) 442 self.job_table = {} # kernel id --> list of job idxs 443 self.kernel_dates = {} # kernel id --> date of nightly test 444 445 vary = self.cgiform.getlist('vary') 446 if vary: 447 platform = self.platforms_filter[0] 448 self.analyze_variants_all_tests_1_platform(platform, vary) 449 elif test: 450 self.analyze_1_test(test, antagonists) 451 else: 452 self.overview_page_all_tests(self.benchmarks, antagonists) 453 454 455 def collect_all_testruns(self, trimmed_kernels, test): 456 # get test_runs run #s for 1 test on some kernels and platforms 457 for kernel in trimmed_kernels: 458 runs = collect_testruns(self.job_table[kernel], test, 459 self.test_attributes, self.platforms_filter, 460 'by_hosts' in self.toggles, self.no_antag) 461 if runs: 462 self.test_runs[kernel] = runs 463 464 465 def table_for_graph_1_test(self, title, metric, plot_data, 466 column_argname, columns, filtered_passthru): 467 # generate detailed html page with 1 graph and corresp numbers 468 # for 1 benchmark metric over all kernels (rows), 469 # over various platforms or various antagonists etc (cols). 470 html_top() 471 print '<h3> %s </h3>' % title 472 print ('%s, machine group %s on //%s server <br>' % 473 (self.kernel_legend, self.test_group, self.tko_server)) 474 if self.test_tag: 475 print '%s test script series <br>' % self.test_tag[1:] 476 477 print "<img src='%s?%s'>" % (self.myurl, '&'.join(self.passthru)) 478 479 link = abs_rel_link(self.myurl, self.passthru+['table']) 480 print "<p><p> <h4> Redraw this with %s performance? </h4>" % link 481 482 heading = "%s, %s metric" % (title, metric) 483 if self.relative: 484 heading += ", relative" 485 print "<p><p> <h3> %s: </h3>" % heading 486 table_1_metric_all_kernels(plot_data, columns, column_argname, 487 self.kernels, self.kernel_dates, 488 self.myurl, filtered_passthru) 489 print "</body></html>" 490 491 492 def graph_1_test_all_platforms(self, test, metric, platforms, plot_data): 493 # generate graph image for one benchmark 494 title = test.capitalize() 495 if 'regress' in self.toggles: 496 title += ' Regressions' 497 if 'table' in self.cgiform: 498 self.table_for_graph_1_test(title, metric, plot_data, 499 'platforms', platforms, 500 filtered_passthru=self.passthru) 501 else: 502 graph_1_test(title, metric, plot_data, 'platforms', platforms, 503 self.kernel_legend, self.relative, 504 self.size, 'dark' in self.toggles) 505 506 507 def testrun_details(self, title, runs, metric): 508 html_top() 509 print '<h3> %s </h3>' % title 510 print ('%s, machine group %s on //%s server' % 511 (self.kernel_legend, self.test_group, self.tko_server)) 512 if self.test_tag: 513 print '<br> %s test script series' % self.test_tag[1:] 514 print '<p>' 515 table_testrun_details(runs, metric, 516 self.tko_server, 'attrs' in self.cgiform) 517 print "</body></html>" 518 519 520 def testrun_details_for_1_test_kernel_platform(self, test, 521 metric, platform): 522 default_kernel = min(self.test_runs.keys()) 523 kernel = self.cgiform.getvalue('kernel', default_kernel) 524 title = '%s on %s using %s' % (test.capitalize(), platform, kernel) 525 runs = self.test_runs[kernel].get(platform, []) 526 self.testrun_details(title, runs, metric) 527 528 529 def analyze_1_metric_all_platforms(self, test, metric): 530 if 'regress' in self.toggles: 531 regressed_platforms = find_regressions(self.kernels, self.test_runs, 532 metric) 533 else: 534 regressed_platforms = None 535 plot_data = collect_scaled_scores(metric, self.test_runs, 536 regressed_platforms, self.relative) 537 platforms = sorted(plot_data.keys()) 538 if not plot_data: 539 html_top() 540 print 'No runs' 541 elif 'runs' in self.cgiform: 542 self.testrun_details_for_1_test_kernel_platform(test, metric, 543 platforms[0]) 544 else: 545 self.graph_1_test_all_platforms(test, metric, platforms, plot_data) 546 547 548 def analyze_all_metrics_1_platform(self, test, platform): 549 # TODO: show #runs in header 550 html_top() 551 heading = "%s %s:  %s" % (self.test_group, self.kernel_legend, 552 test.capitalize()) 553 print "<h2> %s </h2>" % heading 554 print "platform=%s <br>" % platform 555 for attr in self.test_attributes: 556 print "%s=%s " % (attr, self.test_attributes[attr]) 557 print "<p>" 558 table_all_metrics_1_platform(self.test_runs, platform, self.relative) 559 print "</body></html>" 560 561 562 def table_for_variants_all_tests(self, title, plot_data, colkeys, columns, 563 filtered_passthru, test_tag): 564 # generate detailed html page with 1 graph and corresp numbers 565 # for primary metric over all benchmarks (rows), 566 # on one platform and one kernel, 567 # over various combos of test run attribute constraints (cols). 568 html_top() 569 print '<h3> %s </h3>' % title 570 print ('%s, machine group %s on //%s server <br>' % 571 (self.kernel_legend, self.test_group, self.tko_server)) 572 if test_tag: 573 print '%s test script series <br>' % test_tag[1:] 574 575 varies = ['vary='+colkeys[col] for col in columns] 576 print "<img src='%s?%s'>" % (self.myurl, '&'.join(self.passthru+varies)) 577 578 print "<p><p> <h3> %s: </h3>" % title 579 table_variants_all_tests(plot_data, columns, colkeys, self.benchmarks, 580 self.myurl, filtered_passthru) 581 print "</body></html>" 582 583 584 def analyze_variants_all_tests_1_platform(self, platform, vary): 585 # generate one graph image for results of all benchmarks 586 # on one platform and one kernel, comparing effects of 587 # two or more combos of kernel options (test run attributes) 588 # (numa_fake,stale_page,kswapd_merge,sched_idle, etc) 589 kernel = self.cgiform.getvalue('kernel', 'some_kernel') 590 self.passthru.append('kernel=%s' % kernel) 591 592 # two or more vary_groups, one for each plotted line, 593 # each group begins with vary= and ends with next & 594 # each group has comma-separated list of test attribute key=val pairs 595 # eg vary=keyval1,keyval2&vary=keyval3,keyval4 596 vary_groups = [dict(pair.split('=',1) for pair 597 in vary_group.split(',')) 598 for vary_group in vary] 599 600 test = self.benchmarks[0] # pick any test in all jobs 601 kernels, test_tag = self.jobs_selector(test, self.job_table, 602 self.kernel_dates) 603 604 linekeys = {} 605 plot_data = {} 606 baselines = {} 607 for i, vary_group in enumerate(vary_groups): 608 group_attributes = self.test_attributes.copy() 609 group_attributes.update(vary_group) 610 linekey = ','.join('%s=%s' % (attr, vary_group[attr]) 611 for attr in vary_group) 612 linekeys[i] = linekey 613 data = {} 614 for benchmark in self.benchmarks: 615 metric = perf.benchmark_main_metric(benchmark) 616 runs = collect_testruns(self.job_table[kernel], 617 benchmark+test_tag, 618 group_attributes, 619 self.platforms_filter, 620 'by_hosts' in self.toggles, 621 self.no_antag) 622 vals = [] 623 for testrunx in runs[platform]: 624 vals += perf.get_metric_at_point([testrunx], metric) 625 if vals: 626 if benchmark not in baselines: 627 baselines[benchmark], stddev = plotgraph.avg_dev(vals) 628 vals = [val/baselines[benchmark] for val in vals] 629 data[benchmark] = vals 630 plot_data[i] = data 631 632 title = "%s on %s" % (kernel, platform) 633 for attr in self.test_attributes: 634 title += ', %s=%s' % (attr, self.test_attributes[attr]) 635 if 'table' in self.cgiform: 636 self.table_for_variants_all_tests(title, plot_data, linekeys, 637 range(len(linekeys)), 638 filtered_passthru=self.passthru, 639 test_tag=test_tag) 640 else: 641 graph_variants_all_tests(title, plot_data, linekeys, 642 self.size, 'dark' in self.toggles) 643 644 645 def graph_twoway_antagonists_1_test_1_platform( 646 self, test, metric, platform, antagonists, twoway_runs): 647 # generate graph of one benchmark's performance paired with 648 # various antagonists, with one plotted line per antagonist, 649 # over most kernels (X axis), all on one machine type 650 # performance is relative to the no-antag baseline case 651 plot_data = collect_twoway_scores(metric, antagonists, 652 twoway_runs, self.relative) 653 title = "%s vs. an Antagonist on %s:" % (test.capitalize(), platform) 654 if 'table' in self.cgiform: 655 filtered_passthru = [arg for arg in self.passthru 656 if not arg.startswith('antag=')] 657 self.table_for_graph_1_test(title, metric, plot_data, 658 'antag', antagonists, 659 filtered_passthru=filtered_passthru) 660 else: 661 graph_1_test(title, metric, plot_data, 'antag', antagonists, 662 self.kernel_legend, self.relative, 663 self.size, 'dark' in self.toggles) 664 665 666 def testrun_details_for_twoway_test(self, test, metric, platform, 667 antagonist, twoway_runs): 668 default_kernel = min(twoway_runs.keys()) 669 kernel = self.cgiform.getvalue('kernel', default_kernel) 670 title = '%s vs. Antagonist %s on %s using %s' % ( 671 test.capitalize(), antagonist.capitalize(), platform, kernel) 672 runs = twoway_runs[kernel].get(antagonist, []) 673 self.testrun_details(title, runs, metric) 674 675 676 def analyze_twoway_antagonists_1_test_1_platform( 677 self, test, metric, platform, antagonists): 678 twoway_runs, all_antagonists = divide_twoway_testruns(self.test_runs, 679 platform) 680 if antagonists == ['*']: 681 antagonists = all_antagonists 682 if not twoway_runs: 683 html_top() 684 print 'No runs' 685 elif 'runs' in self.cgiform: 686 self.testrun_details_for_twoway_test( 687 test, metric, platform, antagonists[0], twoway_runs) 688 else: 689 self.graph_twoway_antagonists_1_test_1_platform( 690 test, metric, platform, antagonists, twoway_runs) 691 692 693 def get_twoway_default_platform(self): 694 if self.platforms_filter: 695 return self.platforms_filter[0] 696 test = 'unixbench' 697 kernels, test_tag = self.jobs_selector(test, self.job_table, 698 self.kernel_dates) 699 self.collect_all_testruns(kernels, test+test_tag) 700 return all_tested_platforms(self.test_runs)[0] 701 702 703 def overview_page_all_tests(self, benchmarks, antagonists): 704 # generate overview html page with small graphs for each benchmark 705 # linking to detailed html page for that benchmark 706 # recursively link to this same cgi to generate each image 707 html_top() 708 if antagonists is not None: 709 heading = ('Twoway Container Isolation using %s on %s' % 710 (self.kernel_legend, self.get_twoway_default_platform())) 711 else: 712 heading = '%s, %s Benchmarks' % (self.kernel_legend, 713 self.test_group) 714 if 'regress' in self.toggles: 715 heading += ", Regressions Only" 716 print "<h3> %s </h3>" % heading 717 for test in benchmarks: 718 overview_thumb(test, '', self.myurl, self.passthru) 719 if test == 'unixbench': 720 overview_thumb('unixbench', 'Process_creation', 721 self.myurl, self.passthru) 722 723 link = abs_rel_link(self.myurl, self.passthru) 724 print "<p><p> <h4> Redraw this with %s performance? </h4>" % link 725 print "</body></html>" 726 727 728 def analyze_1_test(self, test, antagonists): 729 self.passthru.append('test=%s' % test) 730 metric = self.cgiform.getvalue('metric', '') 731 if metric: 732 self.passthru.append('metric=%s' % metric) 733 else: 734 metric = perf.benchmark_main_metric(test) 735 assert metric, "no default metric for test %s" % test 736 self.kernels, self.test_tag = self.jobs_selector(test, self.job_table, 737 self.kernel_dates) 738 self.collect_all_testruns(self.kernels, test+self.test_tag) 739 if not self.platforms_filter and (metric == '*' or 740 antagonists is not None): 741 # choose default platform 742 self.platforms_filter = all_tested_platforms(self.test_runs)[0:1] 743 self.passthru.append('platforms=%s' % 744 ','.join(self.platforms_filter)) 745 if antagonists is not None: 746 antagonists = antagonists.split(',') 747 if len(antagonists) == 1 and antagonists != ['*']: 748 self.relative = False 749 self.analyze_twoway_antagonists_1_test_1_platform( 750 test, metric, self.platforms_filter[0], antagonists) 751 elif metric == '*': 752 platform = self.platforms_filter[0] 753 self.analyze_all_metrics_1_platform(test, platform) 754 else: 755 self.analyze_1_metric_all_platforms(test, metric) 756 757 758 def parse_most_cgi_args(self): 759 self.myurl = os.path.basename(sys.argv[0]) 760 self.cgiform = cgi.FieldStorage(keep_blank_values=True) 761 self.size = self.cgiform.getvalue('size', '1200,850') 762 all_toggles = set(('absolute', 'regress', 'dark', 'by_hosts')) 763 self.toggles = set(tog for tog in all_toggles if tog in self.cgiform) 764 platforms = self.cgiform.getvalue('platforms', '') 765 if '.' in platforms: 766 self.toggles.add('by_hosts') 767 self.passthru = list(self.toggles) 768 self.relative = 'absolute' not in self.toggles 769 if platforms: 770 self.passthru.append('platforms=%s' % platforms) 771 self.platforms_filter = platforms.split(',') 772 else: 773 self.platforms_filter = [] 774 self.test_attributes = perf.parse_test_attr_args(self.cgiform) 775 perf.append_cgi_args(self.passthru, self.test_attributes) 776 test = self.cgiform.getvalue('test', '') 777 if 'antag' in self.cgiform: 778 antagonists = ','.join(self.cgiform.getlist('antag')) 779 # antag=* 780 # or antag=test1,test2,test3,... 781 # or antag=test1&antag=test2&... 782 # testN is empty for solo case of no antagonist 783 self.passthru.append('antag=%s' % antagonists) 784 else: 785 antagonists = None # not same as '' 786 return test, antagonists 787