1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Helper functions for the layout test analyzer.""" 6 7from datetime import datetime 8from email.mime.multipart import MIMEMultipart 9from email.mime.text import MIMEText 10import fileinput 11import os 12import pickle 13import re 14import smtplib 15import socket 16import sys 17import time 18 19from bug import Bug 20from test_expectations_history import TestExpectationsHistory 21 22DEFAULT_TEST_EXPECTATION_PATH = ('trunk/LayoutTests/TestExpectations') 23LEGACY_DEFAULT_TEST_EXPECTATION_PATH = ( 24 'trunk/LayoutTests/platform/chromium/test_expectations.txt') 25REVISION_LOG_URL = ('http://build.chromium.org/f/chromium/perf/dashboard/ui/' 26 'changelog_blink.html?url=/trunk/LayoutTests/%s&range=%d:%d') 27DEFAULT_REVISION_VIEW_URL = 'http://src.chromium.org/viewvc/blink?revision=%s' 28 29 30class AnalyzerResultMap: 31 """A class to deal with joined result produed by the analyzer. 32 33 The join is done between layouttests and the test_expectations object 34 (based on the test expectation file). The instance variable |result_map| 35 contains the following keys: 'whole','skip','nonskip'. The value of 'whole' 36 contains information about all layouttests. The value of 'skip' contains 37 information about skipped layouttests where it has 'SKIP' in its entry in 38 the test expectation file. The value of 'nonskip' contains all information 39 about non skipped layout tests, which are in the test expectation file but 40 not skipped. The information is exactly same as the one parsed by the 41 analyzer. 42 """ 43 44 def __init__(self, test_info_map): 45 """Initialize the AnalyzerResultMap based on test_info_map. 46 47 Test_info_map contains all layouttest information. The job here is to 48 classify them as 'whole', 'skip' or 'nonskip' based on that information. 49 50 Args: 51 test_info_map: the result map of |layouttests.JoinWithTestExpectation|. 52 The key of the map is test name such as 'media/media-foo.html'. 53 The value of the map is a map that contains the following keys: 54 'desc'(description), 'te_info' (test expectation information), 55 which is a list of test expectation information map. The key of the 56 test expectation information map is test expectation keywords such 57 as "SKIP" and other keywords (for full list of keywords, please 58 refer to |test_expectations.ALL_TE_KEYWORDS|). 59 """ 60 self.result_map = {} 61 self.result_map['whole'] = {} 62 self.result_map['skip'] = {} 63 self.result_map['nonskip'] = {} 64 if test_info_map: 65 for (k, value) in test_info_map.iteritems(): 66 self.result_map['whole'][k] = value 67 if 'te_info' in value: 68 # Don't count SLOW PASS, WONTFIX, or ANDROID tests as failures. 69 if any([True for x in value['te_info'] if set(x.keys()) == 70 set(['SLOW', 'PASS', 'Bugs', 'Comments', 'Platforms']) or 71 'WONTFIX' in x or x['Platforms'] == ['ANDROID']]): 72 continue 73 if any([True for x in value['te_info'] if 'SKIP' in x]): 74 self.result_map['skip'][k] = value 75 else: 76 self.result_map['nonskip'][k] = value 77 78 @staticmethod 79 def GetDiffString(diff_map_element, type_str): 80 """Get difference string out of diff map element. 81 82 The difference string shows difference between two analyzer results 83 (for example, a result for now and a result for sometime in the past) 84 in HTML format (with colors). This is used for generating email messages. 85 86 Args: 87 diff_map_element: An element of the compared map generated by 88 |CompareResultMaps()|. The element has two lists of test cases. One 89 is for test names that are in the current result but NOT in the 90 previous result. The other is for test names that are in the previous 91 results but NOT in the current result. Please refer to comments in 92 |CompareResultMaps()| for details. 93 type_str: a string indicating the test group to which |diff_map_element| 94 belongs; used for color determination. Must be 'whole', 'skip', or 95 'nonskip'. 96 97 Returns: 98 a string in HTML format (with colors) to show difference between two 99 analyzer results. 100 """ 101 if not diff_map_element[0] and not diff_map_element[1]: 102 return 'No Change' 103 color = '' 104 diff = len(diff_map_element[0]) - len(diff_map_element[1]) 105 if diff > 0 and type_str != 'whole': 106 color = 'red' 107 else: 108 color = 'green' 109 diff_sign = '' 110 if diff > 0: 111 diff_sign = '+' 112 if not diff: 113 whole_str = 'No Change' 114 else: 115 whole_str = '<font color="%s">%s%d</font>' % (color, diff_sign, diff) 116 colors = ['red', 'green'] 117 if type_str == 'whole': 118 # Bug 107773 - when we increase the number of tests, 119 # the name of the tests are in red, it should be green 120 # since it is good thing. 121 colors = ['green', 'red'] 122 str1 = '' 123 for (name, _) in diff_map_element[0]: 124 str1 += '<font color="%s">%s,</font>' % (colors[0], name) 125 str2 = '' 126 for (name, _) in diff_map_element[1]: 127 str2 += '<font color="%s">%s,</font>' % (colors[1], name) 128 if str1 or str2: 129 whole_str += ':' 130 if str1: 131 whole_str += str1 132 if str2: 133 whole_str += str2 134 # Remove the last occurrence of ','. 135 whole_str = ''.join(whole_str.rsplit(',', 1)) 136 return whole_str 137 138 def GetPassingRate(self): 139 """Get passing rate. 140 141 Returns: 142 layout test passing rate of this result in percent. 143 144 Raises: 145 ValueEror when the number of tests in test group "whole" is equal 146 or less than that of "skip". 147 """ 148 delta = len(self.result_map['whole'].keys()) - ( 149 len(self.result_map['skip'].keys())) 150 if delta <= 0: 151 raise ValueError('The number of tests in test group "whole" is equal or ' 152 'less than that of "skip"') 153 return 100 - len(self.result_map['nonskip'].keys()) * 100 / delta 154 155 def ConvertToCSVText(self, current_time): 156 """Convert |self.result_map| into stats and issues text in CSV format. 157 158 Both are used as inputs for Google spreadsheet. 159 160 Args: 161 current_time: a string depicting a time in year-month-day-hour 162 format (e.g., 2011-11-08-16). 163 164 Returns: 165 a tuple of stats and issues_txt 166 stats: analyzer result in CSV format that shows: 167 (current_time, the number of tests, the number of skipped tests, 168 the number of failing tests, passing rate) 169 For example, 170 "2011-11-10-15,204,22,12,94" 171 issues_txt: issues listed in CSV format that shows: 172 (BUGWK or BUGCR, bug number, the test expectation entry, 173 the name of the test) 174 For example, 175 "BUGWK,71543,TIMEOUT PASS,media/media-element-play-after-eos.html, 176 BUGCR,97657,IMAGE CPU MAC TIMEOUT PASS,media/audio-repaint.html," 177 """ 178 stats = ','.join([current_time, str(len(self.result_map['whole'].keys())), 179 str(len(self.result_map['skip'].keys())), 180 str(len(self.result_map['nonskip'].keys())), 181 str(self.GetPassingRate())]) 182 issues_txt = '' 183 for bug_txt, test_info_list in ( 184 self.GetListOfBugsForNonSkippedTests().iteritems()): 185 matches = re.match(r'(BUG(CR|WK))(\d+)', bug_txt) 186 bug_suffix = '' 187 bug_no = '' 188 if matches: 189 bug_suffix = matches.group(1) 190 bug_no = matches.group(3) 191 issues_txt += bug_suffix + ',' + bug_no + ',' 192 for test_info in test_info_list: 193 test_name, te_info = test_info 194 issues_txt += ' '.join(te_info.keys()) + ',' + test_name + ',' 195 issues_txt += '\n' 196 return stats, issues_txt 197 198 def ConvertToString(self, prev_time, diff_map, issue_detail_mode): 199 """Convert this result to HTML display for email. 200 201 Args: 202 prev_time: the previous time string that are compared against. 203 diff_map: the compared map generated by |CompareResultMaps()|. 204 issue_detail_mode: includes the issue details in the output string if 205 this is True. 206 207 Returns: 208 a analyzer result string in HTML format. 209 """ 210 return_str = '' 211 if diff_map: 212 return_str += ( 213 '<b>Statistics (Diff Compared to %s):</b><ul>' 214 '<li>The number of tests: %d (%s)</li>' 215 '<li>The number of failing skipped tests: %d (%s)</li>' 216 '<li>The number of failing non-skipped tests: %d (%s)</li>' 217 '<li>Passing rate: %d %%</li></ul>') % ( 218 prev_time, len(self.result_map['whole'].keys()), 219 AnalyzerResultMap.GetDiffString(diff_map['whole'], 'whole'), 220 len(self.result_map['skip'].keys()), 221 AnalyzerResultMap.GetDiffString(diff_map['skip'], 'skip'), 222 len(self.result_map['nonskip'].keys()), 223 AnalyzerResultMap.GetDiffString(diff_map['nonskip'], 'nonskip'), 224 self.GetPassingRate()) 225 if issue_detail_mode: 226 return_str += '<b>Current issues about failing non-skipped tests:</b>' 227 for (bug_txt, test_info_list) in ( 228 self.GetListOfBugsForNonSkippedTests().iteritems()): 229 return_str += '<ul>%s' % Bug(bug_txt) 230 for test_info in test_info_list: 231 (test_name, te_info) = test_info 232 gpu_link = '' 233 if 'GPU' in te_info: 234 gpu_link = 'group=%40ToT%20GPU%20Mesa%20-%20chromium.org&' 235 dashboard_link = ('http://test-results.appspot.com/dashboards/' 236 'flakiness_dashboard.html#%stests=%s') % ( 237 gpu_link, test_name) 238 return_str += '<li><a href="%s">%s</a> (%s) </li>' % ( 239 dashboard_link, test_name, ' '.join( 240 [key for key in te_info.keys() if key != 'Platforms'])) 241 return_str += '</ul>\n' 242 return return_str 243 244 def CompareToOtherResultMap(self, other_result_map): 245 """Compare this result map with the other to see if there are any diff. 246 247 The comparison is done for layouttests which belong to 'whole', 'skip', 248 or 'nonskip'. 249 250 Args: 251 other_result_map: another result map to be compared against the result 252 map of the current object. 253 254 Returns: 255 a map that has 'whole', 'skip' and 'nonskip' as keys. 256 Please refer to |diff_map| in |SendStatusEmail()|. 257 """ 258 comp_result_map = {} 259 for name in ['whole', 'skip', 'nonskip']: 260 if name == 'nonskip': 261 # Look into expectation to get diff only for non-skipped tests. 262 lookIntoTestExpectationInfo = True 263 else: 264 # Otherwise, only test names are compared to get diff. 265 lookIntoTestExpectationInfo = False 266 comp_result_map[name] = GetDiffBetweenMaps( 267 self.result_map[name], other_result_map.result_map[name], 268 lookIntoTestExpectationInfo) 269 return comp_result_map 270 271 @staticmethod 272 def Load(file_path): 273 """Load the object from |file_path| using pickle library. 274 275 Args: 276 file_path: the string path to the file from which to read the result. 277 278 Returns: 279 a AnalyzerResultMap object read from |file_path|. 280 """ 281 file_object = open(file_path) 282 analyzer_result_map = pickle.load(file_object) 283 file_object.close() 284 return analyzer_result_map 285 286 def Save(self, file_path): 287 """Save the object to |file_path| using pickle library. 288 289 Args: 290 file_path: the string path to the file in which to store the result. 291 """ 292 file_object = open(file_path, 'wb') 293 pickle.dump(self, file_object) 294 file_object.close() 295 296 def GetListOfBugsForNonSkippedTests(self): 297 """Get a list of bugs for non-skipped layout tests. 298 299 This is used for generating email content. 300 301 Returns: 302 a mapping from bug modifier text (e.g., BUGCR1111) to a test name and 303 main test information string which excludes comments and bugs. 304 This is used for grouping test names by bug. 305 """ 306 bug_map = {} 307 for (name, value) in self.result_map['nonskip'].iteritems(): 308 for te_info in value['te_info']: 309 main_te_info = {} 310 for k in te_info.keys(): 311 if k != 'Comments' and k != 'Bugs': 312 main_te_info[k] = True 313 if 'Bugs' in te_info: 314 for bug in te_info['Bugs']: 315 if bug not in bug_map: 316 bug_map[bug] = [] 317 bug_map[bug].append((name, main_te_info)) 318 return bug_map 319 320 321def SendStatusEmail(prev_time, analyzer_result_map, diff_map, 322 receiver_email_address, test_group_name, 323 appended_text_to_email, email_content, rev_str, 324 email_only_change_mode): 325 """Send status email. 326 327 Args: 328 prev_time: the date string such as '2011-10-09-11'. This format has been 329 used in this analyzer. 330 analyzer_result_map: current analyzer result. 331 diff_map: a map that has 'whole', 'skip' and 'nonskip' as keys. 332 The values of the map are the result of |GetDiffBetweenMaps()|. 333 The element has two lists of test cases. One (with index 0) is for 334 test names that are in the current result but NOT in the previous 335 result. The other (with index 1) is for test names that are in the 336 previous results but NOT in the current result. 337 For example (test expectation information is omitted for 338 simplicity), 339 comp_result_map['whole'][0] = ['foo1.html'] 340 comp_result_map['whole'][1] = ['foo2.html'] 341 This means that current result has 'foo1.html' but it is NOT in the 342 previous result. This also means the previous result has 'foo2.html' 343 but it is NOT in the current result. 344 receiver_email_address: receiver's email address. 345 test_group_name: string representing the test group name (e.g., 'media'). 346 appended_text_to_email: a text which is appended at the end of the status 347 email. 348 email_content: an email content string that will be shown on the dashboard. 349 rev_str: a revision string that contains revision information that is sent 350 out in the status email. It is obtained by calling 351 |GetRevisionString()|. 352 email_only_change_mode: send email only when there is a change if this is 353 True. Otherwise, always send email after each run. 354 """ 355 if rev_str: 356 email_content += '<br><b>Revision Information:</b>' 357 email_content += rev_str 358 localtime = time.asctime(time.localtime(time.time())) 359 change_str = '' 360 if email_only_change_mode: 361 change_str = 'Status Change ' 362 subject = 'Layout Test Analyzer Result %s(%s): %s' % (change_str, 363 test_group_name, 364 localtime) 365 SendEmail('no-reply@chromium.org', [receiver_email_address], 366 subject, email_content + appended_text_to_email) 367 368 369def GetRevisionString(prev_time, current_time, diff_map): 370 """Get a string for revision information during the specified time period. 371 372 Args: 373 prev_time: the previous time as a floating point number expressed 374 in seconds since the epoch, in UTC. 375 current_time: the current time as a floating point number expressed 376 in seconds since the epoch, in UTC. It is typically obtained by 377 time.time() function. 378 diff_map: a map that has 'whole', 'skip' and 'nonskip' as keys. 379 Please refer to |diff_map| in |SendStatusEmail()|. 380 381 Returns: 382 a tuple of strings: 383 1) full string containing links, author, date, and line for each 384 change in the test expectation file. 385 2) shorter string containing only links to the change. Used for 386 trend graph annotations. 387 3) last revision number for the given test group. 388 4) last revision date for the given test group. 389 """ 390 if not diff_map: 391 return ('', '', '', '') 392 testname_map = {} 393 for test_group in ['skip', 'nonskip']: 394 for i in range(2): 395 for (k, _) in diff_map[test_group][i]: 396 testname_map[k] = True 397 rev_infos = TestExpectationsHistory.GetDiffBetweenTimes(prev_time, 398 current_time, 399 testname_map.keys()) 400 rev_str = '' 401 simple_rev_str = '' 402 rev = '' 403 rev_date = '' 404 if rev_infos: 405 # Get latest revision number and date. 406 rev = rev_infos[-1][1] 407 rev_date = rev_infos[-1][3] 408 for rev_info in rev_infos: 409 (old_rev, new_rev, author, date, _, target_lines) = rev_info 410 411 # test_expectations.txt was renamed to TestExpectations at r119317. 412 new_path = DEFAULT_TEST_EXPECTATION_PATH 413 if new_rev < 119317: 414 new_path = LEGACY_DEFAULT_TEST_EXPECTATION_PATH 415 old_path = DEFAULT_TEST_EXPECTATION_PATH 416 if old_rev < 119317: 417 old_path = LEGACY_DEFAULT_TEST_EXPECTATION_PATH 418 419 link = REVISION_LOG_URL % (new_path, old_rev, new_rev) 420 rev_str += '<ul><a href="%s">%s->%s</a>\n' % (link, old_rev, new_rev) 421 simple_rev_str = '<a href="%s">%s->%s</a>,' % (link, old_rev, new_rev) 422 rev_str += '<li>%s</li>\n' % author 423 rev_str += '<li>%s</li>\n<ul>' % date 424 for line in target_lines: 425 # Find *.html pattern (test name) and replace it with the link to 426 # flakiness dashboard. 427 test_name_pattern = r'(\S+.html)' 428 match = re.search(test_name_pattern, line) 429 if match: 430 test_name = match.group(1) 431 gpu_link = '' 432 if 'GPU' in line: 433 gpu_link = 'group=%40ToT%20GPU%20Mesa%20-%20chromium.org&' 434 dashboard_link = ('http://test-results.appspot.com/dashboards/' 435 'flakiness_dashboard.html#%stests=%s') % ( 436 gpu_link, test_name) 437 line = line.replace(test_name, '<a href="%s">%s</a>' % ( 438 dashboard_link, test_name)) 439 # Find bug text and replace it with the link to the bug. 440 bug = Bug(line) 441 if bug.bug_txt: 442 line = '<li>%s</li>\n' % line.replace(bug.bug_txt, str(bug)) 443 rev_str += line 444 rev_str += '</ul></ul>' 445 return (rev_str, simple_rev_str, rev, rev_date) 446 447 448def SendEmail(sender_email_address, receivers_email_addresses, subject, 449 message): 450 """Send email using localhost's mail server. 451 452 Args: 453 sender_email_address: sender's email address. 454 receivers_email_addresses: receiver's email addresses. 455 subject: subject string. 456 message: email message. 457 """ 458 try: 459 html_top = """ 460 <html> 461 <head></head> 462 <body> 463 """ 464 html_bot = """ 465 </body> 466 </html> 467 """ 468 html = html_top + message + html_bot 469 msg = MIMEMultipart('alternative') 470 msg['Subject'] = subject 471 msg['From'] = sender_email_address 472 msg['To'] = receivers_email_addresses[0] 473 part1 = MIMEText(html, 'html') 474 smtp_obj = smtplib.SMTP('localhost') 475 msg.attach(part1) 476 smtp_obj.sendmail(sender_email_address, receivers_email_addresses, 477 msg.as_string()) 478 print 'Successfully sent email' 479 except smtplib.SMTPException, ex: 480 print 'Authentication failed:', ex 481 print 'Error: unable to send email' 482 except (socket.gaierror, socket.error, socket.herror), ex: 483 print ex 484 print 'Error: unable to send email' 485 486 487def FindLatestTime(time_list): 488 """Find latest time from |time_list|. 489 490 The current status is compared to the status of the latest file in 491 |RESULT_DIR|. 492 493 Args: 494 time_list: a list of time string in the form of 'Year-Month-Day-Hour' 495 (e.g., 2011-10-23-23). Strings not in this format are ignored. 496 497 Returns: 498 a string representing latest time among the time_list or None if 499 |time_list| is empty or no valid date string in |time_list|. 500 """ 501 if not time_list: 502 return None 503 latest_date = None 504 for time_element in time_list: 505 try: 506 item_date = datetime.strptime(time_element, '%Y-%m-%d-%H') 507 if latest_date is None or latest_date < item_date: 508 latest_date = item_date 509 except ValueError: 510 # Do nothing. 511 pass 512 if latest_date: 513 return latest_date.strftime('%Y-%m-%d-%H') 514 else: 515 return None 516 517 518def ReplaceLineInFile(file_path, search_exp, replace_line): 519 """Replace line which has |search_exp| with |replace_line| within a file. 520 521 Args: 522 file_path: the file that is being replaced. 523 search_exp: search expression to find a line to be replaced. 524 replace_line: the new line. 525 """ 526 for line in fileinput.input(file_path, inplace=1): 527 if search_exp in line: 528 line = replace_line 529 sys.stdout.write(line) 530 531 532def FindLatestResult(result_dir): 533 """Find the latest result in |result_dir| and read and return them. 534 535 This is used for comparison of analyzer result between current analyzer 536 and most known latest result. 537 538 Args: 539 result_dir: the result directory. 540 541 Returns: 542 A tuple of filename (latest_time) and the latest analyzer result. 543 Returns None if there is no file or no file that matches the file 544 patterns used ('%Y-%m-%d-%H'). 545 """ 546 dir_list = os.listdir(result_dir) 547 file_name = FindLatestTime(dir_list) 548 if not file_name: 549 return None 550 file_path = os.path.join(result_dir, file_name) 551 return (file_name, AnalyzerResultMap.Load(file_path)) 552 553 554def GetDiffBetweenMaps(map1, map2, lookIntoTestExpectationInfo=False): 555 """Get difference between maps. 556 557 Args: 558 map1: analyzer result map to be compared. 559 map2: analyzer result map to be compared. 560 lookIntoTestExpectationInfo: a boolean to indicate whether to compare 561 test expectation information in addition to just the test case names. 562 563 Returns: 564 a tuple of |name1_list| and |name2_list|. |Name1_list| contains all test 565 name and the test expectation information in |map1| but not in |map2|. 566 |Name2_list| contains all test name and the test expectation 567 information in |map2| but not in |map1|. 568 """ 569 570 def GetDiffBetweenMapsHelper(map1, map2, lookIntoTestExpectationInfo): 571 """A helper function for GetDiffBetweenMaps. 572 573 Args: 574 map1: analyzer result map to be compared. 575 map2: analyzer result map to be compared. 576 lookIntoTestExpectationInfo: a boolean to indicate whether to compare 577 test expectation information in addition to just the test case names. 578 579 Returns: 580 a list of tuples (name, te_info) that are in |map1| but not in |map2|. 581 """ 582 name_list = [] 583 for (name, value1) in map1.iteritems(): 584 if name in map2: 585 if lookIntoTestExpectationInfo and 'te_info' in value1: 586 list1 = value1['te_info'] 587 list2 = map2[name]['te_info'] 588 te_diff = [item for item in list1 if not item in list2] 589 if te_diff: 590 name_list.append((name, te_diff)) 591 else: 592 name_list.append((name, value1)) 593 return name_list 594 595 return (GetDiffBetweenMapsHelper(map1, map2, lookIntoTestExpectationInfo), 596 GetDiffBetweenMapsHelper(map2, map1, lookIntoTestExpectationInfo)) 597