1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import unittest 8 9from telemetry.core import discover 10from telemetry.internal.browser import browser_credentials 11from telemetry.internal import story_runner 12from telemetry import page 13from telemetry import story as story_module 14from telemetry.wpr import archive_info 15 16 17class StorySetSmokeTest(unittest.TestCase): 18 19 def setUp(self): 20 # Make sure the added failure message is appended to the default failure 21 # message. 22 self.longMessage = True 23 24 def GetAllStorySetClasses(self, story_sets_dir, top_level_dir): 25 # We can't test page sets that aren't directly constructible since we 26 # don't know what arguments to put for the constructor. 27 return discover.DiscoverClasses(story_sets_dir, top_level_dir, 28 story_module.StorySet, 29 directly_constructable=True).values() 30 31 def CheckArchive(self, story_set): 32 """Verify that all URLs of pages in story_set have an associated archive.""" 33 # TODO: Eventually these should be fatal. 34 if not story_set.archive_data_file: 35 logging.warning('Skipping %s: no archive data file', story_set.file_path) 36 return 37 38 logging.info('Testing %s', story_set.file_path) 39 40 archive_data_file_path = os.path.join(story_set.base_dir, 41 story_set.archive_data_file) 42 self.assertTrue(os.path.exists(archive_data_file_path), 43 msg='Archive data file not found for %s' % 44 story_set.file_path) 45 46 wpr_archive_info = archive_info.WprArchiveInfo.FromFile( 47 archive_data_file_path, story_set.bucket) 48 for story in story_set.stories: 49 if isinstance(story, page.Page) and story.url.startswith('http'): 50 self.assertTrue(wpr_archive_info.WprFilePathForStory(story), 51 msg='No archive found for %s in %s' % ( 52 story.url, story_set.archive_data_file)) 53 54 def CheckCredentials(self, story_set): 55 """Verify that all pages in story_set use proper credentials""" 56 for story in story_set.stories: 57 if not isinstance(story, page.Page): 58 continue 59 credentials = browser_credentials.BrowserCredentials() 60 if story.credentials_path: 61 credentials.credentials_path = ( 62 os.path.join(story.base_dir, story.credentials_path)) 63 fail_message = ('page %s of %s has invalid credentials %s' % 64 (story.url, story_set.file_path, story.credentials)) 65 if story.credentials: 66 try: 67 self.assertTrue(credentials.CanLogin(story.credentials), fail_message) 68 except browser_credentials.CredentialsError: 69 self.fail(fail_message) 70 71 def CheckAttributes(self, story_set): 72 """Verify that story_set and its stories base attributes have the right 73 types. 74 """ 75 self.CheckAttributesOfStorySetBasicAttributes(story_set) 76 for story in story_set.stories: 77 self.CheckAttributesOfStoryBasicAttributes(story) 78 79 def CheckAttributesOfStorySetBasicAttributes(self, story_set): 80 if story_set.base_dir is not None: 81 self.assertTrue( 82 isinstance(story_set.base_dir, str), 83 msg='story_set %\'s base_dir must have type string') 84 85 self.assertTrue( 86 isinstance(story_set.archive_data_file, str), 87 msg='story_set\'s archive_data_file path must have type string') 88 89 def CheckAttributesOfStoryBasicAttributes(self, story): 90 self.assertTrue(not hasattr(story, 'disabled')) 91 self.assertTrue( 92 isinstance(story.name, str), 93 msg='story %s \'s name field must have type string' % story.display_name) 94 self.assertTrue( 95 isinstance(story.tags, set), 96 msg='story %s \'s tags field must have type set' % story.display_name) 97 for t in story.tags: 98 self.assertTrue( 99 isinstance(t, str), 100 msg='tag %s in story %s \'s tags must have type string' 101 % (str(t), story.display_name)) 102 if not isinstance(story, page.Page): 103 return 104 self.assertTrue( 105 # We use basestring instead of str because story's URL can be string of 106 # unicode. 107 isinstance(story.url, basestring), 108 msg='page %s \'s url must have type string' % story.display_name) 109 self.assertTrue( 110 isinstance(story.startup_url, str), 111 msg=('page %s \'s startup_url field must have type string' 112 % story.display_name)) 113 self.assertIsInstance( 114 story.make_javascript_deterministic, bool, 115 msg='page %s \'s make_javascript_deterministic must have type bool' 116 % story.display_name) 117 118 def CheckSharedStates(self, story_set): 119 if not story_set.allow_mixed_story_states: 120 shared_state_class = ( 121 story_set.stories[0].shared_state_class) 122 for story in story_set: 123 self.assertIs( 124 shared_state_class, 125 story.shared_state_class, 126 msg='story %s\'s shared_state_class field is different ' 127 'from other story\'s shared_state_class whereas ' 128 'story set %s disallows having mixed states' % 129 (story, story_set)) 130 131 def CheckPassingStoryRunnerValidation(self, story_set): 132 errors = [] 133 for s in story_set: 134 try: 135 story_runner.ValidateStory(s) 136 except ValueError as e: 137 errors.append(e) 138 self.assertFalse( 139 errors, 'Errors validating user stories in %s:\n %s' % ( 140 story_set, '\n'.join(e.message for e in errors))) 141 142 def RunSmokeTest(self, story_sets_dir, top_level_dir): 143 """Run smoke test on all story sets in story_sets_dir. 144 145 Subclass of StorySetSmokeTest is supposed to call this in some test 146 method to run smoke test. 147 """ 148 story_sets = self.GetAllStorySetClasses(story_sets_dir, top_level_dir) 149 for story_set_class in story_sets: 150 story_set = story_set_class() 151 self.CheckArchive(story_set) 152 self.CheckCredentials(story_set) 153 self.CheckAttributes(story_set) 154 self.CheckSharedStates(story_set) 155 self.CheckPassingStoryRunnerValidation(story_set) 156