api_data_source.py revision 5c02ac1a9c1b504631c0a3d2b6e737b5d738bae1
1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5from copy import copy 6import logging 7import os 8import posixpath 9 10from data_source import DataSource 11from environment import IsPreviewServer 12from extensions_paths import JSON_TEMPLATES, PRIVATE_TEMPLATES 13from file_system import FileNotFoundError 14from future import Future, Collect 15import third_party.json_schema_compiler.json_parse as json_parse 16import third_party.json_schema_compiler.model as model 17from environment import IsPreviewServer 18from third_party.json_schema_compiler.memoize import memoize 19 20 21def _CreateId(node, prefix): 22 if node.parent is not None and not isinstance(node.parent, model.Namespace): 23 return '-'.join([prefix, node.parent.simple_name, node.simple_name]) 24 return '-'.join([prefix, node.simple_name]) 25 26 27def _FormatValue(value): 28 '''Inserts commas every three digits for integer values. It is magic. 29 ''' 30 s = str(value) 31 return ','.join([s[max(0, i - 3):i] for i in range(len(s), 0, -3)][::-1]) 32 33 34def _GetByNameDict(namespace): 35 '''Returns a dictionary mapping names to named items from |namespace|. 36 37 This lets us render specific API entities rather than the whole thing at once, 38 for example {{apis.manifestTypes.byName.ExternallyConnectable}}. 39 40 Includes items from namespace['types'], namespace['functions'], 41 namespace['events'], and namespace['properties']. 42 ''' 43 by_name = {} 44 for item_type in ('types', 'functions', 'events', 'properties'): 45 if item_type in namespace: 46 old_size = len(by_name) 47 by_name.update( 48 (item['name'], item) for item in namespace[item_type]) 49 assert len(by_name) == old_size + len(namespace[item_type]), ( 50 'Duplicate name in %r' % namespace) 51 return by_name 52 53 54def _GetEventByNameFromEvents(events): 55 '''Parses the dictionary |events| to find the definitions of members of the 56 type Event. Returns a dictionary mapping the name of a member to that 57 member's definition. 58 ''' 59 assert 'types' in events, \ 60 'The dictionary |events| must contain the key "types".' 61 event_list = [t for t in events['types'] if t.get('name') == 'Event'] 62 assert len(event_list) == 1, 'Exactly one type must be called "Event".' 63 return _GetByNameDict(event_list[0]) 64 65 66class _JSCModel(object): 67 '''Uses a Model from the JSON Schema Compiler and generates a dict that 68 a Handlebar template can use for a data source. 69 ''' 70 71 def __init__(self, 72 namespace, 73 availability_finder, 74 json_cache, 75 template_cache, 76 features_bundle, 77 event_byname_function): 78 self._availability_finder = availability_finder 79 self._api_availabilities = json_cache.GetFromFile( 80 posixpath.join(JSON_TEMPLATES, 'api_availabilities.json')) 81 self._intro_tables = json_cache.GetFromFile( 82 posixpath.join(JSON_TEMPLATES, 'intro_tables.json')) 83 self._api_features = features_bundle.GetAPIFeatures() 84 self._template_cache = template_cache 85 self._event_byname_function = event_byname_function 86 self._namespace = namespace 87 88 def _GetLink(self, link): 89 ref = link if '.' in link else (self._namespace.name + '.' + link) 90 return { 'ref': ref, 'text': link, 'name': link } 91 92 def ToDict(self): 93 if self._namespace is None: 94 return {} 95 chrome_dot_name = 'chrome.%s' % self._namespace.name 96 as_dict = { 97 'name': self._namespace.name, 98 'namespace': self._namespace.documentation_options.get('namespace', 99 chrome_dot_name), 100 'title': self._namespace.documentation_options.get('title', 101 chrome_dot_name), 102 'documentationOptions': self._namespace.documentation_options, 103 'types': self._GenerateTypes(self._namespace.types.values()), 104 'functions': self._GenerateFunctions(self._namespace.functions), 105 'events': self._GenerateEvents(self._namespace.events), 106 'domEvents': self._GenerateDomEvents(self._namespace.events), 107 'properties': self._GenerateProperties(self._namespace.properties), 108 'introList': self._GetIntroTableList(), 109 'channelWarning': self._GetChannelWarning(), 110 } 111 if self._namespace.deprecated: 112 as_dict['deprecated'] = self._namespace.deprecated 113 114 as_dict['byName'] = _GetByNameDict(as_dict) 115 return as_dict 116 117 def _GetApiAvailability(self): 118 return self._availability_finder.GetApiAvailability(self._namespace.name) 119 120 def _GetChannelWarning(self): 121 if not self._IsExperimental(): 122 return { self._GetApiAvailability().channel_info.channel: True } 123 return None 124 125 def _IsExperimental(self): 126 return self._namespace.name.startswith('experimental') 127 128 def _GenerateTypes(self, types): 129 return [self._GenerateType(t) for t in types] 130 131 def _GenerateType(self, type_): 132 type_dict = { 133 'name': type_.simple_name, 134 'description': type_.description, 135 'properties': self._GenerateProperties(type_.properties), 136 'functions': self._GenerateFunctions(type_.functions), 137 'events': self._GenerateEvents(type_.events), 138 'id': _CreateId(type_, 'type') 139 } 140 self._RenderTypeInformation(type_, type_dict) 141 return type_dict 142 143 def _GenerateFunctions(self, functions): 144 return [self._GenerateFunction(f) for f in functions.values()] 145 146 def _GenerateFunction(self, function): 147 function_dict = { 148 'name': function.simple_name, 149 'description': function.description, 150 'callback': self._GenerateCallback(function.callback), 151 'parameters': [], 152 'returns': None, 153 'id': _CreateId(function, 'method') 154 } 155 self._AddCommonProperties(function_dict, function) 156 if function.returns: 157 function_dict['returns'] = self._GenerateType(function.returns) 158 for param in function.params: 159 function_dict['parameters'].append(self._GenerateProperty(param)) 160 if function.callback is not None: 161 # Show the callback as an extra parameter. 162 function_dict['parameters'].append( 163 self._GenerateCallbackProperty(function.callback)) 164 if len(function_dict['parameters']) > 0: 165 function_dict['parameters'][-1]['last'] = True 166 return function_dict 167 168 def _GenerateEvents(self, events): 169 return [self._GenerateEvent(e) for e in events.values() 170 if not e.supports_dom] 171 172 def _GenerateDomEvents(self, events): 173 return [self._GenerateEvent(e) for e in events.values() 174 if e.supports_dom] 175 176 def _GenerateEvent(self, event): 177 event_dict = { 178 'name': event.simple_name, 179 'description': event.description, 180 'filters': [self._GenerateProperty(f) for f in event.filters], 181 'conditions': [self._GetLink(condition) 182 for condition in event.conditions], 183 'actions': [self._GetLink(action) for action in event.actions], 184 'supportsRules': event.supports_rules, 185 'supportsListeners': event.supports_listeners, 186 'properties': [], 187 'id': _CreateId(event, 'event'), 188 'byName': {}, 189 } 190 self._AddCommonProperties(event_dict, event) 191 # Add the Event members to each event in this object. 192 if self._event_byname_function: 193 event_dict['byName'].update(self._event_byname_function()) 194 # We need to create the method description for addListener based on the 195 # information stored in |event|. 196 if event.supports_listeners: 197 callback_object = model.Function(parent=event, 198 name='callback', 199 json={}, 200 namespace=event.parent, 201 origin='') 202 callback_object.params = event.params 203 if event.callback: 204 callback_object.callback = event.callback 205 callback_parameters = self._GenerateCallbackProperty(callback_object) 206 callback_parameters['last'] = True 207 event_dict['byName']['addListener'] = { 208 'name': 'addListener', 209 'callback': self._GenerateFunction(callback_object), 210 'parameters': [callback_parameters] 211 } 212 if event.supports_dom: 213 # Treat params as properties of the custom Event object associated with 214 # this DOM Event. 215 event_dict['properties'] += [self._GenerateProperty(param) 216 for param in event.params] 217 return event_dict 218 219 def _GenerateCallback(self, callback): 220 if not callback: 221 return None 222 callback_dict = { 223 'name': callback.simple_name, 224 'simple_type': {'simple_type': 'function'}, 225 'optional': callback.optional, 226 'parameters': [] 227 } 228 for param in callback.params: 229 callback_dict['parameters'].append(self._GenerateProperty(param)) 230 if (len(callback_dict['parameters']) > 0): 231 callback_dict['parameters'][-1]['last'] = True 232 return callback_dict 233 234 def _GenerateProperties(self, properties): 235 return [self._GenerateProperty(v) for v in properties.values()] 236 237 def _GenerateProperty(self, property_): 238 if not hasattr(property_, 'type_'): 239 for d in dir(property_): 240 if not d.startswith('_'): 241 print ('%s -> %s' % (d, getattr(property_, d))) 242 type_ = property_.type_ 243 244 # Make sure we generate property info for arrays, too. 245 # TODO(kalman): what about choices? 246 if type_.property_type == model.PropertyType.ARRAY: 247 properties = type_.item_type.properties 248 else: 249 properties = type_.properties 250 251 property_dict = { 252 'name': property_.simple_name, 253 'optional': property_.optional, 254 'description': property_.description, 255 'properties': self._GenerateProperties(type_.properties), 256 'functions': self._GenerateFunctions(type_.functions), 257 'parameters': [], 258 'returns': None, 259 'id': _CreateId(property_, 'property') 260 } 261 self._AddCommonProperties(property_dict, property_) 262 263 if type_.property_type == model.PropertyType.FUNCTION: 264 function = type_.function 265 for param in function.params: 266 property_dict['parameters'].append(self._GenerateProperty(param)) 267 if function.returns: 268 property_dict['returns'] = self._GenerateType(function.returns) 269 270 value = property_.value 271 if value is not None: 272 if isinstance(value, int): 273 property_dict['value'] = _FormatValue(value) 274 else: 275 property_dict['value'] = value 276 else: 277 self._RenderTypeInformation(type_, property_dict) 278 279 return property_dict 280 281 def _GenerateCallbackProperty(self, callback): 282 property_dict = { 283 'name': callback.simple_name, 284 'description': callback.description, 285 'optional': callback.optional, 286 'is_callback': True, 287 'id': _CreateId(callback, 'property'), 288 'simple_type': 'function', 289 } 290 if (callback.parent is not None and 291 not isinstance(callback.parent, model.Namespace)): 292 property_dict['parentName'] = callback.parent.simple_name 293 return property_dict 294 295 def _RenderTypeInformation(self, type_, dst_dict): 296 dst_dict['is_object'] = type_.property_type == model.PropertyType.OBJECT 297 if type_.property_type == model.PropertyType.CHOICES: 298 dst_dict['choices'] = self._GenerateTypes(type_.choices) 299 # We keep track of which == last for knowing when to add "or" between 300 # choices in templates. 301 if len(dst_dict['choices']) > 0: 302 dst_dict['choices'][-1]['last'] = True 303 elif type_.property_type == model.PropertyType.REF: 304 dst_dict['link'] = self._GetLink(type_.ref_type) 305 elif type_.property_type == model.PropertyType.ARRAY: 306 dst_dict['array'] = self._GenerateType(type_.item_type) 307 elif type_.property_type == model.PropertyType.ENUM: 308 dst_dict['enum_values'] = [ 309 {'name': value.name, 'description': value.description} 310 for value in type_.enum_values] 311 if len(dst_dict['enum_values']) > 0: 312 dst_dict['enum_values'][-1]['last'] = True 313 elif type_.instance_of is not None: 314 dst_dict['simple_type'] = type_.instance_of 315 else: 316 dst_dict['simple_type'] = type_.property_type.name 317 318 def _GetIntroTableList(self): 319 '''Create a generic data structure that can be traversed by the templates 320 to create an API intro table. 321 ''' 322 intro_rows = [ 323 self._GetIntroDescriptionRow(), 324 self._GetIntroAvailabilityRow() 325 ] + self._GetIntroDependencyRows() 326 327 # Add rows using data from intro_tables.json, overriding any existing rows 328 # if they share the same 'title' attribute. 329 row_titles = [row['title'] for row in intro_rows] 330 for misc_row in self._GetMiscIntroRows(): 331 if misc_row['title'] in row_titles: 332 intro_rows[row_titles.index(misc_row['title'])] = misc_row 333 else: 334 intro_rows.append(misc_row) 335 336 return intro_rows 337 338 def _GetIntroDescriptionRow(self): 339 ''' Generates the 'Description' row data for an API intro table. 340 ''' 341 return { 342 'title': 'Description', 343 'content': [ 344 { 'text': self._namespace.description } 345 ] 346 } 347 348 def _GetIntroAvailabilityRow(self): 349 ''' Generates the 'Availability' row data for an API intro table. 350 ''' 351 if self._IsExperimental(): 352 status = 'experimental' 353 version = None 354 scheduled = None 355 else: 356 availability = self._GetApiAvailability() 357 status = availability.channel_info.channel 358 version = availability.channel_info.version 359 scheduled = availability.scheduled 360 return { 361 'title': 'Availability', 362 'content': [{ 363 'partial': self._template_cache.GetFromFile( 364 posixpath.join(PRIVATE_TEMPLATES, 365 'intro_tables', 366 '%s_message.html' % status)).Get(), 367 'version': version, 368 'scheduled': scheduled 369 }] 370 } 371 372 def _GetIntroDependencyRows(self): 373 # Devtools aren't in _api_features. If we're dealing with devtools, bail. 374 if 'devtools' in self._namespace.name: 375 return [] 376 377 api_feature = self._api_features.Get().get(self._namespace.name) 378 if not api_feature: 379 logging.error('"%s" not found in _api_features.json' % 380 self._namespace.name) 381 return [] 382 383 permissions_content = [] 384 manifest_content = [] 385 386 def categorize_dependency(dependency): 387 def make_code_node(text): 388 return { 'class': 'code', 'text': text } 389 390 context, name = dependency.split(':', 1) 391 if context == 'permission': 392 permissions_content.append(make_code_node('"%s"' % name)) 393 elif context == 'manifest': 394 manifest_content.append(make_code_node('"%s": {...}' % name)) 395 elif context == 'api': 396 transitive_dependencies = ( 397 self._api_features.Get().get(name, {}).get('dependencies', [])) 398 for transitive_dependency in transitive_dependencies: 399 categorize_dependency(transitive_dependency) 400 else: 401 logging.error('Unrecognized dependency for %s: %s' % 402 (self._namespace.name, context)) 403 404 for dependency in api_feature.get('dependencies', ()): 405 categorize_dependency(dependency) 406 407 dependency_rows = [] 408 if permissions_content: 409 dependency_rows.append({ 410 'title': 'Permissions', 411 'content': permissions_content 412 }) 413 if manifest_content: 414 dependency_rows.append({ 415 'title': 'Manifest', 416 'content': manifest_content 417 }) 418 return dependency_rows 419 420 def _GetMiscIntroRows(self): 421 ''' Generates miscellaneous intro table row data, such as 'Permissions', 422 'Samples', and 'Learn More', using intro_tables.json. 423 ''' 424 misc_rows = [] 425 # Look up the API name in intro_tables.json, which is structured 426 # similarly to the data structure being created. If the name is found, loop 427 # through the attributes and add them to this structure. 428 table_info = self._intro_tables.Get().get(self._namespace.name) 429 if table_info is None: 430 return misc_rows 431 432 for category in table_info.iterkeys(): 433 content = [] 434 for node in table_info[category]: 435 # If there is a 'partial' argument and it hasn't already been 436 # converted to a Handlebar object, transform it to a template. 437 if 'partial' in node: 438 # Note: it's enough to copy() not deepcopy() because only a single 439 # top-level key is being modified. 440 node = copy(node) 441 node['partial'] = self._template_cache.GetFromFile( 442 posixpath.join(PRIVATE_TEMPLATES, node['partial'])).Get() 443 content.append(node) 444 misc_rows.append({ 'title': category, 'content': content }) 445 return misc_rows 446 447 def _AddCommonProperties(self, target, src): 448 if src.deprecated is not None: 449 target['deprecated'] = src.deprecated 450 if (src.parent is not None and 451 not isinstance(src.parent, model.Namespace)): 452 target['parentName'] = src.parent.simple_name 453 454 455class _LazySamplesGetter(object): 456 '''This class is needed so that an extensions API page does not have to fetch 457 the apps samples page and vice versa. 458 ''' 459 460 def __init__(self, api_name, samples): 461 self._api_name = api_name 462 self._samples = samples 463 464 def get(self, key): 465 return self._samples.FilterSamples(key, self._api_name) 466 467 468class APIDataSource(DataSource): 469 '''This class fetches and loads JSON APIs from the FileSystem passed in with 470 |compiled_fs_factory|, so the APIs can be plugged into templates. 471 ''' 472 def __init__(self, server_instance, request): 473 file_system = server_instance.host_file_system_provider.GetTrunk() 474 self._json_cache = server_instance.compiled_fs_factory.ForJson(file_system) 475 self._template_cache = server_instance.compiled_fs_factory.ForTemplates( 476 file_system) 477 self._availability_finder = server_instance.availability_finder 478 self._api_models = server_instance.api_models 479 self._features_bundle = server_instance.features_bundle 480 self._model_cache = server_instance.object_store_creator.Create( 481 APIDataSource) 482 483 # This caches the result of _LoadEventByName. 484 self._event_byname = None 485 self._samples = server_instance.samples_data_source_factory.Create(request) 486 487 def _LoadEventByName(self): 488 '''All events have some members in common. We source their description 489 from Event in events.json. 490 ''' 491 if self._event_byname is None: 492 self._event_byname = _GetEventByNameFromEvents( 493 self._GetSchemaModel('events').Get()) 494 return self._event_byname 495 496 def _GetSchemaModel(self, api_name): 497 jsc_model_future = self._model_cache.Get(api_name) 498 model_future = self._api_models.GetModel(api_name) 499 def resolve(): 500 jsc_model = jsc_model_future.Get() 501 if jsc_model is None: 502 jsc_model = _JSCModel( 503 model_future.Get(), 504 self._availability_finder, 505 self._json_cache, 506 self._template_cache, 507 self._features_bundle, 508 self._LoadEventByName).ToDict() 509 self._model_cache.Set(api_name, jsc_model) 510 return jsc_model 511 return Future(callback=resolve) 512 513 def _GetImpl(self, api_name): 514 handlebar_dict_future = self._GetSchemaModel(api_name) 515 def resolve(): 516 handlebar_dict = handlebar_dict_future.Get() 517 # Parsing samples on the preview server takes seconds and doesn't add 518 # anything. Don't do it. 519 if not IsPreviewServer(): 520 handlebar_dict['samples'] = _LazySamplesGetter( 521 handlebar_dict['name'], 522 self._samples) 523 return handlebar_dict 524 return Future(callback=resolve) 525 526 def get(self, api_name): 527 return self._GetImpl(api_name).Get() 528 529 def Cron(self): 530 futures = [self._GetImpl(name) for name in self._api_models.GetNames()] 531 return Collect(futures, except_pass=FileNotFoundError) 532