api_data_source.py revision 1e9bf3e0803691d0a228da41fc608347b6db4340
1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import copy 6import json 7import logging 8import os 9from collections import defaultdict, Mapping 10 11import svn_constants 12import third_party.json_schema_compiler.json_parse as json_parse 13import third_party.json_schema_compiler.model as model 14import third_party.json_schema_compiler.idl_schema as idl_schema 15import third_party.json_schema_compiler.idl_parser as idl_parser 16from schema_util import RemoveNoDocs, DetectInlineableTypes, InlineDocs 17from third_party.handlebar import Handlebar 18 19 20def _CreateId(node, prefix): 21 if node.parent is not None and not isinstance(node.parent, model.Namespace): 22 return '-'.join([prefix, node.parent.simple_name, node.simple_name]) 23 return '-'.join([prefix, node.simple_name]) 24 25 26def _FormatValue(value): 27 '''Inserts commas every three digits for integer values. It is magic. 28 ''' 29 s = str(value) 30 return ','.join([s[max(0, i - 3):i] for i in range(len(s), 0, -3)][::-1]) 31 32 33def _GetAddRulesDefinitionFromEvents(events): 34 '''Parses the dictionary |events| to find the definition of the method 35 addRules among functions of the type Event. 36 ''' 37 assert 'types' in events, \ 38 'The dictionary |events| must contain the key "types".' 39 event_list = [t for t in events['types'] 40 if 'name' in t and t['name'] == 'Event'] 41 assert len(event_list) == 1, 'Exactly one type must be called "Event".' 42 event = event_list[0] 43 assert 'functions' in event, 'The type Event must contain "functions".' 44 result_list = [f for f in event['functions'] 45 if 'name' in f and f['name'] == 'addRules'] 46 assert len(result_list) == 1, \ 47 'Exactly one function must be called "addRules".' 48 return result_list[0] 49 50 51class _JSCModel(object): 52 '''Uses a Model from the JSON Schema Compiler and generates a dict that 53 a Handlebar template can use for a data source. 54 ''' 55 56 def __init__(self, 57 json, 58 ref_resolver, 59 disable_refs, 60 availability_finder, 61 branch_utility, 62 parse_cache, 63 template_data_source, 64 add_rules_schema_function, 65 idl=False): 66 self._ref_resolver = ref_resolver 67 self._disable_refs = disable_refs 68 self._availability_finder = availability_finder 69 self._branch_utility = branch_utility 70 self._api_availabilities = parse_cache.GetFromFile( 71 '%s/api_availabilities.json' % svn_constants.JSON_PATH) 72 self._intro_tables = parse_cache.GetFromFile( 73 '%s/intro_tables.json' % svn_constants.JSON_PATH) 74 self._api_features = parse_cache.GetFromFile( 75 '%s/_api_features.json' % svn_constants.API_PATH) 76 self._template_data_source = template_data_source 77 self._add_rules_schema_function = add_rules_schema_function 78 clean_json = copy.deepcopy(json) 79 if RemoveNoDocs(clean_json): 80 self._namespace = None 81 else: 82 if idl: 83 DetectInlineableTypes(clean_json) 84 InlineDocs(clean_json) 85 self._namespace = model.Namespace(clean_json, clean_json['namespace']) 86 87 def _FormatDescription(self, description): 88 if self._disable_refs: 89 return description 90 return self._ref_resolver.ResolveAllLinks(description, 91 namespace=self._namespace.name) 92 93 def _GetLink(self, link): 94 if self._disable_refs: 95 type_name = link.split('.', 1)[-1] 96 return { 'href': '#type-%s' % type_name, 'text': link, 'name': link } 97 return self._ref_resolver.SafeGetLink(link, namespace=self._namespace.name) 98 99 def ToDict(self): 100 if self._namespace is None: 101 return {} 102 as_dict = { 103 'name': self._namespace.name, 104 'documentationOptions': self._namespace.documentation_options, 105 'types': self._GenerateTypes(self._namespace.types.values()), 106 'functions': self._GenerateFunctions(self._namespace.functions), 107 'events': self._GenerateEvents(self._namespace.events), 108 'domEvents': self._GenerateDomEvents(self._namespace.events), 109 'properties': self._GenerateProperties(self._namespace.properties), 110 'introList': self._GetIntroTableList(), 111 'channelWarning': self._GetChannelWarning(), 112 'byName': {}, 113 } 114 # Make every type/function/event/property also accessible by name for 115 # rendering specific API entities rather than the whole thing at once, for 116 # example {{apis.manifestTypes.byName.ExternallyConnectable}}. 117 for item_type in ('types', 'functions', 'events', 'properties'): 118 as_dict['byName'].update( 119 (item['name'], item) for item in as_dict[item_type]) 120 return as_dict 121 122 def _GetApiAvailability(self): 123 # Check for a predetermined availability for this API. 124 api_info = self._api_availabilities.Get().get(self._namespace.name) 125 if api_info is not None: 126 channel = api_info['channel'] 127 if channel == 'stable': 128 return self._branch_utility.GetStableChannelInfo(api_info['version']) 129 return self._branch_utility.GetChannelInfo(channel) 130 return self._availability_finder.GetApiAvailability(self._namespace.name) 131 132 def _GetChannelWarning(self): 133 if not self._IsExperimental(): 134 return { self._GetApiAvailability().channel: True } 135 return None 136 137 def _IsExperimental(self): 138 return self._namespace.name.startswith('experimental') 139 140 def _GenerateTypes(self, types): 141 return [self._GenerateType(t) for t in types] 142 143 def _GenerateType(self, type_): 144 type_dict = { 145 'name': type_.simple_name, 146 'description': self._FormatDescription(type_.description), 147 'properties': self._GenerateProperties(type_.properties), 148 'functions': self._GenerateFunctions(type_.functions), 149 'events': self._GenerateEvents(type_.events), 150 'id': _CreateId(type_, 'type') 151 } 152 self._RenderTypeInformation(type_, type_dict) 153 return type_dict 154 155 def _GenerateFunctions(self, functions): 156 return [self._GenerateFunction(f) for f in functions.values()] 157 158 def _GenerateFunction(self, function): 159 function_dict = { 160 'name': function.simple_name, 161 'description': self._FormatDescription(function.description), 162 'callback': self._GenerateCallback(function.callback), 163 'parameters': [], 164 'returns': None, 165 'id': _CreateId(function, 'method') 166 } 167 if (function.deprecated is not None): 168 function_dict['deprecated'] = self._FormatDescription( 169 function.deprecated) 170 if (function.parent is not None and 171 not isinstance(function.parent, model.Namespace)): 172 function_dict['parentName'] = function.parent.simple_name 173 if function.returns: 174 function_dict['returns'] = self._GenerateType(function.returns) 175 for param in function.params: 176 function_dict['parameters'].append(self._GenerateProperty(param)) 177 if function.callback is not None: 178 # Show the callback as an extra parameter. 179 function_dict['parameters'].append( 180 self._GenerateCallbackProperty(function.callback)) 181 if len(function_dict['parameters']) > 0: 182 function_dict['parameters'][-1]['last'] = True 183 return function_dict 184 185 def _GenerateEvents(self, events): 186 return [self._GenerateEvent(e) for e in events.values() 187 if not e.supports_dom] 188 189 def _GenerateDomEvents(self, events): 190 return [self._GenerateEvent(e) for e in events.values() 191 if e.supports_dom] 192 193 def _GenerateEvent(self, event): 194 event_dict = { 195 'name': event.simple_name, 196 'description': self._FormatDescription(event.description), 197 'filters': [self._GenerateProperty(f) for f in event.filters], 198 'conditions': [self._GetLink(condition) 199 for condition in event.conditions], 200 'actions': [self._GetLink(action) for action in event.actions], 201 'supportsRules': event.supports_rules, 202 'supportsListeners': event.supports_listeners, 203 'properties': [], 204 'id': _CreateId(event, 'event') 205 } 206 if (event.parent is not None and 207 not isinstance(event.parent, model.Namespace)): 208 event_dict['parentName'] = event.parent.simple_name 209 # For the addRules method we can use the common definition, because addRules 210 # has the same signature for every event. 211 if event.supports_rules: 212 event_dict['addRulesFunction'] = self._add_rules_schema_function() 213 # We need to create the method description for addListener based on the 214 # information stored in |event|. 215 if event.supports_listeners: 216 callback_object = model.Function(parent=event, 217 name='callback', 218 json={}, 219 namespace=event.parent, 220 origin='') 221 callback_object.params = event.params 222 if event.callback: 223 callback_object.callback = event.callback 224 callback_parameters = self._GenerateCallbackProperty(callback_object) 225 callback_parameters['last'] = True 226 event_dict['addListenerFunction'] = { 227 'name': 'addListener', 228 'callback': self._GenerateFunction(callback_object), 229 'parameters': [callback_parameters] 230 } 231 if event.supports_dom: 232 # Treat params as properties of the custom Event object associated with 233 # this DOM Event. 234 event_dict['properties'] += [self._GenerateProperty(param) 235 for param in event.params] 236 return event_dict 237 238 def _GenerateCallback(self, callback): 239 if not callback: 240 return None 241 callback_dict = { 242 'name': callback.simple_name, 243 'simple_type': {'simple_type': 'function'}, 244 'optional': callback.optional, 245 'parameters': [] 246 } 247 for param in callback.params: 248 callback_dict['parameters'].append(self._GenerateProperty(param)) 249 if (len(callback_dict['parameters']) > 0): 250 callback_dict['parameters'][-1]['last'] = True 251 return callback_dict 252 253 def _GenerateProperties(self, properties): 254 return [self._GenerateProperty(v) for v in properties.values()] 255 256 def _GenerateProperty(self, property_): 257 if not hasattr(property_, 'type_'): 258 for d in dir(property_): 259 if not d.startswith('_'): 260 print ('%s -> %s' % (d, getattr(property_, d))) 261 type_ = property_.type_ 262 263 # Make sure we generate property info for arrays, too. 264 # TODO(kalman): what about choices? 265 if type_.property_type == model.PropertyType.ARRAY: 266 properties = type_.item_type.properties 267 else: 268 properties = type_.properties 269 270 property_dict = { 271 'name': property_.simple_name, 272 'optional': property_.optional, 273 'description': self._FormatDescription(property_.description), 274 'properties': self._GenerateProperties(type_.properties), 275 'functions': self._GenerateFunctions(type_.functions), 276 'parameters': [], 277 'returns': None, 278 'id': _CreateId(property_, 'property') 279 } 280 281 if type_.property_type == model.PropertyType.FUNCTION: 282 function = type_.function 283 for param in function.params: 284 property_dict['parameters'].append(self._GenerateProperty(param)) 285 if function.returns: 286 property_dict['returns'] = self._GenerateType(function.returns) 287 288 if (property_.parent is not None and 289 not isinstance(property_.parent, model.Namespace)): 290 property_dict['parentName'] = property_.parent.simple_name 291 292 value = property_.value 293 if value is not None: 294 if isinstance(value, int): 295 property_dict['value'] = _FormatValue(value) 296 else: 297 property_dict['value'] = value 298 else: 299 self._RenderTypeInformation(type_, property_dict) 300 301 return property_dict 302 303 def _GenerateCallbackProperty(self, callback): 304 property_dict = { 305 'name': callback.simple_name, 306 'description': self._FormatDescription(callback.description), 307 'optional': callback.optional, 308 'id': _CreateId(callback, 'property'), 309 'simple_type': 'function', 310 } 311 if (callback.parent is not None and 312 not isinstance(callback.parent, model.Namespace)): 313 property_dict['parentName'] = callback.parent.simple_name 314 return property_dict 315 316 def _RenderTypeInformation(self, type_, dst_dict): 317 dst_dict['is_object'] = type_.property_type == model.PropertyType.OBJECT 318 if type_.property_type == model.PropertyType.CHOICES: 319 dst_dict['choices'] = self._GenerateTypes(type_.choices) 320 # We keep track of which == last for knowing when to add "or" between 321 # choices in templates. 322 if len(dst_dict['choices']) > 0: 323 dst_dict['choices'][-1]['last'] = True 324 elif type_.property_type == model.PropertyType.REF: 325 dst_dict['link'] = self._GetLink(type_.ref_type) 326 elif type_.property_type == model.PropertyType.ARRAY: 327 dst_dict['array'] = self._GenerateType(type_.item_type) 328 elif type_.property_type == model.PropertyType.ENUM: 329 dst_dict['enum_values'] = [ 330 {'name': value.name, 'description': value.description} 331 for value in type_.enum_values] 332 if len(dst_dict['enum_values']) > 0: 333 dst_dict['enum_values'][-1]['last'] = True 334 elif type_.instance_of is not None: 335 dst_dict['simple_type'] = type_.instance_of.lower() 336 else: 337 dst_dict['simple_type'] = type_.property_type.name.lower() 338 339 def _GetIntroTableList(self): 340 '''Create a generic data structure that can be traversed by the templates 341 to create an API intro table. 342 ''' 343 intro_rows = [ 344 self._GetIntroDescriptionRow(), 345 self._GetIntroAvailabilityRow() 346 ] + self._GetIntroDependencyRows() 347 348 # Add rows using data from intro_tables.json, overriding any existing rows 349 # if they share the same 'title' attribute. 350 row_titles = [row['title'] for row in intro_rows] 351 for misc_row in self._GetMiscIntroRows(): 352 if misc_row['title'] in row_titles: 353 intro_rows[row_titles.index(misc_row['title'])] = misc_row 354 else: 355 intro_rows.append(misc_row) 356 357 return intro_rows 358 359 def _GetIntroDescriptionRow(self): 360 ''' Generates the 'Description' row data for an API intro table. 361 ''' 362 return { 363 'title': 'Description', 364 'content': [ 365 { 'text': self._FormatDescription(self._namespace.description) } 366 ] 367 } 368 369 def _GetIntroAvailabilityRow(self): 370 ''' Generates the 'Availability' row data for an API intro table. 371 ''' 372 if self._IsExperimental(): 373 status = 'experimental' 374 version = None 375 else: 376 availability = self._GetApiAvailability() 377 status = availability.channel 378 version = availability.version 379 return { 380 'title': 'Availability', 381 'content': [{ 382 'partial': self._template_data_source.get( 383 'intro_tables/%s_message.html' % status), 384 'version': version 385 }] 386 } 387 388 def _GetIntroDependencyRows(self): 389 # Devtools aren't in _api_features. If we're dealing with devtools, bail. 390 if 'devtools' in self._namespace.name: 391 return [] 392 feature = self._api_features.Get().get(self._namespace.name) 393 assert feature, ('"%s" not found in _api_features.json.' 394 % self._namespace.name) 395 396 dependencies = feature.get('dependencies') 397 if dependencies is None: 398 return [] 399 400 def make_code_node(text): 401 return { 'class': 'code', 'text': text } 402 403 permissions_content = [] 404 manifest_content = [] 405 406 def categorize_dependency(dependency): 407 context, name = dependency.split(':', 1) 408 if context == 'permission': 409 permissions_content.append(make_code_node('"%s"' % name)) 410 elif context == 'manifest': 411 manifest_content.append(make_code_node('"%s": {...}' % name)) 412 elif context == 'api': 413 transitive_dependencies = ( 414 self._api_features.Get().get(name, {}).get('dependencies', [])) 415 for transitive_dependency in transitive_dependencies: 416 categorize_dependency(transitive_dependency) 417 else: 418 raise ValueError('Unrecognized dependency for %s: %s' % ( 419 self._namespace.name, context)) 420 421 for dependency in dependencies: 422 categorize_dependency(dependency) 423 424 dependency_rows = [] 425 if permissions_content: 426 dependency_rows.append({ 427 'title': 'Permissions', 428 'content': permissions_content 429 }) 430 if manifest_content: 431 dependency_rows.append({ 432 'title': 'Manifest', 433 'content': manifest_content 434 }) 435 return dependency_rows 436 437 def _GetMiscIntroRows(self): 438 ''' Generates miscellaneous intro table row data, such as 'Permissions', 439 'Samples', and 'Learn More', using intro_tables.json. 440 ''' 441 misc_rows = [] 442 # Look up the API name in intro_tables.json, which is structured 443 # similarly to the data structure being created. If the name is found, loop 444 # through the attributes and add them to this structure. 445 table_info = self._intro_tables.Get().get(self._namespace.name) 446 if table_info is None: 447 return misc_rows 448 449 for category in table_info.keys(): 450 content = copy.deepcopy(table_info[category]) 451 for node in content: 452 # If there is a 'partial' argument and it hasn't already been 453 # converted to a Handlebar object, transform it to a template. 454 if 'partial' in node: 455 node['partial'] = self._template_data_source.get(node['partial']) 456 misc_rows.append({ 'title': category, 'content': content }) 457 return misc_rows 458 459 460class _LazySamplesGetter(object): 461 '''This class is needed so that an extensions API page does not have to fetch 462 the apps samples page and vice versa. 463 ''' 464 465 def __init__(self, api_name, samples): 466 self._api_name = api_name 467 self._samples = samples 468 469 def get(self, key): 470 return self._samples.FilterSamples(key, self._api_name) 471 472 473class APIDataSource(object): 474 '''This class fetches and loads JSON APIs from the FileSystem passed in with 475 |compiled_fs_factory|, so the APIs can be plugged into templates. 476 ''' 477 478 class Factory(object): 479 def __init__(self, 480 compiled_fs_factory, 481 file_system, 482 base_path, 483 availability_finder, 484 branch_utility): 485 def create_compiled_fs(fn, category): 486 return compiled_fs_factory.Create( 487 file_system, fn, APIDataSource, category=category) 488 489 self._json_cache = create_compiled_fs( 490 lambda api_name, api: self._LoadJsonAPI(api, False), 491 'json') 492 self._idl_cache = create_compiled_fs( 493 lambda api_name, api: self._LoadIdlAPI(api, False), 494 'idl') 495 496 # These caches are used if an APIDataSource does not want to resolve the 497 # $refs in an API. This is needed to prevent infinite recursion in 498 # ReferenceResolver. 499 self._json_cache_no_refs = create_compiled_fs( 500 lambda api_name, api: self._LoadJsonAPI(api, True), 501 'json-no-refs') 502 self._idl_cache_no_refs = create_compiled_fs( 503 lambda api_name, api: self._LoadIdlAPI(api, True), 504 'idl-no-refs') 505 506 self._idl_names_cache = create_compiled_fs(self._GetIDLNames, 'idl-names') 507 self._names_cache = create_compiled_fs(self._GetAllNames, 'names') 508 509 self._base_path = base_path 510 self._availability_finder = availability_finder 511 self._branch_utility = branch_utility 512 self._parse_cache = create_compiled_fs( 513 lambda _, json: json_parse.Parse(json), 514 'intro-cache') 515 # These must be set later via the SetFooDataSourceFactory methods. 516 self._ref_resolver_factory = None 517 self._samples_data_source_factory = None 518 519 # This caches the result of _LoadAddRulesSchema. 520 self._add_rules_schema = None 521 522 def SetSamplesDataSourceFactory(self, samples_data_source_factory): 523 self._samples_data_source_factory = samples_data_source_factory 524 525 def SetReferenceResolverFactory(self, ref_resolver_factory): 526 self._ref_resolver_factory = ref_resolver_factory 527 528 def SetTemplateDataSource(self, template_data_source_factory): 529 # This TemplateDataSource is only being used for fetching template data. 530 self._template_data_source = template_data_source_factory.Create( 531 None, {}) 532 533 def Create(self, request, disable_refs=False): 534 '''Create an APIDataSource. |disable_refs| specifies whether $ref's in 535 APIs being processed by the |ToDict| method of _JSCModel follows $ref's 536 in the API. This prevents endless recursion in ReferenceResolver. 537 ''' 538 if self._samples_data_source_factory is None: 539 # Only error if there is a request, which means this APIDataSource is 540 # actually being used to render a page. 541 if request is not None: 542 logging.error('SamplesDataSource.Factory was never set in ' 543 'APIDataSource.Factory.') 544 samples = None 545 else: 546 samples = self._samples_data_source_factory.Create(request) 547 if not disable_refs and self._ref_resolver_factory is None: 548 logging.error('ReferenceResolver.Factory was never set in ' 549 'APIDataSource.Factory.') 550 return APIDataSource(self._json_cache, 551 self._idl_cache, 552 self._json_cache_no_refs, 553 self._idl_cache_no_refs, 554 self._names_cache, 555 self._idl_names_cache, 556 self._base_path, 557 samples, 558 disable_refs) 559 560 def _LoadAddRulesSchema(self): 561 """ All events supporting rules have the addRules method. We source its 562 description from Event in events.json. 563 """ 564 if self._add_rules_schema is None: 565 events_json = self._json_cache.GetFromFile( 566 '%s/events.json' % self._base_path).Get() 567 self._add_rules_schema = _GetAddRulesDefinitionFromEvents(events_json) 568 return self._add_rules_schema 569 570 def _LoadJsonAPI(self, api, disable_refs): 571 return _JSCModel( 572 json_parse.Parse(api)[0], 573 self._ref_resolver_factory.Create() if not disable_refs else None, 574 disable_refs, 575 self._availability_finder, 576 self._branch_utility, 577 self._parse_cache, 578 self._template_data_source, 579 self._LoadAddRulesSchema).ToDict() 580 581 def _LoadIdlAPI(self, api, disable_refs): 582 idl = idl_parser.IDLParser().ParseData(api) 583 return _JSCModel( 584 idl_schema.IDLSchema(idl).process()[0], 585 self._ref_resolver_factory.Create() if not disable_refs else None, 586 disable_refs, 587 self._availability_finder, 588 self._branch_utility, 589 self._parse_cache, 590 self._template_data_source, 591 self._LoadAddRulesSchema, 592 idl=True).ToDict() 593 594 def _GetIDLNames(self, base_dir, apis): 595 return self._GetExtNames(apis, ['idl']) 596 597 def _GetAllNames(self, base_dir, apis): 598 return self._GetExtNames(apis, ['json', 'idl']) 599 600 def _GetExtNames(self, apis, exts): 601 return [model.UnixName(os.path.splitext(api)[0]) for api in apis 602 if os.path.splitext(api)[1][1:] in exts] 603 604 def __init__(self, 605 json_cache, 606 idl_cache, 607 json_cache_no_refs, 608 idl_cache_no_refs, 609 names_cache, 610 idl_names_cache, 611 base_path, 612 samples, 613 disable_refs): 614 self._base_path = base_path 615 self._json_cache = json_cache 616 self._idl_cache = idl_cache 617 self._json_cache_no_refs = json_cache_no_refs 618 self._idl_cache_no_refs = idl_cache_no_refs 619 self._names_cache = names_cache 620 self._idl_names_cache = idl_names_cache 621 self._samples = samples 622 self._disable_refs = disable_refs 623 624 def _GenerateHandlebarContext(self, handlebar_dict): 625 handlebar_dict['samples'] = _LazySamplesGetter( 626 handlebar_dict['name'], 627 self._samples) 628 return handlebar_dict 629 630 def _GetAsSubdirectory(self, name): 631 if name.startswith('experimental_'): 632 parts = name[len('experimental_'):].split('_', 1) 633 if len(parts) > 1: 634 parts[1] = 'experimental_%s' % parts[1] 635 return '/'.join(parts) 636 return '%s/%s' % (parts[0], name) 637 return name.replace('_', '/', 1) 638 639 def get(self, key): 640 if key.endswith('.html') or key.endswith('.json') or key.endswith('.idl'): 641 path, ext = os.path.splitext(key) 642 else: 643 path = key 644 unix_name = model.UnixName(path) 645 idl_names = self._idl_names_cache.GetFromFileListing(self._base_path).Get() 646 names = self._names_cache.GetFromFileListing(self._base_path).Get() 647 if unix_name not in names and self._GetAsSubdirectory(unix_name) in names: 648 unix_name = self._GetAsSubdirectory(unix_name) 649 650 if self._disable_refs: 651 cache, ext = ( 652 (self._idl_cache_no_refs, '.idl') if (unix_name in idl_names) else 653 (self._json_cache_no_refs, '.json')) 654 else: 655 cache, ext = ((self._idl_cache, '.idl') if (unix_name in idl_names) else 656 (self._json_cache, '.json')) 657 return self._GenerateHandlebarContext( 658 cache.GetFromFile('%s/%s%s' % (self._base_path, unix_name, ext)).Get()) 659