api_data_source.py revision 558790d6acca3451cf3a6b497803a5f07d0bec58
1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import copy 6import logging 7import os 8from collections import defaultdict, Mapping 9 10from branch_utility import BranchUtility 11import svn_constants 12from third_party.handlebar import Handlebar 13import third_party.json_schema_compiler.json_parse as json_parse 14import third_party.json_schema_compiler.model as model 15import third_party.json_schema_compiler.idl_schema as idl_schema 16import third_party.json_schema_compiler.idl_parser as idl_parser 17 18def _RemoveNoDocs(item): 19 if json_parse.IsDict(item): 20 if item.get('nodoc', False): 21 return True 22 for key, value in item.items(): 23 if _RemoveNoDocs(value): 24 del item[key] 25 elif type(item) == list: 26 to_remove = [] 27 for i in item: 28 if _RemoveNoDocs(i): 29 to_remove.append(i) 30 for i in to_remove: 31 item.remove(i) 32 return False 33 34def _DetectInlineableTypes(schema): 35 """Look for documents that are only referenced once and mark them as inline. 36 Actual inlining is done by _InlineDocs. 37 """ 38 if not schema.get('types'): 39 return 40 41 ignore = frozenset(('value', 'choices')) 42 refcounts = defaultdict(int) 43 # Use an explicit stack instead of recursion. 44 stack = [schema] 45 46 while stack: 47 node = stack.pop() 48 if isinstance(node, list): 49 stack.extend(node) 50 elif isinstance(node, Mapping): 51 if '$ref' in node: 52 refcounts[node['$ref']] += 1 53 stack.extend(v for k, v in node.iteritems() if k not in ignore) 54 55 for type_ in schema['types']: 56 if not 'noinline_doc' in type_: 57 if refcounts[type_['id']] == 1: 58 type_['inline_doc'] = True 59 60def _InlineDocs(schema): 61 """Replace '$ref's that refer to inline_docs with the json for those docs. 62 """ 63 types = schema.get('types') 64 if types is None: 65 return 66 67 inline_docs = {} 68 types_without_inline_doc = [] 69 70 # Gather the types with inline_doc. 71 for type_ in types: 72 if type_.get('inline_doc'): 73 inline_docs[type_['id']] = type_ 74 for k in ('description', 'id', 'inline_doc'): 75 type_.pop(k, None) 76 else: 77 types_without_inline_doc.append(type_) 78 schema['types'] = types_without_inline_doc 79 80 def apply_inline(node): 81 if isinstance(node, list): 82 for i in node: 83 apply_inline(i) 84 elif isinstance(node, Mapping): 85 ref = node.get('$ref') 86 if ref and ref in inline_docs: 87 node.update(inline_docs[ref]) 88 del node['$ref'] 89 for k, v in node.iteritems(): 90 apply_inline(v) 91 92 apply_inline(schema) 93 94def _CreateId(node, prefix): 95 if node.parent is not None and not isinstance(node.parent, model.Namespace): 96 return '-'.join([prefix, node.parent.simple_name, node.simple_name]) 97 return '-'.join([prefix, node.simple_name]) 98 99def _FormatValue(value): 100 """Inserts commas every three digits for integer values. It is magic. 101 """ 102 s = str(value) 103 return ','.join([s[max(0, i - 3):i] for i in range(len(s), 0, -3)][::-1]) 104 105class _JSCModel(object): 106 """Uses a Model from the JSON Schema Compiler and generates a dict that 107 a Handlebar template can use for a data source. 108 """ 109 def __init__(self, 110 json, 111 ref_resolver, 112 disable_refs, 113 availability_finder, 114 parse_cache, 115 template_data_source, 116 idl=False): 117 self._ref_resolver = ref_resolver 118 self._disable_refs = disable_refs 119 self._availability_finder = availability_finder 120 self._intro_tables = parse_cache.GetFromFile( 121 '%s/intro_tables.json' % svn_constants.JSON_PATH) 122 self._api_features = parse_cache.GetFromFile( 123 '%s/_api_features.json' % svn_constants.API_PATH) 124 self._template_data_source = template_data_source 125 clean_json = copy.deepcopy(json) 126 if _RemoveNoDocs(clean_json): 127 self._namespace = None 128 else: 129 if idl: 130 _DetectInlineableTypes(clean_json) 131 _InlineDocs(clean_json) 132 self._namespace = model.Namespace(clean_json, clean_json['namespace']) 133 134 def _FormatDescription(self, description): 135 if self._disable_refs: 136 return description 137 return self._ref_resolver.ResolveAllLinks(description, 138 namespace=self._namespace.name) 139 140 def _GetLink(self, link): 141 if self._disable_refs: 142 type_name = link.split('.', 1)[-1] 143 return { 'href': '#type-%s' % type_name, 'text': link, 'name': link } 144 return self._ref_resolver.SafeGetLink(link, namespace=self._namespace.name) 145 146 def ToDict(self): 147 if self._namespace is None: 148 return {} 149 return { 150 'name': self._namespace.name, 151 'types': self._GenerateTypes(self._namespace.types.values()), 152 'functions': self._GenerateFunctions(self._namespace.functions), 153 'events': self._GenerateEvents(self._namespace.events), 154 'properties': self._GenerateProperties(self._namespace.properties), 155 'intro_list': self._GetIntroTableList(), 156 'channel_warning': self._GetChannelWarning() 157 } 158 159 def _GetIntroTableList(self): 160 """Create a generic data structure that can be traversed by the templates 161 to create an API intro table. 162 """ 163 intro_rows = [ 164 self._GetIntroDescriptionRow(), 165 self._GetIntroAvailabilityRow() 166 ] + self._GetIntroDependencyRows() 167 168 # Add rows using data from intro_tables.json, overriding any existing rows 169 # if they share the same 'title' attribute. 170 row_titles = [row['title'] for row in intro_rows] 171 for misc_row in self._GetMiscIntroRows(): 172 if misc_row['title'] in row_titles: 173 intro_rows[row_titles.index(misc_row['title'])] = misc_row 174 else: 175 intro_rows.append(misc_row) 176 177 return intro_rows 178 179 def _GetIntroDescriptionRow(self): 180 """ Generates the 'Description' row data for an API intro table. 181 """ 182 return { 183 'title': 'Description', 184 'content': [ 185 { 'text': self._FormatDescription(self._namespace.description) } 186 ] 187 } 188 189 def _GetIntroAvailabilityRow(self): 190 """ Generates the 'Availability' row data for an API intro table. 191 """ 192 if self._IsExperimental(): 193 status = 'experimental' 194 version = None 195 else: 196 availability = self._GetApiAvailability() 197 status = availability.channel 198 version = availability.version 199 return { 200 'title': 'Availability', 201 'content': [{ 202 'partial': self._template_data_source.get( 203 'intro_tables/%s_message.html' % status), 204 'version': version 205 }] 206 } 207 208 def _GetIntroDependencyRows(self): 209 # Devtools aren't in _api_features. If we're dealing with devtools, bail. 210 if 'devtools' in self._namespace.name: 211 return [] 212 feature = self._api_features.get(self._namespace.name) 213 assert feature, ('"%s" not found in _api_features.json.' 214 % self._namespace.name) 215 216 dependencies = feature.get('dependencies') 217 if dependencies is None: 218 return [] 219 220 def make_code_node(text): 221 return { 'class': 'code', 'text': text } 222 223 permissions_content = [] 224 manifest_content = [] 225 226 for dependency in dependencies: 227 context, name = dependency.split(':', 1) 228 if context == 'permission': 229 permissions_content.append(make_code_node('"%s"' % name)) 230 elif context == 'manifest': 231 manifest_content.append(make_code_node('"%s": {...}' % name)) 232 else: 233 raise ValueError('Unrecognized dependency for %s: %s' 234 % (self._namespace.name, context)) 235 236 dependency_rows = [] 237 if permissions_content: 238 dependency_rows.append({ 239 'title': 'Permissions', 240 'content': permissions_content 241 }) 242 if manifest_content: 243 dependency_rows.append({ 244 'title': 'Manifest', 245 'content': manifest_content 246 }) 247 return dependency_rows 248 249 def _GetMiscIntroRows(self): 250 """ Generates miscellaneous intro table row data, such as 'Permissions', 251 'Samples', and 'Learn More', using intro_tables.json. 252 """ 253 misc_rows = [] 254 # Look up the API name in intro_tables.json, which is structured 255 # similarly to the data structure being created. If the name is found, loop 256 # through the attributes and add them to this structure. 257 table_info = self._intro_tables.get(self._namespace.name) 258 if table_info is None: 259 return misc_rows 260 261 for category in table_info.keys(): 262 content = copy.deepcopy(table_info[category]) 263 for node in content: 264 # If there is a 'partial' argument and it hasn't already been 265 # converted to a Handlebar object, transform it to a template. 266 if 'partial' in node: 267 node['partial'] = self._template_data_source.get(node['partial']) 268 misc_rows.append({ 'title': category, 'content': content }) 269 return misc_rows 270 271 def _GetApiAvailability(self): 272 return self._availability_finder.GetApiAvailability(self._namespace.name) 273 274 def _GetChannelWarning(self): 275 if not self._IsExperimental(): 276 return { self._GetApiAvailability().channel: True } 277 return None 278 279 def _IsExperimental(self): 280 return self._namespace.name.startswith('experimental') 281 282 def _GenerateTypes(self, types): 283 return [self._GenerateType(t) for t in types] 284 285 def _GenerateType(self, type_): 286 type_dict = { 287 'name': type_.simple_name, 288 'description': self._FormatDescription(type_.description), 289 'properties': self._GenerateProperties(type_.properties), 290 'functions': self._GenerateFunctions(type_.functions), 291 'events': self._GenerateEvents(type_.events), 292 'id': _CreateId(type_, 'type') 293 } 294 self._RenderTypeInformation(type_, type_dict) 295 return type_dict 296 297 def _GenerateFunctions(self, functions): 298 return [self._GenerateFunction(f) for f in functions.values()] 299 300 def _GenerateFunction(self, function): 301 function_dict = { 302 'name': function.simple_name, 303 'description': self._FormatDescription(function.description), 304 'callback': self._GenerateCallback(function.callback), 305 'parameters': [], 306 'returns': None, 307 'id': _CreateId(function, 'method') 308 } 309 if (function.parent is not None and 310 not isinstance(function.parent, model.Namespace)): 311 function_dict['parent_name'] = function.parent.simple_name 312 if function.returns: 313 function_dict['returns'] = self._GenerateType(function.returns) 314 for param in function.params: 315 function_dict['parameters'].append(self._GenerateProperty(param)) 316 if function.callback is not None: 317 # Show the callback as an extra parameter. 318 function_dict['parameters'].append( 319 self._GenerateCallbackProperty(function.callback)) 320 if len(function_dict['parameters']) > 0: 321 function_dict['parameters'][-1]['last'] = True 322 return function_dict 323 324 def _GenerateEvents(self, events): 325 return [self._GenerateEvent(e) for e in events.values()] 326 327 def _GenerateEvent(self, event): 328 event_dict = { 329 'name': event.simple_name, 330 'description': self._FormatDescription(event.description), 331 'parameters': [self._GenerateProperty(p) for p in event.params], 332 'callback': self._GenerateCallback(event.callback), 333 'filters': [self._GenerateProperty(f) for f in event.filters], 334 'conditions': [self._GetLink(condition) 335 for condition in event.conditions], 336 'actions': [self._GetLink(action) for action in event.actions], 337 'supportsRules': event.supports_rules, 338 'id': _CreateId(event, 'event') 339 } 340 if (event.parent is not None and 341 not isinstance(event.parent, model.Namespace)): 342 event_dict['parent_name'] = event.parent.simple_name 343 if event.callback is not None: 344 # Show the callback as an extra parameter. 345 event_dict['parameters'].append( 346 self._GenerateCallbackProperty(event.callback)) 347 if len(event_dict['parameters']) > 0: 348 event_dict['parameters'][-1]['last'] = True 349 return event_dict 350 351 def _GenerateCallback(self, callback): 352 if not callback: 353 return None 354 callback_dict = { 355 'name': callback.simple_name, 356 'simple_type': {'simple_type': 'function'}, 357 'optional': callback.optional, 358 'parameters': [] 359 } 360 for param in callback.params: 361 callback_dict['parameters'].append(self._GenerateProperty(param)) 362 if (len(callback_dict['parameters']) > 0): 363 callback_dict['parameters'][-1]['last'] = True 364 return callback_dict 365 366 def _GenerateProperties(self, properties): 367 return [self._GenerateProperty(v) for v in properties.values()] 368 369 def _GenerateProperty(self, property_): 370 if not hasattr(property_, 'type_'): 371 for d in dir(property_): 372 if not d.startswith('_'): 373 print ('%s -> %s' % (d, getattr(property_, d))) 374 type_ = property_.type_ 375 376 # Make sure we generate property info for arrays, too. 377 # TODO(kalman): what about choices? 378 if type_.property_type == model.PropertyType.ARRAY: 379 properties = type_.item_type.properties 380 else: 381 properties = type_.properties 382 383 property_dict = { 384 'name': property_.simple_name, 385 'optional': property_.optional, 386 'description': self._FormatDescription(property_.description), 387 'properties': self._GenerateProperties(type_.properties), 388 'functions': self._GenerateFunctions(type_.functions), 389 'parameters': [], 390 'returns': None, 391 'id': _CreateId(property_, 'property') 392 } 393 394 if type_.property_type == model.PropertyType.FUNCTION: 395 function = type_.function 396 for param in function.params: 397 property_dict['parameters'].append(self._GenerateProperty(param)) 398 if function.returns: 399 property_dict['returns'] = self._GenerateType(function.returns) 400 401 if (property_.parent is not None and 402 not isinstance(property_.parent, model.Namespace)): 403 property_dict['parent_name'] = property_.parent.simple_name 404 405 value = property_.value 406 if value is not None: 407 if isinstance(value, int): 408 property_dict['value'] = _FormatValue(value) 409 else: 410 property_dict['value'] = value 411 else: 412 self._RenderTypeInformation(type_, property_dict) 413 414 return property_dict 415 416 def _GenerateCallbackProperty(self, callback): 417 property_dict = { 418 'name': callback.simple_name, 419 'description': self._FormatDescription(callback.description), 420 'optional': callback.optional, 421 'id': _CreateId(callback, 'property'), 422 'simple_type': 'function', 423 } 424 if (callback.parent is not None and 425 not isinstance(callback.parent, model.Namespace)): 426 property_dict['parent_name'] = callback.parent.simple_name 427 return property_dict 428 429 def _RenderTypeInformation(self, type_, dst_dict): 430 dst_dict['is_object'] = type_.property_type == model.PropertyType.OBJECT 431 if type_.property_type == model.PropertyType.CHOICES: 432 dst_dict['choices'] = self._GenerateTypes(type_.choices) 433 # We keep track of which == last for knowing when to add "or" between 434 # choices in templates. 435 if len(dst_dict['choices']) > 0: 436 dst_dict['choices'][-1]['last'] = True 437 elif type_.property_type == model.PropertyType.REF: 438 dst_dict['link'] = self._GetLink(type_.ref_type) 439 elif type_.property_type == model.PropertyType.ARRAY: 440 dst_dict['array'] = self._GenerateType(type_.item_type) 441 elif type_.property_type == model.PropertyType.ENUM: 442 dst_dict['enum_values'] = [] 443 for enum_value in type_.enum_values: 444 dst_dict['enum_values'].append({'name': enum_value}) 445 if len(dst_dict['enum_values']) > 0: 446 dst_dict['enum_values'][-1]['last'] = True 447 elif type_.instance_of is not None: 448 dst_dict['simple_type'] = type_.instance_of.lower() 449 else: 450 dst_dict['simple_type'] = type_.property_type.name.lower() 451 452class _LazySamplesGetter(object): 453 """This class is needed so that an extensions API page does not have to fetch 454 the apps samples page and vice versa. 455 """ 456 def __init__(self, api_name, samples): 457 self._api_name = api_name 458 self._samples = samples 459 460 def get(self, key): 461 return self._samples.FilterSamples(key, self._api_name) 462 463class APIDataSource(object): 464 """This class fetches and loads JSON APIs from the FileSystem passed in with 465 |compiled_fs_factory|, so the APIs can be plugged into templates. 466 """ 467 class Factory(object): 468 def __init__(self, 469 compiled_fs_factory, 470 base_path, 471 availability_finder_factory): 472 def create_compiled_fs(fn, category): 473 return compiled_fs_factory.Create(fn, APIDataSource, category=category) 474 475 self._json_cache = create_compiled_fs( 476 lambda api_name, api: self._LoadJsonAPI(api, False), 477 'json') 478 self._idl_cache = create_compiled_fs( 479 lambda api_name, api: self._LoadIdlAPI(api, False), 480 'idl') 481 482 # These caches are used if an APIDataSource does not want to resolve the 483 # $refs in an API. This is needed to prevent infinite recursion in 484 # ReferenceResolver. 485 self._json_cache_no_refs = create_compiled_fs( 486 lambda api_name, api: self._LoadJsonAPI(api, True), 487 'json-no-refs') 488 self._idl_cache_no_refs = create_compiled_fs( 489 lambda api_name, api: self._LoadIdlAPI(api, True), 490 'idl-no-refs') 491 492 self._idl_names_cache = create_compiled_fs(self._GetIDLNames, 'idl-names') 493 self._names_cache = create_compiled_fs(self._GetAllNames, 'names') 494 495 self._base_path = base_path 496 self._availability_finder = availability_finder_factory.Create() 497 self._parse_cache = create_compiled_fs( 498 lambda _, json: json_parse.Parse(json), 499 'intro-cache') 500 # These must be set later via the SetFooDataSourceFactory methods. 501 self._ref_resolver_factory = None 502 self._samples_data_source_factory = None 503 504 def SetSamplesDataSourceFactory(self, samples_data_source_factory): 505 self._samples_data_source_factory = samples_data_source_factory 506 507 def SetReferenceResolverFactory(self, ref_resolver_factory): 508 self._ref_resolver_factory = ref_resolver_factory 509 510 def SetTemplateDataSource(self, template_data_source_factory): 511 # This TemplateDataSource is only being used for fetching template data. 512 self._template_data_source = template_data_source_factory.Create(None, '') 513 514 def Create(self, request, disable_refs=False): 515 """Create an APIDataSource. |disable_refs| specifies whether $ref's in 516 APIs being processed by the |ToDict| method of _JSCModel follows $ref's 517 in the API. This prevents endless recursion in ReferenceResolver. 518 """ 519 if self._samples_data_source_factory is None: 520 # Only error if there is a request, which means this APIDataSource is 521 # actually being used to render a page. 522 if request is not None: 523 logging.error('SamplesDataSource.Factory was never set in ' 524 'APIDataSource.Factory.') 525 samples = None 526 else: 527 samples = self._samples_data_source_factory.Create(request) 528 if not disable_refs and self._ref_resolver_factory is None: 529 logging.error('ReferenceResolver.Factory was never set in ' 530 'APIDataSource.Factory.') 531 return APIDataSource(self._json_cache, 532 self._idl_cache, 533 self._json_cache_no_refs, 534 self._idl_cache_no_refs, 535 self._names_cache, 536 self._idl_names_cache, 537 self._base_path, 538 samples, 539 disable_refs) 540 541 def _LoadJsonAPI(self, api, disable_refs): 542 return _JSCModel( 543 json_parse.Parse(api)[0], 544 self._ref_resolver_factory.Create() if not disable_refs else None, 545 disable_refs, 546 self._availability_finder, 547 self._parse_cache, 548 self._template_data_source).ToDict() 549 550 def _LoadIdlAPI(self, api, disable_refs): 551 idl = idl_parser.IDLParser().ParseData(api) 552 return _JSCModel( 553 idl_schema.IDLSchema(idl).process()[0], 554 self._ref_resolver_factory.Create() if not disable_refs else None, 555 disable_refs, 556 self._availability_finder, 557 self._parse_cache, 558 self._template_data_source, 559 idl=True).ToDict() 560 561 def _GetIDLNames(self, base_dir, apis): 562 return self._GetExtNames(apis, ['idl']) 563 564 def _GetAllNames(self, base_dir, apis): 565 return self._GetExtNames(apis, ['json', 'idl']) 566 567 def _GetExtNames(self, apis, exts): 568 return [model.UnixName(os.path.splitext(api)[0]) for api in apis 569 if os.path.splitext(api)[1][1:] in exts] 570 571 def __init__(self, 572 json_cache, 573 idl_cache, 574 json_cache_no_refs, 575 idl_cache_no_refs, 576 names_cache, 577 idl_names_cache, 578 base_path, 579 samples, 580 disable_refs): 581 self._base_path = base_path 582 self._json_cache = json_cache 583 self._idl_cache = idl_cache 584 self._json_cache_no_refs = json_cache_no_refs 585 self._idl_cache_no_refs = idl_cache_no_refs 586 self._names_cache = names_cache 587 self._idl_names_cache = idl_names_cache 588 self._samples = samples 589 self._disable_refs = disable_refs 590 591 def _GenerateHandlebarContext(self, handlebar_dict, path): 592 handlebar_dict['samples'] = _LazySamplesGetter(path, self._samples) 593 return handlebar_dict 594 595 def _GetAsSubdirectory(self, name): 596 if name.startswith('experimental_'): 597 parts = name[len('experimental_'):].split('_', 1) 598 if len(parts) > 1: 599 parts[1] = 'experimental_%s' % parts[1] 600 return '/'.join(parts) 601 return '%s/%s' % (parts[0], name) 602 return name.replace('_', '/', 1) 603 604 def get(self, key): 605 if key.endswith('.html') or key.endswith('.json') or key.endswith('.idl'): 606 path, ext = os.path.splitext(key) 607 else: 608 path = key 609 unix_name = model.UnixName(path) 610 idl_names = self._idl_names_cache.GetFromFileListing(self._base_path) 611 names = self._names_cache.GetFromFileListing(self._base_path) 612 if unix_name not in names and self._GetAsSubdirectory(unix_name) in names: 613 unix_name = self._GetAsSubdirectory(unix_name) 614 615 if self._disable_refs: 616 cache, ext = ( 617 (self._idl_cache_no_refs, '.idl') if (unix_name in idl_names) else 618 (self._json_cache_no_refs, '.json')) 619 else: 620 cache, ext = ((self._idl_cache, '.idl') if (unix_name in idl_names) else 621 (self._json_cache, '.json')) 622 return self._GenerateHandlebarContext( 623 cache.GetFromFile('%s/%s%s' % (self._base_path, unix_name, ext)), 624 path) 625