api_data_source.py revision ca12bfac764ba476d6cd062bf1dde12cc64c3f40
1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import copy 6import logging 7import os 8from collections import defaultdict, Mapping 9 10from branch_utility import BranchUtility 11import svn_constants 12from third_party.handlebar import Handlebar 13import third_party.json_schema_compiler.json_parse as json_parse 14import third_party.json_schema_compiler.model as model 15import third_party.json_schema_compiler.idl_schema as idl_schema 16import third_party.json_schema_compiler.idl_parser as idl_parser 17 18def _RemoveNoDocs(item): 19 if json_parse.IsDict(item): 20 if item.get('nodoc', False): 21 return True 22 for key, value in item.items(): 23 if _RemoveNoDocs(value): 24 del item[key] 25 elif type(item) == list: 26 to_remove = [] 27 for i in item: 28 if _RemoveNoDocs(i): 29 to_remove.append(i) 30 for i in to_remove: 31 item.remove(i) 32 return False 33 34def _DetectInlineableTypes(schema): 35 """Look for documents that are only referenced once and mark them as inline. 36 Actual inlining is done by _InlineDocs. 37 """ 38 if not schema.get('types'): 39 return 40 41 ignore = frozenset(('value', 'choices')) 42 refcounts = defaultdict(int) 43 # Use an explicit stack instead of recursion. 44 stack = [schema] 45 46 while stack: 47 node = stack.pop() 48 if isinstance(node, list): 49 stack.extend(node) 50 elif isinstance(node, Mapping): 51 if '$ref' in node: 52 refcounts[node['$ref']] += 1 53 stack.extend(v for k, v in node.iteritems() if k not in ignore) 54 55 for type_ in schema['types']: 56 if not 'noinline_doc' in type_: 57 if refcounts[type_['id']] == 1: 58 type_['inline_doc'] = True 59 60def _InlineDocs(schema): 61 """Replace '$ref's that refer to inline_docs with the json for those docs. 62 """ 63 types = schema.get('types') 64 if types is None: 65 return 66 67 inline_docs = {} 68 types_without_inline_doc = [] 69 70 # Gather the types with inline_doc. 71 for type_ in types: 72 if type_.get('inline_doc'): 73 inline_docs[type_['id']] = type_ 74 for k in ('description', 'id', 'inline_doc'): 75 type_.pop(k, None) 76 else: 77 types_without_inline_doc.append(type_) 78 schema['types'] = types_without_inline_doc 79 80 def apply_inline(node): 81 if isinstance(node, list): 82 for i in node: 83 apply_inline(i) 84 elif isinstance(node, Mapping): 85 ref = node.get('$ref') 86 if ref and ref in inline_docs: 87 node.update(inline_docs[ref]) 88 del node['$ref'] 89 for k, v in node.iteritems(): 90 apply_inline(v) 91 92 apply_inline(schema) 93 94def _CreateId(node, prefix): 95 if node.parent is not None and not isinstance(node.parent, model.Namespace): 96 return '-'.join([prefix, node.parent.simple_name, node.simple_name]) 97 return '-'.join([prefix, node.simple_name]) 98 99def _FormatValue(value): 100 """Inserts commas every three digits for integer values. It is magic. 101 """ 102 s = str(value) 103 return ','.join([s[max(0, i - 3):i] for i in range(len(s), 0, -3)][::-1]) 104 105class _JSCModel(object): 106 """Uses a Model from the JSON Schema Compiler and generates a dict that 107 a Handlebar template can use for a data source. 108 """ 109 def __init__(self, 110 json, 111 ref_resolver, 112 disable_refs, 113 availability_finder, 114 intro_cache, 115 template_data_source, 116 idl=False): 117 self._ref_resolver = ref_resolver 118 self._disable_refs = disable_refs 119 self._availability_finder = availability_finder 120 self._intro_tables = intro_cache.GetFromFile( 121 '%s/intro_tables.json' % svn_constants.JSON_PATH) 122 self._template_data_source = template_data_source 123 clean_json = copy.deepcopy(json) 124 if _RemoveNoDocs(clean_json): 125 self._namespace = None 126 else: 127 if idl: 128 _DetectInlineableTypes(clean_json) 129 _InlineDocs(clean_json) 130 self._namespace = model.Namespace(clean_json, clean_json['namespace']) 131 132 def _FormatDescription(self, description): 133 if self._disable_refs: 134 return description 135 return self._ref_resolver.ResolveAllLinks(description, 136 namespace=self._namespace.name) 137 138 def _GetLink(self, link): 139 if self._disable_refs: 140 type_name = link.split('.', 1)[-1] 141 return { 'href': '#type-%s' % type_name, 'text': link, 'name': link } 142 return self._ref_resolver.SafeGetLink(link, namespace=self._namespace.name) 143 144 def ToDict(self): 145 if self._namespace is None: 146 return {} 147 return { 148 'name': self._namespace.name, 149 'types': self._GenerateTypes(self._namespace.types.values()), 150 'functions': self._GenerateFunctions(self._namespace.functions), 151 'events': self._GenerateEvents(self._namespace.events), 152 'properties': self._GenerateProperties(self._namespace.properties), 153 'intro_list': self._GetIntroTableList(), 154 'channel_warning': self._GetChannelWarning() 155 } 156 157 def _GetIntroTableList(self): 158 """Create a generic data structure that can be traversed by the templates 159 to create an API intro table. 160 """ 161 intro_list = [{ 162 'title': 'Description', 163 'content': [ 164 { 'text': self._FormatDescription(self._namespace.description) } 165 ] 166 }] 167 168 if self._IsExperimental(): 169 status = 'experimental' 170 version = None 171 else: 172 availability = self._GetApiAvailability() 173 status = availability.channel 174 version = availability.version 175 intro_list.append({ 176 'title': 'Availability', 177 'content': [ 178 { 179 'partial': self._template_data_source.get( 180 'intro_tables/%s_message.html' % status), 181 'version': version 182 } 183 ] 184 }) 185 186 # Look up the API name in intro_tables.json, which is structured similarly 187 # to the data structure being created. If the name is found, loop through 188 # the attributes and add them to this structure. 189 table_info = self._intro_tables.get(self._namespace.name) 190 if table_info is None: 191 return intro_list 192 193 # The intro tables have a specific ordering that needs to be followed. 194 ordering = ('Permissions', 'Samples', 'Learn More') 195 196 for category in ordering: 197 if category not in table_info.keys(): 198 continue 199 # Transform the 'partial' argument from the partial name to the 200 # template itself. 201 content = table_info[category] 202 for node in content: 203 # If there is a 'partial' argument and it hasn't already been 204 # converted to a Handlebar object, transform it to a template. 205 # TODO(epeterson/kalman): figure out why this check is necessary 206 # since it should be caching. 207 if 'partial' in node and not isinstance(node['partial'], Handlebar): 208 node['partial'] = self._template_data_source.get(node['partial']) 209 intro_list.append({ 210 'title': category, 211 'content': content 212 }) 213 return intro_list 214 215 def _GetApiAvailability(self): 216 return self._availability_finder.GetApiAvailability(self._namespace.name) 217 218 def _GetChannelWarning(self): 219 if not self._IsExperimental(): 220 return { self._GetApiAvailability().channel: True } 221 return None 222 223 def _IsExperimental(self): 224 return self._namespace.name.startswith('experimental') 225 226 def _GenerateTypes(self, types): 227 return [self._GenerateType(t) for t in types] 228 229 def _GenerateType(self, type_): 230 type_dict = { 231 'name': type_.simple_name, 232 'description': self._FormatDescription(type_.description), 233 'properties': self._GenerateProperties(type_.properties), 234 'functions': self._GenerateFunctions(type_.functions), 235 'events': self._GenerateEvents(type_.events), 236 'id': _CreateId(type_, 'type') 237 } 238 self._RenderTypeInformation(type_, type_dict) 239 return type_dict 240 241 def _GenerateFunctions(self, functions): 242 return [self._GenerateFunction(f) for f in functions.values()] 243 244 def _GenerateFunction(self, function): 245 function_dict = { 246 'name': function.simple_name, 247 'description': self._FormatDescription(function.description), 248 'callback': self._GenerateCallback(function.callback), 249 'parameters': [], 250 'returns': None, 251 'id': _CreateId(function, 'method') 252 } 253 if (function.parent is not None and 254 not isinstance(function.parent, model.Namespace)): 255 function_dict['parent_name'] = function.parent.simple_name 256 if function.returns: 257 function_dict['returns'] = self._GenerateType(function.returns) 258 for param in function.params: 259 function_dict['parameters'].append(self._GenerateProperty(param)) 260 if function.callback is not None: 261 # Show the callback as an extra parameter. 262 function_dict['parameters'].append( 263 self._GenerateCallbackProperty(function.callback)) 264 if len(function_dict['parameters']) > 0: 265 function_dict['parameters'][-1]['last'] = True 266 return function_dict 267 268 def _GenerateEvents(self, events): 269 return [self._GenerateEvent(e) for e in events.values()] 270 271 def _GenerateEvent(self, event): 272 event_dict = { 273 'name': event.simple_name, 274 'description': self._FormatDescription(event.description), 275 'parameters': [self._GenerateProperty(p) for p in event.params], 276 'callback': self._GenerateCallback(event.callback), 277 'filters': [self._GenerateProperty(f) for f in event.filters], 278 'conditions': [self._GetLink(condition) 279 for condition in event.conditions], 280 'actions': [self._GetLink(action) for action in event.actions], 281 'supportsRules': event.supports_rules, 282 'id': _CreateId(event, 'event') 283 } 284 if (event.parent is not None and 285 not isinstance(event.parent, model.Namespace)): 286 event_dict['parent_name'] = event.parent.simple_name 287 if event.callback is not None: 288 # Show the callback as an extra parameter. 289 event_dict['parameters'].append( 290 self._GenerateCallbackProperty(event.callback)) 291 if len(event_dict['parameters']) > 0: 292 event_dict['parameters'][-1]['last'] = True 293 return event_dict 294 295 def _GenerateCallback(self, callback): 296 if not callback: 297 return None 298 callback_dict = { 299 'name': callback.simple_name, 300 'simple_type': {'simple_type': 'function'}, 301 'optional': callback.optional, 302 'parameters': [] 303 } 304 for param in callback.params: 305 callback_dict['parameters'].append(self._GenerateProperty(param)) 306 if (len(callback_dict['parameters']) > 0): 307 callback_dict['parameters'][-1]['last'] = True 308 return callback_dict 309 310 def _GenerateProperties(self, properties): 311 return [self._GenerateProperty(v) for v in properties.values()] 312 313 def _GenerateProperty(self, property_): 314 if not hasattr(property_, 'type_'): 315 for d in dir(property_): 316 if not d.startswith('_'): 317 print ('%s -> %s' % (d, getattr(property_, d))) 318 type_ = property_.type_ 319 320 # Make sure we generate property info for arrays, too. 321 # TODO(kalman): what about choices? 322 if type_.property_type == model.PropertyType.ARRAY: 323 properties = type_.item_type.properties 324 else: 325 properties = type_.properties 326 327 property_dict = { 328 'name': property_.simple_name, 329 'optional': property_.optional, 330 'description': self._FormatDescription(property_.description), 331 'properties': self._GenerateProperties(type_.properties), 332 'functions': self._GenerateFunctions(type_.functions), 333 'parameters': [], 334 'returns': None, 335 'id': _CreateId(property_, 'property') 336 } 337 338 if type_.property_type == model.PropertyType.FUNCTION: 339 function = type_.function 340 for param in function.params: 341 property_dict['parameters'].append(self._GenerateProperty(param)) 342 if function.returns: 343 property_dict['returns'] = self._GenerateType(function.returns) 344 345 if (property_.parent is not None and 346 not isinstance(property_.parent, model.Namespace)): 347 property_dict['parent_name'] = property_.parent.simple_name 348 349 value = property_.value 350 if value is not None: 351 if isinstance(value, int): 352 property_dict['value'] = _FormatValue(value) 353 else: 354 property_dict['value'] = value 355 else: 356 self._RenderTypeInformation(type_, property_dict) 357 358 return property_dict 359 360 def _GenerateCallbackProperty(self, callback): 361 property_dict = { 362 'name': callback.simple_name, 363 'description': self._FormatDescription(callback.description), 364 'optional': callback.optional, 365 'id': _CreateId(callback, 'property'), 366 'simple_type': 'function', 367 } 368 if (callback.parent is not None and 369 not isinstance(callback.parent, model.Namespace)): 370 property_dict['parent_name'] = callback.parent.simple_name 371 return property_dict 372 373 def _RenderTypeInformation(self, type_, dst_dict): 374 dst_dict['is_object'] = type_.property_type == model.PropertyType.OBJECT 375 if type_.property_type == model.PropertyType.CHOICES: 376 dst_dict['choices'] = self._GenerateTypes(type_.choices) 377 # We keep track of which == last for knowing when to add "or" between 378 # choices in templates. 379 if len(dst_dict['choices']) > 0: 380 dst_dict['choices'][-1]['last'] = True 381 elif type_.property_type == model.PropertyType.REF: 382 dst_dict['link'] = self._GetLink(type_.ref_type) 383 elif type_.property_type == model.PropertyType.ARRAY: 384 dst_dict['array'] = self._GenerateType(type_.item_type) 385 elif type_.property_type == model.PropertyType.ENUM: 386 dst_dict['enum_values'] = [] 387 for enum_value in type_.enum_values: 388 dst_dict['enum_values'].append({'name': enum_value}) 389 if len(dst_dict['enum_values']) > 0: 390 dst_dict['enum_values'][-1]['last'] = True 391 elif type_.instance_of is not None: 392 dst_dict['simple_type'] = type_.instance_of.lower() 393 else: 394 dst_dict['simple_type'] = type_.property_type.name.lower() 395 396class _LazySamplesGetter(object): 397 """This class is needed so that an extensions API page does not have to fetch 398 the apps samples page and vice versa. 399 """ 400 def __init__(self, api_name, samples): 401 self._api_name = api_name 402 self._samples = samples 403 404 def get(self, key): 405 return self._samples.FilterSamples(key, self._api_name) 406 407class APIDataSource(object): 408 """This class fetches and loads JSON APIs from the FileSystem passed in with 409 |compiled_fs_factory|, so the APIs can be plugged into templates. 410 """ 411 class Factory(object): 412 def __init__(self, 413 compiled_fs_factory, 414 base_path, 415 availability_finder_factory): 416 def create_compiled_fs(fn, category): 417 return compiled_fs_factory.Create(fn, APIDataSource, category=category) 418 419 self._json_cache = create_compiled_fs( 420 lambda api_name, api: self._LoadJsonAPI(api, False), 421 'json') 422 self._idl_cache = create_compiled_fs( 423 lambda api_name, api: self._LoadIdlAPI(api, False), 424 'idl') 425 426 # These caches are used if an APIDataSource does not want to resolve the 427 # $refs in an API. This is needed to prevent infinite recursion in 428 # ReferenceResolver. 429 self._json_cache_no_refs = create_compiled_fs( 430 lambda api_name, api: self._LoadJsonAPI(api, True), 431 'json-no-refs') 432 self._idl_cache_no_refs = create_compiled_fs( 433 lambda api_name, api: self._LoadIdlAPI(api, True), 434 'idl-no-refs') 435 436 self._idl_names_cache = create_compiled_fs(self._GetIDLNames, 'idl-names') 437 self._names_cache = create_compiled_fs(self._GetAllNames, 'names') 438 439 self._base_path = base_path 440 self._availability_finder = availability_finder_factory.Create() 441 self._intro_cache = create_compiled_fs( 442 lambda _, json: json_parse.Parse(json), 443 'intro-cache') 444 # These must be set later via the SetFooDataSourceFactory methods. 445 self._ref_resolver_factory = None 446 self._samples_data_source_factory = None 447 448 def SetSamplesDataSourceFactory(self, samples_data_source_factory): 449 self._samples_data_source_factory = samples_data_source_factory 450 451 def SetReferenceResolverFactory(self, ref_resolver_factory): 452 self._ref_resolver_factory = ref_resolver_factory 453 454 def SetTemplateDataSource(self, template_data_source_factory): 455 # This TemplateDataSource is only being used for fetching template data. 456 self._template_data_source = template_data_source_factory.Create(None, '') 457 458 def Create(self, request, disable_refs=False): 459 """Create an APIDataSource. |disable_refs| specifies whether $ref's in 460 APIs being processed by the |ToDict| method of _JSCModel follows $ref's 461 in the API. This prevents endless recursion in ReferenceResolver. 462 """ 463 if self._samples_data_source_factory is None: 464 # Only error if there is a request, which means this APIDataSource is 465 # actually being used to render a page. 466 if request is not None: 467 logging.error('SamplesDataSource.Factory was never set in ' 468 'APIDataSource.Factory.') 469 samples = None 470 else: 471 samples = self._samples_data_source_factory.Create(request) 472 if not disable_refs and self._ref_resolver_factory is None: 473 logging.error('ReferenceResolver.Factory was never set in ' 474 'APIDataSource.Factory.') 475 return APIDataSource(self._json_cache, 476 self._idl_cache, 477 self._json_cache_no_refs, 478 self._idl_cache_no_refs, 479 self._names_cache, 480 self._idl_names_cache, 481 self._base_path, 482 samples, 483 disable_refs) 484 485 def _LoadJsonAPI(self, api, disable_refs): 486 return _JSCModel( 487 json_parse.Parse(api)[0], 488 self._ref_resolver_factory.Create() if not disable_refs else None, 489 disable_refs, 490 self._availability_finder, 491 self._intro_cache, 492 self._template_data_source).ToDict() 493 494 def _LoadIdlAPI(self, api, disable_refs): 495 idl = idl_parser.IDLParser().ParseData(api) 496 return _JSCModel( 497 idl_schema.IDLSchema(idl).process()[0], 498 self._ref_resolver_factory.Create() if not disable_refs else None, 499 disable_refs, 500 self._availability_finder, 501 self._intro_cache, 502 self._template_data_source, 503 idl=True).ToDict() 504 505 def _GetIDLNames(self, base_dir, apis): 506 return self._GetExtNames(apis, ['idl']) 507 508 def _GetAllNames(self, base_dir, apis): 509 return self._GetExtNames(apis, ['json', 'idl']) 510 511 def _GetExtNames(self, apis, exts): 512 return [model.UnixName(os.path.splitext(api)[0]) for api in apis 513 if os.path.splitext(api)[1][1:] in exts] 514 515 def __init__(self, 516 json_cache, 517 idl_cache, 518 json_cache_no_refs, 519 idl_cache_no_refs, 520 names_cache, 521 idl_names_cache, 522 base_path, 523 samples, 524 disable_refs): 525 self._base_path = base_path 526 self._json_cache = json_cache 527 self._idl_cache = idl_cache 528 self._json_cache_no_refs = json_cache_no_refs 529 self._idl_cache_no_refs = idl_cache_no_refs 530 self._names_cache = names_cache 531 self._idl_names_cache = idl_names_cache 532 self._samples = samples 533 self._disable_refs = disable_refs 534 535 def _GenerateHandlebarContext(self, handlebar_dict, path): 536 handlebar_dict['samples'] = _LazySamplesGetter(path, self._samples) 537 return handlebar_dict 538 539 def _GetAsSubdirectory(self, name): 540 if name.startswith('experimental_'): 541 parts = name[len('experimental_'):].split('_', 1) 542 if len(parts) > 1: 543 parts[1] = 'experimental_%s' % parts[1] 544 return '/'.join(parts) 545 return '%s/%s' % (parts[0], name) 546 return name.replace('_', '/', 1) 547 548 def get(self, key): 549 if key.endswith('.html') or key.endswith('.json') or key.endswith('.idl'): 550 path, ext = os.path.splitext(key) 551 else: 552 path = key 553 unix_name = model.UnixName(path) 554 idl_names = self._idl_names_cache.GetFromFileListing(self._base_path) 555 names = self._names_cache.GetFromFileListing(self._base_path) 556 if unix_name not in names and self._GetAsSubdirectory(unix_name) in names: 557 unix_name = self._GetAsSubdirectory(unix_name) 558 559 if self._disable_refs: 560 cache, ext = ( 561 (self._idl_cache_no_refs, '.idl') if (unix_name in idl_names) else 562 (self._json_cache_no_refs, '.json')) 563 else: 564 cache, ext = ((self._idl_cache, '.idl') if (unix_name in idl_names) else 565 (self._json_cache, '.json')) 566 return self._GenerateHandlebarContext( 567 cache.GetFromFile('%s/%s%s' % (self._base_path, unix_name, ext)), 568 path) 569