1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import posixpath
6
7from api_models import GetNodeCategories
8from api_schema_graph import APISchemaGraph
9from branch_utility import BranchUtility, ChannelInfo
10from compiled_file_system import CompiledFileSystem, SingleFile, Unicode
11from extensions_paths import API_PATHS, JSON_TEMPLATES
12from features_bundle import FeaturesBundle
13from file_system import FileNotFoundError
14from schema_processor import SchemaProcessor
15from third_party.json_schema_compiler.memoize import memoize
16from third_party.json_schema_compiler.model import UnixName
17
18
19_DEVTOOLS_API = 'devtools_api.json'
20_EXTENSION_API = 'extension_api.json'
21# The version where api_features.json is first available.
22_API_FEATURES_MIN_VERSION = 28
23# The version where permission_ and manifest_features.json are available and
24# presented in the current format.
25_ORIGINAL_FEATURES_MIN_VERSION = 20
26# API schemas are aggregated in extension_api.json up to this version.
27_EXTENSION_API_MAX_VERSION = 17
28# The earliest version for which we have SVN data.
29_SVN_MIN_VERSION = 5
30
31
32def _GetChannelFromFeatures(api_name, features):
33  '''Finds API channel information for |api_name| from |features|.
34  Returns None if channel information for the API cannot be located.
35  '''
36  feature = features.Get().get(api_name)
37  return feature.get('channel') if feature else None
38
39
40def _GetChannelFromAPIFeatures(api_name, features_bundle):
41  return _GetChannelFromFeatures(api_name, features_bundle.GetAPIFeatures())
42
43
44def _GetChannelFromManifestFeatures(api_name, features_bundle):
45  # _manifest_features.json uses unix_style API names.
46  api_name = UnixName(api_name)
47  return _GetChannelFromFeatures(api_name,
48                                 features_bundle.GetManifestFeatures())
49
50
51def _GetChannelFromPermissionFeatures(api_name, features_bundle):
52  return _GetChannelFromFeatures(api_name,
53                                 features_bundle.GetPermissionFeatures())
54
55
56def _GetAPISchemaFilename(api_name, file_system, version):
57  '''Gets the name of the file which may contain the schema for |api_name| in
58  |file_system|, or None if the API is not found. Note that this may be the
59  single _EXTENSION_API file which all APIs share in older versions of Chrome,
60  in which case it is unknown whether the API actually exists there.
61  '''
62  if version == 'master' or version > _ORIGINAL_FEATURES_MIN_VERSION:
63    # API schema filenames switch format to unix_hacker_style.
64    api_name = UnixName(api_name)
65
66  # Devtools API names have 'devtools.' prepended to them.
67  # The corresponding filenames do not.
68  if 'devtools_' in api_name:
69    api_name = api_name.replace('devtools_', '')
70
71  for api_path in API_PATHS:
72    try:
73      for base, _, filenames in file_system.Walk(api_path):
74        for ext in ('json', 'idl'):
75          filename = '%s.%s' % (api_name, ext)
76          if filename in filenames:
77            return posixpath.join(api_path, base, filename)
78          if _EXTENSION_API in filenames:
79            return posixpath.join(api_path, base, _EXTENSION_API)
80    except FileNotFoundError:
81      continue
82  return None
83
84
85class AvailabilityInfo(object):
86  '''Represents availability data for an API. |scheduled| is a version number
87  specifying when dev and beta APIs will become stable, or None if that data
88  is unknown.
89  '''
90  def __init__(self, channel_info, scheduled=None):
91    assert isinstance(channel_info, ChannelInfo)
92    assert isinstance(scheduled, int) or scheduled is None
93    self.channel_info = channel_info
94    self.scheduled = scheduled
95
96  def __eq__(self, other):
97    return self.__dict__ == other.__dict__
98
99  def __ne__(self, other):
100    return not (self == other)
101
102  def __repr__(self):
103    return '%s%s' % (type(self).__name__, repr(self.__dict__))
104
105  def __str__(self):
106    return repr(self)
107
108
109class AvailabilityFinder(object):
110  '''Generates availability information for APIs by looking at API schemas and
111  _features files over multiple release versions of Chrome.
112  '''
113
114  def __init__(self,
115               branch_utility,
116               compiled_fs_factory,
117               file_system_iterator,
118               host_file_system,
119               object_store_creator,
120               platform,
121               schema_processor_factory):
122    self._branch_utility = branch_utility
123    self._compiled_fs_factory = compiled_fs_factory
124    self._file_system_iterator = file_system_iterator
125    self._host_file_system = host_file_system
126    self._object_store_creator = object_store_creator
127    def create_object_store(category):
128      return object_store_creator.Create(
129          AvailabilityFinder, category='/'.join((platform, category)))
130    self._top_level_object_store = create_object_store('top_level')
131    self._node_level_object_store = create_object_store('node_level')
132    self._json_fs = compiled_fs_factory.ForJson(self._host_file_system)
133    self._platform = platform
134    # When processing the API schemas, we retain inlined types in the schema
135    # so that there are not missing nodes in the APISchemaGraphs when trying
136    # to lookup availability.
137    self._schema_processor = schema_processor_factory.Create(True)
138
139  def _GetPredeterminedAvailability(self, api_name):
140    '''Checks a configuration file for hardcoded (i.e. predetermined)
141    availability information for an API.
142    '''
143    api_info = self._json_fs.GetFromFile(
144        JSON_TEMPLATES + 'api_availabilities.json').Get().get(api_name)
145    if api_info is None:
146      return None
147    if api_info['channel'] == 'stable':
148      return AvailabilityInfo(
149          self._branch_utility.GetStableChannelInfo(api_info['version']))
150    return AvailabilityInfo(
151        self._branch_utility.GetChannelInfo(api_info['channel']))
152
153  @memoize
154  def _CreateAPISchemaFileSystem(self, file_system):
155    '''Creates a CompiledFileSystem for parsing raw JSON or IDL API schema
156    data and formatting it so that it can be used to create APISchemaGraphs.
157    '''
158    def process_schema(path, data):
159      return self._schema_processor.Process(path, data)
160    return self._compiled_fs_factory.Create(file_system,
161                                            SingleFile(Unicode(process_schema)),
162                                            CompiledFileSystem,
163                                            category='api-schema')
164
165  def _GetAPISchema(self, api_name, file_system, version):
166    '''Searches |file_system| for |api_name|'s API schema data, and processes
167    and returns it if found.
168    '''
169    api_filename = _GetAPISchemaFilename(api_name, file_system, version)
170    if api_filename is None:
171      # No file for the API could be found in the given |file_system|.
172      return None
173
174    schema_fs = self._CreateAPISchemaFileSystem(file_system)
175    api_schemas = schema_fs.GetFromFile(api_filename).Get()
176    matching_schemas = [api for api in api_schemas
177                        if api['namespace'] == api_name]
178    # There should only be a single matching schema per file, or zero in the
179    # case of no API data being found in _EXTENSION_API.
180    assert len(matching_schemas) <= 1
181    return matching_schemas or None
182
183  def _HasAPISchema(self, api_name, file_system, version):
184    '''Whether or not an API schema for |api_name| exists in the given
185    |file_system|.
186    '''
187    filename = _GetAPISchemaFilename(api_name, file_system, version)
188    if filename is None:
189      return False
190    if filename.endswith(_EXTENSION_API) or filename.endswith(_DEVTOOLS_API):
191      return self._GetAPISchema(api_name, file_system, version) is not None
192    return True
193
194  def _CheckStableAvailability(self,
195                               api_name,
196                               file_system,
197                               version,
198                               earliest_version=None):
199    '''Checks for availability of an API, |api_name|, on the stable channel.
200    Considers several _features.json files, file system existence, and
201    extension_api.json depending on the given |version|.
202    |earliest_version| is the version of Chrome at which |api_name| first became
203    available. It should only be given when checking stable availability for
204    API nodes, so it can be used as an alternative to the check for filesystem
205    existence.
206    '''
207    earliest_version = earliest_version or _SVN_MIN_VERSION
208    if version < earliest_version:
209      # SVN data isn't available below this version.
210      return False
211    features_bundle = self._CreateFeaturesBundle(file_system)
212    available_channel = None
213    if version >= _API_FEATURES_MIN_VERSION:
214      # The _api_features.json file first appears in version 28 and should be
215      # the most reliable for finding API availability.
216      available_channel = _GetChannelFromAPIFeatures(api_name,
217                                                     features_bundle)
218    if version >= _ORIGINAL_FEATURES_MIN_VERSION:
219      # The _permission_features.json and _manifest_features.json files are
220      # present in Chrome 20 and onwards. Use these if no information could be
221      # found using _api_features.json.
222      available_channel = (
223          available_channel or
224          _GetChannelFromPermissionFeatures(api_name, features_bundle) or
225          _GetChannelFromManifestFeatures(api_name, features_bundle))
226      if available_channel is not None:
227        return available_channel == 'stable'
228
229    # |earliest_version| == _SVN_MIN_VERSION implies we're dealing with an API.
230    # Fall back to a check for file system existence if the API is not
231    # stable in any of the _features.json files, or if the _features files
232    # do not exist (version 19 and earlier).
233    if earliest_version == _SVN_MIN_VERSION:
234      return self._HasAPISchema(api_name, file_system, version)
235    # For API nodes, assume it's available if |version| is greater than the
236    # version the node became available (which it is, because of the first
237    # check).
238    return True
239
240  def _CheckChannelAvailability(self, api_name, file_system, channel_info):
241    '''Searches through the _features files in a given |file_system|, falling
242    back to checking the file system for API schema existence, to determine
243    whether or not an API is available on the given channel, |channel_info|.
244    '''
245    features_bundle = self._CreateFeaturesBundle(file_system)
246    available_channel = (
247        _GetChannelFromAPIFeatures(api_name, features_bundle) or
248        _GetChannelFromPermissionFeatures(api_name, features_bundle) or
249        _GetChannelFromManifestFeatures(api_name, features_bundle))
250    if (available_channel is None and
251        self._HasAPISchema(api_name, file_system, channel_info.version)):
252      # If an API is not represented in any of the _features files, but exists
253      # in the filesystem, then assume it is available in this version.
254      # The chrome.windows API is an example of this.
255      available_channel = channel_info.channel
256    # If the channel we're checking is the same as or newer than the
257    # |available_channel| then the API is available at this channel.
258    newest = BranchUtility.NewestChannel((available_channel,
259                                          channel_info.channel))
260    return available_channel is not None and newest == channel_info.channel
261
262  def _CheckChannelAvailabilityForNode(self,
263                                       node_name,
264                                       file_system,
265                                       channel_info,
266                                       earliest_channel_info):
267    '''Searches through the _features files in a given |file_system| to
268    determine whether or not an API node is available on the given channel,
269    |channel_info|. |earliest_channel_info| is the earliest channel the node
270    was introduced.
271    '''
272    features_bundle = self._CreateFeaturesBundle(file_system)
273    available_channel = None
274    # Only API nodes can have their availability overriden on a per-node basis,
275    # so we only need to check _api_features.json.
276    if channel_info.version >= _API_FEATURES_MIN_VERSION:
277      available_channel = _GetChannelFromAPIFeatures(node_name, features_bundle)
278    if (available_channel is None and
279        channel_info.version >= earliest_channel_info.version):
280      # Most API nodes inherit their availabiltity from their parent, so don't
281      # explicitly appear in _api_features.json. For example, "tabs.create"
282      # isn't listed; it inherits from "tabs". Assume these are available at
283      # |channel_info|.
284      available_channel = channel_info.channel
285    newest = BranchUtility.NewestChannel((available_channel,
286                                          channel_info.channel))
287    return available_channel is not None and newest == channel_info.channel
288
289  @memoize
290  def _CreateFeaturesBundle(self, file_system):
291    return FeaturesBundle(file_system,
292                          self._compiled_fs_factory,
293                          self._object_store_creator,
294                          self._platform)
295
296  def _CheckAPIAvailability(self, api_name, file_system, channel_info):
297    '''Determines the availability for an API at a certain version of Chrome.
298    Two branches of logic are used depending on whether or not the API is
299    determined to be 'stable' at the given version.
300    '''
301    if channel_info.channel == 'stable':
302      return self._CheckStableAvailability(api_name,
303                                           file_system,
304                                           channel_info.version)
305    return self._CheckChannelAvailability(api_name,
306                                          file_system,
307                                          channel_info)
308
309  def _FindScheduled(self, api_name, earliest_version=None):
310    '''Determines the earliest version of Chrome where the API is stable.
311    Unlike the code in GetAPIAvailability, this checks if the API is stable
312    even when Chrome is in dev or beta, which shows that the API is scheduled
313    to be stable in that verison of Chrome. |earliest_version| is the version
314    |api_name| became first available. Only use it when finding scheduled
315    availability for nodes.
316    '''
317    def check_scheduled(file_system, channel_info):
318      return self._CheckStableAvailability(api_name,
319                                           file_system,
320                                           channel_info.version,
321                                           earliest_version=earliest_version)
322
323    stable_channel = self._file_system_iterator.Descending(
324        self._branch_utility.GetChannelInfo('dev'), check_scheduled)
325
326    return stable_channel.version if stable_channel else None
327
328  def _CheckAPINodeAvailability(self, node_name, earliest_channel_info):
329    '''Gets availability data for a node by checking _features files.
330    '''
331    def check_node_availability(file_system, channel_info):
332      return self._CheckChannelAvailabilityForNode(node_name,
333                                                   file_system,
334                                                   channel_info,
335                                                   earliest_channel_info)
336    channel_info = (self._file_system_iterator.Descending(
337        self._branch_utility.GetChannelInfo('dev'), check_node_availability) or
338        earliest_channel_info)
339
340    if channel_info.channel == 'stable':
341      scheduled = None
342    else:
343      scheduled = self._FindScheduled(
344          node_name,
345          earliest_version=earliest_channel_info.version)
346
347    return AvailabilityInfo(channel_info, scheduled=scheduled)
348
349  def GetAPIAvailability(self, api_name):
350    '''Performs a search for an API's top-level availability by using a
351    HostFileSystemIterator instance to traverse multiple version of the
352    SVN filesystem.
353    '''
354    availability = self._top_level_object_store.Get(api_name).Get()
355    if availability is not None:
356      return availability
357
358    # Check for predetermined availability and cache this information if found.
359    availability = self._GetPredeterminedAvailability(api_name)
360    if availability is not None:
361      self._top_level_object_store.Set(api_name, availability)
362      return availability
363
364    def check_api_availability(file_system, channel_info):
365      return self._CheckAPIAvailability(api_name, file_system, channel_info)
366
367    channel_info = self._file_system_iterator.Descending(
368        self._branch_utility.GetChannelInfo('dev'),
369        check_api_availability)
370    if channel_info is None:
371      # The API wasn't available on 'dev', so it must be a 'master'-only API.
372      channel_info = self._branch_utility.GetChannelInfo('master')
373
374    # If the API is not stable, check when it will be scheduled to be stable.
375    if channel_info.channel == 'stable':
376      scheduled = None
377    else:
378      scheduled = self._FindScheduled(api_name)
379
380    availability = AvailabilityInfo(channel_info, scheduled=scheduled)
381
382    self._top_level_object_store.Set(api_name, availability)
383    return availability
384
385  def GetAPINodeAvailability(self, api_name):
386    '''Returns an APISchemaGraph annotated with each node's availability (the
387    ChannelInfo at the oldest channel it's available in).
388    '''
389    availability_graph = self._node_level_object_store.Get(api_name).Get()
390    if availability_graph is not None:
391      return availability_graph
392
393    def assert_not_none(value):
394      assert value is not None
395      return value
396
397    availability_graph = APISchemaGraph()
398    host_fs = self._host_file_system
399    master_stat = assert_not_none(host_fs.Stat(_GetAPISchemaFilename(
400        api_name, host_fs, 'master')))
401
402    # Weird object thing here because nonlocal is Python 3.
403    previous = type('previous', (object,), {'stat': None, 'graph': None})
404
405    def update_availability_graph(file_system, channel_info):
406      # If we can't find a filename, skip checking at this branch.
407      # For example, something could have a predetermined availability of 23,
408      # but it doesn't show up in the file system until 26.
409      # We know that the file will become available at some point.
410      #
411      # The problem with this is that at the first version where the API file
412      # exists, we'll get a huge chunk of new objects that don't match
413      # the predetermined API availability.
414      version_filename = _GetAPISchemaFilename(api_name,
415                                               file_system,
416                                               channel_info.version)
417      if version_filename is None:
418        # Continue the loop at the next version.
419        return True
420
421      version_stat = assert_not_none(file_system.Stat(version_filename))
422
423      # Important optimisation: only re-parse the graph if the file changed in
424      # the last revision. Parsing the same schema and forming a graph on every
425      # iteration is really expensive.
426      if version_stat == previous.stat:
427        version_graph = previous.graph
428      else:
429        # Keep track of any new schema elements from this version by adding
430        # them to |availability_graph|.
431        #
432        # Calling |availability_graph|.Lookup() on the nodes being updated
433        # will return the |annotation| object -- the current |channel_info|.
434        version_graph = APISchemaGraph(
435            api_schema=self._GetAPISchema(api_name,
436                                          file_system,
437                                          channel_info.version))
438        def annotator(node_name):
439          return self._CheckAPINodeAvailability('%s.%s' % (api_name, node_name),
440                                                channel_info)
441
442        availability_graph.Update(version_graph.Subtract(availability_graph),
443                                  annotator)
444
445      previous.stat = version_stat
446      previous.graph = version_graph
447
448      # Continue looping until there are no longer differences between this
449      # version and master.
450      return version_stat != master_stat
451
452    self._file_system_iterator.Ascending(
453        self.GetAPIAvailability(api_name).channel_info,
454        update_availability_graph)
455
456    self._node_level_object_store.Set(api_name, availability_graph)
457    return availability_graph
458