samples_data_source.py revision 2a99a7e74a7f215066514fe81d2bfa6639d9eddd 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import hashlib
6import json
7import logging
8import re
9
10import compiled_file_system as compiled_fs
11from file_system import FileNotFoundError
12import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
13import third_party.json_schema_compiler.model as model
14import url_constants
15
16DEFAULT_ICON_PATH = '/images/sample-default-icon.png'
17
18# See api_data_source.py for more info on _VERSION.
19_VERSION = 3
20
21class SamplesDataSource(object):
22  """Constructs a list of samples and their respective files and api calls.
23  """
24
25  class Factory(object):
26    """A factory to create SamplesDataSource instances bound to individual
27    Requests.
28    """
29    def __init__(self,
30                 channel,
31                 file_system,
32                 github_file_system,
33                 cache_factory,
34                 github_cache_factory,
35                 ref_resolver_factory,
36                 samples_path):
37      self._file_system = file_system
38      self._github_file_system = github_file_system
39      self._static_path = '/%s/static' % channel
40      self._extensions_cache = cache_factory.Create(self._MakeSamplesList,
41                                                    compiled_fs.EXTENSIONS,
42                                                    version=_VERSION)
43      self._apps_cache = github_cache_factory.Create(
44          lambda x, y: self._MakeSamplesList(x, y, is_apps=True),
45          compiled_fs.APPS,
46          version=_VERSION)
47      self._ref_resolver = ref_resolver_factory.Create()
48      self._samples_path = samples_path
49
50    def Create(self, request):
51      """Returns a new SamplesDataSource bound to |request|.
52      """
53      return SamplesDataSource(self._extensions_cache,
54                               self._apps_cache,
55                               self._samples_path,
56                               request)
57
58    def _GetAPIItems(self, js_file):
59      chrome_regex = '(chrome\.[a-zA-Z0-9\.]+)'
60      calls = set(re.findall(chrome_regex, js_file))
61      # Find APIs that have been assigned into variables.
62      assigned_vars = dict(re.findall('var\s*([^\s]+)\s*=\s*%s;' % chrome_regex,
63                                      js_file))
64      # Replace the variable name with the full API name.
65      for var_name, value in assigned_vars.iteritems():
66        js_file = js_file.replace(var_name, value)
67      return calls.union(re.findall(chrome_regex, js_file))
68
69    def _GetDataFromManifest(self, path, file_system):
70      manifest = file_system.ReadSingle(path + '/manifest.json')
71      try:
72        manifest_json = json.loads(json_comment_eater.Nom(manifest))
73      except ValueError as e:
74        logging.error('Error parsing manifest.json for %s: %s' % (path, e))
75        return None
76      l10n_data = {
77        'name': manifest_json.get('name', ''),
78        'description': manifest_json.get('description', None),
79        'icon': manifest_json.get('icons', {}).get('128', None),
80        'default_locale': manifest_json.get('default_locale', None),
81        'locales': {}
82      }
83      if not l10n_data['default_locale']:
84        return l10n_data
85      locales_path = path + '/_locales/'
86      locales_dir = file_system.ReadSingle(locales_path)
87      if locales_dir:
88        locales_files = file_system.Read(
89            [locales_path + f + 'messages.json' for f in locales_dir]).Get()
90        try:
91          locales_json = [(locale_path, json.loads(contents))
92                          for locale_path, contents in
93                          locales_files.iteritems()]
94        except ValueError as e:
95          logging.error('Error parsing locales files for %s: %s' % (path, e))
96        else:
97          for path, json_ in locales_json:
98            l10n_data['locales'][path[len(locales_path):].split('/')[0]] = json_
99      return l10n_data
100
101    def _MakeSamplesList(self, base_dir, files, is_apps=False):
102      file_system = self._github_file_system if is_apps else self._file_system
103      samples_list = []
104      for filename in sorted(files):
105        if filename.rsplit('/')[-1] != 'manifest.json':
106          continue
107        # This is a little hacky, but it makes a sample page.
108        sample_path = filename.rsplit('/', 1)[-2]
109        sample_files = [path for path in files
110                        if path.startswith(sample_path + '/')]
111        js_files = [path for path in sample_files if path.endswith('.js')]
112        try:
113          js_contents = file_system.Read(js_files).Get()
114        except FileNotFoundError as e:
115          logging.warning('Error fetching samples files: %s. Was a file '
116                          'deleted from a sample? This warning should go away '
117                          'in 5 minutes.' % e)
118          continue
119        api_items = set()
120        for js in js_contents.values():
121          api_items.update(self._GetAPIItems(js))
122
123        api_calls = []
124        for item in sorted(api_items):
125          if len(item.split('.')) < 3:
126            continue
127          if item.endswith('.removeListener') or item.endswith('.hasListener'):
128            continue
129          if item.endswith('.addListener'):
130            item = item[:-len('.addListener')]
131          if item.startswith('chrome.'):
132            item = item[len('chrome.'):]
133          ref_data = self._ref_resolver.GetLink(item, 'samples')
134          if ref_data is None:
135            continue
136          api_calls.append({
137            'name': ref_data['text'],
138            'link': ref_data['href']
139          })
140        try:
141          manifest_data = self._GetDataFromManifest(sample_path, file_system)
142        except FileNotFoundError as e:
143          logging.warning('Error getting data from samples manifest: %s. If '
144                          'this file was deleted from a sample this message '
145                          'should go away in 5 minutes.' % e)
146          continue
147        if manifest_data is None:
148          continue
149
150        sample_base_path = sample_path.split('/', 1)[1]
151        if is_apps:
152          url = url_constants.GITHUB_BASE + '/' + sample_base_path
153          icon_base = url_constants.RAW_GITHUB_BASE + '/' + sample_base_path
154          download_url = url
155        else:
156          url = sample_base_path
157          icon_base = sample_base_path
158          download_url = sample_base_path + '.zip'
159
160        if manifest_data['icon'] is None:
161          icon_path = self._static_path + DEFAULT_ICON_PATH
162        else:
163          icon_path = icon_base + '/' + manifest_data['icon']
164        manifest_data.update({
165          'icon': icon_path,
166          'id': hashlib.md5(url).hexdigest(),
167          'download_url': download_url,
168          'url': url,
169          'files': [f.replace(sample_path + '/', '') for f in sample_files],
170          'api_calls': api_calls
171        })
172        samples_list.append(manifest_data)
173
174      return samples_list
175
176  def __init__(self, extensions_cache, apps_cache, samples_path, request):
177    self._extensions_cache = extensions_cache
178    self._apps_cache = apps_cache
179    self._samples_path = samples_path
180    self._request = request
181
182  def _GetAcceptedLanguages(self):
183    accept_language = self._request.headers.get('Accept-Language', None)
184    if accept_language is None:
185      return []
186    return [lang_with_q.split(';')[0].strip()
187            for lang_with_q in accept_language.split(',')]
188
189  def FilterSamples(self, key, api_name):
190    """Fetches and filters the list of samples specified by |key|, returning
191    only the samples that use the API |api_name|. |key| is either 'apps' or
192    'extensions'.
193    """
194    api_search = api_name + '_'
195    samples_list = []
196    try:
197      for sample in self.get(key):
198        api_calls_unix = [model.UnixName(call['name'])
199                          for call in sample['api_calls']]
200        for call in api_calls_unix:
201          if call.startswith(api_search):
202            samples_list.append(sample)
203            break
204    except NotImplementedError:
205      # If we're testing, the GithubFileSystem can't fetch samples.
206      # Bug: http://crbug.com/141910
207      return []
208    return samples_list
209
210  def _CreateSamplesDict(self, key):
211    if key == 'apps':
212      samples_list = self._apps_cache.GetFromFileListing('/')
213    else:
214      samples_list = self._extensions_cache.GetFromFileListing(
215          self._samples_path + '/')
216    return_list = []
217    for dict_ in samples_list:
218      name = dict_['name']
219      description = dict_['description']
220      if description is None:
221        description = ''
222      if name.startswith('__MSG_') or description.startswith('__MSG_'):
223        try:
224          # Copy the sample dict so we don't change the dict in the cache.
225          sample_data = dict_.copy()
226          name_key = name[len('__MSG_'):-len('__')]
227          description_key = description[len('__MSG_'):-len('__')]
228          locale = sample_data['default_locale']
229          for lang in self._GetAcceptedLanguages():
230            if lang in sample_data['locales']:
231              locale = lang
232              break
233          locale_data = sample_data['locales'][locale]
234          sample_data['name'] = locale_data[name_key]['message']
235          sample_data['description'] = locale_data[description_key]['message']
236        except Exception as e:
237          logging.error(e)
238          # Revert the sample to the original dict.
239          sample_data = dict_
240        return_list.append(sample_data)
241      else:
242        return_list.append(dict_)
243    return return_list
244
245  def get(self, key):
246    return {
247      'apps': lambda: self._CreateSamplesDict('apps'),
248      'extensions': lambda: self._CreateSamplesDict('extensions')
249    }.get(key, lambda: {})()
250