samples_data_source.py revision a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import hashlib
6import json
7import logging
8import posixpath
9import re
10import traceback
11
12from compiled_file_system import CompiledFileSystem
13from extensions_paths import EXAMPLES
14import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
15import third_party.json_schema_compiler.model as model
16import url_constants
17
18
19_DEFAULT_ICON_PATH = 'images/sample-default-icon.png'
20
21
22class SamplesDataSource(object):
23  '''Constructs a list of samples and their respective files and api calls.
24  '''
25  class Factory(object):
26    '''A factory to create SamplesDataSource instances bound to individual
27    Requests.
28    '''
29    def __init__(self,
30                 host_file_system,
31                 app_samples_file_system,
32                 compiled_fs_factory,
33                 ref_resolver_factory,
34                 base_path):
35      self._host_file_system = host_file_system
36      self._app_samples_file_system = app_samples_file_system
37      self._ref_resolver = ref_resolver_factory.Create()
38      self._base_path = base_path
39      self._extensions_cache = compiled_fs_factory.Create(
40          host_file_system,
41          self._MakeSamplesList,
42          SamplesDataSource,
43          category='extensions')
44      self._extensions_text_cache = compiled_fs_factory.ForUnicode(
45          host_file_system)
46      self._apps_cache = compiled_fs_factory.Create(
47          app_samples_file_system,
48          lambda *args: self._MakeSamplesList(*args, is_apps=True),
49          SamplesDataSource,
50          category='apps')
51      self._apps_text_cache = compiled_fs_factory.ForUnicode(
52          app_samples_file_system)
53
54    def Create(self, request):
55      '''Returns a new SamplesDataSource bound to |request|.
56      '''
57      return SamplesDataSource(self._extensions_cache,
58                               self._apps_cache,
59                               self._base_path,
60                               request)
61
62    def _GetAPIItems(self, js_file):
63      chrome_pattern = r'chrome[\w.]+'
64      # Add API calls that appear normally, like "chrome.runtime.connect".
65      calls = set(re.findall(chrome_pattern, js_file))
66      # Add API calls that have been assigned into variables, like
67      # "var storageArea = chrome.storage.sync; storageArea.get", which should
68      # be expanded like "chrome.storage.sync.get".
69      for match in re.finditer(r'var\s+(\w+)\s*=\s*(%s);' % chrome_pattern,
70                               js_file):
71        var_name, api_prefix = match.groups()
72        for var_match in re.finditer(r'\b%s\.([\w.]+)\b' % re.escape(var_name),
73                                     js_file):
74          api_suffix, = var_match.groups()
75          calls.add('%s.%s' % (api_prefix, api_suffix))
76      return calls
77
78    def _GetDataFromManifest(self, path, text_cache, file_system):
79      manifest = text_cache.GetFromFile(path + '/manifest.json').Get()
80      try:
81        manifest_json = json.loads(json_comment_eater.Nom(manifest))
82      except ValueError as e:
83        logging.error('Error parsing manifest.json for %s: %s' % (path, e))
84        return None
85      l10n_data = {
86        'name': manifest_json.get('name', ''),
87        'description': manifest_json.get('description', None),
88        'icon': manifest_json.get('icons', {}).get('128', None),
89        'default_locale': manifest_json.get('default_locale', None),
90        'locales': {}
91      }
92      if not l10n_data['default_locale']:
93        return l10n_data
94      locales_path = path + '/_locales/'
95      locales_dir = file_system.ReadSingle(locales_path).Get()
96      if locales_dir:
97        def load_locale_json(path):
98          return (path, json.loads(text_cache.GetFromFile(path).Get()))
99
100        try:
101          locales_json = [load_locale_json(locales_path + f + 'messages.json')
102                          for f in locales_dir]
103        except ValueError as e:
104          logging.error('Error parsing locales files for %s: %s' % (path, e))
105        else:
106          for path, json_ in locales_json:
107            l10n_data['locales'][path[len(locales_path):].split('/')[0]] = json_
108      return l10n_data
109
110    def _MakeSamplesList(self, base_path, files, is_apps=False):
111      file_system = (self._app_samples_file_system if is_apps else
112                              self._host_file_system)
113      text_cache = (self._apps_text_cache if is_apps else
114          self._extensions_text_cache)
115      samples_list = []
116      for filename in sorted(files):
117        if filename.rsplit('/')[-1] != 'manifest.json':
118          continue
119
120        # This is a little hacky, but it makes a sample page.
121        sample_path = filename.rsplit('/', 1)[-2]
122        sample_files = [path for path in files
123                        if path.startswith(sample_path + '/')]
124        js_files = [path for path in sample_files if path.endswith('.js')]
125        js_contents = [text_cache.GetFromFile(
126            posixpath.join(base_path, js_file)).Get()
127            for js_file in js_files]
128        api_items = set()
129        for js in js_contents:
130          api_items.update(self._GetAPIItems(js))
131
132        api_calls = []
133        for item in sorted(api_items):
134          if len(item.split('.')) < 3:
135            continue
136          if item.endswith('.removeListener') or item.endswith('.hasListener'):
137            continue
138          if item.endswith('.addListener'):
139            item = item[:-len('.addListener')]
140          if item.startswith('chrome.'):
141            item = item[len('chrome.'):]
142          ref_data = self._ref_resolver.GetLink(item)
143          # TODO(kalman): What about references like chrome.storage.sync.get?
144          # That should link to either chrome.storage.sync or
145          # chrome.storage.StorageArea.get (or probably both).
146          # TODO(kalman): Filter out API-only references? This can happen when
147          # the API namespace is assigned to a variable, but it's very hard to
148          # to disambiguate.
149          if ref_data is None:
150            continue
151          api_calls.append({
152            'name': ref_data['text'],
153            'link': ref_data['href']
154          })
155
156        if is_apps:
157          url = url_constants.GITHUB_BASE + '/' + sample_path
158          icon_base = url_constants.RAW_GITHUB_BASE + '/' + sample_path
159          download_url = url
160        else:
161          extension_sample_path = posixpath.join('examples', sample_path)
162          url = extension_sample_path
163          icon_base = extension_sample_path
164          download_url = extension_sample_path + '.zip'
165
166        manifest_data = self._GetDataFromManifest(
167            posixpath.join(base_path, sample_path),
168            text_cache,
169            file_system)
170        if manifest_data['icon'] is None:
171          icon_path = posixpath.join(
172              self._base_path, 'static', _DEFAULT_ICON_PATH)
173        else:
174          icon_path = '%s/%s' % (icon_base, manifest_data['icon'])
175        manifest_data.update({
176          'icon': icon_path,
177          'download_url': download_url,
178          'url': url,
179          'files': [f.replace(sample_path + '/', '') for f in sample_files],
180          'api_calls': api_calls
181        })
182        samples_list.append(manifest_data)
183
184      return samples_list
185
186  def __init__(self,
187               extensions_cache,
188               apps_cache,
189               base_path,
190               request):
191    self._extensions_cache = extensions_cache
192    self._apps_cache = apps_cache
193    self._base_path = base_path
194    self._request = request
195
196  def _GetSampleId(self, sample_name):
197    return sample_name.lower().replace(' ', '-')
198
199  def _GetAcceptedLanguages(self):
200    accept_language = self._request.headers.get('Accept-Language', None)
201    if accept_language is None:
202      return []
203    return [lang_with_q.split(';')[0].strip()
204            for lang_with_q in accept_language.split(',')]
205
206  def FilterSamples(self, key, api_name):
207    '''Fetches and filters the list of samples specified by |key|, returning
208    only the samples that use the API |api_name|. |key| is either 'apps' or
209    'extensions'.
210    '''
211    return [sample for sample in self.get(key) if any(
212        call['name'].startswith(api_name + '.')
213        for call in sample['api_calls'])]
214
215  def _CreateSamplesDict(self, key):
216    if key == 'apps':
217      samples_list = self._apps_cache.GetFromFileListing('/').Get()
218    else:
219      samples_list = self._extensions_cache.GetFromFileListing(EXAMPLES).Get()
220    return_list = []
221    for dict_ in samples_list:
222      name = dict_['name']
223      description = dict_['description']
224      if description is None:
225        description = ''
226      if name.startswith('__MSG_') or description.startswith('__MSG_'):
227        try:
228          # Copy the sample dict so we don't change the dict in the cache.
229          sample_data = dict_.copy()
230          name_key = name[len('__MSG_'):-len('__')]
231          description_key = description[len('__MSG_'):-len('__')]
232          locale = sample_data['default_locale']
233          for lang in self._GetAcceptedLanguages():
234            if lang in sample_data['locales']:
235              locale = lang
236              break
237          locale_data = sample_data['locales'][locale]
238          sample_data['name'] = locale_data[name_key]['message']
239          sample_data['description'] = locale_data[description_key]['message']
240          sample_data['id'] = self._GetSampleId(sample_data['name'])
241        except Exception as e:
242          logging.error(traceback.format_exc())
243          # Revert the sample to the original dict.
244          sample_data = dict_
245        return_list.append(sample_data)
246      else:
247        dict_['id'] = self._GetSampleId(name)
248        return_list.append(dict_)
249    return return_list
250
251  def get(self, key):
252    return {
253      'apps': lambda: self._CreateSamplesDict('apps'),
254      'extensions': lambda: self._CreateSamplesDict('extensions')
255    }.get(key, lambda: {})()
256