samples_data_source.py revision f2477e01787aa58f445919b809d89e252beef54f 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import hashlib
6import json
7import logging
8import posixpath
9import re
10import traceback
11
12from compiled_file_system import CompiledFileSystem
13from extensions_paths import EXAMPLES
14import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
15import third_party.json_schema_compiler.model as model
16import url_constants
17
18
19_DEFAULT_ICON_PATH = 'images/sample-default-icon.png'
20
21
22class SamplesDataSource(object):
23  '''Constructs a list of samples and their respective files and api calls.
24  '''
25  class Factory(object):
26    '''A factory to create SamplesDataSource instances bound to individual
27    Requests.
28    '''
29    def __init__(self,
30                 host_file_system,
31                 app_samples_file_system,
32                 compiled_fs_factory,
33                 ref_resolver_factory,
34                 base_path):
35      self._host_file_system = host_file_system
36      self._app_samples_file_system = app_samples_file_system
37      self._ref_resolver = ref_resolver_factory.Create()
38      self._base_path = base_path
39      self._extensions_cache = compiled_fs_factory.Create(
40          host_file_system,
41          self._MakeSamplesList,
42          SamplesDataSource,
43          category='extensions')
44      self._apps_cache = compiled_fs_factory.Create(
45          app_samples_file_system,
46          lambda *args: self._MakeSamplesList(*args, is_apps=True),
47          SamplesDataSource,
48          category='apps')
49
50    def Create(self, request):
51      '''Returns a new SamplesDataSource bound to |request|.
52      '''
53      return SamplesDataSource(self._extensions_cache,
54                               self._apps_cache,
55                               self._base_path,
56                               request)
57
58    def _GetAPIItems(self, js_file):
59      chrome_pattern = r'chrome[\w.]+'
60      # Add API calls that appear normally, like "chrome.runtime.connect".
61      calls = set(re.findall(chrome_pattern, js_file))
62      # Add API calls that have been assigned into variables, like
63      # "var storageArea = chrome.storage.sync; storageArea.get", which should
64      # be expanded like "chrome.storage.sync.get".
65      for match in re.finditer(r'var\s+(\w+)\s*=\s*(%s);' % chrome_pattern,
66                               js_file):
67        var_name, api_prefix = match.groups()
68        for var_match in re.finditer(r'\b%s\.([\w.]+)\b' % re.escape(var_name),
69                                     js_file):
70          api_suffix, = var_match.groups()
71          calls.add('%s.%s' % (api_prefix, api_suffix))
72      return calls
73
74    def _GetDataFromManifest(self, path, file_system):
75      manifest = file_system.ReadSingle(path + '/manifest.json').Get()
76      try:
77        manifest_json = json.loads(json_comment_eater.Nom(manifest))
78      except ValueError as e:
79        logging.error('Error parsing manifest.json for %s: %s' % (path, e))
80        return None
81      l10n_data = {
82        'name': manifest_json.get('name', ''),
83        'description': manifest_json.get('description', None),
84        'icon': manifest_json.get('icons', {}).get('128', None),
85        'default_locale': manifest_json.get('default_locale', None),
86        'locales': {}
87      }
88      if not l10n_data['default_locale']:
89        return l10n_data
90      locales_path = path + '/_locales/'
91      locales_dir = file_system.ReadSingle(locales_path).Get()
92      if locales_dir:
93        locales_files = file_system.Read(
94            [locales_path + f + 'messages.json' for f in locales_dir]).Get()
95        try:
96          locales_json = [(locale_path, json.loads(contents))
97                          for locale_path, contents in
98                          locales_files.iteritems()]
99        except ValueError as e:
100          logging.error('Error parsing locales files for %s: %s' % (path, e))
101        else:
102          for path, json_ in locales_json:
103            l10n_data['locales'][path[len(locales_path):].split('/')[0]] = json_
104      return l10n_data
105
106    def _MakeSamplesList(self, base_path, files, is_apps=False):
107      file_system = (self._app_samples_file_system if is_apps else
108                     self._host_file_system)
109      samples_list = []
110      for filename in sorted(files):
111        if filename.rsplit('/')[-1] != 'manifest.json':
112          continue
113
114        # This is a little hacky, but it makes a sample page.
115        sample_path = filename.rsplit('/', 1)[-2]
116        sample_files = [path for path in files
117                        if path.startswith(sample_path + '/')]
118        js_files = [path for path in sample_files if path.endswith('.js')]
119        js_contents = file_system.Read(
120            [posixpath.join(base_path, js_file) for js_file in js_files]).Get()
121        api_items = set()
122        for js in js_contents.itervalues():
123          api_items.update(self._GetAPIItems(js))
124
125        api_calls = []
126        for item in sorted(api_items):
127          if len(item.split('.')) < 3:
128            continue
129          if item.endswith('.removeListener') or item.endswith('.hasListener'):
130            continue
131          if item.endswith('.addListener'):
132            item = item[:-len('.addListener')]
133          if item.startswith('chrome.'):
134            item = item[len('chrome.'):]
135          ref_data = self._ref_resolver.GetLink(item)
136          # TODO(kalman): What about references like chrome.storage.sync.get?
137          # That should link to either chrome.storage.sync or
138          # chrome.storage.StorageArea.get (or probably both).
139          # TODO(kalman): Filter out API-only references? This can happen when
140          # the API namespace is assigned to a variable, but it's very hard to
141          # to disambiguate.
142          if ref_data is None:
143            continue
144          api_calls.append({
145            'name': ref_data['text'],
146            'link': ref_data['href']
147          })
148
149        if is_apps:
150          url = url_constants.GITHUB_BASE + '/' + sample_path
151          icon_base = url_constants.RAW_GITHUB_BASE + '/' + sample_path
152          download_url = url
153        else:
154          extension_sample_path = posixpath.join('examples', sample_path)
155          url = extension_sample_path
156          icon_base = extension_sample_path
157          download_url = extension_sample_path + '.zip'
158
159        manifest_data = self._GetDataFromManifest(
160            posixpath.join(base_path, sample_path), file_system)
161        if manifest_data['icon'] is None:
162          icon_path = posixpath.join(
163              self._base_path, 'static', _DEFAULT_ICON_PATH)
164        else:
165          icon_path = '%s/%s' % (icon_base, manifest_data['icon'])
166        manifest_data.update({
167          'icon': icon_path,
168          'download_url': download_url,
169          'url': url,
170          'files': [f.replace(sample_path + '/', '') for f in sample_files],
171          'api_calls': api_calls
172        })
173        samples_list.append(manifest_data)
174
175      return samples_list
176
177  def __init__(self,
178               extensions_cache,
179               apps_cache,
180               base_path,
181               request):
182    self._extensions_cache = extensions_cache
183    self._apps_cache = apps_cache
184    self._base_path = base_path
185    self._request = request
186
187  def _GetSampleId(self, sample_name):
188    return sample_name.lower().replace(' ', '-')
189
190  def _GetAcceptedLanguages(self):
191    accept_language = self._request.headers.get('Accept-Language', None)
192    if accept_language is None:
193      return []
194    return [lang_with_q.split(';')[0].strip()
195            for lang_with_q in accept_language.split(',')]
196
197  def FilterSamples(self, key, api_name):
198    '''Fetches and filters the list of samples specified by |key|, returning
199    only the samples that use the API |api_name|. |key| is either 'apps' or
200    'extensions'.
201    '''
202    return [sample for sample in self.get(key) if any(
203        call['name'].startswith(api_name + '.')
204        for call in sample['api_calls'])]
205
206  def _CreateSamplesDict(self, key):
207    if key == 'apps':
208      samples_list = self._apps_cache.GetFromFileListing('/').Get()
209    else:
210      samples_list = self._extensions_cache.GetFromFileListing(EXAMPLES).Get()
211    return_list = []
212    for dict_ in samples_list:
213      name = dict_['name']
214      description = dict_['description']
215      if description is None:
216        description = ''
217      if name.startswith('__MSG_') or description.startswith('__MSG_'):
218        try:
219          # Copy the sample dict so we don't change the dict in the cache.
220          sample_data = dict_.copy()
221          name_key = name[len('__MSG_'):-len('__')]
222          description_key = description[len('__MSG_'):-len('__')]
223          locale = sample_data['default_locale']
224          for lang in self._GetAcceptedLanguages():
225            if lang in sample_data['locales']:
226              locale = lang
227              break
228          locale_data = sample_data['locales'][locale]
229          sample_data['name'] = locale_data[name_key]['message']
230          sample_data['description'] = locale_data[description_key]['message']
231          sample_data['id'] = self._GetSampleId(sample_data['name'])
232        except Exception as e:
233          logging.error(traceback.format_exc())
234          # Revert the sample to the original dict.
235          sample_data = dict_
236        return_list.append(sample_data)
237      else:
238        dict_['id'] = self._GetSampleId(name)
239        return_list.append(dict_)
240    return return_list
241
242  def get(self, key):
243    return {
244      'apps': lambda: self._CreateSamplesDict('apps'),
245      'extensions': lambda: self._CreateSamplesDict('extensions')
246    }.get(key, lambda: {})()
247