samples_data_source.py revision 1e9bf3e0803691d0a228da41fc608347b6db4340 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import hashlib
6import json
7import logging
8import posixpath
9import re
10import traceback
11
12from compiled_file_system import CompiledFileSystem
13import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
14import third_party.json_schema_compiler.model as model
15import url_constants
16
17DEFAULT_ICON_PATH = 'images/sample-default-icon.png'
18
19class SamplesDataSource(object):
20  '''Constructs a list of samples and their respective files and api calls.
21  '''
22  class Factory(object):
23    '''A factory to create SamplesDataSource instances bound to individual
24    Requests.
25    '''
26    def __init__(self,
27                 host_file_system,
28                 app_samples_file_system,
29                 compiled_fs_factory,
30                 ref_resolver_factory,
31                 extension_samples_path,
32                 base_path):
33      self._host_file_system = host_file_system
34      self._app_samples_file_system = app_samples_file_system
35      self._ref_resolver = ref_resolver_factory.Create()
36      self._extension_samples_path = extension_samples_path
37      self._base_path = base_path
38      self._extensions_cache = compiled_fs_factory.Create(
39          host_file_system,
40          self._MakeSamplesList,
41          SamplesDataSource,
42          category='extensions')
43      self._apps_cache = compiled_fs_factory.Create(
44          app_samples_file_system,
45          lambda *args: self._MakeSamplesList(*args, is_apps=True),
46          SamplesDataSource,
47          category='apps')
48
49    def Create(self, request):
50      '''Returns a new SamplesDataSource bound to |request|.
51      '''
52      return SamplesDataSource(self._extensions_cache,
53                               self._apps_cache,
54                               self._extension_samples_path,
55                               self._base_path,
56                               request)
57
58    def _GetAPIItems(self, js_file):
59      chrome_regex = '(chrome\.[a-zA-Z0-9\.]+)'
60      calls = set(re.findall(chrome_regex, js_file))
61      # Find APIs that have been assigned into variables.
62      assigned_vars = dict(re.findall('var\s*([^\s]+)\s*=\s*%s;' % chrome_regex,
63                                      js_file))
64      # Replace the variable name with the full API name.
65      for var_name, value in assigned_vars.iteritems():
66        js_file = js_file.replace(var_name, value)
67      return calls.union(re.findall(chrome_regex, js_file))
68
69    def _GetDataFromManifest(self, path, file_system):
70      manifest = file_system.ReadSingle(path + '/manifest.json').Get()
71      try:
72        manifest_json = json.loads(json_comment_eater.Nom(manifest))
73      except ValueError as e:
74        logging.error('Error parsing manifest.json for %s: %s' % (path, e))
75        return None
76      l10n_data = {
77        'name': manifest_json.get('name', ''),
78        'description': manifest_json.get('description', None),
79        'icon': manifest_json.get('icons', {}).get('128', None),
80        'default_locale': manifest_json.get('default_locale', None),
81        'locales': {}
82      }
83      if not l10n_data['default_locale']:
84        return l10n_data
85      locales_path = path + '/_locales/'
86      locales_dir = file_system.ReadSingle(locales_path).Get()
87      if locales_dir:
88        locales_files = file_system.Read(
89            [locales_path + f + 'messages.json' for f in locales_dir]).Get()
90        try:
91          locales_json = [(locale_path, json.loads(contents))
92                          for locale_path, contents in
93                          locales_files.iteritems()]
94        except ValueError as e:
95          logging.error('Error parsing locales files for %s: %s' % (path, e))
96        else:
97          for path, json_ in locales_json:
98            l10n_data['locales'][path[len(locales_path):].split('/')[0]] = json_
99      return l10n_data
100
101    def _MakeSamplesList(self, base_dir, files, is_apps=False):
102      # HACK(kalman): The code here (for legacy reasons) assumes that |files| is
103      # prefixed by |base_dir|, so make it true.
104      files = ['%s%s' % (base_dir, f) for f in files]
105      file_system = (self._app_samples_file_system if is_apps else
106                     self._host_file_system)
107      samples_list = []
108      for filename in sorted(files):
109        if filename.rsplit('/')[-1] != 'manifest.json':
110          continue
111
112        # This is a little hacky, but it makes a sample page.
113        sample_path = filename.rsplit('/', 1)[-2]
114        sample_files = [path for path in files
115                        if path.startswith(sample_path + '/')]
116        js_files = [path for path in sample_files if path.endswith('.js')]
117        js_contents = file_system.Read(js_files).Get()
118        api_items = set()
119        for js in js_contents.values():
120          api_items.update(self._GetAPIItems(js))
121
122        api_calls = []
123        for item in sorted(api_items):
124          if len(item.split('.')) < 3:
125            continue
126          if item.endswith('.removeListener') or item.endswith('.hasListener'):
127            continue
128          if item.endswith('.addListener'):
129            item = item[:-len('.addListener')]
130          if item.startswith('chrome.'):
131            item = item[len('chrome.'):]
132          ref_data = self._ref_resolver.GetLink(item)
133          if ref_data is None:
134            continue
135          api_calls.append({
136            'name': ref_data['text'],
137            'link': ref_data['href']
138          })
139
140        sample_base_path = sample_path.split('/', 1)[1]
141        if is_apps:
142          url = url_constants.GITHUB_BASE + '/' + sample_base_path
143          icon_base = url_constants.RAW_GITHUB_BASE + '/' + sample_base_path
144          download_url = url
145        else:
146          url = sample_base_path
147          icon_base = sample_base_path
148          download_url = sample_base_path + '.zip'
149
150        manifest_data = self._GetDataFromManifest(sample_path, file_system)
151        if manifest_data['icon'] is None:
152          icon_path = posixpath.join(
153              self._base_path, 'static', DEFAULT_ICON_PATH)
154        else:
155          icon_path = '%s/%s' % (icon_base, manifest_data['icon'])
156        manifest_data.update({
157          'icon': icon_path,
158          'download_url': download_url,
159          'url': url,
160          'files': [f.replace(sample_path + '/', '') for f in sample_files],
161          'api_calls': api_calls
162        })
163        samples_list.append(manifest_data)
164
165      return samples_list
166
167  def __init__(self,
168               extensions_cache,
169               apps_cache,
170               extension_samples_path,
171               base_path,
172               request):
173    self._extensions_cache = extensions_cache
174    self._apps_cache = apps_cache
175    self._extension_samples_path = extension_samples_path
176    self._base_path = base_path
177    self._request = request
178
179  def _GetSampleId(self, sample_name):
180    return sample_name.lower().replace(' ', '-')
181
182  def _GetAcceptedLanguages(self):
183    accept_language = self._request.headers.get('Accept-Language', None)
184    if accept_language is None:
185      return []
186    return [lang_with_q.split(';')[0].strip()
187            for lang_with_q in accept_language.split(',')]
188
189  def FilterSamples(self, key, api_name):
190    '''Fetches and filters the list of samples specified by |key|, returning
191    only the samples that use the API |api_name|. |key| is either 'apps' or
192    'extensions'.
193    '''
194    return [sample for sample in self.get(key) if any(
195        call['name'].startswith(api_name + '.')
196        for call in sample['api_calls'])]
197
198  def _CreateSamplesDict(self, key):
199    if key == 'apps':
200      samples_list = self._apps_cache.GetFromFileListing('/').Get()
201    else:
202      samples_list = self._extensions_cache.GetFromFileListing(
203          self._extension_samples_path + '/').Get()
204    return_list = []
205    for dict_ in samples_list:
206      name = dict_['name']
207      description = dict_['description']
208      if description is None:
209        description = ''
210      if name.startswith('__MSG_') or description.startswith('__MSG_'):
211        try:
212          # Copy the sample dict so we don't change the dict in the cache.
213          sample_data = dict_.copy()
214          name_key = name[len('__MSG_'):-len('__')]
215          description_key = description[len('__MSG_'):-len('__')]
216          locale = sample_data['default_locale']
217          for lang in self._GetAcceptedLanguages():
218            if lang in sample_data['locales']:
219              locale = lang
220              break
221          locale_data = sample_data['locales'][locale]
222          sample_data['name'] = locale_data[name_key]['message']
223          sample_data['description'] = locale_data[description_key]['message']
224          sample_data['id'] = self._GetSampleId(sample_data['name'])
225        except Exception as e:
226          logging.error(traceback.format_exc())
227          # Revert the sample to the original dict.
228          sample_data = dict_
229        return_list.append(sample_data)
230      else:
231        dict_['id'] = self._GetSampleId(name)
232        return_list.append(dict_)
233    return return_list
234
235  def get(self, key):
236    return {
237      'apps': lambda: self._CreateSamplesDict('apps'),
238      'extensions': lambda: self._CreateSamplesDict('extensions')
239    }.get(key, lambda: {})()
240