samples_data_source.py revision 2385ea399aae016c0806a4f9ef3c9cfe3d2a39df 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import hashlib
6import json
7import logging
8import re
9
10from compiled_file_system import CompiledFileSystem
11from file_system import FileNotFoundError
12import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
13import third_party.json_schema_compiler.model as model
14import url_constants
15
16DEFAULT_ICON_PATH = 'images/sample-default-icon.png'
17
18class SamplesDataSource(object):
19  '''Constructs a list of samples and their respective files and api calls.
20  '''
21  class Factory(object):
22    '''A factory to create SamplesDataSource instances bound to individual
23    Requests.
24    '''
25    def __init__(self,
26                 host_file_system,
27                 compiled_host_fs_factory,
28                 app_samples_file_system,
29                 compiled_app_samples_fs_factory,
30                 ref_resolver_factory,
31                 extension_samples_path,
32                 base_path):
33      self._host_file_system = host_file_system
34      self._app_samples_file_system = app_samples_file_system
35      self._ref_resolver = ref_resolver_factory.Create()
36      self._extension_samples_path = extension_samples_path
37      self._base_path = base_path
38      self._extensions_cache = compiled_host_fs_factory.Create(
39          self._MakeSamplesList,
40          SamplesDataSource,
41          category='extensions')
42      self._apps_cache = compiled_app_samples_fs_factory.Create(
43          lambda *args: self._MakeSamplesList(*args, is_apps=True),
44          SamplesDataSource,
45          category='apps')
46
47    def Create(self, request):
48      '''Returns a new SamplesDataSource bound to |request|.
49      '''
50      return SamplesDataSource(self._extensions_cache,
51                               self._apps_cache,
52                               self._extension_samples_path,
53                               self._base_path,
54                               request)
55
56    def _GetAPIItems(self, js_file):
57      chrome_regex = '(chrome\.[a-zA-Z0-9\.]+)'
58      calls = set(re.findall(chrome_regex, js_file))
59      # Find APIs that have been assigned into variables.
60      assigned_vars = dict(re.findall('var\s*([^\s]+)\s*=\s*%s;' % chrome_regex,
61                                      js_file))
62      # Replace the variable name with the full API name.
63      for var_name, value in assigned_vars.iteritems():
64        js_file = js_file.replace(var_name, value)
65      return calls.union(re.findall(chrome_regex, js_file))
66
67    def _GetDataFromManifest(self, path, file_system):
68      manifest = file_system.ReadSingle(path + '/manifest.json')
69      try:
70        manifest_json = json.loads(json_comment_eater.Nom(manifest))
71      except ValueError as e:
72        logging.error('Error parsing manifest.json for %s: %s' % (path, e))
73        return None
74      l10n_data = {
75        'name': manifest_json.get('name', ''),
76        'description': manifest_json.get('description', None),
77        'icon': manifest_json.get('icons', {}).get('128', None),
78        'default_locale': manifest_json.get('default_locale', None),
79        'locales': {}
80      }
81      if not l10n_data['default_locale']:
82        return l10n_data
83      locales_path = path + '/_locales/'
84      locales_dir = file_system.ReadSingle(locales_path)
85      if locales_dir:
86        locales_files = file_system.Read(
87            [locales_path + f + 'messages.json' for f in locales_dir]).Get()
88        try:
89          locales_json = [(locale_path, json.loads(contents))
90                          for locale_path, contents in
91                          locales_files.iteritems()]
92        except ValueError as e:
93          logging.error('Error parsing locales files for %s: %s' % (path, e))
94        else:
95          for path, json_ in locales_json:
96            l10n_data['locales'][path[len(locales_path):].split('/')[0]] = json_
97      return l10n_data
98
99    def _MakeSamplesList(self, base_dir, files, is_apps=False):
100      # HACK(kalman): The code here (for legacy reasons) assumes that |files| is
101      # prefixed by |base_dir|, so make it true.
102      files = ['%s%s' % (base_dir, f) for f in files]
103      file_system = (self._app_samples_file_system if is_apps else
104                     self._host_file_system)
105      samples_list = []
106      for filename in sorted(files):
107        if filename.rsplit('/')[-1] != 'manifest.json':
108          continue
109
110        # This is a little hacky, but it makes a sample page.
111        sample_path = filename.rsplit('/', 1)[-2]
112        sample_files = [path for path in files
113                        if path.startswith(sample_path + '/')]
114        js_files = [path for path in sample_files if path.endswith('.js')]
115        js_contents = file_system.Read(js_files).Get()
116        api_items = set()
117        for js in js_contents.values():
118          api_items.update(self._GetAPIItems(js))
119
120        api_calls = []
121        for item in sorted(api_items):
122          if len(item.split('.')) < 3:
123            continue
124          if item.endswith('.removeListener') or item.endswith('.hasListener'):
125            continue
126          if item.endswith('.addListener'):
127            item = item[:-len('.addListener')]
128          if item.startswith('chrome.'):
129            item = item[len('chrome.'):]
130          ref_data = self._ref_resolver.GetLink(item)
131          if ref_data is None:
132            continue
133          api_calls.append({
134            'name': ref_data['text'],
135            'link': ref_data['href']
136          })
137
138        sample_base_path = sample_path.split('/', 1)[1]
139        if is_apps:
140          url = url_constants.GITHUB_BASE + '/' + sample_base_path
141          icon_base = url_constants.RAW_GITHUB_BASE + '/' + sample_base_path
142          download_url = url
143        else:
144          url = sample_base_path
145          icon_base = sample_base_path
146          download_url = sample_base_path + '.zip'
147
148        manifest_data = self._GetDataFromManifest(sample_path, file_system)
149        if manifest_data['icon'] is None:
150          icon_path = '%s/static/%s' % (self._base_path, DEFAULT_ICON_PATH)
151        else:
152          icon_path = '%s/%s' % (icon_base, manifest_data['icon'])
153        manifest_data.update({
154          'icon': icon_path,
155          'id': hashlib.md5(url).hexdigest(),
156          'download_url': download_url,
157          'url': url,
158          'files': [f.replace(sample_path + '/', '') for f in sample_files],
159          'api_calls': api_calls
160        })
161        samples_list.append(manifest_data)
162
163      return samples_list
164
165  def __init__(self,
166               extensions_cache,
167               apps_cache,
168               extension_samples_path,
169               base_path,
170               request):
171    self._extensions_cache = extensions_cache
172    self._apps_cache = apps_cache
173    self._extension_samples_path = extension_samples_path
174    self._base_path = base_path
175    self._request = request
176
177  def _GetAcceptedLanguages(self):
178    accept_language = self._request.headers.get('Accept-Language', None)
179    if accept_language is None:
180      return []
181    return [lang_with_q.split(';')[0].strip()
182            for lang_with_q in accept_language.split(',')]
183
184  def FilterSamples(self, key, api_name):
185    '''Fetches and filters the list of samples specified by |key|, returning
186    only the samples that use the API |api_name|. |key| is either 'apps' or
187    'extensions'.
188    '''
189    api_search = api_name + '_'
190    samples_list = []
191    try:
192      for sample in self.get(key):
193        api_calls_unix = [model.UnixName(call['name'])
194                          for call in sample['api_calls']]
195        for call in api_calls_unix:
196          if call.startswith(api_search):
197            samples_list.append(sample)
198            break
199    except NotImplementedError:
200      # If we're testing, the GithubFileSystem can't fetch samples.
201      # Bug: http://crbug.com/141910
202      return []
203    return samples_list
204
205  def _CreateSamplesDict(self, key):
206    if key == 'apps':
207      samples_list = self._apps_cache.GetFromFileListing('/')
208    else:
209      samples_list = self._extensions_cache.GetFromFileListing(
210          self._extension_samples_path + '/')
211    return_list = []
212    for dict_ in samples_list:
213      name = dict_['name']
214      description = dict_['description']
215      if description is None:
216        description = ''
217      if name.startswith('__MSG_') or description.startswith('__MSG_'):
218        try:
219          # Copy the sample dict so we don't change the dict in the cache.
220          sample_data = dict_.copy()
221          name_key = name[len('__MSG_'):-len('__')]
222          description_key = description[len('__MSG_'):-len('__')]
223          locale = sample_data['default_locale']
224          for lang in self._GetAcceptedLanguages():
225            if lang in sample_data['locales']:
226              locale = lang
227              break
228          locale_data = sample_data['locales'][locale]
229          sample_data['name'] = locale_data[name_key]['message']
230          sample_data['description'] = locale_data[description_key]['message']
231        except Exception as e:
232          logging.error(e)
233          # Revert the sample to the original dict.
234          sample_data = dict_
235        return_list.append(sample_data)
236      else:
237        return_list.append(dict_)
238    return return_list
239
240  def get(self, key):
241    return {
242      'apps': lambda: self._CreateSamplesDict('apps'),
243      'extensions': lambda: self._CreateSamplesDict('extensions')
244    }.get(key, lambda: {})()
245