samples_data_source.py revision d0247b1b59f9c528cb6df88b4f2b9afaf80d181e 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import hashlib
6import json
7import logging
8import re
9
10from compiled_file_system import CompiledFileSystem
11import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
12import third_party.json_schema_compiler.model as model
13import url_constants
14
15DEFAULT_ICON_PATH = 'images/sample-default-icon.png'
16
17class SamplesDataSource(object):
18  '''Constructs a list of samples and their respective files and api calls.
19  '''
20  class Factory(object):
21    '''A factory to create SamplesDataSource instances bound to individual
22    Requests.
23    '''
24    def __init__(self,
25                 host_file_system,
26                 compiled_host_fs_factory,
27                 app_samples_file_system,
28                 compiled_app_samples_fs_factory,
29                 ref_resolver_factory,
30                 extension_samples_path,
31                 base_path):
32      self._host_file_system = host_file_system
33      self._app_samples_file_system = app_samples_file_system
34      self._ref_resolver = ref_resolver_factory.Create()
35      self._extension_samples_path = extension_samples_path
36      self._base_path = base_path
37      self._extensions_cache = compiled_host_fs_factory.Create(
38          self._MakeSamplesList,
39          SamplesDataSource,
40          category='extensions')
41      self._apps_cache = compiled_app_samples_fs_factory.Create(
42          lambda *args: self._MakeSamplesList(*args, is_apps=True),
43          SamplesDataSource,
44          category='apps')
45
46    def Create(self, request):
47      '''Returns a new SamplesDataSource bound to |request|.
48      '''
49      return SamplesDataSource(self._extensions_cache,
50                               self._apps_cache,
51                               self._extension_samples_path,
52                               self._base_path,
53                               request)
54
55    def _GetAPIItems(self, js_file):
56      chrome_regex = '(chrome\.[a-zA-Z0-9\.]+)'
57      calls = set(re.findall(chrome_regex, js_file))
58      # Find APIs that have been assigned into variables.
59      assigned_vars = dict(re.findall('var\s*([^\s]+)\s*=\s*%s;' % chrome_regex,
60                                      js_file))
61      # Replace the variable name with the full API name.
62      for var_name, value in assigned_vars.iteritems():
63        js_file = js_file.replace(var_name, value)
64      return calls.union(re.findall(chrome_regex, js_file))
65
66    def _GetDataFromManifest(self, path, file_system):
67      manifest = file_system.ReadSingle(path + '/manifest.json')
68      try:
69        manifest_json = json.loads(json_comment_eater.Nom(manifest))
70      except ValueError as e:
71        logging.error('Error parsing manifest.json for %s: %s' % (path, e))
72        return None
73      l10n_data = {
74        'name': manifest_json.get('name', ''),
75        'description': manifest_json.get('description', None),
76        'icon': manifest_json.get('icons', {}).get('128', None),
77        'default_locale': manifest_json.get('default_locale', None),
78        'locales': {}
79      }
80      if not l10n_data['default_locale']:
81        return l10n_data
82      locales_path = path + '/_locales/'
83      locales_dir = file_system.ReadSingle(locales_path)
84      if locales_dir:
85        locales_files = file_system.Read(
86            [locales_path + f + 'messages.json' for f in locales_dir]).Get()
87        try:
88          locales_json = [(locale_path, json.loads(contents))
89                          for locale_path, contents in
90                          locales_files.iteritems()]
91        except ValueError as e:
92          logging.error('Error parsing locales files for %s: %s' % (path, e))
93        else:
94          for path, json_ in locales_json:
95            l10n_data['locales'][path[len(locales_path):].split('/')[0]] = json_
96      return l10n_data
97
98    def _MakeSamplesList(self, base_dir, files, is_apps=False):
99      # HACK(kalman): The code here (for legacy reasons) assumes that |files| is
100      # prefixed by |base_dir|, so make it true.
101      files = ['%s%s' % (base_dir, f) for f in files]
102      file_system = (self._app_samples_file_system if is_apps else
103                     self._host_file_system)
104      samples_list = []
105      for filename in sorted(files):
106        if filename.rsplit('/')[-1] != 'manifest.json':
107          continue
108
109        # This is a little hacky, but it makes a sample page.
110        sample_path = filename.rsplit('/', 1)[-2]
111        sample_files = [path for path in files
112                        if path.startswith(sample_path + '/')]
113        js_files = [path for path in sample_files if path.endswith('.js')]
114        js_contents = file_system.Read(js_files).Get()
115        api_items = set()
116        for js in js_contents.values():
117          api_items.update(self._GetAPIItems(js))
118
119        api_calls = []
120        for item in sorted(api_items):
121          if len(item.split('.')) < 3:
122            continue
123          if item.endswith('.removeListener') or item.endswith('.hasListener'):
124            continue
125          if item.endswith('.addListener'):
126            item = item[:-len('.addListener')]
127          if item.startswith('chrome.'):
128            item = item[len('chrome.'):]
129          ref_data = self._ref_resolver.GetLink(item)
130          if ref_data is None:
131            continue
132          api_calls.append({
133            'name': ref_data['text'],
134            'link': ref_data['href']
135          })
136
137        sample_base_path = sample_path.split('/', 1)[1]
138        if is_apps:
139          url = url_constants.GITHUB_BASE + '/' + sample_base_path
140          icon_base = url_constants.RAW_GITHUB_BASE + '/' + sample_base_path
141          download_url = url
142        else:
143          url = sample_base_path
144          icon_base = sample_base_path
145          download_url = sample_base_path + '.zip'
146
147        manifest_data = self._GetDataFromManifest(sample_path, file_system)
148        if manifest_data['icon'] is None:
149          icon_path = '%s/static/%s' % (self._base_path, DEFAULT_ICON_PATH)
150        else:
151          icon_path = '%s/%s' % (icon_base, manifest_data['icon'])
152        manifest_data.update({
153          'icon': icon_path,
154          'id': hashlib.md5(url).hexdigest(),
155          'download_url': download_url,
156          'url': url,
157          'files': [f.replace(sample_path + '/', '') for f in sample_files],
158          'api_calls': api_calls
159        })
160        samples_list.append(manifest_data)
161
162      return samples_list
163
164  def __init__(self,
165               extensions_cache,
166               apps_cache,
167               extension_samples_path,
168               base_path,
169               request):
170    self._extensions_cache = extensions_cache
171    self._apps_cache = apps_cache
172    self._extension_samples_path = extension_samples_path
173    self._base_path = base_path
174    self._request = request
175
176  def _GetAcceptedLanguages(self):
177    accept_language = self._request.headers.get('Accept-Language', None)
178    if accept_language is None:
179      return []
180    return [lang_with_q.split(';')[0].strip()
181            for lang_with_q in accept_language.split(',')]
182
183  def FilterSamples(self, key, api_name):
184    '''Fetches and filters the list of samples specified by |key|, returning
185    only the samples that use the API |api_name|. |key| is either 'apps' or
186    'extensions'.
187    '''
188    api_search = api_name + '_'
189    samples_list = []
190    try:
191      for sample in self.get(key):
192        api_calls_unix = [model.UnixName(call['name'])
193                          for call in sample['api_calls']]
194        for call in api_calls_unix:
195          if call.startswith(api_search):
196            samples_list.append(sample)
197            break
198    except NotImplementedError:
199      # If we're testing, the GithubFileSystem can't fetch samples.
200      # Bug: http://crbug.com/141910
201      return []
202    return samples_list
203
204  def _CreateSamplesDict(self, key):
205    if key == 'apps':
206      samples_list = self._apps_cache.GetFromFileListing('/')
207    else:
208      samples_list = self._extensions_cache.GetFromFileListing(
209          self._extension_samples_path + '/')
210    return_list = []
211    for dict_ in samples_list:
212      name = dict_['name']
213      description = dict_['description']
214      if description is None:
215        description = ''
216      if name.startswith('__MSG_') or description.startswith('__MSG_'):
217        try:
218          # Copy the sample dict so we don't change the dict in the cache.
219          sample_data = dict_.copy()
220          name_key = name[len('__MSG_'):-len('__')]
221          description_key = description[len('__MSG_'):-len('__')]
222          locale = sample_data['default_locale']
223          for lang in self._GetAcceptedLanguages():
224            if lang in sample_data['locales']:
225              locale = lang
226              break
227          locale_data = sample_data['locales'][locale]
228          sample_data['name'] = locale_data[name_key]['message']
229          sample_data['description'] = locale_data[description_key]['message']
230        except Exception as e:
231          logging.error(e)
232          # Revert the sample to the original dict.
233          sample_data = dict_
234        return_list.append(sample_data)
235      else:
236        return_list.append(dict_)
237    return return_list
238
239  def get(self, key):
240    return {
241      'apps': lambda: self._CreateSamplesDict('apps'),
242      'extensions': lambda: self._CreateSamplesDict('extensions')
243    }.get(key, lambda: {})()
244