samples_data_source.py revision c2e0dbddbe15c98d52c4786dac06cb8952a8ae6d 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import hashlib
6import json
7import logging
8import re
9
10from compiled_file_system import CompiledFileSystem
11from file_system import FileNotFoundError
12import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
13import third_party.json_schema_compiler.model as model
14import url_constants
15
16DEFAULT_ICON_PATH = '/images/sample-default-icon.png'
17
18class SamplesDataSource(object):
19  """Constructs a list of samples and their respective files and api calls.
20  """
21  class Factory(object):
22    """A factory to create SamplesDataSource instances bound to individual
23    Requests.
24    """
25    def __init__(self,
26                 channel,
27                 extensions_file_system,
28                 apps_file_system,
29                 ref_resolver_factory,
30                 object_store_creator_factory,
31                 extension_samples_path):
32      self._svn_file_system = extensions_file_system
33      self._github_file_system = apps_file_system
34      self._static_path = '/%s/static' % channel
35      self._ref_resolver = ref_resolver_factory.Create()
36      self._extension_samples_path = extension_samples_path
37      def create_compiled_fs(fs, fn, category):
38        return CompiledFileSystem.Factory(
39            fs,
40            object_store_creator_factory).Create(fn,
41                                                 SamplesDataSource,
42                                                 category=category)
43      self._extensions_cache = create_compiled_fs(extensions_file_system,
44                                                  self._MakeSamplesList,
45                                                  'extensions')
46      self._apps_cache = create_compiled_fs(apps_file_system,
47                                            lambda *args: self._MakeSamplesList(
48                                                *args, is_apps=True),
49                                            'apps')
50
51    def Create(self, request):
52      """Returns a new SamplesDataSource bound to |request|.
53      """
54      return SamplesDataSource(self._extensions_cache,
55                               self._apps_cache,
56                               self._extension_samples_path,
57                               request)
58
59    def _GetAPIItems(self, js_file):
60      chrome_regex = '(chrome\.[a-zA-Z0-9\.]+)'
61      calls = set(re.findall(chrome_regex, js_file))
62      # Find APIs that have been assigned into variables.
63      assigned_vars = dict(re.findall('var\s*([^\s]+)\s*=\s*%s;' % chrome_regex,
64                                      js_file))
65      # Replace the variable name with the full API name.
66      for var_name, value in assigned_vars.iteritems():
67        js_file = js_file.replace(var_name, value)
68      return calls.union(re.findall(chrome_regex, js_file))
69
70    def _GetDataFromManifest(self, path, file_system):
71      manifest = file_system.ReadSingle(path + '/manifest.json')
72      try:
73        manifest_json = json.loads(json_comment_eater.Nom(manifest))
74      except ValueError as e:
75        logging.error('Error parsing manifest.json for %s: %s' % (path, e))
76        return None
77      l10n_data = {
78        'name': manifest_json.get('name', ''),
79        'description': manifest_json.get('description', None),
80        'icon': manifest_json.get('icons', {}).get('128', None),
81        'default_locale': manifest_json.get('default_locale', None),
82        'locales': {}
83      }
84      if not l10n_data['default_locale']:
85        return l10n_data
86      locales_path = path + '/_locales/'
87      locales_dir = file_system.ReadSingle(locales_path)
88      if locales_dir:
89        locales_files = file_system.Read(
90            [locales_path + f + 'messages.json' for f in locales_dir]).Get()
91        try:
92          locales_json = [(locale_path, json.loads(contents))
93                          for locale_path, contents in
94                          locales_files.iteritems()]
95        except ValueError as e:
96          logging.error('Error parsing locales files for %s: %s' % (path, e))
97        else:
98          for path, json_ in locales_json:
99            l10n_data['locales'][path[len(locales_path):].split('/')[0]] = json_
100      return l10n_data
101
102    def _MakeSamplesList(self, base_dir, files, is_apps=False):
103      # HACK(kalman): The code here (for legacy reasons) assumes that |files| is
104      # prefixed by |base_dir|, so make it true.
105      files = ['%s%s' % (base_dir, f) for f in files]
106      file_system = (self._github_file_system if is_apps else
107                     self._svn_file_system)
108      samples_list = []
109      for filename in sorted(files):
110        if filename.rsplit('/')[-1] != 'manifest.json':
111          continue
112        # This is a little hacky, but it makes a sample page.
113        sample_path = filename.rsplit('/', 1)[-2]
114        sample_files = [path for path in files
115                        if path.startswith(sample_path + '/')]
116        js_files = [path for path in sample_files if path.endswith('.js')]
117        try:
118          js_contents = file_system.Read(js_files).Get()
119        except FileNotFoundError as e:
120          logging.warning('Error fetching samples files: %s. Was a file '
121                          'deleted from a sample? This warning should go away '
122                          'in 5 minutes.' % e)
123          continue
124        api_items = set()
125        for js in js_contents.values():
126          api_items.update(self._GetAPIItems(js))
127
128        api_calls = []
129        for item in sorted(api_items):
130          if len(item.split('.')) < 3:
131            continue
132          if item.endswith('.removeListener') or item.endswith('.hasListener'):
133            continue
134          if item.endswith('.addListener'):
135            item = item[:-len('.addListener')]
136          if item.startswith('chrome.'):
137            item = item[len('chrome.'):]
138          ref_data = self._ref_resolver.GetLink(item)
139          if ref_data is None:
140            continue
141          api_calls.append({
142            'name': ref_data['text'],
143            'link': ref_data['href']
144          })
145        try:
146          manifest_data = self._GetDataFromManifest(sample_path, file_system)
147        except FileNotFoundError as e:
148          logging.warning('Error getting data from samples manifest: %s. If '
149                          'this file was deleted from a sample this message '
150                          'should go away in 5 minutes.' % e)
151          continue
152        if manifest_data is None:
153          continue
154
155        sample_base_path = sample_path.split('/', 1)[1]
156        if is_apps:
157          url = url_constants.GITHUB_BASE + '/' + sample_base_path
158          icon_base = url_constants.RAW_GITHUB_BASE + '/' + sample_base_path
159          download_url = url
160        else:
161          url = sample_base_path
162          icon_base = sample_base_path
163          download_url = sample_base_path + '.zip'
164
165        if manifest_data['icon'] is None:
166          icon_path = self._static_path + DEFAULT_ICON_PATH
167        else:
168          icon_path = icon_base + '/' + manifest_data['icon']
169        manifest_data.update({
170          'icon': icon_path,
171          'id': hashlib.md5(url).hexdigest(),
172          'download_url': download_url,
173          'url': url,
174          'files': [f.replace(sample_path + '/', '') for f in sample_files],
175          'api_calls': api_calls
176        })
177        samples_list.append(manifest_data)
178
179      return samples_list
180
181  def __init__(self,
182               extensions_cache,
183               apps_cache,
184               extension_samples_path,
185               request):
186    self._extensions_cache = extensions_cache
187    self._apps_cache = apps_cache
188    self._extension_samples_path = extension_samples_path
189    self._request = request
190
191  def _GetAcceptedLanguages(self):
192    accept_language = self._request.headers.get('Accept-Language', None)
193    if accept_language is None:
194      return []
195    return [lang_with_q.split(';')[0].strip()
196            for lang_with_q in accept_language.split(',')]
197
198  def FilterSamples(self, key, api_name):
199    """Fetches and filters the list of samples specified by |key|, returning
200    only the samples that use the API |api_name|. |key| is either 'apps' or
201    'extensions'.
202    """
203    api_search = api_name + '_'
204    samples_list = []
205    try:
206      for sample in self.get(key):
207        api_calls_unix = [model.UnixName(call['name'])
208                          for call in sample['api_calls']]
209        for call in api_calls_unix:
210          if call.startswith(api_search):
211            samples_list.append(sample)
212            break
213    except NotImplementedError:
214      # If we're testing, the GithubFileSystem can't fetch samples.
215      # Bug: http://crbug.com/141910
216      return []
217    return samples_list
218
219  def _CreateSamplesDict(self, key):
220    if key == 'apps':
221      samples_list = self._apps_cache.GetFromFileListing('/')
222    else:
223      samples_list = self._extensions_cache.GetFromFileListing(
224          self._extension_samples_path + '/')
225    return_list = []
226    for dict_ in samples_list:
227      name = dict_['name']
228      description = dict_['description']
229      if description is None:
230        description = ''
231      if name.startswith('__MSG_') or description.startswith('__MSG_'):
232        try:
233          # Copy the sample dict so we don't change the dict in the cache.
234          sample_data = dict_.copy()
235          name_key = name[len('__MSG_'):-len('__')]
236          description_key = description[len('__MSG_'):-len('__')]
237          locale = sample_data['default_locale']
238          for lang in self._GetAcceptedLanguages():
239            if lang in sample_data['locales']:
240              locale = lang
241              break
242          locale_data = sample_data['locales'][locale]
243          sample_data['name'] = locale_data[name_key]['message']
244          sample_data['description'] = locale_data[description_key]['message']
245        except Exception as e:
246          logging.error(e)
247          # Revert the sample to the original dict.
248          sample_data = dict_
249        return_list.append(sample_data)
250      else:
251        return_list.append(dict_)
252    return return_list
253
254  def get(self, key):
255    return {
256      'apps': lambda: self._CreateSamplesDict('apps'),
257      'extensions': lambda: self._CreateSamplesDict('extensions')
258    }.get(key, lambda: {})()
259