content_providers.py revision effb81e5f8246d0db0270817048dc992db66e9fb
10f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)# Copyright 2013 The Chromium Authors. All rights reserved. 20f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 30f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)# found in the LICENSE file. 40f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 50f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)import logging 65d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import os 75d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import traceback 80f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 90f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)from chroot_file_system import ChrootFileSystem 100f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)from content_provider import ContentProvider 115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import environment 125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from extensions_paths import CONTENT_PROVIDERS, LOCAL_DEBUG_DIR 13effb81e5f8246d0db0270817048dc992db66e9fbBen Murdochfrom future import Future 145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from local_file_system import LocalFileSystem 150f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)from third_party.json_schema_compiler.memoize import memoize 160f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 170f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)_IGNORE_MISSING_CONTENT_PROVIDERS = [False] 195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def IgnoreMissingContentProviders(fn): 225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '''Decorates |fn| to ignore missing content providers during its run. 235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ''' 245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def run(*args, **optargs): 255d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) saved = _IGNORE_MISSING_CONTENT_PROVIDERS[0] 265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) _IGNORE_MISSING_CONTENT_PROVIDERS[0] = True 275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) try: 285d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return fn(*args, **optargs) 295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) finally: 305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) _IGNORE_MISSING_CONTENT_PROVIDERS[0] = saved 315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return run 325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 340f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)class ContentProviders(object): 350f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) '''Implements the content_providers.json configuration; see 360f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) chrome/common/extensions/docs/templates/json/content_providers.json for its 370f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) current state and a description of the format. 380f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 390f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) Returns ContentProvider instances based on how they're configured there. 400f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) ''' 410f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 42f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) def __init__(self, 435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) object_store_creator, 44f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) compiled_fs_factory, 45f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) host_file_system, 465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) github_file_system_provider, 475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) gcs_file_system_provider): 485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._object_store_creator = object_store_creator 490f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) self._compiled_fs_factory = compiled_fs_factory 500f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) self._host_file_system = host_file_system 51f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) self._github_file_system_provider = github_file_system_provider 525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._gcs_file_system_provider = gcs_file_system_provider 535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._cache = None 545d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 555d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # If running the devserver and there is a LOCAL_DEBUG_DIR, we 565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # will read the content_provider configuration from there instead 575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # of fetching it from SVN trunk or patch. 585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if environment.IsDevServer() and os.path.exists(LOCAL_DEBUG_DIR): 595d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) local_fs = LocalFileSystem(LOCAL_DEBUG_DIR) 605d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) conf_stat = None 615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) try: 625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) conf_stat = local_fs.Stat(CONTENT_PROVIDERS) 635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) except: 645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) pass 655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if conf_stat: 675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) logging.warn(("Using local debug folder (%s) for " 685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) "content_provider.json configuration") % LOCAL_DEBUG_DIR) 695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._cache = compiled_fs_factory.ForJson(local_fs) 705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if not self._cache: 725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._cache = compiled_fs_factory.ForJson(host_file_system) 730f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 740f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) @memoize 750f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) def GetByName(self, name): 760f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) '''Gets the ContentProvider keyed by |name| in content_providers.json, or 770f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) None of there is no such content provider. 780f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) ''' 790f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) config = self._GetConfig().get(name) 800f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) if config is None: 810f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) logging.error('No content provider found with name "%s"' % name) 820f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) return None 830f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) return self._CreateContentProvider(name, config) 840f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 850f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) @memoize 860f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) def GetByServeFrom(self, path): 875d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '''Gets a (content_provider, serve_from, path_in_content_provider) tuple, 885d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) where content_provider is the ContentProvider with the longest "serveFrom" 895d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) property that is a subpath of |path|, serve_from is that property, and 905d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) path_in_content_provider is the remainder of |path|. 910f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 920f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) For example, if content provider A serves from "foo" and content provider B 935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) serves from "foo/bar", GetByServeFrom("foo/bar/baz") will return (B, 945d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) "foo/bar", "baz"). 950f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 965d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) Returns (None, '', |path|) if no ContentProvider serves from |path|. 970f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) ''' 980f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) serve_from_to_config = dict( 990f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) (config['serveFrom'], (name, config)) 1000f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) for name, config in self._GetConfig().iteritems()) 1010f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) path_parts = path.split('/') 1020f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) for i in xrange(len(path_parts), -1, -1): 1030f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) name_and_config = serve_from_to_config.get('/'.join(path_parts[:i])) 1040f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) if name_and_config is not None: 1050f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) return (self._CreateContentProvider(name_and_config[0], 1060f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) name_and_config[1]), 1075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '/'.join(path_parts[:i]), 1080f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) '/'.join(path_parts[i:])) 1095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return None, '', path 1100f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 1110f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) def _GetConfig(self): 112f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return self._cache.GetFromFile(CONTENT_PROVIDERS).Get() 1130f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 1140f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) def _CreateContentProvider(self, name, config): 1155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) default_extensions = config.get('defaultExtensions', ()) 1160f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) supports_templates = config.get('supportsTemplates', False) 1170f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) supports_zip = config.get('supportsZip', False) 1180f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 1190f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) if 'chromium' in config: 1200f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) chromium_config = config['chromium'] 1210f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) if 'dir' not in chromium_config: 122f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) logging.error('%s: "chromium" must have a "dir" property' % name) 1230f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) return None 1240f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) file_system = ChrootFileSystem(self._host_file_system, 1250f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) chromium_config['dir']) 1265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) elif 'gcs' in config: 1275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) gcs_config = config['gcs'] 1285d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if 'bucket' not in gcs_config: 1295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) logging.error('%s: "gcs" must have a "bucket" property' % name) 1305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return None 1315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bucket = gcs_config['bucket'] 1325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if not bucket.startswith('gs://'): 1335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) logging.error('%s: bucket %s should start with gs://' % (name, bucket)) 1345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return None 1355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bucket = bucket[len('gs://'):] 1365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) file_system = self._gcs_file_system_provider.Create(bucket) 1375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if 'dir' in gcs_config: 1385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) file_system = ChrootFileSystem(file_system, gcs_config['dir']) 1395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 140f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) elif 'github' in config: 141f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) github_config = config['github'] 142f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if 'owner' not in github_config or 'repo' not in github_config: 143f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) logging.error('%s: "github" must provide an "owner" and "repo"' % name) 144f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return None 145f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) file_system = self._github_file_system_provider.Create( 146f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) github_config['owner'], github_config['repo']) 147f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if 'dir' in github_config: 148f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) file_system = ChrootFileSystem(file_system, github_config['dir']) 1495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1500f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) else: 1515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) logging.error('%s: content provider type not supported' % name) 1520f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) return None 1530f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 1540f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) return ContentProvider(name, 1550f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) self._compiled_fs_factory, 1560f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) file_system, 1575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._object_store_creator, 1585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) default_extensions=default_extensions, 1590f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) supports_templates=supports_templates, 1600f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) supports_zip=supports_zip) 1610f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 1620f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) def Cron(self): 1635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def safe(name, action, callback): 1645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '''Safely runs |callback| for a ContentProvider called |name| by 1655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) swallowing exceptions and turning them into a None return value. It's 1665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) important to run all ContentProvider Crons even if some of them fail. 1675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ''' 1685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) try: 1695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return callback() 1705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) except: 1715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if not _IGNORE_MISSING_CONTENT_PROVIDERS[0]: 1725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) logging.error('Error %s Cron for ContentProvider "%s":\n%s' % 1735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) (action, name, traceback.format_exc())) 1745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return None 1755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) futures = [(name, safe(name, 1775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'initializing', 1785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._CreateContentProvider(name, config).Cron)) 179f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) for name, config in self._GetConfig().iteritems()] 180effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch return Future(callback= 181effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch lambda: [safe(name, 'resolving', f.Get) for name, f in futures if f]) 182