15d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)# Copyright 2014 The Chromium Authors. All rights reserved. 25d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 35d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)# found in the LICENSE file. 45d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 55d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import os 65d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import environment 75d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 85d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from caching_file_system import CachingFileSystem 95d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from empty_dir_file_system import EmptyDirFileSystem 105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from extensions_paths import LOCAL_GCS_DIR, LOCAL_GCS_DEBUG_CONF 115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from local_file_system import LocalFileSystem 12a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)from path_util import IsDirectory, ToDirectory 135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)class CloudStorageFileSystemProvider(object): 155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '''Provides CloudStorageFileSystem bound to a GCS bucket. 165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ''' 175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def __init__(self, object_store_creator): 185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._object_store_creator = object_store_creator 195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def Create(self, bucket): 215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '''Creates a CloudStorageFileSystemProvider. 225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) |bucket| is the name of GCS bucket, eg devtools-docs. It is expected 245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) that this bucket has Read permission for this app in its ACLs. 255d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) Optional configuration can be set in a local_debug/gcs_debug.conf file: 275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) use_local_fs=True|False 285d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) access_token=<token> 295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) remote_bucket_prefix=<prefix> 305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) If running in Preview mode or in Development mode with use_local_fs set to 325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) True, buckets and files are looked inside the local_debug folder instead 335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) of in the real GCS server. Preview server does not support direct GCS 345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) access, so it is always forced to use a LocalFileSystem. 355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) For real GCS access in the Development mode (dev_appserver.py), 375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) access_token and remote_bucket_prefix options can be 385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) used to change the way GCS files are accessed. Both are ignored in a real 395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) appengine instance. 405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) "access_token" is always REQUIRED on dev_appengine, otherwise you will 425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) get 404 (auth) errors. You can get one access_token valid for a few minutes 435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) by typing: 445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) gsutil -d ls 2>&1 | grep "Bearer" | 455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) sed "s/.*Bearer \(.*\).r.nUser-Agent.*/access_token=\1/" )" 465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) A sample output would be: 485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) access_token=ya29.1.AADtN_VW5ibbfLHV5cMIK5ss4bHtVzBXpa4byjd 495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) Now add this line to the local_debug/gcs_debug.conf file and restart the 515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) appengine development server. 525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) Remember that you will need a new access_token every ten minutes or 545d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) so. If you get 404 errors on log, update it. Access token is not 555d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) used for a deployed appengine app, only if you use dev_appengine.py. 565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) remote_bucket_prefix is useful if you want to test on your own GCS buckets 585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) before using the real GCS buckets. 595d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 605d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ''' 615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if not environment.IsReleaseServer() and not environment.IsDevServer(): 625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bucket_local_path = os.path.join(LOCAL_GCS_DIR, bucket) 635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if IsDirectory(bucket_local_path): 645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return LocalFileSystem(bucket_local_path) 655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else: 665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return EmptyDirFileSystem() 675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) debug_access_token = None 695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) debug_bucket_prefix = None 705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) use_local_fs = False 715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if environment.IsDevServer() and os.path.exists(LOCAL_GCS_DEBUG_CONF): 735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) with open(LOCAL_GCS_DEBUG_CONF, "r") as token_file: 745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) properties = dict(line.strip().split('=', 1) for line in token_file) 755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) use_local_fs = properties.get('use_local_fs', 'False')=='True' 765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) debug_access_token = properties.get('access_token', None) 775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) debug_bucket_prefix = properties.get('remote_bucket_prefix', None) 785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if environment.IsDevServer() and use_local_fs: 80a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return LocalFileSystem(ToDirectory(os.path.join(LOCAL_GCS_DIR, bucket))) 815d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # gcs_file_system has strong dependencies on runtime appengine APIs, 835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # so we only import it when we are sure we are not on preview.py or tests. 845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) from gcs_file_system import CloudStorageFileSystem 855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return CachingFileSystem(CloudStorageFileSystem(bucket, 865d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) debug_access_token, debug_bucket_prefix), 875d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._object_store_creator) 885d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 895d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) @staticmethod 905d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def ForEmpty(): 915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) class EmptyImpl(object): 925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def Create(self, bucket): 935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return EmptyDirFileSystem() 945d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return EmptyImpl() 95