boto_bucket.py revision f2477e01787aa58f445919b809d89e252beef54f
17dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch# Copyright 2013 The Chromium Authors. All rights reserved. 21e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 31e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)# found in the LICENSE file. 41e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 51e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)"""Implementation of CloudBucket using Google Cloud Storage as the backend.""" 63551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import os 77dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport sys 83551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 94e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)# boto is located in depot_tools/third_party/ 10f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)if 'DEPOT_TOOLS' in os.environ: 11f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) sys.path.append(os.path.join(os.getenv('DEPOT_TOOLS'), 'third_party')) 12f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)try: 13f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) import boto 14f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)except ImportError: 15f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) print ("Set 'DEPOT_TOOLS' environment variable to the location of " 16f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) "depot_tools.") 17f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) raise 187dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 191e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)from ..common import cloud_bucket 207dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 217dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)class BotoCloudBucket(cloud_bucket.BaseCloudBucket): 234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) """Interfaces with GS using the boto library.""" 247dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 257dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch def __init__(self, key, secret, bucket_name): 267dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """Initializes the bucket with a key, secret, and bucket_name. 277dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 287dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Args: 294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) key: the API key to access GS. 304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) secret: the API secret to access GS. 317dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bucket_name: the name of the bucket to connect to. 327dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """ 337dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch uri = boto.storage_uri('', 'gs') 347dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch conn = uri.connect(key, secret) 357dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch self.bucket = conn.get_bucket(bucket_name) 367dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) def _GetKey(self, path): 384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) key = boto.gs.key.Key(self.bucket) 394e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) key.key = path 404e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return key 414e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 427dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # override 437dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch def UploadFile(self, path, contents, content_type): 444e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) key = self._GetKey(path) 457dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch key.set_metadata('Content-Type', content_type) 467dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch key.set_contents_from_string(contents) 474e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) # Open permissions for the appengine account to read/write. 484e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) key.add_email_grant('FULL_CONTROL', 494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'ispy.google.com@appspot.gserviceaccount.com') 507dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 517dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # override 527dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch def DownloadFile(self, path): 534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) key = self._GetKey(path) 547dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if key.exists(): 557dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch return key.get_contents_as_string() 567dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch else: 577dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch raise cloud_bucket.FileNotFoundError 587dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 597dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # override 604e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) def UpdateFile(self, path, contents): 614e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) key = self._GetKey(path) 624e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if key.exists(): 634e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) key.set_contents_from_string(contents) 644e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) else: 654e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) raise cloud_bucket.FileNotFoundError 664e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 674e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) # override 68bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch def RemoveFile(self, path): 694e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) key = self._GetKey(path) 70bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch key.delete() 71bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch 72bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch # override 73bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch def FileExists(self, path): 744e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) key = self._GetKey(path) 75bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch return key.exists() 76bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch 77bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch # override 784e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) def GetImageURL(self, path): 794e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) key = self._GetKey(path) 807dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if key.exists(): 813551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) # Corrects a bug in boto that incorrectly generates a url 823551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) # to a resource in Google Cloud Storage. 833551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return key.generate_url(3600).replace('AWSAccessKeyId', 'GoogleAccessId') 847dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch else: 857dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch raise cloud_bucket.FileNotFoundError(path) 867dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 877dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # override 887dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch def GetAllPaths(self, prefix): 897dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch return (key.key for key in self.bucket.get_all_keys(prefix=prefix)) 90