17dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch# Copyright 2013 The Chromium Authors. All rights reserved.
21e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be
31e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)# found in the LICENSE file.
41e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)
51e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)"""Implementation of CloudBucket using Google Cloud Storage as the backend."""
63551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import os
77dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport sys
83551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)
94e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)# boto is located in depot_tools/third_party/
10f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)if 'DEPOT_TOOLS' in os.environ:
11f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  sys.path.append(os.path.join(os.getenv('DEPOT_TOOLS'), 'third_party'))
12f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)try:
13f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  import boto
14f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)except ImportError:
15f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  print ("Set 'DEPOT_TOOLS' environment variable to the location of "
16f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)         "depot_tools.")
17f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  raise
187dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch
191e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)from ..common import cloud_bucket
207dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch
217dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch
224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)class BotoCloudBucket(cloud_bucket.BaseCloudBucket):
234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)  """Interfaces with GS using the boto library."""
247dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch
257dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch  def __init__(self, key, secret, bucket_name):
267dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    """Initializes the bucket with a key, secret, and bucket_name.
277dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch
287dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    Args:
294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      key: the API key to access GS.
304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      secret: the API secret to access GS.
317dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch      bucket_name: the name of the bucket to connect to.
327dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    """
337dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    uri = boto.storage_uri('', 'gs')
347dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    conn = uri.connect(key, secret)
357dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    self.bucket = conn.get_bucket(bucket_name)
367dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch
374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)  def _GetKey(self, path):
384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    key = boto.gs.key.Key(self.bucket)
394e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    key.key = path
404e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    return key
414e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)
427dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch  # override
437dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch  def UploadFile(self, path, contents, content_type):
444e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    key = self._GetKey(path)
457dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    key.set_metadata('Content-Type', content_type)
467dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    key.set_contents_from_string(contents)
474e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    # Open permissions for the appengine account to read/write.
484e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    key.add_email_grant('FULL_CONTROL',
494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)        'ispy.google.com@appspot.gserviceaccount.com')
507dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch
517dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch  # override
527dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch  def DownloadFile(self, path):
534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    key = self._GetKey(path)
547dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    if key.exists():
557dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch      return key.get_contents_as_string()
567dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    else:
577dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch      raise cloud_bucket.FileNotFoundError
587dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch
597dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch  # override
604e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)  def UpdateFile(self, path, contents):
614e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    key = self._GetKey(path)
624e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    if key.exists():
634e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      key.set_contents_from_string(contents)
644e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    else:
654e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      raise cloud_bucket.FileNotFoundError
664e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)
674e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)  # override
68bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch  def RemoveFile(self, path):
694e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    key = self._GetKey(path)
70bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch    key.delete()
71bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch
72bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch  # override
73bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch  def FileExists(self, path):
744e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    key = self._GetKey(path)
75bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch    return key.exists()
76bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch
77bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch  # override
784e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)  def GetImageURL(self, path):
794e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    key = self._GetKey(path)
807dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    if key.exists():
813551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)      # Corrects a bug in boto that incorrectly generates a url
823551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)      #  to a resource in Google Cloud Storage.
833551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)      return key.generate_url(3600).replace('AWSAccessKeyId', 'GoogleAccessId')
847dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    else:
857dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch      raise cloud_bucket.FileNotFoundError(path)
867dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch
877dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch  # override
887dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch  def GetAllPaths(self, prefix):
897dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch    return (key.key for key in self.bucket.get_all_keys(prefix=prefix))
90