blob: a2ff2952ccd481737d3e20d7b8527aa2d1c9916c [file] [log] [blame]
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Set of helpers for interacting with Google Cloud Storage."""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import logging
import os
from six.moves import urllib
import uuid
from datetime import datetime, timedelta
from google.appengine.api import app_identity
from google.appengine.api import images
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from third_party import cloudstorage
from third_party.cloudstorage import errors
from framework import filecontent
from framework import framework_constants
from framework import framework_helpers
ATTACHMENT_TTL = timedelta(seconds=30)
IS_DEV_APPSERVER = (
'development' in os.environ.get('SERVER_SOFTWARE', '').lower())
RESIZABLE_MIME_TYPES = [
'image/png', 'image/jpg', 'image/jpeg', 'image/gif', 'image/webp',
]
DEFAULT_THUMB_WIDTH = 250
DEFAULT_THUMB_HEIGHT = 200
LOGO_THUMB_WIDTH = 110
LOGO_THUMB_HEIGHT = 30
MAX_ATTACH_SIZE_TO_COPY = 10 * 1024 * 1024 # 10 MB
# GCS signatures are valid for 10 minutes by default, but cache them for
# 5 minutes just to be on the safe side.
GCS_SIG_TTL = 60 * 5
def _Now():
return datetime.utcnow()
class UnsupportedMimeType(Exception):
pass
def DeleteObjectFromGCS(object_id):
object_path = ('/' + app_identity.get_default_gcs_bucket_name() + object_id)
cloudstorage.delete(object_path)
def StoreObjectInGCS(
content, mime_type, project_id, thumb_width=DEFAULT_THUMB_WIDTH,
thumb_height=DEFAULT_THUMB_HEIGHT, filename=None):
bucket_name = app_identity.get_default_gcs_bucket_name()
guid = uuid.uuid4()
object_id = '/%s/attachments/%s' % (project_id, guid)
object_path = '/' + bucket_name + object_id
options = {}
if filename:
if not framework_constants.FILENAME_RE.match(filename):
logging.info('bad file name: %s' % filename)
filename = 'attachment.dat'
options['Content-Disposition'] = 'inline; filename="%s"' % filename
logging.info('Writing with options %r', options)
with cloudstorage.open(object_path, 'w', mime_type, options=options) as f:
f.write(content)
if mime_type in RESIZABLE_MIME_TYPES:
# Create and save a thumbnail too.
thumb_content = None
try:
thumb_content = images.resize(content, thumb_width, thumb_height)
except images.LargeImageError:
# Don't log the whole exception because we don't need to see
# this on the Cloud Error Reporting page.
logging.info('Got LargeImageError on image with %d bytes', len(content))
except Exception, e:
# Do not raise exception for incorrectly formed images.
# See https://bugs.chromium.org/p/monorail/issues/detail?id=597 for more
# detail.
logging.exception(e)
if thumb_content:
thumb_path = '%s-thumbnail' % object_path
with cloudstorage.open(thumb_path, 'w', 'image/png') as f:
f.write(thumb_content)
return object_id
def CheckMimeTypeResizable(mime_type):
if mime_type not in RESIZABLE_MIME_TYPES:
raise UnsupportedMimeType(
'Please upload a logo with one of the following mime types:\n%s' %
', '.join(RESIZABLE_MIME_TYPES))
def StoreLogoInGCS(file_name, content, project_id):
mime_type = filecontent.GuessContentTypeFromFilename(file_name)
CheckMimeTypeResizable(mime_type)
if '\\' in file_name: # IE insists on giving us the whole path.
file_name = file_name[file_name.rindex('\\') + 1:]
return StoreObjectInGCS(
content, mime_type, project_id, thumb_width=LOGO_THUMB_WIDTH,
thumb_height=LOGO_THUMB_HEIGHT)
@framework_helpers.retry(3, delay=0.25, backoff=1.25)
def _FetchSignedURL(url):
"""Request that devstorage API signs a GCS content URL."""
resp = urlfetch.fetch(url, follow_redirects=False)
redir = resp.headers["Location"]
return redir
def SignUrl(bucket, object_id):
"""Get a signed URL to download a GCS object.
Args:
bucket: string name of the GCS bucket.
object_id: string object ID of the file within that bucket.
Returns:
A signed URL, or '/mising-gcs-url' if signing failed.
"""
try:
cache_key = 'gcs-object-url-%s' % object_id
cached = memcache.get(key=cache_key)
if cached is not None:
return cached
if IS_DEV_APPSERVER:
attachment_url = '/_ah/gcs/%s%s' % (bucket, object_id)
else:
result = ('https://www.googleapis.com/storage/v1/b/'
'{bucket}/o/{object_id}?access_token={token}&alt=media')
scopes = ['https://www.googleapis.com/auth/devstorage.read_only']
if object_id[0] == '/':
object_id = object_id[1:]
url = result.format(
bucket=bucket,
object_id=urllib.parse.quote_plus(object_id),
token=app_identity.get_access_token(scopes)[0])
attachment_url = _FetchSignedURL(url)
if not memcache.set(key=cache_key, value=attachment_url, time=GCS_SIG_TTL):
logging.error('Could not cache gcs url %s for %s', attachment_url,
object_id)
return attachment_url
except Exception as e:
logging.exception(e)
return '/missing-gcs-url'
def MaybeCreateDownload(bucket_name, object_id, filename):
"""If the obj is not huge, and no download version exists, create it."""
src = '/%s%s' % (bucket_name, object_id)
dst = '/%s%s-download' % (bucket_name, object_id)
cloudstorage.validate_file_path(src)
cloudstorage.validate_file_path(dst)
logging.info('Maybe create %r from %r', dst, src)
if IS_DEV_APPSERVER:
logging.info('dev environment never makes download copies.')
return False
# If "Download" object already exists, we are done.
try:
cloudstorage.stat(dst)
logging.info('Download version of attachment already exists')
return True
except errors.NotFoundError:
pass
# If "View" object is huge, give up.
src_stat = cloudstorage.stat(src)
if src_stat.st_size > MAX_ATTACH_SIZE_TO_COPY:
logging.info('Download version of attachment would be too big')
return False
with cloudstorage.open(src, 'r') as infile:
content = infile.read()
logging.info('opened GCS object and read %r bytes', len(content))
content_type = src_stat.content_type
options = {
'Content-Disposition': 'attachment; filename="%s"' % filename,
}
logging.info('Writing with options %r', options)
with cloudstorage.open(dst, 'w', content_type, options=options) as outfile:
outfile.write(content)
logging.info('done writing')
return True