blob: 452ffdddf7a818e2ccb072e6df0a70d2ee90f275 [file] [log] [blame]
#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""A handler that displays information about datastore entities."""
import cgi
import datetime
import math
import time
import types
import urllib
from google.appengine.api import apiproxy_stub_map
from google.appengine.api import datastore
from google.appengine.api import datastore_types
from google.appengine.api import memcache
from google.appengine.api import users
from google.appengine.ext import db
from google.appengine.ext.db import metadata
from google.appengine.tools.devappserver2.admin import admin_request_handler
def _format_datastore_key(key):
"""Return a nicely formatted decomposition of a datastore key.
Args:
key: The datastore_types.Key object to format.
Returns:
A string or Unicode object containing nicely formatted information about
the given key e.g. "ParentKind: name=Animal > ChildKind: id=123".
"""
path = key.to_path() # [kind, id/name, kind, id/name, ...]
parts = []
for i in range(0, len(path)//2):
kind = path[i*2]
value = path[i*2 + 1]
if isinstance(value, (int, long)):
parts.append('%s: id=%d' % (kind, value))
else:
parts.append('%s: name=%s' % (kind, value))
return ' > '.join(parts)
def _property_name_to_values(entities):
"""Returns a a mapping of entity property names to a list of their values.
For example:
_property_name_to_values([{'cat': 5, 'dog': 10},
{'dog': 15, 'mouse': 'happy'}])
=> {'cat': [5], 'dog': [10, 15], 'mouse': ['happy']}
Args:
entities: A sequence of mappings (i.e. datastore.Entity) that represent
datastore properties and their values.
Returns:
A dict whose keys are the union of the keys of the given entities and
whose values are the list of values for those keys.
"""
property_name_to_values = {}
for entity in entities:
for property_name, value in entity.iteritems():
property_name_to_values.setdefault(property_name, []).append(value)
return property_name_to_values
def _property_name_to_value(entities):
"""Returns a a mapping of entity property names to a sample value.
For example:
_property_name_to_value([{'cat': 5, 'dog': 10},
{'dog': 15, 'mouse': 'happy'}])
=> {'cat': 5, 'dog': 10 or 15, 'mouse': 'happy'}
Args:
entities: A sequence of mappings (i.e. datastore.Entity) that represent
datastore properties and their values.
Returns:
A dict whose keys are the union of the keys of the given entities and
whose values are arbitrarily chosen from the key values of the given
entities.
"""
return {key: values[0]
for (key, values) in _property_name_to_values(entities).iteritems()}
def _get_entities(kind, namespace, order, start, count):
"""Returns a list and a count of entities of the given kind.
Args:
kind: A string representing the name of the kind of the entities to
return.
namespace: A string representing the namespace of the entities to return.
order: A string containing the name of the property to sorted the results
by. A "-" prefix indicates descending order e.g. "-age".
start: The number of initial entities to skip in the result set.
count: The maximum number of entities to return.
Returns:
A tuple of (list of datastore.Entity, total entity count).
"""
query = datastore.Query(kind, _namespace=namespace)
if order:
if order.startswith('-'):
direction = datastore.Query.DESCENDING
order = order[1:]
else:
direction = datastore.Query.ASCENDING
query.Order((order, direction))
total = query.Count()
entities = query.Get(count, start)
return entities, total
class DataType(object):
"""A DataType represents a data type in the datastore.
Each DataType subtype defines four methods:
format: returns a formatted string for a datastore value
input_field: returns a string HTML <input> element for this DataType
name: the friendly string name of this DataType
parse: parses the formatted string representation of this DataType
We use DataType instances to display formatted values in our result lists,
and we uses input_field/format/parse to generate forms and parse the results
from those forms to allow editing of entities.
"""
_MAX_SHORT_LENGTH = 30
# The value of the placeholder attribute generated by .input_field e.g.
# <input ... placeholder="11-12-1974 12:12:53">.
PLACEHOLDER = None
@staticmethod
def get(value):
return _DATA_TYPES[value.__class__]
@staticmethod
def get_by_name(name):
return _NAMED_DATA_TYPES[name]
@classmethod
def get_placholder_attribute(cls):
if cls.PLACEHOLDER:
return 'placeholder="%s"' % cgi.escape(cls.PLACEHOLDER)
else:
return ''
def format(self, value):
if isinstance(value, types.StringTypes):
return value
else:
return str(value)
def short_format(self, value):
format = self.format(value)
if len(format) > self._MAX_SHORT_LENGTH:
return format[:self._MAX_SHORT_LENGTH-3] + '...'
else:
return format
def input_field(self, name, value, sample_values, back_uri):
string_value = self.format(value) if value else ''
return (
'<input class="%s" name="%s" type="text" size="%d" value="%s" %s/>' % (
cgi.escape(self.name()),
cgi.escape(name),
self.input_field_size(),
cgi.escape(string_value, True),
self.get_placholder_attribute()))
def input_field_size(self):
return 30
class StringType(DataType):
def input_field(self, name, value, sample_values, back_uri):
string_value = self.format(value) if value else ''
sample_values = [self.format(s) for s in sample_values]
multiline = False
if value:
multiline = len(string_value) > 255 or string_value.find('\n') >= 0
if not multiline:
for sample_value in sample_values:
if sample_value and (len(sample_value) > 255 or
sample_value.find('\n') >= 0):
multiline = True
break
if multiline:
return '<textarea name="%s" rows="5" cols="50" %s>%s</textarea>' % (
cgi.escape(name),
self.get_placholder_attribute(),
cgi.escape(string_value))
else:
return DataType.input_field(self, name, value, sample_values, back_uri)
def name(self):
return 'string'
def parse(self, value):
return value
def input_field_size(self):
return 50
class TextType(StringType):
def name(self):
return 'Text'
def input_field(self, name, value, sample_values, back_uri):
string_value = self.format(value) if value else ''
return '<textarea name="%s" rows="5" cols="50" %s>%s</textarea>' % (
cgi.escape(name),
self.get_placholder_attribute(),
cgi.escape(string_value))
def parse(self, value):
return datastore_types.Text(value)
class ByteStringType(StringType):
def format(self, value):
# Format ByteString values as escaped Python strings.
if value is None:
return 'None'
r = value.encode('string-escape')
return r
def name(self):
return 'ByteString'
def parse(self, value):
# Parse escaped Python strings to ByteString values.
# It is an error if the string contains non-ASCII values.
bytestring = value.encode('ascii').decode('string-escape')
return datastore_types.ByteString(bytestring)
class BlobType(StringType):
def name(self):
return 'Blob'
def input_field(self, name, value, sample_values, back_uri):
return '&lt;binary&gt;'
def format(self, value):
return '<binary>'
class EmbeddedEntityType(BlobType):
def name(self):
return 'entity:proto'
class TimeType(DataType):
_FORMAT = '%Y-%m-%d %H:%M:%S'
PLACEHOLDER = '2009-12-24 23:59:59'
def format(self, value):
return value.isoformat(' ')[0:19]
def name(self):
return 'datetime'
def parse(self, value):
return datetime.datetime(*(time.strptime(value,
TimeType._FORMAT)[0:6]))
class OverflowTimeType(TimeType):
def format(self, value):
return str(value)
def name(self):
return 'overflowdatetime'
def parse(self, value):
try:
return datastore_types._OverflowDateTime(value)
except ValueError:
return super(OverflowTimeType, self).parse(value)
class ListType(DataType):
def name(self):
return 'list'
def input_field(self, name, value, sample_values, back_uri):
string_value = self.format(value) if value else ''
return cgi.escape(string_value)
class BoolType(DataType):
def name(self):
return 'bool'
def input_field(self, name, value, sample_values, back_uri):
selected = {None: '', False: '', True: ''}
selected[value] = 'selected'
return """<select class="%s" name="%s">
<option %s value=''></option>
<option %s value='0'>False</option>
<option %s value='1'>True</option></select>""" % (
cgi.escape(self.name()), cgi.escape(name), selected[None],
selected[False], selected[True])
def parse(self, value):
if value.lower() == 'true':
return True
if value.lower() == 'false':
return False
# Otherwise treat as an int
return bool(int(value))
class NumberType(DataType):
def input_field(self, name, value, sample_values, back_uri):
string_value = self.format(value) if value is not None else ''
return super(NumberType, self).input_field(name,
string_value,
sample_values,
back_uri)
class IntType(NumberType):
PLACEHOLDER = '42'
def input_field_size(self):
return 10
def name(self):
return 'int'
def parse(self, value):
return int(value)
class FloatType(NumberType):
PLACEHOLDER = '3.14159'
def name(self):
return 'float'
def parse(self, value):
return float(value)
class UserType(DataType):
PLACEHOLDER = 'john@example.com'
def name(self):
return 'User'
def parse(self, value):
return users.User(value)
def input_field_size(self):
return 15
# This is incomplete, but enough to make the system still work.
class ReferenceType(DataType):
def name(self):
return 'Key'
def short_format(self, value):
return str(value)[:8] + '...'
def parse(self, value):
return datastore_types.Key(value)
def input_field(self, name, value, sample_values, back_uri):
string_value = self.format(value) if value else ''
html = '<input class="%s" name="%s" type="text" size="%d" value="%s"/>' % (
cgi.escape(self.name()), cgi.escape(name), self.input_field_size(),
cgi.escape(string_value, True))
if value:
html += '<br><a href="/datastore/edit/%s?next=%s">%s</a>' % (
cgi.escape(string_value, True),
urllib.quote_plus(back_uri),
cgi.escape(_format_datastore_key(value), True))
return html
def input_field_size(self):
return 85
class EmailType(StringType):
PLACEHOLDER = 'john@example.com'
def name(self):
return 'Email'
def parse(self, value):
return datastore_types.Email(value)
class CategoryType(StringType):
def name(self):
return 'Category'
def parse(self, value):
return datastore_types.Category(value)
class LinkType(StringType):
PLACEHOLDER = 'http://www.example.com/'
def name(self):
return 'Link'
def parse(self, value):
return datastore_types.Link(value)
class GeoPtType(DataType):
PLACEHOLDER = '33.86,-151.2'
def name(self):
return 'GeoPt'
def parse(self, value):
return datastore_types.GeoPt(value)
class ImType(DataType):
PLACEHOLDER = 'xmpp john@example.com'
def name(self):
return 'IM'
def parse(self, value):
return datastore_types.IM(value)
class PhoneNumberType(StringType):
def name(self):
return 'PhoneNumber'
def parse(self, value):
return datastore_types.PhoneNumber(value)
class PostalAddressType(StringType):
def name(self):
return 'PostalAddress'
def parse(self, value):
return datastore_types.PostalAddress(value)
class RatingType(DataType):
PLACEHOLDER = '93'
def input_field_size(self):
return 5
def name(self):
return 'Rating'
def parse(self, value):
return datastore_types.Rating(value)
class NoneType(DataType):
def name(self):
return 'None'
def parse(self, value):
return None
def format(self, value):
return 'None'
class BlobKeyType(StringType):
def name(self):
return 'BlobKey'
def parse(self, value):
return datastore_types.BlobKey(value)
# Maps Pyathon/datatstore types to DataType instances
_DATA_TYPES = {
types.NoneType: NoneType(),
types.StringType: StringType(),
types.UnicodeType: StringType(),
datastore_types.Text: TextType(),
datastore_types.Blob: BlobType(),
datastore_types.EmbeddedEntity: EmbeddedEntityType(),
types.BooleanType: BoolType(),
types.IntType: IntType(),
types.LongType: IntType(),
types.FloatType: FloatType(),
datetime.datetime: TimeType(),
datastore_types._OverflowDateTime: OverflowTimeType(),
users.User: UserType(),
datastore_types.Key: ReferenceType(),
types.ListType: ListType(),
datastore_types.Email: EmailType(),
datastore_types.Category: CategoryType(),
datastore_types.Link: LinkType(),
datastore_types.GeoPt: GeoPtType(),
datastore_types.IM: ImType(),
datastore_types.PhoneNumber: PhoneNumberType(),
datastore_types.PostalAddress: PostalAddressType(),
datastore_types.Rating: RatingType(),
datastore_types.BlobKey: BlobKeyType(),
datastore_types.ByteString: ByteStringType(),
}
_NAMED_DATA_TYPES = {}
for _data_type in _DATA_TYPES.values():
_NAMED_DATA_TYPES[_data_type.name()] = _data_type
class DatastoreRequestHandler(admin_request_handler.AdminRequestHandler):
"""A handler that displays information about datastore entities."""
NUM_ENTITIES_PER_PAGE = 20
@staticmethod
def _calculate_writes_for_built_in_indices(entity):
writes = 0
for prop_name in entity.keys():
if not prop_name in entity.unindexed_properties():
# 2 writes per property value, one for EntitiesByProperty and one for
# EntitiesbyPropertyDesc
prop_vals = entity[prop_name]
if isinstance(prop_vals, (list)):
num_prop_vals = len(prop_vals)
else:
num_prop_vals = 1
writes += 2 * num_prop_vals
return writes
@staticmethod
def _calculate_writes_for_composite_index(entity, index):
composite_index_value_count = 1
for prop_name, _ in index.Properties():
if not prop_name in entity.keys() or (
prop_name in entity.unindexed_properties()):
return 0
prop_vals = entity[prop_name]
if isinstance(prop_vals, (list)):
composite_index_value_count *= len(prop_vals)
# If this is an ancestor index we're going to duplicate all these index
# writes for every key in the hierarchy. So if the entity key has no
# parent we write the index values once, if the entity key has a depth of
# 2 then we write the index values twice, and so on.
ancestor_count = 1 # A key is its own ancestor.
if index.HasAncestor():
key = entity.key().parent()
while key is not None:
ancestor_count += 1
key = key.parent()
return composite_index_value_count * ancestor_count
@classmethod
def _get_write_ops(cls, entity):
# Minimum 2 writes, one for the entity and one for the EntitiesByKind index.
writes = 2 + cls._calculate_writes_for_built_in_indices(entity)
# Account for composite indices.
for index, _ in datastore.GetIndexes():
if index.Kind() != entity.kind():
continue
writes += cls._calculate_writes_for_composite_index(entity, index)
return writes
@staticmethod
def _get_kinds(namespace):
"""Return a sorted list of kind names present in the given namespace."""
assert namespace is not None
q = metadata.Kind.all(namespace=namespace)
return sorted([x.kind_name for x in q.run()])
@classmethod
def _get_entity_template_data(cls,
request_uri,
kind,
namespace,
order,
start):
entities, total_entities = _get_entities(kind,
namespace,
order,
start,
cls.NUM_ENTITIES_PER_PAGE)
property_name_to_value = _property_name_to_value(entities)
headers = [{'name': property_name}
for property_name in sorted(property_name_to_value)]
template_entities = []
for entity in entities:
attributes = []
for property_name in sorted(property_name_to_value):
if property_name in entity:
raw_value = entity[property_name]
data_type = DataType.get(raw_value)
value = data_type.format(raw_value)
short_value = data_type.short_format(raw_value)
else:
value = ''
short_value = ''
attributes.append({'name': property_name,
'value': value,
'short_value': short_value,
})
edit_uri = '/datastore/edit/%s?next=%s' % (
entity.key(), urllib.quote(request_uri))
template_entities.append(
{'attributes': attributes,
'edit_uri': edit_uri,
'key': entity.key(),
'key_id': entity.key().id(),
'key_name': entity.key().name(),
'shortened_key': str(entity.key())[:8] + '...',
'write_ops': cls._get_write_ops(entity)})
return headers, template_entities, total_entities
def get(self):
# Force all transactions to complete to show the user consistent results.
datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
datastore_stub.Flush()
kind = self.request.get('kind', None)
namespace = self.request.get('namespace', '')
order = self.request.get('order', None)
message = self.request.get('message', None)
try:
page = int(self.request.get('page', '1'))
except ValueError:
page = 1
kinds = self._get_kinds(namespace)
if not kind and kinds:
self.redirect(self._construct_url(add={'kind': kinds[0]}))
return
if kind:
start = (page-1) * self.NUM_ENTITIES_PER_PAGE
headers, template_entities, total_entities = (
self._get_entity_template_data(
self.request.uri,
kind,
namespace,
order,
start))
num_pages = int(math.ceil(float(total_entities) /
self.NUM_ENTITIES_PER_PAGE))
else:
start = 0
headers = []
template_entities = []
total_entities = 0
num_pages = 0
select_namespace_url = self._construct_url(
remove=['message'],
add={'namespace': self.request.get('namespace')})
self.response.write(self.render(
'datastore_viewer.html',
{'entities': template_entities,
'headers': headers,
'kind': kind,
'kinds': kinds,
'message': message,
'namespace': namespace,
'num_pages': num_pages,
'order': order,
'order_base_url': self._construct_url(remove=['message', 'order']),
'page': page,
'paging_base_url': self._construct_url(remove=['message', 'page']),
'select_namespace_url': select_namespace_url,
'show_namespace': self.request.get('namespace', None) is not None,
'start': start,
'total_entities': total_entities}))
def post(self):
"""Handle modifying actions and redirect to a GET page."""
if self.request.get('action:flush_memcache'):
if memcache.flush_all():
message = 'Cache flushed, all keys dropped.'
else:
message = 'Flushing the cache failed. Please try again.'
self.redirect(self._construct_url(remove=['action:flush_memcache'],
add={'message': message}))
elif self.request.get('action:delete_entities'):
entity_keys = self.request.params.getall('entity_key')
db.delete(entity_keys)
self.redirect(self._construct_url(
remove=['action:delete_entities'],
add={'message': '%d entities deleted' % len(entity_keys)}))
else:
self.error(404)
class DatastoreEditRequestHandler(admin_request_handler.AdminRequestHandler):
"""A handler that allows datastore entities to be created and edited."""
def get(self, entity_key_string=None):
if entity_key_string:
entity_key = datastore.Key(entity_key_string)
entity_key_name = entity_key.name()
entity_key_id = entity_key.id()
namespace = entity_key.namespace()
kind = entity_key.kind()
entities = [datastore.Get(entity_key)]
parent_key = entity_key.parent()
if parent_key:
parent_key_string = _format_datastore_key(parent_key)
else:
parent_key_string = None
else:
entity_key = None
entity_key_string = None
entity_key_name = None
entity_key_id = None
namespace = self.request.get('namespace')
kind = self.request.get('kind')
entities, _ = _get_entities(kind,
namespace,
order=None,
start=0,
count=20)
parent_key = None
parent_key_string = None
if not entities:
self.redirect('/datastore?%s' % (
urllib.urlencode(
[('kind', kind),
('message',
'Cannot create the kind "%s" in the "%s" namespace because '
'no template entity exists.' % (kind, namespace)),
('namespace', namespace)])))
return
property_name_to_values = _property_name_to_values(entities)
fields = []
for property_name, values in sorted(property_name_to_values.iteritems()):
data_type = DataType.get(values[0])
field = data_type.input_field('%s|%s' % (data_type.name(), property_name),
values[0] if entity_key else None,
values,
self.request.uri)
fields.append((property_name, data_type.name(), field))
self.response.write(self.render(
'datastore_edit.html',
{'fields': fields,
'key': entity_key_string,
'key_id': entity_key_id,
'key_name': entity_key_name,
'kind': kind,
'namespace': namespace,
'next': self.request.get('next', '/datastore'),
'parent_key': parent_key,
'parent_key_string': parent_key_string}))
def post(self, entity_key_string=None):
if self.request.get('action:delete'):
if entity_key_string:
datastore.Delete(datastore.Key(entity_key_string))
self.redirect(str(self.request.get('next', '/datastore')))
else:
self.response.set_status(400)
return
if entity_key_string:
entity = datastore.Get(datastore.Key(entity_key_string))
else:
kind = self.request.get('kind')
namespace = self.request.get('namespace', None)
entity = datastore.Entity(kind, _namespace=namespace)
for arg_name in self.request.arguments():
# Arguments are in <property_type>|<property_name>=<value> format.
if '|' not in arg_name:
continue
data_type_name, property_name = arg_name.split('|')
form_value = self.request.get(arg_name)
data_type = DataType.get_by_name(data_type_name)
if (entity and
property_name in entity and
data_type.format(entity[property_name]) == form_value):
# If the property is unchanged then don't update it. This will prevent
# empty form values from causing the property to be deleted if the
# property was already empty.
continue
if form_value:
# TODO: Handle parse exceptions.
entity[property_name] = data_type.parse(form_value)
elif property_name in entity:
# TODO: Treating empty input as deletion is a not a good
# interface.
del entity[property_name]
datastore.Put(entity)
self.redirect(str(self.request.get('next', '/datastore')))