| #!/usr/bin/env python |
| # |
| # Copyright 2007 Google Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| |
| |
| |
| """Generate Datastore Stats over Dev mode appserver's datastore.""" |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| import datetime |
| import logging |
| |
| from google.appengine.api import datastore |
| from google.appengine.api import datastore_admin |
| from google.appengine.api import datastore_types |
| from google.appengine.api import users |
| from google.appengine.ext.db import stats |
| |
| DELETE_BATCH_SIZE = 100 |
| |
| |
| _GLOBAL_KEY = (stats.GlobalStat, 'total_entity_usage', '') |
| |
| |
| |
| |
| |
| |
| _PROPERTY_TYPE_TO_DSS_NAME = { |
| unicode: ('String', 'STRING'), |
| bool: ('Boolean', 'BOOLEAN'), |
| long: ('Integer', 'INT64'), |
| type(None): ('NULL', 'NULL'), |
| float: ('Float', 'DOUBLE'), |
| datastore_types.Key: ('Key', 'REFERENCE'), |
| datastore_types.Blob: ('Blob', 'STRING'), |
| datastore_types.EmbeddedEntity: ('EmbeddedEntity', 'STRING'), |
| datastore_types.ByteString: ('ShortBlob', 'STRING'), |
| datastore_types.Text: ('Text', 'STRING'), |
| users.User: ('User', 'USER'), |
| datastore_types.Category: ('Category', 'STRING'), |
| datastore_types.Link: ('Link', 'STRING'), |
| datastore_types.Email: ('Email', 'STRING'), |
| datetime.datetime: ('Date/Time', 'INT64'), |
| datastore_types.GeoPt: ('GeoPt', 'POINT'), |
| datastore_types.IM: ('IM', 'STRING'), |
| datastore_types.PhoneNumber: ('PhoneNumber', 'STRING'), |
| datastore_types.PostalAddress: ('PostalAddress', 'STRING'), |
| datastore_types.Rating: ('Rating', 'INT64'), |
| datastore_types.BlobKey: ('BlobKey', 'STRING'), |
| } |
| |
| |
| |
| |
| class DatastoreStatsProcessor(object): |
| """Generates datastore stats for an app's an datastore entities.""" |
| |
| def __init__(self, _app=None): |
| self.app_id = datastore_types.ResolveAppId(_app) |
| |
| |
| self.whole_app_stats = {} |
| |
| |
| |
| self.namespace_stats = {} |
| self.found_non_empty_namespace = False |
| |
| |
| self.old_stat_keys = [] |
| |
| |
| self.timestamp = datetime.datetime.utcnow() |
| |
| def __ScanAllNamespaces(self): |
| """Scans all the namespaces and processes each namespace.""" |
| namespace_query = datastore.Query('__namespace__', _app=self.app_id) |
| |
| for namespace_entity in namespace_query.Run(): |
| name = namespace_entity.key().name() |
| if name is None: |
| name = '' |
| self.__ProcessNamespace(name) |
| |
| def __ProcessNamespace(self, namespace): |
| """Process all the entities in a given namespace.""" |
| |
| all_query = datastore.Query(namespace=namespace, _app=self.app_id) |
| |
| |
| for entity in all_query.Run(): |
| self.found_non_empty_namespace |= (namespace != '') |
| proto = entity.ToPb() |
| proto_size = len(proto.SerializeToString()) |
| |
| if entity.key().kind() in stats._DATASTORE_STATS_CLASSES_BY_KIND: |
| |
| |
| |
| stat_kind = stats._DATASTORE_STATS_CLASSES_BY_KIND[entity.key().kind()] |
| |
| self.old_stat_keys.append(entity.key()) |
| self.__AggregateTotal(proto_size, entity.key(), proto, namespace, |
| stat_kind) |
| else: |
| self.__ProcessUserEntity(proto_size, entity.key(), proto, namespace) |
| |
| def __GetPropertyIndexStat(self, namespace, kind_name, |
| entity_key_size, prop): |
| """Return the size and count of indexes for a property of an EntityProto.""" |
| |
| property_index_size = (len(self.app_id) + len(kind_name) + |
| len(prop.value().SerializeToString()) + |
| len(namespace) + entity_key_size) |
| |
| return (property_index_size, 2) |
| |
| def __GetTypeIndexStat(self, namespace, kind_name, entity_key_size): |
| """Return the size and count of indexes by type of an EntityProto.""" |
| type_index_size = (len(self.app_id) + len(kind_name) + entity_key_size |
| + len(namespace)) |
| return (type_index_size, 1) |
| |
| def __ProcessUserEntity(self, proto_size, key, proto, namespace): |
| """Increment datastore stats for a non stats record.""" |
| self.__AggregateTotal(proto_size, key, proto, namespace, None) |
| |
| kind_name = key.kind() |
| |
| entity_key_size = (len(proto.key().app()) + len(namespace) + |
| len(proto.key().path().SerializeToString()) + |
| len(proto.entity_group().SerializeToString())) |
| |
| self.__AggregateCompositeIndices(proto, namespace, kind_name, |
| entity_key_size) |
| |
| type_index_size, type_index_count = self.__GetTypeIndexStat(namespace, |
| kind_name, |
| entity_key_size) |
| property_index_count = 0 |
| property_index_size = 0 |
| |
| |
| for prop_list in (proto.property_list(), proto.raw_property_list()): |
| for prop in prop_list: |
| index_size, index_count = self.__GetPropertyIndexStat(namespace, |
| kind_name, |
| entity_key_size, |
| prop) |
| property_index_size += index_size |
| property_index_count += index_count |
| |
| builtin_index_size = type_index_size + property_index_size |
| builtin_index_count = type_index_count + property_index_count |
| |
| self.__Increment(self.whole_app_stats, 1, |
| (stats.KindStat, kind_name, ''), |
| proto_size, |
| builtin_index_count=builtin_index_count, |
| builtin_index_size=builtin_index_size, |
| kind_name=kind_name) |
| |
| self.__Increment(self.namespace_stats, 1, |
| (stats.NamespaceKindStat, kind_name, namespace), |
| proto_size, |
| builtin_index_count=builtin_index_count, |
| builtin_index_size=builtin_index_size, |
| kind_name=kind_name) |
| |
| |
| |
| if key.parent() is None: |
| whole_app_model = stats.KindRootEntityStat |
| namespace_model = stats.NamespaceKindRootEntityStat |
| else: |
| whole_app_model = stats.KindNonRootEntityStat |
| namespace_model = stats.NamespaceKindNonRootEntityStat |
| |
| self.__Increment(self.whole_app_stats, 1, |
| (whole_app_model, kind_name, ''), |
| proto_size, |
| kind_name=kind_name) |
| |
| self.__Increment(self.namespace_stats, 1, |
| (namespace_model, kind_name, namespace), |
| proto_size, |
| kind_name=kind_name) |
| |
| self.__ProcessProperties( |
| kind_name, |
| namespace, |
| entity_key_size, |
| (proto.property_list(), proto.raw_property_list())) |
| |
| def __ProcessProperties(self, kind_name, namespace, entity_key_size, |
| prop_lists): |
| for prop_list in prop_lists: |
| for prop in prop_list: |
| try: |
| value = datastore_types.FromPropertyPb(prop) |
| self.__AggregateProperty(kind_name, namespace, entity_key_size, |
| prop, value) |
| except (AssertionError, AttributeError, TypeError, ValueError), e: |
| logging.error('Cannot process property %r, exception %s' % |
| (prop, e)) |
| |
| def __AggregateProperty(self, kind_name, namespace, entity_key_size, |
| prop, value): |
| property_name = prop.name() |
| property_type = _PROPERTY_TYPE_TO_DSS_NAME[type(value)][0] |
| index_property_type = _PROPERTY_TYPE_TO_DSS_NAME[type(value)][1] |
| size = len(prop.SerializeToString()) |
| |
| |
| index_size, index_count = self.__GetPropertyIndexStat(namespace, kind_name, |
| entity_key_size, prop) |
| |
| |
| |
| |
| |
| self.__Increment(self.whole_app_stats, 1, |
| (stats.PropertyTypeStat, property_type, ''), |
| size, |
| builtin_index_count=0, |
| builtin_index_size=0, |
| property_type=property_type) |
| |
| self.__Increment(self.whole_app_stats, 0, |
| (stats.PropertyTypeStat, index_property_type, ''), |
| 0, |
| builtin_index_count=index_count, |
| builtin_index_size=index_size, |
| property_type=index_property_type) |
| |
| self.__Increment(self.namespace_stats, 1, |
| (stats.NamespacePropertyTypeStat, |
| property_type, namespace), |
| size, |
| builtin_index_count=0, |
| builtin_index_size=0, |
| property_type=property_type) |
| |
| self.__Increment(self.namespace_stats, 0, |
| (stats.NamespacePropertyTypeStat, |
| index_property_type, namespace), |
| 0, |
| builtin_index_count=index_count, |
| builtin_index_size=index_size, |
| property_type=index_property_type) |
| |
| |
| self.__Increment(self.whole_app_stats, 1, |
| (stats.KindPropertyTypeStat, |
| property_type + '_' + kind_name, ''), |
| size, |
| builtin_index_count=0, |
| builtin_index_size=0, |
| property_type=property_type, kind_name=kind_name) |
| |
| self.__Increment(self.whole_app_stats, 0, |
| (stats.KindPropertyTypeStat, |
| index_property_type + '_' + kind_name, ''), |
| 0, |
| builtin_index_count=index_count, |
| builtin_index_size=index_size, |
| property_type=index_property_type, kind_name=kind_name) |
| |
| self.__Increment(self.namespace_stats, 1, |
| (stats.NamespaceKindPropertyTypeStat, |
| property_type + '_' + kind_name, namespace), |
| size, |
| builtin_index_count=0, |
| builtin_index_size=0, |
| property_type=property_type, kind_name=kind_name) |
| |
| self.__Increment(self.namespace_stats, 0, |
| (stats.NamespaceKindPropertyTypeStat, |
| index_property_type + '_' + kind_name, namespace), |
| 0, |
| builtin_index_count=index_count, |
| builtin_index_size=index_size, |
| property_type=index_property_type, kind_name=kind_name) |
| |
| |
| self.__Increment(self.whole_app_stats, 1, |
| (stats.KindPropertyNameStat, |
| property_name + '_' + kind_name, ''), |
| size, |
| builtin_index_count=index_count, |
| builtin_index_size=index_size, |
| property_name=property_name, kind_name=kind_name) |
| |
| self.__Increment(self.namespace_stats, 1, |
| (stats.NamespaceKindPropertyNameStat, |
| property_name + '_' + kind_name, namespace), |
| size, |
| builtin_index_count=index_count, |
| builtin_index_size=index_size, |
| property_name=property_name, kind_name=kind_name) |
| |
| |
| self.__Increment(self.whole_app_stats, 1, |
| (stats.KindPropertyNamePropertyTypeStat, |
| property_type + '_' + property_name + '_' + kind_name, |
| ''), size, |
| builtin_index_count=0, |
| builtin_index_size=0, |
| property_type=property_type, |
| property_name=property_name, kind_name=kind_name) |
| |
| self.__Increment(self.whole_app_stats, 0, |
| (stats.KindPropertyNamePropertyTypeStat, |
| index_property_type + '_' + property_name + '_' + |
| kind_name, |
| ''), 0, |
| builtin_index_count=index_count, |
| builtin_index_size=index_size, |
| property_type=index_property_type, |
| property_name=property_name, kind_name=kind_name) |
| |
| self.__Increment(self.namespace_stats, 1, |
| (stats.NamespaceKindPropertyNamePropertyTypeStat, |
| property_type + '_' + property_name + '_' + kind_name, |
| namespace), |
| size, |
| builtin_index_count=0, |
| builtin_index_size=0, |
| property_type=property_type, |
| property_name=property_name, kind_name=kind_name) |
| |
| self.__Increment(self.namespace_stats, 0, |
| (stats.NamespaceKindPropertyNamePropertyTypeStat, |
| index_property_type + '_' + property_name + '_' + |
| kind_name, |
| namespace), |
| 0, |
| builtin_index_count=index_count, |
| builtin_index_size=index_size, |
| property_type=index_property_type, |
| property_name=property_name, kind_name=kind_name) |
| |
| def __GetCompositeIndexStat(self, definition, proto, namespace, kind_name, |
| entity_key_size): |
| """Get statistics of composite index for a index definition of an entity.""" |
| |
| |
| |
| |
| |
| |
| property_list = proto.property_list() |
| property_count = [] |
| property_size = [] |
| index_count = 1 |
| for indexed_prop in definition.property_list(): |
| name = indexed_prop.name() |
| count = 0 |
| prop_size = 0 |
| for prop in property_list: |
| if prop.name() == name: |
| count += 1 |
| prop_size += len(prop.SerializeToString()) |
| |
| property_count.append(count) |
| property_size.append(prop_size) |
| index_count *= count |
| |
| if index_count == 0: |
| return (0, 0) |
| |
| index_only_size = 0 |
| for i in range(len(property_size)): |
| index_only_size += property_size[i] * (index_count / property_count[i]) |
| |
| |
| |
| |
| |
| index_size = (index_count * (entity_key_size + len(kind_name) + |
| len(self.app_id) + len(namespace)) + |
| index_only_size * 2) |
| |
| return (index_size, index_count) |
| |
| def __AggregateCompositeIndices(self, proto, namespace, kind_name, |
| entity_key_size): |
| """Aggregate statistics of composite indexes for an entity.""" |
| composite_indices = datastore_admin.GetIndices(self.app_id) |
| for index in composite_indices: |
| definition = index.definition() |
| if kind_name != definition.entity_type(): |
| continue |
| |
| index_size, index_count = self.__GetCompositeIndexStat(definition, proto, |
| namespace, |
| kind_name, |
| entity_key_size) |
| |
| if index_count == 0: |
| continue |
| |
| |
| name_id = namespace |
| if not name_id: |
| name_id = 1 |
| |
| |
| self.__Increment(self.whole_app_stats, 0, _GLOBAL_KEY, 0, |
| composite_index_count=index_count, |
| composite_index_size=index_size) |
| |
| self.__Increment(self.whole_app_stats, 0, |
| (stats.NamespaceStat, name_id, ''), 0, |
| composite_index_count=index_count, |
| composite_index_size=index_size, |
| subject_namespace=namespace) |
| |
| self.__Increment(self.namespace_stats, 0, |
| (stats.NamespaceGlobalStat, 'total_entity_usage', |
| namespace), 0, |
| composite_index_count=index_count, |
| composite_index_size=index_size) |
| |
| |
| self.__Increment(self.whole_app_stats, 0, |
| (stats.KindStat, kind_name, ''), 0, |
| composite_index_count=index_count, |
| composite_index_size=index_size, |
| kind_name=kind_name) |
| |
| self.__Increment(self.namespace_stats, 0, |
| (stats.NamespaceKindStat, kind_name, namespace), 0, |
| composite_index_count=index_count, |
| composite_index_size=index_size, |
| kind_name=kind_name) |
| |
| |
| index_id = index.id() |
| self.__Increment(self.whole_app_stats, index_count, |
| (stats.KindCompositeIndexStat, |
| kind_name + '_%s' % index_id, ''), index_size, |
| kind_name=kind_name, index_id=index_id) |
| |
| self.__Increment(self.namespace_stats, index_count, |
| (stats.NamespaceKindCompositeIndexStat, |
| kind_name + '_%s' % index_id, namespace), index_size, |
| kind_name=kind_name, index_id=index_id) |
| |
| def __AggregateTotal(self, size, key, proto, namespace, stat_kind): |
| """Aggregate total datastore stats.""" |
| kind_name = key.kind() |
| |
| entity_key_size = (len(proto.key().app()) + |
| len(proto.key().path().SerializeToString()) + |
| len(proto.entity_group().SerializeToString())) |
| |
| type_index_size, type_index_count = self.__GetTypeIndexStat(namespace, |
| kind_name, |
| entity_key_size) |
| property_index_count = 0 |
| property_index_size = 0 |
| for prop_list in (proto.property_list(), proto.raw_property_list()): |
| for prop in prop_list: |
| index_size, index_count = self.__GetPropertyIndexStat(namespace, |
| kind_name, |
| entity_key_size, |
| prop) |
| property_index_size += index_size |
| property_index_count += index_count |
| |
| builtin_index_size = type_index_size + property_index_size |
| builtin_index_count = type_index_count + property_index_count |
| |
| |
| if stat_kind == stats.GlobalStat: |
| count = 0 |
| else: |
| count = 1 |
| |
| |
| self.__Increment(self.whole_app_stats, count, _GLOBAL_KEY, size, |
| builtin_index_count=builtin_index_count, |
| builtin_index_size=builtin_index_size) |
| |
| |
| name_id = namespace |
| if not name_id: |
| name_id = 1 |
| |
| if (stat_kind == stats.NamespaceStat) and (namespace == ''): |
| count = 0 |
| |
| |
| self.__Increment(self.whole_app_stats, count, |
| (stats.NamespaceStat, name_id, ''), |
| size, |
| builtin_index_count=builtin_index_count, |
| builtin_index_size=builtin_index_size, |
| subject_namespace=namespace) |
| |
| if stat_kind == stats.NamespaceGlobalStat: |
| count = 0 |
| |
| |
| self.__Increment( |
| self.namespace_stats, count, |
| (stats.NamespaceGlobalStat, 'total_entity_usage', namespace), size, |
| builtin_index_count=builtin_index_count, |
| builtin_index_size=builtin_index_size) |
| |
| def __Increment(self, stats_dict, count, stat_key, size, |
| builtin_index_count=0, builtin_index_size=0, |
| composite_index_count=0, composite_index_size=0, **kwds): |
| """Increment stats for a particular kind. |
| |
| Args: |
| stats_dict: The dictionary where the entities are held. |
| The entities are keyed by stat_key. e.g. The |
| __Stat_Total__ entity will be found in stats_dict[_GLOBAL_KEY]. |
| count: The amount to increment the datastore stat by. |
| stat_key: A tuple of (db.Model of the stat, key value, namespace). |
| size: The "bytes" to increment the size by. |
| builtin_index_count: The bytes of builtin index to add in to a stat. |
| builtin_index_size: The count of builtin index to add in to a stat. |
| composite_index_count: The bytes of composite index to add in to a stat. |
| composite_index_size: The count of composite index to add in to a stat. |
| kwds: Name value pairs that are set on the created entities. |
| """ |
| |
| if stat_key not in stats_dict: |
| stat_model = stat_key[0]( |
| key=datastore_types.Key.from_path(stat_key[0].STORED_KIND_NAME, |
| stat_key[1], |
| namespace=stat_key[2], |
| _app=self.app_id), |
| _app=self.app_id) |
| stats_dict[stat_key] = stat_model |
| for field, value in kwds.iteritems(): |
| setattr(stat_model, field, value) |
| stat_model.count = count |
| if size: |
| stat_model.entity_bytes = size |
| if builtin_index_size: |
| stat_model.builtin_index_bytes = builtin_index_size |
| stat_model.builtin_index_count = builtin_index_count |
| if composite_index_size: |
| stat_model.composite_index_bytes = composite_index_size |
| stat_model.composite_index_count = composite_index_count |
| stat_model.bytes = size + builtin_index_size + composite_index_size |
| stat_model.timestamp = self.timestamp |
| else: |
| stat_model = stats_dict[stat_key] |
| stat_model.count += count |
| if size: |
| stat_model.entity_bytes += size |
| if builtin_index_size: |
| stat_model.builtin_index_bytes += builtin_index_size |
| stat_model.builtin_index_count += builtin_index_count |
| if composite_index_size: |
| stat_model.composite_index_bytes += composite_index_size |
| stat_model.composite_index_count += composite_index_count |
| stat_model.bytes += size + builtin_index_size + composite_index_size |
| |
| def __Finalize(self): |
| """Finishes processing, deletes all old stats and writes new ones.""" |
| |
| for i in range(0, len(self.old_stat_keys), DELETE_BATCH_SIZE): |
| datastore.Delete(self.old_stat_keys[i:i+DELETE_BATCH_SIZE]) |
| |
| self.written = 0 |
| |
| for stat in self.whole_app_stats.itervalues(): |
| if stat.count or not (isinstance(stat, stats.GlobalStat) or |
| isinstance(stat, stats.NamespaceStat)): |
| stat.put() |
| self.written += 1 |
| |
| |
| |
| if self.found_non_empty_namespace: |
| for stat in self.namespace_stats.itervalues(): |
| if stat.count or not isinstance(stat, stats.NamespaceGlobalStat): |
| stat.put() |
| self.written += 1 |
| |
| def Run(self): |
| """Scans the datastore, computes new stats and writes them.""" |
| self.__ScanAllNamespaces() |
| self.__Finalize() |
| return self |
| |
| def Report(self): |
| """Produce a small report about the result.""" |
| stat = self.whole_app_stats.get(_GLOBAL_KEY, None) |
| entity_size = 0 |
| entity_count = 0 |
| builtin_index_size = 0 |
| builtin_index_count = 0 |
| composite_index_size = 0 |
| composite_index_count = 0 |
| if stat: |
| entity_size = stat.entity_bytes |
| entity_count = stat.count |
| builtin_index_size = stat.builtin_index_bytes |
| builtin_index_count = stat.builtin_index_count |
| composite_index_size = stat.composite_index_bytes |
| composite_index_count = stat.composite_index_count |
| |
| if not entity_count: |
| entity_count = 1 |
| |
| return ('Scanned %d entities of total %d bytes, %d index entries of total ' |
| '%d bytes and %d composite index entries of total %d bytes. ' |
| 'Inserted %d new records.' |
| % (entity_count, entity_size, builtin_index_count, |
| builtin_index_size, composite_index_count, composite_index_size, |
| self.written)) |
