| #!/usr/bin/env python |
| # |
| # Copyright 2007 Google Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| |
| |
| |
| """Utilities for generating and updating index.yaml.""" |
| |
| |
| |
| |
| |
| |
| |
| __all__ = ['GenerateIndexFromHistory', |
| 'IndexYamlUpdater', |
| ] |
| |
| import os |
| import logging |
| |
| from google.appengine.api import apiproxy_stub_map |
| from google.appengine.api import yaml_errors |
| from google.appengine.datastore import datastore_index |
| |
| import yaml |
| |
| |
| AUTO_MARKER = '\n# AUTOGENERATED\n' |
| |
| |
| AUTO_COMMENT = ''' |
| # This index.yaml is automatically updated whenever the dev_appserver |
| # detects that a new type of query is run. If you want to manage the |
| # index.yaml file manually, remove the above marker line (the line |
| # saying "# AUTOGENERATED"). If you want to manage some indexes |
| # manually, move them above the marker line. The index.yaml file is |
| # automatically uploaded to the admin console when you next deploy |
| # your application using appcfg.py. |
| ''' |
| |
| |
| def GenerateIndexFromHistory(query_history, |
| all_indexes=None, manual_indexes=None): |
| """Generate most of the text for index.yaml from the query history. |
| |
| Args: |
| query_history: Query history, a dict mapping query |
| all_indexes: Optional datastore_index.IndexDefinitions instance |
| representing all the indexes found in the input file. May be None. |
| manual_indexes: Optional datastore_index.IndexDefinitions instance |
| containing indexes for which we should not generate output. May be None. |
| |
| Returns: |
| A string representation that can safely be appended to an existing |
| index.yaml file. Returns the empty string if it would generate no output. |
| """ |
| |
| |
| |
| |
| |
| all_keys = datastore_index.IndexDefinitionsToKeys(all_indexes) |
| manual_keys = datastore_index.IndexDefinitionsToKeys(manual_indexes) |
| |
| |
| indexes = dict((key, 0) for key in all_keys - manual_keys) |
| |
| |
| for query, count in query_history.iteritems(): |
| required, kind, ancestor, props = ( |
| datastore_index.CompositeIndexForQuery(query)) |
| if required: |
| props = datastore_index.GetRecommendedIndexProperties(props) |
| key = (kind, ancestor, props) |
| if key not in manual_keys: |
| if key in indexes: |
| indexes[key] += count |
| else: |
| indexes[key] = count |
| |
| if not indexes: |
| return '' |
| |
| |
| |
| |
| res = [] |
| for (kind, ancestor, props), count in sorted(indexes.iteritems()): |
| |
| res.append('') |
| res.append(datastore_index.IndexYamlForQuery(kind, ancestor, props)) |
| |
| res.append('') |
| return '\n'.join(res) |
| |
| |
| class IndexYamlUpdater(object): |
| """Helper class for updating index.yaml. |
| |
| This class maintains some state about the query history and the |
| index.yaml file in order to minimize the number of times index.yaml |
| is actually overwritten. |
| """ |
| |
| |
| index_yaml_is_manual = False |
| index_yaml_mtime = None |
| last_history_size = 0 |
| |
| def __init__(self, root_path): |
| """Constructor. |
| |
| Args: |
| root_path: Path to the app's root directory. |
| """ |
| self.root_path = root_path |
| |
| def UpdateIndexYaml(self, openfile=open): |
| """Update index.yaml. |
| |
| Args: |
| openfile: Used for dependency injection. |
| |
| We only ever write to index.yaml if either: |
| - it doesn't exist yet; or |
| - it contains an 'AUTOGENERATED' comment. |
| |
| All indexes *before* the AUTOGENERATED comment will be written |
| back unchanged. All indexes *after* the AUTOGENERATED comment |
| will be updated with the latest query counts (query counts are |
| reset by --clear_datastore). Indexes that aren't yet in the file |
| will be appended to the AUTOGENERATED section. |
| |
| We keep track of some data in order to avoid doing repetitive work: |
| - if index.yaml is fully manual, we keep track of its mtime to |
| avoid parsing it over and over; |
| - we keep track of the number of keys in the history dict since |
| the last time we updated index.yaml (or decided there was |
| nothing to update). |
| """ |
| |
| |
| |
| |
| |
| index_yaml_file = os.path.join(self.root_path, 'index.yaml') |
| |
| |
| try: |
| index_yaml_mtime = os.path.getmtime(index_yaml_file) |
| except os.error: |
| index_yaml_mtime = None |
| |
| |
| index_yaml_changed = (index_yaml_mtime != self.index_yaml_mtime) |
| self.index_yaml_mtime = index_yaml_mtime |
| |
| |
| datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') |
| query_ci_history_len = datastore_stub._QueryCompositeIndexHistoryLength() |
| history_changed = (query_ci_history_len != self.last_history_size) |
| self.last_history_size = query_ci_history_len |
| |
| |
| if not (index_yaml_changed or history_changed): |
| logging.debug('No need to update index.yaml') |
| return |
| |
| |
| if self.index_yaml_is_manual and not index_yaml_changed: |
| logging.debug('Will not update manual index.yaml') |
| return |
| |
| |
| if index_yaml_mtime is None: |
| index_yaml_data = None |
| else: |
| try: |
| |
| |
| |
| fh = openfile(index_yaml_file, 'rU') |
| except IOError: |
| index_yaml_data = None |
| else: |
| try: |
| index_yaml_data = fh.read() |
| finally: |
| fh.close() |
| |
| |
| self.index_yaml_is_manual = (index_yaml_data is not None and |
| AUTO_MARKER not in index_yaml_data) |
| if self.index_yaml_is_manual: |
| logging.info('Detected manual index.yaml, will not update') |
| return |
| |
| |
| |
| if index_yaml_data is None: |
| all_indexes = None |
| else: |
| try: |
| all_indexes = datastore_index.ParseIndexDefinitions(index_yaml_data) |
| except yaml_errors.EventListenerError, e: |
| |
| logging.error('Error parsing %s:\n%s', index_yaml_file, e) |
| return |
| except Exception, err: |
| |
| logging.error('Error parsing %s:\n%s.%s: %s', index_yaml_file, |
| err.__class__.__module__, err.__class__.__name__, err) |
| return |
| |
| |
| if index_yaml_data is None: |
| manual_part, prev_automatic_part = 'indexes:\n', '' |
| manual_indexes = None |
| else: |
| manual_part, prev_automatic_part = index_yaml_data.split(AUTO_MARKER, 1) |
| if prev_automatic_part.startswith(AUTO_COMMENT): |
| prev_automatic_part = prev_automatic_part[len(AUTO_COMMENT):] |
| |
| try: |
| manual_indexes = datastore_index.ParseIndexDefinitions(manual_part) |
| except Exception, err: |
| logging.error('Error parsing manual part of %s: %s', |
| index_yaml_file, err) |
| return |
| |
| |
| automatic_part = GenerateIndexFromHistory(datastore_stub.QueryHistory(), |
| all_indexes, manual_indexes) |
| |
| |
| |
| if (index_yaml_mtime is None and automatic_part == '' or |
| automatic_part == prev_automatic_part): |
| logging.debug('No need to update index.yaml') |
| return |
| |
| |
| try: |
| fh = openfile(index_yaml_file, 'w') |
| except IOError, err: |
| logging.error('Can\'t write index.yaml: %s', err) |
| return |
| |
| |
| try: |
| logging.info('Updating %s', index_yaml_file) |
| fh.write(manual_part) |
| fh.write(AUTO_MARKER) |
| fh.write(AUTO_COMMENT) |
| fh.write(automatic_part) |
| finally: |
| fh.close() |
| |
| |
| try: |
| self.index_yaml_mtime = os.path.getmtime(index_yaml_file) |
| except os.error, err: |
| logging.error('Can\'t stat index.yaml we just wrote: %s', err) |
| self.index_yaml_mtime = None |