blob: cc63f9a63b515ac4f8a13f49c259386e8e278b79 [file] [log] [blame]
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is govered by a BSD-style
# license that can be found in the LICENSE file or at
"""Servlet to import a file of issues in JSON format.
import collections
import json
import logging
import time
from third_party import ezt
from features import filterrules_helpers
from framework import framework_helpers
from framework import jsonfeed
from framework import permissions
from framework import servlet
from framework import urls
from proto import tracker_pb2
ParserState = collections.namedtuple(
'user_id_dict, nonexist_emails, issue_list, comments_dict, starrers_dict, '
class IssueImport(servlet.Servlet):
"""IssueImport loads a file of issues in JSON format."""
_PAGE_TEMPLATE = 'tracker/issue-import-page.ezt'
def AssertBasePermission(self, mr):
"""Make sure that the logged in user has permission to view this page."""
super(IssueImport, self).AssertBasePermission(mr)
if not mr.auth.user_pb.is_site_admin:
raise permissions.PermissionException(
'Only site admins may import issues')
def GatherPageData(self, mr):
"""Build up a dictionary of data values to use when rendering the page."""
return {
'issue_tab_mode': None,
'page_perms': self.MakePagePerms(mr, None, permissions.CREATE_ISSUE),
'import_errors': [],
def ProcessFormData(self, mr, post_data):
"""Process the issue entry form.
mr: commonly used info parsed from the request.
post_data: The post_data dict for the current request.
String URL to redirect the user to after processing.
import_errors = []
json_data = None
pre_check_only = 'pre_check_only' in post_data
uploaded_file = post_data.get('jsonfile')
if uploaded_file is None:
import_errors.append('No file uploaded')
json_str = uploaded_file.value
if json_str.startswith(jsonfeed.XSSI_PREFIX):
json_str = json_str[len(jsonfeed.XSSI_PREFIX):]
json_data = json.loads(json_str)
except ValueError:
import_errors.append('error parsing JSON in file')
if uploaded_file and not json_data:
import_errors.append('JSON file was empty')
# Note that the project must already exist in order to even reach
# this servlet because it is hosted in the context of a project.
if json_data and mr.project_name != json_data['metadata']['project']:
'Project name does not match. '
'Edit the file if you want to import into this project anyway.')
if import_errors:
return self.PleaseCorrect(mr, import_errors=import_errors)
event_log = [] # We accumulate a list of messages to display to the user.
# First we parse the JSON into objects, but we don't have DB IDs yet.
state = self._ParseObjects(mr.cnxn, mr.project_id, json_data, event_log)
# If that worked, go ahead and start saving the data to the DB.
if not pre_check_only:
self._SaveObjects(mr.cnxn, mr.project_id, state, event_log)
except JSONImportError:
# just report it to the user by displaying event_log
event_log.append('Aborted import processing')
# This is a little bit of a hack because it always uses the form validation
# error message display logic to show the results of this import run,
# which may include errors or not.
return self.PleaseCorrect(mr, import_errors=event_log)
def _ParseObjects(self, cnxn, project_id, json_data, event_log):
"""Examine JSON data and return a parser state for further processing."""
# Decide which users need to be created.
needed_emails = json_data['emails']
user_id_dict =, needed_emails)
nonexist_emails = [email for email in needed_emails
if email not in user_id_dict]
event_log.append('Need to create %d users: %r' %
(len(nonexist_emails), nonexist_emails))
email.lower(): framework_helpers.MurmurHash3_x86_32(email.lower())
for email in nonexist_emails})
num_comments = 0
num_stars = 0
issue_list = []
comments_dict = collections.defaultdict(list)
starrers_dict = collections.defaultdict(list)
relations_dict = collections.defaultdict(list)
for issue_json in json_data.get('issues', []):
issue, comment_list, starrer_list, relation_list = self._ParseIssue(
cnxn, project_id, user_id_dict, issue_json, event_log)
comments_dict[issue.local_id] = comment_list
starrers_dict[issue.local_id] = starrer_list
relations_dict[issue.local_id] = relation_list
num_comments += len(comment_list)
num_stars += len(starrer_list)
'Found info for %d issues: %r' %
(len(issue_list), sorted([issue.local_id for issue in issue_list])))
'Found %d total comments for %d issues' %
(num_comments, len(comments_dict)))
'Found %d total stars for %d issues' %
(num_stars, len(starrers_dict)))
'Found %d total relationships.' %
sum((len(dsts) for dsts in relations_dict.itervalues())))
event_log.append('Parsing phase finished OK')
return ParserState(
user_id_dict, nonexist_emails, issue_list,
comments_dict, starrers_dict, relations_dict)
def _ParseIssue(self, cnxn, project_id, user_id_dict, issue_json, event_log):
issue = tracker_pb2.Issue(
for cc_email in issue_json.get('cc', [])
if cc_email in user_id_dict],
status=issue_json.get('status', ''),
labels=issue_json.get('labels', []),
field_values=[self._ParseFieldValue(cnxn, project_id, user_id_dict, field)
for field in issue_json.get('fields', [])])
if issue_json.get('owner'):
issue.owner_id = user_id_dict[issue_json['owner']]
if issue_json.get('closed'):
issue.closed_timestamp = issue_json['closed']
comments = [self._ParseComment(
project_id, user_id_dict, comment_json, event_log)
for comment_json in issue_json.get('comments', [])]
starrers = [user_id_dict[starrer] for starrer in issue_json['starrers']]
relations = []
[(i, 'blockedon') for i in issue_json.get('blocked_on', [])])
[(i, 'blocking') for i in issue_json.get('blocking', [])])
if 'merged_into' in issue_json:
relations.append((issue_json['merged_into'], 'mergedinto'))
return issue, comments, starrers, relations
def _ParseFieldValue(self, cnxn, project_id, user_id_dict, field_json):
field = tracker_pb2.FieldValue(, project_id,
if 'int_value' in field_json:
field.int_value = field_json['int_value']
if 'str_value' in field_json:
field.str_value = field_json['str_value']
if 'user_value' in field_json:
field.user_value = user_id_dict.get(field_json['user_value'])
return field
def _ParseComment(self, project_id, user_id_dict, comment_json, event_log):
comment = tracker_pb2.IssueComment(
# Note: issue_id is filled in after the issue is saved.
for amendment in comment_json['amendments']:
self._ParseAmendment(amendment, user_id_dict, event_log))
for attachment in comment_json['attachments']:
self._ParseAttachment(attachment, event_log))
return comment
def _ParseAmendment(self, amendment_json, user_id_dict, _event_log):
amendment = tracker_pb2.Amendment(
if 'new_value' in amendment_json:
amendment.newvalue = amendment_json['new_value']
if 'custom_field_name' in amendment_json:
amendment.custom_field_name = amendment_json['custom_field_name']
if 'added_users' in amendment_json:
[user_id_dict[email] for email in amendment_json['added_users']])
if 'removed_users' in amendment_json:
[user_id_dict[email] for email in amendment_json['removed_users']])
return amendment
def _ParseAttachment(self, attachment_json, _event_log):
attachment = tracker_pb2.Attachment(
return attachment
def _SaveObjects(self, cnxn, project_id, state, event_log):
"""Examine JSON data and create users, issues, and comments."""
created_user_ids =
cnxn, state.nonexist_emails, autocreate=True)
for created_email, created_id in created_user_ids.items():
if created_id != state.user_id_dict[created_email]:
event_log.append('Mismatched user_id for %r' % created_email)
raise JSONImportError()
event_log.append('Created %d users' % len(state.nonexist_emails))
total_comments = 0
total_stars = 0
config =, project_id)
for issue in state.issue_list:
# TODO(jrobbins): renumber issues if there is a local_id conflict.
if issue.local_id not in state.starrers_dict:
# Issues with stars will have filter rules applied in SetStar().
cnxn,, issue, config)
issue_id =, issue)
for comment in state.comments_dict[issue.local_id]:
total_comments += 1
comment.issue_id = issue_id, comment)
for starrer in state.starrers_dict[issue.local_id]:
total_stars += 1
cnxn,, config, issue_id, starrer, True)
event_log.append('Created %d issues' % len(state.issue_list))
event_log.append('Created %d comments for %d issues' % (
total_comments, len(state.comments_dict)))
event_log.append('Set %d stars on %d issues' % (
total_stars, len(state.starrers_dict)))
global_relations_dict = collections.defaultdict(list)
for issue, rels in state.relations_dict.iteritems():
src_iid =
cnxn, project_id, issue).issue_id
dst_iids = [i.issue_id for i in
cnxn, project_id, [rel[0] for rel in rels])]
kinds = [rel[1] for rel in rels]
global_relations_dict[src_iid] = zip(dst_iids, kinds), global_relations_dict), project_id)
event_log.append('Finished import')
class JSONImportError(Exception):
"""Exception to raise if imported JSON is invalid."""