blob: 6e2aada6ba158519b76c951517a55f419f2e94d4 [file] [log] [blame]
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Servlet to import a file of issues in JSON format.
"""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import collections
import json
import logging
import time
from third_party import ezt
from features import filterrules_helpers
from framework import framework_helpers
from framework import jsonfeed
from framework import permissions
from framework import servlet
from framework import urls
from proto import tracker_pb2
ParserState = collections.namedtuple(
'ParserState',
'user_id_dict, nonexist_emails, issue_list, comments_dict, starrers_dict, '
'relations_dict')
class IssueImport(servlet.Servlet):
"""IssueImport loads a file of issues in JSON format."""
_PAGE_TEMPLATE = 'tracker/issue-import-page.ezt'
_MAIN_TAB_MODE = servlet.Servlet.MAIN_TAB_ISSUES
def AssertBasePermission(self, mr):
"""Make sure that the logged in user has permission to view this page."""
super(IssueImport, self).AssertBasePermission(mr)
if not mr.auth.user_pb.is_site_admin:
raise permissions.PermissionException(
'Only site admins may import issues')
def GatherPageData(self, mr):
"""Build up a dictionary of data values to use when rendering the page."""
return {
'issue_tab_mode': None,
'page_perms': self.MakePagePerms(mr, None, permissions.CREATE_ISSUE),
'import_errors': [],
}
def ProcessFormData(self, mr, post_data):
"""Process the issue entry form.
Args:
mr: commonly used info parsed from the request.
post_data: The post_data dict for the current request.
Returns:
String URL to redirect the user to after processing.
"""
import_errors = []
json_data = None
pre_check_only = 'pre_check_only' in post_data
uploaded_file = post_data.get('jsonfile')
if uploaded_file is None:
import_errors.append('No file uploaded')
else:
try:
json_str = uploaded_file.value
if json_str.startswith(jsonfeed.XSSI_PREFIX):
json_str = json_str[len(jsonfeed.XSSI_PREFIX):]
json_data = json.loads(json_str)
except ValueError:
import_errors.append('error parsing JSON in file')
if uploaded_file and not json_data:
import_errors.append('JSON file was empty')
# Note that the project must already exist in order to even reach
# this servlet because it is hosted in the context of a project.
if json_data and mr.project_name != json_data['metadata']['project']:
import_errors.append(
'Project name does not match. '
'Edit the file if you want to import into this project anyway.')
if import_errors:
return self.PleaseCorrect(mr, import_errors=import_errors)
event_log = [] # We accumulate a list of messages to display to the user.
try:
# First we parse the JSON into objects, but we don't have DB IDs yet.
state = self._ParseObjects(mr.cnxn, mr.project_id, json_data, event_log)
# If that worked, go ahead and start saving the data to the DB.
if not pre_check_only:
self._SaveObjects(mr.cnxn, mr.project_id, state, event_log)
except JSONImportError:
# just report it to the user by displaying event_log
event_log.append('Aborted import processing')
# This is a little bit of a hack because it always uses the form validation
# error message display logic to show the results of this import run,
# which may include errors or not.
return self.PleaseCorrect(mr, import_errors=event_log)
def _ParseObjects(self, cnxn, project_id, json_data, event_log):
"""Examine JSON data and return a parser state for further processing."""
# Decide which users need to be created.
needed_emails = json_data['emails']
user_id_dict = self.services.user.LookupExistingUserIDs(cnxn, needed_emails)
nonexist_emails = [email for email in needed_emails
if email not in user_id_dict]
event_log.append('Need to create %d users: %r' %
(len(nonexist_emails), nonexist_emails))
user_id_dict.update({
email.lower(): framework_helpers.MurmurHash3_x86_32(email.lower())
for email in nonexist_emails})
num_comments = 0
num_stars = 0
issue_list = []
comments_dict = collections.defaultdict(list)
starrers_dict = collections.defaultdict(list)
relations_dict = collections.defaultdict(list)
for issue_json in json_data.get('issues', []):
issue, comment_list, starrer_list, relation_list = self._ParseIssue(
cnxn, project_id, user_id_dict, issue_json, event_log)
issue_list.append(issue)
comments_dict[issue.local_id] = comment_list
starrers_dict[issue.local_id] = starrer_list
relations_dict[issue.local_id] = relation_list
num_comments += len(comment_list)
num_stars += len(starrer_list)
event_log.append(
'Found info for %d issues: %r' %
(len(issue_list), sorted([issue.local_id for issue in issue_list])))
event_log.append(
'Found %d total comments for %d issues' %
(num_comments, len(comments_dict)))
event_log.append(
'Found %d total stars for %d issues' %
(num_stars, len(starrers_dict)))
event_log.append(
'Found %d total relationships.' %
sum((len(dsts) for dsts in relations_dict.values())))
event_log.append('Parsing phase finished OK')
return ParserState(
user_id_dict, nonexist_emails, issue_list,
comments_dict, starrers_dict, relations_dict)
def _ParseIssue(self, cnxn, project_id, user_id_dict, issue_json, event_log):
issue = tracker_pb2.Issue(
project_id=project_id,
local_id=issue_json['local_id'],
reporter_id=user_id_dict[issue_json['reporter']],
summary=issue_json['summary'],
opened_timestamp=issue_json['opened'],
modified_timestamp=issue_json['modified'],
cc_ids=[user_id_dict[cc_email]
for cc_email in issue_json.get('cc', [])
if cc_email in user_id_dict],
status=issue_json.get('status', ''),
labels=issue_json.get('labels', []),
field_values=[self._ParseFieldValue(cnxn, project_id, user_id_dict, field)
for field in issue_json.get('fields', [])])
if issue_json.get('owner'):
issue.owner_id = user_id_dict[issue_json['owner']]
if issue_json.get('closed'):
issue.closed_timestamp = issue_json['closed']
comments = [self._ParseComment(
project_id, user_id_dict, comment_json, event_log)
for comment_json in issue_json.get('comments', [])]
starrers = [user_id_dict[starrer] for starrer in issue_json['starrers']]
relations = []
relations.extend(
[(i, 'blockedon') for i in issue_json.get('blocked_on', [])])
relations.extend(
[(i, 'blocking') for i in issue_json.get('blocking', [])])
if 'merged_into' in issue_json:
relations.append((issue_json['merged_into'], 'mergedinto'))
return issue, comments, starrers, relations
def _ParseFieldValue(self, cnxn, project_id, user_id_dict, field_json):
field = tracker_pb2.FieldValue(
field_id=self.services.config.LookupFieldID(cnxn, project_id,
field_json['field']))
if 'int_value' in field_json:
field.int_value = field_json['int_value']
if 'str_value' in field_json:
field.str_value = field_json['str_value']
if 'user_value' in field_json:
field.user_value = user_id_dict.get(field_json['user_value'])
return field
def _ParseComment(self, project_id, user_id_dict, comment_json, event_log):
comment = tracker_pb2.IssueComment(
# Note: issue_id is filled in after the issue is saved.
project_id=project_id,
timestamp=comment_json['timestamp'],
user_id=user_id_dict[comment_json['commenter']],
content=comment_json.get('content'))
for amendment in comment_json['amendments']:
comment.amendments.append(
self._ParseAmendment(amendment, user_id_dict, event_log))
for attachment in comment_json['attachments']:
comment.attachments.append(
self._ParseAttachment(attachment, event_log))
if comment_json['description_num']:
comment.is_description = True
return comment
def _ParseAmendment(self, amendment_json, user_id_dict, _event_log):
amendment = tracker_pb2.Amendment(
field=tracker_pb2.FieldID(amendment_json['field']))
if 'new_value' in amendment_json:
amendment.newvalue = amendment_json['new_value']
if 'custom_field_name' in amendment_json:
amendment.custom_field_name = amendment_json['custom_field_name']
if 'added_users' in amendment_json:
amendment.added_user_ids.extend(
[user_id_dict[email] for email in amendment_json['added_users']])
if 'removed_users' in amendment_json:
amendment.removed_user_ids.extend(
[user_id_dict[email] for email in amendment_json['removed_users']])
return amendment
def _ParseAttachment(self, attachment_json, _event_log):
attachment = tracker_pb2.Attachment(
filename=attachment_json['name'],
filesize=attachment_json['size'],
mimetype=attachment_json['mimetype'],
gcs_object_id=attachment_json['gcs_object_id']
)
return attachment
def _SaveObjects(self, cnxn, project_id, state, event_log):
"""Examine JSON data and create users, issues, and comments."""
created_user_ids = self.services.user.LookupUserIDs(
cnxn, state.nonexist_emails, autocreate=True)
for created_email, created_id in created_user_ids.items():
if created_id != state.user_id_dict[created_email]:
event_log.append('Mismatched user_id for %r' % created_email)
raise JSONImportError()
event_log.append('Created %d users' % len(state.nonexist_emails))
total_comments = 0
total_stars = 0
config = self.services.config.GetProjectConfig(cnxn, project_id)
for issue in state.issue_list:
# TODO(jrobbins): renumber issues if there is a local_id conflict.
if issue.local_id not in state.starrers_dict:
# Issues with stars will have filter rules applied in SetStar().
filterrules_helpers.ApplyFilterRules(
cnxn, self.services, issue, config)
issue_id = self.services.issue.InsertIssue(cnxn, issue)
for comment in state.comments_dict[issue.local_id]:
total_comments += 1
comment.issue_id = issue_id
self.services.issue.InsertComment(cnxn, comment)
self.services.issue_star.SetStarsBatch(
cnxn, self.services, config, issue_id,
state.starrers_dict[issue.local_id], True)
total_stars += len(state.starrers_dict[issue.local_id])
event_log.append('Created %d issues' % len(state.issue_list))
event_log.append('Created %d comments for %d issues' % (
total_comments, len(state.comments_dict)))
event_log.append('Set %d stars on %d issues' % (
total_stars, len(state.starrers_dict)))
global_relations_dict = collections.defaultdict(list)
for issue, rels in state.relations_dict.items():
src_iid = self.services.issue.GetIssueByLocalID(
cnxn, project_id, issue).issue_id
dst_iids = [i.issue_id for i in self.services.issue.GetIssuesByLocalIDs(
cnxn, project_id, [rel[0] for rel in rels])]
kinds = [rel[1] for rel in rels]
global_relations_dict[src_iid] = list(zip(dst_iids, kinds))
self.services.issue.RelateIssues(cnxn, global_relations_dict)
self.services.issue.SetUsedLocalID(cnxn, project_id)
event_log.append('Finished import')
class JSONImportError(Exception):
"""Exception to raise if imported JSON is invalid."""
pass