blob: 930ec94ff2e4e3fed73d91d37388391e619fa9c7 [file] [log] [blame]
// Copyright 2019 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:cocoon_service/src/model/github/checks.dart';
import 'package:cocoon_service/src/service/github_checks_service.dart';
import 'package:cocoon_service/src/service/github_status_service.dart';
import 'package:cocoon_service/src/service/luci_build_service.dart';
import 'package:crypto/crypto.dart';
import 'package:github/github.dart';
import 'package:github/hooks.dart';
import 'package:meta/meta.dart';
import '../datastore/cocoon_config.dart';
import '../request_handling/body.dart';
import '../request_handling/exceptions.dart';
import '../request_handling/request_handler.dart';
import '../service/buildbucket.dart';
/// List of repos that require CQ+1 label.
const Set<String> kNeedsCQLabelList = <String>{'flutter/flutter'};
/// List of repos that require check for golden triage.
const Set<String> kNeedsCheckGoldenTriage = <String>{'flutter/flutter'};
/// List of repos that require check for labels and tests.
const Set<String> kNeedsCheckLabelsAndTests = <String>{
'flutter/flutter',
'flutter/engine'
};
final RegExp kEngineTestRegExp = RegExp(r'tests?\.(dart|java|mm|m|cc)$');
@immutable
class GithubWebhook extends RequestHandler<Body> {
GithubWebhook(Config config, this.buildBucketClient, this.luciBuildService,
this.githubStatusService, this.githubChecksService,
{HttpClient skiaClient})
: assert(buildBucketClient != null),
skiaClient = skiaClient ?? HttpClient(),
super(config: config);
/// A client for querying and scheduling LUCI Builds.
final BuildBucketClient buildBucketClient;
/// An Http Client for querying the Skia Gold API.
final HttpClient skiaClient;
/// Github status service to update the state of the build
/// in the Github UI.
final GithubStatusService githubStatusService;
/// LUCI service class to communicate with buildBucket service.
final LuciBuildService luciBuildService;
/// Github checks service. Used to provide build status to github.
final GithubChecksService githubChecksService;
@override
Future<Body> post() async {
final String gitHubEvent = request.headers.value('X-GitHub-Event');
if (gitHubEvent == null ||
request.headers.value('X-Hub-Signature') == null) {
throw const BadRequestException('Missing required headers.');
}
final List<int> requestBytes = await request.expand((_) => _).toList();
final String hmacSignature = request.headers.value('X-Hub-Signature');
if (!await _validateRequest(hmacSignature, requestBytes)) {
throw const Forbidden();
}
try {
final String stringRequest = utf8.decode(requestBytes);
switch (gitHubEvent) {
case 'pull_request':
await _handlePullRequest(stringRequest);
break;
case 'check_suite':
final CheckSuiteEvent checkSuiteEvent = CheckSuiteEvent.fromJson(
jsonDecode(stringRequest) as Map<String, dynamic>,
);
await githubChecksService.handleCheckSuite(
checkSuiteEvent, luciBuildService);
break;
case 'check_run':
final CheckRunEvent checkRunEvent = CheckRunEvent.fromJson(
jsonDecode(stringRequest) as Map<String, dynamic>,
);
await githubChecksService.handleCheckRun(
checkRunEvent, luciBuildService);
}
return Body.empty;
} on FormatException {
throw const BadRequestException('Could not process input data.');
} on InternalServerError {
rethrow;
}
}
Future<void> _handlePullRequest(
String rawRequest,
) async {
final PullRequestEvent pullRequestEvent =
await _getPullRequestEvent(rawRequest);
if (pullRequestEvent == null) {
throw const BadRequestException('Expected pull request event.');
}
final String eventAction = pullRequestEvent.action;
final PullRequest pr = pullRequestEvent.pullRequest;
// See the API reference:
// https://developer.github.com/v3/activity/events/types/#pullrequestevent
// which unfortunately is a bit light on explanations.
switch (eventAction) {
case 'closed':
// On a successful merge, check for gold.
// If it was closed without merging, cancel any outstanding tryjobs.
// We'll leave unfinished jobs if it was merged since we care about those
// results.
if (pr.merged) {
await _checkForGoldenTriage(pullRequestEvent);
} else {
await luciBuildService.cancelBuilds(
pullRequestEvent.repository.slug(),
pr.number,
pr.head.sha,
'Pull request closed',
);
}
break;
case 'edited':
// Editing a PR should not trigger new jobs, but may update whether
// it has tests.
await _checkForLabelsAndTests(pullRequestEvent);
break;
case 'opened':
case 'ready_for_review':
case 'reopened':
// These cases should trigger LUCI jobs.
await _checkForLabelsAndTests(pullRequestEvent);
await _scheduleIfMergeable(pullRequestEvent);
break;
case 'labeled':
// This should only trigger a LUCI job for flutter/flutter right now,
// since it is in the needsCQLabelList.
if (kNeedsCQLabelList.contains(pr.base.repo.fullName.toLowerCase())) {
await _scheduleIfMergeable(pullRequestEvent);
}
break;
case 'synchronize':
// This indicates the PR has new commits. We need to cancel old jobs
// and schedule new ones.
await _scheduleIfMergeable(pullRequestEvent);
break;
case 'unlabeled':
// Cancel the jobs if someone removed the label on a repo that needs
// them.
if (!kNeedsCQLabelList.contains(pr.base.repo.fullName.toLowerCase())) {
break;
}
if (!await _checkForCqLabel(pr.labels)) {
await luciBuildService.cancelBuilds(
pullRequestEvent.repository.slug(),
pr.number,
pr.head.sha,
'Tryjobs canceled (label removed)',
);
}
break;
// Ignore the rest of the events.
case 'assigned':
case 'locked':
case 'review_request_removed':
case 'review_requested':
case 'unassigned':
case 'unlocked':
break;
}
}
/// This method assumes that jobs should be cancelled if they are already
/// runnning. [githubStatusService] is used to update the status of a build
/// in the GitHub UI. When the build is triggered the status is set to "pending"
/// without a details link. Once the test starts running then the state is set
/// to "pending" with a details link pointing to the build in LUCI infrastructure.
Future<void> _scheduleIfMergeable(
PullRequestEvent pullRequestEvent,
) async {
// The mergeable flag may be null. False indicates there's a merge conflict,
// null indicates unknown. Err on the side of allowing the job to run.
final PullRequest pr = pullRequestEvent.pullRequest;
// For flutter/flutter tests need to be optimized before enforcing CQ.
if (kNeedsCQLabelList.contains(pr.base.repo.fullName.toLowerCase())) {
if (!await _checkForCqLabel(pr.labels)) {
return;
}
}
// Always cancel running builds so we don't ever schedule duplicates.
await luciBuildService.cancelBuilds(
pullRequestEvent.repository.slug(),
pr.number,
pr.head.sha,
'Newer commit available',
);
await luciBuildService.scheduleBuilds(
slug: pullRequestEvent.repository.slug(),
prNumber: pr.number,
commitSha: pr.head.sha,
);
await githubStatusService.setBuildsPendingStatus(
pr.number, pr.head.sha, pr.head.repo.slug());
}
/// Checks the issue in the given repository for `config.cqLabelName`.
Future<bool> _checkForCqLabel(List<IssueLabel> labels) async {
final String cqLabelName = config.cqLabelName;
return labels.any((IssueLabel label) => label.name == cqLabelName);
}
Future<bool> _isIgnoredForGold(String eventAction, PullRequest pr) async {
bool ignored = false;
String rawResponse;
try {
final HttpClientRequest request = await skiaClient
.getUrl(Uri.parse('https://flutter-gold.skia.org/json/ignores'));
final HttpClientResponse response = await request.close();
rawResponse = await utf8.decodeStream(response);
final List<dynamic> ignores = jsonDecode(rawResponse) as List<dynamic>;
for (Map<String, dynamic> ignore
in ignores.cast<Map<String, dynamic>>()) {
if ((ignore['note'] as String).isNotEmpty &&
pr.number.toString() == ignore['note'].split('/').last) {
ignored = true;
break;
}
}
} on IOException catch (e) {
log.error('Request to Flutter Gold for ignores failed for PR '
'#${pr.number} on action: $eventAction.\n'
'error: $e');
} on FormatException catch (_) {
log.error('Format Exception from Flutter Gold ignore request.\n'
'rawResponse: $rawResponse');
rethrow;
}
return ignored;
}
Future<void> _checkForGoldenTriage(PullRequestEvent pullRequestEvent) async {
final PullRequest pr = pullRequestEvent.pullRequest;
final String eventAction = pullRequestEvent.action;
final RepositorySlug slug = pullRequestEvent.repository.slug();
if (kNeedsCheckGoldenTriage.contains(pr.base.repo.fullName.toLowerCase()) &&
await _isIgnoredForGold(eventAction, pr)) {
final GitHub gitHubClient =
await config.createGitHubClient(slug.owner, slug.name);
try {
await _pingForTriage(gitHubClient, pr);
} finally {
gitHubClient.dispose();
}
}
}
Future<void> _pingForTriage(GitHub gitHubClient, PullRequest pr) async {
final String body = config.goldenTriageMessage;
final RepositorySlug slug = pr.base.repo.slug();
await gitHubClient.issues.createComment(slug, pr.number, body);
}
Future<void> _checkForLabelsAndTests(
PullRequestEvent pullRequestEvent) async {
final PullRequest pr = pullRequestEvent.pullRequest;
final String eventAction = pullRequestEvent.action;
final RepositorySlug slug = pullRequestEvent.repository.slug();
final String repo = pr.base.repo.fullName.toLowerCase();
if (kNeedsCheckLabelsAndTests.contains(repo)) {
final GitHub gitHubClient =
await config.createGitHubClient(slug.owner, slug.name);
try {
await _checkBaseRef(gitHubClient, pr);
if (repo == 'flutter/flutter') {
await _applyFrameworkRepoLabels(gitHubClient, eventAction, pr);
} else if (repo == 'flutter/engine') {
await _applyEngineRepoLabels(gitHubClient, eventAction, pr);
}
} finally {
gitHubClient.dispose();
}
}
}
Future<void> _applyFrameworkRepoLabels(
GitHub gitHubClient, String eventAction, PullRequest pr) async {
if (pr.user.login == 'engine-flutter-autoroll') {
return;
}
final RepositorySlug slug = pr.base.repo.slug();
final Stream<PullRequestFile> files =
gitHubClient.pullRequests.listFiles(slug, pr.number);
final Set<String> labels = <String>{};
bool hasTests = false;
bool needsTests = false;
bool isGoldenChange = false;
await for (PullRequestFile file in files) {
if (file.filename.endsWith('pubspec.yaml')) {
// These get updated by a script, and are updated en masse.
labels.add('team');
continue;
}
if (file.filename.endsWith('.dart')) {
needsTests = true;
}
if (file.filename.endsWith('_test.dart')) {
hasTests = true;
}
if (file.filename.startsWith('dev/')) {
labels.add('team');
}
if (file.filename.startsWith('packages/flutter_tools/') ||
file.filename.startsWith('packages/fuchsia_remote_debug_protocol')) {
labels.add('tool');
}
if (file.filename == 'bin/internal/engine.version') {
labels.add('engine');
}
if (await _isIgnoredForGold(eventAction, pr)) {
isGoldenChange = true;
labels.add('will affect goldens');
labels.add('severe: API break');
labels.add('a: tests');
}
if (file.filename.startsWith('packages/flutter/') ||
file.filename.startsWith('packages/flutter_test/') ||
file.filename.startsWith('packages/flutter_driver/')) {
labels.add('framework');
}
if (file.filename.contains('material')) {
labels.add('f: material design');
}
if (file.filename.contains('cupertino')) {
labels.add('f: cupertino');
}
if (file.filename.startsWith('packages/flutter_localizations')) {
labels.add('a: internationalization');
}
if (file.filename.startsWith('packages/flutter_test') ||
file.filename.startsWith('packages/flutter_driver')) {
labels.add('a: tests');
}
if (file.filename.contains('semantics') ||
file.filename.contains('accessibilty')) {
labels.add('a: accessibility');
}
if (file.filename.startsWith('examples/')) {
labels.add('d: examples');
labels.add('team');
if (file.filename.startsWith('examples/flutter_gallery')) {
labels.add('team: gallery');
}
}
}
if (pr.draft) {
labels.add('work in progress; do not review');
}
if (labels.isNotEmpty) {
await gitHubClient.issues
.addLabelsToIssue(slug, pr.number, labels.toList());
}
if (!hasTests && needsTests && !pr.draft) {
final String body = config.missingTestsPullRequestMessage;
if (!await _alreadyCommented(gitHubClient, pr, slug, body)) {
await gitHubClient.issues.createComment(slug, pr.number, body);
}
}
if (isGoldenChange) {
final String body = config.goldenBreakingChangeMessage;
if (!await _alreadyCommented(gitHubClient, pr, slug, body)) {
await gitHubClient.issues.createComment(slug, pr.number, body);
}
}
}
Future<void> _applyEngineRepoLabels(
GitHub gitHubClient, String eventAction, PullRequest pr) async {
if (pr.user.login == 'skia-flutter-autoroll') {
return;
}
final RepositorySlug slug = pr.base.repo.slug();
final Stream<PullRequestFile> files =
gitHubClient.pullRequests.listFiles(slug, pr.number);
final Set<String> labels = <String>{};
bool hasTests = false;
bool needsTests = false;
await for (PullRequestFile file in files) {
final String filename = file.filename.toLowerCase();
if (filename.endsWith('.dart') ||
filename.endsWith('.mm') ||
filename.endsWith('.m') ||
filename.endsWith('.java') ||
filename.endsWith('.cc')) {
needsTests = true;
}
if (kEngineTestRegExp.hasMatch(filename)) {
hasTests = true;
}
if (filename.startsWith('shell/platform/darwin/ios')) {
labels.add('platform-ios');
}
if (filename.startsWith('shell/platform/android')) {
labels.add('platform-android');
}
}
if (labels.isNotEmpty) {
await gitHubClient.issues
.addLabelsToIssue(slug, pr.number, labels.toList());
}
if (!hasTests && needsTests && !pr.draft) {
final String body = config.missingTestsPullRequestMessage;
if (!await _alreadyCommented(gitHubClient, pr, slug, body)) {
await gitHubClient.issues.createComment(slug, pr.number, body);
}
}
}
Future<void> _checkBaseRef(
GitHub gitHubClient,
PullRequest pr,
) async {
if (pr.base.ref != 'master') {
final String body = await _getWrongBaseComment(pr.base.ref);
final RepositorySlug slug = pr.base.repo.slug();
if (!await _alreadyCommented(gitHubClient, pr, slug, body)) {
await gitHubClient.pullRequests.edit(
slug,
pr.number,
base: 'master',
);
await gitHubClient.issues.createComment(slug, pr.number, body);
}
}
}
Future<bool> _alreadyCommented(
GitHub gitHubClient,
PullRequest pr,
RepositorySlug slug,
String message,
) async {
final Stream<IssueComment> comments =
gitHubClient.issues.listCommentsByIssue(slug, pr.number);
await for (IssueComment comment in comments) {
if (comment.body.contains(message)) {
return true;
}
}
return false;
}
Future<String> _getWrongBaseComment(String base) async {
final String messageTemplate = config.nonMasterPullRequestMessage;
return messageTemplate.replaceAll('{{branch}}', base);
}
Future<bool> _validateRequest(
String signature,
List<int> requestBody,
) async {
final String rawKey = await config.webhookKey;
final List<int> key = utf8.encode(rawKey);
final Hmac hmac = Hmac(sha1, key);
final Digest digest = hmac.convert(requestBody);
final String bodySignature = 'sha1=$digest';
return bodySignature == signature;
}
Future<PullRequestEvent> _getPullRequestEvent(String request) async {
if (request == null) {
return null;
}
try {
return PullRequestEvent.fromJson(
json.decode(request) as Map<String, dynamic>);
} on FormatException {
return null;
}
}
}