dev/githubanalysis/lib/issue.dart

// Copyright 2023 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io';
import 'dart:math' as math;

import 'package:github/github.dart';
import 'package:http/http.dart';

import 'cache.dart';
import 'utils.dart';


const List<String> priorities = <String>['P0', 'P1', 'P2', 'P3'];

class FullIssue {
  FullIssue(this.repo, this.issueNumber, this._metadata, this.comments, this.reactions, { this.redirect, this.isDeleted = false }) {
    _labels = _metadata?.labels.map<String>((final IssueLabel label) => label.name).toSet();
    if (_labels != null) {
      final Set<String> matches = priorities.toSet().intersection(_labels!);
      if (matches.length > 1) {
        print('\nWARNING: Too many priority labels on issue #$issueNumber: ${matches.join(', ')}');
        for (final String priority in priorities) {
          if (matches.contains(priority)) {
            _priority = priority;
            break;
          }
        }
      } else if (matches.length == 1) {
        _priority = matches.single;
      } else {
        _priority = null;
      }
    }
  }

  final RepositorySlug repo;
  final int issueNumber;
  final Issue? _metadata;
  Issue get metadata => _metadata!; // only available when isValid
  final List<IssueComment> comments;
  final List<Reaction> reactions;
  final String? redirect; // not null when metadata == null && !isDeleted
  final bool isDeleted;

  late final Set<String>? _labels;
  Set<String> get labels => _labels!; // only available when isValid

  late final String? _priority;
  String? get priority => _priority; // only available when isValid

  bool get isValid => _metadata != null;
  bool get isPullRequest => _metadata?.pullRequest != null;

  static Future<FullIssue> load({
    required final Directory cache,
    required final GitHub github,
    required final RepositorySlug repo,
    required final int issueNumber,
    final DateTime? cacheEpoch,
  }) async {
    final String cacheData = await loadFromCache(cache, github, <String>['issue', repo.owner, repo.name, issueNumber.toString()], cacheEpoch, () async {
      try {
        final Issue issue = await github.issues.get(repo, issueNumber);
        if (issue.url.startsWith(github.endpoint)) {
          if (issue.url != '${github.endpoint}/repos/$repo/issues/$issueNumber') {
            return json.encode(<String, Object?>{ 'redirect': issue.url });
          }
        } else if (issue.url == '') {
          return json.encode(<String, Object?>{ 'deleted': true });
        }
        final List<IssueComment> comments = await github.issues.listCommentsByIssue(repo, issueNumber).toList();
        final List<Reaction> reactions = await github.issues.listReactions(repo, issueNumber).toList();
        return json.encode(<String, Object?>{ 'issue': issue, 'comments': comments, 'reactions': reactions });
      } on ClientException catch (e) {
        // print('\nIssue $issueNumber is a problem child (treating as deleted): $e');
        return json.encode(<String, Object?>{ 'deleted': true, 'error': e.toString() });
      }
    });
    final Map<String, Object?> data = json.decode(cacheData)! as Map<String, Object?>;
    if (data['redirect'] != null) {
      return FullIssue(repo, issueNumber, null, const <IssueComment>[], const <Reaction>[], redirect: data['redirect']! as String);
    }
    if (data['deleted'] == true) {
      return FullIssue(repo, issueNumber, null, const <IssueComment>[], const <Reaction>[], isDeleted: true);
    }
    final Issue issue = Issue.fromJson(data['issue']! as Map<String, Object?>);
    return FullIssue(
      repo,
      issueNumber,
      issue,
      (data['comments']! as List<Object?>)
        .cast<Map<String, Object?>>()
        .map<IssueComment>(IssueComment.fromJson)
        .toList(),
      (data['reactions']! as List<Object?>)
        .cast<Map<String, Object?>>()
        .map<Reaction>(Reaction.fromJson)
        .toList(),
    );
  }

  @override
  String toString() {
    final StringBuffer result = StringBuffer();
    if (isValid) {
      result
        ..writeln('Issue $issueNumber (${metadata.state}): ${metadata.title}')
        ..writeln(metadata.htmlUrl)
        ..writeln('Created by ${metadata.user!.login} on ${metadata.createdAt}, last updated on ${metadata.updatedAt}.');
      if (metadata.isClosed) {
        result.writeln('Closed by ${metadata.closedBy?.login} on ${metadata.closedAt}.');
        if (metadata.closedBy != null) {
          final User user = metadata.closedBy!;
          result
            ..writeln('  avatarUrl: ${user.avatarUrl}')
            ..writeln('  bio: ${user.bio}')
            ..writeln('  blog: ${user.blog}')
            ..writeln('  company: ${user.company}')
            ..writeln('  createdAt: ${user.createdAt}')
            ..writeln('  email: ${user.email}')
            ..writeln('  followersCount: ${user.followersCount}')
            ..writeln('  followingCount: ${user.followingCount}')
            ..writeln('  hirable: ${user.hirable}')
            ..writeln('  htmlUrl: ${user.htmlUrl}')
            ..writeln('  id: ${user.id}')
            ..writeln('  location: ${user.location}')
            ..writeln('  login: ${user.login}')
            ..writeln('  name: ${user.name}')
            ..writeln('  publicGistsCount: ${user.publicGistsCount}')
            ..writeln('  publicReposCount: ${user.publicReposCount}')
            ..writeln('  siteAdmin: ${user.siteAdmin}')
            ..writeln('  twitterUsername: ${user.twitterUsername}')
            ..writeln('  updatedAt: ${user.updatedAt}');
        }
      }
      if (isPullRequest) {
        result.writeln('  Pull request: ${metadata.pullRequest?.diffUrl}');
      }
      result
        ..writeln('Assignees: ${metadata.assignees!.map((final User user) => user.login!).join(', ')}')
        ..writeln('Labels: ${metadata.labels.map((final IssueLabel label) => label.name).join(', ')}')
        ..writeln('Milestone: ${metadata.milestone ?? ""}');
    } else {
      result.writeln('Issue $issueNumber (redirected): see $redirect');
    }
    for (final IssueComment comment in comments) {
      final String line = comment.body!.split('\n').first;
      final String body = line.substring(0, math.min(40, line.length));
      result.writeln('Comment by ${comment.user!.login} at ${comment.createdAt}: $body');
    }
    for (final Reaction reaction in reactions) {
      result.writeln('Reaction: ${reaction.user!.login} ${reaction.content} at ${reaction.createdAt}');
    }
    return result.toString();
  }
}

Future<void> fetchAllIssues(final GitHub github, final Directory cache, final RepositorySlug repo, final Duration issueMaxAge, final Map<int, FullIssue> results) async {
  int index = 1;
  int issues = 0;
  int prs = 0;
  int invalid = 0;
  final File lastIssueNumberFile = cacheFileFor(cache, <String>['issue', repo.owner, repo.name, 'last']);
  int lastIssueNumber = await readFromFile<int>(lastIssueNumberFile, int.tryParse) ?? 0;
  bool maxKnown = false;
  while (true) {
    try {
      final FullIssue issue = await FullIssue.load(
        cache: cache,
        github: github,
        repo: repo,
        issueNumber: index,
        cacheEpoch: maxAge(issueMaxAge),
      );
      if (issue.isValid) {
        if (issue.isPullRequest) {
          prs += 1;
        } else {
          issues += 1;
        }
      } else if (issue.isDeleted) {
        throw NotFound(github, 'Issue $index deleted or not yet filed.');
      } else {
        // probably redirect
        invalid += 1;
      }
      results[index] = issue;
    } on FormatException catch (e) {
      print('\nIssue $index could not be processed: $e');
      invalid += 1;
    } on NotFound {
      if (index > lastIssueNumber) {
        final Issue lastIssue = await github.issues.listByRepo(repo, state: 'all', sort: 'created', direction: 'desc').first;
        lastIssueNumber = lastIssue.number;
        maxKnown = true;
        if (index > lastIssueNumber) {
          break;
        }
        invalid += 1;
      }
    }
    await rateLimit(github, '${repo.fullName}: #$index${ lastIssueNumber > 0 ? " of ${maxKnown ? "" : "~"}$lastIssueNumber" : ""} ($issues issues; $prs PRs; $invalid errors)', 'issue #${index + 1}');
    index += 1;
  }
  await lastIssueNumberFile.writeAsString(lastIssueNumber.toString());
  stdout.write('\x1B[K\r');
}

Future<void> updateAllIssues(final GitHub github, final Directory cache, final RepositorySlug repo, final Map<int, FullIssue> issues) async {
  final Set<int> pendingIssues = issues.isEmpty ? <int>{} : issues.keys.toSet();
  int highestKnownIssue = pendingIssues.isEmpty ? 0 : (pendingIssues.toList()..sort()).last;
  final File updateStampFile = cacheFileFor(cache, <String>['issue', repo.owner, repo.name, 'last-update']);
  final DateTime? lastFullScanStartTime = await readFromFile<DateTime>(updateStampFile, DateTime.tryParse);
  DateTime? thisFullScanStartTime;
  int count = 0;
  await rateLimit(github, '${repo.fullName}: fetching issues with recent changes', 'scan');
  await for (final Issue summary in github.issues.listByRepo(repo, state: 'all', sort: 'updated', direction: 'desc', perPage: 100)) {
    if (summary.updatedAt != null) {
      thisFullScanStartTime ??= summary.updatedAt!;
    }
    if (mode == Mode.abbreviated && summary.updatedAt != null && lastFullScanStartTime != null && summary.updatedAt!.isBefore(lastFullScanStartTime)) {
      stdout.write('\x1B[K\r');
      return;
    }
    const int maxRetry = 5;
    for (int retry = 1; retry <= maxRetry; retry += 1) {
      try {
        final String parenthetical;
        Duration? delta = lastFullScanStartTime?.difference(summary.updatedAt!);
        if (delta != null && delta.inMilliseconds < 0) {
          delta = -delta;
          if (delta.inDays > 2) {
            parenthetical = '${delta.inDays} days remaining';
          } else if (delta.inHours > 2) {
            parenthetical = '${delta.inHours} hours remaining';
          } else if (delta.inMinutes > 2) {
            parenthetical = '${delta.inMinutes} minutes remaining';
          } else {
            parenthetical = '${delta.inMilliseconds}ms remaining';
          }
        } else {
          parenthetical = '${100 * count ~/ (pendingIssues.length + count)}% newer than ${summary.updatedAt}';
        }
        await rateLimit(github, '${repo.fullName}: $count issues updated ($parenthetical)', 'scan');
        final FullIssue issue = await FullIssue.load(
          cache: cache,
          github: github,
          repo: repo,
          issueNumber: summary.number,
          cacheEpoch: summary.updatedAt,
        );
        assert(!issue.isValid || !issue.metadata.updatedAt!.isBefore(summary.updatedAt!), 'invariant violation\nOLD DATA:\n${json.encode(issue.metadata.toJson())}\nNEW DATA:\n${json.encode(summary.toJson())}');
        if (issue.issueNumber > highestKnownIssue) {
          for (int index = highestKnownIssue; index < issue.issueNumber; index += 1) {
            pendingIssues.add(index);
          }
          highestKnownIssue = issue.issueNumber;
        } else {
          pendingIssues.remove(issue.issueNumber);
        }
        issues[issue.issueNumber] = issue;
        break;
      } on FormatException catch (e) {
        print('\nError while updating issue #${summary.number} (attempt $retry/$maxRetry): $e');
      }
    }
    count += 1;
  }
  stdout.write('\x1B[K\r');
  if (pendingIssues.isNotEmpty) {
    // looks like these went away, so force fetch them
    int count = 0;
    for (final int issueNumber in pendingIssues) {
      await rateLimit(github, '${repo.fullName}: $count / ${pendingIssues.length} missing issues checked', 'issue #$issueNumber');
      try {
        issues[issueNumber] = await FullIssue.load(
          cache: cache,
          github: github,
          repo: repo,
          issueNumber: issueNumber,
          cacheEpoch: lastFullScanStartTime,
        );
        count += 1;
      } on Exception catch (e) {
        print('\nError while updating issue #$issueNumber: $e');
      }
    }
    stdout.write('\x1B[K\r');
  }
  if (thisFullScanStartTime != null) {
    await updateStampFile.writeAsString(thisFullScanStartTime.toIso8601String());
  }
}