blob: 26392f94ab3a71646c9299b1a0be88522495328d [file] [edit]
#!/usr/bin/env bash
# Aggregate the results.jsonl artifacts written by parse-flaky-results.sh.
#
# Flaky means a target failed and then passed within the same run, either on a
# retry or in the debug rerun. Every job feeding this report has the rerun, so a
# run that never went green failed every chance it had. That is never counted as
# a flake; a target whose last FAILING_RUNS runs all ended that way is failing.
#
# Both halves of the rate come from the records themselves: every execution is
# one sample, so nothing here assumes how often anything is scheduled. If a
# target is run by five jobs on the same OS, that is five samples; if the
# schedule doubles, the denominator doubles on its own.
#
# A target is tracked per operating system, since the same label passing on Linux
# and flaking on Windows is one flaky test, not two halves of one.
#
# The table covers WINDOW_DAYS so a manual run lists everything known to flake.
#
# A flake that passed on the next attempt is the normal background noise of
# browser tests, and runs per target are too few for a rate to mean much. So a
# test is flagged only when it needed the debug rerun in the last day, meaning
# every retry failed, and it also flaked at least once in the HISTORY_DAYS
# before. A fixed test stops needing the rerun, so it stops being flagged
# without anything to reset. Browser variants of a test count as one test.
set -euo pipefail
WINDOW_DAYS=21
RECENT_DAYS=7
HISTORY_DAYS=14
FAILING_RUNS=2
BROWSER_SUFFIX='-(chrome|firefox|edge|safari)(-beta|-preview)?(-remote|-bidi)?(-alerts)?$'
REPO="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}"
WORK=$(mktemp -d)
trap 'rm -rf "$WORK"' EXIT
days_ago() {
date -u -d "$1 days ago" +"$2" 2>/dev/null || date -u -v-"$1"d +"$2"
}
window_start=$(days_ago "$WINDOW_DAYS" %Y-%m-%dT%H:%M:%SZ)
recent_start=$(days_ago "$RECENT_DAYS" %Y-%m-%dT%H:%M:%SZ)
last_day_start=$(days_ago 1 %Y-%m-%dT%H:%M:%SZ)
history_start=$(days_ago "$((HISTORY_DAYS + 1))" %Y-%m-%dT%H:%M:%SZ)
# A run that merged its per-job results is represented by that one file; any
# other run falls back to the artifacts its jobs uploaded. That covers a workflow
# with a single test job, which has nothing to merge, and a run whose merge job
# failed. Fetch a day of slack and let the record timestamps do the windowing.
fetch_from=$(days_ago "$((WINDOW_DAYS + 1))" %Y-%m-%dT%H:%M:%SZ)
echo "Collecting results created since $fetch_from"
gh api "repos/$REPO/actions/artifacts" --paginate \
--jq '.artifacts[] | select(.expired == false)
| select((.name | startswith("flaky-")) or (.name | startswith("merged-flaky-results-")))
| [.id, .created_at, .workflow_run.id, .name] | @tsv' \
> "$WORK/artifacts.tsv"
awk -F'\t' -v from="$fetch_from" '
$2 > from {
row[++n] = $0
if ($4 ~ /^merged-flaky-results-/) merged[$3] = 1
}
END {
for (i = 1; i <= n; i++) {
split(row[i], f, "\t")
if (f[4] ~ /^merged-flaky-results-/ || !(f[3] in merged)) print f[1]
}
}' "$WORK/artifacts.tsv" > "$WORK/wanted.txt"
available=$(wc -l < "$WORK/wanted.txt" | tr -d ' ')
downloaded=0
: > "$WORK/all.jsonl"
while IFS= read -r id; do
gh api "repos/$REPO/actions/artifacts/$id/zip" > "$WORK/artifact.zip" || continue
unzip -p "$WORK/artifact.zip" results.jsonl >> "$WORK/all.jsonl" || continue
downloaded=$((downloaded + 1))
done < "$WORK/wanted.txt"
if [ "$downloaded" -lt "$available" ]; then
echo "::warning::Only $downloaded of $available result files in the window could be read; rates below are based on an incomplete sample."
fi
echo "Read $downloaded of $available result files"
jq -s --arg lastDay "$last_day_start" --arg history "$history_start" --arg suffix "$BROWSER_SUFFIX" '
map(select(.timestamp >= $history and (.status | endswith("-recovered")))
| . + {test: (.target | sub($suffix; ""))})
| group_by([.test, .os])
| map(select(any(.[]; .status == "rerun-recovered" and .timestamp >= $lastDay)
and any(.[]; .timestamp < $lastDay))
| {test: .[0].test, os: .[0].os, flakes: length,
targets: (map(.target) | unique)})
' "$WORK/all.jsonl" > "$WORK/flagged.json"
jq -s --arg recent "$recent_start" --arg window "$window_start" --slurpfile flagged "$WORK/flagged.json" '
[$flagged[0][] | .os as $os | .targets[] | "\(.)|\($os)"] as $flaggedTargets
| map(select(.timestamp >= $window))
| group_by([.target, .os])
| map(
[.[] | select(.status | endswith("-recovered")) | .timestamp] as $flakes
| {
target: .[0].target,
os: .[0].os,
ran: length,
flaked: ($flakes | length),
reruns: ([.[] | select(.status == "rerun-recovered")] | length),
failed: ([.[] | select(.status == "failed")] | length),
last: ($flakes | max),
# Every flake it has is recent, so it started this week.
new: (($flakes | min // "9999") >= $recent)
}
| . + {rate: (.flaked * 100 / .ran), flagged: ("\(.target)|\(.os)" | IN($flaggedTargets[]))})
| map(select(.flaked > 0))
| sort_by((.flagged | not), -.rate, .target)
' "$WORK/all.jsonl" > "$WORK/report.json"
# The latest run must be recent so a target dropped from the schedule while
# failing is not reported for the rest of the window.
jq -s --arg recent "$recent_start" --arg window "$window_start" --argjson failingRuns "$FAILING_RUNS" '
map(select(.timestamp >= $window))
| group_by([.target, .os])
| map(
sort_by(.timestamp)
| {
target: .[0].target,
os: .[0].os,
streak: ([.[].status == "failed"] | reverse | index(false) // length),
lastRan: .[-1].timestamp,
lastPassed: ([.[] | select(.status != "failed") | .timestamp] | max)
})
| map(select(.streak >= $failingRuns and .lastRan >= $recent))
| sort_by(-.streak, .target)
' "$WORK/all.jsonl" > "$WORK/failing.json"
{
echo "## Failing tests"
echo
echo "Failed every attempt, including the debug rerun, in at least their last $FAILING_RUNS runs."
echo
if jq -e 'length > 0' "$WORK/failing.json" > /dev/null; then
echo "| target | os | failed in a row | last passed |"
echo "|---|---|---|---|"
jq -r '.[] | "| `\(.target)` | \(.os) | \(.streak) | \(.lastPassed // "not in window") |"' "$WORK/failing.json"
else
echo "None."
fi
echo
echo "## Flaky tests"
echo
echo "Every execution counts as one sample, taken from $downloaded scheduled runs over the last $WINDOW_DAYS days."
echo "Flagged when the test needed the debug rerun to pass in the last day and also flaked in the $HISTORY_DAYS days before, counting browser variants together."
echo "Runs that never went green are counted under failed, not as flakes."
echo
echo "| target | os | flaked | needed rerun | failed | of runs | rate | flagged | new | last flaked |"
echo "|---|---|---|---|---|---|---|---|---|---|"
jq -r '.[] | "| `\(.target)` | \(.os) | \(.flaked) | \(.reruns) | \(.failed) | \(.ran) | \(.rate | round)% | \(if .flagged then "yes" else "" end) | \(if .new then "yes" else "" end) | \(.last) |"' "$WORK/report.json"
} > "$WORK/report.md"
# Every flaky target, whatever the thresholds say. The thresholds only decide
# what Slack hears about, so a manual run is a full list of what is known to be
# flaky — the thing to check a suspicious PR failure against.
cat "$WORK/report.md"
cat "$WORK/report.md" >> "${GITHUB_STEP_SUMMARY:-/dev/null}"
failing=$(jq 'length' "$WORK/failing.json")
flagged=$(jq 'length' "$WORK/flagged.json")
flagged_names=$(jq -r 'map("\(.test | sub("^//"; "")) (\(.os))") | join(", ")' "$WORK/flagged.json")
# A target that fails every attempt is broken, not flaky, so it leads.
if [ "$failing" -gt 0 ]; then
color=failure
message="needs investigation, $failing targets failed every one of their last $FAILING_RUNS runs"
[ "$flagged" -eq 0 ] || message="$message; recurring flakes in $flagged_names"
elif [ "$flagged" -gt 0 ]; then
color=warning
message="recurring flakes that needed the debug rerun in $flagged_names"
else
echo "Nothing to report: no target failed its last $FAILING_RUNS runs and no recurring flake needed the debug rerun."
exit 0
fi
echo "$message"
{
echo "color=$color"
echo "message=$message"
} >> "${GITHUB_OUTPUT:-/dev/null}"