| #!/usr/bin/env bash | |
| # Parse the Bazel console log into one record per test target that actually ran, | |
| # so the weekly report has both halves of a flake rate: how often a target ran, | |
| # and how often it only passed because it was retried. | |
| # | |
| # Two ways a test shows up as flaky: | |
| # retry-recovered bazel's own --flaky_test_attempts saved it in place | |
| # rerun-recovered it failed the run outright, then passed in the debug rerun | |
| # | |
| # Cached results are skipped. A replayed result is not an execution and cannot | |
| # flake, so counting it would inflate the denominator. | |
| set -euo pipefail | |
| FAILURES=build/failures | |
| RESULTS="$FAILURES/results.tsv" | |
| mkdir -p "$FAILURES" | |
| : > "$RESULTS" | |
| [ -f build/bazel-console.log ] || exit 0 | |
| # Bazel's test summary, once colours are stripped (it runs with --color=yes): | |
| # //foo:bar PASSED in 1.2s | |
| # //foo:bar (cached) PASSED in 0.0s | |
| # //foo:bar FLAKY, failed in 1 out of 2 in 3.4s | |
| # //foo:bar FAILED in 2 out of 2 in 12.0s | |
| # //foo:bar FAILED in 2.0s <- no retries configured | |
| awk -v out="$RESULTS" ' | |
| { gsub(/\033\[[0-9;]*m/, "") } | |
| $1 !~ /^\/\// { next } | |
| $2 == "(cached)" { next } | |
| $2 ~ /^FLAKY/ { print $1 "\tretry-recovered\t" $5 "\t" $8 > out; next } | |
| $3 != "in" { next } | |
| $2 == "PASSED" { print $1 "\tpassed\t0\t1" > out; next } | |
| $2 ~ /(FAILED|TIMEOUT|INCOMPLETE)/ { | |
| if ($5 == "out") { print $1 "\tfailed\t" $4 "\t" $7 > out } | |
| else { print $1 "\tfailed\t1\t1" > out } | |
| } | |
| ' build/bazel-console.log | |
| # A target counts as recovered only when the rerun actually reported it passing. | |
| # Inferring it from absence in _run2.txt would also sweep up a rerun that timed | |
| # out, came back incomplete, or aborted before printing any summary at all -- | |
| # _run2.txt records FAILED only, so all of those look identical to a pass. | |
| if [ -f build/bazel-console2.log ]; then | |
| awk '{ gsub(/\033\[[0-9;]*m/, "") } | |
| $1 ~ /^\/\// && $2 == "PASSED" && $3 == "in" { print $1 }' build/bazel-console2.log \ | |
| | sort -u > "$FAILURES/_recovered.txt" | |
| awk -F'\t' -v recovered="$FAILURES/_recovered.txt" ' | |
| BEGIN { while ((getline t < recovered) > 0) { recover[t] = 1 } } | |
| $1 in recover { print $1 "\trerun-recovered\t" $3 "\t" ($4 + 1); next } | |
| { print } | |
| ' "$RESULTS" > "$RESULTS.tmp" && mv "$RESULTS.tmp" "$RESULTS" | |
| fi | |
| [ -s "$RESULTS" ] || exit 0 | |
| if awk -F'\t' '$2 ~ /-recovered$/ { found = 1 } END { exit !found }' "$RESULTS"; then | |
| { | |
| echo "### Flaky tests" | |
| echo | |
| awk -F'\t' '$2 ~ /-recovered$/ { printf "- `%s` failed %s of %s attempts (%s)\n", $1, $3, $4, $2 }' "$RESULTS" | |
| echo | |
| } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" | |
| fi | |
| jq -Rc \ | |
| --arg os "${RUNNER_OS:-}" \ | |
| --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ | |
| --arg run "${GITHUB_RUN_ID:-}" \ | |
| --arg job "${JOB_ID:-}" \ | |
| 'select(length > 0) | split("\t") | |
| | { target: .[0], status: .[1], failed: (.[2] | tonumber), attempts: (.[3] | tonumber), | |
| os: $os, timestamp: $timestamp } | |
| | if .status == "passed" then . else . + { run: $run, job: $job } end' \ | |
| "$RESULTS" > "$FAILURES/results.jsonl" | |
| echo "results=true" >> "${GITHUB_OUTPUT:-/dev/null}" | |
| echo "Recorded $(wc -l < "$RESULTS" | tr -d ' ') test results" |