Merge upstream PR #167: Conditionally show more metric details
See https://github.com/WebKit/JetStream/pull/167
- Collect and report subTimes like subScores
- Only show times if the global .details CSS class is set (the default view is unaffected)
- Also use .details for displaying displayCategoryScores
- Change uiFriendlyDuration to only report 2 significant digits
- Add Benchmark.prototype.wallTime accessor and report that as well
- Use # as unit symbol for console printing
Change-Id: Ibc603b29c516060823c39c738069466cff57170a
Reviewed-on: https://chromium-review.googlesource.com/c/external/github.com/WebKit/JetStream/+/6976231
Reviewed-by: Victor Gomes <victorgomes@chromium.org>
diff --git a/JetStream.css b/JetStream.css
index b510f02..c48cc3c 100644
--- a/JetStream.css
+++ b/JetStream.css
@@ -348,6 +348,13 @@
scroll-margin-bottom: 20vh;
}
+.benchmark .result.detail {
+ display: none;
+}
+.details .benchmark .result.detail {
+ display: inline-block;
+}
+
.benchmark h4,
.benchmark .result,
.benchmark label,
diff --git a/JetStreamDriver.js b/JetStreamDriver.js
index 8b74ab2..26f500a 100644
--- a/JetStreamDriver.js
+++ b/JetStreamDriver.js
@@ -37,25 +37,8 @@
this.currentResolve = null;
this.currentReject = null;
-let showScoreDetails = false;
-let categoryScores = null;
-
function displayCategoryScores() {
- if (!categoryScores)
- return;
-
- let scoreDetails = `<div class="benchmark benchmark-done">`;
- for (let [category, scores] of categoryScores) {
- scoreDetails += `<span class="result">
- <span>${uiFriendlyScore(geomeanScore(scores))}</span>
- <label>${category}</label>
- </span>`;
- }
- scoreDetails += "</div>";
- let summaryElement = document.getElementById("result-summary");
- summaryElement.innerHTML += scoreDetails;
-
- categoryScores = null;
+ document.body.classList.add("details");
}
function getIterationCount(plan) {
@@ -81,19 +64,22 @@
if (isInBrowser) {
document.onkeydown = (keyboardEvent) => {
const key = keyboardEvent.key;
- if (key === "d" || key === "D") {
- showScoreDetails = true;
+ if (key === "d" || key === "D")
displayCategoryScores();
- }
};
}
-function mean(values) {
+function sum(values) {
console.assert(values instanceof Array);
let sum = 0;
for (let x of values)
sum += x;
- return sum / values.length;
+ return sum;
+}
+
+function mean(values) {
+ const totalSum = sum(values)
+ return totalSum / values.length;
}
function geomeanScore(values) {
@@ -135,9 +121,25 @@
}
function uiFriendlyDuration(time) {
- return `${time.toFixed(3)} ms`;
+ return `${time.toFixed(2)} ms`;
}
+
+function shellFriendlyLabel(label) {
+ const namePadding = 40;
+ return `${label}:`.padEnd(namePadding);
+}
+
+const valuePadding = 10;
+function shellFriendlyDuration(time) {
+ return `${uiFriendlyDuration(time)}`.padStart(valuePadding);
+}
+
+function shellFriendlyScore(time) {
+ return `${uiFriendlyScore(time)} # `.padStart(valuePadding);
+}
+
+
// TODO: Cleanup / remove / merge. This is only used for caching loads in the
// non-browser setting. In the browser we use exclusively `loadCache`,
// `loadBlob`, `doLoadBlob`, `prefetchResourcesForBrowser` etc., see below.
@@ -226,7 +228,7 @@
if (isInBrowser)
document.getElementById("benchmark-total-time-score").innerHTML = uiFriendlyNumber(totalTime);
else if (!JetStreamParams.dumpJSONResults)
- console.log("Total time:", uiFriendlyNumber(totalTime));
+ console.log("Total-Time:", uiFriendlyNumber(totalTime));
allScores.push(totalTime);
}
@@ -237,10 +239,13 @@
allScores.push(score);
}
- categoryScores = new Map;
+ const categoryScores = new Map();
+ const categoryTimes = new Map();
for (const benchmark of this.benchmarks) {
for (let category of Object.keys(benchmark.subScores()))
categoryScores.set(category, []);
+ for (let category of Object.keys(benchmark.subTimes()))
+ categoryTimes.set(category, []);
}
for (const benchmark of this.benchmarks) {
@@ -249,25 +254,55 @@
console.assert(value > 0, `Invalid ${benchmark.name} ${category} score: ${value}`);
arr.push(value);
}
+ for (let [category, value] of Object.entries(benchmark.subTimes())) {
+ const arr = categoryTimes.get(category);
+ console.assert(value > 0, `Invalid ${benchmark.name} ${category} time: ${value}`);
+ arr.push(value);
+ }
}
const totalScore = geomeanScore(allScores);
console.assert(totalScore > 0, `Invalid total score: ${totalScore}`);
if (isInBrowser) {
+ let summaryHtml = `<div class="score">${uiFriendlyScore(totalScore)}</div>
+ <label>Score</label>`;
+ summaryHtml += `<div class="benchmark benchmark-done">`;
+ for (let [category, scores] of categoryScores) {
+ summaryHtml += `<span class="result detail">
+ <span>${uiFriendlyScore(geomeanScore(scores))}</span>
+ <label>${category}</label>
+ </span>`;
+ }
+ summaryHtml += "<br/>";
+ for (let [category, times] of categoryTimes) {
+ summaryHtml += `<span class="result detail">
+ <span>${uiFriendlyDuration(geomeanScore(times))}</span>
+ <label>${category}</label>
+ </span>`;
+ }
+ summaryHtml += "</div>";
const summaryElement = document.getElementById("result-summary");
summaryElement.classList.add("done");
- summaryElement.innerHTML = `<div class="score">${uiFriendlyScore(totalScore)}</div>
- <label>Score</label>`;
+ summaryElement.innerHTML = summaryHtml;
summaryElement.onclick = displayCategoryScores;
- if (showScoreDetails)
- displayCategoryScores();
statusElement.innerHTML = "";
} else if (!JetStreamParams.dumpJSONResults) {
- console.log("\n");
- for (let [category, scores] of categoryScores)
- console.log(`${category}: ${uiFriendlyScore(geomeanScore(scores))}`);
- console.log("\nTotal Score: ", uiFriendlyScore(totalScore), "\n");
+ console.log("Total:");
+ for (let [category, scores] of categoryScores) {
+ console.log(
+ shellFriendlyLabel(`${category}-Score`),
+ shellFriendlyScore(geomeanScore(scores)));
+ }
+ for (let [category, times] of categoryTimes) {
+ console.log(
+ shellFriendlyLabel(`${category}-Time`),
+ shellFriendlyDuration(geomeanScore(times)));
+ }
+ console.log("");
+ console.log(shellFriendlyLabel("Total-Score"), shellFriendlyScore(totalScore));
+ console.log(shellFriendlyLabel("Total-Time"), shellFriendlyDuration(totalTime));
+ console.log("");
}
this.reportScoreToRunBenchmarkRunner();
@@ -284,33 +319,38 @@
prepareToRun() {
this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1);
- let text = "";
- for (const benchmark of this.benchmarks) {
- const description = Object.keys(benchmark.subScores());
- description.push("Score");
-
- const scoreIds = benchmark.scoreIdentifiers();
- const overallScoreId = scoreIds.pop();
-
- if (isInBrowser) {
- text +=
- `<div class="benchmark" id="benchmark-${benchmark.name}">
- <h3 class="benchmark-name">${benchmark.name} <a class="info" href="in-depth.html#${benchmark.name}">i</a></h3>
- <h4 class="score" id="${overallScoreId}"> </h4>
- <h4 class="plot" id="plot-${benchmark.name}"> </h4>
- <p>`;
- for (let i = 0; i < scoreIds.length; i++) {
- const scoreId = scoreIds[i];
- const label = description[i];
- text += `<span class="result"><span id="${scoreId}"> </span><label>${label}</label></span>`
- }
- text += `</p></div>`;
- }
- }
-
if (!isInBrowser)
return;
+ let text = "";
+ for (const benchmark of this.benchmarks) {
+ const scoreDescription = Object.keys(benchmark.allScores());
+ const timeDescription = Object.keys(benchmark.allTimes());
+
+ const scoreIds = benchmark.allScoreIdentifiers();
+ const overallScoreId = scoreIds.pop();
+ const timeIds = benchmark.allTimeIdentifiers();
+
+ text +=
+ `<div class="benchmark" id="benchmark-${benchmark.name}">
+ <h3 class="benchmark-name">${benchmark.name} <a class="info" href="in-depth.html#${benchmark.name}">i</a></h3>
+ <h4 class="score" id="${overallScoreId}"> </h4>
+ <h4 class="plot" id="plot-${benchmark.name}"> </h4>
+ <p>`;
+ for (let i = 0; i < scoreIds.length; i++) {
+ const scoreId = scoreIds[i];
+ const label = scoreDescription[i];
+ text += `<span class="result score"><span id="${scoreId}"> </span><label>${label}</label></span>`
+ }
+ text += "<br/>";
+ for (let i = 0; i < timeIds.length; i++) {
+ const timeId = timeIds[i];
+ const label = timeDescription[i];
+ text += `<span class="result detail"><span id="${timeId}"> </span><label>${label}</label></span>`
+ }
+ text += `</p></div>`;
+ }
+
const timestamp = performance.now();
document.getElementById('jetstreams').style.backgroundImage = `url('jetstreams.svg?${timestamp}')`;
const resultsTable = document.getElementById("results");
@@ -329,12 +369,13 @@
if (!isInBrowser)
return;
- for (const id of benchmark.scoreIdentifiers()) {
+ for (const id of benchmark.allScoreIdentifiers())
document.getElementById(id).innerHTML = "error";
- const benchmarkResultsUI = document.getElementById(`benchmark-${benchmark.name}`);
- benchmarkResultsUI.classList.remove("benchmark-running");
- benchmarkResultsUI.classList.add("benchmark-error");
- }
+ for (const id of benchmark.allTimeIdentifiers())
+ document.getElementById(id).innerHTML = "error";
+ const benchmarkResultsUI = document.getElementById(`benchmark-${benchmark.name}`);
+ benchmarkResultsUI.classList.remove("benchmark-running");
+ benchmarkResultsUI.classList.add("benchmark-error");
}
pushError(name, error) {
@@ -731,16 +772,36 @@
return geomeanScore(subScores);
}
+ get totalTime() {
+ const subTimes = Object.values(this.subTimes());
+ return sum(subTimes);
+ }
+
+ get wallTime() {
+ return this.endTime - this.startTime;
+ }
+
subScores() {
throw new Error("Subclasses need to implement this");
}
+ subTimes() {
+ throw new Error("Subclasses need to implement this");
+ }
+
allScores() {
const allScores = this.subScores();
allScores["Score"] = this.score;
return allScores;
}
+ allTimes() {
+ const allTimes = this.subTimes();
+ allTimes["Wall"] = this.wallTime;
+ allTimes["Total"] = this.totalTime;
+ return allTimes;
+ }
+
get prerunCode() { return null; }
get preIterationCode() {
@@ -1001,7 +1062,7 @@
this.preloads = Object.entries(this.plan.preload ?? {});
}
- scoreIdentifiers() {
+ allScoreIdentifiers() {
const ids = Object.keys(this.allScores()).map(name => this.scoreIdentifier(name));
return ids;
}
@@ -1010,6 +1071,15 @@
return `results-cell-${this.name}-${scoreName}`;
}
+ allTimeIdentifiers() {
+ const ids = Object.keys(this.allTimes()).map(name => this.timeIdentifier(name));
+ return ids;
+ }
+
+ timeIdentifier(scoreName) {
+ return `results-cell-${this.name}-${scoreName}-time`;
+ }
+
updateUIBeforeRun() {
if (!JetStreamParams.dumpJSONResults)
console.log(`Running ${this.name}:`);
@@ -1022,30 +1092,56 @@
resultsBenchmarkUI.classList.add("benchmark-running");
resultsBenchmarkUI.scrollIntoView({ block: "nearest" });
- for (const id of this.scoreIdentifiers())
+ for (const id of this.allScoreIdentifiers())
+ document.getElementById(id).innerHTML = "...";
+ for (const id of this.allTimeIdentifiers())
document.getElementById(id).innerHTML = "...";
}
updateUIAfterRun() {
- const scoreEntries = Object.entries(this.allScores());
if (isInBrowser)
- this.updateUIAfterRunInBrowser(scoreEntries);
+ this.updateUIAfterRunInBrowser();
if (JetStreamParams.dumpJSONResults)
return;
- this.updateConsoleAfterRun(scoreEntries);
+ this.updateConsoleAfterRun();
}
- updateUIAfterRunInBrowser(scoreEntries) {
+ updateUIAfterRunInBrowser() {
const benchmarkResultsUI = document.getElementById(`benchmark-${this.name}`);
benchmarkResultsUI.classList.remove("benchmark-running");
benchmarkResultsUI.classList.add("benchmark-done");
- for (const [name, value] of scoreEntries)
+ for (const [name, value] of Object.entries(this.allScores()))
document.getElementById(this.scoreIdentifier(name)).innerHTML = uiFriendlyScore(value);
+ for (const [name, value] of Object.entries(this.allTimes()))
+ document.getElementById(this.timeIdentifier(name)).innerHTML = uiFriendlyDuration(value);
this.renderScatterPlot();
}
+ updateConsoleAfterRun() {
+ for (let [name, value] of Object.entries(this.allScores())) {
+ if (!name.endsWith("Score"))
+ name = `${name}-Score`;
+
+ this.logMetric(name, shellFriendlyScore(value));
+ }
+ for (let [name, value] of Object.entries(this.allTimes())) {
+ this.logMetric(`${name}-Time`, shellFriendlyDuration(value));
+ }
+ if (JetStreamParams.RAMification) {
+ this.logMetric("Current Footprint", uiFriendlyNumber(this.currentFootprint));
+ this.logMetric("Peak Footprint", uiFriendlyNumber(this.peakFootprint));
+ }
+ console.log("");
+ }
+
+ logMetric(name, value) {
+ console.log(
+ shellFriendlyLabel(`${this.name} ${name}`),
+ value);
+ }
+
renderScatterPlot() {
const plotContainer = document.getElementById(`plot-${this.name}`);
if (!plotContainer || !this.results || this.results.length === 0)
@@ -1073,17 +1169,6 @@
}
plotContainer.innerHTML = `<svg width="${width}px" height="${height}px">${circlesSVG}</svg>`;
}
-
- updateConsoleAfterRun(scoreEntries) {
- for (let [name, value] of scoreEntries) {
- console.log(` ${name}:`, uiFriendlyScore(value));
- }
- if (JetStreamParams.RAMification) {
- console.log(" Current Footprint:", uiFriendlyNumber(this.currentFootprint));
- console.log(" Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
- }
- console.log(" Wall-Time:", uiFriendlyDuration(this.endTime - this.startTime));
- }
};
class GroupedBenchmark extends Benchmark {
@@ -1167,6 +1252,22 @@
results[subScore] = geomeanScore(results[subScore]);
return results;
}
+
+ subTimes() {
+ const results = {};
+
+ for (const benchmark of this.benchmarks) {
+ let times = benchmark.subTimes();
+ for (let subTime in times) {
+ results[subTime] ??= [];
+ results[subTime].push(times[subTime]);
+ }
+ }
+
+ for (let subTimes in results)
+ results[subTimes] = sum(results[subTimes]);
+ return results;
+ }
};
class DefaultBenchmark extends Benchmark {
@@ -1215,6 +1316,17 @@
scores["Average"] = this.averageScore;
return scores;
}
+
+ subTimes() {
+ const times = {
+ "First": this.firstIterationTime,
+ };
+ if (this.worstCaseCount)
+ times["Worst"] = this.worstTime;
+ if (this.iterations > 1)
+ times["Average"] = this.averageTime;
+ return times;
+ }
}
class AsyncBenchmark extends DefaultBenchmark {
@@ -1379,6 +1491,13 @@
}`;
}
+ subTimes() {
+ return {
+ "Stdlib": this.stdlibTime,
+ "MainRun": this.mainRunTime,
+ };
+ }
+
subScores() {
return {
"Stdlib": this.stdlibScore,
@@ -1516,6 +1635,13 @@
"Runtime": this.runScore,
};
}
+
+ subTimes() {
+ return {
+ "Startup": this.startupTime,
+ "Runtime": this.runTime,
+ };
+ }
};
function dotnetPreloads(type)
