Split off ExperimentalSuites (#519)
- Add experimental/tests.mjs which exports ExperimentalSuites
- Add "TodoMVC-LocalStorage" and "NewsSite-PostMessage" to ExperimentalSuites
- ExperimentalSuites is merged into the default Suites
- Add unit tests for Suites and ExperimentalSuites to ensure all tags are set correctly
diff --git a/experimental/tests.mjs b/experimental/tests.mjs
new file mode 100644
index 0000000..ca4fa57
--- /dev/null
+++ b/experimental/tests.mjs
@@ -0,0 +1,145 @@
+import { BenchmarkTestStep } from "../resources/benchmark-runner.mjs";
+import { getTodoText } from "../resources/shared/translations.mjs";
+import { numberOfItemsToAdd } from "../resources/shared/todomvc-utils.mjs";
+import { freezeSuites } from "../resources/suites-helper.mjs";
+
+export const ExperimentalSuites = freezeSuites([
+ {
+ name: "TodoMVC-LocalStorage",
+ url: "experimental/todomvc-localstorage/dist/index.html",
+ tags: ["todomvc", "experimental"],
+ async prepare(page) {
+ (await page.waitForElement(".new-todo")).focus();
+ page.getLocalStorage().getItem("javascript-es5");
+ },
+ tests: [
+ new BenchmarkTestStep(`Adding${numberOfItemsToAdd}Items`, (page) => {
+ const newTodo = page.querySelector(".new-todo");
+ for (let i = 0; i < numberOfItemsToAdd; i++) {
+ newTodo.setValue(getTodoText("ja", i));
+ newTodo.dispatchEvent("change");
+ newTodo.enter("keypress");
+ }
+ }),
+ new BenchmarkTestStep("CompletingAllItems", (page) => {
+ const checkboxes = page.querySelectorAll(".toggle");
+ for (let i = 0; i < numberOfItemsToAdd; i++)
+ checkboxes[i].click();
+ }),
+ new BenchmarkTestStep("DeletingAllItems", (page) => {
+ const deleteButtons = page.querySelectorAll(".destroy");
+ for (let i = numberOfItemsToAdd - 1; i >= 0; i--)
+ deleteButtons[i].click();
+ }),
+ ],
+ },
+ {
+ name: "TodoMVC-Emoji",
+ url: "resources/todomvc/vanilla-examples/javascript-web-components/dist/index.html",
+ tags: ["todomvc", "experimental"],
+ async prepare(page) {
+ await page.waitForElement("todo-app");
+ },
+ tests: [
+ new BenchmarkTestStep(`Adding${numberOfItemsToAdd}Items`, (page) => {
+ const input = page.querySelector(".new-todo-input", ["todo-app", "todo-topbar"]);
+ for (let i = 0; i < numberOfItemsToAdd; i++) {
+ input.setValue(getTodoText("emoji", i));
+ input.dispatchEvent("input");
+ input.enter("keyup");
+ }
+ }),
+ new BenchmarkTestStep("CompletingAllItems", (page) => {
+ const items = page.querySelectorAll("todo-item", ["todo-app", "todo-list"]);
+ for (let i = 0; i < numberOfItemsToAdd; i++) {
+ const item = items[i].querySelectorInShadowRoot(".toggle-todo-input");
+ item.click();
+ }
+ }),
+ new BenchmarkTestStep("DeletingAllItems", (page) => {
+ const items = page.querySelectorAll("todo-item", ["todo-app", "todo-list"]);
+ for (let i = numberOfItemsToAdd - 1; i >= 0; i--) {
+ const item = items[i].querySelectorInShadowRoot(".remove-todo-button");
+ item.click();
+ }
+ }),
+ ],
+ },
+ {
+ name: "TodoMVC-WebComponents-PostMessage",
+ url: "resources/todomvc/vanilla-examples/javascript-web-components/dist/index.html",
+ tags: ["experimental", "todomvc", "webcomponents"],
+ async prepare() {},
+ type: "remote",
+ /* config: {
+ name: "default", // optional param to target non-default tests locally
+ }, */
+ },
+ {
+ name: "TodoMVC-Jaspr-Dart2JS-O4",
+ url: "experimental/todomvc-dart-jaspr/dist/out-dart2js-O4/index.html",
+ tags: ["todomvc", "experimental"],
+ async prepare(page) {
+ (await page.waitForElement(".new-todo")).focus();
+ },
+ tests: [
+ new BenchmarkTestStep(`Adding${numberOfItemsToAdd}Items`, (page) => {
+ const newTodo = page.querySelector(".new-todo");
+ for (let i = 0; i < numberOfItemsToAdd; i++) {
+ newTodo.setValue(getTodoText("ja", i));
+ newTodo.dispatchEvent("change");
+ newTodo.enter("keypress");
+ }
+ }),
+ new BenchmarkTestStep("CompletingAllItems", (page) => {
+ const checkboxes = page.querySelectorAll(".toggle");
+ for (let i = 0; i < numberOfItemsToAdd; i++)
+ checkboxes[i].click();
+ }),
+ new BenchmarkTestStep("DeletingAllItems", (page) => {
+ const deleteButtons = page.querySelectorAll(".destroy");
+ for (let i = numberOfItemsToAdd - 1; i >= 0; i--)
+ deleteButtons[i].click();
+ }),
+ ],
+ },
+ {
+ name: "TodoMVC-Jaspr-Dart2Wasm-O2",
+ url: "experimental/todomvc-dart-jaspr/dist/out-dart2wasm-O2/index.html",
+ tags: ["todomvc", "experimental"],
+ disabled: true,
+ async prepare(page) {
+ (await page.waitForElement(".new-todo")).focus();
+ },
+ tests: [
+ new BenchmarkTestStep(`Adding${numberOfItemsToAdd}Items`, (page) => {
+ const newTodo = page.querySelector(".new-todo");
+ for (let i = 0; i < numberOfItemsToAdd; i++) {
+ newTodo.setValue(getTodoText("ja", i));
+ newTodo.dispatchEvent("change");
+ newTodo.enter("keypress");
+ }
+ }),
+ new BenchmarkTestStep("CompletingAllItems", (page) => {
+ const checkboxes = page.querySelectorAll(".toggle");
+ for (let i = 0; i < numberOfItemsToAdd; i++)
+ checkboxes[i].click();
+ }),
+ new BenchmarkTestStep("DeletingAllItems", (page) => {
+ const deleteButtons = page.querySelectorAll(".destroy");
+ for (let i = numberOfItemsToAdd - 1; i >= 0; i--)
+ deleteButtons[i].click();
+ }),
+ ],
+ },
+ {
+ name: "NewsSite-PostMessage",
+ url: "resources/newssite/news-next/dist/index.html",
+ tags: ["experimental", "newssite", "language"],
+ async prepare() {},
+ type: "remote",
+ /* config: {
+ name: "default", // optional param to target non-default tests locally
+ }, */
+ },
+]);
diff --git a/resources/benchmark-configurator.mjs b/resources/benchmark-configurator.mjs
index 2546505..ee250a7 100644
--- a/resources/benchmark-configurator.mjs
+++ b/resources/benchmark-configurator.mjs
@@ -1,7 +1,8 @@
// example url for local testing:
// http://localhost:8080/?developerMode=true&config=http://localhost:8080/resources/config.json
// since the json doesn't contain a default suite, dismiss warning popups and select from the developerMenu
-import { defaultSuites } from "./default-tests.mjs";
+import { DefaultSuites } from "./default-tests.mjs";
+import { ExperimentalSuites } from "../experimental/tests.mjs";
import { params } from "./shared/params.mjs";
const DEFAULT_TAGS = ["all", "default", "experimental"];
@@ -75,7 +76,7 @@
const benchmarkUrl = new URL(window.location);
if (DISALLOWED_DOMAINS.some((domain) => benchmarkUrl.hostname.endsWith(domain))) {
console.warn("Configuration fetch not allowed. Loading default suites.");
- this._loadDefaultSuites();
+ this._loadSuites();
return;
}
@@ -98,19 +99,24 @@
});
} catch (error) {
console.warn(`Error loading custom configuration: ${error.message}. Loading default suites.`);
- this._loadDefaultSuites();
+ this._loadSuites();
}
} else {
- this._loadDefaultSuites();
+ this._loadSuites();
}
this._freezeTags();
this._freezeSuites();
}
- _loadDefaultSuites() {
- defaultSuites.flatMap((suite) => suite.tags).forEach((tag) => this.#tags.add(tag));
- defaultSuites.forEach((suite) => this.#suites.push(suite));
+ _loadSuites() {
+ DefaultSuites.forEach((suite) => this._loadSuite(suite));
+ ExperimentalSuites.forEach((suite) => this._loadSuite(suite));
+ }
+
+ _loadSuite(suite) {
+ suite.tags.forEach((tag) => this.#tags.add(tag));
+ this.#suites.push(suite);
}
enableSuites(names, tags) {
diff --git a/resources/default-tests.mjs b/resources/default-tests.mjs
index 2b677d2..ecfbc33 100644
--- a/resources/default-tests.mjs
+++ b/resources/default-tests.mjs
@@ -1,69 +1,9 @@
import { BenchmarkTestStep } from "./benchmark-runner.mjs";
import { getTodoText, defaultLanguage } from "./shared/translations.mjs";
import { numberOfItemsToAdd } from "./shared/todomvc-utils.mjs";
+import { freezeSuites } from "../resources/suites-helper.mjs";
-export const defaultSuites = [
- {
- name: "TodoMVC-LocalStorage",
- url: "experimental/todomvc-localstorage/dist/index.html",
- tags: ["todomvc"],
- async prepare(page) {
- (await page.waitForElement(".new-todo")).focus();
- page.getLocalStorage().getItem("javascript-es5");
- },
- tests: [
- new BenchmarkTestStep(`Adding${numberOfItemsToAdd}Items`, (page) => {
- const newTodo = page.querySelector(".new-todo");
- for (let i = 0; i < numberOfItemsToAdd; i++) {
- newTodo.setValue(getTodoText("ja", i));
- newTodo.dispatchEvent("change");
- newTodo.enter("keypress");
- }
- }),
- new BenchmarkTestStep("CompletingAllItems", (page) => {
- const checkboxes = page.querySelectorAll(".toggle");
- for (let i = 0; i < numberOfItemsToAdd; i++)
- checkboxes[i].click();
- }),
- new BenchmarkTestStep("DeletingAllItems", (page) => {
- const deleteButtons = page.querySelectorAll(".destroy");
- for (let i = numberOfItemsToAdd - 1; i >= 0; i--)
- deleteButtons[i].click();
- }),
- ],
- },
- {
- name: "TodoMVC-Emoji",
- url: "resources/todomvc/vanilla-examples/javascript-web-components/dist/index.html",
- tags: ["todomvc", "experimental"],
- async prepare(page) {
- await page.waitForElement("todo-app");
- },
- tests: [
- new BenchmarkTestStep(`Adding${numberOfItemsToAdd}Items`, (page) => {
- const input = page.querySelector(".new-todo-input", ["todo-app", "todo-topbar"]);
- for (let i = 0; i < numberOfItemsToAdd; i++) {
- input.setValue(getTodoText("emoji", i));
- input.dispatchEvent("input");
- input.enter("keyup");
- }
- }),
- new BenchmarkTestStep("CompletingAllItems", (page) => {
- const items = page.querySelectorAll("todo-item", ["todo-app", "todo-list"]);
- for (let i = 0; i < numberOfItemsToAdd; i++) {
- const item = items[i].querySelectorInShadowRoot(".toggle-todo-input");
- item.click();
- }
- }),
- new BenchmarkTestStep("DeletingAllItems", (page) => {
- const items = page.querySelectorAll("todo-item", ["todo-app", "todo-list"]);
- for (let i = numberOfItemsToAdd - 1; i >= 0; i--) {
- const item = items[i].querySelectorInShadowRoot(".remove-todo-button");
- item.click();
- }
- }),
- ],
- },
+export const DefaultSuites = freezeSuites([
{
name: "TodoMVC-JavaScript-ES5",
url: "resources/todomvc/vanilla-examples/javascript-es5/dist/index.html",
@@ -211,16 +151,6 @@
],
},
{
- name: "TodoMVC-WebComponents-PostMessage",
- url: "resources/todomvc/vanilla-examples/javascript-web-components/dist/index.html",
- tags: ["experimental", "todomvc", "webcomponents"],
- async prepare() {},
- type: "remote",
- /* config: {
- name: "default", // optional param to target non-default tests locally
- }, */
- },
- {
name: "TodoMVC-WebComponents-Complex-DOM",
url: "resources/todomvc/vanilla-examples/javascript-web-components-complex/dist/index.html",
tags: ["todomvc", "webcomponents", "complex"],
@@ -771,63 +701,6 @@
],
},
{
- name: "TodoMVC-Jaspr-Dart2JS-O4",
- url: "experimental/todomvc-dart-jaspr/dist/out-dart2js-O4/index.html",
- tags: ["todomvc", "experimental"],
- async prepare(page) {
- (await page.waitForElement(".new-todo")).focus();
- },
- tests: [
- new BenchmarkTestStep(`Adding${numberOfItemsToAdd}Items`, (page) => {
- const newTodo = page.querySelector(".new-todo");
- for (let i = 0; i < numberOfItemsToAdd; i++) {
- newTodo.setValue(getTodoText("ja", i));
- newTodo.dispatchEvent("change");
- newTodo.enter("keypress");
- }
- }),
- new BenchmarkTestStep("CompletingAllItems", (page) => {
- const checkboxes = page.querySelectorAll(".toggle");
- for (let i = 0; i < numberOfItemsToAdd; i++)
- checkboxes[i].click();
- }),
- new BenchmarkTestStep("DeletingAllItems", (page) => {
- const deleteButtons = page.querySelectorAll(".destroy");
- for (let i = numberOfItemsToAdd - 1; i >= 0; i--)
- deleteButtons[i].click();
- }),
- ],
- },
- {
- name: "TodoMVC-Jaspr-Dart2Wasm-O2",
- url: "experimental/todomvc-dart-jaspr/dist/out-dart2wasm-O2/index.html",
- tags: ["todomvc", "experimental"],
- disabled: true,
- async prepare(page) {
- (await page.waitForElement(".new-todo")).focus();
- },
- tests: [
- new BenchmarkTestStep(`Adding${numberOfItemsToAdd}Items`, (page) => {
- const newTodo = page.querySelector(".new-todo");
- for (let i = 0; i < numberOfItemsToAdd; i++) {
- newTodo.setValue(getTodoText("ja", i));
- newTodo.dispatchEvent("change");
- newTodo.enter("keypress");
- }
- }),
- new BenchmarkTestStep("CompletingAllItems", (page) => {
- const checkboxes = page.querySelectorAll(".toggle");
- for (let i = 0; i < numberOfItemsToAdd; i++)
- checkboxes[i].click();
- }),
- new BenchmarkTestStep("DeletingAllItems", (page) => {
- const deleteButtons = page.querySelectorAll(".destroy");
- for (let i = numberOfItemsToAdd - 1; i >= 0; i--)
- deleteButtons[i].click();
- }),
- ],
- },
- {
name: "NewsSite-Next",
url: "resources/newssite/news-next/dist/index.html",
tags: ["default", "newssite", "language"],
@@ -868,16 +741,6 @@
],
},
{
- name: "NewsSite-PostMessage",
- url: "resources/newssite/news-next/dist/index.html",
- tags: ["experimental", "newssite", "language"],
- async prepare() {},
- type: "remote",
- /* config: {
- name: "default", // optional param to target non-default tests locally
- }, */
- },
- {
name: "NewsSite-Nuxt",
url: "resources/newssite/news-nuxt/dist/index.html",
tags: ["default", "newssite"],
@@ -1077,4 +940,4 @@
}),
],
},
-];
+]);
diff --git a/resources/suites-helper.mjs b/resources/suites-helper.mjs
new file mode 100644
index 0000000..a50e53c
--- /dev/null
+++ b/resources/suites-helper.mjs
@@ -0,0 +1,11 @@
+export function freezeSuites(suites) {
+ suites = suites.map((suite) => {
+ // FIXME: freeze tags
+ // suite.tags = Object.freeze(suite.tags);
+ suite.steps = Object.freeze(suite.steps);
+ // FIXME: freeze suite after fixing the benchmark-configurator.
+ // return Object.freeze(suite);
+ return suite;
+ });
+ return suites;
+}
diff --git a/tests/index.html b/tests/index.html
index d11801e..a6bc60a 100644
--- a/tests/index.html
+++ b/tests/index.html
@@ -29,6 +29,7 @@
await import("./unittests/benchmark-runner.mjs");
await import("./unittests/params.mjs");
+ await import("./unittests/suites.mjs");
globalThis.testResults = undefined;
globalThis.testRunner = mocha.run();
diff --git a/tests/unittests/suites.mjs b/tests/unittests/suites.mjs
new file mode 100644
index 0000000..243b3a9
--- /dev/null
+++ b/tests/unittests/suites.mjs
@@ -0,0 +1,94 @@
+import { ExperimentalSuites } from "../../experimental/tests.mjs";
+import { DefaultSuites } from "../../resources/default-tests.mjs";
+
+const Suites = {
+ ExperimentalSuites,
+ DefaultSuites,
+};
+
+for (const [name, suites] of Object.entries(Suites)) {
+ describe(`${name}-common`, () => {
+ it("should be frozen", () => {
+ // FIXME: freeze suite
+ // expect(Object.isFrozen(suites)).to.be(true);
+ });
+ it("should have tags array", () => {
+ suites.forEach((suite) => {
+ expect(suite.tags).to.be.an("array");
+ });
+ });
+ it("should have frozen tags array", () => {
+ // FIXME: freeze suite and tags
+ // suites.forEach((suite) => {
+ // expect(Object.isFrozen(suite.tags)).to.be(true);
+ // });
+ });
+ it("should have frozen steps array", () => {
+ suites.forEach((suite) => {
+ expect(Object.isFrozen(suite.steps)).to.be(true);
+ });
+ });
+ it("should not have duplicate tags", () => {
+ suites.forEach((suite) => {
+ const uniqueTags = new Set(suite.tags);
+ expect(suite.tags).to.eql(Array.from(uniqueTags));
+ });
+ });
+ it("should have 'all' tag", () => {
+ // FIXME: freeze suite and tags
+ // suites.forEach((suite) => {
+ // expect(suite.tags.includes("all")).to.be(true);
+ // });
+ });
+ it("should not have enabled property", () => {
+ suites.forEach((suite) => {
+ expect(suite.enabled).to.be(undefined);
+ });
+ });
+ it("should have a name string", () => {
+ suites.forEach((suite) => {
+ expect(suite.name).to.be.a("string");
+ expect(suite.name.length).to.be.greaterThan(0);
+ });
+ });
+ it("should have unique names", () => {
+ const uniqueNames = new Set();
+ suites.forEach((suite) => {
+ expect(uniqueNames.has(suite.name)).to.be(false);
+ uniqueNames.add(suite.name);
+ });
+ });
+ it("should have a url string", () => {
+ suites.forEach((suite) => {
+ expect(suite.url).to.be.a("string");
+ expect(suite.url.length).to.be.greaterThan(0);
+ });
+ });
+ });
+}
+
+describe("ExperimentalSuites", () => {
+ it("should have 'experimental' tag", () => {
+ ExperimentalSuites.forEach((suite) => {
+ expect(suite.tags.includes("experimental")).to.be(true);
+ });
+ });
+ it("should not have 'default' tag", () => {
+ ExperimentalSuites.forEach((suite) => {
+ expect(suite.tags.includes("default")).to.be(false);
+ });
+ });
+});
+
+describe("Suites", () => {
+ it("should not have 'experimental' tag", () => {
+ DefaultSuites.forEach((suite) => {
+ expect(suite.tags.includes("experimental")).to.be(false);
+ });
+ });
+ it("should not have enabled property", () => {
+ ExperimentalSuites.forEach((suite) => {
+ expect(suite.enabled).to.be(undefined);
+ });
+ });
+});
