Mitigate mysterious Vitest + Webkit + Vue environment flakiness (#165)

* Fix: dedupe Vite dependency in yarn.lock

It’s unclear why both versions made it through the last round of updates. This is *possibly* related to some of the flakiness we’re seeing, but it’s safe either way.

Presumably `postcss` is also deduped as a transitive dep somewhere? Shrug emoji!

* Mitigation: web-forms/vue/vitest/webkit flakiness (1 of 2)

Adds a bit of delay in global setup if the test run is in webkit, and if there’s not already a `.vite/deps` cache (as a heuristic to differentiate first/subsequent test runs).

* Mitigation: web-forms/vue/vitest/webkit flakiness (2 of 2)

For `web-forms` only (as it seems to consistently be the package showing flakiness), and for `webkit` runs only (same reasoning), attempt test run a second time if first run fails.

.github/workflows/ci.yml

@@ -550,5 +550,11 @@ jobs:
  - if: ${{ matrix.target == 'Web' }}
  run: 'yarn playwright install ${{ matrix.browser }} --with-deps'
 
- - if: ${{ matrix.target == 'Web' }}
+ # TODO: hopefully temporary! Attempt to mitigate flakiness in webkit by
+ # running twice when first run fails. At least in local testing, this has
+ # been near (if not at) 100% success.
+ - if: ${{ matrix.target == 'Web' && matrix.browser == 'webkit' }}
+ run: 'yarn workspace @getodk/web-forms test-browser:${{ matrix.browser }} || yarn workspace @getodk/web-forms test-browser:${{ matrix.browser }}'
+
+ - if: ${{ matrix.target == 'Web' && matrix.browser != 'webkit' }}
  run: 'yarn workspace @getodk/web-forms test-browser:${{ matrix.browser }}'

packages/web-forms/tests/globalSetup/mitigate-webkit-flakiness.ts

@@ -0,0 +1,7 @@
+const sleep = async (milliseconds: number) => {
+ return new Promise((resolve) => setTimeout(resolve, milliseconds));
+};
+
+export default async function () {
+ await sleep(250);
+}

packages/web-forms/vitest.config.ts

@@ -1,5 +1,6 @@
 import type { CollectionValues } from '@getodk/common/types/collections/CollectionValues.ts';
 import type { VitestTestConfig } from '@getodk/common/types/vitest-config.ts';
+import { existsSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import { configDefaults, defineConfig, mergeConfig } from 'vitest/config';
 import viteConfig from './vite.config';
@@ -29,6 +30,22 @@ const BROWSER_ENABLED = BROWSER_NAME != null;
 
 const TEST_ENVIRONMENT = BROWSER_ENABLED ? 'node' : 'jsdom';
 
+const globalSetup: string[] = [];
+
+/**
+ * @todo this is (hopefully!) temporary. Adds a delay when testing in
+ * `webkit`, to help mitigate flakiness that seems to be rooted in
+ * first-run timing issues (where "first" = "no Vite cache"; the issue was
+ * much more consistently reproducible in a state where
+ * `node_modules/.vite` is not present).
+ */
+const webkitFlakinessMitigations =
+ BROWSER_NAME === 'webkit' && !existsSync('./node_modules/.vite/deps');
+
+if (webkitFlakinessMitigations) {
+ globalSetup.push('./tests/globalSetup/mitigate-webkit-flakiness.ts');
+}
+
 export default mergeConfig(
  viteConfig,
  defineConfig({
@@ -37,12 +54,17 @@ export default mergeConfig(
  enabled: BROWSER_ENABLED,
  name: BROWSER_NAME!,
  provider: 'playwright',
+ fileParallelism: false,
  headless: true,
  screenshotFailures: false,
  },
  environment: TEST_ENVIRONMENT,
  exclude: [...configDefaults.exclude, 'e2e/**'],
  root: fileURLToPath(new URL('./', import.meta.url)),
+
+ /** @see {@link webkitFlakinessMitigations} */
+ globalSetup,
+
  // Suppress the console error log about parsing CSS stylesheet
  // This is an open issue of jsdom
  // see primefaces/primevue#4512 and jsdom/jsdom#2177

yarn.lock

@@ -5225,7 +5225,7 @@ postcss-selector-parser@^6.0.15:
  cssesc "^3.0.0"
  util-deprecate "^1.0.2"
 
-postcss@^8.1.10, postcss@^8.4.38:
+postcss@^8.1.10:
  version "8.4.38"
  resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.38.tgz#b387d533baf2054288e337066d81c6bee9db9e0e"
  integrity sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==
@@ -6363,18 +6363,7 @@ vite-plugin-solid@^2.10.2:
  solid-refresh "^0.6.3"
  vitefu "^0.2.5"
 
-vite@^5.0.0:
- version "5.3.1"
- resolved "https://registry.yarnpkg.com/vite/-/vite-5.3.1.tgz#bb2ca6b5fd7483249d3e86b25026e27ba8a663e6"
- integrity sha512-XBmSKRLXLxiaPYamLv3/hnP/KXDai1NDexN0FpkTaZXTfycHvkRHoenpgl/fvuK/kPbB6xAgoyiryAhQNxYmAQ==
- dependencies:
- esbuild "^0.21.3"
- postcss "^8.4.38"
- rollup "^4.13.0"
- optionalDependencies:
- fsevents "~2.3.3"
-
-vite@^5.3.3:
+vite@^5.0.0, vite@^5.3.3:
  version "5.3.3"
  resolved "https://registry.yarnpkg.com/vite/-/vite-5.3.3.tgz#5265b1f0a825b3b6564c2d07524777c83e3c04c2"
  integrity sha512-NPQdeCU0Dv2z5fu+ULotpuq5yfCS1BzKUIPhNbP3YBfAMGJXbt2nS+sbTFu+qchaqWTD+H3JK++nRwr6XIcp6A==