From d1d31ea6e0a8db0eeb59df135982a14ad143938f Mon Sep 17 00:00:00 2001 From: eyelidlessness Date: Fri, 12 Jul 2024 11:41:44 -0700 Subject: [PATCH] Mitigate mysterious Vitest + Webkit + Vue environment flakiness (#165) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix: dedupe Vite dependency in yarn.lock It’s unclear why both versions made it through the last round of updates. This is *possibly* related to some of the flakiness we’re seeing, but it’s safe either way. Presumably `postcss` is also deduped as a transitive dep somewhere? Shrug emoji! * Mitigation: web-forms/vue/vitest/webkit flakiness (1 of 2) Adds a bit of delay in global setup if the test run is in webkit, and if there’s not already a `.vite/deps` cache (as a heuristic to differentiate first/subsequent test runs). * Mitigation: web-forms/vue/vitest/webkit flakiness (2 of 2) For `web-forms` only (as it seems to consistently be the package showing flakiness), and for `webkit` runs only (same reasoning), attempt test run a second time if first run fails. --- .github/workflows/ci.yml | 8 ++++++- .../globalSetup/mitigate-webkit-flakiness.ts | 7 ++++++ packages/web-forms/vitest.config.ts | 22 +++++++++++++++++++ yarn.lock | 15 ++----------- 4 files changed, 38 insertions(+), 14 deletions(-) create mode 100644 packages/web-forms/tests/globalSetup/mitigate-webkit-flakiness.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f426c609..f2a87aa0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -550,5 +550,11 @@ jobs: - if: ${{ matrix.target == 'Web' }} run: 'yarn playwright install ${{ matrix.browser }} --with-deps' - - if: ${{ matrix.target == 'Web' }} + # TODO: hopefully temporary! Attempt to mitigate flakiness in webkit by + # running twice when first run fails. At least in local testing, this has + # been near (if not at) 100% success. + - if: ${{ matrix.target == 'Web' && matrix.browser == 'webkit' }} + run: 'yarn workspace @getodk/web-forms test-browser:${{ matrix.browser }} || yarn workspace @getodk/web-forms test-browser:${{ matrix.browser }}' + + - if: ${{ matrix.target == 'Web' && matrix.browser != 'webkit' }} run: 'yarn workspace @getodk/web-forms test-browser:${{ matrix.browser }}' diff --git a/packages/web-forms/tests/globalSetup/mitigate-webkit-flakiness.ts b/packages/web-forms/tests/globalSetup/mitigate-webkit-flakiness.ts new file mode 100644 index 00000000..b2664312 --- /dev/null +++ b/packages/web-forms/tests/globalSetup/mitigate-webkit-flakiness.ts @@ -0,0 +1,7 @@ +const sleep = async (milliseconds: number) => { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +}; + +export default async function () { + await sleep(250); +} diff --git a/packages/web-forms/vitest.config.ts b/packages/web-forms/vitest.config.ts index 520b4a33..2858b374 100644 --- a/packages/web-forms/vitest.config.ts +++ b/packages/web-forms/vitest.config.ts @@ -1,5 +1,6 @@ import type { CollectionValues } from '@getodk/common/types/collections/CollectionValues.ts'; import type { VitestTestConfig } from '@getodk/common/types/vitest-config.ts'; +import { existsSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { configDefaults, defineConfig, mergeConfig } from 'vitest/config'; import viteConfig from './vite.config'; @@ -29,6 +30,22 @@ const BROWSER_ENABLED = BROWSER_NAME != null; const TEST_ENVIRONMENT = BROWSER_ENABLED ? 'node' : 'jsdom'; +const globalSetup: string[] = []; + +/** + * @todo this is (hopefully!) temporary. Adds a delay when testing in + * `webkit`, to help mitigate flakiness that seems to be rooted in + * first-run timing issues (where "first" = "no Vite cache"; the issue was + * much more consistently reproducible in a state where + * `node_modules/.vite` is not present). + */ +const webkitFlakinessMitigations = + BROWSER_NAME === 'webkit' && !existsSync('./node_modules/.vite/deps'); + +if (webkitFlakinessMitigations) { + globalSetup.push('./tests/globalSetup/mitigate-webkit-flakiness.ts'); +} + export default mergeConfig( viteConfig, defineConfig({ @@ -37,12 +54,17 @@ export default mergeConfig( enabled: BROWSER_ENABLED, name: BROWSER_NAME!, provider: 'playwright', + fileParallelism: false, headless: true, screenshotFailures: false, }, environment: TEST_ENVIRONMENT, exclude: [...configDefaults.exclude, 'e2e/**'], root: fileURLToPath(new URL('./', import.meta.url)), + + /** @see {@link webkitFlakinessMitigations} */ + globalSetup, + // Suppress the console error log about parsing CSS stylesheet // This is an open issue of jsdom // see primefaces/primevue#4512 and jsdom/jsdom#2177 diff --git a/yarn.lock b/yarn.lock index a6689b75..f8c47980 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5225,7 +5225,7 @@ postcss-selector-parser@^6.0.15: cssesc "^3.0.0" util-deprecate "^1.0.2" -postcss@^8.1.10, postcss@^8.4.38: +postcss@^8.1.10: version "8.4.38" resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.38.tgz#b387d533baf2054288e337066d81c6bee9db9e0e" integrity sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A== @@ -6363,18 +6363,7 @@ vite-plugin-solid@^2.10.2: solid-refresh "^0.6.3" vitefu "^0.2.5" -vite@^5.0.0: - version "5.3.1" - resolved "https://registry.yarnpkg.com/vite/-/vite-5.3.1.tgz#bb2ca6b5fd7483249d3e86b25026e27ba8a663e6" - integrity sha512-XBmSKRLXLxiaPYamLv3/hnP/KXDai1NDexN0FpkTaZXTfycHvkRHoenpgl/fvuK/kPbB6xAgoyiryAhQNxYmAQ== - dependencies: - esbuild "^0.21.3" - postcss "^8.4.38" - rollup "^4.13.0" - optionalDependencies: - fsevents "~2.3.3" - -vite@^5.3.3: +vite@^5.0.0, vite@^5.3.3: version "5.3.3" resolved "https://registry.yarnpkg.com/vite/-/vite-5.3.3.tgz#5265b1f0a825b3b6564c2d07524777c83e3c04c2" integrity sha512-NPQdeCU0Dv2z5fu+ULotpuq5yfCS1BzKUIPhNbP3YBfAMGJXbt2nS+sbTFu+qchaqWTD+H3JK++nRwr6XIcp6A==