fix(optimizer): use temp dirs to ensure content of deps cache dir is consistent

packages/vite/src/node/optimizer/index.ts

@@ -27,7 +27,6 @@ import { transformWithEsbuild } from '../plugins/esbuild'
 import { ESBUILD_MODULES_TARGET } from '../constants'
 import { resolvePackageData } from '../packages'
 import type { ViteDevServer } from '../server'
-import type { Logger } from '../logger'
 import { esbuildCjsExternalPlugin, esbuildDepPlugin } from './esbuildDepPlugin'
 import { scanImports } from './scan'
 export {
@@ -360,11 +359,6 @@ export async function loadCachedDepOptimizationMetadata(
 
  const depsCacheDir = getDepsCacheDir(config, ssr)
 
- // If the lock timed out, we cancel and return undefined
- if (!(await waitOptimizerWriteLock(depsCacheDir, config.logger))) {
- return
- }
-
  if (!force) {
  let cachedMetadata: DepOptimizationMetadata | undefined
  try {
@@ -595,58 +589,52 @@ export function runOptimizeDeps(
 
  // Write this run of pre-bundled dependencies to the deps cache
  async function commitFiles() {
- // Get a list of old files in the deps directory to delete the stale ones
- const oldFilesPaths: string[] = []
- // File used to tell other processes that we're writing the deps cache directory
- const writingFilePath = path.resolve(depsCacheDir, '_writing')
-
- if (
- !fs.existsSync(depsCacheDir) ||
- !(await waitOptimizerWriteLock(depsCacheDir, config.logger)) // unlock timed out
- ) {
- fs.mkdirSync(depsCacheDir, { recursive: true })
- fs.writeFileSync(writingFilePath, '')
- } else {
- fs.writeFileSync(writingFilePath, '')
- oldFilesPaths.push(
- ...(await fsp.readdir(depsCacheDir)).map((f) =>
- path.join(depsCacheDir, f),
- ),
- )
- }
-
- const newFilesPaths = new Set<string>()
- newFilesPaths.add(writingFilePath)
- const files: Promise<void>[] = []
- const write = (filePath: string, content: string | Uint8Array) => {
- newFilesPaths.add(filePath)
- files.push(fsp.writeFile(filePath, content))
- }
-
- path.join(depsCacheDir, '_metadata.json'),
- // a hint for Node.js
- // all files in the cache directory should be recognized as ES modules
- write(
- path.resolve(depsCacheDir, 'package.json'),
+ // Write this run of pre-bundled dependencies to the deps cache
+
+ // create temp sibling dir
+ const tempDir = getTempSiblingName(depsCacheDir, 'temp')
+ fs.mkdirSync(tempDir, { recursive: true })
+
+ // write files with callback api, use write count to resolve
+ await new Promise<void>((resolve, reject) => {
+ let numWrites = result.outputFiles!.length + 2 // two metadata files
+ const callback = (err: any) => {
+ if (err) {
+ reject(err)
+ } else {
+ if (--numWrites === 0) {
+ resolve()
+ }
+ }
+ }
+ fs.writeFile(
+ `${tempDir}/package.json`,
  '{\n "type": "module"\n}\n',
+ callback,
  )
+ fs.writeFile(
+ `${tempDir}/_metadata.json`,
+ stringifyDepsOptimizerMetadata(metadata, depsCacheDir),
+ callback,
+ )
+ result.outputFiles!.forEach(({ path, contents }) => {
+ fs.writeFile(
+ path.replace(depsCacheDir, tempDir),
+ contents,
+ callback,
+ )
+ })
+ })
 
- write(
- path.join(depsCacheDir, '_metadata.json'),
- stringifyDepsOptimizerMetadata(metadata, depsCacheDir),
- )
-
- for (const outputFile of result.outputFiles!)
- write(outputFile.path, outputFile.contents)
-
- // Clean up old files in the background
- for (const filePath of oldFilesPaths)
- if (!newFilesPaths.has(filePath)) fs.unlink(filePath, () => {}) // ignore errors
-
- await Promise.all(files)
-
- // Successful write
- fsp.unlink(writingFilePath)
+ // replace depsCacheDir with newly written dir
+ if (fs.existsSync(depsCacheDir)) {
+ const deleteMe = getTempSiblingName(depsCacheDir, 'delete')
+ fs.renameSync(depsCacheDir, deleteMe)
+ fs.renameSync(tempDir, depsCacheDir)
+ fs.rmdir(deleteMe, () => {}) // ignore errors
+ } else {
+ fs.renameSync(tempDir, depsCacheDir)
+ }
 
  setTimeout(() => {
  // Free up memory, these files aren't going to be re-requested because
@@ -1346,47 +1334,7 @@ export async function loadOptimizedDep(
  return fsp.readFile(file, 'utf-8')
 }
 
-/**
- * Processes that write to the deps cache directory adds a `_writing` lock to
- * inform other processes of so. So before doing any work on it, they can wait
- * for the file to be removed to know it's ready.
- *
- * Returns true if successfully waited for unlock, false if lock timed out.
- */
-async function waitOptimizerWriteLock(depsCacheDir: string, logger: Logger) {
- const writingPath = path.join(depsCacheDir, '_writing')
- const tryAgainMs = 100
-
- // if _writing exist, we wait for a maximum of 500ms before assuming something
- // is not right
- let maxWaitTime = 500
- let waited = 0
- let filesLength: number
-
- while (fs.existsSync(writingPath)) {
- // on the first run, we check the number of files it started with for later use
- filesLength ??= (await fsp.readdir(depsCacheDir)).length
-
- await new Promise((r) => setTimeout(r, tryAgainMs))
- waited += tryAgainMs
-
- if (waited >= maxWaitTime) {
- const newFilesLength = (await fsp.readdir(depsCacheDir)).length
-
- // after 500ms, if the number of files is the same, assume previous process
- // terminated and didn't cleanup `_writing` lock. clear the directory.
- if (filesLength === newFilesLength) {
- logger.info('Outdated deps cache, forcing re-optimization...')
- await fsp.rm(depsCacheDir, { recursive: true, force: true })
- return false
- }
- // new files were saved, wait a bit longer to decide again.
- else {
- maxWaitTime += 500
- filesLength = newFilesLength
- }
- }
- }
-
- return true
+function getTempSiblingName(file: string, suffix: string): string {
+ // timestamp + pid is unique, we never create the same suffix in the same process twice in the same ms
+ return `${file}_${suffix}_${Date.now()}_${process.pid}`
 }