[devtools] Port cache invalidation logic from turbopack to webpack

Author: bgw

packages/next/errors.json

@@ -693,5 +693,6 @@
   "692": "Expected clientReferenceManifest to be defined.",
   "693": "%s must not be used within a client component. Next.js should be preventing %s from being included in client components statically, but did not in this case.",
   "694": "createPrerenderPathname was called inside a client component scope.",
-  "695": "Expected workUnitAsyncStorage to have a store."
+  "695": "Expected workUnitAsyncStorage to have a store.",
+  "696": "Unable to remove an invalidated webpack cache. If this issue persists you can work around it by deleting %s"
 }

packages/next/src/build/webpack-config.ts

@@ -289,6 +289,21 @@ export function hasExternalOtelApiPackage(): boolean {
 
 const UNSAFE_CACHE_REGEX = /[\\/]pages[\\/][^\\/]+(?:$|\?|#)/
 
+export function getCacheDirectories(
+  configs: webpack.Configuration[]
+): Set<string> {
+  return new Set(
+    configs
+      .map((cfg) => {
+        if (typeof cfg.cache === 'object' && cfg.cache.type === 'filesystem') {
+          return cfg.cache.cacheDirectory
+        }
+        return null
+      })
+      .filter((dir) => dir != null)
+  )
+}
+
 export default async function getBaseWebpackConfig(
   dir: string,
   {

packages/next/src/build/webpack/cache-invalidation.ts

@@ -0,0 +1,91 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+
+const INVALIDATION_MARKER = '__nextjs_invalidated_cache'
+
+/**
+ * Atomically write an invalidation marker.
+ *
+ * Because attempting to delete currently open cache files could cause issues,
+ * actual deletion of files is deferred until the next start-up (in
+ * `checkPersistentCacheInvalidationAndCleanup`).
+ *
+ * In the case that no database is currently open (e.g. via a separate CLI
+ * subcommand), you should call `cleanupPersistentCache` *after* this to eagerly
+ * remove the cache files.
+ */
+export async function invalidatePersistentCache(cacheDirectory: string) {
+  let file
+  try {
+    // We're just opening it so that `open()` creates the file.
+    file = await fs.open(path.join(cacheDirectory, INVALIDATION_MARKER), 'w')
+    // We don't currently write anything to the file, but we could choose to
+    // later, e.g. a reason for the invalidation.
+  } finally {
+    file?.close()
+  }
+}
+
+/**
+ * Called during startup. See if the cache is in a partially-completed
+ * invalidation state. Finds and delete any invalidated cache files.
+ */
+export async function checkPersistentCacheInvalidationAndCleanup(
+  cacheDirectory: string
+) {
+  const invalidated = await fs
+    .access(path.join(cacheDirectory, INVALIDATION_MARKER))
+    .then(
+      () => true,
+      () => false
+    )
+  if (invalidated) {
+    await cleanupPersistentCache(cacheDirectory)
+  }
+}
+
+/**
+ * Helper for `checkPersistentCacheInvalidationAndCleanup`. You can call this to
+ * explicitly clean up a database after running `invalidatePersistentCache` when
+ * webpack is not running.
+ *
+ * You should not run this if the cache has not yet been invalidated, as this
+ * operation is not atomic and could result in a partially-deleted and corrupted
+ * database.
+ */
+async function cleanupPersistentCache(cacheDirectory: string) {
+  try {
+    await cleanupPersistentCacheInner(cacheDirectory)
+  } catch (e) {
+    // generate a user-friendly error message
+    throw new Error(
+      `Unable to remove an invalidated webpack cache. If this issue persists ` +
+        `you can work around it by deleting ${cacheDirectory}`,
+      { cause: e }
+    )
+  }
+}
+
+async function cleanupPersistentCacheInner(cacheDirectory: string) {
+  const files = await fs.readdir(cacheDirectory)
+
+  // delete everything except the invalidation marker
+  await Promise.all(
+    files.map((name) =>
+      name !== INVALIDATION_MARKER
+        ? fs.rm(path.join(cacheDirectory, name), {
+            force: true, // ignore errors if path does not exist
+            recursive: true,
+            maxRetries: 2, // windows prevents deletion of open files
+          })
+        : null
+    )
+  )
+
+  // delete the invalidation marker last, once we're sure everything is cleaned
+  // up
+  await fs.rm(path.join(cacheDirectory, INVALIDATION_MARKER), {
+    force: true,
+    maxRetries: 2,
+  })
+}

packages/next/src/client/components/react-dev-overlay/server/restart-dev-server-middleware.ts

@@ -3,17 +3,20 @@ import type { Telemetry } from '../../../../telemetry/storage'
 import { RESTART_EXIT_CODE } from '../../../../server/lib/utils'
 import { middlewareResponse } from './middleware-response'
 import type { Project } from '../../../../build/swc/types'
+import { invalidatePersistentCache } from '../../../../build/webpack/cache-invalidation'
 
 const EVENT_DEV_OVERLAY_RESTART_SERVER = 'DEV_OVERLAY_RESTART_SERVER'
 
 interface RestartDevServerMiddlewareConfig {
   telemetry: Telemetry
   turbopackProject?: Project
+  webpackCacheDirectories?: Set<string>
 }
 
 export function getRestartDevServerMiddleware({
   telemetry,
   turbopackProject,
+  webpackCacheDirectories,
 }: RestartDevServerMiddlewareConfig) {
   /**
    * Some random value between 1 and Number.MAX_SAFE_INTEGER (inclusive). The same value is returned
@@ -34,11 +37,18 @@ export function getRestartDevServerMiddleware({
       return middlewareResponse.methodNotAllowed(res)
     }
 
-    const invalidatePersistentCache = searchParams.has(
+    const shouldInvalidatePersistentCache = searchParams.has(
       'invalidatePersistentCache'
     )
-    if (invalidatePersistentCache) {
-      await turbopackProject?.invalidatePersistentCache()
+    if (shouldInvalidatePersistentCache) {
+      if (webpackCacheDirectories != null) {
+        await Promise.all(
+          Array.from(webpackCacheDirectories).map(invalidatePersistentCache)
+        )
+      }
+      if (turbopackProject != null) {
+        await turbopackProject.invalidatePersistentCache()
+      }
     }
 
     telemetry.record({

packages/next/src/server/dev/hot-reloader-webpack.ts

@@ -27,6 +27,7 @@ import {
 import { watchCompilers } from '../../build/output'
 import * as Log from '../../build/output/log'
 import getBaseWebpackConfig, {
+  getCacheDirectories,
   loadProjectInfo,
 } from '../../build/webpack-config'
 import { APP_DIR_ALIAS, WEBPACK_LAYERS } from '../../lib/constants'
@@ -88,6 +89,7 @@ import { getDevOverlayFontMiddleware } from '../../client/components/react-dev-o
 import { getDisableDevIndicatorMiddleware } from './dev-indicator-middleware'
 import getWebpackBundler from '../../shared/lib/get-webpack-bundler'
 import { getRestartDevServerMiddleware } from '../../client/components/react-dev-overlay/server/restart-dev-server-middleware'
+import { checkPersistentCacheInvalidationAndCleanup } from '../../build/webpack/cache-invalidation'
 
 const MILLISECONDS_IN_NANOSECOND = BigInt(1_000_000)
 
@@ -1155,6 +1157,11 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
     // @ts-ignore webpack 5
     this.activeWebpackConfigs.parallelism = 1
 
+    await Promise.all(
+      Array.from(getCacheDirectories(this.activeWebpackConfigs)).map(
+        checkPersistentCacheInvalidationAndCleanup
+      )
+    )
     this.multiCompiler = getWebpackBundler()(
       this.activeWebpackConfigs
     ) as unknown as webpack.MultiCompiler
@@ -1572,6 +1579,10 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
       getDisableDevIndicatorMiddleware(),
       getRestartDevServerMiddleware({
         telemetry: this.telemetry,
+        webpackCacheDirectories:
+          this.activeWebpackConfigs != null
+            ? getCacheDirectories(this.activeWebpackConfigs)
+            : undefined,
       }),
     ]
   }