[devtools] Port cache invalidation logic from turbopack to webpack

Author: bgw

packages/next/errors.json

@@ -706,5 +706,6 @@
   "705": "Route is configured with dynamic = error that cannot be statically generated.",
   "706": "Invariant: static responses cannot be streamed %s",
   "707": "Invariant app-page handler received invalid cache entry %s",
-  "708": "Failed to persist Chrome DevTools workspace UUID. The Chrome DevTools Workspace needs to be reconnected after the next page reload."
+  "708": "Failed to persist Chrome DevTools workspace UUID. The Chrome DevTools Workspace needs to be reconnected after the next page reload.",
+  "709": "Unable to remove an invalidated webpack cache. If this issue persists you can work around it by deleting %s"
 }

packages/next/src/build/define-env.ts

@@ -289,8 +289,21 @@ export function getDefineEnv({
       : {}),
     'process.env.__NEXT_DEVTOOL_SEGMENT_EXPLORER':
       config.experimental.devtoolSegmentExplorer ?? false,
-    'process.env.__NEXT_TURBOPACK_PERSISTENT_CACHE':
-      config.experimental.turbopackPersistentCaching ?? false,
+
+    // The devtools need to know whether or not to show an option to clear the
+    // bundler cache. This option may be removed later once Turbopack's
+    // persistent cache feature is more stable.
+    //
+    // This environment value is currently best-effort:
+    // - It's possible to disable the webpack filesystem cache, but it's
+    //   unlikely for a user to do that.
+    // - Rspack's persistent cache is unstable and requires a different
+    //   configuration than webpack to enable (which we don't do).
+    //
+    // In the worst case we'll show an option to clear the cache, but it'll be a
+    // no-op that just restarts the development server.
+    'process.env.__NEXT_BUNDLER_HAS_PERSISTENT_CACHE':
+      !isTurbopack || (config.experimental.turbopackPersistentCaching ?? false),
   }
 
   const userDefines = config.compiler?.define ?? {}

packages/next/src/build/webpack-config.ts

@@ -289,6 +289,21 @@ export function hasExternalOtelApiPackage(): boolean {
 
 const UNSAFE_CACHE_REGEX = /[\\/]pages[\\/][^\\/]+(?:$|\?|#)/
 
+export function getCacheDirectories(
+  configs: webpack.Configuration[]
+): Set<string> {
+  return new Set(
+    configs
+      .map((cfg) => {
+        if (typeof cfg.cache === 'object' && cfg.cache.type === 'filesystem') {
+          return cfg.cache.cacheDirectory
+        }
+        return null
+      })
+      .filter((dir) => dir != null)
+  )
+}
+
 export default async function getBaseWebpackConfig(
   dir: string,
   {

packages/next/src/build/webpack/cache-invalidation.ts

@@ -0,0 +1,96 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+
+const INVALIDATION_MARKER = '__nextjs_invalidated_cache'
+
+/**
+ * Atomically write an invalidation marker.
+ *
+ * Because attempting to delete currently open cache files could cause issues,
+ * actual deletion of files is deferred until the next start-up (in
+ * `checkPersistentCacheInvalidationAndCleanup`).
+ *
+ * In the case that no database is currently open (e.g. via a separate CLI
+ * subcommand), you should call `cleanupPersistentCache` *after* this to eagerly
+ * remove the cache files.
+ */
+export async function invalidatePersistentCache(cacheDirectory: string) {
+  let file
+  try {
+    // We're just opening it so that `open()` creates the file.
+    file = await fs.open(path.join(cacheDirectory, INVALIDATION_MARKER), 'w')
+    // We don't currently write anything to the file, but we could choose to
+    // later, e.g. a reason for the invalidation.
+  } catch (err: any) {
+    // it's valid for the cache to not exist at all
+    if (err.errno === 'ENOENT') {
+      return
+    }
+  } finally {
+    file?.close()
+  }
+}
+
+/**
+ * Called during startup. See if the cache is in a partially-completed
+ * invalidation state. Finds and delete any invalidated cache files.
+ */
+export async function checkPersistentCacheInvalidationAndCleanup(
+  cacheDirectory: string
+) {
+  const invalidated = await fs
+    .access(path.join(cacheDirectory, INVALIDATION_MARKER))
+    .then(
+      () => true,
+      () => false
+    )
+  if (invalidated) {
+    await cleanupPersistentCache(cacheDirectory)
+  }
+}
+
+/**
+ * Helper for `checkPersistentCacheInvalidationAndCleanup`. You can call this to
+ * explicitly clean up a database after running `invalidatePersistentCache` when
+ * webpack is not running.
+ *
+ * You should not run this if the cache has not yet been invalidated, as this
+ * operation is not atomic and could result in a partially-deleted and corrupted
+ * database.
+ */
+async function cleanupPersistentCache(cacheDirectory: string) {
+  try {
+    await cleanupPersistentCacheInner(cacheDirectory)
+  } catch (e) {
+    // generate a user-friendly error message
+    throw new Error(
+      `Unable to remove an invalidated webpack cache. If this issue persists ` +
+        `you can work around it by deleting ${cacheDirectory}`,
+      { cause: e }
+    )
+  }
+}
+
+async function cleanupPersistentCacheInner(cacheDirectory: string) {
+  const files = await fs.readdir(cacheDirectory)
+
+  // delete everything except the invalidation marker
+  await Promise.all(
+    files.map((name) =>
+      name !== INVALIDATION_MARKER
+        ? fs.rm(path.join(cacheDirectory, name), {
+            force: true, // ignore errors if path does not exist
+            recursive: true,
+            maxRetries: 2, // windows prevents deletion of open files
+          })
+        : null
+    )
+  )
+
+  // delete the invalidation marker last, once we're sure everything is cleaned
+  // up
+  await fs.rm(path.join(cacheDirectory, INVALIDATION_MARKER), {
+    force: true,
+    maxRetries: 2,
+  })
+}

packages/next/src/next-devtools/dev-overlay/components/errors/dev-tools-indicator/dev-tools-info/user-preferences.tsx

@@ -66,10 +66,10 @@ export function UserPreferences({
     setScale(value)
   }
 
-  function handleRestartDevServer() {
+  function handleRestartDevServer(invalidatePersistentCache: boolean) {
     let endpoint = '/__nextjs_restart_dev'
 
-    if (process.env.__NEXT_TURBOPACK_PERSISTENT_CACHE) {
+    if (invalidatePersistentCache) {
       endpoint = '/__nextjs_restart_dev?invalidatePersistentCache'
     }
 
@@ -198,14 +198,16 @@ export function UserPreferences({
               name="restart-dev-server"
               data-restart-dev-server
               className="action-button"
-              onClick={handleRestartDevServer}
+              onClick={() =>
+                handleRestartDevServer(/*invalidatePersistentCache*/ false)
+              }
             >
               <span>Restart</span>
             </button>
           </div>
         </div>
       </div>
-      {process.env.__NEXT_TURBOPACK_PERSISTENT_CACHE ? (
+      {process.env.__NEXT_BUNDLER_HAS_PERSISTENT_CACHE ? (
         <div className="preferences-container">
           <div className="preference-section">
             <div className="preference-header">
@@ -222,7 +224,9 @@ export function UserPreferences({
                 name="reset-bundler-cache"
                 data-reset-bundler-cache
                 className="action-button"
-                onClick={handleRestartDevServer}
+                onClick={() =>
+                  handleRestartDevServer(/*invalidatePersistentCache*/ true)
+                }
               >
                 <span>Reset Cache</span>
               </button>

packages/next/src/next-devtools/dev-overlay/components/errors/error-overlay-toolbar/restart-server-button.tsx

@@ -34,7 +34,7 @@ export function RestartServerButton({ error }: { error: Error }) {
   function handleClick() {
     let endpoint = '/__nextjs_restart_dev'
 
-    if (process.env.__NEXT_TURBOPACK_PERSISTENT_CACHE) {
+    if (process.env.__NEXT_BUNDLER_HAS_PERSISTENT_CACHE) {
       endpoint = '/__nextjs_restart_dev?invalidatePersistentCache'
     }
 
@@ -52,7 +52,7 @@ export function RestartServerButton({ error }: { error: Error }) {
       className="restart-dev-server-button"
       onClick={handleClick}
       title={
-        process.env.__NEXT_TURBOPACK_PERSISTENT_CACHE
+        process.env.__NEXT_BUNDLER_HAS_PERSISTENT_CACHE
           ? 'Clears the bundler cache and restarts the dev server. Helpful if you are seeing stale errors or changes are not appearing.'
           : 'Restarts the development server without needing to leave the browser.'
       }

packages/next/src/next-devtools/server/restart-dev-server-middleware.ts

@@ -3,17 +3,20 @@ import type { Telemetry } from '../../telemetry/storage'
 import { RESTART_EXIT_CODE } from '../../server/lib/utils'
 import { middlewareResponse } from './middleware-response'
 import type { Project } from '../../build/swc/types'
+import { invalidatePersistentCache as invalidateWebpackPersistentCache } from '../../build/webpack/cache-invalidation'
 
 const EVENT_DEV_OVERLAY_RESTART_SERVER = 'DEV_OVERLAY_RESTART_SERVER'
 
 interface RestartDevServerMiddlewareConfig {
   telemetry: Telemetry
   turbopackProject?: Project
+  webpackCacheDirectories?: Set<string>
 }
 
 export function getRestartDevServerMiddleware({
   telemetry,
   turbopackProject,
+  webpackCacheDirectories,
 }: RestartDevServerMiddlewareConfig) {
   /**
    * Some random value between 1 and Number.MAX_SAFE_INTEGER (inclusive). The same value is returned
@@ -34,16 +37,25 @@ export function getRestartDevServerMiddleware({
       return middlewareResponse.methodNotAllowed(res)
     }
 
-    const invalidatePersistentCache = searchParams.has(
+    const shouldInvalidatePersistentCache = searchParams.has(
       'invalidatePersistentCache'
     )
-    if (invalidatePersistentCache) {
-      await turbopackProject?.invalidatePersistentCache()
+    if (shouldInvalidatePersistentCache) {
+      if (webpackCacheDirectories != null) {
+        await Promise.all(
+          Array.from(webpackCacheDirectories).map(
+            invalidateWebpackPersistentCache
+          )
+        )
+      }
+      if (turbopackProject != null) {
+        await turbopackProject.invalidatePersistentCache()
+      }
     }
 
     telemetry.record({
       eventName: EVENT_DEV_OVERLAY_RESTART_SERVER,
-      payload: { invalidatePersistentCache },
+      payload: { invalidatePersistentCache: shouldInvalidatePersistentCache },
     })
 
     // TODO: Use flushDetached

packages/next/src/server/dev/hot-reloader-webpack.ts

@@ -27,6 +27,7 @@ import {
 import { watchCompilers } from '../../build/output'
 import * as Log from '../../build/output/log'
 import getBaseWebpackConfig, {
+  getCacheDirectories,
   loadProjectInfo,
 } from '../../build/webpack-config'
 import { APP_DIR_ALIAS, WEBPACK_LAYERS } from '../../lib/constants'
@@ -88,6 +89,7 @@ import { getDevOverlayFontMiddleware } from '../../next-devtools/server/font/get
 import { getDisableDevIndicatorMiddleware } from '../../next-devtools/server/dev-indicator-middleware'
 import getWebpackBundler from '../../shared/lib/get-webpack-bundler'
 import { getRestartDevServerMiddleware } from '../../next-devtools/server/restart-dev-server-middleware'
+import { checkPersistentCacheInvalidationAndCleanup } from '../../build/webpack/cache-invalidation'
 
 const MILLISECONDS_IN_NANOSECOND = BigInt(1_000_000)
 
@@ -1155,6 +1157,11 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
     // @ts-ignore webpack 5
     this.activeWebpackConfigs.parallelism = 1
 
+    await Promise.all(
+      Array.from(getCacheDirectories(this.activeWebpackConfigs)).map(
+        checkPersistentCacheInvalidationAndCleanup
+      )
+    )
     this.multiCompiler = getWebpackBundler()(
       this.activeWebpackConfigs
     ) as unknown as webpack.MultiCompiler
@@ -1574,6 +1581,10 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
       getDisableDevIndicatorMiddleware(),
       getRestartDevServerMiddleware({
         telemetry: this.telemetry,
+        webpackCacheDirectories:
+          this.activeWebpackConfigs != null
+            ? getCacheDirectories(this.activeWebpackConfigs)
+            : undefined,
       }),
     ]
   }

turbopack/crates/turbo-tasks-backend/src/database/db_invalidation.rs

@@ -1,6 +1,6 @@
 use std::{
     fs::{self, read_dir},
-    io,
+    io::{self, ErrorKind},
     path::Path,
 };
 
@@ -19,8 +19,12 @@ const INVALIDATION_MARKER: &str = "__turbo_tasks_invalidated_db";
 /// This should be run with the base (non-versioned) path, as that likely aligns closest with user
 /// expectations (e.g. if they're clearing the cache for disk space reasons).
 pub fn invalidate_db(base_path: &Path) -> anyhow::Result<()> {
-    fs::write(base_path.join(INVALIDATION_MARKER), [0u8; 0])?;
-    Ok(())
+    match fs::write(base_path.join(INVALIDATION_MARKER), [0u8; 0]) {
+        Ok(_) => Ok(()),
+        // just ignore if the cache directory doesn't exist at all
+        Err(err) if err.kind() == ErrorKind::NotFound => Ok(()),
+        Err(err) => Err(err).context("Failed to invalidate database"),
+    }
 }
 
 /// Called during startup. See if the db is in a partially-completed invalidation state. Find and