Add experimental trace file field (#65071)

This adds a new field to our `.nft.json` trace files when enabled behind
a feature flag to output the hashes for the traced files.

Closes NEXT-3232

Author: ijjk

packages/next/src/build/collect-build-traces.ts

@@ -5,6 +5,7 @@ import {
   TRACE_IGNORES,
   type BuildTraceContext,
   getFilesMapFromReasons,
+  getHash,
 } from './webpack/plugins/next-trace-entrypoints-plugin'
 
 import {
@@ -29,6 +30,8 @@ import type { RoutesUsingEdgeRuntime } from './utils'
 
 const debug = debugOriginal('next:build:build-traces')
 
+const hashCache: Record<string, string> = {}
+
 function shouldIgnore(
   file: string,
   serverIgnoreFn: (file: string) => boolean,
@@ -88,11 +91,13 @@ export async function collectBuildTraces({
   hasSsrAmpPages,
   buildTraceContext,
   outputFileTracingRoot,
+  isFlyingShuttle,
 }: {
   dir: string
   distDir: string
   staticPages: string[]
   hasSsrAmpPages: boolean
+  isFlyingShuttle?: boolean
   outputFileTracingRoot: string
   // pageInfos is serialized when this function runs in a worker.
   edgeRuntimeRoutes: RoutesUsingEdgeRuntime
@@ -496,9 +501,11 @@ export async function collectBuildTraces({
           [minimalServerEntries, minimalServerTracedFiles],
         ] as Array<[string[], Set<string>]>) {
           for (const file of entries) {
-            const curFiles = parentFilesMap.get(
-              path.relative(outputFileTracingRoot, file)
-            )
+            const curFiles = [
+              ...(parentFilesMap
+                .get(path.relative(outputFileTracingRoot, file))
+                ?.keys() || []),
+            ]
             tracedFiles.add(path.relative(distDir, file).replace(/\\/g, '/'))
 
             for (const curFile of curFiles || []) {
@@ -545,8 +552,10 @@ export async function collectBuildTraces({
             }
 
             // we don't need to trace for automatically statically optimized
-            // pages as they don't have server bundles
-            if (staticPages.includes(route)) {
+            // pages as they don't have server bundles, note there is
+            // the caveat with flying shuttle mode as it needs this for
+            // detecting changed entries
+            if (staticPages.includes(route) && !isFlyingShuttle) {
               return
             }
             const entryOutputPath = path.join(
@@ -557,14 +566,22 @@ export async function collectBuildTraces({
             const traceOutputPath = `${entryOutputPath}.nft.json`
             const existingTrace = JSON.parse(
               await fs.readFile(traceOutputPath, 'utf8')
-            )
+            ) as {
+              version: number
+              files: string[]
+              fileHashes: Record<string, string>
+            }
             const traceOutputDir = path.dirname(traceOutputPath)
             const curTracedFiles = new Set<string>()
+            const curFileHashes: Record<string, string> =
+              existingTrace.fileHashes
 
             for (const file of [...entryNameFiles, entryOutputPath]) {
-              const curFiles = parentFilesMap.get(
-                path.relative(outputFileTracingRoot, file)
-              )
+              const curFiles = [
+                ...(parentFilesMap
+                  .get(path.relative(outputFileTracingRoot, file))
+                  ?.keys() || []),
+              ]
               for (const curFile of curFiles || []) {
                 if (
                   !shouldIgnore(
@@ -579,6 +596,19 @@ export async function collectBuildTraces({
                     .relative(traceOutputDir, filePath)
                     .replace(/\\/g, '/')
                   curTracedFiles.add(outputFile)
+
+                  if (isFlyingShuttle) {
+                    try {
+                      let hash = hashCache[filePath]
+
+                      if (!hash) {
+                        hash = getHash(await fs.readFile(filePath))
+                      }
+                      curFileHashes[outputFile] = hash
+                    } catch (err: any) {
+                      // handle symlink errors or similar
+                    }
+                  }
                 }
               }
             }
@@ -592,6 +622,11 @@ export async function collectBuildTraces({
               JSON.stringify({
                 ...existingTrace,
                 files: [...curTracedFiles].sort(),
+                ...(isFlyingShuttle
+                  ? {
+                      fileHashes: curFileHashes,
+                    }
+                  : {}),
               })
             )
           })

packages/next/src/build/index.ts

@@ -1658,6 +1658,7 @@ export default async function build(
                   hasSsrAmpPages: false,
                   buildTraceContext,
                   outputFileTracingRoot,
+                  isFlyingShuttle: !!config.experimental.flyingShuttle,
                 })
                 .catch((err) => {
                   console.error(err)
@@ -2396,6 +2397,7 @@ export default async function build(
           hasSsrAmpPages,
           buildTraceContext,
           outputFileTracingRoot,
+          isFlyingShuttle: !!config.experimental.flyingShuttle,
         }).catch((err) => {
           console.error(err)
           process.exit(1)

packages/next/src/build/webpack-config.ts

@@ -1762,6 +1762,8 @@ export default async function getBaseWebpackConfig(
             turbotrace: config.experimental.turbotrace,
             optOutBundlingPackages,
             traceIgnores: config.experimental.outputFileTracingIgnores || [],
+            flyingShuttle: !!config.experimental.flyingShuttle,
+            compilerType,
           }
         ),
       // Moment.js is an extremely popular library that bundles large locale files