chore(swc-wasm): Fix and clean up various issues with swc-wasm tests

Author: bgw

.github/workflows/build_and_test.yml

@@ -473,15 +473,12 @@ jobs:
       skipNativeInstall: 'yes'
       afterBuild: |
         rustup target add wasm32-unknown-unknown
-        curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
         node ./scripts/normalize-version-bump.js
         pnpm dlx turbo@${TURBO_VERSION} run build-wasm -- --target nodejs
         git checkout .
-        mv crates/wasm/pkg crates/wasm/pkg-nodejs
-        node ./scripts/setup-wasm.mjs
 
         export NEXT_TEST_MODE=start
-        export TEST_WASM=true
+        export NEXT_TEST_WASM=true
         node run-tests.js \
           test/production/pages-dir/production/test/index.test.ts \
           test/e2e/streaming-ssr/index.test.ts

packages/next/errors.json

@@ -706,5 +706,6 @@
   "705": "Route is configured with dynamic = error that cannot be statically generated.",
   "706": "Invariant: static responses cannot be streamed %s",
   "707": "Invariant app-page handler received invalid cache entry %s",
-  "708": "Failed to persist Chrome DevTools workspace UUID. The Chrome DevTools Workspace needs to be reconnected after the next page reload."
+  "708": "Failed to persist Chrome DevTools workspace UUID. The Chrome DevTools Workspace needs to be reconnected after the next page reload.",
+  "709": "cannot run loadNative when `NEXT_TEST_WASM` is set"
 }

packages/next/src/build/swc/index.ts

@@ -159,15 +159,27 @@ let lastNativeBindingsLoadErrorCode:
   | 'unsupported_target'
   | string
   | undefined = undefined
+// Used to cache calls to `loadBindings`
+let pendingBindings: Promise<Binding>
+// some things call `loadNative` directly instead of `loadBindings`... Cache calls to that
+// separately.
 let nativeBindings: Binding
+// can allow hacky sync access to bindings for loadBindingsSync
 let wasmBindings: Binding
 let downloadWasmPromise: any
-let pendingBindings: any
 let swcTraceFlushGuard: any
 let downloadNativeBindingsPromise: Promise<void> | undefined = undefined
 
 export const lockfilePatchPromise: { cur?: Promise<void> } = {}
 
+/**
+ * Attempts to load a native or wasm binding.
+ *
+ * By default, this first tries to use a native binding, falling back to a wasm binding if that
+ * fails.
+ *
+ * This function is `async` as wasm requires an asynchronous import in browsers.
+ */
 export async function loadBindings(
   useWasmBinary: boolean = false
 ): Promise<Binding> {
@@ -180,6 +192,10 @@ export async function loadBindings(
     return pendingBindings
   }
 
+  if (process.env.NEXT_TEST_WASM) {
+    useWasmBinary = true
+  }
+
   // rust needs stdout to be blocking, otherwise it will throw an error (on macOS at least) when writing a lot of data (logs) to it
   // see https://github.com/napi-rs/napi-rs/issues/1630
   // and https://github.com/nodejs/node/blob/main/doc/api/process.md#a-note-on-process-io
@@ -291,7 +307,10 @@ async function tryLoadNativeWithFallback(attempts: Array<string>) {
   return undefined
 }
 
-async function tryLoadWasmWithFallback(attempts: any[]) {
+// helper for loadBindings
+async function tryLoadWasmWithFallback(
+  attempts: any[]
+): Promise<Binding | undefined> {
   try {
     let bindings = await loadWasm('')
     // @ts-expect-error TODO: this event has a wrong type.
@@ -343,8 +362,8 @@ function loadBindingsSync() {
     attempts = attempts.concat(a)
   }
 
-  // we can leverage the wasm bindings if they are already
-  // loaded
+  // HACK: we can leverage the wasm bindings if they are already loaded
+  // this may introduce race conditions
   if (wasmBindings) {
     return wasmBindings
   }
@@ -1059,122 +1078,165 @@ function bindingToApi(
 }
 
 async function loadWasm(importPath = '') {
-  if (wasmBindings) {
-    return wasmBindings
-  }
-
   let attempts = []
-  for (let pkg of ['@next/swc-wasm-nodejs', '@next/swc-wasm-web']) {
-    try {
-      let pkgPath = pkg
+  let rawBindings: RawWasmBindings | null = null
 
-      if (importPath) {
-        // the import path must be exact when not in node_modules
-        pkgPath = path.join(importPath, pkg, 'wasm.js')
-      }
-      let bindings: RawWasmBindings = await import(
-        pathToFileURL(pkgPath).toString()
-      )
-      if (pkg === '@next/swc-wasm-web') {
-        bindings = await bindings.default!()
-      }
-      infoLog('next-swc build: wasm build @next/swc-wasm-web')
-
-      // Note wasm binary does not support async intefaces yet, all async
-      // interface coereces to sync interfaces.
-      wasmBindings = {
-        css: {
-          lightning: {
-            transform: function (_options: any) {
-              throw new Error(
-                '`css.lightning.transform` is not supported by the wasm bindings.'
-              )
-            },
-            transformStyleAttr: function (_options: any) {
-              throw new Error(
-                '`css.lightning.transformStyleAttr` is not supported by the wasm bindings.'
-              )
-            },
-          },
-        },
-        isWasm: true,
-        transform(src: string, options: any) {
-          // TODO: we can remove fallback to sync interface once new stable version of next-swc gets published (current v12.2)
-          return bindings?.transform
-            ? bindings.transform(src.toString(), options)
-            : Promise.resolve(bindings.transformSync(src.toString(), options))
-        },
-        transformSync(src: string, options: any) {
-          return bindings.transformSync(src.toString(), options)
-        },
-        minify(src: string, options: any) {
-          return bindings?.minify
-            ? bindings.minify(src.toString(), options)
-            : Promise.resolve(bindings.minifySync(src.toString(), options))
-        },
-        minifySync(src: string, options: any) {
-          return bindings.minifySync(src.toString(), options)
-        },
-        parse(src: string, options: any) {
-          return bindings?.parse
-            ? bindings.parse(src.toString(), options)
-            : Promise.resolve(bindings.parseSync(src.toString(), options))
-        },
-        getTargetTriple() {
-          return undefined
-        },
-        turbo: {
-          createProject: function (
-            _options: ProjectOptions,
-            _turboEngineOptions?: TurboEngineOptions | undefined
-          ): Promise<Project> {
-            throw new Error(
-              '`turbo.createProject` is not supported by the wasm bindings.'
-            )
-          },
-          startTurbopackTraceServer: function (_traceFilePath: string): void {
-            throw new Error(
-              '`turbo.startTurbopackTraceServer` is not supported by the wasm bindings.'
-            )
-          },
-        },
-        mdx: {
-          compile(src: string, options: any) {
-            return bindings.mdxCompile(src, getMdxOptions(options))
-          },
-          compileSync(src: string, options: any) {
-            return bindings.mdxCompileSync(src, getMdxOptions(options))
-          },
-        },
-        reactCompiler: {
-          isReactCompilerRequired(_filename: string) {
-            return Promise.resolve(true)
-          },
-        },
-      }
-      return wasmBindings
-    } catch (e: any) {
-      // Only log attempts for loading wasm when loading as fallback
-      if (importPath) {
-        if (e?.code === 'ERR_MODULE_NOT_FOUND') {
-          attempts.push(`Attempted to load ${pkg}, but it was not installed`)
+  // Used by `run-tests` to force use of a locally-built wasm binary. This environment variable is
+  // unstable and subject to change.
+  const testWasmDir = process.env.NEXT_TEST_WASM_DIR
+
+  if (testWasmDir != null) {
+    // assume these are node.js bindings and don't need a call to `.default()`
+    rawBindings = await import(
+      pathToFileURL(path.join(testWasmDir, 'wasm.js')).toString()
+    )
+    infoLog(`next-swc build: wasm build ${testWasmDir}`)
+  } else {
+    for (let pkg of ['@next/swc-wasm-nodejs', '@next/swc-wasm-web']) {
+      try {
+        let pkgPath = pkg
+
+        if (importPath) {
+          // the import path must be exact when not in node_modules
+          pkgPath = path.join(importPath, pkg, 'wasm.js')
+        }
+        const importedRawBindings = await import(
+          pathToFileURL(pkgPath).toString()
+        )
+        if (pkg === '@next/swc-wasm-web') {
+          // https://rustwasm.github.io/docs/wasm-bindgen/examples/without-a-bundler.html
+          // `default` must be called to initialize the module
+          rawBindings = await importedRawBindings.default!()
         } else {
-          attempts.push(
-            `Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}`
-          )
+          rawBindings = importedRawBindings
+        }
+        infoLog(`next-swc build: wasm build ${pkg}`)
+      } catch (e: any) {
+        // Only log attempts for loading wasm when loading as fallback
+        if (importPath) {
+          if (e?.code === 'ERR_MODULE_NOT_FOUND') {
+            attempts.push(`Attempted to load ${pkg}, but it was not installed`)
+          } else {
+            attempts.push(
+              `Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}`
+            )
+          }
         }
       }
     }
   }
 
-  throw attempts
+  if (rawBindings == null) {
+    throw attempts
+  }
+
+  function removeUndefined(obj: any): any {
+    // serde-wasm-bindgen expect that `undefined` values map to `()` in rust, but we want to treat
+    // those fields as non-existent, so remove them before passing them to rust.
+    //
+    // The native (non-wasm) bindings use `JSON.stringify`, which strips undefined values.
+    if (typeof obj !== 'object') {
+      return
+    }
+    if (Array.isArray(obj)) {
+      return obj.map(removeUndefined)
+    }
+    for (const [k, v] of Object.entries(obj)) {
+      if (typeof v === 'undefined') {
+        delete obj[k]
+      } else {
+        obj[k] = removeUndefined(v)
+      }
+    }
+  }
+
+  // Note wasm binary does not support async intefaces yet, all async
+  // interface coereces to sync interfaces.
+  wasmBindings = {
+    css: {
+      lightning: {
+        transform: function (_options: any) {
+          throw new Error(
+            '`css.lightning.transform` is not supported by the wasm bindings.'
+          )
+        },
+        transformStyleAttr: function (_options: any) {
+          throw new Error(
+            '`css.lightning.transformStyleAttr` is not supported by the wasm bindings.'
+          )
+        },
+      },
+    },
+    isWasm: true,
+    transform(src: string, options: any): Promise<any> {
+      return rawBindings.transform(src.toString(), removeUndefined(options))
+    },
+    transformSync(src: string, options: any) {
+      return rawBindings.transformSync(src.toString(), removeUndefined(options))
+    },
+    minify(src: string, options: any): Promise<any> {
+      return rawBindings.minify(src.toString(), removeUndefined(options))
+    },
+    minifySync(src: string, options: any) {
+      return rawBindings.minifySync(src.toString(), removeUndefined(options))
+    },
+    parse(src: string, options: any): Promise<any> {
+      return rawBindings.parse(src.toString(), removeUndefined(options))
+    },
+    getTargetTriple() {
+      return undefined
+    },
+    turbo: {
+      createProject: function (
+        _options: ProjectOptions,
+        _turboEngineOptions?: TurboEngineOptions | undefined
+      ): Promise<Project> {
+        throw new Error(
+          '`turbo.createProject` is not supported by the wasm bindings.'
+        )
+      },
+      startTurbopackTraceServer: function (_traceFilePath: string): void {
+        throw new Error(
+          '`turbo.startTurbopackTraceServer` is not supported by the wasm bindings.'
+        )
+      },
+    },
+    mdx: {
+      compile(src: string, options: any) {
+        return rawBindings.mdxCompile(
+          src,
+          removeUndefined(getMdxOptions(options))
+        )
+      },
+      compileSync(src: string, options: any) {
+        return rawBindings.mdxCompileSync(
+          src,
+          removeUndefined(getMdxOptions(options))
+        )
+      },
+    },
+    reactCompiler: {
+      isReactCompilerRequired(_filename: string) {
+        return Promise.resolve(true)
+      },
+    },
+  }
+  return wasmBindings
 }
 
+/**
+ * Loads the native (non-wasm) bindings. Prefer `loadBindings` over this API, as that includes a
+ * wasm fallback.
+ */
 function loadNative(importPath?: string) {
   if (nativeBindings) {
     return nativeBindings
   }
 
+  if (process.env.NEXT_TEST_WASM) {
+    throw new Error('cannot run loadNative when `NEXT_TEST_WASM` is set')
+  }
+
   const customBindings: RawBindings = !!__INTERNAL_CUSTOM_TURBOPACK_BINDINGS
     ? require(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS)
     : null