Inline pthread worker.js file into the main output file

This method has many advantages over the previous method of generating
a separate file:

- Avoids separate output file simplifying deployment.
- Avoids confusing laying of global scopes
- Avoids exporting symbols on the Module simply for visibility within
  the worker file.
- Avoids code duplication
- Avoids the needs to importScripts call, and the node polyfill for
  this.
- Allows optimizers such as closure and JSDCE to operate on the combined
  code.
- `-sSINGLE_FILE` now works with pthreads
- Fewer network requests
- No need for locateFile logic to run on the worker to find the worker.js

Fixes: #9796

Author: sbc100

src/library_async.js

@@ -38,6 +38,8 @@ addToLibrary({
     rewindArguments: {},
 #endif
     instrumentWasmImports(imports) {
+      if (imports.instrumented) return;
+      imports.instrumented = true;
 #if ASYNCIFY_DEBUG
       dbg('asyncify instrumenting imports');
 #endif

src/library_pthread.js

@@ -75,7 +75,7 @@ var LibraryPThread = {
         ) {
           t = _pthread_self();
         }
-        return 'w:' + (Module['workerID'] || 0) + ',t:' + ptrToString(t) + ': ';
+        return 'w:' + (workerID || 0) + ',t:' + ptrToString(t) + ': ';
       }
 
       // Prefix all err()/dbg() messages with the calling thread ID.
@@ -127,16 +127,6 @@ var LibraryPThread = {
     },
 
     initWorker() {
-#if MAYBE_CLOSURE_COMPILER
-      // worker.js is not compiled together with us, and must access certain
-      // things.
-      PThread['receiveObjectTransfer'] = PThread.receiveObjectTransfer;
-      PThread['threadInitTLS'] = PThread.threadInitTLS;
-#if !MINIMAL_RUNTIME
-      PThread['setExitStatus'] = PThread.setExitStatus;
-#endif
-#endif
-
 #if isSymbolNeeded('$noExitRuntime')
       // The default behaviour for pthreads is always to exit once they return
       // from their entry point (or call pthread_exit).  If we set noExitRuntime
@@ -440,34 +430,49 @@ var LibraryPThread = {
     // Creates a new web Worker and places it in the unused worker pool to wait for its use.
     allocateUnusedWorker() {
       var worker;
+      var workerOptions = {
+        {{{ EXPORT_ES6 ? "type: 'module'," : "" }}}
+#if ENVIRONMENT_MAY_BE_NODE
+        // This is the way that we signal to the node worker that it is hosting
+        // a pthread.  On the web we append `?pthread=1` to the URL instead.
+        'workerData': 'pthread',
+#endif
+      };
 #if MINIMAL_RUNTIME
-      var pthreadMainJs = Module['worker'] || './{{{ PTHREAD_WORKER_FILE }}}';
+      var pthreadMainJs = Module['worker'] || './{{{ TARGET_JS_NAME }}}';
 #else
 #if EXPORT_ES6 && USE_ES6_IMPORT_META
       // If we're using module output and there's no explicit override, use bundler-friendly pattern.
       if (!Module['locateFile']) {
 #if PTHREADS_DEBUG
-        dbg('Allocating a new web worker from ' + new URL('{{{ PTHREAD_WORKER_FILE }}}', import.meta.url));
+        dbg('Allocating a new web worker from ' + import.meta.url);
 #endif
 #if TRUSTED_TYPES
         // Use Trusted Types compatible wrappers.
         if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) {
           var p = trustedTypes.createPolicy(
             'emscripten#workerPolicy1',
             {
-              createScriptURL: (ignored) => new URL('{{{ PTHREAD_WORKER_FILE }}}', import.meta.url);
+              createScriptURL: (ignored) => new URL(import.meta.url);
             }
           );
-          worker = new Worker(p.createScriptURL('ignored'), {type: 'module'});
+          worker = new Worker(p.createScriptURL('ignored'), workerOptions);
         } else
 #endif
-        worker = new Worker(new URL('{{{ PTHREAD_WORKER_FILE }}}', import.meta.url), {type: 'module'});
+        worker = new Worker(new URL(import.meta.url), workerOptions);
       } else {
 #endif
       // Allow HTML module to configure the location where the 'worker.js' file will be loaded from,
       // via Module.locateFile() function. If not specified, then the default URL 'worker.js' relative
       // to the main html file is loaded.
-      var pthreadMainJs = locateFile('{{{ PTHREAD_WORKER_FILE }}}');
+      var pthreadMainJs = locateFile('{{{ TARGET_JS_NAME }}}');
+#if ENVIRONMENT_MAY_BE_WEB
+      // On the web we use a URL parameter to determine if we are running a
+      // pthread or not.
+      if (ENVIRONMENT_IS_WEB) {
+        pthreadMainJs += '?pthread=1';
+      }
+#endif
 #endif
 #if PTHREADS_DEBUG
       dbg(`Allocating a new web worker from ${pthreadMainJs}`);
@@ -476,10 +481,10 @@ var LibraryPThread = {
       // Use Trusted Types compatible wrappers.
       if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) {
         var p = trustedTypes.createPolicy('emscripten#workerPolicy2', { createScriptURL: (ignored) => pthreadMainJs });
-        worker = new Worker(p.createScriptURL('ignored'){{{ EXPORT_ES6 ? ", {type: 'module'}" : '' }}});
+        worker = new Worker(p.createScriptURL('ignored'), workerOptions);
       } else
 #endif
-      worker = new Worker(pthreadMainJs{{{ EXPORT_ES6 ? ", {type: 'module'}" : '' }}});
+      worker = new Worker(pthreadMainJs, workerOptions);
 #if EXPORT_ES6 && USE_ES6_IMPORT_META
     }
 #endif

src/library_sdl.js

@@ -2905,14 +2905,14 @@ var LibrarySDL = {
     return SDL.setGetVolume(SDL.music, volume);
   },
 
-  Mix_LoadMUS_RW__docs: '/** @param {number} a1 */',
-  Mix_LoadMUS_RW: 'Mix_LoadWAV_RW',
+  Mix_LoadMUS_RW__deps: ['Mix_LoadWAV_RW'],
+  Mix_LoadMUS_RW: (filename) => _Mix_LoadWAV_RW(filename, 0),
 
   Mix_LoadMUS__deps: ['Mix_LoadMUS_RW', 'SDL_RWFromFile', 'SDL_FreeRW'],
   Mix_LoadMUS__proxy: 'sync',
   Mix_LoadMUS: (filename) => {
     var rwops = _SDL_RWFromFile(filename, 0);
-    var result = _Mix_LoadMUS_RW(rwops, 0);
+    var result = _Mix_LoadMUS_RW(rwops);
     _SDL_FreeRW(rwops);
     return result;
   },

src/parseTools.mjs

@@ -888,21 +888,6 @@ function buildStringArray(array) {
   }
 }
 
-// Generates access to a JS imports scope variable in pthreads worker.js. In MODULARIZE mode these flow into the imports object for the Module.
-// In non-MODULARIZE mode, we can directly access the variables in global scope.
-function makeAsmImportsAccessInPthread(variable) {
-  if (!MINIMAL_RUNTIME) {
-    // Regular runtime uses the name "Module" for both imports and exports.
-    return `Module['${variable}']`;
-  }
-  if (MODULARIZE) {
-    // MINIMAL_RUNTIME uses 'imports' as the name for the imports object in MODULARIZE builds.
-    return `imports['${variable}']`;
-  }
-  // In non-MODULARIZE builds, can access the imports from global scope.
-  return `self.${variable}`;
-}
-
 function _asmjsDemangle(symbol) {
   if (symbol.startsWith('dynCall_')) {
     return symbol;
@@ -1130,7 +1115,6 @@ addToCompileTimeContext({
   hasExportedSymbol,
   implicitSelf,
   isSymbolNeeded,
-  makeAsmImportsAccessInPthread,
   makeDynCall,
   makeEval,
   makeGetValue,

src/postamble.js

@@ -144,8 +144,8 @@ function stackCheckInit() {
 
 #if MAIN_MODULE && PTHREADS
 // Map of modules to be shared with new threads.  This gets populated by the
-// main thread and shared with all new workers.
-var sharedModules = Module['sharedModules'] || [];
+// main thread and shared with all new workers via the initial `load` message.
+var sharedModules = {};
 #endif
 
 #if MAIN_READS_PARAMS

src/postamble_minimal.js

@@ -89,15 +89,6 @@ function initRuntime(wasmExports) {
 
 // Initialize wasm (asynchronous)
 
-var imports = {
-#if MINIFY_WASM_IMPORTED_MODULES
-  'a': wasmImports,
-#else // MINIFY_WASM_IMPORTED_MODULES
-  'env': wasmImports,
-  '{{{ WASI_MODULE_NAME }}}': wasmImports,
-#endif // MINIFY_WASM_IMPORTED_MODULES
-};
-
 // In non-fastcomp non-asm.js builds, grab wasm exports to outer scope
 // for emscripten_get_exported_function() to be able to access them.
 #if LibraryManager.has('library_exports.js')
@@ -112,6 +103,20 @@ var wasmModule;
 <<< WASM_MODULE_EXPORTS_DECLARES >>>
 #endif
 
+#if PTHREADS
+function loadModule() {
+  assignWasmImports();
+#endif
+
+var imports = {
+#if MINIFY_WASM_IMPORTED_MODULES
+  'a': wasmImports,
+#else // MINIFY_WASM_IMPORTED_MODULES
+  'env': wasmImports,
+  '{{{ WASI_MODULE_NAME }}}': wasmImports,
+#endif // MINIFY_WASM_IMPORTED_MODULES
+};
+
 #if MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION
 // https://caniuse.com/#feat=wasm and https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/instantiateStreaming
 // Firefox 52 added Wasm support, but only Firefox 58 added instantiateStreaming.
@@ -252,3 +257,13 @@ WebAssembly.instantiate(Module['wasm'], imports).then((output) => {
 }
 #endif // ASSERTIONS || WASM == 2
 );
+
+#if PTHREADS
+}
+
+if (!ENVIRONMENT_IS_PTHREAD) {
+  // When running in a pthread we delay module loading untill we have
+  // received the module via postMessage
+  loadModule();
+}
+#endif

src/preamble.js

@@ -14,6 +14,10 @@
 // An online HTML version (which may be of a different version of Emscripten)
 //    is up at http://kripken.github.io/emscripten-site/docs/api_reference/preamble.js.html
 
+#if PTHREADS
+#include "runtime_pthread.js"
+#endif
+
 #if RELOCATABLE
 {{{ makeModuleReceiveWithVar('dynamicLibraries', undefined, '[]', true) }}}
 #endif
@@ -897,11 +901,18 @@ function instantiateAsync(binary, binaryFile, imports, callback) {
 }
 #endif // WASM_ASYNC_COMPILATION
 
-// Create the wasm instance.
-// Receives the wasm imports, returns the exports.
-function createWasm() {
+function getWasmImports() {
+#if PTHREADS
+  assignWasmImports();
+#endif
+#if ASYNCIFY && (ASSERTIONS || ASYNCIFY == 2)
+  // instrumenting imports is used in asyncify in two ways: to add assertions
+  // that check for proper import use, and for ASYNCIFY=2 we use them to set up
+  // the Promise API on the import side.
+  Asyncify.instrumentWasmImports(wasmImports);
+#endif
   // prepare imports
-  var info = {
+  return {
 #if MINIFY_WASM_IMPORTED_MODULES
     'a': wasmImports,
 #else // MINIFY_WASM_IMPORTED_MODULES
@@ -915,7 +926,13 @@ function createWasm() {
     'GOT.mem': new Proxy(wasmImports, GOTHandler),
     'GOT.func': new Proxy(wasmImports, GOTHandler),
 #endif
-  };
+  }
+}
+
+// Create the wasm instance.
+// Receives the wasm imports, returns the exports.
+function createWasm() {
+  var info = getWasmImports();
   // Load the wasm module and create an instance of using native support in the JS engine.
   // handle a generated wasm instance, receiving its exports and
   // performing other necessary setup
@@ -1047,21 +1064,6 @@ function createWasm() {
   // Also pthreads and wasm workers initialize the wasm instance through this
   // path.
   if (Module['instantiateWasm']) {
-
-#if USE_OFFSET_CONVERTER
-#if ASSERTIONS
-{{{ runIfWorkerThread("assert(Module['wasmOffsetData'], 'wasmOffsetData not found on Module object');") }}}
-#endif
-{{{ runIfWorkerThread("wasmOffsetConverter = resetPrototype(WasmOffsetConverter, Module['wasmOffsetData']);") }}}
-#endif
-
-#if LOAD_SOURCE_MAP
-#if ASSERTIONS
-{{{ runIfWorkerThread("assert(Module['wasmSourceMapData'], 'wasmSourceMapData not found on Module object');") }}}
-#endif
-{{{ runIfWorkerThread("wasmSourceMap = resetPrototype(WasmSourceMap, Module['wasmSourceMapData']);") }}}
-#endif
-
     try {
       return Module['instantiateWasm'](info, receiveInstance);
     } catch(e) {

src/preamble_minimal.js

@@ -12,6 +12,10 @@
 #include "runtime_asan.js"
 #endif
 
+#if PTHREADS
+#include "runtime_pthread.js"
+#endif
+
 #if ASSERTIONS
 /** @type {function(*, string=)} */
 function assert(condition, text) {
@@ -85,11 +89,19 @@ else {
 #endif // MODULARIZE
 #endif // PTHREADS
 
+#if PTHREADS
+if (!ENVIRONMENT_IS_PTHREAD) {
+#endif
+
 #if ASSERTIONS && SHARED_MEMORY
 assert(wasmMemory.buffer instanceof SharedArrayBuffer, 'requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag');
 #endif
 
 updateMemoryViews();
+
+#if PTHREADS
+}
+#endif
 #endif // IMPORTED_MEMORY
 
 #include "runtime_stack_check.js"

src/runtime_init_memory.js

@@ -9,18 +9,10 @@
 {{{ throw "this file should not be be included when IMPORTED_MEMORY is set"; }}}
 #endif
 
-{{{ makeModuleReceiveWithVar('INITIAL_MEMORY', undefined, INITIAL_MEMORY) }}}
-
-#if ASSERTIONS
-assert(INITIAL_MEMORY >= {{{STACK_SIZE}}}, 'INITIAL_MEMORY should be larger than STACK_SIZE, was ' + INITIAL_MEMORY + '! (STACK_SIZE=' + {{{STACK_SIZE}}} + ')');
-#endif
-  
 // check for full engine support (use string 'subarray' to avoid closure compiler confusion)
 
 #if PTHREADS
-if (ENVIRONMENT_IS_PTHREAD) {
-  wasmMemory = Module['wasmMemory'];
-} else {
+if (!ENVIRONMENT_IS_PTHREAD) {
 #endif // PTHREADS
 
 #if expectToReceiveOnModule('wasmMemory')
@@ -29,6 +21,11 @@ if (ENVIRONMENT_IS_PTHREAD) {
   } else
 #endif
   {
+    {{{ makeModuleReceiveWithVar('INITIAL_MEMORY', undefined, INITIAL_MEMORY) }}}
+
+    #if ASSERTIONS
+    assert(INITIAL_MEMORY >= {{{STACK_SIZE}}}, 'INITIAL_MEMORY should be larger than STACK_SIZE, was ' + INITIAL_MEMORY + '! (STACK_SIZE=' + {{{STACK_SIZE}}} + ')');
+    #endif
     wasmMemory = new WebAssembly.Memory({
       'initial': INITIAL_MEMORY / {{{ WASM_PAGE_SIZE }}},
 #if ALLOW_MEMORY_GROWTH
@@ -59,15 +56,8 @@ if (ENVIRONMENT_IS_PTHREAD) {
 #endif
   }
 
+  updateMemoryViews();
 #if PTHREADS
 }
 #endif
 
-updateMemoryViews();
-
-// If the user provides an incorrect length, just use that length instead rather than providing the user to
-// specifically provide the memory length with Module['INITIAL_MEMORY'].
-INITIAL_MEMORY = wasmMemory.buffer.byteLength;
-#if ASSERTIONS
-assert(INITIAL_MEMORY % {{{ WASM_PAGE_SIZE }}} === 0);
-#endif