follow-up to #4109

Author: evanw

CHANGELOG.md

@@ -34,6 +34,16 @@
     });
     ```
 
+* Basic support for index source maps ([#3439](https://github.com/evanw/esbuild/issues/3439), [#4109](https://github.com/evanw/esbuild/pull/4109))
+
+    The source map specification has an optional mode called [index source maps](https://tc39.es/ecma426/#sec-index-source-map) that makes it easier for tools to create an aggregate JavaScript file by concatenating many smaller JavaScript files with source maps, and then generate an aggregate source map by simply providing the original source maps along with some offset information. My understanding is that this is rarely used in practice. I'm only aware of two uses of it in the wild: [ClojureScript](https://clojurescript.org/) and [Turbopack](https://turbo.build/pack/).
+
+    This release provides basic support for indexed source maps. However, the implementation has not been tested on a real app (just on very simple test input). If you are using index source maps in a real app, please try this out and report back if anything isn't working for you.
+
+    Note that this is also not a complete implementation. For example, index source maps technically allows nesting source maps to an arbitrary depth, while esbuild's implementation in this release only supports a single level of nesting. It's unclear whether supporting more than one level of nesting is important or not given the lack of available test cases.
+
+    This feature was contributed by [@clyfish](https://github.com/clyfish).
+
 ## 0.25.1
 
 * Fix incorrect paths in inline source maps ([#4070](https://github.com/evanw/esbuild/issues/4070), [#4075](https://github.com/evanw/esbuild/issues/4075), [#4105](https://github.com/evanw/esbuild/issues/4105))

internal/js_parser/sourcemap_parser.go

@@ -38,7 +38,7 @@ func ParseSourceMap(log logger.Log, source logger.Source) *sourcemap.SourceMap {
 	hasSections := false
 
 	for _, prop := range obj.Properties {
-		if helpers.UTF16ToString(prop.Key.Data.(*js_ast.EString).Value) != "sections" {
+		if !helpers.UTF16EqualsString(prop.Key.Data.(*js_ast.EString).Value, "sections") {
 			continue
 		}
 
@@ -66,11 +66,17 @@ func ParseSourceMap(log logger.Log, source logger.Source) *sourcemap.SourceMap {
 										}
 									}
 								}
+							} else {
+								log.AddError(&tracker, logger.Range{Loc: sectionProp.ValueOrNil.Loc}, "Expected \"offset\" to be an object")
+								return nil
 							}
 
 						case "map":
 							if mapValue, ok := sectionProp.ValueOrNil.Data.(*js_ast.EObject); ok {
 								sectionSourceMap = mapValue
+							} else {
+								log.AddError(&tracker, logger.Range{Loc: sectionProp.ValueOrNil.Loc}, "Expected \"map\" to be an object")
+								return nil
 							}
 						}
 					}
@@ -85,7 +91,7 @@ func ParseSourceMap(log logger.Log, source logger.Source) *sourcemap.SourceMap {
 				}
 			}
 		} else {
-			log.AddError(&tracker, logger.Range{Loc: prop.Key.Loc}, "Invalid \"sections\"")
+			log.AddError(&tracker, logger.Range{Loc: prop.ValueOrNil.Loc}, "Expected \"sections\" to be an array")
 			return nil
 		}
 
@@ -105,10 +111,6 @@ func ParseSourceMap(log logger.Log, source logger.Source) *sourcemap.SourceMap {
 	var mappings mappingArray
 	var generatedLine int32
 	var generatedColumn int32
-	var sourceIndex int32
-	var originalLine int32
-	var originalColumn int32
-	var originalName int32
 	needSort := false
 
 	for _, section := range sections {
@@ -180,10 +182,10 @@ func ParseSourceMap(log logger.Log, source logger.Source) *sourcemap.SourceMap {
 
 		generatedLine = lineOffset
 		generatedColumn = columnOffset
-		sourceIndex = sourceOffset
-		originalLine = 0
-		originalColumn = 0
-		originalName = nameOffset
+		sourceIndex := sourceOffset
+		var originalLine int32
+		var originalColumn int32
+		originalName := nameOffset
 
 		current := 0
 		errorText := ""
@@ -362,9 +364,25 @@ func ParseSourceMap(log logger.Log, source logger.Source) *sourcemap.SourceMap {
 		}
 
 		if len(sourcesContentArray) > 0 {
+			// It's possible that one of the source maps inside "sections" has
+			// different lengths for the "sources" and "sourcesContent" arrays.
+			// This is bad because we need to us a single index to get the name
+			// of the source from "sources[i]" and the content of the source
+			// from "sourcesContent[i]".
+			//
+			// So if a previous source map had a shorter "sourcesContent" array
+			// than its "sources" array (or if the previous source map just had
+			// no "sourcesContent" array), expand our aggregated array to the
+			// right length by padding it out with empty entries.
 			sourcesContent = append(sourcesContent, make([]sourcemap.SourceContent, int(sourceOffset)-len(sourcesContent))...)
 
 			for i, item := range sourcesContentArray {
+				// Make sure we don't ever record more "sourcesContent" entries
+				// than there are "sources" entries, which is possible because
+				// these are two separate arrays in the source map JSON. We need
+				// to avoid this because that would mess up our shared indexing
+				// of the "sources" and "sourcesContent" arrays. See the above
+				// comment for more details.
 				if i == sourcesLen {
 					break
 				}

scripts/verify-source-map.js

@@ -496,6 +496,69 @@ const toSearchAbsoluteSourcesURL = {
   test: 'input.js',
 }
 
+// This test case was generated using the "shadow-cljs" tool by someone who has
+// no idea how to write Clojure code (i.e. me). See the following GitHub issue
+// for more details: https://github.com/evanw/esbuild/issues/3439
+//
+// Note that the mappings in the Clojure output strangely seem to be really
+// buggy. Many sub-expressions with two operands map the operands switched,
+// strings are way off, and there's even one mapping that's floating off in
+// space past the end of the line. This appears to just be bad output from the
+// Clojure tooling itself though, and not a problem with esbuild.
+//
+// For the example code below, I manually edited the mapping for the "done"
+// string to line up correctly so that this test can pass (it was off by
+// five lines).
+const testCaseIndexSourceMap = {
+  'entry.js': `
+    import './app.main.js'
+    console.log('testing')
+  `,
+  'app.main.js': `export const $APP = {};
+export const shadow$provide = {};
+export const $jscomp = {};
+/*
+
+ Copyright The Closure Library Authors.
+ SPDX-License-Identifier: Apache-2.0
+*/
+console.log(function $app$lib$log_many$$($G__6268$jscomp$1_i$jscomp$282$$, $collection$$) {
+  return $G__6268$jscomp$1_i$jscomp$282$$ < $collection$$.length ? (console.` +
+    `log($collection$$.at($G__6268$jscomp$1_i$jscomp$282$$)), $G__6268$jscom` +
+    `p$1_i$jscomp$282$$ += 1, $app$lib$log_many$$.$cljs$core$IFn$_invoke$ari` +
+    `ty$2$ ? $app$lib$log_many$$.$cljs$core$IFn$_invoke$arity$2$($G__6268$js` +
+    `comp$1_i$jscomp$282$$, $collection$$) : $app$lib$log_many$$.call(null, ` +
+    `$G__6268$jscomp$1_i$jscomp$282$$, $collection$$)) : "done";
+}(0, Object.keys(console)));
+export const render = {}.render;
+
+//# sourceMappingURL=app.main.js.map`,
+  'app.main.js.map': `{"version":3,"file":"app.main.js","sections":[{"offset` +
+    `":{"line":3,"column":0},"map":{"version":3,"file":"app.main.js","lineCo` +
+    `unt":10,"mappings":"A;;;;;AAGMA,OAAAA,CAAAA,GAAAA,CCDAC,QAAAA,oBAAAA,CA` +
+    `AUC,gCAAVD,EAAYE,aAAZF,CAAYE;AAAlB,SACSD,gCADT,GACWC,aAAUA,CAAAA,MADrB,` +
+    `IAGYH,OAAAA,CAAAA,GAAAA,CAAgBG,aAAAA,CAAAA,EAAAA,CAAWD,gCAAXC,CAAhBH,CA` +
+    `CN,EAAUE,gCAAV,IAAaA,CAAb,EAAAE,mBAAAC,CAAAA,+BAAA,GAAAD,mBAAAC,CAAAA,+` +
+    `BAAA,CAAAC,gCAAA,EAAkBH,aAAlB,CAAA,GAAAI,mBAAAA,CAAAA,IAAAA,CAAAA,IAAAA` +
+    `,EAAAD,gCAAAC,EAAkBJ,aAAlBI,CAJN,IAKI,MALJ;AAAkBJ,CDCD,CAACF,CAAD,EAAgB` +
+    `O,MAAOC,CAAAA,IAAP,CAAiBT,OAAjB,CAAhB,CAAXA,CAAAA;AEFN,sBFDkBU,EECaC,CA` +
+    `AAA,MAA/B;;","sources":["app/app.cljs","app/lib.cljs","shadow/module/ap` +
+    `p.main/append.js"],"sourcesContent":["(ns app.app\\n  (:require [app.li` +
+    `b :as lib]))\\n\\n(.log js/console (lib/log-many 0 (.keys js/Object js/` +
+    `console)))\\n","(ns app.lib)\\n\\n(defn log-many [i collection]\\n  (if` +
+    ` (< i (.-length collection))\\n    (do\\n      (.log js/console (.at co` +
+    `llection i))\\n      (log-many (+ i 1) collection))\\n    \\"done\\"))` +
+    `\\n","\\nshadow$export(\\"render\\",app.app.render);"],"names":["js/con` +
+    `sole","app.lib/log-many","i","collection","app.lib.log_manycljs$core$IF` +
+    `n$_invoke$arity$2","cljs$core$IFn$_invoke$arity$2","G__6268","G__6269",` +
+    `"Object","js/Object","app.apprender","render"],"x_google_ignoreList":[2` +
+    `]}}]}`,
+}
+
+const toSearchIndexSourceMap = {
+  'done': 'app/lib.cljs',
+}
+
 // This case covers a crash when esbuild would generate an invalid source map
 // containing a mapping with an index of a source that's out of bounds of the
 // "sources" array. This happened when generating the namespace exports chunk
@@ -1149,6 +1212,12 @@ async function main() {
           entryPoints: ['entry.js'],
           crlf,
         }),
+        check('indexed-source-map' + suffix, testCaseIndexSourceMap, toSearchIndexSourceMap, {
+          outfile: 'out.js',
+          flags: flags.concat('--bundle'),
+          entryPoints: ['entry.js'],
+          crlf,
+        }),
         check('issue-4070' + suffix, testCaseNestedFoldersIssue4070, toSearchNestedFoldersIssue4070, {
           outfile: 'out.js',
           flags: flags.concat('--bundle'),