internal/astinternal: make it easier to debug references

The `astutil.Sanitize` logic tries to respect `Ident.Node` references,
but when debugging generated syntax, it's hard to be sure whether the
Node references are pointing to the correct places.

This adds functionality to `astinternal.AppendDebug` to cause it to
print references and their referred-to nodes in a somewhat friendly
manner.

Here's a (slightly abbreviated) example of its output from a recent
debugging session:

	Value: *ast.StructLit@ref001{
		Elts: []ast.Decl{
			*ast.Field{
				Label: *ast.Ident{
					Name: "next"
				}
				Value: *ast.Ident{
					Name: "_schema"
					Node: @ref001 (*ast.StructLit)
				}
			}
		}
	}

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: I2f859f3b8d365c6ba3408902039f2aa9cbd8a679
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1205069
Reviewed-by: Daniel Martí <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>

Author: rogpeppe

Diff for: internal/astinternal/debug.go

@@ -33,6 +33,10 @@ func AppendDebug(dst []byte, node ast.Node, config DebugConfig) []byte {
 		cfg: config,
 		buf: dst,
 	}
+	if config.IncludeNodeRefs {
+		d.nodeRefs = make(map[ast.Node]int)
+		d.addNodeRefs(reflect.ValueOf(node))
+	}
 	if d.value(reflect.ValueOf(node), nil) {
 		d.newline()
 	}
@@ -48,12 +52,18 @@ type DebugConfig struct {
 	// OmitEmpty causes empty strings, empty structs, empty lists,
 	// nil pointers, invalid positions, and missing tokens to be omitted.
 	OmitEmpty bool
+
+	// IncludeNodeRefs causes a Node reference in an identifier
+	// to indicate which (if any) ast.Node it refers to.
+	IncludeNodeRefs bool
 }
 
 type debugPrinter struct {
-	buf   []byte
-	cfg   DebugConfig
-	level int
+	buf      []byte
+	cfg      DebugConfig
+	level    int
+	nodeRefs map[ast.Node]int
+	refID    int
 }
 
 // value produces the given value, omitting type information if
@@ -71,6 +81,7 @@ func (d *debugPrinter) value0(v reflect.Value, impliedType reflect.Type) {
 	}
 	// Skip over interfaces and pointers, stopping early if nil.
 	concreteType := v.Type()
+	refName := ""
 	for {
 		k := v.Kind()
 		if k != reflect.Interface && k != reflect.Pointer {
@@ -82,6 +93,13 @@ func (d *debugPrinter) value0(v reflect.Value, impliedType reflect.Type) {
 			}
 			return
 		}
+		if k == reflect.Pointer {
+			if n, ok := v.Interface().(ast.Node); ok {
+				if id, ok := d.nodeRefs[n]; ok {
+					refName = refIDToName(id)
+				}
+			}
+		}
 		v = v.Elem()
 		if k == reflect.Interface {
 			// For example, *ast.Ident can be the concrete type behind an ast.Expr.
@@ -125,6 +143,9 @@ func (d *debugPrinter) value0(v reflect.Value, impliedType reflect.Type) {
 		if concreteType != impliedType {
 			d.printf("%s", concreteType)
 		}
+		if refName != "" {
+			d.printf("@%s", refName)
+		}
 		d.printf("{")
 		d.level++
 		var anyElems bool
@@ -169,9 +190,18 @@ func (d *debugPrinter) structFields(v reflect.Value) (anyElems bool) {
 		if !gotoken.IsExported(f.Name) {
 			continue
 		}
+		if f.Name == "Node" {
+			nodeVal := v.Field(i)
+			if !d.cfg.IncludeNodeRefs || nodeVal.IsNil() {
+				continue
+			}
+			d.newline()
+			d.printf("Node: @%s (%v)", refIDToName(d.nodeRefs[nodeVal.Interface().(ast.Node)]), nodeVal.Elem().Type())
+			continue
+		}
 		switch f.Name {
 		// These fields are cyclic, and they don't represent the syntax anyway.
-		case "Scope", "Node", "Unresolved":
+		case "Scope", "Unresolved":
 			continue
 		}
 		elemStart := d.pos()
@@ -213,6 +243,62 @@ func (d *debugPrinter) truncate(pos int) {
 	d.buf = d.buf[:pos]
 }
 
+// addNodeRefs does a first pass over the value looking for
+// [ast.Ident] nodes that refer to other nodes.
+// This means when we find such a node, we can include
+// an anchor name for it
+func (d *debugPrinter) addNodeRefs(v reflect.Value) {
+	// Skip over interfaces and pointers, stopping early if nil.
+	for ; v.Kind() == reflect.Interface || v.Kind() == reflect.Pointer; v = v.Elem() {
+		if v.IsNil() {
+			return
+		}
+	}
+
+	t := v.Type()
+	switch v := v.Interface().(type) {
+	case token.Pos, token.Token:
+		// Simple types which can't contain an ast.Node.
+		return
+	case ast.Ident:
+		if v.Node != nil {
+			if _, ok := d.nodeRefs[v.Node]; !ok {
+				d.refID++
+				d.nodeRefs[v.Node] = d.refID
+			}
+		}
+		return
+	}
+
+	switch t.Kind() {
+	case reflect.Slice:
+		for i := 0; i < v.Len(); i++ {
+			d.addNodeRefs(v.Index(i))
+		}
+	case reflect.Struct:
+		t := v.Type()
+		for i := 0; i < v.NumField(); i++ {
+			f := t.Field(i)
+			if !gotoken.IsExported(f.Name) {
+				continue
+			}
+			switch f.Name {
+			// These fields don't point to any nodes that Node can refer to.
+			case "Scope", "Node", "Unresolved":
+				continue
+			}
+			d.addNodeRefs(v.Field(i))
+		}
+	}
+}
+
+func refIDToName(id int) string {
+	if id == 0 {
+		return "unknown"
+	}
+	return fmt.Sprintf("ref%03d", id)
+}
+
 func DebugStr(x interface{}) (out string) {
 	if n, ok := x.(ast.Node); ok {
 		comments := ""

Diff for: internal/astinternal/debug_test.go

@@ -43,7 +43,13 @@ func TestDebugPrint(t *testing.T) {
 			qt.Assert(t, qt.IsNil(err))
 
 			// The full syntax tree, as printed by default.
-			full := astinternal.AppendDebug(nil, f, astinternal.DebugConfig{})
+			// We enable IncludeNodeRefs because it only adds information
+			// that would not otherwise be present.
+			// The syntax tree does not contain any maps, so
+			// the generated reference names should be deterministic.
+			full := astinternal.AppendDebug(nil, f, astinternal.DebugConfig{
+				IncludeNodeRefs: true,
+			})
 			t.Writer(file.Name).Write(full)
 
 			// A syntax tree which omits any empty values,

Diff for: internal/astinternal/testdata/debugprint/comprehensions.txtar

@@ -46,12 +46,12 @@ for k, v in input if v > 2 {
 			Clauses: []ast.Clause{
 				*ast.ForClause{
 					For: token.Pos("comprehensions.cue:4:1", newline)
-					Key: *ast.Ident{
+					Key: *ast.Ident@ref002{
 						NamePos: token.Pos("comprehensions.cue:4:5", blank)
 						Name: "k"
 					}
 					Colon: token.Pos("comprehensions.cue:4:6", nospace)
-					Value: *ast.Ident{
+					Value: *ast.Ident@ref001{
 						NamePos: token.Pos("comprehensions.cue:4:8", blank)
 						Name: "v"
 					}
@@ -67,6 +67,7 @@ for k, v in input if v > 2 {
 						X: *ast.Ident{
 							NamePos: token.Pos("comprehensions.cue:4:22", blank)
 							Name: "v"
+							Node: @ref001 (*ast.Ident)
 						}
 						OpPos: token.Pos("comprehensions.cue:4:24", blank)
 						Op: token.Token(">")
@@ -87,6 +88,7 @@ for k, v in input if v > 2 {
 							X: *ast.Ident{
 								NamePos: token.Pos("comprehensions.cue:5:3", nospace)
 								Name: "k"
+								Node: @ref002 (*ast.Ident)
 							}
 							Rparen: token.Pos("comprehensions.cue:5:4", nospace)
 						}
@@ -97,6 +99,7 @@ for k, v in input if v > 2 {
 						Value: *ast.Ident{
 							NamePos: token.Pos("comprehensions.cue:5:7", blank)
 							Name: "v"
+							Node: @ref001 (*ast.Ident)
 						}
 						Attrs: []*ast.Attribute{}
 					}

Diff for: internal/astinternal/testdata/debugprint/fields.txtar

@@ -153,7 +153,7 @@ embed: {
 			Constraint: token.Token("ILLEGAL")
 			TokenPos: token.Pos("fields.cue:5:8", nospace)
 			Token: token.Token(":")
-			Value: *ast.StructLit{
+			Value: *ast.StructLit@ref001{
 				Lbrace: token.Pos("fields.cue:5:10", blank)
 				Elts: []ast.Decl{
 					*ast.Field{
@@ -196,6 +196,7 @@ embed: {
 						Expr: *ast.Ident{
 							NamePos: token.Pos("fields.cue:10:2", newline)
 							Name: "#Schema"
+							Node: @ref001 (*ast.StructLit)
 						}
 					}
 				}