cmd/compile/internal/types2: port CL 576975 to types2

This CL ports to types2 the (passing) test from CL 576975,
which fixed a bug in go/types.

Updates #66704
Updates #65294

Change-Id: Icdf77e39ed177d9f9ecc435d5125f02f2ee4dd0f
Reviewed-on: https://go-review.googlesource.com/c/go/+/579015
Auto-Submit: Alan Donovan <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Robert Findley <[email protected]>

Author: adonovan

src/cmd/compile/internal/types2/api_test.go

@@ -2985,3 +2985,28 @@ func TestTooNew(t *testing.T) {
 		}
 	}
 }
+
+// This is a regression test for #66704.
+func TestUnaliasTooSoonInCycle(t *testing.T) {
+	t.Setenv("GODEBUG", "gotypesalias=1")
+	const src = `package a
+
+var x T[B] // this appears to cause Unalias to be called on B while still Invalid
+
+type T[_ any] struct{}
+type A T[B]
+type B = T[A]
+`
+
+	f := mustParse(src)
+	pkg, err := new(Config).Check("a", []*syntax.File{f}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	B := pkg.Scope().Lookup("B")
+	got, want := Unalias(B.Type()).String(), "a.T[a.A]"
+	if got != want {
+		t.Errorf("Unalias(type B = T[A]) = %q, want %q", got, want)
+	}
+}

src/cmd/compile/internal/types2/decl.go

@@ -512,6 +512,11 @@ func (check *Checker) typeDecl(obj *TypeName, tdecl *syntax.TypeDecl, def *TypeN
 			// TODO(gri) Should be able to use nil instead of Typ[Invalid] to mark
 			//           the alias as incomplete. Currently this causes problems
 			//           with certain cycles. Investigate.
+			//
+			// NOTE(adonovan): to avoid the Invalid being prematurely observed
+			// by (e.g.) a var whose type is an unfinished cycle,
+			// Unalias does not memoize if Invalid. Perhaps we should use a
+			// special sentinel distinct from Invalid.
 			alias := check.newAlias(obj, Typ[Invalid])
 			setDefType(def, alias)