encoding/jsonschema: add location information to test output

It's useful to know exactly where in the test data a test is,
so log that information. Also print a txtar file to make it
easier to reproduce test failures.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: Ibef118835a429437825eb443e265066737c5fcb1
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1200518
TryBot-Result: CUEcueckoo <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>
Reviewed-by: Daniel Martí <[email protected]>

Author: rogpeppe

encoding/jsonschema/external_test.go

@@ -15,6 +15,7 @@
 package jsonschema_test
 
 import (
+	stdjson "encoding/json"
 	"fmt"
 	"os"
 	"path"
@@ -28,6 +29,7 @@ import (
 	"cuelang.org/go/cue/cuecontext"
 	"cuelang.org/go/cue/errors"
 	"cuelang.org/go/cue/format"
+	"cuelang.org/go/cue/token"
 	"cuelang.org/go/encoding/json"
 	"cuelang.org/go/encoding/jsonschema"
 	"cuelang.org/go/encoding/jsonschema/internal/externaltest"
@@ -100,23 +102,32 @@ func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schem
 		qt.Assert(t, qt.IsNil(err))
 		schemaValue = ctx.CompileBytes(b, cue.Filename("generated.cue"))
 		if err := schemaValue.Err(); err != nil {
+			t.Logf("extracted schema: %q", b)
 			extractErr = fmt.Errorf("cannot compile resulting schema: %v", errors.Details(err, nil))
 		}
 	}
 
 	if extractErr != nil {
+		t.Logf("location: %v", testdataPos(s))
+		t.Logf("txtar:\n%s", schemaFailureTxtar(s))
 		for _, test := range s.Tests {
 			t.Run("", func(t *testing.T) {
-				testFailed(t, &test.Skip, "could not compile schema")
+				testFailed(t, &test.Skip, test, "could not compile schema")
 			})
 		}
-		testFailed(t, &s.Skip, fmt.Sprintf("extract error: %v", extractErr))
+		testFailed(t, &s.Skip, s, fmt.Sprintf("extract error: %v", extractErr))
 		return
 	}
-	testSucceeded(t, &s.Skip)
+	testSucceeded(t, &s.Skip, s)
 
 	for _, test := range s.Tests {
 		t.Run(testName(test.Description), func(t *testing.T) {
+			defer func() {
+				if t.Failed() || testing.Verbose() {
+					t.Logf("txtar:\n%s", testCaseTxtar(s, test))
+				}
+			}()
+			t.Logf("location: %v", testdataPos(test))
 			instAST, err := json.Extract("instance.json", test.Data)
 			if err != nil {
 				t.Fatal(err)
@@ -129,21 +140,60 @@ func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schem
 			err = instValue.Unify(schemaValue).Err()
 			if test.Valid {
 				if err != nil {
-					testFailed(t, &test.Skip, errors.Details(err, nil))
+					testFailed(t, &test.Skip, test, errors.Details(err, nil))
 				} else {
-					testSucceeded(t, &test.Skip)
+					testSucceeded(t, &test.Skip, test)
 				}
 			} else {
 				if err == nil {
-					testFailed(t, &test.Skip, "unexpected success")
+					testFailed(t, &test.Skip, test, "unexpected success")
 				} else {
-					testSucceeded(t, &test.Skip)
+					testSucceeded(t, &test.Skip, test)
 				}
 			}
 		})
 	}
 }
 
+// testCaseTxtar returns a testscript that runs the given test.
+func testCaseTxtar(s *externaltest.Schema, test *externaltest.Test) string {
+	var buf strings.Builder
+	fmt.Fprintf(&buf, "env CUE_EXPERIMENT=evalv3\n")
+	fmt.Fprintf(&buf, "exec cue def json+jsonschema: schema.json\n")
+	if !test.Valid {
+		buf.WriteString("! ")
+	}
+	// TODO add $schema when one isn't already present?
+	fmt.Fprintf(&buf, "exec cue vet -c instance.json json+jsonschema: schema.json\n")
+	fmt.Fprintf(&buf, "\n")
+	fmt.Fprintf(&buf, "-- schema.json --\n%s\n", indentJSON(s.Schema))
+	fmt.Fprintf(&buf, "-- instance.json --\n%s\n", indentJSON(test.Data))
+	return buf.String()
+}
+
+// testCaseTxtar returns a testscript that decodes the given schema.
+func schemaFailureTxtar(s *externaltest.Schema) string {
+	var buf strings.Builder
+	fmt.Fprintf(&buf, "env CUE_EXPERIMENT=evalv3\n")
+	fmt.Fprintf(&buf, "exec cue def -o schema.cue json+jsonschema: schema.json\n")
+	fmt.Fprintf(&buf, "exec cat schema.cue\n")
+	fmt.Fprintf(&buf, "exec cue vet schema.cue\n")
+	fmt.Fprintf(&buf, "-- schema.json --\n%s\n", indentJSON(s.Schema))
+	return buf.String()
+}
+
+func indentJSON(x stdjson.RawMessage) []byte {
+	data, err := stdjson.MarshalIndent(x, "", "\t")
+	if err != nil {
+		panic(err)
+	}
+	return data
+}
+
+type positioner interface {
+	Pos() token.Pos
+}
+
 // testName returns a test name that doesn't contain any
 // slashes because slashes muck with matching.
 func testName(s string) string {
@@ -153,7 +203,7 @@ func testName(s string) string {
 // testFailed marks the current test as failed with the
 // given error message, and updates the
 // skip field pointed to by skipField if necessary.
-func testFailed(t *testing.T, skipField *string, errStr string) {
+func testFailed(t *testing.T, skipField *string, p positioner, errStr string) {
 	if cuetest.UpdateGoldenFiles {
 		if *skipField == "" && !allowRegressions() {
 			t.Fatalf("test regression; was succeeding, now failing: %v", errStr)
@@ -169,7 +219,7 @@ func testFailed(t *testing.T, skipField *string, errStr string) {
 
 // testFails marks the current test as succeeded and updates the
 // skip field pointed to by skipField if necessary.
-func testSucceeded(t *testing.T, skipField *string) {
+func testSucceeded(t *testing.T, skipField *string, p positioner) {
 	if cuetest.UpdateGoldenFiles {
 		*skipField = ""
 		return
@@ -179,6 +229,12 @@ func testSucceeded(t *testing.T, skipField *string) {
 	}
 }
 
+func testdataPos(p positioner) token.Position {
+	pp := p.Pos().Position()
+	pp.Filename = path.Join(testDir, pp.Filename)
+	return pp
+}
+
 func allowRegressions() bool {
 	return os.Getenv("CUE_ALLOW_REGRESSIONS") != ""
 }

encoding/jsonschema/internal/externaltest/tests.go

@@ -12,9 +12,11 @@ import (
 	"cuelang.org/go/cue/cuecontext"
 	"cuelang.org/go/cue/interpreter/embed"
 	"cuelang.org/go/cue/load"
+	"cuelang.org/go/cue/token"
 )
 
 type Schema struct {
+	location
 	Description string             `json:"description"`
 	Comment     string             `json:"comment,omitempty"`
 	Schema      stdjson.RawMessage `json:"schema"`
@@ -23,13 +25,23 @@ type Schema struct {
 }
 
 type Test struct {
+	location
 	Description string             `json:"description"`
 	Comment     string             `json:"comment,omitempty"`
 	Data        stdjson.RawMessage `json:"data"`
 	Valid       bool               `json:"valid"`
 	Skip        string             `json:"skip,omitempty"`
 }
 
+type location struct {
+	root cue.Value
+	path cue.Path
+}
+
+func (loc location) Pos() token.Pos {
+	return loc.root.LookupPath(loc.path).Pos()
+}
+
 func ParseTestData(data []byte) ([]*Schema, error) {
 	var schemas []*Schema
 	if err := json.Unmarshal(data, &schemas); err != nil {
@@ -81,7 +93,7 @@ func ReadTestDir(dir string) (tests map[string][]*Schema, err error) {
 	inst := load.Instances([]string{"."}, &load.Config{
 		Dir: dir,
 	})[0]
-	if err != nil {
+	if err := inst.Err; err != nil {
 		return nil, err
 	}
 	ctx := cuecontext.New(cuecontext.Interpreter(embed.New()))
@@ -97,9 +109,17 @@ func ReadTestDir(dir string) (tests map[string][]*Schema, err error) {
 		return nil, err
 	}
 	// Fix up the raw JSON data to avoid running into some decode issues.
-	for _, schemas := range tests {
-		for _, schema := range schemas {
-			for _, test := range schema.Tests {
+	for filename, schemas := range tests {
+		for i, schema := range schemas {
+			schema.location = location{
+				root: val,
+				path: cue.MakePath(cue.Str(filename), cue.Index(i)),
+			}
+			for j, test := range schema.Tests {
+				test.location = location{
+					root: val,
+					path: cue.MakePath(cue.Str(filename), cue.Index(i), cue.Str("tests"), cue.Index(j)),
+				}
 				if len(test.Data) == 0 {
 					// See https://github.com/cue-lang/cue/issues/3397
 					test.Data = []byte("null")