encoding/jsonschema: prepare for matrix test runs

Currently we are just running the external tests on the
old evaluator, but we want confidence in what's happening
in both evaluators.

In order to do so, we will record skip reasons for both
old and new evaluators, so prepare for that by making
the `skip` field into a struct and moving all the
existing skip fields into a field inside that, corresponding
to the point in the matrix that is being run currently.

In the next CL, we will actually run the matrix tests
and populate the `skip.v3` field.

Most of this CL has been accomplished with an automated script.

Also, in passing, remove the `externaltest.ParseTestData`
function as it's not used.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: I6f59c6eb837a7c15eb7467f6860167534b6d1e75
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1200519
TryBot-Result: CUEcueckoo <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>
Reviewed-by: Daniel Martí <[email protected]>

Author: rogpeppe

encoding/jsonschema/external_test.go

@@ -203,28 +203,28 @@ func testName(s string) string {
 // testFailed marks the current test as failed with the
 // given error message, and updates the
 // skip field pointed to by skipField if necessary.
-func testFailed(t *testing.T, skipField *string, p positioner, errStr string) {
+func testFailed(t *testing.T, skipField *externaltest.Skip, p positioner, errStr string) {
 	if cuetest.UpdateGoldenFiles {
-		if *skipField == "" && !allowRegressions() {
+		if *skipField == nil && !allowRegressions() {
 			t.Fatalf("test regression; was succeeding, now failing: %v", errStr)
 		}
-		*skipField = errStr
+		*skipField = externaltest.Skip{"v2": errStr}
 		return
 	}
-	if *skipField != "" {
+	if *skipField != nil {
 		t.Skipf("skipping due to known error: %v", *skipField)
 	}
 	t.Fatal(errStr)
 }
 
 // testFails marks the current test as succeeded and updates the
 // skip field pointed to by skipField if necessary.
-func testSucceeded(t *testing.T, skipField *string, p positioner) {
+func testSucceeded(t *testing.T, skipField *externaltest.Skip, p positioner) {
 	if cuetest.UpdateGoldenFiles {
-		*skipField = ""
+		*skipField = nil
 		return
 	}
-	if *skipField != "" {
+	if *skipField != nil {
 		t.Fatalf("unexpectedly more correct behavior (test success) on skipped test")
 	}
 }

encoding/jsonschema/internal/externaltest/tests.go

@@ -2,7 +2,6 @@ package externaltest
 
 import (
 	"bytes"
-	"encoding/json"
 	stdjson "encoding/json"
 	"fmt"
 	"os"
@@ -20,7 +19,7 @@ type Schema struct {
 	Description string             `json:"description"`
 	Comment     string             `json:"comment,omitempty"`
 	Schema      stdjson.RawMessage `json:"schema"`
-	Skip        string             `json:"skip,omitempty"`
+	Skip        Skip               `json:"skip,omitempty"`
 	Tests       []*Test            `json:"tests"`
 }
 
@@ -30,9 +29,17 @@ type Test struct {
 	Comment     string             `json:"comment,omitempty"`
 	Data        stdjson.RawMessage `json:"data"`
 	Valid       bool               `json:"valid"`
-	Skip        string             `json:"skip,omitempty"`
+	Skip        Skip               `json:"skip,omitempty"`
 }
 
+// Skip records information about whether a given schema
+// or test will be skipped when testing. If not present,
+// the test will be expected to pass.
+//
+// Each key in the map represents the name of a point
+// in the cuetdtest matrix.
+type Skip map[string]string
+
 type location struct {
 	root cue.Value
 	path cue.Path
@@ -42,14 +49,6 @@ func (loc location) Pos() token.Pos {
 	return loc.root.LookupPath(loc.path).Pos()
 }
 
-func ParseTestData(data []byte) ([]*Schema, error) {
-	var schemas []*Schema
-	if err := json.Unmarshal(data, &schemas); err != nil {
-		return nil, err
-	}
-	return schemas, nil
-}
-
 // WriteTestDir writes test data files as read by ReadTestDir
 // to the given directory. The keys of tests are filenames relative
 // to dir.

encoding/jsonschema/testdata/external/config.cue

@@ -19,9 +19,8 @@ allTests: [_]: [... #Schema]
 	// skip is not part of the orginal test data, but
 	// is inserted by our test logic (when CUE_UPDATE=1)
 	// to indicate which tests are passing and which
-	// are failing. The text indicates some reason as to
-	// why the schema is skipped.
-	skip?: string
+	// are failing. See #Skip for details.
+	skip?: #Skip
 }
 
 #Test: {
@@ -35,5 +34,15 @@ allTests: [_]: [... #Schema]
 	// to indicate which tests are passing and which
 	// are failing. The text indicates some reason as to
 	// why the test is skipped.
-	skip?: string
+	skip?: #Skip
+}
+
+// #Skip indicates that a test or schema should be skipped
+// in the external test. The text indicates some reason as to
+// why the schema is skipped for a given evaluator version.
+//
+// If all fields are empty, the skip field itself should be omitted.
+#Skip: {
+	v2?: string
+	v3?: string
 }

encoding/jsonschema/testdata/external/tests/draft2019-09/additionalItems.json

@@ -102,15 +102,19 @@
 				"description": "empty array",
 				"data": [],
 				"valid": true,
-				"skip": "5 errors in empty disjunction:\nconflicting values [] and {...} (mismatched types list and struct):\n    generated.cue:2:1\n    generated.cue:2:45\n    instance.json:1:1\nconflicting values bool and [] (mismatched types bool and list):\n    generated.cue:2:8\n    instance.json:1:1\nconflicting values null and [] (mismatched types null and list):\n    generated.cue:2:1\n    instance.json:1:1\nconflicting values number and [] (mismatched types number and list):\n    generated.cue:2:15\n    instance.json:1:1\nconflicting values string and [] (mismatched types string and list):\n    generated.cue:2:24\n    instance.json:1:1\n"
+				"skip": {
+					"v2": "5 errors in empty disjunction:\nconflicting values [] and {...} (mismatched types list and struct):\n    generated.cue:2:1\n    generated.cue:2:45\n    instance.json:1:1\nconflicting values bool and [] (mismatched types bool and list):\n    generated.cue:2:8\n    instance.json:1:1\nconflicting values null and [] (mismatched types null and list):\n    generated.cue:2:1\n    instance.json:1:1\nconflicting values number and [] (mismatched types number and list):\n    generated.cue:2:15\n    instance.json:1:1\nconflicting values string and [] (mismatched types string and list):\n    generated.cue:2:24\n    instance.json:1:1\n"
+				}
 			},
 			{
 				"description": "fewer number of items present (1)",
 				"data": [
 					1
 				],
 				"valid": true,
-				"skip": "5 errors in empty disjunction:\nconflicting values [1] and {...} (mismatched types list and struct):\n    generated.cue:2:1\n    generated.cue:2:45\n    instance.json:1:1\nconflicting values bool and [1] (mismatched types bool and list):\n    generated.cue:2:8\n    instance.json:1:1\nconflicting values null and [1] (mismatched types null and list):\n    generated.cue:2:1\n    instance.json:1:1\nconflicting values number and [1] (mismatched types number and list):\n    generated.cue:2:15\n    instance.json:1:1\nconflicting values string and [1] (mismatched types string and list):\n    generated.cue:2:24\n    instance.json:1:1\n"
+				"skip": {
+					"v2": "5 errors in empty disjunction:\nconflicting values [1] and {...} (mismatched types list and struct):\n    generated.cue:2:1\n    generated.cue:2:45\n    instance.json:1:1\nconflicting values bool and [1] (mismatched types bool and list):\n    generated.cue:2:8\n    instance.json:1:1\nconflicting values null and [1] (mismatched types null and list):\n    generated.cue:2:1\n    instance.json:1:1\nconflicting values number and [1] (mismatched types number and list):\n    generated.cue:2:15\n    instance.json:1:1\nconflicting values string and [1] (mismatched types string and list):\n    generated.cue:2:24\n    instance.json:1:1\n"
+				}
 			},
 			{
 				"description": "fewer number of items present (2)",
@@ -119,7 +123,9 @@
 					2
 				],
 				"valid": true,
-				"skip": "5 errors in empty disjunction:\nconflicting values [1,2] and {...} (mismatched types list and struct):\n    generated.cue:2:1\n    generated.cue:2:45\n    instance.json:1:1\nconflicting values bool and [1,2] (mismatched types bool and list):\n    generated.cue:2:8\n    instance.json:1:1\nconflicting values null and [1,2] (mismatched types null and list):\n    generated.cue:2:1\n    instance.json:1:1\nconflicting values number and [1,2] (mismatched types number and list):\n    generated.cue:2:15\n    instance.json:1:1\nconflicting values string and [1,2] (mismatched types string and list):\n    generated.cue:2:24\n    instance.json:1:1\n"
+				"skip": {
+					"v2": "5 errors in empty disjunction:\nconflicting values [1,2] and {...} (mismatched types list and struct):\n    generated.cue:2:1\n    generated.cue:2:45\n    instance.json:1:1\nconflicting values bool and [1,2] (mismatched types bool and list):\n    generated.cue:2:8\n    instance.json:1:1\nconflicting values null and [1,2] (mismatched types null and list):\n    generated.cue:2:1\n    instance.json:1:1\nconflicting values number and [1,2] (mismatched types number and list):\n    generated.cue:2:15\n    instance.json:1:1\nconflicting values string and [1,2] (mismatched types string and list):\n    generated.cue:2:24\n    instance.json:1:1\n"
+				}
 			},
 			{
 				"description": "equal number of items present",
@@ -188,7 +194,9 @@
 					false
 				],
 				"valid": true,
-				"skip": "5 errors in empty disjunction:\nconflicting values [1,\"foo\",false] and {...} (mismatched types list and struct):\n    generated.cue:2:1\n    generated.cue:2:41\n    instance.json:1:1\nconflicting values bool and [1,\"foo\",false] (mismatched types bool and list):\n    generated.cue:2:8\n    instance.json:1:1\nconflicting values null and [1,\"foo\",false] (mismatched types null and list):\n    generated.cue:2:1\n    instance.json:1:1\nconflicting values number and [1,\"foo\",false] (mismatched types number and list):\n    generated.cue:2:15\n    instance.json:1:1\nconflicting values string and [1,\"foo\",false] (mismatched types string and list):\n    generated.cue:2:24\n    instance.json:1:1\n"
+				"skip": {
+					"v2": "5 errors in empty disjunction:\nconflicting values [1,\"foo\",false] and {...} (mismatched types list and struct):\n    generated.cue:2:1\n    generated.cue:2:41\n    instance.json:1:1\nconflicting values bool and [1,\"foo\",false] (mismatched types bool and list):\n    generated.cue:2:8\n    instance.json:1:1\nconflicting values null and [1,\"foo\",false] (mismatched types null and list):\n    generated.cue:2:1\n    instance.json:1:1\nconflicting values number and [1,\"foo\",false] (mismatched types number and list):\n    generated.cue:2:15\n    instance.json:1:1\nconflicting values string and [1,\"foo\",false] (mismatched types string and list):\n    generated.cue:2:24\n    instance.json:1:1\n"
+				}
 			}
 		]
 	},
@@ -217,7 +225,9 @@
 					null
 				],
 				"valid": true,
-				"skip": "5 errors in empty disjunction:\nconflicting values [1,null] and {...} (mismatched types list and struct):\n    generated.cue:2:1\n    generated.cue:2:41\n    instance.json:1:1\nconflicting values bool and [1,null] (mismatched types bool and list):\n    generated.cue:2:8\n    instance.json:1:1\nconflicting values null and [1,null] (mismatched types null and list):\n    generated.cue:2:1\n    instance.json:1:1\nconflicting values number and [1,null] (mismatched types number and list):\n    generated.cue:2:15\n    instance.json:1:1\nconflicting values string and [1,null] (mismatched types string and list):\n    generated.cue:2:24\n    instance.json:1:1\n"
+				"skip": {
+					"v2": "5 errors in empty disjunction:\nconflicting values [1,null] and {...} (mismatched types list and struct):\n    generated.cue:2:1\n    generated.cue:2:41\n    instance.json:1:1\nconflicting values bool and [1,null] (mismatched types bool and list):\n    generated.cue:2:8\n    instance.json:1:1\nconflicting values null and [1,null] (mismatched types null and list):\n    generated.cue:2:1\n    instance.json:1:1\nconflicting values number and [1,null] (mismatched types number and list):\n    generated.cue:2:15\n    instance.json:1:1\nconflicting values string and [1,null] (mismatched types string and list):\n    generated.cue:2:24\n    instance.json:1:1\n"
+				}
 			}
 		]
 	},

encoding/jsonschema/testdata/external/tests/draft2019-09/additionalProperties.json

@@ -28,7 +28,9 @@
 					"quux": "boom"
 				},
 				"valid": false,
-				"skip": "unexpected success"
+				"skip": {
+					"v2": "unexpected success"
+				}
 			},
 			{
 				"description": "ignores arrays",
@@ -82,7 +84,9 @@
 					"élmény": 2
 				},
 				"valid": false,
-				"skip": "unexpected success"
+				"skip": {
+					"v2": "unexpected success"
+				}
 			}
 		]
 	},
@@ -227,15 +231,19 @@
 				"type": "number"
 			}
 		},
-		"skip": "extract error: cannot compile resulting schema: reference \"strings\" in label expression refers to field against which it would be matched:\n    generated.cue:5:3\n",
+		"skip": {
+			"v2": "extract error: cannot compile resulting schema: reference \"strings\" in label expression refers to field against which it would be matched:\n    generated.cue:5:3\n"
+		},
 		"tests": [
 			{
 				"description": "Valid against both keywords",
 				"data": {
 					"apple": 4
 				},
 				"valid": true,
-				"skip": "could not compile schema"
+				"skip": {
+					"v2": "could not compile schema"
+				}
 			},
 			{
 				"description": "Valid against propertyNames, but not additionalProperties",
@@ -244,7 +252,9 @@
 					"pear": "available"
 				},
 				"valid": false,
-				"skip": "could not compile schema"
+				"skip": {
+					"v2": "could not compile schema"
+				}
 			}
 		]
 	},
@@ -265,23 +275,29 @@
 			},
 			"additionalProperties": false
 		},
-		"skip": "extract error: unsupported constraint \"dependentSchemas\"",
+		"skip": {
+			"v2": "extract error: unsupported constraint \"dependentSchemas\""
+		},
 		"tests": [
 			{
 				"description": "additionalProperties doesn't consider dependentSchemas",
 				"data": {
 					"foo": ""
 				},
 				"valid": false,
-				"skip": "could not compile schema"
+				"skip": {
+					"v2": "could not compile schema"
+				}
 			},
 			{
 				"description": "additionalProperties can't see bar",
 				"data": {
 					"bar": ""
 				},
 				"valid": false,
-				"skip": "could not compile schema"
+				"skip": {
+					"v2": "could not compile schema"
+				}
 			},
 			{
 				"description": "additionalProperties can't see bar even when foo2 is present",
@@ -290,7 +306,9 @@
 					"bar": ""
 				},
 				"valid": false,
-				"skip": "could not compile schema"
+				"skip": {
+					"v2": "could not compile schema"
+				}
 			}
 		]
 	}

encoding/jsonschema/testdata/external/tests/draft2019-09/allOf.json

@@ -41,15 +41,19 @@
 					"foo": "baz"
 				},
 				"valid": false,
-				"skip": "unexpected success"
+				"skip": {
+					"v2": "unexpected success"
+				}
 			},
 			{
 				"description": "mismatch first",
 				"data": {
 					"bar": 2
 				},
 				"valid": false,
-				"skip": "unexpected success"
+				"skip": {
+					"v2": "unexpected success"
+				}
 			},
 			{
 				"description": "wrong type",
@@ -113,7 +117,9 @@
 					"baz": null
 				},
 				"valid": false,
-				"skip": "unexpected success"
+				"skip": {
+					"v2": "unexpected success"
+				}
 			},
 			{
 				"description": "mismatch first allOf",
@@ -122,7 +128,9 @@
 					"baz": null
 				},
 				"valid": false,
-				"skip": "unexpected success"
+				"skip": {
+					"v2": "unexpected success"
+				}
 			},
 			{
 				"description": "mismatch second allOf",
@@ -131,15 +139,19 @@
 					"bar": 2
 				},
 				"valid": false,
-				"skip": "unexpected success"
+				"skip": {
+					"v2": "unexpected success"
+				}
 			},
 			{
 				"description": "mismatch both",
 				"data": {
 					"bar": 2
 				},
 				"valid": false,
-				"skip": "unexpected success"
+				"skip": {
+					"v2": "unexpected success"
+				}
 			}
 		]
 	},
@@ -200,7 +212,9 @@
 				"description": "any value is invalid",
 				"data": "foo",
 				"valid": false,
-				"skip": "unexpected success"
+				"skip": {
+					"v2": "unexpected success"
+				}
 			}
 		]
 	},
@@ -218,7 +232,9 @@
 				"description": "any value is invalid",
 				"data": "foo",
 				"valid": false,
-				"skip": "unexpected success"
+				"skip": {
+					"v2": "unexpected success"
+				}
 			}
 		]
 	},