Skip to content

Commit 2e19c1f

Browse files
rogpepperogpeppe
committed
encoding/jsonschema: verify result is concrete in external tests
The result of unifying a JSON Schema with a concrete instance should itself be concrete, but that's not necessarily the case. Verify this by checking for concreteness in the test. The test statistics before were: ``` v2: schema extract (pass / total): 971 / 1637 = 59.3% tests (pass / total): 3032 / 7175 = 42.3% tests on extracted schemas (pass / total): 3032 / 3542 = 85.6% v3: schema extract (pass / total): 971 / 1637 = 59.3% tests (pass / total): 3014 / 7175 = 42.0% tests on extracted schemas (pass / total): 3014 / 3542 = 85.1% ``` With this change applied they are: ``` v2: schema extract (pass / total): 971 / 1637 = 59.3% tests (pass / total): 3081 / 7175 = 42.9% tests on extracted schemas (pass / total): 3081 / 3542 = 87.0% v3: schema extract (pass / total): 971 / 1637 = 59.3% tests (pass / total): 3063 / 7175 = 42.7% tests on extracted schemas (pass / total): 3063 / 3542 = 86.5% ``` That is, overall, 49 more tests pass on both v2 and v3. This is (almost?) entirely down to the fact that with out concreteness checking enabled, we won't fail a test that is checking for a required field that isn't present. Signed-off-by: Roger Peppe <[email protected]> Change-Id: I6c2a8cceb454826aa02c32525070d093160fec9e Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1200522 Reviewed-by: Daniel Martí <[email protected]> TryBot-Result: CUEcueckoo <[email protected]> Unity-Result: CUE porcuepine <[email protected]>
1 parent e2685dd commit 2e19c1f

30 files changed

+294
-490
lines changed

Diff for: ‎encoding/jsonschema/external_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ func runExternalSchemaTests(t *testing.T, m *cuetdtest.M, filename string, s *ex
137137

138138
instValue := ctx.BuildExpr(instAST)
139139
qt.Assert(t, qt.IsNil(instValue.Err()))
140-
err = instValue.Unify(schemaValue).Err()
140+
err = instValue.Unify(schemaValue).Validate(cue.Concrete(true))
141141
if test.Valid {
142142
if err != nil {
143143
testFailed(t, m, &test.Skip, test, errors.Details(err, nil))

Diff for: ‎encoding/jsonschema/testdata/external/tests/draft2019-09/allOf.json

+6-30
Original file line numberDiff line numberDiff line change
@@ -40,22 +40,14 @@
4040
"data": {
4141
"foo": "baz"
4242
},
43-
"valid": false,
44-
"skip": {
45-
"v2": "unexpected success",
46-
"v3": "unexpected success"
47-
}
43+
"valid": false
4844
},
4945
{
5046
"description": "mismatch first",
5147
"data": {
5248
"bar": 2
5349
},
54-
"valid": false,
55-
"skip": {
56-
"v2": "unexpected success",
57-
"v3": "unexpected success"
58-
}
50+
"valid": false
5951
},
6052
{
6153
"description": "wrong type",
@@ -118,46 +110,30 @@
118110
"foo": "quux",
119111
"baz": null
120112
},
121-
"valid": false,
122-
"skip": {
123-
"v2": "unexpected success",
124-
"v3": "unexpected success"
125-
}
113+
"valid": false
126114
},
127115
{
128116
"description": "mismatch first allOf",
129117
"data": {
130118
"bar": 2,
131119
"baz": null
132120
},
133-
"valid": false,
134-
"skip": {
135-
"v2": "unexpected success",
136-
"v3": "unexpected success"
137-
}
121+
"valid": false
138122
},
139123
{
140124
"description": "mismatch second allOf",
141125
"data": {
142126
"foo": "quux",
143127
"bar": 2
144128
},
145-
"valid": false,
146-
"skip": {
147-
"v2": "unexpected success",
148-
"v3": "unexpected success"
149-
}
129+
"valid": false
150130
},
151131
{
152132
"description": "mismatch both",
153133
"data": {
154134
"bar": 2
155135
},
156-
"valid": false,
157-
"skip": {
158-
"v2": "unexpected success",
159-
"v3": "unexpected success"
160-
}
136+
"valid": false
161137
}
162138
]
163139
},

Diff for: ‎encoding/jsonschema/testdata/external/tests/draft2019-09/anyOf.json

+10-2
Original file line numberDiff line numberDiff line change
@@ -159,14 +159,22 @@
159159
"data": {
160160
"bar": 2
161161
},
162-
"valid": true
162+
"valid": true,
163+
"skip": {
164+
"v2": "incomplete value {bar:2} | {bar:2,foo!:string}\n",
165+
"v3": "incomplete value {bar:2} | {bar:2,foo!:string}\n"
166+
}
163167
},
164168
{
165169
"description": "second anyOf valid (complex)",
166170
"data": {
167171
"foo": "baz"
168172
},
169-
"valid": true
173+
"valid": true,
174+
"skip": {
175+
"v2": "incomplete value {foo:\"baz\",bar!:int} | {foo:\"baz\"}\n",
176+
"v3": "incomplete value {foo:\"baz\",bar!:int} | {foo:\"baz\"}\n"
177+
}
170178
},
171179
{
172180
"description": "both anyOf valid (complex)",

Diff for: ‎encoding/jsonschema/testdata/external/tests/draft2019-09/enum.json

+2-10
Original file line numberDiff line numberDiff line change
@@ -160,20 +160,12 @@
160160
"data": {
161161
"foo": "foo"
162162
},
163-
"valid": false,
164-
"skip": {
165-
"v2": "unexpected success",
166-
"v3": "unexpected success"
167-
}
163+
"valid": false
168164
},
169165
{
170166
"description": "missing all properties is invalid",
171167
"data": {},
172-
"valid": false,
173-
"skip": {
174-
"v2": "unexpected success",
175-
"v3": "unexpected success"
176-
}
168+
"valid": false
177169
}
178170
]
179171
},

Diff for: ‎encoding/jsonschema/testdata/external/tests/draft2019-09/items.json

+2-6
Original file line numberDiff line numberDiff line change
@@ -417,11 +417,7 @@
417417
}
418418
]
419419
],
420-
"valid": false,
421-
"skip": {
422-
"v2": "unexpected success",
423-
"v3": "unexpected success"
424-
}
420+
"valid": false
425421
},
426422
{
427423
"description": "fewer items is valid",
@@ -439,7 +435,7 @@
439435
],
440436
"valid": true,
441437
"skip": {
442-
"v2": "incompatible list lengths (2 and 3)\n",
438+
"v2": "incompatible list lengths (2 and 3)\n0: incompatible list lengths (1 and 2)\n1: incompatible list lengths (1 and 2)\n",
443439
"v3": "incompatible list lengths (2 and 3):\n generated.cue:2:1\n"
444440
}
445441
}

Diff for: ‎encoding/jsonschema/testdata/external/tests/draft2019-09/oneOf.json

+33-21
Original file line numberDiff line numberDiff line change
@@ -192,14 +192,22 @@
192192
"data": {
193193
"bar": 2
194194
},
195-
"valid": true
195+
"valid": true,
196+
"skip": {
197+
"v2": "incomplete value {bar:2} | {bar:2,foo!:string}\n",
198+
"v3": "incomplete value {bar:2} | {bar:2,foo!:string}\n"
199+
}
196200
},
197201
{
198202
"description": "second oneOf valid (complex)",
199203
"data": {
200204
"foo": "baz"
201205
},
202-
"valid": true
206+
"valid": true,
207+
"skip": {
208+
"v2": "incomplete value {foo:\"baz\",bar!:int} | {foo:\"baz\"}\n",
209+
"v3": "incomplete value {foo:\"baz\",bar!:int} | {foo:\"baz\"}\n"
210+
}
203211
},
204212
{
205213
"description": "both oneOf valid (complex)",
@@ -277,27 +285,31 @@
277285
"data": {
278286
"bar": 2
279287
},
280-
"valid": false,
281-
"skip": {
282-
"v2": "unexpected success",
283-
"v3": "unexpected success"
284-
}
288+
"valid": false
285289
},
286290
{
287291
"description": "first valid - valid",
288292
"data": {
289293
"foo": 1,
290294
"bar": 2
291295
},
292-
"valid": true
296+
"valid": true,
297+
"skip": {
298+
"v2": "incomplete value {foo:1,bar:2} | {foo:1,bar:2,baz!:_}\n",
299+
"v3": "incomplete value {foo:1,bar:2} | {foo:1,bar:2,baz!:_}\n"
300+
}
293301
},
294302
{
295303
"description": "second valid - valid",
296304
"data": {
297305
"foo": 1,
298306
"baz": 3
299307
},
300-
"valid": true
308+
"valid": true,
309+
"skip": {
310+
"v2": "incomplete value {foo:1,baz:3,bar!:_} | {foo:1,baz:3}\n",
311+
"v3": "incomplete value {foo:1,baz:3,bar!:_} | {foo:1,baz:3}\n"
312+
}
301313
},
302314
{
303315
"description": "both valid - invalid",
@@ -344,37 +356,37 @@
344356
"data": {
345357
"bar": 8
346358
},
347-
"valid": true
359+
"valid": true,
360+
"skip": {
361+
"v2": "incomplete value {bar:8,baz?:_} | {bar:8,foo!:_}\n",
362+
"v3": "incomplete value {bar:8,baz?:_} | {bar:8,foo!:_}\n"
363+
}
348364
},
349365
{
350366
"description": "second oneOf valid",
351367
"data": {
352368
"foo": "foo"
353369
},
354-
"valid": true
370+
"valid": true,
371+
"skip": {
372+
"v2": "incomplete value {foo:\"foo\",bar!:_,baz?:_} | {foo:\"foo\"}\n",
373+
"v3": "incomplete value {foo:\"foo\",bar!:_,baz?:_} | {foo:\"foo\"}\n"
374+
}
355375
},
356376
{
357377
"description": "both oneOf valid",
358378
"data": {
359379
"foo": "foo",
360380
"bar": 8
361381
},
362-
"valid": false,
363-
"skip": {
364-
"v2": "unexpected success",
365-
"v3": "unexpected success"
366-
}
382+
"valid": false
367383
},
368384
{
369385
"description": "neither oneOf valid",
370386
"data": {
371387
"baz": "quux"
372388
},
373-
"valid": false,
374-
"skip": {
375-
"v2": "unexpected success",
376-
"v3": "unexpected success"
377-
}
389+
"valid": false
378390
}
379391
]
380392
},

Diff for: ‎encoding/jsonschema/testdata/external/tests/draft2019-09/required.json

+6-30
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,7 @@
2424
"data": {
2525
"bar": 1
2626
},
27-
"valid": false,
28-
"skip": {
29-
"v2": "unexpected success",
30-
"v3": "unexpected success"
31-
}
27+
"valid": false
3228
},
3329
{
3430
"description": "ignores arrays",
@@ -112,11 +108,7 @@
112108
"foo\nbar": "1",
113109
"foo\"bar": "1"
114110
},
115-
"valid": false,
116-
"skip": {
117-
"v2": "unexpected success",
118-
"v3": "unexpected success"
119-
}
111+
"valid": false
120112
}
121113
]
122114
},
@@ -145,22 +137,14 @@
145137
{
146138
"description": "none of the properties mentioned",
147139
"data": {},
148-
"valid": false,
149-
"skip": {
150-
"v2": "unexpected success",
151-
"v3": "unexpected success"
152-
}
140+
"valid": false
153141
},
154142
{
155143
"description": "__proto__ present",
156144
"data": {
157145
"__proto__": "foo"
158146
},
159-
"valid": false,
160-
"skip": {
161-
"v2": "unexpected success",
162-
"v3": "unexpected success"
163-
}
147+
"valid": false
164148
},
165149
{
166150
"description": "toString present",
@@ -169,11 +153,7 @@
169153
"length": 37
170154
}
171155
},
172-
"valid": false,
173-
"skip": {
174-
"v2": "unexpected success",
175-
"v3": "unexpected success"
176-
}
156+
"valid": false
177157
},
178158
{
179159
"description": "constructor present",
@@ -182,11 +162,7 @@
182162
"length": 37
183163
}
184164
},
185-
"valid": false,
186-
"skip": {
187-
"v2": "unexpected success",
188-
"v3": "unexpected success"
189-
}
165+
"valid": false
190166
},
191167
{
192168
"description": "all present",

0 commit comments

Comments
 (0)