this repo has no description
encoding/jsonschema: add location information to test output
It's useful to know exactly where in the test data a test is,
so log that information. Also print a txtar file to make it
easier to reproduce test failures.
Signed-off-by: Roger Peppe <rogpeppe@gmail.com>
Change-Id: Ibef118835a429437825eb443e265066737c5fcb1
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1200518
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
Unity-Result: CUE porcuepine <cue.porcuepine@gmail.com>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
+89
-13
+65
-9
encoding/jsonschema/external_test.go
+65
-9
encoding/jsonschema/external_test.go
···
15
15
package jsonschema_test
16
16
17
17
import (
18
+
stdjson "encoding/json"
18
19
"fmt"
19
20
"os"
20
21
"path"
···
28
29
"cuelang.org/go/cue/cuecontext"
29
30
"cuelang.org/go/cue/errors"
30
31
"cuelang.org/go/cue/format"
32
+
"cuelang.org/go/cue/token"
31
33
"cuelang.org/go/encoding/json"
32
34
"cuelang.org/go/encoding/jsonschema"
33
35
"cuelang.org/go/encoding/jsonschema/internal/externaltest"
···
100
102
qt.Assert(t, qt.IsNil(err))
101
103
schemaValue = ctx.CompileBytes(b, cue.Filename("generated.cue"))
102
104
if err := schemaValue.Err(); err != nil {
105
+
t.Logf("extracted schema: %q", b)
103
106
extractErr = fmt.Errorf("cannot compile resulting schema: %v", errors.Details(err, nil))
104
107
}
105
108
}
106
109
107
110
if extractErr != nil {
111
+
t.Logf("location: %v", testdataPos(s))
112
+
t.Logf("txtar:\n%s", schemaFailureTxtar(s))
108
113
for _, test := range s.Tests {
109
114
t.Run("", func(t *testing.T) {
110
-
testFailed(t, &test.Skip, "could not compile schema")
115
+
testFailed(t, &test.Skip, test, "could not compile schema")
111
116
})
112
117
}
113
-
testFailed(t, &s.Skip, fmt.Sprintf("extract error: %v", extractErr))
118
+
testFailed(t, &s.Skip, s, fmt.Sprintf("extract error: %v", extractErr))
114
119
return
115
120
}
116
-
testSucceeded(t, &s.Skip)
121
+
testSucceeded(t, &s.Skip, s)
117
122
118
123
for _, test := range s.Tests {
119
124
t.Run(testName(test.Description), func(t *testing.T) {
125
+
defer func() {
126
+
if t.Failed() || testing.Verbose() {
127
+
t.Logf("txtar:\n%s", testCaseTxtar(s, test))
128
+
}
129
+
}()
130
+
t.Logf("location: %v", testdataPos(test))
120
131
instAST, err := json.Extract("instance.json", test.Data)
121
132
if err != nil {
122
133
t.Fatal(err)
···
129
140
err = instValue.Unify(schemaValue).Err()
130
141
if test.Valid {
131
142
if err != nil {
132
-
testFailed(t, &test.Skip, errors.Details(err, nil))
143
+
testFailed(t, &test.Skip, test, errors.Details(err, nil))
133
144
} else {
134
-
testSucceeded(t, &test.Skip)
145
+
testSucceeded(t, &test.Skip, test)
135
146
}
136
147
} else {
137
148
if err == nil {
138
-
testFailed(t, &test.Skip, "unexpected success")
149
+
testFailed(t, &test.Skip, test, "unexpected success")
139
150
} else {
140
-
testSucceeded(t, &test.Skip)
151
+
testSucceeded(t, &test.Skip, test)
141
152
}
142
153
}
143
154
})
144
155
}
145
156
}
146
157
158
+
// testCaseTxtar returns a testscript that runs the given test.
159
+
func testCaseTxtar(s *externaltest.Schema, test *externaltest.Test) string {
160
+
var buf strings.Builder
161
+
fmt.Fprintf(&buf, "env CUE_EXPERIMENT=evalv3\n")
162
+
fmt.Fprintf(&buf, "exec cue def json+jsonschema: schema.json\n")
163
+
if !test.Valid {
164
+
buf.WriteString("! ")
165
+
}
166
+
// TODO add $schema when one isn't already present?
167
+
fmt.Fprintf(&buf, "exec cue vet -c instance.json json+jsonschema: schema.json\n")
168
+
fmt.Fprintf(&buf, "\n")
169
+
fmt.Fprintf(&buf, "-- schema.json --\n%s\n", indentJSON(s.Schema))
170
+
fmt.Fprintf(&buf, "-- instance.json --\n%s\n", indentJSON(test.Data))
171
+
return buf.String()
172
+
}
173
+
174
+
// testCaseTxtar returns a testscript that decodes the given schema.
175
+
func schemaFailureTxtar(s *externaltest.Schema) string {
176
+
var buf strings.Builder
177
+
fmt.Fprintf(&buf, "env CUE_EXPERIMENT=evalv3\n")
178
+
fmt.Fprintf(&buf, "exec cue def -o schema.cue json+jsonschema: schema.json\n")
179
+
fmt.Fprintf(&buf, "exec cat schema.cue\n")
180
+
fmt.Fprintf(&buf, "exec cue vet schema.cue\n")
181
+
fmt.Fprintf(&buf, "-- schema.json --\n%s\n", indentJSON(s.Schema))
182
+
return buf.String()
183
+
}
184
+
185
+
func indentJSON(x stdjson.RawMessage) []byte {
186
+
data, err := stdjson.MarshalIndent(x, "", "\t")
187
+
if err != nil {
188
+
panic(err)
189
+
}
190
+
return data
191
+
}
192
+
193
+
type positioner interface {
194
+
Pos() token.Pos
195
+
}
196
+
147
197
// testName returns a test name that doesn't contain any
148
198
// slashes because slashes muck with matching.
149
199
func testName(s string) string {
···
153
203
// testFailed marks the current test as failed with the
154
204
// given error message, and updates the
155
205
// skip field pointed to by skipField if necessary.
156
-
func testFailed(t *testing.T, skipField *string, errStr string) {
206
+
func testFailed(t *testing.T, skipField *string, p positioner, errStr string) {
157
207
if cuetest.UpdateGoldenFiles {
158
208
if *skipField == "" && !allowRegressions() {
159
209
t.Fatalf("test regression; was succeeding, now failing: %v", errStr)
···
169
219
170
220
// testFails marks the current test as succeeded and updates the
171
221
// skip field pointed to by skipField if necessary.
172
-
func testSucceeded(t *testing.T, skipField *string) {
222
+
func testSucceeded(t *testing.T, skipField *string, p positioner) {
173
223
if cuetest.UpdateGoldenFiles {
174
224
*skipField = ""
175
225
return
···
177
227
if *skipField != "" {
178
228
t.Fatalf("unexpectedly more correct behavior (test success) on skipped test")
179
229
}
230
+
}
231
+
232
+
func testdataPos(p positioner) token.Position {
233
+
pp := p.Pos().Position()
234
+
pp.Filename = path.Join(testDir, pp.Filename)
235
+
return pp
180
236
}
181
237
182
238
func allowRegressions() bool {
+24
-4
encoding/jsonschema/internal/externaltest/tests.go
+24
-4
encoding/jsonschema/internal/externaltest/tests.go
···
12
12
"cuelang.org/go/cue/cuecontext"
13
13
"cuelang.org/go/cue/interpreter/embed"
14
14
"cuelang.org/go/cue/load"
15
+
"cuelang.org/go/cue/token"
15
16
)
16
17
17
18
type Schema struct {
19
+
location
18
20
Description string `json:"description"`
19
21
Comment string `json:"comment,omitempty"`
20
22
Schema stdjson.RawMessage `json:"schema"`
···
23
25
}
24
26
25
27
type Test struct {
28
+
location
26
29
Description string `json:"description"`
27
30
Comment string `json:"comment,omitempty"`
28
31
Data stdjson.RawMessage `json:"data"`
29
32
Valid bool `json:"valid"`
30
33
Skip string `json:"skip,omitempty"`
34
+
}
35
+
36
+
type location struct {
37
+
root cue.Value
38
+
path cue.Path
39
+
}
40
+
41
+
func (loc location) Pos() token.Pos {
42
+
return loc.root.LookupPath(loc.path).Pos()
31
43
}
32
44
33
45
func ParseTestData(data []byte) ([]*Schema, error) {
···
81
93
inst := load.Instances([]string{"."}, &load.Config{
82
94
Dir: dir,
83
95
})[0]
84
-
if err != nil {
96
+
if err := inst.Err; err != nil {
85
97
return nil, err
86
98
}
87
99
ctx := cuecontext.New(cuecontext.Interpreter(embed.New()))
···
97
109
return nil, err
98
110
}
99
111
// Fix up the raw JSON data to avoid running into some decode issues.
100
-
for _, schemas := range tests {
101
-
for _, schema := range schemas {
102
-
for _, test := range schema.Tests {
112
+
for filename, schemas := range tests {
113
+
for i, schema := range schemas {
114
+
schema.location = location{
115
+
root: val,
116
+
path: cue.MakePath(cue.Str(filename), cue.Index(i)),
117
+
}
118
+
for j, test := range schema.Tests {
119
+
test.location = location{
120
+
root: val,
121
+
path: cue.MakePath(cue.Str(filename), cue.Index(i), cue.Str("tests"), cue.Index(j)),
122
+
}
103
123
if len(test.Data) == 0 {
104
124
// See https://github.com/cue-lang/cue/issues/3397
105
125
test.Data = []byte("null")