this repo has no description
encoding/jsonschema: commit external test stats
This makes the JSON Schema external test statistics a little more
obvious in code review.
Signed-off-by: Roger Peppe <rogpeppe@gmail.com>
Change-Id: Id508a24d0636e9af6a1a44d5fed98bf89947d986
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1200529
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
+40
-15
+1
encoding/jsonschema/external_test.go
+1
encoding/jsonschema/external_test.go
···
40
40
// The commit below references the JSON schema test main branch as of Sun May 19 19:01:03 2024 +0300
41
41
42
42
//go:generate go run vendor_external.go -- 9fc880bfb6d8ccd093bc82431f17d13681ffae8e
43
+
//go:generate go run teststats.go -o external_teststats.txt
43
44
44
45
const testDir = "testdata/external"
45
46
+10
encoding/jsonschema/external_teststats.txt
+10
encoding/jsonschema/external_teststats.txt
···
1
+
# Generated by teststats. DO NOT EDIT
2
+
v2:
3
+
schema extract (pass / total): 975 / 1637 = 59.6%
4
+
tests (pass / total): 3140 / 7175 = 43.8%
5
+
tests on extracted schemas (pass / total): 3140 / 3546 = 88.6%
6
+
7
+
v3:
8
+
schema extract (pass / total): 967 / 1637 = 59.1%
9
+
tests (pass / total): 3074 / 7175 = 42.8%
10
+
tests on extracted schemas (pass / total): 3074 / 3538 = 86.9%
+29
-15
encoding/jsonschema/teststats.go
+29
-15
encoding/jsonschema/teststats.go
···
20
20
import (
21
21
"flag"
22
22
"fmt"
23
+
"io"
23
24
"log"
25
+
"os"
24
26
"path"
25
27
"sort"
26
28
···
28
30
"cuelang.org/go/encoding/jsonschema/internal/externaltest"
29
31
)
30
32
31
-
var list = flag.String("list", "", "list all failed tests for a given evaluator version")
33
+
var (
34
+
list = flag.String("list", "", "list all failed tests for a given evaluator version")
35
+
out = flag.String("o", "", "write output to a file")
36
+
)
32
37
33
38
const testDir = "testdata/external"
34
39
35
40
func main() {
41
+
flag.Parse()
36
42
tests, err := externaltest.ReadTestDir(testDir)
37
43
if err != nil {
38
44
log.Fatal(err)
39
45
}
40
-
flag.Parse()
46
+
outw := os.Stdout
47
+
if *out != "" {
48
+
var err error
49
+
outw, err = os.Create(*out)
50
+
if err != nil {
51
+
log.Fatal(err)
52
+
}
53
+
fmt.Fprintf(outw, "# Generated by teststats. DO NOT EDIT\n")
54
+
}
41
55
if *list != "" {
42
-
listFailures(*list, tests)
56
+
listFailures(outw, *list, tests)
43
57
} else {
44
-
fmt.Printf("v2:\n")
45
-
showStats("v2", tests)
46
-
fmt.Println()
47
-
fmt.Printf("v3:\n")
48
-
showStats("v3", tests)
58
+
fmt.Fprintf(outw, "v2:\n")
59
+
showStats(outw, "v2", tests)
60
+
fmt.Fprintf(outw, "\n")
61
+
fmt.Fprintf(outw, "v3:\n")
62
+
showStats(outw, "v3", tests)
49
63
}
50
64
}
51
65
52
-
func showStats(version string, tests map[string][]*externaltest.Schema) {
66
+
func showStats(outw io.Writer, version string, tests map[string][]*externaltest.Schema) {
53
67
schemaOK := 0
54
68
schemaTot := 0
55
69
testOK := 0
···
76
90
}
77
91
}
78
92
}
79
-
fmt.Printf("\tschema extract (pass / total): %d / %d = %.1f%%\n", schemaOK, schemaTot, percent(schemaOK, schemaTot))
80
-
fmt.Printf("\ttests (pass / total): %d / %d = %.1f%%\n", testOK, testTot, percent(testOK, testTot))
81
-
fmt.Printf("\ttests on extracted schemas (pass / total): %d / %d = %.1f%%\n", schemaOKTestOK, schemaOKTestTot, percent(schemaOKTestOK, schemaOKTestTot))
93
+
fmt.Fprintf(outw, "\tschema extract (pass / total): %d / %d = %.1f%%\n", schemaOK, schemaTot, percent(schemaOK, schemaTot))
94
+
fmt.Fprintf(outw, "\ttests (pass / total): %d / %d = %.1f%%\n", testOK, testTot, percent(testOK, testTot))
95
+
fmt.Fprintf(outw, "\ttests on extracted schemas (pass / total): %d / %d = %.1f%%\n", schemaOKTestOK, schemaOKTestTot, percent(schemaOKTestOK, schemaOKTestTot))
82
96
}
83
97
84
-
func listFailures(version string, tests map[string][]*externaltest.Schema) {
98
+
func listFailures(outw io.Writer, version string, tests map[string][]*externaltest.Schema) {
85
99
for _, filename := range sortedKeys(tests) {
86
100
schemas := tests[filename]
87
101
for _, schema := range schemas {
88
102
if schema.Skip[version] != "" {
89
-
fmt.Printf("%s: schema fail (%s)\n", testdataPos(schema), schema.Description)
103
+
fmt.Fprintf(outw, "%s: schema fail (%s)\n", testdataPos(schema), schema.Description)
90
104
continue
91
105
}
92
106
for _, test := range schema.Tests {
···
95
109
if !test.Valid {
96
110
reason = "unexpected success"
97
111
}
98
-
fmt.Printf("%s: %s (%s; %s)\n", testdataPos(test), reason, schema.Description, test.Description)
112
+
fmt.Fprintf(outw, "%s: %s (%s; %s)\n", testdataPos(test), reason, schema.Description, test.Description)
99
113
}
100
114
}
101
115
}