this repo has no description
internal/core/adt: also use strings.ToValidUTF8 over x/text/encoding/unicode
Mimicking the change in the last commit in internal/core/convert.
Note that there is one tiny difference in test output here;
given the byte sequence 'a\x80\x95a',
x/text/encoding/unicode inserts two replacement characters for the
two invalid bytes, whereas strings.ToValidUTF8 uses one.
As long as the "rune error" sequence is inserted once, it's fine.
Signed-off-by: Daniel Martí <mvdan@mvdan.cc>
Change-Id: Idc2f78e2f9ca03c0889bda80ce402d1acab9e95a
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1229429
Reviewed-by: Matthew Sackman <matthew@cue.works>
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
Unity-Result: CUE porcuepine <cue.porcuepine@gmail.com>
+29
-3
+26
cue/testdata/interpolation/scalars.txtar
+26
cue/testdata/interpolation/scalars.txtar
···
28
28
Unifications: 11
29
29
Conjuncts: 12
30
30
Disjuncts: 11
31
+
-- out/evalalpha --
32
+
(struct){
33
+
bool1: (string){ "1+1=2: true" }
34
+
bool2: (string){ "1+1=1: false" }
35
+
b1: (bytes){ 'a\xed\x95a' }
36
+
bytes1s: (string){ "a�a" }
37
+
bytes1b: (bytes){ 'a\xed\x95a' }
38
+
b2: (bytes){ 'a\x80\x95a' }
39
+
bytes2s: (string){ "a�a" }
40
+
bytes2b: (bytes){ 'a\x80\x95a' }
41
+
n1: (string){ "1 2.00" }
42
+
n2: (string){ "1E+2" }
43
+
}
44
+
-- diff/-out/evalalpha<==>+out/eval --
45
+
diff old new
46
+
--- old
47
+
+++ new
48
+
@@ -5,7 +5,7 @@
49
+
bytes1s: (string){ "a�a" }
50
+
bytes1b: (bytes){ 'a\xed\x95a' }
51
+
b2: (bytes){ 'a\x80\x95a' }
52
+
- bytes2s: (string){ "a��a" }
53
+
+ bytes2s: (string){ "a�a" }
54
+
bytes2b: (bytes){ 'a\x80\x95a' }
55
+
n1: (string){ "1 2.00" }
56
+
n2: (string){ "1E+2" }
31
57
-- out/eval --
32
58
(struct){
33
59
bool1: (string){ "1+1=2: true" }
+3
-3
internal/core/adt/context.go
+3
-3
internal/core/adt/context.go
···
19
19
"iter"
20
20
"reflect"
21
21
"regexp"
22
+
"strings"
22
23
"sync/atomic"
24
+
"unicode/utf8"
23
25
24
26
"github.com/cockroachdb/apd/v3"
25
-
"golang.org/x/text/encoding/unicode"
26
27
27
28
"cuelang.org/go/cue/ast"
28
29
"cuelang.org/go/cue/errors"
···
1208
1209
}
1209
1210
1210
1211
func bytesToString(b []byte) string {
1211
-
b, _ = unicode.UTF8.NewDecoder().Bytes(b)
1212
-
return string(b)
1212
+
return strings.ToValidUTF8(string(b), string(utf8.RuneError))
1213
1213
}
1214
1214
1215
1215
func (c *OpContext) bytesValue(v Value, as interface{}) []byte {