Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
unittests: test_tokenizer: check if the tokenizer works
Add extra tests to check if the tokenizer is working properly.
Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
Signed-off-by: Jonathan Corbet <corbet@lwn.net>
Message-ID: <e4b6e880565f40287a69667652adb393879e039b.1773770483.git.mchehab+huawei@kernel.org>
authored by
Mauro Carvalho Chehab
and committed by
Jonathan Corbet
5f6fc8ed
fe79f85f
+106
-2
+106
-2
tools/unittests/test_tokenizer.py
+106
-2
tools/unittests/test_tokenizer.py
···
15
15
SRC_DIR = os.path.dirname(os.path.realpath(__file__))
16
16
sys.path.insert(0, os.path.join(SRC_DIR, "../lib/python"))
17
17
18
-
from kdoc.c_lex import CTokenizer
18
+
from kdoc.c_lex import CToken, CTokenizer
19
19
from unittest_helper import run_unittest
20
-
21
20
22
21
#
23
22
# List of tests.
24
23
#
25
24
# The code will dynamically generate one test for each key on this dictionary.
26
25
#
26
+
def tokens_to_list(tokens):
27
+
tuples = []
28
+
29
+
for tok in tokens:
30
+
if tok.kind == CToken.SPACE:
31
+
continue
32
+
33
+
tuples += [(tok.kind, tok.value, tok.level)]
34
+
35
+
return tuples
36
+
37
+
38
+
def make_tokenizer_test(name, data):
39
+
"""
40
+
Create a test named ``name`` using parameters given by ``data`` dict.
41
+
"""
42
+
43
+
def test(self):
44
+
"""In-lined lambda-like function to run the test"""
45
+
46
+
#
47
+
# Check if logger is working
48
+
#
49
+
if "log_level" in data:
50
+
with self.assertLogs('kdoc.c_lex', level='ERROR') as cm:
51
+
tokenizer = CTokenizer(data["source"])
52
+
53
+
return
54
+
55
+
#
56
+
# Check if tokenizer is producing expected results
57
+
#
58
+
tokens = CTokenizer(data["source"]).tokens
59
+
60
+
result = tokens_to_list(tokens)
61
+
expected = tokens_to_list(data["expected"])
62
+
63
+
self.assertEqual(result, expected, msg=f"{name}")
64
+
65
+
return test
66
+
67
+
#: Tokenizer tests.
68
+
TESTS_TOKENIZER = {
69
+
"__run__": make_tokenizer_test,
70
+
71
+
"basic_tokens": {
72
+
"source": """
73
+
int a; // comment
74
+
float b = 1.23;
75
+
""",
76
+
"expected": [
77
+
CToken(CToken.NAME, "int"),
78
+
CToken(CToken.NAME, "a"),
79
+
CToken(CToken.ENDSTMT, ";"),
80
+
CToken(CToken.COMMENT, "// comment"),
81
+
CToken(CToken.NAME, "float"),
82
+
CToken(CToken.NAME, "b"),
83
+
CToken(CToken.OP, "="),
84
+
CToken(CToken.NUMBER, "1.23"),
85
+
CToken(CToken.ENDSTMT, ";"),
86
+
],
87
+
},
88
+
89
+
"depth_counters": {
90
+
"source": """
91
+
struct X {
92
+
int arr[10];
93
+
func(a[0], (b + c));
94
+
}
95
+
""",
96
+
"expected": [
97
+
CToken(CToken.STRUCT, "struct"),
98
+
CToken(CToken.NAME, "X"),
99
+
CToken(CToken.BEGIN, "{", brace_level=1),
100
+
101
+
CToken(CToken.NAME, "int", brace_level=1),
102
+
CToken(CToken.NAME, "arr", brace_level=1),
103
+
CToken(CToken.BEGIN, "[", brace_level=1, bracket_level=1),
104
+
CToken(CToken.NUMBER, "10", brace_level=1, bracket_level=1),
105
+
CToken(CToken.END, "]", brace_level=1),
106
+
CToken(CToken.ENDSTMT, ";", brace_level=1),
107
+
CToken(CToken.NAME, "func", brace_level=1),
108
+
CToken(CToken.BEGIN, "(", brace_level=1, paren_level=1),
109
+
CToken(CToken.NAME, "a", brace_level=1, paren_level=1),
110
+
CToken(CToken.BEGIN, "[", brace_level=1, paren_level=1, bracket_level=1),
111
+
CToken(CToken.NUMBER, "0", brace_level=1, paren_level=1, bracket_level=1),
112
+
CToken(CToken.END, "]", brace_level=1, paren_level=1),
113
+
CToken(CToken.PUNC, ",", brace_level=1, paren_level=1),
114
+
CToken(CToken.BEGIN, "(", brace_level=1, paren_level=2),
115
+
CToken(CToken.NAME, "b", brace_level=1, paren_level=2),
116
+
CToken(CToken.OP, "+", brace_level=1, paren_level=2),
117
+
CToken(CToken.NAME, "c", brace_level=1, paren_level=2),
118
+
CToken(CToken.END, ")", brace_level=1, paren_level=1),
119
+
CToken(CToken.END, ")", brace_level=1),
120
+
CToken(CToken.ENDSTMT, ";", brace_level=1),
121
+
CToken(CToken.END, "}"),
122
+
],
123
+
},
124
+
125
+
"mismatch_error": {
126
+
"source": "int a$ = 5;", # $ is illegal
127
+
"log_level": "ERROR",
128
+
},
129
+
}
27
130
28
131
def make_private_test(name, data):
29
132
"""
···
417
314
#: Dict containing all test groups fror CTokenizer
418
315
TESTS = {
419
316
"TestPublicPrivate": TESTS_PRIVATE,
317
+
"TestTokenizer": TESTS_TOKENIZER,
420
318
}
421
319
422
320
def setUp(self):