Native OCaml Rego/OPA policy engine
ocaml-rego: lexer suppresses synthetic SEMICOLON before continuation tokens
Multi-line set/array/object literals fail to parse when the lexer
inserts a synthetic SEMICOLON between the trailing element and the
following separator. Peek the next real token after a newline and
suppress the synthetic semicolon when it would split a continuation:
closing brackets/parens/braces, a comma, else, a dot, or any binary
operator that has to bind to the preceding expression.
+17
-12
+17
-12
lib/lexer.ml
+17
-12
lib/lexer.ml
···
163
163
| `Newline -> next_real_token lexbuf
164
164
| `Token t -> t
165
165
166
+
(** After a stmt-ending token and a newline, we usually emit a synthetic
167
+
SEMICOLON. But certain following tokens always continue the current
168
+
construct (the closing brackets/parens, a separator, an [else], or binary
169
+
operators that have to bind to the previous expression), so suppress the
170
+
SEMICOLON in those cases. *)
171
+
let suppress_semi_before = function
172
+
| Parser.ELSE | Parser.RBRACE | Parser.RBRACK | Parser.RPAREN | Parser.COMMA
173
+
| Parser.PIPE | Parser.AMPERSAND | Parser.PLUS | Parser.MINUS | Parser.STAR
174
+
| Parser.SLASH | Parser.PERCENT | Parser.EQEQ | Parser.BANGEQ | Parser.LT
175
+
| Parser.LE | Parser.GT | Parser.GE | Parser.ASSIGN | Parser.COLONEQ
176
+
| Parser.IN | Parser.DOT ->
177
+
true
178
+
| _ -> false
179
+
166
180
let rec token st =
167
181
match st.buffered with
168
182
| Some tok ->
···
175
189
st.prev <- tok;
176
190
tok
177
191
| `Newline ->
178
-
if can_end_stmt st.prev then
192
+
if can_end_stmt st.prev then (
179
193
let next = next_real_token st.lexbuf in
180
-
(* A rule body's [}] is always followed by a newline before
181
-
[else]; suppressing the synthetic [;] there keeps the
182
-
grammar simpler than absorbing it inside [else_chain]. *)
183
-
if next = Parser.ELSE then begin
184
-
st.buffered <- Some next;
185
-
token st
186
-
end
187
-
else begin
188
-
st.buffered <- Some next;
189
-
Parser.SEMICOLON
190
-
end
194
+
st.buffered <- Some next;
195
+
if suppress_semi_before next then token st else Parser.SEMICOLON)
191
196
else token st)