forked from
jonmsterling.com/ocaml-forester
ocaml
simplify lexer
+11
-11
+11
-11
lib/compiler/Lexer.mll
+11
-11
lib/compiler/Lexer.mll
···
1
1
{
2
2
open Forester_prelude
3
-
let drop_sigil c str = 1 |> List.nth @@ String.split_on_char c str
4
3
let raise_err lexbuf =
5
4
let loc = Asai.Range.of_lexbuf lexbuf in
6
5
Forester_core.Reporter.fatalf ~loc Forester_core.Reporter.Message.Parse_error "unrecognized token `%s`" @@
···
10
9
let digit = ['0'-'9']
11
10
let alpha = ['a'-'z' 'A'-'Z']
12
11
let int = '-'? digit+
13
-
let ident = '\\' (alpha) (alpha | digit | '-' | '/')*
14
-
let hash_ident = '#' (alpha | digit | '-')*
15
-
let dx_var = '?' (alpha | digit | '-')*
12
+
13
+
let hierarchical_name = (alpha) (alpha | digit | '-' | '/')*
14
+
let simple_name = (alpha | digit | '-')*
15
+
16
16
let xml_base_ident = (alpha) (alpha | digit | '-' | '_')*
17
17
let xml_qname = (xml_base_ident ':' xml_base_ident) | xml_base_ident
18
18
let addr = (alpha) (alpha | digit | '_' | '-')*
···
42
42
| "\\}" { Grammar.IDENT {|}|} }
43
43
| "\\[" { Grammar.IDENT {|[|} }
44
44
| "\\]" { Grammar.IDENT {|]|} }
45
+
| "\\ " { Grammar.IDENT {| |} }
45
46
| "-:" { Grammar.DX_ENTAILED }
46
47
| "#" { Grammar.HASH }
47
48
| "\\verb" { custom_verbatim_herald lexbuf }
48
49
| "\\datalog" { Grammar.DATALOG }
49
50
| "\\startverb" { custom_verbatim "\\stopverb" (Buffer.create 2000) lexbuf }
50
-
| "\\ " { Grammar.IDENT {| |} }
51
51
| "\\scope" { Grammar.SCOPE }
52
52
| "\\put" { Grammar.PUT }
53
53
| "\\put?" { Grammar.DEFAULT }
···
64
64
| "\\object" { Grammar.OBJECT }
65
65
| "\\patch" { Grammar.PATCH }
66
66
| "\\call" { Grammar.CALL }
67
-
| hash_ident { Grammar.HASH_IDENT (drop_sigil '#' (Lexing.lexeme lexbuf)) }
68
-
| dx_var { Grammar.DX_VAR (drop_sigil '?' (Lexing.lexeme lexbuf)) }
67
+
| '#' (simple_name as name) { Grammar.HASH_IDENT name }
68
+
| '?' (simple_name as name) { Grammar.DX_VAR name }
69
69
|
70
70
"\\<"
71
71
{
···
74
74
XML_ELT_IDENT qname
75
75
}
76
76
| "\\xmlns:" { DECL_XMLNS (xml_base_ident lexbuf) }
77
-
| ident { Grammar.IDENT (drop_sigil '\\' (Lexing.lexeme lexbuf)) }
77
+
| '\\' (hierarchical_name as name) { Grammar.IDENT name }
78
78
| '{' { Grammar.LBRACE }
79
79
| '}' { Grammar.RBRACE }
80
80
| '[' { Grammar.LSQUARE }
81
81
| ']' { Grammar.RSQUARE }
82
82
| '(' { Grammar.LPAREN }
83
83
| ')' { Grammar.RPAREN }
84
-
| text { Grammar.TEXT (Lexing.lexeme lexbuf) }
85
-
| wschar+ { Grammar.WHITESPACE (Lexing.lexeme lexbuf) }
86
-
| newline { Lexing.new_line lexbuf; Grammar.WHITESPACE (Lexing.lexeme lexbuf) }
84
+
| text as str { Grammar.TEXT str }
85
+
| wschar+ as str { Grammar.WHITESPACE str }
86
+
| newline as str { Lexing.new_line lexbuf; Grammar.WHITESPACE str }
87
87
| eof { Grammar.EOF }
88
88
| _ { raise_err lexbuf }
89
89