this repo has no description
more
+119
-49
+119
-49
stack/river/lib/markdown_converter.ml
+119
-49
stack/river/lib/markdown_converter.ml
···
15
15
|> List.rev
16
16
with _ -> []
17
17
18
-
(** Normalize whitespace while preserving boundary spaces *)
19
-
let normalize_whitespace s =
20
-
let len = String.length s in
21
-
if len = 0 then ""
22
-
else
23
-
(* Check if original had leading/trailing whitespace *)
24
-
let has_leading = len > 0 && (s.[0] = ' ' || s.[0] = '\t' || s.[0] = '\n' || s.[0] = '\r') in
25
-
let has_trailing = len > 0 && (s.[len-1] = ' ' || s.[len-1] = '\t' || s.[len-1] = '\n' || s.[len-1] = '\r') in
26
-
27
-
(* Collapse multiple whitespace to single space *)
28
-
let normalized = Str.global_replace (Str.regexp "[ \t\n\r]+") " " s in
29
-
let trimmed = String.trim normalized in
30
-
31
-
(* If the trimmed result is empty but there was whitespace, return single space *)
32
-
if trimmed = "" && (has_leading || has_trailing) then " "
33
-
else
34
-
(* Add back boundary spaces if they existed *)
35
-
let result =
36
-
(if has_leading && trimmed <> "" then " " else "") ^
37
-
trimmed ^
38
-
(if has_trailing && trimmed <> "" then " " else "")
39
-
in
40
-
result
18
+
(** Check if string contains any whitespace *)
19
+
let has_whitespace s =
20
+
try
21
+
let _ = Str.search_forward (Str.regexp "[ \t\n\r]") s 0 in
22
+
true
23
+
with Not_found -> false
41
24
42
25
(** Clean up excessive newlines and normalize spacing *)
43
26
let cleanup_markdown s =
···
68
51
(* Trim leading and trailing whitespace *)
69
52
String.trim s
70
53
71
-
(** Convert HTML to Markdown *)
54
+
(** Convert HTML to Markdown using state-based whitespace handling *)
72
55
let html_to_markdown html_str =
73
56
try
74
57
let soup = Soup.parse html_str in
75
58
let buffer = Buffer.create 256 in
76
59
60
+
(* State: track if we need to insert a space before next text *)
61
+
let need_space = ref false in
62
+
63
+
(* Get last character in buffer, if any *)
64
+
let last_char () =
65
+
let len = Buffer.length buffer in
66
+
if len = 0 then None
67
+
else Some (Buffer.nth buffer (len - 1))
68
+
in
69
+
70
+
(* Add text with proper spacing *)
71
+
let add_text text =
72
+
let trimmed = String.trim text in
73
+
if trimmed <> "" then begin
74
+
(* Add space if needed and last char isn't already whitespace *)
75
+
if !need_space then begin
76
+
match last_char () with
77
+
| Some (' ' | '\n') -> ()
78
+
| _ -> Buffer.add_char buffer ' '
79
+
end;
80
+
Buffer.add_string buffer trimmed;
81
+
need_space := false
82
+
end
83
+
in
84
+
85
+
(* Mark that we need space before next text (for inline elements) *)
86
+
let mark_space_needed () =
87
+
need_space := has_whitespace (Buffer.contents buffer) || Buffer.length buffer > 0
88
+
in
89
+
77
90
let rec process_node node =
78
91
match Soup.element node with
79
92
| Some elem ->
80
93
let tag = Soup.name elem in
81
94
(match tag with
95
+
(* Block elements - reset space tracking *)
82
96
| "h1" ->
97
+
need_space := false;
83
98
Buffer.add_string buffer "\n# ";
84
99
Soup.children elem |> Soup.iter process_node;
85
-
Buffer.add_string buffer "\n\n"
100
+
Buffer.add_string buffer "\n\n";
101
+
need_space := false
86
102
| "h2" ->
103
+
need_space := false;
87
104
Buffer.add_string buffer "\n## ";
88
105
Soup.children elem |> Soup.iter process_node;
89
-
Buffer.add_string buffer "\n\n"
106
+
Buffer.add_string buffer "\n\n";
107
+
need_space := false
90
108
| "h3" ->
109
+
need_space := false;
91
110
Buffer.add_string buffer "\n### ";
92
111
Soup.children elem |> Soup.iter process_node;
93
-
Buffer.add_string buffer "\n\n"
112
+
Buffer.add_string buffer "\n\n";
113
+
need_space := false
94
114
| "h4" ->
115
+
need_space := false;
95
116
Buffer.add_string buffer "\n#### ";
96
117
Soup.children elem |> Soup.iter process_node;
97
-
Buffer.add_string buffer "\n\n"
118
+
Buffer.add_string buffer "\n\n";
119
+
need_space := false
98
120
| "h5" ->
121
+
need_space := false;
99
122
Buffer.add_string buffer "\n##### ";
100
123
Soup.children elem |> Soup.iter process_node;
101
-
Buffer.add_string buffer "\n\n"
124
+
Buffer.add_string buffer "\n\n";
125
+
need_space := false
102
126
| "h6" ->
127
+
need_space := false;
103
128
Buffer.add_string buffer "\n###### ";
104
129
Soup.children elem |> Soup.iter process_node;
105
-
Buffer.add_string buffer "\n\n"
130
+
Buffer.add_string buffer "\n\n";
131
+
need_space := false
106
132
| "p" ->
133
+
need_space := false;
107
134
Soup.children elem |> Soup.iter process_node;
108
-
Buffer.add_string buffer "\n\n"
135
+
Buffer.add_string buffer "\n\n";
136
+
need_space := false
109
137
| "br" ->
110
-
Buffer.add_string buffer "\n"
138
+
Buffer.add_string buffer "\n";
139
+
need_space := false
140
+
(* Inline elements - preserve space tracking *)
111
141
| "strong" | "b" ->
112
142
Buffer.add_string buffer "**";
143
+
need_space := false;
113
144
Soup.children elem |> Soup.iter process_node;
114
-
Buffer.add_string buffer "**"
145
+
Buffer.add_string buffer "**";
146
+
mark_space_needed ()
115
147
| "em" | "i" ->
116
148
Buffer.add_string buffer "*";
149
+
need_space := false;
117
150
Soup.children elem |> Soup.iter process_node;
118
-
Buffer.add_string buffer "*"
151
+
Buffer.add_string buffer "*";
152
+
mark_space_needed ()
119
153
| "code" ->
120
154
Buffer.add_string buffer "`";
155
+
need_space := false;
121
156
Soup.children elem |> Soup.iter process_node;
122
-
Buffer.add_string buffer "`"
157
+
Buffer.add_string buffer "`";
158
+
mark_space_needed ()
123
159
| "pre" ->
160
+
need_space := false;
124
161
Buffer.add_string buffer "\n```\n";
125
162
Soup.children elem |> Soup.iter process_node;
126
-
Buffer.add_string buffer "\n```\n\n"
163
+
Buffer.add_string buffer "\n```\n\n";
164
+
need_space := false
127
165
| "a" ->
128
-
let text = Soup.texts elem |> String.concat "" |> String.trim in
129
-
(match Soup.attribute "href" elem with
166
+
let text = Soup.texts elem |> String.concat " " |> String.trim in
167
+
let href = Soup.attribute "href" elem in
168
+
(match href with
130
169
| Some href ->
131
170
if text = "" then
132
171
Buffer.add_string buffer (Printf.sprintf "<%s>" href)
133
172
else
134
-
Buffer.add_string buffer (Printf.sprintf "[%s](%s)" text href)
135
-
| None -> Buffer.add_string buffer text)
173
+
Buffer.add_string buffer (Printf.sprintf "[%s](%s)" text href);
174
+
mark_space_needed ()
175
+
| None ->
176
+
add_text text)
136
177
| "ul" | "ol" ->
178
+
need_space := false;
137
179
Buffer.add_string buffer "\n";
138
180
let is_ordered = tag = "ol" in
139
181
let items = Soup.children elem |> Soup.to_list in
140
182
List.iteri (fun i item ->
141
183
match Soup.element item with
142
184
| Some li when Soup.name li = "li" ->
185
+
need_space := false;
143
186
if is_ordered then
144
187
Buffer.add_string buffer (Printf.sprintf "%d. " (i + 1))
145
188
else
···
148
191
Buffer.add_string buffer "\n"
149
192
| _ -> ()
150
193
) items;
151
-
Buffer.add_string buffer "\n"
194
+
Buffer.add_string buffer "\n";
195
+
need_space := false
152
196
| "blockquote" ->
197
+
need_space := false;
153
198
Buffer.add_string buffer "\n> ";
154
199
Soup.children elem |> Soup.iter process_node;
155
-
Buffer.add_string buffer "\n\n"
200
+
Buffer.add_string buffer "\n\n";
201
+
need_space := false
156
202
| "img" ->
157
203
let alt = Soup.attribute "alt" elem |> Option.value ~default:"" in
158
204
let src = Soup.attribute "src" elem |> Option.value ~default:"" in
159
-
Buffer.add_string buffer (Printf.sprintf "" alt src)
205
+
Buffer.add_string buffer (Printf.sprintf "" alt src);
206
+
mark_space_needed ()
160
207
| "hr" ->
161
-
Buffer.add_string buffer "\n---\n\n"
208
+
need_space := false;
209
+
Buffer.add_string buffer "\n---\n\n";
210
+
need_space := false
162
211
(* Strip these tags but keep content *)
163
212
| "div" | "span" | "article" | "section" | "header" | "footer"
164
-
| "main" | "nav" | "aside" | "figure" | "figcaption" ->
213
+
| "main" | "nav" | "aside" | "figure" | "figcaption" | "details" | "summary" ->
165
214
Soup.children elem |> Soup.iter process_node
166
215
(* Ignore script, style, etc *)
167
216
| "script" | "style" | "noscript" -> ()
···
169
218
| _ ->
170
219
Soup.children elem |> Soup.iter process_node)
171
220
| None ->
172
-
(* Text node *)
221
+
(* Text node - handle whitespace properly *)
173
222
match Soup.leaf_text node with
174
223
| Some text ->
175
-
let normalized = normalize_whitespace text in
176
-
if normalized <> "" then
177
-
Buffer.add_string buffer normalized
224
+
(* If text is only whitespace, mark that we need space *)
225
+
let trimmed = String.trim text in
226
+
if trimmed = "" then begin
227
+
if has_whitespace text then
228
+
need_space := true
229
+
end else begin
230
+
(* Text has content - check if it had leading/trailing whitespace *)
231
+
let had_leading_ws = has_whitespace text &&
232
+
(String.length text > 0 &&
233
+
(text.[0] = ' ' || text.[0] = '\t' || text.[0] = '\n' || text.[0] = '\r')) in
234
+
235
+
(* If had leading whitespace, mark we need space *)
236
+
if had_leading_ws then need_space := true;
237
+
238
+
(* Add the text content *)
239
+
add_text trimmed;
240
+
241
+
(* If had trailing whitespace, mark we need space for next *)
242
+
let had_trailing_ws = has_whitespace text &&
243
+
(String.length text > 0 &&
244
+
let last = text.[String.length text - 1] in
245
+
last = ' ' || last = '\t' || last = '\n' || last = '\r') in
246
+
if had_trailing_ws then need_space := true
247
+
end
178
248
| None -> ()
179
249
in
180
250