this repo has no description
Store markdown docstrings alongside HTML in sherlodoc database
Add a doc_markdown field to Entry.t, populated at index time using
odoc's markdown renderer. The CLI gains --print-docstring for markdown
output (useful for LLM consumption and terminal display) while the
existing HTML output moves to --print-docstring-html.
Also exports Odoc_markdown.Generator.block for external use and adds
the UnboxedField case to url.ml's name_aux (needed after cherry-picked
fully-qualified-paths changes).
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
+118
-17
+29
-11
sherlodoc/cli/search.ml
+29
-11
sherlodoc/cli/search.ml
···
20
20
| Page -> "page"
21
21
| Impl -> "source"
22
22
23
-
let print_result ~print_cost ~print_docstring ~no_rhs (elt : Db.Entry.t) =
23
+
type docstring_format = No_docstring | Markdown | Html
24
+
25
+
let print_result ~print_cost ~docstring_format ~no_rhs (elt : Db.Entry.t) =
24
26
let cost = if print_cost then string_of_int elt.cost ^ " " else "" in
25
27
let typedecl_params =
26
28
(match elt.kind with
···
36
38
| Some _ when no_rhs -> ()
37
39
| Some rhs -> Format.fprintf h "%s" (Unescape.string rhs)
38
40
in
39
-
let docstring = if print_docstring then "\n" ^ elt.doc_html else "" in
41
+
let docstring =
42
+
match docstring_format with
43
+
| No_docstring -> ""
44
+
| Markdown -> "\n" ^ elt.doc_markdown
45
+
| Html -> "\n" ^ elt.doc_html
46
+
in
40
47
Format.printf "%s%s %s%s%a%s@." cost kind typedecl_params name pp_rhs elt.rhs docstring
41
48
42
49
let search
···
47
54
~no_rhs
48
55
~pretty_query
49
56
~time
50
-
~print_docstring
57
+
~docstring_format
51
58
query
52
59
=
53
60
let query = Query.{ query; packages = []; limit } in
···
58
65
match r with
59
66
| [] -> print_endline "[No results]"
60
67
| _ :: _ as results ->
61
-
List.iter (print_result ~print_cost ~print_docstring ~no_rhs) results ;
68
+
List.iter (print_result ~print_cost ~docstring_format ~no_rhs) results ;
62
69
flush stdout ;
63
70
if time then Format.printf "Search in %f@." (t1 -. t0)
64
71
···
69
76
~static_sort
70
77
~limit
71
78
~time
72
-
~print_docstring
79
+
~docstring_format
73
80
~db
74
81
=
75
82
Printf.printf "%ssearch>%s %!" "\027[0;36m" "\027[0;0m" ;
···
83
90
~no_rhs
84
91
~pretty_query
85
92
~time
86
-
~print_docstring
93
+
~docstring_format
87
94
query ;
88
95
search_loop
89
96
~print_cost
···
92
99
~static_sort
93
100
~limit
94
101
~time
95
-
~print_docstring
102
+
~docstring_format
96
103
~db
97
104
| exception End_of_file -> Printf.printf "\n%!"
98
105
···
105
112
pretty_query
106
113
time
107
114
print_docstring
115
+
print_docstring_html
108
116
db_format
109
117
db_filename
110
118
=
119
+
let docstring_format =
120
+
if print_docstring then Markdown
121
+
else if print_docstring_html then Html
122
+
else No_docstring
123
+
in
111
124
let module Storage = (val Db_store.storage_module db_format) in
112
125
let db = Storage.load db_filename in
113
126
match query with
···
120
133
~static_sort
121
134
~limit
122
135
~time
123
-
~print_docstring
136
+
~docstring_format
124
137
~db
125
138
| Some query ->
126
139
search
···
130
143
~static_sort
131
144
~limit
132
145
~time
133
-
~print_docstring
146
+
~docstring_format
134
147
~db
135
148
query
136
149
···
169
182
Arg.(value & flag & info [ "pretty-query" ] ~doc)
170
183
171
184
let print_docstring =
172
-
let doc = "Print the HTML of the docstring of the results" in
185
+
let doc = "Print the markdown docstring of the results" in
186
+
Arg.(value & flag & info [ "print-docstring" ] ~doc)
187
+
188
+
let print_docstring_html =
189
+
let doc = "Print the HTML docstring of the results" in
173
190
Arg.(value & flag & info [ "print-docstring-html" ] ~doc)
174
191
175
192
let term =
···
182
199
$ limit
183
200
$ pretty_query
184
201
$ print_time
185
-
$ print_docstring)
202
+
$ print_docstring
203
+
$ print_docstring_html)
+4
-2
sherlodoc/db/entry.ml
+4
-2
sherlodoc/db/entry.ml
···
53
53
; kind : Kind.t
54
54
; cost : int
55
55
; doc_html : string
56
+
; doc_markdown : string
56
57
; pkg : Package.t
57
58
}
58
59
59
-
let pp fmt { name; rhs; url; kind = _; cost; doc_html; pkg = _ } =
60
+
let pp fmt { name; rhs; url; kind = _; cost; doc_html; doc_markdown = _; pkg = _ } =
60
61
Format.fprintf
61
62
fmt
62
63
"{ name = %s ; rhs = %a ; url = %s ; kind = . ; cost = %d ; doc_html = %s ; pkg = . }\n"
···
150
151
let path = String.concat "/" (List.rev path) in
151
152
pkg_link ^ "/doc/" ^ path
152
153
153
-
let v ~name ~kind ~cost ~rhs ~doc_html ~url ~pkg () =
154
+
let v ~name ~kind ~cost ~rhs ~doc_html ~doc_markdown ~url ~pkg () =
154
155
{ name = non_empty_string name
155
156
; kind
156
157
; url = non_empty_string url
157
158
; cost
158
159
; doc_html = non_empty_string doc_html
160
+
; doc_markdown = non_empty_string doc_markdown
159
161
; pkg
160
162
; rhs = Option.map non_empty_string rhs
161
163
}
+2
sherlodoc/db/entry.mli
+2
sherlodoc/db/entry.mli
+1
sherlodoc/index/dune
+1
sherlodoc/index/dune
+13
-1
sherlodoc/index/load_doc.ml
+13
-1
sherlodoc/index/load_doc.ml
···
185
185
| "" -> ""
186
186
| _ -> string_of_html (Html.of_doc doc)
187
187
in
188
+
let doc_markdown =
189
+
match doc_txt with
190
+
| "" -> ""
191
+
| _ ->
192
+
let config =
193
+
Odoc_markdown.Config.make ~root_url:None ~allow_html:false ()
194
+
in
195
+
let ir = Odoc_document.Comment.to_ir doc in
196
+
let blocks = Odoc_markdown.Generator.block ~config ~xref_base_uri:"" ir in
197
+
Odoc_markdown.Renderer.to_string
198
+
(Odoc_markdown.Renderer.Block.Blocks blocks)
199
+
in
188
200
let rhs = Html.rhs_of_kind kind in
189
201
let kind = convert_kind ~db entry in
190
202
let cost = cost ~name ~kind ~doc_html ~rhs ~cat ~favourite ~favoured_prefixes in
191
203
let url = Html.url entry in
192
-
let elt = Sherlodoc_entry.v ~name ~kind ~rhs ~doc_html ~cost ~url ~pkg () in
204
+
let elt = Sherlodoc_entry.v ~name ~kind ~rhs ~doc_html ~doc_markdown ~cost ~url ~pkg () in
193
205
if index_docstring then register_doc ~db elt doc_txt ;
194
206
if index_name && kind <> Doc then register_full_name ~db elt ;
195
207
if type_search then register_kind ~db elt
+1
-1
sherlodoc/query/test/test_succ.ml
+1
-1
sherlodoc/query/test/test_succ.ml
···
3
3
let pkg = Db.Entry.Package.v ~name:"" ~version:""
4
4
5
5
let elt cost =
6
-
Db.Entry.v ~cost ~name:"" ~kind:Db.Entry.Kind.Doc ~rhs:None ~doc_html:"" ~url:"" ~pkg ()
6
+
Db.Entry.v ~cost ~name:"" ~kind:Db.Entry.Kind.Doc ~rhs:None ~doc_html:"" ~doc_markdown:"" ~url:"" ~pkg ()
7
7
8
8
(** This module does the same thing as Succ, but its correctness is obvious
9
9
and its performance terrible. *)
+2
-2
sherlodoc/test/cram/link_in_docstring.t/run.t
+2
-2
sherlodoc/test/cram/link_in_docstring.t/run.t
···
4
4
$ export SHERLODOC_DB=db.bin
5
5
$ export SHERLODOC_FORMAT=marshal
6
6
$ sherlodoc index $(find . -name '*.odocl')
7
-
$ sherlodoc search --print-docstring "foo"
7
+
$ sherlodoc search --print-docstring-html "foo"
8
8
val A.foo : int
9
9
<div><p>This is a docstring with a <span>link</span></p></div>
10
-
$ sherlodoc search --print-docstring "bar"
10
+
$ sherlodoc search --print-docstring-html "bar"
11
11
val A.bar : int
12
12
<div><p>This is a docstring with a ref to <span><code>foo</code></span></p></div>
+15
sherlodoc/test/cram/markdown_docstring.t/a.mli
+15
sherlodoc/test/cram/markdown_docstring.t/a.mli
···
1
+
2
+
(** This is a docstring with a {{:https://sherlocode.com}link} *)
3
+
val foo : int
4
+
5
+
(** This is a docstring with a ref to {!foo} *)
6
+
val bar : int
7
+
8
+
(** [hello name] returns a greeting for [name].
9
+
10
+
For example:
11
+
{[
12
+
hello "world" = "Hello, world!"
13
+
]}
14
+
*)
15
+
val hello : string -> string
+40
sherlodoc/test/cram/markdown_docstring.t/run.t
+40
sherlodoc/test/cram/markdown_docstring.t/run.t
···
1
+
$ ocamlc -c a.mli -bin-annot -I .
2
+
$ odoc compile -I . a.cmti
3
+
$ odoc link -I . a.odoc
4
+
$ export SHERLODOC_DB=db.bin
5
+
$ export SHERLODOC_FORMAT=marshal
6
+
$ sherlodoc index $(find . -name '*.odocl')
7
+
8
+
Search without docstrings:
9
+
10
+
$ sherlodoc search "foo" 2>&1
11
+
val A.foo : int
12
+
13
+
Search with markdown docstrings:
14
+
15
+
$ sherlodoc search --print-docstring "foo" 2>&1
16
+
val A.foo : int
17
+
This is a docstring with a [link](https://sherlocode.com)
18
+
19
+
20
+
$ sherlodoc search --print-docstring "bar" 2>&1
21
+
val A.bar : int
22
+
This is a docstring with a ref to [`foo`](A.md#val-foo)
23
+
24
+
25
+
$ sherlodoc search --print-docstring "hello" 2>&1
26
+
val A.hello : string -> string
27
+
`hello name` returns a greeting for `name`.
28
+
29
+
For example:
30
+
31
+
```ocaml
32
+
hello "world" = "Hello, world!"
33
+
```
34
+
35
+
36
+
Search with HTML docstrings:
37
+
38
+
$ sherlodoc search --print-docstring-html "foo" 2>&1
39
+
val A.foo : int
40
+
<div><p>This is a docstring with a <span>link</span></p></div>
+1
src/document/url.ml
+1
src/document/url.ml
···
24
24
| `Type (_, name) -> [TypeName.to_string name]
25
25
| `Constructor (_, name) -> [ConstructorName.to_string name]
26
26
| `Field (_, name) -> [FieldName.to_string name]
27
+
| `UnboxedField (_, name) -> [UnboxedFieldName.to_string name]
27
28
| `Extension (_, name) -> [ExtensionName.to_string name]
28
29
| `ExtensionDecl (_, _, name) -> [ExtensionName.to_string name]
29
30
| `Exception (_, name) -> [ExceptionName.to_string name]
+4
src/markdown2/generator.ml
+4
src/markdown2/generator.ml