···11+# This file is generated by dune, edit dune-project instead
22+opam-version: "2.0"
33+synopsis: "Pure OCaml B-tree implementation for persistent storage"
44+description:
55+ "A B-tree implementation supporting both table B-trees (integer keys with data) and index B-trees (arbitrary keys). Designed for SQLite-compatible persistent storage with page-based I/O."
66+maintainer: ["Thomas Gazagnaire"]
77+authors: ["Thomas Gazagnaire"]
88+license: "MIT"
99+depends: [
1010+ "dune" {>= "3.0"}
1111+ "ocaml" {>= "5.1"}
1212+ "eio" {>= "1.0"}
1313+ "cstruct"
1414+ "alcotest" {with-test}
1515+ "eio_main" {with-test}
1616+ "odoc" {with-doc}
1717+]
1818+build: [
1919+ ["dune" "subst"] {dev}
2020+ [
2121+ "dune"
2222+ "build"
2323+ "-p"
2424+ name
2525+ "-j"
2626+ jobs
2727+ "@install"
2828+ "@runtest" {with-test}
2929+ "@doc" {with-doc}
3030+ ]
3131+]
3232+dev-repo: "https://tangled.org/gazagnaire.org/ocaml-btree"
+415-92
lib/btree.ml
···1616 else if shift >= 56 then
1717 (* 9th byte - use all 8 bits *)
1818 let byte9 = Char.code buf.[i + 1] in
1919- let acc = Int64.logor acc (Int64.shift_left (Int64.of_int byte9) 56) in
1919+ let acc =
2020+ Int64.logor acc (Int64.shift_left (Int64.of_int byte9) 56)
2121+ in
2022 (acc, i - off + 2)
2123 else loop acc (shift + 7) (i + 1)
2224 in
···4951end
50525153(* Page types *)
5252-type page_type =
5353- | Interior_index
5454- | Interior_table
5555- | Leaf_index
5656- | Leaf_table
5454+type page_type = Interior_index | Interior_table | Leaf_index | Leaf_table
57555856let pp_page_type ppf = function
5957 | Interior_index -> Format.pp_print_string ppf "interior_index"
···9290 right_child : int option;
9391}
94929595-let get_u16_be buf off =
9696- (Char.code buf.[off] lsl 8) lor Char.code buf.[off + 1]
9393+let get_u16_be buf off = (Char.code buf.[off] lsl 8) lor Char.code buf.[off + 1]
97949895let get_u32_be buf off =
9996 (Char.code buf.[off] lsl 24)
···141138 }
142139143140 type table_interior = { left_child : int; rowid : int64 }
144144-145141 type index_leaf = { payload : string; overflow_page : int option }
146142147143 type index_interior = {
···154150 let max_local ~usable_size ~is_table =
155151 if is_table then usable_size - 35 else ((usable_size - 12) * 64 / 255) - 23
156152157157- let min_local ~usable_size =
158158- ((usable_size - 12) * 32 / 255) - 23
153153+ let min_local ~usable_size = ((usable_size - 12) * 32 / 255) - 23
159154160155 let parse_table_leaf buf off ~usable_size =
161156 let payload_size, consumed1 = Varint.decode buf off in
···167162 let local_size, overflow_page =
168163 if payload_size <= max_local then (payload_size, None)
169164 else
170170- let k = min_local + ((payload_size - min_local) mod (usable_size - 4)) in
165165+ let k =
166166+ min_local + ((payload_size - min_local) mod (usable_size - 4))
167167+ in
171168 let local = if k <= max_local then k else min_local in
172169 let overflow = get_u32_be buf (off + header_len + local) in
173170 (local, Some overflow)
174171 in
175172 let payload = String.sub buf (off + header_len) local_size in
176173 let total_consumed =
177177- header_len + local_size + (if overflow_page = None then 0 else 4)
174174+ header_len + local_size + if overflow_page = None then 0 else 4
178175 in
179176 ({ rowid; payload; overflow_page }, total_consumed)
180177···191188 let local_size, overflow_page =
192189 if payload_size <= max_local then (payload_size, None)
193190 else
194194- let k = min_local + ((payload_size - min_local) mod (usable_size - 4)) in
191191+ let k =
192192+ min_local + ((payload_size - min_local) mod (usable_size - 4))
193193+ in
195194 let local = if k <= max_local then k else min_local in
196195 let overflow = get_u32_be buf (off + consumed + local) in
197196 (local, Some overflow)
198197 in
199198 let payload = String.sub buf (off + consumed) local_size in
200200- let total = consumed + local_size + (if overflow_page = None then 0 else 4) in
199199+ let total = consumed + local_size + if overflow_page = None then 0 else 4 in
201200 ({ payload; overflow_page }, total)
202201203202 let parse_index_interior buf off ~usable_size =
···209208 let local_size, overflow_page =
210209 if payload_size <= max_local then (payload_size, None)
211210 else
212212- let k = min_local + ((payload_size - min_local) mod (usable_size - 4)) in
211211+ let k =
212212+ min_local + ((payload_size - min_local) mod (usable_size - 4))
213213+ in
213214 let local = if k <= max_local then k else min_local in
214215 let overflow = get_u32_be buf (off + 4 + consumed + local) in
215216 (local, Some overflow)
216217 in
217218 let payload = String.sub buf (off + 4 + consumed) local_size in
218219 let total =
219219- 4 + consumed + local_size + (if overflow_page = None then 0 else 4)
220220+ 4 + consumed + local_size + if overflow_page = None then 0 else 4
220221 in
221222 ({ left_child; payload; overflow_page }, total)
222223end
···237238 | Blob of int
238239 | Text of int
239240240240- type value = Vnull | Vint of int64 | Vfloat of float | Vblob of string | Vtext of string
241241+ type value =
242242+ | Vnull
243243+ | Vint of int64
244244+ | Vfloat of float
245245+ | Vblob of string
246246+ | Vtext of string
241247242248 let serial_type_of_int = function
243249 | 0 -> Null
···254260 | n when n >= 13 -> Text ((n - 13) / 2)
255261 | n -> failwith (Printf.sprintf "Invalid serial type: %d" n)
256262257257- let serial_type_size = function
258258- | Null | Zero | One -> 0
259259- | Int8 -> 1
260260- | Int16 -> 2
261261- | Int24 -> 3
262262- | Int32 -> 4
263263- | Int48 -> 6
264264- | Int64 | Float64 -> 8
265265- | Blob n | Text n -> n
266266-267263 let decode_int buf off len =
268264 let rec loop acc i =
269265 if i >= len then acc
···329325 else if n >= -140737488355328L && n <= 140737488355327L then (5, 6)
330326 else (6, 8)
331327 | Vfloat _ -> (7, 8)
332332- | Vblob s -> (12 + String.length s * 2, String.length s)
333333- | Vtext s -> (13 + String.length s * 2, String.length s)
328328+ | Vblob s -> (12 + (String.length s * 2), String.length s)
329329+ | Vtext s -> (13 + (String.length s * 2), String.length s)
334330335331 let encode_int buf off n len =
336332 for i = 0 to len - 1 do
···348344 let header_body = String.concat "" header_types in
349345 let header_size = 1 + String.length header_body in
350346 (* header size varint + types *)
351351- let body_size = List.fold_left (fun acc (_, sz) -> acc + sz) 0 types_and_sizes in
347347+ let body_size =
348348+ List.fold_left (fun acc (_, sz) -> acc + sz) 0 types_and_sizes
349349+ in
352350 let total = header_size + body_size in
353351 let buf = Bytes.create total in
354352 (* Write header size *)
···425423 data)
426424427425 let write t page_num data =
428428- if String.length data <> t.page_size then
429429- failwith "Invalid page size";
426426+ if String.length data <> t.page_size then failwith "Invalid page size";
430427 Hashtbl.replace t.dirty page_num data;
431428 Hashtbl.replace t.cache page_num data
432429···451448module Table = struct
452449 type t = { pager : Pager.t; mutable root_page : int }
453450454454- let create pager =
455455- let root = Pager.allocate pager in
456456- (* Initialize as empty leaf page *)
457457- let page_size = Pager.page_size pager in
451451+ (* Initialize a page as empty leaf or interior *)
452452+ let init_page ~page_size ~page_type =
458453 let buf = Bytes.create page_size in
459459- Bytes.set_uint8 buf 0 (byte_of_page_type Leaf_table);
454454+ Bytes.set_uint8 buf 0 (byte_of_page_type page_type);
460455 set_u16_be buf 1 0;
461456 (* first freeblock *)
462457 set_u16_be buf 3 0;
···465460 (* cell content start *)
466461 Bytes.set_uint8 buf 7 0;
467462 (* fragmented bytes *)
463463+ if is_interior page_type then set_u32_be buf 8 0;
464464+ (* right child *)
465465+ buf
466466+467467+ let create pager =
468468+ let root = Pager.allocate pager in
469469+ let page_size = Pager.page_size pager in
470470+ let buf = init_page ~page_size ~page_type:Leaf_table in
468471 Pager.write pager root (Bytes.unsafe_to_string buf);
469472 { pager; root_page = root }
470473471474 let open_ pager ~root_page = { pager; root_page }
472475 let root_page t = t.root_page
473473-474476 let usable_size t = Pager.page_size t.pager
475477476478 (* Find cell pointers in a page *)
···478480 let ptrs = Array.make header.cell_count 0 in
479481 let ptr_start = header_offset + page_header_size header.page_type in
480482 for i = 0 to header.cell_count - 1 do
481481- ptrs.(i) <- get_u16_be page (ptr_start + i * 2)
483483+ ptrs.(i) <- get_u16_be page (ptr_start + (i * 2))
482484 done;
483485 ptrs
484486487487+ (* Calculate free space in a page *)
488488+ let free_space header ~page_type =
489489+ let header_size = page_header_size page_type in
490490+ let ptr_area_end = header_size + (header.cell_count * 2) in
491491+ header.cell_content_start - ptr_area_end - header.fragmented_bytes
492492+493493+ (* Encode a table leaf cell *)
494494+ let encode_table_leaf_cell ~rowid ~data =
495495+ let rowid_varint = Varint.encode rowid in
496496+ let payload_size_varint =
497497+ Varint.encode (Int64.of_int (String.length data))
498498+ in
499499+ let cell =
500500+ Bytes.create
501501+ (String.length payload_size_varint
502502+ + String.length rowid_varint + String.length data)
503503+ in
504504+ Bytes.blit_string payload_size_varint 0 cell 0
505505+ (String.length payload_size_varint);
506506+ Bytes.blit_string rowid_varint 0 cell
507507+ (String.length payload_size_varint)
508508+ (String.length rowid_varint);
509509+ Bytes.blit_string data 0 cell
510510+ (String.length payload_size_varint + String.length rowid_varint)
511511+ (String.length data);
512512+ Bytes.unsafe_to_string cell
513513+514514+ (* Encode a table interior cell *)
515515+ let encode_table_interior_cell ~left_child ~rowid =
516516+ let rowid_varint = Varint.encode rowid in
517517+ let cell = Bytes.create (4 + String.length rowid_varint) in
518518+ set_u32_be cell 0 left_child;
519519+ Bytes.blit_string rowid_varint 0 cell 4 (String.length rowid_varint);
520520+ Bytes.unsafe_to_string cell
521521+522522+ (* Write a cell into a page buffer, returns new cell_content_start *)
523523+ let write_cell buf ~cell_content_start ~cell =
524524+ let cell_len = String.length cell in
525525+ let new_start = cell_content_start - cell_len in
526526+ Bytes.blit_string cell 0 buf new_start cell_len;
527527+ new_start
528528+529529+ (* Insert a cell pointer at index, shifting others *)
530530+ let insert_cell_pointer buf ~header_offset ~page_type ~cell_count ~index ~ptr
531531+ =
532532+ let ptr_start = header_offset + page_header_size page_type in
533533+ (* Shift existing pointers right *)
534534+ for i = cell_count - 1 downto index do
535535+ let old_ptr =
536536+ get_u16_be (Bytes.unsafe_to_string buf) (ptr_start + (i * 2))
537537+ in
538538+ set_u16_be buf (ptr_start + ((i + 1) * 2)) old_ptr
539539+ done;
540540+ set_u16_be buf (ptr_start + (index * 2)) ptr
541541+485542 (* Binary search for rowid in leaf page *)
486543 let search_leaf t page header rowid =
487544 let ptrs = cell_pointers page 0 header in
···490547 if lo > hi then None
491548 else
492549 let mid = (lo + hi) / 2 in
493493- let cell, _ = Cell.parse_table_leaf page ptrs.(mid) ~usable_size:usable in
550550+ let cell, _ =
551551+ Cell.parse_table_leaf page ptrs.(mid) ~usable_size:usable
552552+ in
494553 if cell.rowid = rowid then Some cell.payload
495554 else if cell.rowid < rowid then loop (mid + 1) hi
496555 else loop lo (mid - 1)
497556 in
498557 loop 0 (header.cell_count - 1)
499558559559+ (* Find insertion index for rowid in leaf page *)
560560+ let find_insert_idx t page header rowid =
561561+ let ptrs = cell_pointers page 0 header in
562562+ let usable = usable_size t in
563563+ let rec find i =
564564+ if i >= header.cell_count then i
565565+ else
566566+ let cell, _ = Cell.parse_table_leaf page ptrs.(i) ~usable_size:usable in
567567+ if rowid < cell.rowid then i else find (i + 1)
568568+ in
569569+ find 0
570570+500571 (* Find child page for rowid in interior page *)
501501- let find_child t page header rowid =
572572+ let find_child _t page header rowid =
502573 let ptrs = cell_pointers page 0 header in
503574 let rec loop i =
504575 if i >= header.cell_count then Option.get header.right_child
···508579 in
509580 loop 0
510581582582+ (* Find child index for rowid in interior page *)
583583+ let find_child_idx page header rowid =
584584+ let ptrs = cell_pointers page 0 header in
585585+ let rec loop i =
586586+ if i >= header.cell_count then i (* right child *)
587587+ else
588588+ let cell, _ = Cell.parse_table_interior page ptrs.(i) in
589589+ if rowid <= cell.rowid then i else loop (i + 1)
590590+ in
591591+ loop 0
592592+511593 let rec find_in_page t page_num rowid =
512594 let page = Pager.read t.pager page_num in
513595 let header = parse_page_header page 0 in
···520602521603 let find t rowid = find_in_page t t.root_page rowid
522604523523- (* Insert - simplified, doesn't handle page splits *)
524524- let insert t ~rowid data =
525525- let page = Pager.read t.pager t.root_page in
605605+ (* Split result: new page number and separator rowid *)
606606+ type split_result = { new_page : int; separator_rowid : int64 }
607607+608608+ (* Split a leaf page, returns info about the new page *)
609609+ let split_leaf t page_num =
610610+ let page = Pager.read t.pager page_num in
611611+ let header = parse_page_header page 0 in
612612+ let ptrs = cell_pointers page 0 header in
613613+ let usable = usable_size t in
614614+ let page_size = Pager.page_size t.pager in
615615+616616+ (* Find split point (middle) *)
617617+ let split_idx = header.cell_count / 2 in
618618+619619+ (* Get separator rowid (first key that goes to right page) *)
620620+ let sep_cell, _ =
621621+ Cell.parse_table_leaf page ptrs.(split_idx) ~usable_size:usable
622622+ in
623623+ let separator_rowid = sep_cell.rowid in
624624+625625+ (* Create new right page *)
626626+ let new_page_num = Pager.allocate t.pager in
627627+ let new_buf = init_page ~page_size ~page_type:Leaf_table in
628628+629629+ (* Copy cells [split_idx..cell_count-1] to new page *)
630630+ let new_cell_content_start = ref page_size in
631631+ for i = split_idx to header.cell_count - 1 do
632632+ let cell_off = ptrs.(i) in
633633+ let cell, cell_len =
634634+ Cell.parse_table_leaf page cell_off ~usable_size:usable
635635+ in
636636+ let cell_data =
637637+ encode_table_leaf_cell ~rowid:cell.rowid ~data:cell.payload
638638+ in
639639+ new_cell_content_start :=
640640+ write_cell new_buf ~cell_content_start:!new_cell_content_start
641641+ ~cell:cell_data;
642642+ let new_idx = i - split_idx in
643643+ let ptr_off = page_header_size Leaf_table + (new_idx * 2) in
644644+ set_u16_be new_buf ptr_off !new_cell_content_start;
645645+ ignore cell_len
646646+ done;
647647+ let new_cell_count = header.cell_count - split_idx in
648648+ set_u16_be new_buf 3 new_cell_count;
649649+ set_u16_be new_buf 5 !new_cell_content_start;
650650+ Pager.write t.pager new_page_num (Bytes.unsafe_to_string new_buf);
651651+652652+ (* Update original page to only have cells [0..split_idx-1] *)
653653+ let old_buf = Bytes.of_string page in
654654+ set_u16_be old_buf 3 split_idx;
655655+ (* Recalculate cell content start for remaining cells *)
656656+ if split_idx > 0 then begin
657657+ let min_ptr = ref page_size in
658658+ for i = 0 to split_idx - 1 do
659659+ if ptrs.(i) < !min_ptr then min_ptr := ptrs.(i)
660660+ done;
661661+ set_u16_be old_buf 5 !min_ptr
662662+ end
663663+ else set_u16_be old_buf 5 page_size;
664664+ Pager.write t.pager page_num (Bytes.unsafe_to_string old_buf);
665665+666666+ { new_page = new_page_num; separator_rowid }
667667+668668+ (* Split an interior page *)
669669+ let split_interior t page_num =
670670+ let page = Pager.read t.pager page_num in
671671+ let header = parse_page_header page 0 in
672672+ let ptrs = cell_pointers page 0 header in
673673+ let page_size = Pager.page_size t.pager in
674674+675675+ (* Split point - middle cell becomes separator, doesn't go to either page *)
676676+ let split_idx = header.cell_count / 2 in
677677+ let sep_cell, _ = Cell.parse_table_interior page ptrs.(split_idx) in
678678+ let separator_rowid = sep_cell.rowid in
679679+680680+ (* Create new right page *)
681681+ let new_page_num = Pager.allocate t.pager in
682682+ let new_buf = init_page ~page_size ~page_type:Interior_table in
683683+684684+ (* The right child of split cell becomes the left-most child of new page *)
685685+ (* Cells [split_idx+1..cell_count-1] go to new page *)
686686+ let new_cell_content_start = ref page_size in
687687+ for i = split_idx + 1 to header.cell_count - 1 do
688688+ let cell_off = ptrs.(i) in
689689+ let cell, _ = Cell.parse_table_interior page cell_off in
690690+ let cell_data =
691691+ encode_table_interior_cell ~left_child:cell.left_child ~rowid:cell.rowid
692692+ in
693693+ new_cell_content_start :=
694694+ write_cell new_buf ~cell_content_start:!new_cell_content_start
695695+ ~cell:cell_data;
696696+ let new_idx = i - split_idx - 1 in
697697+ let ptr_off = page_header_size Interior_table + (new_idx * 2) in
698698+ set_u16_be new_buf ptr_off !new_cell_content_start
699699+ done;
700700+ let new_cell_count = header.cell_count - split_idx - 1 in
701701+ set_u16_be new_buf 3 new_cell_count;
702702+ set_u16_be new_buf 5 !new_cell_content_start;
703703+ (* Right child of new page is same as original *)
704704+ set_u32_be new_buf 8 (Option.get header.right_child);
705705+ Pager.write t.pager new_page_num (Bytes.unsafe_to_string new_buf);
706706+707707+ (* Update original page: keep cells [0..split_idx-1], right child = sep_cell.left_child *)
708708+ let old_buf = Bytes.of_string page in
709709+ set_u16_be old_buf 3 split_idx;
710710+ set_u32_be old_buf 8 sep_cell.left_child;
711711+ if split_idx > 0 then begin
712712+ let min_ptr = ref page_size in
713713+ for i = 0 to split_idx - 1 do
714714+ if ptrs.(i) < !min_ptr then min_ptr := ptrs.(i)
715715+ done;
716716+ set_u16_be old_buf 5 !min_ptr
717717+ end
718718+ else set_u16_be old_buf 5 page_size;
719719+ Pager.write t.pager page_num (Bytes.unsafe_to_string old_buf);
720720+721721+ { new_page = new_page_num; separator_rowid }
722722+723723+ (* Insert a separator into an interior page, potentially splitting *)
724724+ let rec insert_into_interior t page_num ~left_child ~separator_rowid
725725+ ~right_child =
726726+ let page = Pager.read t.pager page_num in
727727+ let header = parse_page_header page 0 in
728728+ let cell = encode_table_interior_cell ~left_child ~rowid:separator_rowid in
729729+ let cell_len = String.length cell in
730730+ let space_needed = cell_len + 2 in
731731+ (* cell + pointer *)
732732+733733+ if free_space header ~page_type:Interior_table >= space_needed then begin
734734+ (* Fits - insert directly *)
735735+ let buf = Bytes.of_string page in
736736+ let ptrs = cell_pointers page 0 header in
737737+738738+ (* Find insert position *)
739739+ let insert_idx =
740740+ let rec find i =
741741+ if i >= header.cell_count then i
742742+ else
743743+ let c, _ = Cell.parse_table_interior page ptrs.(i) in
744744+ if separator_rowid < c.rowid then i else find (i + 1)
745745+ in
746746+ find 0
747747+ in
748748+749749+ (* Write cell *)
750750+ let cell_start =
751751+ write_cell buf ~cell_content_start:header.cell_content_start ~cell
752752+ in
753753+ set_u16_be buf 5 cell_start;
754754+755755+ (* Insert pointer *)
756756+ insert_cell_pointer buf ~header_offset:0 ~page_type:Interior_table
757757+ ~cell_count:header.cell_count ~index:insert_idx ~ptr:cell_start;
758758+759759+ (* Update cell count *)
760760+ set_u16_be buf 3 (header.cell_count + 1);
761761+762762+ (* Update right child if inserting at end *)
763763+ if insert_idx = header.cell_count then set_u32_be buf 8 right_child;
764764+765765+ Pager.write t.pager page_num (Bytes.unsafe_to_string buf);
766766+ None
767767+ end
768768+ else begin
769769+ (* Need to split interior page *)
770770+ let split = split_interior t page_num in
771771+772772+ (* Determine which page gets the new separator *)
773773+ if separator_rowid < split.separator_rowid then begin
774774+ (* Insert into left (original) page *)
775775+ ignore
776776+ (insert_into_interior t page_num ~left_child ~separator_rowid
777777+ ~right_child)
778778+ end
779779+ else begin
780780+ (* Insert into right (new) page *)
781781+ ignore
782782+ (insert_into_interior t split.new_page ~left_child ~separator_rowid
783783+ ~right_child)
784784+ end;
785785+786786+ (* Return split info to propagate up *)
787787+ Some { split with separator_rowid = split.separator_rowid }
788788+ end
789789+790790+ (* Insert into a leaf page, potentially splitting *)
791791+ let rec insert_into_leaf t page_num ~rowid ~data ~parent_stack =
792792+ let page = Pager.read t.pager page_num in
526793 let header = parse_page_header page 0 in
527527- match header.page_type with
528528- | Leaf_table ->
794794+ let cell = encode_table_leaf_cell ~rowid ~data in
795795+ let cell_len = String.length cell in
796796+ let space_needed = cell_len + 2 in
797797+ (* cell + pointer *)
798798+799799+ if free_space header ~page_type:Leaf_table >= space_needed then begin
800800+ (* Fits - insert directly *)
801801+ let buf = Bytes.of_string page in
802802+ let insert_idx = find_insert_idx t page header rowid in
803803+804804+ (* Write cell *)
805805+ let cell_start =
806806+ write_cell buf ~cell_content_start:header.cell_content_start ~cell
807807+ in
808808+ set_u16_be buf 5 cell_start;
809809+810810+ (* Insert pointer *)
811811+ insert_cell_pointer buf ~header_offset:0 ~page_type:Leaf_table
812812+ ~cell_count:header.cell_count ~index:insert_idx ~ptr:cell_start;
813813+814814+ (* Update cell count *)
815815+ set_u16_be buf 3 (header.cell_count + 1);
816816+817817+ Pager.write t.pager page_num (Bytes.unsafe_to_string buf)
818818+ end
819819+ else begin
820820+ (* Need to split *)
821821+ let split = split_leaf t page_num in
822822+823823+ (* Insert into appropriate page *)
824824+ if rowid < split.separator_rowid then
825825+ insert_into_leaf t page_num ~rowid ~data ~parent_stack:[]
826826+ else insert_into_leaf t split.new_page ~rowid ~data ~parent_stack:[];
827827+828828+ (* Propagate split up *)
829829+ propagate_split t ~parent_stack ~left_page:page_num
830830+ ~separator_rowid:split.separator_rowid ~right_page:split.new_page
831831+ end
832832+833833+ and propagate_split t ~parent_stack ~left_page ~separator_rowid ~right_page =
834834+ match parent_stack with
835835+ | [] ->
836836+ (* Splitting root - create new root *)
529837 let page_size = Pager.page_size t.pager in
530530- let buf = Bytes.of_string page in
531531- (* Create cell *)
532532- let rowid_varint = Varint.encode rowid in
533533- let payload_size_varint = Varint.encode (Int64.of_int (String.length data)) in
534534- let cell_size =
535535- String.length payload_size_varint + String.length rowid_varint + String.length data
838838+ let new_root = Pager.allocate t.pager in
839839+ let buf = init_page ~page_size ~page_type:Interior_table in
840840+841841+ (* Single cell pointing to left page *)
842842+ let cell =
843843+ encode_table_interior_cell ~left_child:left_page
844844+ ~rowid:separator_rowid
536845 in
537537- (* Find insertion point *)
538538- let ptrs = cell_pointers page 0 header in
539539- let usable = usable_size t in
540540- let insert_idx =
541541- let rec find i =
542542- if i >= header.cell_count then i
846846+ let cell_start = write_cell buf ~cell_content_start:page_size ~cell in
847847+ set_u16_be buf 5 cell_start;
848848+ set_u16_be buf (page_header_size Interior_table) cell_start;
849849+ set_u16_be buf 3 1;
850850+ set_u32_be buf 8 right_page;
851851+852852+ Pager.write t.pager new_root (Bytes.unsafe_to_string buf);
853853+ t.root_page <- new_root
854854+ | parent_page :: rest -> (
855855+ match
856856+ insert_into_interior t parent_page ~left_child:left_page
857857+ ~separator_rowid ~right_child:right_page
858858+ with
859859+ | None -> () (* Fit in parent *)
860860+ | Some split ->
861861+ (* Parent also split, propagate up *)
862862+ propagate_split t ~parent_stack:rest ~left_page:parent_page
863863+ ~separator_rowid:split.separator_rowid ~right_page:split.new_page)
864864+865865+ (* Main insert - traverses tree and handles splits *)
866866+ let insert t ~rowid data =
867867+ let rec traverse page_num parent_stack =
868868+ let page = Pager.read t.pager page_num in
869869+ let header = parse_page_header page 0 in
870870+ match header.page_type with
871871+ | Leaf_table -> insert_into_leaf t page_num ~rowid ~data ~parent_stack
872872+ | Interior_table ->
873873+ let child_idx = find_child_idx page header rowid in
874874+ let child_page =
875875+ if child_idx >= header.cell_count then Option.get header.right_child
543876 else
544544- let cell, _ = Cell.parse_table_leaf page ptrs.(i) ~usable_size:usable in
545545- if rowid < cell.rowid then i else find (i + 1)
877877+ let ptrs = cell_pointers page 0 header in
878878+ let cell, _ = Cell.parse_table_interior page ptrs.(child_idx) in
879879+ cell.left_child
546880 in
547547- find 0
548548- in
549549- (* Write cell at end of cell content area *)
550550- let cell_start = header.cell_content_start - cell_size in
551551- Bytes.blit_string payload_size_varint 0 buf cell_start
552552- (String.length payload_size_varint);
553553- let off = cell_start + String.length payload_size_varint in
554554- Bytes.blit_string rowid_varint 0 buf off (String.length rowid_varint);
555555- let off = off + String.length rowid_varint in
556556- Bytes.blit_string data 0 buf off (String.length data);
557557- (* Update cell content start *)
558558- set_u16_be buf 5 cell_start;
559559- (* Insert cell pointer *)
560560- let ptr_area_start = page_header_size Leaf_table in
561561- (* Shift existing pointers *)
562562- for i = header.cell_count - 1 downto insert_idx do
563563- set_u16_be buf (ptr_area_start + (i + 1) * 2) (get_u16_be page (ptr_area_start + i * 2))
564564- done;
565565- set_u16_be buf (ptr_area_start + insert_idx * 2) cell_start;
566566- (* Update cell count *)
567567- set_u16_be buf 3 (header.cell_count + 1);
568568- Pager.write t.pager t.root_page (Bytes.unsafe_to_string buf)
569569- | _ -> failwith "Insert into interior page not yet implemented"
881881+ traverse child_page (page_num :: parent_stack)
882882+ | _ -> failwith "Invalid page type in table B-tree"
883883+ in
884884+ traverse t.root_page []
570885571886 let delete _t _rowid = failwith "Delete not yet implemented"
572887···579894 match header.page_type with
580895 | Leaf_table ->
581896 for i = 0 to header.cell_count - 1 do
582582- let cell, _ = Cell.parse_table_leaf page ptrs.(i) ~usable_size:usable in
897897+ let cell, _ =
898898+ Cell.parse_table_leaf page ptrs.(i) ~usable_size:usable
899899+ in
583900 f cell.rowid cell.payload
584901 done
585902 | Interior_table ->
···616933617934 let open_ pager ~root_page = { pager; root_page }
618935 let root_page t = t.root_page
619619-620936 let usable_size t = Pager.page_size t.pager
621937622938 let cell_pointers page header =
623939 let ptrs = Array.make header.cell_count 0 in
624940 let ptr_start = page_header_size header.page_type in
625941 for i = 0 to header.cell_count - 1 do
626626- ptrs.(i) <- get_u16_be page (ptr_start + i * 2)
942942+ ptrs.(i) <- get_u16_be page (ptr_start + (i * 2))
627943 done;
628944 ptrs
629945···637953 let rec search i =
638954 if i >= header.cell_count then false
639955 else
640640- let cell, _ = Cell.parse_index_leaf page ptrs.(i) ~usable_size:usable in
956956+ let cell, _ =
957957+ Cell.parse_index_leaf page ptrs.(i) ~usable_size:usable
958958+ in
641959 if cell.payload = key then true
642960 else if cell.payload > key then false
643961 else search (i + 1)
···647965 let rec find_child i =
648966 if i >= header.cell_count then Option.get header.right_child
649967 else
650650- let cell, _ = Cell.parse_index_interior page ptrs.(i) ~usable_size:usable in
968968+ let cell, _ =
969969+ Cell.parse_index_interior page ptrs.(i) ~usable_size:usable
970970+ in
651971 if key <= cell.payload then cell.left_child else find_child (i + 1)
652972 in
653973 mem_in_page t (find_child 0) key
654974 | _ -> failwith "Invalid page type in index B-tree"
655975656976 let mem t key = mem_in_page t t.root_page key
657657-658977 let insert _t _key = failwith "Index insert not yet implemented"
659978 let delete _t _key = failwith "Index delete not yet implemented"
660979···667986 match header.page_type with
668987 | Leaf_index ->
669988 for i = 0 to header.cell_count - 1 do
670670- let cell, _ = Cell.parse_index_leaf page ptrs.(i) ~usable_size:usable in
989989+ let cell, _ =
990990+ Cell.parse_index_leaf page ptrs.(i) ~usable_size:usable
991991+ in
671992 f cell.payload
672993 done
673994 | Interior_index ->
674995 for i = 0 to header.cell_count - 1 do
675675- let cell, _ = Cell.parse_index_interior page ptrs.(i) ~usable_size:usable in
996996+ let cell, _ =
997997+ Cell.parse_index_interior page ptrs.(i) ~usable_size:usable
998998+ in
676999 iter_page cell.left_child
6771000 done;
6781001 Option.iter iter_page header.right_child
+16-20
lib/btree.mli
···13131414module Varint : sig
1515 val decode : string -> int -> int64 * int
1616- (** [decode buf off] decodes a varint starting at [off] in [buf].
1717- Returns [(value, bytes_consumed)]. *)
1616+ (** [decode buf off] decodes a varint starting at [off] in [buf]. Returns
1717+ [(value, bytes_consumed)]. *)
18181919 val encode : int64 -> string
2020 (** [encode n] encodes [n] as a varint. *)
···2828type page_type =
2929 | Interior_index (** 0x02 *)
3030 | Interior_table (** 0x05 *)
3131- | Leaf_index (** 0x0a *)
3232- | Leaf_table (** 0x0d *)
3131+ | Leaf_index (** 0x0a *)
3232+ | Leaf_table (** 0x0d *)
33333434val pp_page_type : Format.formatter -> page_type -> unit
3535···4545}
46464747val parse_page_header : string -> int -> page_header
4848-(** [parse_page_header buf off] parses a page header starting at [off].
4949- For page 1, [off] should be 100 (after database header). *)
4848+(** [parse_page_header buf off] parses a page header starting at [off]. For page
4949+ 1, [off] should be 100 (after database header). *)
50505151val page_header_size : page_type -> int
5252(** [page_header_size typ] is 8 for leaf pages, 12 for interior pages. *)
···5656 Cells contain the actual data in B-tree pages. *)
57575858module Cell : sig
5959- (** Table leaf cell: rowid + payload *)
6059 type table_leaf = {
6160 rowid : int64;
6261 payload : string;
6362 overflow_page : int option;
6463 }
6464+ (** Table leaf cell: rowid + payload *)
65656666+ type table_interior = { left_child : int; rowid : int64 }
6667 (** Table interior cell: child page + rowid *)
6767- type table_interior = {
6868- left_child : int;
6969- rowid : int64;
7070- }
71686969+ type index_leaf = { payload : string; overflow_page : int option }
7270 (** Index leaf cell: payload only *)
7373- type index_leaf = {
7474- payload : string;
7575- overflow_page : int option;
7676- }
77717878- (** Index interior cell: child page + payload *)
7972 type index_interior = {
8073 left_child : int;
8174 payload : string;
8275 overflow_page : int option;
8376 }
7777+ (** Index interior cell: child page + payload *)
84788579 val parse_table_leaf : string -> int -> usable_size:int -> table_leaf * int
8686- (** [parse_table_leaf buf off ~usable_size] parses a table leaf cell.
8787- Returns [(cell, bytes_consumed)]. *)
8080+ (** [parse_table_leaf buf off ~usable_size] parses a table leaf cell. Returns
8181+ [(cell, bytes_consumed)]. *)
88828983 val parse_table_interior : string -> int -> table_interior * int
9084 (** [parse_table_interior buf off] parses a table interior cell. *)
···9286 val parse_index_leaf : string -> int -> usable_size:int -> index_leaf * int
9387 (** [parse_index_leaf buf off ~usable_size] parses an index leaf cell. *)
94889595- val parse_index_interior : string -> int -> usable_size:int -> index_interior * int
9696- (** [parse_index_interior buf off ~usable_size] parses an index interior cell. *)
8989+ val parse_index_interior :
9090+ string -> int -> usable_size:int -> index_interior * int
9191+ (** [parse_index_interior buf off ~usable_size] parses an index interior cell.
9292+ *)
9793end
98949995(** {1 Record Format}
+231-10
test/test_btree.ml
···97979898let test_page_header_leaf () =
9999 let page = Bytes.create 4096 in
100100- Bytes.set_uint8 page 0 0x0d; (* Leaf table *)
100100+ Bytes.set_uint8 page 0 0x0d;
101101+ (* Leaf table *)
101102 Bytes.set_uint8 page 1 0;
102102- Bytes.set_uint8 page 2 0; (* first freeblock = 0 *)
103103+ Bytes.set_uint8 page 2 0;
104104+ (* first freeblock = 0 *)
103105 Bytes.set_uint8 page 3 0;
104104- Bytes.set_uint8 page 4 5; (* cell count = 5 *)
106106+ Bytes.set_uint8 page 4 5;
107107+ (* cell count = 5 *)
105108 Bytes.set_uint8 page 5 0x0f;
106106- Bytes.set_uint8 page 6 0x00; (* cell content start = 0x0f00 *)
107107- Bytes.set_uint8 page 7 0; (* fragmented = 0 *)
109109+ Bytes.set_uint8 page 6 0x00;
110110+ (* cell content start = 0x0f00 *)
111111+ Bytes.set_uint8 page 7 0;
112112+ (* fragmented = 0 *)
108113 let header = Btree.parse_page_header (Bytes.unsafe_to_string page) 0 in
109114 Alcotest.(check int) "cell_count" 5 header.cell_count;
110115 Alcotest.(check int) "cell_content_start" 0x0f00 header.cell_content_start;
111111- Alcotest.(check bool) "no right_child" true (Option.is_none header.right_child)
116116+ Alcotest.(check bool)
117117+ "no right_child" true
118118+ (Option.is_none header.right_child)
112119113120let test_page_header_interior () =
114121 let page = Bytes.create 4096 in
115115- Bytes.set_uint8 page 0 0x05; (* Interior table *)
122122+ Bytes.set_uint8 page 0 0x05;
123123+ (* Interior table *)
116124 Bytes.set_uint8 page 1 0;
117125 Bytes.set_uint8 page 2 0;
118126 Bytes.set_uint8 page 3 0;
···165173 Btree.Table.insert tree ~rowid:3L "Three";
166174 Btree.Table.insert tree ~rowid:1L "One";
167175 Btree.Table.insert tree ~rowid:2L "Two";
168168- Alcotest.(check (option string)) "find 1" (Some "One") (Btree.Table.find tree 1L);
169169- Alcotest.(check (option string)) "find 2" (Some "Two") (Btree.Table.find tree 2L);
170170- Alcotest.(check (option string)) "find 3" (Some "Three") (Btree.Table.find tree 3L);
176176+ Alcotest.(check (option string))
177177+ "find 1" (Some "One") (Btree.Table.find tree 1L);
178178+ Alcotest.(check (option string))
179179+ "find 2" (Some "Two") (Btree.Table.find tree 2L);
180180+ Alcotest.(check (option string))
181181+ "find 3" (Some "Three") (Btree.Table.find tree 3L);
171182 Alcotest.(check (option string)) "find 4" None (Btree.Table.find tree 4L)
172183173184let test_table_iter () =
···185196 [ (1L, "A"); (2L, "B"); (3L, "C") ]
186197 sorted
187198199199+(* Page split tests *)
200200+201201+let test_leaf_split () =
202202+ (* Use small page size to trigger splits quickly *)
203203+ with_temp_file @@ fun file ->
204204+ let pager = Btree.Pager.create ~page_size:512 file in
205205+ let tree = Btree.Table.create pager in
206206+ (* Insert enough records to trigger a split *)
207207+ (* With 512 byte pages, ~20-30 small records should trigger a split *)
208208+ for i = 1 to 50 do
209209+ let data = Printf.sprintf "value_%d" i in
210210+ Btree.Table.insert tree ~rowid:(Int64.of_int i) data
211211+ done;
212212+ (* Verify all records are findable *)
213213+ for i = 1 to 50 do
214214+ let expected = Printf.sprintf "value_%d" i in
215215+ let actual = Btree.Table.find tree (Int64.of_int i) in
216216+ Alcotest.(check (option string))
217217+ (Printf.sprintf "find %d" i)
218218+ (Some expected) actual
219219+ done;
220220+ (* Verify iteration returns all records in order *)
221221+ let items = ref [] in
222222+ Btree.Table.iter tree (fun rowid data -> items := (rowid, data) :: !items);
223223+ Alcotest.(check int) "count after split" 50 (List.length !items)
224224+225225+let test_multi_level_tree () =
226226+ (* Force multiple levels by inserting many records *)
227227+ with_temp_file @@ fun file ->
228228+ let pager = Btree.Pager.create ~page_size:512 file in
229229+ let tree = Btree.Table.create pager in
230230+ let n = 200 in
231231+ for i = 1 to n do
232232+ Btree.Table.insert tree ~rowid:(Int64.of_int i) (Printf.sprintf "v%d" i)
233233+ done;
234234+ (* Verify all records *)
235235+ for i = 1 to n do
236236+ let r = Btree.Table.find tree (Int64.of_int i) in
237237+ Alcotest.(check (option string))
238238+ (Printf.sprintf "find %d" i)
239239+ (Some (Printf.sprintf "v%d" i))
240240+ r
241241+ done;
242242+ (* Check count *)
243243+ let count = Btree.Table.fold tree ~init:0 ~f:(fun _ _ acc -> acc + 1) in
244244+ Alcotest.(check int) "total count" n count
245245+246246+let test_reverse_insert_order () =
247247+ (* Insert in reverse order to stress different split patterns *)
248248+ with_temp_file @@ fun file ->
249249+ let pager = Btree.Pager.create ~page_size:512 file in
250250+ let tree = Btree.Table.create pager in
251251+ for i = 100 downto 1 do
252252+ Btree.Table.insert tree ~rowid:(Int64.of_int i) (Printf.sprintf "r%d" i)
253253+ done;
254254+ for i = 1 to 100 do
255255+ let r = Btree.Table.find tree (Int64.of_int i) in
256256+ Alcotest.(check (option string))
257257+ (Printf.sprintf "find %d" i)
258258+ (Some (Printf.sprintf "r%d" i))
259259+ r
260260+ done
261261+262262+let test_random_insert_order () =
263263+ (* Pseudo-random insertion order *)
264264+ with_temp_file @@ fun file ->
265265+ let pager = Btree.Pager.create ~page_size:512 file in
266266+ let tree = Btree.Table.create pager in
267267+ (* LCG for deterministic "random" order *)
268268+ let shuffle n =
269269+ let a = Array.init n (fun i -> i + 1) in
270270+ let state = ref 42 in
271271+ for i = n - 1 downto 1 do
272272+ state := ((!state * 1103515245) + 12345) land 0x7fffffff;
273273+ let j = !state mod (i + 1) in
274274+ let tmp = a.(i) in
275275+ a.(i) <- a.(j);
276276+ a.(j) <- tmp
277277+ done;
278278+ a
279279+ in
280280+ let order = shuffle 150 in
281281+ Array.iter
282282+ (fun i ->
283283+ Btree.Table.insert tree ~rowid:(Int64.of_int i) (Printf.sprintf "x%d" i))
284284+ order;
285285+ for i = 1 to 150 do
286286+ let r = Btree.Table.find tree (Int64.of_int i) in
287287+ Alcotest.(check (option string))
288288+ (Printf.sprintf "find %d" i)
289289+ (Some (Printf.sprintf "x%d" i))
290290+ r
291291+ done
292292+293293+let test_large_values () =
294294+ (* Test with larger values that take more space *)
295295+ with_temp_file @@ fun file ->
296296+ let pager = Btree.Pager.create ~page_size:1024 file in
297297+ let tree = Btree.Table.create pager in
298298+ for i = 1 to 30 do
299299+ let data = String.make 50 (Char.chr (65 + (i mod 26))) in
300300+ Btree.Table.insert tree ~rowid:(Int64.of_int i) data
301301+ done;
302302+ for i = 1 to 30 do
303303+ let expected = String.make 50 (Char.chr (65 + (i mod 26))) in
304304+ let r = Btree.Table.find tree (Int64.of_int i) in
305305+ Alcotest.(check (option string))
306306+ (Printf.sprintf "find %d" i)
307307+ (Some expected) r
308308+ done
309309+310310+(* SQLite varint test vectors from spec *)
311311+312312+let test_varint_sqlite_vectors () =
313313+ (* Test vectors from SQLite file format spec *)
314314+ let test_encode n expected_bytes =
315315+ let encoded = Btree.Varint.encode n in
316316+ let bytes =
317317+ List.init (String.length encoded) (fun i -> Char.code encoded.[i])
318318+ in
319319+ Alcotest.(check (list int))
320320+ (Printf.sprintf "encode %Ld" n)
321321+ expected_bytes bytes
322322+ in
323323+ (* Single byte: 0-127 *)
324324+ test_encode 0L [ 0x00 ];
325325+ test_encode 1L [ 0x01 ];
326326+ test_encode 127L [ 0x7f ];
327327+ (* Two bytes: 128-16383 *)
328328+ test_encode 128L [ 0x80; 0x01 ];
329329+ test_encode 255L [ 0xff; 0x01 ];
330330+ test_encode 16383L [ 0xff; 0x7f ]
331331+332332+let test_varint_boundaries () =
333333+ (* Test at varint size boundaries *)
334334+ let boundaries =
335335+ [
336336+ (0L, 1);
337337+ (127L, 1);
338338+ (128L, 2);
339339+ (16383L, 2);
340340+ (16384L, 3);
341341+ (2097151L, 3);
342342+ (2097152L, 4);
343343+ (268435455L, 4);
344344+ (268435456L, 5);
345345+ ]
346346+ in
347347+ List.iter
348348+ (fun (n, expected_size) ->
349349+ let encoded = Btree.Varint.encode n in
350350+ Alcotest.(check int)
351351+ (Printf.sprintf "size of %Ld" n)
352352+ expected_size (String.length encoded);
353353+ let decoded, consumed = Btree.Varint.decode encoded 0 in
354354+ Alcotest.(check int64) (Printf.sprintf "decode %Ld" n) n decoded;
355355+ Alcotest.(check int)
356356+ (Printf.sprintf "consumed for %Ld" n)
357357+ expected_size consumed)
358358+ boundaries
359359+360360+(* Record format test vectors *)
361361+362362+let test_record_serial_types () =
363363+ (* Test specific serial type encodings *)
364364+ let open Btree.Record in
365365+ (* Zero and One are special-cased *)
366366+ let r0 = encode [ Vint 0L ] in
367367+ let d0 = decode r0 in
368368+ (match d0 with [ Vint 0L ] -> () | _ -> Alcotest.fail "expected 0");
369369+ let r1 = encode [ Vint 1L ] in
370370+ let d1 = decode r1 in
371371+ (match d1 with [ Vint 1L ] -> () | _ -> Alcotest.fail "expected 1");
372372+ (* Int8 range *)
373373+ let r127 = encode [ Vint 127L ] in
374374+ let d127 = decode r127 in
375375+ (match d127 with [ Vint 127L ] -> () | _ -> Alcotest.fail "expected 127");
376376+ (* Negative *)
377377+ let rm1 = encode [ Vint (-1L) ] in
378378+ let dm1 = decode rm1 in
379379+ match dm1 with [ Vint -1L ] -> () | _ -> Alcotest.fail "expected -1"
380380+381381+let test_iter_order_after_splits () =
382382+ (* Verify iteration order is correct after splits *)
383383+ with_temp_file @@ fun file ->
384384+ let pager = Btree.Pager.create ~page_size:512 file in
385385+ let tree = Btree.Table.create pager in
386386+ let n = 100 in
387387+ for i = 1 to n do
388388+ Btree.Table.insert tree ~rowid:(Int64.of_int i) (string_of_int i)
389389+ done;
390390+ let prev = ref 0L in
391391+ Btree.Table.iter tree (fun rowid _ ->
392392+ if rowid <= !prev then
393393+ Alcotest.fail (Printf.sprintf "out of order: %Ld after %Ld" rowid !prev);
394394+ prev := rowid);
395395+ Alcotest.(check int64) "last rowid" (Int64.of_int n) !prev
396396+188397let suite =
189398 [
190399 ( "varint",
191400 [
192401 Alcotest.test_case "roundtrip" `Quick test_varint_roundtrip;
193402 Alcotest.test_case "size" `Quick test_varint_size;
403403+ Alcotest.test_case "sqlite_vectors" `Quick test_varint_sqlite_vectors;
404404+ Alcotest.test_case "boundaries" `Quick test_varint_boundaries;
194405 ] );
195406 ( "record",
196407 [
···199410 Alcotest.test_case "text" `Quick test_record_text;
200411 Alcotest.test_case "blob" `Quick test_record_blob;
201412 Alcotest.test_case "mixed" `Quick test_record_mixed;
413413+ Alcotest.test_case "serial_types" `Quick test_record_serial_types;
202414 ] );
203415 ( "page_header",
204416 [
···211423 Alcotest.test_case "insert/find" `Quick test_table_insert_find;
212424 Alcotest.test_case "multiple inserts" `Quick test_table_multiple_inserts;
213425 Alcotest.test_case "iter" `Quick test_table_iter;
426426+ ] );
427427+ ( "splits",
428428+ [
429429+ Alcotest.test_case "leaf_split" `Quick test_leaf_split;
430430+ Alcotest.test_case "multi_level" `Quick test_multi_level_tree;
431431+ Alcotest.test_case "reverse_order" `Quick test_reverse_insert_order;
432432+ Alcotest.test_case "random_order" `Quick test_random_insert_order;
433433+ Alcotest.test_case "large_values" `Quick test_large_values;
434434+ Alcotest.test_case "iter_order" `Quick test_iter_order_after_splits;
214435 ] );
215436 ]