Pure OCaml B-tree implementation for persistent storage
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

pds, btree, sqlite: Add opam files, fix formatting, remove unused code

+695 -123
+1 -1
.ocamlformat
··· 1 - version=0.27.0 1 + version = 0.28.1
+32
btree.opam
··· 1 + # This file is generated by dune, edit dune-project instead 2 + opam-version: "2.0" 3 + synopsis: "Pure OCaml B-tree implementation for persistent storage" 4 + description: 5 + "A B-tree implementation supporting both table B-trees (integer keys with data) and index B-trees (arbitrary keys). Designed for SQLite-compatible persistent storage with page-based I/O." 6 + maintainer: ["Thomas Gazagnaire"] 7 + authors: ["Thomas Gazagnaire"] 8 + license: "MIT" 9 + depends: [ 10 + "dune" {>= "3.0"} 11 + "ocaml" {>= "5.1"} 12 + "eio" {>= "1.0"} 13 + "cstruct" 14 + "alcotest" {with-test} 15 + "eio_main" {with-test} 16 + "odoc" {with-doc} 17 + ] 18 + build: [ 19 + ["dune" "subst"] {dev} 20 + [ 21 + "dune" 22 + "build" 23 + "-p" 24 + name 25 + "-j" 26 + jobs 27 + "@install" 28 + "@runtest" {with-test} 29 + "@doc" {with-doc} 30 + ] 31 + ] 32 + dev-repo: "https://tangled.org/gazagnaire.org/ocaml-btree"
+415 -92
lib/btree.ml
··· 16 16 else if shift >= 56 then 17 17 (* 9th byte - use all 8 bits *) 18 18 let byte9 = Char.code buf.[i + 1] in 19 - let acc = Int64.logor acc (Int64.shift_left (Int64.of_int byte9) 56) in 19 + let acc = 20 + Int64.logor acc (Int64.shift_left (Int64.of_int byte9) 56) 21 + in 20 22 (acc, i - off + 2) 21 23 else loop acc (shift + 7) (i + 1) 22 24 in ··· 49 51 end 50 52 51 53 (* Page types *) 52 - type page_type = 53 - | Interior_index 54 - | Interior_table 55 - | Leaf_index 56 - | Leaf_table 54 + type page_type = Interior_index | Interior_table | Leaf_index | Leaf_table 57 55 58 56 let pp_page_type ppf = function 59 57 | Interior_index -> Format.pp_print_string ppf "interior_index" ··· 92 90 right_child : int option; 93 91 } 94 92 95 - let get_u16_be buf off = 96 - (Char.code buf.[off] lsl 8) lor Char.code buf.[off + 1] 93 + let get_u16_be buf off = (Char.code buf.[off] lsl 8) lor Char.code buf.[off + 1] 97 94 98 95 let get_u32_be buf off = 99 96 (Char.code buf.[off] lsl 24) ··· 141 138 } 142 139 143 140 type table_interior = { left_child : int; rowid : int64 } 144 - 145 141 type index_leaf = { payload : string; overflow_page : int option } 146 142 147 143 type index_interior = { ··· 154 150 let max_local ~usable_size ~is_table = 155 151 if is_table then usable_size - 35 else ((usable_size - 12) * 64 / 255) - 23 156 152 157 - let min_local ~usable_size = 158 - ((usable_size - 12) * 32 / 255) - 23 153 + let min_local ~usable_size = ((usable_size - 12) * 32 / 255) - 23 159 154 160 155 let parse_table_leaf buf off ~usable_size = 161 156 let payload_size, consumed1 = Varint.decode buf off in ··· 167 162 let local_size, overflow_page = 168 163 if payload_size <= max_local then (payload_size, None) 169 164 else 170 - let k = min_local + ((payload_size - min_local) mod (usable_size - 4)) in 165 + let k = 166 + min_local + ((payload_size - min_local) mod (usable_size - 4)) 167 + in 171 168 let local = if k <= max_local then k else min_local in 172 169 let overflow = get_u32_be buf (off + header_len + local) in 173 170 (local, Some overflow) 174 171 in 175 172 let payload = String.sub buf (off + header_len) local_size in 176 173 let total_consumed = 177 - header_len + local_size + (if overflow_page = None then 0 else 4) 174 + header_len + local_size + if overflow_page = None then 0 else 4 178 175 in 179 176 ({ rowid; payload; overflow_page }, total_consumed) 180 177 ··· 191 188 let local_size, overflow_page = 192 189 if payload_size <= max_local then (payload_size, None) 193 190 else 194 - let k = min_local + ((payload_size - min_local) mod (usable_size - 4)) in 191 + let k = 192 + min_local + ((payload_size - min_local) mod (usable_size - 4)) 193 + in 195 194 let local = if k <= max_local then k else min_local in 196 195 let overflow = get_u32_be buf (off + consumed + local) in 197 196 (local, Some overflow) 198 197 in 199 198 let payload = String.sub buf (off + consumed) local_size in 200 - let total = consumed + local_size + (if overflow_page = None then 0 else 4) in 199 + let total = consumed + local_size + if overflow_page = None then 0 else 4 in 201 200 ({ payload; overflow_page }, total) 202 201 203 202 let parse_index_interior buf off ~usable_size = ··· 209 208 let local_size, overflow_page = 210 209 if payload_size <= max_local then (payload_size, None) 211 210 else 212 - let k = min_local + ((payload_size - min_local) mod (usable_size - 4)) in 211 + let k = 212 + min_local + ((payload_size - min_local) mod (usable_size - 4)) 213 + in 213 214 let local = if k <= max_local then k else min_local in 214 215 let overflow = get_u32_be buf (off + 4 + consumed + local) in 215 216 (local, Some overflow) 216 217 in 217 218 let payload = String.sub buf (off + 4 + consumed) local_size in 218 219 let total = 219 - 4 + consumed + local_size + (if overflow_page = None then 0 else 4) 220 + 4 + consumed + local_size + if overflow_page = None then 0 else 4 220 221 in 221 222 ({ left_child; payload; overflow_page }, total) 222 223 end ··· 237 238 | Blob of int 238 239 | Text of int 239 240 240 - type value = Vnull | Vint of int64 | Vfloat of float | Vblob of string | Vtext of string 241 + type value = 242 + | Vnull 243 + | Vint of int64 244 + | Vfloat of float 245 + | Vblob of string 246 + | Vtext of string 241 247 242 248 let serial_type_of_int = function 243 249 | 0 -> Null ··· 254 260 | n when n >= 13 -> Text ((n - 13) / 2) 255 261 | n -> failwith (Printf.sprintf "Invalid serial type: %d" n) 256 262 257 - let serial_type_size = function 258 - | Null | Zero | One -> 0 259 - | Int8 -> 1 260 - | Int16 -> 2 261 - | Int24 -> 3 262 - | Int32 -> 4 263 - | Int48 -> 6 264 - | Int64 | Float64 -> 8 265 - | Blob n | Text n -> n 266 - 267 263 let decode_int buf off len = 268 264 let rec loop acc i = 269 265 if i >= len then acc ··· 329 325 else if n >= -140737488355328L && n <= 140737488355327L then (5, 6) 330 326 else (6, 8) 331 327 | Vfloat _ -> (7, 8) 332 - | Vblob s -> (12 + String.length s * 2, String.length s) 333 - | Vtext s -> (13 + String.length s * 2, String.length s) 328 + | Vblob s -> (12 + (String.length s * 2), String.length s) 329 + | Vtext s -> (13 + (String.length s * 2), String.length s) 334 330 335 331 let encode_int buf off n len = 336 332 for i = 0 to len - 1 do ··· 348 344 let header_body = String.concat "" header_types in 349 345 let header_size = 1 + String.length header_body in 350 346 (* header size varint + types *) 351 - let body_size = List.fold_left (fun acc (_, sz) -> acc + sz) 0 types_and_sizes in 347 + let body_size = 348 + List.fold_left (fun acc (_, sz) -> acc + sz) 0 types_and_sizes 349 + in 352 350 let total = header_size + body_size in 353 351 let buf = Bytes.create total in 354 352 (* Write header size *) ··· 425 423 data) 426 424 427 425 let write t page_num data = 428 - if String.length data <> t.page_size then 429 - failwith "Invalid page size"; 426 + if String.length data <> t.page_size then failwith "Invalid page size"; 430 427 Hashtbl.replace t.dirty page_num data; 431 428 Hashtbl.replace t.cache page_num data 432 429 ··· 451 448 module Table = struct 452 449 type t = { pager : Pager.t; mutable root_page : int } 453 450 454 - let create pager = 455 - let root = Pager.allocate pager in 456 - (* Initialize as empty leaf page *) 457 - let page_size = Pager.page_size pager in 451 + (* Initialize a page as empty leaf or interior *) 452 + let init_page ~page_size ~page_type = 458 453 let buf = Bytes.create page_size in 459 - Bytes.set_uint8 buf 0 (byte_of_page_type Leaf_table); 454 + Bytes.set_uint8 buf 0 (byte_of_page_type page_type); 460 455 set_u16_be buf 1 0; 461 456 (* first freeblock *) 462 457 set_u16_be buf 3 0; ··· 465 460 (* cell content start *) 466 461 Bytes.set_uint8 buf 7 0; 467 462 (* fragmented bytes *) 463 + if is_interior page_type then set_u32_be buf 8 0; 464 + (* right child *) 465 + buf 466 + 467 + let create pager = 468 + let root = Pager.allocate pager in 469 + let page_size = Pager.page_size pager in 470 + let buf = init_page ~page_size ~page_type:Leaf_table in 468 471 Pager.write pager root (Bytes.unsafe_to_string buf); 469 472 { pager; root_page = root } 470 473 471 474 let open_ pager ~root_page = { pager; root_page } 472 475 let root_page t = t.root_page 473 - 474 476 let usable_size t = Pager.page_size t.pager 475 477 476 478 (* Find cell pointers in a page *) ··· 478 480 let ptrs = Array.make header.cell_count 0 in 479 481 let ptr_start = header_offset + page_header_size header.page_type in 480 482 for i = 0 to header.cell_count - 1 do 481 - ptrs.(i) <- get_u16_be page (ptr_start + i * 2) 483 + ptrs.(i) <- get_u16_be page (ptr_start + (i * 2)) 482 484 done; 483 485 ptrs 484 486 487 + (* Calculate free space in a page *) 488 + let free_space header ~page_type = 489 + let header_size = page_header_size page_type in 490 + let ptr_area_end = header_size + (header.cell_count * 2) in 491 + header.cell_content_start - ptr_area_end - header.fragmented_bytes 492 + 493 + (* Encode a table leaf cell *) 494 + let encode_table_leaf_cell ~rowid ~data = 495 + let rowid_varint = Varint.encode rowid in 496 + let payload_size_varint = 497 + Varint.encode (Int64.of_int (String.length data)) 498 + in 499 + let cell = 500 + Bytes.create 501 + (String.length payload_size_varint 502 + + String.length rowid_varint + String.length data) 503 + in 504 + Bytes.blit_string payload_size_varint 0 cell 0 505 + (String.length payload_size_varint); 506 + Bytes.blit_string rowid_varint 0 cell 507 + (String.length payload_size_varint) 508 + (String.length rowid_varint); 509 + Bytes.blit_string data 0 cell 510 + (String.length payload_size_varint + String.length rowid_varint) 511 + (String.length data); 512 + Bytes.unsafe_to_string cell 513 + 514 + (* Encode a table interior cell *) 515 + let encode_table_interior_cell ~left_child ~rowid = 516 + let rowid_varint = Varint.encode rowid in 517 + let cell = Bytes.create (4 + String.length rowid_varint) in 518 + set_u32_be cell 0 left_child; 519 + Bytes.blit_string rowid_varint 0 cell 4 (String.length rowid_varint); 520 + Bytes.unsafe_to_string cell 521 + 522 + (* Write a cell into a page buffer, returns new cell_content_start *) 523 + let write_cell buf ~cell_content_start ~cell = 524 + let cell_len = String.length cell in 525 + let new_start = cell_content_start - cell_len in 526 + Bytes.blit_string cell 0 buf new_start cell_len; 527 + new_start 528 + 529 + (* Insert a cell pointer at index, shifting others *) 530 + let insert_cell_pointer buf ~header_offset ~page_type ~cell_count ~index ~ptr 531 + = 532 + let ptr_start = header_offset + page_header_size page_type in 533 + (* Shift existing pointers right *) 534 + for i = cell_count - 1 downto index do 535 + let old_ptr = 536 + get_u16_be (Bytes.unsafe_to_string buf) (ptr_start + (i * 2)) 537 + in 538 + set_u16_be buf (ptr_start + ((i + 1) * 2)) old_ptr 539 + done; 540 + set_u16_be buf (ptr_start + (index * 2)) ptr 541 + 485 542 (* Binary search for rowid in leaf page *) 486 543 let search_leaf t page header rowid = 487 544 let ptrs = cell_pointers page 0 header in ··· 490 547 if lo > hi then None 491 548 else 492 549 let mid = (lo + hi) / 2 in 493 - let cell, _ = Cell.parse_table_leaf page ptrs.(mid) ~usable_size:usable in 550 + let cell, _ = 551 + Cell.parse_table_leaf page ptrs.(mid) ~usable_size:usable 552 + in 494 553 if cell.rowid = rowid then Some cell.payload 495 554 else if cell.rowid < rowid then loop (mid + 1) hi 496 555 else loop lo (mid - 1) 497 556 in 498 557 loop 0 (header.cell_count - 1) 499 558 559 + (* Find insertion index for rowid in leaf page *) 560 + let find_insert_idx t page header rowid = 561 + let ptrs = cell_pointers page 0 header in 562 + let usable = usable_size t in 563 + let rec find i = 564 + if i >= header.cell_count then i 565 + else 566 + let cell, _ = Cell.parse_table_leaf page ptrs.(i) ~usable_size:usable in 567 + if rowid < cell.rowid then i else find (i + 1) 568 + in 569 + find 0 570 + 500 571 (* Find child page for rowid in interior page *) 501 - let find_child t page header rowid = 572 + let find_child _t page header rowid = 502 573 let ptrs = cell_pointers page 0 header in 503 574 let rec loop i = 504 575 if i >= header.cell_count then Option.get header.right_child ··· 508 579 in 509 580 loop 0 510 581 582 + (* Find child index for rowid in interior page *) 583 + let find_child_idx page header rowid = 584 + let ptrs = cell_pointers page 0 header in 585 + let rec loop i = 586 + if i >= header.cell_count then i (* right child *) 587 + else 588 + let cell, _ = Cell.parse_table_interior page ptrs.(i) in 589 + if rowid <= cell.rowid then i else loop (i + 1) 590 + in 591 + loop 0 592 + 511 593 let rec find_in_page t page_num rowid = 512 594 let page = Pager.read t.pager page_num in 513 595 let header = parse_page_header page 0 in ··· 520 602 521 603 let find t rowid = find_in_page t t.root_page rowid 522 604 523 - (* Insert - simplified, doesn't handle page splits *) 524 - let insert t ~rowid data = 525 - let page = Pager.read t.pager t.root_page in 605 + (* Split result: new page number and separator rowid *) 606 + type split_result = { new_page : int; separator_rowid : int64 } 607 + 608 + (* Split a leaf page, returns info about the new page *) 609 + let split_leaf t page_num = 610 + let page = Pager.read t.pager page_num in 611 + let header = parse_page_header page 0 in 612 + let ptrs = cell_pointers page 0 header in 613 + let usable = usable_size t in 614 + let page_size = Pager.page_size t.pager in 615 + 616 + (* Find split point (middle) *) 617 + let split_idx = header.cell_count / 2 in 618 + 619 + (* Get separator rowid (first key that goes to right page) *) 620 + let sep_cell, _ = 621 + Cell.parse_table_leaf page ptrs.(split_idx) ~usable_size:usable 622 + in 623 + let separator_rowid = sep_cell.rowid in 624 + 625 + (* Create new right page *) 626 + let new_page_num = Pager.allocate t.pager in 627 + let new_buf = init_page ~page_size ~page_type:Leaf_table in 628 + 629 + (* Copy cells [split_idx..cell_count-1] to new page *) 630 + let new_cell_content_start = ref page_size in 631 + for i = split_idx to header.cell_count - 1 do 632 + let cell_off = ptrs.(i) in 633 + let cell, cell_len = 634 + Cell.parse_table_leaf page cell_off ~usable_size:usable 635 + in 636 + let cell_data = 637 + encode_table_leaf_cell ~rowid:cell.rowid ~data:cell.payload 638 + in 639 + new_cell_content_start := 640 + write_cell new_buf ~cell_content_start:!new_cell_content_start 641 + ~cell:cell_data; 642 + let new_idx = i - split_idx in 643 + let ptr_off = page_header_size Leaf_table + (new_idx * 2) in 644 + set_u16_be new_buf ptr_off !new_cell_content_start; 645 + ignore cell_len 646 + done; 647 + let new_cell_count = header.cell_count - split_idx in 648 + set_u16_be new_buf 3 new_cell_count; 649 + set_u16_be new_buf 5 !new_cell_content_start; 650 + Pager.write t.pager new_page_num (Bytes.unsafe_to_string new_buf); 651 + 652 + (* Update original page to only have cells [0..split_idx-1] *) 653 + let old_buf = Bytes.of_string page in 654 + set_u16_be old_buf 3 split_idx; 655 + (* Recalculate cell content start for remaining cells *) 656 + if split_idx > 0 then begin 657 + let min_ptr = ref page_size in 658 + for i = 0 to split_idx - 1 do 659 + if ptrs.(i) < !min_ptr then min_ptr := ptrs.(i) 660 + done; 661 + set_u16_be old_buf 5 !min_ptr 662 + end 663 + else set_u16_be old_buf 5 page_size; 664 + Pager.write t.pager page_num (Bytes.unsafe_to_string old_buf); 665 + 666 + { new_page = new_page_num; separator_rowid } 667 + 668 + (* Split an interior page *) 669 + let split_interior t page_num = 670 + let page = Pager.read t.pager page_num in 671 + let header = parse_page_header page 0 in 672 + let ptrs = cell_pointers page 0 header in 673 + let page_size = Pager.page_size t.pager in 674 + 675 + (* Split point - middle cell becomes separator, doesn't go to either page *) 676 + let split_idx = header.cell_count / 2 in 677 + let sep_cell, _ = Cell.parse_table_interior page ptrs.(split_idx) in 678 + let separator_rowid = sep_cell.rowid in 679 + 680 + (* Create new right page *) 681 + let new_page_num = Pager.allocate t.pager in 682 + let new_buf = init_page ~page_size ~page_type:Interior_table in 683 + 684 + (* The right child of split cell becomes the left-most child of new page *) 685 + (* Cells [split_idx+1..cell_count-1] go to new page *) 686 + let new_cell_content_start = ref page_size in 687 + for i = split_idx + 1 to header.cell_count - 1 do 688 + let cell_off = ptrs.(i) in 689 + let cell, _ = Cell.parse_table_interior page cell_off in 690 + let cell_data = 691 + encode_table_interior_cell ~left_child:cell.left_child ~rowid:cell.rowid 692 + in 693 + new_cell_content_start := 694 + write_cell new_buf ~cell_content_start:!new_cell_content_start 695 + ~cell:cell_data; 696 + let new_idx = i - split_idx - 1 in 697 + let ptr_off = page_header_size Interior_table + (new_idx * 2) in 698 + set_u16_be new_buf ptr_off !new_cell_content_start 699 + done; 700 + let new_cell_count = header.cell_count - split_idx - 1 in 701 + set_u16_be new_buf 3 new_cell_count; 702 + set_u16_be new_buf 5 !new_cell_content_start; 703 + (* Right child of new page is same as original *) 704 + set_u32_be new_buf 8 (Option.get header.right_child); 705 + Pager.write t.pager new_page_num (Bytes.unsafe_to_string new_buf); 706 + 707 + (* Update original page: keep cells [0..split_idx-1], right child = sep_cell.left_child *) 708 + let old_buf = Bytes.of_string page in 709 + set_u16_be old_buf 3 split_idx; 710 + set_u32_be old_buf 8 sep_cell.left_child; 711 + if split_idx > 0 then begin 712 + let min_ptr = ref page_size in 713 + for i = 0 to split_idx - 1 do 714 + if ptrs.(i) < !min_ptr then min_ptr := ptrs.(i) 715 + done; 716 + set_u16_be old_buf 5 !min_ptr 717 + end 718 + else set_u16_be old_buf 5 page_size; 719 + Pager.write t.pager page_num (Bytes.unsafe_to_string old_buf); 720 + 721 + { new_page = new_page_num; separator_rowid } 722 + 723 + (* Insert a separator into an interior page, potentially splitting *) 724 + let rec insert_into_interior t page_num ~left_child ~separator_rowid 725 + ~right_child = 726 + let page = Pager.read t.pager page_num in 727 + let header = parse_page_header page 0 in 728 + let cell = encode_table_interior_cell ~left_child ~rowid:separator_rowid in 729 + let cell_len = String.length cell in 730 + let space_needed = cell_len + 2 in 731 + (* cell + pointer *) 732 + 733 + if free_space header ~page_type:Interior_table >= space_needed then begin 734 + (* Fits - insert directly *) 735 + let buf = Bytes.of_string page in 736 + let ptrs = cell_pointers page 0 header in 737 + 738 + (* Find insert position *) 739 + let insert_idx = 740 + let rec find i = 741 + if i >= header.cell_count then i 742 + else 743 + let c, _ = Cell.parse_table_interior page ptrs.(i) in 744 + if separator_rowid < c.rowid then i else find (i + 1) 745 + in 746 + find 0 747 + in 748 + 749 + (* Write cell *) 750 + let cell_start = 751 + write_cell buf ~cell_content_start:header.cell_content_start ~cell 752 + in 753 + set_u16_be buf 5 cell_start; 754 + 755 + (* Insert pointer *) 756 + insert_cell_pointer buf ~header_offset:0 ~page_type:Interior_table 757 + ~cell_count:header.cell_count ~index:insert_idx ~ptr:cell_start; 758 + 759 + (* Update cell count *) 760 + set_u16_be buf 3 (header.cell_count + 1); 761 + 762 + (* Update right child if inserting at end *) 763 + if insert_idx = header.cell_count then set_u32_be buf 8 right_child; 764 + 765 + Pager.write t.pager page_num (Bytes.unsafe_to_string buf); 766 + None 767 + end 768 + else begin 769 + (* Need to split interior page *) 770 + let split = split_interior t page_num in 771 + 772 + (* Determine which page gets the new separator *) 773 + if separator_rowid < split.separator_rowid then begin 774 + (* Insert into left (original) page *) 775 + ignore 776 + (insert_into_interior t page_num ~left_child ~separator_rowid 777 + ~right_child) 778 + end 779 + else begin 780 + (* Insert into right (new) page *) 781 + ignore 782 + (insert_into_interior t split.new_page ~left_child ~separator_rowid 783 + ~right_child) 784 + end; 785 + 786 + (* Return split info to propagate up *) 787 + Some { split with separator_rowid = split.separator_rowid } 788 + end 789 + 790 + (* Insert into a leaf page, potentially splitting *) 791 + let rec insert_into_leaf t page_num ~rowid ~data ~parent_stack = 792 + let page = Pager.read t.pager page_num in 526 793 let header = parse_page_header page 0 in 527 - match header.page_type with 528 - | Leaf_table -> 794 + let cell = encode_table_leaf_cell ~rowid ~data in 795 + let cell_len = String.length cell in 796 + let space_needed = cell_len + 2 in 797 + (* cell + pointer *) 798 + 799 + if free_space header ~page_type:Leaf_table >= space_needed then begin 800 + (* Fits - insert directly *) 801 + let buf = Bytes.of_string page in 802 + let insert_idx = find_insert_idx t page header rowid in 803 + 804 + (* Write cell *) 805 + let cell_start = 806 + write_cell buf ~cell_content_start:header.cell_content_start ~cell 807 + in 808 + set_u16_be buf 5 cell_start; 809 + 810 + (* Insert pointer *) 811 + insert_cell_pointer buf ~header_offset:0 ~page_type:Leaf_table 812 + ~cell_count:header.cell_count ~index:insert_idx ~ptr:cell_start; 813 + 814 + (* Update cell count *) 815 + set_u16_be buf 3 (header.cell_count + 1); 816 + 817 + Pager.write t.pager page_num (Bytes.unsafe_to_string buf) 818 + end 819 + else begin 820 + (* Need to split *) 821 + let split = split_leaf t page_num in 822 + 823 + (* Insert into appropriate page *) 824 + if rowid < split.separator_rowid then 825 + insert_into_leaf t page_num ~rowid ~data ~parent_stack:[] 826 + else insert_into_leaf t split.new_page ~rowid ~data ~parent_stack:[]; 827 + 828 + (* Propagate split up *) 829 + propagate_split t ~parent_stack ~left_page:page_num 830 + ~separator_rowid:split.separator_rowid ~right_page:split.new_page 831 + end 832 + 833 + and propagate_split t ~parent_stack ~left_page ~separator_rowid ~right_page = 834 + match parent_stack with 835 + | [] -> 836 + (* Splitting root - create new root *) 529 837 let page_size = Pager.page_size t.pager in 530 - let buf = Bytes.of_string page in 531 - (* Create cell *) 532 - let rowid_varint = Varint.encode rowid in 533 - let payload_size_varint = Varint.encode (Int64.of_int (String.length data)) in 534 - let cell_size = 535 - String.length payload_size_varint + String.length rowid_varint + String.length data 838 + let new_root = Pager.allocate t.pager in 839 + let buf = init_page ~page_size ~page_type:Interior_table in 840 + 841 + (* Single cell pointing to left page *) 842 + let cell = 843 + encode_table_interior_cell ~left_child:left_page 844 + ~rowid:separator_rowid 536 845 in 537 - (* Find insertion point *) 538 - let ptrs = cell_pointers page 0 header in 539 - let usable = usable_size t in 540 - let insert_idx = 541 - let rec find i = 542 - if i >= header.cell_count then i 846 + let cell_start = write_cell buf ~cell_content_start:page_size ~cell in 847 + set_u16_be buf 5 cell_start; 848 + set_u16_be buf (page_header_size Interior_table) cell_start; 849 + set_u16_be buf 3 1; 850 + set_u32_be buf 8 right_page; 851 + 852 + Pager.write t.pager new_root (Bytes.unsafe_to_string buf); 853 + t.root_page <- new_root 854 + | parent_page :: rest -> ( 855 + match 856 + insert_into_interior t parent_page ~left_child:left_page 857 + ~separator_rowid ~right_child:right_page 858 + with 859 + | None -> () (* Fit in parent *) 860 + | Some split -> 861 + (* Parent also split, propagate up *) 862 + propagate_split t ~parent_stack:rest ~left_page:parent_page 863 + ~separator_rowid:split.separator_rowid ~right_page:split.new_page) 864 + 865 + (* Main insert - traverses tree and handles splits *) 866 + let insert t ~rowid data = 867 + let rec traverse page_num parent_stack = 868 + let page = Pager.read t.pager page_num in 869 + let header = parse_page_header page 0 in 870 + match header.page_type with 871 + | Leaf_table -> insert_into_leaf t page_num ~rowid ~data ~parent_stack 872 + | Interior_table -> 873 + let child_idx = find_child_idx page header rowid in 874 + let child_page = 875 + if child_idx >= header.cell_count then Option.get header.right_child 543 876 else 544 - let cell, _ = Cell.parse_table_leaf page ptrs.(i) ~usable_size:usable in 545 - if rowid < cell.rowid then i else find (i + 1) 877 + let ptrs = cell_pointers page 0 header in 878 + let cell, _ = Cell.parse_table_interior page ptrs.(child_idx) in 879 + cell.left_child 546 880 in 547 - find 0 548 - in 549 - (* Write cell at end of cell content area *) 550 - let cell_start = header.cell_content_start - cell_size in 551 - Bytes.blit_string payload_size_varint 0 buf cell_start 552 - (String.length payload_size_varint); 553 - let off = cell_start + String.length payload_size_varint in 554 - Bytes.blit_string rowid_varint 0 buf off (String.length rowid_varint); 555 - let off = off + String.length rowid_varint in 556 - Bytes.blit_string data 0 buf off (String.length data); 557 - (* Update cell content start *) 558 - set_u16_be buf 5 cell_start; 559 - (* Insert cell pointer *) 560 - let ptr_area_start = page_header_size Leaf_table in 561 - (* Shift existing pointers *) 562 - for i = header.cell_count - 1 downto insert_idx do 563 - set_u16_be buf (ptr_area_start + (i + 1) * 2) (get_u16_be page (ptr_area_start + i * 2)) 564 - done; 565 - set_u16_be buf (ptr_area_start + insert_idx * 2) cell_start; 566 - (* Update cell count *) 567 - set_u16_be buf 3 (header.cell_count + 1); 568 - Pager.write t.pager t.root_page (Bytes.unsafe_to_string buf) 569 - | _ -> failwith "Insert into interior page not yet implemented" 881 + traverse child_page (page_num :: parent_stack) 882 + | _ -> failwith "Invalid page type in table B-tree" 883 + in 884 + traverse t.root_page [] 570 885 571 886 let delete _t _rowid = failwith "Delete not yet implemented" 572 887 ··· 579 894 match header.page_type with 580 895 | Leaf_table -> 581 896 for i = 0 to header.cell_count - 1 do 582 - let cell, _ = Cell.parse_table_leaf page ptrs.(i) ~usable_size:usable in 897 + let cell, _ = 898 + Cell.parse_table_leaf page ptrs.(i) ~usable_size:usable 899 + in 583 900 f cell.rowid cell.payload 584 901 done 585 902 | Interior_table -> ··· 616 933 617 934 let open_ pager ~root_page = { pager; root_page } 618 935 let root_page t = t.root_page 619 - 620 936 let usable_size t = Pager.page_size t.pager 621 937 622 938 let cell_pointers page header = 623 939 let ptrs = Array.make header.cell_count 0 in 624 940 let ptr_start = page_header_size header.page_type in 625 941 for i = 0 to header.cell_count - 1 do 626 - ptrs.(i) <- get_u16_be page (ptr_start + i * 2) 942 + ptrs.(i) <- get_u16_be page (ptr_start + (i * 2)) 627 943 done; 628 944 ptrs 629 945 ··· 637 953 let rec search i = 638 954 if i >= header.cell_count then false 639 955 else 640 - let cell, _ = Cell.parse_index_leaf page ptrs.(i) ~usable_size:usable in 956 + let cell, _ = 957 + Cell.parse_index_leaf page ptrs.(i) ~usable_size:usable 958 + in 641 959 if cell.payload = key then true 642 960 else if cell.payload > key then false 643 961 else search (i + 1) ··· 647 965 let rec find_child i = 648 966 if i >= header.cell_count then Option.get header.right_child 649 967 else 650 - let cell, _ = Cell.parse_index_interior page ptrs.(i) ~usable_size:usable in 968 + let cell, _ = 969 + Cell.parse_index_interior page ptrs.(i) ~usable_size:usable 970 + in 651 971 if key <= cell.payload then cell.left_child else find_child (i + 1) 652 972 in 653 973 mem_in_page t (find_child 0) key 654 974 | _ -> failwith "Invalid page type in index B-tree" 655 975 656 976 let mem t key = mem_in_page t t.root_page key 657 - 658 977 let insert _t _key = failwith "Index insert not yet implemented" 659 978 let delete _t _key = failwith "Index delete not yet implemented" 660 979 ··· 667 986 match header.page_type with 668 987 | Leaf_index -> 669 988 for i = 0 to header.cell_count - 1 do 670 - let cell, _ = Cell.parse_index_leaf page ptrs.(i) ~usable_size:usable in 989 + let cell, _ = 990 + Cell.parse_index_leaf page ptrs.(i) ~usable_size:usable 991 + in 671 992 f cell.payload 672 993 done 673 994 | Interior_index -> 674 995 for i = 0 to header.cell_count - 1 do 675 - let cell, _ = Cell.parse_index_interior page ptrs.(i) ~usable_size:usable in 996 + let cell, _ = 997 + Cell.parse_index_interior page ptrs.(i) ~usable_size:usable 998 + in 676 999 iter_page cell.left_child 677 1000 done; 678 1001 Option.iter iter_page header.right_child
+16 -20
lib/btree.mli
··· 13 13 14 14 module Varint : sig 15 15 val decode : string -> int -> int64 * int 16 - (** [decode buf off] decodes a varint starting at [off] in [buf]. 17 - Returns [(value, bytes_consumed)]. *) 16 + (** [decode buf off] decodes a varint starting at [off] in [buf]. Returns 17 + [(value, bytes_consumed)]. *) 18 18 19 19 val encode : int64 -> string 20 20 (** [encode n] encodes [n] as a varint. *) ··· 28 28 type page_type = 29 29 | Interior_index (** 0x02 *) 30 30 | Interior_table (** 0x05 *) 31 - | Leaf_index (** 0x0a *) 32 - | Leaf_table (** 0x0d *) 31 + | Leaf_index (** 0x0a *) 32 + | Leaf_table (** 0x0d *) 33 33 34 34 val pp_page_type : Format.formatter -> page_type -> unit 35 35 ··· 45 45 } 46 46 47 47 val parse_page_header : string -> int -> page_header 48 - (** [parse_page_header buf off] parses a page header starting at [off]. 49 - For page 1, [off] should be 100 (after database header). *) 48 + (** [parse_page_header buf off] parses a page header starting at [off]. For page 49 + 1, [off] should be 100 (after database header). *) 50 50 51 51 val page_header_size : page_type -> int 52 52 (** [page_header_size typ] is 8 for leaf pages, 12 for interior pages. *) ··· 56 56 Cells contain the actual data in B-tree pages. *) 57 57 58 58 module Cell : sig 59 - (** Table leaf cell: rowid + payload *) 60 59 type table_leaf = { 61 60 rowid : int64; 62 61 payload : string; 63 62 overflow_page : int option; 64 63 } 64 + (** Table leaf cell: rowid + payload *) 65 65 66 + type table_interior = { left_child : int; rowid : int64 } 66 67 (** Table interior cell: child page + rowid *) 67 - type table_interior = { 68 - left_child : int; 69 - rowid : int64; 70 - } 71 68 69 + type index_leaf = { payload : string; overflow_page : int option } 72 70 (** Index leaf cell: payload only *) 73 - type index_leaf = { 74 - payload : string; 75 - overflow_page : int option; 76 - } 77 71 78 - (** Index interior cell: child page + payload *) 79 72 type index_interior = { 80 73 left_child : int; 81 74 payload : string; 82 75 overflow_page : int option; 83 76 } 77 + (** Index interior cell: child page + payload *) 84 78 85 79 val parse_table_leaf : string -> int -> usable_size:int -> table_leaf * int 86 - (** [parse_table_leaf buf off ~usable_size] parses a table leaf cell. 87 - Returns [(cell, bytes_consumed)]. *) 80 + (** [parse_table_leaf buf off ~usable_size] parses a table leaf cell. Returns 81 + [(cell, bytes_consumed)]. *) 88 82 89 83 val parse_table_interior : string -> int -> table_interior * int 90 84 (** [parse_table_interior buf off] parses a table interior cell. *) ··· 92 86 val parse_index_leaf : string -> int -> usable_size:int -> index_leaf * int 93 87 (** [parse_index_leaf buf off ~usable_size] parses an index leaf cell. *) 94 88 95 - val parse_index_interior : string -> int -> usable_size:int -> index_interior * int 96 - (** [parse_index_interior buf off ~usable_size] parses an index interior cell. *) 89 + val parse_index_interior : 90 + string -> int -> usable_size:int -> index_interior * int 91 + (** [parse_index_interior buf off ~usable_size] parses an index interior cell. 92 + *) 97 93 end 98 94 99 95 (** {1 Record Format}
+231 -10
test/test_btree.ml
··· 97 97 98 98 let test_page_header_leaf () = 99 99 let page = Bytes.create 4096 in 100 - Bytes.set_uint8 page 0 0x0d; (* Leaf table *) 100 + Bytes.set_uint8 page 0 0x0d; 101 + (* Leaf table *) 101 102 Bytes.set_uint8 page 1 0; 102 - Bytes.set_uint8 page 2 0; (* first freeblock = 0 *) 103 + Bytes.set_uint8 page 2 0; 104 + (* first freeblock = 0 *) 103 105 Bytes.set_uint8 page 3 0; 104 - Bytes.set_uint8 page 4 5; (* cell count = 5 *) 106 + Bytes.set_uint8 page 4 5; 107 + (* cell count = 5 *) 105 108 Bytes.set_uint8 page 5 0x0f; 106 - Bytes.set_uint8 page 6 0x00; (* cell content start = 0x0f00 *) 107 - Bytes.set_uint8 page 7 0; (* fragmented = 0 *) 109 + Bytes.set_uint8 page 6 0x00; 110 + (* cell content start = 0x0f00 *) 111 + Bytes.set_uint8 page 7 0; 112 + (* fragmented = 0 *) 108 113 let header = Btree.parse_page_header (Bytes.unsafe_to_string page) 0 in 109 114 Alcotest.(check int) "cell_count" 5 header.cell_count; 110 115 Alcotest.(check int) "cell_content_start" 0x0f00 header.cell_content_start; 111 - Alcotest.(check bool) "no right_child" true (Option.is_none header.right_child) 116 + Alcotest.(check bool) 117 + "no right_child" true 118 + (Option.is_none header.right_child) 112 119 113 120 let test_page_header_interior () = 114 121 let page = Bytes.create 4096 in 115 - Bytes.set_uint8 page 0 0x05; (* Interior table *) 122 + Bytes.set_uint8 page 0 0x05; 123 + (* Interior table *) 116 124 Bytes.set_uint8 page 1 0; 117 125 Bytes.set_uint8 page 2 0; 118 126 Bytes.set_uint8 page 3 0; ··· 165 173 Btree.Table.insert tree ~rowid:3L "Three"; 166 174 Btree.Table.insert tree ~rowid:1L "One"; 167 175 Btree.Table.insert tree ~rowid:2L "Two"; 168 - Alcotest.(check (option string)) "find 1" (Some "One") (Btree.Table.find tree 1L); 169 - Alcotest.(check (option string)) "find 2" (Some "Two") (Btree.Table.find tree 2L); 170 - Alcotest.(check (option string)) "find 3" (Some "Three") (Btree.Table.find tree 3L); 176 + Alcotest.(check (option string)) 177 + "find 1" (Some "One") (Btree.Table.find tree 1L); 178 + Alcotest.(check (option string)) 179 + "find 2" (Some "Two") (Btree.Table.find tree 2L); 180 + Alcotest.(check (option string)) 181 + "find 3" (Some "Three") (Btree.Table.find tree 3L); 171 182 Alcotest.(check (option string)) "find 4" None (Btree.Table.find tree 4L) 172 183 173 184 let test_table_iter () = ··· 185 196 [ (1L, "A"); (2L, "B"); (3L, "C") ] 186 197 sorted 187 198 199 + (* Page split tests *) 200 + 201 + let test_leaf_split () = 202 + (* Use small page size to trigger splits quickly *) 203 + with_temp_file @@ fun file -> 204 + let pager = Btree.Pager.create ~page_size:512 file in 205 + let tree = Btree.Table.create pager in 206 + (* Insert enough records to trigger a split *) 207 + (* With 512 byte pages, ~20-30 small records should trigger a split *) 208 + for i = 1 to 50 do 209 + let data = Printf.sprintf "value_%d" i in 210 + Btree.Table.insert tree ~rowid:(Int64.of_int i) data 211 + done; 212 + (* Verify all records are findable *) 213 + for i = 1 to 50 do 214 + let expected = Printf.sprintf "value_%d" i in 215 + let actual = Btree.Table.find tree (Int64.of_int i) in 216 + Alcotest.(check (option string)) 217 + (Printf.sprintf "find %d" i) 218 + (Some expected) actual 219 + done; 220 + (* Verify iteration returns all records in order *) 221 + let items = ref [] in 222 + Btree.Table.iter tree (fun rowid data -> items := (rowid, data) :: !items); 223 + Alcotest.(check int) "count after split" 50 (List.length !items) 224 + 225 + let test_multi_level_tree () = 226 + (* Force multiple levels by inserting many records *) 227 + with_temp_file @@ fun file -> 228 + let pager = Btree.Pager.create ~page_size:512 file in 229 + let tree = Btree.Table.create pager in 230 + let n = 200 in 231 + for i = 1 to n do 232 + Btree.Table.insert tree ~rowid:(Int64.of_int i) (Printf.sprintf "v%d" i) 233 + done; 234 + (* Verify all records *) 235 + for i = 1 to n do 236 + let r = Btree.Table.find tree (Int64.of_int i) in 237 + Alcotest.(check (option string)) 238 + (Printf.sprintf "find %d" i) 239 + (Some (Printf.sprintf "v%d" i)) 240 + r 241 + done; 242 + (* Check count *) 243 + let count = Btree.Table.fold tree ~init:0 ~f:(fun _ _ acc -> acc + 1) in 244 + Alcotest.(check int) "total count" n count 245 + 246 + let test_reverse_insert_order () = 247 + (* Insert in reverse order to stress different split patterns *) 248 + with_temp_file @@ fun file -> 249 + let pager = Btree.Pager.create ~page_size:512 file in 250 + let tree = Btree.Table.create pager in 251 + for i = 100 downto 1 do 252 + Btree.Table.insert tree ~rowid:(Int64.of_int i) (Printf.sprintf "r%d" i) 253 + done; 254 + for i = 1 to 100 do 255 + let r = Btree.Table.find tree (Int64.of_int i) in 256 + Alcotest.(check (option string)) 257 + (Printf.sprintf "find %d" i) 258 + (Some (Printf.sprintf "r%d" i)) 259 + r 260 + done 261 + 262 + let test_random_insert_order () = 263 + (* Pseudo-random insertion order *) 264 + with_temp_file @@ fun file -> 265 + let pager = Btree.Pager.create ~page_size:512 file in 266 + let tree = Btree.Table.create pager in 267 + (* LCG for deterministic "random" order *) 268 + let shuffle n = 269 + let a = Array.init n (fun i -> i + 1) in 270 + let state = ref 42 in 271 + for i = n - 1 downto 1 do 272 + state := ((!state * 1103515245) + 12345) land 0x7fffffff; 273 + let j = !state mod (i + 1) in 274 + let tmp = a.(i) in 275 + a.(i) <- a.(j); 276 + a.(j) <- tmp 277 + done; 278 + a 279 + in 280 + let order = shuffle 150 in 281 + Array.iter 282 + (fun i -> 283 + Btree.Table.insert tree ~rowid:(Int64.of_int i) (Printf.sprintf "x%d" i)) 284 + order; 285 + for i = 1 to 150 do 286 + let r = Btree.Table.find tree (Int64.of_int i) in 287 + Alcotest.(check (option string)) 288 + (Printf.sprintf "find %d" i) 289 + (Some (Printf.sprintf "x%d" i)) 290 + r 291 + done 292 + 293 + let test_large_values () = 294 + (* Test with larger values that take more space *) 295 + with_temp_file @@ fun file -> 296 + let pager = Btree.Pager.create ~page_size:1024 file in 297 + let tree = Btree.Table.create pager in 298 + for i = 1 to 30 do 299 + let data = String.make 50 (Char.chr (65 + (i mod 26))) in 300 + Btree.Table.insert tree ~rowid:(Int64.of_int i) data 301 + done; 302 + for i = 1 to 30 do 303 + let expected = String.make 50 (Char.chr (65 + (i mod 26))) in 304 + let r = Btree.Table.find tree (Int64.of_int i) in 305 + Alcotest.(check (option string)) 306 + (Printf.sprintf "find %d" i) 307 + (Some expected) r 308 + done 309 + 310 + (* SQLite varint test vectors from spec *) 311 + 312 + let test_varint_sqlite_vectors () = 313 + (* Test vectors from SQLite file format spec *) 314 + let test_encode n expected_bytes = 315 + let encoded = Btree.Varint.encode n in 316 + let bytes = 317 + List.init (String.length encoded) (fun i -> Char.code encoded.[i]) 318 + in 319 + Alcotest.(check (list int)) 320 + (Printf.sprintf "encode %Ld" n) 321 + expected_bytes bytes 322 + in 323 + (* Single byte: 0-127 *) 324 + test_encode 0L [ 0x00 ]; 325 + test_encode 1L [ 0x01 ]; 326 + test_encode 127L [ 0x7f ]; 327 + (* Two bytes: 128-16383 *) 328 + test_encode 128L [ 0x80; 0x01 ]; 329 + test_encode 255L [ 0xff; 0x01 ]; 330 + test_encode 16383L [ 0xff; 0x7f ] 331 + 332 + let test_varint_boundaries () = 333 + (* Test at varint size boundaries *) 334 + let boundaries = 335 + [ 336 + (0L, 1); 337 + (127L, 1); 338 + (128L, 2); 339 + (16383L, 2); 340 + (16384L, 3); 341 + (2097151L, 3); 342 + (2097152L, 4); 343 + (268435455L, 4); 344 + (268435456L, 5); 345 + ] 346 + in 347 + List.iter 348 + (fun (n, expected_size) -> 349 + let encoded = Btree.Varint.encode n in 350 + Alcotest.(check int) 351 + (Printf.sprintf "size of %Ld" n) 352 + expected_size (String.length encoded); 353 + let decoded, consumed = Btree.Varint.decode encoded 0 in 354 + Alcotest.(check int64) (Printf.sprintf "decode %Ld" n) n decoded; 355 + Alcotest.(check int) 356 + (Printf.sprintf "consumed for %Ld" n) 357 + expected_size consumed) 358 + boundaries 359 + 360 + (* Record format test vectors *) 361 + 362 + let test_record_serial_types () = 363 + (* Test specific serial type encodings *) 364 + let open Btree.Record in 365 + (* Zero and One are special-cased *) 366 + let r0 = encode [ Vint 0L ] in 367 + let d0 = decode r0 in 368 + (match d0 with [ Vint 0L ] -> () | _ -> Alcotest.fail "expected 0"); 369 + let r1 = encode [ Vint 1L ] in 370 + let d1 = decode r1 in 371 + (match d1 with [ Vint 1L ] -> () | _ -> Alcotest.fail "expected 1"); 372 + (* Int8 range *) 373 + let r127 = encode [ Vint 127L ] in 374 + let d127 = decode r127 in 375 + (match d127 with [ Vint 127L ] -> () | _ -> Alcotest.fail "expected 127"); 376 + (* Negative *) 377 + let rm1 = encode [ Vint (-1L) ] in 378 + let dm1 = decode rm1 in 379 + match dm1 with [ Vint -1L ] -> () | _ -> Alcotest.fail "expected -1" 380 + 381 + let test_iter_order_after_splits () = 382 + (* Verify iteration order is correct after splits *) 383 + with_temp_file @@ fun file -> 384 + let pager = Btree.Pager.create ~page_size:512 file in 385 + let tree = Btree.Table.create pager in 386 + let n = 100 in 387 + for i = 1 to n do 388 + Btree.Table.insert tree ~rowid:(Int64.of_int i) (string_of_int i) 389 + done; 390 + let prev = ref 0L in 391 + Btree.Table.iter tree (fun rowid _ -> 392 + if rowid <= !prev then 393 + Alcotest.fail (Printf.sprintf "out of order: %Ld after %Ld" rowid !prev); 394 + prev := rowid); 395 + Alcotest.(check int64) "last rowid" (Int64.of_int n) !prev 396 + 188 397 let suite = 189 398 [ 190 399 ( "varint", 191 400 [ 192 401 Alcotest.test_case "roundtrip" `Quick test_varint_roundtrip; 193 402 Alcotest.test_case "size" `Quick test_varint_size; 403 + Alcotest.test_case "sqlite_vectors" `Quick test_varint_sqlite_vectors; 404 + Alcotest.test_case "boundaries" `Quick test_varint_boundaries; 194 405 ] ); 195 406 ( "record", 196 407 [ ··· 199 410 Alcotest.test_case "text" `Quick test_record_text; 200 411 Alcotest.test_case "blob" `Quick test_record_blob; 201 412 Alcotest.test_case "mixed" `Quick test_record_mixed; 413 + Alcotest.test_case "serial_types" `Quick test_record_serial_types; 202 414 ] ); 203 415 ( "page_header", 204 416 [ ··· 211 423 Alcotest.test_case "insert/find" `Quick test_table_insert_find; 212 424 Alcotest.test_case "multiple inserts" `Quick test_table_multiple_inserts; 213 425 Alcotest.test_case "iter" `Quick test_table_iter; 426 + ] ); 427 + ( "splits", 428 + [ 429 + Alcotest.test_case "leaf_split" `Quick test_leaf_split; 430 + Alcotest.test_case "multi_level" `Quick test_multi_level_tree; 431 + Alcotest.test_case "reverse_order" `Quick test_reverse_insert_order; 432 + Alcotest.test_case "random_order" `Quick test_random_insert_order; 433 + Alcotest.test_case "large_values" `Quick test_large_values; 434 + Alcotest.test_case "iter_order" `Quick test_iter_order_after_splits; 214 435 ] ); 215 436 ]