+2 -2

pegasus/lib/api/sync/subscribeRepos.ml

reviewed

··· 4 4 let cursor = 5 5 match Dream.query req "cursor" with 6 6 | Some s -> 7 7 - max 0 (Option.value (int_of_string_opt s) ~default:0) 7 7 + Some (max 0 (Option.value (int_of_string_opt s) ~default:0)) 8 8 | None -> 9 9 - 0 9 9 + None 10 10 in 11 11 let closed = ref false in 12 12 let send (bytes : bytes) =

+104 -91

pegasus/lib/sequencer.ml

reviewed

··· 621 621 end 622 622 623 623 module Live = struct 624 624 - let stream_with_backfill ~(conn : Data_store.t) ~(cursor : int) 625 625 - ~(send : bytes -> unit Lwt.t) : unit Lwt.t = 626 626 - let%lwt sub = Bus.subscribe () in 627 627 - let send_consumer_too_slow () = 628 628 - let err = 629 629 - { error= "ConsumerTooSlow" 630 630 - ; message= 631 631 - Some 632 632 - "you're not consuming messages fast enough! maybe \ 633 633 - com.atproto.sync.getRepo is more your speed?" } 634 634 - in 635 635 - send (Frame.encode_error err) 624 624 + let send_consumer_too_slow ~send = 625 625 + let err = 626 626 + { error= "ConsumerTooSlow" 627 627 + ; message= 628 628 + Some 629 629 + "you're not consuming messages fast enough! maybe \ 630 630 + com.atproto.sync.getRepo is more your speed?" } 636 631 in 632 632 + send (Frame.encode_error err) 633 633 + 634 634 + let live_loop ~(conn : Data_store.t) ~(sub : Bus.subscriber) ~(send : bytes -> unit Lwt.t) 635 635 + ~(start_seq : int) : unit Lwt.t = 636 636 + let rec loop last = 637 637 + if sub.Bus.closed then 638 638 + match sub.Bus.close_reason with 639 639 + | Some "ConsumerTooSlow" -> 640 640 + send_consumer_too_slow ~send 641 641 + | _ -> 642 642 + Lwt.return_unit 643 643 + else 644 644 + Lwt.catch 645 645 + (fun () -> 646 646 + let%lwt it = Bus.wait_next sub in 647 647 + if it.seq <= last then loop last 648 648 + else if it.seq > last + 1 then 649 649 + let%lwt gap = 650 650 + DB.request_seq_range ~earliest_seq:last ~latest_seq:(it.seq - 1) 651 651 + ~limit:1000 conn 652 652 + in 653 653 + let%lwt () = 654 654 + Lwt_list.iter_s 655 655 + (fun ev -> 656 656 + if ev.seq <= last then Lwt.return_unit 657 657 + else 658 658 + send 659 659 + ( match ev.kind with 660 660 + | Message (m, _) -> 661 661 + Frame.encode_message ~seq:ev.seq ~time:ev.time m 662 662 + | Error e -> 663 663 + Frame.encode_error e ) ) 664 664 + gap 665 665 + in 666 666 + send it.bytes >>= fun () -> loop it.seq 667 667 + else send it.bytes >>= fun () -> loop it.seq ) 668 668 + (fun _exn -> 669 669 + (* check if any failure was due to slow consumer *) 670 670 + match sub.Bus.close_reason with 671 671 + | Some "ConsumerTooSlow" -> 672 672 + send_consumer_too_slow ~send 673 673 + | _ -> 674 674 + Lwt.return_unit ) 675 675 + in 676 676 + loop start_seq 677 677 + 678 678 + let stream_live ~(conn : Data_store.t) ~(send : bytes -> unit Lwt.t) : unit Lwt.t = 679 679 + let%lwt sub = Bus.subscribe () in 637 680 Lwt.finalize 638 681 (fun () -> 639 639 - let%lwt head_db = DB.latest_seq conn in 640 640 - let cutoff = head_db in 641 641 - (* try backfill from buffer first *) 642 642 - let%lwt ring = Bus.ring_after cursor in 643 643 - let ring_covers = 644 644 - match ring with 645 645 - | [] -> 646 646 - false 647 647 - | first :: _ -> ( 648 648 - match List.rev ring with 649 649 - | [] -> 650 650 - false 651 651 - | last :: _ -> 652 652 - (* ring covers if it goes from <= cursor all the way to >= cutoff *) 653 653 - first.seq <= cursor && last.seq >= cutoff ) 654 654 - in 655 655 - ( if ring_covers then 656 656 - Lwt_list.iter_s (fun (it : Bus.item) -> send it.bytes) ring 657 657 - else 658 658 - let%lwt events = 659 659 - DB.request_seq_range ~earliest_seq:cursor ~latest_seq:cutoff 660 660 - ~limit:1000 conn 682 682 + let%lwt start_seq = DB.latest_seq conn in 683 683 + live_loop ~conn ~sub ~send ~start_seq ) 684 684 + (fun () -> Bus.unsubscribe sub) 685 685 + 686 686 + let stream_with_backfill ~(conn : Data_store.t) ~(cursor : int option) 687 687 + ~(send : bytes -> unit Lwt.t) : unit Lwt.t = 688 688 + match cursor with 689 689 + | None -> 690 690 + stream_live ~conn ~send 691 691 + | Some cursor -> 692 692 + let%lwt sub = Bus.subscribe () in 693 693 + Lwt.finalize 694 694 + (fun () -> 695 695 + let%lwt head_db = DB.latest_seq conn in 696 696 + let cutoff = head_db in 697 697 + (* try backfill from buffer first *) 698 698 + let%lwt ring = Bus.ring_after cursor in 699 699 + let ring_covers = 700 700 + match ring with 701 701 + | [] -> 702 702 + false 703 703 + | first :: _ -> ( 704 704 + match List.rev ring with 705 705 + | [] -> 706 706 + false 707 707 + | last :: _ -> 708 708 + (* ring covers if it goes from <= cursor all the way to >= cutoff *) 709 709 + first.seq <= cursor && last.seq >= cutoff ) 661 710 in 662 662 - Lwt_list.iter_s 663 663 - (fun ev -> 664 664 - match ev.kind with 665 665 - | Error _ -> 666 666 - Lwt.return_unit 667 667 - | Message (payload, _) -> 668 668 - send 669 669 - (Frame.encode_message ~seq:ev.seq ~time:ev.time payload) ) 670 670 - events ) 671 671 - >>= fun () -> 672 672 - (* bail if consumer too slow *) 673 673 - if sub.Bus.closed then 674 674 - match sub.Bus.close_reason with 675 675 - | Some "ConsumerTooSlow" -> 676 676 - send_consumer_too_slow () 677 677 - | _ -> 678 678 - Lwt.return_unit 679 679 - else 680 680 - (* live tail *) 681 681 - let rec loop last = 711 711 + ( if ring_covers then 712 712 + Lwt_list.iter_s (fun (it : Bus.item) -> send it.bytes) ring 713 713 + else 714 714 + let%lwt events = 715 715 + DB.request_seq_range ~earliest_seq:cursor ~latest_seq:cutoff 716 716 + ~limit:1000 conn 717 717 + in 718 718 + Lwt_list.iter_s 719 719 + (fun ev -> 720 720 + match ev.kind with 721 721 + | Error _ -> 722 722 + Lwt.return_unit 723 723 + | Message (payload, _) -> 724 724 + send 725 725 + (Frame.encode_message ~seq:ev.seq ~time:ev.time payload) ) 726 726 + events ) 727 727 + >>= fun () -> 728 728 + (* bail if consumer too slow *) 682 729 if sub.Bus.closed then 683 730 match sub.Bus.close_reason with 684 731 | Some "ConsumerTooSlow" -> 685 685 - send_consumer_too_slow () 732 732 + send_consumer_too_slow ~send 686 733 | _ -> 687 734 Lwt.return_unit 688 688 - else 689 689 - Lwt.catch 690 690 - (fun () -> 691 691 - let%lwt it = Bus.wait_next sub in 692 692 - if it.seq <= last then loop last 693 693 - else if it.seq > last + 1 then 694 694 - let%lwt gap = 695 695 - DB.request_seq_range ~earliest_seq:last ~latest_seq:it.seq 696 696 - ~limit:1000 conn 697 697 - in 698 698 - let%lwt () = 699 699 - Lwt_list.iter_s 700 700 - (fun ev -> 701 701 - if ev.seq <= last then Lwt.return_unit 702 702 - else 703 703 - send 704 704 - ( match ev.kind with 705 705 - | Message (m, _) -> 706 706 - Frame.encode_message ~seq:ev.seq ~time:ev.time 707 707 - m 708 708 - | Error e -> 709 709 - Frame.encode_error e ) ) 710 710 - gap 711 711 - in 712 712 - send it.bytes >>= fun () -> loop it.seq 713 713 - else send it.bytes >>= fun () -> loop it.seq ) 714 714 - (fun _exn -> 715 715 - (* check if any failure was due to slow consumer *) 716 716 - match sub.Bus.close_reason with 717 717 - | Some "ConsumerTooSlow" -> 718 718 - send_consumer_too_slow () 719 719 - | _ -> 720 720 - Lwt.return_unit ) 721 721 - in 722 722 - loop cutoff ) 723 723 - (fun () -> Bus.unsubscribe sub) 735 735 + else live_loop ~conn ~sub ~send ~start_seq:cutoff ) 736 736 + (fun () -> Bus.unsubscribe sub) 724 737 end 725 738 726 739 let sequence_commit (conn : Data_store.t) ~(did : string) ~(commit : Cid.t)

+2 -2

pegasus/test/test_sequencer.ml

reviewed

··· 107 107 in 108 108 let stream = 109 109 Lwt.catch 110 110 - (fun () -> Sequencer.Live.stream_with_backfill ~conn ~cursor:0 ~send) 110 110 + (fun () -> Sequencer.Live.stream_with_backfill ~conn ~cursor:(Some 0) ~send) 111 111 (fun _ -> Lwt.return_unit) 112 112 in 113 113 let _ = Lwt.async (fun () -> stream) in ··· 162 162 Lwt.async (fun () -> 163 163 Lwt.catch 164 164 (fun () -> 165 165 - Sequencer.Live.stream_with_backfill ~conn ~cursor:0 ~send ) 165 165 + Sequencer.Live.stream_with_backfill ~conn ~cursor:(Some 0) ~send ) 166 166 (fun _ -> Lwt.return_unit) ) 167 167 in 168 168 (* after stream starts, directly insert identity row with seq=3, then publish identity for seq=4 via bus *)