this repo has no description
2
fork

Configure Feed

Select the types of activity you want to include in your feed.

Flush FreeList

garrison d534a62b be75c0d2

+159 -4
+17
lib/btree/btree.ex
··· 27 27 :free_list, 28 28 :root_store, 29 29 :write_buffer, 30 + 31 + :free_list_tail_address, 32 + :free_list_max_index, 30 33 ] 31 34 defstruct @enforce_keys 32 35 ··· 48 51 free_list: FreeList.new(), 49 52 root_store: :ets.new(__MODULE__, [:ordered_set, :protected]), 50 53 write_buffer: :ets.new(__MODULE__, [:ordered_set, :private]), 54 + free_list_tail_address: nil, 55 + free_list_max_index: nil, 51 56 } 52 57 :ok = create_root_pages(btree) 53 58 54 59 btree 60 + end 61 + 62 + def commit(%BTree{} = btree) do 63 + %{ 64 + free_list_tail_address: free_list_tail_address, 65 + free_list_max_index: free_list_max_index, 66 + } = FreeList.flush(btree) 67 + 68 + %{btree | 69 + free_list_tail_address: free_list_tail_address, 70 + free_list_max_index: free_list_max_index, 71 + } 55 72 end 56 73 57 74 @keyspace_end_key "\xFF\xFF\xFF\xFF"
+107
lib/btree/free_list.ex
··· 1 1 defmodule Hobbes.BTree.FreeList do 2 + alias Hobbes.BTree 3 + 4 + import ExUnit.Assertions, only: [assert: 1] 5 + import Hobbes.BTree.Utils 6 + 7 + @dialyzer :no_improper_lists 8 + 2 9 @type t :: :ets.table 3 10 4 11 @spec new :: t ··· 9 16 free_list 10 17 end 11 18 19 + def flush(%BTree{} = btree) do 20 + %{ 21 + page_store: page_store, 22 + free_list: free_list, 23 + opts: %{ 24 + page_size: opt_page_size, 25 + }, 26 + } = btree 27 + 28 + page_acc = "" 29 + last_address = c_null_address() 30 + 31 + {page_acc, last_address} = do_flush(page_store, free_list, opt_page_size, :pending, page_acc, last_address, [:pending | -1]) 32 + {page_acc, last_address} = do_flush(page_store, free_list, opt_page_size, :free, page_acc, last_address, [:free | -1]) 33 + 34 + free_list_tail_address = write_free_list_page(page_store, free_list, opt_page_size, page_acc, last_address) 35 + [{:max_index, free_list_max_index}] = :ets.lookup(free_list, :max_index) 36 + 37 + :ok = do_release_pending(free_list) 38 + 39 + %{ 40 + free_list_tail_address: free_list_tail_address, 41 + free_list_max_index: free_list_max_index, 42 + } 43 + end 44 + 45 + defp do_flush(page_store, free_list, opt_page_size, subspace, page_acc, last_address, last_key) do 46 + entries_size_max = opt_page_size - c_address_bytes() 47 + 48 + case :ets.next(free_list, last_key) do 49 + [^subspace | index] = key -> 50 + case (byte_size(page_acc) + c_free_list_entry_size()) > entries_size_max do 51 + true -> 52 + new_address = write_free_list_page(page_store, free_list, opt_page_size, page_acc, last_address) 53 + 54 + page_acc = "" 55 + last_address = new_address 56 + do_flush(page_store, free_list, opt_page_size, subspace, page_acc, last_address, last_key) 57 + 58 + false -> 59 + # See constant: c_free_list_entry_size() 60 + page_acc = << 61 + page_acc::binary, 62 + index::integer-64, 63 + >> 64 + 65 + last_key = key 66 + do_flush(page_store, free_list, opt_page_size, subspace, page_acc, last_address, last_key) 67 + end 68 + 69 + _ -> 70 + {page_acc, last_address} 71 + end 72 + 73 + end 74 + 75 + defp write_free_list_page(page_store, free_list, opt_page_size, page_data, last_address) do 76 + {last_index, last_checksum} = last_address 77 + 78 + pad_bytes = opt_page_size - byte_size(page_data) - c_address_bytes() 79 + assert pad_bytes >= 0 80 + 81 + page_data = << 82 + page_data::binary, 83 + 0::integer-unit(8)-size(pad_bytes), 84 + last_index::integer-64, 85 + last_checksum::binary-16, 86 + >> 87 + 88 + assert byte_size(page_data) == opt_page_size 89 + 90 + [index] = reserve_pages(free_list, 1) 91 + checksum = BTree.checksum(page_data) 92 + 93 + :ok = BTree.write_page(page_store, index, page_data) 94 + {index, checksum} 95 + end 96 + 97 + defp do_release_pending(free_list) do 98 + case :ets.next(free_list, [:pending | -1]) do 99 + [:pending | index] = key -> 100 + :ets.delete(free_list, key) 101 + :ets.insert(free_list, {[:free | index], nil}) 102 + do_release_pending(free_list) 103 + 104 + _ -> :ok 105 + end 106 + end 107 + 12 108 @spec reserve_pages(t, non_neg_integer) :: [non_neg_integer] 13 109 def reserve_pages(free_list, count) do 14 110 [{:max_index, max_index}] = :ets.lookup(free_list, :max_index) 15 111 16 112 :ets.insert(free_list, {:max_index, max_index + count}) 17 113 Enum.to_list((max_index + 1)..(max_index + count)) 114 + end 115 + 116 + @spec free_page(t, non_neg_integer) :: :ok 117 + def free_page(free_list, page_index) do 118 + :ets.insert(free_list, {[:pending | page_index], nil}) 119 + :ok 120 + end 121 + 122 + @doc false 123 + def dump(free_list) do 124 + :ets.tab2list(free_list) 18 125 end 19 126 end
+19 -3
lib/btree/fuzz/model_fuzz.ex
··· 60 60 end 61 61 62 62 defp random_op do 63 - case Enum.random(1..2) do 64 - 1 -> :apply_batch 65 - 2 -> :scan 63 + case Enum.random(1..20) do 64 + 1 -> :commit 65 + _ -> 66 + case Enum.random(1..2) do 67 + 1 -> :apply_batch 68 + 2 -> :scan 69 + end 66 70 end 71 + end 72 + 73 + defp execute(:commit, %State{} = state) do 74 + %{ 75 + btree: btree, 76 + } = state 77 + 78 + btree = BTree.commit(btree) 79 + 80 + %{state | 81 + btree: btree, 82 + } 67 83 end 68 84 69 85 defp execute(:apply_batch, %State{} = state) do
+8
lib/btree/utils.ex
··· 1 1 defmodule Hobbes.BTree.Utils do 2 + defmacro c_null_address, do: {0, <<0::integer-128>>} 3 + 4 + # index + checksum 5 + defmacro c_address_bytes, do: 8 + 16 6 + 7 + # index 8 + defmacro c_free_list_entry_size, do: 8 9 + 2 10 # pair_count + type_byte 3 11 defmacro c_page_trailer_bytes, do: 2 + 1 4 12 # key_size + value_size
+4
lib/btree/writer.ex
··· 90 90 0x01 -> do_flush_leaf(btree, page_data, page_sk, page_ek) 91 91 end 92 92 93 + # Free old page 94 + :ok = FreeList.free_page(free_list, page_index) 95 + 96 + # Write new pages 93 97 new_indices = FreeList.reserve_pages(free_list, length(new_pages)) 94 98 Enum.zip(new_pages, new_indices) 95 99 |> Enum.map(fn {{ek, data}, index} ->
+4 -1
test/btree_test.exs
··· 13 13 14 14 test "fuzz" do 15 15 Hobbes.BTree.Fuzz.ModelFuzz.run(100, [ 16 - iterations: 100, 16 + iterations: 200, 17 17 key_bits: 32, 18 18 ]) 19 19 end ··· 40 40 {:write, str, String.reverse(str)} 41 41 end) 42 42 :ok = BTree.Writer.apply_batch(btree, mutations) 43 + 44 + btree = BTree.commit(btree) 45 + _btree = BTree.commit(btree) 43 46 end 44 47 end 45 48 end