this repo has no description
Use BTree.Store to read and write pages
+49
-54
+6
-29
lib/btree/btree.ex
+6
-29
lib/btree/btree.ex
···
16
16
17
17
@type t :: %__MODULE__{
18
18
opts: Opts.t,
19
-
store: :ets.table,
20
-
page_store: :ets.table,
19
+
store: Store.t,
21
20
free_list: FreeList.t,
22
21
root_store: :ets.table,
23
22
···
31
30
@enforce_keys [
32
31
:opts,
33
32
:store,
34
-
:page_store,
35
33
:free_list,
36
34
:root_store,
37
35
···
60
58
btree = %BTree{
61
59
opts: opts,
62
60
store: Store.new(path, opts.page_size),
63
-
page_store: :ets.new(__MODULE__, [:set, :protected]),
64
61
free_list: FreeList.new(),
65
62
root_store: :ets.new(__MODULE__, [:ordered_set, :protected]),
66
63
···
92
89
93
90
defp create_root_pages(%BTree{} = btree) do
94
91
%{
92
+
store: store,
95
93
free_list: free_list,
96
-
page_store: page_store,
97
94
root_store: root_store,
98
95
opts: %{
99
96
page_size: opt_page_size,
···
105
102
[root_index, leaf_index] = FreeList.reserve_pages(free_list, 2)
106
103
107
104
[{_ek, leaf_data}] = Page.encode_page_from_pairs(:leaf, opt_page_size, [], @keyspace_end_key)
108
-
leaf_checksum = checksum(leaf_data)
109
-
:ok = write_page(page_store, leaf_index, leaf_data)
105
+
leaf_checksum = Store.checksum(leaf_data)
106
+
:ok = Store.write_page(store, leaf_index, leaf_data)
110
107
111
108
leaf_end_key = "\xFF\xFF\xFF\xFF"
112
109
leaf_address = <<leaf_index::integer-64, leaf_checksum::binary-16>>
113
110
114
111
root_pairs = [{leaf_end_key, leaf_address}]
115
112
[{_ek, root_data}] = Writer.write_root(btree, root_pairs)
116
-
root_checksum = checksum(root_data)
117
-
:ok = write_page(page_store, root_index, root_data)
113
+
root_checksum = Store.checksum(root_data)
114
+
:ok = Store.write_page(store, root_index, root_data)
118
115
119
116
:ets.insert(root_store, {:root_address, {root_index, root_checksum}})
120
117
:ok
121
-
end
122
-
123
-
def checksum(data) do
124
-
<<hash::binary-16, _rest::binary-16>> = :crypto.hash(:sha256, data)
125
-
hash
126
-
end
127
-
128
-
def write_page(page_store, page_index, data) do
129
-
:ets.insert(page_store, {page_index, data})
130
-
:ok
131
-
end
132
-
133
-
def read_page(page_store, page_index, page_checksum) do
134
-
case :ets.lookup(page_store, page_index) do
135
-
[{_index, data}] ->
136
-
assert checksum(data) == page_checksum
137
-
data
138
-
[] ->
139
-
assert false
140
-
end
141
118
end
142
119
143
120
def write_pair(pairs, key, value, acc \\ [])
+12
-11
lib/btree/free_list.ex
+12
-11
lib/btree/free_list.ex
···
1
1
defmodule Hobbes.BTree.FreeList do
2
2
alias Hobbes.BTree
3
+
alias Hobbes.BTree.Store
3
4
4
5
import ExUnit.Assertions, only: [assert: 1]
5
6
import Hobbes.BTree.Utils
···
18
19
19
20
def flush(%BTree{} = btree) do
20
21
%{
21
-
page_store: page_store,
22
+
store: store,
22
23
free_list: free_list,
23
24
opts: %{
24
25
page_size: opt_page_size,
···
28
29
page_acc = ""
29
30
last_address = c_null_address()
30
31
31
-
{page_acc, last_address} = do_flush(page_store, free_list, opt_page_size, :pending, page_acc, last_address, [:pending | -1])
32
-
{page_acc, last_address} = do_flush(page_store, free_list, opt_page_size, :free, page_acc, last_address, [:free | -1])
32
+
{page_acc, last_address} = do_flush(store, free_list, opt_page_size, :pending, page_acc, last_address, [:pending | -1])
33
+
{page_acc, last_address} = do_flush(store, free_list, opt_page_size, :free, page_acc, last_address, [:free | -1])
33
34
34
-
free_list_tail_address = write_free_list_page(page_store, free_list, opt_page_size, page_acc, last_address)
35
+
free_list_tail_address = write_free_list_page(store, free_list, opt_page_size, page_acc, last_address)
35
36
[{:max_index, free_list_max_index}] = :ets.lookup(free_list, :max_index)
36
37
37
38
:ok = do_release_pending(free_list)
···
42
43
}
43
44
end
44
45
45
-
defp do_flush(page_store, free_list, opt_page_size, subspace, page_acc, last_address, last_key) do
46
+
defp do_flush(store, free_list, opt_page_size, subspace, page_acc, last_address, last_key) do
46
47
entries_size_max = opt_page_size - c_address_bytes()
47
48
48
49
case :ets.next(free_list, last_key) do
49
50
[^subspace | index] = key ->
50
51
case (byte_size(page_acc) + c_free_list_entry_size()) > entries_size_max do
51
52
true ->
52
-
new_address = write_free_list_page(page_store, free_list, opt_page_size, page_acc, last_address)
53
+
new_address = write_free_list_page(store, free_list, opt_page_size, page_acc, last_address)
53
54
54
55
page_acc = ""
55
56
last_address = new_address
56
-
do_flush(page_store, free_list, opt_page_size, subspace, page_acc, last_address, last_key)
57
+
do_flush(store, free_list, opt_page_size, subspace, page_acc, last_address, last_key)
57
58
58
59
false ->
59
60
# See constant: c_free_list_entry_size()
···
63
64
>>
64
65
65
66
last_key = key
66
-
do_flush(page_store, free_list, opt_page_size, subspace, page_acc, last_address, last_key)
67
+
do_flush(store, free_list, opt_page_size, subspace, page_acc, last_address, last_key)
67
68
end
68
69
69
70
_ ->
···
72
73
73
74
end
74
75
75
-
defp write_free_list_page(page_store, free_list, opt_page_size, page_data, last_address) do
76
+
defp write_free_list_page(store, free_list, opt_page_size, page_data, last_address) do
76
77
{last_index, last_checksum} = last_address
77
78
78
79
pad_bytes = opt_page_size - byte_size(page_data) - c_address_bytes()
···
88
89
assert byte_size(page_data) == opt_page_size
89
90
90
91
[index] = reserve_pages(free_list, 1)
91
-
checksum = BTree.checksum(page_data)
92
+
checksum = Store.checksum(page_data)
92
93
93
-
:ok = BTree.write_page(page_store, index, page_data)
94
+
:ok = Store.write_page(store, index, page_data)
94
95
{index, checksum}
95
96
end
96
97
+8
-7
lib/btree/reader.ex
+8
-7
lib/btree/reader.ex
···
1
1
defmodule Hobbes.BTree.Reader do
2
2
alias Hobbes.BTree
3
+
alias Hobbes.BTree.Store
3
4
4
5
import ExUnit.Assertions, only: [assert: 1]
5
6
import Hobbes.BTree.Utils
···
252
253
@spec new(BTree.t, binary, boolean) :: StorageIterator.t
253
254
def new(%BTree{} = btree, start_key, reverse?) do
254
255
%{
255
-
page_store: page_store,
256
+
store: store,
256
257
root_store: root_store,
257
258
} = btree
258
259
259
260
[{:root_address, {root_index, root_checksum}}] = :ets.lookup(root_store, :root_address)
260
-
root_data = BTree.read_page(page_store, root_index, root_checksum)
261
+
root_data = Store.read_page(store, root_index, root_checksum)
261
262
262
263
it = %StorageIterator{
263
264
btree: btree,
···
310
311
0x00 ->
311
312
{i, {_end_key, child_address}} = seek_inner(page_data, search_key)
312
313
<<child_index::integer-64, child_checksum::binary-16>> = child_address
313
-
child_data = BTree.read_page(it.btree.page_store, child_index, child_checksum)
314
+
child_data = Store.read_page(it.btree.store, child_index, child_checksum)
314
315
315
316
it = %{it |
316
317
page_stack: [{page_data, i} | it.page_stack],
···
433
434
current_i = current_i - 1
434
435
case type_byte do
435
436
0x00 ->
436
-
child_page = inner_child_page_at(it.btree.page_store, current_page, pair_count, current_i)
437
+
child_page = inner_child_page_at(it.btree.store, current_page, pair_count, current_i)
437
438
<<
438
439
_::binary-size(byte_size(current_page) - c_page_trailer_bytes()),
439
440
child_pair_count::integer-16,
···
492
493
current_i = current_i + 1
493
494
case type_byte do
494
495
0x00 ->
495
-
child_page = inner_child_page_at(it.btree.page_store, current_page, pair_count, current_i)
496
+
child_page = inner_child_page_at(it.btree.store, current_page, pair_count, current_i)
496
497
497
498
it = %{it |
498
499
page_stack: [{current_page, current_i} | it.page_stack],
···
529
530
end
530
531
end
531
532
532
-
defp inner_child_page_at(page_store, page_data, pointer_count, i) do
533
+
defp inner_child_page_at(store, page_data, pointer_count, i) do
533
534
page_size = byte_size(page_data)
534
535
key_count = pointer_count - 1
535
536
slots_start = page_size - (key_count * c_page_slot_entry_bytes()) - c_page_trailer_bytes()
···
542
543
_::binary,
543
544
>> = page_data
544
545
545
-
BTree.read_page(page_store, child_index, child_checksum)
546
+
Store.read_page(store, child_index, child_checksum)
546
547
end
547
548
548
549
defp leaf_pair_at(page_data, pair_count, i) do
+16
lib/btree/store.ex
+16
lib/btree/store.ex
···
4
4
import ExUnit.Assertions, only: [assert: 1]
5
5
6
6
@type t :: BlockDevice.t
7
+
@type checksum :: <<_::128>>
7
8
8
9
@spec new(String.t | :memory, pos_integer) :: t
9
10
def new(path, block_size) do
10
11
case path do
11
12
:memory -> BlockDevice.new(block_size)
12
13
end
14
+
end
15
+
16
+
@spec checksum(binary) :: checksum
17
+
def checksum(binary) do
18
+
<<hash::binary-16, _rest::binary-16>> = :crypto.hash(:sha256, binary)
19
+
hash
20
+
end
21
+
22
+
@spec read_page(t, non_neg_integer, checksum) :: binary
23
+
def read_page({:block_device, opt_block_size, _} = bd, page_index, checksum) do
24
+
pos = page_index * opt_block_size
25
+
page_data = BlockDevice.read(bd, pos, opt_block_size)
26
+
27
+
assert checksum(page_data) == checksum
28
+
page_data
13
29
end
14
30
15
31
@spec write_page(t, non_neg_integer, binary) :: :ok
+7
-7
lib/btree/writer.ex
+7
-7
lib/btree/writer.ex
···
1
1
defmodule Hobbes.BTree.Writer do
2
2
alias Hobbes.BTree
3
-
alias Hobbes.BTree.{FreeList, Page}
3
+
alias Hobbes.BTree.{Store, FreeList, Page}
4
4
5
5
import ExUnit.Assertions, only: [assert: 1]
6
6
import Hobbes.BTree.Utils
···
123
123
new_root_addresses =
124
124
Enum.zip(new_root_data, new_indices)
125
125
|> Enum.map(fn {{ek, data}, index} ->
126
-
:ok = BTree.write_page(btree.page_store, index, data)
126
+
:ok = Store.write_page(btree.store, index, data)
127
127
128
-
checksum = BTree.checksum(data)
128
+
checksum = Store.checksum(data)
129
129
{ek, <<index::integer-64, checksum::binary-16>>}
130
130
end)
131
131
···
137
137
138
138
defp do_flush(%BTree{} = btree, page_index, page_checksum, page_sk, page_ek) do
139
139
%{
140
-
page_store: page_store,
140
+
store: store,
141
141
free_list: free_list,
142
142
opts: %{
143
143
page_size: opt_page_size,
144
144
},
145
145
} = btree
146
146
147
-
page_data = BTree.read_page(page_store, page_index, page_checksum)
147
+
page_data = Store.read_page(store, page_index, page_checksum)
148
148
prefix_size = opt_page_size - c_page_trailer_bytes()
149
149
# See constant: c_page_trailer_bytes()
150
150
<<
···
166
166
new_indices = FreeList.reserve_pages(free_list, length(new_pages))
167
167
Enum.zip(new_pages, new_indices)
168
168
|> Enum.map(fn {{ek, data}, index} ->
169
-
:ok = BTree.write_page(page_store, index, data)
169
+
:ok = Store.write_page(store, index, data)
170
170
171
-
checksum = BTree.checksum(data)
171
+
checksum = Store.checksum(data)
172
172
{ek, <<index::integer-64, checksum::binary-16>>}
173
173
end)
174
174
end