this repo has no description
Add range deletes to HybridKV with support for gets
+156
-51
+24
-5
lib/hybrid_kv.ex
+24
-5
lib/hybrid_kv.ex
···
1
1
defmodule Hobbes.HybridKV do
2
-
alias Hobbes.{HybridKV, MemKV}
2
+
alias Hobbes.{HybridKV, MemKV, RangeForest}
3
+
alias Hobbes.RangeForest.RangeTree
3
4
alias Hobbes.KV.FlatKV
4
5
alias Hobbes.Structs.RangeResult
5
6
6
-
@enforce_keys [:mem_kv, :storage_kv]
7
+
@enforce_keys [:mem_kv, :storage_kv, :deleted_forest]
7
8
defstruct @enforce_keys
8
9
9
10
def new do
10
-
%__MODULE__{
11
+
%HybridKV{
11
12
mem_kv: MemKV.new(),
12
13
storage_kv: FlatKV.new(),
14
+
deleted_forest: RangeForest.new(),
13
15
}
14
16
end
15
17
16
-
@spec put(%HybridKV{}, non_neg_integer, binary, binary) :: :ok
18
+
@spec put(%HybridKV{}, non_neg_integer, binary, binary) :: %HybridKV{}
17
19
def put(%HybridKV{} = kv, version, key, value)
18
20
when is_integer(version) and version >= 0 and is_binary(key) and is_binary(value) do
19
21
:ok = MemKV.put(kv.mem_kv, version, key, value)
22
+
23
+
case RangeForest.split_at(kv.deleted_forest, version, key) do
24
+
{:updated, deleted_forest} -> %HybridKV{kv | deleted_forest: deleted_forest}
25
+
:noop -> kv
26
+
end
20
27
end
21
28
22
29
@spec delete(%HybridKV{}, non_neg_integer, binary) :: :ok
···
24
31
:ok = MemKV.delete(kv.mem_kv, version, key)
25
32
end
26
33
34
+
@spec delete_range(%HybridKV{}, non_neg_integer, binary, binary) :: %HybridKV{}
35
+
def delete_range(%HybridKV{} = kv, version, start_key, end_key) do
36
+
%HybridKV{kv | deleted_forest: RangeForest.add_range(kv.deleted_forest, version, start_key, end_key)}
37
+
end
38
+
27
39
@spec get(%HybridKV{}, non_neg_integer, binary) :: binary | nil
28
40
def get(%HybridKV{} = kv, version, key)
29
41
when is_integer(version) and version >= 0 and is_binary(key) do
30
-
case MemKV.get(kv.mem_kv, version, key) do
42
+
floor_version =
43
+
case RangeForest.tree_at(kv.deleted_forest, version) |> RangeTree.intersect_key(0, key) do
44
+
{_sk, _ek, v} -> v + 1
45
+
nil -> 0
46
+
end
47
+
48
+
case MemKV.get(kv.mem_kv, version, floor_version, key) do
31
49
nil -> FlatKV.get(kv.storage_kv, key)
32
50
:deleted -> nil
33
51
value -> value
···
219
237
%{
220
238
mem_kv: MemKV.dump(kv.mem_kv),
221
239
storage_kv: FlatKV.dump(kv.storage_kv),
240
+
deleted_forest: RangeForest.dump(kv.deleted_forest),
222
241
}
223
242
end
224
243
end
+11
lib/kv/test_kv.ex
+11
lib/kv/test_kv.ex
···
38
38
%TestKV{kv | maps: Map.put(kv.maps, version, updated), version: version}
39
39
end
40
40
41
+
def delete_range(%TestKV{} = kv, version, start_key, end_key) do
42
+
if version < kv.version, do: raise "TestKV cannot write at old versions!"
43
+
44
+
updated =
45
+
Map.fetch!(kv.maps, kv.version)
46
+
|> Enum.reject(fn {k, _v} -> k >= start_key and k < end_key end)
47
+
|> Map.new()
48
+
49
+
%TestKV{kv | maps: Map.put(kv.maps, version, updated), version: version}
50
+
end
51
+
41
52
defp read_version_for(%TestKV{maps: maps}, version) do
42
53
# Newest version which is <= `version`
43
54
maps
+4
-4
lib/mem_kv.ex
+4
-4
lib/mem_kv.ex
···
26
26
:ok
27
27
end
28
28
29
-
@spec get(:ets.table, non_neg_integer, binary) :: binary | :deleted | nil
30
-
def get(table, version, key)
29
+
@spec get(:ets.table, non_neg_integer, non_neg_integer, binary) :: binary | :deleted | nil
30
+
def get(table, version, floor_version, key)
31
31
when is_integer(version) and version >= 0 and is_binary(key) do
32
32
case :ets.prev(table, {key, version + 1}) do
33
-
{^key, _version} = k ->
34
-
[{^k, value}] = :ets.lookup(table, k)
33
+
{^key, version} = full_key when version >= floor_version ->
34
+
[{^full_key, value}] = :ets.lookup(table, full_key)
35
35
value
36
36
37
37
_ ->
+1
-1
lib/meta_store.ex
+1
-1
lib/meta_store.ex
+8
-1
lib/range_forest.ex
+8
-1
lib/range_forest.ex
···
82
82
def split_at(tree, version, key) do
83
83
next = key <> "\x00"
84
84
case :gb_trees.smaller(next, tree) do
85
-
{sk, {ek, ^version}} ->
85
+
{^key, {ek, ^version}} ->
86
+
tree = :gb_trees.delete(key, tree)
87
+
case ek do
88
+
^next -> {:updated, tree}
89
+
ek -> {:updated, :gb_trees.insert(next, {ek, version}, tree)}
90
+
end
91
+
92
+
{sk, {ek, ^version}} when ek > key ->
86
93
tree = :gb_trees.update(sk, {key, version}, tree)
87
94
case ek do
88
95
^next -> {:updated, tree}
+30
-10
test/hybrid_kv_test.exs
+30
-10
test/hybrid_kv_test.exs
···
22
22
23
23
alias Verifier
24
24
25
-
@enforce_keys [:hybrid_kv, :test_kv, :version, :durable_version]
25
+
@enforce_keys [:hybrid_kv, :test_kv, :version, :durable_version, :seed]
26
26
defstruct @enforce_keys
27
27
28
-
def new do
28
+
def new(seed) do
29
+
:rand.seed(:exsss, seed)
30
+
29
31
%Verifier{
30
32
hybrid_kv: HybridKV.new(),
31
33
test_kv: TestKV.new(),
32
34
version: 1,
33
35
durable_version: 0,
36
+
seed: seed,
34
37
}
35
38
end
36
39
···
42
45
perform(random_op(), verifier)
43
46
rescue
44
47
e in [ExUnit.AssertionError] ->
45
-
e = Map.update!(e, :message, &(&1 <> " (at op #{i})"))
48
+
e = Map.update!(e, :message, &(&1 <> " (at op #{i}, seed=#{inspect(verifier.seed)})"))
46
49
reraise e, __STACKTRACE__
47
50
end
48
51
end)
···
56
59
], limit: :infinity
57
60
end
58
61
59
-
@ops [:put, :delete, :get, :scan]
62
+
@ops [:put, :delete, :delete_range, :get]
60
63
defp random_op do
61
64
case Enum.random(1..100) do
62
-
1 -> :flush
65
+
#1 -> :flush
63
66
_ -> Enum.random(@ops)
64
67
end
65
68
end
···
71
74
key = rand_hash()
72
75
value = rand_hash()
73
76
74
-
:ok = HybridKV.put(hkv, version, key, value)
77
+
hkv = HybridKV.put(hkv, version, key, value)
75
78
tkv = TestKV.put(tkv, version, key, value)
76
79
77
80
%Verifier{verifier | hybrid_kv: hkv, test_kv: tkv}
···
89
92
%Verifier{verifier | hybrid_kv: hkv, test_kv: tkv}
90
93
end
91
94
95
+
defp perform(:delete_range, %Verifier{hybrid_kv: hkv, test_kv: tkv} = verifier) do
96
+
verifier = inc_version(verifier)
97
+
98
+
version = verifier.version
99
+
start_key = rand_hash()
100
+
end_key = rand_hash()
101
+
{start_key, end_key} =
102
+
cond do
103
+
start_key < end_key -> {start_key, end_key}
104
+
start_key > end_key -> {end_key, start_key}
105
+
start_key == end_key -> {start_key, start_key <> "\x00"}
106
+
end
107
+
108
+
hkv = HybridKV.delete_range(hkv, version, start_key, end_key)
109
+
tkv = TestKV.delete_range(tkv, version, start_key, end_key)
110
+
111
+
%Verifier{verifier | hybrid_kv: hkv, test_kv: tkv}
112
+
end
113
+
92
114
defp perform(:get, %Verifier{hybrid_kv: hkv, test_kv: tkv} = verifier) do
93
115
read_version = rand_read_version(verifier)
94
116
···
166
188
describe "verify HybridKV" do
167
189
@tag :hkv_verify
168
190
test "operations" do
169
-
:rand.seed(:exsss, {100, 101, 102})
170
-
Verifier.new()
191
+
Verifier.new({101, 101, 102})
171
192
|> Verifier.run(1000)
172
193
end
173
194
···
176
197
test "operations multi (slow)" do
177
198
# 300 verification tests (takes about 1.3s)
178
199
Enum.map(1..300, fn s ->
179
-
:rand.seed(:exsss, {100 + s, 101, 102})
180
-
Verifier.new()
200
+
Verifier.new({100 + s, 101, 102})
181
201
|> Verifier.run(1000)
182
202
end)
183
203
end
+16
test/kv/test_kv_test.exs
+16
test/kv/test_kv_test.exs
···
35
35
assert TestKV.get(kv, 5, "foo_c") == nil
36
36
end
37
37
38
+
test "delete_range/4" do
39
+
kv = Enum.reduce(1..9, TestKV.new(), fn i, kv ->
40
+
TestKV.put(kv, 1, "foo_#{i}", "bar_#{i}")
41
+
end)
42
+
43
+
assert kv = TestKV.delete_range(kv, 1, "foo_3", "foo_7")
44
+
45
+
assert TestKV.scan(kv, 1, "", "\xFF").pairs == [
46
+
{"foo_1", "bar_1"},
47
+
{"foo_2", "bar_2"},
48
+
{"foo_7", "bar_7"},
49
+
{"foo_8", "bar_8"},
50
+
{"foo_9", "bar_9"},
51
+
]
52
+
end
53
+
38
54
test "scan/5 forward", %{kv: kv} do
39
55
assert %RangeResult{pairs: [
40
56
{"foo_a", "bar_a_1"},
+30
-30
test/mem_kv_test.exs
+30
-30
test/mem_kv_test.exs
···
19
19
20
20
test "puts value", %{kv: kv} do
21
21
assert :ok = MemKV.put(kv, 1, "foo", "bar")
22
-
assert MemKV.get(kv, 1, "foo") == "bar"
22
+
assert MemKV.get(kv, 1, 0, "foo") == "bar"
23
23
end
24
24
25
25
test "overwrites value", %{kv: kv} do
26
26
assert :ok = MemKV.put(kv, 1, "foo", "bar")
27
27
assert :ok = MemKV.put(kv, 1, "foo", "baz")
28
-
assert MemKV.get(kv, 1, "foo") == "baz"
28
+
assert MemKV.get(kv, 1, 0, "foo") == "baz"
29
29
end
30
30
31
31
test "puts multiple versions", %{kv: kv} do
32
32
assert :ok = MemKV.put(kv, 1, "foo", "bar")
33
33
assert :ok = MemKV.put(kv, 2, "foo", "baz")
34
34
35
-
assert MemKV.get(kv, 1, "foo") == "bar"
36
-
assert MemKV.get(kv, 2, "foo") == "baz"
35
+
assert MemKV.get(kv, 1, 0, "foo") == "bar"
36
+
assert MemKV.get(kv, 2, 0, "foo") == "baz"
37
37
end
38
38
39
39
test "puts multiple keys", %{kv: kv} do
···
41
41
assert :ok = MemKV.put(kv, 1, "foo2", "bar2")
42
42
assert :ok = MemKV.put(kv, 1, "foo3", "bar3")
43
43
44
-
assert MemKV.get(kv, 1, "foo1") == "bar1"
45
-
assert MemKV.get(kv, 1, "foo2") == "bar2"
46
-
assert MemKV.get(kv, 1, "foo3") == "bar3"
44
+
assert MemKV.get(kv, 1, 0, "foo1") == "bar1"
45
+
assert MemKV.get(kv, 1, 0, "foo2") == "bar2"
46
+
assert MemKV.get(kv, 1, 0, "foo3") == "bar3"
47
47
end
48
48
end
49
49
···
53
53
assert :ok = MemKV.put(kv, 2, "foo", "baz")
54
54
assert :ok = MemKV.delete(kv, 3, "foo")
55
55
56
-
assert MemKV.get(kv, 0, "foo") == nil
57
-
assert MemKV.get(kv, 1, "foo") == "bar"
58
-
assert MemKV.get(kv, 2, "foo") == "baz"
59
-
assert MemKV.get(kv, 3, "foo") == :deleted
56
+
assert MemKV.get(kv, 0, 0, "foo") == nil
57
+
assert MemKV.get(kv, 1, 0, "foo") == "bar"
58
+
assert MemKV.get(kv, 2, 0, "foo") == "baz"
59
+
assert MemKV.get(kv, 3, 0, "foo") == :deleted
60
60
end
61
61
end
62
62
63
63
describe "get/3" do
64
64
test "raises for invalid version", %{kv: kv} do
65
65
assert_raise FunctionClauseError, fn ->
66
-
MemKV.get(kv, -1, "foo")
66
+
MemKV.get(kv, -1, 0, "foo")
67
67
end
68
68
end
69
69
70
70
test "returns nil if there is no value", %{kv: kv} do
71
-
assert MemKV.get(kv, 0, "foo") == nil
72
-
assert MemKV.get(kv, 1, "foo") == nil
71
+
assert MemKV.get(kv, 0, 0, "foo") == nil
72
+
assert MemKV.get(kv, 1, 0, "foo") == nil
73
73
end
74
74
75
75
test "returns nil if there is no value at version", %{kv: kv} do
76
76
MemKV.put(kv, 2, "foo", "bar")
77
77
78
-
assert MemKV.get(kv, 1, "foo") == nil
79
-
assert MemKV.get(kv, 2, "foo") == "bar"
78
+
assert MemKV.get(kv, 1, 0, "foo") == nil
79
+
assert MemKV.get(kv, 2, 0, "foo") == "bar"
80
80
end
81
81
82
82
test "returns :deleted if value is deleted", %{kv: kv} do
83
83
:ok = MemKV.put(kv, 1, "foo", "bar")
84
84
:ok = MemKV.delete(kv, 2, "foo")
85
85
86
-
assert MemKV.get(kv, 1, "foo") == "bar"
87
-
assert MemKV.get(kv, 2, "foo") == :deleted
86
+
assert MemKV.get(kv, 1, 0, "foo") == "bar"
87
+
assert MemKV.get(kv, 2, 0, "foo") == :deleted
88
88
end
89
89
90
90
test "returns correct version", %{kv: kv} do
91
91
MemKV.put(kv, 1, "foo", "bar")
92
92
MemKV.put(kv, 2, "foo", "baz")
93
93
94
-
assert MemKV.get(kv, 1, "foo") == "bar"
95
-
assert MemKV.get(kv, 2, "foo") == "baz"
96
-
assert MemKV.get(kv, 1000, "foo") == "baz"
94
+
assert MemKV.get(kv, 1, 0, "foo") == "bar"
95
+
assert MemKV.get(kv, 2, 0, "foo") == "baz"
96
+
assert MemKV.get(kv, 1000, 0, "foo") == "baz"
97
97
end
98
98
end
99
99
···
200
200
MemKV.put(kv, 2, "foo_b", "bar_b_2")
201
201
MemKV.put(kv, 2, "foo_c", "bar_c_2")
202
202
203
-
assert MemKV.get(kv, 1, "foo") == "bar"
204
-
assert MemKV.get(kv, 3, "foo") == "bar_3"
203
+
assert MemKV.get(kv, 1, 0, "foo") == "bar"
204
+
assert MemKV.get(kv, 3, 0, "foo") == "bar_3"
205
205
206
206
assert MemKV.flush(kv, 2) == %{
207
207
"foo" => "bar_2",
···
209
209
"foo_c" => "bar_c_2",
210
210
}
211
211
212
-
assert MemKV.get(kv, 1, "foo") == nil
213
-
assert MemKV.get(kv, 2, "foo") == nil
214
-
assert MemKV.get(kv, 3, "foo") == "bar_3"
212
+
assert MemKV.get(kv, 1, 0, "foo") == nil
213
+
assert MemKV.get(kv, 2, 0, "foo") == nil
214
+
assert MemKV.get(kv, 3, 0, "foo") == "bar_3"
215
215
216
-
assert MemKV.get(kv, 3, "foo_b") == nil
217
-
assert MemKV.get(kv, 3, "foo_c") == nil
216
+
assert MemKV.get(kv, 3, 0, "foo_b") == nil
217
+
assert MemKV.get(kv, 3, 0, "foo_c") == nil
218
218
end
219
219
220
220
test "does not flush special keyspace", %{kv: kv} do
···
225
225
"foo" => "bar",
226
226
}
227
227
228
-
assert MemKV.get(kv, 1, "foo") == nil
229
-
assert MemKV.get(kv, 1, "\xFF\xFFhello") == "world"
228
+
assert MemKV.get(kv, 1, 0, "foo") == nil
229
+
assert MemKV.get(kv, 1, 0, "\xFF\xFFhello") == "world"
230
230
end
231
231
end
232
232
+32
test/range_forest_test.exs
+32
test/range_forest_test.exs
···
77
77
]
78
78
end
79
79
80
+
test "splits for key in middle", %{rt: rt} do
81
+
assert {:updated, rt} = RangeTree.split_at(rt, 1, "foo_b")
82
+
assert RangeTree.dump(rt) == [
83
+
{"foo_a", "foo_b", 1},
84
+
{"foo_b\x00", "foo_d", 1},
85
+
{"foo_e", "foo_g", 2},
86
+
{"foo_g", "foo_z", 1},
87
+
]
88
+
end
89
+
90
+
test "splits for key at the start", %{rt: rt} do
91
+
assert {:updated, rt} = RangeTree.split_at(rt, 2, "foo_e")
92
+
assert RangeTree.dump(rt) == [
93
+
{"foo_a", "foo_d", 1},
94
+
{"foo_e\x00", "foo_g", 2},
95
+
{"foo_g", "foo_z", 1},
96
+
]
97
+
end
98
+
99
+
test "removes single key range" do
100
+
rt = RangeTree.new()
101
+
rt = RangeTree.insert_range(rt, 1, "foo_a", "foo_b")
102
+
rt = RangeTree.insert_range(rt, 1, "foo_b", "foo_b\x00")
103
+
rt = RangeTree.insert_range(rt, 1, "foo_d", "foo_z")
104
+
105
+
assert {:updated, rt} = RangeTree.split_at(rt, 1, "foo_b")
106
+
assert RangeTree.dump(rt) == [
107
+
{"foo_a", "foo_b", 1},
108
+
{"foo_d", "foo_z", 1},
109
+
]
110
+
end
111
+
80
112
test "intersects keys", %{rt: rt} do
81
113
assert RangeTree.intersect_key(rt, 1, "foo_a") == {"foo_a", "foo_d", 1}
82
114
assert RangeTree.intersect_key(rt, 1, "foo_c") == {"foo_a", "foo_d", 1}