this repo has no description
Scan manifest level at epoch (abandon all hope)
+97
-6
+79
lib/xks/manifest.ex
+79
lib/xks/manifest.ex
···
68
68
:ok
69
69
end
70
70
71
+
@spec list_overlapping_tables(t, non_neg_integer, non_neg_integer, {binary, non_neg_integer}, {binary, non_neg_integer}) :: [tuple]
72
+
def list_overlapping_tables(manifest, epoch, level, start_key, end_key) do
73
+
{sk_key, sk_ver} = start_key
74
+
# Scan backwards to find the first table visible at `epoch` which is outside of the range (below)
75
+
prev_key =
76
+
case find_prev_table_key(manifest, epoch, level, start_key, {level, sk_key, sk_ver, epoch + 1}, false) do
77
+
{:ok, key} -> key
78
+
# If we hit the start of the level, we use a sentinel at epoch -1
79
+
:error -> {level, "", 0, -1}
80
+
end
81
+
82
+
# Start the scan at the `prev_key` found above, which guarantees
83
+
# that any overlapping tables will be found by `next()`
84
+
scan_tables(manifest, epoch, level, end_key, prev_key, [])
85
+
|> Enum.reverse()
86
+
end
87
+
88
+
# Scan backwards to find the key of the first table *below* (disjoint from) `start_key` that is visible at `epoch`
89
+
# This key can then be used to start a forward scan and accumulate
90
+
#
91
+
# TODO: there is absolutely no way this is correct right now
92
+
defp find_prev_table_key(manifest, epoch, level, start_key, prev_key, prev_tombstone?) do
93
+
case :ets.prev_lookup(manifest, prev_key) do
94
+
{key, _obj} when prev_tombstone? ->
95
+
# The previous key we saw (i.e. the next key in the keyspace) was a tombstone,
96
+
# so this entry has been deleted
97
+
find_prev_table_key(manifest, epoch, level, start_key, key, false)
98
+
99
+
{{^level, _k, _ver, ep} = key, _obj} when ep > epoch ->
100
+
# This entry is not visible at `epoch`
101
+
find_prev_table_key(manifest, epoch, level, start_key, key, false)
102
+
103
+
{{^level, _k, _ver, _ep} = key, [{_key, :tombstone}]} ->
104
+
# This entry is a tombstone, so we will ignore the next entry we see (which is previous in the keyspace)
105
+
find_prev_table_key(manifest, epoch, level, start_key, key, true)
106
+
107
+
{{^level, _k, _ver, _ep} = key, [{_key, {ek_key, ek_ver, _i, _ck, _id}}]} when {ek_key, ek_ver} > start_key ->
108
+
# This table actually *contains* `start_key`:
109
+
# - table_sk < start_key due to prev()
110
+
# - table_ek > start_key due to guard
111
+
# Therefore: sk < start_key < ek
112
+
find_prev_table_key(manifest, epoch, level, start_key, key, false)
113
+
114
+
{{^level, _k, _ver, _ep} = key, _obj} ->
115
+
# This is the first table found which has an ek <= `start_key`,
116
+
# which means it is the first table with a range *below* (disjoint) start_key
117
+
{:ok, key}
118
+
119
+
_ ->
120
+
:error
121
+
end
122
+
end
123
+
124
+
defp scan_tables(manifest, epoch, level, end_key, prev_key, acc) do
125
+
case :ets.next_lookup(manifest, prev_key) do
126
+
{{^level, _k, _ver, ep} = key, [{_key, _value}]} when ep > epoch ->
127
+
# This entry is not visible at epoch
128
+
scan_tables(manifest, epoch, level, end_key, key, acc)
129
+
130
+
{{^level, _k, _ver, _ep} = key, [{_key, :tombstone}]} ->
131
+
# This entry is a tombstone and cancels out the previous entry
132
+
# TODO: assert that it was the same key?
133
+
[_deleted | acc] = acc
134
+
scan_tables(manifest, epoch, level, end_key, key, acc)
135
+
136
+
{{^level, sk_key, sk_ver, _ep} = key, [{_key, value}]} when {sk_key, sk_ver} < end_key ->
137
+
# This entry is visible and overlaps, accumulate
138
+
{ek_key, ek_ver, block_index, block_checksum, _id} = value
139
+
table = {:table, block_index, block_checksum, {sk_key, sk_ver}, {ek_key, ek_ver}}
140
+
141
+
acc = [table | acc]
142
+
scan_tables(manifest, epoch, level, end_key, key, acc)
143
+
144
+
_ ->
145
+
# Reached the end of either the ets table, the level, or the range (end_key)
146
+
acc
147
+
end
148
+
end
149
+
71
150
@doc false
72
151
def dump(manifest) do
73
152
:ets.tab2list(manifest)
+9
-6
lib/xks/merge.ex
+9
-6
lib/xks/merge.ex
···
14
14
{_k, _ver} = key -> :gb_trees.insert(key, it, tree)
15
15
end
16
16
end)
17
+
|> then(&coerce_empty/1)
17
18
end
18
19
19
20
@spec next(merge_state) :: {term, merge_state | :empty}
···
28
29
{_k, _ver} = key -> :gb_trees.insert(key, iterator, tree)
29
30
end
30
31
31
-
tree =
32
-
case :gb_trees.is_empty(tree) do
33
-
true -> :empty
34
-
false -> tree
35
-
end
36
-
32
+
tree = coerce_empty(tree)
37
33
{pair, tree}
34
+
end
35
+
36
+
defp coerce_empty(state) do
37
+
case :gb_trees.is_empty(state) do
38
+
true -> :empty
39
+
false -> state
40
+
end
38
41
end
39
42
40
43
@spec iterator_for_memtable(Memtable.t, binary) :: memtable_iterator
+8
lib/xks/xks.ex
+8
lib/xks/xks.ex
···
60
60
61
61
@spec get(t, non_neg_integer, binary) :: binary | nil
62
62
def get(%XKS{} = xks, version, key) do
63
+
manifest = xks.manifest
63
64
epoch = :atomics.get(xks.epoch_atomic, 1)
65
+
66
+
_tables = Manifest.list_overlapping_tables(manifest, epoch, 1, {key, version}, {key, version + 1})
64
67
65
68
Manifest.list_memtables(xks.manifest, epoch)
66
69
|> Enum.find_value(:error, fn {_id, memtable} ->
···
78
81
79
82
@spec scan(t, non_neg_integer, binary, binary) :: [{binary, binary}]
80
83
def scan(%XKS{} = xks, version, start_key, end_key) do
84
+
manifest = xks.manifest
81
85
epoch = :atomics.get(xks.epoch_atomic, 1)
86
+
87
+
# start_key has version=0 because we have no way of knowing its latest version in the DB
88
+
# end_key has version=0 because the end key is exclusive so we won't actually read it
89
+
_tables = Manifest.list_overlapping_tables(manifest, epoch, 1, {start_key, 0}, {end_key, 0})
82
90
83
91
iterators =
84
92
Manifest.list_memtables(xks.manifest, epoch)