this repo has no description
Add FileStore to XKS
+132
-14
+17
-9
lib/xks/blocks.ex
+17
-9
lib/xks/blocks.ex
···
1
1
defmodule Hobbes.XKS.Blocks do
2
2
alias Hobbes.XKS
3
-
alias Hobbes.XKS.MemoryStore
3
+
alias Hobbes.XKS.{MemoryStore, FileStore}
4
4
5
5
import ExUnit.Assertions, only: [assert: 1]
6
6
···
17
17
MemoryStore.put_superblock(memory_store, copy, block_data)
18
18
end
19
19
20
+
defp do_write_superblock({:file, file_store}, copy, block_data) do
21
+
FileStore.put_superblock(file_store, copy, block_data)
22
+
end
23
+
20
24
@spec read_superblock(XKS.block_store, non_neg_integer) :: binary
21
25
def read_superblock(store, copy) do
22
26
do_read_superblock(store, copy)
···
29
33
end
30
34
end
31
35
36
+
defp do_read_superblock({:file, file_store}, copy) do
37
+
FileStore.get_superblock(file_store, copy)
38
+
end
39
+
32
40
@spec write(XKS.block_store, non_neg_integer, iodata) :: checksum
33
41
def write(store, index, block_data) do
34
42
assert is_integer(index)
···
47
55
MemoryStore.put(memory_store, index, block_data)
48
56
end
49
57
58
+
defp do_write({:file, file_store}, index, block_data) do
59
+
FileStore.put(file_store, index, block_data)
60
+
end
61
+
50
62
@spec read(XKS.block_store, non_neg_integer, checksum) :: binary
51
63
def read(store, index, checksum) do
52
64
block_data = do_read(store, index)
···
69
81
end
70
82
end
71
83
72
-
def read_raw({:memory, memory_store}, index) do
73
-
[{:block_size, block_size}] = :ets.lookup(memory_store, :block_size)
74
-
case MemoryStore.fetch(memory_store, index) do
75
-
{:ok, block_data} -> block_data
76
-
:error -> <<0::integer-unit(8)-size(block_size)>>
77
-
end
78
-
end
79
-
80
84
defp do_read({:memory, memory_store}, index) do
81
85
# TODO: return zeros for missing blocks in memory store?
82
86
case MemoryStore.fetch(memory_store, index) do
83
87
{:ok, block_data} -> block_data
84
88
end
89
+
end
90
+
91
+
defp do_read({:file, file_store}, index) do
92
+
FileStore.get(file_store, index)
85
93
end
86
94
87
95
@spec checksum(binary) :: checksum
+82
lib/xks/file_store.ex
+82
lib/xks/file_store.ex
···
1
+
defmodule Hobbes.XKS.FileStore do
2
+
alias Hobbes.XKS.FileStore
3
+
4
+
alias Trinity.SimFile
5
+
6
+
import ExUnit.Assertions, only: [assert: 1]
7
+
import Hobbes.XKS.Utils
8
+
9
+
@type t :: %__MODULE__{
10
+
fd: term,
11
+
block_size: pos_integer,
12
+
}
13
+
@enforce_keys [
14
+
:fd,
15
+
:block_size,
16
+
]
17
+
defstruct @enforce_keys
18
+
19
+
@spec new(keyword) :: term
20
+
def new(opts) do
21
+
block_size = Keyword.fetch!(opts, :block_size)
22
+
assert is_integer(block_size)
23
+
assert block_size >= 1
24
+
25
+
path = Keyword.fetch!(opts, :path)
26
+
assert is_binary(path)
27
+
28
+
{:ok, fd} = SimFile.open(path, [:read, :write])
29
+
30
+
%FileStore{
31
+
fd: fd,
32
+
block_size: block_size,
33
+
}
34
+
end
35
+
36
+
@spec destroy(t) :: :ok
37
+
def destroy(%FileStore{} = _file_store) do
38
+
# TODO: delete file
39
+
:ok
40
+
end
41
+
42
+
@spec put_superblock(t, non_neg_integer, binary) :: :ok
43
+
def put_superblock(%FileStore{} = file_store, copy, block_data) do
44
+
%{fd: fd} = file_store
45
+
assert byte_size(block_data) == c_superblock_size()
46
+
47
+
loc = copy * c_superblock_size()
48
+
:ok = SimFile.pwrite(fd, loc, block_data)
49
+
end
50
+
51
+
@spec get_superblock(t, non_neg_integer) :: binary
52
+
def get_superblock(%FileStore{} = file_store, copy) do
53
+
%{fd: fd} = file_store
54
+
55
+
loc = copy * c_superblock_size()
56
+
case SimFile.pread(fd, loc, c_superblock_size()) do
57
+
{:ok, binary} -> binary
58
+
end
59
+
end
60
+
61
+
@spec put(t, pos_integer, binary) :: :ok
62
+
def put(%FileStore{} = file_store, index, block_data) do
63
+
%{fd: fd, block_size: block_size} = file_store
64
+
65
+
assert index >= 1
66
+
assert byte_size(block_data) == block_size
67
+
68
+
loc = index * block_size
69
+
:ok = SimFile.pwrite(fd, loc, block_data)
70
+
end
71
+
72
+
@spec get(t, pos_integer) :: binary
73
+
def get(%FileStore{} = file_store, index) do
74
+
%{fd: fd, block_size: block_size} = file_store
75
+
assert index >= 1
76
+
77
+
loc = index * block_size
78
+
case SimFile.pread(fd, loc, block_size) do
79
+
{:ok, binary} -> binary
80
+
end
81
+
end
82
+
end
+9
-5
lib/xks/xks.ex
+9
-5
lib/xks/xks.ex
···
1
1
defmodule Hobbes.XKS do
2
2
alias Hobbes.XKS
3
-
alias Hobbes.XKS.{MemoryStore, Superblock, FreeList, Manifest, ManifestLog, WorkQueue, Memtable, Merge, Compaction, Table}
3
+
alias Hobbes.XKS.{MemoryStore, FileStore, Superblock, FreeList, Manifest, ManifestLog, WorkQueue, Memtable, Merge, Compaction, Table}
4
4
alias Hobbes.XKS.Manifest.MemtableInfo
5
5
6
6
import Hobbes.XKS.Utils
···
94
94
end
95
95
96
96
@spec new(keyword) :: t
97
-
def new(opts \\ []) do
98
-
opts = parse_opts(opts)
97
+
def new(kw_opts \\ []) do
98
+
opts = parse_opts(kw_opts)
99
99
100
-
block_store = {:memory, MemoryStore.new(opts.block_size)}
100
+
block_store =
101
+
case Keyword.get(kw_opts, :path, :memory) do
102
+
:memory -> {:memory, MemoryStore.new(opts.block_size)}
103
+
path -> {:file, FileStore.new(path: path, block_size: opts.block_size)}
104
+
end
101
105
102
106
xks = %XKS{
103
107
opts: opts,
···
129
133
if !Keyword.get(opts, :preserve_block_store) do
130
134
case xks.block_store do
131
135
{:memory, store} -> MemoryStore.destroy(store)
132
-
_ -> :noop
136
+
{:file, file_store} -> FileStore.destroy(file_store)
133
137
end
134
138
end
135
139
FreeList.destroy(xks.free_list)
+24
test/xks_test.exs
+24
test/xks_test.exs
···
138
138
|> Task.await_many()
139
139
end
140
140
end
141
+
142
+
defmodule ModelFileFuzzTest do
143
+
use ExUnit.Case, async: true
144
+
@moduletag :xks
145
+
@moduletag :xks_fuzz
146
+
@moduletag :model_file_fuzz
147
+
@moduletag :disable
148
+
149
+
test "fuzz" do
150
+
Trinity.Sim.run_simulation(fn ->
151
+
block_kib = 64
152
+
sub_kib = 8
153
+
Hobbes.XKS.Fuzz.ModelFuzz.run(100, [
154
+
iterations: 1000,
155
+
#key_bits: 32,
156
+
xks_opts: [
157
+
block_size: block_kib * 1024,
158
+
lsm_subtable_size: sub_kib * 1024,
159
+
path: "/fuzz.xks",
160
+
],
161
+
])
162
+
end)
163
+
end
164
+
end
141
165
end