tracks lexicons and how many times they appeared on the jetstream
feat(server): use miniz for compression
+10
-13
+1
-7
server/Cargo.lock
+1
-7
server/Cargo.lock
···
1353
1353
"guardian",
1354
1354
"interval-heap",
1355
1355
"log",
1356
-
"lz4_flex",
1356
+
"miniz_oxide",
1357
1357
"path-absolutize",
1358
1358
"quick_cache",
1359
1359
"rustc-hash",
···
1363
1363
"varint-rs",
1364
1364
"xxhash-rust",
1365
1365
]
1366
-
1367
-
[[package]]
1368
-
name = "lz4_flex"
1369
-
version = "0.11.3"
1370
-
source = "registry+https://github.com/rust-lang/crates.io-index"
1371
-
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
1372
1366
1373
1367
[[package]]
1374
1368
name = "matchers"
+1
-1
server/Cargo.toml
+1
-1
server/Cargo.toml
···
12
12
axum = { version = "0.8", features = ["json", "ws"] }
13
13
tower-http = {version = "0.6", features = ["request-id"]}
14
14
atproto-jetstream = "0.9"
15
-
fjall = "2"
15
+
fjall = { version = "2", default-features = false, features = ["miniz"] }
16
16
rkyv = {version = "0.8", features = ["unaligned"]}
17
17
smol_str = { version = "0.3", features = ["serde"] }
18
18
papaya = "0.2"
+8
-5
server/src/db.rs
+8
-5
server/src/db.rs
···
65
65
.open()?;
66
66
Ok(Self {
67
67
hits: Default::default(),
68
-
counts: ks.open_partition("_counts", PartitionCreateOptions::default())?,
68
+
counts: ks.open_partition(
69
+
"_counts",
70
+
PartitionCreateOptions::default().compression(fjall::CompressionType::None),
71
+
)?,
69
72
inner: ks,
70
73
event_broadcaster: broadcast::channel(1000).0,
71
74
})
···
82
85
f: impl FnOnce(&Partition) -> AppResult<()>,
83
86
) -> AppResult<()> {
84
87
f(self.hits.pin().get_or_insert_with(SmolStr::new(nsid), || {
85
-
let opts = PartitionCreateOptions::default().compaction_strategy(
86
-
fjall::compaction::Strategy::Fifo(fjall::compaction::Fifo {
88
+
let opts = PartitionCreateOptions::default()
89
+
.compression(fjall::CompressionType::Miniz(9))
90
+
.compaction_strategy(fjall::compaction::Strategy::Fifo(fjall::compaction::Fifo {
87
91
limit: 5 * 1024 * 1024 * 1024, // 5 gb
88
92
ttl_seconds: Some(60 * 60 * 24 * 30), // 30 days
89
-
}),
90
-
);
93
+
}));
91
94
self.inner.open_partition(nsid, opts).unwrap()
92
95
}))
93
96
}