Parakeet is a Rust-based Bluesky AppServer aiming to implement most of the functionality required to support the Bluesky client
appview
atproto
bluesky
rust
appserver
fix(consumer): remove old downloaded duplicate check
-7
-7
crates/consumer/src/backfill/mod.rs
-7
crates/consumer/src/backfill/mod.rs
···
26
26
mod utils;
27
27
28
28
const BF_QUEUE: &str = "backfill_queue";
29
-
const DL_DUP_KEY: &str = "bf_completed";
30
29
// There's a 4MiB limit on parakeet-index, so break delta batches up if there's loads.
31
30
// this should be plenty low enough to not trigger the size limit. (59k did slightly)
32
31
const DELTA_BATCH_SIZE: usize = 32 * 1024;
···
133
132
did: &str,
134
133
) -> eyre::Result<()> {
135
134
// has the repo already been downloaded?
136
-
if rc.sismember(DL_DUP_KEY, did).await.unwrap_or_default() {
137
-
tracing::info!("skipping duplicate repo {did}");
138
-
return Ok(());
139
-
}
140
-
141
-
// check if they're already synced in DB too
142
135
match db::actor_get_statuses(conn, did).await {
143
136
Ok(Some((_, state))) => {
144
137
if state == ActorSyncState::Synced || state == ActorSyncState::Processing {