-1

README.md

reviewed

··· 62 62 | `ENABLE_DEBUG` | `false` | enable debug endpoints. | 63 63 | `DEBUG_PORT` | `API_PORT + 1` | port for debug endpoints (if enabled). | 64 64 | `ENABLE_FIREHOSE` | `true` | whether to ingest relay subscriptions. | 65 65 - | `ENABLE_BACKFILL` | `true` | whether to backfill from PDS instances. | 66 65 | `ENABLE_CRAWLER` | `false` (if Filter), `true` (if Full) | whether to actively query the network for unknown repositories. | 67 66 | `CRAWLER_MAX_PENDING_REPOS` | `2000` | max pending repos for crawler. | 68 67 | `CRAWLER_RESUME_PENDING_REPOS` | `1000` | resume threshold for crawler pending repos. |

-8

src/config.rs

reviewed

··· 82 82 pub verify_signatures: SignatureVerification, 83 83 pub identity_cache_size: u64, 84 84 pub enable_firehose: bool, 85 85 - pub enable_backfill: bool, 86 85 pub enable_crawler: Option<bool>, 87 86 pub firehose_workers: usize, 88 88 - pub db_compact: bool, 89 87 pub db_worker_threads: usize, 90 88 pub db_max_journaling_size_mb: u64, 91 89 pub db_blocks_memtable_size_mb: u64, ··· 174 172 let verify_signatures = cfg!("VERIFY_SIGNATURES", SignatureVerification::Full); 175 173 let identity_cache_size = cfg!("IDENTITY_CACHE_SIZE", 1_000_000u64); 176 174 let enable_firehose = cfg!("ENABLE_FIREHOSE", true); 177 177 - let enable_backfill = cfg!("ENABLE_BACKFILL", true); 178 175 let enable_crawler = std::env::var("HYDRANT_ENABLE_CRAWLER") 179 176 .ok() 180 177 .and_then(|s| s.parse().ok()); ··· 188 185 full_network.then_some(24usize).unwrap_or(8usize) 189 186 ); 190 187 191 191 - let db_compact = cfg!("COMPACT_DB", false); 192 192 - 193 188 let ( 194 189 default_db_worker_threads, 195 190 default_db_max_journaling_size_mb, ··· 257 252 verify_signatures, 258 253 identity_cache_size, 259 254 enable_firehose, 260 260 - enable_backfill, 261 255 enable_crawler, 262 256 firehose_workers, 263 263 - db_compact, 264 257 db_worker_threads, 265 258 db_max_journaling_size_mb, 266 259 db_blocks_memtable_size_mb, ··· 310 303 config_line!(f, "journal compression", self.journal_compression)?; 311 304 config_line!(f, "api port", self.api_port)?; 312 305 config_line!(f, "firehose workers", self.firehose_workers)?; 313 313 - config_line!(f, "db compact", self.db_compact)?; 314 306 config_line!(f, "db worker threads", self.db_worker_threads)?; 315 307 config_line!( 316 308 f,

+16 -23

src/main.rs

reviewed

··· 31 31 32 32 let state = AppState::new(&cfg)?; 33 33 34 34 - if cfg.db_compact { 35 35 - info!("compacting database..."); 36 36 - state.db.compact().await?; 37 37 - } 38 38 - 39 34 if cfg.full_network 40 35 || cfg.filter_signals.is_some() 41 36 || cfg.filter_collections.is_some() ··· 93 88 .into_diagnostic()?; 94 89 } 95 90 96 96 - if cfg.enable_backfill { 97 97 - tokio::spawn({ 98 98 - let state = state.clone(); 99 99 - let timeout = cfg.repo_fetch_timeout; 100 100 - BackfillWorker::new( 101 101 - state, 102 102 - buffer_tx.clone(), 103 103 - timeout, 104 104 - cfg.backfill_concurrency_limit, 105 105 - matches!( 106 106 - cfg.verify_signatures, 107 107 - SignatureVerification::Full | SignatureVerification::BackfillOnly 108 108 - ), 109 109 - cfg.ephemeral, 110 110 - ) 111 111 - .run() 112 112 - }); 113 113 - } 91 91 + tokio::spawn({ 92 92 + let state = state.clone(); 93 93 + let timeout = cfg.repo_fetch_timeout; 94 94 + BackfillWorker::new( 95 95 + state, 96 96 + buffer_tx.clone(), 97 97 + timeout, 98 98 + cfg.backfill_concurrency_limit, 99 99 + matches!( 100 100 + cfg.verify_signatures, 101 101 + SignatureVerification::Full | SignatureVerification::BackfillOnly 102 102 + ), 103 103 + cfg.ephemeral, 104 104 + ) 105 105 + .run() 106 106 + }); 114 107 115 108 if let Err(e) = spawn_blocking({ 116 109 let state = state.clone();