tracks lexicons and how many times they appeared on the jetstream
fix(server): max_items wasnt working in get_hits
+37
-16
+37
-16
server/src/db/mod.rs
+37
-16
server/src/db/mod.rs
···
402
402
};
403
403
404
404
// let mut ts = CLOCK.now();
405
-
let mut current_item_count = 0;
406
-
let map_block = move |(key, val)| {
405
+
let map_block = move |(res, current_item_count)| -> AppResult<(Option<_>, usize)> {
407
406
if current_item_count >= max_items {
408
-
return Ok(None);
407
+
return Ok((None, current_item_count));
409
408
}
409
+
let (key, val) = res?;
410
410
let mut key_reader = Cursor::new(key);
411
411
let start_timestamp = key_reader.read_varint::<u64>()?;
412
412
// let end_timestamp = key_reader.read_varint::<u64>()?;
···
414
414
// tracing::info!(
415
415
// "stopped at block with timestamps {start_timestamp}..{end_timestamp} because {start_limit} is greater"
416
416
// );
417
-
return Ok(None);
417
+
return Ok((None, current_item_count));
418
418
}
419
419
let decoder = handle::ItemDecoder::new(Cursor::new(val), start_timestamp)?;
420
-
current_item_count += decoder.item_count();
420
+
let current_item_count = current_item_count + decoder.item_count();
421
421
// tracing::info!(
422
422
// "took {}ns to get block with size {}",
423
423
// ts.elapsed().as_nanos(),
424
424
// decoder.item_count()
425
425
// );
426
426
// ts = CLOCK.now();
427
-
Ok(Some(
428
-
decoder
429
-
.take_while(move |item| {
430
-
item.as_ref().map_or(true, |item| {
431
-
item.timestamp <= end_limit && item.timestamp >= start_limit
427
+
Ok((
428
+
Some(
429
+
decoder
430
+
.take_while(move |item| {
431
+
item.as_ref().map_or(true, |item| {
432
+
item.timestamp <= end_limit && item.timestamp >= start_limit
433
+
})
432
434
})
433
-
})
434
-
.map(|res| res.map_err(AppError::from)),
435
+
.map(|res| res.map_err(AppError::from)),
436
+
),
437
+
current_item_count,
435
438
))
436
439
};
437
440
438
-
let blocks = handle
441
+
let (blocks, counted) = handle
439
442
.range(..end_key)
443
+
.map(|res| res.map_err(AppError::from))
440
444
.rev()
441
-
.map_while(move |res| res.map_err(AppError::from).and_then(map_block).transpose())
442
-
.collect_vec();
445
+
.fold_while(
446
+
(Vec::with_capacity(20), 0),
447
+
|(mut blocks, current_item_count), res| {
448
+
use itertools::FoldWhile::*;
449
+
450
+
match map_block((res, current_item_count)) {
451
+
Ok((Some(block), current_item_count)) => {
452
+
blocks.push(Ok(block));
453
+
Continue((blocks, current_item_count))
454
+
}
455
+
Ok((None, current_item_count)) => Done((blocks, current_item_count)),
456
+
Err(err) => {
457
+
blocks.push(Err(err));
458
+
Done((blocks, current_item_count))
459
+
}
460
+
}
461
+
},
462
+
)
463
+
.into_inner();
443
464
444
465
tracing::info!(
445
-
"got blocks with size {}, item count {current_item_count}",
466
+
"got blocks with size {}, item count {counted}",
446
467
blocks.len()
447
468
);
448
469