A local-first private AI assistant for everyday use. Runs on-device models with encrypted P2P sync, and supports sharing chats publicly on ATProto.
feat: Added aggregation to benchmark, needed for memory models
+25
-3
+25
-3
tiles/src/runtime/mlx.rs
+25
-3
tiles/src/runtime/mlx.rs
···
33
33
total_latency_s: f64,
34
34
}
35
35
36
+
impl BenchmarkMetrics {
37
+
fn update(&mut self, metrics: BenchmarkMetrics) -> &Self {
38
+
if self.ttft_ms == 0.0 {
39
+
self.ttft_ms += metrics.ttft_ms;
40
+
}
41
+
self.total_tokens += metrics.total_tokens;
42
+
self.tokens_per_second += metrics.tokens_per_second;
43
+
self.total_latency_s += metrics.total_latency_s;
44
+
self
45
+
}
46
+
}
36
47
pub struct MLXRuntime {}
37
48
38
49
impl MLXRuntime {}
···
399
410
}
400
411
let mut remaining_count = run_args.relay_count;
401
412
let mut python_code: String = "".to_owned();
413
+
let mut bench_metrics: BenchmarkMetrics = BenchmarkMetrics {
414
+
ttft_ms: 0.0,
415
+
total_tokens: 0,
416
+
tokens_per_second: 0.0,
417
+
total_latency_s: 0.0,
418
+
};
402
419
loop {
403
420
if remaining_count > 0 {
404
421
let chat_start = remaining_count == run_args.relay_count;
···
415
432
if response.reply.is_empty() {
416
433
if !response.code.is_empty() {
417
434
python_code = response.code;
435
+
}
436
+
if let Some(metrics) = response.metrics {
437
+
bench_metrics.update(metrics);
418
438
}
419
439
remaining_count -= 1;
420
440
} else {
···
426
446
}
427
447
// Display benchmark metrics if available
428
448
if let Some(metrics) = response.metrics {
449
+
bench_metrics.update(metrics);
429
450
println!(
430
451
"{}",
431
452
format!(
432
453
"\n{} {:.1} tok/s | {} tokens | {:.0}ms TTFT",
433
454
"💡".yellow(),
434
-
metrics.tokens_per_second,
435
-
metrics.total_tokens,
436
-
metrics.ttft_ms
455
+
bench_metrics.total_tokens
456
+
/ bench_metrics.total_latency_s as i32,
457
+
bench_metrics.total_tokens,
458
+
bench_metrics.ttft_ms
437
459
)
438
460
.dimmed()
439
461
);