delaytop: add psi info to show system delay

Support showing whole delay of system by reading PSI, just like the first
few lines of information output by the top command. the output of
delaytop includes both system-wide delay and delay of individual tasks,
providing a more comprehensive reflection of system latency status.

Use case
========
bash# ./delaytop
System Pressure Information: (avg10/avg60/avg300/total)
CPU: full: 0.0%/ 0.0%/ 0.0%/0 some: 0.1%/ 0.0%/ 0.0%/14216596
Memory: full: 0.0%/ 0.0%/ 0.0%/34010659 some: 0.0%/ 0.0%/ 0.0%/35406492
IO: full: 0.1%/ 0.0%/ 0.0%/51029453 some: 0.1%/ 0.0%/ 0.0%/55330465
IRQ: full: 0.0%/ 0.0%/ 0.0%/0

Top 20 processes (sorted by CPU delay):

PID TGID COMMAND CPU(ms) IO(ms) SWAP(ms) RCL(ms) THR(ms) CMP(ms) WP(ms) IRQ(ms)
---------------------------------------------------------------------------------------------
32 32 kworker/2:0H-sy 23.65 0.00 0.00 0.00 0.00 0.00 0.00 0.00
497 497 kworker/R-scsi_ 1.20 0.00 0.00 0.00 0.00 0.00 0.00 0.00
495 495 kworker/R-scsi_ 1.13 0.00 0.00 0.00 0.00 0.00 0.00 0.00
494 494 scsi_eh_0 1.12 0.00 0.00 0.00 0.00 0.00 0.00 0.00
485 485 kworker/R-ata_s 0.90 0.00 0.00 0.00 0.00 0.00 0.00 0.00
574 574 kworker/R-kdmfl 0.36 0.00 0.00 0.00 0.00 0.00 0.00 0.00
34 34 idle_inject/3 0.33 0.00 0.00 0.00 0.00 0.00 0.00 0.00
1123 1123 nde-netfilter 0.28 0.00 0.00 0.00 0.00 0.00 0.00 0.00
60 60 ksoftirqd/7 0.25 0.00 0.00 0.00 0.00 0.00 0.00 0.00
114 114 kworker/0:2-cgr 0.25 0.00 0.00 0.00 0.00 0.00 0.00 0.00
496 496 scsi_eh_1 0.24 0.00 0.00 0.00 0.00 0.00 0.00 0.00
51 51 cpuhp/6 0.24 0.00 0.00 0.00 0.00 0.00 0.00 0.00
1667 1667 atd 0.24 0.00 0.00 0.00 0.00 0.00 0.00 0.00
45 45 cpuhp/5 0.23 0.00 0.00 0.00 0.00 0.00 0.00 0.00
1102 1102 nde-backupservi 0.22 0.00 0.00 0.00 0.00 0.00 0.00 0.00
1098 1098 systemsettings 0.21 0.00 0.00 0.00 0.00 0.00 0.00 0.00
1100 1100 audit-monitor 0.20 0.00 0.00 0.00 0.00 0.00 0.00 0.00
53 53 migration/6 0.20 0.00 0.00 0.00 0.00 0.00 0.00 0.00
1482 1482 sshd 0.19 0.00 0.00 0.00 0.00 0.00 0.00 0.00
39 39 cpuhp/4 0.19 0.00 0.00 0.00 0.00 0.00 0.00 0.00

Link: https://lkml.kernel.org/r/20250710135451340_5pOgpIFi0M5AE7H44W1D@zte.com.cn
Co-developed-by: Fan Yu <fan.yu9@zte.com.cn>
Signed-off-by: Fan Yu <fan.yu9@zte.com.cn>
Signed-off-by: Wang Yaxin <wang.yaxin@zte.com.cn>
Signed-off-by: Jiang Kun <jiang.kun2@zte.com.cn>
Cc: Balbir Singh <bsingharora@gmail.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Peilin He <he.peilin@zte.com.cn>
Cc: Qiang Tu <tu.qiang35@zte.com.cn>
Cc: wangyong <wang.yong12@zte.com.cn>
Cc: xu xin <xu.xin16@zte.com.cn>
Cc: Yang Yang <yang.yang29@zte.com.cn>
Cc: Yunkai Zhang <zhang.yunkai@zte.com.cn>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Wang Yaxin and committed by

Andrew Morton 10 months ago 6b47c9f8 599579e8

+148 -13

1 changed file

expand all

tools

accounting

delaytop.c

+148 -13

tools/accounting/delaytop.c

··· 10 10 * individual tasks (PIDs). 11 11 * 12 12 * Key features: 13 - * - Collects per-task delay accounting statistics via taskstats. 14 - * - Supports sorting, filtering. 15 - * - Supports both interactive (screen refresh). 13 + * - Collects per-task delay accounting statistics via taskstats. 14 + * - Supports sorting, filtering. 15 + * - Supports both interactive (screen refresh). 16 16 * 17 17 * Copyright (C) Fan Yu, ZTE Corp. 2025 18 18 * Copyright (C) Wang Yaxin, ZTE Corp. 2025 ··· 43 43 #include <linux/cgroupstats.h> 44 44 #include <ncurses.h> 45 45 46 + #define PSI_CPU_SOME "/proc/pressure/cpu" 47 + #define PSI_CPU_FULL "/proc/pressure/cpu" 48 + #define PSI_MEMORY_SOME "/proc/pressure/memory" 49 + #define PSI_MEMORY_FULL "/proc/pressure/memory" 50 + #define PSI_IO_SOME "/proc/pressure/io" 51 + #define PSI_IO_FULL "/proc/pressure/io" 52 + #define PSI_IRQ_FULL "/proc/pressure/irq" 53 + 46 54 #define NLA_NEXT(na) ((struct nlattr *)((char *)(na) + NLA_ALIGN((na)->nla_len))) 47 55 #define NLA_DATA(na) ((void *)((char *)(na) + NLA_HDRLEN)) 48 56 #define NLA_PAYLOAD(len) (len - NLA_HDRLEN) ··· 72 64 int output_one_time; /* Output once and exit */ 73 65 int monitor_pid; /* Monitor specific PID */ 74 66 char *container_path; /* Path to container cgroup */ 67 + }; 68 + 69 + /* PSI statistics structure */ 70 + struct psi_stats { 71 + double cpu_some_avg10, cpu_some_avg60, cpu_some_avg300; 72 + unsigned long long cpu_some_total; 73 + double cpu_full_avg10, cpu_full_avg60, cpu_full_avg300; 74 + unsigned long long cpu_full_total; 75 + double memory_some_avg10, memory_some_avg60, memory_some_avg300; 76 + unsigned long long memory_some_total; 77 + double memory_full_avg10, memory_full_avg60, memory_full_avg300; 78 + unsigned long long memory_full_total; 79 + double io_some_avg10, io_some_avg60, io_some_avg300; 80 + unsigned long long io_some_total; 81 + double io_full_avg10, io_full_avg60, io_full_avg300; 82 + unsigned long long io_full_total; 83 + double irq_full_avg10, irq_full_avg60, irq_full_avg300; 84 + unsigned long long irq_full_total; 75 85 }; 76 86 77 87 /* Task delay information structure */ ··· 126 100 127 101 /* Global variables */ 128 102 static struct config cfg; 103 + static struct psi_stats psi; 129 104 static struct task_info tasks[MAX_TASKS]; 130 105 static int task_count; 131 106 static int running = 1; ··· 157 130 { 158 131 printf("Usage: delaytop [Options]\n" 159 132 "Options:\n" 160 - " -h, --help Show this help message and exit\n" 161 - " -d, --delay=SECONDS Set refresh interval (default: 2 seconds, min: 1)\n" 162 - " -n, --iterations=COUNT Set number of updates (default: 0 = infinite)\n" 163 - " -P, --processes=NUMBER Set maximum number of processes to show (default: 20, max: 1000)\n" 164 - " -o, --once Display once and exit\n" 165 - " -p, --pid=PID Monitor only the specified PID\n" 166 - " -C, --container=PATH Monitor the container at specified cgroup path\n"); 133 + " -h, --help Show this help message and exit\n" 134 + " -d, --delay=SECONDS Set refresh interval (default: 2 seconds, min: 1)\n" 135 + " -n, --iterations=COUNT Set number of updates (default: 0 = infinite)\n" 136 + " -P, --processes=NUMBER Set maximum number of processes to show (default: 20, max: 1000)\n" 137 + " -o, --once Display once and exit\n" 138 + " -p, --pid=PID Monitor only the specified PID\n" 139 + " -C, --container=PATH Monitor the container at specified cgroup path\n"); 167 140 exit(0); 168 141 } 169 142 ··· 303 276 memset(&nladdr, 0, sizeof(nladdr)); 304 277 nladdr.nl_family = AF_NETLINK; 305 278 while ((r = sendto(sd, buf, buflen, 0, (struct sockaddr *) &nladdr, 306 - sizeof(nladdr))) < buflen) { 279 + sizeof(nladdr))) < buflen) { 307 280 if (r > 0) { 308 281 buf += r; 309 282 buflen -= r; ··· 345 318 if (na->nla_type == CTRL_ATTR_FAMILY_ID) 346 319 id = *(__u16 *) NLA_DATA(na); 347 320 return id; 321 + } 322 + 323 + static void read_psi_stats(void) 324 + { 325 + FILE *fp; 326 + char line[256]; 327 + int ret = 0; 328 + /* Zero all fields */ 329 + memset(&psi, 0, sizeof(psi)); 330 + /* CPU pressure */ 331 + fp = fopen(PSI_CPU_SOME, "r"); 332 + if (fp) { 333 + while (fgets(line, sizeof(line), fp)) { 334 + if (strncmp(line, "some", 4) == 0) { 335 + ret = sscanf(line, "some avg10=%lf avg60=%lf avg300=%lf total=%llu", 336 + &psi.cpu_some_avg10, &psi.cpu_some_avg60, 337 + &psi.cpu_some_avg300, &psi.cpu_some_total); 338 + if (ret != 4) 339 + fprintf(stderr, "Failed to parse CPU some PSI data\n"); 340 + } else if (strncmp(line, "full", 4) == 0) { 341 + ret = sscanf(line, "full avg10=%lf avg60=%lf avg300=%lf total=%llu", 342 + &psi.cpu_full_avg10, &psi.cpu_full_avg60, 343 + &psi.cpu_full_avg300, &psi.cpu_full_total); 344 + if (ret != 4) 345 + fprintf(stderr, "Failed to parse CPU full PSI data\n"); 346 + } 347 + } 348 + fclose(fp); 349 + } 350 + /* Memory pressure */ 351 + fp = fopen(PSI_MEMORY_SOME, "r"); 352 + if (fp) { 353 + while (fgets(line, sizeof(line), fp)) { 354 + if (strncmp(line, "some", 4) == 0) { 355 + ret = sscanf(line, "some avg10=%lf avg60=%lf avg300=%lf total=%llu", 356 + &psi.memory_some_avg10, &psi.memory_some_avg60, 357 + &psi.memory_some_avg300, &psi.memory_some_total); 358 + if (ret != 4) 359 + fprintf(stderr, "Failed to parse Memory some PSI data\n"); 360 + } else if (strncmp(line, "full", 4) == 0) { 361 + ret = sscanf(line, "full avg10=%lf avg60=%lf avg300=%lf total=%llu", 362 + &psi.memory_full_avg10, &psi.memory_full_avg60, 363 + &psi.memory_full_avg300, &psi.memory_full_total); 364 + } 365 + if (ret != 4) 366 + fprintf(stderr, "Failed to parse Memory full PSI data\n"); 367 + } 368 + fclose(fp); 369 + } 370 + /* IO pressure */ 371 + fp = fopen(PSI_IO_SOME, "r"); 372 + if (fp) { 373 + while (fgets(line, sizeof(line), fp)) { 374 + if (strncmp(line, "some", 4) == 0) { 375 + ret = sscanf(line, "some avg10=%lf avg60=%lf avg300=%lf total=%llu", 376 + &psi.io_some_avg10, &psi.io_some_avg60, 377 + &psi.io_some_avg300, &psi.io_some_total); 378 + if (ret != 4) 379 + fprintf(stderr, "Failed to parse IO some PSI data\n"); 380 + } else if (strncmp(line, "full", 4) == 0) { 381 + ret = sscanf(line, "full avg10=%lf avg60=%lf avg300=%lf total=%llu", 382 + &psi.io_full_avg10, &psi.io_full_avg60, 383 + &psi.io_full_avg300, &psi.io_full_total); 384 + if (ret != 4) 385 + fprintf(stderr, "Failed to parse IO full PSI data\n"); 386 + } 387 + } 388 + fclose(fp); 389 + } 390 + /* IRQ pressure (only full) */ 391 + fp = fopen(PSI_IRQ_FULL, "r"); 392 + if (fp) { 393 + while (fgets(line, sizeof(line), fp)) { 394 + if (strncmp(line, "full", 4) == 0) { 395 + ret = sscanf(line, "full avg10=%lf avg60=%lf avg300=%lf total=%llu", 396 + &psi.irq_full_avg10, &psi.irq_full_avg60, 397 + &psi.irq_full_avg300, &psi.irq_full_total); 398 + if (ret != 4) 399 + fprintf(stderr, "Failed to parse IRQ full PSI data\n"); 400 + } 401 + } 402 + fclose(fp); 403 + } 348 404 } 349 405 350 406 static int read_comm(int pid, char *comm_buf, size_t buf_size) ··· 659 549 FILE *out = stdout; 660 550 661 551 fprintf(out, "\033[H\033[J"); 552 + /* PSI output (one-line, no cat style) */ 553 + fprintf(out, "System Pressure Information: "); 554 + fprintf(out, "(avg10/avg60/avg300/total)\n"); 555 + fprintf(out, "CPU:"); 556 + fprintf(out, " full: %6.1f%%/%6.1f%%/%6.1f%%/%-10llu", psi.cpu_full_avg10, 557 + psi.cpu_full_avg60, psi.cpu_full_avg300, psi.cpu_full_total); 558 + fprintf(out, " some: %6.1f%%/%6.1f%%/%6.1f%%/%-10llu\n", psi.cpu_some_avg10, 559 + psi.cpu_some_avg60, psi.cpu_some_avg300, psi.cpu_some_total); 662 560 561 + fprintf(out, "Memory:"); 562 + fprintf(out, " full: %6.1f%%/%6.1f%%/%6.1f%%/%-10llu", psi.memory_full_avg10, 563 + psi.memory_full_avg60, psi.memory_full_avg300, psi.memory_full_total); 564 + fprintf(out, " some: %6.1f%%/%6.1f%%/%6.1f%%/%-10llu\n", psi.memory_some_avg10, 565 + psi.memory_some_avg60, psi.memory_some_avg300, psi.memory_some_total); 566 + 567 + fprintf(out, "IO:"); 568 + fprintf(out, " full: %6.1f%%/%6.1f%%/%6.1f%%/%-10llu", psi.io_full_avg10, 569 + psi.io_full_avg60, psi.io_full_avg300, psi.io_full_total); 570 + fprintf(out, " some: %6.1f%%/%6.1f%%/%6.1f%%/%-10llu\n", psi.io_some_avg10, 571 + psi.io_some_avg60, psi.io_some_avg300, psi.io_some_total); 572 + fprintf(out, "IRQ:"); 573 + fprintf(out, " full: %6.1f%%/%6.1f%%/%6.1f%%/%-10llu\n\n", psi.irq_full_avg10, 574 + psi.irq_full_avg60, psi.irq_full_avg300, psi.irq_full_total); 663 575 if (cfg.container_path) { 664 576 fprintf(out, "Container Information (%s):\n", cfg.container_path); 665 577 fprintf(out, "Processes: running=%d, sleeping=%d, ", ··· 691 559 container_stats.nr_io_wait); 692 560 } 693 561 fprintf(out, "Top %d processes (sorted by CPU delay):\n\n", 694 - cfg.max_processes); 695 - fprintf(out, " PID TGID COMMAND CPU(ms) IO(ms) "); 562 + cfg.max_processes); 563 + fprintf(out, " PID TGID COMMAND CPU(ms) IO(ms) "); 696 564 fprintf(out, "SWAP(ms) RCL(ms) THR(ms) CMP(ms) WP(ms) IRQ(ms)\n"); 697 565 fprintf(out, "-----------------------------------------------"); 698 566 fprintf(out, "----------------------------------------------\n"); ··· 748 616 749 617 /* Main loop */ 750 618 while (running) { 619 + /* Read PSI statistics */ 620 + read_psi_stats(); 621 + 751 622 /* Get container stats if container path provided */ 752 623 if (cfg.container_path) 753 624 get_container_stats();

Configure Feed

Configure Feed