sched/fair: Fair server interface · tjh.dev/kernel@d741f29

+86 -17

kernel/sched/deadline.c

··· 320 320 __sub_running_bw(dl_se->dl_bw, dl_rq); 321 321 } 322 322 323 - static void dl_change_utilization(struct task_struct *p, u64 new_bw) 323 + static void dl_rq_change_utilization(struct rq *rq, struct sched_dl_entity *dl_se, u64 new_bw) 324 324 { 325 - struct rq *rq; 325 + if (dl_se->dl_non_contending) { 326 + sub_running_bw(dl_se, &rq->dl); 327 + dl_se->dl_non_contending = 0; 326 328 327 - WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV); 328 - 329 - if (task_on_rq_queued(p)) 330 - return; 331 - 332 - rq = task_rq(p); 333 - if (p->dl.dl_non_contending) { 334 - sub_running_bw(&p->dl, &rq->dl); 335 - p->dl.dl_non_contending = 0; 336 329 /* 337 330 * If the timer handler is currently running and the 338 331 * timer cannot be canceled, inactive_task_timer() ··· 333 340 * will not touch the rq's active utilization, 334 341 * so we are still safe. 335 342 */ 336 - if (hrtimer_try_to_cancel(&p->dl.inactive_timer) == 1) 337 - put_task_struct(p); 343 + if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1) { 344 + if (!dl_server(dl_se)) 345 + put_task_struct(dl_task_of(dl_se)); 346 + } 338 347 } 339 - __sub_rq_bw(p->dl.dl_bw, &rq->dl); 348 + __sub_rq_bw(dl_se->dl_bw, &rq->dl); 340 349 __add_rq_bw(new_bw, &rq->dl); 350 + } 351 + 352 + static void dl_change_utilization(struct task_struct *p, u64 new_bw) 353 + { 354 + WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV); 355 + 356 + if (task_on_rq_queued(p)) 357 + return; 358 + 359 + dl_rq_change_utilization(task_rq(p), &p->dl, new_bw); 341 360 } 342 361 343 362 static void __dl_clear_params(struct sched_dl_entity *dl_se); ··· 1626 1621 { 1627 1622 struct rq *rq = dl_se->rq; 1628 1623 1624 + /* 1625 + * XXX: the apply do not work fine at the init phase for the 1626 + * fair server because things are not yet set. We need to improve 1627 + * this before getting generic. 1628 + */ 1629 1629 if (!dl_server(dl_se)) { 1630 1630 /* Disabled */ 1631 - dl_se->dl_runtime = 0; 1632 - dl_se->dl_deadline = 1000 * NSEC_PER_MSEC; 1633 - dl_se->dl_period = 1000 * NSEC_PER_MSEC; 1631 + u64 runtime = 0; 1632 + u64 period = 1000 * NSEC_PER_MSEC; 1633 + 1634 + dl_server_apply_params(dl_se, runtime, period, 1); 1634 1635 1635 1636 dl_se->dl_server = 1; 1636 1637 dl_se->dl_defer = 1; ··· 1669 1658 dl_se->rq = rq; 1670 1659 dl_se->server_has_tasks = has_tasks; 1671 1660 dl_se->server_pick = pick; 1661 + } 1662 + 1663 + void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq) 1664 + { 1665 + u64 new_bw = dl_se->dl_bw; 1666 + int cpu = cpu_of(rq); 1667 + struct dl_bw *dl_b; 1668 + 1669 + dl_b = dl_bw_of(cpu_of(rq)); 1670 + guard(raw_spinlock)(&dl_b->lock); 1671 + 1672 + if (!dl_bw_cpus(cpu)) 1673 + return; 1674 + 1675 + __dl_add(dl_b, new_bw, dl_bw_cpus(cpu)); 1676 + } 1677 + 1678 + int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 period, bool init) 1679 + { 1680 + u64 old_bw = init ? 0 : to_ratio(dl_se->dl_period, dl_se->dl_runtime); 1681 + u64 new_bw = to_ratio(period, runtime); 1682 + struct rq *rq = dl_se->rq; 1683 + int cpu = cpu_of(rq); 1684 + struct dl_bw *dl_b; 1685 + unsigned long cap; 1686 + int retval = 0; 1687 + int cpus; 1688 + 1689 + dl_b = dl_bw_of(cpu); 1690 + guard(raw_spinlock)(&dl_b->lock); 1691 + 1692 + cpus = dl_bw_cpus(cpu); 1693 + cap = dl_bw_capacity(cpu); 1694 + 1695 + if (__dl_overflow(dl_b, cap, old_bw, new_bw)) 1696 + return -EBUSY; 1697 + 1698 + if (init) { 1699 + __add_rq_bw(new_bw, &rq->dl); 1700 + __dl_add(dl_b, new_bw, cpus); 1701 + } else { 1702 + __dl_sub(dl_b, dl_se->dl_bw, cpus); 1703 + __dl_add(dl_b, new_bw, cpus); 1704 + 1705 + dl_rq_change_utilization(rq, dl_se, new_bw); 1706 + } 1707 + 1708 + dl_se->dl_runtime = runtime; 1709 + dl_se->dl_deadline = period; 1710 + dl_se->dl_period = period; 1711 + 1712 + dl_se->runtime = 0; 1713 + dl_se->deadline = 0; 1714 + 1715 + dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); 1716 + dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime); 1717 + 1718 + return retval; 1672 1719 } 1673 1720 1674 1721 /*

+159

kernel/sched/debug.c

··· 333 333 .release = seq_release, 334 334 }; 335 335 336 + enum dl_param { 337 + DL_RUNTIME = 0, 338 + DL_PERIOD, 339 + }; 340 + 341 + static unsigned long fair_server_period_max = (1 << 22) * NSEC_PER_USEC; /* ~4 seconds */ 342 + static unsigned long fair_server_period_min = (100) * NSEC_PER_USEC; /* 100 us */ 343 + 344 + static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubuf, 345 + size_t cnt, loff_t *ppos, enum dl_param param) 346 + { 347 + long cpu = (long) ((struct seq_file *) filp->private_data)->private; 348 + struct rq *rq = cpu_rq(cpu); 349 + u64 runtime, period; 350 + size_t err; 351 + int retval; 352 + u64 value; 353 + 354 + err = kstrtoull_from_user(ubuf, cnt, 10, &value); 355 + if (err) 356 + return err; 357 + 358 + scoped_guard (rq_lock_irqsave, rq) { 359 + runtime = rq->fair_server.dl_runtime; 360 + period = rq->fair_server.dl_period; 361 + 362 + switch (param) { 363 + case DL_RUNTIME: 364 + if (runtime == value) 365 + break; 366 + runtime = value; 367 + break; 368 + case DL_PERIOD: 369 + if (value == period) 370 + break; 371 + period = value; 372 + break; 373 + } 374 + 375 + if (runtime > period || 376 + period > fair_server_period_max || 377 + period < fair_server_period_min) { 378 + return -EINVAL; 379 + } 380 + 381 + if (rq->cfs.h_nr_running) { 382 + update_rq_clock(rq); 383 + dl_server_stop(&rq->fair_server); 384 + } 385 + 386 + retval = dl_server_apply_params(&rq->fair_server, runtime, period, 0); 387 + if (retval) 388 + cnt = retval; 389 + 390 + if (!runtime) 391 + printk_deferred("Fair server disabled in CPU %d, system may crash due to starvation.\n", 392 + cpu_of(rq)); 393 + 394 + if (rq->cfs.h_nr_running) 395 + dl_server_start(&rq->fair_server); 396 + } 397 + 398 + *ppos += cnt; 399 + return cnt; 400 + } 401 + 402 + static size_t sched_fair_server_show(struct seq_file *m, void *v, enum dl_param param) 403 + { 404 + unsigned long cpu = (unsigned long) m->private; 405 + struct rq *rq = cpu_rq(cpu); 406 + u64 value; 407 + 408 + switch (param) { 409 + case DL_RUNTIME: 410 + value = rq->fair_server.dl_runtime; 411 + break; 412 + case DL_PERIOD: 413 + value = rq->fair_server.dl_period; 414 + break; 415 + } 416 + 417 + seq_printf(m, "%llu\n", value); 418 + return 0; 419 + 420 + } 421 + 422 + static ssize_t 423 + sched_fair_server_runtime_write(struct file *filp, const char __user *ubuf, 424 + size_t cnt, loff_t *ppos) 425 + { 426 + return sched_fair_server_write(filp, ubuf, cnt, ppos, DL_RUNTIME); 427 + } 428 + 429 + static int sched_fair_server_runtime_show(struct seq_file *m, void *v) 430 + { 431 + return sched_fair_server_show(m, v, DL_RUNTIME); 432 + } 433 + 434 + static int sched_fair_server_runtime_open(struct inode *inode, struct file *filp) 435 + { 436 + return single_open(filp, sched_fair_server_runtime_show, inode->i_private); 437 + } 438 + 439 + static const struct file_operations fair_server_runtime_fops = { 440 + .open = sched_fair_server_runtime_open, 441 + .write = sched_fair_server_runtime_write, 442 + .read = seq_read, 443 + .llseek = seq_lseek, 444 + .release = single_release, 445 + }; 446 + 447 + static ssize_t 448 + sched_fair_server_period_write(struct file *filp, const char __user *ubuf, 449 + size_t cnt, loff_t *ppos) 450 + { 451 + return sched_fair_server_write(filp, ubuf, cnt, ppos, DL_PERIOD); 452 + } 453 + 454 + static int sched_fair_server_period_show(struct seq_file *m, void *v) 455 + { 456 + return sched_fair_server_show(m, v, DL_PERIOD); 457 + } 458 + 459 + static int sched_fair_server_period_open(struct inode *inode, struct file *filp) 460 + { 461 + return single_open(filp, sched_fair_server_period_show, inode->i_private); 462 + } 463 + 464 + static const struct file_operations fair_server_period_fops = { 465 + .open = sched_fair_server_period_open, 466 + .write = sched_fair_server_period_write, 467 + .read = seq_read, 468 + .llseek = seq_lseek, 469 + .release = single_release, 470 + }; 471 + 336 472 static struct dentry *debugfs_sched; 473 + 474 + static void debugfs_fair_server_init(void) 475 + { 476 + struct dentry *d_fair; 477 + unsigned long cpu; 478 + 479 + d_fair = debugfs_create_dir("fair_server", debugfs_sched); 480 + if (!d_fair) 481 + return; 482 + 483 + for_each_possible_cpu(cpu) { 484 + struct dentry *d_cpu; 485 + char buf[32]; 486 + 487 + snprintf(buf, sizeof(buf), "cpu%lu", cpu); 488 + d_cpu = debugfs_create_dir(buf, d_fair); 489 + 490 + debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &fair_server_runtime_fops); 491 + debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &fair_server_period_fops); 492 + } 493 + } 337 494 338 495 static __init int sched_init_debug(void) 339 496 { ··· 530 373 #endif 531 374 532 375 debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); 376 + 377 + debugfs_fair_server_init(); 533 378 534 379 return 0; 535 380 }

+3

kernel/sched/sched.h

··· 366 366 extern void dl_server_update_idle_time(struct rq *rq, 367 367 struct task_struct *p); 368 368 extern void fair_server_init(struct rq *rq); 369 + extern void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq); 370 + extern int dl_server_apply_params(struct sched_dl_entity *dl_se, 371 + u64 runtime, u64 period, bool init); 369 372 370 373 #ifdef CONFIG_CGROUP_SCHED 371 374

+8

kernel/sched/topology.c

··· 516 516 if (cpumask_test_cpu(rq->cpu, cpu_active_mask)) 517 517 set_rq_online(rq); 518 518 519 + /* 520 + * Because the rq is not a task, dl_add_task_root_domain() did not 521 + * move the fair server bw to the rd if it already started. 522 + * Add it now. 523 + */ 524 + if (rq->fair_server.dl_server) 525 + __dl_server_attach_root(&rq->fair_server, rq); 526 + 519 527 rq_unlock_irqrestore(rq, &rf); 520 528 521 529 if (old_rd)

Configure Feed

Configure Feed