sched_ext: Rename pnt_seq to kick_sync · tjh.dev/kernel@987e000

+41 -41

2 changed files

expand all

kernel

sched

ext.c

sched.h

+40 -40

kernel/sched/ext.c

··· 68 68 static struct delayed_work scx_watchdog_work; 69 69 70 70 /* 71 - * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of pick_task sequence 71 + * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence 72 72 * numbers. The arrays are allocated with kvzalloc() as size can exceed percpu 73 73 * allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated 74 74 * lazily when enabling and freed when disabling to avoid waste when sched_ext 75 75 * isn't active. 76 76 */ 77 - struct scx_kick_pseqs { 77 + struct scx_kick_syncs { 78 78 struct rcu_head rcu; 79 - unsigned long seqs[]; 79 + unsigned long syncs[]; 80 80 }; 81 81 82 - static DEFINE_PER_CPU(struct scx_kick_pseqs __rcu *, scx_kick_pseqs); 82 + static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs); 83 83 84 84 /* 85 85 * Direct dispatch marker. ··· 2301 2301 struct scx_sched *sch = scx_root; 2302 2302 2303 2303 /* see kick_cpus_irq_workfn() */ 2304 - smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1); 2304 + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); 2305 2305 2306 2306 update_curr_scx(rq); 2307 2307 ··· 2357 2357 struct task_struct *p; 2358 2358 2359 2359 /* see kick_cpus_irq_workfn() */ 2360 - smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1); 2360 + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); 2361 2361 2362 2362 rq_modified_clear(rq); 2363 2363 ··· 3883 3883 } 3884 3884 } 3885 3885 3886 - static void free_kick_pseqs_rcu(struct rcu_head *rcu) 3886 + static void free_kick_syncs_rcu(struct rcu_head *rcu) 3887 3887 { 3888 - struct scx_kick_pseqs *pseqs = container_of(rcu, struct scx_kick_pseqs, rcu); 3888 + struct scx_kick_syncs *ksyncs = container_of(rcu, struct scx_kick_syncs, rcu); 3889 3889 3890 - kvfree(pseqs); 3890 + kvfree(ksyncs); 3891 3891 } 3892 3892 3893 - static void free_kick_pseqs(void) 3893 + static void free_kick_syncs(void) 3894 3894 { 3895 3895 int cpu; 3896 3896 3897 3897 for_each_possible_cpu(cpu) { 3898 - struct scx_kick_pseqs **pseqs = per_cpu_ptr(&scx_kick_pseqs, cpu); 3899 - struct scx_kick_pseqs *to_free; 3898 + struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); 3899 + struct scx_kick_syncs *to_free; 3900 3900 3901 - to_free = rcu_replace_pointer(*pseqs, NULL, true); 3901 + to_free = rcu_replace_pointer(*ksyncs, NULL, true); 3902 3902 if (to_free) 3903 - call_rcu(&to_free->rcu, free_kick_pseqs_rcu); 3903 + call_rcu(&to_free->rcu, free_kick_syncs_rcu); 3904 3904 } 3905 3905 } 3906 3906 ··· 4038 4038 free_percpu(scx_dsp_ctx); 4039 4039 scx_dsp_ctx = NULL; 4040 4040 scx_dsp_max_batch = 0; 4041 - free_kick_pseqs(); 4041 + free_kick_syncs(); 4042 4042 4043 4043 mutex_unlock(&scx_enable_mutex); 4044 4044 ··· 4287 4287 seq_buf_init(&ns, buf, avail); 4288 4288 4289 4289 dump_newline(&ns); 4290 - dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu pnt_seq=%lu", 4290 + dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", 4291 4291 cpu, rq->scx.nr_running, rq->scx.flags, 4292 4292 rq->scx.cpu_released, rq->scx.ops_qseq, 4293 - rq->scx.pnt_seq); 4293 + rq->scx.kick_sync); 4294 4294 dump_line(&ns, " curr=%s[%d] class=%ps", 4295 4295 rq->curr->comm, rq->curr->pid, 4296 4296 rq->curr->sched_class); ··· 4401 4401 irq_work_queue(&sch->error_irq_work); 4402 4402 } 4403 4403 4404 - static int alloc_kick_pseqs(void) 4404 + static int alloc_kick_syncs(void) 4405 4405 { 4406 4406 int cpu; 4407 4407 ··· 4410 4410 * can exceed percpu allocator limits on large machines. 4411 4411 */ 4412 4412 for_each_possible_cpu(cpu) { 4413 - struct scx_kick_pseqs **pseqs = per_cpu_ptr(&scx_kick_pseqs, cpu); 4414 - struct scx_kick_pseqs *new_pseqs; 4413 + struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); 4414 + struct scx_kick_syncs *new_ksyncs; 4415 4415 4416 - WARN_ON_ONCE(rcu_access_pointer(*pseqs)); 4416 + WARN_ON_ONCE(rcu_access_pointer(*ksyncs)); 4417 4417 4418 - new_pseqs = kvzalloc_node(struct_size(new_pseqs, seqs, nr_cpu_ids), 4419 - GFP_KERNEL, cpu_to_node(cpu)); 4420 - if (!new_pseqs) { 4421 - free_kick_pseqs(); 4418 + new_ksyncs = kvzalloc_node(struct_size(new_ksyncs, syncs, nr_cpu_ids), 4419 + GFP_KERNEL, cpu_to_node(cpu)); 4420 + if (!new_ksyncs) { 4421 + free_kick_syncs(); 4422 4422 return -ENOMEM; 4423 4423 } 4424 4424 4425 - rcu_assign_pointer(*pseqs, new_pseqs); 4425 + rcu_assign_pointer(*ksyncs, new_ksyncs); 4426 4426 } 4427 4427 4428 4428 return 0; ··· 4578 4578 goto err_unlock; 4579 4579 } 4580 4580 4581 - ret = alloc_kick_pseqs(); 4581 + ret = alloc_kick_syncs(); 4582 4582 if (ret) 4583 4583 goto err_unlock; 4584 4584 4585 4585 sch = scx_alloc_and_add_sched(ops); 4586 4586 if (IS_ERR(sch)) { 4587 4587 ret = PTR_ERR(sch); 4588 - goto err_free_pseqs; 4588 + goto err_free_ksyncs; 4589 4589 } 4590 4590 4591 4591 /* ··· 4788 4788 4789 4789 return 0; 4790 4790 4791 - err_free_pseqs: 4792 - free_kick_pseqs(); 4791 + err_free_ksyncs: 4792 + free_kick_syncs(); 4793 4793 err_unlock: 4794 4794 mutex_unlock(&scx_enable_mutex); 4795 4795 return ret; ··· 5119 5119 return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); 5120 5120 } 5121 5121 5122 - static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *pseqs) 5122 + static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) 5123 5123 { 5124 5124 struct rq *rq = cpu_rq(cpu); 5125 5125 struct scx_rq *this_scx = &this_rq->scx; ··· 5146 5146 5147 5147 if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { 5148 5148 if (cur_class == &ext_sched_class) { 5149 - pseqs[cpu] = rq->scx.pnt_seq; 5149 + ksyncs[cpu] = rq->scx.kick_sync; 5150 5150 should_wait = true; 5151 5151 } else { 5152 5152 cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); ··· 5182 5182 { 5183 5183 struct rq *this_rq = this_rq(); 5184 5184 struct scx_rq *this_scx = &this_rq->scx; 5185 - struct scx_kick_pseqs __rcu *pseqs_pcpu = __this_cpu_read(scx_kick_pseqs); 5185 + struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs); 5186 5186 bool should_wait = false; 5187 - unsigned long *pseqs; 5187 + unsigned long *ksyncs; 5188 5188 s32 cpu; 5189 5189 5190 - if (unlikely(!pseqs_pcpu)) { 5191 - pr_warn_once("kick_cpus_irq_workfn() called with NULL scx_kick_pseqs"); 5190 + if (unlikely(!ksyncs_pcpu)) { 5191 + pr_warn_once("kick_cpus_irq_workfn() called with NULL scx_kick_syncs"); 5192 5192 return; 5193 5193 } 5194 5194 5195 - pseqs = rcu_dereference_bh(pseqs_pcpu)->seqs; 5195 + ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs; 5196 5196 5197 5197 for_each_cpu(cpu, this_scx->cpus_to_kick) { 5198 - should_wait |= kick_one_cpu(cpu, this_rq, pseqs); 5198 + should_wait |= kick_one_cpu(cpu, this_rq, ksyncs); 5199 5199 cpumask_clear_cpu(cpu, this_scx->cpus_to_kick); 5200 5200 cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); 5201 5201 } ··· 5209 5209 return; 5210 5210 5211 5211 for_each_cpu(cpu, this_scx->cpus_to_wait) { 5212 - unsigned long *wait_pnt_seq = &cpu_rq(cpu)->scx.pnt_seq; 5212 + unsigned long *wait_kick_sync = &cpu_rq(cpu)->scx.kick_sync; 5213 5213 5214 5214 /* 5215 5215 * Busy-wait until the task running at the time of kicking is no ··· 5223 5223 * the wait when $cpu is taken by a higher sched class. 5224 5224 */ 5225 5225 if (cpu != cpu_of(this_rq)) 5226 - smp_cond_load_acquire(wait_pnt_seq, VAL != pseqs[cpu]); 5226 + smp_cond_load_acquire(wait_kick_sync, VAL != ksyncs[cpu]); 5227 5227 5228 5228 cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); 5229 5229 }

+1 -1

kernel/sched/sched.h

··· 803 803 cpumask_var_t cpus_to_kick_if_idle; 804 804 cpumask_var_t cpus_to_preempt; 805 805 cpumask_var_t cpus_to_wait; 806 - unsigned long pnt_seq; 806 + unsigned long kick_sync; 807 807 struct balance_callback deferred_bal_cb; 808 808 struct irq_work deferred_irq_work; 809 809 struct irq_work kick_cpus_irq_work;

Configure Feed

Configure Feed