Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'timers_urgent_for_v6.4_rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull timer fix from Borislav Petkov:
- Prevent CPU state corruption when an active clockevent broadcast
device is replaced while the system is already in oneshot mode
* tag 'timers_urgent_for_v6.4_rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
tick/broadcast: Make broadcast device replacement work correctly
+93
-37
+93
-37
kernel/time/tick-broadcast.c
+93
-37
kernel/time/tick-broadcast.c
···
35
35
#ifdef CONFIG_TICK_ONESHOT
36
36
static DEFINE_PER_CPU(struct clock_event_device *, tick_oneshot_wakeup_device);
37
37
38
-
static void tick_broadcast_setup_oneshot(struct clock_event_device *bc);
38
+
static void tick_broadcast_setup_oneshot(struct clock_event_device *bc, bool from_periodic);
39
39
static void tick_broadcast_clear_oneshot(int cpu);
40
40
static void tick_resume_broadcast_oneshot(struct clock_event_device *bc);
41
41
# ifdef CONFIG_HOTPLUG_CPU
42
42
static void tick_broadcast_oneshot_offline(unsigned int cpu);
43
43
# endif
44
44
#else
45
-
static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) { BUG(); }
45
+
static inline void
46
+
tick_broadcast_setup_oneshot(struct clock_event_device *bc, bool from_periodic) { BUG(); }
46
47
static inline void tick_broadcast_clear_oneshot(int cpu) { }
47
48
static inline void tick_resume_broadcast_oneshot(struct clock_event_device *bc) { }
48
49
# ifdef CONFIG_HOTPLUG_CPU
···
265
264
if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
266
265
tick_broadcast_start_periodic(bc);
267
266
else
268
-
tick_broadcast_setup_oneshot(bc);
267
+
tick_broadcast_setup_oneshot(bc, false);
269
268
ret = 1;
270
269
} else {
271
270
/*
···
501
500
if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
502
501
tick_broadcast_start_periodic(bc);
503
502
else
504
-
tick_broadcast_setup_oneshot(bc);
503
+
tick_broadcast_setup_oneshot(bc, false);
505
504
}
506
505
}
507
506
out:
···
1021
1020
/**
1022
1021
* tick_broadcast_setup_oneshot - setup the broadcast device
1023
1022
*/
1024
-
static void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
1023
+
static void tick_broadcast_setup_oneshot(struct clock_event_device *bc,
1024
+
bool from_periodic)
1025
1025
{
1026
1026
int cpu = smp_processor_id();
1027
+
ktime_t nexttick = 0;
1027
1028
1028
1029
if (!bc)
1029
1030
return;
1030
1031
1031
-
/* Set it up only once ! */
1032
-
if (bc->event_handler != tick_handle_oneshot_broadcast) {
1033
-
int was_periodic = clockevent_state_periodic(bc);
1034
-
1035
-
bc->event_handler = tick_handle_oneshot_broadcast;
1036
-
1032
+
/*
1033
+
* When the broadcast device was switched to oneshot by the first
1034
+
* CPU handling the NOHZ change, the other CPUs will reach this
1035
+
* code via hrtimer_run_queues() -> tick_check_oneshot_change()
1036
+
* too. Set up the broadcast device only once!
1037
+
*/
1038
+
if (bc->event_handler == tick_handle_oneshot_broadcast) {
1037
1039
/*
1038
-
* We must be careful here. There might be other CPUs
1039
-
* waiting for periodic broadcast. We need to set the
1040
-
* oneshot_mask bits for those and program the
1041
-
* broadcast device to fire.
1042
-
*/
1043
-
cpumask_copy(tmpmask, tick_broadcast_mask);
1044
-
cpumask_clear_cpu(cpu, tmpmask);
1045
-
cpumask_or(tick_broadcast_oneshot_mask,
1046
-
tick_broadcast_oneshot_mask, tmpmask);
1047
-
1048
-
if (was_periodic && !cpumask_empty(tmpmask)) {
1049
-
ktime_t nextevt = tick_get_next_period();
1050
-
1051
-
clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
1052
-
tick_broadcast_init_next_event(tmpmask, nextevt);
1053
-
tick_broadcast_set_event(bc, cpu, nextevt);
1054
-
} else
1055
-
bc->next_event = KTIME_MAX;
1056
-
} else {
1057
-
/*
1058
-
* The first cpu which switches to oneshot mode sets
1059
-
* the bit for all other cpus which are in the general
1060
-
* (periodic) broadcast mask. So the bit is set and
1061
-
* would prevent the first broadcast enter after this
1062
-
* to program the bc device.
1040
+
* The CPU which switched from periodic to oneshot mode
1041
+
* set the broadcast oneshot bit for all other CPUs which
1042
+
* are in the general (periodic) broadcast mask to ensure
1043
+
* that CPUs which wait for the periodic broadcast are
1044
+
* woken up.
1045
+
*
1046
+
* Clear the bit for the local CPU as the set bit would
1047
+
* prevent the first tick_broadcast_enter() after this CPU
1048
+
* switched to oneshot state to program the broadcast
1049
+
* device.
1050
+
*
1051
+
* This code can also be reached via tick_broadcast_control(),
1052
+
* but this cannot avoid the tick_broadcast_clear_oneshot()
1053
+
* as that would break the periodic to oneshot transition of
1054
+
* secondary CPUs. But that's harmless as the below only
1055
+
* clears already cleared bits.
1063
1056
*/
1064
1057
tick_broadcast_clear_oneshot(cpu);
1058
+
return;
1065
1059
}
1060
+
1061
+
1062
+
bc->event_handler = tick_handle_oneshot_broadcast;
1063
+
bc->next_event = KTIME_MAX;
1064
+
1065
+
/*
1066
+
* When the tick mode is switched from periodic to oneshot it must
1067
+
* be ensured that CPUs which are waiting for periodic broadcast
1068
+
* get their wake-up at the next tick. This is achieved by ORing
1069
+
* tick_broadcast_mask into tick_broadcast_oneshot_mask.
1070
+
*
1071
+
* For other callers, e.g. broadcast device replacement,
1072
+
* tick_broadcast_oneshot_mask must not be touched as this would
1073
+
* set bits for CPUs which are already NOHZ, but not idle. Their
1074
+
* next tick_broadcast_enter() would observe the bit set and fail
1075
+
* to update the expiry time and the broadcast event device.
1076
+
*/
1077
+
if (from_periodic) {
1078
+
cpumask_copy(tmpmask, tick_broadcast_mask);
1079
+
/* Remove the local CPU as it is obviously not idle */
1080
+
cpumask_clear_cpu(cpu, tmpmask);
1081
+
cpumask_or(tick_broadcast_oneshot_mask, tick_broadcast_oneshot_mask, tmpmask);
1082
+
1083
+
/*
1084
+
* Ensure that the oneshot broadcast handler will wake the
1085
+
* CPUs which are still waiting for periodic broadcast.
1086
+
*/
1087
+
nexttick = tick_get_next_period();
1088
+
tick_broadcast_init_next_event(tmpmask, nexttick);
1089
+
1090
+
/*
1091
+
* If the underlying broadcast clock event device is
1092
+
* already in oneshot state, then there is nothing to do.
1093
+
* The device was already armed for the next tick
1094
+
* in tick_handle_broadcast_periodic()
1095
+
*/
1096
+
if (clockevent_state_oneshot(bc))
1097
+
return;
1098
+
}
1099
+
1100
+
/*
1101
+
* When switching from periodic to oneshot mode arm the broadcast
1102
+
* device for the next tick.
1103
+
*
1104
+
* If the broadcast device has been replaced in oneshot mode and
1105
+
* the oneshot broadcast mask is not empty, then arm it to expire
1106
+
* immediately in order to reevaluate the next expiring timer.
1107
+
* @nexttick is 0 and therefore in the past which will cause the
1108
+
* clockevent code to force an event.
1109
+
*
1110
+
* For both cases the programming can be avoided when the oneshot
1111
+
* broadcast mask is empty.
1112
+
*
1113
+
* tick_broadcast_set_event() implicitly switches the broadcast
1114
+
* device to oneshot state.
1115
+
*/
1116
+
if (!cpumask_empty(tick_broadcast_oneshot_mask))
1117
+
tick_broadcast_set_event(bc, cpu, nexttick);
1066
1118
}
1067
1119
1068
1120
/*
···
1124
1070
void tick_broadcast_switch_to_oneshot(void)
1125
1071
{
1126
1072
struct clock_event_device *bc;
1073
+
enum tick_device_mode oldmode;
1127
1074
unsigned long flags;
1128
1075
1129
1076
raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
1130
1077
1078
+
oldmode = tick_broadcast_device.mode;
1131
1079
tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
1132
1080
bc = tick_broadcast_device.evtdev;
1133
1081
if (bc)
1134
-
tick_broadcast_setup_oneshot(bc);
1082
+
tick_broadcast_setup_oneshot(bc, oldmode == TICKDEV_MODE_PERIODIC);
1135
1083
1136
1084
raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
1137
1085
}