Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'perf_urgent_for_v5.13_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 perf fix from Borislav Petkov:
"Handle power-gating of AMD IOMMU perf counters properly when they are
used"
* tag 'perf_urgent_for_v5.13_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
x86/events/amd/iommu: Fix invalid Perf result due to IOMMU PMC power-gating
+26
-21
+26
-21
arch/x86/events/amd/iommu.c
+26
-21
arch/x86/events/amd/iommu.c
···
19
19
#include "../perf_event.h"
20
20
#include "iommu.h"
21
21
22
-
#define COUNTER_SHIFT 16
23
-
24
22
/* iommu pmu conf masks */
25
23
#define GET_CSOURCE(x) ((x)->conf & 0xFFULL)
26
24
#define GET_DEVID(x) (((x)->conf >> 8) & 0xFFFFULL)
···
284
286
WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
285
287
hwc->state = 0;
286
288
289
+
/*
290
+
* To account for power-gating, which prevents write to
291
+
* the counter, we need to enable the counter
292
+
* before setting up counter register.
293
+
*/
294
+
perf_iommu_enable_event(event);
295
+
287
296
if (flags & PERF_EF_RELOAD) {
288
-
u64 prev_raw_count = local64_read(&hwc->prev_count);
297
+
u64 count = 0;
289
298
struct amd_iommu *iommu = perf_event_2_iommu(event);
290
299
300
+
/*
301
+
* Since the IOMMU PMU only support counting mode,
302
+
* the counter always start with value zero.
303
+
*/
291
304
amd_iommu_pc_set_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr,
292
-
IOMMU_PC_COUNTER_REG, &prev_raw_count);
305
+
IOMMU_PC_COUNTER_REG, &count);
293
306
}
294
307
295
-
perf_iommu_enable_event(event);
296
308
perf_event_update_userpage(event);
297
-
298
309
}
299
310
300
311
static void perf_iommu_read(struct perf_event *event)
301
312
{
302
-
u64 count, prev, delta;
313
+
u64 count;
303
314
struct hw_perf_event *hwc = &event->hw;
304
315
struct amd_iommu *iommu = perf_event_2_iommu(event);
305
316
···
319
312
/* IOMMU pc counter register is only 48 bits */
320
313
count &= GENMASK_ULL(47, 0);
321
314
322
-
prev = local64_read(&hwc->prev_count);
323
-
if (local64_cmpxchg(&hwc->prev_count, prev, count) != prev)
324
-
return;
325
-
326
-
/* Handle 48-bit counter overflow */
327
-
delta = (count << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
328
-
delta >>= COUNTER_SHIFT;
329
-
local64_add(delta, &event->count);
315
+
/*
316
+
* Since the counter always start with value zero,
317
+
* simply just accumulate the count for the event.
318
+
*/
319
+
local64_add(count, &event->count);
330
320
}
331
321
332
322
static void perf_iommu_stop(struct perf_event *event, int flags)
···
333
329
if (hwc->state & PERF_HES_UPTODATE)
334
330
return;
335
331
332
+
/*
333
+
* To account for power-gating, in which reading the counter would
334
+
* return zero, we need to read the register before disabling.
335
+
*/
336
+
perf_iommu_read(event);
337
+
hwc->state |= PERF_HES_UPTODATE;
338
+
336
339
perf_iommu_disable_event(event);
337
340
WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
338
341
hwc->state |= PERF_HES_STOPPED;
339
-
340
-
if (hwc->state & PERF_HES_UPTODATE)
341
-
return;
342
-
343
-
perf_iommu_read(event);
344
-
hwc->state |= PERF_HES_UPTODATE;
345
342
}
346
343
347
344
static int perf_iommu_add(struct perf_event *event, int flags)