Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'trace-ringbuffer-v6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace
Pull trace ring-buffer cleanup from Steven Rostedt:
- Add helper functions for allocations
The allocation of the per CPU buffer descriptor, the buffer page
descriptors and the buffer page data itself can be pretty ugly.
Add some helper macros and a function to have the code that allocates
buffer pages and such look a little cleaner.
* tag 'trace-ringbuffer-v6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace:
ring-buffer: Add helper functions for allocations
+53
-44
+53
-44
kernel/trace/ring_buffer.c
+53
-44
kernel/trace/ring_buffer.c
···
402
402
}
403
403
404
404
/*
405
+
* For best performance, allocate cpu buffer data cache line sized
406
+
* and per CPU.
407
+
*/
408
+
#define alloc_cpu_buffer(cpu) (struct ring_buffer_per_cpu *) \
409
+
kzalloc_node(ALIGN(sizeof(struct ring_buffer_per_cpu), \
410
+
cache_line_size()), GFP_KERNEL, cpu_to_node(cpu));
411
+
412
+
#define alloc_cpu_page(cpu) (struct buffer_page *) \
413
+
kzalloc_node(ALIGN(sizeof(struct buffer_page), \
414
+
cache_line_size()), GFP_KERNEL, cpu_to_node(cpu));
415
+
416
+
static struct buffer_data_page *alloc_cpu_data(int cpu, int order)
417
+
{
418
+
struct buffer_data_page *dpage;
419
+
struct page *page;
420
+
gfp_t mflags;
421
+
422
+
/*
423
+
* __GFP_RETRY_MAYFAIL flag makes sure that the allocation fails
424
+
* gracefully without invoking oom-killer and the system is not
425
+
* destabilized.
426
+
*/
427
+
mflags = GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_COMP | __GFP_ZERO;
428
+
429
+
page = alloc_pages_node(cpu_to_node(cpu), mflags, order);
430
+
if (!page)
431
+
return NULL;
432
+
433
+
dpage = page_address(page);
434
+
rb_init_page(dpage);
435
+
436
+
return dpage;
437
+
}
438
+
439
+
/*
405
440
* We need to fit the time_stamp delta into 27 bits.
406
441
*/
407
442
static inline bool test_time_stamp(u64 delta)
···
2239
2204
struct ring_buffer_cpu_meta *meta = NULL;
2240
2205
struct buffer_page *bpage, *tmp;
2241
2206
bool user_thread = current->mm != NULL;
2242
-
gfp_t mflags;
2243
2207
long i;
2244
2208
2245
2209
/*
···
2251
2217
i = si_mem_available();
2252
2218
if (i < nr_pages)
2253
2219
return -ENOMEM;
2254
-
2255
-
/*
2256
-
* __GFP_RETRY_MAYFAIL flag makes sure that the allocation fails
2257
-
* gracefully without invoking oom-killer and the system is not
2258
-
* destabilized.
2259
-
*/
2260
-
mflags = GFP_KERNEL | __GFP_RETRY_MAYFAIL;
2261
2220
2262
2221
/*
2263
2222
* If a user thread allocates too much, and si_mem_available()
···
2268
2241
meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu);
2269
2242
2270
2243
for (i = 0; i < nr_pages; i++) {
2271
-
struct page *page;
2272
2244
2273
-
bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
2274
-
mflags, cpu_to_node(cpu_buffer->cpu));
2245
+
bpage = alloc_cpu_page(cpu_buffer->cpu);
2275
2246
if (!bpage)
2276
2247
goto free_pages;
2277
2248
···
2292
2267
bpage->range = 1;
2293
2268
bpage->id = i + 1;
2294
2269
} else {
2295
-
page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu),
2296
-
mflags | __GFP_COMP | __GFP_ZERO,
2297
-
cpu_buffer->buffer->subbuf_order);
2298
-
if (!page)
2270
+
int order = cpu_buffer->buffer->subbuf_order;
2271
+
bpage->page = alloc_cpu_data(cpu_buffer->cpu, order);
2272
+
if (!bpage->page)
2299
2273
goto free_pages;
2300
-
bpage->page = page_address(page);
2301
-
rb_init_page(bpage->page);
2302
2274
}
2303
2275
bpage->order = cpu_buffer->buffer->subbuf_order;
2304
2276
···
2346
2324
static struct ring_buffer_per_cpu *
2347
2325
rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu)
2348
2326
{
2349
-
struct ring_buffer_per_cpu *cpu_buffer __free(kfree) = NULL;
2327
+
struct ring_buffer_per_cpu *cpu_buffer __free(kfree) =
2328
+
alloc_cpu_buffer(cpu);
2350
2329
struct ring_buffer_cpu_meta *meta;
2351
2330
struct buffer_page *bpage;
2352
-
struct page *page;
2353
2331
int ret;
2354
2332
2355
-
cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()),
2356
-
GFP_KERNEL, cpu_to_node(cpu));
2357
2333
if (!cpu_buffer)
2358
2334
return NULL;
2359
2335
···
2367
2347
init_waitqueue_head(&cpu_buffer->irq_work.full_waiters);
2368
2348
mutex_init(&cpu_buffer->mapping_lock);
2369
2349
2370
-
bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
2371
-
GFP_KERNEL, cpu_to_node(cpu));
2350
+
bpage = alloc_cpu_page(cpu);
2372
2351
if (!bpage)
2373
2352
return NULL;
2374
2353
···
2389
2370
rb_meta_buffer_update(cpu_buffer, bpage);
2390
2371
bpage->range = 1;
2391
2372
} else {
2392
-
page = alloc_pages_node(cpu_to_node(cpu),
2393
-
GFP_KERNEL | __GFP_COMP | __GFP_ZERO,
2394
-
cpu_buffer->buffer->subbuf_order);
2395
-
if (!page)
2373
+
int order = cpu_buffer->buffer->subbuf_order;
2374
+
bpage->page = alloc_cpu_data(cpu, order);
2375
+
if (!bpage->page)
2396
2376
goto fail_free_reader;
2397
-
bpage->page = page_address(page);
2398
-
rb_init_page(bpage->page);
2399
2377
}
2400
2378
2401
2379
INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
···
6480
6464
struct ring_buffer_per_cpu *cpu_buffer;
6481
6465
struct buffer_data_read_page *bpage = NULL;
6482
6466
unsigned long flags;
6483
-
struct page *page;
6484
6467
6485
6468
if (!cpumask_test_cpu(cpu, buffer->cpumask))
6486
6469
return ERR_PTR(-ENODEV);
···
6501
6486
arch_spin_unlock(&cpu_buffer->lock);
6502
6487
local_irq_restore(flags);
6503
6488
6504
-
if (bpage->data)
6505
-
goto out;
6506
-
6507
-
page = alloc_pages_node(cpu_to_node(cpu),
6508
-
GFP_KERNEL | __GFP_NORETRY | __GFP_COMP | __GFP_ZERO,
6509
-
cpu_buffer->buffer->subbuf_order);
6510
-
if (!page) {
6511
-
kfree(bpage);
6512
-
return ERR_PTR(-ENOMEM);
6489
+
if (bpage->data) {
6490
+
rb_init_page(bpage->data);
6491
+
} else {
6492
+
bpage->data = alloc_cpu_data(cpu, cpu_buffer->buffer->subbuf_order);
6493
+
if (!bpage->data) {
6494
+
kfree(bpage);
6495
+
return ERR_PTR(-ENOMEM);
6496
+
}
6513
6497
}
6514
-
6515
-
bpage->data = page_address(page);
6516
-
6517
-
out:
6518
-
rb_init_page(bpage->data);
6519
6498
6520
6499
return bpage;
6521
6500
}