Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'akpm' (patches from Andrew)
Merge misc fixes from Andrew Morton:
"23 fixes"
* emailed patches from Andrew Morton <akpm@linux-foundation.org>: (23 commits)
mm, memory_hotplug: fix off-by-one in is_pageblock_removable
mm: don't let userspace spam allocations warnings
slub: fix a crash with SLUB_DEBUG + KASAN_SW_TAGS
kasan, slab: remove redundant kasan_slab_alloc hooks
kasan, slab: make freelist stored without tags
kasan, slab: fix conflicts with CONFIG_HARDENED_USERCOPY
kasan: prevent tracing of tags.c
kasan: fix random seed generation for tag-based mode
tmpfs: fix link accounting when a tmpfile is linked in
psi: avoid divide-by-zero crash inside virtual machines
mm: handle lru_add_drain_all for UP properly
mm, page_alloc: fix a division by zero error when boosting watermarks v2
mm/debug.c: fix __dump_page() for poisoned pages
proc, oom: do not report alien mms when setting oom_score_adj
slub: fix SLAB_CONSISTENCY_CHECKS + KASAN_SW_TAGS
kasan, slub: fix more conflicts with CONFIG_SLAB_FREELIST_HARDENED
kasan, slub: fix conflicts with CONFIG_SLAB_FREELIST_HARDENED
kasan, slub: move kasan_poison_slab hook before page_address
kmemleak: account for tagged pointers when calculating pointer range
kasan, kmemleak: pass tagged pointers to kmemleak
...
+140
-82
+3
arch/arm64/kernel/setup.c
+3
arch/arm64/kernel/setup.c
-2
arch/arm64/mm/kasan_init.c
-2
arch/arm64/mm/kasan_init.c
···
252
252
memset(kasan_early_shadow_page, KASAN_SHADOW_INIT, PAGE_SIZE);
253
253
cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
254
254
255
-
kasan_init_tags();
256
-
257
255
/* At this point kasan is fully initialized. Enable error messages */
258
256
init_task.kasan_depth = 0;
259
257
pr_info("KernelAddressSanitizer initialized\n");
-4
fs/proc/base.c
-4
fs/proc/base.c
···
1086
1086
1087
1087
task_lock(p);
1088
1088
if (!p->vfork_done && process_shares_mm(p, mm)) {
1089
-
pr_info("updating oom_score_adj for %d (%s) from %d to %d because it shares mm with %d (%s). Report if this is unexpected.\n",
1090
-
task_pid_nr(p), p->comm,
1091
-
p->signal->oom_score_adj, oom_adj,
1092
-
task_pid_nr(task), task->comm);
1093
1089
p->signal->oom_score_adj = oom_adj;
1094
1090
if (!legacy && has_capability_noaudit(current, CAP_SYS_RESOURCE))
1095
1091
p->signal->oom_score_adj_min = (short)oom_adj;
+3
-3
init/initramfs.c
+3
-3
init/initramfs.c
···
550
550
initrd_end = 0;
551
551
}
552
552
553
+
#ifdef CONFIG_BLK_DEV_RAM
553
554
#define BUF_SIZE 1024
554
555
static void __init clean_rootfs(void)
555
556
{
···
597
596
ksys_close(fd);
598
597
kfree(buf);
599
598
}
599
+
#endif
600
600
601
601
static int __init populate_rootfs(void)
602
602
{
···
640
638
printk(KERN_INFO "Unpacking initramfs...\n");
641
639
err = unpack_to_rootfs((char *)initrd_start,
642
640
initrd_end - initrd_start);
643
-
if (err) {
641
+
if (err)
644
642
printk(KERN_EMERG "Initramfs unpacking failed: %s\n", err);
645
-
clean_rootfs();
646
-
}
647
643
free_initrd();
648
644
#endif
649
645
}
+1
-1
kernel/sched/psi.c
+1
-1
kernel/sched/psi.c
+3
-1
mm/debug.c
+3
-1
mm/debug.c
···
44
44
45
45
void __dump_page(struct page *page, const char *reason)
46
46
{
47
-
struct address_space *mapping = page_mapping(page);
47
+
struct address_space *mapping;
48
48
bool page_poisoned = PagePoisoned(page);
49
49
int mapcount;
50
50
···
57
57
pr_warn("page:%px is uninitialized and poisoned", page);
58
58
goto hex_only;
59
59
}
60
+
61
+
mapping = page_mapping(page);
60
62
61
63
/*
62
64
* Avoid VM_BUG_ON() in page_mapcount().
+2
mm/kasan/Makefile
+2
mm/kasan/Makefile
+17
-12
mm/kasan/common.c
+17
-12
mm/kasan/common.c
···
361
361
* get different tags.
362
362
*/
363
363
static u8 assign_tag(struct kmem_cache *cache, const void *object,
364
-
bool init, bool krealloc)
364
+
bool init, bool keep_tag)
365
365
{
366
-
/* Reuse the same tag for krealloc'ed objects. */
367
-
if (krealloc)
366
+
/*
367
+
* 1. When an object is kmalloc()'ed, two hooks are called:
368
+
* kasan_slab_alloc() and kasan_kmalloc(). We assign the
369
+
* tag only in the first one.
370
+
* 2. We reuse the same tag for krealloc'ed objects.
371
+
*/
372
+
if (keep_tag)
368
373
return get_tag(object);
369
374
370
375
/*
···
408
403
assign_tag(cache, object, true, false));
409
404
410
405
return (void *)object;
411
-
}
412
-
413
-
void * __must_check kasan_slab_alloc(struct kmem_cache *cache, void *object,
414
-
gfp_t flags)
415
-
{
416
-
return kasan_kmalloc(cache, object, cache->object_size, flags);
417
406
}
418
407
419
408
static inline bool shadow_invalid(u8 tag, s8 shadow_byte)
···
466
467
}
467
468
468
469
static void *__kasan_kmalloc(struct kmem_cache *cache, const void *object,
469
-
size_t size, gfp_t flags, bool krealloc)
470
+
size_t size, gfp_t flags, bool keep_tag)
470
471
{
471
472
unsigned long redzone_start;
472
473
unsigned long redzone_end;
···
484
485
KASAN_SHADOW_SCALE_SIZE);
485
486
486
487
if (IS_ENABLED(CONFIG_KASAN_SW_TAGS))
487
-
tag = assign_tag(cache, object, false, krealloc);
488
+
tag = assign_tag(cache, object, false, keep_tag);
488
489
489
490
/* Tag is ignored in set_tag without CONFIG_KASAN_SW_TAGS */
490
491
kasan_unpoison_shadow(set_tag(object, tag), size);
···
497
498
return set_tag(object, tag);
498
499
}
499
500
501
+
void * __must_check kasan_slab_alloc(struct kmem_cache *cache, void *object,
502
+
gfp_t flags)
503
+
{
504
+
return __kasan_kmalloc(cache, object, cache->object_size, flags, false);
505
+
}
506
+
500
507
void * __must_check kasan_kmalloc(struct kmem_cache *cache, const void *object,
501
508
size_t size, gfp_t flags)
502
509
{
503
-
return __kasan_kmalloc(cache, object, size, flags, false);
510
+
return __kasan_kmalloc(cache, object, size, flags, true);
504
511
}
505
512
EXPORT_SYMBOL(kasan_kmalloc);
506
513
+7
-3
mm/kmemleak.c
+7
-3
mm/kmemleak.c
···
574
574
unsigned long flags;
575
575
struct kmemleak_object *object, *parent;
576
576
struct rb_node **link, *rb_parent;
577
+
unsigned long untagged_ptr;
577
578
578
579
object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
579
580
if (!object) {
···
620
619
621
620
write_lock_irqsave(&kmemleak_lock, flags);
622
621
623
-
min_addr = min(min_addr, ptr);
624
-
max_addr = max(max_addr, ptr + size);
622
+
untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
623
+
min_addr = min(min_addr, untagged_ptr);
624
+
max_addr = max(max_addr, untagged_ptr + size);
625
625
link = &object_tree_root.rb_node;
626
626
rb_parent = NULL;
627
627
while (*link) {
···
1335
1333
unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1336
1334
unsigned long *end = _end - (BYTES_PER_POINTER - 1);
1337
1335
unsigned long flags;
1336
+
unsigned long untagged_ptr;
1338
1337
1339
1338
read_lock_irqsave(&kmemleak_lock, flags);
1340
1339
for (ptr = start; ptr < end; ptr++) {
···
1350
1347
pointer = *ptr;
1351
1348
kasan_enable_current();
1352
1349
1353
-
if (pointer < min_addr || pointer >= max_addr)
1350
+
untagged_ptr = (unsigned long)kasan_reset_tag((void *)pointer);
1351
+
if (untagged_ptr < min_addr || untagged_ptr >= max_addr)
1354
1352
continue;
1355
1353
1356
1354
/*
+15
-12
mm/memory_hotplug.c
+15
-12
mm/memory_hotplug.c
···
1188
1188
return PageBuddy(page) && page_order(page) >= pageblock_order;
1189
1189
}
1190
1190
1191
-
/* Return the start of the next active pageblock after a given page */
1192
-
static struct page *next_active_pageblock(struct page *page)
1191
+
/* Return the pfn of the start of the next active pageblock after a given pfn */
1192
+
static unsigned long next_active_pageblock(unsigned long pfn)
1193
1193
{
1194
+
struct page *page = pfn_to_page(pfn);
1195
+
1194
1196
/* Ensure the starting page is pageblock-aligned */
1195
-
BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1));
1197
+
BUG_ON(pfn & (pageblock_nr_pages - 1));
1196
1198
1197
1199
/* If the entire pageblock is free, move to the end of free page */
1198
1200
if (pageblock_free(page)) {
···
1202
1200
/* be careful. we don't have locks, page_order can be changed.*/
1203
1201
order = page_order(page);
1204
1202
if ((order < MAX_ORDER) && (order >= pageblock_order))
1205
-
return page + (1 << order);
1203
+
return pfn + (1 << order);
1206
1204
}
1207
1205
1208
-
return page + pageblock_nr_pages;
1206
+
return pfn + pageblock_nr_pages;
1209
1207
}
1210
1208
1211
-
static bool is_pageblock_removable_nolock(struct page *page)
1209
+
static bool is_pageblock_removable_nolock(unsigned long pfn)
1212
1210
{
1211
+
struct page *page = pfn_to_page(pfn);
1213
1212
struct zone *zone;
1214
-
unsigned long pfn;
1215
1213
1216
1214
/*
1217
1215
* We have to be careful here because we are iterating over memory
···
1234
1232
/* Checks if this range of memory is likely to be hot-removable. */
1235
1233
bool is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages)
1236
1234
{
1237
-
struct page *page = pfn_to_page(start_pfn);
1238
-
unsigned long end_pfn = min(start_pfn + nr_pages, zone_end_pfn(page_zone(page)));
1239
-
struct page *end_page = pfn_to_page(end_pfn);
1235
+
unsigned long end_pfn, pfn;
1236
+
1237
+
end_pfn = min(start_pfn + nr_pages,
1238
+
zone_end_pfn(page_zone(pfn_to_page(start_pfn))));
1240
1239
1241
1240
/* Check the starting page of each pageblock within the range */
1242
-
for (; page < end_page; page = next_active_pageblock(page)) {
1243
-
if (!is_pageblock_removable_nolock(page))
1241
+
for (pfn = start_pfn; pfn < end_pfn; pfn = next_active_pageblock(pfn)) {
1242
+
if (!is_pageblock_removable_nolock(pfn))
1244
1243
return false;
1245
1244
cond_resched();
1246
1245
}
+3
-3
mm/mempolicy.c
+3
-3
mm/mempolicy.c
···
1314
1314
nodemask_t *nodes)
1315
1315
{
1316
1316
unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1317
-
const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
1317
+
unsigned int nbytes = BITS_TO_LONGS(nr_node_ids) * sizeof(long);
1318
1318
1319
1319
if (copy > nbytes) {
1320
1320
if (copy > PAGE_SIZE)
···
1491
1491
int uninitialized_var(pval);
1492
1492
nodemask_t nodes;
1493
1493
1494
-
if (nmask != NULL && maxnode < MAX_NUMNODES)
1494
+
if (nmask != NULL && maxnode < nr_node_ids)
1495
1495
return -EINVAL;
1496
1496
1497
1497
err = do_get_mempolicy(&pval, &nodes, addr, flags);
···
1527
1527
unsigned long nr_bits, alloc_size;
1528
1528
DECLARE_BITMAP(bm, MAX_NUMNODES);
1529
1529
1530
-
nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1530
+
nr_bits = min_t(unsigned long, maxnode-1, nr_node_ids);
1531
1531
alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1532
1532
1533
1533
if (nmask)
+12
mm/page_alloc.c
+12
mm/page_alloc.c
···
2170
2170
2171
2171
max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
2172
2172
watermark_boost_factor, 10000);
2173
+
2174
+
/*
2175
+
* high watermark may be uninitialised if fragmentation occurs
2176
+
* very early in boot so do not boost. We do not fall
2177
+
* through and boost by pageblock_nr_pages as failing
2178
+
* allocations that early means that reclaim is not going
2179
+
* to help and it may even be impossible to reclaim the
2180
+
* boosted watermark resulting in a hang.
2181
+
*/
2182
+
if (!max_boost)
2183
+
return;
2184
+
2173
2185
max_boost = max(pageblock_nr_pages, max_boost);
2174
2186
2175
2187
zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
+7
-3
mm/shmem.c
+7
-3
mm/shmem.c
···
2854
2854
* No ordinary (disk based) filesystem counts links as inodes;
2855
2855
* but each new link needs a new dentry, pinning lowmem, and
2856
2856
* tmpfs dentries cannot be pruned until they are unlinked.
2857
+
* But if an O_TMPFILE file is linked into the tmpfs, the
2858
+
* first link must skip that, to get the accounting right.
2857
2859
*/
2858
-
ret = shmem_reserve_inode(inode->i_sb);
2859
-
if (ret)
2860
-
goto out;
2860
+
if (inode->i_nlink) {
2861
+
ret = shmem_reserve_inode(inode->i_sb);
2862
+
if (ret)
2863
+
goto out;
2864
+
}
2861
2865
2862
2866
dir->i_size += BOGO_DIRENT_SIZE;
2863
2867
inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
+11
-4
mm/slab.c
+11
-4
mm/slab.c
···
2359
2359
void *freelist;
2360
2360
void *addr = page_address(page);
2361
2361
2362
-
page->s_mem = kasan_reset_tag(addr) + colour_off;
2362
+
page->s_mem = addr + colour_off;
2363
2363
page->active = 0;
2364
2364
2365
2365
if (OBJFREELIST_SLAB(cachep))
···
2368
2368
/* Slab management obj is off-slab. */
2369
2369
freelist = kmem_cache_alloc_node(cachep->freelist_cache,
2370
2370
local_flags, nodeid);
2371
+
freelist = kasan_reset_tag(freelist);
2371
2372
if (!freelist)
2372
2373
return NULL;
2373
2374
} else {
···
2682
2681
2683
2682
offset *= cachep->colour_off;
2684
2683
2684
+
/*
2685
+
* Call kasan_poison_slab() before calling alloc_slabmgmt(), so
2686
+
* page_address() in the latter returns a non-tagged pointer,
2687
+
* as it should be for slab pages.
2688
+
*/
2689
+
kasan_poison_slab(page);
2690
+
2685
2691
/* Get slab management. */
2686
2692
freelist = alloc_slabmgmt(cachep, page, offset,
2687
2693
local_flags & ~GFP_CONSTRAINT_MASK, page_node);
···
2697
2689
2698
2690
slab_map_pages(cachep, page, freelist);
2699
2691
2700
-
kasan_poison_slab(page);
2701
2692
cache_init_objs(cachep, page);
2702
2693
2703
2694
if (gfpflags_allow_blocking(local_flags))
···
3547
3540
{
3548
3541
void *ret = slab_alloc(cachep, flags, _RET_IP_);
3549
3542
3550
-
ret = kasan_slab_alloc(cachep, ret, flags);
3551
3543
trace_kmem_cache_alloc(_RET_IP_, ret,
3552
3544
cachep->object_size, cachep->size, flags);
3553
3545
···
3636
3630
{
3637
3631
void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3638
3632
3639
-
ret = kasan_slab_alloc(cachep, ret, flags);
3640
3633
trace_kmem_cache_alloc_node(_RET_IP_, ret,
3641
3634
cachep->object_size, cachep->size,
3642
3635
flags, nodeid);
···
4412
4407
struct kmem_cache *cachep;
4413
4408
unsigned int objnr;
4414
4409
unsigned long offset;
4410
+
4411
+
ptr = kasan_reset_tag(ptr);
4415
4412
4416
4413
/* Find and validate object. */
4417
4414
cachep = page->slab_cache;
+3
-4
mm/slab.h
+3
-4
mm/slab.h
···
437
437
438
438
flags &= gfp_allowed_mask;
439
439
for (i = 0; i < size; i++) {
440
-
void *object = p[i];
441
-
442
-
kmemleak_alloc_recursive(object, s->object_size, 1,
440
+
p[i] = kasan_slab_alloc(s, p[i], flags);
441
+
/* As p[i] might get tagged, call kmemleak hook after KASAN. */
442
+
kmemleak_alloc_recursive(p[i], s->object_size, 1,
443
443
s->flags, flags);
444
-
p[i] = kasan_slab_alloc(s, object, flags);
445
444
}
446
445
447
446
if (memcg_kmem_enabled())
+2
-1
mm/slab_common.c
+2
-1
mm/slab_common.c
···
1228
1228
flags |= __GFP_COMP;
1229
1229
page = alloc_pages(flags, order);
1230
1230
ret = page ? page_address(page) : NULL;
1231
-
kmemleak_alloc(ret, size, 1, flags);
1232
1231
ret = kasan_kmalloc_large(ret, size, flags);
1232
+
/* As ret might get tagged, call kmemleak hook after KASAN. */
1233
+
kmemleak_alloc(ret, size, 1, flags);
1233
1234
return ret;
1234
1235
}
1235
1236
EXPORT_SYMBOL(kmalloc_order);
+39
-20
mm/slub.c
+39
-20
mm/slub.c
···
249
249
unsigned long ptr_addr)
250
250
{
251
251
#ifdef CONFIG_SLAB_FREELIST_HARDENED
252
-
return (void *)((unsigned long)ptr ^ s->random ^ ptr_addr);
252
+
/*
253
+
* When CONFIG_KASAN_SW_TAGS is enabled, ptr_addr might be tagged.
254
+
* Normally, this doesn't cause any issues, as both set_freepointer()
255
+
* and get_freepointer() are called with a pointer with the same tag.
256
+
* However, there are some issues with CONFIG_SLUB_DEBUG code. For
257
+
* example, when __free_slub() iterates over objects in a cache, it
258
+
* passes untagged pointers to check_object(). check_object() in turns
259
+
* calls get_freepointer() with an untagged pointer, which causes the
260
+
* freepointer to be restored incorrectly.
261
+
*/
262
+
return (void *)((unsigned long)ptr ^ s->random ^
263
+
(unsigned long)kasan_reset_tag((void *)ptr_addr));
253
264
#else
254
265
return ptr;
255
266
#endif
···
314
303
__p < (__addr) + (__objects) * (__s)->size; \
315
304
__p += (__s)->size)
316
305
317
-
#define for_each_object_idx(__p, __idx, __s, __addr, __objects) \
318
-
for (__p = fixup_red_left(__s, __addr), __idx = 1; \
319
-
__idx <= __objects; \
320
-
__p += (__s)->size, __idx++)
321
-
322
306
/* Determine object index from a given position */
323
307
static inline unsigned int slab_index(void *p, struct kmem_cache *s, void *addr)
324
308
{
325
-
return (p - addr) / s->size;
309
+
return (kasan_reset_tag(p) - addr) / s->size;
326
310
}
327
311
328
312
static inline unsigned int order_objects(unsigned int order, unsigned int size)
···
513
507
return 1;
514
508
515
509
base = page_address(page);
510
+
object = kasan_reset_tag(object);
516
511
object = restore_red_left(s, object);
517
512
if (object < base || object >= base + page->objects * s->size ||
518
513
(object - base) % s->size) {
···
1082
1075
init_tracking(s, object);
1083
1076
}
1084
1077
1078
+
static void setup_page_debug(struct kmem_cache *s, void *addr, int order)
1079
+
{
1080
+
if (!(s->flags & SLAB_POISON))
1081
+
return;
1082
+
1083
+
metadata_access_enable();
1084
+
memset(addr, POISON_INUSE, PAGE_SIZE << order);
1085
+
metadata_access_disable();
1086
+
}
1087
+
1085
1088
static inline int alloc_consistency_checks(struct kmem_cache *s,
1086
1089
struct page *page,
1087
1090
void *object, unsigned long addr)
···
1347
1330
#else /* !CONFIG_SLUB_DEBUG */
1348
1331
static inline void setup_object_debug(struct kmem_cache *s,
1349
1332
struct page *page, void *object) {}
1333
+
static inline void setup_page_debug(struct kmem_cache *s,
1334
+
void *addr, int order) {}
1350
1335
1351
1336
static inline int alloc_debug_processing(struct kmem_cache *s,
1352
1337
struct page *page, void *object, unsigned long addr) { return 0; }
···
1393
1374
*/
1394
1375
static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
1395
1376
{
1377
+
ptr = kasan_kmalloc_large(ptr, size, flags);
1378
+
/* As ptr might get tagged, call kmemleak hook after KASAN. */
1396
1379
kmemleak_alloc(ptr, size, 1, flags);
1397
-
return kasan_kmalloc_large(ptr, size, flags);
1380
+
return ptr;
1398
1381
}
1399
1382
1400
1383
static __always_inline void kfree_hook(void *x)
···
1662
1641
if (page_is_pfmemalloc(page))
1663
1642
SetPageSlabPfmemalloc(page);
1664
1643
1644
+
kasan_poison_slab(page);
1645
+
1665
1646
start = page_address(page);
1666
1647
1667
-
if (unlikely(s->flags & SLAB_POISON))
1668
-
memset(start, POISON_INUSE, PAGE_SIZE << order);
1669
-
1670
-
kasan_poison_slab(page);
1648
+
setup_page_debug(s, start, order);
1671
1649
1672
1650
shuffle = shuffle_freelist(s, page);
1673
1651
1674
1652
if (!shuffle) {
1675
-
for_each_object_idx(p, idx, s, start, page->objects) {
1676
-
if (likely(idx < page->objects)) {
1677
-
next = p + s->size;
1678
-
next = setup_object(s, page, next);
1679
-
set_freepointer(s, p, next);
1680
-
} else
1681
-
set_freepointer(s, p, NULL);
1682
-
}
1683
1653
start = fixup_red_left(s, start);
1684
1654
start = setup_object(s, page, start);
1685
1655
page->freelist = start;
1656
+
for (idx = 0, p = start; idx < page->objects - 1; idx++) {
1657
+
next = p + s->size;
1658
+
next = setup_object(s, page, next);
1659
+
set_freepointer(s, p, next);
1660
+
p = next;
1661
+
}
1662
+
set_freepointer(s, p, NULL);
1686
1663
}
1687
1664
1688
1665
page->inuse = page->objects;
+10
-7
mm/swap.c
+10
-7
mm/swap.c
···
320
320
{
321
321
}
322
322
323
-
static bool need_activate_page_drain(int cpu)
324
-
{
325
-
return false;
326
-
}
327
-
328
323
void activate_page(struct page *page)
329
324
{
330
325
struct zone *zone = page_zone(page);
···
648
653
put_cpu();
649
654
}
650
655
656
+
#ifdef CONFIG_SMP
657
+
658
+
static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
659
+
651
660
static void lru_add_drain_per_cpu(struct work_struct *dummy)
652
661
{
653
662
lru_add_drain();
654
663
}
655
-
656
-
static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
657
664
658
665
/*
659
666
* Doesn't need any cpu hotplug locking because we do rely on per-cpu
···
699
702
700
703
mutex_unlock(&lock);
701
704
}
705
+
#else
706
+
void lru_add_drain_all(void)
707
+
{
708
+
lru_add_drain();
709
+
}
710
+
#endif
702
711
703
712
/**
704
713
* release_pages - batched put_page()