Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
vfs: fix dentry LRU list handling and nr_dentry_unused accounting
The LRU list changes interacted badly with our nr_dentry_unused
accounting, and even worse with the new DCACHE_LRU_LIST bit logic.
This introduces helper functions to make sure everything follows the
proper dcache d_lru list rules: the dentry cache is complicated by the
fact that some of the hotpaths don't even want to look at the LRU list
at all, and the fact that we use the same list entry in the dentry for
both the LRU list and for our temporary shrinking lists when removing
things from the LRU.
The helper functions temporarily have some extra sanity checking for the
flag bits that have to match the current LRU state of the dentry. We'll
remove that before the final 3.12 release, but considering how easy it
is to get wrong, this first cleanup version has some very particular
sanity checking.
Acked-by: Al Viro <viro@zeniv.linux.org.uk>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
+101
-27
+101
-27
fs/dcache.c
+101
-27
fs/dcache.c
···
357
357
}
358
358
359
359
/*
360
+
* The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
361
+
* is in use - which includes both the "real" per-superblock
362
+
* LRU list _and_ the DCACHE_SHRINK_LIST use.
363
+
*
364
+
* The DCACHE_SHRINK_LIST bit is set whenever the dentry is
365
+
* on the shrink list (ie not on the superblock LRU list).
366
+
*
367
+
* The per-cpu "nr_dentry_unused" counters are updated with
368
+
* the DCACHE_LRU_LIST bit.
369
+
*
370
+
* These helper functions make sure we always follow the
371
+
* rules. d_lock must be held by the caller.
372
+
*/
373
+
#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
374
+
static void d_lru_add(struct dentry *dentry)
375
+
{
376
+
D_FLAG_VERIFY(dentry, 0);
377
+
dentry->d_flags |= DCACHE_LRU_LIST;
378
+
this_cpu_inc(nr_dentry_unused);
379
+
WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
380
+
}
381
+
382
+
static void d_lru_del(struct dentry *dentry)
383
+
{
384
+
D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
385
+
dentry->d_flags &= ~DCACHE_LRU_LIST;
386
+
this_cpu_dec(nr_dentry_unused);
387
+
WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
388
+
}
389
+
390
+
static void d_shrink_del(struct dentry *dentry)
391
+
{
392
+
D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
393
+
list_del_init(&dentry->d_lru);
394
+
dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
395
+
this_cpu_dec(nr_dentry_unused);
396
+
}
397
+
398
+
static void d_shrink_add(struct dentry *dentry, struct list_head *list)
399
+
{
400
+
D_FLAG_VERIFY(dentry, 0);
401
+
list_add(&dentry->d_lru, list);
402
+
dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
403
+
this_cpu_inc(nr_dentry_unused);
404
+
}
405
+
406
+
/*
407
+
* These can only be called under the global LRU lock, ie during the
408
+
* callback for freeing the LRU list. "isolate" removes it from the
409
+
* LRU lists entirely, while shrink_move moves it to the indicated
410
+
* private list.
411
+
*/
412
+
static void d_lru_isolate(struct dentry *dentry)
413
+
{
414
+
D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
415
+
dentry->d_flags &= ~DCACHE_LRU_LIST;
416
+
this_cpu_dec(nr_dentry_unused);
417
+
list_del_init(&dentry->d_lru);
418
+
}
419
+
420
+
static void d_lru_shrink_move(struct dentry *dentry, struct list_head *list)
421
+
{
422
+
D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
423
+
dentry->d_flags |= DCACHE_SHRINK_LIST;
424
+
list_move_tail(&dentry->d_lru, list);
425
+
}
426
+
427
+
/*
360
428
* dentry_lru_(add|del)_list) must be called with d_lock held.
361
429
*/
362
430
static void dentry_lru_add(struct dentry *dentry)
363
431
{
364
-
if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST))) {
365
-
if (list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru))
366
-
this_cpu_inc(nr_dentry_unused);
367
-
dentry->d_flags |= DCACHE_LRU_LIST;
368
-
}
432
+
if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
433
+
d_lru_add(dentry);
369
434
}
370
435
371
436
/*
···
442
377
*/
443
378
static void dentry_lru_del(struct dentry *dentry)
444
379
{
445
-
if (dentry->d_flags & DCACHE_SHRINK_LIST) {
446
-
list_del_init(&dentry->d_lru);
447
-
dentry->d_flags &= ~DCACHE_SHRINK_LIST;
448
-
return;
380
+
if (dentry->d_flags & DCACHE_LRU_LIST) {
381
+
if (dentry->d_flags & DCACHE_SHRINK_LIST)
382
+
return d_shrink_del(dentry);
383
+
d_lru_del(dentry);
449
384
}
450
-
451
-
if (list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru))
452
-
this_cpu_dec(nr_dentry_unused);
453
-
dentry->d_flags &= ~DCACHE_LRU_LIST;
454
385
}
455
386
456
387
/**
···
898
837
dentry = list_entry_rcu(list->prev, struct dentry, d_lru);
899
838
if (&dentry->d_lru == list)
900
839
break; /* empty */
840
+
841
+
/*
842
+
* Get the dentry lock, and re-verify that the dentry is
843
+
* this on the shrinking list. If it is, we know that
844
+
* DCACHE_SHRINK_LIST and DCACHE_LRU_LIST are set.
845
+
*/
901
846
spin_lock(&dentry->d_lock);
902
847
if (dentry != list_entry(list->prev, struct dentry, d_lru)) {
903
848
spin_unlock(&dentry->d_lock);
···
915
848
* to the LRU here, so we can simply remove it from the list
916
849
* here regardless of whether it is referenced or not.
917
850
*/
918
-
list_del_init(&dentry->d_lru);
919
-
dentry->d_flags &= ~DCACHE_SHRINK_LIST;
851
+
d_shrink_del(dentry);
920
852
921
853
/*
922
854
* We found an inuse dentry which was not removed from
···
927
861
}
928
862
rcu_read_unlock();
929
863
864
+
/*
865
+
* If 'try_to_prune()' returns a dentry, it will
866
+
* be the same one we passed in, and d_lock will
867
+
* have been held the whole time, so it will not
868
+
* have been added to any other lists. We failed
869
+
* to get the inode lock.
870
+
*
871
+
* We just add it back to the shrink list.
872
+
*/
930
873
dentry = try_prune_one_dentry(dentry);
931
874
932
875
rcu_read_lock();
933
876
if (dentry) {
934
-
dentry->d_flags |= DCACHE_SHRINK_LIST;
935
-
list_add(&dentry->d_lru, list);
877
+
d_shrink_add(dentry, list);
936
878
spin_unlock(&dentry->d_lock);
937
879
}
938
880
}
···
968
894
* another pass through the LRU.
969
895
*/
970
896
if (dentry->d_lockref.count) {
971
-
list_del_init(&dentry->d_lru);
897
+
d_lru_isolate(dentry);
972
898
spin_unlock(&dentry->d_lock);
973
899
return LRU_REMOVED;
974
900
}
···
999
925
return LRU_ROTATE;
1000
926
}
1001
927
1002
-
dentry->d_flags |= DCACHE_SHRINK_LIST;
1003
-
list_move_tail(&dentry->d_lru, freeable);
1004
-
this_cpu_dec(nr_dentry_unused);
928
+
d_lru_shrink_move(dentry, freeable);
1005
929
spin_unlock(&dentry->d_lock);
1006
930
1007
931
return LRU_REMOVED;
···
1044
972
if (!spin_trylock(&dentry->d_lock))
1045
973
return LRU_SKIP;
1046
974
1047
-
dentry->d_flags |= DCACHE_SHRINK_LIST;
1048
-
list_move_tail(&dentry->d_lru, freeable);
1049
-
this_cpu_dec(nr_dentry_unused);
975
+
d_lru_shrink_move(dentry, freeable);
1050
976
spin_unlock(&dentry->d_lock);
1051
977
1052
978
return LRU_REMOVED;
···
1432
1362
if (dentry->d_lockref.count) {
1433
1363
dentry_lru_del(dentry);
1434
1364
} else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
1435
-
dentry_lru_del(dentry);
1436
-
list_add_tail(&dentry->d_lru, &data->dispose);
1437
-
dentry->d_flags |= DCACHE_SHRINK_LIST;
1365
+
/*
1366
+
* We can't use d_lru_shrink_move() because we
1367
+
* need to get the global LRU lock and do the
1368
+
* RLU accounting.
1369
+
*/
1370
+
d_lru_del(dentry);
1371
+
d_shrink_add(dentry, &data->dispose);
1438
1372
data->found++;
1439
1373
ret = D_WALK_NORETRY;
1440
1374
}