Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
slab: make __slab_free() more clear
The function is tricky and many of its tests are hard to understand. Try
to improve that by using more descriptively named variables and added
comments.
- rename 'prior' to 'old_head' to match the head and tail parameters
- introduce a 'bool was_full' to make it more obvious what we are
testing instead of the !prior and prior tests
- add or improve comments in various places to explain what we're doing
Also replace kmem_cache_has_cpu_partial() tests with
IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) which are compile-time constants.
We can do that because the kmem_cache_debug(s) case is handled upfront
via free_to_partial_list().
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Link: https://patch.msgid.link/20251105-sheaves-cleanups-v1-1-b8218e1ac7ef@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
+45
-17
+45
-17
mm/slub.c
+45
-17
mm/slub.c
···
5859
5859
unsigned long addr)
5860
5860
5861
5861
{
5862
-
void *prior;
5863
-
int was_frozen;
5862
+
void *old_head;
5863
+
bool was_frozen, was_full;
5864
5864
struct slab new;
5865
5865
unsigned long counters;
5866
5866
struct kmem_cache_node *n = NULL;
···
5874
5874
return;
5875
5875
}
5876
5876
5877
+
/*
5878
+
* It is enough to test IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) below
5879
+
* instead of kmem_cache_has_cpu_partial(s), because kmem_cache_debug(s)
5880
+
* is the only other reason it can be false, and it is already handled
5881
+
* above.
5882
+
*/
5883
+
5877
5884
do {
5878
5885
if (unlikely(n)) {
5879
5886
spin_unlock_irqrestore(&n->list_lock, flags);
5880
5887
n = NULL;
5881
5888
}
5882
-
prior = slab->freelist;
5889
+
old_head = slab->freelist;
5883
5890
counters = slab->counters;
5884
-
set_freepointer(s, tail, prior);
5891
+
set_freepointer(s, tail, old_head);
5885
5892
new.counters = counters;
5886
-
was_frozen = new.frozen;
5893
+
was_frozen = !!new.frozen;
5894
+
was_full = (old_head == NULL);
5887
5895
new.inuse -= cnt;
5888
-
if ((!new.inuse || !prior) && !was_frozen) {
5889
-
/* Needs to be taken off a list */
5890
-
if (!kmem_cache_has_cpu_partial(s) || prior) {
5896
+
/*
5897
+
* Might need to be taken off (due to becoming empty) or added
5898
+
* to (due to not being full anymore) the partial list.
5899
+
* Unless it's frozen.
5900
+
*/
5901
+
if ((!new.inuse || was_full) && !was_frozen) {
5902
+
/*
5903
+
* If slab becomes non-full and we have cpu partial
5904
+
* lists, we put it there unconditionally to avoid
5905
+
* taking the list_lock. Otherwise we need it.
5906
+
*/
5907
+
if (!(IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full)) {
5891
5908
5892
5909
n = get_node(s, slab_nid(slab));
5893
5910
/*
···
5922
5905
}
5923
5906
5924
5907
} while (!slab_update_freelist(s, slab,
5925
-
prior, counters,
5908
+
old_head, counters,
5926
5909
head, new.counters,
5927
5910
"__slab_free"));
5928
5911
···
5934
5917
* activity can be necessary.
5935
5918
*/
5936
5919
stat(s, FREE_FROZEN);
5937
-
} else if (kmem_cache_has_cpu_partial(s) && !prior) {
5920
+
} else if (IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full) {
5938
5921
/*
5939
5922
* If we started with a full slab then put it onto the
5940
5923
* per cpu partial list.
···
5943
5926
stat(s, CPU_PARTIAL_FREE);
5944
5927
}
5945
5928
5929
+
/*
5930
+
* In other cases we didn't take the list_lock because the slab
5931
+
* was already on the partial list and will remain there.
5932
+
*/
5933
+
5946
5934
return;
5947
5935
}
5948
5936
···
5955
5933
* This slab was partially empty but not on the per-node partial list,
5956
5934
* in which case we shouldn't manipulate its list, just return.
5957
5935
*/
5958
-
if (prior && !on_node_partial) {
5936
+
if (!was_full && !on_node_partial) {
5959
5937
spin_unlock_irqrestore(&n->list_lock, flags);
5960
5938
return;
5961
5939
}
5962
5940
5941
+
/*
5942
+
* If slab became empty, should we add/keep it on the partial list or we
5943
+
* have enough?
5944
+
*/
5963
5945
if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
5964
5946
goto slab_empty;
5965
5947
5966
5948
/*
5967
5949
* Objects left in the slab. If it was not on the partial list before
5968
-
* then add it.
5950
+
* then add it. This can only happen when cache has no per cpu partial
5951
+
* list otherwise we would have put it there.
5969
5952
*/
5970
-
if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
5953
+
if (!IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && unlikely(was_full)) {
5971
5954
add_partial(n, slab, DEACTIVATE_TO_TAIL);
5972
5955
stat(s, FREE_ADD_PARTIAL);
5973
5956
}
···
5980
5953
return;
5981
5954
5982
5955
slab_empty:
5983
-
if (prior) {
5984
-
/*
5985
-
* Slab on the partial list.
5986
-
*/
5956
+
/*
5957
+
* The slab could have a single object and thus go from full to empty in
5958
+
* a single free, but more likely it was on the partial list. Remove it.
5959
+
*/
5960
+
if (likely(!was_full)) {
5987
5961
remove_partial(n, slab);
5988
5962
stat(s, FREE_REMOVE_PARTIAL);
5989
5963
}