Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
zram: introduce writeback bio batching
Patch series "zram: introduce writeback bio batching", v6.
As writeback is becoming more and more common the longstanding limitations
of zram writeback throughput are becoming more visible. Introduce
writeback bio batching so that multiple writeback bios can be processed
simultaneously.
This patch (of 6):
As was stated in a comment [1] a single page writeback IO is not
efficient, but it works. It's time to address this throughput limitation
as writeback becomes used more often. Introduce batched (multiple) bio
writeback support to take advantage of parallel requests processing and
better requests scheduling.
Approach used in this patch doesn't use a dedicated kthread like in [2],
or blk-plug like in [3]. Dedicated kthread adds complexity, which can be
avoided. Apart from that not all zram setups use writeback, so having
numerous per-device kthreads (on systems that create multiple zram
devices) hanging around is not the most optimal thing to do. blk-plug, on
the other hand, works best when request are sequential, which doesn't
particularly fit zram writebck IO patterns: zram writeback IO patterns are
expected to be random, due to how bdev block reservation/release are
handled. blk-plug approach also works in cycles: idle IO, when zram sets
up requests in a batch, is followed by bursts of IO, when zram submits the
entire batch.
Instead we use a batch of requests and submit new bio as soon as one of
the in-flight requests completes.
For the time being the writeback batch size (maximum number of in-flight
bio requests) is set to 32 for all devices. A follow up patch adds a
writeback_batch_size device attribute, so the batch size becomes run-time
configurable.
Link: https://lkml.kernel.org/r/20251122074029.3948921-1-senozhatsky@chromium.org
Link: https://lkml.kernel.org/r/20251122074029.3948921-2-senozhatsky@chromium.org
Link: https://lore.kernel.org/all/20181203024045.153534-6-minchan@kernel.org/ [1]
Link: https://lore.kernel.org/all/20250731064949.1690732-1-richardycc@google.com/ [2]
Link: https://lore.kernel.org/all/tencent_78FC2C4FE16BA1EBAF0897DB60FCD675ED05@qq.com/ [3]
Signed-off-by: Sergey Senozhatsky <senozhatsky@chromium.org>
Co-developed-by: Yuwen Chen <ywen.chen@foxmail.com>
Co-developed-by: Richard Chang <richardycc@google.com>
Suggested-by: Minchan Kim <minchan@google.com>
Cc: Brian Geffon <bgeffon@google.com>
Cc: Richard Chang <richardycc@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Sergey Senozhatsky
and committed by
Andrew Morton
f405066a
9e014077
+302
-69
+302
-69
drivers/block/zram/zram_drv.c
+302
-69
drivers/block/zram/zram_drv.c
···
500
500
}
501
501
502
502
#ifdef CONFIG_ZRAM_WRITEBACK
503
+
struct zram_wb_ctl {
504
+
/* idle list is accessed only by the writeback task, no concurency */
505
+
struct list_head idle_reqs;
506
+
/* done list is accessed concurrently, protect by done_lock */
507
+
struct list_head done_reqs;
508
+
wait_queue_head_t done_wait;
509
+
spinlock_t done_lock;
510
+
atomic_t num_inflight;
511
+
};
512
+
513
+
struct zram_wb_req {
514
+
unsigned long blk_idx;
515
+
struct page *page;
516
+
struct zram_pp_slot *pps;
517
+
struct bio_vec bio_vec;
518
+
struct bio bio;
519
+
520
+
struct list_head entry;
521
+
};
522
+
503
523
static ssize_t writeback_limit_enable_store(struct device *dev,
504
524
struct device_attribute *attr, const char *buf, size_t len)
505
525
{
···
754
734
submit_bio(bio);
755
735
}
756
736
757
-
static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl)
737
+
static void release_wb_req(struct zram_wb_req *req)
758
738
{
759
-
unsigned long blk_idx = 0;
760
-
struct page *page = NULL;
761
-
struct zram_pp_slot *pps;
762
-
struct bio_vec bio_vec;
763
-
struct bio bio;
764
-
int ret = 0, err;
765
-
u32 index;
739
+
__free_page(req->page);
740
+
kfree(req);
741
+
}
766
742
767
-
page = alloc_page(GFP_KERNEL);
768
-
if (!page)
769
-
return -ENOMEM;
743
+
static void release_wb_ctl(struct zram_wb_ctl *wb_ctl)
744
+
{
745
+
if (!wb_ctl)
746
+
return;
747
+
748
+
/* We should never have inflight requests at this point */
749
+
WARN_ON(atomic_read(&wb_ctl->num_inflight));
750
+
WARN_ON(!list_empty(&wb_ctl->done_reqs));
751
+
752
+
while (!list_empty(&wb_ctl->idle_reqs)) {
753
+
struct zram_wb_req *req;
754
+
755
+
req = list_first_entry(&wb_ctl->idle_reqs,
756
+
struct zram_wb_req, entry);
757
+
list_del(&req->entry);
758
+
release_wb_req(req);
759
+
}
760
+
761
+
kfree(wb_ctl);
762
+
}
763
+
764
+
/* XXX: should be a per-device sysfs attr */
765
+
#define ZRAM_WB_REQ_CNT 32
766
+
767
+
static struct zram_wb_ctl *init_wb_ctl(void)
768
+
{
769
+
struct zram_wb_ctl *wb_ctl;
770
+
int i;
771
+
772
+
wb_ctl = kmalloc(sizeof(*wb_ctl), GFP_KERNEL);
773
+
if (!wb_ctl)
774
+
return NULL;
775
+
776
+
INIT_LIST_HEAD(&wb_ctl->idle_reqs);
777
+
INIT_LIST_HEAD(&wb_ctl->done_reqs);
778
+
atomic_set(&wb_ctl->num_inflight, 0);
779
+
init_waitqueue_head(&wb_ctl->done_wait);
780
+
spin_lock_init(&wb_ctl->done_lock);
781
+
782
+
for (i = 0; i < ZRAM_WB_REQ_CNT; i++) {
783
+
struct zram_wb_req *req;
784
+
785
+
/*
786
+
* This is fatal condition only if we couldn't allocate
787
+
* any requests at all. Otherwise we just work with the
788
+
* requests that we have successfully allocated, so that
789
+
* writeback can still proceed, even if there is only one
790
+
* request on the idle list.
791
+
*/
792
+
req = kzalloc(sizeof(*req), GFP_KERNEL | __GFP_NOWARN);
793
+
if (!req)
794
+
break;
795
+
796
+
req->page = alloc_page(GFP_KERNEL | __GFP_NOWARN);
797
+
if (!req->page) {
798
+
kfree(req);
799
+
break;
800
+
}
801
+
802
+
list_add(&req->entry, &wb_ctl->idle_reqs);
803
+
}
804
+
805
+
/* We couldn't allocate any requests, so writeabck is not possible */
806
+
if (list_empty(&wb_ctl->idle_reqs))
807
+
goto release_wb_ctl;
808
+
809
+
return wb_ctl;
810
+
811
+
release_wb_ctl:
812
+
release_wb_ctl(wb_ctl);
813
+
return NULL;
814
+
}
815
+
816
+
static void zram_account_writeback_rollback(struct zram *zram)
817
+
{
818
+
spin_lock(&zram->wb_limit_lock);
819
+
if (zram->wb_limit_enable)
820
+
zram->bd_wb_limit += 1UL << (PAGE_SHIFT - 12);
821
+
spin_unlock(&zram->wb_limit_lock);
822
+
}
823
+
824
+
static void zram_account_writeback_submit(struct zram *zram)
825
+
{
826
+
spin_lock(&zram->wb_limit_lock);
827
+
if (zram->wb_limit_enable && zram->bd_wb_limit > 0)
828
+
zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12);
829
+
spin_unlock(&zram->wb_limit_lock);
830
+
}
831
+
832
+
static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req)
833
+
{
834
+
u32 index = req->pps->index;
835
+
int err;
836
+
837
+
err = blk_status_to_errno(req->bio.bi_status);
838
+
if (err) {
839
+
/*
840
+
* Failed wb requests should not be accounted in wb_limit
841
+
* (if enabled).
842
+
*/
843
+
zram_account_writeback_rollback(zram);
844
+
free_block_bdev(zram, req->blk_idx);
845
+
return err;
846
+
}
847
+
848
+
atomic64_inc(&zram->stats.bd_writes);
849
+
zram_slot_lock(zram, index);
850
+
/*
851
+
* We release slot lock during writeback so slot can change under us:
852
+
* slot_free() or slot_free() and zram_write_page(). In both cases
853
+
* slot loses ZRAM_PP_SLOT flag. No concurrent post-processing can
854
+
* set ZRAM_PP_SLOT on such slots until current post-processing
855
+
* finishes.
856
+
*/
857
+
if (!zram_test_flag(zram, index, ZRAM_PP_SLOT)) {
858
+
free_block_bdev(zram, req->blk_idx);
859
+
goto out;
860
+
}
861
+
862
+
zram_free_page(zram, index);
863
+
zram_set_flag(zram, index, ZRAM_WB);
864
+
zram_set_handle(zram, index, req->blk_idx);
865
+
atomic64_inc(&zram->stats.pages_stored);
866
+
867
+
out:
868
+
zram_slot_unlock(zram, index);
869
+
return 0;
870
+
}
871
+
872
+
static void zram_writeback_endio(struct bio *bio)
873
+
{
874
+
struct zram_wb_req *req = container_of(bio, struct zram_wb_req, bio);
875
+
struct zram_wb_ctl *wb_ctl = bio->bi_private;
876
+
unsigned long flags;
877
+
878
+
spin_lock_irqsave(&wb_ctl->done_lock, flags);
879
+
list_add(&req->entry, &wb_ctl->done_reqs);
880
+
spin_unlock_irqrestore(&wb_ctl->done_lock, flags);
881
+
882
+
wake_up(&wb_ctl->done_wait);
883
+
}
884
+
885
+
static void zram_submit_wb_request(struct zram *zram,
886
+
struct zram_wb_ctl *wb_ctl,
887
+
struct zram_wb_req *req)
888
+
{
889
+
/*
890
+
* wb_limit (if enabled) should be adjusted before submission,
891
+
* so that we don't over-submit.
892
+
*/
893
+
zram_account_writeback_submit(zram);
894
+
atomic_inc(&wb_ctl->num_inflight);
895
+
req->bio.bi_private = wb_ctl;
896
+
submit_bio(&req->bio);
897
+
}
898
+
899
+
static int zram_complete_done_reqs(struct zram *zram,
900
+
struct zram_wb_ctl *wb_ctl)
901
+
{
902
+
struct zram_wb_req *req;
903
+
unsigned long flags;
904
+
int ret = 0, err;
905
+
906
+
while (atomic_read(&wb_ctl->num_inflight) > 0) {
907
+
spin_lock_irqsave(&wb_ctl->done_lock, flags);
908
+
req = list_first_entry_or_null(&wb_ctl->done_reqs,
909
+
struct zram_wb_req, entry);
910
+
if (req)
911
+
list_del(&req->entry);
912
+
spin_unlock_irqrestore(&wb_ctl->done_lock, flags);
913
+
914
+
/* ->num_inflight > 0 doesn't mean we have done requests */
915
+
if (!req)
916
+
break;
917
+
918
+
err = zram_writeback_complete(zram, req);
919
+
if (err)
920
+
ret = err;
921
+
922
+
atomic_dec(&wb_ctl->num_inflight);
923
+
release_pp_slot(zram, req->pps);
924
+
req->pps = NULL;
925
+
926
+
list_add(&req->entry, &wb_ctl->idle_reqs);
927
+
}
928
+
929
+
return ret;
930
+
}
931
+
932
+
static struct zram_wb_req *zram_select_idle_req(struct zram_wb_ctl *wb_ctl)
933
+
{
934
+
struct zram_wb_req *req;
935
+
936
+
req = list_first_entry_or_null(&wb_ctl->idle_reqs,
937
+
struct zram_wb_req, entry);
938
+
if (req)
939
+
list_del(&req->entry);
940
+
return req;
941
+
}
942
+
943
+
static int zram_writeback_slots(struct zram *zram,
944
+
struct zram_pp_ctl *ctl,
945
+
struct zram_wb_ctl *wb_ctl)
946
+
{
947
+
struct zram_wb_req *req = NULL;
948
+
unsigned long blk_idx = 0;
949
+
struct zram_pp_slot *pps;
950
+
int ret = 0, err = 0;
951
+
u32 index = 0;
770
952
771
953
while ((pps = select_pp_slot(ctl))) {
772
954
spin_lock(&zram->wb_limit_lock);
···
978
756
break;
979
757
}
980
758
spin_unlock(&zram->wb_limit_lock);
759
+
760
+
while (!req) {
761
+
req = zram_select_idle_req(wb_ctl);
762
+
if (req)
763
+
break;
764
+
765
+
wait_event(wb_ctl->done_wait,
766
+
!list_empty(&wb_ctl->done_reqs));
767
+
768
+
err = zram_complete_done_reqs(zram, wb_ctl);
769
+
/*
770
+
* BIO errors are not fatal, we continue and simply
771
+
* attempt to writeback the remaining objects (pages).
772
+
* At the same time we need to signal user-space that
773
+
* some writes (at least one, but also could be all of
774
+
* them) were not successful and we do so by returning
775
+
* the most recent BIO error.
776
+
*/
777
+
if (err)
778
+
ret = err;
779
+
}
981
780
982
781
if (!blk_idx) {
983
782
blk_idx = alloc_block_bdev(zram);
···
1018
775
*/
1019
776
if (!zram_test_flag(zram, index, ZRAM_PP_SLOT))
1020
777
goto next;
1021
-
if (zram_read_from_zspool(zram, page, index))
778
+
if (zram_read_from_zspool(zram, req->page, index))
1022
779
goto next;
1023
780
zram_slot_unlock(zram, index);
1024
781
1025
-
bio_init(&bio, zram->bdev, &bio_vec, 1,
1026
-
REQ_OP_WRITE | REQ_SYNC);
1027
-
bio.bi_iter.bi_sector = blk_idx * (PAGE_SIZE >> 9);
1028
-
__bio_add_page(&bio, page, PAGE_SIZE, 0);
1029
-
1030
782
/*
1031
-
* XXX: A single page IO would be inefficient for write
1032
-
* but it would be not bad as starter.
783
+
* From now on pp-slot is owned by the req, remove it from
784
+
* its pp bucket.
1033
785
*/
1034
-
err = submit_bio_wait(&bio);
1035
-
if (err) {
1036
-
release_pp_slot(zram, pps);
1037
-
/*
1038
-
* BIO errors are not fatal, we continue and simply
1039
-
* attempt to writeback the remaining objects (pages).
1040
-
* At the same time we need to signal user-space that
1041
-
* some writes (at least one, but also could be all of
1042
-
* them) were not successful and we do so by returning
1043
-
* the most recent BIO error.
1044
-
*/
1045
-
ret = err;
1046
-
continue;
1047
-
}
786
+
list_del_init(&pps->entry);
1048
787
1049
-
atomic64_inc(&zram->stats.bd_writes);
1050
-
zram_slot_lock(zram, index);
1051
-
/*
1052
-
* Same as above, we release slot lock during writeback so
1053
-
* slot can change under us: slot_free() or slot_free() and
1054
-
* reallocation (zram_write_page()). In both cases slot loses
1055
-
* ZRAM_PP_SLOT flag. No concurrent post-processing can set
1056
-
* ZRAM_PP_SLOT on such slots until current post-processing
1057
-
* finishes.
1058
-
*/
1059
-
if (!zram_test_flag(zram, index, ZRAM_PP_SLOT))
1060
-
goto next;
788
+
req->blk_idx = blk_idx;
789
+
req->pps = pps;
790
+
bio_init(&req->bio, zram->bdev, &req->bio_vec, 1, REQ_OP_WRITE);
791
+
req->bio.bi_iter.bi_sector = req->blk_idx * (PAGE_SIZE >> 9);
792
+
req->bio.bi_end_io = zram_writeback_endio;
793
+
__bio_add_page(&req->bio, req->page, PAGE_SIZE, 0);
1061
794
1062
-
zram_free_page(zram, index);
1063
-
zram_set_flag(zram, index, ZRAM_WB);
1064
-
zram_set_handle(zram, index, blk_idx);
795
+
zram_submit_wb_request(zram, wb_ctl, req);
1065
796
blk_idx = 0;
1066
-
atomic64_inc(&zram->stats.pages_stored);
1067
-
spin_lock(&zram->wb_limit_lock);
1068
-
if (zram->wb_limit_enable && zram->bd_wb_limit > 0)
1069
-
zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12);
1070
-
spin_unlock(&zram->wb_limit_lock);
797
+
req = NULL;
798
+
cond_resched();
799
+
continue;
800
+
1071
801
next:
1072
802
zram_slot_unlock(zram, index);
1073
803
release_pp_slot(zram, pps);
1074
-
1075
-
cond_resched();
1076
804
}
1077
805
1078
-
if (blk_idx)
1079
-
free_block_bdev(zram, blk_idx);
1080
-
if (page)
1081
-
__free_page(page);
806
+
/*
807
+
* Selected idle req, but never submitted it due to some error or
808
+
* wb limit.
809
+
*/
810
+
if (req)
811
+
release_wb_req(req);
812
+
813
+
while (atomic_read(&wb_ctl->num_inflight) > 0) {
814
+
wait_event(wb_ctl->done_wait, !list_empty(&wb_ctl->done_reqs));
815
+
err = zram_complete_done_reqs(zram, wb_ctl);
816
+
if (err)
817
+
ret = err;
818
+
}
1082
819
1083
820
return ret;
1084
821
}
···
1171
948
struct zram *zram = dev_to_zram(dev);
1172
949
u64 nr_pages = zram->disksize >> PAGE_SHIFT;
1173
950
unsigned long lo = 0, hi = nr_pages;
1174
-
struct zram_pp_ctl *ctl = NULL;
951
+
struct zram_pp_ctl *pp_ctl = NULL;
952
+
struct zram_wb_ctl *wb_ctl = NULL;
1175
953
char *args, *param, *val;
1176
954
ssize_t ret = len;
1177
955
int err, mode = 0;
···
1194
970
goto release_init_lock;
1195
971
}
1196
972
1197
-
ctl = init_pp_ctl();
1198
-
if (!ctl) {
973
+
pp_ctl = init_pp_ctl();
974
+
if (!pp_ctl) {
975
+
ret = -ENOMEM;
976
+
goto release_init_lock;
977
+
}
978
+
979
+
wb_ctl = init_wb_ctl();
980
+
if (!wb_ctl) {
1199
981
ret = -ENOMEM;
1200
982
goto release_init_lock;
1201
983
}
···
1230
1000
goto release_init_lock;
1231
1001
}
1232
1002
1233
-
scan_slots_for_writeback(zram, mode, lo, hi, ctl);
1003
+
scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);
1234
1004
break;
1235
1005
}
1236
1006
···
1241
1011
goto release_init_lock;
1242
1012
}
1243
1013
1244
-
scan_slots_for_writeback(zram, mode, lo, hi, ctl);
1014
+
scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);
1245
1015
break;
1246
1016
}
1247
1017
···
1252
1022
goto release_init_lock;
1253
1023
}
1254
1024
1255
-
scan_slots_for_writeback(zram, mode, lo, hi, ctl);
1025
+
scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);
1256
1026
continue;
1257
1027
}
1258
1028
···
1263
1033
goto release_init_lock;
1264
1034
}
1265
1035
1266
-
scan_slots_for_writeback(zram, mode, lo, hi, ctl);
1036
+
scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);
1267
1037
continue;
1268
1038
}
1269
1039
}
1270
1040
1271
-
err = zram_writeback_slots(zram, ctl);
1041
+
err = zram_writeback_slots(zram, pp_ctl, wb_ctl);
1272
1042
if (err)
1273
1043
ret = err;
1274
1044
1275
1045
release_init_lock:
1276
-
release_pp_ctl(zram, ctl);
1046
+
release_pp_ctl(zram, pp_ctl);
1047
+
release_wb_ctl(wb_ctl);
1277
1048
atomic_set(&zram->pp_in_progress, 0);
1278
1049
up_read(&zram->init_lock);
1279
1050
···
1343
1112
return -EIO;
1344
1113
}
1345
1114
1346
-
static void free_block_bdev(struct zram *zram, unsigned long blk_idx) {};
1115
+
static void free_block_bdev(struct zram *zram, unsigned long blk_idx)
1116
+
{
1117
+
}
1347
1118
#endif
1348
1119
1349
1120
#ifdef CONFIG_ZRAM_MEMORY_TRACKING