Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'for-linus-5.10c-rc8-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip
Pull xen fixes from Juergen Gross:
"A short series fixing a regression introduced in 5.9 for running as
Xen dom0 on a system with NVMe backed storage"
* tag 'for-linus-5.10c-rc8-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip:
xen: don't use page->lru for ZONE_DEVICE memory
xen: add helpers for caching grant mapping pages
+182
-137
+17
-72
drivers/block/xen-blkback/blkback.c
+17
-72
drivers/block/xen-blkback/blkback.c
···
132
132
133
133
#define BLKBACK_INVALID_HANDLE (~0)
134
134
135
-
/* Number of free pages to remove on each call to gnttab_free_pages */
136
-
#define NUM_BATCH_FREE_PAGES 10
137
-
138
135
static inline bool persistent_gnt_timeout(struct persistent_gnt *persistent_gnt)
139
136
{
140
137
return pgrant_timeout && (jiffies - persistent_gnt->last_used >=
141
138
HZ * pgrant_timeout);
142
-
}
143
-
144
-
static inline int get_free_page(struct xen_blkif_ring *ring, struct page **page)
145
-
{
146
-
unsigned long flags;
147
-
148
-
spin_lock_irqsave(&ring->free_pages_lock, flags);
149
-
if (list_empty(&ring->free_pages)) {
150
-
BUG_ON(ring->free_pages_num != 0);
151
-
spin_unlock_irqrestore(&ring->free_pages_lock, flags);
152
-
return gnttab_alloc_pages(1, page);
153
-
}
154
-
BUG_ON(ring->free_pages_num == 0);
155
-
page[0] = list_first_entry(&ring->free_pages, struct page, lru);
156
-
list_del(&page[0]->lru);
157
-
ring->free_pages_num--;
158
-
spin_unlock_irqrestore(&ring->free_pages_lock, flags);
159
-
160
-
return 0;
161
-
}
162
-
163
-
static inline void put_free_pages(struct xen_blkif_ring *ring, struct page **page,
164
-
int num)
165
-
{
166
-
unsigned long flags;
167
-
int i;
168
-
169
-
spin_lock_irqsave(&ring->free_pages_lock, flags);
170
-
for (i = 0; i < num; i++)
171
-
list_add(&page[i]->lru, &ring->free_pages);
172
-
ring->free_pages_num += num;
173
-
spin_unlock_irqrestore(&ring->free_pages_lock, flags);
174
-
}
175
-
176
-
static inline void shrink_free_pagepool(struct xen_blkif_ring *ring, int num)
177
-
{
178
-
/* Remove requested pages in batches of NUM_BATCH_FREE_PAGES */
179
-
struct page *page[NUM_BATCH_FREE_PAGES];
180
-
unsigned int num_pages = 0;
181
-
unsigned long flags;
182
-
183
-
spin_lock_irqsave(&ring->free_pages_lock, flags);
184
-
while (ring->free_pages_num > num) {
185
-
BUG_ON(list_empty(&ring->free_pages));
186
-
page[num_pages] = list_first_entry(&ring->free_pages,
187
-
struct page, lru);
188
-
list_del(&page[num_pages]->lru);
189
-
ring->free_pages_num--;
190
-
if (++num_pages == NUM_BATCH_FREE_PAGES) {
191
-
spin_unlock_irqrestore(&ring->free_pages_lock, flags);
192
-
gnttab_free_pages(num_pages, page);
193
-
spin_lock_irqsave(&ring->free_pages_lock, flags);
194
-
num_pages = 0;
195
-
}
196
-
}
197
-
spin_unlock_irqrestore(&ring->free_pages_lock, flags);
198
-
if (num_pages != 0)
199
-
gnttab_free_pages(num_pages, page);
200
139
}
201
140
202
141
#define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page)))
···
270
331
unmap_data.count = segs_to_unmap;
271
332
BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
272
333
273
-
put_free_pages(ring, pages, segs_to_unmap);
334
+
gnttab_page_cache_put(&ring->free_pages, pages,
335
+
segs_to_unmap);
274
336
segs_to_unmap = 0;
275
337
}
276
338
···
311
371
if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
312
372
unmap_data.count = segs_to_unmap;
313
373
BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
314
-
put_free_pages(ring, pages, segs_to_unmap);
374
+
gnttab_page_cache_put(&ring->free_pages, pages,
375
+
segs_to_unmap);
315
376
segs_to_unmap = 0;
316
377
}
317
378
kfree(persistent_gnt);
···
320
379
if (segs_to_unmap > 0) {
321
380
unmap_data.count = segs_to_unmap;
322
381
BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
323
-
put_free_pages(ring, pages, segs_to_unmap);
382
+
gnttab_page_cache_put(&ring->free_pages, pages, segs_to_unmap);
324
383
}
325
384
}
326
385
···
605
664
606
665
/* Shrink the free pages pool if it is too large. */
607
666
if (time_before(jiffies, blkif->buffer_squeeze_end))
608
-
shrink_free_pagepool(ring, 0);
667
+
gnttab_page_cache_shrink(&ring->free_pages, 0);
609
668
else
610
-
shrink_free_pagepool(ring, max_buffer_pages);
669
+
gnttab_page_cache_shrink(&ring->free_pages,
670
+
max_buffer_pages);
611
671
612
672
if (log_stats && time_after(jiffies, ring->st_print))
613
673
print_stats(ring);
···
639
697
ring->persistent_gnt_c = 0;
640
698
641
699
/* Since we are shutting down remove all pages from the buffer */
642
-
shrink_free_pagepool(ring, 0 /* All */);
700
+
gnttab_page_cache_shrink(&ring->free_pages, 0 /* All */);
643
701
}
644
702
645
703
static unsigned int xen_blkbk_unmap_prepare(
···
678
736
but is this the best way to deal with this? */
679
737
BUG_ON(result);
680
738
681
-
put_free_pages(ring, data->pages, data->count);
739
+
gnttab_page_cache_put(&ring->free_pages, data->pages, data->count);
682
740
make_response(ring, pending_req->id,
683
741
pending_req->operation, pending_req->status);
684
742
free_req(ring, pending_req);
···
745
803
if (invcount) {
746
804
ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount);
747
805
BUG_ON(ret);
748
-
put_free_pages(ring, unmap_pages, invcount);
806
+
gnttab_page_cache_put(&ring->free_pages, unmap_pages,
807
+
invcount);
749
808
}
750
809
pages += batch;
751
810
num -= batch;
···
793
850
pages[i]->page = persistent_gnt->page;
794
851
pages[i]->persistent_gnt = persistent_gnt;
795
852
} else {
796
-
if (get_free_page(ring, &pages[i]->page))
853
+
if (gnttab_page_cache_get(&ring->free_pages,
854
+
&pages[i]->page))
797
855
goto out_of_memory;
798
856
addr = vaddr(pages[i]->page);
799
857
pages_to_gnt[segs_to_map] = pages[i]->page;
···
827
883
BUG_ON(new_map_idx >= segs_to_map);
828
884
if (unlikely(map[new_map_idx].status != 0)) {
829
885
pr_debug("invalid buffer -- could not remap it\n");
830
-
put_free_pages(ring, &pages[seg_idx]->page, 1);
886
+
gnttab_page_cache_put(&ring->free_pages,
887
+
&pages[seg_idx]->page, 1);
831
888
pages[seg_idx]->handle = BLKBACK_INVALID_HANDLE;
832
889
ret |= 1;
833
890
goto next;
···
889
944
890
945
out_of_memory:
891
946
pr_alert("%s: out of memory\n", __func__);
892
-
put_free_pages(ring, pages_to_gnt, segs_to_map);
947
+
gnttab_page_cache_put(&ring->free_pages, pages_to_gnt, segs_to_map);
893
948
for (i = last_map; i < num; i++)
894
949
pages[i]->handle = BLKBACK_INVALID_HANDLE;
895
950
return -ENOMEM;
+1
-3
drivers/block/xen-blkback/common.h
+1
-3
drivers/block/xen-blkback/common.h
···
288
288
struct work_struct persistent_purge_work;
289
289
290
290
/* Buffer of free pages to map grant refs. */
291
-
spinlock_t free_pages_lock;
292
-
int free_pages_num;
293
-
struct list_head free_pages;
291
+
struct gnttab_page_cache free_pages;
294
292
295
293
struct work_struct free_work;
296
294
/* Thread shutdown wait queue. */
+2
-4
drivers/block/xen-blkback/xenbus.c
+2
-4
drivers/block/xen-blkback/xenbus.c
···
144
144
INIT_LIST_HEAD(&ring->pending_free);
145
145
INIT_LIST_HEAD(&ring->persistent_purge_list);
146
146
INIT_WORK(&ring->persistent_purge_work, xen_blkbk_unmap_purged_grants);
147
-
spin_lock_init(&ring->free_pages_lock);
148
-
INIT_LIST_HEAD(&ring->free_pages);
147
+
gnttab_page_cache_init(&ring->free_pages);
149
148
150
149
spin_lock_init(&ring->pending_free_lock);
151
150
init_waitqueue_head(&ring->pending_free_wq);
···
316
317
BUG_ON(atomic_read(&ring->persistent_gnt_in_use) != 0);
317
318
BUG_ON(!list_empty(&ring->persistent_purge_list));
318
319
BUG_ON(!RB_EMPTY_ROOT(&ring->persistent_gnts));
319
-
BUG_ON(!list_empty(&ring->free_pages));
320
-
BUG_ON(ring->free_pages_num != 0);
320
+
BUG_ON(ring->free_pages.num_pages != 0);
321
321
BUG_ON(ring->persistent_gnt_c != 0);
322
322
WARN_ON(i != (XEN_BLKIF_REQS_PER_PAGE * blkif->nr_ring_pages));
323
323
ring->active = false;
+123
drivers/xen/grant-table.c
+123
drivers/xen/grant-table.c
···
813
813
}
814
814
EXPORT_SYMBOL_GPL(gnttab_alloc_pages);
815
815
816
+
#ifdef CONFIG_XEN_UNPOPULATED_ALLOC
817
+
static inline void cache_init(struct gnttab_page_cache *cache)
818
+
{
819
+
cache->pages = NULL;
820
+
}
821
+
822
+
static inline bool cache_empty(struct gnttab_page_cache *cache)
823
+
{
824
+
return !cache->pages;
825
+
}
826
+
827
+
static inline struct page *cache_deq(struct gnttab_page_cache *cache)
828
+
{
829
+
struct page *page;
830
+
831
+
page = cache->pages;
832
+
cache->pages = page->zone_device_data;
833
+
834
+
return page;
835
+
}
836
+
837
+
static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)
838
+
{
839
+
page->zone_device_data = cache->pages;
840
+
cache->pages = page;
841
+
}
842
+
#else
843
+
static inline void cache_init(struct gnttab_page_cache *cache)
844
+
{
845
+
INIT_LIST_HEAD(&cache->pages);
846
+
}
847
+
848
+
static inline bool cache_empty(struct gnttab_page_cache *cache)
849
+
{
850
+
return list_empty(&cache->pages);
851
+
}
852
+
853
+
static inline struct page *cache_deq(struct gnttab_page_cache *cache)
854
+
{
855
+
struct page *page;
856
+
857
+
page = list_first_entry(&cache->pages, struct page, lru);
858
+
list_del(&page->lru);
859
+
860
+
return page;
861
+
}
862
+
863
+
static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)
864
+
{
865
+
list_add(&page->lru, &cache->pages);
866
+
}
867
+
#endif
868
+
869
+
void gnttab_page_cache_init(struct gnttab_page_cache *cache)
870
+
{
871
+
spin_lock_init(&cache->lock);
872
+
cache_init(cache);
873
+
cache->num_pages = 0;
874
+
}
875
+
EXPORT_SYMBOL_GPL(gnttab_page_cache_init);
876
+
877
+
int gnttab_page_cache_get(struct gnttab_page_cache *cache, struct page **page)
878
+
{
879
+
unsigned long flags;
880
+
881
+
spin_lock_irqsave(&cache->lock, flags);
882
+
883
+
if (cache_empty(cache)) {
884
+
spin_unlock_irqrestore(&cache->lock, flags);
885
+
return gnttab_alloc_pages(1, page);
886
+
}
887
+
888
+
page[0] = cache_deq(cache);
889
+
cache->num_pages--;
890
+
891
+
spin_unlock_irqrestore(&cache->lock, flags);
892
+
893
+
return 0;
894
+
}
895
+
EXPORT_SYMBOL_GPL(gnttab_page_cache_get);
896
+
897
+
void gnttab_page_cache_put(struct gnttab_page_cache *cache, struct page **page,
898
+
unsigned int num)
899
+
{
900
+
unsigned long flags;
901
+
unsigned int i;
902
+
903
+
spin_lock_irqsave(&cache->lock, flags);
904
+
905
+
for (i = 0; i < num; i++)
906
+
cache_enq(cache, page[i]);
907
+
cache->num_pages += num;
908
+
909
+
spin_unlock_irqrestore(&cache->lock, flags);
910
+
}
911
+
EXPORT_SYMBOL_GPL(gnttab_page_cache_put);
912
+
913
+
void gnttab_page_cache_shrink(struct gnttab_page_cache *cache, unsigned int num)
914
+
{
915
+
struct page *page[10];
916
+
unsigned int i = 0;
917
+
unsigned long flags;
918
+
919
+
spin_lock_irqsave(&cache->lock, flags);
920
+
921
+
while (cache->num_pages > num) {
922
+
page[i] = cache_deq(cache);
923
+
cache->num_pages--;
924
+
if (++i == ARRAY_SIZE(page)) {
925
+
spin_unlock_irqrestore(&cache->lock, flags);
926
+
gnttab_free_pages(i, page);
927
+
i = 0;
928
+
spin_lock_irqsave(&cache->lock, flags);
929
+
}
930
+
}
931
+
932
+
spin_unlock_irqrestore(&cache->lock, flags);
933
+
934
+
if (i != 0)
935
+
gnttab_free_pages(i, page);
936
+
}
937
+
EXPORT_SYMBOL_GPL(gnttab_page_cache_shrink);
938
+
816
939
void gnttab_pages_clear_private(int nr_pages, struct page **pages)
817
940
{
818
941
int i;
+11
-9
drivers/xen/unpopulated-alloc.c
+11
-9
drivers/xen/unpopulated-alloc.c
···
12
12
#include <xen/xen.h>
13
13
14
14
static DEFINE_MUTEX(list_lock);
15
-
static LIST_HEAD(page_list);
15
+
static struct page *page_list;
16
16
static unsigned int list_count;
17
17
18
18
static int fill_list(unsigned int nr_pages)
···
84
84
struct page *pg = virt_to_page(vaddr + PAGE_SIZE * i);
85
85
86
86
BUG_ON(!virt_addr_valid(vaddr + PAGE_SIZE * i));
87
-
list_add(&pg->lru, &page_list);
87
+
pg->zone_device_data = page_list;
88
+
page_list = pg;
88
89
list_count++;
89
90
}
90
91
···
119
118
}
120
119
121
120
for (i = 0; i < nr_pages; i++) {
122
-
struct page *pg = list_first_entry_or_null(&page_list,
123
-
struct page,
124
-
lru);
121
+
struct page *pg = page_list;
125
122
126
123
BUG_ON(!pg);
127
-
list_del(&pg->lru);
124
+
page_list = pg->zone_device_data;
128
125
list_count--;
129
126
pages[i] = pg;
130
127
···
133
134
unsigned int j;
134
135
135
136
for (j = 0; j <= i; j++) {
136
-
list_add(&pages[j]->lru, &page_list);
137
+
pages[j]->zone_device_data = page_list;
138
+
page_list = pages[j];
137
139
list_count++;
138
140
}
139
141
goto out;
···
160
160
161
161
mutex_lock(&list_lock);
162
162
for (i = 0; i < nr_pages; i++) {
163
-
list_add(&pages[i]->lru, &page_list);
163
+
pages[i]->zone_device_data = page_list;
164
+
page_list = pages[i];
164
165
list_count++;
165
166
}
166
167
mutex_unlock(&list_lock);
···
190
189
struct page *pg =
191
190
pfn_to_page(xen_extra_mem[i].start_pfn + j);
192
191
193
-
list_add(&pg->lru, &page_list);
192
+
pg->zone_device_data = page_list;
193
+
page_list = pg;
194
194
list_count++;
195
195
}
196
196
}
+11
-49
drivers/xen/xen-scsiback.c
+11
-49
drivers/xen/xen-scsiback.c
···
99
99
struct list_head v2p_entry_lists;
100
100
101
101
wait_queue_head_t waiting_to_free;
102
+
103
+
struct gnttab_page_cache free_pages;
102
104
};
103
105
104
106
/* theoretical maximum of grants for one request */
···
190
188
MODULE_PARM_DESC(max_buffer_pages,
191
189
"Maximum number of free pages to keep in backend buffer");
192
190
193
-
static DEFINE_SPINLOCK(free_pages_lock);
194
-
static int free_pages_num;
195
-
static LIST_HEAD(scsiback_free_pages);
196
-
197
191
/* Global spinlock to protect scsiback TPG list */
198
192
static DEFINE_MUTEX(scsiback_mutex);
199
193
static LIST_HEAD(scsiback_list);
···
203
205
{
204
206
if (atomic_dec_and_test(&info->nr_unreplied_reqs))
205
207
wake_up(&info->waiting_to_free);
206
-
}
207
-
208
-
static void put_free_pages(struct page **page, int num)
209
-
{
210
-
unsigned long flags;
211
-
int i = free_pages_num + num, n = num;
212
-
213
-
if (num == 0)
214
-
return;
215
-
if (i > scsiback_max_buffer_pages) {
216
-
n = min(num, i - scsiback_max_buffer_pages);
217
-
gnttab_free_pages(n, page + num - n);
218
-
n = num - n;
219
-
}
220
-
spin_lock_irqsave(&free_pages_lock, flags);
221
-
for (i = 0; i < n; i++)
222
-
list_add(&page[i]->lru, &scsiback_free_pages);
223
-
free_pages_num += n;
224
-
spin_unlock_irqrestore(&free_pages_lock, flags);
225
-
}
226
-
227
-
static int get_free_page(struct page **page)
228
-
{
229
-
unsigned long flags;
230
-
231
-
spin_lock_irqsave(&free_pages_lock, flags);
232
-
if (list_empty(&scsiback_free_pages)) {
233
-
spin_unlock_irqrestore(&free_pages_lock, flags);
234
-
return gnttab_alloc_pages(1, page);
235
-
}
236
-
page[0] = list_first_entry(&scsiback_free_pages, struct page, lru);
237
-
list_del(&page[0]->lru);
238
-
free_pages_num--;
239
-
spin_unlock_irqrestore(&free_pages_lock, flags);
240
-
return 0;
241
208
}
242
209
243
210
static unsigned long vaddr_page(struct page *page)
···
265
302
BUG_ON(err);
266
303
}
267
304
268
-
put_free_pages(req->pages, req->n_grants);
305
+
gnttab_page_cache_put(&req->info->free_pages, req->pages,
306
+
req->n_grants);
269
307
req->n_grants = 0;
270
308
}
271
309
···
409
445
struct vscsibk_info *info = pending_req->info;
410
446
411
447
for (i = 0; i < cnt; i++) {
412
-
if (get_free_page(pg + mapcount)) {
413
-
put_free_pages(pg, mapcount);
448
+
if (gnttab_page_cache_get(&info->free_pages, pg + mapcount)) {
449
+
gnttab_page_cache_put(&info->free_pages, pg, mapcount);
414
450
pr_err("no grant page\n");
415
451
return -ENOMEM;
416
452
}
···
759
795
/* Yield point for this unbounded loop. */
760
796
cond_resched();
761
797
}
798
+
799
+
gnttab_page_cache_shrink(&info->free_pages, scsiback_max_buffer_pages);
762
800
763
801
RING_FINAL_CHECK_FOR_REQUESTS(&info->ring, more_to_do);
764
802
return more_to_do;
···
1199
1233
1200
1234
scsiback_release_translation_entry(info);
1201
1235
1236
+
gnttab_page_cache_shrink(&info->free_pages, 0);
1237
+
1202
1238
dev_set_drvdata(&dev->dev, NULL);
1203
1239
1204
1240
return 0;
···
1231
1263
info->irq = 0;
1232
1264
INIT_LIST_HEAD(&info->v2p_entry_lists);
1233
1265
spin_lock_init(&info->v2p_lock);
1266
+
gnttab_page_cache_init(&info->free_pages);
1234
1267
1235
1268
err = xenbus_printf(XBT_NIL, dev->nodename, "feature-sg-grant", "%u",
1236
1269
SG_ALL);
···
1848
1879
1849
1880
static void __exit scsiback_exit(void)
1850
1881
{
1851
-
struct page *page;
1852
-
1853
-
while (free_pages_num) {
1854
-
if (get_free_page(&page))
1855
-
BUG();
1856
-
gnttab_free_pages(1, &page);
1857
-
}
1858
1882
target_unregister_template(&scsiback_ops);
1859
1883
xenbus_unregister_driver(&scsiback_driver);
1860
1884
}
+17
include/xen/grant_table.h
+17
include/xen/grant_table.h
···
198
198
int gnttab_alloc_pages(int nr_pages, struct page **pages);
199
199
void gnttab_free_pages(int nr_pages, struct page **pages);
200
200
201
+
struct gnttab_page_cache {
202
+
spinlock_t lock;
203
+
#ifdef CONFIG_XEN_UNPOPULATED_ALLOC
204
+
struct page *pages;
205
+
#else
206
+
struct list_head pages;
207
+
#endif
208
+
unsigned int num_pages;
209
+
};
210
+
211
+
void gnttab_page_cache_init(struct gnttab_page_cache *cache);
212
+
int gnttab_page_cache_get(struct gnttab_page_cache *cache, struct page **page);
213
+
void gnttab_page_cache_put(struct gnttab_page_cache *cache, struct page **page,
214
+
unsigned int num);
215
+
void gnttab_page_cache_shrink(struct gnttab_page_cache *cache,
216
+
unsigned int num);
217
+
201
218
#ifdef CONFIG_XEN_GRANT_DMA_ALLOC
202
219
struct gnttab_dma_alloc_args {
203
220
/* Device for which DMA memory will be/was allocated. */