iov_iter: extract a iov_iter_extract_bvecs helper from bio code

+18 -102

block/bio.c

··· 1172 1172 bio_set_flag(bio, BIO_CLONED); 1173 1173 } 1174 1174 1175 - static unsigned int get_contig_folio_len(struct page **pages, 1176 - unsigned int *num_pages, size_t left, 1177 - size_t offset) 1178 - { 1179 - struct folio *folio = page_folio(pages[0]); 1180 - size_t contig_sz = min_t(size_t, PAGE_SIZE - offset, left); 1181 - unsigned int max_pages, i; 1182 - size_t folio_offset, len; 1183 - 1184 - folio_offset = PAGE_SIZE * folio_page_idx(folio, pages[0]) + offset; 1185 - len = min(folio_size(folio) - folio_offset, left); 1186 - 1187 - /* 1188 - * We might COW a single page in the middle of a large folio, so we have 1189 - * to check that all pages belong to the same folio. 1190 - */ 1191 - left -= contig_sz; 1192 - max_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); 1193 - for (i = 1; i < max_pages; i++) { 1194 - size_t next = min_t(size_t, PAGE_SIZE, left); 1195 - 1196 - if (page_folio(pages[i]) != folio || 1197 - pages[i] != pages[i - 1] + 1) 1198 - break; 1199 - contig_sz += next; 1200 - left -= next; 1201 - } 1202 - 1203 - *num_pages = i; 1204 - return contig_sz; 1205 - } 1206 - 1207 - #define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) 1208 - 1209 - /** 1210 - * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio 1211 - * @bio: bio to add pages to 1212 - * @iter: iov iterator describing the region to be mapped 1213 - * 1214 - * Extracts pages from *iter and appends them to @bio's bvec array. The pages 1215 - * will have to be cleaned up in the way indicated by the BIO_PAGE_PINNED flag. 1216 - * For a multi-segment *iter, this function only adds pages from the next 1217 - * non-empty segment of the iov iterator. 1218 - */ 1219 - static ssize_t __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) 1220 - { 1221 - iov_iter_extraction_t extraction_flags = 0; 1222 - unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; 1223 - unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; 1224 - struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; 1225 - struct page **pages = (struct page **)bv; 1226 - ssize_t size; 1227 - unsigned int i = 0; 1228 - size_t offset, left, len; 1229 - 1230 - /* 1231 - * Move page array up in the allocated memory for the bio vecs as far as 1232 - * possible so that we can start filling biovecs from the beginning 1233 - * without overwriting the temporary page array. 1234 - */ 1235 - BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2); 1236 - pages += entries_left * (PAGE_PTRS_PER_BVEC - 1); 1237 - 1238 - if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) 1239 - extraction_flags |= ITER_ALLOW_P2PDMA; 1240 - 1241 - size = iov_iter_extract_pages(iter, &pages, 1242 - BIO_MAX_SIZE - bio->bi_iter.bi_size, 1243 - nr_pages, extraction_flags, &offset); 1244 - if (unlikely(size <= 0)) 1245 - return size ? size : -EFAULT; 1246 - 1247 - nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE); 1248 - for (left = size; left > 0; left -= len) { 1249 - unsigned int nr_to_add; 1250 - 1251 - if (bio->bi_vcnt > 0) { 1252 - struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1]; 1253 - 1254 - if (!zone_device_pages_have_same_pgmap(prev->bv_page, 1255 - pages[i])) 1256 - break; 1257 - } 1258 - 1259 - len = get_contig_folio_len(&pages[i], &nr_to_add, left, offset); 1260 - __bio_add_page(bio, pages[i], len, offset); 1261 - i += nr_to_add; 1262 - offset = 0; 1263 - } 1264 - 1265 - iov_iter_revert(iter, left); 1266 - while (i < nr_pages) 1267 - bio_release_page(bio, pages[i++]); 1268 - return size - left; 1269 - } 1270 - 1271 1175 /* 1272 1176 * Aligns the bio size to the len_align_mask, releasing excessive bio vecs that 1273 1177 * __bio_iov_iter_get_pages may have inserted, and reverts the trimmed length ··· 1229 1325 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, 1230 1326 unsigned len_align_mask) 1231 1327 { 1232 - ssize_t ret; 1328 + iov_iter_extraction_t flags = 0; 1233 1329 1234 1330 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) 1235 1331 return -EIO; ··· 1242 1338 1243 1339 if (iov_iter_extract_will_pin(iter)) 1244 1340 bio_set_flag(bio, BIO_PAGE_PINNED); 1341 + if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) 1342 + flags |= ITER_ALLOW_P2PDMA; 1245 1343 1246 1344 do { 1247 - ret = __bio_iov_iter_get_pages(bio, iter); 1248 - } while (ret > 0 && iov_iter_count(iter) && !bio_full(bio, 0)); 1345 + ssize_t ret; 1249 1346 1250 - if (bio->bi_vcnt) 1251 - return bio_iov_iter_align_down(bio, iter, len_align_mask); 1252 - return ret; 1347 + ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec, 1348 + BIO_MAX_SIZE - bio->bi_iter.bi_size, 1349 + &bio->bi_vcnt, bio->bi_max_vecs, flags); 1350 + if (ret <= 0) { 1351 + if (!bio->bi_vcnt) 1352 + return ret; 1353 + break; 1354 + } 1355 + bio->bi_iter.bi_size += ret; 1356 + } while (iov_iter_count(iter) && !bio_full(bio, 0)); 1357 + 1358 + if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page)) 1359 + bio->bi_opf |= REQ_NOMERGE; 1360 + return bio_iov_iter_align_down(bio, iter, len_align_mask); 1253 1361 } 1254 1362 1255 1363 static void submit_bio_wait_endio(struct bio *bio)

+3

include/linux/uio.h

··· 389 389 size_t maxsize, unsigned int maxpages, 390 390 iov_iter_extraction_t extraction_flags, 391 391 size_t *offset0); 392 + ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, 393 + size_t max_size, unsigned short *nr_vecs, 394 + unsigned short max_vecs, iov_iter_extraction_t extraction_flags); 392 395 393 396 /** 394 397 * iov_iter_extract_will_pin - Indicate how pages from the iterator will be retained

+98

lib/iov_iter.c

··· 1845 1845 return -EFAULT; 1846 1846 } 1847 1847 EXPORT_SYMBOL_GPL(iov_iter_extract_pages); 1848 + 1849 + static unsigned int get_contig_folio_len(struct page **pages, 1850 + unsigned int *num_pages, size_t left, size_t offset) 1851 + { 1852 + struct folio *folio = page_folio(pages[0]); 1853 + size_t contig_sz = min_t(size_t, PAGE_SIZE - offset, left); 1854 + unsigned int max_pages, i; 1855 + size_t folio_offset, len; 1856 + 1857 + folio_offset = PAGE_SIZE * folio_page_idx(folio, pages[0]) + offset; 1858 + len = min(folio_size(folio) - folio_offset, left); 1859 + 1860 + /* 1861 + * We might COW a single page in the middle of a large folio, so we have 1862 + * to check that all pages belong to the same folio. 1863 + */ 1864 + left -= contig_sz; 1865 + max_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); 1866 + for (i = 1; i < max_pages; i++) { 1867 + size_t next = min_t(size_t, PAGE_SIZE, left); 1868 + 1869 + if (page_folio(pages[i]) != folio || 1870 + pages[i] != pages[i - 1] + 1) 1871 + break; 1872 + contig_sz += next; 1873 + left -= next; 1874 + } 1875 + 1876 + *num_pages = i; 1877 + return contig_sz; 1878 + } 1879 + 1880 + #define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) 1881 + 1882 + /** 1883 + * iov_iter_extract_bvecs - Extract bvecs from an iterator 1884 + * @iter: the iterator to extract from 1885 + * @bv: bvec return array 1886 + * @max_size: maximum size to extract from @iter 1887 + * @nr_vecs: number of vectors in @bv (on in and output) 1888 + * @max_vecs: maximum vectors in @bv, including those filled before calling 1889 + * @extraction_flags: flags to qualify request 1890 + * 1891 + * Like iov_iter_extract_pages(), but returns physically contiguous ranges 1892 + * contained in a single folio as a single bvec instead of multiple entries. 1893 + * 1894 + * Returns the number of bytes extracted when successful, or a negative errno. 1895 + * If @nr_vecs was non-zero on entry, the number of successfully extracted bytes 1896 + * can be 0. 1897 + */ 1898 + ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, 1899 + size_t max_size, unsigned short *nr_vecs, 1900 + unsigned short max_vecs, iov_iter_extraction_t extraction_flags) 1901 + { 1902 + unsigned short entries_left = max_vecs - *nr_vecs; 1903 + unsigned short nr_pages, i = 0; 1904 + size_t left, offset, len; 1905 + struct page **pages; 1906 + ssize_t size; 1907 + 1908 + /* 1909 + * Move page array up in the allocated memory for the bio vecs as far as 1910 + * possible so that we can start filling biovecs from the beginning 1911 + * without overwriting the temporary page array. 1912 + */ 1913 + BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2); 1914 + pages = (struct page **)(bv + *nr_vecs) + 1915 + entries_left * (PAGE_PTRS_PER_BVEC - 1); 1916 + 1917 + size = iov_iter_extract_pages(iter, &pages, max_size, entries_left, 1918 + extraction_flags, &offset); 1919 + if (unlikely(size <= 0)) 1920 + return size ? size : -EFAULT; 1921 + 1922 + nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE); 1923 + for (left = size; left > 0; left -= len) { 1924 + unsigned int nr_to_add; 1925 + 1926 + if (*nr_vecs > 0 && 1927 + !zone_device_pages_have_same_pgmap(bv[*nr_vecs - 1].bv_page, 1928 + pages[i])) 1929 + break; 1930 + 1931 + len = get_contig_folio_len(&pages[i], &nr_to_add, left, offset); 1932 + bvec_set_page(&bv[*nr_vecs], pages[i], len, offset); 1933 + i += nr_to_add; 1934 + (*nr_vecs)++; 1935 + offset = 0; 1936 + } 1937 + 1938 + iov_iter_revert(iter, left); 1939 + if (iov_iter_extract_will_pin(iter)) { 1940 + while (i < nr_pages) 1941 + unpin_user_page(pages[i++]); 1942 + } 1943 + return size - left; 1944 + } 1945 + EXPORT_SYMBOL_GPL(iov_iter_extract_bvecs);

Configure Feed

Configure Feed