ublk: move cold paths out of __ublk_batch_dispatch() for icache efficiency

Mark ublk_filter_unused_tags() as noinline since it is only called from
the unlikely(needs_filter) branch. Extract the error-handling block from
__ublk_batch_dispatch() into a new noinline ublk_batch_dispatch_fail()
function to keep the hot path compact and icache-friendly. This also
makes __ublk_batch_dispatch() more readable by separating the error
recovery logic from the normal dispatch flow.

Before: __ublk_batch_dispatch is ~1419 bytes
After: __ublk_batch_dispatch is ~1090 bytes (-329 bytes, -23%)

Signed-off-by: Ming Lei <ming.lei@redhat.com>
Link: https://patch.msgid.link/20260318014112.3125432-1-ming.lei@redhat.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Ming Lei and committed by

Jens Axboe 1 month ago 24d4c902 713db70d

+38 -32

1 changed file

expand all

drivers

block

ublk_drv.c

+38 -32

drivers/block/ublk_drv.c

··· 1789 1789 * Filter out UBLK_BATCH_IO_UNUSED_TAG entries from tag_buf. 1790 1790 * Returns the new length after filtering. 1791 1791 */ 1792 - static unsigned int ublk_filter_unused_tags(unsigned short *tag_buf, 1792 + static noinline unsigned int ublk_filter_unused_tags(unsigned short *tag_buf, 1793 1793 unsigned int len) 1794 1794 { 1795 1795 unsigned int i, j; ··· 1803 1803 } 1804 1804 1805 1805 return j; 1806 + } 1807 + 1808 + static noinline void ublk_batch_dispatch_fail(struct ublk_queue *ubq, 1809 + const struct ublk_batch_io_data *data, 1810 + unsigned short *tag_buf, size_t len, int ret) 1811 + { 1812 + int i, res; 1813 + 1814 + /* 1815 + * Undo prep state for all IOs since userspace never received them. 1816 + * This restores IOs to pre-prepared state so they can be cleanly 1817 + * re-prepared when tags are pulled from FIFO again. 1818 + */ 1819 + for (i = 0; i < len; i++) { 1820 + struct ublk_io *io = &ubq->ios[tag_buf[i]]; 1821 + int index = -1; 1822 + 1823 + ublk_io_lock(io); 1824 + if (io->flags & UBLK_IO_FLAG_AUTO_BUF_REG) 1825 + index = io->buf.auto_reg.index; 1826 + io->flags &= ~(UBLK_IO_FLAG_OWNED_BY_SRV | UBLK_IO_FLAG_AUTO_BUF_REG); 1827 + io->flags |= UBLK_IO_FLAG_ACTIVE; 1828 + ublk_io_unlock(io); 1829 + 1830 + if (index != -1) 1831 + io_buffer_unregister_bvec(data->cmd, index, 1832 + data->issue_flags); 1833 + } 1834 + 1835 + res = kfifo_in_spinlocked_noirqsave(&ubq->evts_fifo, 1836 + tag_buf, len, &ubq->evts_lock); 1837 + 1838 + pr_warn_ratelimited("%s: copy tags or post CQE failure, move back " 1839 + "tags(%d %zu) ret %d\n", __func__, res, len, 1840 + ret); 1806 1841 } 1807 1842 1808 1843 #define MAX_NR_TAG 128 ··· 1883 1848 1884 1849 sel.val = ublk_batch_copy_io_tags(fcmd, sel.addr, tag_buf, len * tag_sz); 1885 1850 ret = ublk_batch_fetch_post_cqe(fcmd, &sel, data->issue_flags); 1886 - if (unlikely(ret < 0)) { 1887 - int i, res; 1888 - 1889 - /* 1890 - * Undo prep state for all IOs since userspace never received them. 1891 - * This restores IOs to pre-prepared state so they can be cleanly 1892 - * re-prepared when tags are pulled from FIFO again. 1893 - */ 1894 - for (i = 0; i < len; i++) { 1895 - struct ublk_io *io = &ubq->ios[tag_buf[i]]; 1896 - int index = -1; 1897 - 1898 - ublk_io_lock(io); 1899 - if (io->flags & UBLK_IO_FLAG_AUTO_BUF_REG) 1900 - index = io->buf.auto_reg.index; 1901 - io->flags &= ~(UBLK_IO_FLAG_OWNED_BY_SRV | UBLK_IO_FLAG_AUTO_BUF_REG); 1902 - io->flags |= UBLK_IO_FLAG_ACTIVE; 1903 - ublk_io_unlock(io); 1904 - 1905 - if (index != -1) 1906 - io_buffer_unregister_bvec(data->cmd, index, 1907 - data->issue_flags); 1908 - } 1909 - 1910 - res = kfifo_in_spinlocked_noirqsave(&ubq->evts_fifo, 1911 - tag_buf, len, &ubq->evts_lock); 1912 - 1913 - pr_warn_ratelimited("%s: copy tags or post CQE failure, move back " 1914 - "tags(%d %zu) ret %d\n", __func__, res, len, 1915 - ret); 1916 - } 1851 + if (unlikely(ret < 0)) 1852 + ublk_batch_dispatch_fail(ubq, data, tag_buf, len, ret); 1917 1853 return ret; 1918 1854 } 1919 1855

Configure Feed

Configure Feed