virtio_ring: add in order support · tjh.dev/kernel@f6a15d8

+430 -18

1 changed file

expand all

drivers

virtio

virtio_ring.c

+430 -18

drivers/virtio/virtio_ring.c

··· 70 70 enum vq_layout { 71 71 VQ_LAYOUT_SPLIT = 0, 72 72 VQ_LAYOUT_PACKED, 73 + VQ_LAYOUT_SPLIT_IN_ORDER, 74 + VQ_LAYOUT_PACKED_IN_ORDER, 73 75 }; 74 76 75 77 struct vring_desc_state_split { ··· 81 79 * allocated together. So we won't stress more to the memory allocator. 82 80 */ 83 81 struct vring_desc *indir_desc; 82 + u32 total_in_len; 84 83 }; 85 84 86 85 struct vring_desc_state_packed { ··· 93 90 struct vring_packed_desc *indir_desc; 94 91 u16 num; /* Descriptor list length. */ 95 92 u16 last; /* The last desc state in a list. */ 93 + u32 total_in_len; /* In length for the skipped buffer. */ 96 94 }; 97 95 98 96 struct vring_desc_extra { ··· 208 204 209 205 enum vq_layout layout; 210 206 211 - /* Head of free buffer list. */ 207 + /* 208 + * Without IN_ORDER it's the head of free buffer list. With 209 + * IN_ORDER and SPLIT, it's the next available buffer 210 + * index. With IN_ORDER and PACKED, it's unused. 211 + */ 212 212 unsigned int free_head; 213 + 214 + /* 215 + * With IN_ORDER, once we see an in-order batch, this stores 216 + * this last entry, and until we return the last buffer. 217 + * After this, id is set to UINT_MAX to mark it invalid. 218 + * Unused without IN_ORDER. 219 + */ 220 + struct used_entry { 221 + u32 id; 222 + u32 len; 223 + } batch_last; 224 + 213 225 /* Number we've added since last sync. */ 214 226 unsigned int num_added; 215 227 ··· 236 216 * bits from VRING_PACKED_EVENT_F_WRAP_CTR include the used wrap counter. 237 217 */ 238 218 u16 last_used_idx; 219 + 220 + /* With IN_ORDER and SPLIT, last descriptor id we used to 221 + * detach buffer. 222 + */ 223 + u16 last_used; 239 224 240 225 /* Hint for event idx: already triggered no need to disable. */ 241 226 bool event_triggered; ··· 283 258 284 259 static inline bool virtqueue_is_packed(const struct vring_virtqueue *vq) 285 260 { 286 - return vq->layout == VQ_LAYOUT_PACKED; 261 + return vq->layout == VQ_LAYOUT_PACKED || 262 + vq->layout == VQ_LAYOUT_PACKED_IN_ORDER; 263 + } 264 + 265 + static inline bool virtqueue_is_in_order(const struct vring_virtqueue *vq) 266 + { 267 + return vq->layout == VQ_LAYOUT_SPLIT_IN_ORDER || 268 + vq->layout == VQ_LAYOUT_PACKED_IN_ORDER; 287 269 } 288 270 289 271 static bool virtqueue_use_indirect(const struct vring_virtqueue *vq, ··· 500 468 else 501 469 vq->last_used_idx = 0; 502 470 471 + vq->last_used = 0; 472 + 503 473 vq->event_triggered = false; 504 474 vq->num_added = 0; 505 475 ··· 609 575 struct scatterlist *sg; 610 576 struct vring_desc *desc; 611 577 unsigned int i, n, avail, descs_used, err_idx, sg_count = 0; 578 + /* Total length for in-order */ 579 + unsigned int total_in_len = 0; 612 580 int head; 613 581 bool indirect; 614 582 ··· 702 666 */ 703 667 i = virtqueue_add_desc_split(vq, desc, extra, i, addr, 704 668 len, flags, premapped); 669 + total_in_len += len; 705 670 } 706 671 } 707 672 ··· 725 688 vq->vq.num_free -= descs_used; 726 689 727 690 /* Update free pointer */ 728 - if (indirect) 691 + if (virtqueue_is_in_order(vq)) { 692 + vq->free_head += descs_used; 693 + if (vq->free_head >= vq->split.vring.num) 694 + vq->free_head -= vq->split.vring.num; 695 + vq->split.desc_state[head].total_in_len = total_in_len; 696 + } else if (indirect) 729 697 vq->free_head = vq->split.desc_extra[head].next; 730 698 else 731 699 vq->free_head = i; ··· 902 860 return virtqueue_poll_split(vq, vq->last_used_idx); 903 861 } 904 862 863 + static bool more_used_split_in_order(const struct vring_virtqueue *vq) 864 + { 865 + if (vq->batch_last.id != UINT_MAX) 866 + return true; 867 + 868 + return virtqueue_poll_split(vq, vq->last_used_idx); 869 + } 870 + 905 871 static void *virtqueue_get_buf_ctx_split(struct vring_virtqueue *vq, 906 872 unsigned int *len, 907 873 void **ctx) ··· 953 903 ret = vq->split.desc_state[i].data; 954 904 detach_buf_split(vq, i, ctx); 955 905 vq->last_used_idx++; 906 + /* If we expect an interrupt for the next entry, tell host 907 + * by writing event index and flush out the write before 908 + * the read in the next get_buf call. */ 909 + if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) 910 + virtio_store_mb(vq->weak_barriers, 911 + &vring_used_event(&vq->split.vring), 912 + cpu_to_virtio16(vq->vq.vdev, vq->last_used_idx)); 913 + 914 + LAST_ADD_TIME_INVALID(vq); 915 + 916 + END_USE(vq); 917 + return ret; 918 + } 919 + 920 + static void *virtqueue_get_buf_ctx_split_in_order(struct vring_virtqueue *vq, 921 + unsigned int *len, 922 + void **ctx) 923 + { 924 + void *ret; 925 + unsigned int num = vq->split.vring.num; 926 + unsigned int num_free = vq->vq.num_free; 927 + u16 last_used, last_used_idx; 928 + 929 + START_USE(vq); 930 + 931 + if (unlikely(vq->broken)) { 932 + END_USE(vq); 933 + return NULL; 934 + } 935 + 936 + last_used = vq->last_used & (num - 1); 937 + last_used_idx = vq->last_used_idx & (num - 1); 938 + 939 + if (vq->batch_last.id == UINT_MAX) { 940 + if (!more_used_split_in_order(vq)) { 941 + pr_debug("No more buffers in queue\n"); 942 + END_USE(vq); 943 + return NULL; 944 + } 945 + 946 + /* 947 + * Only get used array entries after they have been 948 + * exposed by host. 949 + */ 950 + virtio_rmb(vq->weak_barriers); 951 + 952 + vq->batch_last.id = virtio32_to_cpu(vq->vq.vdev, 953 + vq->split.vring.used->ring[last_used_idx].id); 954 + vq->batch_last.len = virtio32_to_cpu(vq->vq.vdev, 955 + vq->split.vring.used->ring[last_used_idx].len); 956 + } 957 + 958 + if (vq->batch_last.id == last_used) { 959 + vq->batch_last.id = UINT_MAX; 960 + *len = vq->batch_last.len; 961 + } else { 962 + *len = vq->split.desc_state[last_used].total_in_len; 963 + } 964 + 965 + if (unlikely(!vq->split.desc_state[last_used].data)) { 966 + BAD_RING(vq, "id %u is not a head!\n", last_used); 967 + return NULL; 968 + } 969 + 970 + /* detach_buf_split clears data, so grab it now. */ 971 + ret = vq->split.desc_state[last_used].data; 972 + detach_buf_split_in_order(vq, last_used, ctx); 973 + 974 + vq->last_used_idx++; 975 + vq->last_used += (vq->vq.num_free - num_free); 956 976 /* If we expect an interrupt for the next entry, tell host 957 977 * by writing event index and flush out the write before 958 978 * the read in the next get_buf call. */ ··· 1130 1010 continue; 1131 1011 /* detach_buf_split clears data, so grab it now. */ 1132 1012 buf = vq->split.desc_state[i].data; 1133 - detach_buf_split(vq, i, NULL); 1013 + if (virtqueue_is_in_order(vq)) 1014 + detach_buf_split_in_order(vq, i, NULL); 1015 + else 1016 + detach_buf_split(vq, i, NULL); 1134 1017 vq->split.avail_idx_shadow--; 1135 1018 vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev, 1136 1019 vq->split.avail_idx_shadow); ··· 1196 1073 1197 1074 /* Put everything in free lists. */ 1198 1075 vq->free_head = 0; 1076 + vq->batch_last.id = UINT_MAX; 1199 1077 } 1200 1078 1201 1079 static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split) ··· 1308 1184 if (!vq) 1309 1185 return NULL; 1310 1186 1311 - vq->layout = VQ_LAYOUT_SPLIT; 1312 1187 vq->vq.callback = callback; 1313 1188 vq->vq.vdev = vdev; 1314 1189 vq->vq.name = name; ··· 1327 1204 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && 1328 1205 !context; 1329 1206 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); 1207 + vq->layout = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ? 1208 + VQ_LAYOUT_SPLIT_IN_ORDER : VQ_LAYOUT_SPLIT; 1330 1209 1331 1210 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) 1332 1211 vq->weak_barriers = false; ··· 1486 1361 unsigned int in_sgs, 1487 1362 void *data, 1488 1363 bool premapped, 1489 - gfp_t gfp) 1364 + gfp_t gfp, 1365 + u16 id) 1490 1366 { 1491 1367 struct vring_desc_extra *extra; 1492 1368 struct vring_packed_desc *desc; 1493 1369 struct scatterlist *sg; 1494 - unsigned int i, n, err_idx, len; 1495 - u16 head, id; 1370 + unsigned int i, n, err_idx, len, total_in_len = 0; 1371 + u16 head; 1496 1372 dma_addr_t addr; 1497 1373 1498 1374 head = vq->packed.next_avail_idx; ··· 1511 1385 } 1512 1386 1513 1387 i = 0; 1514 - id = vq->free_head; 1515 - BUG_ON(id == vq->packed.vring.num); 1516 1388 1517 1389 for (n = 0; n < out_sgs + in_sgs; n++) { 1518 1390 for (sg = sgs[n]; sg; sg = sg_next(sg)) { ··· 1530 1406 extra[i].flags = n < out_sgs ? 0 : VRING_DESC_F_WRITE; 1531 1407 } 1532 1408 1409 + if (n >= out_sgs) 1410 + total_in_len += len; 1533 1411 i++; 1534 1412 } 1535 1413 } ··· 1578 1452 1 << VRING_PACKED_DESC_F_USED; 1579 1453 } 1580 1454 vq->packed.next_avail_idx = n; 1581 - vq->free_head = vq->packed.desc_extra[id].next; 1455 + if (!virtqueue_is_in_order(vq)) 1456 + vq->free_head = vq->packed.desc_extra[id].next; 1582 1457 1583 1458 /* Store token and indirect buffer state. */ 1584 1459 vq->packed.desc_state[id].num = 1; 1585 1460 vq->packed.desc_state[id].data = data; 1586 1461 vq->packed.desc_state[id].indir_desc = desc; 1587 1462 vq->packed.desc_state[id].last = id; 1463 + vq->packed.desc_state[id].total_in_len = total_in_len; 1588 1464 1589 1465 vq->num_added += 1; 1590 1466 ··· 1639 1511 BUG_ON(total_sg == 0); 1640 1512 1641 1513 if (virtqueue_use_indirect(vq, total_sg)) { 1514 + id = vq->free_head; 1515 + BUG_ON(id == vq->packed.vring.num); 1642 1516 err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs, 1643 - in_sgs, data, premapped, gfp); 1517 + in_sgs, data, premapped, 1518 + gfp, id); 1644 1519 if (err != -ENOMEM) { 1645 1520 END_USE(vq); 1646 1521 return err; ··· 1755 1624 break; 1756 1625 vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]); 1757 1626 curr = vq->packed.desc_extra[curr].next; 1627 + i++; 1628 + if (i >= vq->packed.vring.num) 1629 + i = 0; 1630 + } 1631 + 1632 + END_USE(vq); 1633 + return -EIO; 1634 + } 1635 + 1636 + static inline int virtqueue_add_packed_in_order(struct vring_virtqueue *vq, 1637 + struct scatterlist *sgs[], 1638 + unsigned int total_sg, 1639 + unsigned int out_sgs, 1640 + unsigned int in_sgs, 1641 + void *data, 1642 + void *ctx, 1643 + bool premapped, 1644 + gfp_t gfp) 1645 + { 1646 + struct vring_packed_desc *desc; 1647 + struct scatterlist *sg; 1648 + unsigned int i, n, sg_count, err_idx, total_in_len = 0; 1649 + __le16 head_flags, flags; 1650 + u16 head, avail_used_flags; 1651 + bool avail_wrap_counter; 1652 + int err; 1653 + 1654 + START_USE(vq); 1655 + 1656 + BUG_ON(data == NULL); 1657 + BUG_ON(ctx && vq->indirect); 1658 + 1659 + if (unlikely(vq->broken)) { 1660 + END_USE(vq); 1661 + return -EIO; 1662 + } 1663 + 1664 + LAST_ADD_TIME_UPDATE(vq); 1665 + 1666 + BUG_ON(total_sg == 0); 1667 + 1668 + if (virtqueue_use_indirect(vq, total_sg)) { 1669 + err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs, 1670 + in_sgs, data, premapped, gfp, 1671 + vq->packed.next_avail_idx); 1672 + if (err != -ENOMEM) { 1673 + END_USE(vq); 1674 + return err; 1675 + } 1676 + 1677 + /* fall back on direct */ 1678 + } 1679 + 1680 + head = vq->packed.next_avail_idx; 1681 + avail_used_flags = vq->packed.avail_used_flags; 1682 + avail_wrap_counter = vq->packed.avail_wrap_counter; 1683 + 1684 + WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect); 1685 + 1686 + desc = vq->packed.vring.desc; 1687 + i = head; 1688 + 1689 + if (unlikely(vq->vq.num_free < total_sg)) { 1690 + pr_debug("Can't add buf len %i - avail = %i\n", 1691 + total_sg, vq->vq.num_free); 1692 + END_USE(vq); 1693 + return -ENOSPC; 1694 + } 1695 + 1696 + sg_count = 0; 1697 + for (n = 0; n < out_sgs + in_sgs; n++) { 1698 + for (sg = sgs[n]; sg; sg = sg_next(sg)) { 1699 + dma_addr_t addr; 1700 + u32 len; 1701 + 1702 + flags = 0; 1703 + if (++sg_count != total_sg) 1704 + flags |= cpu_to_le16(VRING_DESC_F_NEXT); 1705 + if (n >= out_sgs) 1706 + flags |= cpu_to_le16(VRING_DESC_F_WRITE); 1707 + 1708 + if (vring_map_one_sg(vq, sg, n < out_sgs ? 1709 + DMA_TO_DEVICE : DMA_FROM_DEVICE, 1710 + &addr, &len, premapped)) 1711 + goto unmap_release; 1712 + 1713 + flags |= cpu_to_le16(vq->packed.avail_used_flags); 1714 + 1715 + if (i == head) 1716 + head_flags = flags; 1717 + else 1718 + desc[i].flags = flags; 1719 + 1720 + desc[i].addr = cpu_to_le64(addr); 1721 + desc[i].len = cpu_to_le32(len); 1722 + desc[i].id = cpu_to_le16(head); 1723 + 1724 + if (unlikely(vq->use_map_api)) { 1725 + vq->packed.desc_extra[i].addr = premapped ? 1726 + DMA_MAPPING_ERROR : addr; 1727 + vq->packed.desc_extra[i].len = len; 1728 + vq->packed.desc_extra[i].flags = 1729 + le16_to_cpu(flags); 1730 + } 1731 + 1732 + if ((unlikely(++i >= vq->packed.vring.num))) { 1733 + i = 0; 1734 + vq->packed.avail_used_flags ^= 1735 + 1 << VRING_PACKED_DESC_F_AVAIL | 1736 + 1 << VRING_PACKED_DESC_F_USED; 1737 + vq->packed.avail_wrap_counter ^= 1; 1738 + } 1739 + 1740 + if (n >= out_sgs) 1741 + total_in_len += len; 1742 + } 1743 + } 1744 + 1745 + /* We're using some buffers from the free list. */ 1746 + vq->vq.num_free -= total_sg; 1747 + 1748 + /* Update free pointer */ 1749 + vq->packed.next_avail_idx = i; 1750 + 1751 + /* Store token. */ 1752 + vq->packed.desc_state[head].num = total_sg; 1753 + vq->packed.desc_state[head].data = data; 1754 + vq->packed.desc_state[head].indir_desc = ctx; 1755 + vq->packed.desc_state[head].total_in_len = total_in_len; 1756 + 1757 + /* 1758 + * A driver MUST NOT make the first descriptor in the list 1759 + * available before all subsequent descriptors comprising 1760 + * the list are made available. 1761 + */ 1762 + virtio_wmb(vq->weak_barriers); 1763 + vq->packed.vring.desc[head].flags = head_flags; 1764 + vq->num_added += total_sg; 1765 + 1766 + pr_debug("Added buffer head %i to %p\n", head, vq); 1767 + END_USE(vq); 1768 + 1769 + return 0; 1770 + 1771 + unmap_release: 1772 + err_idx = i; 1773 + i = head; 1774 + vq->packed.avail_used_flags = avail_used_flags; 1775 + vq->packed.avail_wrap_counter = avail_wrap_counter; 1776 + 1777 + for (n = 0; n < total_sg; n++) { 1778 + if (i == err_idx) 1779 + break; 1780 + vring_unmap_extra_packed(vq, &vq->packed.desc_extra[i]); 1758 1781 i++; 1759 1782 if (i >= vq->packed.vring.num) 1760 1783 i = 0; ··· 2079 1794 cpu_to_le16(vq->last_used_idx)); 2080 1795 } 2081 1796 1797 + static bool more_used_packed_in_order(const struct vring_virtqueue *vq) 1798 + { 1799 + if (vq->batch_last.id != UINT_MAX) 1800 + return true; 1801 + 1802 + return virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx)); 1803 + } 1804 + 1805 + static void *virtqueue_get_buf_ctx_packed_in_order(struct vring_virtqueue *vq, 1806 + unsigned int *len, 1807 + void **ctx) 1808 + { 1809 + unsigned int num = vq->packed.vring.num; 1810 + u16 last_used, last_used_idx; 1811 + bool used_wrap_counter; 1812 + void *ret; 1813 + 1814 + START_USE(vq); 1815 + 1816 + if (unlikely(vq->broken)) { 1817 + END_USE(vq); 1818 + return NULL; 1819 + } 1820 + 1821 + last_used_idx = vq->last_used_idx; 1822 + used_wrap_counter = packed_used_wrap_counter(last_used_idx); 1823 + last_used = packed_last_used(last_used_idx); 1824 + 1825 + if (vq->batch_last.id == UINT_MAX) { 1826 + if (!more_used_packed_in_order(vq)) { 1827 + pr_debug("No more buffers in queue\n"); 1828 + END_USE(vq); 1829 + return NULL; 1830 + } 1831 + /* Only get used elements after they have been exposed by host. */ 1832 + virtio_rmb(vq->weak_barriers); 1833 + vq->batch_last.id = 1834 + le16_to_cpu(vq->packed.vring.desc[last_used].id); 1835 + vq->batch_last.len = 1836 + le32_to_cpu(vq->packed.vring.desc[last_used].len); 1837 + } 1838 + 1839 + if (vq->batch_last.id == last_used) { 1840 + vq->batch_last.id = UINT_MAX; 1841 + *len = vq->batch_last.len; 1842 + } else { 1843 + *len = vq->packed.desc_state[last_used].total_in_len; 1844 + } 1845 + 1846 + if (unlikely(last_used >= num)) { 1847 + BAD_RING(vq, "id %u out of range\n", last_used); 1848 + return NULL; 1849 + } 1850 + if (unlikely(!vq->packed.desc_state[last_used].data)) { 1851 + BAD_RING(vq, "id %u is not a head!\n", last_used); 1852 + return NULL; 1853 + } 1854 + 1855 + /* detach_buf_packed clears data, so grab it now. */ 1856 + ret = vq->packed.desc_state[last_used].data; 1857 + detach_buf_packed_in_order(vq, last_used, ctx); 1858 + 1859 + update_last_used_idx_packed(vq, last_used, last_used, 1860 + used_wrap_counter); 1861 + 1862 + LAST_ADD_TIME_INVALID(vq); 1863 + 1864 + END_USE(vq); 1865 + return ret; 1866 + } 1867 + 2082 1868 static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq, 2083 1869 unsigned int *len, 2084 1870 void **ctx) 2085 1871 { 1872 + unsigned int num = vq->packed.vring.num; 2086 1873 u16 last_used, id, last_used_idx; 2087 1874 bool used_wrap_counter; 2088 1875 void *ret; ··· 2181 1824 id = le16_to_cpu(vq->packed.vring.desc[last_used].id); 2182 1825 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len); 2183 1826 2184 - if (unlikely(id >= vq->packed.vring.num)) { 1827 + if (unlikely(id >= num)) { 2185 1828 BAD_RING(vq, "id %u out of range\n", id); 2186 1829 return NULL; 2187 1830 } ··· 2322 1965 continue; 2323 1966 /* detach_buf clears data, so grab it now. */ 2324 1967 buf = vq->packed.desc_state[i].data; 2325 - detach_buf_packed(vq, i, NULL); 1968 + if (virtqueue_is_in_order(vq)) 1969 + detach_buf_packed_in_order(vq, i, NULL); 1970 + else 1971 + detach_buf_packed(vq, i, NULL); 2326 1972 END_USE(vq); 2327 1973 return buf; 2328 1974 } ··· 2350 1990 2351 1991 for (i = 0; i < num - 1; i++) 2352 1992 desc_extra[i].next = i + 1; 1993 + 1994 + desc_extra[num - 1].next = 0; 2353 1995 2354 1996 return desc_extra; 2355 1997 } ··· 2484 2122 { 2485 2123 vq->packed = *vring_packed; 2486 2124 2487 - /* Put everything in free lists. */ 2488 - vq->free_head = 0; 2125 + if (virtqueue_is_in_order(vq)) { 2126 + vq->batch_last.id = UINT_MAX; 2127 + } else { 2128 + /* 2129 + * Put everything in free lists. Note that 2130 + * next_avail_idx is sufficient with IN_ORDER so 2131 + * free_head is unused. 2132 + */ 2133 + vq->free_head = 0; 2134 + } 2489 2135 } 2490 - 2491 2136 static void virtqueue_reset_packed(struct vring_virtqueue *vq) 2492 2137 { 2493 2138 memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes); ··· 2538 2169 #else 2539 2170 vq->broken = false; 2540 2171 #endif 2541 - vq->layout = VQ_LAYOUT_PACKED; 2542 2172 vq->map = map; 2543 2173 vq->use_map_api = vring_use_map_api(vdev); 2544 2174 2545 2175 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && 2546 2176 !context; 2547 2177 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); 2178 + vq->layout = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ? 2179 + VQ_LAYOUT_PACKED_IN_ORDER : VQ_LAYOUT_PACKED; 2548 2180 2549 2181 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) 2550 2182 vq->weak_barriers = false; ··· 2655 2285 .reset = virtqueue_reset_packed, 2656 2286 }; 2657 2287 2288 + static const struct virtqueue_ops split_in_order_ops = { 2289 + .add = virtqueue_add_split, 2290 + .get = virtqueue_get_buf_ctx_split_in_order, 2291 + .kick_prepare = virtqueue_kick_prepare_split, 2292 + .disable_cb = virtqueue_disable_cb_split, 2293 + .enable_cb_delayed = virtqueue_enable_cb_delayed_split, 2294 + .enable_cb_prepare = virtqueue_enable_cb_prepare_split, 2295 + .poll = virtqueue_poll_split, 2296 + .detach_unused_buf = virtqueue_detach_unused_buf_split, 2297 + .more_used = more_used_split_in_order, 2298 + .resize = virtqueue_resize_split, 2299 + .reset = virtqueue_reset_split, 2300 + }; 2301 + 2302 + static const struct virtqueue_ops packed_in_order_ops = { 2303 + .add = virtqueue_add_packed_in_order, 2304 + .get = virtqueue_get_buf_ctx_packed_in_order, 2305 + .kick_prepare = virtqueue_kick_prepare_packed, 2306 + .disable_cb = virtqueue_disable_cb_packed, 2307 + .enable_cb_delayed = virtqueue_enable_cb_delayed_packed, 2308 + .enable_cb_prepare = virtqueue_enable_cb_prepare_packed, 2309 + .poll = virtqueue_poll_packed, 2310 + .detach_unused_buf = virtqueue_detach_unused_buf_packed, 2311 + .more_used = more_used_packed_in_order, 2312 + .resize = virtqueue_resize_packed, 2313 + .reset = virtqueue_reset_packed, 2314 + }; 2315 + 2658 2316 static int virtqueue_disable_and_recycle(struct virtqueue *_vq, 2659 2317 void (*recycle)(struct virtqueue *vq, void *buf)) 2660 2318 { ··· 2737 2339 case VQ_LAYOUT_PACKED: \ 2738 2340 ret = packed_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__);\ 2739 2341 break; \ 2342 + case VQ_LAYOUT_SPLIT_IN_ORDER: \ 2343 + ret = split_in_order_ops.op(vq, ##__VA_ARGS__); \ 2344 + break; \ 2345 + case VQ_LAYOUT_PACKED_IN_ORDER: \ 2346 + ret = packed_in_order_ops.op(vq, ##__VA_ARGS__); \ 2347 + break; \ 2740 2348 default: \ 2741 2349 BUG(); \ 2742 2350 break; \ ··· 2760 2356 break; \ 2761 2357 case VQ_LAYOUT_PACKED: \ 2762 2358 packed_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__); \ 2359 + break; \ 2360 + case VQ_LAYOUT_SPLIT_IN_ORDER: \ 2361 + split_in_order_ops.op(vq, ##__VA_ARGS__); \ 2362 + break; \ 2363 + case VQ_LAYOUT_PACKED_IN_ORDER: \ 2364 + packed_in_order_ops.op(vq, ##__VA_ARGS__); \ 2763 2365 break; \ 2764 2366 default: \ 2765 2367 BUG(); \ ··· 3485 3075 case VIRTIO_F_ORDER_PLATFORM: 3486 3076 break; 3487 3077 case VIRTIO_F_NOTIFICATION_DATA: 3078 + break; 3079 + case VIRTIO_F_IN_ORDER: 3488 3080 break; 3489 3081 default: 3490 3082 /* We don't understand this bit. */

Configure Feed

Configure Feed