virtio_ring: introduce virtqueue ops · tjh.dev/kernel@1208473

+127 -44

1 changed file

expand all

drivers

virtio

virtio_ring.c

+127 -44

drivers/virtio/virtio_ring.c

··· 67 67 #define LAST_ADD_TIME_INVALID(vq) 68 68 #endif 69 69 70 + enum vq_layout { 71 + VQ_LAYOUT_SPLIT = 0, 72 + VQ_LAYOUT_PACKED, 73 + }; 74 + 70 75 struct vring_desc_state_split { 71 76 void *data; /* Data for callback. */ 72 77 ··· 164 159 size_t event_size_in_bytes; 165 160 }; 166 161 162 + struct vring_virtqueue; 163 + 164 + struct virtqueue_ops { 165 + int (*add)(struct vring_virtqueue *vq, struct scatterlist *sgs[], 166 + unsigned int total_sg, unsigned int out_sgs, 167 + unsigned int in_sgs, void *data, 168 + void *ctx, bool premapped, gfp_t gfp); 169 + void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx); 170 + bool (*kick_prepare)(struct vring_virtqueue *vq); 171 + void (*disable_cb)(struct vring_virtqueue *vq); 172 + bool (*enable_cb_delayed)(struct vring_virtqueue *vq); 173 + unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq); 174 + bool (*poll)(const struct vring_virtqueue *vq, 175 + unsigned int last_used_idx); 176 + void *(*detach_unused_buf)(struct vring_virtqueue *vq); 177 + bool (*more_used)(const struct vring_virtqueue *vq); 178 + int (*resize)(struct vring_virtqueue *vq, u32 num); 179 + void (*reset)(struct vring_virtqueue *vq); 180 + }; 181 + 167 182 struct vring_virtqueue { 168 183 struct virtqueue vq; 169 - 170 - /* Is this a packed ring? */ 171 - bool packed_ring; 172 184 173 185 /* Is DMA API used? */ 174 186 bool use_map_api; ··· 201 179 202 180 /* Host publishes avail event idx */ 203 181 bool event; 182 + 183 + enum vq_layout layout; 204 184 205 185 /* Head of free buffer list. */ 206 186 unsigned int free_head; ··· 254 230 */ 255 231 256 232 #define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq) 233 + 234 + 235 + static inline bool virtqueue_is_packed(const struct vring_virtqueue *vq) 236 + { 237 + return vq->layout == VQ_LAYOUT_PACKED; 238 + } 257 239 258 240 static bool virtqueue_use_indirect(const struct vring_virtqueue *vq, 259 241 unsigned int total_sg) ··· 463 433 { 464 434 vq->vq.num_free = num; 465 435 466 - if (vq->packed_ring) 436 + if (virtqueue_is_packed(vq)) 467 437 vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR); 468 438 else 469 439 vq->last_used_idx = 0; ··· 1151 1121 return 0; 1152 1122 } 1153 1123 1124 + static const struct virtqueue_ops split_ops; 1125 + 1154 1126 static struct virtqueue *__vring_new_virtqueue_split(unsigned int index, 1155 1127 struct vring_virtqueue_split *vring_split, 1156 1128 struct virtio_device *vdev, ··· 1170 1138 if (!vq) 1171 1139 return NULL; 1172 1140 1173 - vq->packed_ring = false; 1141 + vq->layout = VQ_LAYOUT_SPLIT; 1174 1142 vq->vq.callback = callback; 1175 1143 vq->vq.vdev = vdev; 1176 1144 vq->vq.name = name; ··· 2103 2071 2104 2072 /* we need to reset the desc.flags. For more, see is_used_desc_packed() */ 2105 2073 memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes); 2106 - 2107 2074 virtqueue_init(vq, vq->packed.vring.num); 2108 2075 virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback); 2109 2076 } 2077 + 2078 + static const struct virtqueue_ops packed_ops; 2110 2079 2111 2080 static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index, 2112 2081 struct vring_virtqueue_packed *vring_packed, ··· 2139 2106 #else 2140 2107 vq->broken = false; 2141 2108 #endif 2142 - vq->packed_ring = true; 2109 + vq->layout = VQ_LAYOUT_PACKED; 2143 2110 vq->map = map; 2144 2111 vq->use_map_api = vring_use_map_api(vdev); 2145 2112 ··· 2227 2194 return -ENOMEM; 2228 2195 } 2229 2196 2197 + static const struct virtqueue_ops split_ops = { 2198 + .add = virtqueue_add_split, 2199 + .get = virtqueue_get_buf_ctx_split, 2200 + .kick_prepare = virtqueue_kick_prepare_split, 2201 + .disable_cb = virtqueue_disable_cb_split, 2202 + .enable_cb_delayed = virtqueue_enable_cb_delayed_split, 2203 + .enable_cb_prepare = virtqueue_enable_cb_prepare_split, 2204 + .poll = virtqueue_poll_split, 2205 + .detach_unused_buf = virtqueue_detach_unused_buf_split, 2206 + .more_used = more_used_split, 2207 + .resize = virtqueue_resize_split, 2208 + .reset = virtqueue_reset_split, 2209 + }; 2210 + 2211 + static const struct virtqueue_ops packed_ops = { 2212 + .add = virtqueue_add_packed, 2213 + .get = virtqueue_get_buf_ctx_packed, 2214 + .kick_prepare = virtqueue_kick_prepare_packed, 2215 + .disable_cb = virtqueue_disable_cb_packed, 2216 + .enable_cb_delayed = virtqueue_enable_cb_delayed_packed, 2217 + .enable_cb_prepare = virtqueue_enable_cb_prepare_packed, 2218 + .poll = virtqueue_poll_packed, 2219 + .detach_unused_buf = virtqueue_detach_unused_buf_packed, 2220 + .more_used = more_used_packed, 2221 + .resize = virtqueue_resize_packed, 2222 + .reset = virtqueue_reset_packed, 2223 + }; 2224 + 2230 2225 static int virtqueue_disable_and_recycle(struct virtqueue *_vq, 2231 2226 void (*recycle)(struct virtqueue *vq, void *buf)) 2232 2227 { ··· 2297 2236 * Generic functions and exported symbols. 2298 2237 */ 2299 2238 2239 + #define VIRTQUEUE_CALL(vq, op, ...) \ 2240 + ({ \ 2241 + typeof(vq) __VIRTQUEUE_CALL_vq = (vq); \ 2242 + typeof(split_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__)) ret; \ 2243 + \ 2244 + switch (__VIRTQUEUE_CALL_vq->layout) { \ 2245 + case VQ_LAYOUT_SPLIT: \ 2246 + ret = split_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__); \ 2247 + break; \ 2248 + case VQ_LAYOUT_PACKED: \ 2249 + ret = packed_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__);\ 2250 + break; \ 2251 + default: \ 2252 + BUG(); \ 2253 + break; \ 2254 + } \ 2255 + ret; \ 2256 + }) 2257 + 2258 + #define VOID_VIRTQUEUE_CALL(vq, op, ...) \ 2259 + ({ \ 2260 + typeof(vq) __VIRTQUEUE_CALL_vq = (vq); \ 2261 + \ 2262 + switch (__VIRTQUEUE_CALL_vq->layout) { \ 2263 + case VQ_LAYOUT_SPLIT: \ 2264 + split_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__); \ 2265 + break; \ 2266 + case VQ_LAYOUT_PACKED: \ 2267 + packed_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__); \ 2268 + break; \ 2269 + default: \ 2270 + BUG(); \ 2271 + break; \ 2272 + } \ 2273 + }) 2274 + 2300 2275 static inline int virtqueue_add(struct virtqueue *_vq, 2301 2276 struct scatterlist *sgs[], 2302 2277 unsigned int total_sg, ··· 2345 2248 { 2346 2249 struct vring_virtqueue *vq = to_vvq(_vq); 2347 2250 2348 - return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg, 2349 - out_sgs, in_sgs, data, ctx, premapped, gfp) : 2350 - virtqueue_add_split(vq, sgs, total_sg, 2351 - out_sgs, in_sgs, data, ctx, premapped, gfp); 2251 + return VIRTQUEUE_CALL(vq, add, sgs, total_sg, 2252 + out_sgs, in_sgs, data, 2253 + ctx, premapped, gfp); 2352 2254 } 2353 2255 2354 2256 /** ··· 2537 2441 { 2538 2442 struct vring_virtqueue *vq = to_vvq(_vq); 2539 2443 2540 - return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) : 2541 - virtqueue_kick_prepare_split(vq); 2444 + return VIRTQUEUE_CALL(vq, kick_prepare); 2542 2445 } 2543 2446 EXPORT_SYMBOL_GPL(virtqueue_kick_prepare); 2544 2447 ··· 2607 2512 { 2608 2513 struct vring_virtqueue *vq = to_vvq(_vq); 2609 2514 2610 - return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) : 2611 - virtqueue_get_buf_ctx_split(vq, len, ctx); 2515 + return VIRTQUEUE_CALL(vq, get, len, ctx); 2612 2516 } 2613 2517 EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx); 2614 2518 ··· 2629 2535 { 2630 2536 struct vring_virtqueue *vq = to_vvq(_vq); 2631 2537 2632 - if (vq->packed_ring) 2633 - virtqueue_disable_cb_packed(vq); 2634 - else 2635 - virtqueue_disable_cb_split(vq); 2538 + VOID_VIRTQUEUE_CALL(vq, disable_cb); 2636 2539 } 2637 2540 EXPORT_SYMBOL_GPL(virtqueue_disable_cb); 2638 2541 ··· 2652 2561 if (vq->event_triggered) 2653 2562 vq->event_triggered = false; 2654 2563 2655 - return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) : 2656 - virtqueue_enable_cb_prepare_split(vq); 2564 + return VIRTQUEUE_CALL(vq, enable_cb_prepare); 2657 2565 } 2658 2566 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare); 2659 2567 ··· 2673 2583 return false; 2674 2584 2675 2585 virtio_mb(vq->weak_barriers); 2676 - return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) : 2677 - virtqueue_poll_split(vq, last_used_idx); 2586 + 2587 + return VIRTQUEUE_CALL(vq, poll, last_used_idx); 2678 2588 } 2679 2589 EXPORT_SYMBOL_GPL(virtqueue_poll); 2680 2590 ··· 2717 2627 if (vq->event_triggered) 2718 2628 data_race(vq->event_triggered = false); 2719 2629 2720 - return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) : 2721 - virtqueue_enable_cb_delayed_split(vq); 2630 + return VIRTQUEUE_CALL(vq, enable_cb_delayed); 2722 2631 } 2723 2632 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed); 2724 2633 ··· 2733 2644 { 2734 2645 struct vring_virtqueue *vq = to_vvq(_vq); 2735 2646 2736 - return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) : 2737 - virtqueue_detach_unused_buf_split(vq); 2647 + return VIRTQUEUE_CALL(vq, detach_unused_buf); 2738 2648 } 2739 2649 EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf); 2740 2650 2741 2651 static inline bool more_used(const struct vring_virtqueue *vq) 2742 2652 { 2743 - return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq); 2653 + return VIRTQUEUE_CALL(vq, more_used); 2744 2654 } 2745 2655 2746 2656 /** ··· 2869 2781 if (!num) 2870 2782 return -EINVAL; 2871 2783 2872 - if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num) 2784 + if (virtqueue_get_vring_size(_vq) == num) 2873 2785 return 0; 2874 2786 2875 2787 err = virtqueue_disable_and_recycle(_vq, recycle); ··· 2878 2790 if (recycle_done) 2879 2791 recycle_done(_vq); 2880 2792 2881 - if (vq->packed_ring) 2882 - err = virtqueue_resize_packed(vq, num); 2883 - else 2884 - err = virtqueue_resize_split(vq, num); 2793 + err = VIRTQUEUE_CALL(vq, resize, num); 2885 2794 2886 2795 err_reset = virtqueue_enable_after_reset(_vq); 2887 2796 if (err_reset) ··· 2916 2831 if (recycle_done) 2917 2832 recycle_done(_vq); 2918 2833 2919 - if (vq->packed_ring) 2920 - virtqueue_reset_packed(vq); 2921 - else 2922 - virtqueue_reset_split(vq); 2834 + VOID_VIRTQUEUE_CALL(vq, reset); 2923 2835 2924 2836 return virtqueue_enable_after_reset(_vq); 2925 2837 } ··· 2959 2877 struct vring_virtqueue *vq = to_vvq(_vq); 2960 2878 2961 2879 if (vq->we_own_ring) { 2962 - if (vq->packed_ring) { 2880 + if (virtqueue_is_packed(vq)) { 2963 2881 vring_free_queue(vq->vq.vdev, 2964 2882 vq->packed.ring_size_in_bytes, 2965 2883 vq->packed.vring.desc, ··· 2988 2906 vq->map); 2989 2907 } 2990 2908 } 2991 - if (!vq->packed_ring) { 2909 + if (!virtqueue_is_packed(vq)) { 2992 2910 kfree(vq->split.desc_state); 2993 2911 kfree(vq->split.desc_extra); 2994 2912 } ··· 3013 2931 struct vring_virtqueue *vq = to_vvq(_vq); 3014 2932 u16 next; 3015 2933 3016 - if (vq->packed_ring) 2934 + if (virtqueue_is_packed(vq)) 3017 2935 next = (vq->packed.next_avail_idx & 3018 2936 ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) | 3019 2937 vq->packed.avail_wrap_counter << ··· 3066 2984 3067 2985 const struct vring_virtqueue *vq = to_vvq(_vq); 3068 2986 3069 - return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; 2987 + return virtqueue_is_packed(vq) ? vq->packed.vring.num : 2988 + vq->split.vring.num; 3070 2989 } 3071 2990 EXPORT_SYMBOL_GPL(virtqueue_get_vring_size); 3072 2991 ··· 3150 3067 3151 3068 BUG_ON(!vq->we_own_ring); 3152 3069 3153 - if (vq->packed_ring) 3070 + if (virtqueue_is_packed(vq)) 3154 3071 return vq->packed.ring_dma_addr; 3155 3072 3156 3073 return vq->split.queue_dma_addr; ··· 3163 3080 3164 3081 BUG_ON(!vq->we_own_ring); 3165 3082 3166 - if (vq->packed_ring) 3083 + if (virtqueue_is_packed(vq)) 3167 3084 return vq->packed.driver_event_dma_addr; 3168 3085 3169 3086 return vq->split.queue_dma_addr + ··· 3177 3094 3178 3095 BUG_ON(!vq->we_own_ring); 3179 3096 3180 - if (vq->packed_ring) 3097 + if (virtqueue_is_packed(vq)) 3181 3098 return vq->packed.device_event_dma_addr; 3182 3099 3183 3100 return vq->split.queue_dma_addr +

Configure Feed

Configure Feed