virtio/vsock: replace virtio_vsock_pkt with sk_buff

+90 -126

drivers/vhost/vsock.c

··· 51 51 struct hlist_node hash; 52 52 53 53 struct vhost_work send_pkt_work; 54 - spinlock_t send_pkt_list_lock; 55 - struct list_head send_pkt_list; /* host->guest pending packets */ 54 + struct sk_buff_head send_pkt_queue; /* host->guest pending packets */ 56 55 57 56 atomic_t queued_replies; 58 57 ··· 107 108 vhost_disable_notify(&vsock->dev, vq); 108 109 109 110 do { 110 - struct virtio_vsock_pkt *pkt; 111 + struct virtio_vsock_hdr *hdr; 112 + size_t iov_len, payload_len; 111 113 struct iov_iter iov_iter; 114 + u32 flags_to_restore = 0; 115 + struct sk_buff *skb; 112 116 unsigned out, in; 113 117 size_t nbytes; 114 - size_t iov_len, payload_len; 115 118 int head; 116 - u32 flags_to_restore = 0; 117 119 118 - spin_lock_bh(&vsock->send_pkt_list_lock); 119 - if (list_empty(&vsock->send_pkt_list)) { 120 - spin_unlock_bh(&vsock->send_pkt_list_lock); 120 + skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue); 121 + 122 + if (!skb) { 121 123 vhost_enable_notify(&vsock->dev, vq); 122 124 break; 123 125 } 124 126 125 - pkt = list_first_entry(&vsock->send_pkt_list, 126 - struct virtio_vsock_pkt, list); 127 - list_del_init(&pkt->list); 128 - spin_unlock_bh(&vsock->send_pkt_list_lock); 129 - 130 127 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), 131 128 &out, &in, NULL, NULL); 132 129 if (head < 0) { 133 - spin_lock_bh(&vsock->send_pkt_list_lock); 134 - list_add(&pkt->list, &vsock->send_pkt_list); 135 - spin_unlock_bh(&vsock->send_pkt_list_lock); 130 + virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 136 131 break; 137 132 } 138 133 139 134 if (head == vq->num) { 140 - spin_lock_bh(&vsock->send_pkt_list_lock); 141 - list_add(&pkt->list, &vsock->send_pkt_list); 142 - spin_unlock_bh(&vsock->send_pkt_list_lock); 143 - 135 + virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 144 136 /* We cannot finish yet if more buffers snuck in while 145 137 * re-enabling notify. 146 138 */ ··· 143 153 } 144 154 145 155 if (out) { 146 - virtio_transport_free_pkt(pkt); 156 + kfree_skb(skb); 147 157 vq_err(vq, "Expected 0 output buffers, got %u\n", out); 148 158 break; 149 159 } 150 160 151 161 iov_len = iov_length(&vq->iov[out], in); 152 - if (iov_len < sizeof(pkt->hdr)) { 153 - virtio_transport_free_pkt(pkt); 162 + if (iov_len < sizeof(*hdr)) { 163 + kfree_skb(skb); 154 164 vq_err(vq, "Buffer len [%zu] too small\n", iov_len); 155 165 break; 156 166 } 157 167 158 168 iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[out], in, iov_len); 159 - payload_len = pkt->len - pkt->off; 169 + payload_len = skb->len; 170 + hdr = virtio_vsock_hdr(skb); 160 171 161 172 /* If the packet is greater than the space available in the 162 173 * buffer, we split it using multiple buffers. 163 174 */ 164 - if (payload_len > iov_len - sizeof(pkt->hdr)) { 165 - payload_len = iov_len - sizeof(pkt->hdr); 175 + if (payload_len > iov_len - sizeof(*hdr)) { 176 + payload_len = iov_len - sizeof(*hdr); 166 177 167 178 /* As we are copying pieces of large packet's buffer to 168 179 * small rx buffers, headers of packets in rx queue are ··· 176 185 * bits set. After initialized header will be copied to 177 186 * rx buffer, these required bits will be restored. 178 187 */ 179 - if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SEQ_EOM) { 180 - pkt->hdr.flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM); 188 + if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) { 189 + hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM); 181 190 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOM; 182 191 183 - if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SEQ_EOR) { 184 - pkt->hdr.flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR); 192 + if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) { 193 + hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR); 185 194 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOR; 186 195 } 187 196 } 188 197 } 189 198 190 199 /* Set the correct length in the header */ 191 - pkt->hdr.len = cpu_to_le32(payload_len); 200 + hdr->len = cpu_to_le32(payload_len); 192 201 193 - nbytes = copy_to_iter(&pkt->hdr, sizeof(pkt->hdr), &iov_iter); 194 - if (nbytes != sizeof(pkt->hdr)) { 195 - virtio_transport_free_pkt(pkt); 202 + nbytes = copy_to_iter(hdr, sizeof(*hdr), &iov_iter); 203 + if (nbytes != sizeof(*hdr)) { 204 + kfree_skb(skb); 196 205 vq_err(vq, "Faulted on copying pkt hdr\n"); 197 206 break; 198 207 } 199 208 200 - nbytes = copy_to_iter(pkt->buf + pkt->off, payload_len, 201 - &iov_iter); 209 + nbytes = copy_to_iter(skb->data, payload_len, &iov_iter); 202 210 if (nbytes != payload_len) { 203 - virtio_transport_free_pkt(pkt); 211 + kfree_skb(skb); 204 212 vq_err(vq, "Faulted on copying pkt buf\n"); 205 213 break; 206 214 } ··· 207 217 /* Deliver to monitoring devices all packets that we 208 218 * will transmit. 209 219 */ 210 - virtio_transport_deliver_tap_pkt(pkt); 220 + virtio_transport_deliver_tap_pkt(skb); 211 221 212 - vhost_add_used(vq, head, sizeof(pkt->hdr) + payload_len); 222 + vhost_add_used(vq, head, sizeof(*hdr) + payload_len); 213 223 added = true; 214 224 215 - pkt->off += payload_len; 225 + skb_pull(skb, payload_len); 216 226 total_len += payload_len; 217 227 218 228 /* If we didn't send all the payload we can requeue the packet 219 229 * to send it with the next available buffer. 220 230 */ 221 - if (pkt->off < pkt->len) { 222 - pkt->hdr.flags |= cpu_to_le32(flags_to_restore); 231 + if (skb->len > 0) { 232 + hdr->flags |= cpu_to_le32(flags_to_restore); 223 233 224 - /* We are queueing the same virtio_vsock_pkt to handle 234 + /* We are queueing the same skb to handle 225 235 * the remaining bytes, and we want to deliver it 226 236 * to monitoring devices in the next iteration. 227 237 */ 228 - pkt->tap_delivered = false; 229 - 230 - spin_lock_bh(&vsock->send_pkt_list_lock); 231 - list_add(&pkt->list, &vsock->send_pkt_list); 232 - spin_unlock_bh(&vsock->send_pkt_list_lock); 238 + virtio_vsock_skb_clear_tap_delivered(skb); 239 + virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 233 240 } else { 234 - if (pkt->reply) { 241 + if (virtio_vsock_skb_reply(skb)) { 235 242 int val; 236 243 237 244 val = atomic_dec_return(&vsock->queued_replies); ··· 240 253 restart_tx = true; 241 254 } 242 255 243 - virtio_transport_free_pkt(pkt); 256 + consume_skb(skb); 244 257 } 245 258 } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len))); 246 259 if (added) ··· 265 278 } 266 279 267 280 static int 268 - vhost_transport_send_pkt(struct virtio_vsock_pkt *pkt) 281 + vhost_transport_send_pkt(struct sk_buff *skb) 269 282 { 283 + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 270 284 struct vhost_vsock *vsock; 271 - int len = pkt->len; 285 + int len = skb->len; 272 286 273 287 rcu_read_lock(); 274 288 275 289 /* Find the vhost_vsock according to guest context id */ 276 - vsock = vhost_vsock_get(le64_to_cpu(pkt->hdr.dst_cid)); 290 + vsock = vhost_vsock_get(le64_to_cpu(hdr->dst_cid)); 277 291 if (!vsock) { 278 292 rcu_read_unlock(); 279 - virtio_transport_free_pkt(pkt); 293 + kfree_skb(skb); 280 294 return -ENODEV; 281 295 } 282 296 283 - if (pkt->reply) 297 + if (virtio_vsock_skb_reply(skb)) 284 298 atomic_inc(&vsock->queued_replies); 285 299 286 - spin_lock_bh(&vsock->send_pkt_list_lock); 287 - list_add_tail(&pkt->list, &vsock->send_pkt_list); 288 - spin_unlock_bh(&vsock->send_pkt_list_lock); 289 - 300 + virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb); 290 301 vhost_work_queue(&vsock->dev, &vsock->send_pkt_work); 291 302 292 303 rcu_read_unlock(); ··· 295 310 vhost_transport_cancel_pkt(struct vsock_sock *vsk) 296 311 { 297 312 struct vhost_vsock *vsock; 298 - struct virtio_vsock_pkt *pkt, *n; 299 313 int cnt = 0; 300 314 int ret = -ENODEV; 301 - LIST_HEAD(freeme); 302 315 303 316 rcu_read_lock(); 304 317 ··· 305 322 if (!vsock) 306 323 goto out; 307 324 308 - spin_lock_bh(&vsock->send_pkt_list_lock); 309 - list_for_each_entry_safe(pkt, n, &vsock->send_pkt_list, list) { 310 - if (pkt->vsk != vsk) 311 - continue; 312 - list_move(&pkt->list, &freeme); 313 - } 314 - spin_unlock_bh(&vsock->send_pkt_list_lock); 315 - 316 - list_for_each_entry_safe(pkt, n, &freeme, list) { 317 - if (pkt->reply) 318 - cnt++; 319 - list_del(&pkt->list); 320 - virtio_transport_free_pkt(pkt); 321 - } 325 + cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue); 322 326 323 327 if (cnt) { 324 328 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX]; ··· 322 352 return ret; 323 353 } 324 354 325 - static struct virtio_vsock_pkt * 326 - vhost_vsock_alloc_pkt(struct vhost_virtqueue *vq, 355 + static struct sk_buff * 356 + vhost_vsock_alloc_skb(struct vhost_virtqueue *vq, 327 357 unsigned int out, unsigned int in) 328 358 { 329 - struct virtio_vsock_pkt *pkt; 359 + struct virtio_vsock_hdr *hdr; 330 360 struct iov_iter iov_iter; 361 + struct sk_buff *skb; 362 + size_t payload_len; 331 363 size_t nbytes; 332 364 size_t len; 333 365 ··· 338 366 return NULL; 339 367 } 340 368 341 - pkt = kzalloc(sizeof(*pkt), GFP_KERNEL); 342 - if (!pkt) 369 + len = iov_length(vq->iov, out); 370 + 371 + /* len contains both payload and hdr */ 372 + skb = virtio_vsock_alloc_skb(len, GFP_KERNEL); 373 + if (!skb) 343 374 return NULL; 344 375 345 - len = iov_length(vq->iov, out); 346 376 iov_iter_init(&iov_iter, ITER_SOURCE, vq->iov, out, len); 347 377 348 - nbytes = copy_from_iter(&pkt->hdr, sizeof(pkt->hdr), &iov_iter); 349 - if (nbytes != sizeof(pkt->hdr)) { 378 + hdr = virtio_vsock_hdr(skb); 379 + nbytes = copy_from_iter(hdr, sizeof(*hdr), &iov_iter); 380 + if (nbytes != sizeof(*hdr)) { 350 381 vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n", 351 - sizeof(pkt->hdr), nbytes); 352 - kfree(pkt); 382 + sizeof(*hdr), nbytes); 383 + kfree_skb(skb); 353 384 return NULL; 354 385 } 355 386 356 - pkt->len = le32_to_cpu(pkt->hdr.len); 387 + payload_len = le32_to_cpu(hdr->len); 357 388 358 389 /* No payload */ 359 - if (!pkt->len) 360 - return pkt; 390 + if (!payload_len) 391 + return skb; 361 392 362 - /* The pkt is too big */ 363 - if (pkt->len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) { 364 - kfree(pkt); 393 + /* The pkt is too big or the length in the header is invalid */ 394 + if (payload_len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE || 395 + payload_len + sizeof(*hdr) > len) { 396 + kfree_skb(skb); 365 397 return NULL; 366 398 } 367 399 368 - pkt->buf = kvmalloc(pkt->len, GFP_KERNEL); 369 - if (!pkt->buf) { 370 - kfree(pkt); 400 + virtio_vsock_skb_rx_put(skb); 401 + 402 + nbytes = copy_from_iter(skb->data, payload_len, &iov_iter); 403 + if (nbytes != payload_len) { 404 + vq_err(vq, "Expected %zu byte payload, got %zu bytes\n", 405 + payload_len, nbytes); 406 + kfree_skb(skb); 371 407 return NULL; 372 408 } 373 409 374 - pkt->buf_len = pkt->len; 375 - 376 - nbytes = copy_from_iter(pkt->buf, pkt->len, &iov_iter); 377 - if (nbytes != pkt->len) { 378 - vq_err(vq, "Expected %u byte payload, got %zu bytes\n", 379 - pkt->len, nbytes); 380 - virtio_transport_free_pkt(pkt); 381 - return NULL; 382 - } 383 - 384 - return pkt; 410 + return skb; 385 411 } 386 412 387 413 /* Is there space left for replies to rx packets? */ ··· 466 496 poll.work); 467 497 struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock, 468 498 dev); 469 - struct virtio_vsock_pkt *pkt; 470 499 int head, pkts = 0, total_len = 0; 471 500 unsigned int out, in; 501 + struct sk_buff *skb; 472 502 bool added = false; 473 503 474 504 mutex_lock(&vq->mutex); ··· 481 511 482 512 vhost_disable_notify(&vsock->dev, vq); 483 513 do { 514 + struct virtio_vsock_hdr *hdr; 515 + 484 516 if (!vhost_vsock_more_replies(vsock)) { 485 517 /* Stop tx until the device processes already 486 518 * pending replies. Leave tx virtqueue ··· 504 532 break; 505 533 } 506 534 507 - pkt = vhost_vsock_alloc_pkt(vq, out, in); 508 - if (!pkt) { 535 + skb = vhost_vsock_alloc_skb(vq, out, in); 536 + if (!skb) { 509 537 vq_err(vq, "Faulted on pkt\n"); 510 538 continue; 511 539 } 512 540 513 - total_len += sizeof(pkt->hdr) + pkt->len; 541 + total_len += sizeof(*hdr) + skb->len; 514 542 515 543 /* Deliver to monitoring devices all received packets */ 516 - virtio_transport_deliver_tap_pkt(pkt); 544 + virtio_transport_deliver_tap_pkt(skb); 545 + 546 + hdr = virtio_vsock_hdr(skb); 517 547 518 548 /* Only accept correctly addressed packets */ 519 - if (le64_to_cpu(pkt->hdr.src_cid) == vsock->guest_cid && 520 - le64_to_cpu(pkt->hdr.dst_cid) == 549 + if (le64_to_cpu(hdr->src_cid) == vsock->guest_cid && 550 + le64_to_cpu(hdr->dst_cid) == 521 551 vhost_transport_get_local_cid()) 522 - virtio_transport_recv_pkt(&vhost_transport, pkt); 552 + virtio_transport_recv_pkt(&vhost_transport, skb); 523 553 else 524 - virtio_transport_free_pkt(pkt); 554 + kfree_skb(skb); 525 555 526 556 vhost_add_used(vq, head, 0); 527 557 added = true; ··· 667 693 VHOST_VSOCK_WEIGHT, true, NULL); 668 694 669 695 file->private_data = vsock; 670 - spin_lock_init(&vsock->send_pkt_list_lock); 671 - INIT_LIST_HEAD(&vsock->send_pkt_list); 696 + skb_queue_head_init(&vsock->send_pkt_queue); 672 697 vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work); 673 698 return 0; 674 699 ··· 733 760 vhost_vsock_flush(vsock); 734 761 vhost_dev_stop(&vsock->dev); 735 762 736 - spin_lock_bh(&vsock->send_pkt_list_lock); 737 - while (!list_empty(&vsock->send_pkt_list)) { 738 - struct virtio_vsock_pkt *pkt; 739 - 740 - pkt = list_first_entry(&vsock->send_pkt_list, 741 - struct virtio_vsock_pkt, list); 742 - list_del_init(&pkt->list); 743 - virtio_transport_free_pkt(pkt); 744 - } 745 - spin_unlock_bh(&vsock->send_pkt_list_lock); 763 + virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue); 746 764 747 765 vhost_dev_cleanup(&vsock->dev); 748 766 kfree(vsock->dev.vqs);

+109 -20

include/linux/virtio_vsock.h

··· 7 7 #include <net/sock.h> 8 8 #include <net/af_vsock.h> 9 9 10 + #define VIRTIO_VSOCK_SKB_HEADROOM (sizeof(struct virtio_vsock_hdr)) 11 + 12 + struct virtio_vsock_skb_cb { 13 + bool reply; 14 + bool tap_delivered; 15 + }; 16 + 17 + #define VIRTIO_VSOCK_SKB_CB(skb) ((struct virtio_vsock_skb_cb *)((skb)->cb)) 18 + 19 + static inline struct virtio_vsock_hdr *virtio_vsock_hdr(struct sk_buff *skb) 20 + { 21 + return (struct virtio_vsock_hdr *)skb->head; 22 + } 23 + 24 + static inline bool virtio_vsock_skb_reply(struct sk_buff *skb) 25 + { 26 + return VIRTIO_VSOCK_SKB_CB(skb)->reply; 27 + } 28 + 29 + static inline void virtio_vsock_skb_set_reply(struct sk_buff *skb) 30 + { 31 + VIRTIO_VSOCK_SKB_CB(skb)->reply = true; 32 + } 33 + 34 + static inline bool virtio_vsock_skb_tap_delivered(struct sk_buff *skb) 35 + { 36 + return VIRTIO_VSOCK_SKB_CB(skb)->tap_delivered; 37 + } 38 + 39 + static inline void virtio_vsock_skb_set_tap_delivered(struct sk_buff *skb) 40 + { 41 + VIRTIO_VSOCK_SKB_CB(skb)->tap_delivered = true; 42 + } 43 + 44 + static inline void virtio_vsock_skb_clear_tap_delivered(struct sk_buff *skb) 45 + { 46 + VIRTIO_VSOCK_SKB_CB(skb)->tap_delivered = false; 47 + } 48 + 49 + static inline void virtio_vsock_skb_rx_put(struct sk_buff *skb) 50 + { 51 + u32 len; 52 + 53 + len = le32_to_cpu(virtio_vsock_hdr(skb)->len); 54 + 55 + if (len > 0) 56 + skb_put(skb, len); 57 + } 58 + 59 + static inline struct sk_buff *virtio_vsock_alloc_skb(unsigned int size, gfp_t mask) 60 + { 61 + struct sk_buff *skb; 62 + 63 + if (size < VIRTIO_VSOCK_SKB_HEADROOM) 64 + return NULL; 65 + 66 + skb = alloc_skb(size, mask); 67 + if (!skb) 68 + return NULL; 69 + 70 + skb_reserve(skb, VIRTIO_VSOCK_SKB_HEADROOM); 71 + return skb; 72 + } 73 + 74 + static inline void 75 + virtio_vsock_skb_queue_head(struct sk_buff_head *list, struct sk_buff *skb) 76 + { 77 + spin_lock_bh(&list->lock); 78 + __skb_queue_head(list, skb); 79 + spin_unlock_bh(&list->lock); 80 + } 81 + 82 + static inline void 83 + virtio_vsock_skb_queue_tail(struct sk_buff_head *list, struct sk_buff *skb) 84 + { 85 + spin_lock_bh(&list->lock); 86 + __skb_queue_tail(list, skb); 87 + spin_unlock_bh(&list->lock); 88 + } 89 + 90 + static inline struct sk_buff *virtio_vsock_skb_dequeue(struct sk_buff_head *list) 91 + { 92 + struct sk_buff *skb; 93 + 94 + spin_lock_bh(&list->lock); 95 + skb = __skb_dequeue(list); 96 + spin_unlock_bh(&list->lock); 97 + 98 + return skb; 99 + } 100 + 101 + static inline void virtio_vsock_skb_queue_purge(struct sk_buff_head *list) 102 + { 103 + spin_lock_bh(&list->lock); 104 + __skb_queue_purge(list); 105 + spin_unlock_bh(&list->lock); 106 + } 107 + 108 + static inline size_t virtio_vsock_skb_len(struct sk_buff *skb) 109 + { 110 + return (size_t)(skb_end_pointer(skb) - skb->head); 111 + } 112 + 10 113 #define VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE (1024 * 4) 11 114 #define VIRTIO_VSOCK_MAX_BUF_SIZE 0xFFFFFFFFUL 12 115 #define VIRTIO_VSOCK_MAX_PKT_BUF_SIZE (1024 * 64) ··· 138 35 u32 last_fwd_cnt; 139 36 u32 rx_bytes; 140 37 u32 buf_alloc; 141 - struct list_head rx_queue; 38 + struct sk_buff_head rx_queue; 142 39 u32 msg_count; 143 - }; 144 - 145 - struct virtio_vsock_pkt { 146 - struct virtio_vsock_hdr hdr; 147 - struct list_head list; 148 - /* socket refcnt not held, only use for cancellation */ 149 - struct vsock_sock *vsk; 150 - void *buf; 151 - u32 buf_len; 152 - u32 len; 153 - u32 off; 154 - bool reply; 155 - bool tap_delivered; 156 40 }; 157 41 158 42 struct virtio_vsock_pkt_info { ··· 158 68 struct vsock_transport transport; 159 69 160 70 /* Takes ownership of the packet */ 161 - int (*send_pkt)(struct virtio_vsock_pkt *pkt); 71 + int (*send_pkt)(struct sk_buff *skb); 162 72 }; 163 73 164 74 ssize_t ··· 239 149 void virtio_transport_destruct(struct vsock_sock *vsk); 240 150 241 151 void virtio_transport_recv_pkt(struct virtio_transport *t, 242 - struct virtio_vsock_pkt *pkt); 243 - void virtio_transport_free_pkt(struct virtio_vsock_pkt *pkt); 244 - void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt); 152 + struct sk_buff *skb); 153 + void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct sk_buff *skb); 245 154 u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 wanted); 246 155 void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit); 247 - void virtio_transport_deliver_tap_pkt(struct virtio_vsock_pkt *pkt); 248 - 156 + void virtio_transport_deliver_tap_pkt(struct sk_buff *skb); 157 + int virtio_transport_purge_skbs(void *vsk, struct sk_buff_head *list); 249 158 #endif /* _LINUX_VIRTIO_VSOCK_H */

+53 -98

net/vmw_vsock/virtio_transport.c

··· 42 42 bool tx_run; 43 43 44 44 struct work_struct send_pkt_work; 45 - spinlock_t send_pkt_list_lock; 46 - struct list_head send_pkt_list; 45 + struct sk_buff_head send_pkt_queue; 47 46 48 47 atomic_t queued_replies; 49 48 ··· 100 101 vq = vsock->vqs[VSOCK_VQ_TX]; 101 102 102 103 for (;;) { 103 - struct virtio_vsock_pkt *pkt; 104 104 struct scatterlist hdr, buf, *sgs[2]; 105 105 int ret, in_sg = 0, out_sg = 0; 106 + struct sk_buff *skb; 106 107 bool reply; 107 108 108 - spin_lock_bh(&vsock->send_pkt_list_lock); 109 - if (list_empty(&vsock->send_pkt_list)) { 110 - spin_unlock_bh(&vsock->send_pkt_list_lock); 109 + skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue); 110 + if (!skb) 111 111 break; 112 - } 113 112 114 - pkt = list_first_entry(&vsock->send_pkt_list, 115 - struct virtio_vsock_pkt, list); 116 - list_del_init(&pkt->list); 117 - spin_unlock_bh(&vsock->send_pkt_list_lock); 113 + virtio_transport_deliver_tap_pkt(skb); 114 + reply = virtio_vsock_skb_reply(skb); 118 115 119 - virtio_transport_deliver_tap_pkt(pkt); 120 - 121 - reply = pkt->reply; 122 - 123 - sg_init_one(&hdr, &pkt->hdr, sizeof(pkt->hdr)); 116 + sg_init_one(&hdr, virtio_vsock_hdr(skb), sizeof(*virtio_vsock_hdr(skb))); 124 117 sgs[out_sg++] = &hdr; 125 - if (pkt->buf) { 126 - sg_init_one(&buf, pkt->buf, pkt->len); 118 + if (skb->len > 0) { 119 + sg_init_one(&buf, skb->data, skb->len); 127 120 sgs[out_sg++] = &buf; 128 121 } 129 122 130 - ret = virtqueue_add_sgs(vq, sgs, out_sg, in_sg, pkt, GFP_KERNEL); 123 + ret = virtqueue_add_sgs(vq, sgs, out_sg, in_sg, skb, GFP_KERNEL); 131 124 /* Usually this means that there is no more space available in 132 125 * the vq 133 126 */ 134 127 if (ret < 0) { 135 - spin_lock_bh(&vsock->send_pkt_list_lock); 136 - list_add(&pkt->list, &vsock->send_pkt_list); 137 - spin_unlock_bh(&vsock->send_pkt_list_lock); 128 + virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 138 129 break; 139 130 } 140 131 ··· 153 164 } 154 165 155 166 static int 156 - virtio_transport_send_pkt(struct virtio_vsock_pkt *pkt) 167 + virtio_transport_send_pkt(struct sk_buff *skb) 157 168 { 169 + struct virtio_vsock_hdr *hdr; 158 170 struct virtio_vsock *vsock; 159 - int len = pkt->len; 171 + int len = skb->len; 172 + 173 + hdr = virtio_vsock_hdr(skb); 160 174 161 175 rcu_read_lock(); 162 176 vsock = rcu_dereference(the_virtio_vsock); 163 177 if (!vsock) { 164 - virtio_transport_free_pkt(pkt); 178 + kfree_skb(skb); 165 179 len = -ENODEV; 166 180 goto out_rcu; 167 181 } 168 182 169 - if (le64_to_cpu(pkt->hdr.dst_cid) == vsock->guest_cid) { 170 - virtio_transport_free_pkt(pkt); 183 + if (le64_to_cpu(hdr->dst_cid) == vsock->guest_cid) { 184 + kfree_skb(skb); 171 185 len = -ENODEV; 172 186 goto out_rcu; 173 187 } 174 188 175 - if (pkt->reply) 189 + if (virtio_vsock_skb_reply(skb)) 176 190 atomic_inc(&vsock->queued_replies); 177 191 178 - spin_lock_bh(&vsock->send_pkt_list_lock); 179 - list_add_tail(&pkt->list, &vsock->send_pkt_list); 180 - spin_unlock_bh(&vsock->send_pkt_list_lock); 181 - 192 + virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb); 182 193 queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work); 183 194 184 195 out_rcu: ··· 190 201 virtio_transport_cancel_pkt(struct vsock_sock *vsk) 191 202 { 192 203 struct virtio_vsock *vsock; 193 - struct virtio_vsock_pkt *pkt, *n; 194 204 int cnt = 0, ret; 195 - LIST_HEAD(freeme); 196 205 197 206 rcu_read_lock(); 198 207 vsock = rcu_dereference(the_virtio_vsock); ··· 199 212 goto out_rcu; 200 213 } 201 214 202 - spin_lock_bh(&vsock->send_pkt_list_lock); 203 - list_for_each_entry_safe(pkt, n, &vsock->send_pkt_list, list) { 204 - if (pkt->vsk != vsk) 205 - continue; 206 - list_move(&pkt->list, &freeme); 207 - } 208 - spin_unlock_bh(&vsock->send_pkt_list_lock); 209 - 210 - list_for_each_entry_safe(pkt, n, &freeme, list) { 211 - if (pkt->reply) 212 - cnt++; 213 - list_del(&pkt->list); 214 - virtio_transport_free_pkt(pkt); 215 - } 215 + cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue); 216 216 217 217 if (cnt) { 218 218 struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX]; ··· 220 246 221 247 static void virtio_vsock_rx_fill(struct virtio_vsock *vsock) 222 248 { 223 - int buf_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE; 224 - struct virtio_vsock_pkt *pkt; 225 - struct scatterlist hdr, buf, *sgs[2]; 249 + int total_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM; 250 + struct scatterlist pkt, *p; 226 251 struct virtqueue *vq; 252 + struct sk_buff *skb; 227 253 int ret; 228 254 229 255 vq = vsock->vqs[VSOCK_VQ_RX]; 230 256 231 257 do { 232 - pkt = kzalloc(sizeof(*pkt), GFP_KERNEL); 233 - if (!pkt) 258 + skb = virtio_vsock_alloc_skb(total_len, GFP_KERNEL); 259 + if (!skb) 234 260 break; 235 261 236 - pkt->buf = kmalloc(buf_len, GFP_KERNEL); 237 - if (!pkt->buf) { 238 - virtio_transport_free_pkt(pkt); 239 - break; 240 - } 241 - 242 - pkt->buf_len = buf_len; 243 - pkt->len = buf_len; 244 - 245 - sg_init_one(&hdr, &pkt->hdr, sizeof(pkt->hdr)); 246 - sgs[0] = &hdr; 247 - 248 - sg_init_one(&buf, pkt->buf, buf_len); 249 - sgs[1] = &buf; 250 - ret = virtqueue_add_sgs(vq, sgs, 0, 2, pkt, GFP_KERNEL); 251 - if (ret) { 252 - virtio_transport_free_pkt(pkt); 262 + memset(skb->head, 0, VIRTIO_VSOCK_SKB_HEADROOM); 263 + sg_init_one(&pkt, virtio_vsock_hdr(skb), total_len); 264 + p = &pkt; 265 + ret = virtqueue_add_sgs(vq, &p, 0, 1, skb, GFP_KERNEL); 266 + if (ret < 0) { 267 + kfree_skb(skb); 253 268 break; 254 269 } 270 + 255 271 vsock->rx_buf_nr++; 256 272 } while (vq->num_free); 257 273 if (vsock->rx_buf_nr > vsock->rx_buf_max_nr) ··· 263 299 goto out; 264 300 265 301 do { 266 - struct virtio_vsock_pkt *pkt; 302 + struct sk_buff *skb; 267 303 unsigned int len; 268 304 269 305 virtqueue_disable_cb(vq); 270 - while ((pkt = virtqueue_get_buf(vq, &len)) != NULL) { 271 - virtio_transport_free_pkt(pkt); 306 + while ((skb = virtqueue_get_buf(vq, &len)) != NULL) { 307 + consume_skb(skb); 272 308 added = true; 273 309 } 274 310 } while (!virtqueue_enable_cb(vq)); ··· 493 529 do { 494 530 virtqueue_disable_cb(vq); 495 531 for (;;) { 496 - struct virtio_vsock_pkt *pkt; 532 + struct sk_buff *skb; 497 533 unsigned int len; 498 534 499 535 if (!virtio_transport_more_replies(vsock)) { ··· 504 540 goto out; 505 541 } 506 542 507 - pkt = virtqueue_get_buf(vq, &len); 508 - if (!pkt) { 543 + skb = virtqueue_get_buf(vq, &len); 544 + if (!skb) 509 545 break; 510 - } 511 546 512 547 vsock->rx_buf_nr--; 513 548 514 549 /* Drop short/long packets */ 515 - if (unlikely(len < sizeof(pkt->hdr) || 516 - len > sizeof(pkt->hdr) + pkt->len)) { 517 - virtio_transport_free_pkt(pkt); 550 + if (unlikely(len < sizeof(struct virtio_vsock_hdr) || 551 + len > virtio_vsock_skb_len(skb))) { 552 + kfree_skb(skb); 518 553 continue; 519 554 } 520 555 521 - pkt->len = len - sizeof(pkt->hdr); 522 - virtio_transport_deliver_tap_pkt(pkt); 523 - virtio_transport_recv_pkt(&virtio_transport, pkt); 556 + virtio_vsock_skb_rx_put(skb); 557 + virtio_transport_deliver_tap_pkt(skb); 558 + virtio_transport_recv_pkt(&virtio_transport, skb); 524 559 } 525 560 } while (!virtqueue_enable_cb(vq)); 526 561 ··· 573 610 static void virtio_vsock_vqs_del(struct virtio_vsock *vsock) 574 611 { 575 612 struct virtio_device *vdev = vsock->vdev; 576 - struct virtio_vsock_pkt *pkt; 613 + struct sk_buff *skb; 577 614 578 615 /* Reset all connected sockets when the VQs disappear */ 579 616 vsock_for_each_connected_socket(&virtio_transport.transport, ··· 600 637 virtio_reset_device(vdev); 601 638 602 639 mutex_lock(&vsock->rx_lock); 603 - while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX]))) 604 - virtio_transport_free_pkt(pkt); 640 + while ((skb = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX]))) 641 + kfree_skb(skb); 605 642 mutex_unlock(&vsock->rx_lock); 606 643 607 644 mutex_lock(&vsock->tx_lock); 608 - while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX]))) 609 - virtio_transport_free_pkt(pkt); 645 + while ((skb = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX]))) 646 + kfree_skb(skb); 610 647 mutex_unlock(&vsock->tx_lock); 611 648 612 - spin_lock_bh(&vsock->send_pkt_list_lock); 613 - while (!list_empty(&vsock->send_pkt_list)) { 614 - pkt = list_first_entry(&vsock->send_pkt_list, 615 - struct virtio_vsock_pkt, list); 616 - list_del(&pkt->list); 617 - virtio_transport_free_pkt(pkt); 618 - } 619 - spin_unlock_bh(&vsock->send_pkt_list_lock); 649 + virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue); 620 650 621 651 /* Delete virtqueues and flush outstanding callbacks if any */ 622 652 vdev->config->del_vqs(vdev); ··· 646 690 mutex_init(&vsock->tx_lock); 647 691 mutex_init(&vsock->rx_lock); 648 692 mutex_init(&vsock->event_lock); 649 - spin_lock_init(&vsock->send_pkt_list_lock); 650 - INIT_LIST_HEAD(&vsock->send_pkt_list); 693 + skb_queue_head_init(&vsock->send_pkt_queue); 651 694 INIT_WORK(&vsock->rx_work, virtio_transport_rx_work); 652 695 INIT_WORK(&vsock->tx_work, virtio_transport_tx_work); 653 696 INIT_WORK(&vsock->event_work, virtio_transport_event_work);

+232 -188

net/vmw_vsock/virtio_transport_common.c

··· 37 37 return container_of(t, struct virtio_transport, transport); 38 38 } 39 39 40 - static struct virtio_vsock_pkt * 41 - virtio_transport_alloc_pkt(struct virtio_vsock_pkt_info *info, 40 + /* Returns a new packet on success, otherwise returns NULL. 41 + * 42 + * If NULL is returned, errp is set to a negative errno. 43 + */ 44 + static struct sk_buff * 45 + virtio_transport_alloc_skb(struct virtio_vsock_pkt_info *info, 42 46 size_t len, 43 47 u32 src_cid, 44 48 u32 src_port, 45 49 u32 dst_cid, 46 50 u32 dst_port) 47 51 { 48 - struct virtio_vsock_pkt *pkt; 52 + const size_t skb_len = VIRTIO_VSOCK_SKB_HEADROOM + len; 53 + struct virtio_vsock_hdr *hdr; 54 + struct sk_buff *skb; 55 + void *payload; 49 56 int err; 50 57 51 - pkt = kzalloc(sizeof(*pkt), GFP_KERNEL); 52 - if (!pkt) 58 + skb = virtio_vsock_alloc_skb(skb_len, GFP_KERNEL); 59 + if (!skb) 53 60 return NULL; 54 61 55 - pkt->hdr.type = cpu_to_le16(info->type); 56 - pkt->hdr.op = cpu_to_le16(info->op); 57 - pkt->hdr.src_cid = cpu_to_le64(src_cid); 58 - pkt->hdr.dst_cid = cpu_to_le64(dst_cid); 59 - pkt->hdr.src_port = cpu_to_le32(src_port); 60 - pkt->hdr.dst_port = cpu_to_le32(dst_port); 61 - pkt->hdr.flags = cpu_to_le32(info->flags); 62 - pkt->len = len; 63 - pkt->hdr.len = cpu_to_le32(len); 64 - pkt->reply = info->reply; 65 - pkt->vsk = info->vsk; 62 + hdr = virtio_vsock_hdr(skb); 63 + hdr->type = cpu_to_le16(info->type); 64 + hdr->op = cpu_to_le16(info->op); 65 + hdr->src_cid = cpu_to_le64(src_cid); 66 + hdr->dst_cid = cpu_to_le64(dst_cid); 67 + hdr->src_port = cpu_to_le32(src_port); 68 + hdr->dst_port = cpu_to_le32(dst_port); 69 + hdr->flags = cpu_to_le32(info->flags); 70 + hdr->len = cpu_to_le32(len); 66 71 67 72 if (info->msg && len > 0) { 68 - pkt->buf = kmalloc(len, GFP_KERNEL); 69 - if (!pkt->buf) 70 - goto out_pkt; 71 - 72 - pkt->buf_len = len; 73 - 74 - err = memcpy_from_msg(pkt->buf, info->msg, len); 73 + payload = skb_put(skb, len); 74 + err = memcpy_from_msg(payload, info->msg, len); 75 75 if (err) 76 76 goto out; 77 77 78 78 if (msg_data_left(info->msg) == 0 && 79 79 info->type == VIRTIO_VSOCK_TYPE_SEQPACKET) { 80 - pkt->hdr.flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM); 80 + hdr->flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM); 81 81 82 82 if (info->msg->msg_flags & MSG_EOR) 83 - pkt->hdr.flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR); 83 + hdr->flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR); 84 84 } 85 85 } 86 + 87 + if (info->reply) 88 + virtio_vsock_skb_set_reply(skb); 86 89 87 90 trace_virtio_transport_alloc_pkt(src_cid, src_port, 88 91 dst_cid, dst_port, ··· 94 91 info->op, 95 92 info->flags); 96 93 97 - return pkt; 94 + return skb; 98 95 99 96 out: 100 - kfree(pkt->buf); 101 - out_pkt: 102 - kfree(pkt); 97 + kfree_skb(skb); 103 98 return NULL; 104 99 } 105 100 106 101 /* Packet capture */ 107 102 static struct sk_buff *virtio_transport_build_skb(void *opaque) 108 103 { 109 - struct virtio_vsock_pkt *pkt = opaque; 104 + struct virtio_vsock_hdr *pkt_hdr; 105 + struct sk_buff *pkt = opaque; 110 106 struct af_vsockmon_hdr *hdr; 111 107 struct sk_buff *skb; 112 108 size_t payload_len; ··· 115 113 * the payload length from the header and the buffer pointer taking 116 114 * care of the offset in the original packet. 117 115 */ 118 - payload_len = le32_to_cpu(pkt->hdr.len); 119 - payload_buf = pkt->buf + pkt->off; 116 + pkt_hdr = virtio_vsock_hdr(pkt); 117 + payload_len = pkt->len; 118 + payload_buf = pkt->data; 120 119 121 - skb = alloc_skb(sizeof(*hdr) + sizeof(pkt->hdr) + payload_len, 120 + skb = alloc_skb(sizeof(*hdr) + sizeof(*pkt_hdr) + payload_len, 122 121 GFP_ATOMIC); 123 122 if (!skb) 124 123 return NULL; ··· 127 124 hdr = skb_put(skb, sizeof(*hdr)); 128 125 129 126 /* pkt->hdr is little-endian so no need to byteswap here */ 130 - hdr->src_cid = pkt->hdr.src_cid; 131 - hdr->src_port = pkt->hdr.src_port; 132 - hdr->dst_cid = pkt->hdr.dst_cid; 133 - hdr->dst_port = pkt->hdr.dst_port; 127 + hdr->src_cid = pkt_hdr->src_cid; 128 + hdr->src_port = pkt_hdr->src_port; 129 + hdr->dst_cid = pkt_hdr->dst_cid; 130 + hdr->dst_port = pkt_hdr->dst_port; 134 131 135 132 hdr->transport = cpu_to_le16(AF_VSOCK_TRANSPORT_VIRTIO); 136 - hdr->len = cpu_to_le16(sizeof(pkt->hdr)); 133 + hdr->len = cpu_to_le16(sizeof(*pkt_hdr)); 137 134 memset(hdr->reserved, 0, sizeof(hdr->reserved)); 138 135 139 - switch (le16_to_cpu(pkt->hdr.op)) { 136 + switch (le16_to_cpu(pkt_hdr->op)) { 140 137 case VIRTIO_VSOCK_OP_REQUEST: 141 138 case VIRTIO_VSOCK_OP_RESPONSE: 142 139 hdr->op = cpu_to_le16(AF_VSOCK_OP_CONNECT); ··· 157 154 break; 158 155 } 159 156 160 - skb_put_data(skb, &pkt->hdr, sizeof(pkt->hdr)); 157 + skb_put_data(skb, pkt_hdr, sizeof(*pkt_hdr)); 161 158 162 159 if (payload_len) { 163 160 skb_put_data(skb, payload_buf, payload_len); ··· 166 163 return skb; 167 164 } 168 165 169 - void virtio_transport_deliver_tap_pkt(struct virtio_vsock_pkt *pkt) 166 + void virtio_transport_deliver_tap_pkt(struct sk_buff *skb) 170 167 { 171 - if (pkt->tap_delivered) 168 + if (virtio_vsock_skb_tap_delivered(skb)) 172 169 return; 173 170 174 - vsock_deliver_tap(virtio_transport_build_skb, pkt); 175 - pkt->tap_delivered = true; 171 + vsock_deliver_tap(virtio_transport_build_skb, skb); 172 + virtio_vsock_skb_set_tap_delivered(skb); 176 173 } 177 174 EXPORT_SYMBOL_GPL(virtio_transport_deliver_tap_pkt); 178 175 ··· 195 192 u32 src_cid, src_port, dst_cid, dst_port; 196 193 const struct virtio_transport *t_ops; 197 194 struct virtio_vsock_sock *vvs; 198 - struct virtio_vsock_pkt *pkt; 199 195 u32 pkt_len = info->pkt_len; 196 + struct sk_buff *skb; 200 197 201 198 info->type = virtio_transport_get_type(sk_vsock(vsk)); 202 199 ··· 227 224 if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW) 228 225 return pkt_len; 229 226 230 - pkt = virtio_transport_alloc_pkt(info, pkt_len, 227 + skb = virtio_transport_alloc_skb(info, pkt_len, 231 228 src_cid, src_port, 232 229 dst_cid, dst_port); 233 - if (!pkt) { 230 + if (!skb) { 234 231 virtio_transport_put_credit(vvs, pkt_len); 235 232 return -ENOMEM; 236 233 } 237 234 238 - virtio_transport_inc_tx_pkt(vvs, pkt); 235 + virtio_transport_inc_tx_pkt(vvs, skb); 239 236 240 - return t_ops->send_pkt(pkt); 237 + return t_ops->send_pkt(skb); 241 238 } 242 239 243 240 static bool virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs, 244 - struct virtio_vsock_pkt *pkt) 241 + struct sk_buff *skb) 245 242 { 246 - if (vvs->rx_bytes + pkt->len > vvs->buf_alloc) 243 + if (vvs->rx_bytes + skb->len > vvs->buf_alloc) 247 244 return false; 248 245 249 - vvs->rx_bytes += pkt->len; 246 + vvs->rx_bytes += skb->len; 250 247 return true; 251 248 } 252 249 253 250 static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs, 254 - struct virtio_vsock_pkt *pkt) 251 + struct sk_buff *skb) 255 252 { 256 - vvs->rx_bytes -= pkt->len; 257 - vvs->fwd_cnt += pkt->len; 253 + int len; 254 + 255 + len = skb_headroom(skb) - sizeof(struct virtio_vsock_hdr) - skb->len; 256 + vvs->rx_bytes -= len; 257 + vvs->fwd_cnt += len; 258 258 } 259 259 260 - void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt) 260 + void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct sk_buff *skb) 261 261 { 262 + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 263 + 262 264 spin_lock_bh(&vvs->rx_lock); 263 265 vvs->last_fwd_cnt = vvs->fwd_cnt; 264 - pkt->hdr.fwd_cnt = cpu_to_le32(vvs->fwd_cnt); 265 - pkt->hdr.buf_alloc = cpu_to_le32(vvs->buf_alloc); 266 + hdr->fwd_cnt = cpu_to_le32(vvs->fwd_cnt); 267 + hdr->buf_alloc = cpu_to_le32(vvs->buf_alloc); 266 268 spin_unlock_bh(&vvs->rx_lock); 267 269 } 268 270 EXPORT_SYMBOL_GPL(virtio_transport_inc_tx_pkt); ··· 311 303 size_t len) 312 304 { 313 305 struct virtio_vsock_sock *vvs = vsk->trans; 314 - struct virtio_vsock_pkt *pkt; 315 306 size_t bytes, total = 0, off; 307 + struct sk_buff *skb, *tmp; 316 308 int err = -EFAULT; 317 309 318 310 spin_lock_bh(&vvs->rx_lock); 319 311 320 - list_for_each_entry(pkt, &vvs->rx_queue, list) { 321 - off = pkt->off; 312 + skb_queue_walk_safe(&vvs->rx_queue, skb, tmp) { 313 + off = 0; 322 314 323 315 if (total == len) 324 316 break; 325 317 326 - while (total < len && off < pkt->len) { 318 + while (total < len && off < skb->len) { 327 319 bytes = len - total; 328 - if (bytes > pkt->len - off) 329 - bytes = pkt->len - off; 320 + if (bytes > skb->len - off) 321 + bytes = skb->len - off; 330 322 331 323 /* sk_lock is held by caller so no one else can dequeue. 332 324 * Unlock rx_lock since memcpy_to_msg() may sleep. 333 325 */ 334 326 spin_unlock_bh(&vvs->rx_lock); 335 327 336 - err = memcpy_to_msg(msg, pkt->buf + off, bytes); 328 + err = memcpy_to_msg(msg, skb->data + off, bytes); 337 329 if (err) 338 330 goto out; 339 331 ··· 360 352 size_t len) 361 353 { 362 354 struct virtio_vsock_sock *vvs = vsk->trans; 363 - struct virtio_vsock_pkt *pkt; 364 355 size_t bytes, total = 0; 365 - u32 free_space; 356 + struct sk_buff *skb; 366 357 int err = -EFAULT; 358 + u32 free_space; 367 359 368 360 spin_lock_bh(&vvs->rx_lock); 369 - while (total < len && !list_empty(&vvs->rx_queue)) { 370 - pkt = list_first_entry(&vvs->rx_queue, 371 - struct virtio_vsock_pkt, list); 361 + while (total < len && !skb_queue_empty(&vvs->rx_queue)) { 362 + skb = __skb_dequeue(&vvs->rx_queue); 372 363 373 364 bytes = len - total; 374 - if (bytes > pkt->len - pkt->off) 375 - bytes = pkt->len - pkt->off; 365 + if (bytes > skb->len) 366 + bytes = skb->len; 376 367 377 368 /* sk_lock is held by caller so no one else can dequeue. 378 369 * Unlock rx_lock since memcpy_to_msg() may sleep. 379 370 */ 380 371 spin_unlock_bh(&vvs->rx_lock); 381 372 382 - err = memcpy_to_msg(msg, pkt->buf + pkt->off, bytes); 373 + err = memcpy_to_msg(msg, skb->data, bytes); 383 374 if (err) 384 375 goto out; 385 376 386 377 spin_lock_bh(&vvs->rx_lock); 387 378 388 379 total += bytes; 389 - pkt->off += bytes; 390 - if (pkt->off == pkt->len) { 391 - virtio_transport_dec_rx_pkt(vvs, pkt); 392 - list_del(&pkt->list); 393 - virtio_transport_free_pkt(pkt); 380 + skb_pull(skb, bytes); 381 + 382 + if (skb->len == 0) { 383 + virtio_transport_dec_rx_pkt(vvs, skb); 384 + consume_skb(skb); 385 + } else { 386 + __skb_queue_head(&vvs->rx_queue, skb); 394 387 } 395 388 } 396 389 ··· 423 414 int flags) 424 415 { 425 416 struct virtio_vsock_sock *vvs = vsk->trans; 426 - struct virtio_vsock_pkt *pkt; 427 417 int dequeued_len = 0; 428 418 size_t user_buf_len = msg_data_left(msg); 429 419 bool msg_ready = false; 420 + struct sk_buff *skb; 430 421 431 422 spin_lock_bh(&vvs->rx_lock); 432 423 ··· 436 427 } 437 428 438 429 while (!msg_ready) { 439 - pkt = list_first_entry(&vvs->rx_queue, struct virtio_vsock_pkt, list); 430 + struct virtio_vsock_hdr *hdr; 431 + 432 + skb = __skb_dequeue(&vvs->rx_queue); 433 + if (!skb) 434 + break; 435 + hdr = virtio_vsock_hdr(skb); 440 436 441 437 if (dequeued_len >= 0) { 442 438 size_t pkt_len; 443 439 size_t bytes_to_copy; 444 440 445 - pkt_len = (size_t)le32_to_cpu(pkt->hdr.len); 441 + pkt_len = (size_t)le32_to_cpu(hdr->len); 446 442 bytes_to_copy = min(user_buf_len, pkt_len); 447 443 448 444 if (bytes_to_copy) { ··· 458 444 */ 459 445 spin_unlock_bh(&vvs->rx_lock); 460 446 461 - err = memcpy_to_msg(msg, pkt->buf, bytes_to_copy); 447 + err = memcpy_to_msg(msg, skb->data, bytes_to_copy); 462 448 if (err) { 463 449 /* Copy of message failed. Rest of 464 450 * fragments will be freed without copy. ··· 466 452 dequeued_len = err; 467 453 } else { 468 454 user_buf_len -= bytes_to_copy; 455 + skb_pull(skb, bytes_to_copy); 469 456 } 470 457 471 458 spin_lock_bh(&vvs->rx_lock); ··· 476 461 dequeued_len += pkt_len; 477 462 } 478 463 479 - if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SEQ_EOM) { 464 + if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) { 480 465 msg_ready = true; 481 466 vvs->msg_count--; 482 467 483 - if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SEQ_EOR) 468 + if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) 484 469 msg->msg_flags |= MSG_EOR; 485 470 } 486 471 487 - virtio_transport_dec_rx_pkt(vvs, pkt); 488 - list_del(&pkt->list); 489 - virtio_transport_free_pkt(pkt); 472 + virtio_transport_dec_rx_pkt(vvs, skb); 473 + kfree_skb(skb); 490 474 } 491 475 492 476 spin_unlock_bh(&vvs->rx_lock); ··· 623 609 624 610 spin_lock_init(&vvs->rx_lock); 625 611 spin_lock_init(&vvs->tx_lock); 626 - INIT_LIST_HEAD(&vvs->rx_queue); 612 + skb_queue_head_init(&vvs->rx_queue); 627 613 628 614 return 0; 629 615 } ··· 820 806 EXPORT_SYMBOL_GPL(virtio_transport_destruct); 821 807 822 808 static int virtio_transport_reset(struct vsock_sock *vsk, 823 - struct virtio_vsock_pkt *pkt) 809 + struct sk_buff *skb) 824 810 { 825 811 struct virtio_vsock_pkt_info info = { 826 812 .op = VIRTIO_VSOCK_OP_RST, 827 - .reply = !!pkt, 813 + .reply = !!skb, 828 814 .vsk = vsk, 829 815 }; 830 816 831 817 /* Send RST only if the original pkt is not a RST pkt */ 832 - if (pkt && le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 818 + if (skb && le16_to_cpu(virtio_vsock_hdr(skb)->op) == VIRTIO_VSOCK_OP_RST) 833 819 return 0; 834 820 835 821 return virtio_transport_send_pkt_info(vsk, &info); ··· 839 825 * attempt was made to connect to a socket that does not exist. 840 826 */ 841 827 static int virtio_transport_reset_no_sock(const struct virtio_transport *t, 842 - struct virtio_vsock_pkt *pkt) 828 + struct sk_buff *skb) 843 829 { 844 - struct virtio_vsock_pkt *reply; 830 + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 845 831 struct virtio_vsock_pkt_info info = { 846 832 .op = VIRTIO_VSOCK_OP_RST, 847 - .type = le16_to_cpu(pkt->hdr.type), 833 + .type = le16_to_cpu(hdr->type), 848 834 .reply = true, 849 835 }; 836 + struct sk_buff *reply; 850 837 851 838 /* Send RST only if the original pkt is not a RST pkt */ 852 - if (le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 839 + if (le16_to_cpu(hdr->op) == VIRTIO_VSOCK_OP_RST) 853 840 return 0; 854 841 855 - reply = virtio_transport_alloc_pkt(&info, 0, 856 - le64_to_cpu(pkt->hdr.dst_cid), 857 - le32_to_cpu(pkt->hdr.dst_port), 858 - le64_to_cpu(pkt->hdr.src_cid), 859 - le32_to_cpu(pkt->hdr.src_port)); 842 + reply = virtio_transport_alloc_skb(&info, 0, 843 + le64_to_cpu(hdr->dst_cid), 844 + le32_to_cpu(hdr->dst_port), 845 + le64_to_cpu(hdr->src_cid), 846 + le32_to_cpu(hdr->src_port)); 860 847 if (!reply) 861 848 return -ENOMEM; 862 849 863 850 if (!t) { 864 - virtio_transport_free_pkt(reply); 851 + kfree_skb(reply); 865 852 return -ENOTCONN; 866 853 } 867 854 ··· 873 858 static void virtio_transport_remove_sock(struct vsock_sock *vsk) 874 859 { 875 860 struct virtio_vsock_sock *vvs = vsk->trans; 876 - struct virtio_vsock_pkt *pkt, *tmp; 877 861 878 862 /* We don't need to take rx_lock, as the socket is closing and we are 879 863 * removing it. 880 864 */ 881 - list_for_each_entry_safe(pkt, tmp, &vvs->rx_queue, list) { 882 - list_del(&pkt->list); 883 - virtio_transport_free_pkt(pkt); 884 - } 885 - 865 + __skb_queue_purge(&vvs->rx_queue); 886 866 vsock_remove_sock(vsk); 887 867 } 888 868 ··· 991 981 992 982 static int 993 983 virtio_transport_recv_connecting(struct sock *sk, 994 - struct virtio_vsock_pkt *pkt) 984 + struct sk_buff *skb) 995 985 { 986 + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 996 987 struct vsock_sock *vsk = vsock_sk(sk); 997 - int err; 998 988 int skerr; 989 + int err; 999 990 1000 - switch (le16_to_cpu(pkt->hdr.op)) { 991 + switch (le16_to_cpu(hdr->op)) { 1001 992 case VIRTIO_VSOCK_OP_RESPONSE: 1002 993 sk->sk_state = TCP_ESTABLISHED; 1003 994 sk->sk_socket->state = SS_CONNECTED; ··· 1019 1008 return 0; 1020 1009 1021 1010 destroy: 1022 - virtio_transport_reset(vsk, pkt); 1011 + virtio_transport_reset(vsk, skb); 1023 1012 sk->sk_state = TCP_CLOSE; 1024 1013 sk->sk_err = skerr; 1025 1014 sk_error_report(sk); ··· 1028 1017 1029 1018 static void 1030 1019 virtio_transport_recv_enqueue(struct vsock_sock *vsk, 1031 - struct virtio_vsock_pkt *pkt) 1020 + struct sk_buff *skb) 1032 1021 { 1033 1022 struct virtio_vsock_sock *vvs = vsk->trans; 1034 1023 bool can_enqueue, free_pkt = false; 1024 + struct virtio_vsock_hdr *hdr; 1025 + u32 len; 1035 1026 1036 - pkt->len = le32_to_cpu(pkt->hdr.len); 1037 - pkt->off = 0; 1027 + hdr = virtio_vsock_hdr(skb); 1028 + len = le32_to_cpu(hdr->len); 1038 1029 1039 1030 spin_lock_bh(&vvs->rx_lock); 1040 1031 1041 - can_enqueue = virtio_transport_inc_rx_pkt(vvs, pkt); 1032 + can_enqueue = virtio_transport_inc_rx_pkt(vvs, skb); 1042 1033 if (!can_enqueue) { 1043 1034 free_pkt = true; 1044 1035 goto out; 1045 1036 } 1046 1037 1047 - if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SEQ_EOM) 1038 + if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) 1048 1039 vvs->msg_count++; 1049 1040 1050 1041 /* Try to copy small packets into the buffer of last packet queued, 1051 1042 * to avoid wasting memory queueing the entire buffer with a small 1052 1043 * payload. 1053 1044 */ 1054 - if (pkt->len <= GOOD_COPY_LEN && !list_empty(&vvs->rx_queue)) { 1055 - struct virtio_vsock_pkt *last_pkt; 1045 + if (len <= GOOD_COPY_LEN && !skb_queue_empty(&vvs->rx_queue)) { 1046 + struct virtio_vsock_hdr *last_hdr; 1047 + struct sk_buff *last_skb; 1056 1048 1057 - last_pkt = list_last_entry(&vvs->rx_queue, 1058 - struct virtio_vsock_pkt, list); 1049 + last_skb = skb_peek_tail(&vvs->rx_queue); 1050 + last_hdr = virtio_vsock_hdr(last_skb); 1059 1051 1060 1052 /* If there is space in the last packet queued, we copy the 1061 1053 * new packet in its buffer. We avoid this if the last packet ··· 1066 1052 * delimiter of SEQPACKET message, so 'pkt' is the first packet 1067 1053 * of a new message. 1068 1054 */ 1069 - if ((pkt->len <= last_pkt->buf_len - last_pkt->len) && 1070 - !(le32_to_cpu(last_pkt->hdr.flags) & VIRTIO_VSOCK_SEQ_EOM)) { 1071 - memcpy(last_pkt->buf + last_pkt->len, pkt->buf, 1072 - pkt->len); 1073 - last_pkt->len += pkt->len; 1055 + if (skb->len < skb_tailroom(last_skb) && 1056 + !(le32_to_cpu(last_hdr->flags) & VIRTIO_VSOCK_SEQ_EOM)) { 1057 + memcpy(skb_put(last_skb, skb->len), skb->data, skb->len); 1074 1058 free_pkt = true; 1075 - last_pkt->hdr.flags |= pkt->hdr.flags; 1059 + last_hdr->flags |= hdr->flags; 1060 + last_hdr->len = cpu_to_le32(last_skb->len); 1076 1061 goto out; 1077 1062 } 1078 1063 } 1079 1064 1080 - list_add_tail(&pkt->list, &vvs->rx_queue); 1065 + __skb_queue_tail(&vvs->rx_queue, skb); 1081 1066 1082 1067 out: 1083 1068 spin_unlock_bh(&vvs->rx_lock); 1084 1069 if (free_pkt) 1085 - virtio_transport_free_pkt(pkt); 1070 + kfree_skb(skb); 1086 1071 } 1087 1072 1088 1073 static int 1089 1074 virtio_transport_recv_connected(struct sock *sk, 1090 - struct virtio_vsock_pkt *pkt) 1075 + struct sk_buff *skb) 1091 1076 { 1077 + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1092 1078 struct vsock_sock *vsk = vsock_sk(sk); 1093 1079 int err = 0; 1094 1080 1095 - switch (le16_to_cpu(pkt->hdr.op)) { 1081 + switch (le16_to_cpu(hdr->op)) { 1096 1082 case VIRTIO_VSOCK_OP_RW: 1097 - virtio_transport_recv_enqueue(vsk, pkt); 1083 + virtio_transport_recv_enqueue(vsk, skb); 1098 1084 vsock_data_ready(sk); 1099 1085 return err; 1100 1086 case VIRTIO_VSOCK_OP_CREDIT_REQUEST: ··· 1104 1090 sk->sk_write_space(sk); 1105 1091 break; 1106 1092 case VIRTIO_VSOCK_OP_SHUTDOWN: 1107 - if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SHUTDOWN_RCV) 1093 + if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_RCV) 1108 1094 vsk->peer_shutdown |= RCV_SHUTDOWN; 1109 - if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SHUTDOWN_SEND) 1095 + if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_SEND) 1110 1096 vsk->peer_shutdown |= SEND_SHUTDOWN; 1111 1097 if (vsk->peer_shutdown == SHUTDOWN_MASK && 1112 1098 vsock_stream_has_data(vsk) <= 0 && 1113 1099 !sock_flag(sk, SOCK_DONE)) { 1114 1100 (void)virtio_transport_reset(vsk, NULL); 1115 - 1116 1101 virtio_transport_do_close(vsk, true); 1117 1102 } 1118 - if (le32_to_cpu(pkt->hdr.flags)) 1103 + if (le32_to_cpu(virtio_vsock_hdr(skb)->flags)) 1119 1104 sk->sk_state_change(sk); 1120 1105 break; 1121 1106 case VIRTIO_VSOCK_OP_RST: ··· 1125 1112 break; 1126 1113 } 1127 1114 1128 - virtio_transport_free_pkt(pkt); 1115 + kfree_skb(skb); 1129 1116 return err; 1130 1117 } 1131 1118 1132 1119 static void 1133 1120 virtio_transport_recv_disconnecting(struct sock *sk, 1134 - struct virtio_vsock_pkt *pkt) 1121 + struct sk_buff *skb) 1135 1122 { 1123 + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1136 1124 struct vsock_sock *vsk = vsock_sk(sk); 1137 1125 1138 - if (le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 1126 + if (le16_to_cpu(hdr->op) == VIRTIO_VSOCK_OP_RST) 1139 1127 virtio_transport_do_close(vsk, true); 1140 1128 } 1141 1129 1142 1130 static int 1143 1131 virtio_transport_send_response(struct vsock_sock *vsk, 1144 - struct virtio_vsock_pkt *pkt) 1132 + struct sk_buff *skb) 1145 1133 { 1134 + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1146 1135 struct virtio_vsock_pkt_info info = { 1147 1136 .op = VIRTIO_VSOCK_OP_RESPONSE, 1148 - .remote_cid = le64_to_cpu(pkt->hdr.src_cid), 1149 - .remote_port = le32_to_cpu(pkt->hdr.src_port), 1137 + .remote_cid = le64_to_cpu(hdr->src_cid), 1138 + .remote_port = le32_to_cpu(hdr->src_port), 1150 1139 .reply = true, 1151 1140 .vsk = vsk, 1152 1141 }; ··· 1157 1142 } 1158 1143 1159 1144 static bool virtio_transport_space_update(struct sock *sk, 1160 - struct virtio_vsock_pkt *pkt) 1145 + struct sk_buff *skb) 1161 1146 { 1147 + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1162 1148 struct vsock_sock *vsk = vsock_sk(sk); 1163 1149 struct virtio_vsock_sock *vvs = vsk->trans; 1164 1150 bool space_available; ··· 1174 1158 1175 1159 /* buf_alloc and fwd_cnt is always included in the hdr */ 1176 1160 spin_lock_bh(&vvs->tx_lock); 1177 - vvs->peer_buf_alloc = le32_to_cpu(pkt->hdr.buf_alloc); 1178 - vvs->peer_fwd_cnt = le32_to_cpu(pkt->hdr.fwd_cnt); 1161 + vvs->peer_buf_alloc = le32_to_cpu(hdr->buf_alloc); 1162 + vvs->peer_fwd_cnt = le32_to_cpu(hdr->fwd_cnt); 1179 1163 space_available = virtio_transport_has_space(vsk); 1180 1164 spin_unlock_bh(&vvs->tx_lock); 1181 1165 return space_available; ··· 1183 1167 1184 1168 /* Handle server socket */ 1185 1169 static int 1186 - virtio_transport_recv_listen(struct sock *sk, struct virtio_vsock_pkt *pkt, 1170 + virtio_transport_recv_listen(struct sock *sk, struct sk_buff *skb, 1187 1171 struct virtio_transport *t) 1188 1172 { 1173 + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1189 1174 struct vsock_sock *vsk = vsock_sk(sk); 1190 1175 struct vsock_sock *vchild; 1191 1176 struct sock *child; 1192 1177 int ret; 1193 1178 1194 - if (le16_to_cpu(pkt->hdr.op) != VIRTIO_VSOCK_OP_REQUEST) { 1195 - virtio_transport_reset_no_sock(t, pkt); 1179 + if (le16_to_cpu(hdr->op) != VIRTIO_VSOCK_OP_REQUEST) { 1180 + virtio_transport_reset_no_sock(t, skb); 1196 1181 return -EINVAL; 1197 1182 } 1198 1183 1199 1184 if (sk_acceptq_is_full(sk)) { 1200 - virtio_transport_reset_no_sock(t, pkt); 1185 + virtio_transport_reset_no_sock(t, skb); 1201 1186 return -ENOMEM; 1202 1187 } 1203 1188 1204 1189 child = vsock_create_connected(sk); 1205 1190 if (!child) { 1206 - virtio_transport_reset_no_sock(t, pkt); 1191 + virtio_transport_reset_no_sock(t, skb); 1207 1192 return -ENOMEM; 1208 1193 } 1209 1194 ··· 1215 1198 child->sk_state = TCP_ESTABLISHED; 1216 1199 1217 1200 vchild = vsock_sk(child); 1218 - vsock_addr_init(&vchild->local_addr, le64_to_cpu(pkt->hdr.dst_cid), 1219 - le32_to_cpu(pkt->hdr.dst_port)); 1220 - vsock_addr_init(&vchild->remote_addr, le64_to_cpu(pkt->hdr.src_cid), 1221 - le32_to_cpu(pkt->hdr.src_port)); 1201 + vsock_addr_init(&vchild->local_addr, le64_to_cpu(hdr->dst_cid), 1202 + le32_to_cpu(hdr->dst_port)); 1203 + vsock_addr_init(&vchild->remote_addr, le64_to_cpu(hdr->src_cid), 1204 + le32_to_cpu(hdr->src_port)); 1222 1205 1223 1206 ret = vsock_assign_transport(vchild, vsk); 1224 1207 /* Transport assigned (looking at remote_addr) must be the same ··· 1226 1209 */ 1227 1210 if (ret || vchild->transport != &t->transport) { 1228 1211 release_sock(child); 1229 - virtio_transport_reset_no_sock(t, pkt); 1212 + virtio_transport_reset_no_sock(t, skb); 1230 1213 sock_put(child); 1231 1214 return ret; 1232 1215 } 1233 1216 1234 - if (virtio_transport_space_update(child, pkt)) 1217 + if (virtio_transport_space_update(child, skb)) 1235 1218 child->sk_write_space(child); 1236 1219 1237 1220 vsock_insert_connected(vchild); 1238 1221 vsock_enqueue_accept(sk, child); 1239 - virtio_transport_send_response(vchild, pkt); 1222 + virtio_transport_send_response(vchild, skb); 1240 1223 1241 1224 release_sock(child); 1242 1225 ··· 1254 1237 * lock. 1255 1238 */ 1256 1239 void virtio_transport_recv_pkt(struct virtio_transport *t, 1257 - struct virtio_vsock_pkt *pkt) 1240 + struct sk_buff *skb) 1258 1241 { 1242 + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1259 1243 struct sockaddr_vm src, dst; 1260 1244 struct vsock_sock *vsk; 1261 1245 struct sock *sk; 1262 1246 bool space_available; 1263 1247 1264 - vsock_addr_init(&src, le64_to_cpu(pkt->hdr.src_cid), 1265 - le32_to_cpu(pkt->hdr.src_port)); 1266 - vsock_addr_init(&dst, le64_to_cpu(pkt->hdr.dst_cid), 1267 - le32_to_cpu(pkt->hdr.dst_port)); 1248 + vsock_addr_init(&src, le64_to_cpu(hdr->src_cid), 1249 + le32_to_cpu(hdr->src_port)); 1250 + vsock_addr_init(&dst, le64_to_cpu(hdr->dst_cid), 1251 + le32_to_cpu(hdr->dst_port)); 1268 1252 1269 1253 trace_virtio_transport_recv_pkt(src.svm_cid, src.svm_port, 1270 1254 dst.svm_cid, dst.svm_port, 1271 - le32_to_cpu(pkt->hdr.len), 1272 - le16_to_cpu(pkt->hdr.type), 1273 - le16_to_cpu(pkt->hdr.op), 1274 - le32_to_cpu(pkt->hdr.flags), 1275 - le32_to_cpu(pkt->hdr.buf_alloc), 1276 - le32_to_cpu(pkt->hdr.fwd_cnt)); 1255 + le32_to_cpu(hdr->len), 1256 + le16_to_cpu(hdr->type), 1257 + le16_to_cpu(hdr->op), 1258 + le32_to_cpu(hdr->flags), 1259 + le32_to_cpu(hdr->buf_alloc), 1260 + le32_to_cpu(hdr->fwd_cnt)); 1277 1261 1278 - if (!virtio_transport_valid_type(le16_to_cpu(pkt->hdr.type))) { 1279 - (void)virtio_transport_reset_no_sock(t, pkt); 1262 + if (!virtio_transport_valid_type(le16_to_cpu(hdr->type))) { 1263 + (void)virtio_transport_reset_no_sock(t, skb); 1280 1264 goto free_pkt; 1281 1265 } 1282 1266 ··· 1288 1270 if (!sk) { 1289 1271 sk = vsock_find_bound_socket(&dst); 1290 1272 if (!sk) { 1291 - (void)virtio_transport_reset_no_sock(t, pkt); 1273 + (void)virtio_transport_reset_no_sock(t, skb); 1292 1274 goto free_pkt; 1293 1275 } 1294 1276 } 1295 1277 1296 - if (virtio_transport_get_type(sk) != le16_to_cpu(pkt->hdr.type)) { 1297 - (void)virtio_transport_reset_no_sock(t, pkt); 1278 + if (virtio_transport_get_type(sk) != le16_to_cpu(hdr->type)) { 1279 + (void)virtio_transport_reset_no_sock(t, skb); 1298 1280 sock_put(sk); 1299 1281 goto free_pkt; 1300 1282 } ··· 1305 1287 1306 1288 /* Check if sk has been closed before lock_sock */ 1307 1289 if (sock_flag(sk, SOCK_DONE)) { 1308 - (void)virtio_transport_reset_no_sock(t, pkt); 1290 + (void)virtio_transport_reset_no_sock(t, skb); 1309 1291 release_sock(sk); 1310 1292 sock_put(sk); 1311 1293 goto free_pkt; 1312 1294 } 1313 1295 1314 - space_available = virtio_transport_space_update(sk, pkt); 1296 + space_available = virtio_transport_space_update(sk, skb); 1315 1297 1316 1298 /* Update CID in case it has changed after a transport reset event */ 1317 1299 if (vsk->local_addr.svm_cid != VMADDR_CID_ANY) ··· 1322 1304 1323 1305 switch (sk->sk_state) { 1324 1306 case TCP_LISTEN: 1325 - virtio_transport_recv_listen(sk, pkt, t); 1326 - virtio_transport_free_pkt(pkt); 1307 + virtio_transport_recv_listen(sk, skb, t); 1308 + kfree_skb(skb); 1327 1309 break; 1328 1310 case TCP_SYN_SENT: 1329 - virtio_transport_recv_connecting(sk, pkt); 1330 - virtio_transport_free_pkt(pkt); 1311 + virtio_transport_recv_connecting(sk, skb); 1312 + kfree_skb(skb); 1331 1313 break; 1332 1314 case TCP_ESTABLISHED: 1333 - virtio_transport_recv_connected(sk, pkt); 1315 + virtio_transport_recv_connected(sk, skb); 1334 1316 break; 1335 1317 case TCP_CLOSING: 1336 - virtio_transport_recv_disconnecting(sk, pkt); 1337 - virtio_transport_free_pkt(pkt); 1318 + virtio_transport_recv_disconnecting(sk, skb); 1319 + kfree_skb(skb); 1338 1320 break; 1339 1321 default: 1340 - (void)virtio_transport_reset_no_sock(t, pkt); 1341 - virtio_transport_free_pkt(pkt); 1322 + (void)virtio_transport_reset_no_sock(t, skb); 1323 + kfree_skb(skb); 1342 1324 break; 1343 1325 } 1344 1326 ··· 1351 1333 return; 1352 1334 1353 1335 free_pkt: 1354 - virtio_transport_free_pkt(pkt); 1336 + kfree_skb(skb); 1355 1337 } 1356 1338 EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt); 1357 1339 1358 - void virtio_transport_free_pkt(struct virtio_vsock_pkt *pkt) 1340 + /* Remove skbs found in a queue that have a vsk that matches. 1341 + * 1342 + * Each skb is freed. 1343 + * 1344 + * Returns the count of skbs that were reply packets. 1345 + */ 1346 + int virtio_transport_purge_skbs(void *vsk, struct sk_buff_head *queue) 1359 1347 { 1360 - kvfree(pkt->buf); 1361 - kfree(pkt); 1348 + struct sk_buff_head freeme; 1349 + struct sk_buff *skb, *tmp; 1350 + int cnt = 0; 1351 + 1352 + skb_queue_head_init(&freeme); 1353 + 1354 + spin_lock_bh(&queue->lock); 1355 + skb_queue_walk_safe(queue, skb, tmp) { 1356 + if (vsock_sk(skb->sk) != vsk) 1357 + continue; 1358 + 1359 + __skb_unlink(skb, queue); 1360 + __skb_queue_tail(&freeme, skb); 1361 + 1362 + if (virtio_vsock_skb_reply(skb)) 1363 + cnt++; 1364 + } 1365 + spin_unlock_bh(&queue->lock); 1366 + 1367 + __skb_queue_purge(&freeme); 1368 + 1369 + return cnt; 1362 1370 } 1363 - EXPORT_SYMBOL_GPL(virtio_transport_free_pkt); 1371 + EXPORT_SYMBOL_GPL(virtio_transport_purge_skbs); 1364 1372 1365 1373 MODULE_LICENSE("GPL v2"); 1366 1374 MODULE_AUTHOR("Asias He");

+15 -36

net/vmw_vsock/vsock_loopback.c

··· 16 16 struct workqueue_struct *workqueue; 17 17 18 18 spinlock_t pkt_list_lock; /* protects pkt_list */ 19 - struct list_head pkt_list; 19 + struct sk_buff_head pkt_queue; 20 20 struct work_struct pkt_work; 21 21 }; 22 22 ··· 27 27 return VMADDR_CID_LOCAL; 28 28 } 29 29 30 - static int vsock_loopback_send_pkt(struct virtio_vsock_pkt *pkt) 30 + static int vsock_loopback_send_pkt(struct sk_buff *skb) 31 31 { 32 32 struct vsock_loopback *vsock = &the_vsock_loopback; 33 - int len = pkt->len; 33 + int len = skb->len; 34 34 35 35 spin_lock_bh(&vsock->pkt_list_lock); 36 - list_add_tail(&pkt->list, &vsock->pkt_list); 36 + skb_queue_tail(&vsock->pkt_queue, skb); 37 37 spin_unlock_bh(&vsock->pkt_list_lock); 38 38 39 39 queue_work(vsock->workqueue, &vsock->pkt_work); ··· 44 44 static int vsock_loopback_cancel_pkt(struct vsock_sock *vsk) 45 45 { 46 46 struct vsock_loopback *vsock = &the_vsock_loopback; 47 - struct virtio_vsock_pkt *pkt, *n; 48 - LIST_HEAD(freeme); 49 47 50 - spin_lock_bh(&vsock->pkt_list_lock); 51 - list_for_each_entry_safe(pkt, n, &vsock->pkt_list, list) { 52 - if (pkt->vsk != vsk) 53 - continue; 54 - list_move(&pkt->list, &freeme); 55 - } 56 - spin_unlock_bh(&vsock->pkt_list_lock); 57 - 58 - list_for_each_entry_safe(pkt, n, &freeme, list) { 59 - list_del(&pkt->list); 60 - virtio_transport_free_pkt(pkt); 61 - } 48 + virtio_transport_purge_skbs(vsk, &vsock->pkt_queue); 62 49 63 50 return 0; 64 51 } ··· 108 121 { 109 122 struct vsock_loopback *vsock = 110 123 container_of(work, struct vsock_loopback, pkt_work); 111 - LIST_HEAD(pkts); 124 + struct sk_buff_head pkts; 125 + struct sk_buff *skb; 126 + 127 + skb_queue_head_init(&pkts); 112 128 113 129 spin_lock_bh(&vsock->pkt_list_lock); 114 - list_splice_init(&vsock->pkt_list, &pkts); 130 + skb_queue_splice_init(&vsock->pkt_queue, &pkts); 115 131 spin_unlock_bh(&vsock->pkt_list_lock); 116 132 117 - while (!list_empty(&pkts)) { 118 - struct virtio_vsock_pkt *pkt; 119 - 120 - pkt = list_first_entry(&pkts, struct virtio_vsock_pkt, list); 121 - list_del_init(&pkt->list); 122 - 123 - virtio_transport_deliver_tap_pkt(pkt); 124 - virtio_transport_recv_pkt(&loopback_transport, pkt); 133 + while ((skb = __skb_dequeue(&pkts))) { 134 + virtio_transport_deliver_tap_pkt(skb); 135 + virtio_transport_recv_pkt(&loopback_transport, skb); 125 136 } 126 137 } 127 138 ··· 133 148 return -ENOMEM; 134 149 135 150 spin_lock_init(&vsock->pkt_list_lock); 136 - INIT_LIST_HEAD(&vsock->pkt_list); 151 + skb_queue_head_init(&vsock->pkt_queue); 137 152 INIT_WORK(&vsock->pkt_work, vsock_loopback_work); 138 153 139 154 ret = vsock_core_register(&loopback_transport.transport, ··· 151 166 static void __exit vsock_loopback_exit(void) 152 167 { 153 168 struct vsock_loopback *vsock = &the_vsock_loopback; 154 - struct virtio_vsock_pkt *pkt; 155 169 156 170 vsock_core_unregister(&loopback_transport.transport); 157 171 158 172 flush_work(&vsock->pkt_work); 159 173 160 174 spin_lock_bh(&vsock->pkt_list_lock); 161 - while (!list_empty(&vsock->pkt_list)) { 162 - pkt = list_first_entry(&vsock->pkt_list, 163 - struct virtio_vsock_pkt, list); 164 - list_del(&pkt->list); 165 - virtio_transport_free_pkt(pkt); 166 - } 175 + virtio_vsock_skb_queue_purge(&vsock->pkt_queue); 167 176 spin_unlock_bh(&vsock->pkt_list_lock); 168 177 169 178 destroy_workqueue(vsock->workqueue);

Configure Feed

Configure Feed