2 * virtio transport for vsock
4 * Copyright (C) 2013-2015 Red Hat, Inc.
5 * Author: Asias He <asias@redhat.com>
6 * Stefan Hajnoczi <stefanha@redhat.com>
8 * Some of the code is take from Gerd Hoffmann <kraxel@redhat.com>'s
9 * early virtio-vsock proof-of-concept bits.
11 * This work is licensed under the terms of the GNU GPL, version 2.
13 #include <linux/spinlock.h>
14 #include <linux/module.h>
15 #include <linux/list.h>
16 #include <linux/virtio.h>
17 #include <linux/virtio_ids.h>
18 #include <linux/virtio_config.h>
19 #include <linux/virtio_vsock.h>
21 #include <linux/mutex.h>
22 #include <net/af_vsock.h>
24 static struct workqueue_struct *virtio_vsock_workqueue;
25 static struct virtio_vsock *the_virtio_vsock;
26 static DEFINE_MUTEX(the_virtio_vsock_mutex); /* protects the_virtio_vsock */
27 static void virtio_vsock_rx_fill(struct virtio_vsock *vsock);
31 struct virtio_device *vdev;
32 /* Virtio virtqueue */
33 struct virtqueue *vqs[VSOCK_VQ_MAX];
34 /* Wait queue for send pkt */
35 wait_queue_head_t queue_wait;
36 /* Work item to send pkt */
37 struct work_struct tx_work;
38 /* Work item to recv pkt */
39 struct work_struct rx_work;
40 /* Mutex to protect send pkt*/
42 /* Mutex to protect recv pkt*/
44 /* Number of recv buffers */
46 /* Number of max recv buffers */
48 /* Used for global tx buf limitation */
50 /* Guest context id, just like guest ip address */
54 static struct virtio_vsock *virtio_vsock_get(void)
56 return the_virtio_vsock;
59 static u32 virtio_transport_get_local_cid(void)
61 struct virtio_vsock *vsock = virtio_vsock_get();
63 return vsock->guest_cid;
67 virtio_transport_send_pkt(struct vsock_sock *vsk,
68 struct virtio_vsock_pkt_info *info)
70 u32 src_cid, src_port, dst_cid, dst_port;
71 int ret, in_sg = 0, out_sg = 0;
72 struct virtio_transport *trans;
73 struct virtio_vsock_pkt *pkt;
74 struct virtio_vsock *vsock;
75 struct scatterlist hdr, buf, *sgs[2];
77 u32 pkt_len = info->pkt_len;
80 vsock = virtio_vsock_get();
84 src_cid = virtio_transport_get_local_cid();
85 src_port = vsk->local_addr.svm_port;
86 if (!info->remote_cid) {
87 dst_cid = vsk->remote_addr.svm_cid;
88 dst_port = vsk->remote_addr.svm_port;
90 dst_cid = info->remote_cid;
91 dst_port = info->remote_port;
95 vq = vsock->vqs[VSOCK_VQ_TX];
97 if (pkt_len > VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE)
98 pkt_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE;
99 pkt_len = virtio_transport_get_credit(trans, pkt_len);
100 /* Do not send zero length OP_RW pkt*/
101 if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW)
104 /* Respect global tx buf limitation */
105 mutex_lock(&vsock->tx_lock);
106 while (pkt_len + vsock->total_tx_buf > VIRTIO_VSOCK_MAX_TX_BUF_SIZE) {
107 prepare_to_wait_exclusive(&vsock->queue_wait, &wait,
108 TASK_UNINTERRUPTIBLE);
109 mutex_unlock(&vsock->tx_lock);
111 mutex_lock(&vsock->tx_lock);
112 finish_wait(&vsock->queue_wait, &wait);
114 vsock->total_tx_buf += pkt_len;
115 mutex_unlock(&vsock->tx_lock);
117 pkt = virtio_transport_alloc_pkt(vsk, info, pkt_len,
121 mutex_lock(&vsock->tx_lock);
122 vsock->total_tx_buf -= pkt_len;
123 mutex_unlock(&vsock->tx_lock);
124 virtio_transport_put_credit(trans, pkt_len);
128 pr_debug("%s:info->pkt_len= %d\n", __func__, info->pkt_len);
130 /* Will be released in virtio_transport_send_pkt_work */
131 sock_hold(&trans->vsk->sk);
132 virtio_transport_inc_tx_pkt(pkt);
134 /* Put pkt in the virtqueue */
135 sg_init_one(&hdr, &pkt->hdr, sizeof(pkt->hdr));
136 sgs[out_sg++] = &hdr;
137 if (info->msg && info->pkt_len > 0) {
138 sg_init_one(&buf, pkt->buf, pkt->len);
139 sgs[out_sg++] = &buf;
142 mutex_lock(&vsock->tx_lock);
143 while ((ret = virtqueue_add_sgs(vq, sgs, out_sg, in_sg, pkt,
145 prepare_to_wait_exclusive(&vsock->queue_wait, &wait,
146 TASK_UNINTERRUPTIBLE);
147 mutex_unlock(&vsock->tx_lock);
149 mutex_lock(&vsock->tx_lock);
150 finish_wait(&vsock->queue_wait, &wait);
153 mutex_unlock(&vsock->tx_lock);
158 static struct virtio_transport_pkt_ops virtio_ops = {
159 .send_pkt = virtio_transport_send_pkt,
162 static void virtio_vsock_rx_fill(struct virtio_vsock *vsock)
164 int buf_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE;
165 struct virtio_vsock_pkt *pkt;
166 struct scatterlist hdr, buf, *sgs[2];
167 struct virtqueue *vq;
170 vq = vsock->vqs[VSOCK_VQ_RX];
173 pkt = kzalloc(sizeof(*pkt), GFP_KERNEL);
175 pr_debug("%s: fail to allocate pkt\n", __func__);
179 /* TODO: use mergeable rx buffer */
180 pkt->buf = kmalloc(buf_len, GFP_KERNEL);
182 pr_debug("%s: fail to allocate pkt->buf\n", __func__);
186 sg_init_one(&hdr, &pkt->hdr, sizeof(pkt->hdr));
189 sg_init_one(&buf, pkt->buf, buf_len);
191 ret = virtqueue_add_sgs(vq, sgs, 0, 2, pkt, GFP_KERNEL);
195 } while (vq->num_free);
196 if (vsock->rx_buf_nr > vsock->rx_buf_max_nr)
197 vsock->rx_buf_max_nr = vsock->rx_buf_nr;
203 virtio_transport_free_pkt(pkt);
207 static void virtio_transport_send_pkt_work(struct work_struct *work)
209 struct virtio_vsock *vsock =
210 container_of(work, struct virtio_vsock, tx_work);
211 struct virtio_vsock_pkt *pkt;
213 struct virtqueue *vq;
217 vq = vsock->vqs[VSOCK_VQ_TX];
218 mutex_lock(&vsock->tx_lock);
220 virtqueue_disable_cb(vq);
221 while ((pkt = virtqueue_get_buf(vq, &len)) != NULL) {
222 sk = &pkt->trans->vsk->sk;
223 virtio_transport_dec_tx_pkt(pkt);
224 /* Release refcnt taken in virtio_transport_send_pkt */
226 vsock->total_tx_buf -= pkt->len;
227 virtio_transport_free_pkt(pkt);
230 } while (!virtqueue_enable_cb(vq));
231 mutex_unlock(&vsock->tx_lock);
234 wake_up(&vsock->queue_wait);
237 static void virtio_transport_recv_pkt_work(struct work_struct *work)
239 struct virtio_vsock *vsock =
240 container_of(work, struct virtio_vsock, rx_work);
241 struct virtio_vsock_pkt *pkt;
242 struct virtqueue *vq;
245 vq = vsock->vqs[VSOCK_VQ_RX];
246 mutex_lock(&vsock->rx_lock);
248 virtqueue_disable_cb(vq);
249 while ((pkt = virtqueue_get_buf(vq, &len)) != NULL) {
251 virtio_transport_recv_pkt(pkt);
254 } while (!virtqueue_enable_cb(vq));
256 if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2)
257 virtio_vsock_rx_fill(vsock);
258 mutex_unlock(&vsock->rx_lock);
261 static void virtio_vsock_ctrl_done(struct virtqueue *vq)
265 static void virtio_vsock_tx_done(struct virtqueue *vq)
267 struct virtio_vsock *vsock = vq->vdev->priv;
271 queue_work(virtio_vsock_workqueue, &vsock->tx_work);
274 static void virtio_vsock_rx_done(struct virtqueue *vq)
276 struct virtio_vsock *vsock = vq->vdev->priv;
280 queue_work(virtio_vsock_workqueue, &vsock->rx_work);
284 virtio_transport_socket_init(struct vsock_sock *vsk, struct vsock_sock *psk)
286 struct virtio_transport *trans;
289 ret = virtio_transport_do_socket_init(vsk, psk);
294 trans->ops = &virtio_ops;
298 static struct vsock_transport virtio_transport = {
299 .get_local_cid = virtio_transport_get_local_cid,
301 .init = virtio_transport_socket_init,
302 .destruct = virtio_transport_destruct,
303 .release = virtio_transport_release,
304 .connect = virtio_transport_connect,
305 .shutdown = virtio_transport_shutdown,
307 .dgram_bind = virtio_transport_dgram_bind,
308 .dgram_dequeue = virtio_transport_dgram_dequeue,
309 .dgram_enqueue = virtio_transport_dgram_enqueue,
310 .dgram_allow = virtio_transport_dgram_allow,
312 .stream_dequeue = virtio_transport_stream_dequeue,
313 .stream_enqueue = virtio_transport_stream_enqueue,
314 .stream_has_data = virtio_transport_stream_has_data,
315 .stream_has_space = virtio_transport_stream_has_space,
316 .stream_rcvhiwat = virtio_transport_stream_rcvhiwat,
317 .stream_is_active = virtio_transport_stream_is_active,
318 .stream_allow = virtio_transport_stream_allow,
320 .notify_poll_in = virtio_transport_notify_poll_in,
321 .notify_poll_out = virtio_transport_notify_poll_out,
322 .notify_recv_init = virtio_transport_notify_recv_init,
323 .notify_recv_pre_block = virtio_transport_notify_recv_pre_block,
324 .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue,
325 .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue,
326 .notify_send_init = virtio_transport_notify_send_init,
327 .notify_send_pre_block = virtio_transport_notify_send_pre_block,
328 .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue,
329 .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue,
331 .set_buffer_size = virtio_transport_set_buffer_size,
332 .set_min_buffer_size = virtio_transport_set_min_buffer_size,
333 .set_max_buffer_size = virtio_transport_set_max_buffer_size,
334 .get_buffer_size = virtio_transport_get_buffer_size,
335 .get_min_buffer_size = virtio_transport_get_min_buffer_size,
336 .get_max_buffer_size = virtio_transport_get_max_buffer_size,
339 static int virtio_vsock_probe(struct virtio_device *vdev)
341 vq_callback_t *callbacks[] = {
342 virtio_vsock_ctrl_done,
343 virtio_vsock_rx_done,
344 virtio_vsock_tx_done,
346 const char *names[] = {
351 struct virtio_vsock *vsock = NULL;
355 ret = mutex_lock_interruptible(&the_virtio_vsock_mutex);
359 /* Only one virtio-vsock device per guest is supported */
360 if (the_virtio_vsock) {
365 vsock = kzalloc(sizeof(*vsock), GFP_KERNEL);
373 ret = vsock->vdev->config->find_vqs(vsock->vdev, VSOCK_VQ_MAX,
374 vsock->vqs, callbacks, names);
378 vdev->config->get(vdev, offsetof(struct virtio_vsock_config, guest_cid),
379 &guest_cid, sizeof(guest_cid));
380 vsock->guest_cid = le32_to_cpu(guest_cid);
381 pr_debug("%s:guest_cid=%d\n", __func__, vsock->guest_cid);
383 ret = vsock_core_init(&virtio_transport);
387 vsock->rx_buf_nr = 0;
388 vsock->rx_buf_max_nr = 0;
390 vdev->priv = the_virtio_vsock = vsock;
391 init_waitqueue_head(&vsock->queue_wait);
392 mutex_init(&vsock->tx_lock);
393 mutex_init(&vsock->rx_lock);
394 INIT_WORK(&vsock->rx_work, virtio_transport_recv_pkt_work);
395 INIT_WORK(&vsock->tx_work, virtio_transport_send_pkt_work);
397 mutex_lock(&vsock->rx_lock);
398 virtio_vsock_rx_fill(vsock);
399 mutex_unlock(&vsock->rx_lock);
401 mutex_unlock(&the_virtio_vsock_mutex);
405 vsock->vdev->config->del_vqs(vsock->vdev);
408 mutex_unlock(&the_virtio_vsock_mutex);
412 static void virtio_vsock_remove(struct virtio_device *vdev)
414 struct virtio_vsock *vsock = vdev->priv;
416 mutex_lock(&the_virtio_vsock_mutex);
417 the_virtio_vsock = NULL;
419 mutex_unlock(&the_virtio_vsock_mutex);
424 static struct virtio_device_id id_table[] = {
425 { VIRTIO_ID_VSOCK, VIRTIO_DEV_ANY_ID },
429 static unsigned int features[] = {
432 static struct virtio_driver virtio_vsock_driver = {
433 .feature_table = features,
434 .feature_table_size = ARRAY_SIZE(features),
435 .driver.name = KBUILD_MODNAME,
436 .driver.owner = THIS_MODULE,
437 .id_table = id_table,
438 .probe = virtio_vsock_probe,
439 .remove = virtio_vsock_remove,
442 static int __init virtio_vsock_init(void)
446 virtio_vsock_workqueue = alloc_workqueue("virtio_vsock", 0, 0);
447 if (!virtio_vsock_workqueue)
449 ret = register_virtio_driver(&virtio_vsock_driver);
451 destroy_workqueue(virtio_vsock_workqueue);
455 static void __exit virtio_vsock_exit(void)
457 unregister_virtio_driver(&virtio_vsock_driver);
458 destroy_workqueue(virtio_vsock_workqueue);
461 module_init(virtio_vsock_init);
462 module_exit(virtio_vsock_exit);
463 MODULE_LICENSE("GPL v2");
464 MODULE_AUTHOR("Asias He");
465 MODULE_DESCRIPTION("virtio transport for vsock");
466 MODULE_DEVICE_TABLE(virtio, id_table);