tools/virtio/ringtest/virtio_ring_0_9.c

   1 /*
   2  * Copyright (C) 2016 Red Hat, Inc.
   3  * Author: Michael S. Tsirkin <mst@redhat.com>
   4  * This work is licensed under the terms of the GNU GPL, version 2.
   5  *
   6  * Partial implementation of virtio 0.9. event index is used for signalling,
   7  * unconditionally. Design roughly follows linux kernel implementation in order
   8  * to be able to judge its performance.
   9  */
  10 #define _GNU_SOURCE
  11 #include "main.h"
  12 #include <stdlib.h>
  13 #include <stdio.h>
  14 #include <assert.h>
  15 #include <string.h>
  16 #include <linux/virtio_ring.h>
  17
  18 struct data {
  19         void *data;
  20 } *data;
  21
  22 struct vring ring;
  23
  24 /* enabling the below activates experimental ring polling code
  25  * (which skips index reads on consumer in favor of looking at
  26  * high bits of ring id ^ 0x8000).
  27  */
  28 /* #ifdef RING_POLL */
  29 /* enabling the below activates experimental in-order code
  30  * (which skips ring updates and reads and writes len in descriptor).
  31  */
  32 /* #ifdef INORDER */
  33
  34 #if defined(RING_POLL) && defined(INORDER)
  35 #error "RING_POLL and INORDER are mutually exclusive"
  36 #endif
  37
  38 /* how much padding is needed to avoid false cache sharing */
  39 #define HOST_GUEST_PADDING 0x80
  40
  41 struct guest {
  42         unsigned short avail_idx;
  43         unsigned short last_used_idx;
  44         unsigned short num_free;
  45         unsigned short kicked_avail_idx;
  46 #ifndef INORDER
  47         unsigned short free_head;
  48 #else
  49         unsigned short reserved_free_head;
  50 #endif
  51         unsigned char reserved[HOST_GUEST_PADDING - 10];
  52 } guest;
  53
  54 struct host {
  55         /* we do not need to track last avail index
  56          * unless we have more than one in flight.
  57          */
  58         unsigned short used_idx;
  59         unsigned short called_used_idx;
  60         unsigned char reserved[HOST_GUEST_PADDING - 4];
  61 } host;
  62
  63 /* implemented by ring */
  64 void alloc_ring(void)
  65 {
  66         int ret;
  67         int i;
  68         void *p;
  69
  70         ret = posix_memalign(&p, 0x1000, vring_size(ring_size, 0x1000));
  71         if (ret) {
  72                 perror("Unable to allocate ring buffer.\n");
  73                 exit(3);
  74         }
  75         memset(p, 0, vring_size(ring_size, 0x1000));
  76         vring_init(&ring, ring_size, p, 0x1000);
  77
  78         guest.avail_idx = 0;
  79         guest.kicked_avail_idx = -1;
  80         guest.last_used_idx = 0;
  81 #ifndef INORDER
  82         /* Put everything in free lists. */
  83         guest.free_head = 0;
  84 #endif
  85         for (i = 0; i < ring_size - 1; i++)
  86                 ring.desc[i].next = i + 1;
  87         host.used_idx = 0;
  88         host.called_used_idx = -1;
  89         guest.num_free = ring_size;
  90         data = malloc(ring_size * sizeof *data);
  91         if (!data) {
  92                 perror("Unable to allocate data buffer.\n");
  93                 exit(3);
  94         }
  95         memset(data, 0, ring_size * sizeof *data);
  96 }
  97
  98 /* guest side */
  99 int add_inbuf(unsigned len, void *buf, void *datap)
 100 {
 101         unsigned head;
 102 #ifndef INORDER
 103         unsigned avail;
 104 #endif
 105         struct vring_desc *desc;
 106
 107         if (!guest.num_free)
 108                 return -1;
 109
 110 #ifdef INORDER
 111         head = (ring_size - 1) & (guest.avail_idx++);
 112 #else
 113         head = guest.free_head;
 114 #endif
 115         guest.num_free--;
 116
 117         desc = ring.desc;
 118         desc[head].flags = VRING_DESC_F_NEXT;
 119         desc[head].addr = (unsigned long)(void *)buf;
 120         desc[head].len = len;
 121         /* We do it like this to simulate the way
 122          * we'd have to flip it if we had multiple
 123          * descriptors.
 124          */
 125         desc[head].flags &= ~VRING_DESC_F_NEXT;
 126 #ifndef INORDER
 127         guest.free_head = desc[head].next;
 128 #endif
 129
 130         data[head].data = datap;
 131
 132 #ifdef RING_POLL
 133         /* Barrier A (for pairing) */
 134         smp_release();
 135         avail = guest.avail_idx++;
 136         ring.avail->ring[avail & (ring_size - 1)] =
 137                 (head | (avail & ~(ring_size - 1))) ^ 0x8000;
 138 #else
 139 #ifndef INORDER
 140         /* Barrier A (for pairing) */
 141         smp_release();
 142         avail = (ring_size - 1) & (guest.avail_idx++);
 143         ring.avail->ring[avail] = head;
 144 #endif
 145         /* Barrier A (for pairing) */
 146         smp_release();
 147 #endif
 148         ring.avail->idx = guest.avail_idx;
 149         return 0;
 150 }
 151
 152 void *get_buf(unsigned *lenp, void **bufp)
 153 {
 154         unsigned head;
 155         unsigned index;
 156         void *datap;
 157
 158 #ifdef RING_POLL
 159         head = (ring_size - 1) & guest.last_used_idx;
 160         index = ring.used->ring[head].id;
 161         if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1))
 162                 return NULL;
 163         /* Barrier B (for pairing) */
 164         smp_acquire();
 165         index &= ring_size - 1;
 166 #else
 167         if (ring.used->idx == guest.last_used_idx)
 168                 return NULL;
 169         /* Barrier B (for pairing) */
 170         smp_acquire();
 171 #ifdef INORDER
 172         head = (ring_size - 1) & guest.last_used_idx;
 173         index = head;
 174 #else
 175         head = (ring_size - 1) & guest.last_used_idx;
 176         index = ring.used->ring[head].id;
 177 #endif
 178
 179 #endif
 180 #ifdef INORDER
 181         *lenp = ring.desc[index].len;
 182 #else
 183         *lenp = ring.used->ring[head].len;
 184 #endif
 185         datap = data[index].data;
 186         *bufp = (void*)(unsigned long)ring.desc[index].addr;
 187         data[index].data = NULL;
 188 #ifndef INORDER
 189         ring.desc[index].next = guest.free_head;
 190         guest.free_head = index;
 191 #endif
 192         guest.num_free++;
 193         guest.last_used_idx++;
 194         return datap;
 195 }
 196
 197 void poll_used(void)
 198 {
 199 #ifdef RING_POLL
 200         unsigned head = (ring_size - 1) & guest.last_used_idx;
 201
 202         for (;;) {
 203                 unsigned index = ring.used->ring[head].id;
 204
 205                 if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1))
 206                         busy_wait();
 207                 else
 208                         break;
 209         }
 210 #else
 211         unsigned head = guest.last_used_idx;
 212
 213         while (ring.used->idx == head)
 214                 busy_wait();
 215 #endif
 216 }
 217
 218 void disable_call()
 219 {
 220         /* Doing nothing to disable calls might cause
 221          * extra interrupts, but reduces the number of cache misses.
 222          */
 223 }
 224
 225 bool enable_call()
 226 {
 227         unsigned short last_used_idx;
 228
 229         vring_used_event(&ring) = (last_used_idx = guest.last_used_idx);
 230         /* Flush call index write */
 231         /* Barrier D (for pairing) */
 232         smp_mb();
 233 #ifdef RING_POLL
 234         {
 235                 unsigned short head = last_used_idx & (ring_size - 1);
 236                 unsigned index = ring.used->ring[head].id;
 237
 238                 return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1);
 239         }
 240 #else
 241         return ring.used->idx == last_used_idx;
 242 #endif
 243 }
 244
 245 void kick_available(void)
 246 {
 247         /* Flush in previous flags write */
 248         /* Barrier C (for pairing) */
 249         smp_mb();
 250         if (!vring_need_event(vring_avail_event(&ring),
 251                               guest.avail_idx,
 252                               guest.kicked_avail_idx))
 253                 return;
 254
 255         guest.kicked_avail_idx = guest.avail_idx;
 256         kick();
 257 }
 258
 259 /* host side */
 260 void disable_kick()
 261 {
 262         /* Doing nothing to disable kicks might cause
 263          * extra interrupts, but reduces the number of cache misses.
 264          */
 265 }
 266
 267 bool enable_kick()
 268 {
 269         unsigned head = host.used_idx;
 270
 271         vring_avail_event(&ring) = head;
 272         /* Barrier C (for pairing) */
 273         smp_mb();
 274 #ifdef RING_POLL
 275         {
 276                 unsigned index = ring.avail->ring[head & (ring_size - 1)];
 277
 278                 return (index ^ head ^ 0x8000) & ~(ring_size - 1);
 279         }
 280 #else
 281         return head == ring.avail->idx;
 282 #endif
 283 }
 284
 285 void poll_avail(void)
 286 {
 287         unsigned head = host.used_idx;
 288 #ifdef RING_POLL
 289         for (;;) {
 290                 unsigned index = ring.avail->ring[head & (ring_size - 1)];
 291                 if ((index ^ head ^ 0x8000) & ~(ring_size - 1))
 292                         busy_wait();
 293                 else
 294                         break;
 295         }
 296 #else
 297         while (ring.avail->idx == head)
 298                 busy_wait();
 299 #endif
 300 }
 301
 302 bool use_buf(unsigned *lenp, void **bufp)
 303 {
 304         unsigned used_idx = host.used_idx;
 305         struct vring_desc *desc;
 306         unsigned head;
 307
 308 #ifdef RING_POLL
 309         head = ring.avail->ring[used_idx & (ring_size - 1)];
 310         if ((used_idx ^ head ^ 0x8000) & ~(ring_size - 1))
 311                 return false;
 312         /* Barrier A (for pairing) */
 313         smp_acquire();
 314
 315         used_idx &= ring_size - 1;
 316         desc = &ring.desc[head & (ring_size - 1)];
 317 #else
 318         if (used_idx == ring.avail->idx)
 319                 return false;
 320
 321         /* Barrier A (for pairing) */
 322         smp_acquire();
 323
 324         used_idx &= ring_size - 1;
 325 #ifdef INORDER
 326         head = used_idx;
 327 #else
 328         head = ring.avail->ring[used_idx];
 329 #endif
 330         desc = &ring.desc[head];
 331 #endif
 332
 333         *lenp = desc->len;
 334         *bufp = (void *)(unsigned long)desc->addr;
 335
 336 #ifdef INORDER
 337         desc->len = desc->len - 1;
 338 #else
 339         /* now update used ring */
 340         ring.used->ring[used_idx].id = head;
 341         ring.used->ring[used_idx].len = desc->len - 1;
 342 #endif
 343         /* Barrier B (for pairing) */
 344         smp_release();
 345         host.used_idx++;
 346         ring.used->idx = host.used_idx;
 347
 348         return true;
 349 }
 350
 351 void call_used(void)
 352 {
 353         /* Flush in previous flags write */
 354         /* Barrier D (for pairing) */
 355         smp_mb();
 356         if (!vring_need_event(vring_used_event(&ring),
 357                               host.used_idx,
 358                               host.called_used_idx))
 359                 return;
 360
 361         host.called_used_idx = host.used_idx;
 362         call();
 363 }