drivers/misc/mic/vop/vop_vringh.c

   1 /*
   2  * Intel MIC Platform Software Stack (MPSS)
   3  *
   4  * Copyright(c) 2016 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License, version 2, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13  * General Public License for more details.
  14  *
  15  * The full GNU General Public License is included in this distribution in
  16  * the file called "COPYING".
  17  *
  18  * Intel Virtio Over PCIe (VOP) driver.
  19  *
  20  */
  21 #include <linux/sched.h>
  22 #include <linux/poll.h>
  23 #include <linux/dma-mapping.h>
  24
  25 #include <linux/mic_common.h>
  26 #include "../common/mic_dev.h"
  27
  28 #include <linux/mic_ioctl.h>
  29 #include "vop_main.h"
  30
  31 /* Helper API to obtain the VOP PCIe device */
  32 static inline struct device *vop_dev(struct vop_vdev *vdev)
  33 {
  34         return vdev->vpdev->dev.parent;
  35 }
  36
  37 /* Helper API to check if a virtio device is initialized */
  38 static inline int vop_vdev_inited(struct vop_vdev *vdev)
  39 {
  40         if (!vdev)
  41                 return -EINVAL;
  42         /* Device has not been created yet */
  43         if (!vdev->dd || !vdev->dd->type) {
  44                 dev_err(vop_dev(vdev), "%s %d err %d\n",
  45                         __func__, __LINE__, -EINVAL);
  46                 return -EINVAL;
  47         }
  48         /* Device has been removed/deleted */
  49         if (vdev->dd->type == -1) {
  50                 dev_dbg(vop_dev(vdev), "%s %d err %d\n",
  51                         __func__, __LINE__, -ENODEV);
  52                 return -ENODEV;
  53         }
  54         return 0;
  55 }
  56
  57 static void _vop_notify(struct vringh *vrh)
  58 {
  59         struct vop_vringh *vvrh = container_of(vrh, struct vop_vringh, vrh);
  60         struct vop_vdev *vdev = vvrh->vdev;
  61         struct vop_device *vpdev = vdev->vpdev;
  62         s8 db = vdev->dc->h2c_vdev_db;
  63
  64         if (db != -1)
  65                 vpdev->hw_ops->send_intr(vpdev, db);
  66 }
  67
  68 static void vop_virtio_init_post(struct vop_vdev *vdev)
  69 {
  70         struct mic_vqconfig *vqconfig = mic_vq_config(vdev->dd);
  71         struct vop_device *vpdev = vdev->vpdev;
  72         int i, used_size;
  73
  74         for (i = 0; i < vdev->dd->num_vq; i++) {
  75                 used_size = PAGE_ALIGN(sizeof(u16) * 3 +
  76                                 sizeof(struct vring_used_elem) *
  77                                 le16_to_cpu(vqconfig->num));
  78                 if (!le64_to_cpu(vqconfig[i].used_address)) {
  79                         dev_warn(vop_dev(vdev), "used_address zero??\n");
  80                         continue;
  81                 }
  82                 vdev->vvr[i].vrh.vring.used =
  83                         (void __force *)vpdev->hw_ops->ioremap(
  84                         vpdev,
  85                         le64_to_cpu(vqconfig[i].used_address),
  86                         used_size);
  87         }
  88
  89         vdev->dc->used_address_updated = 0;
  90
  91         dev_info(vop_dev(vdev), "%s: device type %d LINKUP\n",
  92                  __func__, vdev->virtio_id);
  93 }
  94
  95 static inline void vop_virtio_device_reset(struct vop_vdev *vdev)
  96 {
  97         int i;
  98
  99         dev_dbg(vop_dev(vdev), "%s: status %d device type %d RESET\n",
 100                 __func__, vdev->dd->status, vdev->virtio_id);
 101
 102         for (i = 0; i < vdev->dd->num_vq; i++)
 103                 /*
 104                  * Avoid lockdep false positive. The + 1 is for the vop
 105                  * mutex which is held in the reset devices code path.
 106                  */
 107                 mutex_lock_nested(&vdev->vvr[i].vr_mutex, i + 1);
 108
 109         /* 0 status means "reset" */
 110         vdev->dd->status = 0;
 111         vdev->dc->vdev_reset = 0;
 112         vdev->dc->host_ack = 1;
 113
 114         for (i = 0; i < vdev->dd->num_vq; i++) {
 115                 struct vringh *vrh = &vdev->vvr[i].vrh;
 116
 117                 vdev->vvr[i].vring.info->avail_idx = 0;
 118                 vrh->completed = 0;
 119                 vrh->last_avail_idx = 0;
 120                 vrh->last_used_idx = 0;
 121         }
 122
 123         for (i = 0; i < vdev->dd->num_vq; i++)
 124                 mutex_unlock(&vdev->vvr[i].vr_mutex);
 125 }
 126
 127 static void vop_virtio_reset_devices(struct vop_info *vi)
 128 {
 129         struct list_head *pos, *tmp;
 130         struct vop_vdev *vdev;
 131
 132         list_for_each_safe(pos, tmp, &vi->vdev_list) {
 133                 vdev = list_entry(pos, struct vop_vdev, list);
 134                 vop_virtio_device_reset(vdev);
 135                 vdev->poll_wake = 1;
 136                 wake_up(&vdev->waitq);
 137         }
 138 }
 139
 140 static void vop_bh_handler(struct work_struct *work)
 141 {
 142         struct vop_vdev *vdev = container_of(work, struct vop_vdev,
 143                         virtio_bh_work);
 144
 145         if (vdev->dc->used_address_updated)
 146                 vop_virtio_init_post(vdev);
 147
 148         if (vdev->dc->vdev_reset)
 149                 vop_virtio_device_reset(vdev);
 150
 151         vdev->poll_wake = 1;
 152         wake_up(&vdev->waitq);
 153 }
 154
 155 static irqreturn_t _vop_virtio_intr_handler(int irq, void *data)
 156 {
 157         struct vop_vdev *vdev = data;
 158         struct vop_device *vpdev = vdev->vpdev;
 159
 160         vpdev->hw_ops->ack_interrupt(vpdev, vdev->virtio_db);
 161         schedule_work(&vdev->virtio_bh_work);
 162         return IRQ_HANDLED;
 163 }
 164
 165 static int vop_virtio_config_change(struct vop_vdev *vdev, void *argp)
 166 {
 167         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wake);
 168         int ret = 0, retry, i;
 169         struct vop_device *vpdev = vdev->vpdev;
 170         struct vop_info *vi = dev_get_drvdata(&vpdev->dev);
 171         struct mic_bootparam *bootparam = vpdev->hw_ops->get_dp(vpdev);
 172         s8 db = bootparam->h2c_config_db;
 173
 174         mutex_lock(&vi->vop_mutex);
 175         for (i = 0; i < vdev->dd->num_vq; i++)
 176                 mutex_lock_nested(&vdev->vvr[i].vr_mutex, i + 1);
 177
 178         if (db == -1 || vdev->dd->type == -1) {
 179                 ret = -EIO;
 180                 goto exit;
 181         }
 182
 183         memcpy(mic_vq_configspace(vdev->dd), argp, vdev->dd->config_len);
 184         vdev->dc->config_change = MIC_VIRTIO_PARAM_CONFIG_CHANGED;
 185         vpdev->hw_ops->send_intr(vpdev, db);
 186
 187         for (retry = 100; retry--;) {
 188                 ret = wait_event_timeout(wake, vdev->dc->guest_ack,
 189                                          msecs_to_jiffies(100));
 190                 if (ret)
 191                         break;
 192         }
 193
 194         dev_dbg(vop_dev(vdev),
 195                 "%s %d retry: %d\n", __func__, __LINE__, retry);
 196         vdev->dc->config_change = 0;
 197         vdev->dc->guest_ack = 0;
 198 exit:
 199         for (i = 0; i < vdev->dd->num_vq; i++)
 200                 mutex_unlock(&vdev->vvr[i].vr_mutex);
 201         mutex_unlock(&vi->vop_mutex);
 202         return ret;
 203 }
 204
 205 static int vop_copy_dp_entry(struct vop_vdev *vdev,
 206                              struct mic_device_desc *argp, __u8 *type,
 207                              struct mic_device_desc **devpage)
 208 {
 209         struct vop_device *vpdev = vdev->vpdev;
 210         struct mic_device_desc *devp;
 211         struct mic_vqconfig *vqconfig;
 212         int ret = 0, i;
 213         bool slot_found = false;
 214
 215         vqconfig = mic_vq_config(argp);
 216         for (i = 0; i < argp->num_vq; i++) {
 217                 if (le16_to_cpu(vqconfig[i].num) > MIC_MAX_VRING_ENTRIES) {
 218                         ret =  -EINVAL;
 219                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 220                                 __func__, __LINE__, ret);
 221                         goto exit;
 222                 }
 223         }
 224
 225         /* Find the first free device page entry */
 226         for (i = sizeof(struct mic_bootparam);
 227                 i < MIC_DP_SIZE - mic_total_desc_size(argp);
 228                 i += mic_total_desc_size(devp)) {
 229                 devp = vpdev->hw_ops->get_dp(vpdev) + i;
 230                 if (devp->type == 0 || devp->type == -1) {
 231                         slot_found = true;
 232                         break;
 233                 }
 234         }
 235         if (!slot_found) {
 236                 ret =  -EINVAL;
 237                 dev_err(vop_dev(vdev), "%s %d err %d\n",
 238                         __func__, __LINE__, ret);
 239                 goto exit;
 240         }
 241         /*
 242          * Save off the type before doing the memcpy. Type will be set in the
 243          * end after completing all initialization for the new device.
 244          */
 245         *type = argp->type;
 246         argp->type = 0;
 247         memcpy(devp, argp, mic_desc_size(argp));
 248
 249         *devpage = devp;
 250 exit:
 251         return ret;
 252 }
 253
 254 static void vop_init_device_ctrl(struct vop_vdev *vdev,
 255                                  struct mic_device_desc *devpage)
 256 {
 257         struct mic_device_ctrl *dc;
 258
 259         dc = (void *)devpage + mic_aligned_desc_size(devpage);
 260
 261         dc->config_change = 0;
 262         dc->guest_ack = 0;
 263         dc->vdev_reset = 0;
 264         dc->host_ack = 0;
 265         dc->used_address_updated = 0;
 266         dc->c2h_vdev_db = -1;
 267         dc->h2c_vdev_db = -1;
 268         vdev->dc = dc;
 269 }
 270
 271 static int vop_virtio_add_device(struct vop_vdev *vdev,
 272                                  struct mic_device_desc *argp)
 273 {
 274         struct vop_info *vi = vdev->vi;
 275         struct vop_device *vpdev = vi->vpdev;
 276         struct mic_device_desc *dd = NULL;
 277         struct mic_vqconfig *vqconfig;
 278         int vr_size, i, j, ret;
 279         u8 type = 0;
 280         s8 db = -1;
 281         char irqname[16];
 282         struct mic_bootparam *bootparam;
 283         u16 num;
 284         dma_addr_t vr_addr;
 285
 286         bootparam = vpdev->hw_ops->get_dp(vpdev);
 287         init_waitqueue_head(&vdev->waitq);
 288         INIT_LIST_HEAD(&vdev->list);
 289         vdev->vpdev = vpdev;
 290
 291         ret = vop_copy_dp_entry(vdev, argp, &type, &dd);
 292         if (ret) {
 293                 dev_err(vop_dev(vdev), "%s %d err %d\n",
 294                         __func__, __LINE__, ret);
 295                 kfree(vdev);
 296                 return ret;
 297         }
 298
 299         vop_init_device_ctrl(vdev, dd);
 300
 301         vdev->dd = dd;
 302         vdev->virtio_id = type;
 303         vqconfig = mic_vq_config(dd);
 304         INIT_WORK(&vdev->virtio_bh_work, vop_bh_handler);
 305
 306         for (i = 0; i < dd->num_vq; i++) {
 307                 struct vop_vringh *vvr = &vdev->vvr[i];
 308                 struct mic_vring *vr = &vdev->vvr[i].vring;
 309
 310                 num = le16_to_cpu(vqconfig[i].num);
 311                 mutex_init(&vvr->vr_mutex);
 312                 vr_size = PAGE_ALIGN(vring_size(num, MIC_VIRTIO_RING_ALIGN) +
 313                         sizeof(struct _mic_vring_info));
 314                 vr->va = (void *)
 315                         __get_free_pages(GFP_KERNEL | __GFP_ZERO,
 316                                          get_order(vr_size));
 317                 if (!vr->va) {
 318                         ret = -ENOMEM;
 319                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 320                                 __func__, __LINE__, ret);
 321                         goto err;
 322                 }
 323                 vr->len = vr_size;
 324                 vr->info = vr->va + vring_size(num, MIC_VIRTIO_RING_ALIGN);
 325                 vr->info->magic = cpu_to_le32(MIC_MAGIC + vdev->virtio_id + i);
 326                 vr_addr = dma_map_single(&vpdev->dev, vr->va, vr_size,
 327                                          DMA_BIDIRECTIONAL);
 328                 if (dma_mapping_error(&vpdev->dev, vr_addr)) {
 329                         free_pages((unsigned long)vr->va, get_order(vr_size));
 330                         ret = -ENOMEM;
 331                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 332                                 __func__, __LINE__, ret);
 333                         goto err;
 334                 }
 335                 vqconfig[i].address = cpu_to_le64(vr_addr);
 336
 337                 vring_init(&vr->vr, num, vr->va, MIC_VIRTIO_RING_ALIGN);
 338                 ret = vringh_init_kern(&vvr->vrh,
 339                                        *(u32 *)mic_vq_features(vdev->dd),
 340                                        num, false, vr->vr.desc, vr->vr.avail,
 341                                        vr->vr.used);
 342                 if (ret) {
 343                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 344                                 __func__, __LINE__, ret);
 345                         goto err;
 346                 }
 347                 vringh_kiov_init(&vvr->riov, NULL, 0);
 348                 vringh_kiov_init(&vvr->wiov, NULL, 0);
 349                 vvr->head = USHRT_MAX;
 350                 vvr->vdev = vdev;
 351                 vvr->vrh.notify = _vop_notify;
 352                 dev_dbg(&vpdev->dev,
 353                         "%s %d index %d va %p info %p vr_size 0x%x\n",
 354                         __func__, __LINE__, i, vr->va, vr->info, vr_size);
 355                 vvr->buf = (void *)__get_free_pages(GFP_KERNEL,
 356                                         get_order(VOP_INT_DMA_BUF_SIZE));
 357                 vvr->buf_da = dma_map_single(&vpdev->dev,
 358                                           vvr->buf, VOP_INT_DMA_BUF_SIZE,
 359                                           DMA_BIDIRECTIONAL);
 360         }
 361
 362         snprintf(irqname, sizeof(irqname), "vop%dvirtio%d", vpdev->index,
 363                  vdev->virtio_id);
 364         vdev->virtio_db = vpdev->hw_ops->next_db(vpdev);
 365         vdev->virtio_cookie = vpdev->hw_ops->request_irq(vpdev,
 366                         _vop_virtio_intr_handler, irqname, vdev,
 367                         vdev->virtio_db);
 368         if (IS_ERR(vdev->virtio_cookie)) {
 369                 ret = PTR_ERR(vdev->virtio_cookie);
 370                 dev_dbg(&vpdev->dev, "request irq failed\n");
 371                 goto err;
 372         }
 373
 374         vdev->dc->c2h_vdev_db = vdev->virtio_db;
 375
 376         /*
 377          * Order the type update with previous stores. This write barrier
 378          * is paired with the corresponding read barrier before the uncached
 379          * system memory read of the type, on the card while scanning the
 380          * device page.
 381          */
 382         smp_wmb();
 383         dd->type = type;
 384         argp->type = type;
 385
 386         if (bootparam) {
 387                 db = bootparam->h2c_config_db;
 388                 if (db != -1)
 389                         vpdev->hw_ops->send_intr(vpdev, db);
 390         }
 391         dev_dbg(&vpdev->dev, "Added virtio id %d db %d\n", dd->type, db);
 392         return 0;
 393 err:
 394         vqconfig = mic_vq_config(dd);
 395         for (j = 0; j < i; j++) {
 396                 struct vop_vringh *vvr = &vdev->vvr[j];
 397
 398                 dma_unmap_single(&vpdev->dev, le64_to_cpu(vqconfig[j].address),
 399                                  vvr->vring.len, DMA_BIDIRECTIONAL);
 400                 free_pages((unsigned long)vvr->vring.va,
 401                            get_order(vvr->vring.len));
 402         }
 403         return ret;
 404 }
 405
 406 static void vop_dev_remove(struct vop_info *pvi, struct mic_device_ctrl *devp,
 407                            struct vop_device *vpdev)
 408 {
 409         struct mic_bootparam *bootparam = vpdev->hw_ops->get_dp(vpdev);
 410         s8 db;
 411         int ret, retry;
 412         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wake);
 413
 414         devp->config_change = MIC_VIRTIO_PARAM_DEV_REMOVE;
 415         db = bootparam->h2c_config_db;
 416         if (db != -1)
 417                 vpdev->hw_ops->send_intr(vpdev, db);
 418         else
 419                 goto done;
 420         for (retry = 15; retry--;) {
 421                 ret = wait_event_timeout(wake, devp->guest_ack,
 422                                          msecs_to_jiffies(1000));
 423                 if (ret)
 424                         break;
 425         }
 426 done:
 427         devp->config_change = 0;
 428         devp->guest_ack = 0;
 429 }
 430
 431 static void vop_virtio_del_device(struct vop_vdev *vdev)
 432 {
 433         struct vop_info *vi = vdev->vi;
 434         struct vop_device *vpdev = vdev->vpdev;
 435         int i;
 436         struct mic_vqconfig *vqconfig;
 437         struct mic_bootparam *bootparam = vpdev->hw_ops->get_dp(vpdev);
 438
 439         if (!bootparam)
 440                 goto skip_hot_remove;
 441         vop_dev_remove(vi, vdev->dc, vpdev);
 442 skip_hot_remove:
 443         vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev);
 444         flush_work(&vdev->virtio_bh_work);
 445         vqconfig = mic_vq_config(vdev->dd);
 446         for (i = 0; i < vdev->dd->num_vq; i++) {
 447                 struct vop_vringh *vvr = &vdev->vvr[i];
 448
 449                 dma_unmap_single(&vpdev->dev,
 450                                  vvr->buf_da, VOP_INT_DMA_BUF_SIZE,
 451                                  DMA_BIDIRECTIONAL);
 452                 free_pages((unsigned long)vvr->buf,
 453                            get_order(VOP_INT_DMA_BUF_SIZE));
 454                 vringh_kiov_cleanup(&vvr->riov);
 455                 vringh_kiov_cleanup(&vvr->wiov);
 456                 dma_unmap_single(&vpdev->dev, le64_to_cpu(vqconfig[i].address),
 457                                  vvr->vring.len, DMA_BIDIRECTIONAL);
 458                 free_pages((unsigned long)vvr->vring.va,
 459                            get_order(vvr->vring.len));
 460         }
 461         /*
 462          * Order the type update with previous stores. This write barrier
 463          * is paired with the corresponding read barrier before the uncached
 464          * system memory read of the type, on the card while scanning the
 465          * device page.
 466          */
 467         smp_wmb();
 468         vdev->dd->type = -1;
 469 }
 470
 471 /*
 472  * vop_sync_dma - Wrapper for synchronous DMAs.
 473  *
 474  * @dev - The address of the pointer to the device instance used
 475  * for DMA registration.
 476  * @dst - destination DMA address.
 477  * @src - source DMA address.
 478  * @len - size of the transfer.
 479  *
 480  * Return DMA_SUCCESS on success
 481  */
 482 static int vop_sync_dma(struct vop_vdev *vdev, dma_addr_t dst, dma_addr_t src,
 483                         size_t len)
 484 {
 485         int err = 0;
 486         struct dma_device *ddev;
 487         struct dma_async_tx_descriptor *tx;
 488         struct vop_info *vi = dev_get_drvdata(&vdev->vpdev->dev);
 489         struct dma_chan *vop_ch = vi->dma_ch;
 490
 491         if (!vop_ch) {
 492                 err = -EBUSY;
 493                 goto error;
 494         }
 495         ddev = vop_ch->device;
 496         tx = ddev->device_prep_dma_memcpy(vop_ch, dst, src, len,
 497                 DMA_PREP_FENCE);
 498         if (!tx) {
 499                 err = -ENOMEM;
 500                 goto error;
 501         } else {
 502                 dma_cookie_t cookie;
 503
 504                 cookie = tx->tx_submit(tx);
 505                 if (dma_submit_error(cookie)) {
 506                         err = -ENOMEM;
 507                         goto error;
 508                 }
 509                 dma_async_issue_pending(vop_ch);
 510                 err = dma_sync_wait(vop_ch, cookie);
 511         }
 512 error:
 513         if (err)
 514                 dev_err(&vi->vpdev->dev, "%s %d err %d\n",
 515                         __func__, __LINE__, err);
 516         return err;
 517 }
 518
 519 #define VOP_USE_DMA true
 520
 521 /*
 522  * Initiates the copies across the PCIe bus from card memory to a user
 523  * space buffer. When transfers are done using DMA, source/destination
 524  * addresses and transfer length must follow the alignment requirements of
 525  * the MIC DMA engine.
 526  */
 527 static int vop_virtio_copy_to_user(struct vop_vdev *vdev, void __user *ubuf,
 528                                    size_t len, u64 daddr, size_t dlen,
 529                                    int vr_idx)
 530 {
 531         struct vop_device *vpdev = vdev->vpdev;
 532         void __iomem *dbuf = vpdev->hw_ops->ioremap(vpdev, daddr, len);
 533         struct vop_vringh *vvr = &vdev->vvr[vr_idx];
 534         struct vop_info *vi = dev_get_drvdata(&vpdev->dev);
 535         size_t dma_alignment = 1 << vi->dma_ch->device->copy_align;
 536         bool x200 = is_dma_copy_aligned(vi->dma_ch->device, 1, 1, 1);
 537         size_t dma_offset, partlen;
 538         int err;
 539
 540         if (!VOP_USE_DMA) {
 541                 if (copy_to_user(ubuf, (void __force *)dbuf, len)) {
 542                         err = -EFAULT;
 543                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 544                                 __func__, __LINE__, err);
 545                         goto err;
 546                 }
 547                 vdev->in_bytes += len;
 548                 err = 0;
 549                 goto err;
 550         }
 551
 552         dma_offset = daddr - round_down(daddr, dma_alignment);
 553         daddr -= dma_offset;
 554         len += dma_offset;
 555         /*
 556          * X100 uses DMA addresses as seen by the card so adding
 557          * the aperture base is not required for DMA. However x200
 558          * requires DMA addresses to be an offset into the bar so
 559          * add the aperture base for x200.
 560          */
 561         if (x200)
 562                 daddr += vpdev->aper->pa;
 563         while (len) {
 564                 partlen = min_t(size_t, len, VOP_INT_DMA_BUF_SIZE);
 565                 err = vop_sync_dma(vdev, vvr->buf_da, daddr,
 566                                    ALIGN(partlen, dma_alignment));
 567                 if (err) {
 568                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 569                                 __func__, __LINE__, err);
 570                         goto err;
 571                 }
 572                 if (copy_to_user(ubuf, vvr->buf + dma_offset,
 573                                  partlen - dma_offset)) {
 574                         err = -EFAULT;
 575                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 576                                 __func__, __LINE__, err);
 577                         goto err;
 578                 }
 579                 daddr += partlen;
 580                 ubuf += partlen;
 581                 dbuf += partlen;
 582                 vdev->in_bytes_dma += partlen;
 583                 vdev->in_bytes += partlen;
 584                 len -= partlen;
 585                 dma_offset = 0;
 586         }
 587         err = 0;
 588 err:
 589         vpdev->hw_ops->iounmap(vpdev, dbuf);
 590         dev_dbg(vop_dev(vdev),
 591                 "%s: ubuf %p dbuf %p len 0x%lx vr_idx 0x%x\n",
 592                 __func__, ubuf, dbuf, len, vr_idx);
 593         return err;
 594 }
 595
 596 /*
 597  * Initiates copies across the PCIe bus from a user space buffer to card
 598  * memory. When transfers are done using DMA, source/destination addresses
 599  * and transfer length must follow the alignment requirements of the MIC
 600  * DMA engine.
 601  */
 602 static int vop_virtio_copy_from_user(struct vop_vdev *vdev, void __user *ubuf,
 603                                      size_t len, u64 daddr, size_t dlen,
 604                                      int vr_idx)
 605 {
 606         struct vop_device *vpdev = vdev->vpdev;
 607         void __iomem *dbuf = vpdev->hw_ops->ioremap(vpdev, daddr, len);
 608         struct vop_vringh *vvr = &vdev->vvr[vr_idx];
 609         struct vop_info *vi = dev_get_drvdata(&vdev->vpdev->dev);
 610         size_t dma_alignment = 1 << vi->dma_ch->device->copy_align;
 611         bool x200 = is_dma_copy_aligned(vi->dma_ch->device, 1, 1, 1);
 612         size_t partlen;
 613         bool dma = VOP_USE_DMA;
 614         int err = 0;
 615
 616         if (daddr & (dma_alignment - 1)) {
 617                 vdev->tx_dst_unaligned += len;
 618                 dma = false;
 619         } else if (ALIGN(len, dma_alignment) > dlen) {
 620                 vdev->tx_len_unaligned += len;
 621                 dma = false;
 622         }
 623
 624         if (!dma)
 625                 goto memcpy;
 626
 627         /*
 628          * X100 uses DMA addresses as seen by the card so adding
 629          * the aperture base is not required for DMA. However x200
 630          * requires DMA addresses to be an offset into the bar so
 631          * add the aperture base for x200.
 632          */
 633         if (x200)
 634                 daddr += vpdev->aper->pa;
 635         while (len) {
 636                 partlen = min_t(size_t, len, VOP_INT_DMA_BUF_SIZE);
 637
 638                 if (copy_from_user(vvr->buf, ubuf, partlen)) {
 639                         err = -EFAULT;
 640                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 641                                 __func__, __LINE__, err);
 642                         goto err;
 643                 }
 644                 err = vop_sync_dma(vdev, daddr, vvr->buf_da,
 645                                    ALIGN(partlen, dma_alignment));
 646                 if (err) {
 647                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 648                                 __func__, __LINE__, err);
 649                         goto err;
 650                 }
 651                 daddr += partlen;
 652                 ubuf += partlen;
 653                 dbuf += partlen;
 654                 vdev->out_bytes_dma += partlen;
 655                 vdev->out_bytes += partlen;
 656                 len -= partlen;
 657         }
 658 memcpy:
 659         /*
 660          * We are copying to IO below and should ideally use something
 661          * like copy_from_user_toio(..) if it existed.
 662          */
 663         if (copy_from_user((void __force *)dbuf, ubuf, len)) {
 664                 err = -EFAULT;
 665                 dev_err(vop_dev(vdev), "%s %d err %d\n",
 666                         __func__, __LINE__, err);
 667                 goto err;
 668         }
 669         vdev->out_bytes += len;
 670         err = 0;
 671 err:
 672         vpdev->hw_ops->iounmap(vpdev, dbuf);
 673         dev_dbg(vop_dev(vdev),
 674                 "%s: ubuf %p dbuf %p len 0x%lx vr_idx 0x%x\n",
 675                 __func__, ubuf, dbuf, len, vr_idx);
 676         return err;
 677 }
 678
 679 #define MIC_VRINGH_READ true
 680
 681 /* Determine the total number of bytes consumed in a VRINGH KIOV */
 682 static inline u32 vop_vringh_iov_consumed(struct vringh_kiov *iov)
 683 {
 684         int i;
 685         u32 total = iov->consumed;
 686
 687         for (i = 0; i < iov->i; i++)
 688                 total += iov->iov[i].iov_len;
 689         return total;
 690 }
 691
 692 /*
 693  * Traverse the VRINGH KIOV and issue the APIs to trigger the copies.
 694  * This API is heavily based on the vringh_iov_xfer(..) implementation
 695  * in vringh.c. The reason we cannot reuse vringh_iov_pull_kern(..)
 696  * and vringh_iov_push_kern(..) directly is because there is no
 697  * way to override the VRINGH xfer(..) routines as of v3.10.
 698  */
 699 static int vop_vringh_copy(struct vop_vdev *vdev, struct vringh_kiov *iov,
 700                            void __user *ubuf, size_t len, bool read, int vr_idx,
 701                            size_t *out_len)
 702 {
 703         int ret = 0;
 704         size_t partlen, tot_len = 0;
 705
 706         while (len && iov->i < iov->used) {
 707                 struct kvec *kiov = &iov->iov[iov->i];
 708
 709                 partlen = min(kiov->iov_len, len);
 710                 if (read)
 711                         ret = vop_virtio_copy_to_user(vdev, ubuf, partlen,
 712                                                       (u64)kiov->iov_base,
 713                                                       kiov->iov_len,
 714                                                       vr_idx);
 715                 else
 716                         ret = vop_virtio_copy_from_user(vdev, ubuf, partlen,
 717                                                         (u64)kiov->iov_base,
 718                                                         kiov->iov_len,
 719                                                         vr_idx);
 720                 if (ret) {
 721                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 722                                 __func__, __LINE__, ret);
 723                         break;
 724                 }
 725                 len -= partlen;
 726                 ubuf += partlen;
 727                 tot_len += partlen;
 728                 iov->consumed += partlen;
 729                 kiov->iov_len -= partlen;
 730                 kiov->iov_base += partlen;
 731                 if (!kiov->iov_len) {
 732                         /* Fix up old iov element then increment. */
 733                         kiov->iov_len = iov->consumed;
 734                         kiov->iov_base -= iov->consumed;
 735
 736                         iov->consumed = 0;
 737                         iov->i++;
 738                 }
 739         }
 740         *out_len = tot_len;
 741         return ret;
 742 }
 743
 744 /*
 745  * Use the standard VRINGH infrastructure in the kernel to fetch new
 746  * descriptors, initiate the copies and update the used ring.
 747  */
 748 static int _vop_virtio_copy(struct vop_vdev *vdev, struct mic_copy_desc *copy)
 749 {
 750         int ret = 0;
 751         u32 iovcnt = copy->iovcnt;
 752         struct iovec iov;
 753         struct iovec __user *u_iov = copy->iov;
 754         void __user *ubuf = NULL;
 755         struct vop_vringh *vvr = &vdev->vvr[copy->vr_idx];
 756         struct vringh_kiov *riov = &vvr->riov;
 757         struct vringh_kiov *wiov = &vvr->wiov;
 758         struct vringh *vrh = &vvr->vrh;
 759         u16 *head = &vvr->head;
 760         struct mic_vring *vr = &vvr->vring;
 761         size_t len = 0, out_len;
 762
 763         copy->out_len = 0;
 764         /* Fetch a new IOVEC if all previous elements have been processed */
 765         if (riov->i == riov->used && wiov->i == wiov->used) {
 766                 ret = vringh_getdesc_kern(vrh, riov, wiov,
 767                                           head, GFP_KERNEL);
 768                 /* Check if there are available descriptors */
 769                 if (ret <= 0)
 770                         return ret;
 771         }
 772         while (iovcnt) {
 773                 if (!len) {
 774                         /* Copy over a new iovec from user space. */
 775                         ret = copy_from_user(&iov, u_iov, sizeof(*u_iov));
 776                         if (ret) {
 777                                 ret = -EINVAL;
 778                                 dev_err(vop_dev(vdev), "%s %d err %d\n",
 779                                         __func__, __LINE__, ret);
 780                                 break;
 781                         }
 782                         len = iov.iov_len;
 783                         ubuf = iov.iov_base;
 784                 }
 785                 /* Issue all the read descriptors first */
 786                 ret = vop_vringh_copy(vdev, riov, ubuf, len,
 787                                       MIC_VRINGH_READ, copy->vr_idx, &out_len);
 788                 if (ret) {
 789                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 790                                 __func__, __LINE__, ret);
 791                         break;
 792                 }
 793                 len -= out_len;
 794                 ubuf += out_len;
 795                 copy->out_len += out_len;
 796                 /* Issue the write descriptors next */
 797                 ret = vop_vringh_copy(vdev, wiov, ubuf, len,
 798                                       !MIC_VRINGH_READ, copy->vr_idx, &out_len);
 799                 if (ret) {
 800                         dev_err(vop_dev(vdev), "%s %d err %d\n",
 801                                 __func__, __LINE__, ret);
 802                         break;
 803                 }
 804                 len -= out_len;
 805                 ubuf += out_len;
 806                 copy->out_len += out_len;
 807                 if (!len) {
 808                         /* One user space iovec is now completed */
 809                         iovcnt--;
 810                         u_iov++;
 811                 }
 812                 /* Exit loop if all elements in KIOVs have been processed. */
 813                 if (riov->i == riov->used && wiov->i == wiov->used)
 814                         break;
 815         }
 816         /*
 817          * Update the used ring if a descriptor was available and some data was
 818          * copied in/out and the user asked for a used ring update.
 819          */
 820         if (*head != USHRT_MAX && copy->out_len && copy->update_used) {
 821                 u32 total = 0;
 822
 823                 /* Determine the total data consumed */
 824                 total += vop_vringh_iov_consumed(riov);
 825                 total += vop_vringh_iov_consumed(wiov);
 826                 vringh_complete_kern(vrh, *head, total);
 827                 *head = USHRT_MAX;
 828                 if (vringh_need_notify_kern(vrh) > 0)
 829                         vringh_notify(vrh);
 830                 vringh_kiov_cleanup(riov);
 831                 vringh_kiov_cleanup(wiov);
 832                 /* Update avail idx for user space */
 833                 vr->info->avail_idx = vrh->last_avail_idx;
 834         }
 835         return ret;
 836 }
 837
 838 static inline int vop_verify_copy_args(struct vop_vdev *vdev,
 839                                        struct mic_copy_desc *copy)
 840 {
 841         if (!vdev || copy->vr_idx >= vdev->dd->num_vq)
 842                 return -EINVAL;
 843         return 0;
 844 }
 845
 846 /* Copy a specified number of virtio descriptors in a chain */
 847 static int vop_virtio_copy_desc(struct vop_vdev *vdev,
 848                                 struct mic_copy_desc *copy)
 849 {
 850         int err;
 851         struct vop_vringh *vvr;
 852
 853         err = vop_verify_copy_args(vdev, copy);
 854         if (err)
 855                 return err;
 856
 857         vvr = &vdev->vvr[copy->vr_idx];
 858         mutex_lock(&vvr->vr_mutex);
 859         if (!vop_vdevup(vdev)) {
 860                 err = -ENODEV;
 861                 dev_err(vop_dev(vdev), "%s %d err %d\n",
 862                         __func__, __LINE__, err);
 863                 goto err;
 864         }
 865         err = _vop_virtio_copy(vdev, copy);
 866         if (err) {
 867                 dev_err(vop_dev(vdev), "%s %d err %d\n",
 868                         __func__, __LINE__, err);
 869         }
 870 err:
 871         mutex_unlock(&vvr->vr_mutex);
 872         return err;
 873 }
 874
 875 static int vop_open(struct inode *inode, struct file *f)
 876 {
 877         struct vop_vdev *vdev;
 878         struct vop_info *vi = container_of(f->private_data,
 879                 struct vop_info, miscdev);
 880
 881         vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
 882         if (!vdev)
 883                 return -ENOMEM;
 884         vdev->vi = vi;
 885         mutex_init(&vdev->vdev_mutex);
 886         f->private_data = vdev;
 887         init_completion(&vdev->destroy);
 888         complete(&vdev->destroy);
 889         return 0;
 890 }
 891
 892 static int vop_release(struct inode *inode, struct file *f)
 893 {
 894         struct vop_vdev *vdev = f->private_data, *vdev_tmp;
 895         struct vop_info *vi = vdev->vi;
 896         struct list_head *pos, *tmp;
 897         bool found = false;
 898
 899         mutex_lock(&vdev->vdev_mutex);
 900         if (vdev->deleted)
 901                 goto unlock;
 902         mutex_lock(&vi->vop_mutex);
 903         list_for_each_safe(pos, tmp, &vi->vdev_list) {
 904                 vdev_tmp = list_entry(pos, struct vop_vdev, list);
 905                 if (vdev == vdev_tmp) {
 906                         vop_virtio_del_device(vdev);
 907                         list_del(pos);
 908                         found = true;
 909                         break;
 910                 }
 911         }
 912         mutex_unlock(&vi->vop_mutex);
 913 unlock:
 914         mutex_unlock(&vdev->vdev_mutex);
 915         if (!found)
 916                 wait_for_completion(&vdev->destroy);
 917         f->private_data = NULL;
 918         kfree(vdev);
 919         return 0;
 920 }
 921
 922 static long vop_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
 923 {
 924         struct vop_vdev *vdev = f->private_data;
 925         struct vop_info *vi = vdev->vi;
 926         void __user *argp = (void __user *)arg;
 927         int ret;
 928
 929         switch (cmd) {
 930         case MIC_VIRTIO_ADD_DEVICE:
 931         {
 932                 struct mic_device_desc dd, *dd_config;
 933
 934                 if (copy_from_user(&dd, argp, sizeof(dd)))
 935                         return -EFAULT;
 936
 937                 if (mic_aligned_desc_size(&dd) > MIC_MAX_DESC_BLK_SIZE ||
 938                     dd.num_vq > MIC_MAX_VRINGS)
 939                         return -EINVAL;
 940
 941                 dd_config = kzalloc(mic_desc_size(&dd), GFP_KERNEL);
 942                 if (!dd_config)
 943                         return -ENOMEM;
 944                 if (copy_from_user(dd_config, argp, mic_desc_size(&dd))) {
 945                         ret = -EFAULT;
 946                         goto free_ret;
 947                 }
 948                 mutex_lock(&vdev->vdev_mutex);
 949                 mutex_lock(&vi->vop_mutex);
 950                 ret = vop_virtio_add_device(vdev, dd_config);
 951                 if (ret)
 952                         goto unlock_ret;
 953                 list_add_tail(&vdev->list, &vi->vdev_list);
 954 unlock_ret:
 955                 mutex_unlock(&vi->vop_mutex);
 956                 mutex_unlock(&vdev->vdev_mutex);
 957 free_ret:
 958                 kfree(dd_config);
 959                 return ret;
 960         }
 961         case MIC_VIRTIO_COPY_DESC:
 962         {
 963                 struct mic_copy_desc copy;
 964
 965                 mutex_lock(&vdev->vdev_mutex);
 966                 ret = vop_vdev_inited(vdev);
 967                 if (ret)
 968                         goto _unlock_ret;
 969
 970                 if (copy_from_user(&copy, argp, sizeof(copy))) {
 971                         ret = -EFAULT;
 972                         goto _unlock_ret;
 973                 }
 974
 975                 ret = vop_virtio_copy_desc(vdev, &copy);
 976                 if (ret < 0)
 977                         goto _unlock_ret;
 978                 if (copy_to_user(
 979                         &((struct mic_copy_desc __user *)argp)->out_len,
 980                         &copy.out_len, sizeof(copy.out_len)))
 981                         ret = -EFAULT;
 982 _unlock_ret:
 983                 mutex_unlock(&vdev->vdev_mutex);
 984                 return ret;
 985         }
 986         case MIC_VIRTIO_CONFIG_CHANGE:
 987         {
 988                 void *buf;
 989
 990                 mutex_lock(&vdev->vdev_mutex);
 991                 ret = vop_vdev_inited(vdev);
 992                 if (ret)
 993                         goto __unlock_ret;
 994                 buf = kzalloc(vdev->dd->config_len, GFP_KERNEL);
 995                 if (!buf) {
 996                         ret = -ENOMEM;
 997                         goto __unlock_ret;
 998                 }
 999                 if (copy_from_user(buf, argp, vdev->dd->config_len)) {
1000                         ret = -EFAULT;
1001                         goto done;
1002                 }
1003                 ret = vop_virtio_config_change(vdev, buf);
1004 done:
1005                 kfree(buf);
1006 __unlock_ret:
1007                 mutex_unlock(&vdev->vdev_mutex);
1008                 return ret;
1009         }
1010         default:
1011                 return -ENOIOCTLCMD;
1012         };
1013         return 0;
1014 }
1015
1016 /*
1017  * We return POLLIN | POLLOUT from poll when new buffers are enqueued, and
1018  * not when previously enqueued buffers may be available. This means that
1019  * in the card->host (TX) path, when userspace is unblocked by poll it
1020  * must drain all available descriptors or it can stall.
1021  */
1022 static unsigned int vop_poll(struct file *f, poll_table *wait)
1023 {
1024         struct vop_vdev *vdev = f->private_data;
1025         int mask = 0;
1026
1027         mutex_lock(&vdev->vdev_mutex);
1028         if (vop_vdev_inited(vdev)) {
1029                 mask = POLLERR;
1030                 goto done;
1031         }
1032         poll_wait(f, &vdev->waitq, wait);
1033         if (vop_vdev_inited(vdev)) {
1034                 mask = POLLERR;
1035         } else if (vdev->poll_wake) {
1036                 vdev->poll_wake = 0;
1037                 mask = POLLIN | POLLOUT;
1038         }
1039 done:
1040         mutex_unlock(&vdev->vdev_mutex);
1041         return mask;
1042 }
1043
1044 static inline int
1045 vop_query_offset(struct vop_vdev *vdev, unsigned long offset,
1046                  unsigned long *size, unsigned long *pa)
1047 {
1048         struct vop_device *vpdev = vdev->vpdev;
1049         unsigned long start = MIC_DP_SIZE;
1050         int i;
1051
1052         /*
1053          * MMAP interface is as follows:
1054          * offset                               region
1055          * 0x0                                  virtio device_page
1056          * 0x1000                               first vring
1057          * 0x1000 + size of 1st vring           second vring
1058          * ....
1059          */
1060         if (!offset) {
1061                 *pa = virt_to_phys(vpdev->hw_ops->get_dp(vpdev));
1062                 *size = MIC_DP_SIZE;
1063                 return 0;
1064         }
1065
1066         for (i = 0; i < vdev->dd->num_vq; i++) {
1067                 struct vop_vringh *vvr = &vdev->vvr[i];
1068
1069                 if (offset == start) {
1070                         *pa = virt_to_phys(vvr->vring.va);
1071                         *size = vvr->vring.len;
1072                         return 0;
1073                 }
1074                 start += vvr->vring.len;
1075         }
1076         return -1;
1077 }
1078
1079 /*
1080  * Maps the device page and virtio rings to user space for readonly access.
1081  */
1082 static int vop_mmap(struct file *f, struct vm_area_struct *vma)
1083 {
1084         struct vop_vdev *vdev = f->private_data;
1085         unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
1086         unsigned long pa, size = vma->vm_end - vma->vm_start, size_rem = size;
1087         int i, err;
1088
1089         err = vop_vdev_inited(vdev);
1090         if (err)
1091                 goto ret;
1092         if (vma->vm_flags & VM_WRITE) {
1093                 err = -EACCES;
1094                 goto ret;
1095         }
1096         while (size_rem) {
1097                 i = vop_query_offset(vdev, offset, &size, &pa);
1098                 if (i < 0) {
1099                         err = -EINVAL;
1100                         goto ret;
1101                 }
1102                 err = remap_pfn_range(vma, vma->vm_start + offset,
1103                                       pa >> PAGE_SHIFT, size,
1104                                       vma->vm_page_prot);
1105                 if (err)
1106                         goto ret;
1107                 size_rem -= size;
1108                 offset += size;
1109         }
1110 ret:
1111         return err;
1112 }
1113
1114 static const struct file_operations vop_fops = {
1115         .open = vop_open,
1116         .release = vop_release,
1117         .unlocked_ioctl = vop_ioctl,
1118         .poll = vop_poll,
1119         .mmap = vop_mmap,
1120         .owner = THIS_MODULE,
1121 };
1122
1123 int vop_host_init(struct vop_info *vi)
1124 {
1125         int rc;
1126         struct miscdevice *mdev;
1127         struct vop_device *vpdev = vi->vpdev;
1128
1129         INIT_LIST_HEAD(&vi->vdev_list);
1130         vi->dma_ch = vpdev->dma_ch;
1131         mdev = &vi->miscdev;
1132         mdev->minor = MISC_DYNAMIC_MINOR;
1133         snprintf(vi->name, sizeof(vi->name), "vop_virtio%d", vpdev->index);
1134         mdev->name = vi->name;
1135         mdev->fops = &vop_fops;
1136         mdev->parent = &vpdev->dev;
1137
1138         rc = misc_register(mdev);
1139         if (rc)
1140                 dev_err(&vpdev->dev, "%s failed rc %d\n", __func__, rc);
1141         return rc;
1142 }
1143
1144 void vop_host_uninit(struct vop_info *vi)
1145 {
1146         struct list_head *pos, *tmp;
1147         struct vop_vdev *vdev;
1148
1149         mutex_lock(&vi->vop_mutex);
1150         vop_virtio_reset_devices(vi);
1151         list_for_each_safe(pos, tmp, &vi->vdev_list) {
1152                 vdev = list_entry(pos, struct vop_vdev, list);
1153                 list_del(pos);
1154                 reinit_completion(&vdev->destroy);
1155                 mutex_unlock(&vi->vop_mutex);
1156                 mutex_lock(&vdev->vdev_mutex);
1157                 vop_virtio_del_device(vdev);
1158                 vdev->deleted = true;
1159                 mutex_unlock(&vdev->vdev_mutex);
1160                 complete(&vdev->destroy);
1161                 mutex_lock(&vi->vop_mutex);
1162         }
1163         mutex_unlock(&vi->vop_mutex);
1164         misc_deregister(&vi->miscdev);
1165 }