drivers/infiniband/hw/i40iw/i40iw_main.c

   1 /*******************************************************************************
   2 *
   3 * Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenFabrics.org BSD license below:
  10 *
  11 *   Redistribution and use in source and binary forms, with or
  12 *   without modification, are permitted provided that the following
  13 *   conditions are met:
  14 *
  15 *    - Redistributions of source code must retain the above
  16 *       copyright notice, this list of conditions and the following
  17 *       disclaimer.
  18 *
  19 *    - Redistributions in binary form must reproduce the above
  20 *       copyright notice, this list of conditions and the following
  21 *       disclaimer in the documentation and/or other materials
  22 *       provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 *
  33 *******************************************************************************/
  34
  35 #include <linux/module.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/ip.h>
  40 #include <linux/tcp.h>
  41 #include <linux/if_vlan.h>
  42 #include <net/addrconf.h>
  43
  44 #include "i40iw.h"
  45 #include "i40iw_register.h"
  46 #include <net/netevent.h>
  47 #define CLIENT_IW_INTERFACE_VERSION_MAJOR 0
  48 #define CLIENT_IW_INTERFACE_VERSION_MINOR 01
  49 #define CLIENT_IW_INTERFACE_VERSION_BUILD 00
  50
  51 #define DRV_VERSION_MAJOR 0
  52 #define DRV_VERSION_MINOR 5
  53 #define DRV_VERSION_BUILD 123
  54 #define DRV_VERSION     __stringify(DRV_VERSION_MAJOR) "."              \
  55         __stringify(DRV_VERSION_MINOR) "." __stringify(DRV_VERSION_BUILD)
  56
  57 static int push_mode;
  58 module_param(push_mode, int, 0644);
  59 MODULE_PARM_DESC(push_mode, "Low latency mode: 0=disabled (default), 1=enabled)");
  60
  61 static int debug;
  62 module_param(debug, int, 0644);
  63 MODULE_PARM_DESC(debug, "debug flags: 0=disabled (default), 0x7fffffff=all");
  64
  65 static int resource_profile;
  66 module_param(resource_profile, int, 0644);
  67 MODULE_PARM_DESC(resource_profile,
  68                  "Resource Profile: 0=no VF RDMA support (default), 1=Weighted VF, 2=Even Distribution");
  69
  70 static int max_rdma_vfs = 32;
  71 module_param(max_rdma_vfs, int, 0644);
  72 MODULE_PARM_DESC(max_rdma_vfs, "Maximum VF count: 0-32 32=default");
  73 static int mpa_version = 2;
  74 module_param(mpa_version, int, 0644);
  75 MODULE_PARM_DESC(mpa_version, "MPA version to be used in MPA Req/Resp 1 or 2");
  76
  77 MODULE_AUTHOR("Intel Corporation, <e1000-rdma@lists.sourceforge.net>");
  78 MODULE_DESCRIPTION("Intel(R) Ethernet Connection X722 iWARP RDMA Driver");
  79 MODULE_LICENSE("Dual BSD/GPL");
  80 MODULE_VERSION(DRV_VERSION);
  81
  82 static struct i40e_client i40iw_client;
  83 static char i40iw_client_name[I40E_CLIENT_STR_LENGTH] = "i40iw";
  84
  85 static LIST_HEAD(i40iw_handlers);
  86 static spinlock_t i40iw_handler_lock;
  87
  88 static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,
  89                                                   u32 vf_id, u8 *msg, u16 len);
  90
  91 static struct notifier_block i40iw_inetaddr_notifier = {
  92         .notifier_call = i40iw_inetaddr_event
  93 };
  94
  95 static struct notifier_block i40iw_inetaddr6_notifier = {
  96         .notifier_call = i40iw_inet6addr_event
  97 };
  98
  99 static struct notifier_block i40iw_net_notifier = {
 100         .notifier_call = i40iw_net_event
 101 };
 102
 103 static atomic_t i40iw_notifiers_registered;
 104
 105 /**
 106  * i40iw_find_i40e_handler - find a handler given a client info
 107  * @ldev: pointer to a client info
 108  */
 109 static struct i40iw_handler *i40iw_find_i40e_handler(struct i40e_info *ldev)
 110 {
 111         struct i40iw_handler *hdl;
 112         unsigned long flags;
 113
 114         spin_lock_irqsave(&i40iw_handler_lock, flags);
 115         list_for_each_entry(hdl, &i40iw_handlers, list) {
 116                 if (hdl->ldev.netdev == ldev->netdev) {
 117                         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 118                         return hdl;
 119                 }
 120         }
 121         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 122         return NULL;
 123 }
 124
 125 /**
 126  * i40iw_find_netdev - find a handler given a netdev
 127  * @netdev: pointer to net_device
 128  */
 129 struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev)
 130 {
 131         struct i40iw_handler *hdl;
 132         unsigned long flags;
 133
 134         spin_lock_irqsave(&i40iw_handler_lock, flags);
 135         list_for_each_entry(hdl, &i40iw_handlers, list) {
 136                 if (hdl->ldev.netdev == netdev) {
 137                         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 138                         return hdl;
 139                 }
 140         }
 141         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 142         return NULL;
 143 }
 144
 145 /**
 146  * i40iw_add_handler - add a handler to the list
 147  * @hdl: handler to be added to the handler list
 148  */
 149 static void i40iw_add_handler(struct i40iw_handler *hdl)
 150 {
 151         unsigned long flags;
 152
 153         spin_lock_irqsave(&i40iw_handler_lock, flags);
 154         list_add(&hdl->list, &i40iw_handlers);
 155         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 156 }
 157
 158 /**
 159  * i40iw_del_handler - delete a handler from the list
 160  * @hdl: handler to be deleted from the handler list
 161  */
 162 static int i40iw_del_handler(struct i40iw_handler *hdl)
 163 {
 164         unsigned long flags;
 165
 166         spin_lock_irqsave(&i40iw_handler_lock, flags);
 167         list_del(&hdl->list);
 168         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 169         return 0;
 170 }
 171
 172 /**
 173  * i40iw_enable_intr - set up device interrupts
 174  * @dev: hardware control device structure
 175  * @msix_id: id of the interrupt to be enabled
 176  */
 177 static void i40iw_enable_intr(struct i40iw_sc_dev *dev, u32 msix_id)
 178 {
 179         u32 val;
 180
 181         val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
 182                 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
 183                 (3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
 184         if (dev->is_pf)
 185                 i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_id - 1), val);
 186         else
 187                 i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_id - 1), val);
 188 }
 189
 190 /**
 191  * i40iw_dpc - tasklet for aeq and ceq 0
 192  * @data: iwarp device
 193  */
 194 static void i40iw_dpc(unsigned long data)
 195 {
 196         struct i40iw_device *iwdev = (struct i40iw_device *)data;
 197
 198         if (iwdev->msix_shared)
 199                 i40iw_process_ceq(iwdev, iwdev->ceqlist);
 200         i40iw_process_aeq(iwdev);
 201         i40iw_enable_intr(&iwdev->sc_dev, iwdev->iw_msixtbl[0].idx);
 202 }
 203
 204 /**
 205  * i40iw_ceq_dpc - dpc handler for CEQ
 206  * @data: data points to CEQ
 207  */
 208 static void i40iw_ceq_dpc(unsigned long data)
 209 {
 210         struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;
 211         struct i40iw_device *iwdev = iwceq->iwdev;
 212
 213         i40iw_process_ceq(iwdev, iwceq);
 214         i40iw_enable_intr(&iwdev->sc_dev, iwceq->msix_idx);
 215 }
 216
 217 /**
 218  * i40iw_irq_handler - interrupt handler for aeq and ceq0
 219  * @irq: Interrupt request number
 220  * @data: iwarp device
 221  */
 222 static irqreturn_t i40iw_irq_handler(int irq, void *data)
 223 {
 224         struct i40iw_device *iwdev = (struct i40iw_device *)data;
 225
 226         tasklet_schedule(&iwdev->dpc_tasklet);
 227         return IRQ_HANDLED;
 228 }
 229
 230 /**
 231  * i40iw_destroy_cqp  - destroy control qp
 232  * @iwdev: iwarp device
 233  * @create_done: 1 if cqp create poll was success
 234  *
 235  * Issue destroy cqp request and
 236  * free the resources associated with the cqp
 237  */
 238 static void i40iw_destroy_cqp(struct i40iw_device *iwdev, bool free_hwcqp)
 239 {
 240         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 241         struct i40iw_cqp *cqp = &iwdev->cqp;
 242
 243         if (free_hwcqp)
 244                 dev->cqp_ops->cqp_destroy(dev->cqp);
 245
 246         i40iw_cleanup_pending_cqp_op(iwdev);
 247
 248         i40iw_free_dma_mem(dev->hw, &cqp->sq);
 249         kfree(cqp->scratch_array);
 250         iwdev->cqp.scratch_array = NULL;
 251
 252         kfree(cqp->cqp_requests);
 253         cqp->cqp_requests = NULL;
 254 }
 255
 256 /**
 257  * i40iw_disable_irqs - disable device interrupts
 258  * @dev: hardware control device structure
 259  * @msic_vec: msix vector to disable irq
 260  * @dev_id: parameter to pass to free_irq (used during irq setup)
 261  *
 262  * The function is called when destroying aeq/ceq
 263  */
 264 static void i40iw_disable_irq(struct i40iw_sc_dev *dev,
 265                               struct i40iw_msix_vector *msix_vec,
 266                               void *dev_id)
 267 {
 268         if (dev->is_pf)
 269                 i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_vec->idx - 1), 0);
 270         else
 271                 i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_vec->idx - 1), 0);
 272         irq_set_affinity_hint(msix_vec->irq, NULL);
 273         free_irq(msix_vec->irq, dev_id);
 274 }
 275
 276 /**
 277  * i40iw_destroy_aeq - destroy aeq
 278  * @iwdev: iwarp device
 279  *
 280  * Issue a destroy aeq request and
 281  * free the resources associated with the aeq
 282  * The function is called during driver unload
 283  */
 284 static void i40iw_destroy_aeq(struct i40iw_device *iwdev)
 285 {
 286         enum i40iw_status_code status = I40IW_ERR_NOT_READY;
 287         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 288         struct i40iw_aeq *aeq = &iwdev->aeq;
 289
 290         if (!iwdev->msix_shared)
 291                 i40iw_disable_irq(dev, iwdev->iw_msixtbl, (void *)iwdev);
 292         if (iwdev->reset)
 293                 goto exit;
 294
 295         if (!dev->aeq_ops->aeq_destroy(&aeq->sc_aeq, 0, 1))
 296                 status = dev->aeq_ops->aeq_destroy_done(&aeq->sc_aeq);
 297         if (status)
 298                 i40iw_pr_err("destroy aeq failed %d\n", status);
 299
 300 exit:
 301         i40iw_free_dma_mem(dev->hw, &aeq->mem);
 302 }
 303
 304 /**
 305  * i40iw_destroy_ceq - destroy ceq
 306  * @iwdev: iwarp device
 307  * @iwceq: ceq to be destroyed
 308  *
 309  * Issue a destroy ceq request and
 310  * free the resources associated with the ceq
 311  */
 312 static void i40iw_destroy_ceq(struct i40iw_device *iwdev,
 313                               struct i40iw_ceq *iwceq)
 314 {
 315         enum i40iw_status_code status;
 316         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 317
 318         if (iwdev->reset)
 319                 goto exit;
 320
 321         status = dev->ceq_ops->ceq_destroy(&iwceq->sc_ceq, 0, 1);
 322         if (status) {
 323                 i40iw_pr_err("ceq destroy command failed %d\n", status);
 324                 goto exit;
 325         }
 326
 327         status = dev->ceq_ops->cceq_destroy_done(&iwceq->sc_ceq);
 328         if (status)
 329                 i40iw_pr_err("ceq destroy completion failed %d\n", status);
 330 exit:
 331         i40iw_free_dma_mem(dev->hw, &iwceq->mem);
 332 }
 333
 334 /**
 335  * i40iw_dele_ceqs - destroy all ceq's
 336  * @iwdev: iwarp device
 337  *
 338  * Go through all of the device ceq's and for each ceq
 339  * disable the ceq interrupt and destroy the ceq
 340  */
 341 static void i40iw_dele_ceqs(struct i40iw_device *iwdev)
 342 {
 343         u32 i = 0;
 344         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 345         struct i40iw_ceq *iwceq = iwdev->ceqlist;
 346         struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl;
 347
 348         if (iwdev->msix_shared) {
 349                 i40iw_disable_irq(dev, msix_vec, (void *)iwdev);
 350                 i40iw_destroy_ceq(iwdev, iwceq);
 351                 iwceq++;
 352                 i++;
 353         }
 354
 355         for (msix_vec++; i < iwdev->ceqs_count; i++, msix_vec++, iwceq++) {
 356                 i40iw_disable_irq(dev, msix_vec, (void *)iwceq);
 357                 i40iw_destroy_ceq(iwdev, iwceq);
 358         }
 359 }
 360
 361 /**
 362  * i40iw_destroy_ccq - destroy control cq
 363  * @iwdev: iwarp device
 364  *
 365  * Issue destroy ccq request and
 366  * free the resources associated with the ccq
 367  */
 368 static void i40iw_destroy_ccq(struct i40iw_device *iwdev)
 369 {
 370         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 371         struct i40iw_ccq *ccq = &iwdev->ccq;
 372         enum i40iw_status_code status = 0;
 373
 374         if (!iwdev->reset)
 375                 status = dev->ccq_ops->ccq_destroy(dev->ccq, 0, true);
 376         if (status)
 377                 i40iw_pr_err("ccq destroy failed %d\n", status);
 378         i40iw_free_dma_mem(dev->hw, &ccq->mem_cq);
 379 }
 380
 381 /* types of hmc objects */
 382 static enum i40iw_hmc_rsrc_type iw_hmc_obj_types[] = {
 383         I40IW_HMC_IW_QP,
 384         I40IW_HMC_IW_CQ,
 385         I40IW_HMC_IW_HTE,
 386         I40IW_HMC_IW_ARP,
 387         I40IW_HMC_IW_APBVT_ENTRY,
 388         I40IW_HMC_IW_MR,
 389         I40IW_HMC_IW_XF,
 390         I40IW_HMC_IW_XFFL,
 391         I40IW_HMC_IW_Q1,
 392         I40IW_HMC_IW_Q1FL,
 393         I40IW_HMC_IW_TIMER,
 394 };
 395
 396 /**
 397  * i40iw_close_hmc_objects_type - delete hmc objects of a given type
 398  * @iwdev: iwarp device
 399  * @obj_type: the hmc object type to be deleted
 400  * @is_pf: true if the function is PF otherwise false
 401  * @reset: true if called before reset
 402  */
 403 static void i40iw_close_hmc_objects_type(struct i40iw_sc_dev *dev,
 404                                          enum i40iw_hmc_rsrc_type obj_type,
 405                                          struct i40iw_hmc_info *hmc_info,
 406                                          bool is_pf,
 407                                          bool reset)
 408 {
 409         struct i40iw_hmc_del_obj_info info;
 410
 411         memset(&info, 0, sizeof(info));
 412         info.hmc_info = hmc_info;
 413         info.rsrc_type = obj_type;
 414         info.count = hmc_info->hmc_obj[obj_type].cnt;
 415         info.is_pf = is_pf;
 416         if (dev->hmc_ops->del_hmc_object(dev, &info, reset))
 417                 i40iw_pr_err("del obj of type %d failed\n", obj_type);
 418 }
 419
 420 /**
 421  * i40iw_del_hmc_objects - remove all device hmc objects
 422  * @dev: iwarp device
 423  * @hmc_info: hmc_info to free
 424  * @is_pf: true if hmc_info belongs to PF, not vf nor allocated
 425  *         by PF on behalf of VF
 426  * @reset: true if called before reset
 427  */
 428 static void i40iw_del_hmc_objects(struct i40iw_sc_dev *dev,
 429                                   struct i40iw_hmc_info *hmc_info,
 430                                   bool is_pf,
 431                                   bool reset)
 432 {
 433         unsigned int i;
 434
 435         for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++)
 436                 i40iw_close_hmc_objects_type(dev, iw_hmc_obj_types[i], hmc_info, is_pf, reset);
 437 }
 438
 439 /**
 440  * i40iw_ceq_handler - interrupt handler for ceq
 441  * @data: ceq pointer
 442  */
 443 static irqreturn_t i40iw_ceq_handler(int irq, void *data)
 444 {
 445         struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;
 446
 447         if (iwceq->irq != irq)
 448                 i40iw_pr_err("expected irq = %d received irq = %d\n", iwceq->irq, irq);
 449         tasklet_schedule(&iwceq->dpc_tasklet);
 450         return IRQ_HANDLED;
 451 }
 452
 453 /**
 454  * i40iw_create_hmc_obj_type - create hmc object of a given type
 455  * @dev: hardware control device structure
 456  * @info: information for the hmc object to create
 457  */
 458 static enum i40iw_status_code i40iw_create_hmc_obj_type(struct i40iw_sc_dev *dev,
 459                                                         struct i40iw_hmc_create_obj_info *info)
 460 {
 461         return dev->hmc_ops->create_hmc_object(dev, info);
 462 }
 463
 464 /**
 465  * i40iw_create_hmc_objs - create all hmc objects for the device
 466  * @iwdev: iwarp device
 467  * @is_pf: true if the function is PF otherwise false
 468  *
 469  * Create the device hmc objects and allocate hmc pages
 470  * Return 0 if successful, otherwise clean up and return error
 471  */
 472 static enum i40iw_status_code i40iw_create_hmc_objs(struct i40iw_device *iwdev,
 473                                                     bool is_pf)
 474 {
 475         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 476         struct i40iw_hmc_create_obj_info info;
 477         enum i40iw_status_code status;
 478         int i;
 479
 480         memset(&info, 0, sizeof(info));
 481         info.hmc_info = dev->hmc_info;
 482         info.is_pf = is_pf;
 483         info.entry_type = iwdev->sd_type;
 484         for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
 485                 info.rsrc_type = iw_hmc_obj_types[i];
 486                 info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt;
 487                 status = i40iw_create_hmc_obj_type(dev, &info);
 488                 if (status) {
 489                         i40iw_pr_err("create obj type %d status = %d\n",
 490                                      iw_hmc_obj_types[i], status);
 491                         break;
 492                 }
 493         }
 494         if (!status)
 495                 return (dev->cqp_misc_ops->static_hmc_pages_allocated(dev->cqp, 0,
 496                                                                       dev->hmc_fn_id,
 497                                                                       true, true));
 498
 499         while (i) {
 500                 i--;
 501                 /* destroy the hmc objects of a given type */
 502                 i40iw_close_hmc_objects_type(dev,
 503                                              iw_hmc_obj_types[i],
 504                                              dev->hmc_info,
 505                                              is_pf,
 506                                              false);
 507         }
 508         return status;
 509 }
 510
 511 /**
 512  * i40iw_obj_aligned_mem - get aligned memory from device allocated memory
 513  * @iwdev: iwarp device
 514  * @memptr: points to the memory addresses
 515  * @size: size of memory needed
 516  * @mask: mask for the aligned memory
 517  *
 518  * Get aligned memory of the requested size and
 519  * update the memptr to point to the new aligned memory
 520  * Return 0 if successful, otherwise return no memory error
 521  */
 522 enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev,
 523                                              struct i40iw_dma_mem *memptr,
 524                                              u32 size,
 525                                              u32 mask)
 526 {
 527         unsigned long va, newva;
 528         unsigned long extra;
 529
 530         va = (unsigned long)iwdev->obj_next.va;
 531         newva = va;
 532         if (mask)
 533                 newva = ALIGN(va, (mask + 1));
 534         extra = newva - va;
 535         memptr->va = (u8 *)va + extra;
 536         memptr->pa = iwdev->obj_next.pa + extra;
 537         memptr->size = size;
 538         if ((memptr->va + size) > (iwdev->obj_mem.va + iwdev->obj_mem.size))
 539                 return I40IW_ERR_NO_MEMORY;
 540
 541         iwdev->obj_next.va = memptr->va + size;
 542         iwdev->obj_next.pa = memptr->pa + size;
 543         return 0;
 544 }
 545
 546 /**
 547  * i40iw_create_cqp - create control qp
 548  * @iwdev: iwarp device
 549  *
 550  * Return 0, if the cqp and all the resources associated with it
 551  * are successfully created, otherwise return error
 552  */
 553 static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev)
 554 {
 555         enum i40iw_status_code status;
 556         u32 sqsize = I40IW_CQP_SW_SQSIZE_2048;
 557         struct i40iw_dma_mem mem;
 558         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 559         struct i40iw_cqp_init_info cqp_init_info;
 560         struct i40iw_cqp *cqp = &iwdev->cqp;
 561         u16 maj_err, min_err;
 562         int i;
 563
 564         cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
 565         if (!cqp->cqp_requests)
 566                 return I40IW_ERR_NO_MEMORY;
 567         cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
 568         if (!cqp->scratch_array) {
 569                 kfree(cqp->cqp_requests);
 570                 return I40IW_ERR_NO_MEMORY;
 571         }
 572         dev->cqp = &cqp->sc_cqp;
 573         dev->cqp->dev = dev;
 574         memset(&cqp_init_info, 0, sizeof(cqp_init_info));
 575         status = i40iw_allocate_dma_mem(dev->hw, &cqp->sq,
 576                                         (sizeof(struct i40iw_cqp_sq_wqe) * sqsize),
 577                                         I40IW_CQP_ALIGNMENT);
 578         if (status)
 579                 goto exit;
 580         status = i40iw_obj_aligned_mem(iwdev, &mem, sizeof(struct i40iw_cqp_ctx),
 581                                        I40IW_HOST_CTX_ALIGNMENT_MASK);
 582         if (status)
 583                 goto exit;
 584         dev->cqp->host_ctx_pa = mem.pa;
 585         dev->cqp->host_ctx = mem.va;
 586         /* populate the cqp init info */
 587         cqp_init_info.dev = dev;
 588         cqp_init_info.sq_size = sqsize;
 589         cqp_init_info.sq = cqp->sq.va;
 590         cqp_init_info.sq_pa = cqp->sq.pa;
 591         cqp_init_info.host_ctx_pa = mem.pa;
 592         cqp_init_info.host_ctx = mem.va;
 593         cqp_init_info.hmc_profile = iwdev->resource_profile;
 594         cqp_init_info.enabled_vf_count = iwdev->max_rdma_vfs;
 595         cqp_init_info.scratch_array = cqp->scratch_array;
 596         status = dev->cqp_ops->cqp_init(dev->cqp, &cqp_init_info);
 597         if (status) {
 598                 i40iw_pr_err("cqp init status %d\n", status);
 599                 goto exit;
 600         }
 601         status = dev->cqp_ops->cqp_create(dev->cqp, &maj_err, &min_err);
 602         if (status) {
 603                 i40iw_pr_err("cqp create status %d maj_err %d min_err %d\n",
 604                              status, maj_err, min_err);
 605                 goto exit;
 606         }
 607         spin_lock_init(&cqp->req_lock);
 608         INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
 609         INIT_LIST_HEAD(&cqp->cqp_pending_reqs);
 610         /* init the waitq of the cqp_requests and add them to the list */
 611         for (i = 0; i < I40IW_CQP_SW_SQSIZE_2048; i++) {
 612                 init_waitqueue_head(&cqp->cqp_requests[i].waitq);
 613                 list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs);
 614         }
 615         return 0;
 616 exit:
 617         /* clean up the created resources */
 618         i40iw_destroy_cqp(iwdev, false);
 619         return status;
 620 }
 621
 622 /**
 623  * i40iw_create_ccq - create control cq
 624  * @iwdev: iwarp device
 625  *
 626  * Return 0, if the ccq and the resources associated with it
 627  * are successfully created, otherwise return error
 628  */
 629 static enum i40iw_status_code i40iw_create_ccq(struct i40iw_device *iwdev)
 630 {
 631         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 632         struct i40iw_dma_mem mem;
 633         enum i40iw_status_code status;
 634         struct i40iw_ccq_init_info info;
 635         struct i40iw_ccq *ccq = &iwdev->ccq;
 636
 637         memset(&info, 0, sizeof(info));
 638         dev->ccq = &ccq->sc_cq;
 639         dev->ccq->dev = dev;
 640         info.dev = dev;
 641         ccq->shadow_area.size = sizeof(struct i40iw_cq_shadow_area);
 642         ccq->mem_cq.size = sizeof(struct i40iw_cqe) * IW_CCQ_SIZE;
 643         status = i40iw_allocate_dma_mem(dev->hw, &ccq->mem_cq,
 644                                         ccq->mem_cq.size, I40IW_CQ0_ALIGNMENT);
 645         if (status)
 646                 goto exit;
 647         status = i40iw_obj_aligned_mem(iwdev, &mem, ccq->shadow_area.size,
 648                                        I40IW_SHADOWAREA_MASK);
 649         if (status)
 650                 goto exit;
 651         ccq->sc_cq.back_cq = (void *)ccq;
 652         /* populate the ccq init info */
 653         info.cq_base = ccq->mem_cq.va;
 654         info.cq_pa = ccq->mem_cq.pa;
 655         info.num_elem = IW_CCQ_SIZE;
 656         info.shadow_area = mem.va;
 657         info.shadow_area_pa = mem.pa;
 658         info.ceqe_mask = false;
 659         info.ceq_id_valid = true;
 660         info.shadow_read_threshold = 16;
 661         status = dev->ccq_ops->ccq_init(dev->ccq, &info);
 662         if (!status)
 663                 status = dev->ccq_ops->ccq_create(dev->ccq, 0, true, true);
 664 exit:
 665         if (status)
 666                 i40iw_free_dma_mem(dev->hw, &ccq->mem_cq);
 667         return status;
 668 }
 669
 670 /**
 671  * i40iw_configure_ceq_vector - set up the msix interrupt vector for ceq
 672  * @iwdev: iwarp device
 673  * @msix_vec: interrupt vector information
 674  * @iwceq: ceq associated with the vector
 675  * @ceq_id: the id number of the iwceq
 676  *
 677  * Allocate interrupt resources and enable irq handling
 678  * Return 0 if successful, otherwise return error
 679  */
 680 static enum i40iw_status_code i40iw_configure_ceq_vector(struct i40iw_device *iwdev,
 681                                                          struct i40iw_ceq *iwceq,
 682                                                          u32 ceq_id,
 683                                                          struct i40iw_msix_vector *msix_vec)
 684 {
 685         enum i40iw_status_code status;
 686         cpumask_t mask;
 687
 688         if (iwdev->msix_shared && !ceq_id) {
 689                 tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);
 690                 status = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "AEQCEQ", iwdev);
 691         } else {
 692                 tasklet_init(&iwceq->dpc_tasklet, i40iw_ceq_dpc, (unsigned long)iwceq);
 693                 status = request_irq(msix_vec->irq, i40iw_ceq_handler, 0, "CEQ", iwceq);
 694         }
 695
 696         cpumask_clear(&mask);
 697         cpumask_set_cpu(msix_vec->cpu_affinity, &mask);
 698         irq_set_affinity_hint(msix_vec->irq, &mask);
 699
 700         if (status) {
 701                 i40iw_pr_err("ceq irq config fail\n");
 702                 return I40IW_ERR_CONFIG;
 703         }
 704         msix_vec->ceq_id = ceq_id;
 705
 706         return 0;
 707 }
 708
 709 /**
 710  * i40iw_create_ceq - create completion event queue
 711  * @iwdev: iwarp device
 712  * @iwceq: pointer to the ceq resources to be created
 713  * @ceq_id: the id number of the iwceq
 714  *
 715  * Return 0, if the ceq and the resources associated with it
 716  * are successfully created, otherwise return error
 717  */
 718 static enum i40iw_status_code i40iw_create_ceq(struct i40iw_device *iwdev,
 719                                                struct i40iw_ceq *iwceq,
 720                                                u32 ceq_id)
 721 {
 722         enum i40iw_status_code status;
 723         struct i40iw_ceq_init_info info;
 724         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 725         u64 scratch;
 726
 727         memset(&info, 0, sizeof(info));
 728         info.ceq_id = ceq_id;
 729         iwceq->iwdev = iwdev;
 730         iwceq->mem.size = sizeof(struct i40iw_ceqe) *
 731                 iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
 732         status = i40iw_allocate_dma_mem(dev->hw, &iwceq->mem, iwceq->mem.size,
 733                                         I40IW_CEQ_ALIGNMENT);
 734         if (status)
 735                 goto exit;
 736         info.ceq_id = ceq_id;
 737         info.ceqe_base = iwceq->mem.va;
 738         info.ceqe_pa = iwceq->mem.pa;
 739
 740         info.elem_cnt = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
 741         iwceq->sc_ceq.ceq_id = ceq_id;
 742         info.dev = dev;
 743         scratch = (uintptr_t)&iwdev->cqp.sc_cqp;
 744         status = dev->ceq_ops->ceq_init(&iwceq->sc_ceq, &info);
 745         if (!status)
 746                 status = dev->ceq_ops->cceq_create(&iwceq->sc_ceq, scratch);
 747
 748 exit:
 749         if (status)
 750                 i40iw_free_dma_mem(dev->hw, &iwceq->mem);
 751         return status;
 752 }
 753
 754 void i40iw_request_reset(struct i40iw_device *iwdev)
 755 {
 756         struct i40e_info *ldev = iwdev->ldev;
 757
 758         ldev->ops->request_reset(ldev, iwdev->client, 1);
 759 }
 760
 761 /**
 762  * i40iw_setup_ceqs - manage the device ceq's and their interrupt resources
 763  * @iwdev: iwarp device
 764  * @ldev: i40e lan device
 765  *
 766  * Allocate a list for all device completion event queues
 767  * Create the ceq's and configure their msix interrupt vectors
 768  * Return 0, if at least one ceq is successfully set up, otherwise return error
 769  */
 770 static enum i40iw_status_code i40iw_setup_ceqs(struct i40iw_device *iwdev,
 771                                                struct i40e_info *ldev)
 772 {
 773         u32 i;
 774         u32 ceq_id;
 775         struct i40iw_ceq *iwceq;
 776         struct i40iw_msix_vector *msix_vec;
 777         enum i40iw_status_code status = 0;
 778         u32 num_ceqs;
 779
 780         if (ldev && ldev->ops && ldev->ops->setup_qvlist) {
 781                 status = ldev->ops->setup_qvlist(ldev, &i40iw_client,
 782                                                  iwdev->iw_qvlist);
 783                 if (status)
 784                         goto exit;
 785         } else {
 786                 status = I40IW_ERR_BAD_PTR;
 787                 goto exit;
 788         }
 789
 790         num_ceqs = min(iwdev->msix_count, iwdev->sc_dev.hmc_fpm_misc.max_ceqs);
 791         iwdev->ceqlist = kcalloc(num_ceqs, sizeof(*iwdev->ceqlist), GFP_KERNEL);
 792         if (!iwdev->ceqlist) {
 793                 status = I40IW_ERR_NO_MEMORY;
 794                 goto exit;
 795         }
 796         i = (iwdev->msix_shared) ? 0 : 1;
 797         for (ceq_id = 0; i < num_ceqs; i++, ceq_id++) {
 798                 iwceq = &iwdev->ceqlist[ceq_id];
 799                 status = i40iw_create_ceq(iwdev, iwceq, ceq_id);
 800                 if (status) {
 801                         i40iw_pr_err("create ceq status = %d\n", status);
 802                         break;
 803                 }
 804
 805                 msix_vec = &iwdev->iw_msixtbl[i];
 806                 iwceq->irq = msix_vec->irq;
 807                 iwceq->msix_idx = msix_vec->idx;
 808                 status = i40iw_configure_ceq_vector(iwdev, iwceq, ceq_id, msix_vec);
 809                 if (status) {
 810                         i40iw_destroy_ceq(iwdev, iwceq);
 811                         break;
 812                 }
 813                 i40iw_enable_intr(&iwdev->sc_dev, msix_vec->idx);
 814                 iwdev->ceqs_count++;
 815         }
 816
 817 exit:
 818         if (status) {
 819                 if (!iwdev->ceqs_count) {
 820                         kfree(iwdev->ceqlist);
 821                         iwdev->ceqlist = NULL;
 822                 } else {
 823                         status = 0;
 824                 }
 825         }
 826         return status;
 827 }
 828
 829 /**
 830  * i40iw_configure_aeq_vector - set up the msix vector for aeq
 831  * @iwdev: iwarp device
 832  *
 833  * Allocate interrupt resources and enable irq handling
 834  * Return 0 if successful, otherwise return error
 835  */
 836 static enum i40iw_status_code i40iw_configure_aeq_vector(struct i40iw_device *iwdev)
 837 {
 838         struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl;
 839         u32 ret = 0;
 840
 841         if (!iwdev->msix_shared) {
 842                 tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);
 843                 ret = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "i40iw", iwdev);
 844         }
 845         if (ret) {
 846                 i40iw_pr_err("aeq irq config fail\n");
 847                 return I40IW_ERR_CONFIG;
 848         }
 849
 850         return 0;
 851 }
 852
 853 /**
 854  * i40iw_create_aeq - create async event queue
 855  * @iwdev: iwarp device
 856  *
 857  * Return 0, if the aeq and the resources associated with it
 858  * are successfully created, otherwise return error
 859  */
 860 static enum i40iw_status_code i40iw_create_aeq(struct i40iw_device *iwdev)
 861 {
 862         enum i40iw_status_code status;
 863         struct i40iw_aeq_init_info info;
 864         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 865         struct i40iw_aeq *aeq = &iwdev->aeq;
 866         u64 scratch = 0;
 867         u32 aeq_size;
 868
 869         aeq_size = 2 * iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt +
 870                 iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
 871         memset(&info, 0, sizeof(info));
 872         aeq->mem.size = sizeof(struct i40iw_sc_aeqe) * aeq_size;
 873         status = i40iw_allocate_dma_mem(dev->hw, &aeq->mem, aeq->mem.size,
 874                                         I40IW_AEQ_ALIGNMENT);
 875         if (status)
 876                 goto exit;
 877
 878         info.aeqe_base = aeq->mem.va;
 879         info.aeq_elem_pa = aeq->mem.pa;
 880         info.elem_cnt = aeq_size;
 881         info.dev = dev;
 882         status = dev->aeq_ops->aeq_init(&aeq->sc_aeq, &info);
 883         if (status)
 884                 goto exit;
 885         status = dev->aeq_ops->aeq_create(&aeq->sc_aeq, scratch, 1);
 886         if (!status)
 887                 status = dev->aeq_ops->aeq_create_done(&aeq->sc_aeq);
 888 exit:
 889         if (status)
 890                 i40iw_free_dma_mem(dev->hw, &aeq->mem);
 891         return status;
 892 }
 893
 894 /**
 895  * i40iw_setup_aeq - set up the device aeq
 896  * @iwdev: iwarp device
 897  *
 898  * Create the aeq and configure its msix interrupt vector
 899  * Return 0 if successful, otherwise return error
 900  */
 901 static enum i40iw_status_code i40iw_setup_aeq(struct i40iw_device *iwdev)
 902 {
 903         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 904         enum i40iw_status_code status;
 905
 906         status = i40iw_create_aeq(iwdev);
 907         if (status)
 908                 return status;
 909
 910         status = i40iw_configure_aeq_vector(iwdev);
 911         if (status) {
 912                 i40iw_destroy_aeq(iwdev);
 913                 return status;
 914         }
 915
 916         if (!iwdev->msix_shared)
 917                 i40iw_enable_intr(dev, iwdev->iw_msixtbl[0].idx);
 918         return 0;
 919 }
 920
 921 /**
 922  * i40iw_initialize_ilq - create iwarp local queue for cm
 923  * @iwdev: iwarp device
 924  *
 925  * Return 0 if successful, otherwise return error
 926  */
 927 static enum i40iw_status_code i40iw_initialize_ilq(struct i40iw_device *iwdev)
 928 {
 929         struct i40iw_puda_rsrc_info info;
 930         enum i40iw_status_code status;
 931
 932         memset(&info, 0, sizeof(info));
 933         info.type = I40IW_PUDA_RSRC_TYPE_ILQ;
 934         info.cq_id = 1;
 935         info.qp_id = 0;
 936         info.count = 1;
 937         info.pd_id = 1;
 938         info.sq_size = 8192;
 939         info.rq_size = 8192;
 940         info.buf_size = 1024;
 941         info.tx_buf_cnt = 16384;
 942         info.receive = i40iw_receive_ilq;
 943         info.xmit_complete = i40iw_free_sqbuf;
 944         status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
 945         if (status)
 946                 i40iw_pr_err("ilq create fail\n");
 947         return status;
 948 }
 949
 950 /**
 951  * i40iw_initialize_ieq - create iwarp exception queue
 952  * @iwdev: iwarp device
 953  *
 954  * Return 0 if successful, otherwise return error
 955  */
 956 static enum i40iw_status_code i40iw_initialize_ieq(struct i40iw_device *iwdev)
 957 {
 958         struct i40iw_puda_rsrc_info info;
 959         enum i40iw_status_code status;
 960
 961         memset(&info, 0, sizeof(info));
 962         info.type = I40IW_PUDA_RSRC_TYPE_IEQ;
 963         info.cq_id = 2;
 964         info.qp_id = iwdev->sc_dev.exception_lan_queue;
 965         info.count = 1;
 966         info.pd_id = 2;
 967         info.sq_size = 8192;
 968         info.rq_size = 8192;
 969         info.buf_size = 2048;
 970         info.tx_buf_cnt = 16384;
 971         status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
 972         if (status)
 973                 i40iw_pr_err("ieq create fail\n");
 974         return status;
 975 }
 976
 977 /**
 978  * i40iw_hmc_setup - create hmc objects for the device
 979  * @iwdev: iwarp device
 980  *
 981  * Set up the device private memory space for the number and size of
 982  * the hmc objects and create the objects
 983  * Return 0 if successful, otherwise return error
 984  */
 985 static enum i40iw_status_code i40iw_hmc_setup(struct i40iw_device *iwdev)
 986 {
 987         enum i40iw_status_code status;
 988
 989         iwdev->sd_type = I40IW_SD_TYPE_DIRECT;
 990         status = i40iw_config_fpm_values(&iwdev->sc_dev, IW_CFG_FPM_QP_COUNT);
 991         if (status)
 992                 goto exit;
 993         status = i40iw_create_hmc_objs(iwdev, true);
 994         if (status)
 995                 goto exit;
 996         iwdev->init_state = HMC_OBJS_CREATED;
 997 exit:
 998         return status;
 999 }
1000
1001 /**
1002  * i40iw_del_init_mem - deallocate memory resources
1003  * @iwdev: iwarp device
1004  */
1005 static void i40iw_del_init_mem(struct i40iw_device *iwdev)
1006 {
1007         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1008
1009         i40iw_free_dma_mem(&iwdev->hw, &iwdev->obj_mem);
1010         kfree(dev->hmc_info->sd_table.sd_entry);
1011         dev->hmc_info->sd_table.sd_entry = NULL;
1012         kfree(iwdev->mem_resources);
1013         iwdev->mem_resources = NULL;
1014         kfree(iwdev->ceqlist);
1015         iwdev->ceqlist = NULL;
1016         kfree(iwdev->iw_msixtbl);
1017         iwdev->iw_msixtbl = NULL;
1018         kfree(iwdev->hmc_info_mem);
1019         iwdev->hmc_info_mem = NULL;
1020 }
1021
1022 /**
1023  * i40iw_del_macip_entry - remove a mac ip address entry from the hw table
1024  * @iwdev: iwarp device
1025  * @idx: the index of the mac ip address to delete
1026  */
1027 static void i40iw_del_macip_entry(struct i40iw_device *iwdev, u8 idx)
1028 {
1029         struct i40iw_cqp *iwcqp = &iwdev->cqp;
1030         struct i40iw_cqp_request *cqp_request;
1031         struct cqp_commands_info *cqp_info;
1032         enum i40iw_status_code status = 0;
1033
1034         cqp_request = i40iw_get_cqp_request(iwcqp, true);
1035         if (!cqp_request) {
1036                 i40iw_pr_err("cqp_request memory failed\n");
1037                 return;
1038         }
1039         cqp_info = &cqp_request->info;
1040         cqp_info->cqp_cmd = OP_DELETE_LOCAL_MAC_IPADDR_ENTRY;
1041         cqp_info->post_sq = 1;
1042         cqp_info->in.u.del_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1043         cqp_info->in.u.del_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1044         cqp_info->in.u.del_local_mac_ipaddr_entry.entry_idx = idx;
1045         cqp_info->in.u.del_local_mac_ipaddr_entry.ignore_ref_count = 0;
1046         status = i40iw_handle_cqp_op(iwdev, cqp_request);
1047         if (status)
1048                 i40iw_pr_err("CQP-OP Del MAC Ip entry fail");
1049 }
1050
1051 /**
1052  * i40iw_add_mac_ipaddr_entry - add a mac ip address entry to the hw table
1053  * @iwdev: iwarp device
1054  * @mac_addr: pointer to mac address
1055  * @idx: the index of the mac ip address to add
1056  */
1057 static enum i40iw_status_code i40iw_add_mac_ipaddr_entry(struct i40iw_device *iwdev,
1058                                                          u8 *mac_addr,
1059                                                          u8 idx)
1060 {
1061         struct i40iw_local_mac_ipaddr_entry_info *info;
1062         struct i40iw_cqp *iwcqp = &iwdev->cqp;
1063         struct i40iw_cqp_request *cqp_request;
1064         struct cqp_commands_info *cqp_info;
1065         enum i40iw_status_code status = 0;
1066
1067         cqp_request = i40iw_get_cqp_request(iwcqp, true);
1068         if (!cqp_request) {
1069                 i40iw_pr_err("cqp_request memory failed\n");
1070                 return I40IW_ERR_NO_MEMORY;
1071         }
1072
1073         cqp_info = &cqp_request->info;
1074
1075         cqp_info->post_sq = 1;
1076         info = &cqp_info->in.u.add_local_mac_ipaddr_entry.info;
1077         ether_addr_copy(info->mac_addr, mac_addr);
1078         info->entry_idx = idx;
1079         cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1080         cqp_info->cqp_cmd = OP_ADD_LOCAL_MAC_IPADDR_ENTRY;
1081         cqp_info->in.u.add_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1082         cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1083         status = i40iw_handle_cqp_op(iwdev, cqp_request);
1084         if (status)
1085                 i40iw_pr_err("CQP-OP Add MAC Ip entry fail");
1086         return status;
1087 }
1088
1089 /**
1090  * i40iw_alloc_local_mac_ipaddr_entry - allocate a mac ip address entry
1091  * @iwdev: iwarp device
1092  * @mac_ip_tbl_idx: the index of the new mac ip address
1093  *
1094  * Allocate a mac ip address entry and update the mac_ip_tbl_idx
1095  * to hold the index of the newly created mac ip address
1096  * Return 0 if successful, otherwise return error
1097  */
1098 static enum i40iw_status_code i40iw_alloc_local_mac_ipaddr_entry(struct i40iw_device *iwdev,
1099                                                                  u16 *mac_ip_tbl_idx)
1100 {
1101         struct i40iw_cqp *iwcqp = &iwdev->cqp;
1102         struct i40iw_cqp_request *cqp_request;
1103         struct cqp_commands_info *cqp_info;
1104         enum i40iw_status_code status = 0;
1105
1106         cqp_request = i40iw_get_cqp_request(iwcqp, true);
1107         if (!cqp_request) {
1108                 i40iw_pr_err("cqp_request memory failed\n");
1109                 return I40IW_ERR_NO_MEMORY;
1110         }
1111
1112         /* increment refcount, because we need the cqp request ret value */
1113         atomic_inc(&cqp_request->refcount);
1114
1115         cqp_info = &cqp_request->info;
1116         cqp_info->cqp_cmd = OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY;
1117         cqp_info->post_sq = 1;
1118         cqp_info->in.u.alloc_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1119         cqp_info->in.u.alloc_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1120         status = i40iw_handle_cqp_op(iwdev, cqp_request);
1121         if (!status)
1122                 *mac_ip_tbl_idx = cqp_request->compl_info.op_ret_val;
1123         else
1124                 i40iw_pr_err("CQP-OP Alloc MAC Ip entry fail");
1125         /* decrement refcount and free the cqp request, if no longer used */
1126         i40iw_put_cqp_request(iwcqp, cqp_request);
1127         return status;
1128 }
1129
1130 /**
1131  * i40iw_alloc_set_mac_ipaddr - set up a mac ip address table entry
1132  * @iwdev: iwarp device
1133  * @macaddr: pointer to mac address
1134  *
1135  * Allocate a mac ip address entry and add it to the hw table
1136  * Return 0 if successful, otherwise return error
1137  */
1138 static enum i40iw_status_code i40iw_alloc_set_mac_ipaddr(struct i40iw_device *iwdev,
1139                                                          u8 *macaddr)
1140 {
1141         enum i40iw_status_code status;
1142
1143         status = i40iw_alloc_local_mac_ipaddr_entry(iwdev, &iwdev->mac_ip_table_idx);
1144         if (!status) {
1145                 status = i40iw_add_mac_ipaddr_entry(iwdev, macaddr,
1146                                                     (u8)iwdev->mac_ip_table_idx);
1147                 if (status)
1148                         i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
1149         }
1150         return status;
1151 }
1152
1153 /**
1154  * i40iw_add_ipv6_addr - add ipv6 address to the hw arp table
1155  * @iwdev: iwarp device
1156  */
1157 static void i40iw_add_ipv6_addr(struct i40iw_device *iwdev)
1158 {
1159         struct net_device *ip_dev;
1160         struct inet6_dev *idev;
1161         struct inet6_ifaddr *ifp, *tmp;
1162         u32 local_ipaddr6[4];
1163
1164         rcu_read_lock();
1165         for_each_netdev_rcu(&init_net, ip_dev) {
1166                 if ((((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF) &&
1167                       (rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev)) ||
1168                      (ip_dev == iwdev->netdev)) && (ip_dev->flags & IFF_UP)) {
1169                         idev = __in6_dev_get(ip_dev);
1170                         if (!idev) {
1171                                 i40iw_pr_err("ipv6 inet device not found\n");
1172                                 break;
1173                         }
1174                         list_for_each_entry_safe(ifp, tmp, &idev->addr_list, if_list) {
1175                                 i40iw_pr_info("IP=%pI6, vlan_id=%d, MAC=%pM\n", &ifp->addr,
1176                                               rdma_vlan_dev_vlan_id(ip_dev), ip_dev->dev_addr);
1177                                 i40iw_copy_ip_ntohl(local_ipaddr6,
1178                                                     ifp->addr.in6_u.u6_addr32);
1179                                 i40iw_manage_arp_cache(iwdev,
1180                                                        ip_dev->dev_addr,
1181                                                        local_ipaddr6,
1182                                                        false,
1183                                                        I40IW_ARP_ADD);
1184                         }
1185                 }
1186         }
1187         rcu_read_unlock();
1188 }
1189
1190 /**
1191  * i40iw_add_ipv4_addr - add ipv4 address to the hw arp table
1192  * @iwdev: iwarp device
1193  */
1194 static void i40iw_add_ipv4_addr(struct i40iw_device *iwdev)
1195 {
1196         struct net_device *dev;
1197         struct in_device *idev;
1198         bool got_lock = true;
1199         u32 ip_addr;
1200
1201         if (!rtnl_trylock())
1202                 got_lock = false;
1203
1204         for_each_netdev(&init_net, dev) {
1205                 if ((((rdma_vlan_dev_vlan_id(dev) < 0xFFFF) &&
1206                       (rdma_vlan_dev_real_dev(dev) == iwdev->netdev)) ||
1207                     (dev == iwdev->netdev)) && (dev->flags & IFF_UP)) {
1208                         idev = in_dev_get(dev);
1209                         for_ifa(idev) {
1210                                 i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
1211                                             "IP=%pI4, vlan_id=%d, MAC=%pM\n", &ifa->ifa_address,
1212                                              rdma_vlan_dev_vlan_id(dev), dev->dev_addr);
1213
1214                                 ip_addr = ntohl(ifa->ifa_address);
1215                                 i40iw_manage_arp_cache(iwdev,
1216                                                        dev->dev_addr,
1217                                                        &ip_addr,
1218                                                        true,
1219                                                        I40IW_ARP_ADD);
1220                         }
1221                         endfor_ifa(idev);
1222                         in_dev_put(idev);
1223                 }
1224         }
1225         if (got_lock)
1226                 rtnl_unlock();
1227 }
1228
1229 /**
1230  * i40iw_add_mac_ip - add mac and ip addresses
1231  * @iwdev: iwarp device
1232  *
1233  * Create and add a mac ip address entry to the hw table and
1234  * ipv4/ipv6 addresses to the arp cache
1235  * Return 0 if successful, otherwise return error
1236  */
1237 static enum i40iw_status_code i40iw_add_mac_ip(struct i40iw_device *iwdev)
1238 {
1239         struct net_device *netdev = iwdev->netdev;
1240         enum i40iw_status_code status;
1241
1242         status = i40iw_alloc_set_mac_ipaddr(iwdev, (u8 *)netdev->dev_addr);
1243         if (status)
1244                 return status;
1245         i40iw_add_ipv4_addr(iwdev);
1246         i40iw_add_ipv6_addr(iwdev);
1247         return 0;
1248 }
1249
1250 /**
1251  * i40iw_wait_pe_ready - Check if firmware is ready
1252  * @hw: provides access to registers
1253  */
1254 static void i40iw_wait_pe_ready(struct i40iw_hw *hw)
1255 {
1256         u32 statusfw;
1257         u32 statuscpu0;
1258         u32 statuscpu1;
1259         u32 statuscpu2;
1260         u32 retrycount = 0;
1261
1262         do {
1263                 statusfw = i40iw_rd32(hw, I40E_GLPE_FWLDSTATUS);
1264                 i40iw_pr_info("[%04d] fm load status[x%04X]\n", __LINE__, statusfw);
1265                 statuscpu0 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS0);
1266                 i40iw_pr_info("[%04d] CSR_CQP status[x%04X]\n", __LINE__, statuscpu0);
1267                 statuscpu1 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS1);
1268                 i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS1 status[x%04X]\n",
1269                               __LINE__, statuscpu1);
1270                 statuscpu2 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS2);
1271                 i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS2 status[x%04X]\n",
1272                               __LINE__, statuscpu2);
1273                 if ((statuscpu0 == 0x80) && (statuscpu1 == 0x80) && (statuscpu2 == 0x80))
1274                         break;  /* SUCCESS */
1275                 mdelay(1000);
1276                 retrycount++;
1277         } while (retrycount < 14);
1278         i40iw_wr32(hw, 0xb4040, 0x4C104C5);
1279 }
1280
1281 /**
1282  * i40iw_initialize_dev - initialize device
1283  * @iwdev: iwarp device
1284  * @ldev: lan device information
1285  *
1286  * Allocate memory for the hmc objects and initialize iwdev
1287  * Return 0 if successful, otherwise clean up the resources
1288  * and return error
1289  */
1290 static enum i40iw_status_code i40iw_initialize_dev(struct i40iw_device *iwdev,
1291                                                    struct i40e_info *ldev)
1292 {
1293         enum i40iw_status_code status;
1294         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1295         struct i40iw_device_init_info info;
1296         struct i40iw_vsi_init_info vsi_info;
1297         struct i40iw_dma_mem mem;
1298         struct i40iw_l2params l2params;
1299         u32 size;
1300         struct i40iw_vsi_stats_info stats_info;
1301         u16 last_qset = I40IW_NO_QSET;
1302         u16 qset;
1303         u32 i;
1304
1305         memset(&l2params, 0, sizeof(l2params));
1306         memset(&info, 0, sizeof(info));
1307         size = sizeof(struct i40iw_hmc_pble_rsrc) + sizeof(struct i40iw_hmc_info) +
1308                                 (sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX);
1309         iwdev->hmc_info_mem = kzalloc(size, GFP_KERNEL);
1310         if (!iwdev->hmc_info_mem)
1311                 return I40IW_ERR_NO_MEMORY;
1312
1313         iwdev->pble_rsrc = (struct i40iw_hmc_pble_rsrc *)iwdev->hmc_info_mem;
1314         dev->hmc_info = &iwdev->hw.hmc;
1315         dev->hmc_info->hmc_obj = (struct i40iw_hmc_obj_info *)(iwdev->pble_rsrc + 1);
1316         status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_QUERY_FPM_BUF_SIZE,
1317                                        I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK);
1318         if (status)
1319                 goto error;
1320         info.fpm_query_buf_pa = mem.pa;
1321         info.fpm_query_buf = mem.va;
1322         status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_COMMIT_FPM_BUF_SIZE,
1323                                        I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK);
1324         if (status)
1325                 goto error;
1326         info.fpm_commit_buf_pa = mem.pa;
1327         info.fpm_commit_buf = mem.va;
1328         info.hmc_fn_id = ldev->fid;
1329         info.is_pf = (ldev->ftype) ? false : true;
1330         info.bar0 = ldev->hw_addr;
1331         info.hw = &iwdev->hw;
1332         info.debug_mask = debug;
1333         l2params.mss =
1334                 (ldev->params.mtu) ? ldev->params.mtu - I40IW_MTU_TO_MSS : I40IW_DEFAULT_MSS;
1335         for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++) {
1336                 qset = ldev->params.qos.prio_qos[i].qs_handle;
1337                 l2params.qs_handle_list[i] = qset;
1338                 if (last_qset == I40IW_NO_QSET)
1339                         last_qset = qset;
1340                 else if ((qset != last_qset) && (qset != I40IW_NO_QSET))
1341                         iwdev->dcb = true;
1342         }
1343         i40iw_pr_info("DCB is set/clear = %d\n", iwdev->dcb);
1344         info.exception_lan_queue = 1;
1345         info.vchnl_send = i40iw_virtchnl_send;
1346         status = i40iw_device_init(&iwdev->sc_dev, &info);
1347
1348         if (status)
1349                 goto error;
1350         memset(&vsi_info, 0, sizeof(vsi_info));
1351         vsi_info.dev = &iwdev->sc_dev;
1352         vsi_info.back_vsi = (void *)iwdev;
1353         vsi_info.params = &l2params;
1354         i40iw_sc_vsi_init(&iwdev->vsi, &vsi_info);
1355
1356         if (dev->is_pf) {
1357                 memset(&stats_info, 0, sizeof(stats_info));
1358                 stats_info.fcn_id = ldev->fid;
1359                 stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);
1360                 if (!stats_info.pestat) {
1361                         status = I40IW_ERR_NO_MEMORY;
1362                         goto error;
1363                 }
1364                 stats_info.stats_initialize = true;
1365                 if (stats_info.pestat)
1366                         i40iw_vsi_stats_init(&iwdev->vsi, &stats_info);
1367         }
1368         return status;
1369 error:
1370         kfree(iwdev->hmc_info_mem);
1371         iwdev->hmc_info_mem = NULL;
1372         return status;
1373 }
1374
1375 /**
1376  * i40iw_register_notifiers - register tcp ip notifiers
1377  */
1378 static void i40iw_register_notifiers(void)
1379 {
1380         if (atomic_inc_return(&i40iw_notifiers_registered) == 1) {
1381                 register_inetaddr_notifier(&i40iw_inetaddr_notifier);
1382                 register_inet6addr_notifier(&i40iw_inetaddr6_notifier);
1383                 register_netevent_notifier(&i40iw_net_notifier);
1384         }
1385 }
1386
1387 /**
1388  * i40iw_save_msix_info - copy msix vector information to iwarp device
1389  * @iwdev: iwarp device
1390  * @ldev: lan device information
1391  *
1392  * Allocate iwdev msix table and copy the ldev msix info to the table
1393  * Return 0 if successful, otherwise return error
1394  */
1395 static enum i40iw_status_code i40iw_save_msix_info(struct i40iw_device *iwdev,
1396                                                    struct i40e_info *ldev)
1397 {
1398         struct i40e_qvlist_info *iw_qvlist;
1399         struct i40e_qv_info *iw_qvinfo;
1400         u32 ceq_idx;
1401         u32 i;
1402         u32 size;
1403
1404         iwdev->msix_count = ldev->msix_count;
1405
1406         size = sizeof(struct i40iw_msix_vector) * iwdev->msix_count;
1407         size += sizeof(struct i40e_qvlist_info);
1408         size +=  sizeof(struct i40e_qv_info) * iwdev->msix_count - 1;
1409         iwdev->iw_msixtbl = kzalloc(size, GFP_KERNEL);
1410
1411         if (!iwdev->iw_msixtbl)
1412                 return I40IW_ERR_NO_MEMORY;
1413         iwdev->iw_qvlist = (struct i40e_qvlist_info *)(&iwdev->iw_msixtbl[iwdev->msix_count]);
1414         iw_qvlist = iwdev->iw_qvlist;
1415         iw_qvinfo = iw_qvlist->qv_info;
1416         iw_qvlist->num_vectors = iwdev->msix_count;
1417         if (iwdev->msix_count <= num_online_cpus())
1418                 iwdev->msix_shared = true;
1419         for (i = 0, ceq_idx = 0; i < iwdev->msix_count; i++, iw_qvinfo++) {
1420                 iwdev->iw_msixtbl[i].idx = ldev->msix_entries[i].entry;
1421                 iwdev->iw_msixtbl[i].irq = ldev->msix_entries[i].vector;
1422                 iwdev->iw_msixtbl[i].cpu_affinity = ceq_idx;
1423                 if (i == 0) {
1424                         iw_qvinfo->aeq_idx = 0;
1425                         if (iwdev->msix_shared)
1426                                 iw_qvinfo->ceq_idx = ceq_idx++;
1427                         else
1428                                 iw_qvinfo->ceq_idx = I40E_QUEUE_INVALID_IDX;
1429                 } else {
1430                         iw_qvinfo->aeq_idx = I40E_QUEUE_INVALID_IDX;
1431                         iw_qvinfo->ceq_idx = ceq_idx++;
1432                 }
1433                 iw_qvinfo->itr_idx = 3;
1434                 iw_qvinfo->v_idx = iwdev->iw_msixtbl[i].idx;
1435         }
1436         return 0;
1437 }
1438
1439 /**
1440  * i40iw_deinit_device - clean up the device resources
1441  * @iwdev: iwarp device
1442  *
1443  * Destroy the ib device interface, remove the mac ip entry and ipv4/ipv6 addresses,
1444  * destroy the device queues and free the pble and the hmc objects
1445  */
1446 static void i40iw_deinit_device(struct i40iw_device *iwdev)
1447 {
1448         struct i40e_info *ldev = iwdev->ldev;
1449
1450         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1451
1452         i40iw_pr_info("state = %d\n", iwdev->init_state);
1453         if (iwdev->param_wq)
1454                 destroy_workqueue(iwdev->param_wq);
1455
1456         switch (iwdev->init_state) {
1457         case RDMA_DEV_REGISTERED:
1458                 iwdev->iw_status = 0;
1459                 i40iw_port_ibevent(iwdev);
1460                 i40iw_destroy_rdma_device(iwdev->iwibdev);
1461                 /* fallthrough */
1462         case IP_ADDR_REGISTERED:
1463                 if (!iwdev->reset)
1464                         i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
1465                 /* fallthrough */
1466         case INET_NOTIFIER:
1467                 if (!atomic_dec_return(&i40iw_notifiers_registered)) {
1468                         unregister_netevent_notifier(&i40iw_net_notifier);
1469                         unregister_inetaddr_notifier(&i40iw_inetaddr_notifier);
1470                         unregister_inet6addr_notifier(&i40iw_inetaddr6_notifier);
1471                 }
1472                 /* fallthrough */
1473         case PBLE_CHUNK_MEM:
1474                 i40iw_destroy_pble_pool(dev, iwdev->pble_rsrc);
1475                 /* fallthrough */
1476         case CEQ_CREATED:
1477                 i40iw_dele_ceqs(iwdev);
1478                 /* fallthrough */
1479         case AEQ_CREATED:
1480                 i40iw_destroy_aeq(iwdev);
1481                 /* fallthrough */
1482         case IEQ_CREATED:
1483                 i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_IEQ, iwdev->reset);
1484                 /* fallthrough */
1485         case ILQ_CREATED:
1486                 i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_ILQ, iwdev->reset);
1487                 /* fallthrough */
1488         case CCQ_CREATED:
1489                 i40iw_destroy_ccq(iwdev);
1490                 /* fallthrough */
1491         case HMC_OBJS_CREATED:
1492                 i40iw_del_hmc_objects(dev, dev->hmc_info, true, iwdev->reset);
1493                 /* fallthrough */
1494         case CQP_CREATED:
1495                 i40iw_destroy_cqp(iwdev, true);
1496                 /* fallthrough */
1497         case INITIAL_STATE:
1498                 i40iw_cleanup_cm_core(&iwdev->cm_core);
1499                 if (iwdev->vsi.pestat) {
1500                         i40iw_vsi_stats_free(&iwdev->vsi);
1501                         kfree(iwdev->vsi.pestat);
1502                 }
1503                 i40iw_del_init_mem(iwdev);
1504                 break;
1505         case INVALID_STATE:
1506                 /* fallthrough */
1507         default:
1508                 i40iw_pr_err("bad init_state = %d\n", iwdev->init_state);
1509                 break;
1510         }
1511
1512         i40iw_del_handler(i40iw_find_i40e_handler(ldev));
1513         kfree(iwdev->hdl);
1514 }
1515
1516 /**
1517  * i40iw_setup_init_state - set up the initial device struct
1518  * @hdl: handler for iwarp device - one per instance
1519  * @ldev: lan device information
1520  * @client: iwarp client information, provided during registration
1521  *
1522  * Initialize the iwarp device and its hdl information
1523  * using the ldev and client information
1524  * Return 0 if successful, otherwise return error
1525  */
1526 static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl,
1527                                                      struct i40e_info *ldev,
1528                                                      struct i40e_client *client)
1529 {
1530         struct i40iw_device *iwdev = &hdl->device;
1531         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1532         enum i40iw_status_code status;
1533
1534         memcpy(&hdl->ldev, ldev, sizeof(*ldev));
1535         if (resource_profile == 1)
1536                 resource_profile = 2;
1537
1538         iwdev->mpa_version = mpa_version;
1539         iwdev->resource_profile = (resource_profile < I40IW_HMC_PROFILE_EQUAL) ?
1540             (u8)resource_profile + I40IW_HMC_PROFILE_DEFAULT :
1541             I40IW_HMC_PROFILE_DEFAULT;
1542         iwdev->max_rdma_vfs =
1543                 (iwdev->resource_profile != I40IW_HMC_PROFILE_DEFAULT) ?  max_rdma_vfs : 0;
1544         iwdev->max_enabled_vfs = iwdev->max_rdma_vfs;
1545         iwdev->netdev = ldev->netdev;
1546         hdl->client = client;
1547         if (!ldev->ftype)
1548                 iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_DB_ADDR_OFFSET;
1549         else
1550                 iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_VF_DB_ADDR_OFFSET;
1551
1552         status = i40iw_save_msix_info(iwdev, ldev);
1553         if (status)
1554                 goto exit;
1555         iwdev->hw.dev_context = (void *)ldev->pcidev;
1556         iwdev->hw.hw_addr = ldev->hw_addr;
1557         status = i40iw_allocate_dma_mem(&iwdev->hw,
1558                                         &iwdev->obj_mem, 8192, 4096);
1559         if (status)
1560                 goto exit;
1561         iwdev->obj_next = iwdev->obj_mem;
1562         iwdev->push_mode = push_mode;
1563
1564         init_waitqueue_head(&iwdev->vchnl_waitq);
1565         init_waitqueue_head(&dev->vf_reqs);
1566         init_waitqueue_head(&iwdev->close_wq);
1567
1568         status = i40iw_initialize_dev(iwdev, ldev);
1569 exit:
1570         if (status) {
1571                 kfree(iwdev->iw_msixtbl);
1572                 i40iw_free_dma_mem(dev->hw, &iwdev->obj_mem);
1573                 iwdev->iw_msixtbl = NULL;
1574         }
1575         return status;
1576 }
1577
1578 /**
1579  * i40iw_get_used_rsrc - determine resources used internally
1580  * @iwdev: iwarp device
1581  *
1582  * Called after internal allocations
1583  */
1584 static void i40iw_get_used_rsrc(struct i40iw_device *iwdev)
1585 {
1586         iwdev->used_pds = find_next_zero_bit(iwdev->allocated_pds, iwdev->max_pd, 0);
1587         iwdev->used_qps = find_next_zero_bit(iwdev->allocated_qps, iwdev->max_qp, 0);
1588         iwdev->used_cqs = find_next_zero_bit(iwdev->allocated_cqs, iwdev->max_cq, 0);
1589         iwdev->used_mrs = find_next_zero_bit(iwdev->allocated_mrs, iwdev->max_mr, 0);
1590 }
1591
1592 /**
1593  * i40iw_open - client interface operation open for iwarp/uda device
1594  * @ldev: lan device information
1595  * @client: iwarp client information, provided during registration
1596  *
1597  * Called by the lan driver during the processing of client register
1598  * Create device resources, set up queues, pble and hmc objects and
1599  * register the device with the ib verbs interface
1600  * Return 0 if successful, otherwise return error
1601  */
1602 static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
1603 {
1604         struct i40iw_device *iwdev;
1605         struct i40iw_sc_dev *dev;
1606         enum i40iw_status_code status;
1607         struct i40iw_handler *hdl;
1608
1609         hdl = i40iw_find_netdev(ldev->netdev);
1610         if (hdl)
1611                 return 0;
1612
1613         hdl = kzalloc(sizeof(*hdl), GFP_KERNEL);
1614         if (!hdl)
1615                 return -ENOMEM;
1616         iwdev = &hdl->device;
1617         iwdev->hdl = hdl;
1618         dev = &iwdev->sc_dev;
1619         i40iw_setup_cm_core(iwdev);
1620
1621         dev->back_dev = (void *)iwdev;
1622         iwdev->ldev = &hdl->ldev;
1623         iwdev->client = client;
1624         mutex_init(&iwdev->pbl_mutex);
1625         i40iw_add_handler(hdl);
1626
1627         do {
1628                 status = i40iw_setup_init_state(hdl, ldev, client);
1629                 if (status)
1630                         break;
1631                 iwdev->init_state = INITIAL_STATE;
1632                 if (dev->is_pf)
1633                         i40iw_wait_pe_ready(dev->hw);
1634                 status = i40iw_create_cqp(iwdev);
1635                 if (status)
1636                         break;
1637                 iwdev->init_state = CQP_CREATED;
1638                 status = i40iw_hmc_setup(iwdev);
1639                 if (status)
1640                         break;
1641                 status = i40iw_create_ccq(iwdev);
1642                 if (status)
1643                         break;
1644                 iwdev->init_state = CCQ_CREATED;
1645                 status = i40iw_initialize_ilq(iwdev);
1646                 if (status)
1647                         break;
1648                 iwdev->init_state = ILQ_CREATED;
1649                 status = i40iw_initialize_ieq(iwdev);
1650                 if (status)
1651                         break;
1652                 iwdev->init_state = IEQ_CREATED;
1653                 status = i40iw_setup_aeq(iwdev);
1654                 if (status)
1655                         break;
1656                 iwdev->init_state = AEQ_CREATED;
1657                 status = i40iw_setup_ceqs(iwdev, ldev);
1658                 if (status)
1659                         break;
1660                 iwdev->init_state = CEQ_CREATED;
1661                 status = i40iw_initialize_hw_resources(iwdev);
1662                 if (status)
1663                         break;
1664                 i40iw_get_used_rsrc(iwdev);
1665                 dev->ccq_ops->ccq_arm(dev->ccq);
1666                 status = i40iw_hmc_init_pble(&iwdev->sc_dev, iwdev->pble_rsrc);
1667                 if (status)
1668                         break;
1669                 iwdev->init_state = PBLE_CHUNK_MEM;
1670                 iwdev->virtchnl_wq = alloc_ordered_workqueue("iwvch", WQ_MEM_RECLAIM);
1671                 i40iw_register_notifiers();
1672                 iwdev->init_state = INET_NOTIFIER;
1673                 status = i40iw_add_mac_ip(iwdev);
1674                 if (status)
1675                         break;
1676                 iwdev->init_state = IP_ADDR_REGISTERED;
1677                 if (i40iw_register_rdma_device(iwdev)) {
1678                         i40iw_pr_err("register rdma device fail\n");
1679                         break;
1680                 };
1681
1682                 iwdev->init_state = RDMA_DEV_REGISTERED;
1683                 iwdev->iw_status = 1;
1684                 i40iw_port_ibevent(iwdev);
1685                 iwdev->param_wq = alloc_ordered_workqueue("l2params", WQ_MEM_RECLAIM);
1686                 if(iwdev->param_wq == NULL)
1687                         break;
1688                 i40iw_pr_info("i40iw_open completed\n");
1689                 return 0;
1690         } while (0);
1691
1692         i40iw_pr_err("status = %d last completion = %d\n", status, iwdev->init_state);
1693         i40iw_deinit_device(iwdev);
1694         return -ERESTART;
1695 }
1696
1697 /**
1698  * i40iw_l2params_worker - worker for l2 params change
1699  * @work: work pointer for l2 params
1700  */
1701 static void i40iw_l2params_worker(struct work_struct *work)
1702 {
1703         struct l2params_work *dwork =
1704             container_of(work, struct l2params_work, work);
1705         struct i40iw_device *iwdev = dwork->iwdev;
1706
1707         i40iw_change_l2params(&iwdev->vsi, &dwork->l2params);
1708         atomic_dec(&iwdev->params_busy);
1709         kfree(work);
1710 }
1711
1712 /**
1713  * i40iw_l2param_change - handle qs handles for qos and mss change
1714  * @ldev: lan device information
1715  * @client: client for paramater change
1716  * @params: new parameters from L2
1717  */
1718 static void i40iw_l2param_change(struct i40e_info *ldev, struct i40e_client *client,
1719                                  struct i40e_params *params)
1720 {
1721         struct i40iw_handler *hdl;
1722         struct i40iw_l2params *l2params;
1723         struct l2params_work *work;
1724         struct i40iw_device *iwdev;
1725         int i;
1726
1727         hdl = i40iw_find_i40e_handler(ldev);
1728         if (!hdl)
1729                 return;
1730
1731         iwdev = &hdl->device;
1732
1733         if (atomic_read(&iwdev->params_busy))
1734                 return;
1735
1736
1737         work = kzalloc(sizeof(*work), GFP_ATOMIC);
1738         if (!work)
1739                 return;
1740
1741         atomic_inc(&iwdev->params_busy);
1742
1743         work->iwdev = iwdev;
1744         l2params = &work->l2params;
1745         for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++)
1746                 l2params->qs_handle_list[i] = params->qos.prio_qos[i].qs_handle;
1747
1748         l2params->mss = (params->mtu) ? params->mtu - I40IW_MTU_TO_MSS : iwdev->vsi.mss;
1749
1750         INIT_WORK(&work->work, i40iw_l2params_worker);
1751         queue_work(iwdev->param_wq, &work->work);
1752 }
1753
1754 /**
1755  * i40iw_close - client interface operation close for iwarp/uda device
1756  * @ldev: lan device information
1757  * @client: client to close
1758  *
1759  * Called by the lan driver during the processing of client unregister
1760  * Destroy and clean up the driver resources
1761  */
1762 static void i40iw_close(struct i40e_info *ldev, struct i40e_client *client, bool reset)
1763 {
1764         struct i40iw_device *iwdev;
1765         struct i40iw_handler *hdl;
1766
1767         hdl = i40iw_find_i40e_handler(ldev);
1768         if (!hdl)
1769                 return;
1770
1771         iwdev = &hdl->device;
1772         iwdev->closing = true;
1773
1774         if (reset)
1775                 iwdev->reset = true;
1776
1777         i40iw_cm_disconnect_all(iwdev);
1778         destroy_workqueue(iwdev->virtchnl_wq);
1779         i40iw_deinit_device(iwdev);
1780 }
1781
1782 /**
1783  * i40iw_vf_reset - process VF reset
1784  * @ldev: lan device information
1785  * @client: client interface instance
1786  * @vf_id: virtual function id
1787  *
1788  * Called when a VF is reset by the PF
1789  * Destroy and clean up the VF resources
1790  */
1791 static void i40iw_vf_reset(struct i40e_info *ldev, struct i40e_client *client, u32 vf_id)
1792 {
1793         struct i40iw_handler *hdl;
1794         struct i40iw_sc_dev *dev;
1795         struct i40iw_hmc_fcn_info hmc_fcn_info;
1796         struct i40iw_virt_mem vf_dev_mem;
1797         struct i40iw_vfdev *tmp_vfdev;
1798         unsigned int i;
1799         unsigned long flags;
1800         struct i40iw_device *iwdev;
1801
1802         hdl = i40iw_find_i40e_handler(ldev);
1803         if (!hdl)
1804                 return;
1805
1806         dev = &hdl->device.sc_dev;
1807         iwdev = (struct i40iw_device *)dev->back_dev;
1808
1809         for (i = 0; i < I40IW_MAX_PE_ENABLED_VF_COUNT; i++) {
1810                 if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id != vf_id))
1811                         continue;
1812                 /* free all resources allocated on behalf of vf */
1813                 tmp_vfdev = dev->vf_dev[i];
1814                 spin_lock_irqsave(&iwdev->vsi.pestat->lock, flags);
1815                 dev->vf_dev[i] = NULL;
1816                 spin_unlock_irqrestore(&iwdev->vsi.pestat->lock, flags);
1817                 i40iw_del_hmc_objects(dev, &tmp_vfdev->hmc_info, false, false);
1818                 /* remove vf hmc function */
1819                 memset(&hmc_fcn_info, 0, sizeof(hmc_fcn_info));
1820                 hmc_fcn_info.vf_id = vf_id;
1821                 hmc_fcn_info.iw_vf_idx = tmp_vfdev->iw_vf_idx;
1822                 hmc_fcn_info.free_fcn = true;
1823                 i40iw_cqp_manage_hmc_fcn_cmd(dev, &hmc_fcn_info);
1824                 /* free vf_dev */
1825                 vf_dev_mem.va = tmp_vfdev;
1826                 vf_dev_mem.size = sizeof(struct i40iw_vfdev) +
1827                                         sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX;
1828                 i40iw_free_virt_mem(dev->hw, &vf_dev_mem);
1829                 break;
1830         }
1831 }
1832
1833 /**
1834  * i40iw_vf_enable - enable a number of VFs
1835  * @ldev: lan device information
1836  * @client: client interface instance
1837  * @num_vfs: number of VFs for the PF
1838  *
1839  * Called when the number of VFs changes
1840  */
1841 static void i40iw_vf_enable(struct i40e_info *ldev,
1842                             struct i40e_client *client,
1843                             u32 num_vfs)
1844 {
1845         struct i40iw_handler *hdl;
1846
1847         hdl = i40iw_find_i40e_handler(ldev);
1848         if (!hdl)
1849                 return;
1850
1851         if (num_vfs > I40IW_MAX_PE_ENABLED_VF_COUNT)
1852                 hdl->device.max_enabled_vfs = I40IW_MAX_PE_ENABLED_VF_COUNT;
1853         else
1854                 hdl->device.max_enabled_vfs = num_vfs;
1855 }
1856
1857 /**
1858  * i40iw_vf_capable - check if VF capable
1859  * @ldev: lan device information
1860  * @client: client interface instance
1861  * @vf_id: virtual function id
1862  *
1863  * Return 1 if a VF slot is available or if VF is already RDMA enabled
1864  * Return 0 otherwise
1865  */
1866 static int i40iw_vf_capable(struct i40e_info *ldev,
1867                             struct i40e_client *client,
1868                             u32 vf_id)
1869 {
1870         struct i40iw_handler *hdl;
1871         struct i40iw_sc_dev *dev;
1872         unsigned int i;
1873
1874         hdl = i40iw_find_i40e_handler(ldev);
1875         if (!hdl)
1876                 return 0;
1877
1878         dev = &hdl->device.sc_dev;
1879
1880         for (i = 0; i < hdl->device.max_enabled_vfs; i++) {
1881                 if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id == vf_id))
1882                         return 1;
1883         }
1884
1885         return 0;
1886 }
1887
1888 /**
1889  * i40iw_virtchnl_receive - receive a message through the virtual channel
1890  * @ldev: lan device information
1891  * @client: client interface instance
1892  * @vf_id: virtual function id associated with the message
1893  * @msg: message buffer pointer
1894  * @len: length of the message
1895  *
1896  * Invoke virtual channel receive operation for the given msg
1897  * Return 0 if successful, otherwise return error
1898  */
1899 static int i40iw_virtchnl_receive(struct i40e_info *ldev,
1900                                   struct i40e_client *client,
1901                                   u32 vf_id,
1902                                   u8 *msg,
1903                                   u16 len)
1904 {
1905         struct i40iw_handler *hdl;
1906         struct i40iw_sc_dev *dev;
1907         struct i40iw_device *iwdev;
1908         int ret_code = I40IW_NOT_SUPPORTED;
1909
1910         if (!len || !msg)
1911                 return I40IW_ERR_PARAM;
1912
1913         hdl = i40iw_find_i40e_handler(ldev);
1914         if (!hdl)
1915                 return I40IW_ERR_PARAM;
1916
1917         dev = &hdl->device.sc_dev;
1918         iwdev = dev->back_dev;
1919
1920         if (dev->vchnl_if.vchnl_recv) {
1921                 ret_code = dev->vchnl_if.vchnl_recv(dev, vf_id, msg, len);
1922                 if (!dev->is_pf) {
1923                         atomic_dec(&iwdev->vchnl_msgs);
1924                         wake_up(&iwdev->vchnl_waitq);
1925                 }
1926         }
1927         return ret_code;
1928 }
1929
1930 /**
1931  * i40iw_vf_clear_to_send - wait to send virtual channel message
1932  * @dev: iwarp device *
1933  * Wait for until virtual channel is clear
1934  * before sending the next message
1935  *
1936  * Returns false if error
1937  * Returns true if clear to send
1938  */
1939 bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev)
1940 {
1941         struct i40iw_device *iwdev;
1942         wait_queue_entry_t wait;
1943
1944         iwdev = dev->back_dev;
1945
1946         if (!wq_has_sleeper(&dev->vf_reqs) &&
1947             (atomic_read(&iwdev->vchnl_msgs) == 0))
1948                 return true; /* virtual channel is clear */
1949
1950         init_wait(&wait);
1951         add_wait_queue_exclusive(&dev->vf_reqs, &wait);
1952
1953         if (!wait_event_timeout(dev->vf_reqs,
1954                                 (atomic_read(&iwdev->vchnl_msgs) == 0),
1955                                 I40IW_VCHNL_EVENT_TIMEOUT))
1956                 dev->vchnl_up = false;
1957
1958         remove_wait_queue(&dev->vf_reqs, &wait);
1959
1960         return dev->vchnl_up;
1961 }
1962
1963 /**
1964  * i40iw_virtchnl_send - send a message through the virtual channel
1965  * @dev: iwarp device
1966  * @vf_id: virtual function id associated with the message
1967  * @msg: virtual channel message buffer pointer
1968  * @len: length of the message
1969  *
1970  * Invoke virtual channel send operation for the given msg
1971  * Return 0 if successful, otherwise return error
1972  */
1973 static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,
1974                                                   u32 vf_id,
1975                                                   u8 *msg,
1976                                                   u16 len)
1977 {
1978         struct i40iw_device *iwdev;
1979         struct i40e_info *ldev;
1980
1981         if (!dev || !dev->back_dev)
1982                 return I40IW_ERR_BAD_PTR;
1983
1984         iwdev = dev->back_dev;
1985         ldev = iwdev->ldev;
1986
1987         if (ldev && ldev->ops && ldev->ops->virtchnl_send)
1988                 return ldev->ops->virtchnl_send(ldev, &i40iw_client, vf_id, msg, len);
1989         return I40IW_ERR_BAD_PTR;
1990 }
1991
1992 /* client interface functions */
1993 static const struct i40e_client_ops i40e_ops = {
1994         .open = i40iw_open,
1995         .close = i40iw_close,
1996         .l2_param_change = i40iw_l2param_change,
1997         .virtchnl_receive = i40iw_virtchnl_receive,
1998         .vf_reset = i40iw_vf_reset,
1999         .vf_enable = i40iw_vf_enable,
2000         .vf_capable = i40iw_vf_capable
2001 };
2002
2003 /**
2004  * i40iw_init_module - driver initialization function
2005  *
2006  * First function to call when the driver is loaded
2007  * Register the driver as i40e client and port mapper client
2008  */
2009 static int __init i40iw_init_module(void)
2010 {
2011         int ret;
2012
2013         memset(&i40iw_client, 0, sizeof(i40iw_client));
2014         i40iw_client.version.major = CLIENT_IW_INTERFACE_VERSION_MAJOR;
2015         i40iw_client.version.minor = CLIENT_IW_INTERFACE_VERSION_MINOR;
2016         i40iw_client.version.build = CLIENT_IW_INTERFACE_VERSION_BUILD;
2017         i40iw_client.ops = &i40e_ops;
2018         memcpy(i40iw_client.name, i40iw_client_name, I40E_CLIENT_STR_LENGTH);
2019         i40iw_client.type = I40E_CLIENT_IWARP;
2020         spin_lock_init(&i40iw_handler_lock);
2021         ret = i40e_register_client(&i40iw_client);
2022         return ret;
2023 }
2024
2025 /**
2026  * i40iw_exit_module - driver exit clean up function
2027  *
2028  * The function is called just before the driver is unloaded
2029  * Unregister the driver as i40e client and port mapper client
2030  */
2031 static void __exit i40iw_exit_module(void)
2032 {
2033         i40e_unregister_client(&i40iw_client);
2034 }
2035
2036 module_init(i40iw_init_module);
2037 module_exit(i40iw_exit_module);