drivers/infiniband/hw/i40iw/i40iw_ctrl.c

   1 /*******************************************************************************
   2 *
   3 * Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenFabrics.org BSD license below:
  10 *
  11 *   Redistribution and use in source and binary forms, with or
  12 *   without modification, are permitted provided that the following
  13 *   conditions are met:
  14 *
  15 *    - Redistributions of source code must retain the above
  16 *       copyright notice, this list of conditions and the following
  17 *       disclaimer.
  18 *
  19 *    - Redistributions in binary form must reproduce the above
  20 *       copyright notice, this list of conditions and the following
  21 *       disclaimer in the documentation and/or other materials
  22 *       provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 *
  33 *******************************************************************************/
  34
  35 #include "i40iw_osdep.h"
  36 #include "i40iw_register.h"
  37 #include "i40iw_status.h"
  38 #include "i40iw_hmc.h"
  39
  40 #include "i40iw_d.h"
  41 #include "i40iw_type.h"
  42 #include "i40iw_p.h"
  43 #include "i40iw_vf.h"
  44 #include "i40iw_virtchnl.h"
  45
  46 /**
  47  * i40iw_insert_wqe_hdr - write wqe header
  48  * @wqe: cqp wqe for header
  49  * @header: header for the cqp wqe
  50  */
  51 static inline void i40iw_insert_wqe_hdr(u64 *wqe, u64 header)
  52 {
  53         wmb();            /* make sure WQE is populated before polarity is set */
  54         set_64bit_val(wqe, 24, header);
  55 }
  56
  57 /**
  58  * i40iw_get_cqp_reg_info - get head and tail for cqp using registers
  59  * @cqp: struct for cqp hw
  60  * @val: cqp tail register value
  61  * @tail:wqtail register value
  62  * @error: cqp processing err
  63  */
  64 static inline void i40iw_get_cqp_reg_info(struct i40iw_sc_cqp *cqp,
  65                                           u32 *val,
  66                                           u32 *tail,
  67                                           u32 *error)
  68 {
  69         if (cqp->dev->is_pf) {
  70                 *val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPTAIL);
  71                 *tail = RS_32(*val, I40E_PFPE_CQPTAIL_WQTAIL);
  72                 *error = RS_32(*val, I40E_PFPE_CQPTAIL_CQP_OP_ERR);
  73         } else {
  74                 *val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPTAIL1);
  75                 *tail = RS_32(*val, I40E_VFPE_CQPTAIL_WQTAIL);
  76                 *error = RS_32(*val, I40E_VFPE_CQPTAIL_CQP_OP_ERR);
  77         }
  78 }
  79
  80 /**
  81  * i40iw_cqp_poll_registers - poll cqp registers
  82  * @cqp: struct for cqp hw
  83  * @tail:wqtail register value
  84  * @count: how many times to try for completion
  85  */
  86 static enum i40iw_status_code i40iw_cqp_poll_registers(
  87                                                 struct i40iw_sc_cqp *cqp,
  88                                                 u32 tail,
  89                                                 u32 count)
  90 {
  91         u32 i = 0;
  92         u32 newtail, error, val;
  93
  94         while (i < count) {
  95                 i++;
  96                 i40iw_get_cqp_reg_info(cqp, &val, &newtail, &error);
  97                 if (error) {
  98                         error = (cqp->dev->is_pf) ?
  99                                  i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES) :
 100                                  i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
 101                         return I40IW_ERR_CQP_COMPL_ERROR;
 102                 }
 103                 if (newtail != tail) {
 104                         /* SUCCESS */
 105                         I40IW_RING_MOVE_TAIL(cqp->sq_ring);
 106                         cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
 107                         return 0;
 108                 }
 109                 udelay(I40IW_SLEEP_COUNT);
 110         }
 111         return I40IW_ERR_TIMEOUT;
 112 }
 113
 114 /**
 115  * i40iw_sc_parse_fpm_commit_buf - parse fpm commit buffer
 116  * @buf: ptr to fpm commit buffer
 117  * @info: ptr to i40iw_hmc_obj_info struct
 118  * @sd: number of SDs for HMC objects
 119  *
 120  * parses fpm commit info and copy base value
 121  * of hmc objects in hmc_info
 122  */
 123 static enum i40iw_status_code i40iw_sc_parse_fpm_commit_buf(
 124                                 u64 *buf,
 125                                 struct i40iw_hmc_obj_info *info,
 126                                 u32 *sd)
 127 {
 128         u64 temp;
 129         u64 size;
 130         u64 base = 0;
 131         u32 i, j;
 132         u32 k = 0;
 133
 134         /* copy base values in obj_info */
 135         for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE; i++, j += 8) {
 136                 if ((i == I40IW_HMC_IW_SRQ) ||
 137                         (i == I40IW_HMC_IW_FSIMC) ||
 138                         (i == I40IW_HMC_IW_FSIAV)) {
 139                         info[i].base = 0;
 140                         info[i].cnt = 0;
 141                         continue;
 142                 }
 143                 get_64bit_val(buf, j, &temp);
 144                 info[i].base = RS_64_1(temp, 32) * 512;
 145                 if (info[i].base > base) {
 146                         base = info[i].base;
 147                         k = i;
 148                 }
 149                 if (i == I40IW_HMC_IW_APBVT_ENTRY) {
 150                         info[i].cnt = 1;
 151                         continue;
 152                 }
 153                 if (i == I40IW_HMC_IW_QP)
 154                         info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
 155                 else if (i == I40IW_HMC_IW_CQ)
 156                         info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
 157                 else
 158                         info[i].cnt = (u32)(temp);
 159         }
 160         size = info[k].cnt * info[k].size + info[k].base;
 161         if (size & 0x1FFFFF)
 162                 *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */
 163         else
 164                 *sd = (u32)(size >> 21);
 165
 166         return 0;
 167 }
 168
 169 /**
 170  * i40iw_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
 171  * @buf: ptr to fpm query buffer
 172  * @buf_idx: index into buf
 173  * @info: ptr to i40iw_hmc_obj_info struct
 174  * @rsrc_idx: resource index into info
 175  *
 176  * Decode a 64 bit value from fpm query buffer into max count and size
 177  */
 178 static u64 i40iw_sc_decode_fpm_query(u64 *buf,
 179                                             u32 buf_idx,
 180                                             struct i40iw_hmc_obj_info *obj_info,
 181                                             u32 rsrc_idx)
 182 {
 183         u64 temp;
 184         u32 size;
 185
 186         get_64bit_val(buf, buf_idx, &temp);
 187         obj_info[rsrc_idx].max_cnt = (u32)temp;
 188         size = (u32)RS_64_1(temp, 32);
 189         obj_info[rsrc_idx].size = LS_64_1(1, size);
 190
 191         return temp;
 192 }
 193
 194 /**
 195  * i40iw_sc_parse_fpm_query_buf() - parses fpm query buffer
 196  * @buf: ptr to fpm query buffer
 197  * @info: ptr to i40iw_hmc_obj_info struct
 198  * @hmc_fpm_misc: ptr to fpm data
 199  *
 200  * parses fpm query buffer and copy max_cnt and
 201  * size value of hmc objects in hmc_info
 202  */
 203 static enum i40iw_status_code i40iw_sc_parse_fpm_query_buf(
 204                                 u64 *buf,
 205                                 struct i40iw_hmc_info *hmc_info,
 206                                 struct i40iw_hmc_fpm_misc *hmc_fpm_misc)
 207 {
 208         struct i40iw_hmc_obj_info *obj_info;
 209         u64 temp;
 210         u32 size;
 211         u16 max_pe_sds;
 212
 213         obj_info = hmc_info->hmc_obj;
 214
 215         get_64bit_val(buf, 0, &temp);
 216         hmc_info->first_sd_index = (u16)RS_64(temp, I40IW_QUERY_FPM_FIRST_PE_SD_INDEX);
 217         max_pe_sds = (u16)RS_64(temp, I40IW_QUERY_FPM_MAX_PE_SDS);
 218
 219         /* Reduce SD count for VFs by 1 to account for PBLE backing page rounding */
 220         if (hmc_info->hmc_fn_id >= I40IW_FIRST_VF_FPM_ID)
 221                 max_pe_sds--;
 222         hmc_fpm_misc->max_sds = max_pe_sds;
 223         hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
 224
 225         get_64bit_val(buf, 8, &temp);
 226         obj_info[I40IW_HMC_IW_QP].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
 227         size = (u32)RS_64_1(temp, 32);
 228         obj_info[I40IW_HMC_IW_QP].size = LS_64_1(1, size);
 229
 230         get_64bit_val(buf, 16, &temp);
 231         obj_info[I40IW_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
 232         size = (u32)RS_64_1(temp, 32);
 233         obj_info[I40IW_HMC_IW_CQ].size = LS_64_1(1, size);
 234
 235         i40iw_sc_decode_fpm_query(buf, 32, obj_info, I40IW_HMC_IW_HTE);
 236         i40iw_sc_decode_fpm_query(buf, 40, obj_info, I40IW_HMC_IW_ARP);
 237
 238         obj_info[I40IW_HMC_IW_APBVT_ENTRY].size = 8192;
 239         obj_info[I40IW_HMC_IW_APBVT_ENTRY].max_cnt = 1;
 240
 241         i40iw_sc_decode_fpm_query(buf, 48, obj_info, I40IW_HMC_IW_MR);
 242         i40iw_sc_decode_fpm_query(buf, 56, obj_info, I40IW_HMC_IW_XF);
 243
 244         get_64bit_val(buf, 64, &temp);
 245         obj_info[I40IW_HMC_IW_XFFL].max_cnt = (u32)temp;
 246         obj_info[I40IW_HMC_IW_XFFL].size = 4;
 247         hmc_fpm_misc->xf_block_size = RS_64(temp, I40IW_QUERY_FPM_XFBLOCKSIZE);
 248         if (!hmc_fpm_misc->xf_block_size)
 249                 return I40IW_ERR_INVALID_SIZE;
 250
 251         i40iw_sc_decode_fpm_query(buf, 72, obj_info, I40IW_HMC_IW_Q1);
 252
 253         get_64bit_val(buf, 80, &temp);
 254         obj_info[I40IW_HMC_IW_Q1FL].max_cnt = (u32)temp;
 255         obj_info[I40IW_HMC_IW_Q1FL].size = 4;
 256         hmc_fpm_misc->q1_block_size = RS_64(temp, I40IW_QUERY_FPM_Q1BLOCKSIZE);
 257         if (!hmc_fpm_misc->q1_block_size)
 258                 return I40IW_ERR_INVALID_SIZE;
 259
 260         i40iw_sc_decode_fpm_query(buf, 88, obj_info, I40IW_HMC_IW_TIMER);
 261
 262         get_64bit_val(buf, 112, &temp);
 263         obj_info[I40IW_HMC_IW_PBLE].max_cnt = (u32)temp;
 264         obj_info[I40IW_HMC_IW_PBLE].size = 8;
 265
 266         get_64bit_val(buf, 120, &temp);
 267         hmc_fpm_misc->max_ceqs = (u8)RS_64(temp, I40IW_QUERY_FPM_MAX_CEQS);
 268         hmc_fpm_misc->ht_multiplier = RS_64(temp, I40IW_QUERY_FPM_HTMULTIPLIER);
 269         hmc_fpm_misc->timer_bucket = RS_64(temp, I40IW_QUERY_FPM_TIMERBUCKET);
 270
 271         return 0;
 272 }
 273
 274 /**
 275  * i40iw_fill_qos_list - Change all unknown qs handles to available ones
 276  * @qs_list: list of qs_handles to be fixed with valid qs_handles
 277  */
 278 static void i40iw_fill_qos_list(u16 *qs_list)
 279 {
 280         u16 qshandle = qs_list[0];
 281         int i;
 282
 283         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
 284                 if (qs_list[i] == QS_HANDLE_UNKNOWN)
 285                         qs_list[i] = qshandle;
 286                 else
 287                         qshandle = qs_list[i];
 288         }
 289 }
 290
 291 /**
 292  * i40iw_qp_from_entry - Given entry, get to the qp structure
 293  * @entry: Points to list of qp structure
 294  */
 295 static struct i40iw_sc_qp *i40iw_qp_from_entry(struct list_head *entry)
 296 {
 297         if (!entry)
 298                 return NULL;
 299
 300         return (struct i40iw_sc_qp *)((char *)entry - offsetof(struct i40iw_sc_qp, list));
 301 }
 302
 303 /**
 304  * i40iw_get_qp - get the next qp from the list given current qp
 305  * @head: Listhead of qp's
 306  * @qp: current qp
 307  */
 308 static struct i40iw_sc_qp *i40iw_get_qp(struct list_head *head, struct i40iw_sc_qp *qp)
 309 {
 310         struct list_head *entry = NULL;
 311         struct list_head *lastentry;
 312
 313         if (list_empty(head))
 314                 return NULL;
 315
 316         if (!qp) {
 317                 entry = head->next;
 318         } else {
 319                 lastentry = &qp->list;
 320                 entry = (lastentry != head) ? lastentry->next : NULL;
 321         }
 322
 323         return i40iw_qp_from_entry(entry);
 324 }
 325
 326 /**
 327  * i40iw_change_l2params - given the new l2 parameters, change all qp
 328  * @vsi: pointer to the vsi structure
 329  * @l2params: New paramaters from l2
 330  */
 331 void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params)
 332 {
 333         struct i40iw_sc_dev *dev = vsi->dev;
 334         struct i40iw_sc_qp *qp = NULL;
 335         bool qs_handle_change = false;
 336         unsigned long flags;
 337         u16 qs_handle;
 338         int i;
 339
 340         vsi->mss = l2params->mss;
 341
 342         i40iw_fill_qos_list(l2params->qs_handle_list);
 343         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
 344                 qs_handle = l2params->qs_handle_list[i];
 345                 if (vsi->qos[i].qs_handle != qs_handle)
 346                         qs_handle_change = true;
 347                 spin_lock_irqsave(&vsi->qos[i].lock, flags);
 348                 qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
 349                 while (qp) {
 350                         if (qs_handle_change) {
 351                                 qp->qs_handle = qs_handle;
 352                                 /* issue cqp suspend command */
 353                                 i40iw_qp_suspend_resume(dev, qp, true);
 354                         }
 355                         qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
 356                 }
 357                 spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
 358                 vsi->qos[i].qs_handle = qs_handle;
 359         }
 360 }
 361
 362 /**
 363  * i40iw_qp_rem_qos - remove qp from qos lists during destroy qp
 364  * @qp: qp to be removed from qos
 365  */
 366 static void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp)
 367 {
 368         struct i40iw_sc_vsi *vsi = qp->vsi;
 369         unsigned long flags;
 370
 371         if (!qp->on_qoslist)
 372                 return;
 373         spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
 374         list_del(&qp->list);
 375         spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
 376 }
 377
 378 /**
 379  * i40iw_qp_add_qos - called during setctx fot qp to be added to qos
 380  * @qp: qp to be added to qos
 381  */
 382 void i40iw_qp_add_qos(struct i40iw_sc_qp *qp)
 383 {
 384         struct i40iw_sc_vsi *vsi = qp->vsi;
 385         unsigned long flags;
 386
 387         if (qp->on_qoslist)
 388                 return;
 389         spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
 390         qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
 391         list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
 392         qp->on_qoslist = true;
 393         spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
 394 }
 395
 396 /**
 397  * i40iw_sc_pd_init - initialize sc pd struct
 398  * @dev: sc device struct
 399  * @pd: sc pd ptr
 400  * @pd_id: pd_id for allocated pd
 401  * @abi_ver: ABI version from user context, -1 if not valid
 402  */
 403 static void i40iw_sc_pd_init(struct i40iw_sc_dev *dev,
 404                              struct i40iw_sc_pd *pd,
 405                              u16 pd_id,
 406                              int abi_ver)
 407 {
 408         pd->size = sizeof(*pd);
 409         pd->pd_id = pd_id;
 410         pd->abi_ver = abi_ver;
 411         pd->dev = dev;
 412 }
 413
 414 /**
 415  * i40iw_get_encoded_wqe_size - given wq size, returns hardware encoded size
 416  * @wqsize: size of the wq (sq, rq, srq) to encoded_size
 417  * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's
 418  */
 419 u8 i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq)
 420 {
 421         u8 encoded_size = 0;
 422
 423         /* cqp sq's hw coded value starts from 1 for size of 4
 424          * while it starts from 0 for qp' wq's.
 425          */
 426         if (cqpsq)
 427                 encoded_size = 1;
 428         wqsize >>= 2;
 429         while (wqsize >>= 1)
 430                 encoded_size++;
 431         return encoded_size;
 432 }
 433
 434 /**
 435  * i40iw_sc_cqp_init - Initialize buffers for a control Queue Pair
 436  * @cqp: IWARP control queue pair pointer
 437  * @info: IWARP control queue pair init info pointer
 438  *
 439  * Initializes the object and context buffers for a control Queue Pair.
 440  */
 441 static enum i40iw_status_code i40iw_sc_cqp_init(struct i40iw_sc_cqp *cqp,
 442                                                 struct i40iw_cqp_init_info *info)
 443 {
 444         u8 hw_sq_size;
 445
 446         if ((info->sq_size > I40IW_CQP_SW_SQSIZE_2048) ||
 447             (info->sq_size < I40IW_CQP_SW_SQSIZE_4) ||
 448             ((info->sq_size & (info->sq_size - 1))))
 449                 return I40IW_ERR_INVALID_SIZE;
 450
 451         hw_sq_size = i40iw_get_encoded_wqe_size(info->sq_size, true);
 452         cqp->size = sizeof(*cqp);
 453         cqp->sq_size = info->sq_size;
 454         cqp->hw_sq_size = hw_sq_size;
 455         cqp->sq_base = info->sq;
 456         cqp->host_ctx = info->host_ctx;
 457         cqp->sq_pa = info->sq_pa;
 458         cqp->host_ctx_pa = info->host_ctx_pa;
 459         cqp->dev = info->dev;
 460         cqp->struct_ver = info->struct_ver;
 461         cqp->scratch_array = info->scratch_array;
 462         cqp->polarity = 0;
 463         cqp->en_datacenter_tcp = info->en_datacenter_tcp;
 464         cqp->enabled_vf_count = info->enabled_vf_count;
 465         cqp->hmc_profile = info->hmc_profile;
 466         info->dev->cqp = cqp;
 467
 468         I40IW_RING_INIT(cqp->sq_ring, cqp->sq_size);
 469         cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] = 0;
 470         cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS] = 0;
 471
 472         i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
 473                     "%s: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04X]\n",
 474                     __func__, cqp->sq_size, cqp->hw_sq_size,
 475                     cqp->sq_base, cqp->sq_pa, cqp, cqp->polarity);
 476         return 0;
 477 }
 478
 479 /**
 480  * i40iw_sc_cqp_create - create cqp during bringup
 481  * @cqp: struct for cqp hw
 482  * @maj_err: If error, major err number
 483  * @min_err: If error, minor err number
 484  */
 485 static enum i40iw_status_code i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp,
 486                                                   u16 *maj_err,
 487                                                   u16 *min_err)
 488 {
 489         u64 temp;
 490         u32 cnt = 0, p1, p2, val = 0, err_code;
 491         enum i40iw_status_code ret_code;
 492
 493         *maj_err = 0;
 494         *min_err = 0;
 495
 496         ret_code = i40iw_allocate_dma_mem(cqp->dev->hw,
 497                                           &cqp->sdbuf,
 498                                           128,
 499                                           I40IW_SD_BUF_ALIGNMENT);
 500
 501         if (ret_code)
 502                 goto exit;
 503
 504         temp = LS_64(cqp->hw_sq_size, I40IW_CQPHC_SQSIZE) |
 505                LS_64(cqp->struct_ver, I40IW_CQPHC_SVER);
 506
 507         set_64bit_val(cqp->host_ctx, 0, temp);
 508         set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);
 509         temp = LS_64(cqp->enabled_vf_count, I40IW_CQPHC_ENABLED_VFS) |
 510                LS_64(cqp->hmc_profile, I40IW_CQPHC_HMC_PROFILE);
 511         set_64bit_val(cqp->host_ctx, 16, temp);
 512         set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp);
 513         set_64bit_val(cqp->host_ctx, 32, 0);
 514         set_64bit_val(cqp->host_ctx, 40, 0);
 515         set_64bit_val(cqp->host_ctx, 48, 0);
 516         set_64bit_val(cqp->host_ctx, 56, 0);
 517
 518         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQP_HOST_CTX",
 519                         cqp->host_ctx, I40IW_CQP_CTX_SIZE * 8);
 520
 521         p1 = RS_32_1(cqp->host_ctx_pa, 32);
 522         p2 = (u32)cqp->host_ctx_pa;
 523
 524         if (cqp->dev->is_pf) {
 525                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, p1);
 526                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, p2);
 527         } else {
 528                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, p1);
 529                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, p2);
 530         }
 531         do {
 532                 if (cnt++ > I40IW_DONE_COUNT) {
 533                         i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
 534                         ret_code = I40IW_ERR_TIMEOUT;
 535                         /*
 536                          * read PFPE_CQPERRORCODES register to get the minor
 537                          * and major error code
 538                          */
 539                         if (cqp->dev->is_pf)
 540                                 err_code = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES);
 541                         else
 542                                 err_code = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
 543                         *min_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE);
 544                         *maj_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE);
 545                         goto exit;
 546                 }
 547                 udelay(I40IW_SLEEP_COUNT);
 548                 if (cqp->dev->is_pf)
 549                         val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CCQPSTATUS);
 550                 else
 551                         val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CCQPSTATUS1);
 552         } while (!val);
 553
 554 exit:
 555         if (!ret_code)
 556                 cqp->process_cqp_sds = i40iw_update_sds_noccq;
 557         return ret_code;
 558 }
 559
 560 /**
 561  * i40iw_sc_cqp_post_sq - post of cqp's sq
 562  * @cqp: struct for cqp hw
 563  */
 564 void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp)
 565 {
 566         if (cqp->dev->is_pf)
 567                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPDB, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
 568         else
 569                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CQPDB1, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
 570
 571         i40iw_debug(cqp->dev,
 572                     I40IW_DEBUG_WQE,
 573                     "%s: HEAD_TAIL[%04d,%04d,%04d]\n",
 574                     __func__,
 575                     cqp->sq_ring.head,
 576                     cqp->sq_ring.tail,
 577                     cqp->sq_ring.size);
 578 }
 579
 580 /**
 581  * i40iw_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
 582  * @cqp: struct for cqp hw
 583  * @wqe_idx: we index of cqp ring
 584  */
 585 u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch)
 586 {
 587         u64 *wqe = NULL;
 588         u32     wqe_idx;
 589         enum i40iw_status_code ret_code;
 590
 591         if (I40IW_RING_FULL_ERR(cqp->sq_ring)) {
 592                 i40iw_debug(cqp->dev,
 593                             I40IW_DEBUG_WQE,
 594                             "%s: ring is full head %x tail %x size %x\n",
 595                             __func__,
 596                             cqp->sq_ring.head,
 597                             cqp->sq_ring.tail,
 598                             cqp->sq_ring.size);
 599                 return NULL;
 600         }
 601         I40IW_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, wqe_idx, ret_code);
 602         cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS]++;
 603         if (ret_code)
 604                 return NULL;
 605         if (!wqe_idx)
 606                 cqp->polarity = !cqp->polarity;
 607
 608         wqe = cqp->sq_base[wqe_idx].elem;
 609         cqp->scratch_array[wqe_idx] = scratch;
 610         I40IW_CQP_INIT_WQE(wqe);
 611
 612         return wqe;
 613 }
 614
 615 /**
 616  * i40iw_sc_cqp_destroy - destroy cqp during close
 617  * @cqp: struct for cqp hw
 618  */
 619 static enum i40iw_status_code i40iw_sc_cqp_destroy(struct i40iw_sc_cqp *cqp)
 620 {
 621         u32 cnt = 0, val = 1;
 622         enum i40iw_status_code ret_code = 0;
 623         u32 cqpstat_addr;
 624
 625         if (cqp->dev->is_pf) {
 626                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, 0);
 627                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, 0);
 628                 cqpstat_addr = I40E_PFPE_CCQPSTATUS;
 629         } else {
 630                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, 0);
 631                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, 0);
 632                 cqpstat_addr = I40E_VFPE_CCQPSTATUS1;
 633         }
 634         do {
 635                 if (cnt++ > I40IW_DONE_COUNT) {
 636                         ret_code = I40IW_ERR_TIMEOUT;
 637                         break;
 638                 }
 639                 udelay(I40IW_SLEEP_COUNT);
 640                 val = i40iw_rd32(cqp->dev->hw, cqpstat_addr);
 641         } while (val);
 642
 643         i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
 644         return ret_code;
 645 }
 646
 647 /**
 648  * i40iw_sc_ccq_arm - enable intr for control cq
 649  * @ccq: ccq sc struct
 650  */
 651 static void i40iw_sc_ccq_arm(struct i40iw_sc_cq *ccq)
 652 {
 653         u64 temp_val;
 654         u16 sw_cq_sel;
 655         u8 arm_next_se;
 656         u8 arm_seq_num;
 657
 658         /* write to cq doorbell shadow area */
 659         /* arm next se should always be zero */
 660         get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val);
 661
 662         sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT);
 663         arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE);
 664
 665         arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM);
 666         arm_seq_num++;
 667
 668         temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) |
 669                    LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) |
 670                    LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) |
 671                    LS_64(1, I40IW_CQ_DBSA_ARM_NEXT);
 672
 673         set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val);
 674
 675         wmb();       /* make sure shadow area is updated before arming */
 676
 677         if (ccq->dev->is_pf)
 678                 i40iw_wr32(ccq->dev->hw, I40E_PFPE_CQARM, ccq->cq_uk.cq_id);
 679         else
 680                 i40iw_wr32(ccq->dev->hw, I40E_VFPE_CQARM1, ccq->cq_uk.cq_id);
 681 }
 682
 683 /**
 684  * i40iw_sc_ccq_get_cqe_info - get ccq's cq entry
 685  * @ccq: ccq sc struct
 686  * @info: completion q entry to return
 687  */
 688 static enum i40iw_status_code i40iw_sc_ccq_get_cqe_info(
 689                                         struct i40iw_sc_cq *ccq,
 690                                         struct i40iw_ccq_cqe_info *info)
 691 {
 692         u64 qp_ctx, temp, temp1;
 693         u64 *cqe;
 694         struct i40iw_sc_cqp *cqp;
 695         u32 wqe_idx;
 696         u8 polarity;
 697         enum i40iw_status_code ret_code = 0;
 698
 699         if (ccq->cq_uk.avoid_mem_cflct)
 700                 cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(&ccq->cq_uk);
 701         else
 702                 cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&ccq->cq_uk);
 703
 704         get_64bit_val(cqe, 24, &temp);
 705         polarity = (u8)RS_64(temp, I40IW_CQ_VALID);
 706         if (polarity != ccq->cq_uk.polarity)
 707                 return I40IW_ERR_QUEUE_EMPTY;
 708
 709         get_64bit_val(cqe, 8, &qp_ctx);
 710         cqp = (struct i40iw_sc_cqp *)(unsigned long)qp_ctx;
 711         info->error = (bool)RS_64(temp, I40IW_CQ_ERROR);
 712         info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
 713         if (info->error) {
 714                 info->maj_err_code = (u16)RS_64(temp, I40IW_CQ_MAJERR);
 715                 info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
 716         }
 717         wqe_idx = (u32)RS_64(temp, I40IW_CQ_WQEIDX);
 718         info->scratch = cqp->scratch_array[wqe_idx];
 719
 720         get_64bit_val(cqe, 16, &temp1);
 721         info->op_ret_val = (u32)RS_64(temp1, I40IW_CCQ_OPRETVAL);
 722         get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1);
 723         info->op_code = (u8)RS_64(temp1, I40IW_CQPSQ_OPCODE);
 724         info->cqp = cqp;
 725
 726         /*  move the head for cq */
 727         I40IW_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);
 728         if (I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring) == 0)
 729                 ccq->cq_uk.polarity ^= 1;
 730
 731         /* update cq tail in cq shadow memory also */
 732         I40IW_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);
 733         set_64bit_val(ccq->cq_uk.shadow_area,
 734                       0,
 735                       I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring));
 736         wmb(); /* write shadow area before tail */
 737         I40IW_RING_MOVE_TAIL(cqp->sq_ring);
 738         ccq->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
 739
 740         return ret_code;
 741 }
 742
 743 /**
 744  * i40iw_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
 745  * @cqp: struct for cqp hw
 746  * @op_code: cqp opcode for completion
 747  * @info: completion q entry to return
 748  */
 749 static enum i40iw_status_code i40iw_sc_poll_for_cqp_op_done(
 750                                         struct i40iw_sc_cqp *cqp,
 751                                         u8 op_code,
 752                                         struct i40iw_ccq_cqe_info *compl_info)
 753 {
 754         struct i40iw_ccq_cqe_info info;
 755         struct i40iw_sc_cq *ccq;
 756         enum i40iw_status_code ret_code = 0;
 757         u32 cnt = 0;
 758
 759         memset(&info, 0, sizeof(info));
 760         ccq = cqp->dev->ccq;
 761         while (1) {
 762                 if (cnt++ > I40IW_DONE_COUNT)
 763                         return I40IW_ERR_TIMEOUT;
 764
 765                 if (i40iw_sc_ccq_get_cqe_info(ccq, &info)) {
 766                         udelay(I40IW_SLEEP_COUNT);
 767                         continue;
 768                 }
 769
 770                 if (info.error) {
 771                         ret_code = I40IW_ERR_CQP_COMPL_ERROR;
 772                         break;
 773                 }
 774                 /* check if opcode is cq create */
 775                 if (op_code != info.op_code) {
 776                         i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
 777                                     "%s: opcode mismatch for my op code 0x%x, returned opcode %x\n",
 778                                     __func__, op_code, info.op_code);
 779                 }
 780                 /* success, exit out of the loop */
 781                 if (op_code == info.op_code)
 782                         break;
 783         }
 784
 785         if (compl_info)
 786                 memcpy(compl_info, &info, sizeof(*compl_info));
 787
 788         return ret_code;
 789 }
 790
 791 /**
 792  * i40iw_sc_manage_push_page - Handle push page
 793  * @cqp: struct for cqp hw
 794  * @info: push page info
 795  * @scratch: u64 saved to be used during cqp completion
 796  * @post_sq: flag for cqp db to ring
 797  */
 798 static enum i40iw_status_code i40iw_sc_manage_push_page(
 799                                 struct i40iw_sc_cqp *cqp,
 800                                 struct i40iw_cqp_manage_push_page_info *info,
 801                                 u64 scratch,
 802                                 bool post_sq)
 803 {
 804         u64 *wqe;
 805         u64 header;
 806
 807         if (info->push_idx >= I40IW_MAX_PUSH_PAGE_COUNT)
 808                 return I40IW_ERR_INVALID_PUSH_PAGE_INDEX;
 809
 810         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 811         if (!wqe)
 812                 return I40IW_ERR_RING_FULL;
 813
 814         set_64bit_val(wqe, 16, info->qs_handle);
 815
 816         header = LS_64(info->push_idx, I40IW_CQPSQ_MPP_PPIDX) |
 817                  LS_64(I40IW_CQP_OP_MANAGE_PUSH_PAGES, I40IW_CQPSQ_OPCODE) |
 818                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
 819                  LS_64(info->free_page, I40IW_CQPSQ_MPP_FREE_PAGE);
 820
 821         i40iw_insert_wqe_hdr(wqe, header);
 822
 823         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE",
 824                         wqe, I40IW_CQP_WQE_SIZE * 8);
 825
 826         if (post_sq)
 827                 i40iw_sc_cqp_post_sq(cqp);
 828         return 0;
 829 }
 830
 831 /**
 832  * i40iw_sc_manage_hmc_pm_func_table - manage of function table
 833  * @cqp: struct for cqp hw
 834  * @scratch: u64 saved to be used during cqp completion
 835  * @vf_index: vf index for cqp
 836  * @free_pm_fcn: function number
 837  * @post_sq: flag for cqp db to ring
 838  */
 839 static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table(
 840                                 struct i40iw_sc_cqp *cqp,
 841                                 u64 scratch,
 842                                 u8 vf_index,
 843                                 bool free_pm_fcn,
 844                                 bool post_sq)
 845 {
 846         u64 *wqe;
 847         u64 header;
 848
 849         if (vf_index >= I40IW_MAX_VF_PER_PF)
 850                 return I40IW_ERR_INVALID_VF_ID;
 851         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 852         if (!wqe)
 853                 return I40IW_ERR_RING_FULL;
 854
 855         header = LS_64(vf_index, I40IW_CQPSQ_MHMC_VFIDX) |
 856                  LS_64(I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, I40IW_CQPSQ_OPCODE) |
 857                  LS_64(free_pm_fcn, I40IW_CQPSQ_MHMC_FREEPMFN) |
 858                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
 859
 860         i40iw_insert_wqe_hdr(wqe, header);
 861         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
 862                         wqe, I40IW_CQP_WQE_SIZE * 8);
 863         if (post_sq)
 864                 i40iw_sc_cqp_post_sq(cqp);
 865         return 0;
 866 }
 867
 868 /**
 869  * i40iw_sc_set_hmc_resource_profile - cqp wqe for hmc profile
 870  * @cqp: struct for cqp hw
 871  * @scratch: u64 saved to be used during cqp completion
 872  * @hmc_profile_type: type of profile to set
 873  * @vf_num: vf number for profile
 874  * @post_sq: flag for cqp db to ring
 875  * @poll_registers: flag to poll register for cqp completion
 876  */
 877 static enum i40iw_status_code i40iw_sc_set_hmc_resource_profile(
 878                                 struct i40iw_sc_cqp *cqp,
 879                                 u64 scratch,
 880                                 u8 hmc_profile_type,
 881                                 u8 vf_num, bool post_sq,
 882                                 bool poll_registers)
 883 {
 884         u64 *wqe;
 885         u64 header;
 886         u32 val, tail, error;
 887         enum i40iw_status_code ret_code = 0;
 888
 889         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 890         if (!wqe)
 891                 return I40IW_ERR_RING_FULL;
 892
 893         set_64bit_val(wqe, 16,
 894                       (LS_64(hmc_profile_type, I40IW_CQPSQ_SHMCRP_HMC_PROFILE) |
 895                                 LS_64(vf_num, I40IW_CQPSQ_SHMCRP_VFNUM)));
 896
 897         header = LS_64(I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE, I40IW_CQPSQ_OPCODE) |
 898                        LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
 899
 900         i40iw_insert_wqe_hdr(wqe, header);
 901
 902         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
 903                         wqe, I40IW_CQP_WQE_SIZE * 8);
 904
 905         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
 906         if (error)
 907                 return I40IW_ERR_CQP_COMPL_ERROR;
 908
 909         if (post_sq) {
 910                 i40iw_sc_cqp_post_sq(cqp);
 911                 if (poll_registers)
 912                         ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000000);
 913                 else
 914                         ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,
 915                                                                  I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,
 916                                                                  NULL);
 917         }
 918
 919         return ret_code;
 920 }
 921
 922 /**
 923  * i40iw_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table
 924  * @cqp: struct for cqp hw
 925  */
 926 static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table_done(struct i40iw_sc_cqp *cqp)
 927 {
 928         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, NULL);
 929 }
 930
 931 /**
 932  * i40iw_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit
 933  * @cqp: struct for cqp hw
 934  */
 935 static enum i40iw_status_code i40iw_sc_commit_fpm_values_done(struct i40iw_sc_cqp *cqp)
 936 {
 937         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_COMMIT_FPM_VALUES, NULL);
 938 }
 939
 940 /**
 941  * i40iw_sc_commit_fpm_values - cqp wqe for commit fpm values
 942  * @cqp: struct for cqp hw
 943  * @scratch: u64 saved to be used during cqp completion
 944  * @hmc_fn_id: hmc function id
 945  * @commit_fpm_mem; Memory for fpm values
 946  * @post_sq: flag for cqp db to ring
 947  * @wait_type: poll ccq or cqp registers for cqp completion
 948  */
 949 static enum i40iw_status_code i40iw_sc_commit_fpm_values(
 950                                         struct i40iw_sc_cqp *cqp,
 951                                         u64 scratch,
 952                                         u8 hmc_fn_id,
 953                                         struct i40iw_dma_mem *commit_fpm_mem,
 954                                         bool post_sq,
 955                                         u8 wait_type)
 956 {
 957         u64 *wqe;
 958         u64 header;
 959         u32 tail, val, error;
 960         enum i40iw_status_code ret_code = 0;
 961
 962         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 963         if (!wqe)
 964                 return I40IW_ERR_RING_FULL;
 965
 966         set_64bit_val(wqe, 16, hmc_fn_id);
 967         set_64bit_val(wqe, 32, commit_fpm_mem->pa);
 968
 969         header = LS_64(I40IW_CQP_OP_COMMIT_FPM_VALUES, I40IW_CQPSQ_OPCODE) |
 970                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
 971
 972         i40iw_insert_wqe_hdr(wqe, header);
 973
 974         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "COMMIT_FPM_VALUES WQE",
 975                         wqe, I40IW_CQP_WQE_SIZE * 8);
 976
 977         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
 978         if (error)
 979                 return I40IW_ERR_CQP_COMPL_ERROR;
 980
 981         if (post_sq) {
 982                 i40iw_sc_cqp_post_sq(cqp);
 983
 984                 if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
 985                         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
 986                 else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
 987                         ret_code = i40iw_sc_commit_fpm_values_done(cqp);
 988         }
 989
 990         return ret_code;
 991 }
 992
 993 /**
 994  * i40iw_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm
 995  * @cqp: struct for cqp hw
 996  */
 997 static enum i40iw_status_code i40iw_sc_query_fpm_values_done(struct i40iw_sc_cqp *cqp)
 998 {
 999         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_QUERY_FPM_VALUES, NULL);
1000 }
1001
1002 /**
1003  * i40iw_sc_query_fpm_values - cqp wqe query fpm values
1004  * @cqp: struct for cqp hw
1005  * @scratch: u64 saved to be used during cqp completion
1006  * @hmc_fn_id: hmc function id
1007  * @query_fpm_mem: memory for return fpm values
1008  * @post_sq: flag for cqp db to ring
1009  * @wait_type: poll ccq or cqp registers for cqp completion
1010  */
1011 static enum i40iw_status_code i40iw_sc_query_fpm_values(
1012                                         struct i40iw_sc_cqp *cqp,
1013                                         u64 scratch,
1014                                         u8 hmc_fn_id,
1015                                         struct i40iw_dma_mem *query_fpm_mem,
1016                                         bool post_sq,
1017                                         u8 wait_type)
1018 {
1019         u64 *wqe;
1020         u64 header;
1021         u32 tail, val, error;
1022         enum i40iw_status_code ret_code = 0;
1023
1024         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1025         if (!wqe)
1026                 return I40IW_ERR_RING_FULL;
1027
1028         set_64bit_val(wqe, 16, hmc_fn_id);
1029         set_64bit_val(wqe, 32, query_fpm_mem->pa);
1030
1031         header = LS_64(I40IW_CQP_OP_QUERY_FPM_VALUES, I40IW_CQPSQ_OPCODE) |
1032                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1033
1034         i40iw_insert_wqe_hdr(wqe, header);
1035
1036         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_FPM WQE",
1037                         wqe, I40IW_CQP_WQE_SIZE * 8);
1038
1039         /* read the tail from CQP_TAIL register */
1040         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
1041
1042         if (error)
1043                 return I40IW_ERR_CQP_COMPL_ERROR;
1044
1045         if (post_sq) {
1046                 i40iw_sc_cqp_post_sq(cqp);
1047                 if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
1048                         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
1049                 else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
1050                         ret_code = i40iw_sc_query_fpm_values_done(cqp);
1051         }
1052
1053         return ret_code;
1054 }
1055
1056 /**
1057  * i40iw_sc_add_arp_cache_entry - cqp wqe add arp cache entry
1058  * @cqp: struct for cqp hw
1059  * @info: arp entry information
1060  * @scratch: u64 saved to be used during cqp completion
1061  * @post_sq: flag for cqp db to ring
1062  */
1063 static enum i40iw_status_code i40iw_sc_add_arp_cache_entry(
1064                                 struct i40iw_sc_cqp *cqp,
1065                                 struct i40iw_add_arp_cache_entry_info *info,
1066                                 u64 scratch,
1067                                 bool post_sq)
1068 {
1069         u64 *wqe;
1070         u64 temp, header;
1071
1072         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1073         if (!wqe)
1074                 return I40IW_ERR_RING_FULL;
1075         set_64bit_val(wqe, 8, info->reach_max);
1076
1077         temp = info->mac_addr[5] |
1078                LS_64_1(info->mac_addr[4], 8) |
1079                LS_64_1(info->mac_addr[3], 16) |
1080                LS_64_1(info->mac_addr[2], 24) |
1081                LS_64_1(info->mac_addr[1], 32) |
1082                LS_64_1(info->mac_addr[0], 40);
1083
1084         set_64bit_val(wqe, 16, temp);
1085
1086         header = info->arp_index |
1087                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1088                  LS_64((info->permanent ? 1 : 0), I40IW_CQPSQ_MAT_PERMANENT) |
1089                  LS_64(1, I40IW_CQPSQ_MAT_ENTRYVALID) |
1090                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1091
1092         i40iw_insert_wqe_hdr(wqe, header);
1093
1094         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_ENTRY WQE",
1095                         wqe, I40IW_CQP_WQE_SIZE * 8);
1096
1097         if (post_sq)
1098                 i40iw_sc_cqp_post_sq(cqp);
1099         return 0;
1100 }
1101
1102 /**
1103  * i40iw_sc_del_arp_cache_entry - dele arp cache entry
1104  * @cqp: struct for cqp hw
1105  * @scratch: u64 saved to be used during cqp completion
1106  * @arp_index: arp index to delete arp entry
1107  * @post_sq: flag for cqp db to ring
1108  */
1109 static enum i40iw_status_code i40iw_sc_del_arp_cache_entry(
1110                                         struct i40iw_sc_cqp *cqp,
1111                                         u64 scratch,
1112                                         u16 arp_index,
1113                                         bool post_sq)
1114 {
1115         u64 *wqe;
1116         u64 header;
1117
1118         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1119         if (!wqe)
1120                 return I40IW_ERR_RING_FULL;
1121
1122         header = arp_index |
1123                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1124                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1125         i40iw_insert_wqe_hdr(wqe, header);
1126
1127         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE",
1128                         wqe, I40IW_CQP_WQE_SIZE * 8);
1129
1130         if (post_sq)
1131                 i40iw_sc_cqp_post_sq(cqp);
1132         return 0;
1133 }
1134
1135 /**
1136  * i40iw_sc_query_arp_cache_entry - cqp wqe to query arp and arp index
1137  * @cqp: struct for cqp hw
1138  * @scratch: u64 saved to be used during cqp completion
1139  * @arp_index: arp index to delete arp entry
1140  * @post_sq: flag for cqp db to ring
1141  */
1142 static enum i40iw_status_code i40iw_sc_query_arp_cache_entry(
1143                                 struct i40iw_sc_cqp *cqp,
1144                                 u64 scratch,
1145                                 u16 arp_index,
1146                                 bool post_sq)
1147 {
1148         u64 *wqe;
1149         u64 header;
1150
1151         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1152         if (!wqe)
1153                 return I40IW_ERR_RING_FULL;
1154
1155         header = arp_index |
1156                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1157                  LS_64(1, I40IW_CQPSQ_MAT_QUERY) |
1158                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1159
1160         i40iw_insert_wqe_hdr(wqe, header);
1161
1162         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_ARP_CACHE_ENTRY WQE",
1163                         wqe, I40IW_CQP_WQE_SIZE * 8);
1164
1165         if (post_sq)
1166                 i40iw_sc_cqp_post_sq(cqp);
1167         return 0;
1168 }
1169
1170 /**
1171  * i40iw_sc_manage_apbvt_entry - for adding and deleting apbvt entries
1172  * @cqp: struct for cqp hw
1173  * @info: info for apbvt entry to add or delete
1174  * @scratch: u64 saved to be used during cqp completion
1175  * @post_sq: flag for cqp db to ring
1176  */
1177 static enum i40iw_status_code i40iw_sc_manage_apbvt_entry(
1178                                 struct i40iw_sc_cqp *cqp,
1179                                 struct i40iw_apbvt_info *info,
1180                                 u64 scratch,
1181                                 bool post_sq)
1182 {
1183         u64 *wqe;
1184         u64 header;
1185
1186         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1187         if (!wqe)
1188                 return I40IW_ERR_RING_FULL;
1189
1190         set_64bit_val(wqe, 16, info->port);
1191
1192         header = LS_64(I40IW_CQP_OP_MANAGE_APBVT, I40IW_CQPSQ_OPCODE) |
1193                  LS_64(info->add, I40IW_CQPSQ_MAPT_ADDPORT) |
1194                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1195
1196         i40iw_insert_wqe_hdr(wqe, header);
1197
1198         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_APBVT WQE",
1199                         wqe, I40IW_CQP_WQE_SIZE * 8);
1200
1201         if (post_sq)
1202                 i40iw_sc_cqp_post_sq(cqp);
1203         return 0;
1204 }
1205
1206 /**
1207  * i40iw_sc_manage_qhash_table_entry - manage quad hash entries
1208  * @cqp: struct for cqp hw
1209  * @info: info for quad hash to manage
1210  * @scratch: u64 saved to be used during cqp completion
1211  * @post_sq: flag for cqp db to ring
1212  *
1213  * This is called before connection establishment is started. For passive connections, when
1214  * listener is created, it will call with entry type of  I40IW_QHASH_TYPE_TCP_SYN with local
1215  * ip address and tcp port. When SYN is received (passive connections) or
1216  * sent (active connections), this routine is called with entry type of
1217  * I40IW_QHASH_TYPE_TCP_ESTABLISHED and quad is passed in info.
1218  *
1219  * When iwarp connection is done and its state moves to RTS, the quad hash entry in
1220  * the hardware will point to iwarp's qp number and requires no calls from the driver.
1221  */
1222 static enum i40iw_status_code i40iw_sc_manage_qhash_table_entry(
1223                                         struct i40iw_sc_cqp *cqp,
1224                                         struct i40iw_qhash_table_info *info,
1225                                         u64 scratch,
1226                                         bool post_sq)
1227 {
1228         u64 *wqe;
1229         u64 qw1 = 0;
1230         u64 qw2 = 0;
1231         u64 temp;
1232         struct i40iw_sc_vsi *vsi = info->vsi;
1233
1234         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1235         if (!wqe)
1236                 return I40IW_ERR_RING_FULL;
1237
1238         temp = info->mac_addr[5] |
1239                 LS_64_1(info->mac_addr[4], 8) |
1240                 LS_64_1(info->mac_addr[3], 16) |
1241                 LS_64_1(info->mac_addr[2], 24) |
1242                 LS_64_1(info->mac_addr[1], 32) |
1243                 LS_64_1(info->mac_addr[0], 40);
1244
1245         set_64bit_val(wqe, 0, temp);
1246
1247         qw1 = LS_64(info->qp_num, I40IW_CQPSQ_QHASH_QPN) |
1248               LS_64(info->dest_port, I40IW_CQPSQ_QHASH_DEST_PORT);
1249         if (info->ipv4_valid) {
1250                 set_64bit_val(wqe,
1251                               48,
1252                               LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
1253         } else {
1254                 set_64bit_val(wqe,
1255                               56,
1256                               LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |
1257                               LS_64(info->dest_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
1258
1259                 set_64bit_val(wqe,
1260                               48,
1261                               LS_64(info->dest_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |
1262                               LS_64(info->dest_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
1263         }
1264         qw2 = LS_64(vsi->qos[info->user_pri].qs_handle, I40IW_CQPSQ_QHASH_QS_HANDLE);
1265         if (info->vlan_valid)
1266                 qw2 |= LS_64(info->vlan_id, I40IW_CQPSQ_QHASH_VLANID);
1267         set_64bit_val(wqe, 16, qw2);
1268         if (info->entry_type == I40IW_QHASH_TYPE_TCP_ESTABLISHED) {
1269                 qw1 |= LS_64(info->src_port, I40IW_CQPSQ_QHASH_SRC_PORT);
1270                 if (!info->ipv4_valid) {
1271                         set_64bit_val(wqe,
1272                                       40,
1273                                       LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |
1274                                       LS_64(info->src_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
1275                         set_64bit_val(wqe,
1276                                       32,
1277                                       LS_64(info->src_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |
1278                                       LS_64(info->src_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
1279                 } else {
1280                         set_64bit_val(wqe,
1281                                       32,
1282                                       LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
1283                 }
1284         }
1285
1286         set_64bit_val(wqe, 8, qw1);
1287         temp = LS_64(cqp->polarity, I40IW_CQPSQ_QHASH_WQEVALID) |
1288                LS_64(I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY, I40IW_CQPSQ_QHASH_OPCODE) |
1289                LS_64(info->manage, I40IW_CQPSQ_QHASH_MANAGE) |
1290                LS_64(info->ipv4_valid, I40IW_CQPSQ_QHASH_IPV4VALID) |
1291                LS_64(info->vlan_valid, I40IW_CQPSQ_QHASH_VLANVALID) |
1292                LS_64(info->entry_type, I40IW_CQPSQ_QHASH_ENTRYTYPE);
1293
1294         i40iw_insert_wqe_hdr(wqe, temp);
1295
1296         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_QHASH WQE",
1297                         wqe, I40IW_CQP_WQE_SIZE * 8);
1298
1299         if (post_sq)
1300                 i40iw_sc_cqp_post_sq(cqp);
1301         return 0;
1302 }
1303
1304 /**
1305  * i40iw_sc_alloc_local_mac_ipaddr_entry - cqp wqe for loc mac entry
1306  * @cqp: struct for cqp hw
1307  * @scratch: u64 saved to be used during cqp completion
1308  * @post_sq: flag for cqp db to ring
1309  */
1310 static enum i40iw_status_code i40iw_sc_alloc_local_mac_ipaddr_entry(
1311                                         struct i40iw_sc_cqp *cqp,
1312                                         u64 scratch,
1313                                         bool post_sq)
1314 {
1315         u64 *wqe;
1316         u64 header;
1317
1318         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1319         if (!wqe)
1320                 return I40IW_ERR_RING_FULL;
1321         header = LS_64(I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY, I40IW_CQPSQ_OPCODE) |
1322                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1323
1324         i40iw_insert_wqe_hdr(wqe, header);
1325         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ALLOCATE_LOCAL_MAC_IPADDR WQE",
1326                         wqe, I40IW_CQP_WQE_SIZE * 8);
1327         if (post_sq)
1328                 i40iw_sc_cqp_post_sq(cqp);
1329         return 0;
1330 }
1331
1332 /**
1333  * i40iw_sc_add_local_mac_ipaddr_entry - add mac enry
1334  * @cqp: struct for cqp hw
1335  * @info:mac addr info
1336  * @scratch: u64 saved to be used during cqp completion
1337  * @post_sq: flag for cqp db to ring
1338  */
1339 static enum i40iw_status_code i40iw_sc_add_local_mac_ipaddr_entry(
1340                                 struct i40iw_sc_cqp *cqp,
1341                                 struct i40iw_local_mac_ipaddr_entry_info *info,
1342                                 u64 scratch,
1343                                 bool post_sq)
1344 {
1345         u64 *wqe;
1346         u64 temp, header;
1347
1348         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1349         if (!wqe)
1350                 return I40IW_ERR_RING_FULL;
1351         temp = info->mac_addr[5] |
1352                 LS_64_1(info->mac_addr[4], 8) |
1353                 LS_64_1(info->mac_addr[3], 16) |
1354                 LS_64_1(info->mac_addr[2], 24) |
1355                 LS_64_1(info->mac_addr[1], 32) |
1356                 LS_64_1(info->mac_addr[0], 40);
1357
1358         set_64bit_val(wqe, 32, temp);
1359
1360         header = LS_64(info->entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |
1361                  LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |
1362                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1363
1364         i40iw_insert_wqe_hdr(wqe, header);
1365
1366         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ADD_LOCAL_MAC_IPADDR WQE",
1367                         wqe, I40IW_CQP_WQE_SIZE * 8);
1368
1369         if (post_sq)
1370                 i40iw_sc_cqp_post_sq(cqp);
1371         return 0;
1372 }
1373
1374 /**
1375  * i40iw_sc_del_local_mac_ipaddr_entry - cqp wqe to dele local mac
1376  * @cqp: struct for cqp hw
1377  * @scratch: u64 saved to be used during cqp completion
1378  * @entry_idx: index of mac entry
1379  * @ ignore_ref_count: to force mac adde delete
1380  * @post_sq: flag for cqp db to ring
1381  */
1382 static enum i40iw_status_code i40iw_sc_del_local_mac_ipaddr_entry(
1383                                 struct i40iw_sc_cqp *cqp,
1384                                 u64 scratch,
1385                                 u8 entry_idx,
1386                                 u8 ignore_ref_count,
1387                                 bool post_sq)
1388 {
1389         u64 *wqe;
1390         u64 header;
1391
1392         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1393         if (!wqe)
1394                 return I40IW_ERR_RING_FULL;
1395         header = LS_64(entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |
1396                  LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |
1397                  LS_64(1, I40IW_CQPSQ_MLIPA_FREEENTRY) |
1398                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
1399                  LS_64(ignore_ref_count, I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT);
1400
1401         i40iw_insert_wqe_hdr(wqe, header);
1402
1403         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE",
1404                         wqe, I40IW_CQP_WQE_SIZE * 8);
1405
1406         if (post_sq)
1407                 i40iw_sc_cqp_post_sq(cqp);
1408         return 0;
1409 }
1410
1411 /**
1412  * i40iw_sc_cqp_nop - send a nop wqe
1413  * @cqp: struct for cqp hw
1414  * @scratch: u64 saved to be used during cqp completion
1415  * @post_sq: flag for cqp db to ring
1416  */
1417 static enum i40iw_status_code i40iw_sc_cqp_nop(struct i40iw_sc_cqp *cqp,
1418                                                u64 scratch,
1419                                                bool post_sq)
1420 {
1421         u64 *wqe;
1422         u64 header;
1423
1424         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1425         if (!wqe)
1426                 return I40IW_ERR_RING_FULL;
1427         header = LS_64(I40IW_CQP_OP_NOP, I40IW_CQPSQ_OPCODE) |
1428                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1429         i40iw_insert_wqe_hdr(wqe, header);
1430         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "NOP WQE",
1431                         wqe, I40IW_CQP_WQE_SIZE * 8);
1432
1433         if (post_sq)
1434                 i40iw_sc_cqp_post_sq(cqp);
1435         return 0;
1436 }
1437
1438 /**
1439  * i40iw_sc_ceq_init - initialize ceq
1440  * @ceq: ceq sc structure
1441  * @info: ceq initialization info
1442  */
1443 static enum i40iw_status_code i40iw_sc_ceq_init(struct i40iw_sc_ceq *ceq,
1444                                                 struct i40iw_ceq_init_info *info)
1445 {
1446         u32 pble_obj_cnt;
1447
1448         if ((info->elem_cnt < I40IW_MIN_CEQ_ENTRIES) ||
1449             (info->elem_cnt > I40IW_MAX_CEQ_ENTRIES))
1450                 return I40IW_ERR_INVALID_SIZE;
1451
1452         if (info->ceq_id >= I40IW_MAX_CEQID)
1453                 return I40IW_ERR_INVALID_CEQ_ID;
1454
1455         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
1456
1457         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
1458                 return I40IW_ERR_INVALID_PBLE_INDEX;
1459
1460         ceq->size = sizeof(*ceq);
1461         ceq->ceqe_base = (struct i40iw_ceqe *)info->ceqe_base;
1462         ceq->ceq_id = info->ceq_id;
1463         ceq->dev = info->dev;
1464         ceq->elem_cnt = info->elem_cnt;
1465         ceq->ceq_elem_pa = info->ceqe_pa;
1466         ceq->virtual_map = info->virtual_map;
1467
1468         ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);
1469         ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);
1470         ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);
1471
1472         ceq->tph_en = info->tph_en;
1473         ceq->tph_val = info->tph_val;
1474         ceq->polarity = 1;
1475         I40IW_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);
1476         ceq->dev->ceq[info->ceq_id] = ceq;
1477
1478         return 0;
1479 }
1480
1481 /**
1482  * i40iw_sc_ceq_create - create ceq wqe
1483  * @ceq: ceq sc structure
1484  * @scratch: u64 saved to be used during cqp completion
1485  * @post_sq: flag for cqp db to ring
1486  */
1487 static enum i40iw_status_code i40iw_sc_ceq_create(struct i40iw_sc_ceq *ceq,
1488                                                   u64 scratch,
1489                                                   bool post_sq)
1490 {
1491         struct i40iw_sc_cqp *cqp;
1492         u64 *wqe;
1493         u64 header;
1494
1495         cqp = ceq->dev->cqp;
1496         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1497         if (!wqe)
1498                 return I40IW_ERR_RING_FULL;
1499         set_64bit_val(wqe, 16, ceq->elem_cnt);
1500         set_64bit_val(wqe, 32, (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));
1501         set_64bit_val(wqe, 48, (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));
1502         set_64bit_val(wqe, 56, LS_64(ceq->tph_val, I40IW_CQPSQ_TPHVAL));
1503
1504         header = ceq->ceq_id |
1505                  LS_64(I40IW_CQP_OP_CREATE_CEQ, I40IW_CQPSQ_OPCODE) |
1506                  LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |
1507                  LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) |
1508                  LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) |
1509                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1510
1511         i40iw_insert_wqe_hdr(wqe, header);
1512
1513         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_CREATE WQE",
1514                         wqe, I40IW_CQP_WQE_SIZE * 8);
1515
1516         if (post_sq)
1517                 i40iw_sc_cqp_post_sq(cqp);
1518         return 0;
1519 }
1520
1521 /**
1522  * i40iw_sc_cceq_create_done - poll for control ceq wqe to complete
1523  * @ceq: ceq sc structure
1524  */
1525 static enum i40iw_status_code i40iw_sc_cceq_create_done(struct i40iw_sc_ceq *ceq)
1526 {
1527         struct i40iw_sc_cqp *cqp;
1528
1529         cqp = ceq->dev->cqp;
1530         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CEQ, NULL);
1531 }
1532
1533 /**
1534  * i40iw_sc_cceq_destroy_done - poll for destroy cceq to complete
1535  * @ceq: ceq sc structure
1536  */
1537 static enum i40iw_status_code i40iw_sc_cceq_destroy_done(struct i40iw_sc_ceq *ceq)
1538 {
1539         struct i40iw_sc_cqp *cqp;
1540
1541         cqp = ceq->dev->cqp;
1542         cqp->process_cqp_sds = i40iw_update_sds_noccq;
1543         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_CEQ, NULL);
1544 }
1545
1546 /**
1547  * i40iw_sc_cceq_create - create cceq
1548  * @ceq: ceq sc structure
1549  * @scratch: u64 saved to be used during cqp completion
1550  */
1551 static enum i40iw_status_code i40iw_sc_cceq_create(struct i40iw_sc_ceq *ceq, u64 scratch)
1552 {
1553         enum i40iw_status_code ret_code;
1554
1555         ret_code = i40iw_sc_ceq_create(ceq, scratch, true);
1556         if (!ret_code)
1557                 ret_code = i40iw_sc_cceq_create_done(ceq);
1558         return ret_code;
1559 }
1560
1561 /**
1562  * i40iw_sc_ceq_destroy - destroy ceq
1563  * @ceq: ceq sc structure
1564  * @scratch: u64 saved to be used during cqp completion
1565  * @post_sq: flag for cqp db to ring
1566  */
1567 static enum i40iw_status_code i40iw_sc_ceq_destroy(struct i40iw_sc_ceq *ceq,
1568                                                    u64 scratch,
1569                                                    bool post_sq)
1570 {
1571         struct i40iw_sc_cqp *cqp;
1572         u64 *wqe;
1573         u64 header;
1574
1575         cqp = ceq->dev->cqp;
1576         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1577         if (!wqe)
1578                 return I40IW_ERR_RING_FULL;
1579         set_64bit_val(wqe, 16, ceq->elem_cnt);
1580         set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);
1581         header = ceq->ceq_id |
1582                  LS_64(I40IW_CQP_OP_DESTROY_CEQ, I40IW_CQPSQ_OPCODE) |
1583                  LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |
1584                  LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) |
1585                  LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) |
1586                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1587         i40iw_insert_wqe_hdr(wqe, header);
1588         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_DESTROY WQE",
1589                         wqe, I40IW_CQP_WQE_SIZE * 8);
1590
1591         if (post_sq)
1592                 i40iw_sc_cqp_post_sq(cqp);
1593         return 0;
1594 }
1595
1596 /**
1597  * i40iw_sc_process_ceq - process ceq
1598  * @dev: sc device struct
1599  * @ceq: ceq sc structure
1600  */
1601 static void *i40iw_sc_process_ceq(struct i40iw_sc_dev *dev, struct i40iw_sc_ceq *ceq)
1602 {
1603         u64 temp;
1604         u64 *ceqe;
1605         struct i40iw_sc_cq *cq = NULL;
1606         u8 polarity;
1607
1608         ceqe = (u64 *)I40IW_GET_CURRENT_CEQ_ELEMENT(ceq);
1609         get_64bit_val(ceqe, 0, &temp);
1610         polarity = (u8)RS_64(temp, I40IW_CEQE_VALID);
1611         if (polarity != ceq->polarity)
1612                 return cq;
1613
1614         cq = (struct i40iw_sc_cq *)(unsigned long)LS_64_1(temp, 1);
1615
1616         I40IW_RING_MOVE_TAIL(ceq->ceq_ring);
1617         if (I40IW_RING_GETCURRENT_TAIL(ceq->ceq_ring) == 0)
1618                 ceq->polarity ^= 1;
1619
1620         if (dev->is_pf)
1621                 i40iw_wr32(dev->hw, I40E_PFPE_CQACK, cq->cq_uk.cq_id);
1622         else
1623                 i40iw_wr32(dev->hw, I40E_VFPE_CQACK1, cq->cq_uk.cq_id);
1624
1625         return cq;
1626 }
1627
1628 /**
1629  * i40iw_sc_aeq_init - initialize aeq
1630  * @aeq: aeq structure ptr
1631  * @info: aeq initialization info
1632  */
1633 static enum i40iw_status_code i40iw_sc_aeq_init(struct i40iw_sc_aeq *aeq,
1634                                                 struct i40iw_aeq_init_info *info)
1635 {
1636         u32 pble_obj_cnt;
1637
1638         if ((info->elem_cnt < I40IW_MIN_AEQ_ENTRIES) ||
1639             (info->elem_cnt > I40IW_MAX_AEQ_ENTRIES))
1640                 return I40IW_ERR_INVALID_SIZE;
1641         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
1642
1643         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
1644                 return I40IW_ERR_INVALID_PBLE_INDEX;
1645
1646         aeq->size = sizeof(*aeq);
1647         aeq->polarity = 1;
1648         aeq->aeqe_base = (struct i40iw_sc_aeqe *)info->aeqe_base;
1649         aeq->dev = info->dev;
1650         aeq->elem_cnt = info->elem_cnt;
1651
1652         aeq->aeq_elem_pa = info->aeq_elem_pa;
1653         I40IW_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);
1654         info->dev->aeq = aeq;
1655
1656         aeq->virtual_map = info->virtual_map;
1657         aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);
1658         aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);
1659         aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);
1660         info->dev->aeq = aeq;
1661         return 0;
1662 }
1663
1664 /**
1665  * i40iw_sc_aeq_create - create aeq
1666  * @aeq: aeq structure ptr
1667  * @scratch: u64 saved to be used during cqp completion
1668  * @post_sq: flag for cqp db to ring
1669  */
1670 static enum i40iw_status_code i40iw_sc_aeq_create(struct i40iw_sc_aeq *aeq,
1671                                                   u64 scratch,
1672                                                   bool post_sq)
1673 {
1674         u64 *wqe;
1675         struct i40iw_sc_cqp *cqp;
1676         u64 header;
1677
1678         cqp = aeq->dev->cqp;
1679         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1680         if (!wqe)
1681                 return I40IW_ERR_RING_FULL;
1682         set_64bit_val(wqe, 16, aeq->elem_cnt);
1683         set_64bit_val(wqe, 32,
1684                       (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));
1685         set_64bit_val(wqe, 48,
1686                       (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));
1687
1688         header = LS_64(I40IW_CQP_OP_CREATE_AEQ, I40IW_CQPSQ_OPCODE) |
1689                  LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |
1690                  LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) |
1691                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1692
1693         i40iw_insert_wqe_hdr(wqe, header);
1694         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_CREATE WQE",
1695                         wqe, I40IW_CQP_WQE_SIZE * 8);
1696         if (post_sq)
1697                 i40iw_sc_cqp_post_sq(cqp);
1698         return 0;
1699 }
1700
1701 /**
1702  * i40iw_sc_aeq_destroy - destroy aeq during close
1703  * @aeq: aeq structure ptr
1704  * @scratch: u64 saved to be used during cqp completion
1705  * @post_sq: flag for cqp db to ring
1706  */
1707 static enum i40iw_status_code i40iw_sc_aeq_destroy(struct i40iw_sc_aeq *aeq,
1708                                                    u64 scratch,
1709                                                    bool post_sq)
1710 {
1711         u64 *wqe;
1712         struct i40iw_sc_cqp *cqp;
1713         u64 header;
1714
1715         cqp = aeq->dev->cqp;
1716         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1717         if (!wqe)
1718                 return I40IW_ERR_RING_FULL;
1719         set_64bit_val(wqe, 16, aeq->elem_cnt);
1720         set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx);
1721         header = LS_64(I40IW_CQP_OP_DESTROY_AEQ, I40IW_CQPSQ_OPCODE) |
1722                  LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |
1723                  LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) |
1724                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1725         i40iw_insert_wqe_hdr(wqe, header);
1726
1727         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_DESTROY WQE",
1728                         wqe, I40IW_CQP_WQE_SIZE * 8);
1729         if (post_sq)
1730                 i40iw_sc_cqp_post_sq(cqp);
1731         return 0;
1732 }
1733
1734 /**
1735  * i40iw_sc_get_next_aeqe - get next aeq entry
1736  * @aeq: aeq structure ptr
1737  * @info: aeqe info to be returned
1738  */
1739 static enum i40iw_status_code i40iw_sc_get_next_aeqe(struct i40iw_sc_aeq *aeq,
1740                                                      struct i40iw_aeqe_info *info)
1741 {
1742         u64 temp, compl_ctx;
1743         u64 *aeqe;
1744         u16 wqe_idx;
1745         u8 ae_src;
1746         u8 polarity;
1747
1748         aeqe = (u64 *)I40IW_GET_CURRENT_AEQ_ELEMENT(aeq);
1749         get_64bit_val(aeqe, 0, &compl_ctx);
1750         get_64bit_val(aeqe, 8, &temp);
1751         polarity = (u8)RS_64(temp, I40IW_AEQE_VALID);
1752
1753         if (aeq->polarity != polarity)
1754                 return I40IW_ERR_QUEUE_EMPTY;
1755
1756         i40iw_debug_buf(aeq->dev, I40IW_DEBUG_WQE, "AEQ_ENTRY", aeqe, 16);
1757
1758         ae_src = (u8)RS_64(temp, I40IW_AEQE_AESRC);
1759         wqe_idx = (u16)RS_64(temp, I40IW_AEQE_WQDESCIDX);
1760         info->qp_cq_id = (u32)RS_64(temp, I40IW_AEQE_QPCQID);
1761         info->ae_id = (u16)RS_64(temp, I40IW_AEQE_AECODE);
1762         info->tcp_state = (u8)RS_64(temp, I40IW_AEQE_TCPSTATE);
1763         info->iwarp_state = (u8)RS_64(temp, I40IW_AEQE_IWSTATE);
1764         info->q2_data_written = (u8)RS_64(temp, I40IW_AEQE_Q2DATA);
1765         info->aeqe_overflow = (bool)RS_64(temp, I40IW_AEQE_OVERFLOW);
1766         switch (ae_src) {
1767         case I40IW_AE_SOURCE_RQ:
1768         case I40IW_AE_SOURCE_RQ_0011:
1769                 info->qp = true;
1770                 info->wqe_idx = wqe_idx;
1771                 info->compl_ctx = compl_ctx;
1772                 break;
1773         case I40IW_AE_SOURCE_CQ:
1774         case I40IW_AE_SOURCE_CQ_0110:
1775         case I40IW_AE_SOURCE_CQ_1010:
1776         case I40IW_AE_SOURCE_CQ_1110:
1777                 info->cq = true;
1778                 info->compl_ctx = LS_64_1(compl_ctx, 1);
1779                 break;
1780         case I40IW_AE_SOURCE_SQ:
1781         case I40IW_AE_SOURCE_SQ_0111:
1782                 info->qp = true;
1783                 info->sq = true;
1784                 info->wqe_idx = wqe_idx;
1785                 info->compl_ctx = compl_ctx;
1786                 break;
1787         case I40IW_AE_SOURCE_IN_RR_WR:
1788         case I40IW_AE_SOURCE_IN_RR_WR_1011:
1789                 info->qp = true;
1790                 info->compl_ctx = compl_ctx;
1791                 info->in_rdrsp_wr = true;
1792                 break;
1793         case I40IW_AE_SOURCE_OUT_RR:
1794         case I40IW_AE_SOURCE_OUT_RR_1111:
1795                 info->qp = true;
1796                 info->compl_ctx = compl_ctx;
1797                 info->out_rdrsp = true;
1798                 break;
1799         default:
1800                 break;
1801         }
1802         I40IW_RING_MOVE_TAIL(aeq->aeq_ring);
1803         if (I40IW_RING_GETCURRENT_TAIL(aeq->aeq_ring) == 0)
1804                 aeq->polarity ^= 1;
1805         return 0;
1806 }
1807
1808 /**
1809  * i40iw_sc_repost_aeq_entries - repost completed aeq entries
1810  * @dev: sc device struct
1811  * @count: allocate count
1812  */
1813 static enum i40iw_status_code i40iw_sc_repost_aeq_entries(struct i40iw_sc_dev *dev,
1814                                                           u32 count)
1815 {
1816         if (count > I40IW_MAX_AEQ_ALLOCATE_COUNT)
1817                 return I40IW_ERR_INVALID_SIZE;
1818
1819         if (dev->is_pf)
1820                 i40iw_wr32(dev->hw, I40E_PFPE_AEQALLOC, count);
1821         else
1822                 i40iw_wr32(dev->hw, I40E_VFPE_AEQALLOC1, count);
1823
1824         return 0;
1825 }
1826
1827 /**
1828  * i40iw_sc_aeq_create_done - create aeq
1829  * @aeq: aeq structure ptr
1830  */
1831 static enum i40iw_status_code i40iw_sc_aeq_create_done(struct i40iw_sc_aeq *aeq)
1832 {
1833         struct i40iw_sc_cqp *cqp;
1834
1835         cqp = aeq->dev->cqp;
1836         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_AEQ, NULL);
1837 }
1838
1839 /**
1840  * i40iw_sc_aeq_destroy_done - destroy of aeq during close
1841  * @aeq: aeq structure ptr
1842  */
1843 static enum i40iw_status_code i40iw_sc_aeq_destroy_done(struct i40iw_sc_aeq *aeq)
1844 {
1845         struct i40iw_sc_cqp *cqp;
1846
1847         cqp = aeq->dev->cqp;
1848         return  i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_AEQ, NULL);
1849 }
1850
1851 /**
1852  * i40iw_sc_ccq_init - initialize control cq
1853  * @cq: sc's cq ctruct
1854  * @info: info for control cq initialization
1855  */
1856 static enum i40iw_status_code i40iw_sc_ccq_init(struct i40iw_sc_cq *cq,
1857                                                 struct i40iw_ccq_init_info *info)
1858 {
1859         u32 pble_obj_cnt;
1860
1861         if (info->num_elem < I40IW_MIN_CQ_SIZE || info->num_elem > I40IW_MAX_CQ_SIZE)
1862                 return I40IW_ERR_INVALID_SIZE;
1863
1864         if (info->ceq_id > I40IW_MAX_CEQID)
1865                 return I40IW_ERR_INVALID_CEQ_ID;
1866
1867         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
1868
1869         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
1870                 return I40IW_ERR_INVALID_PBLE_INDEX;
1871
1872         cq->cq_pa = info->cq_pa;
1873         cq->cq_uk.cq_base = info->cq_base;
1874         cq->shadow_area_pa = info->shadow_area_pa;
1875         cq->cq_uk.shadow_area = info->shadow_area;
1876         cq->shadow_read_threshold = info->shadow_read_threshold;
1877         cq->dev = info->dev;
1878         cq->ceq_id = info->ceq_id;
1879         cq->cq_uk.cq_size = info->num_elem;
1880         cq->cq_type = I40IW_CQ_TYPE_CQP;
1881         cq->ceqe_mask = info->ceqe_mask;
1882         I40IW_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);
1883
1884         cq->cq_uk.cq_id = 0;    /* control cq is id 0 always */
1885         cq->ceq_id_valid = info->ceq_id_valid;
1886         cq->tph_en = info->tph_en;
1887         cq->tph_val = info->tph_val;
1888         cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;
1889
1890         cq->pbl_list = info->pbl_list;
1891         cq->virtual_map = info->virtual_map;
1892         cq->pbl_chunk_size = info->pbl_chunk_size;
1893         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
1894         cq->cq_uk.polarity = true;
1895
1896         /* following are only for iw cqs so initialize them to zero */
1897         cq->cq_uk.cqe_alloc_reg = NULL;
1898         info->dev->ccq = cq;
1899         return 0;
1900 }
1901
1902 /**
1903  * i40iw_sc_ccq_create_done - poll cqp for ccq create
1904  * @ccq: ccq sc struct
1905  */
1906 static enum i40iw_status_code i40iw_sc_ccq_create_done(struct i40iw_sc_cq *ccq)
1907 {
1908         struct i40iw_sc_cqp *cqp;
1909
1910         cqp = ccq->dev->cqp;
1911         return  i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CQ, NULL);
1912 }
1913
1914 /**
1915  * i40iw_sc_ccq_create - create control cq
1916  * @ccq: ccq sc struct
1917  * @scratch: u64 saved to be used during cqp completion
1918  * @check_overflow: overlow flag for ccq
1919  * @post_sq: flag for cqp db to ring
1920  */
1921 static enum i40iw_status_code i40iw_sc_ccq_create(struct i40iw_sc_cq *ccq,
1922                                                   u64 scratch,
1923                                                   bool check_overflow,
1924                                                   bool post_sq)
1925 {
1926         u64 *wqe;
1927         struct i40iw_sc_cqp *cqp;
1928         u64 header;
1929         enum i40iw_status_code ret_code;
1930
1931         cqp = ccq->dev->cqp;
1932         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1933         if (!wqe)
1934                 return I40IW_ERR_RING_FULL;
1935         set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
1936         set_64bit_val(wqe, 8, RS_64_1(ccq, 1));
1937         set_64bit_val(wqe, 16,
1938                       LS_64(ccq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
1939         set_64bit_val(wqe, 32, (ccq->virtual_map ? 0 : ccq->cq_pa));
1940         set_64bit_val(wqe, 40, ccq->shadow_area_pa);
1941         set_64bit_val(wqe, 48,
1942                       (ccq->virtual_map ? ccq->first_pm_pbl_idx : 0));
1943         set_64bit_val(wqe, 56,
1944                       LS_64(ccq->tph_val, I40IW_CQPSQ_TPHVAL));
1945
1946         header = ccq->cq_uk.cq_id |
1947                  LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
1948                  LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
1949                  LS_64(ccq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
1950                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
1951                  LS_64(ccq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
1952                  LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
1953                  LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
1954                  LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) |
1955                  LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
1956                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1957
1958         i40iw_insert_wqe_hdr(wqe, header);
1959
1960         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_CREATE WQE",
1961                         wqe, I40IW_CQP_WQE_SIZE * 8);
1962
1963         if (post_sq) {
1964                 i40iw_sc_cqp_post_sq(cqp);
1965                 ret_code = i40iw_sc_ccq_create_done(ccq);
1966                 if (ret_code)
1967                         return ret_code;
1968         }
1969         cqp->process_cqp_sds = i40iw_cqp_sds_cmd;
1970
1971         return 0;
1972 }
1973
1974 /**
1975  * i40iw_sc_ccq_destroy - destroy ccq during close
1976  * @ccq: ccq sc struct
1977  * @scratch: u64 saved to be used during cqp completion
1978  * @post_sq: flag for cqp db to ring
1979  */
1980 static enum i40iw_status_code i40iw_sc_ccq_destroy(struct i40iw_sc_cq *ccq,
1981                                                    u64 scratch,
1982                                                    bool post_sq)
1983 {
1984         struct i40iw_sc_cqp *cqp;
1985         u64 *wqe;
1986         u64 header;
1987         enum i40iw_status_code ret_code = 0;
1988         u32 tail, val, error;
1989
1990         cqp = ccq->dev->cqp;
1991         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1992         if (!wqe)
1993                 return I40IW_ERR_RING_FULL;
1994         set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
1995         set_64bit_val(wqe, 8, RS_64_1(ccq, 1));
1996         set_64bit_val(wqe, 40, ccq->shadow_area_pa);
1997
1998         header = ccq->cq_uk.cq_id |
1999                  LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2000                  LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |
2001                  LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2002                  LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2003                  LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) |
2004                  LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2005                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2006
2007         i40iw_insert_wqe_hdr(wqe, header);
2008
2009         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_DESTROY WQE",
2010                         wqe, I40IW_CQP_WQE_SIZE * 8);
2011
2012         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
2013         if (error)
2014                 return I40IW_ERR_CQP_COMPL_ERROR;
2015
2016         if (post_sq) {
2017                 i40iw_sc_cqp_post_sq(cqp);
2018                 ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
2019         }
2020
2021         cqp->process_cqp_sds = i40iw_update_sds_noccq;
2022
2023         return ret_code;
2024 }
2025
2026 /**
2027  * i40iw_sc_cq_init - initialize completion q
2028  * @cq: cq struct
2029  * @info: cq initialization info
2030  */
2031 static enum i40iw_status_code i40iw_sc_cq_init(struct i40iw_sc_cq *cq,
2032                                                struct i40iw_cq_init_info *info)
2033 {
2034         u32 __iomem *cqe_alloc_reg = NULL;
2035         enum i40iw_status_code ret_code;
2036         u32 pble_obj_cnt;
2037         u32 arm_offset;
2038
2039         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2040
2041         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
2042                 return I40IW_ERR_INVALID_PBLE_INDEX;
2043
2044         cq->cq_pa = info->cq_base_pa;
2045         cq->dev = info->dev;
2046         cq->ceq_id = info->ceq_id;
2047         arm_offset = (info->dev->is_pf) ? I40E_PFPE_CQARM : I40E_VFPE_CQARM1;
2048         if (i40iw_get_hw_addr(cq->dev))
2049                 cqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(cq->dev) +
2050                                               arm_offset);
2051         info->cq_uk_init_info.cqe_alloc_reg = cqe_alloc_reg;
2052         ret_code = i40iw_cq_uk_init(&cq->cq_uk, &info->cq_uk_init_info);
2053         if (ret_code)
2054                 return ret_code;
2055         cq->virtual_map = info->virtual_map;
2056         cq->pbl_chunk_size = info->pbl_chunk_size;
2057         cq->ceqe_mask = info->ceqe_mask;
2058         cq->cq_type = (info->type) ? info->type : I40IW_CQ_TYPE_IWARP;
2059
2060         cq->shadow_area_pa = info->shadow_area_pa;
2061         cq->shadow_read_threshold = info->shadow_read_threshold;
2062
2063         cq->ceq_id_valid = info->ceq_id_valid;
2064         cq->tph_en = info->tph_en;
2065         cq->tph_val = info->tph_val;
2066
2067         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2068
2069         return 0;
2070 }
2071
2072 /**
2073  * i40iw_sc_cq_create - create completion q
2074  * @cq: cq struct
2075  * @scratch: u64 saved to be used during cqp completion
2076  * @check_overflow: flag for overflow check
2077  * @post_sq: flag for cqp db to ring
2078  */
2079 static enum i40iw_status_code i40iw_sc_cq_create(struct i40iw_sc_cq *cq,
2080                                                  u64 scratch,
2081                                                  bool check_overflow,
2082                                                  bool post_sq)
2083 {
2084         u64 *wqe;
2085         struct i40iw_sc_cqp *cqp;
2086         u64 header;
2087
2088         if (cq->cq_uk.cq_id > I40IW_MAX_CQID)
2089                 return I40IW_ERR_INVALID_CQ_ID;
2090
2091         if (cq->ceq_id > I40IW_MAX_CEQID)
2092                 return I40IW_ERR_INVALID_CEQ_ID;
2093
2094         cqp = cq->dev->cqp;
2095         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2096         if (!wqe)
2097                 return I40IW_ERR_RING_FULL;
2098
2099         set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2100         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2101         set_64bit_val(wqe,
2102                       16,
2103                       LS_64(cq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
2104
2105         set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
2106
2107         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2108         set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2109         set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
2110
2111         header = cq->cq_uk.cq_id |
2112                  LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2113                  LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
2114                  LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2115                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
2116                  LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2117                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2118                  LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2119                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2120                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2121                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2122
2123         i40iw_insert_wqe_hdr(wqe, header);
2124
2125         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_CREATE WQE",
2126                         wqe, I40IW_CQP_WQE_SIZE * 8);
2127
2128         if (post_sq)
2129                 i40iw_sc_cqp_post_sq(cqp);
2130         return 0;
2131 }
2132
2133 /**
2134  * i40iw_sc_cq_destroy - destroy completion q
2135  * @cq: cq struct
2136  * @scratch: u64 saved to be used during cqp completion
2137  * @post_sq: flag for cqp db to ring
2138  */
2139 static enum i40iw_status_code i40iw_sc_cq_destroy(struct i40iw_sc_cq *cq,
2140                                                   u64 scratch,
2141                                                   bool post_sq)
2142 {
2143         struct i40iw_sc_cqp *cqp;
2144         u64 *wqe;
2145         u64 header;
2146
2147         cqp = cq->dev->cqp;
2148         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2149         if (!wqe)
2150                 return I40IW_ERR_RING_FULL;
2151         set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2152         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2153         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2154         set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2155
2156         header = cq->cq_uk.cq_id |
2157                  LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2158                  LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |
2159                  LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2160                  LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2161                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2162                  LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2163                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2164                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2165                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2166
2167         i40iw_insert_wqe_hdr(wqe, header);
2168
2169         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_DESTROY WQE",
2170                         wqe, I40IW_CQP_WQE_SIZE * 8);
2171
2172         if (post_sq)
2173                 i40iw_sc_cqp_post_sq(cqp);
2174         return 0;
2175 }
2176
2177 /**
2178  * i40iw_sc_cq_modify - modify a Completion Queue
2179  * @cq: cq struct
2180  * @info: modification info struct
2181  * @scratch:
2182  * @post_sq: flag to post to sq
2183  */
2184 static enum i40iw_status_code i40iw_sc_cq_modify(struct i40iw_sc_cq *cq,
2185                                                  struct i40iw_modify_cq_info *info,
2186                                                  u64 scratch,
2187                                                  bool post_sq)
2188 {
2189         struct i40iw_sc_cqp *cqp;
2190         u64 *wqe;
2191         u64 header;
2192         u32 cq_size, ceq_id, first_pm_pbl_idx;
2193         u8 pbl_chunk_size;
2194         bool virtual_map, ceq_id_valid, check_overflow;
2195         u32 pble_obj_cnt;
2196
2197         if (info->ceq_valid && (info->ceq_id > I40IW_MAX_CEQID))
2198                 return I40IW_ERR_INVALID_CEQ_ID;
2199
2200         pble_obj_cnt = cq->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2201
2202         if (info->cq_resize && info->virtual_map &&
2203             (info->first_pm_pbl_idx >= pble_obj_cnt))
2204                 return I40IW_ERR_INVALID_PBLE_INDEX;
2205
2206         cqp = cq->dev->cqp;
2207         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2208         if (!wqe)
2209                 return I40IW_ERR_RING_FULL;
2210
2211         cq->pbl_list = info->pbl_list;
2212         cq->cq_pa = info->cq_pa;
2213         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2214
2215         cq_size = info->cq_resize ? info->cq_size : cq->cq_uk.cq_size;
2216         if (info->ceq_change) {
2217                 ceq_id_valid = true;
2218                 ceq_id = info->ceq_id;
2219         } else {
2220                 ceq_id_valid = cq->ceq_id_valid;
2221                 ceq_id = ceq_id_valid ? cq->ceq_id : 0;
2222         }
2223         virtual_map = info->cq_resize ? info->virtual_map : cq->virtual_map;
2224         first_pm_pbl_idx = (info->cq_resize ?
2225                             (info->virtual_map ? info->first_pm_pbl_idx : 0) :
2226                             (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2227         pbl_chunk_size = (info->cq_resize ?
2228                           (info->virtual_map ? info->pbl_chunk_size : 0) :
2229                           (cq->virtual_map ? cq->pbl_chunk_size : 0));
2230         check_overflow = info->check_overflow_change ? info->check_overflow :
2231                          cq->check_overflow;
2232         cq->cq_uk.cq_size = cq_size;
2233         cq->ceq_id_valid = ceq_id_valid;
2234         cq->ceq_id = ceq_id;
2235         cq->virtual_map = virtual_map;
2236         cq->first_pm_pbl_idx = first_pm_pbl_idx;
2237         cq->pbl_chunk_size = pbl_chunk_size;
2238         cq->check_overflow = check_overflow;
2239
2240         set_64bit_val(wqe, 0, cq_size);
2241         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2242         set_64bit_val(wqe, 16,
2243                       LS_64(info->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
2244         set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
2245         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2246         set_64bit_val(wqe, 48, (cq->virtual_map ? first_pm_pbl_idx : 0));
2247         set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
2248
2249         header = cq->cq_uk.cq_id |
2250                  LS_64(ceq_id, I40IW_CQPSQ_CQ_CEQID) |
2251                  LS_64(I40IW_CQP_OP_MODIFY_CQ, I40IW_CQPSQ_OPCODE) |
2252                  LS_64(info->cq_resize, I40IW_CQPSQ_CQ_CQRESIZE) |
2253                  LS_64(pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2254                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
2255                  LS_64(virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2256                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2257                  LS_64(ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2258                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2259                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2260                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2261
2262         i40iw_insert_wqe_hdr(wqe, header);
2263
2264         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_MODIFY WQE",
2265                         wqe, I40IW_CQP_WQE_SIZE * 8);
2266
2267         if (post_sq)
2268                 i40iw_sc_cqp_post_sq(cqp);
2269         return 0;
2270 }
2271
2272 /**
2273  * i40iw_sc_qp_init - initialize qp
2274  * @qp: sc qp
2275  * @info: initialization qp info
2276  */
2277 static enum i40iw_status_code i40iw_sc_qp_init(struct i40iw_sc_qp *qp,
2278                                                struct i40iw_qp_init_info *info)
2279 {
2280         u32 __iomem *wqe_alloc_reg = NULL;
2281         enum i40iw_status_code ret_code;
2282         u32 pble_obj_cnt;
2283         u8 wqe_size;
2284         u32 offset;
2285
2286         qp->dev = info->pd->dev;
2287         qp->vsi = info->vsi;
2288         qp->sq_pa = info->sq_pa;
2289         qp->rq_pa = info->rq_pa;
2290         qp->hw_host_ctx_pa = info->host_ctx_pa;
2291         qp->q2_pa = info->q2_pa;
2292         qp->shadow_area_pa = info->shadow_area_pa;
2293
2294         qp->q2_buf = info->q2;
2295         qp->pd = info->pd;
2296         qp->hw_host_ctx = info->host_ctx;
2297         offset = (qp->pd->dev->is_pf) ? I40E_PFPE_WQEALLOC : I40E_VFPE_WQEALLOC1;
2298         if (i40iw_get_hw_addr(qp->pd->dev))
2299                 wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) +
2300                                               offset);
2301
2302         info->qp_uk_init_info.wqe_alloc_reg = wqe_alloc_reg;
2303         info->qp_uk_init_info.abi_ver = qp->pd->abi_ver;
2304         ret_code = i40iw_qp_uk_init(&qp->qp_uk, &info->qp_uk_init_info);
2305         if (ret_code)
2306                 return ret_code;
2307         qp->virtual_map = info->virtual_map;
2308
2309         pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2310
2311         if ((info->virtual_map && (info->sq_pa >= pble_obj_cnt)) ||
2312             (info->virtual_map && (info->rq_pa >= pble_obj_cnt)))
2313                 return I40IW_ERR_INVALID_PBLE_INDEX;
2314
2315         qp->llp_stream_handle = (void *)(-1);
2316         qp->qp_type = (info->type) ? info->type : I40IW_QP_TYPE_IWARP;
2317
2318         qp->hw_sq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.sq_ring.size,
2319                                                     false);
2320         i40iw_debug(qp->dev, I40IW_DEBUG_WQE, "%s: hw_sq_size[%04d] sq_ring.size[%04d]\n",
2321                     __func__, qp->hw_sq_size, qp->qp_uk.sq_ring.size);
2322
2323         switch (qp->pd->abi_ver) {
2324         case 4:
2325                 ret_code = i40iw_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,
2326                                                        &wqe_size);
2327                 if (ret_code)
2328                         return ret_code;
2329                 break;
2330         case 5: /* fallthrough until next ABI version */
2331         default:
2332                 if (qp->qp_uk.max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT)
2333                         return I40IW_ERR_INVALID_FRAG_COUNT;
2334                 wqe_size = I40IW_MAX_WQE_SIZE_RQ;
2335                 break;
2336         }
2337         qp->hw_rq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.rq_size *
2338                                 (wqe_size / I40IW_QP_WQE_MIN_SIZE), false);
2339         i40iw_debug(qp->dev, I40IW_DEBUG_WQE,
2340                     "%s: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
2341                     __func__, qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);
2342         qp->sq_tph_val = info->sq_tph_val;
2343         qp->rq_tph_val = info->rq_tph_val;
2344         qp->sq_tph_en = info->sq_tph_en;
2345         qp->rq_tph_en = info->rq_tph_en;
2346         qp->rcv_tph_en = info->rcv_tph_en;
2347         qp->xmit_tph_en = info->xmit_tph_en;
2348         qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
2349         qp->exception_lan_queue = qp->pd->dev->exception_lan_queue;
2350
2351         return 0;
2352 }
2353
2354 /**
2355  * i40iw_sc_qp_create - create qp
2356  * @qp: sc qp
2357  * @info: qp create info
2358  * @scratch: u64 saved to be used during cqp completion
2359  * @post_sq: flag for cqp db to ring
2360  */
2361 static enum i40iw_status_code i40iw_sc_qp_create(
2362                                 struct i40iw_sc_qp *qp,
2363                                 struct i40iw_create_qp_info *info,
2364                                 u64 scratch,
2365                                 bool post_sq)
2366 {
2367         struct i40iw_sc_cqp *cqp;
2368         u64 *wqe;
2369         u64 header;
2370
2371         if ((qp->qp_uk.qp_id < I40IW_MIN_IW_QP_ID) ||
2372             (qp->qp_uk.qp_id > I40IW_MAX_IW_QP_ID))
2373                 return I40IW_ERR_INVALID_QP_ID;
2374
2375         cqp = qp->pd->dev->cqp;
2376         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2377         if (!wqe)
2378                 return I40IW_ERR_RING_FULL;
2379
2380         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2381
2382         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2383
2384         header = qp->qp_uk.qp_id |
2385                  LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) |
2386                  LS_64((info->ord_valid ? 1 : 0), I40IW_CQPSQ_QP_ORDVALID) |
2387                  LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |
2388                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2389                  LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) |
2390                  LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |
2391                  LS_64(info->static_rsrc, I40IW_CQPSQ_QP_STATRSRC) |
2392                  LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |
2393                  LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |
2394                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2395
2396         i40iw_insert_wqe_hdr(wqe, header);
2397         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_CREATE WQE",
2398                         wqe, I40IW_CQP_WQE_SIZE * 8);
2399
2400         if (post_sq)
2401                 i40iw_sc_cqp_post_sq(cqp);
2402         return 0;
2403 }
2404
2405 /**
2406  * i40iw_sc_qp_modify - modify qp cqp wqe
2407  * @qp: sc qp
2408  * @info: modify qp info
2409  * @scratch: u64 saved to be used during cqp completion
2410  * @post_sq: flag for cqp db to ring
2411  */
2412 static enum i40iw_status_code i40iw_sc_qp_modify(
2413                                 struct i40iw_sc_qp *qp,
2414                                 struct i40iw_modify_qp_info *info,
2415                                 u64 scratch,
2416                                 bool post_sq)
2417 {
2418         u64 *wqe;
2419         struct i40iw_sc_cqp *cqp;
2420         u64 header;
2421         u8 term_actions = 0;
2422         u8 term_len = 0;
2423
2424         cqp = qp->pd->dev->cqp;
2425         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2426         if (!wqe)
2427                 return I40IW_ERR_RING_FULL;
2428         if (info->next_iwarp_state == I40IW_QP_STATE_TERMINATE) {
2429                 if (info->dont_send_fin)
2430                         term_actions += I40IWQP_TERM_SEND_TERM_ONLY;
2431                 if (info->dont_send_term)
2432                         term_actions += I40IWQP_TERM_SEND_FIN_ONLY;
2433                 if ((term_actions == I40IWQP_TERM_SEND_TERM_AND_FIN) ||
2434                     (term_actions == I40IWQP_TERM_SEND_TERM_ONLY))
2435                         term_len = info->termlen;
2436         }
2437
2438         set_64bit_val(wqe,
2439                       8,
2440                       LS_64(term_len, I40IW_CQPSQ_QP_TERMLEN));
2441
2442         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2443         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2444
2445         header = qp->qp_uk.qp_id |
2446                  LS_64(I40IW_CQP_OP_MODIFY_QP, I40IW_CQPSQ_OPCODE) |
2447                  LS_64(info->ord_valid, I40IW_CQPSQ_QP_ORDVALID) |
2448                  LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |
2449                  LS_64(info->cached_var_valid, I40IW_CQPSQ_QP_CACHEDVARVALID) |
2450                  LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) |
2451                  LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |
2452                  LS_64(info->force_loopback, I40IW_CQPSQ_QP_FORCELOOPBACK) |
2453                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2454                  LS_64(info->static_rsrc, I40IW_CQPSQ_QP_STATRSRC) |
2455                  LS_64(info->remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |
2456                  LS_64(term_actions, I40IW_CQPSQ_QP_TERMACT) |
2457                  LS_64(info->reset_tcp_conn, I40IW_CQPSQ_QP_RESETCON) |
2458                  LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |
2459                  LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |
2460                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2461
2462         i40iw_insert_wqe_hdr(wqe, header);
2463
2464         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_MODIFY WQE",
2465                         wqe, I40IW_CQP_WQE_SIZE * 8);
2466
2467         if (post_sq)
2468                 i40iw_sc_cqp_post_sq(cqp);
2469         return 0;
2470 }
2471
2472 /**
2473  * i40iw_sc_qp_destroy - cqp destroy qp
2474  * @qp: sc qp
2475  * @scratch: u64 saved to be used during cqp completion
2476  * @remove_hash_idx: flag if to remove hash idx
2477  * @ignore_mw_bnd: memory window bind flag
2478  * @post_sq: flag for cqp db to ring
2479  */
2480 static enum i40iw_status_code i40iw_sc_qp_destroy(
2481                                         struct i40iw_sc_qp *qp,
2482                                         u64 scratch,
2483                                         bool remove_hash_idx,
2484                                         bool ignore_mw_bnd,
2485                                         bool post_sq)
2486 {
2487         u64 *wqe;
2488         struct i40iw_sc_cqp *cqp;
2489         u64 header;
2490
2491         i40iw_qp_rem_qos(qp);
2492         cqp = qp->pd->dev->cqp;
2493         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2494         if (!wqe)
2495                 return I40IW_ERR_RING_FULL;
2496         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2497         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2498
2499         header = qp->qp_uk.qp_id |
2500                  LS_64(I40IW_CQP_OP_DESTROY_QP, I40IW_CQPSQ_OPCODE) |
2501                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2502                  LS_64(ignore_mw_bnd, I40IW_CQPSQ_QP_IGNOREMWBOUND) |
2503                  LS_64(remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |
2504                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2505
2506         i40iw_insert_wqe_hdr(wqe, header);
2507         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_DESTROY WQE",
2508                         wqe, I40IW_CQP_WQE_SIZE * 8);
2509
2510         if (post_sq)
2511                 i40iw_sc_cqp_post_sq(cqp);
2512         return 0;
2513 }
2514
2515 /**
2516  * i40iw_sc_qp_flush_wqes - flush qp's wqe
2517  * @qp: sc qp
2518  * @info: dlush information
2519  * @scratch: u64 saved to be used during cqp completion
2520  * @post_sq: flag for cqp db to ring
2521  */
2522 static enum i40iw_status_code i40iw_sc_qp_flush_wqes(
2523                                 struct i40iw_sc_qp *qp,
2524                                 struct i40iw_qp_flush_info *info,
2525                                 u64 scratch,
2526                                 bool post_sq)
2527 {
2528         u64 temp = 0;
2529         u64 *wqe;
2530         struct i40iw_sc_cqp *cqp;
2531         u64 header;
2532         bool flush_sq = false, flush_rq = false;
2533
2534         if (info->rq && !qp->flush_rq)
2535                 flush_rq = true;
2536
2537         if (info->sq && !qp->flush_sq)
2538                 flush_sq = true;
2539
2540         qp->flush_sq |= flush_sq;
2541         qp->flush_rq |= flush_rq;
2542         if (!flush_sq && !flush_rq) {
2543                 if (info->ae_code != I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR)
2544                         return 0;
2545         }
2546
2547         cqp = qp->pd->dev->cqp;
2548         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2549         if (!wqe)
2550                 return I40IW_ERR_RING_FULL;
2551         if (info->userflushcode) {
2552                 if (flush_rq) {
2553                         temp |= LS_64(info->rq_minor_code, I40IW_CQPSQ_FWQE_RQMNERR) |
2554                                 LS_64(info->rq_major_code, I40IW_CQPSQ_FWQE_RQMJERR);
2555                 }
2556                 if (flush_sq) {
2557                         temp |= LS_64(info->sq_minor_code, I40IW_CQPSQ_FWQE_SQMNERR) |
2558                                 LS_64(info->sq_major_code, I40IW_CQPSQ_FWQE_SQMJERR);
2559                 }
2560         }
2561         set_64bit_val(wqe, 16, temp);
2562
2563         temp = (info->generate_ae) ?
2564                 info->ae_code | LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE) : 0;
2565
2566         set_64bit_val(wqe, 8, temp);
2567
2568         header = qp->qp_uk.qp_id |
2569                  LS_64(I40IW_CQP_OP_FLUSH_WQES, I40IW_CQPSQ_OPCODE) |
2570                  LS_64(info->generate_ae, I40IW_CQPSQ_FWQE_GENERATE_AE) |
2571                  LS_64(info->userflushcode, I40IW_CQPSQ_FWQE_USERFLCODE) |
2572                  LS_64(flush_sq, I40IW_CQPSQ_FWQE_FLUSHSQ) |
2573                  LS_64(flush_rq, I40IW_CQPSQ_FWQE_FLUSHRQ) |
2574                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2575
2576         i40iw_insert_wqe_hdr(wqe, header);
2577
2578         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_FLUSH WQE",
2579                         wqe, I40IW_CQP_WQE_SIZE * 8);
2580
2581         if (post_sq)
2582                 i40iw_sc_cqp_post_sq(cqp);
2583         return 0;
2584 }
2585
2586 /**
2587  * i40iw_sc_qp_upload_context - upload qp's context
2588  * @dev: sc device struct
2589  * @info: upload context info ptr for return
2590  * @scratch: u64 saved to be used during cqp completion
2591  * @post_sq: flag for cqp db to ring
2592  */
2593 static enum i40iw_status_code i40iw_sc_qp_upload_context(
2594                                         struct i40iw_sc_dev *dev,
2595                                         struct i40iw_upload_context_info *info,
2596                                         u64 scratch,
2597                                         bool post_sq)
2598 {
2599         u64 *wqe;
2600         struct i40iw_sc_cqp *cqp;
2601         u64 header;
2602
2603         cqp = dev->cqp;
2604         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2605         if (!wqe)
2606                 return I40IW_ERR_RING_FULL;
2607         set_64bit_val(wqe, 16, info->buf_pa);
2608
2609         header = LS_64(info->qp_id, I40IW_CQPSQ_UCTX_QPID) |
2610                  LS_64(I40IW_CQP_OP_UPLOAD_CONTEXT, I40IW_CQPSQ_OPCODE) |
2611                  LS_64(info->qp_type, I40IW_CQPSQ_UCTX_QPTYPE) |
2612                  LS_64(info->raw_format, I40IW_CQPSQ_UCTX_RAWFORMAT) |
2613                  LS_64(info->freeze_qp, I40IW_CQPSQ_UCTX_FREEZEQP) |
2614                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2615
2616         i40iw_insert_wqe_hdr(wqe, header);
2617
2618         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QP_UPLOAD_CTX WQE",
2619                         wqe, I40IW_CQP_WQE_SIZE * 8);
2620
2621         if (post_sq)
2622                 i40iw_sc_cqp_post_sq(cqp);
2623         return 0;
2624 }
2625
2626 /**
2627  * i40iw_sc_qp_setctx - set qp's context
2628  * @qp: sc qp
2629  * @qp_ctx: context ptr
2630  * @info: ctx info
2631  */
2632 static enum i40iw_status_code i40iw_sc_qp_setctx(
2633                                 struct i40iw_sc_qp *qp,
2634                                 u64 *qp_ctx,
2635                                 struct i40iw_qp_host_ctx_info *info)
2636 {
2637         struct i40iwarp_offload_info *iw;
2638         struct i40iw_tcp_offload_info *tcp;
2639         struct i40iw_sc_vsi *vsi;
2640         struct i40iw_sc_dev *dev;
2641         u64 qw0, qw3, qw7 = 0;
2642
2643         iw = info->iwarp_info;
2644         tcp = info->tcp_info;
2645         vsi = qp->vsi;
2646         dev = qp->dev;
2647         if (info->add_to_qoslist) {
2648                 qp->user_pri = info->user_pri;
2649                 i40iw_qp_add_qos(qp);
2650                 i40iw_debug(qp->dev, I40IW_DEBUG_DCB, "%s qp[%d] UP[%d] qset[%d]\n",
2651                             __func__, qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle);
2652         }
2653         qw0 = LS_64(qp->qp_uk.rq_wqe_size, I40IWQPC_RQWQESIZE) |
2654               LS_64(info->err_rq_idx_valid, I40IWQPC_ERR_RQ_IDX_VALID) |
2655               LS_64(qp->rcv_tph_en, I40IWQPC_RCVTPHEN) |
2656               LS_64(qp->xmit_tph_en, I40IWQPC_XMITTPHEN) |
2657               LS_64(qp->rq_tph_en, I40IWQPC_RQTPHEN) |
2658               LS_64(qp->sq_tph_en, I40IWQPC_SQTPHEN) |
2659               LS_64(info->push_idx, I40IWQPC_PPIDX) |
2660               LS_64(info->push_mode_en, I40IWQPC_PMENA);
2661
2662         set_64bit_val(qp_ctx, 8, qp->sq_pa);
2663         set_64bit_val(qp_ctx, 16, qp->rq_pa);
2664
2665         qw3 = LS_64(qp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |
2666               LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) |
2667               LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE);
2668
2669         set_64bit_val(qp_ctx,
2670                       128,
2671                       LS_64(info->err_rq_idx, I40IWQPC_ERR_RQ_IDX));
2672
2673         set_64bit_val(qp_ctx,
2674                       136,
2675                       LS_64(info->send_cq_num, I40IWQPC_TXCQNUM) |
2676                       LS_64(info->rcv_cq_num, I40IWQPC_RXCQNUM));
2677
2678         set_64bit_val(qp_ctx,
2679                       168,
2680                       LS_64(info->qp_compl_ctx, I40IWQPC_QPCOMPCTX));
2681         set_64bit_val(qp_ctx,
2682                       176,
2683                       LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) |
2684                       LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) |
2685                       LS_64(qp->qs_handle, I40IWQPC_QSHANDLE) |
2686                       LS_64(qp->exception_lan_queue, I40IWQPC_EXCEPTION_LAN_QUEUE));
2687
2688         if (info->iwarp_info_valid) {
2689                 qw0 |= LS_64(iw->ddp_ver, I40IWQPC_DDP_VER) |
2690                        LS_64(iw->rdmap_ver, I40IWQPC_RDMAP_VER);
2691
2692                 qw7 |= LS_64(iw->pd_id, I40IWQPC_PDIDX);
2693                 set_64bit_val(qp_ctx,
2694                               144,
2695                               LS_64(qp->q2_pa, I40IWQPC_Q2ADDR) |
2696                               LS_64(vsi->fcn_id, I40IWQPC_STAT_INDEX));
2697                 set_64bit_val(qp_ctx,
2698                               152,
2699                               LS_64(iw->last_byte_sent, I40IWQPC_LASTBYTESENT));
2700
2701                 set_64bit_val(qp_ctx,
2702                               160,
2703                               LS_64(iw->ord_size, I40IWQPC_ORDSIZE) |
2704                               LS_64(iw->ird_size, I40IWQPC_IRDSIZE) |
2705                               LS_64(iw->wr_rdresp_en, I40IWQPC_WRRDRSPOK) |
2706                               LS_64(iw->rd_enable, I40IWQPC_RDOK) |
2707                               LS_64(iw->snd_mark_en, I40IWQPC_SNDMARKERS) |
2708                               LS_64(iw->bind_en, I40IWQPC_BINDEN) |
2709                               LS_64(iw->fast_reg_en, I40IWQPC_FASTREGEN) |
2710                               LS_64(iw->priv_mode_en, I40IWQPC_PRIVEN) |
2711                               LS_64((((vsi->stats_fcn_id_alloc) &&
2712                                       (dev->is_pf) && (vsi->fcn_id >= I40IW_FIRST_NON_PF_STAT)) ? 1 : 0),
2713                                     I40IWQPC_USESTATSINSTANCE) |
2714                               LS_64(1, I40IWQPC_IWARPMODE) |
2715                               LS_64(iw->rcv_mark_en, I40IWQPC_RCVMARKERS) |
2716                               LS_64(iw->align_hdrs, I40IWQPC_ALIGNHDRS) |
2717                               LS_64(iw->rcv_no_mpa_crc, I40IWQPC_RCVNOMPACRC) |
2718                               LS_64(iw->rcv_mark_offset, I40IWQPC_RCVMARKOFFSET) |
2719                               LS_64(iw->snd_mark_offset, I40IWQPC_SNDMARKOFFSET));
2720         }
2721         if (info->tcp_info_valid) {
2722                 qw0 |= LS_64(tcp->ipv4, I40IWQPC_IPV4) |
2723                        LS_64(tcp->no_nagle, I40IWQPC_NONAGLE) |
2724                        LS_64(tcp->insert_vlan_tag, I40IWQPC_INSERTVLANTAG) |
2725                        LS_64(tcp->time_stamp, I40IWQPC_TIMESTAMP) |
2726                        LS_64(tcp->cwnd_inc_limit, I40IWQPC_LIMIT) |
2727                        LS_64(tcp->drop_ooo_seg, I40IWQPC_DROPOOOSEG) |
2728                        LS_64(tcp->dup_ack_thresh, I40IWQPC_DUPACK_THRESH);
2729
2730                 qw3 |= LS_64(tcp->ttl, I40IWQPC_TTL) |
2731                        LS_64(tcp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |
2732                        LS_64(tcp->avoid_stretch_ack, I40IWQPC_AVOIDSTRETCHACK) |
2733                        LS_64(tcp->tos, I40IWQPC_TOS) |
2734                        LS_64(tcp->src_port, I40IWQPC_SRCPORTNUM) |
2735                        LS_64(tcp->dst_port, I40IWQPC_DESTPORTNUM);
2736
2737                 qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
2738                 set_64bit_val(qp_ctx,
2739                               32,
2740                               LS_64(tcp->dest_ip_addr2, I40IWQPC_DESTIPADDR2) |
2741                               LS_64(tcp->dest_ip_addr3, I40IWQPC_DESTIPADDR3));
2742
2743                 set_64bit_val(qp_ctx,
2744                               40,
2745                               LS_64(tcp->dest_ip_addr0, I40IWQPC_DESTIPADDR0) |
2746                               LS_64(tcp->dest_ip_addr1, I40IWQPC_DESTIPADDR1));
2747
2748                 set_64bit_val(qp_ctx,
2749                               48,
2750                               LS_64(tcp->snd_mss, I40IWQPC_SNDMSS) |
2751                                 LS_64(tcp->vlan_tag, I40IWQPC_VLANTAG) |
2752                                 LS_64(tcp->arp_idx, I40IWQPC_ARPIDX));
2753
2754                 qw7 |= LS_64(tcp->flow_label, I40IWQPC_FLOWLABEL) |
2755                        LS_64(tcp->wscale, I40IWQPC_WSCALE) |
2756                        LS_64(tcp->ignore_tcp_opt, I40IWQPC_IGNORE_TCP_OPT) |
2757                        LS_64(tcp->ignore_tcp_uns_opt, I40IWQPC_IGNORE_TCP_UNS_OPT) |
2758                        LS_64(tcp->tcp_state, I40IWQPC_TCPSTATE) |
2759                        LS_64(tcp->rcv_wscale, I40IWQPC_RCVSCALE) |
2760                        LS_64(tcp->snd_wscale, I40IWQPC_SNDSCALE);
2761
2762                 set_64bit_val(qp_ctx,
2763                               72,
2764                               LS_64(tcp->time_stamp_recent, I40IWQPC_TIMESTAMP_RECENT) |
2765                               LS_64(tcp->time_stamp_age, I40IWQPC_TIMESTAMP_AGE));
2766                 set_64bit_val(qp_ctx,
2767                               80,
2768                               LS_64(tcp->snd_nxt, I40IWQPC_SNDNXT) |
2769                               LS_64(tcp->snd_wnd, I40IWQPC_SNDWND));
2770
2771                 set_64bit_val(qp_ctx,
2772                               88,
2773                               LS_64(tcp->rcv_nxt, I40IWQPC_RCVNXT) |
2774                               LS_64(tcp->rcv_wnd, I40IWQPC_RCVWND));
2775                 set_64bit_val(qp_ctx,
2776                               96,
2777                               LS_64(tcp->snd_max, I40IWQPC_SNDMAX) |
2778                               LS_64(tcp->snd_una, I40IWQPC_SNDUNA));
2779                 set_64bit_val(qp_ctx,
2780                               104,
2781                               LS_64(tcp->srtt, I40IWQPC_SRTT) |
2782                               LS_64(tcp->rtt_var, I40IWQPC_RTTVAR));
2783                 set_64bit_val(qp_ctx,
2784                               112,
2785                               LS_64(tcp->ss_thresh, I40IWQPC_SSTHRESH) |
2786                               LS_64(tcp->cwnd, I40IWQPC_CWND));
2787                 set_64bit_val(qp_ctx,
2788                               120,
2789                               LS_64(tcp->snd_wl1, I40IWQPC_SNDWL1) |
2790                               LS_64(tcp->snd_wl2, I40IWQPC_SNDWL2));
2791                 set_64bit_val(qp_ctx,
2792                               128,
2793                               LS_64(tcp->max_snd_window, I40IWQPC_MAXSNDWND) |
2794                               LS_64(tcp->rexmit_thresh, I40IWQPC_REXMIT_THRESH));
2795                 set_64bit_val(qp_ctx,
2796                               184,
2797                               LS_64(tcp->local_ipaddr3, I40IWQPC_LOCAL_IPADDR3) |
2798                               LS_64(tcp->local_ipaddr2, I40IWQPC_LOCAL_IPADDR2));
2799                 set_64bit_val(qp_ctx,
2800                               192,
2801                               LS_64(tcp->local_ipaddr1, I40IWQPC_LOCAL_IPADDR1) |
2802                               LS_64(tcp->local_ipaddr0, I40IWQPC_LOCAL_IPADDR0));
2803         }
2804
2805         set_64bit_val(qp_ctx, 0, qw0);
2806         set_64bit_val(qp_ctx, 24, qw3);
2807         set_64bit_val(qp_ctx, 56, qw7);
2808
2809         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "QP_HOST)CTX WQE",
2810                         qp_ctx, I40IW_QP_CTX_SIZE);
2811         return 0;
2812 }
2813
2814 /**
2815  * i40iw_sc_alloc_stag - mr stag alloc
2816  * @dev: sc device struct
2817  * @info: stag info
2818  * @scratch: u64 saved to be used during cqp completion
2819  * @post_sq: flag for cqp db to ring
2820  */
2821 static enum i40iw_status_code i40iw_sc_alloc_stag(
2822                                 struct i40iw_sc_dev *dev,
2823                                 struct i40iw_allocate_stag_info *info,
2824                                 u64 scratch,
2825                                 bool post_sq)
2826 {
2827         u64 *wqe;
2828         struct i40iw_sc_cqp *cqp;
2829         u64 header;
2830         enum i40iw_page_size page_size;
2831
2832         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
2833         cqp = dev->cqp;
2834         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2835         if (!wqe)
2836                 return I40IW_ERR_RING_FULL;
2837         set_64bit_val(wqe,
2838                       8,
2839                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID) |
2840                       LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN));
2841         set_64bit_val(wqe,
2842                       16,
2843                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
2844         set_64bit_val(wqe,
2845                       40,
2846                       LS_64(info->hmc_fcn_index, I40IW_CQPSQ_STAG_HMCFNIDX));
2847
2848         header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |
2849                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
2850                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
2851                  LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
2852                  LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
2853                  LS_64(info->remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
2854                  LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |
2855                  LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |
2856                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2857
2858         i40iw_insert_wqe_hdr(wqe, header);
2859
2860         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "ALLOC_STAG WQE",
2861                         wqe, I40IW_CQP_WQE_SIZE * 8);
2862
2863         if (post_sq)
2864                 i40iw_sc_cqp_post_sq(cqp);
2865         return 0;
2866 }
2867
2868 /**
2869  * i40iw_sc_mr_reg_non_shared - non-shared mr registration
2870  * @dev: sc device struct
2871  * @info: mr info
2872  * @scratch: u64 saved to be used during cqp completion
2873  * @post_sq: flag for cqp db to ring
2874  */
2875 static enum i40iw_status_code i40iw_sc_mr_reg_non_shared(
2876                                 struct i40iw_sc_dev *dev,
2877                                 struct i40iw_reg_ns_stag_info *info,
2878                                 u64 scratch,
2879                                 bool post_sq)
2880 {
2881         u64 *wqe;
2882         u64 temp;
2883         struct i40iw_sc_cqp *cqp;
2884         u64 header;
2885         u32 pble_obj_cnt;
2886         bool remote_access;
2887         u8 addr_type;
2888         enum i40iw_page_size page_size;
2889
2890         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
2891         if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |
2892                                    I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
2893                 remote_access = true;
2894         else
2895                 remote_access = false;
2896
2897         pble_obj_cnt = dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2898
2899         if (info->chunk_size && (info->first_pm_pbl_index >= pble_obj_cnt))
2900                 return I40IW_ERR_INVALID_PBLE_INDEX;
2901
2902         cqp = dev->cqp;
2903         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2904         if (!wqe)
2905                 return I40IW_ERR_RING_FULL;
2906
2907         temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
2908         set_64bit_val(wqe, 0, temp);
2909
2910         set_64bit_val(wqe,
2911                       8,
2912                       LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN) |
2913                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
2914
2915         set_64bit_val(wqe,
2916                       16,
2917                       LS_64(info->stag_key, I40IW_CQPSQ_STAG_KEY) |
2918                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
2919         if (!info->chunk_size) {
2920                 set_64bit_val(wqe, 32, info->reg_addr_pa);
2921                 set_64bit_val(wqe, 48, 0);
2922         } else {
2923                 set_64bit_val(wqe, 32, 0);
2924                 set_64bit_val(wqe, 48, info->first_pm_pbl_index);
2925         }
2926         set_64bit_val(wqe, 40, info->hmc_fcn_index);
2927         set_64bit_val(wqe, 56, 0);
2928
2929         addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
2930         header = LS_64(I40IW_CQP_OP_REG_MR, I40IW_CQPSQ_OPCODE) |
2931                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
2932                  LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
2933                  LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
2934                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
2935                  LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
2936                  LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |
2937                  LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |
2938                  LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |
2939                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2940
2941         i40iw_insert_wqe_hdr(wqe, header);
2942
2943         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_NS WQE",
2944                         wqe, I40IW_CQP_WQE_SIZE * 8);
2945
2946         if (post_sq)
2947                 i40iw_sc_cqp_post_sq(cqp);
2948         return 0;
2949 }
2950
2951 /**
2952  * i40iw_sc_mr_reg_shared - registered shared memory region
2953  * @dev: sc device struct
2954  * @info: info for shared memory registeration
2955  * @scratch: u64 saved to be used during cqp completion
2956  * @post_sq: flag for cqp db to ring
2957  */
2958 static enum i40iw_status_code i40iw_sc_mr_reg_shared(
2959                                         struct i40iw_sc_dev *dev,
2960                                         struct i40iw_register_shared_stag *info,
2961                                         u64 scratch,
2962                                         bool post_sq)
2963 {
2964         u64 *wqe;
2965         struct i40iw_sc_cqp *cqp;
2966         u64 temp, va64, fbo, header;
2967         u32 va32;
2968         bool remote_access;
2969         u8 addr_type;
2970
2971         if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |
2972                                    I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
2973                 remote_access = true;
2974         else
2975                 remote_access = false;
2976         cqp = dev->cqp;
2977         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2978         if (!wqe)
2979                 return I40IW_ERR_RING_FULL;
2980         va64 = (uintptr_t)(info->va);
2981         va32 = (u32)(va64 & 0x00000000FFFFFFFF);
2982         fbo = (u64)(va32 & (4096 - 1));
2983
2984         set_64bit_val(wqe,
2985                       0,
2986                       (info->addr_type == I40IW_ADDR_TYPE_VA_BASED ? (uintptr_t)info->va : fbo));
2987
2988         set_64bit_val(wqe,
2989                       8,
2990                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
2991         temp = LS_64(info->new_stag_key, I40IW_CQPSQ_STAG_KEY) |
2992                LS_64(info->new_stag_idx, I40IW_CQPSQ_STAG_IDX) |
2993                LS_64(info->parent_stag_idx, I40IW_CQPSQ_STAG_PARENTSTAGIDX);
2994         set_64bit_val(wqe, 16, temp);
2995
2996         addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
2997         header = LS_64(I40IW_CQP_OP_REG_SMR, I40IW_CQPSQ_OPCODE) |
2998                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
2999                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
3000                  LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
3001                  LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |
3002                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3003
3004         i40iw_insert_wqe_hdr(wqe, header);
3005
3006         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_SHARED WQE",
3007                         wqe, I40IW_CQP_WQE_SIZE * 8);
3008
3009         if (post_sq)
3010                 i40iw_sc_cqp_post_sq(cqp);
3011         return 0;
3012 }
3013
3014 /**
3015  * i40iw_sc_dealloc_stag - deallocate stag
3016  * @dev: sc device struct
3017  * @info: dealloc stag info
3018  * @scratch: u64 saved to be used during cqp completion
3019  * @post_sq: flag for cqp db to ring
3020  */
3021 static enum i40iw_status_code i40iw_sc_dealloc_stag(
3022                                         struct i40iw_sc_dev *dev,
3023                                         struct i40iw_dealloc_stag_info *info,
3024                                         u64 scratch,
3025                                         bool post_sq)
3026 {
3027         u64 header;
3028         u64 *wqe;
3029         struct i40iw_sc_cqp *cqp;
3030
3031         cqp = dev->cqp;
3032         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3033         if (!wqe)
3034                 return I40IW_ERR_RING_FULL;
3035         set_64bit_val(wqe,
3036                       8,
3037                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
3038         set_64bit_val(wqe,
3039                       16,
3040                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
3041
3042         header = LS_64(I40IW_CQP_OP_DEALLOC_STAG, I40IW_CQPSQ_OPCODE) |
3043                  LS_64(info->mr, I40IW_CQPSQ_STAG_MR) |
3044                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3045
3046         i40iw_insert_wqe_hdr(wqe, header);
3047
3048         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "DEALLOC_STAG WQE",
3049                         wqe, I40IW_CQP_WQE_SIZE * 8);
3050
3051         if (post_sq)
3052                 i40iw_sc_cqp_post_sq(cqp);
3053         return 0;
3054 }
3055
3056 /**
3057  * i40iw_sc_query_stag - query hardware for stag
3058  * @dev: sc device struct
3059  * @scratch: u64 saved to be used during cqp completion
3060  * @stag_index: stag index for query
3061  * @post_sq: flag for cqp db to ring
3062  */
3063 static enum i40iw_status_code i40iw_sc_query_stag(struct i40iw_sc_dev *dev,
3064                                                   u64 scratch,
3065                                                   u32 stag_index,
3066                                                   bool post_sq)
3067 {
3068         u64 header;
3069         u64 *wqe;
3070         struct i40iw_sc_cqp *cqp;
3071
3072         cqp = dev->cqp;
3073         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3074         if (!wqe)
3075                 return I40IW_ERR_RING_FULL;
3076         set_64bit_val(wqe,
3077                       16,
3078                       LS_64(stag_index, I40IW_CQPSQ_QUERYSTAG_IDX));
3079
3080         header = LS_64(I40IW_CQP_OP_QUERY_STAG, I40IW_CQPSQ_OPCODE) |
3081                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3082
3083         i40iw_insert_wqe_hdr(wqe, header);
3084
3085         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QUERY_STAG WQE",
3086                         wqe, I40IW_CQP_WQE_SIZE * 8);
3087
3088         if (post_sq)
3089                 i40iw_sc_cqp_post_sq(cqp);
3090         return 0;
3091 }
3092
3093 /**
3094  * i40iw_sc_mw_alloc - mw allocate
3095  * @dev: sc device struct
3096  * @scratch: u64 saved to be used during cqp completion
3097  * @mw_stag_index:stag index
3098  * @pd_id: pd is for this mw
3099  * @post_sq: flag for cqp db to ring
3100  */
3101 static enum i40iw_status_code i40iw_sc_mw_alloc(
3102                                         struct i40iw_sc_dev *dev,
3103                                         u64 scratch,
3104                                         u32 mw_stag_index,
3105                                         u16 pd_id,
3106                                         bool post_sq)
3107 {
3108         u64 header;
3109         struct i40iw_sc_cqp *cqp;
3110         u64 *wqe;
3111
3112         cqp = dev->cqp;
3113         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3114         if (!wqe)
3115                 return I40IW_ERR_RING_FULL;
3116         set_64bit_val(wqe, 8, LS_64(pd_id, I40IW_CQPSQ_STAG_PDID));
3117         set_64bit_val(wqe,
3118                       16,
3119                       LS_64(mw_stag_index, I40IW_CQPSQ_STAG_IDX));
3120
3121         header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |
3122                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3123
3124         i40iw_insert_wqe_hdr(wqe, header);
3125
3126         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MW_ALLOC WQE",
3127                         wqe, I40IW_CQP_WQE_SIZE * 8);
3128
3129         if (post_sq)
3130                 i40iw_sc_cqp_post_sq(cqp);
3131         return 0;
3132 }
3133
3134 /**
3135  * i40iw_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
3136  * @qp: sc qp struct
3137  * @info: fast mr info
3138  * @post_sq: flag for cqp db to ring
3139  */
3140 enum i40iw_status_code i40iw_sc_mr_fast_register(
3141                                 struct i40iw_sc_qp *qp,
3142                                 struct i40iw_fast_reg_stag_info *info,
3143                                 bool post_sq)
3144 {
3145         u64 temp, header;
3146         u64 *wqe;
3147         u32 wqe_idx;
3148         enum i40iw_page_size page_size;
3149
3150         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
3151         wqe = i40iw_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
3152                                          0, info->wr_id);
3153         if (!wqe)
3154                 return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
3155
3156         i40iw_debug(qp->dev, I40IW_DEBUG_MR, "%s: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
3157                     __func__, info->wr_id, wqe_idx,
3158                     &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
3159         temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
3160         set_64bit_val(wqe, 0, temp);
3161
3162         temp = RS_64(info->first_pm_pbl_index >> 16, I40IWQPSQ_FIRSTPMPBLIDXHI);
3163         set_64bit_val(wqe,
3164                       8,
3165                       LS_64(temp, I40IWQPSQ_FIRSTPMPBLIDXHI) |
3166                       LS_64(info->reg_addr_pa >> I40IWQPSQ_PBLADDR_SHIFT, I40IWQPSQ_PBLADDR));
3167
3168         set_64bit_val(wqe,
3169                       16,
3170                       info->total_len |
3171                       LS_64(info->first_pm_pbl_index, I40IWQPSQ_FIRSTPMPBLIDXLO));
3172
3173         header = LS_64(info->stag_key, I40IWQPSQ_STAGKEY) |
3174                  LS_64(info->stag_idx, I40IWQPSQ_STAGINDEX) |
3175                  LS_64(I40IWQP_OP_FAST_REGISTER, I40IWQPSQ_OPCODE) |
3176                  LS_64(info->chunk_size, I40IWQPSQ_LPBLSIZE) |
3177                  LS_64(page_size, I40IWQPSQ_HPAGESIZE) |
3178                  LS_64(info->access_rights, I40IWQPSQ_STAGRIGHTS) |
3179                  LS_64(info->addr_type, I40IWQPSQ_VABASEDTO) |
3180                  LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
3181                  LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
3182                  LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
3183                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3184
3185         i40iw_insert_wqe_hdr(wqe, header);
3186
3187         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "FAST_REG WQE",
3188                         wqe, I40IW_QP_WQE_MIN_SIZE);
3189
3190         if (post_sq)
3191                 i40iw_qp_post_wr(&qp->qp_uk);
3192         return 0;
3193 }
3194
3195 /**
3196  * i40iw_sc_send_lsmm - send last streaming mode message
3197  * @qp: sc qp struct
3198  * @lsmm_buf: buffer with lsmm message
3199  * @size: size of lsmm buffer
3200  * @stag: stag of lsmm buffer
3201  */
3202 static void i40iw_sc_send_lsmm(struct i40iw_sc_qp *qp,
3203                                void *lsmm_buf,
3204                                u32 size,
3205                                i40iw_stag stag)
3206 {
3207         u64 *wqe;
3208         u64 header;
3209         struct i40iw_qp_uk *qp_uk;
3210
3211         qp_uk = &qp->qp_uk;
3212         wqe = qp_uk->sq_base->elem;
3213
3214         set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
3215
3216         set_64bit_val(wqe, 8, (size | LS_64(stag, I40IWQPSQ_FRAG_STAG)));
3217
3218         set_64bit_val(wqe, 16, 0);
3219
3220         header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3221                  LS_64(1, I40IWQPSQ_STREAMMODE) |
3222                  LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |
3223                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3224
3225         i40iw_insert_wqe_hdr(wqe, header);
3226
3227         i40iw_debug_buf(qp->dev, I40IW_DEBUG_QP, "SEND_LSMM WQE",
3228                         wqe, I40IW_QP_WQE_MIN_SIZE);
3229 }
3230
3231 /**
3232  * i40iw_sc_send_lsmm_nostag - for privilege qp
3233  * @qp: sc qp struct
3234  * @lsmm_buf: buffer with lsmm message
3235  * @size: size of lsmm buffer
3236  */
3237 static void i40iw_sc_send_lsmm_nostag(struct i40iw_sc_qp *qp,
3238                                       void *lsmm_buf,
3239                                       u32 size)
3240 {
3241         u64 *wqe;
3242         u64 header;
3243         struct i40iw_qp_uk *qp_uk;
3244
3245         qp_uk = &qp->qp_uk;
3246         wqe = qp_uk->sq_base->elem;
3247
3248         set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
3249
3250         set_64bit_val(wqe, 8, size);
3251
3252         set_64bit_val(wqe, 16, 0);
3253
3254         header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3255                  LS_64(1, I40IWQPSQ_STREAMMODE) |
3256                  LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |
3257                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3258
3259         i40iw_insert_wqe_hdr(wqe, header);
3260
3261         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE",
3262                         wqe, I40IW_QP_WQE_MIN_SIZE);
3263 }
3264
3265 /**
3266  * i40iw_sc_send_rtt - send last read0 or write0
3267  * @qp: sc qp struct
3268  * @read: Do read0 or write0
3269  */
3270 static void i40iw_sc_send_rtt(struct i40iw_sc_qp *qp, bool read)
3271 {
3272         u64 *wqe;
3273         u64 header;
3274         struct i40iw_qp_uk *qp_uk;
3275
3276         qp_uk = &qp->qp_uk;
3277         wqe = qp_uk->sq_base->elem;
3278
3279         set_64bit_val(wqe, 0, 0);
3280         set_64bit_val(wqe, 8, 0);
3281         set_64bit_val(wqe, 16, 0);
3282         if (read) {
3283                 header = LS_64(0x1234, I40IWQPSQ_REMSTAG) |
3284                          LS_64(I40IWQP_OP_RDMA_READ, I40IWQPSQ_OPCODE) |
3285                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3286                 set_64bit_val(wqe, 8, ((u64)0xabcd << 32));
3287         } else {
3288                 header = LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |
3289                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3290         }
3291
3292         i40iw_insert_wqe_hdr(wqe, header);
3293
3294         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "RTR WQE",
3295                         wqe, I40IW_QP_WQE_MIN_SIZE);
3296 }
3297
3298 /**
3299  * i40iw_sc_post_wqe0 - send wqe with opcode
3300  * @qp: sc qp struct
3301  * @opcode: opcode to use for wqe0
3302  */
3303 static enum i40iw_status_code i40iw_sc_post_wqe0(struct i40iw_sc_qp *qp, u8 opcode)
3304 {
3305         u64 *wqe;
3306         u64 header;
3307         struct i40iw_qp_uk *qp_uk;
3308
3309         qp_uk = &qp->qp_uk;
3310         wqe = qp_uk->sq_base->elem;
3311
3312         if (!wqe)
3313                 return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
3314         switch (opcode) {
3315         case I40IWQP_OP_NOP:
3316                 set_64bit_val(wqe, 0, 0);
3317                 set_64bit_val(wqe, 8, 0);
3318                 set_64bit_val(wqe, 16, 0);
3319                 header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |
3320                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3321
3322                 i40iw_insert_wqe_hdr(wqe, header);
3323                 break;
3324         case I40IWQP_OP_RDMA_SEND:
3325                 set_64bit_val(wqe, 0, 0);
3326                 set_64bit_val(wqe, 8, 0);
3327                 set_64bit_val(wqe, 16, 0);
3328                 header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3329                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID) |
3330                          LS_64(1, I40IWQPSQ_STREAMMODE) |
3331                          LS_64(1, I40IWQPSQ_WAITFORRCVPDU);
3332
3333                 i40iw_insert_wqe_hdr(wqe, header);
3334                 break;
3335         default:
3336                 i40iw_debug(qp->dev, I40IW_DEBUG_QP, "%s: Invalid WQE zero opcode\n",
3337                             __func__);
3338                 break;
3339         }
3340         return 0;
3341 }
3342
3343 /**
3344  * i40iw_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
3345  * @dev : ptr to i40iw_dev struct
3346  * @hmc_fn_id: hmc function id
3347  */
3348 enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev, u8 hmc_fn_id)
3349 {
3350         struct i40iw_hmc_info *hmc_info;
3351         struct i40iw_dma_mem query_fpm_mem;
3352         struct i40iw_virt_mem virt_mem;
3353         struct i40iw_vfdev *vf_dev = NULL;
3354         u32 mem_size;
3355         enum i40iw_status_code ret_code = 0;
3356         bool poll_registers = true;
3357         u16 iw_vf_idx;
3358         u8 wait_type;
3359
3360         if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID ||
3361             (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
3362                 return I40IW_ERR_INVALID_HMCFN_ID;
3363
3364         i40iw_debug(dev, I40IW_DEBUG_HMC, "hmc_fn_id %u, dev->hmc_fn_id %u\n", hmc_fn_id,
3365                     dev->hmc_fn_id);
3366         if (hmc_fn_id == dev->hmc_fn_id) {
3367                 hmc_info = dev->hmc_info;
3368                 query_fpm_mem.pa = dev->fpm_query_buf_pa;
3369                 query_fpm_mem.va = dev->fpm_query_buf;
3370         } else {
3371                 vf_dev = i40iw_vfdev_from_fpm(dev, hmc_fn_id);
3372                 if (!vf_dev)
3373                         return I40IW_ERR_INVALID_VF_ID;
3374
3375                 hmc_info = &vf_dev->hmc_info;
3376                 iw_vf_idx = vf_dev->iw_vf_idx;
3377                 i40iw_debug(dev, I40IW_DEBUG_HMC, "vf_dev %p, hmc_info %p, hmc_obj %p\n", vf_dev,
3378                             hmc_info, hmc_info->hmc_obj);
3379                 if (!vf_dev->fpm_query_buf) {
3380                         if (!dev->vf_fpm_query_buf[iw_vf_idx].va) {
3381                                 ret_code = i40iw_alloc_query_fpm_buf(dev,
3382                                                                      &dev->vf_fpm_query_buf[iw_vf_idx]);
3383                                 if (ret_code)
3384                                         return ret_code;
3385                         }
3386                         vf_dev->fpm_query_buf = dev->vf_fpm_query_buf[iw_vf_idx].va;
3387                         vf_dev->fpm_query_buf_pa = dev->vf_fpm_query_buf[iw_vf_idx].pa;
3388                 }
3389                 query_fpm_mem.pa = vf_dev->fpm_query_buf_pa;
3390                 query_fpm_mem.va = vf_dev->fpm_query_buf;
3391                 /**
3392                  * It is HARDWARE specific:
3393                  * this call is done by PF for VF and
3394                  * i40iw_sc_query_fpm_values needs ccq poll
3395                  * because PF ccq is already created.
3396                  */
3397                 poll_registers = false;
3398         }
3399
3400         hmc_info->hmc_fn_id = hmc_fn_id;
3401
3402         if (hmc_fn_id != dev->hmc_fn_id) {
3403                 ret_code =
3404                         i40iw_cqp_query_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id);
3405         } else {
3406                 wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
3407                             (u8)I40IW_CQP_WAIT_POLL_CQ;
3408
3409                 ret_code = i40iw_sc_query_fpm_values(
3410                                         dev->cqp,
3411                                         0,
3412                                         hmc_info->hmc_fn_id,
3413                                         &query_fpm_mem,
3414                                         true,
3415                                         wait_type);
3416         }
3417         if (ret_code)
3418                 return ret_code;
3419
3420         /* parse the fpm_query_buf and fill hmc obj info */
3421         ret_code =
3422                 i40iw_sc_parse_fpm_query_buf((u64 *)query_fpm_mem.va,
3423                                              hmc_info,
3424                                              &dev->hmc_fpm_misc);
3425         if (ret_code)
3426                 return ret_code;
3427         i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "QUERY FPM BUFFER",
3428                         query_fpm_mem.va, I40IW_QUERY_FPM_BUF_SIZE);
3429
3430         if (hmc_fn_id != dev->hmc_fn_id) {
3431                 i40iw_cqp_commit_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id);
3432
3433                 /* parse the fpm_commit_buf and fill hmc obj info */
3434                 i40iw_sc_parse_fpm_commit_buf((u64 *)query_fpm_mem.va, hmc_info->hmc_obj, &hmc_info->sd_table.sd_cnt);
3435                 mem_size = sizeof(struct i40iw_hmc_sd_entry) *
3436                            (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index);
3437                 ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
3438                 if (ret_code)
3439                         return ret_code;
3440                 hmc_info->sd_table.sd_entry = virt_mem.va;
3441         }
3442
3443         return ret_code;
3444 }
3445
3446 /**
3447  * i40iw_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and
3448  * populates fpm base address in hmc_info
3449  * @dev : ptr to i40iw_dev struct
3450  * @hmc_fn_id: hmc function id
3451  */
3452 static enum i40iw_status_code i40iw_sc_configure_iw_fpm(struct i40iw_sc_dev *dev,
3453                                                         u8 hmc_fn_id)
3454 {
3455         struct i40iw_hmc_info *hmc_info;
3456         struct i40iw_hmc_obj_info *obj_info;
3457         u64 *buf;
3458         struct i40iw_dma_mem commit_fpm_mem;
3459         u32 i, j;
3460         enum i40iw_status_code ret_code = 0;
3461         bool poll_registers = true;
3462         u8 wait_type;
3463
3464         if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID ||
3465             (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
3466                 return I40IW_ERR_INVALID_HMCFN_ID;
3467
3468         if (hmc_fn_id == dev->hmc_fn_id) {
3469                 hmc_info = dev->hmc_info;
3470         } else {
3471                 hmc_info = i40iw_vf_hmcinfo_from_fpm(dev, hmc_fn_id);
3472                 poll_registers = false;
3473         }
3474         if (!hmc_info)
3475                 return I40IW_ERR_BAD_PTR;
3476
3477         obj_info = hmc_info->hmc_obj;
3478         buf = dev->fpm_commit_buf;
3479
3480         /* copy cnt values in commit buf */
3481         for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE;
3482              i++, j += 8)
3483                 set_64bit_val(buf, j, (u64)obj_info[i].cnt);
3484
3485         set_64bit_val(buf, 40, 0);   /* APBVT rsvd */
3486
3487         commit_fpm_mem.pa = dev->fpm_commit_buf_pa;
3488         commit_fpm_mem.va = dev->fpm_commit_buf;
3489         wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
3490                         (u8)I40IW_CQP_WAIT_POLL_CQ;
3491         ret_code = i40iw_sc_commit_fpm_values(
3492                                         dev->cqp,
3493                                         0,
3494                                         hmc_info->hmc_fn_id,
3495                                         &commit_fpm_mem,
3496                                         true,
3497                                         wait_type);
3498
3499         /* parse the fpm_commit_buf and fill hmc obj info */
3500         if (!ret_code)
3501                 ret_code = i40iw_sc_parse_fpm_commit_buf(dev->fpm_commit_buf,
3502                                                          hmc_info->hmc_obj,
3503                                                          &hmc_info->sd_table.sd_cnt);
3504
3505         i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "COMMIT FPM BUFFER",
3506                         commit_fpm_mem.va, I40IW_COMMIT_FPM_BUF_SIZE);
3507
3508         return ret_code;
3509 }
3510
3511 /**
3512  * cqp_sds_wqe_fill - fill cqp wqe doe sd
3513  * @cqp: struct for cqp hw
3514  * @info; sd info for wqe
3515  * @scratch: u64 saved to be used during cqp completion
3516  */
3517 static enum i40iw_status_code cqp_sds_wqe_fill(struct i40iw_sc_cqp *cqp,
3518                                                struct i40iw_update_sds_info *info,
3519                                                u64 scratch)
3520 {
3521         u64 data;
3522         u64 header;
3523         u64 *wqe;
3524         int mem_entries, wqe_entries;
3525         struct i40iw_dma_mem *sdbuf = &cqp->sdbuf;
3526
3527         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3528         if (!wqe)
3529                 return I40IW_ERR_RING_FULL;
3530
3531         I40IW_CQP_INIT_WQE(wqe);
3532         wqe_entries = (info->cnt > 3) ? 3 : info->cnt;
3533         mem_entries = info->cnt - wqe_entries;
3534
3535         header = LS_64(I40IW_CQP_OP_UPDATE_PE_SDS, I40IW_CQPSQ_OPCODE) |
3536                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
3537                  LS_64(mem_entries, I40IW_CQPSQ_UPESD_ENTRY_COUNT);
3538
3539         if (mem_entries) {
3540                 memcpy(sdbuf->va, &info->entry[3], (mem_entries << 4));
3541                 data = sdbuf->pa;
3542         } else {
3543                 data = 0;
3544         }
3545         data |= LS_64(info->hmc_fn_id, I40IW_CQPSQ_UPESD_HMCFNID);
3546
3547         set_64bit_val(wqe, 16, data);
3548
3549         switch (wqe_entries) {
3550         case 3:
3551                 set_64bit_val(wqe, 48,
3552                               (LS_64(info->entry[2].cmd, I40IW_CQPSQ_UPESD_SDCMD) |
3553                                         LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
3554
3555                 set_64bit_val(wqe, 56, info->entry[2].data);
3556                 /* fallthrough */
3557         case 2:
3558                 set_64bit_val(wqe, 32,
3559                               (LS_64(info->entry[1].cmd, I40IW_CQPSQ_UPESD_SDCMD) |
3560                                         LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
3561
3562                 set_64bit_val(wqe, 40, info->entry[1].data);
3563                 /* fallthrough */
3564         case 1:
3565                 set_64bit_val(wqe, 0,
3566                               LS_64(info->entry[0].cmd, I40IW_CQPSQ_UPESD_SDCMD));
3567
3568                 set_64bit_val(wqe, 8, info->entry[0].data);
3569                 break;
3570         default:
3571                 break;
3572         }
3573
3574         i40iw_insert_wqe_hdr(wqe, header);
3575
3576         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "UPDATE_PE_SDS WQE",
3577                         wqe, I40IW_CQP_WQE_SIZE * 8);
3578         return 0;
3579 }
3580
3581 /**
3582  * i40iw_update_pe_sds - cqp wqe for sd
3583  * @dev: ptr to i40iw_dev struct
3584  * @info: sd info for sd's
3585  * @scratch: u64 saved to be used during cqp completion
3586  */
3587 static enum i40iw_status_code i40iw_update_pe_sds(struct i40iw_sc_dev *dev,
3588                                                   struct i40iw_update_sds_info *info,
3589                                                   u64 scratch)
3590 {
3591         struct i40iw_sc_cqp *cqp = dev->cqp;
3592         enum i40iw_status_code ret_code;
3593
3594         ret_code = cqp_sds_wqe_fill(cqp, info, scratch);
3595         if (!ret_code)
3596                 i40iw_sc_cqp_post_sq(cqp);
3597
3598         return ret_code;
3599 }
3600
3601 /**
3602  * i40iw_update_sds_noccq - update sd before ccq created
3603  * @dev: sc device struct
3604  * @info: sd info for sd's
3605  */
3606 enum i40iw_status_code i40iw_update_sds_noccq(struct i40iw_sc_dev *dev,
3607                                               struct i40iw_update_sds_info *info)
3608 {
3609         u32 error, val, tail;
3610         struct i40iw_sc_cqp *cqp = dev->cqp;
3611         enum i40iw_status_code ret_code;
3612
3613         ret_code = cqp_sds_wqe_fill(cqp, info, 0);
3614         if (ret_code)
3615                 return ret_code;
3616         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
3617         if (error)
3618                 return I40IW_ERR_CQP_COMPL_ERROR;
3619
3620         i40iw_sc_cqp_post_sq(cqp);
3621         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
3622
3623         return ret_code;
3624 }
3625
3626 /**
3627  * i40iw_sc_suspend_qp - suspend qp for param change
3628  * @cqp: struct for cqp hw
3629  * @qp: sc qp struct
3630  * @scratch: u64 saved to be used during cqp completion
3631  */
3632 enum i40iw_status_code i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp,
3633                                            struct i40iw_sc_qp *qp,
3634                                            u64 scratch)
3635 {
3636         u64 header;
3637         u64 *wqe;
3638
3639         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3640         if (!wqe)
3641                 return I40IW_ERR_RING_FULL;
3642         header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_SUSPENDQP_QPID) |
3643                  LS_64(I40IW_CQP_OP_SUSPEND_QP, I40IW_CQPSQ_OPCODE) |
3644                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3645
3646         i40iw_insert_wqe_hdr(wqe, header);
3647
3648         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SUSPEND_QP WQE",
3649                         wqe, I40IW_CQP_WQE_SIZE * 8);
3650
3651         i40iw_sc_cqp_post_sq(cqp);
3652         return 0;
3653 }
3654
3655 /**
3656  * i40iw_sc_resume_qp - resume qp after suspend
3657  * @cqp: struct for cqp hw
3658  * @qp: sc qp struct
3659  * @scratch: u64 saved to be used during cqp completion
3660  */
3661 enum i40iw_status_code i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp,
3662                                           struct i40iw_sc_qp *qp,
3663                                           u64 scratch)
3664 {
3665         u64 header;
3666         u64 *wqe;
3667
3668         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3669         if (!wqe)
3670                 return I40IW_ERR_RING_FULL;
3671         set_64bit_val(wqe,
3672                       16,
3673                         LS_64(qp->qs_handle, I40IW_CQPSQ_RESUMEQP_QSHANDLE));
3674
3675         header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_RESUMEQP_QPID) |
3676                  LS_64(I40IW_CQP_OP_RESUME_QP, I40IW_CQPSQ_OPCODE) |
3677                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3678
3679         i40iw_insert_wqe_hdr(wqe, header);
3680
3681         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "RESUME_QP WQE",
3682                         wqe, I40IW_CQP_WQE_SIZE * 8);
3683
3684         i40iw_sc_cqp_post_sq(cqp);
3685         return 0;
3686 }
3687
3688 /**
3689  * i40iw_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
3690  * @cqp: struct for cqp hw
3691  * @scratch: u64 saved to be used during cqp completion
3692  * @hmc_fn_id: hmc function id
3693  * @post_sq: flag for cqp db to ring
3694  * @poll_registers: flag to poll register for cqp completion
3695  */
3696 enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated(
3697                                         struct i40iw_sc_cqp *cqp,
3698                                         u64 scratch,
3699                                         u8 hmc_fn_id,
3700                                         bool post_sq,
3701                                         bool poll_registers)
3702 {
3703         u64 header;
3704         u64 *wqe;
3705         u32 tail, val, error;
3706         enum i40iw_status_code ret_code = 0;
3707
3708         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3709         if (!wqe)
3710                 return I40IW_ERR_RING_FULL;
3711         set_64bit_val(wqe,
3712                       16,
3713                       LS_64(hmc_fn_id, I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID));
3714
3715         header = LS_64(I40IW_CQP_OP_SHMC_PAGES_ALLOCATED, I40IW_CQPSQ_OPCODE) |
3716                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3717
3718         i40iw_insert_wqe_hdr(wqe, header);
3719
3720         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SHMC_PAGES_ALLOCATED WQE",
3721                         wqe, I40IW_CQP_WQE_SIZE * 8);
3722         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
3723         if (error) {
3724                 ret_code = I40IW_ERR_CQP_COMPL_ERROR;
3725                 return ret_code;
3726         }
3727         if (post_sq) {
3728                 i40iw_sc_cqp_post_sq(cqp);
3729                 if (poll_registers)
3730                         /* check for cqp sq tail update */
3731                         ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
3732                 else
3733                         ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,
3734                                                                  I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,
3735                                                                  NULL);
3736         }
3737
3738         return ret_code;
3739 }
3740
3741 /**
3742  * i40iw_ring_full - check if cqp ring is full
3743  * @cqp: struct for cqp hw
3744  */
3745 static bool i40iw_ring_full(struct i40iw_sc_cqp *cqp)
3746 {
3747         return I40IW_RING_FULL_ERR(cqp->sq_ring);
3748 }
3749
3750 /**
3751  * i40iw_est_sd - returns approximate number of SDs for HMC
3752  * @dev: sc device struct
3753  * @hmc_info: hmc structure, size and count for HMC objects
3754  */
3755 static u64 i40iw_est_sd(struct i40iw_sc_dev *dev, struct i40iw_hmc_info *hmc_info)
3756 {
3757         int i;
3758         u64 size = 0;
3759         u64 sd;
3760
3761         for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_PBLE; i++)
3762                 size += hmc_info->hmc_obj[i].cnt * hmc_info->hmc_obj[i].size;
3763
3764         if (dev->is_pf)
3765                 size += hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
3766
3767         if (size & 0x1FFFFF)
3768                 sd = (size >> 21) + 1; /* add 1 for remainder */
3769         else
3770                 sd = size >> 21;
3771
3772         if (!dev->is_pf) {
3773                 /* 2MB alignment for VF PBLE HMC */
3774                 size = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
3775                 if (size & 0x1FFFFF)
3776                         sd += (size >> 21) + 1; /* add 1 for remainder */
3777                 else
3778                         sd += size >> 21;
3779         }
3780
3781         return sd;
3782 }
3783
3784 /**
3785  * i40iw_config_fpm_values - configure HMC objects
3786  * @dev: sc device struct
3787  * @qp_count: desired qp count
3788  */
3789 enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count)
3790 {
3791         struct i40iw_virt_mem virt_mem;
3792         u32 i, mem_size;
3793         u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted;
3794         u32 powerof2;
3795         u64 sd_needed;
3796         u32 loop_count = 0;
3797
3798         struct i40iw_hmc_info *hmc_info;
3799         struct i40iw_hmc_fpm_misc *hmc_fpm_misc;
3800         enum i40iw_status_code ret_code = 0;
3801
3802         hmc_info = dev->hmc_info;
3803         hmc_fpm_misc = &dev->hmc_fpm_misc;
3804
3805         ret_code = i40iw_sc_init_iw_hmc(dev, dev->hmc_fn_id);
3806         if (ret_code) {
3807                 i40iw_debug(dev, I40IW_DEBUG_HMC,
3808                             "i40iw_sc_init_iw_hmc returned error_code = %d\n",
3809                             ret_code);
3810                 return ret_code;
3811         }
3812
3813         for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++)
3814                 hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;
3815         sd_needed = i40iw_est_sd(dev, hmc_info);
3816         i40iw_debug(dev, I40IW_DEBUG_HMC,
3817                     "%s: FW initial max sd_count[%08lld] first_sd_index[%04d]\n",
3818                     __func__, sd_needed, hmc_info->first_sd_index);
3819         i40iw_debug(dev, I40IW_DEBUG_HMC,
3820                     "%s: sd count %d where max sd is %d\n",
3821                     __func__, hmc_info->sd_table.sd_cnt,
3822                     hmc_fpm_misc->max_sds);
3823
3824         qpwanted = min(qp_count, hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt);
3825         qpwantedoriginal = qpwanted;
3826         mrwanted = hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt;
3827         pblewanted = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt;
3828
3829         i40iw_debug(dev, I40IW_DEBUG_HMC,
3830                     "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d\n",
3831                     qp_count, hmc_fpm_misc->max_sds,
3832                     hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt,
3833                     hmc_info->hmc_obj[I40IW_HMC_IW_CQ].max_cnt,
3834                     hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt,
3835                     hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt);
3836
3837         do {
3838                 ++loop_count;
3839                 hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt = qpwanted;
3840                 hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt =
3841                         min(2 * qpwanted, hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt);
3842                 hmc_info->hmc_obj[I40IW_HMC_IW_SRQ].cnt = 0x00; /* Reserved */
3843                 hmc_info->hmc_obj[I40IW_HMC_IW_HTE].cnt =
3844                                         qpwanted * hmc_fpm_misc->ht_multiplier;
3845                 hmc_info->hmc_obj[I40IW_HMC_IW_ARP].cnt =
3846                         hmc_info->hmc_obj[I40IW_HMC_IW_ARP].max_cnt;
3847                 hmc_info->hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].cnt = 1;
3848                 hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt = mrwanted;
3849
3850                 hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt = I40IW_MAX_WQ_ENTRIES * qpwanted;
3851                 hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt = 4 * I40IW_MAX_IRD_SIZE * qpwanted;
3852                 hmc_info->hmc_obj[I40IW_HMC_IW_XFFL].cnt =
3853                         hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;
3854                 hmc_info->hmc_obj[I40IW_HMC_IW_Q1FL].cnt =
3855                         hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;
3856                 hmc_info->hmc_obj[I40IW_HMC_IW_TIMER].cnt =
3857                         ((qpwanted) / 512 + 1) * hmc_fpm_misc->timer_bucket;
3858                 hmc_info->hmc_obj[I40IW_HMC_IW_FSIMC].cnt = 0x00;
3859                 hmc_info->hmc_obj[I40IW_HMC_IW_FSIAV].cnt = 0x00;
3860                 hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt = pblewanted;
3861
3862                 /* How much memory is needed for all the objects. */
3863                 sd_needed = i40iw_est_sd(dev, hmc_info);
3864                 if ((loop_count > 1000) ||
3865                     ((!(loop_count % 10)) &&
3866                     (qpwanted > qpwantedoriginal * 2 / 3))) {
3867                         if (qpwanted > FPM_MULTIPLIER) {
3868                                 qpwanted -= FPM_MULTIPLIER;
3869                                 powerof2 = 1;
3870                                 while (powerof2 < qpwanted)
3871                                         powerof2 *= 2;
3872                                 powerof2 /= 2;
3873                                 qpwanted = powerof2;
3874                         } else {
3875                                 qpwanted /= 2;
3876                         }
3877                 }
3878                 if (mrwanted > FPM_MULTIPLIER * 10)
3879                         mrwanted -= FPM_MULTIPLIER * 10;
3880                 if (pblewanted > FPM_MULTIPLIER * 1000)
3881                         pblewanted -= FPM_MULTIPLIER * 1000;
3882         } while (sd_needed > hmc_fpm_misc->max_sds && loop_count < 2000);
3883
3884         sd_needed = i40iw_est_sd(dev, hmc_info);
3885
3886         i40iw_debug(dev, I40IW_DEBUG_HMC,
3887                     "loop_cnt=%d, sd_needed=%lld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n",
3888                     loop_count, sd_needed,
3889                     hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt,
3890                     hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt,
3891                     hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt,
3892                     hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt);
3893
3894         ret_code = i40iw_sc_configure_iw_fpm(dev, dev->hmc_fn_id);
3895         if (ret_code) {
3896                 i40iw_debug(dev, I40IW_DEBUG_HMC,
3897                             "configure_iw_fpm returned error_code[x%08X]\n",
3898                             i40iw_rd32(dev->hw, dev->is_pf ? I40E_PFPE_CQPERRCODES : I40E_VFPE_CQPERRCODES1));
3899                 return ret_code;
3900         }
3901
3902         mem_size = sizeof(struct i40iw_hmc_sd_entry) *
3903                    (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);
3904         ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
3905         if (ret_code) {
3906                 i40iw_debug(dev, I40IW_DEBUG_HMC,
3907                             "%s: failed to allocate memory for sd_entry buffer\n",
3908                             __func__);
3909                 return ret_code;
3910         }
3911         hmc_info->sd_table.sd_entry = virt_mem.va;
3912
3913         return ret_code;
3914 }
3915
3916 /**
3917  * i40iw_exec_cqp_cmd - execute cqp cmd when wqe are available
3918  * @dev: rdma device
3919  * @pcmdinfo: cqp command info
3920  */
3921 static enum i40iw_status_code i40iw_exec_cqp_cmd(struct i40iw_sc_dev *dev,
3922                                                  struct cqp_commands_info *pcmdinfo)
3923 {
3924         enum i40iw_status_code status;
3925         struct i40iw_dma_mem values_mem;
3926
3927         dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;
3928         switch (pcmdinfo->cqp_cmd) {
3929         case OP_DELETE_LOCAL_MAC_IPADDR_ENTRY:
3930                 status = i40iw_sc_del_local_mac_ipaddr_entry(
3931                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.cqp,
3932                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.scratch,
3933                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.entry_idx,
3934                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.ignore_ref_count,
3935                                 pcmdinfo->post_sq);
3936                 break;
3937         case OP_CEQ_DESTROY:
3938                 status = i40iw_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,
3939                                               pcmdinfo->in.u.ceq_destroy.scratch,
3940                                               pcmdinfo->post_sq);
3941                 break;
3942         case OP_AEQ_DESTROY:
3943                 status = i40iw_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,
3944                                               pcmdinfo->in.u.aeq_destroy.scratch,
3945                                               pcmdinfo->post_sq);
3946
3947                 break;
3948         case OP_DELETE_ARP_CACHE_ENTRY:
3949                 status = i40iw_sc_del_arp_cache_entry(
3950                                 pcmdinfo->in.u.del_arp_cache_entry.cqp,
3951                                 pcmdinfo->in.u.del_arp_cache_entry.scratch,
3952                                 pcmdinfo->in.u.del_arp_cache_entry.arp_index,
3953                                 pcmdinfo->post_sq);
3954                 break;
3955         case OP_MANAGE_APBVT_ENTRY:
3956                 status = i40iw_sc_manage_apbvt_entry(
3957                                 pcmdinfo->in.u.manage_apbvt_entry.cqp,
3958                                 &pcmdinfo->in.u.manage_apbvt_entry.info,
3959                                 pcmdinfo->in.u.manage_apbvt_entry.scratch,
3960                                 pcmdinfo->post_sq);
3961                 break;
3962         case OP_CEQ_CREATE:
3963                 status = i40iw_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,
3964                                              pcmdinfo->in.u.ceq_create.scratch,
3965                                              pcmdinfo->post_sq);
3966                 break;
3967         case OP_AEQ_CREATE:
3968                 status = i40iw_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,
3969                                              pcmdinfo->in.u.aeq_create.scratch,
3970                                              pcmdinfo->post_sq);
3971                 break;
3972         case OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY:
3973                 status = i40iw_sc_alloc_local_mac_ipaddr_entry(
3974                                 pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.cqp,
3975                                 pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.scratch,
3976                                 pcmdinfo->post_sq);
3977                 break;
3978         case OP_ADD_LOCAL_MAC_IPADDR_ENTRY:
3979                 status = i40iw_sc_add_local_mac_ipaddr_entry(
3980                                 pcmdinfo->in.u.add_local_mac_ipaddr_entry.cqp,
3981                                 &pcmdinfo->in.u.add_local_mac_ipaddr_entry.info,
3982                                 pcmdinfo->in.u.add_local_mac_ipaddr_entry.scratch,
3983                                 pcmdinfo->post_sq);
3984                 break;
3985         case OP_MANAGE_QHASH_TABLE_ENTRY:
3986                 status = i40iw_sc_manage_qhash_table_entry(
3987                                 pcmdinfo->in.u.manage_qhash_table_entry.cqp,
3988                                 &pcmdinfo->in.u.manage_qhash_table_entry.info,
3989                                 pcmdinfo->in.u.manage_qhash_table_entry.scratch,
3990                                 pcmdinfo->post_sq);
3991
3992                 break;
3993         case OP_QP_MODIFY:
3994                 status = i40iw_sc_qp_modify(
3995                                 pcmdinfo->in.u.qp_modify.qp,
3996                                 &pcmdinfo->in.u.qp_modify.info,
3997                                 pcmdinfo->in.u.qp_modify.scratch,
3998                                 pcmdinfo->post_sq);
3999
4000                 break;
4001         case OP_QP_UPLOAD_CONTEXT:
4002                 status = i40iw_sc_qp_upload_context(
4003                                 pcmdinfo->in.u.qp_upload_context.dev,
4004                                 &pcmdinfo->in.u.qp_upload_context.info,
4005                                 pcmdinfo->in.u.qp_upload_context.scratch,
4006                                 pcmdinfo->post_sq);
4007
4008                 break;
4009         case OP_CQ_CREATE:
4010                 status = i40iw_sc_cq_create(
4011                                 pcmdinfo->in.u.cq_create.cq,
4012                                 pcmdinfo->in.u.cq_create.scratch,
4013                                 pcmdinfo->in.u.cq_create.check_overflow,
4014                                 pcmdinfo->post_sq);
4015                 break;
4016         case OP_CQ_DESTROY:
4017                 status = i40iw_sc_cq_destroy(
4018                                 pcmdinfo->in.u.cq_destroy.cq,
4019                                 pcmdinfo->in.u.cq_destroy.scratch,
4020                                 pcmdinfo->post_sq);
4021
4022                 break;
4023         case OP_QP_CREATE:
4024                 status = i40iw_sc_qp_create(
4025                                 pcmdinfo->in.u.qp_create.qp,
4026                                 &pcmdinfo->in.u.qp_create.info,
4027                                 pcmdinfo->in.u.qp_create.scratch,
4028                                 pcmdinfo->post_sq);
4029                 break;
4030         case OP_QP_DESTROY:
4031                 status = i40iw_sc_qp_destroy(
4032                                 pcmdinfo->in.u.qp_destroy.qp,
4033                                 pcmdinfo->in.u.qp_destroy.scratch,
4034                                 pcmdinfo->in.u.qp_destroy.remove_hash_idx,
4035                                 pcmdinfo->in.u.qp_destroy.
4036                                 ignore_mw_bnd,
4037                                 pcmdinfo->post_sq);
4038
4039                 break;
4040         case OP_ALLOC_STAG:
4041                 status = i40iw_sc_alloc_stag(
4042                                 pcmdinfo->in.u.alloc_stag.dev,
4043                                 &pcmdinfo->in.u.alloc_stag.info,
4044                                 pcmdinfo->in.u.alloc_stag.scratch,
4045                                 pcmdinfo->post_sq);
4046                 break;
4047         case OP_MR_REG_NON_SHARED:
4048                 status = i40iw_sc_mr_reg_non_shared(
4049                                 pcmdinfo->in.u.mr_reg_non_shared.dev,
4050                                 &pcmdinfo->in.u.mr_reg_non_shared.info,
4051                                 pcmdinfo->in.u.mr_reg_non_shared.scratch,
4052                                 pcmdinfo->post_sq);
4053
4054                 break;
4055         case OP_DEALLOC_STAG:
4056                 status = i40iw_sc_dealloc_stag(
4057                                 pcmdinfo->in.u.dealloc_stag.dev,
4058                                 &pcmdinfo->in.u.dealloc_stag.info,
4059                                 pcmdinfo->in.u.dealloc_stag.scratch,
4060                                 pcmdinfo->post_sq);
4061
4062                 break;
4063         case OP_MW_ALLOC:
4064                 status = i40iw_sc_mw_alloc(
4065                                 pcmdinfo->in.u.mw_alloc.dev,
4066                                 pcmdinfo->in.u.mw_alloc.scratch,
4067                                 pcmdinfo->in.u.mw_alloc.mw_stag_index,
4068                                 pcmdinfo->in.u.mw_alloc.pd_id,
4069                                 pcmdinfo->post_sq);
4070
4071                 break;
4072         case OP_QP_FLUSH_WQES:
4073                 status = i40iw_sc_qp_flush_wqes(
4074                                 pcmdinfo->in.u.qp_flush_wqes.qp,
4075                                 &pcmdinfo->in.u.qp_flush_wqes.info,
4076                                 pcmdinfo->in.u.qp_flush_wqes.
4077                                 scratch, pcmdinfo->post_sq);
4078                 break;
4079         case OP_ADD_ARP_CACHE_ENTRY:
4080                 status = i40iw_sc_add_arp_cache_entry(
4081                                 pcmdinfo->in.u.add_arp_cache_entry.cqp,
4082                                 &pcmdinfo->in.u.add_arp_cache_entry.info,
4083                                 pcmdinfo->in.u.add_arp_cache_entry.scratch,
4084                                 pcmdinfo->post_sq);
4085                 break;
4086         case OP_MANAGE_PUSH_PAGE:
4087                 status = i40iw_sc_manage_push_page(
4088                                 pcmdinfo->in.u.manage_push_page.cqp,
4089                                 &pcmdinfo->in.u.manage_push_page.info,
4090                                 pcmdinfo->in.u.manage_push_page.scratch,
4091                                 pcmdinfo->post_sq);
4092                 break;
4093         case OP_UPDATE_PE_SDS:
4094                 /* case I40IW_CQP_OP_UPDATE_PE_SDS */
4095                 status = i40iw_update_pe_sds(
4096                                 pcmdinfo->in.u.update_pe_sds.dev,
4097                                 &pcmdinfo->in.u.update_pe_sds.info,
4098                                 pcmdinfo->in.u.update_pe_sds.
4099                                 scratch);
4100
4101                 break;
4102         case OP_MANAGE_HMC_PM_FUNC_TABLE:
4103                 status = i40iw_sc_manage_hmc_pm_func_table(
4104                                 pcmdinfo->in.u.manage_hmc_pm.dev->cqp,
4105                                 pcmdinfo->in.u.manage_hmc_pm.scratch,
4106                                 (u8)pcmdinfo->in.u.manage_hmc_pm.info.vf_id,
4107                                 pcmdinfo->in.u.manage_hmc_pm.info.free_fcn,
4108                                 true);
4109                 break;
4110         case OP_SUSPEND:
4111                 status = i40iw_sc_suspend_qp(
4112                                 pcmdinfo->in.u.suspend_resume.cqp,
4113                                 pcmdinfo->in.u.suspend_resume.qp,
4114                                 pcmdinfo->in.u.suspend_resume.scratch);
4115                 break;
4116         case OP_RESUME:
4117                 status = i40iw_sc_resume_qp(
4118                                 pcmdinfo->in.u.suspend_resume.cqp,
4119                                 pcmdinfo->in.u.suspend_resume.qp,
4120                                 pcmdinfo->in.u.suspend_resume.scratch);
4121                 break;
4122         case OP_MANAGE_VF_PBLE_BP:
4123                 status = i40iw_manage_vf_pble_bp(
4124                                 pcmdinfo->in.u.manage_vf_pble_bp.cqp,
4125                                 &pcmdinfo->in.u.manage_vf_pble_bp.info,
4126                                 pcmdinfo->in.u.manage_vf_pble_bp.scratch, true);
4127                 break;
4128         case OP_QUERY_FPM_VALUES:
4129                 values_mem.pa = pcmdinfo->in.u.query_fpm_values.fpm_values_pa;
4130                 values_mem.va = pcmdinfo->in.u.query_fpm_values.fpm_values_va;
4131                 status = i40iw_sc_query_fpm_values(
4132                                 pcmdinfo->in.u.query_fpm_values.cqp,
4133                                 pcmdinfo->in.u.query_fpm_values.scratch,
4134                                 pcmdinfo->in.u.query_fpm_values.hmc_fn_id,
4135                                 &values_mem, true, I40IW_CQP_WAIT_EVENT);
4136                 break;
4137         case OP_COMMIT_FPM_VALUES:
4138                 values_mem.pa = pcmdinfo->in.u.commit_fpm_values.fpm_values_pa;
4139                 values_mem.va = pcmdinfo->in.u.commit_fpm_values.fpm_values_va;
4140                 status = i40iw_sc_commit_fpm_values(
4141                                 pcmdinfo->in.u.commit_fpm_values.cqp,
4142                                 pcmdinfo->in.u.commit_fpm_values.scratch,
4143                                 pcmdinfo->in.u.commit_fpm_values.hmc_fn_id,
4144                                 &values_mem,
4145                                 true,
4146                                 I40IW_CQP_WAIT_EVENT);
4147                 break;
4148         default:
4149                 status = I40IW_NOT_SUPPORTED;
4150                 break;
4151         }
4152
4153         return status;
4154 }
4155
4156 /**
4157  * i40iw_process_cqp_cmd - process all cqp commands
4158  * @dev: sc device struct
4159  * @pcmdinfo: cqp command info
4160  */
4161 enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev,
4162                                              struct cqp_commands_info *pcmdinfo)
4163 {
4164         enum i40iw_status_code status = 0;
4165         unsigned long flags;
4166
4167         spin_lock_irqsave(&dev->cqp_lock, flags);
4168         if (list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp))
4169                 status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
4170         else
4171                 list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);
4172         spin_unlock_irqrestore(&dev->cqp_lock, flags);
4173         return status;
4174 }
4175
4176 /**
4177  * i40iw_process_bh - called from tasklet for cqp list
4178  * @dev: sc device struct
4179  */
4180 enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev)
4181 {
4182         enum i40iw_status_code status = 0;
4183         struct cqp_commands_info *pcmdinfo;
4184         unsigned long flags;
4185
4186         spin_lock_irqsave(&dev->cqp_lock, flags);
4187         while (!list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp)) {
4188                 pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&dev->cqp_cmd_head);
4189
4190                 status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
4191                 if (status)
4192                         break;
4193         }
4194         spin_unlock_irqrestore(&dev->cqp_lock, flags);
4195         return status;
4196 }
4197
4198 /**
4199  * i40iw_iwarp_opcode - determine if incoming is rdma layer
4200  * @info: aeq info for the packet
4201  * @pkt: packet for error
4202  */
4203 static u32 i40iw_iwarp_opcode(struct i40iw_aeqe_info *info, u8 *pkt)
4204 {
4205         __be16 *mpa;
4206         u32 opcode = 0xffffffff;
4207
4208         if (info->q2_data_written) {
4209                 mpa = (__be16 *)pkt;
4210                 opcode = ntohs(mpa[1]) & 0xf;
4211         }
4212         return opcode;
4213 }
4214
4215 /**
4216  * i40iw_locate_mpa - return pointer to mpa in the pkt
4217  * @pkt: packet with data
4218  */
4219 static u8 *i40iw_locate_mpa(u8 *pkt)
4220 {
4221         /* skip over ethernet header */
4222         pkt += I40IW_MAC_HLEN;
4223
4224         /* Skip over IP and TCP headers */
4225         pkt += 4 * (pkt[0] & 0x0f);
4226         pkt += 4 * ((pkt[12] >> 4) & 0x0f);
4227         return pkt;
4228 }
4229
4230 /**
4231  * i40iw_setup_termhdr - termhdr for terminate pkt
4232  * @qp: sc qp ptr for pkt
4233  * @hdr: term hdr
4234  * @opcode: flush opcode for termhdr
4235  * @layer_etype: error layer + error type
4236  * @err: error cod ein the header
4237  */
4238 static void i40iw_setup_termhdr(struct i40iw_sc_qp *qp,
4239                                 struct i40iw_terminate_hdr *hdr,
4240                                 enum i40iw_flush_opcode opcode,
4241                                 u8 layer_etype,
4242                                 u8 err)
4243 {
4244         qp->flush_code = opcode;
4245         hdr->layer_etype = layer_etype;
4246         hdr->error_code = err;
4247 }
4248
4249 /**
4250  * i40iw_bld_terminate_hdr - build terminate message header
4251  * @qp: qp associated with received terminate AE
4252  * @info: the struct contiaing AE information
4253  */
4254 static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp,
4255                                    struct i40iw_aeqe_info *info)
4256 {
4257         u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
4258         u16 ddp_seg_len;
4259         int copy_len = 0;
4260         u8 is_tagged = 0;
4261         u32 opcode;
4262         struct i40iw_terminate_hdr *termhdr;
4263
4264         termhdr = (struct i40iw_terminate_hdr *)qp->q2_buf;
4265         memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);
4266
4267         if (info->q2_data_written) {
4268                 /* Use data from offending packet to fill in ddp & rdma hdrs */
4269                 pkt = i40iw_locate_mpa(pkt);
4270                 ddp_seg_len = ntohs(*(__be16 *)pkt);
4271                 if (ddp_seg_len) {
4272                         copy_len = 2;
4273                         termhdr->hdrct = DDP_LEN_FLAG;
4274                         if (pkt[2] & 0x80) {
4275                                 is_tagged = 1;
4276                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
4277                                         copy_len += TERM_DDP_LEN_TAGGED;
4278                                         termhdr->hdrct |= DDP_HDR_FLAG;
4279                                 }
4280                         } else {
4281                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
4282                                         copy_len += TERM_DDP_LEN_UNTAGGED;
4283                                         termhdr->hdrct |= DDP_HDR_FLAG;
4284                                 }
4285
4286                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
4287                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
4288                                                 copy_len += TERM_RDMA_LEN;
4289                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
4290                                         }
4291                                 }
4292                         }
4293                 }
4294         }
4295
4296         opcode = i40iw_iwarp_opcode(info, pkt);
4297
4298         switch (info->ae_id) {
4299         case I40IW_AE_AMP_UNALLOCATED_STAG:
4300                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4301                 if (opcode == I40IW_OP_TYPE_RDMA_WRITE)
4302                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4303                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_STAG);
4304                 else
4305                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4306                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
4307                 break;
4308         case I40IW_AE_AMP_BOUNDS_VIOLATION:
4309                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4310                 if (info->q2_data_written)
4311                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4312                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_BOUNDS);
4313                 else
4314                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4315                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_BOUNDS);
4316                 break;
4317         case I40IW_AE_AMP_BAD_PD:
4318                 switch (opcode) {
4319                 case I40IW_OP_TYPE_RDMA_WRITE:
4320                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4321                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_UNASSOC_STAG);
4322                         break;
4323                 case I40IW_OP_TYPE_SEND_INV:
4324                 case I40IW_OP_TYPE_SEND_SOL_INV:
4325                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4326                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_CANT_INV_STAG);
4327                         break;
4328                 default:
4329                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4330                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_UNASSOC_STAG);
4331                 }
4332                 break;
4333         case I40IW_AE_AMP_INVALID_STAG:
4334                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4335                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4336                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
4337                 break;
4338         case I40IW_AE_AMP_BAD_QP:
4339                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
4340                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
4341                 break;
4342         case I40IW_AE_AMP_BAD_STAG_KEY:
4343         case I40IW_AE_AMP_BAD_STAG_INDEX:
4344                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4345                 switch (opcode) {
4346                 case I40IW_OP_TYPE_SEND_INV:
4347                 case I40IW_OP_TYPE_SEND_SOL_INV:
4348                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
4349                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_CANT_INV_STAG);
4350                         break;
4351                 default:
4352                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4353                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_STAG);
4354                 }
4355                 break;
4356         case I40IW_AE_AMP_RIGHTS_VIOLATION:
4357         case I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
4358         case I40IW_AE_PRIV_OPERATION_DENIED:
4359                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4360                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4361                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_ACCESS);
4362                 break;
4363         case I40IW_AE_AMP_TO_WRAP:
4364                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4365                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4366                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_TO_WRAP);
4367                 break;
4368         case I40IW_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
4369                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
4370                                     (LAYER_MPA << 4) | DDP_LLP, MPA_MARKER);
4371                 break;
4372         case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
4373                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4374                                     (LAYER_MPA << 4) | DDP_LLP, MPA_CRC);
4375                 break;
4376         case I40IW_AE_LLP_SEGMENT_TOO_LARGE:
4377         case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
4378                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
4379                                     (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
4380                 break;
4381         case I40IW_AE_LCE_QP_CATASTROPHIC:
4382         case I40IW_AE_DDP_NO_L_BIT:
4383                 i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
4384                                     (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
4385                 break;
4386         case I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN:
4387         case I40IW_AE_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
4388                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4389                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_RANGE);
4390                 break;
4391         case I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
4392                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4393                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
4394                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_TOO_LONG);
4395                 break;
4396         case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
4397                 if (is_tagged)
4398                         i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4399                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_DDP_VER);
4400                 else
4401                         i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4402                                             (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_DDP_VER);
4403                 break;
4404         case I40IW_AE_DDP_UBE_INVALID_MO:
4405                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4406                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MO);
4407                 break;
4408         case I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
4409                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
4410                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_NO_BUF);
4411                 break;
4412         case I40IW_AE_DDP_UBE_INVALID_QN:
4413                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4414                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
4415                 break;
4416         case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
4417                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4418                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_RDMAP_VER);
4419                 break;
4420         case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
4421                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
4422                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNEXPECTED_OP);
4423                 break;
4424         default:
4425                 i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
4426                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNSPECIFIED);
4427                 break;
4428         }
4429
4430         if (copy_len)
4431                 memcpy(termhdr + 1, pkt, copy_len);
4432
4433         return sizeof(struct i40iw_terminate_hdr) + copy_len;
4434 }
4435
4436 /**
4437  * i40iw_terminate_send_fin() - Send fin for terminate message
4438  * @qp: qp associated with received terminate AE
4439  */
4440 void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp)
4441 {
4442         /* Send the fin only */
4443         i40iw_term_modify_qp(qp,
4444                              I40IW_QP_STATE_TERMINATE,
4445                              I40IWQP_TERM_SEND_FIN_ONLY,
4446                              0);
4447 }
4448
4449 /**
4450  * i40iw_terminate_connection() - Bad AE and send terminate to remote QP
4451  * @qp: qp associated with received terminate AE
4452  * @info: the struct contiaing AE information
4453  */
4454 void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)
4455 {
4456         u8 termlen = 0;
4457
4458         if (qp->term_flags & I40IW_TERM_SENT)
4459                 return;         /* Sanity check */
4460
4461         /* Eventtype can change from bld_terminate_hdr */
4462         qp->eventtype = TERM_EVENT_QP_FATAL;
4463         termlen = i40iw_bld_terminate_hdr(qp, info);
4464         i40iw_terminate_start_timer(qp);
4465         qp->term_flags |= I40IW_TERM_SENT;
4466         i40iw_term_modify_qp(qp, I40IW_QP_STATE_TERMINATE,
4467                              I40IWQP_TERM_SEND_TERM_ONLY, termlen);
4468 }
4469
4470 /**
4471  * i40iw_terminate_received - handle terminate received AE
4472  * @qp: qp associated with received terminate AE
4473  * @info: the struct contiaing AE information
4474  */
4475 void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)
4476 {
4477         u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
4478         __be32 *mpa;
4479         u8 ddp_ctl;
4480         u8 rdma_ctl;
4481         u16 aeq_id = 0;
4482         struct i40iw_terminate_hdr *termhdr;
4483
4484         mpa = (__be32 *)i40iw_locate_mpa(pkt);
4485         if (info->q2_data_written) {
4486                 /* did not validate the frame - do it now */
4487                 ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff;
4488                 rdma_ctl = ntohl(mpa[0]) & 0xff;
4489                 if ((ddp_ctl & 0xc0) != 0x40)
4490                         aeq_id = I40IW_AE_LCE_QP_CATASTROPHIC;
4491                 else if ((ddp_ctl & 0x03) != 1)
4492                         aeq_id = I40IW_AE_DDP_UBE_INVALID_DDP_VERSION;
4493                 else if (ntohl(mpa[2]) != 2)
4494                         aeq_id = I40IW_AE_DDP_UBE_INVALID_QN;
4495                 else if (ntohl(mpa[3]) != 1)
4496                         aeq_id = I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN;
4497                 else if (ntohl(mpa[4]) != 0)
4498                         aeq_id = I40IW_AE_DDP_UBE_INVALID_MO;
4499                 else if ((rdma_ctl & 0xc0) != 0x40)
4500                         aeq_id = I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;
4501
4502                 info->ae_id = aeq_id;
4503                 if (info->ae_id) {
4504                         /* Bad terminate recvd - send back a terminate */
4505                         i40iw_terminate_connection(qp, info);
4506                         return;
4507                 }
4508         }
4509
4510         qp->term_flags |= I40IW_TERM_RCVD;
4511         qp->eventtype = TERM_EVENT_QP_FATAL;
4512         termhdr = (struct i40iw_terminate_hdr *)&mpa[5];
4513         if (termhdr->layer_etype == RDMAP_REMOTE_PROT ||
4514             termhdr->layer_etype == RDMAP_REMOTE_OP) {
4515                 i40iw_terminate_done(qp, 0);
4516         } else {
4517                 i40iw_terminate_start_timer(qp);
4518                 i40iw_terminate_send_fin(qp);
4519         }
4520 }
4521
4522 /**
4523  * i40iw_sc_vsi_init - Initialize virtual device
4524  * @vsi: pointer to the vsi structure
4525  * @info: parameters to initialize vsi
4526  **/
4527 void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info)
4528 {
4529         int i;
4530
4531         vsi->dev = info->dev;
4532         vsi->back_vsi = info->back_vsi;
4533         vsi->mss = info->params->mss;
4534         i40iw_fill_qos_list(info->params->qs_handle_list);
4535
4536         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
4537                 vsi->qos[i].qs_handle = info->params->qs_handle_list[i];
4538                 i40iw_debug(vsi->dev, I40IW_DEBUG_DCB, "qset[%d]: %d\n", i,
4539                             vsi->qos[i].qs_handle);
4540                 spin_lock_init(&vsi->qos[i].lock);
4541                 INIT_LIST_HEAD(&vsi->qos[i].qplist);
4542         }
4543 }
4544
4545 /**
4546  * i40iw_hw_stats_init - Initiliaze HW stats table
4547  * @stats: pestat struct
4548  * @fcn_idx: PCI fn id
4549  * @is_pf: Is it a PF?
4550  *
4551  * Populate the HW stats table with register offset addr for each
4552  * stats. And start the perioidic stats timer.
4553  */
4554 void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 fcn_idx, bool is_pf)
4555 {
4556         u32 stats_reg_offset;
4557         u32 stats_index;
4558         struct i40iw_dev_hw_stats_offsets *stats_table =
4559                 &stats->hw_stats_offsets;
4560         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4561
4562         if (is_pf) {
4563                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
4564                                 I40E_GLPES_PFIP4RXDISCARD(fcn_idx);
4565                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
4566                                 I40E_GLPES_PFIP4RXTRUNC(fcn_idx);
4567                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
4568                                 I40E_GLPES_PFIP4TXNOROUTE(fcn_idx);
4569                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
4570                                 I40E_GLPES_PFIP6RXDISCARD(fcn_idx);
4571                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
4572                                 I40E_GLPES_PFIP6RXTRUNC(fcn_idx);
4573                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
4574                                 I40E_GLPES_PFIP6TXNOROUTE(fcn_idx);
4575                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
4576                                 I40E_GLPES_PFTCPRTXSEG(fcn_idx);
4577                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
4578                                 I40E_GLPES_PFTCPRXOPTERR(fcn_idx);
4579                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
4580                                 I40E_GLPES_PFTCPRXPROTOERR(fcn_idx);
4581
4582                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
4583                                 I40E_GLPES_PFIP4RXOCTSLO(fcn_idx);
4584                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
4585                                 I40E_GLPES_PFIP4RXPKTSLO(fcn_idx);
4586                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
4587                                 I40E_GLPES_PFIP4RXFRAGSLO(fcn_idx);
4588                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
4589                                 I40E_GLPES_PFIP4RXMCPKTSLO(fcn_idx);
4590                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
4591                                 I40E_GLPES_PFIP4TXOCTSLO(fcn_idx);
4592                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
4593                                 I40E_GLPES_PFIP4TXPKTSLO(fcn_idx);
4594                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
4595                                 I40E_GLPES_PFIP4TXFRAGSLO(fcn_idx);
4596                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
4597                                 I40E_GLPES_PFIP4TXMCPKTSLO(fcn_idx);
4598                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
4599                                 I40E_GLPES_PFIP6RXOCTSLO(fcn_idx);
4600                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
4601                                 I40E_GLPES_PFIP6RXPKTSLO(fcn_idx);
4602                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
4603                                 I40E_GLPES_PFIP6RXFRAGSLO(fcn_idx);
4604                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
4605                                 I40E_GLPES_PFIP6RXMCPKTSLO(fcn_idx);
4606                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
4607                                 I40E_GLPES_PFIP6TXOCTSLO(fcn_idx);
4608                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4609                                 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
4610                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4611                                 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
4612                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
4613                                 I40E_GLPES_PFIP6TXFRAGSLO(fcn_idx);
4614                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
4615                                 I40E_GLPES_PFTCPRXSEGSLO(fcn_idx);
4616                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
4617                                 I40E_GLPES_PFTCPTXSEGLO(fcn_idx);
4618                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
4619                                 I40E_GLPES_PFRDMARXRDSLO(fcn_idx);
4620                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
4621                                 I40E_GLPES_PFRDMARXSNDSLO(fcn_idx);
4622                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
4623                                 I40E_GLPES_PFRDMARXWRSLO(fcn_idx);
4624                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
4625                                 I40E_GLPES_PFRDMATXRDSLO(fcn_idx);
4626                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
4627                                 I40E_GLPES_PFRDMATXSNDSLO(fcn_idx);
4628                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
4629                                 I40E_GLPES_PFRDMATXWRSLO(fcn_idx);
4630                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
4631                                 I40E_GLPES_PFRDMAVBNDLO(fcn_idx);
4632                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
4633                                 I40E_GLPES_PFRDMAVINVLO(fcn_idx);
4634         } else {
4635                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
4636                                 I40E_GLPES_VFIP4RXDISCARD(fcn_idx);
4637                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
4638                                 I40E_GLPES_VFIP4RXTRUNC(fcn_idx);
4639                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
4640                                 I40E_GLPES_VFIP4TXNOROUTE(fcn_idx);
4641                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
4642                                 I40E_GLPES_VFIP6RXDISCARD(fcn_idx);
4643                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
4644                                 I40E_GLPES_VFIP6RXTRUNC(fcn_idx);
4645                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
4646                                 I40E_GLPES_VFIP6TXNOROUTE(fcn_idx);
4647                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
4648                                 I40E_GLPES_VFTCPRTXSEG(fcn_idx);
4649                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
4650                                 I40E_GLPES_VFTCPRXOPTERR(fcn_idx);
4651                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
4652                                 I40E_GLPES_VFTCPRXPROTOERR(fcn_idx);
4653
4654                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
4655                                 I40E_GLPES_VFIP4RXOCTSLO(fcn_idx);
4656                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
4657                                 I40E_GLPES_VFIP4RXPKTSLO(fcn_idx);
4658                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
4659                                 I40E_GLPES_VFIP4RXFRAGSLO(fcn_idx);
4660                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
4661                                 I40E_GLPES_VFIP4RXMCPKTSLO(fcn_idx);
4662                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
4663                                 I40E_GLPES_VFIP4TXOCTSLO(fcn_idx);
4664                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
4665                                 I40E_GLPES_VFIP4TXPKTSLO(fcn_idx);
4666                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
4667                                 I40E_GLPES_VFIP4TXFRAGSLO(fcn_idx);
4668                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
4669                                 I40E_GLPES_VFIP4TXMCPKTSLO(fcn_idx);
4670                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
4671                                 I40E_GLPES_VFIP6RXOCTSLO(fcn_idx);
4672                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
4673                                 I40E_GLPES_VFIP6RXPKTSLO(fcn_idx);
4674                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
4675                                 I40E_GLPES_VFIP6RXFRAGSLO(fcn_idx);
4676                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
4677                                 I40E_GLPES_VFIP6RXMCPKTSLO(fcn_idx);
4678                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
4679                                 I40E_GLPES_VFIP6TXOCTSLO(fcn_idx);
4680                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4681                                 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
4682                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4683                                 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
4684                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
4685                                 I40E_GLPES_VFIP6TXFRAGSLO(fcn_idx);
4686                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
4687                                 I40E_GLPES_VFTCPRXSEGSLO(fcn_idx);
4688                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
4689                                 I40E_GLPES_VFTCPTXSEGLO(fcn_idx);
4690                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
4691                                 I40E_GLPES_VFRDMARXRDSLO(fcn_idx);
4692                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
4693                                 I40E_GLPES_VFRDMARXSNDSLO(fcn_idx);
4694                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
4695                                 I40E_GLPES_VFRDMARXWRSLO(fcn_idx);
4696                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
4697                                 I40E_GLPES_VFRDMATXRDSLO(fcn_idx);
4698                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
4699                                 I40E_GLPES_VFRDMATXSNDSLO(fcn_idx);
4700                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
4701                                 I40E_GLPES_VFRDMATXWRSLO(fcn_idx);
4702                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
4703                                 I40E_GLPES_VFRDMAVBNDLO(fcn_idx);
4704                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
4705                                 I40E_GLPES_VFRDMAVINVLO(fcn_idx);
4706         }
4707
4708         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
4709              stats_index++) {
4710                 stats_reg_offset = stats_table->stats_offset_64[stats_index];
4711                 last_rd_stats->stats_value_64[stats_index] =
4712                         readq(stats->hw->hw_addr + stats_reg_offset);
4713         }
4714
4715         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
4716              stats_index++) {
4717                 stats_reg_offset = stats_table->stats_offset_32[stats_index];
4718                 last_rd_stats->stats_value_32[stats_index] =
4719                         i40iw_rd32(stats->hw, stats_reg_offset);
4720         }
4721 }
4722
4723 /**
4724  * i40iw_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates for roll-overs.
4725  * @stat: pestat struct
4726  * @index: index in HW stats table which contains offset reg-addr
4727  * @value: hw stats value
4728  */
4729 void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,
4730                             enum i40iw_hw_stats_index_32b index,
4731                             u64 *value)
4732 {
4733         struct i40iw_dev_hw_stats_offsets *stats_table =
4734                 &stats->hw_stats_offsets;
4735         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4736         struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
4737         u64 new_stats_value = 0;
4738         u32 stats_reg_offset = stats_table->stats_offset_32[index];
4739
4740         new_stats_value = i40iw_rd32(stats->hw, stats_reg_offset);
4741         /*roll-over case */
4742         if (new_stats_value < last_rd_stats->stats_value_32[index])
4743                 hw_stats->stats_value_32[index] += new_stats_value;
4744         else
4745                 hw_stats->stats_value_32[index] +=
4746                         new_stats_value - last_rd_stats->stats_value_32[index];
4747         last_rd_stats->stats_value_32[index] = new_stats_value;
4748         *value = hw_stats->stats_value_32[index];
4749 }
4750
4751 /**
4752  * i40iw_hw_stats_read_64 - Read HW stats counters (greater than 32-bit) and accommodates for roll-overs.
4753  * @stats: pestat struct
4754  * @index: index in HW stats table which contains offset reg-addr
4755  * @value: hw stats value
4756  */
4757 void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,
4758                             enum i40iw_hw_stats_index_64b index,
4759                             u64 *value)
4760 {
4761         struct i40iw_dev_hw_stats_offsets *stats_table =
4762                 &stats->hw_stats_offsets;
4763         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4764         struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
4765         u64 new_stats_value = 0;
4766         u32 stats_reg_offset = stats_table->stats_offset_64[index];
4767
4768         new_stats_value = readq(stats->hw->hw_addr + stats_reg_offset);
4769         /*roll-over case */
4770         if (new_stats_value < last_rd_stats->stats_value_64[index])
4771                 hw_stats->stats_value_64[index] += new_stats_value;
4772         else
4773                 hw_stats->stats_value_64[index] +=
4774                         new_stats_value - last_rd_stats->stats_value_64[index];
4775         last_rd_stats->stats_value_64[index] = new_stats_value;
4776         *value = hw_stats->stats_value_64[index];
4777 }
4778
4779 /**
4780  * i40iw_hw_stats_read_all - read all HW stat counters
4781  * @stats: pestat struct
4782  * @stats_values: hw stats structure
4783  *
4784  * Read all the HW stat counters and populates hw_stats structure
4785  * of passed-in vsi's pestat as well as copy created in stat_values.
4786  */
4787 void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats,
4788                              struct i40iw_dev_hw_stats *stats_values)
4789 {
4790         u32 stats_index;
4791         unsigned long flags;
4792
4793         spin_lock_irqsave(&stats->lock, flags);
4794
4795         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
4796              stats_index++)
4797                 i40iw_hw_stats_read_32(stats, stats_index,
4798                                        &stats_values->stats_value_32[stats_index]);
4799         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
4800              stats_index++)
4801                 i40iw_hw_stats_read_64(stats, stats_index,
4802                                        &stats_values->stats_value_64[stats_index]);
4803         spin_unlock_irqrestore(&stats->lock, flags);
4804 }
4805
4806 /**
4807  * i40iw_hw_stats_refresh_all - Update all HW stats structs
4808  * @stats: pestat struct
4809  *
4810  * Read all the HW stats counters to refresh values in hw_stats structure
4811  * of passed-in dev's pestat
4812  */
4813 void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats)
4814 {
4815         u64 stats_value;
4816         u32 stats_index;
4817         unsigned long flags;
4818
4819         spin_lock_irqsave(&stats->lock, flags);
4820
4821         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
4822              stats_index++)
4823                 i40iw_hw_stats_read_32(stats, stats_index, &stats_value);
4824         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
4825              stats_index++)
4826                 i40iw_hw_stats_read_64(stats, stats_index, &stats_value);
4827         spin_unlock_irqrestore(&stats->lock, flags);
4828 }
4829
4830 /**
4831  * i40iw_get_fcn_id - Return the function id
4832  * @dev: pointer to the device
4833  */
4834 static u8 i40iw_get_fcn_id(struct i40iw_sc_dev *dev)
4835 {
4836         u8 fcn_id = I40IW_INVALID_FCN_ID;
4837         u8 i;
4838
4839         for (i = I40IW_FIRST_NON_PF_STAT; i < I40IW_MAX_STATS_COUNT; i++)
4840                 if (!dev->fcn_id_array[i]) {
4841                         fcn_id = i;
4842                         dev->fcn_id_array[i] = true;
4843                         break;
4844                 }
4845         return fcn_id;
4846 }
4847
4848 /**
4849  * i40iw_vsi_stats_init - Initialize the vsi statistics
4850  * @vsi: pointer to the vsi structure
4851  * @info: The info structure used for initialization
4852  */
4853 enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info)
4854 {
4855         u8 fcn_id = info->fcn_id;
4856
4857         if (info->alloc_fcn_id)
4858                 fcn_id = i40iw_get_fcn_id(vsi->dev);
4859
4860         if (fcn_id == I40IW_INVALID_FCN_ID)
4861                 return I40IW_ERR_NOT_READY;
4862
4863         vsi->pestat = info->pestat;
4864         vsi->pestat->hw = vsi->dev->hw;
4865
4866         if (info->stats_initialize) {
4867                 i40iw_hw_stats_init(vsi->pestat, fcn_id, true);
4868                 spin_lock_init(&vsi->pestat->lock);
4869                 i40iw_hw_stats_start_timer(vsi);
4870         }
4871         vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
4872         vsi->fcn_id = fcn_id;
4873         return I40IW_SUCCESS;
4874 }
4875
4876 /**
4877  * i40iw_vsi_stats_free - Free the vsi stats
4878  * @vsi: pointer to the vsi structure
4879  */
4880 void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi)
4881 {
4882         u8 fcn_id = vsi->fcn_id;
4883
4884         if ((vsi->stats_fcn_id_alloc) && (fcn_id != I40IW_INVALID_FCN_ID))
4885                 vsi->dev->fcn_id_array[fcn_id] = false;
4886         i40iw_hw_stats_stop_timer(vsi);
4887 }
4888
4889 static struct i40iw_cqp_ops iw_cqp_ops = {
4890         .cqp_init = i40iw_sc_cqp_init,
4891         .cqp_create = i40iw_sc_cqp_create,
4892         .cqp_post_sq = i40iw_sc_cqp_post_sq,
4893         .cqp_get_next_send_wqe = i40iw_sc_cqp_get_next_send_wqe,
4894         .cqp_destroy = i40iw_sc_cqp_destroy,
4895         .poll_for_cqp_op_done = i40iw_sc_poll_for_cqp_op_done
4896 };
4897
4898 static struct i40iw_ccq_ops iw_ccq_ops = {
4899         .ccq_init = i40iw_sc_ccq_init,
4900         .ccq_create = i40iw_sc_ccq_create,
4901         .ccq_destroy = i40iw_sc_ccq_destroy,
4902         .ccq_create_done = i40iw_sc_ccq_create_done,
4903         .ccq_get_cqe_info = i40iw_sc_ccq_get_cqe_info,
4904         .ccq_arm = i40iw_sc_ccq_arm
4905 };
4906
4907 static struct i40iw_ceq_ops iw_ceq_ops = {
4908         .ceq_init = i40iw_sc_ceq_init,
4909         .ceq_create = i40iw_sc_ceq_create,
4910         .cceq_create_done = i40iw_sc_cceq_create_done,
4911         .cceq_destroy_done = i40iw_sc_cceq_destroy_done,
4912         .cceq_create = i40iw_sc_cceq_create,
4913         .ceq_destroy = i40iw_sc_ceq_destroy,
4914         .process_ceq = i40iw_sc_process_ceq
4915 };
4916
4917 static struct i40iw_aeq_ops iw_aeq_ops = {
4918         .aeq_init = i40iw_sc_aeq_init,
4919         .aeq_create = i40iw_sc_aeq_create,
4920         .aeq_destroy = i40iw_sc_aeq_destroy,
4921         .get_next_aeqe = i40iw_sc_get_next_aeqe,
4922         .repost_aeq_entries = i40iw_sc_repost_aeq_entries,
4923         .aeq_create_done = i40iw_sc_aeq_create_done,
4924         .aeq_destroy_done = i40iw_sc_aeq_destroy_done
4925 };
4926
4927 /* iwarp pd ops */
4928 static struct i40iw_pd_ops iw_pd_ops = {
4929         .pd_init = i40iw_sc_pd_init,
4930 };
4931
4932 static struct i40iw_priv_qp_ops iw_priv_qp_ops = {
4933         .qp_init = i40iw_sc_qp_init,
4934         .qp_create = i40iw_sc_qp_create,
4935         .qp_modify = i40iw_sc_qp_modify,
4936         .qp_destroy = i40iw_sc_qp_destroy,
4937         .qp_flush_wqes = i40iw_sc_qp_flush_wqes,
4938         .qp_upload_context = i40iw_sc_qp_upload_context,
4939         .qp_setctx = i40iw_sc_qp_setctx,
4940         .qp_send_lsmm = i40iw_sc_send_lsmm,
4941         .qp_send_lsmm_nostag = i40iw_sc_send_lsmm_nostag,
4942         .qp_send_rtt = i40iw_sc_send_rtt,
4943         .qp_post_wqe0 = i40iw_sc_post_wqe0,
4944         .iw_mr_fast_register = i40iw_sc_mr_fast_register
4945 };
4946
4947 static struct i40iw_priv_cq_ops iw_priv_cq_ops = {
4948         .cq_init = i40iw_sc_cq_init,
4949         .cq_create = i40iw_sc_cq_create,
4950         .cq_destroy = i40iw_sc_cq_destroy,
4951         .cq_modify = i40iw_sc_cq_modify,
4952 };
4953
4954 static struct i40iw_mr_ops iw_mr_ops = {
4955         .alloc_stag = i40iw_sc_alloc_stag,
4956         .mr_reg_non_shared = i40iw_sc_mr_reg_non_shared,
4957         .mr_reg_shared = i40iw_sc_mr_reg_shared,
4958         .dealloc_stag = i40iw_sc_dealloc_stag,
4959         .query_stag = i40iw_sc_query_stag,
4960         .mw_alloc = i40iw_sc_mw_alloc
4961 };
4962
4963 static struct i40iw_cqp_misc_ops iw_cqp_misc_ops = {
4964         .manage_push_page = i40iw_sc_manage_push_page,
4965         .manage_hmc_pm_func_table = i40iw_sc_manage_hmc_pm_func_table,
4966         .set_hmc_resource_profile = i40iw_sc_set_hmc_resource_profile,
4967         .commit_fpm_values = i40iw_sc_commit_fpm_values,
4968         .query_fpm_values = i40iw_sc_query_fpm_values,
4969         .static_hmc_pages_allocated = i40iw_sc_static_hmc_pages_allocated,
4970         .add_arp_cache_entry = i40iw_sc_add_arp_cache_entry,
4971         .del_arp_cache_entry = i40iw_sc_del_arp_cache_entry,
4972         .query_arp_cache_entry = i40iw_sc_query_arp_cache_entry,
4973         .manage_apbvt_entry = i40iw_sc_manage_apbvt_entry,
4974         .manage_qhash_table_entry = i40iw_sc_manage_qhash_table_entry,
4975         .alloc_local_mac_ipaddr_table_entry = i40iw_sc_alloc_local_mac_ipaddr_entry,
4976         .add_local_mac_ipaddr_entry = i40iw_sc_add_local_mac_ipaddr_entry,
4977         .del_local_mac_ipaddr_entry = i40iw_sc_del_local_mac_ipaddr_entry,
4978         .cqp_nop = i40iw_sc_cqp_nop,
4979         .commit_fpm_values_done = i40iw_sc_commit_fpm_values_done,
4980         .query_fpm_values_done = i40iw_sc_query_fpm_values_done,
4981         .manage_hmc_pm_func_table_done = i40iw_sc_manage_hmc_pm_func_table_done,
4982         .update_suspend_qp = i40iw_sc_suspend_qp,
4983         .update_resume_qp = i40iw_sc_resume_qp
4984 };
4985
4986 static struct i40iw_hmc_ops iw_hmc_ops = {
4987         .init_iw_hmc = i40iw_sc_init_iw_hmc,
4988         .parse_fpm_query_buf = i40iw_sc_parse_fpm_query_buf,
4989         .configure_iw_fpm = i40iw_sc_configure_iw_fpm,
4990         .parse_fpm_commit_buf = i40iw_sc_parse_fpm_commit_buf,
4991         .create_hmc_object = i40iw_sc_create_hmc_obj,
4992         .del_hmc_object = i40iw_sc_del_hmc_obj
4993 };
4994
4995 /**
4996  * i40iw_device_init - Initialize IWARP device
4997  * @dev: IWARP device pointer
4998  * @info: IWARP init info
4999  */
5000 enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,
5001                                          struct i40iw_device_init_info *info)
5002 {
5003         u32 val;
5004         u32 vchnl_ver = 0;
5005         u16 hmc_fcn = 0;
5006         enum i40iw_status_code ret_code = 0;
5007         u8 db_size;
5008
5009         spin_lock_init(&dev->cqp_lock);
5010         INIT_LIST_HEAD(&dev->cqp_cmd_head);             /* for the cqp commands backlog. */
5011
5012         i40iw_device_init_uk(&dev->dev_uk);
5013
5014         dev->debug_mask = info->debug_mask;
5015
5016         dev->hmc_fn_id = info->hmc_fn_id;
5017         dev->exception_lan_queue = info->exception_lan_queue;
5018         dev->is_pf = info->is_pf;
5019
5020         dev->fpm_query_buf_pa = info->fpm_query_buf_pa;
5021         dev->fpm_query_buf = info->fpm_query_buf;
5022
5023         dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;
5024         dev->fpm_commit_buf = info->fpm_commit_buf;
5025
5026         dev->hw = info->hw;
5027         dev->hw->hw_addr = info->bar0;
5028
5029         if (dev->is_pf) {
5030                 val = i40iw_rd32(dev->hw, I40E_GLPCI_DREVID);
5031                 dev->hw_rev = (u8)RS_32(val, I40E_GLPCI_DREVID_DEFAULT_REVID);
5032
5033                 val = i40iw_rd32(dev->hw, I40E_GLPCI_LBARCTRL);
5034                 db_size = (u8)RS_32(val, I40E_GLPCI_LBARCTRL_PE_DB_SIZE);
5035                 if ((db_size != I40IW_PE_DB_SIZE_4M) &&
5036                     (db_size != I40IW_PE_DB_SIZE_8M)) {
5037                         i40iw_debug(dev, I40IW_DEBUG_DEV,
5038                                     "%s: PE doorbell is not enabled in CSR val 0x%x\n",
5039                                     __func__, val);
5040                         ret_code = I40IW_ERR_PE_DOORBELL_NOT_ENABLED;
5041                         return ret_code;
5042                 }
5043                 dev->db_addr = dev->hw->hw_addr + I40IW_DB_ADDR_OFFSET;
5044                 dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_pf;
5045         } else {
5046                 dev->db_addr = dev->hw->hw_addr + I40IW_VF_DB_ADDR_OFFSET;
5047         }
5048
5049         dev->cqp_ops = &iw_cqp_ops;
5050         dev->ccq_ops = &iw_ccq_ops;
5051         dev->ceq_ops = &iw_ceq_ops;
5052         dev->aeq_ops = &iw_aeq_ops;
5053         dev->cqp_misc_ops = &iw_cqp_misc_ops;
5054         dev->iw_pd_ops = &iw_pd_ops;
5055         dev->iw_priv_qp_ops = &iw_priv_qp_ops;
5056         dev->iw_priv_cq_ops = &iw_priv_cq_ops;
5057         dev->mr_ops = &iw_mr_ops;
5058         dev->hmc_ops = &iw_hmc_ops;
5059         dev->vchnl_if.vchnl_send = info->vchnl_send;
5060         if (dev->vchnl_if.vchnl_send)
5061                 dev->vchnl_up = true;
5062         else
5063                 dev->vchnl_up = false;
5064         if (!dev->is_pf) {
5065                 dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_vf;
5066                 ret_code = i40iw_vchnl_vf_get_ver(dev, &vchnl_ver);
5067                 if (!ret_code) {
5068                         i40iw_debug(dev, I40IW_DEBUG_DEV,
5069                                     "%s: Get Channel version rc = 0x%0x, version is %u\n",
5070                                 __func__, ret_code, vchnl_ver);
5071                         ret_code = i40iw_vchnl_vf_get_hmc_fcn(dev, &hmc_fcn);
5072                         if (!ret_code) {
5073                                 i40iw_debug(dev, I40IW_DEBUG_DEV,
5074                                             "%s Get HMC function rc = 0x%0x, hmc fcn is %u\n",
5075                                             __func__, ret_code, hmc_fcn);
5076                                 dev->hmc_fn_id = (u8)hmc_fcn;
5077                         }
5078                 }
5079         }
5080         dev->iw_vf_cqp_ops = &iw_vf_cqp_ops;
5081
5082         return ret_code;
5083 }