drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2013 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <linux/crash_dump.h>
  25 #include <net/tcp.h>
  26 #include <net/ipv6.h>
  27 #include <net/ip6_checksum.h>
  28 #include <net/busy_poll.h>
  29 #include <linux/prefetch.h>
  30 #include "bnx2x_cmn.h"
  31 #include "bnx2x_init.h"
  32 #include "bnx2x_sp.h"
  33
  34 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
  35 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
  36 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
  37 static int bnx2x_poll(struct napi_struct *napi, int budget);
  38
  39 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
  40 {
  41         int i;
  42
  43         /* Add NAPI objects */
  44         for_each_rx_queue_cnic(bp, i) {
  45                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
  46                                bnx2x_poll, NAPI_POLL_WEIGHT);
  47                 napi_hash_add(&bnx2x_fp(bp, i, napi));
  48         }
  49 }
  50
  51 static void bnx2x_add_all_napi(struct bnx2x *bp)
  52 {
  53         int i;
  54
  55         /* Add NAPI objects */
  56         for_each_eth_queue(bp, i) {
  57                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
  58                                bnx2x_poll, NAPI_POLL_WEIGHT);
  59                 napi_hash_add(&bnx2x_fp(bp, i, napi));
  60         }
  61 }
  62
  63 static int bnx2x_calc_num_queues(struct bnx2x *bp)
  64 {
  65         int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
  66
  67         /* Reduce memory usage in kdump environment by using only one queue */
  68         if (is_kdump_kernel())
  69                 nq = 1;
  70
  71         nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
  72         return nq;
  73 }
  74
  75 /**
  76  * bnx2x_move_fp - move content of the fastpath structure.
  77  *
  78  * @bp:         driver handle
  79  * @from:       source FP index
  80  * @to:         destination FP index
  81  *
  82  * Makes sure the contents of the bp->fp[to].napi is kept
  83  * intact. This is done by first copying the napi struct from
  84  * the target to the source, and then mem copying the entire
  85  * source onto the target. Update txdata pointers and related
  86  * content.
  87  */
  88 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  89 {
  90         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  91         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  92         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  93         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  94         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  95         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  96         int old_max_eth_txqs, new_max_eth_txqs;
  97         int old_txdata_index = 0, new_txdata_index = 0;
  98         struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
  99
 100         /* Copy the NAPI object as it has been already initialized */
 101         from_fp->napi = to_fp->napi;
 102
 103         /* Move bnx2x_fastpath contents */
 104         memcpy(to_fp, from_fp, sizeof(*to_fp));
 105         to_fp->index = to;
 106
 107         /* Retain the tpa_info of the original `to' version as we don't want
 108          * 2 FPs to contain the same tpa_info pointer.
 109          */
 110         to_fp->tpa_info = old_tpa_info;
 111
 112         /* move sp_objs contents as well, as their indices match fp ones */
 113         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
 114
 115         /* move fp_stats contents as well, as their indices match fp ones */
 116         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
 117
 118         /* Update txdata pointers in fp and move txdata content accordingly:
 119          * Each fp consumes 'max_cos' txdata structures, so the index should be
 120          * decremented by max_cos x delta.
 121          */
 122
 123         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
 124         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
 125                                 (bp)->max_cos;
 126         if (from == FCOE_IDX(bp)) {
 127                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
 128                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
 129         }
 130
 131         memcpy(&bp->bnx2x_txq[new_txdata_index],
 132                &bp->bnx2x_txq[old_txdata_index],
 133                sizeof(struct bnx2x_fp_txdata));
 134         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
 135 }
 136
 137 /**
 138  * bnx2x_fill_fw_str - Fill buffer with FW version string.
 139  *
 140  * @bp:        driver handle
 141  * @buf:       character buffer to fill with the fw name
 142  * @buf_len:   length of the above buffer
 143  *
 144  */
 145 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
 146 {
 147         if (IS_PF(bp)) {
 148                 u8 phy_fw_ver[PHY_FW_VER_LEN];
 149
 150                 phy_fw_ver[0] = '\0';
 151                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
 152                                              phy_fw_ver, PHY_FW_VER_LEN);
 153                 strlcpy(buf, bp->fw_ver, buf_len);
 154                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 155                          "bc %d.%d.%d%s%s",
 156                          (bp->common.bc_ver & 0xff0000) >> 16,
 157                          (bp->common.bc_ver & 0xff00) >> 8,
 158                          (bp->common.bc_ver & 0xff),
 159                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 160         } else {
 161                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 162         }
 163 }
 164
 165 /**
 166  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 167  *
 168  * @bp: driver handle
 169  * @delta:      number of eth queues which were not allocated
 170  */
 171 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 172 {
 173         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 174
 175         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 176          * backward along the array could cause memory to be overridden
 177          */
 178         for (cos = 1; cos < bp->max_cos; cos++) {
 179                 for (i = 0; i < old_eth_num - delta; i++) {
 180                         struct bnx2x_fastpath *fp = &bp->fp[i];
 181                         int new_idx = cos * (old_eth_num - delta) + i;
 182
 183                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 184                                sizeof(struct bnx2x_fp_txdata));
 185                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 186                 }
 187         }
 188 }
 189
 190 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 191
 192 /* free skb in the packet ring at pos idx
 193  * return idx of last bd freed
 194  */
 195 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 196                              u16 idx, unsigned int *pkts_compl,
 197                              unsigned int *bytes_compl)
 198 {
 199         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 200         struct eth_tx_start_bd *tx_start_bd;
 201         struct eth_tx_bd *tx_data_bd;
 202         struct sk_buff *skb = tx_buf->skb;
 203         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 204         int nbd;
 205         u16 split_bd_len = 0;
 206
 207         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 208         prefetch(&skb->end);
 209
 210         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 211            txdata->txq_index, idx, tx_buf, skb);
 212
 213         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 214
 215         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 216 #ifdef BNX2X_STOP_ON_ERROR
 217         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 218                 BNX2X_ERR("BAD nbd!\n");
 219                 bnx2x_panic();
 220         }
 221 #endif
 222         new_cons = nbd + tx_buf->first_bd;
 223
 224         /* Get the next bd */
 225         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 226
 227         /* Skip a parse bd... */
 228         --nbd;
 229         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 230
 231         if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) {
 232                 /* Skip second parse bd... */
 233                 --nbd;
 234                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 235         }
 236
 237         /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
 238         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 239                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 240                 split_bd_len = BD_UNMAP_LEN(tx_data_bd);
 241                 --nbd;
 242                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 243         }
 244
 245         /* unmap first bd */
 246         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 247                          BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
 248                          DMA_TO_DEVICE);
 249
 250         /* now free frags */
 251         while (nbd > 0) {
 252
 253                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 254                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 255                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 256                 if (--nbd)
 257                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 258         }
 259
 260         /* release skb */
 261         WARN_ON(!skb);
 262         if (likely(skb)) {
 263                 (*pkts_compl)++;
 264                 (*bytes_compl) += skb->len;
 265         }
 266
 267         dev_kfree_skb_any(skb);
 268         tx_buf->first_bd = 0;
 269         tx_buf->skb = NULL;
 270
 271         return new_cons;
 272 }
 273
 274 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 275 {
 276         struct netdev_queue *txq;
 277         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 278         unsigned int pkts_compl = 0, bytes_compl = 0;
 279
 280 #ifdef BNX2X_STOP_ON_ERROR
 281         if (unlikely(bp->panic))
 282                 return -1;
 283 #endif
 284
 285         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 286         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 287         sw_cons = txdata->tx_pkt_cons;
 288
 289         while (sw_cons != hw_cons) {
 290                 u16 pkt_cons;
 291
 292                 pkt_cons = TX_BD(sw_cons);
 293
 294                 DP(NETIF_MSG_TX_DONE,
 295                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 296                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 297
 298                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 299                                             &pkts_compl, &bytes_compl);
 300
 301                 sw_cons++;
 302         }
 303
 304         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 305
 306         txdata->tx_pkt_cons = sw_cons;
 307         txdata->tx_bd_cons = bd_cons;
 308
 309         /* Need to make the tx_bd_cons update visible to start_xmit()
 310          * before checking for netif_tx_queue_stopped().  Without the
 311          * memory barrier, there is a small possibility that
 312          * start_xmit() will miss it and cause the queue to be stopped
 313          * forever.
 314          * On the other hand we need an rmb() here to ensure the proper
 315          * ordering of bit testing in the following
 316          * netif_tx_queue_stopped(txq) call.
 317          */
 318         smp_mb();
 319
 320         if (unlikely(netif_tx_queue_stopped(txq))) {
 321                 /* Taking tx_lock() is needed to prevent re-enabling the queue
 322                  * while it's empty. This could have happen if rx_action() gets
 323                  * suspended in bnx2x_tx_int() after the condition before
 324                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 325                  *
 326                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 327                  * sends some packets consuming the whole queue again->
 328                  * stops the queue
 329                  */
 330
 331                 __netif_tx_lock(txq, smp_processor_id());
 332
 333                 if ((netif_tx_queue_stopped(txq)) &&
 334                     (bp->state == BNX2X_STATE_OPEN) &&
 335                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 336                         netif_tx_wake_queue(txq);
 337
 338                 __netif_tx_unlock(txq);
 339         }
 340         return 0;
 341 }
 342
 343 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 344                                              u16 idx)
 345 {
 346         u16 last_max = fp->last_max_sge;
 347
 348         if (SUB_S16(idx, last_max) > 0)
 349                 fp->last_max_sge = idx;
 350 }
 351
 352 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 353                                          u16 sge_len,
 354                                          struct eth_end_agg_rx_cqe *cqe)
 355 {
 356         struct bnx2x *bp = fp->bp;
 357         u16 last_max, last_elem, first_elem;
 358         u16 delta = 0;
 359         u16 i;
 360
 361         if (!sge_len)
 362                 return;
 363
 364         /* First mark all used pages */
 365         for (i = 0; i < sge_len; i++)
 366                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 367                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 368
 369         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 370            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 371
 372         /* Here we assume that the last SGE index is the biggest */
 373         prefetch((void *)(fp->sge_mask));
 374         bnx2x_update_last_max_sge(fp,
 375                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 376
 377         last_max = RX_SGE(fp->last_max_sge);
 378         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 379         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 380
 381         /* If ring is not full */
 382         if (last_elem + 1 != first_elem)
 383                 last_elem++;
 384
 385         /* Now update the prod */
 386         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 387                 if (likely(fp->sge_mask[i]))
 388                         break;
 389
 390                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 391                 delta += BIT_VEC64_ELEM_SZ;
 392         }
 393
 394         if (delta > 0) {
 395                 fp->rx_sge_prod += delta;
 396                 /* clear page-end entries */
 397                 bnx2x_clear_sge_mask_next_elems(fp);
 398         }
 399
 400         DP(NETIF_MSG_RX_STATUS,
 401            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 402            fp->last_max_sge, fp->rx_sge_prod);
 403 }
 404
 405 /* Get Toeplitz hash value in the skb using the value from the
 406  * CQE (calculated by HW).
 407  */
 408 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 409                             const struct eth_fast_path_rx_cqe *cqe,
 410                             enum pkt_hash_types *rxhash_type)
 411 {
 412         /* Get Toeplitz hash from CQE */
 413         if ((bp->dev->features & NETIF_F_RXHASH) &&
 414             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 415                 enum eth_rss_hash_type htype;
 416
 417                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 418                 *rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
 419                                 (htype == TCP_IPV6_HASH_TYPE)) ?
 420                                PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
 421
 422                 return le32_to_cpu(cqe->rss_hash_result);
 423         }
 424         *rxhash_type = PKT_HASH_TYPE_NONE;
 425         return 0;
 426 }
 427
 428 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 429                             u16 cons, u16 prod,
 430                             struct eth_fast_path_rx_cqe *cqe)
 431 {
 432         struct bnx2x *bp = fp->bp;
 433         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 434         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 435         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 436         dma_addr_t mapping;
 437         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 438         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 439
 440         /* print error if current state != stop */
 441         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 442                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 443
 444         /* Try to map an empty data buffer from the aggregation info  */
 445         mapping = dma_map_single(&bp->pdev->dev,
 446                                  first_buf->data + NET_SKB_PAD,
 447                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 448         /*
 449          *  ...if it fails - move the skb from the consumer to the producer
 450          *  and set the current aggregation state as ERROR to drop it
 451          *  when TPA_STOP arrives.
 452          */
 453
 454         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 455                 /* Move the BD from the consumer to the producer */
 456                 bnx2x_reuse_rx_data(fp, cons, prod);
 457                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 458                 return;
 459         }
 460
 461         /* move empty data from pool to prod */
 462         prod_rx_buf->data = first_buf->data;
 463         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 464         /* point prod_bd to new data */
 465         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 466         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 467
 468         /* move partial skb from cons to pool (don't unmap yet) */
 469         *first_buf = *cons_rx_buf;
 470
 471         /* mark bin state as START */
 472         tpa_info->parsing_flags =
 473                 le16_to_cpu(cqe->pars_flags.flags);
 474         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 475         tpa_info->tpa_state = BNX2X_TPA_START;
 476         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 477         tpa_info->placement_offset = cqe->placement_offset;
 478         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
 479         if (fp->mode == TPA_MODE_GRO) {
 480                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 481                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 482                 tpa_info->gro_size = gro_size;
 483         }
 484
 485 #ifdef BNX2X_STOP_ON_ERROR
 486         fp->tpa_queue_used |= (1 << queue);
 487         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 488            fp->tpa_queue_used);
 489 #endif
 490 }
 491
 492 /* Timestamp option length allowed for TPA aggregation:
 493  *
 494  *              nop nop kind length echo val
 495  */
 496 #define TPA_TSTAMP_OPT_LEN      12
 497 /**
 498  * bnx2x_set_gro_params - compute GRO values
 499  *
 500  * @skb:                packet skb
 501  * @parsing_flags:      parsing flags from the START CQE
 502  * @len_on_bd:          total length of the first packet for the
 503  *                      aggregation.
 504  * @pkt_len:            length of all segments
 505  *
 506  * Approximate value of the MSS for this aggregation calculated using
 507  * the first packet of it.
 508  * Compute number of aggregated segments, and gso_type.
 509  */
 510 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 511                                  u16 len_on_bd, unsigned int pkt_len,
 512                                  u16 num_of_coalesced_segs)
 513 {
 514         /* TPA aggregation won't have either IP options or TCP options
 515          * other than timestamp or IPv6 extension headers.
 516          */
 517         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 518
 519         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 520             PRS_FLAG_OVERETH_IPV6) {
 521                 hdrs_len += sizeof(struct ipv6hdr);
 522                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 523         } else {
 524                 hdrs_len += sizeof(struct iphdr);
 525                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 526         }
 527
 528         /* Check if there was a TCP timestamp, if there is it's will
 529          * always be 12 bytes length: nop nop kind length echo val.
 530          *
 531          * Otherwise FW would close the aggregation.
 532          */
 533         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 534                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 535
 536         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 537
 538         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 539          * to skb_shinfo(skb)->gso_segs
 540          */
 541         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 542 }
 543
 544 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 545                               u16 index, gfp_t gfp_mask)
 546 {
 547         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 548         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 549         struct bnx2x_alloc_pool *pool = &fp->page_pool;
 550         dma_addr_t mapping;
 551
 552         if (!pool->page || (PAGE_SIZE - pool->offset) < SGE_PAGE_SIZE) {
 553
 554                 /* put page reference used by the memory pool, since we
 555                  * won't be using this page as the mempool anymore.
 556                  */
 557                 if (pool->page)
 558                         put_page(pool->page);
 559
 560                 pool->page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
 561                 if (unlikely(!pool->page)) {
 562                         BNX2X_ERR("Can't alloc sge\n");
 563                         return -ENOMEM;
 564                 }
 565
 566                 pool->dma = dma_map_page(&bp->pdev->dev, pool->page, 0,
 567                                          PAGE_SIZE, DMA_FROM_DEVICE);
 568                 if (unlikely(dma_mapping_error(&bp->pdev->dev,
 569                                                pool->dma))) {
 570                         __free_pages(pool->page, PAGES_PER_SGE_SHIFT);
 571                         pool->page = NULL;
 572                         BNX2X_ERR("Can't map sge\n");
 573                         return -ENOMEM;
 574                 }
 575                 pool->offset = 0;
 576         }
 577
 578         get_page(pool->page);
 579         sw_buf->page = pool->page;
 580         sw_buf->offset = pool->offset;
 581
 582         mapping = pool->dma + sw_buf->offset;
 583         dma_unmap_addr_set(sw_buf, mapping, mapping);
 584
 585         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 586         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 587
 588         pool->offset += SGE_PAGE_SIZE;
 589
 590         return 0;
 591 }
 592
 593 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 594                                struct bnx2x_agg_info *tpa_info,
 595                                u16 pages,
 596                                struct sk_buff *skb,
 597                                struct eth_end_agg_rx_cqe *cqe,
 598                                u16 cqe_idx)
 599 {
 600         struct sw_rx_page *rx_pg, old_rx_pg;
 601         u32 i, frag_len, frag_size;
 602         int err, j, frag_id = 0;
 603         u16 len_on_bd = tpa_info->len_on_bd;
 604         u16 full_page = 0, gro_size = 0;
 605
 606         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 607
 608         if (fp->mode == TPA_MODE_GRO) {
 609                 gro_size = tpa_info->gro_size;
 610                 full_page = tpa_info->full_page;
 611         }
 612
 613         /* This is needed in order to enable forwarding support */
 614         if (frag_size)
 615                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 616                                      le16_to_cpu(cqe->pkt_len),
 617                                      le16_to_cpu(cqe->num_of_coalesced_segs));
 618
 619 #ifdef BNX2X_STOP_ON_ERROR
 620         if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 621                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 622                           pages, cqe_idx);
 623                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 624                 bnx2x_panic();
 625                 return -EINVAL;
 626         }
 627 #endif
 628
 629         /* Run through the SGL and compose the fragmented skb */
 630         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 631                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 632
 633                 /* FW gives the indices of the SGE as if the ring is an array
 634                    (meaning that "next" element will consume 2 indices) */
 635                 if (fp->mode == TPA_MODE_GRO)
 636                         frag_len = min_t(u32, frag_size, (u32)full_page);
 637                 else /* LRO */
 638                         frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 639
 640                 rx_pg = &fp->rx_page_ring[sge_idx];
 641                 old_rx_pg = *rx_pg;
 642
 643                 /* If we fail to allocate a substitute page, we simply stop
 644                    where we are and drop the whole packet */
 645                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
 646                 if (unlikely(err)) {
 647                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 648                         return err;
 649                 }
 650
 651                 dma_unmap_single(&bp->pdev->dev,
 652                                  dma_unmap_addr(&old_rx_pg, mapping),
 653                                  SGE_PAGE_SIZE, DMA_FROM_DEVICE);
 654                 /* Add one frag and update the appropriate fields in the skb */
 655                 if (fp->mode == TPA_MODE_LRO)
 656                         skb_fill_page_desc(skb, j, old_rx_pg.page,
 657                                            old_rx_pg.offset, frag_len);
 658                 else { /* GRO */
 659                         int rem;
 660                         int offset = 0;
 661                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 662                                 int len = rem > gro_size ? gro_size : rem;
 663                                 skb_fill_page_desc(skb, frag_id++,
 664                                                    old_rx_pg.page,
 665                                                    old_rx_pg.offset + offset,
 666                                                    len);
 667                                 if (offset)
 668                                         get_page(old_rx_pg.page);
 669                                 offset += len;
 670                         }
 671                 }
 672
 673                 skb->data_len += frag_len;
 674                 skb->truesize += SGE_PAGES;
 675                 skb->len += frag_len;
 676
 677                 frag_size -= frag_len;
 678         }
 679
 680         return 0;
 681 }
 682
 683 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 684 {
 685         if (fp->rx_frag_size)
 686                 skb_free_frag(data);
 687         else
 688                 kfree(data);
 689 }
 690
 691 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
 692 {
 693         if (fp->rx_frag_size) {
 694                 /* GFP_KERNEL allocations are used only during initialization */
 695                 if (unlikely(gfp_mask & __GFP_WAIT))
 696                         return (void *)__get_free_page(gfp_mask);
 697
 698                 return netdev_alloc_frag(fp->rx_frag_size);
 699         }
 700
 701         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
 702 }
 703
 704 #ifdef CONFIG_INET
 705 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 706 {
 707         const struct iphdr *iph = ip_hdr(skb);
 708         struct tcphdr *th;
 709
 710         skb_set_transport_header(skb, sizeof(struct iphdr));
 711         th = tcp_hdr(skb);
 712
 713         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 714                                   iph->saddr, iph->daddr, 0);
 715 }
 716
 717 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 718 {
 719         struct ipv6hdr *iph = ipv6_hdr(skb);
 720         struct tcphdr *th;
 721
 722         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 723         th = tcp_hdr(skb);
 724
 725         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 726                                   &iph->saddr, &iph->daddr, 0);
 727 }
 728
 729 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
 730                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
 731 {
 732         skb_set_network_header(skb, 0);
 733         gro_func(bp, skb);
 734         tcp_gro_complete(skb);
 735 }
 736 #endif
 737
 738 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 739                                struct sk_buff *skb)
 740 {
 741 #ifdef CONFIG_INET
 742         if (skb_shinfo(skb)->gso_size) {
 743                 switch (be16_to_cpu(skb->protocol)) {
 744                 case ETH_P_IP:
 745                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
 746                         break;
 747                 case ETH_P_IPV6:
 748                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 749                         break;
 750                 default:
 751                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 752                                   be16_to_cpu(skb->protocol));
 753                 }
 754         }
 755 #endif
 756         skb_record_rx_queue(skb, fp->rx_queue);
 757         napi_gro_receive(&fp->napi, skb);
 758 }
 759
 760 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 761                            struct bnx2x_agg_info *tpa_info,
 762                            u16 pages,
 763                            struct eth_end_agg_rx_cqe *cqe,
 764                            u16 cqe_idx)
 765 {
 766         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 767         u8 pad = tpa_info->placement_offset;
 768         u16 len = tpa_info->len_on_bd;
 769         struct sk_buff *skb = NULL;
 770         u8 *new_data, *data = rx_buf->data;
 771         u8 old_tpa_state = tpa_info->tpa_state;
 772
 773         tpa_info->tpa_state = BNX2X_TPA_STOP;
 774
 775         /* If we there was an error during the handling of the TPA_START -
 776          * drop this aggregation.
 777          */
 778         if (old_tpa_state == BNX2X_TPA_ERROR)
 779                 goto drop;
 780
 781         /* Try to allocate the new data */
 782         new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
 783         /* Unmap skb in the pool anyway, as we are going to change
 784            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 785            fails. */
 786         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 787                          fp->rx_buf_size, DMA_FROM_DEVICE);
 788         if (likely(new_data))
 789                 skb = build_skb(data, fp->rx_frag_size);
 790
 791         if (likely(skb)) {
 792 #ifdef BNX2X_STOP_ON_ERROR
 793                 if (pad + len > fp->rx_buf_size) {
 794                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 795                                   pad, len, fp->rx_buf_size);
 796                         bnx2x_panic();
 797                         return;
 798                 }
 799 #endif
 800
 801                 skb_reserve(skb, pad + NET_SKB_PAD);
 802                 skb_put(skb, len);
 803                 skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
 804
 805                 skb->protocol = eth_type_trans(skb, bp->dev);
 806                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 807
 808                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 809                                          skb, cqe, cqe_idx)) {
 810                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 811                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
 812                         bnx2x_gro_receive(bp, fp, skb);
 813                 } else {
 814                         DP(NETIF_MSG_RX_STATUS,
 815                            "Failed to allocate new pages - dropping packet!\n");
 816                         dev_kfree_skb_any(skb);
 817                 }
 818
 819                 /* put new data in bin */
 820                 rx_buf->data = new_data;
 821
 822                 return;
 823         }
 824         if (new_data)
 825                 bnx2x_frag_free(fp, new_data);
 826 drop:
 827         /* drop the packet and keep the buffer in the bin */
 828         DP(NETIF_MSG_RX_STATUS,
 829            "Failed to allocate or map a new skb - dropping packet!\n");
 830         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 831 }
 832
 833 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 834                                u16 index, gfp_t gfp_mask)
 835 {
 836         u8 *data;
 837         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 838         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 839         dma_addr_t mapping;
 840
 841         data = bnx2x_frag_alloc(fp, gfp_mask);
 842         if (unlikely(data == NULL))
 843                 return -ENOMEM;
 844
 845         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 846                                  fp->rx_buf_size,
 847                                  DMA_FROM_DEVICE);
 848         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 849                 bnx2x_frag_free(fp, data);
 850                 BNX2X_ERR("Can't map rx data\n");
 851                 return -ENOMEM;
 852         }
 853
 854         rx_buf->data = data;
 855         dma_unmap_addr_set(rx_buf, mapping, mapping);
 856
 857         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 858         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 859
 860         return 0;
 861 }
 862
 863 static
 864 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 865                                  struct bnx2x_fastpath *fp,
 866                                  struct bnx2x_eth_q_stats *qstats)
 867 {
 868         /* Do nothing if no L4 csum validation was done.
 869          * We do not check whether IP csum was validated. For IPv4 we assume
 870          * that if the card got as far as validating the L4 csum, it also
 871          * validated the IP csum. IPv6 has no IP csum.
 872          */
 873         if (cqe->fast_path_cqe.status_flags &
 874             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 875                 return;
 876
 877         /* If L4 validation was done, check if an error was found. */
 878
 879         if (cqe->fast_path_cqe.type_error_flags &
 880             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 881              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 882                 qstats->hw_csum_err++;
 883         else
 884                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 885 }
 886
 887 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 888 {
 889         struct bnx2x *bp = fp->bp;
 890         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 891         u16 sw_comp_cons, sw_comp_prod;
 892         int rx_pkt = 0;
 893         union eth_rx_cqe *cqe;
 894         struct eth_fast_path_rx_cqe *cqe_fp;
 895
 896 #ifdef BNX2X_STOP_ON_ERROR
 897         if (unlikely(bp->panic))
 898                 return 0;
 899 #endif
 900         if (budget <= 0)
 901                 return rx_pkt;
 902
 903         bd_cons = fp->rx_bd_cons;
 904         bd_prod = fp->rx_bd_prod;
 905         bd_prod_fw = bd_prod;
 906         sw_comp_cons = fp->rx_comp_cons;
 907         sw_comp_prod = fp->rx_comp_prod;
 908
 909         comp_ring_cons = RCQ_BD(sw_comp_cons);
 910         cqe = &fp->rx_comp_ring[comp_ring_cons];
 911         cqe_fp = &cqe->fast_path_cqe;
 912
 913         DP(NETIF_MSG_RX_STATUS,
 914            "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
 915
 916         while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
 917                 struct sw_rx_bd *rx_buf = NULL;
 918                 struct sk_buff *skb;
 919                 u8 cqe_fp_flags;
 920                 enum eth_rx_cqe_type cqe_fp_type;
 921                 u16 len, pad, queue;
 922                 u8 *data;
 923                 u32 rxhash;
 924                 enum pkt_hash_types rxhash_type;
 925
 926 #ifdef BNX2X_STOP_ON_ERROR
 927                 if (unlikely(bp->panic))
 928                         return 0;
 929 #endif
 930
 931                 bd_prod = RX_BD(bd_prod);
 932                 bd_cons = RX_BD(bd_cons);
 933
 934                 /* A rmb() is required to ensure that the CQE is not read
 935                  * before it is written by the adapter DMA.  PCI ordering
 936                  * rules will make sure the other fields are written before
 937                  * the marker at the end of struct eth_fast_path_rx_cqe
 938                  * but without rmb() a weakly ordered processor can process
 939                  * stale data.  Without the barrier TPA state-machine might
 940                  * enter inconsistent state and kernel stack might be
 941                  * provided with incorrect packet description - these lead
 942                  * to various kernel crashed.
 943                  */
 944                 rmb();
 945
 946                 cqe_fp_flags = cqe_fp->type_error_flags;
 947                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 948
 949                 DP(NETIF_MSG_RX_STATUS,
 950                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 951                    CQE_TYPE(cqe_fp_flags),
 952                    cqe_fp_flags, cqe_fp->status_flags,
 953                    le32_to_cpu(cqe_fp->rss_hash_result),
 954                    le16_to_cpu(cqe_fp->vlan_tag),
 955                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 956
 957                 /* is this a slowpath msg? */
 958                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 959                         bnx2x_sp_event(fp, cqe);
 960                         goto next_cqe;
 961                 }
 962
 963                 rx_buf = &fp->rx_buf_ring[bd_cons];
 964                 data = rx_buf->data;
 965
 966                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 967                         struct bnx2x_agg_info *tpa_info;
 968                         u16 frag_size, pages;
 969 #ifdef BNX2X_STOP_ON_ERROR
 970                         /* sanity check */
 971                         if (fp->mode == TPA_MODE_DISABLED &&
 972                             (CQE_TYPE_START(cqe_fp_type) ||
 973                              CQE_TYPE_STOP(cqe_fp_type)))
 974                                 BNX2X_ERR("START/STOP packet while TPA disabled, type %x\n",
 975                                           CQE_TYPE(cqe_fp_type));
 976 #endif
 977
 978                         if (CQE_TYPE_START(cqe_fp_type)) {
 979                                 u16 queue = cqe_fp->queue_index;
 980                                 DP(NETIF_MSG_RX_STATUS,
 981                                    "calling tpa_start on queue %d\n",
 982                                    queue);
 983
 984                                 bnx2x_tpa_start(fp, queue,
 985                                                 bd_cons, bd_prod,
 986                                                 cqe_fp);
 987
 988                                 goto next_rx;
 989                         }
 990                         queue = cqe->end_agg_cqe.queue_index;
 991                         tpa_info = &fp->tpa_info[queue];
 992                         DP(NETIF_MSG_RX_STATUS,
 993                            "calling tpa_stop on queue %d\n",
 994                            queue);
 995
 996                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 997                                     tpa_info->len_on_bd;
 998
 999                         if (fp->mode == TPA_MODE_GRO)
1000                                 pages = (frag_size + tpa_info->full_page - 1) /
1001                                          tpa_info->full_page;
1002                         else
1003                                 pages = SGE_PAGE_ALIGN(frag_size) >>
1004                                         SGE_PAGE_SHIFT;
1005
1006                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
1007                                        &cqe->end_agg_cqe, comp_ring_cons);
1008 #ifdef BNX2X_STOP_ON_ERROR
1009                         if (bp->panic)
1010                                 return 0;
1011 #endif
1012
1013                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
1014                         goto next_cqe;
1015                 }
1016                 /* non TPA */
1017                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
1018                 pad = cqe_fp->placement_offset;
1019                 dma_sync_single_for_cpu(&bp->pdev->dev,
1020                                         dma_unmap_addr(rx_buf, mapping),
1021                                         pad + RX_COPY_THRESH,
1022                                         DMA_FROM_DEVICE);
1023                 pad += NET_SKB_PAD;
1024                 prefetch(data + pad); /* speedup eth_type_trans() */
1025                 /* is this an error packet? */
1026                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1027                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1028                            "ERROR  flags %x  rx packet %u\n",
1029                            cqe_fp_flags, sw_comp_cons);
1030                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1031                         goto reuse_rx;
1032                 }
1033
1034                 /* Since we don't have a jumbo ring
1035                  * copy small packets if mtu > 1500
1036                  */
1037                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1038                     (len <= RX_COPY_THRESH)) {
1039                         skb = napi_alloc_skb(&fp->napi, len);
1040                         if (skb == NULL) {
1041                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1042                                    "ERROR  packet dropped because of alloc failure\n");
1043                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1044                                 goto reuse_rx;
1045                         }
1046                         memcpy(skb->data, data + pad, len);
1047                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1048                 } else {
1049                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1050                                                        GFP_ATOMIC) == 0)) {
1051                                 dma_unmap_single(&bp->pdev->dev,
1052                                                  dma_unmap_addr(rx_buf, mapping),
1053                                                  fp->rx_buf_size,
1054                                                  DMA_FROM_DEVICE);
1055                                 skb = build_skb(data, fp->rx_frag_size);
1056                                 if (unlikely(!skb)) {
1057                                         bnx2x_frag_free(fp, data);
1058                                         bnx2x_fp_qstats(bp, fp)->
1059                                                         rx_skb_alloc_failed++;
1060                                         goto next_rx;
1061                                 }
1062                                 skb_reserve(skb, pad);
1063                         } else {
1064                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1065                                    "ERROR  packet dropped because of alloc failure\n");
1066                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1067 reuse_rx:
1068                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1069                                 goto next_rx;
1070                         }
1071                 }
1072
1073                 skb_put(skb, len);
1074                 skb->protocol = eth_type_trans(skb, bp->dev);
1075
1076                 /* Set Toeplitz hash for a none-LRO skb */
1077                 rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1078                 skb_set_hash(skb, rxhash, rxhash_type);
1079
1080                 skb_checksum_none_assert(skb);
1081
1082                 if (bp->dev->features & NETIF_F_RXCSUM)
1083                         bnx2x_csum_validate(skb, cqe, fp,
1084                                             bnx2x_fp_qstats(bp, fp));
1085
1086                 skb_record_rx_queue(skb, fp->rx_queue);
1087
1088                 /* Check if this packet was timestamped */
1089                 if (unlikely(cqe->fast_path_cqe.type_error_flags &
1090                              (1 << ETH_FAST_PATH_RX_CQE_PTP_PKT_SHIFT)))
1091                         bnx2x_set_rx_ts(bp, skb);
1092
1093                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1094                     PARSING_FLAGS_VLAN)
1095                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1096                                                le16_to_cpu(cqe_fp->vlan_tag));
1097
1098                 skb_mark_napi_id(skb, &fp->napi);
1099
1100                 if (bnx2x_fp_ll_polling(fp))
1101                         netif_receive_skb(skb);
1102                 else
1103                         napi_gro_receive(&fp->napi, skb);
1104 next_rx:
1105                 rx_buf->data = NULL;
1106
1107                 bd_cons = NEXT_RX_IDX(bd_cons);
1108                 bd_prod = NEXT_RX_IDX(bd_prod);
1109                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1110                 rx_pkt++;
1111 next_cqe:
1112                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1113                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1114
1115                 /* mark CQE as free */
1116                 BNX2X_SEED_CQE(cqe_fp);
1117
1118                 if (rx_pkt == budget)
1119                         break;
1120
1121                 comp_ring_cons = RCQ_BD(sw_comp_cons);
1122                 cqe = &fp->rx_comp_ring[comp_ring_cons];
1123                 cqe_fp = &cqe->fast_path_cqe;
1124         } /* while */
1125
1126         fp->rx_bd_cons = bd_cons;
1127         fp->rx_bd_prod = bd_prod_fw;
1128         fp->rx_comp_cons = sw_comp_cons;
1129         fp->rx_comp_prod = sw_comp_prod;
1130
1131         /* Update producers */
1132         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1133                              fp->rx_sge_prod);
1134
1135         fp->rx_pkt += rx_pkt;
1136         fp->rx_calls++;
1137
1138         return rx_pkt;
1139 }
1140
1141 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1142 {
1143         struct bnx2x_fastpath *fp = fp_cookie;
1144         struct bnx2x *bp = fp->bp;
1145         u8 cos;
1146
1147         DP(NETIF_MSG_INTR,
1148            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1149            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1150
1151         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1152
1153 #ifdef BNX2X_STOP_ON_ERROR
1154         if (unlikely(bp->panic))
1155                 return IRQ_HANDLED;
1156 #endif
1157
1158         /* Handle Rx and Tx according to MSI-X vector */
1159         for_each_cos_in_tx_queue(fp, cos)
1160                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1161
1162         prefetch(&fp->sb_running_index[SM_RX_ID]);
1163         napi_schedule_irqoff(&bnx2x_fp(bp, fp->index, napi));
1164
1165         return IRQ_HANDLED;
1166 }
1167
1168 /* HW Lock for shared dual port PHYs */
1169 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1170 {
1171         mutex_lock(&bp->port.phy_mutex);
1172
1173         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1174 }
1175
1176 void bnx2x_release_phy_lock(struct bnx2x *bp)
1177 {
1178         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1179
1180         mutex_unlock(&bp->port.phy_mutex);
1181 }
1182
1183 /* calculates MF speed according to current linespeed and MF configuration */
1184 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1185 {
1186         u16 line_speed = bp->link_vars.line_speed;
1187         if (IS_MF(bp)) {
1188                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1189                                                    bp->mf_config[BP_VN(bp)]);
1190
1191                 /* Calculate the current MAX line speed limit for the MF
1192                  * devices
1193                  */
1194                 if (IS_MF_SI(bp))
1195                         line_speed = (line_speed * maxCfg) / 100;
1196                 else { /* SD mode */
1197                         u16 vn_max_rate = maxCfg * 100;
1198
1199                         if (vn_max_rate < line_speed)
1200                                 line_speed = vn_max_rate;
1201                 }
1202         }
1203
1204         return line_speed;
1205 }
1206
1207 /**
1208  * bnx2x_fill_report_data - fill link report data to report
1209  *
1210  * @bp:         driver handle
1211  * @data:       link state to update
1212  *
1213  * It uses a none-atomic bit operations because is called under the mutex.
1214  */
1215 static void bnx2x_fill_report_data(struct bnx2x *bp,
1216                                    struct bnx2x_link_report_data *data)
1217 {
1218         memset(data, 0, sizeof(*data));
1219
1220         if (IS_PF(bp)) {
1221                 /* Fill the report data: effective line speed */
1222                 data->line_speed = bnx2x_get_mf_speed(bp);
1223
1224                 /* Link is down */
1225                 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1226                         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1227                                   &data->link_report_flags);
1228
1229                 if (!BNX2X_NUM_ETH_QUEUES(bp))
1230                         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1231                                   &data->link_report_flags);
1232
1233                 /* Full DUPLEX */
1234                 if (bp->link_vars.duplex == DUPLEX_FULL)
1235                         __set_bit(BNX2X_LINK_REPORT_FD,
1236                                   &data->link_report_flags);
1237
1238                 /* Rx Flow Control is ON */
1239                 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1240                         __set_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1241                                   &data->link_report_flags);
1242
1243                 /* Tx Flow Control is ON */
1244                 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1245                         __set_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1246                                   &data->link_report_flags);
1247         } else { /* VF */
1248                 *data = bp->vf_link_vars;
1249         }
1250 }
1251
1252 /**
1253  * bnx2x_link_report - report link status to OS.
1254  *
1255  * @bp:         driver handle
1256  *
1257  * Calls the __bnx2x_link_report() under the same locking scheme
1258  * as a link/PHY state managing code to ensure a consistent link
1259  * reporting.
1260  */
1261
1262 void bnx2x_link_report(struct bnx2x *bp)
1263 {
1264         bnx2x_acquire_phy_lock(bp);
1265         __bnx2x_link_report(bp);
1266         bnx2x_release_phy_lock(bp);
1267 }
1268
1269 /**
1270  * __bnx2x_link_report - report link status to OS.
1271  *
1272  * @bp:         driver handle
1273  *
1274  * None atomic implementation.
1275  * Should be called under the phy_lock.
1276  */
1277 void __bnx2x_link_report(struct bnx2x *bp)
1278 {
1279         struct bnx2x_link_report_data cur_data;
1280
1281         /* reread mf_cfg */
1282         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1283                 bnx2x_read_mf_cfg(bp);
1284
1285         /* Read the current link report info */
1286         bnx2x_fill_report_data(bp, &cur_data);
1287
1288         /* Don't report link down or exactly the same link status twice */
1289         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1290             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1291                       &bp->last_reported_link.link_report_flags) &&
1292              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1293                       &cur_data.link_report_flags)))
1294                 return;
1295
1296         bp->link_cnt++;
1297
1298         /* We are going to report a new link parameters now -
1299          * remember the current data for the next time.
1300          */
1301         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1302
1303         /* propagate status to VFs */
1304         if (IS_PF(bp))
1305                 bnx2x_iov_link_update(bp);
1306
1307         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1308                      &cur_data.link_report_flags)) {
1309                 netif_carrier_off(bp->dev);
1310                 netdev_err(bp->dev, "NIC Link is Down\n");
1311                 return;
1312         } else {
1313                 const char *duplex;
1314                 const char *flow;
1315
1316                 netif_carrier_on(bp->dev);
1317
1318                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1319                                        &cur_data.link_report_flags))
1320                         duplex = "full";
1321                 else
1322                         duplex = "half";
1323
1324                 /* Handle the FC at the end so that only these flags would be
1325                  * possibly set. This way we may easily check if there is no FC
1326                  * enabled.
1327                  */
1328                 if (cur_data.link_report_flags) {
1329                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1330                                      &cur_data.link_report_flags)) {
1331                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1332                                      &cur_data.link_report_flags))
1333                                         flow = "ON - receive & transmit";
1334                                 else
1335                                         flow = "ON - receive";
1336                         } else {
1337                                 flow = "ON - transmit";
1338                         }
1339                 } else {
1340                         flow = "none";
1341                 }
1342                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1343                             cur_data.line_speed, duplex, flow);
1344         }
1345 }
1346
1347 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1348 {
1349         int i;
1350
1351         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1352                 struct eth_rx_sge *sge;
1353
1354                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1355                 sge->addr_hi =
1356                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1357                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1358
1359                 sge->addr_lo =
1360                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1361                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1362         }
1363 }
1364
1365 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1366                                 struct bnx2x_fastpath *fp, int last)
1367 {
1368         int i;
1369
1370         for (i = 0; i < last; i++) {
1371                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1372                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1373                 u8 *data = first_buf->data;
1374
1375                 if (data == NULL) {
1376                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1377                         continue;
1378                 }
1379                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1380                         dma_unmap_single(&bp->pdev->dev,
1381                                          dma_unmap_addr(first_buf, mapping),
1382                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1383                 bnx2x_frag_free(fp, data);
1384                 first_buf->data = NULL;
1385         }
1386 }
1387
1388 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1389 {
1390         int j;
1391
1392         for_each_rx_queue_cnic(bp, j) {
1393                 struct bnx2x_fastpath *fp = &bp->fp[j];
1394
1395                 fp->rx_bd_cons = 0;
1396
1397                 /* Activate BD ring */
1398                 /* Warning!
1399                  * this will generate an interrupt (to the TSTORM)
1400                  * must only be done after chip is initialized
1401                  */
1402                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1403                                      fp->rx_sge_prod);
1404         }
1405 }
1406
1407 void bnx2x_init_rx_rings(struct bnx2x *bp)
1408 {
1409         int func = BP_FUNC(bp);
1410         u16 ring_prod;
1411         int i, j;
1412
1413         /* Allocate TPA resources */
1414         for_each_eth_queue(bp, j) {
1415                 struct bnx2x_fastpath *fp = &bp->fp[j];
1416
1417                 DP(NETIF_MSG_IFUP,
1418                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1419
1420                 if (fp->mode != TPA_MODE_DISABLED) {
1421                         /* Fill the per-aggregation pool */
1422                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1423                                 struct bnx2x_agg_info *tpa_info =
1424                                         &fp->tpa_info[i];
1425                                 struct sw_rx_bd *first_buf =
1426                                         &tpa_info->first_buf;
1427
1428                                 first_buf->data =
1429                                         bnx2x_frag_alloc(fp, GFP_KERNEL);
1430                                 if (!first_buf->data) {
1431                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1432                                                   j);
1433                                         bnx2x_free_tpa_pool(bp, fp, i);
1434                                         fp->mode = TPA_MODE_DISABLED;
1435                                         break;
1436                                 }
1437                                 dma_unmap_addr_set(first_buf, mapping, 0);
1438                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1439                         }
1440
1441                         /* "next page" elements initialization */
1442                         bnx2x_set_next_page_sgl(fp);
1443
1444                         /* set SGEs bit mask */
1445                         bnx2x_init_sge_ring_bit_mask(fp);
1446
1447                         /* Allocate SGEs and initialize the ring elements */
1448                         for (i = 0, ring_prod = 0;
1449                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1450
1451                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1452                                                        GFP_KERNEL) < 0) {
1453                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1454                                                   i);
1455                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1456                                                   j);
1457                                         /* Cleanup already allocated elements */
1458                                         bnx2x_free_rx_sge_range(bp, fp,
1459                                                                 ring_prod);
1460                                         bnx2x_free_tpa_pool(bp, fp,
1461                                                             MAX_AGG_QS(bp));
1462                                         fp->mode = TPA_MODE_DISABLED;
1463                                         ring_prod = 0;
1464                                         break;
1465                                 }
1466                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1467                         }
1468
1469                         fp->rx_sge_prod = ring_prod;
1470                 }
1471         }
1472
1473         for_each_eth_queue(bp, j) {
1474                 struct bnx2x_fastpath *fp = &bp->fp[j];
1475
1476                 fp->rx_bd_cons = 0;
1477
1478                 /* Activate BD ring */
1479                 /* Warning!
1480                  * this will generate an interrupt (to the TSTORM)
1481                  * must only be done after chip is initialized
1482                  */
1483                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1484                                      fp->rx_sge_prod);
1485
1486                 if (j != 0)
1487                         continue;
1488
1489                 if (CHIP_IS_E1(bp)) {
1490                         REG_WR(bp, BAR_USTRORM_INTMEM +
1491                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1492                                U64_LO(fp->rx_comp_mapping));
1493                         REG_WR(bp, BAR_USTRORM_INTMEM +
1494                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1495                                U64_HI(fp->rx_comp_mapping));
1496                 }
1497         }
1498 }
1499
1500 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1501 {
1502         u8 cos;
1503         struct bnx2x *bp = fp->bp;
1504
1505         for_each_cos_in_tx_queue(fp, cos) {
1506                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1507                 unsigned pkts_compl = 0, bytes_compl = 0;
1508
1509                 u16 sw_prod = txdata->tx_pkt_prod;
1510                 u16 sw_cons = txdata->tx_pkt_cons;
1511
1512                 while (sw_cons != sw_prod) {
1513                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1514                                           &pkts_compl, &bytes_compl);
1515                         sw_cons++;
1516                 }
1517
1518                 netdev_tx_reset_queue(
1519                         netdev_get_tx_queue(bp->dev,
1520                                             txdata->txq_index));
1521         }
1522 }
1523
1524 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1525 {
1526         int i;
1527
1528         for_each_tx_queue_cnic(bp, i) {
1529                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1530         }
1531 }
1532
1533 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1534 {
1535         int i;
1536
1537         for_each_eth_queue(bp, i) {
1538                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1539         }
1540 }
1541
1542 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1543 {
1544         struct bnx2x *bp = fp->bp;
1545         int i;
1546
1547         /* ring wasn't allocated */
1548         if (fp->rx_buf_ring == NULL)
1549                 return;
1550
1551         for (i = 0; i < NUM_RX_BD; i++) {
1552                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1553                 u8 *data = rx_buf->data;
1554
1555                 if (data == NULL)
1556                         continue;
1557                 dma_unmap_single(&bp->pdev->dev,
1558                                  dma_unmap_addr(rx_buf, mapping),
1559                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1560
1561                 rx_buf->data = NULL;
1562                 bnx2x_frag_free(fp, data);
1563         }
1564 }
1565
1566 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1567 {
1568         int j;
1569
1570         for_each_rx_queue_cnic(bp, j) {
1571                 bnx2x_free_rx_bds(&bp->fp[j]);
1572         }
1573 }
1574
1575 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1576 {
1577         int j;
1578
1579         for_each_eth_queue(bp, j) {
1580                 struct bnx2x_fastpath *fp = &bp->fp[j];
1581
1582                 bnx2x_free_rx_bds(fp);
1583
1584                 if (fp->mode != TPA_MODE_DISABLED)
1585                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1586         }
1587 }
1588
1589 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1590 {
1591         bnx2x_free_tx_skbs_cnic(bp);
1592         bnx2x_free_rx_skbs_cnic(bp);
1593 }
1594
1595 void bnx2x_free_skbs(struct bnx2x *bp)
1596 {
1597         bnx2x_free_tx_skbs(bp);
1598         bnx2x_free_rx_skbs(bp);
1599 }
1600
1601 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1602 {
1603         /* load old values */
1604         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1605
1606         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1607                 /* leave all but MAX value */
1608                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1609
1610                 /* set new MAX value */
1611                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1612                                 & FUNC_MF_CFG_MAX_BW_MASK;
1613
1614                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1615         }
1616 }
1617
1618 /**
1619  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1620  *
1621  * @bp:         driver handle
1622  * @nvecs:      number of vectors to be released
1623  */
1624 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1625 {
1626         int i, offset = 0;
1627
1628         if (nvecs == offset)
1629                 return;
1630
1631         /* VFs don't have a default SB */
1632         if (IS_PF(bp)) {
1633                 free_irq(bp->msix_table[offset].vector, bp->dev);
1634                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1635                    bp->msix_table[offset].vector);
1636                 offset++;
1637         }
1638
1639         if (CNIC_SUPPORT(bp)) {
1640                 if (nvecs == offset)
1641                         return;
1642                 offset++;
1643         }
1644
1645         for_each_eth_queue(bp, i) {
1646                 if (nvecs == offset)
1647                         return;
1648                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1649                    i, bp->msix_table[offset].vector);
1650
1651                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1652         }
1653 }
1654
1655 void bnx2x_free_irq(struct bnx2x *bp)
1656 {
1657         if (bp->flags & USING_MSIX_FLAG &&
1658             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1659                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1660
1661                 /* vfs don't have a default status block */
1662                 if (IS_PF(bp))
1663                         nvecs++;
1664
1665                 bnx2x_free_msix_irqs(bp, nvecs);
1666         } else {
1667                 free_irq(bp->dev->irq, bp->dev);
1668         }
1669 }
1670
1671 int bnx2x_enable_msix(struct bnx2x *bp)
1672 {
1673         int msix_vec = 0, i, rc;
1674
1675         /* VFs don't have a default status block */
1676         if (IS_PF(bp)) {
1677                 bp->msix_table[msix_vec].entry = msix_vec;
1678                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1679                                bp->msix_table[0].entry);
1680                 msix_vec++;
1681         }
1682
1683         /* Cnic requires an msix vector for itself */
1684         if (CNIC_SUPPORT(bp)) {
1685                 bp->msix_table[msix_vec].entry = msix_vec;
1686                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1687                                msix_vec, bp->msix_table[msix_vec].entry);
1688                 msix_vec++;
1689         }
1690
1691         /* We need separate vectors for ETH queues only (not FCoE) */
1692         for_each_eth_queue(bp, i) {
1693                 bp->msix_table[msix_vec].entry = msix_vec;
1694                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1695                                msix_vec, msix_vec, i);
1696                 msix_vec++;
1697         }
1698
1699         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1700            msix_vec);
1701
1702         rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1703                                    BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1704         /*
1705          * reconfigure number of tx/rx queues according to available
1706          * MSI-X vectors
1707          */
1708         if (rc == -ENOSPC) {
1709                 /* Get by with single vector */
1710                 rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1711                 if (rc < 0) {
1712                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1713                                        rc);
1714                         goto no_msix;
1715                 }
1716
1717                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1718                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1719
1720                 BNX2X_DEV_INFO("set number of queues to 1\n");
1721                 bp->num_ethernet_queues = 1;
1722                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1723         } else if (rc < 0) {
1724                 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1725                 goto no_msix;
1726         } else if (rc < msix_vec) {
1727                 /* how less vectors we will have? */
1728                 int diff = msix_vec - rc;
1729
1730                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1731
1732                 /*
1733                  * decrease number of queues by number of unallocated entries
1734                  */
1735                 bp->num_ethernet_queues -= diff;
1736                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1737
1738                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1739                                bp->num_queues);
1740         }
1741
1742         bp->flags |= USING_MSIX_FLAG;
1743
1744         return 0;
1745
1746 no_msix:
1747         /* fall to INTx if not enough memory */
1748         if (rc == -ENOMEM)
1749                 bp->flags |= DISABLE_MSI_FLAG;
1750
1751         return rc;
1752 }
1753
1754 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1755 {
1756         int i, rc, offset = 0;
1757
1758         /* no default status block for vf */
1759         if (IS_PF(bp)) {
1760                 rc = request_irq(bp->msix_table[offset++].vector,
1761                                  bnx2x_msix_sp_int, 0,
1762                                  bp->dev->name, bp->dev);
1763                 if (rc) {
1764                         BNX2X_ERR("request sp irq failed\n");
1765                         return -EBUSY;
1766                 }
1767         }
1768
1769         if (CNIC_SUPPORT(bp))
1770                 offset++;
1771
1772         for_each_eth_queue(bp, i) {
1773                 struct bnx2x_fastpath *fp = &bp->fp[i];
1774                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1775                          bp->dev->name, i);
1776
1777                 rc = request_irq(bp->msix_table[offset].vector,
1778                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1779                 if (rc) {
1780                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1781                               bp->msix_table[offset].vector, rc);
1782                         bnx2x_free_msix_irqs(bp, offset);
1783                         return -EBUSY;
1784                 }
1785
1786                 offset++;
1787         }
1788
1789         i = BNX2X_NUM_ETH_QUEUES(bp);
1790         if (IS_PF(bp)) {
1791                 offset = 1 + CNIC_SUPPORT(bp);
1792                 netdev_info(bp->dev,
1793                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1794                             bp->msix_table[0].vector,
1795                             0, bp->msix_table[offset].vector,
1796                             i - 1, bp->msix_table[offset + i - 1].vector);
1797         } else {
1798                 offset = CNIC_SUPPORT(bp);
1799                 netdev_info(bp->dev,
1800                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1801                             0, bp->msix_table[offset].vector,
1802                             i - 1, bp->msix_table[offset + i - 1].vector);
1803         }
1804         return 0;
1805 }
1806
1807 int bnx2x_enable_msi(struct bnx2x *bp)
1808 {
1809         int rc;
1810
1811         rc = pci_enable_msi(bp->pdev);
1812         if (rc) {
1813                 BNX2X_DEV_INFO("MSI is not attainable\n");
1814                 return -1;
1815         }
1816         bp->flags |= USING_MSI_FLAG;
1817
1818         return 0;
1819 }
1820
1821 static int bnx2x_req_irq(struct bnx2x *bp)
1822 {
1823         unsigned long flags;
1824         unsigned int irq;
1825
1826         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1827                 flags = 0;
1828         else
1829                 flags = IRQF_SHARED;
1830
1831         if (bp->flags & USING_MSIX_FLAG)
1832                 irq = bp->msix_table[0].vector;
1833         else
1834                 irq = bp->pdev->irq;
1835
1836         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1837 }
1838
1839 static int bnx2x_setup_irqs(struct bnx2x *bp)
1840 {
1841         int rc = 0;
1842         if (bp->flags & USING_MSIX_FLAG &&
1843             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1844                 rc = bnx2x_req_msix_irqs(bp);
1845                 if (rc)
1846                         return rc;
1847         } else {
1848                 rc = bnx2x_req_irq(bp);
1849                 if (rc) {
1850                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1851                         return rc;
1852                 }
1853                 if (bp->flags & USING_MSI_FLAG) {
1854                         bp->dev->irq = bp->pdev->irq;
1855                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1856                                     bp->dev->irq);
1857                 }
1858                 if (bp->flags & USING_MSIX_FLAG) {
1859                         bp->dev->irq = bp->msix_table[0].vector;
1860                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1861                                     bp->dev->irq);
1862                 }
1863         }
1864
1865         return 0;
1866 }
1867
1868 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1869 {
1870         int i;
1871
1872         for_each_rx_queue_cnic(bp, i) {
1873                 bnx2x_fp_busy_poll_init(&bp->fp[i]);
1874                 napi_enable(&bnx2x_fp(bp, i, napi));
1875         }
1876 }
1877
1878 static void bnx2x_napi_enable(struct bnx2x *bp)
1879 {
1880         int i;
1881
1882         for_each_eth_queue(bp, i) {
1883                 bnx2x_fp_busy_poll_init(&bp->fp[i]);
1884                 napi_enable(&bnx2x_fp(bp, i, napi));
1885         }
1886 }
1887
1888 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1889 {
1890         int i;
1891
1892         for_each_rx_queue_cnic(bp, i) {
1893                 napi_disable(&bnx2x_fp(bp, i, napi));
1894                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1895                         usleep_range(1000, 2000);
1896         }
1897 }
1898
1899 static void bnx2x_napi_disable(struct bnx2x *bp)
1900 {
1901         int i;
1902
1903         for_each_eth_queue(bp, i) {
1904                 napi_disable(&bnx2x_fp(bp, i, napi));
1905                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1906                         usleep_range(1000, 2000);
1907         }
1908 }
1909
1910 void bnx2x_netif_start(struct bnx2x *bp)
1911 {
1912         if (netif_running(bp->dev)) {
1913                 bnx2x_napi_enable(bp);
1914                 if (CNIC_LOADED(bp))
1915                         bnx2x_napi_enable_cnic(bp);
1916                 bnx2x_int_enable(bp);
1917                 if (bp->state == BNX2X_STATE_OPEN)
1918                         netif_tx_wake_all_queues(bp->dev);
1919         }
1920 }
1921
1922 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1923 {
1924         bnx2x_int_disable_sync(bp, disable_hw);
1925         bnx2x_napi_disable(bp);
1926         if (CNIC_LOADED(bp))
1927                 bnx2x_napi_disable_cnic(bp);
1928 }
1929
1930 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1931                        void *accel_priv, select_queue_fallback_t fallback)
1932 {
1933         struct bnx2x *bp = netdev_priv(dev);
1934
1935         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1936                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1937                 u16 ether_type = ntohs(hdr->h_proto);
1938
1939                 /* Skip VLAN tag if present */
1940                 if (ether_type == ETH_P_8021Q) {
1941                         struct vlan_ethhdr *vhdr =
1942                                 (struct vlan_ethhdr *)skb->data;
1943
1944                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1945                 }
1946
1947                 /* If ethertype is FCoE or FIP - use FCoE ring */
1948                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1949                         return bnx2x_fcoe_tx(bp, txq_index);
1950         }
1951
1952         /* select a non-FCoE queue */
1953         return fallback(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1954 }
1955
1956 void bnx2x_set_num_queues(struct bnx2x *bp)
1957 {
1958         /* RSS queues */
1959         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1960
1961         /* override in STORAGE SD modes */
1962         if (IS_MF_STORAGE_ONLY(bp))
1963                 bp->num_ethernet_queues = 1;
1964
1965         /* Add special queues */
1966         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1967         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1968
1969         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1970 }
1971
1972 /**
1973  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1974  *
1975  * @bp:         Driver handle
1976  *
1977  * We currently support for at most 16 Tx queues for each CoS thus we will
1978  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1979  * bp->max_cos.
1980  *
1981  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1982  * index after all ETH L2 indices.
1983  *
1984  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1985  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1986  * 16..31,...) with indices that are not coupled with any real Tx queue.
1987  *
1988  * The proper configuration of skb->queue_mapping is handled by
1989  * bnx2x_select_queue() and __skb_tx_hash().
1990  *
1991  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1992  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1993  */
1994 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1995 {
1996         int rc, tx, rx;
1997
1998         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1999         rx = BNX2X_NUM_ETH_QUEUES(bp);
2000
2001 /* account for fcoe queue */
2002         if (include_cnic && !NO_FCOE(bp)) {
2003                 rx++;
2004                 tx++;
2005         }
2006
2007         rc = netif_set_real_num_tx_queues(bp->dev, tx);
2008         if (rc) {
2009                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
2010                 return rc;
2011         }
2012         rc = netif_set_real_num_rx_queues(bp->dev, rx);
2013         if (rc) {
2014                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
2015                 return rc;
2016         }
2017
2018         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
2019                           tx, rx);
2020
2021         return rc;
2022 }
2023
2024 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
2025 {
2026         int i;
2027
2028         for_each_queue(bp, i) {
2029                 struct bnx2x_fastpath *fp = &bp->fp[i];
2030                 u32 mtu;
2031
2032                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
2033                 if (IS_FCOE_IDX(i))
2034                         /*
2035                          * Although there are no IP frames expected to arrive to
2036                          * this ring we still want to add an
2037                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
2038                          * overrun attack.
2039                          */
2040                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
2041                 else
2042                         mtu = bp->dev->mtu;
2043                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2044                                   IP_HEADER_ALIGNMENT_PADDING +
2045                                   ETH_OVREHEAD +
2046                                   mtu +
2047                                   BNX2X_FW_RX_ALIGN_END;
2048                 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2049                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2050                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2051                 else
2052                         fp->rx_frag_size = 0;
2053         }
2054 }
2055
2056 static int bnx2x_init_rss(struct bnx2x *bp)
2057 {
2058         int i;
2059         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2060
2061         /* Prepare the initial contents for the indirection table if RSS is
2062          * enabled
2063          */
2064         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2065                 bp->rss_conf_obj.ind_table[i] =
2066                         bp->fp->cl_id +
2067                         ethtool_rxfh_indir_default(i, num_eth_queues);
2068
2069         /*
2070          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2071          * per-port, so if explicit configuration is needed , do it only
2072          * for a PMF.
2073          *
2074          * For 57712 and newer on the other hand it's a per-function
2075          * configuration.
2076          */
2077         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2078 }
2079
2080 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2081               bool config_hash, bool enable)
2082 {
2083         struct bnx2x_config_rss_params params = {NULL};
2084
2085         /* Although RSS is meaningless when there is a single HW queue we
2086          * still need it enabled in order to have HW Rx hash generated.
2087          *
2088          * if (!is_eth_multi(bp))
2089          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
2090          */
2091
2092         params.rss_obj = rss_obj;
2093
2094         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2095
2096         if (enable) {
2097                 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2098
2099                 /* RSS configuration */
2100                 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2101                 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2102                 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2103                 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2104                 if (rss_obj->udp_rss_v4)
2105                         __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2106                 if (rss_obj->udp_rss_v6)
2107                         __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2108
2109                 if (!CHIP_IS_E1x(bp))
2110                         /* valid only for TUNN_MODE_GRE tunnel mode */
2111                         __set_bit(BNX2X_RSS_GRE_INNER_HDRS, &params.rss_flags);
2112         } else {
2113                 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2114         }
2115
2116         /* Hash bits */
2117         params.rss_result_mask = MULTI_MASK;
2118
2119         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2120
2121         if (config_hash) {
2122                 /* RSS keys */
2123                 netdev_rss_key_fill(params.rss_key, T_ETH_RSS_KEY * 4);
2124                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2125         }
2126
2127         if (IS_PF(bp))
2128                 return bnx2x_config_rss(bp, &params);
2129         else
2130                 return bnx2x_vfpf_config_rss(bp, &params);
2131 }
2132
2133 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2134 {
2135         struct bnx2x_func_state_params func_params = {NULL};
2136
2137         /* Prepare parameters for function state transitions */
2138         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2139
2140         func_params.f_obj = &bp->func_obj;
2141         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2142
2143         func_params.params.hw_init.load_phase = load_code;
2144
2145         return bnx2x_func_state_change(bp, &func_params);
2146 }
2147
2148 /*
2149  * Cleans the object that have internal lists without sending
2150  * ramrods. Should be run when interrupts are disabled.
2151  */
2152 void bnx2x_squeeze_objects(struct bnx2x *bp)
2153 {
2154         int rc;
2155         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2156         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2157         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2158
2159         /***************** Cleanup MACs' object first *************************/
2160
2161         /* Wait for completion of requested */
2162         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2163         /* Perform a dry cleanup */
2164         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2165
2166         /* Clean ETH primary MAC */
2167         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2168         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2169                                  &ramrod_flags);
2170         if (rc != 0)
2171                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2172
2173         /* Cleanup UC list */
2174         vlan_mac_flags = 0;
2175         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2176         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2177                                  &ramrod_flags);
2178         if (rc != 0)
2179                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2180
2181         /***************** Now clean mcast object *****************************/
2182         rparam.mcast_obj = &bp->mcast_obj;
2183         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2184
2185         /* Add a DEL command... - Since we're doing a driver cleanup only,
2186          * we take a lock surrounding both the initial send and the CONTs,
2187          * as we don't want a true completion to disrupt us in the middle.
2188          */
2189         netif_addr_lock_bh(bp->dev);
2190         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2191         if (rc < 0)
2192                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2193                           rc);
2194
2195         /* ...and wait until all pending commands are cleared */
2196         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2197         while (rc != 0) {
2198                 if (rc < 0) {
2199                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2200                                   rc);
2201                         netif_addr_unlock_bh(bp->dev);
2202                         return;
2203                 }
2204
2205                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2206         }
2207         netif_addr_unlock_bh(bp->dev);
2208 }
2209
2210 #ifndef BNX2X_STOP_ON_ERROR
2211 #define LOAD_ERROR_EXIT(bp, label) \
2212         do { \
2213                 (bp)->state = BNX2X_STATE_ERROR; \
2214                 goto label; \
2215         } while (0)
2216
2217 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2218         do { \
2219                 bp->cnic_loaded = false; \
2220                 goto label; \
2221         } while (0)
2222 #else /*BNX2X_STOP_ON_ERROR*/
2223 #define LOAD_ERROR_EXIT(bp, label) \
2224         do { \
2225                 (bp)->state = BNX2X_STATE_ERROR; \
2226                 (bp)->panic = 1; \
2227                 return -EBUSY; \
2228         } while (0)
2229 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2230         do { \
2231                 bp->cnic_loaded = false; \
2232                 (bp)->panic = 1; \
2233                 return -EBUSY; \
2234         } while (0)
2235 #endif /*BNX2X_STOP_ON_ERROR*/
2236
2237 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2238 {
2239         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2240                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2241         return;
2242 }
2243
2244 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2245 {
2246         int num_groups, vf_headroom = 0;
2247         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2248
2249         /* number of queues for statistics is number of eth queues + FCoE */
2250         u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2251
2252         /* Total number of FW statistics requests =
2253          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2254          * and fcoe l2 queue) stats + num of queues (which includes another 1
2255          * for fcoe l2 queue if applicable)
2256          */
2257         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2258
2259         /* vf stats appear in the request list, but their data is allocated by
2260          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2261          * it is used to determine where to place the vf stats queries in the
2262          * request struct
2263          */
2264         if (IS_SRIOV(bp))
2265                 vf_headroom = bnx2x_vf_headroom(bp);
2266
2267         /* Request is built from stats_query_header and an array of
2268          * stats_query_cmd_group each of which contains
2269          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2270          * configured in the stats_query_header.
2271          */
2272         num_groups =
2273                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2274                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2275                  1 : 0));
2276
2277         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2278            bp->fw_stats_num, vf_headroom, num_groups);
2279         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2280                 num_groups * sizeof(struct stats_query_cmd_group);
2281
2282         /* Data for statistics requests + stats_counter
2283          * stats_counter holds per-STORM counters that are incremented
2284          * when STORM has finished with the current request.
2285          * memory for FCoE offloaded statistics are counted anyway,
2286          * even if they will not be sent.
2287          * VF stats are not accounted for here as the data of VF stats is stored
2288          * in memory allocated by the VF, not here.
2289          */
2290         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2291                 sizeof(struct per_pf_stats) +
2292                 sizeof(struct fcoe_statistics_params) +
2293                 sizeof(struct per_queue_stats) * num_queue_stats +
2294                 sizeof(struct stats_counter);
2295
2296         bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2297                                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2298         if (!bp->fw_stats)
2299                 goto alloc_mem_err;
2300
2301         /* Set shortcuts */
2302         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2303         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2304         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2305                 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2306         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2307                 bp->fw_stats_req_sz;
2308
2309         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2310            U64_HI(bp->fw_stats_req_mapping),
2311            U64_LO(bp->fw_stats_req_mapping));
2312         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2313            U64_HI(bp->fw_stats_data_mapping),
2314            U64_LO(bp->fw_stats_data_mapping));
2315         return 0;
2316
2317 alloc_mem_err:
2318         bnx2x_free_fw_stats_mem(bp);
2319         BNX2X_ERR("Can't allocate FW stats memory\n");
2320         return -ENOMEM;
2321 }
2322
2323 /* send load request to mcp and analyze response */
2324 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2325 {
2326         u32 param;
2327
2328         /* init fw_seq */
2329         bp->fw_seq =
2330                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2331                  DRV_MSG_SEQ_NUMBER_MASK);
2332         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2333
2334         /* Get current FW pulse sequence */
2335         bp->fw_drv_pulse_wr_seq =
2336                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2337                  DRV_PULSE_SEQ_MASK);
2338         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2339
2340         param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2341
2342         if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2343                 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2344
2345         /* load request */
2346         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2347
2348         /* if mcp fails to respond we must abort */
2349         if (!(*load_code)) {
2350                 BNX2X_ERR("MCP response failure, aborting\n");
2351                 return -EBUSY;
2352         }
2353
2354         /* If mcp refused (e.g. other port is in diagnostic mode) we
2355          * must abort
2356          */
2357         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2358                 BNX2X_ERR("MCP refused load request, aborting\n");
2359                 return -EBUSY;
2360         }
2361         return 0;
2362 }
2363
2364 /* check whether another PF has already loaded FW to chip. In
2365  * virtualized environments a pf from another VM may have already
2366  * initialized the device including loading FW
2367  */
2368 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2369 {
2370         /* is another pf loaded on this engine? */
2371         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2372             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2373                 /* build my FW version dword */
2374                 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2375                         (BCM_5710_FW_MINOR_VERSION << 8) +
2376                         (BCM_5710_FW_REVISION_VERSION << 16) +
2377                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2378
2379                 /* read loaded FW from chip */
2380                 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2381
2382                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2383                    loaded_fw, my_fw);
2384
2385                 /* abort nic load if version mismatch */
2386                 if (my_fw != loaded_fw) {
2387                         if (print_err)
2388                                 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2389                                           loaded_fw, my_fw);
2390                         else
2391                                 BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2392                                                loaded_fw, my_fw);
2393                         return -EBUSY;
2394                 }
2395         }
2396         return 0;
2397 }
2398
2399 /* returns the "mcp load_code" according to global load_count array */
2400 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2401 {
2402         int path = BP_PATH(bp);
2403
2404         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2405            path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2406            bnx2x_load_count[path][2]);
2407         bnx2x_load_count[path][0]++;
2408         bnx2x_load_count[path][1 + port]++;
2409         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2410            path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2411            bnx2x_load_count[path][2]);
2412         if (bnx2x_load_count[path][0] == 1)
2413                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2414         else if (bnx2x_load_count[path][1 + port] == 1)
2415                 return FW_MSG_CODE_DRV_LOAD_PORT;
2416         else
2417                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2418 }
2419
2420 /* mark PMF if applicable */
2421 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2422 {
2423         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2424             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2425             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2426                 bp->port.pmf = 1;
2427                 /* We need the barrier to ensure the ordering between the
2428                  * writing to bp->port.pmf here and reading it from the
2429                  * bnx2x_periodic_task().
2430                  */
2431                 smp_mb();
2432         } else {
2433                 bp->port.pmf = 0;
2434         }
2435
2436         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2437 }
2438
2439 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2440 {
2441         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2442              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2443             (bp->common.shmem2_base)) {
2444                 if (SHMEM2_HAS(bp, dcc_support))
2445                         SHMEM2_WR(bp, dcc_support,
2446                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2447                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2448                 if (SHMEM2_HAS(bp, afex_driver_support))
2449                         SHMEM2_WR(bp, afex_driver_support,
2450                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2451         }
2452
2453         /* Set AFEX default VLAN tag to an invalid value */
2454         bp->afex_def_vlan_tag = -1;
2455 }
2456
2457 /**
2458  * bnx2x_bz_fp - zero content of the fastpath structure.
2459  *
2460  * @bp:         driver handle
2461  * @index:      fastpath index to be zeroed
2462  *
2463  * Makes sure the contents of the bp->fp[index].napi is kept
2464  * intact.
2465  */
2466 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2467 {
2468         struct bnx2x_fastpath *fp = &bp->fp[index];
2469         int cos;
2470         struct napi_struct orig_napi = fp->napi;
2471         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2472
2473         /* bzero bnx2x_fastpath contents */
2474         if (fp->tpa_info)
2475                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2476                        sizeof(struct bnx2x_agg_info));
2477         memset(fp, 0, sizeof(*fp));
2478
2479         /* Restore the NAPI object as it has been already initialized */
2480         fp->napi = orig_napi;
2481         fp->tpa_info = orig_tpa_info;
2482         fp->bp = bp;
2483         fp->index = index;
2484         if (IS_ETH_FP(fp))
2485                 fp->max_cos = bp->max_cos;
2486         else
2487                 /* Special queues support only one CoS */
2488                 fp->max_cos = 1;
2489
2490         /* Init txdata pointers */
2491         if (IS_FCOE_FP(fp))
2492                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2493         if (IS_ETH_FP(fp))
2494                 for_each_cos_in_tx_queue(fp, cos)
2495                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2496                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2497
2498         /* set the tpa flag for each queue. The tpa flag determines the queue
2499          * minimal size so it must be set prior to queue memory allocation
2500          */
2501         if (bp->dev->features & NETIF_F_LRO)
2502                 fp->mode = TPA_MODE_LRO;
2503         else if (bp->dev->features & NETIF_F_GRO &&
2504                  bnx2x_mtu_allows_gro(bp->dev->mtu))
2505                 fp->mode = TPA_MODE_GRO;
2506         else
2507                 fp->mode = TPA_MODE_DISABLED;
2508
2509         /* We don't want TPA if it's disabled in bp
2510          * or if this is an FCoE L2 ring.
2511          */
2512         if (bp->disable_tpa || IS_FCOE_FP(fp))
2513                 fp->mode = TPA_MODE_DISABLED;
2514 }
2515
2516 int bnx2x_load_cnic(struct bnx2x *bp)
2517 {
2518         int i, rc, port = BP_PORT(bp);
2519
2520         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2521
2522         mutex_init(&bp->cnic_mutex);
2523
2524         if (IS_PF(bp)) {
2525                 rc = bnx2x_alloc_mem_cnic(bp);
2526                 if (rc) {
2527                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2528                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2529                 }
2530         }
2531
2532         rc = bnx2x_alloc_fp_mem_cnic(bp);
2533         if (rc) {
2534                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2535                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2536         }
2537
2538         /* Update the number of queues with the cnic queues */
2539         rc = bnx2x_set_real_num_queues(bp, 1);
2540         if (rc) {
2541                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2542                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2543         }
2544
2545         /* Add all CNIC NAPI objects */
2546         bnx2x_add_all_napi_cnic(bp);
2547         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2548         bnx2x_napi_enable_cnic(bp);
2549
2550         rc = bnx2x_init_hw_func_cnic(bp);
2551         if (rc)
2552                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2553
2554         bnx2x_nic_init_cnic(bp);
2555
2556         if (IS_PF(bp)) {
2557                 /* Enable Timer scan */
2558                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2559
2560                 /* setup cnic queues */
2561                 for_each_cnic_queue(bp, i) {
2562                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2563                         if (rc) {
2564                                 BNX2X_ERR("Queue setup failed\n");
2565                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2566                         }
2567                 }
2568         }
2569
2570         /* Initialize Rx filter. */
2571         bnx2x_set_rx_mode_inner(bp);
2572
2573         /* re-read iscsi info */
2574         bnx2x_get_iscsi_info(bp);
2575         bnx2x_setup_cnic_irq_info(bp);
2576         bnx2x_setup_cnic_info(bp);
2577         bp->cnic_loaded = true;
2578         if (bp->state == BNX2X_STATE_OPEN)
2579                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2580
2581         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2582
2583         return 0;
2584
2585 #ifndef BNX2X_STOP_ON_ERROR
2586 load_error_cnic2:
2587         /* Disable Timer scan */
2588         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2589
2590 load_error_cnic1:
2591         bnx2x_napi_disable_cnic(bp);
2592         /* Update the number of queues without the cnic queues */
2593         if (bnx2x_set_real_num_queues(bp, 0))
2594                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2595 load_error_cnic0:
2596         BNX2X_ERR("CNIC-related load failed\n");
2597         bnx2x_free_fp_mem_cnic(bp);
2598         bnx2x_free_mem_cnic(bp);
2599         return rc;
2600 #endif /* ! BNX2X_STOP_ON_ERROR */
2601 }
2602
2603 /* must be called with rtnl_lock */
2604 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2605 {
2606         int port = BP_PORT(bp);
2607         int i, rc = 0, load_code = 0;
2608
2609         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2610         DP(NETIF_MSG_IFUP,
2611            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2612
2613 #ifdef BNX2X_STOP_ON_ERROR
2614         if (unlikely(bp->panic)) {
2615                 BNX2X_ERR("Can't load NIC when there is panic\n");
2616                 return -EPERM;
2617         }
2618 #endif
2619
2620         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2621
2622         /* zero the structure w/o any lock, before SP handler is initialized */
2623         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2624         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2625                 &bp->last_reported_link.link_report_flags);
2626
2627         if (IS_PF(bp))
2628                 /* must be called before memory allocation and HW init */
2629                 bnx2x_ilt_set_info(bp);
2630
2631         /*
2632          * Zero fastpath structures preserving invariants like napi, which are
2633          * allocated only once, fp index, max_cos, bp pointer.
2634          * Also set fp->mode and txdata_ptr.
2635          */
2636         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2637         for_each_queue(bp, i)
2638                 bnx2x_bz_fp(bp, i);
2639         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2640                                   bp->num_cnic_queues) *
2641                                   sizeof(struct bnx2x_fp_txdata));
2642
2643         bp->fcoe_init = false;
2644
2645         /* Set the receive queues buffer size */
2646         bnx2x_set_rx_buf_size(bp);
2647
2648         if (IS_PF(bp)) {
2649                 rc = bnx2x_alloc_mem(bp);
2650                 if (rc) {
2651                         BNX2X_ERR("Unable to allocate bp memory\n");
2652                         return rc;
2653                 }
2654         }
2655
2656         /* need to be done after alloc mem, since it's self adjusting to amount
2657          * of memory available for RSS queues
2658          */
2659         rc = bnx2x_alloc_fp_mem(bp);
2660         if (rc) {
2661                 BNX2X_ERR("Unable to allocate memory for fps\n");
2662                 LOAD_ERROR_EXIT(bp, load_error0);
2663         }
2664
2665         /* Allocated memory for FW statistics  */
2666         if (bnx2x_alloc_fw_stats_mem(bp))
2667                 LOAD_ERROR_EXIT(bp, load_error0);
2668
2669         /* request pf to initialize status blocks */
2670         if (IS_VF(bp)) {
2671                 rc = bnx2x_vfpf_init(bp);
2672                 if (rc)
2673                         LOAD_ERROR_EXIT(bp, load_error0);
2674         }
2675
2676         /* As long as bnx2x_alloc_mem() may possibly update
2677          * bp->num_queues, bnx2x_set_real_num_queues() should always
2678          * come after it. At this stage cnic queues are not counted.
2679          */
2680         rc = bnx2x_set_real_num_queues(bp, 0);
2681         if (rc) {
2682                 BNX2X_ERR("Unable to set real_num_queues\n");
2683                 LOAD_ERROR_EXIT(bp, load_error0);
2684         }
2685
2686         /* configure multi cos mappings in kernel.
2687          * this configuration may be overridden by a multi class queue
2688          * discipline or by a dcbx negotiation result.
2689          */
2690         bnx2x_setup_tc(bp->dev, bp->max_cos);
2691
2692         /* Add all NAPI objects */
2693         bnx2x_add_all_napi(bp);
2694         DP(NETIF_MSG_IFUP, "napi added\n");
2695         bnx2x_napi_enable(bp);
2696
2697         if (IS_PF(bp)) {
2698                 /* set pf load just before approaching the MCP */
2699                 bnx2x_set_pf_load(bp);
2700
2701                 /* if mcp exists send load request and analyze response */
2702                 if (!BP_NOMCP(bp)) {
2703                         /* attempt to load pf */
2704                         rc = bnx2x_nic_load_request(bp, &load_code);
2705                         if (rc)
2706                                 LOAD_ERROR_EXIT(bp, load_error1);
2707
2708                         /* what did mcp say? */
2709                         rc = bnx2x_compare_fw_ver(bp, load_code, true);
2710                         if (rc) {
2711                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2712                                 LOAD_ERROR_EXIT(bp, load_error2);
2713                         }
2714                 } else {
2715                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2716                 }
2717
2718                 /* mark pmf if applicable */
2719                 bnx2x_nic_load_pmf(bp, load_code);
2720
2721                 /* Init Function state controlling object */
2722                 bnx2x__init_func_obj(bp);
2723
2724                 /* Initialize HW */
2725                 rc = bnx2x_init_hw(bp, load_code);
2726                 if (rc) {
2727                         BNX2X_ERR("HW init failed, aborting\n");
2728                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2729                         LOAD_ERROR_EXIT(bp, load_error2);
2730                 }
2731         }
2732
2733         bnx2x_pre_irq_nic_init(bp);
2734
2735         /* Connect to IRQs */
2736         rc = bnx2x_setup_irqs(bp);
2737         if (rc) {
2738                 BNX2X_ERR("setup irqs failed\n");
2739                 if (IS_PF(bp))
2740                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2741                 LOAD_ERROR_EXIT(bp, load_error2);
2742         }
2743
2744         /* Init per-function objects */
2745         if (IS_PF(bp)) {
2746                 /* Setup NIC internals and enable interrupts */
2747                 bnx2x_post_irq_nic_init(bp, load_code);
2748
2749                 bnx2x_init_bp_objs(bp);
2750                 bnx2x_iov_nic_init(bp);
2751
2752                 /* Set AFEX default VLAN tag to an invalid value */
2753                 bp->afex_def_vlan_tag = -1;
2754                 bnx2x_nic_load_afex_dcc(bp, load_code);
2755                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2756                 rc = bnx2x_func_start(bp);
2757                 if (rc) {
2758                         BNX2X_ERR("Function start failed!\n");
2759                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2760
2761                         LOAD_ERROR_EXIT(bp, load_error3);
2762                 }
2763
2764                 /* Send LOAD_DONE command to MCP */
2765                 if (!BP_NOMCP(bp)) {
2766                         load_code = bnx2x_fw_command(bp,
2767                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2768                         if (!load_code) {
2769                                 BNX2X_ERR("MCP response failure, aborting\n");
2770                                 rc = -EBUSY;
2771                                 LOAD_ERROR_EXIT(bp, load_error3);
2772                         }
2773                 }
2774
2775                 /* initialize FW coalescing state machines in RAM */
2776                 bnx2x_update_coalesce(bp);
2777         }
2778
2779         /* setup the leading queue */
2780         rc = bnx2x_setup_leading(bp);
2781         if (rc) {
2782                 BNX2X_ERR("Setup leading failed!\n");
2783                 LOAD_ERROR_EXIT(bp, load_error3);
2784         }
2785
2786         /* set up the rest of the queues */
2787         for_each_nondefault_eth_queue(bp, i) {
2788                 if (IS_PF(bp))
2789                         rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2790                 else /* VF */
2791                         rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2792                 if (rc) {
2793                         BNX2X_ERR("Queue %d setup failed\n", i);
2794                         LOAD_ERROR_EXIT(bp, load_error3);
2795                 }
2796         }
2797
2798         /* setup rss */
2799         rc = bnx2x_init_rss(bp);
2800         if (rc) {
2801                 BNX2X_ERR("PF RSS init failed\n");
2802                 LOAD_ERROR_EXIT(bp, load_error3);
2803         }
2804
2805         /* Now when Clients are configured we are ready to work */
2806         bp->state = BNX2X_STATE_OPEN;
2807
2808         /* Configure a ucast MAC */
2809         if (IS_PF(bp))
2810                 rc = bnx2x_set_eth_mac(bp, true);
2811         else /* vf */
2812                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2813                                            true);
2814         if (rc) {
2815                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2816                 LOAD_ERROR_EXIT(bp, load_error3);
2817         }
2818
2819         if (IS_PF(bp) && bp->pending_max) {
2820                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2821                 bp->pending_max = 0;
2822         }
2823
2824         if (bp->port.pmf) {
2825                 rc = bnx2x_initial_phy_init(bp, load_mode);
2826                 if (rc)
2827                         LOAD_ERROR_EXIT(bp, load_error3);
2828         }
2829         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2830
2831         /* Start fast path */
2832
2833         /* Initialize Rx filter. */
2834         bnx2x_set_rx_mode_inner(bp);
2835
2836         if (bp->flags & PTP_SUPPORTED) {
2837                 bnx2x_init_ptp(bp);
2838                 bnx2x_configure_ptp_filters(bp);
2839         }
2840         /* Start Tx */
2841         switch (load_mode) {
2842         case LOAD_NORMAL:
2843                 /* Tx queue should be only re-enabled */
2844                 netif_tx_wake_all_queues(bp->dev);
2845                 break;
2846
2847         case LOAD_OPEN:
2848                 netif_tx_start_all_queues(bp->dev);
2849                 smp_mb__after_atomic();
2850                 break;
2851
2852         case LOAD_DIAG:
2853         case LOAD_LOOPBACK_EXT:
2854                 bp->state = BNX2X_STATE_DIAG;
2855                 break;
2856
2857         default:
2858                 break;
2859         }
2860
2861         if (bp->port.pmf)
2862                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2863         else
2864                 bnx2x__link_status_update(bp);
2865
2866         /* start the timer */
2867         mod_timer(&bp->timer, jiffies + bp->current_interval);
2868
2869         if (CNIC_ENABLED(bp))
2870                 bnx2x_load_cnic(bp);
2871
2872         if (IS_PF(bp))
2873                 bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2874
2875         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2876                 /* mark driver is loaded in shmem2 */
2877                 u32 val;
2878                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2879                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2880                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2881                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2882         }
2883
2884         /* Wait for all pending SP commands to complete */
2885         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2886                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2887                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2888                 return -EBUSY;
2889         }
2890
2891         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2892         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2893                 bnx2x_dcbx_init(bp, false);
2894
2895         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2896
2897         return 0;
2898
2899 #ifndef BNX2X_STOP_ON_ERROR
2900 load_error3:
2901         if (IS_PF(bp)) {
2902                 bnx2x_int_disable_sync(bp, 1);
2903
2904                 /* Clean queueable objects */
2905                 bnx2x_squeeze_objects(bp);
2906         }
2907
2908         /* Free SKBs, SGEs, TPA pool and driver internals */
2909         bnx2x_free_skbs(bp);
2910         for_each_rx_queue(bp, i)
2911                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2912
2913         /* Release IRQs */
2914         bnx2x_free_irq(bp);
2915 load_error2:
2916         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2917                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2918                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2919         }
2920
2921         bp->port.pmf = 0;
2922 load_error1:
2923         bnx2x_napi_disable(bp);
2924         bnx2x_del_all_napi(bp);
2925
2926         /* clear pf_load status, as it was already set */
2927         if (IS_PF(bp))
2928                 bnx2x_clear_pf_load(bp);
2929 load_error0:
2930         bnx2x_free_fw_stats_mem(bp);
2931         bnx2x_free_fp_mem(bp);
2932         bnx2x_free_mem(bp);
2933
2934         return rc;
2935 #endif /* ! BNX2X_STOP_ON_ERROR */
2936 }
2937
2938 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2939 {
2940         u8 rc = 0, cos, i;
2941
2942         /* Wait until tx fastpath tasks complete */
2943         for_each_tx_queue(bp, i) {
2944                 struct bnx2x_fastpath *fp = &bp->fp[i];
2945
2946                 for_each_cos_in_tx_queue(fp, cos)
2947                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2948                 if (rc)
2949                         return rc;
2950         }
2951         return 0;
2952 }
2953
2954 /* must be called with rtnl_lock */
2955 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2956 {
2957         int i;
2958         bool global = false;
2959
2960         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2961
2962         /* mark driver is unloaded in shmem2 */
2963         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2964                 u32 val;
2965                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2966                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2967                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2968         }
2969
2970         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2971             (bp->state == BNX2X_STATE_CLOSED ||
2972              bp->state == BNX2X_STATE_ERROR)) {
2973                 /* We can get here if the driver has been unloaded
2974                  * during parity error recovery and is either waiting for a
2975                  * leader to complete or for other functions to unload and
2976                  * then ifdown has been issued. In this case we want to
2977                  * unload and let other functions to complete a recovery
2978                  * process.
2979                  */
2980                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2981                 bp->is_leader = 0;
2982                 bnx2x_release_leader_lock(bp);
2983                 smp_mb();
2984
2985                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2986                 BNX2X_ERR("Can't unload in closed or error state\n");
2987                 return -EINVAL;
2988         }
2989
2990         /* Nothing to do during unload if previous bnx2x_nic_load()
2991          * have not completed successfully - all resources are released.
2992          *
2993          * we can get here only after unsuccessful ndo_* callback, during which
2994          * dev->IFF_UP flag is still on.
2995          */
2996         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2997                 return 0;
2998
2999         /* It's important to set the bp->state to the value different from
3000          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
3001          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
3002          */
3003         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
3004         smp_mb();
3005
3006         /* indicate to VFs that the PF is going down */
3007         bnx2x_iov_channel_down(bp);
3008
3009         if (CNIC_LOADED(bp))
3010                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
3011
3012         /* Stop Tx */
3013         bnx2x_tx_disable(bp);
3014         netdev_reset_tc(bp->dev);
3015
3016         bp->rx_mode = BNX2X_RX_MODE_NONE;
3017
3018         del_timer_sync(&bp->timer);
3019
3020         if (IS_PF(bp)) {
3021                 /* Set ALWAYS_ALIVE bit in shmem */
3022                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
3023                 bnx2x_drv_pulse(bp);
3024                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
3025                 bnx2x_save_statistics(bp);
3026         }
3027
3028         /* wait till consumers catch up with producers in all queues */
3029         bnx2x_drain_tx_queues(bp);
3030
3031         /* if VF indicate to PF this function is going down (PF will delete sp
3032          * elements and clear initializations
3033          */
3034         if (IS_VF(bp))
3035                 bnx2x_vfpf_close_vf(bp);
3036         else if (unload_mode != UNLOAD_RECOVERY)
3037                 /* if this is a normal/close unload need to clean up chip*/
3038                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
3039         else {
3040                 /* Send the UNLOAD_REQUEST to the MCP */
3041                 bnx2x_send_unload_req(bp, unload_mode);
3042
3043                 /* Prevent transactions to host from the functions on the
3044                  * engine that doesn't reset global blocks in case of global
3045                  * attention once global blocks are reset and gates are opened
3046                  * (the engine which leader will perform the recovery
3047                  * last).
3048                  */
3049                 if (!CHIP_IS_E1x(bp))
3050                         bnx2x_pf_disable(bp);
3051
3052                 /* Disable HW interrupts, NAPI */
3053                 bnx2x_netif_stop(bp, 1);
3054                 /* Delete all NAPI objects */
3055                 bnx2x_del_all_napi(bp);
3056                 if (CNIC_LOADED(bp))
3057                         bnx2x_del_all_napi_cnic(bp);
3058                 /* Release IRQs */
3059                 bnx2x_free_irq(bp);
3060
3061                 /* Report UNLOAD_DONE to MCP */
3062                 bnx2x_send_unload_done(bp, false);
3063         }
3064
3065         /*
3066          * At this stage no more interrupts will arrive so we may safely clean
3067          * the queueable objects here in case they failed to get cleaned so far.
3068          */
3069         if (IS_PF(bp))
3070                 bnx2x_squeeze_objects(bp);
3071
3072         /* There should be no more pending SP commands at this stage */
3073         bp->sp_state = 0;
3074
3075         bp->port.pmf = 0;
3076
3077         /* clear pending work in rtnl task */
3078         bp->sp_rtnl_state = 0;
3079         smp_mb();
3080
3081         /* Free SKBs, SGEs, TPA pool and driver internals */
3082         bnx2x_free_skbs(bp);
3083         if (CNIC_LOADED(bp))
3084                 bnx2x_free_skbs_cnic(bp);
3085         for_each_rx_queue(bp, i)
3086                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3087
3088         bnx2x_free_fp_mem(bp);
3089         if (CNIC_LOADED(bp))
3090                 bnx2x_free_fp_mem_cnic(bp);
3091
3092         if (IS_PF(bp)) {
3093                 if (CNIC_LOADED(bp))
3094                         bnx2x_free_mem_cnic(bp);
3095         }
3096         bnx2x_free_mem(bp);
3097
3098         bp->state = BNX2X_STATE_CLOSED;
3099         bp->cnic_loaded = false;
3100
3101         /* Clear driver version indication in shmem */
3102         if (IS_PF(bp))
3103                 bnx2x_update_mng_version(bp);
3104
3105         /* Check if there are pending parity attentions. If there are - set
3106          * RECOVERY_IN_PROGRESS.
3107          */
3108         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3109                 bnx2x_set_reset_in_progress(bp);
3110
3111                 /* Set RESET_IS_GLOBAL if needed */
3112                 if (global)
3113                         bnx2x_set_reset_global(bp);
3114         }
3115
3116         /* The last driver must disable a "close the gate" if there is no
3117          * parity attention or "process kill" pending.
3118          */
3119         if (IS_PF(bp) &&
3120             !bnx2x_clear_pf_load(bp) &&
3121             bnx2x_reset_is_done(bp, BP_PATH(bp)))
3122                 bnx2x_disable_close_the_gate(bp);
3123
3124         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3125
3126         return 0;
3127 }
3128
3129 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3130 {
3131         u16 pmcsr;
3132
3133         /* If there is no power capability, silently succeed */
3134         if (!bp->pdev->pm_cap) {
3135                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3136                 return 0;
3137         }
3138
3139         pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3140
3141         switch (state) {
3142         case PCI_D0:
3143                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3144                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3145                                        PCI_PM_CTRL_PME_STATUS));
3146
3147                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3148                         /* delay required during transition out of D3hot */
3149                         msleep(20);
3150                 break;
3151
3152         case PCI_D3hot:
3153                 /* If there are other clients above don't
3154                    shut down the power */
3155                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3156                         return 0;
3157                 /* Don't shut down the power for emulation and FPGA */
3158                 if (CHIP_REV_IS_SLOW(bp))
3159                         return 0;
3160
3161                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3162                 pmcsr |= 3;
3163
3164                 if (bp->wol)
3165                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3166
3167                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3168                                       pmcsr);
3169
3170                 /* No more memory access after this point until
3171                 * device is brought back to D0.
3172                 */
3173                 break;
3174
3175         default:
3176                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3177                 return -EINVAL;
3178         }
3179         return 0;
3180 }
3181
3182 /*
3183  * net_device service functions
3184  */
3185 static int bnx2x_poll(struct napi_struct *napi, int budget)
3186 {
3187         int work_done = 0;
3188         u8 cos;
3189         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3190                                                  napi);
3191         struct bnx2x *bp = fp->bp;
3192
3193         while (1) {
3194 #ifdef BNX2X_STOP_ON_ERROR
3195                 if (unlikely(bp->panic)) {
3196                         napi_complete(napi);
3197                         return 0;
3198                 }
3199 #endif
3200                 if (!bnx2x_fp_lock_napi(fp))
3201                         return budget;
3202
3203                 for_each_cos_in_tx_queue(fp, cos)
3204                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3205                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3206
3207                 if (bnx2x_has_rx_work(fp)) {
3208                         work_done += bnx2x_rx_int(fp, budget - work_done);
3209
3210                         /* must not complete if we consumed full budget */
3211                         if (work_done >= budget) {
3212                                 bnx2x_fp_unlock_napi(fp);
3213                                 break;
3214                         }
3215                 }
3216
3217                 bnx2x_fp_unlock_napi(fp);
3218
3219                 /* Fall out from the NAPI loop if needed */
3220                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3221
3222                         /* No need to update SB for FCoE L2 ring as long as
3223                          * it's connected to the default SB and the SB
3224                          * has been updated when NAPI was scheduled.
3225                          */
3226                         if (IS_FCOE_FP(fp)) {
3227                                 napi_complete(napi);
3228                                 break;
3229                         }
3230                         bnx2x_update_fpsb_idx(fp);
3231                         /* bnx2x_has_rx_work() reads the status block,
3232                          * thus we need to ensure that status block indices
3233                          * have been actually read (bnx2x_update_fpsb_idx)
3234                          * prior to this check (bnx2x_has_rx_work) so that
3235                          * we won't write the "newer" value of the status block
3236                          * to IGU (if there was a DMA right after
3237                          * bnx2x_has_rx_work and if there is no rmb, the memory
3238                          * reading (bnx2x_update_fpsb_idx) may be postponed
3239                          * to right before bnx2x_ack_sb). In this case there
3240                          * will never be another interrupt until there is
3241                          * another update of the status block, while there
3242                          * is still unhandled work.
3243                          */
3244                         rmb();
3245
3246                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3247                                 napi_complete(napi);
3248                                 /* Re-enable interrupts */
3249                                 DP(NETIF_MSG_RX_STATUS,
3250                                    "Update index to %d\n", fp->fp_hc_idx);
3251                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3252                                              le16_to_cpu(fp->fp_hc_idx),
3253                                              IGU_INT_ENABLE, 1);
3254                                 break;
3255                         }
3256                 }
3257         }
3258
3259         return work_done;
3260 }
3261
3262 #ifdef CONFIG_NET_RX_BUSY_POLL
3263 /* must be called with local_bh_disable()d */
3264 int bnx2x_low_latency_recv(struct napi_struct *napi)
3265 {
3266         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3267                                                  napi);
3268         struct bnx2x *bp = fp->bp;
3269         int found = 0;
3270
3271         if ((bp->state == BNX2X_STATE_CLOSED) ||
3272             (bp->state == BNX2X_STATE_ERROR) ||
3273             (bp->dev->features & (NETIF_F_LRO | NETIF_F_GRO)))
3274                 return LL_FLUSH_FAILED;
3275
3276         if (!bnx2x_fp_lock_poll(fp))
3277                 return LL_FLUSH_BUSY;
3278
3279         if (bnx2x_has_rx_work(fp))
3280                 found = bnx2x_rx_int(fp, 4);
3281
3282         bnx2x_fp_unlock_poll(fp);
3283
3284         return found;
3285 }
3286 #endif
3287
3288 /* we split the first BD into headers and data BDs
3289  * to ease the pain of our fellow microcode engineers
3290  * we use one mapping for both BDs
3291  */
3292 static u16 bnx2x_tx_split(struct bnx2x *bp,
3293                           struct bnx2x_fp_txdata *txdata,
3294                           struct sw_tx_bd *tx_buf,
3295                           struct eth_tx_start_bd **tx_bd, u16 hlen,
3296                           u16 bd_prod)
3297 {
3298         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3299         struct eth_tx_bd *d_tx_bd;
3300         dma_addr_t mapping;
3301         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3302
3303         /* first fix first BD */
3304         h_tx_bd->nbytes = cpu_to_le16(hlen);
3305
3306         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3307            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3308
3309         /* now get a new data BD
3310          * (after the pbd) and fill it */
3311         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3312         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3313
3314         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3315                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3316
3317         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3318         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3319         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3320
3321         /* this marks the BD as one that has no individual mapping */
3322         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3323
3324         DP(NETIF_MSG_TX_QUEUED,
3325            "TSO split data size is %d (%x:%x)\n",
3326            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3327
3328         /* update tx_bd */
3329         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3330
3331         return bd_prod;
3332 }
3333
3334 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3335 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3336 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3337 {
3338         __sum16 tsum = (__force __sum16) csum;
3339
3340         if (fix > 0)
3341                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3342                                   csum_partial(t_header - fix, fix, 0)));
3343
3344         else if (fix < 0)
3345                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3346                                   csum_partial(t_header, -fix, 0)));
3347
3348         return bswab16(tsum);
3349 }
3350
3351 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3352 {
3353         u32 rc;
3354         __u8 prot = 0;
3355         __be16 protocol;
3356
3357         if (skb->ip_summed != CHECKSUM_PARTIAL)
3358                 return XMIT_PLAIN;
3359
3360         protocol = vlan_get_protocol(skb);
3361         if (protocol == htons(ETH_P_IPV6)) {
3362                 rc = XMIT_CSUM_V6;
3363                 prot = ipv6_hdr(skb)->nexthdr;
3364         } else {
3365                 rc = XMIT_CSUM_V4;
3366                 prot = ip_hdr(skb)->protocol;
3367         }
3368
3369         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3370                 if (inner_ip_hdr(skb)->version == 6) {
3371                         rc |= XMIT_CSUM_ENC_V6;
3372                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3373                                 rc |= XMIT_CSUM_TCP;
3374                 } else {
3375                         rc |= XMIT_CSUM_ENC_V4;
3376                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3377                                 rc |= XMIT_CSUM_TCP;
3378                 }
3379         }
3380         if (prot == IPPROTO_TCP)
3381                 rc |= XMIT_CSUM_TCP;
3382
3383         if (skb_is_gso(skb)) {
3384                 if (skb_is_gso_v6(skb)) {
3385                         rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3386                         if (rc & XMIT_CSUM_ENC)
3387                                 rc |= XMIT_GSO_ENC_V6;
3388                 } else {
3389                         rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3390                         if (rc & XMIT_CSUM_ENC)
3391                                 rc |= XMIT_GSO_ENC_V4;
3392                 }
3393         }
3394
3395         return rc;
3396 }
3397
3398 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3399 /* check if packet requires linearization (packet is too fragmented)
3400    no need to check fragmentation if page size > 8K (there will be no
3401    violation to FW restrictions) */
3402 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3403                              u32 xmit_type)
3404 {
3405         int to_copy = 0;
3406         int hlen = 0;
3407         int first_bd_sz = 0;
3408
3409         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3410         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3411
3412                 if (xmit_type & XMIT_GSO) {
3413                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3414                         /* Check if LSO packet needs to be copied:
3415                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3416                         int wnd_size = MAX_FETCH_BD - 3;
3417                         /* Number of windows to check */
3418                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3419                         int wnd_idx = 0;
3420                         int frag_idx = 0;
3421                         u32 wnd_sum = 0;
3422
3423                         /* Headers length */
3424                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3425                                 tcp_hdrlen(skb);
3426
3427                         /* Amount of data (w/o headers) on linear part of SKB*/
3428                         first_bd_sz = skb_headlen(skb) - hlen;
3429
3430                         wnd_sum  = first_bd_sz;
3431
3432                         /* Calculate the first sum - it's special */
3433                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3434                                 wnd_sum +=
3435                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3436
3437                         /* If there was data on linear skb data - check it */
3438                         if (first_bd_sz > 0) {
3439                                 if (unlikely(wnd_sum < lso_mss)) {
3440                                         to_copy = 1;
3441                                         goto exit_lbl;
3442                                 }
3443
3444                                 wnd_sum -= first_bd_sz;
3445                         }
3446
3447                         /* Others are easier: run through the frag list and
3448                            check all windows */
3449                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3450                                 wnd_sum +=
3451                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3452
3453                                 if (unlikely(wnd_sum < lso_mss)) {
3454                                         to_copy = 1;
3455                                         break;
3456                                 }
3457                                 wnd_sum -=
3458                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3459                         }
3460                 } else {
3461                         /* in non-LSO too fragmented packet should always
3462                            be linearized */
3463                         to_copy = 1;
3464                 }
3465         }
3466
3467 exit_lbl:
3468         if (unlikely(to_copy))
3469                 DP(NETIF_MSG_TX_QUEUED,
3470                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3471                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3472                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3473
3474         return to_copy;
3475 }
3476 #endif
3477
3478 /**
3479  * bnx2x_set_pbd_gso - update PBD in GSO case.
3480  *
3481  * @skb:        packet skb
3482  * @pbd:        parse BD
3483  * @xmit_type:  xmit flags
3484  */
3485 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3486                               struct eth_tx_parse_bd_e1x *pbd,
3487                               u32 xmit_type)
3488 {
3489         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3490         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3491         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3492
3493         if (xmit_type & XMIT_GSO_V4) {
3494                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3495                 pbd->tcp_pseudo_csum =
3496                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3497                                                    ip_hdr(skb)->daddr,
3498                                                    0, IPPROTO_TCP, 0));
3499         } else {
3500                 pbd->tcp_pseudo_csum =
3501                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3502                                                  &ipv6_hdr(skb)->daddr,
3503                                                  0, IPPROTO_TCP, 0));
3504         }
3505
3506         pbd->global_data |=
3507                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3508 }
3509
3510 /**
3511  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3512  *
3513  * @bp:                 driver handle
3514  * @skb:                packet skb
3515  * @parsing_data:       data to be updated
3516  * @xmit_type:          xmit flags
3517  *
3518  * 57712/578xx related, when skb has encapsulation
3519  */
3520 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3521                                  u32 *parsing_data, u32 xmit_type)
3522 {
3523         *parsing_data |=
3524                 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3525                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3526                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3527
3528         if (xmit_type & XMIT_CSUM_TCP) {
3529                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3530                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3531                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3532
3533                 return skb_inner_transport_header(skb) +
3534                         inner_tcp_hdrlen(skb) - skb->data;
3535         }
3536
3537         /* We support checksum offload for TCP and UDP only.
3538          * No need to pass the UDP header length - it's a constant.
3539          */
3540         return skb_inner_transport_header(skb) +
3541                 sizeof(struct udphdr) - skb->data;
3542 }
3543
3544 /**
3545  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3546  *
3547  * @bp:                 driver handle
3548  * @skb:                packet skb
3549  * @parsing_data:       data to be updated
3550  * @xmit_type:          xmit flags
3551  *
3552  * 57712/578xx related
3553  */
3554 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3555                                 u32 *parsing_data, u32 xmit_type)
3556 {
3557         *parsing_data |=
3558                 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3559                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3560                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3561
3562         if (xmit_type & XMIT_CSUM_TCP) {
3563                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3564                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3565                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3566
3567                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3568         }
3569         /* We support checksum offload for TCP and UDP only.
3570          * No need to pass the UDP header length - it's a constant.
3571          */
3572         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3573 }
3574
3575 /* set FW indication according to inner or outer protocols if tunneled */
3576 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3577                                struct eth_tx_start_bd *tx_start_bd,
3578                                u32 xmit_type)
3579 {
3580         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3581
3582         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3583                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3584
3585         if (!(xmit_type & XMIT_CSUM_TCP))
3586                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3587 }
3588
3589 /**
3590  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3591  *
3592  * @bp:         driver handle
3593  * @skb:        packet skb
3594  * @pbd:        parse BD to be updated
3595  * @xmit_type:  xmit flags
3596  */
3597 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3598                              struct eth_tx_parse_bd_e1x *pbd,
3599                              u32 xmit_type)
3600 {
3601         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3602
3603         /* for now NS flag is not used in Linux */
3604         pbd->global_data =
3605                 cpu_to_le16(hlen |
3606                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3607                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3608
3609         pbd->ip_hlen_w = (skb_transport_header(skb) -
3610                         skb_network_header(skb)) >> 1;
3611
3612         hlen += pbd->ip_hlen_w;
3613
3614         /* We support checksum offload for TCP and UDP only */
3615         if (xmit_type & XMIT_CSUM_TCP)
3616                 hlen += tcp_hdrlen(skb) / 2;
3617         else
3618                 hlen += sizeof(struct udphdr) / 2;
3619
3620         pbd->total_hlen_w = cpu_to_le16(hlen);
3621         hlen = hlen*2;
3622
3623         if (xmit_type & XMIT_CSUM_TCP) {
3624                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3625
3626         } else {
3627                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3628
3629                 DP(NETIF_MSG_TX_QUEUED,
3630                    "hlen %d  fix %d  csum before fix %x\n",
3631                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3632
3633                 /* HW bug: fixup the CSUM */
3634                 pbd->tcp_pseudo_csum =
3635                         bnx2x_csum_fix(skb_transport_header(skb),
3636                                        SKB_CS(skb), fix);
3637
3638                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3639                    pbd->tcp_pseudo_csum);
3640         }
3641
3642         return hlen;
3643 }
3644
3645 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3646                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3647                                       struct eth_tx_parse_2nd_bd *pbd2,
3648                                       u16 *global_data,
3649                                       u32 xmit_type)
3650 {
3651         u16 hlen_w = 0;
3652         u8 outerip_off, outerip_len = 0;
3653
3654         /* from outer IP to transport */
3655         hlen_w = (skb_inner_transport_header(skb) -
3656                   skb_network_header(skb)) >> 1;
3657
3658         /* transport len */
3659         hlen_w += inner_tcp_hdrlen(skb) >> 1;
3660
3661         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3662
3663         /* outer IP header info */
3664         if (xmit_type & XMIT_CSUM_V4) {
3665                 struct iphdr *iph = ip_hdr(skb);
3666                 u32 csum = (__force u32)(~iph->check) -
3667                            (__force u32)iph->tot_len -
3668                            (__force u32)iph->frag_off;
3669
3670                 outerip_len = iph->ihl << 1;
3671
3672                 pbd2->fw_ip_csum_wo_len_flags_frag =
3673                         bswab16(csum_fold((__force __wsum)csum));
3674         } else {
3675                 pbd2->fw_ip_hdr_to_payload_w =
3676                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3677                 pbd_e2->data.tunnel_data.flags |=
3678                         ETH_TUNNEL_DATA_IP_HDR_TYPE_OUTER;
3679         }
3680
3681         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3682
3683         pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3684
3685         /* inner IP header info */
3686         if (xmit_type & XMIT_CSUM_ENC_V4) {
3687                 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3688
3689                 pbd_e2->data.tunnel_data.pseudo_csum =
3690                         bswab16(~csum_tcpudp_magic(
3691                                         inner_ip_hdr(skb)->saddr,
3692                                         inner_ip_hdr(skb)->daddr,
3693                                         0, IPPROTO_TCP, 0));
3694         } else {
3695                 pbd_e2->data.tunnel_data.pseudo_csum =
3696                         bswab16(~csum_ipv6_magic(
3697                                         &inner_ipv6_hdr(skb)->saddr,
3698                                         &inner_ipv6_hdr(skb)->daddr,
3699                                         0, IPPROTO_TCP, 0));
3700         }
3701
3702         outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3703
3704         *global_data |=
3705                 outerip_off |
3706                 (outerip_len <<
3707                         ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3708                 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3709                         ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3710
3711         if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3712                 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3713                 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3714         }
3715 }
3716
3717 static inline void bnx2x_set_ipv6_ext_e2(struct sk_buff *skb, u32 *parsing_data,
3718                                          u32 xmit_type)
3719 {
3720         struct ipv6hdr *ipv6;
3721
3722         if (!(xmit_type & (XMIT_GSO_ENC_V6 | XMIT_GSO_V6)))
3723                 return;
3724
3725         if (xmit_type & XMIT_GSO_ENC_V6)
3726                 ipv6 = inner_ipv6_hdr(skb);
3727         else /* XMIT_GSO_V6 */
3728                 ipv6 = ipv6_hdr(skb);
3729
3730         if (ipv6->nexthdr == NEXTHDR_IPV6)
3731                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3732 }
3733
3734 /* called with netif_tx_lock
3735  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3736  * netif_wake_queue()
3737  */
3738 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3739 {
3740         struct bnx2x *bp = netdev_priv(dev);
3741
3742         struct netdev_queue *txq;
3743         struct bnx2x_fp_txdata *txdata;
3744         struct sw_tx_bd *tx_buf;
3745         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3746         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3747         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3748         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3749         struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3750         u32 pbd_e2_parsing_data = 0;
3751         u16 pkt_prod, bd_prod;
3752         int nbd, txq_index;
3753         dma_addr_t mapping;
3754         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3755         int i;
3756         u8 hlen = 0;
3757         __le16 pkt_size = 0;
3758         struct ethhdr *eth;
3759         u8 mac_type = UNICAST_ADDRESS;
3760
3761 #ifdef BNX2X_STOP_ON_ERROR
3762         if (unlikely(bp->panic))
3763                 return NETDEV_TX_BUSY;
3764 #endif
3765
3766         txq_index = skb_get_queue_mapping(skb);
3767         txq = netdev_get_tx_queue(dev, txq_index);
3768
3769         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3770
3771         txdata = &bp->bnx2x_txq[txq_index];
3772
3773         /* enable this debug print to view the transmission queue being used
3774         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3775            txq_index, fp_index, txdata_index); */
3776
3777         /* enable this debug print to view the transmission details
3778         DP(NETIF_MSG_TX_QUEUED,
3779            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3780            txdata->cid, fp_index, txdata_index, txdata, fp); */
3781
3782         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3783                         skb_shinfo(skb)->nr_frags +
3784                         BDS_PER_TX_PKT +
3785                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3786                 /* Handle special storage cases separately */
3787                 if (txdata->tx_ring_size == 0) {
3788                         struct bnx2x_eth_q_stats *q_stats =
3789                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3790                         q_stats->driver_filtered_tx_pkt++;
3791                         dev_kfree_skb(skb);
3792                         return NETDEV_TX_OK;
3793                 }
3794                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3795                 netif_tx_stop_queue(txq);
3796                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3797
3798                 return NETDEV_TX_BUSY;
3799         }
3800
3801         DP(NETIF_MSG_TX_QUEUED,
3802            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3803            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3804            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3805            skb->len);
3806
3807         eth = (struct ethhdr *)skb->data;
3808
3809         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3810         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3811                 if (is_broadcast_ether_addr(eth->h_dest))
3812                         mac_type = BROADCAST_ADDRESS;
3813                 else
3814                         mac_type = MULTICAST_ADDRESS;
3815         }
3816
3817 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3818         /* First, check if we need to linearize the skb (due to FW
3819            restrictions). No need to check fragmentation if page size > 8K
3820            (there will be no violation to FW restrictions) */
3821         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3822                 /* Statistics of linearization */
3823                 bp->lin_cnt++;
3824                 if (skb_linearize(skb) != 0) {
3825                         DP(NETIF_MSG_TX_QUEUED,
3826                            "SKB linearization failed - silently dropping this SKB\n");
3827                         dev_kfree_skb_any(skb);
3828                         return NETDEV_TX_OK;
3829                 }
3830         }
3831 #endif
3832         /* Map skb linear data for DMA */
3833         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3834                                  skb_headlen(skb), DMA_TO_DEVICE);
3835         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3836                 DP(NETIF_MSG_TX_QUEUED,
3837                    "SKB mapping failed - silently dropping this SKB\n");
3838                 dev_kfree_skb_any(skb);
3839                 return NETDEV_TX_OK;
3840         }
3841         /*
3842         Please read carefully. First we use one BD which we mark as start,
3843         then we have a parsing info BD (used for TSO or xsum),
3844         and only then we have the rest of the TSO BDs.
3845         (don't forget to mark the last one as last,
3846         and to unmap only AFTER you write to the BD ...)
3847         And above all, all pdb sizes are in words - NOT DWORDS!
3848         */
3849
3850         /* get current pkt produced now - advance it just before sending packet
3851          * since mapping of pages may fail and cause packet to be dropped
3852          */
3853         pkt_prod = txdata->tx_pkt_prod;
3854         bd_prod = TX_BD(txdata->tx_bd_prod);
3855
3856         /* get a tx_buf and first BD
3857          * tx_start_bd may be changed during SPLIT,
3858          * but first_bd will always stay first
3859          */
3860         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3861         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3862         first_bd = tx_start_bd;
3863
3864         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3865
3866         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
3867                 if (!(bp->flags & TX_TIMESTAMPING_EN)) {
3868                         BNX2X_ERR("Tx timestamping was not enabled, this packet will not be timestamped\n");
3869                 } else if (bp->ptp_tx_skb) {
3870                         BNX2X_ERR("The device supports only a single outstanding packet to timestamp, this packet will not be timestamped\n");
3871                 } else {
3872                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3873                         /* schedule check for Tx timestamp */
3874                         bp->ptp_tx_skb = skb_get(skb);
3875                         bp->ptp_tx_start = jiffies;
3876                         schedule_work(&bp->ptp_task);
3877                 }
3878         }
3879
3880         /* header nbd: indirectly zero other flags! */
3881         tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3882
3883         /* remember the first BD of the packet */
3884         tx_buf->first_bd = txdata->tx_bd_prod;
3885         tx_buf->skb = skb;
3886         tx_buf->flags = 0;
3887
3888         DP(NETIF_MSG_TX_QUEUED,
3889            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3890            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3891
3892         if (skb_vlan_tag_present(skb)) {
3893                 tx_start_bd->vlan_or_ethertype =
3894                     cpu_to_le16(skb_vlan_tag_get(skb));
3895                 tx_start_bd->bd_flags.as_bitfield |=
3896                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3897         } else {
3898                 /* when transmitting in a vf, start bd must hold the ethertype
3899                  * for fw to enforce it
3900                  */
3901 #ifndef BNX2X_STOP_ON_ERROR
3902                 if (IS_VF(bp))
3903 #endif
3904                         tx_start_bd->vlan_or_ethertype =
3905                                 cpu_to_le16(ntohs(eth->h_proto));
3906 #ifndef BNX2X_STOP_ON_ERROR
3907                 else
3908                         /* used by FW for packet accounting */
3909                         tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3910 #endif
3911         }
3912
3913         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3914
3915         /* turn on parsing and get a BD */
3916         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3917
3918         if (xmit_type & XMIT_CSUM)
3919                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3920
3921         if (!CHIP_IS_E1x(bp)) {
3922                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3923                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3924
3925                 if (xmit_type & XMIT_CSUM_ENC) {
3926                         u16 global_data = 0;
3927
3928                         /* Set PBD in enc checksum offload case */
3929                         hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3930                                                       &pbd_e2_parsing_data,
3931                                                       xmit_type);
3932
3933                         /* turn on 2nd parsing and get a BD */
3934                         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3935
3936                         pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3937
3938                         memset(pbd2, 0, sizeof(*pbd2));
3939
3940                         pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3941                                 (skb_inner_network_header(skb) -
3942                                  skb->data) >> 1;
3943
3944                         if (xmit_type & XMIT_GSO_ENC)
3945                                 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3946                                                           &global_data,
3947                                                           xmit_type);
3948
3949                         pbd2->global_data = cpu_to_le16(global_data);
3950
3951                         /* add addition parse BD indication to start BD */
3952                         SET_FLAG(tx_start_bd->general_data,
3953                                  ETH_TX_START_BD_PARSE_NBDS, 1);
3954                         /* set encapsulation flag in start BD */
3955                         SET_FLAG(tx_start_bd->general_data,
3956                                  ETH_TX_START_BD_TUNNEL_EXIST, 1);
3957
3958                         tx_buf->flags |= BNX2X_HAS_SECOND_PBD;
3959
3960                         nbd++;
3961                 } else if (xmit_type & XMIT_CSUM) {
3962                         /* Set PBD in checksum offload case w/o encapsulation */
3963                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3964                                                      &pbd_e2_parsing_data,
3965                                                      xmit_type);
3966                 }
3967
3968                 bnx2x_set_ipv6_ext_e2(skb, &pbd_e2_parsing_data, xmit_type);
3969                 /* Add the macs to the parsing BD if this is a vf or if
3970                  * Tx Switching is enabled.
3971                  */
3972                 if (IS_VF(bp)) {
3973                         /* override GRE parameters in BD */
3974                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3975                                               &pbd_e2->data.mac_addr.src_mid,
3976                                               &pbd_e2->data.mac_addr.src_lo,
3977                                               eth->h_source);
3978
3979                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3980                                               &pbd_e2->data.mac_addr.dst_mid,
3981                                               &pbd_e2->data.mac_addr.dst_lo,
3982                                               eth->h_dest);
3983                 } else {
3984                         if (bp->flags & TX_SWITCHING)
3985                                 bnx2x_set_fw_mac_addr(
3986                                                 &pbd_e2->data.mac_addr.dst_hi,
3987                                                 &pbd_e2->data.mac_addr.dst_mid,
3988                                                 &pbd_e2->data.mac_addr.dst_lo,
3989                                                 eth->h_dest);
3990 #ifdef BNX2X_STOP_ON_ERROR
3991                         /* Enforce security is always set in Stop on Error -
3992                          * source mac should be present in the parsing BD
3993                          */
3994                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3995                                               &pbd_e2->data.mac_addr.src_mid,
3996                                               &pbd_e2->data.mac_addr.src_lo,
3997                                               eth->h_source);
3998 #endif
3999                 }
4000
4001                 SET_FLAG(pbd_e2_parsing_data,
4002                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
4003         } else {
4004                 u16 global_data = 0;
4005                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
4006                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
4007                 /* Set PBD in checksum offload case */
4008                 if (xmit_type & XMIT_CSUM)
4009                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
4010
4011                 SET_FLAG(global_data,
4012                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
4013                 pbd_e1x->global_data |= cpu_to_le16(global_data);
4014         }
4015
4016         /* Setup the data pointer of the first BD of the packet */
4017         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4018         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4019         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
4020         pkt_size = tx_start_bd->nbytes;
4021
4022         DP(NETIF_MSG_TX_QUEUED,
4023            "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
4024            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
4025            le16_to_cpu(tx_start_bd->nbytes),
4026            tx_start_bd->bd_flags.as_bitfield,
4027            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
4028
4029         if (xmit_type & XMIT_GSO) {
4030
4031                 DP(NETIF_MSG_TX_QUEUED,
4032                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
4033                    skb->len, hlen, skb_headlen(skb),
4034                    skb_shinfo(skb)->gso_size);
4035
4036                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
4037
4038                 if (unlikely(skb_headlen(skb) > hlen)) {
4039                         nbd++;
4040                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
4041                                                  &tx_start_bd, hlen,
4042                                                  bd_prod);
4043                 }
4044                 if (!CHIP_IS_E1x(bp))
4045                         pbd_e2_parsing_data |=
4046                                 (skb_shinfo(skb)->gso_size <<
4047                                  ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
4048                                  ETH_TX_PARSE_BD_E2_LSO_MSS;
4049                 else
4050                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
4051         }
4052
4053         /* Set the PBD's parsing_data field if not zero
4054          * (for the chips newer than 57711).
4055          */
4056         if (pbd_e2_parsing_data)
4057                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
4058
4059         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
4060
4061         /* Handle fragmented skb */
4062         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4063                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4064
4065                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
4066                                            skb_frag_size(frag), DMA_TO_DEVICE);
4067                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
4068                         unsigned int pkts_compl = 0, bytes_compl = 0;
4069
4070                         DP(NETIF_MSG_TX_QUEUED,
4071                            "Unable to map page - dropping packet...\n");
4072
4073                         /* we need unmap all buffers already mapped
4074                          * for this SKB;
4075                          * first_bd->nbd need to be properly updated
4076                          * before call to bnx2x_free_tx_pkt
4077                          */
4078                         first_bd->nbd = cpu_to_le16(nbd);
4079                         bnx2x_free_tx_pkt(bp, txdata,
4080                                           TX_BD(txdata->tx_pkt_prod),
4081                                           &pkts_compl, &bytes_compl);
4082                         return NETDEV_TX_OK;
4083                 }
4084
4085                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4086                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4087                 if (total_pkt_bd == NULL)
4088                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4089
4090                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4091                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4092                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
4093                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
4094                 nbd++;
4095
4096                 DP(NETIF_MSG_TX_QUEUED,
4097                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
4098                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
4099                    le16_to_cpu(tx_data_bd->nbytes));
4100         }
4101
4102         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
4103
4104         /* update with actual num BDs */
4105         first_bd->nbd = cpu_to_le16(nbd);
4106
4107         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4108
4109         /* now send a tx doorbell, counting the next BD
4110          * if the packet contains or ends with it
4111          */
4112         if (TX_BD_POFF(bd_prod) < nbd)
4113                 nbd++;
4114
4115         /* total_pkt_bytes should be set on the first data BD if
4116          * it's not an LSO packet and there is more than one
4117          * data BD. In this case pkt_size is limited by an MTU value.
4118          * However we prefer to set it for an LSO packet (while we don't
4119          * have to) in order to save some CPU cycles in a none-LSO
4120          * case, when we much more care about them.
4121          */
4122         if (total_pkt_bd != NULL)
4123                 total_pkt_bd->total_pkt_bytes = pkt_size;
4124
4125         if (pbd_e1x)
4126                 DP(NETIF_MSG_TX_QUEUED,
4127                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
4128                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4129                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4130                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4131                     le16_to_cpu(pbd_e1x->total_hlen_w));
4132         if (pbd_e2)
4133                 DP(NETIF_MSG_TX_QUEUED,
4134                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
4135                    pbd_e2,
4136                    pbd_e2->data.mac_addr.dst_hi,
4137                    pbd_e2->data.mac_addr.dst_mid,
4138                    pbd_e2->data.mac_addr.dst_lo,
4139                    pbd_e2->data.mac_addr.src_hi,
4140                    pbd_e2->data.mac_addr.src_mid,
4141                    pbd_e2->data.mac_addr.src_lo,
4142                    pbd_e2->parsing_data);
4143         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
4144
4145         netdev_tx_sent_queue(txq, skb->len);
4146
4147         skb_tx_timestamp(skb);
4148
4149         txdata->tx_pkt_prod++;
4150         /*
4151          * Make sure that the BD data is updated before updating the producer
4152          * since FW might read the BD right after the producer is updated.
4153          * This is only applicable for weak-ordered memory model archs such
4154          * as IA-64. The following barrier is also mandatory since FW will
4155          * assumes packets must have BDs.
4156          */
4157         wmb();
4158
4159         txdata->tx_db.data.prod += nbd;
4160         barrier();
4161
4162         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4163
4164         mmiowb();
4165
4166         txdata->tx_bd_prod += nbd;
4167
4168         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4169                 netif_tx_stop_queue(txq);
4170
4171                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4172                  * ordering of set_bit() in netif_tx_stop_queue() and read of
4173                  * fp->bd_tx_cons */
4174                 smp_mb();
4175
4176                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4177                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4178                         netif_tx_wake_queue(txq);
4179         }
4180         txdata->tx_pkt++;
4181
4182         return NETDEV_TX_OK;
4183 }
4184
4185 /**
4186  * bnx2x_setup_tc - routine to configure net_device for multi tc
4187  *
4188  * @netdev: net device to configure
4189  * @tc: number of traffic classes to enable
4190  *
4191  * callback connected to the ndo_setup_tc function pointer
4192  */
4193 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4194 {
4195         int cos, prio, count, offset;
4196         struct bnx2x *bp = netdev_priv(dev);
4197
4198         /* setup tc must be called under rtnl lock */
4199         ASSERT_RTNL();
4200
4201         /* no traffic classes requested. Aborting */
4202         if (!num_tc) {
4203                 netdev_reset_tc(dev);
4204                 return 0;
4205         }
4206
4207         /* requested to support too many traffic classes */
4208         if (num_tc > bp->max_cos) {
4209                 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4210                           num_tc, bp->max_cos);
4211                 return -EINVAL;
4212         }
4213
4214         /* declare amount of supported traffic classes */
4215         if (netdev_set_num_tc(dev, num_tc)) {
4216                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4217                 return -EINVAL;
4218         }
4219
4220         /* configure priority to traffic class mapping */
4221         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4222                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4223                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4224                    "mapping priority %d to tc %d\n",
4225                    prio, bp->prio_to_cos[prio]);
4226         }
4227
4228         /* Use this configuration to differentiate tc0 from other COSes
4229            This can be used for ets or pfc, and save the effort of setting
4230            up a multio class queue disc or negotiating DCBX with a switch
4231         netdev_set_prio_tc_map(dev, 0, 0);
4232         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4233         for (prio = 1; prio < 16; prio++) {
4234                 netdev_set_prio_tc_map(dev, prio, 1);
4235                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4236         } */
4237
4238         /* configure traffic class to transmission queue mapping */
4239         for (cos = 0; cos < bp->max_cos; cos++) {
4240                 count = BNX2X_NUM_ETH_QUEUES(bp);
4241                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4242                 netdev_set_tc_queue(dev, cos, count, offset);
4243                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4244                    "mapping tc %d to offset %d count %d\n",
4245                    cos, offset, count);
4246         }
4247
4248         return 0;
4249 }
4250
4251 /* called with rtnl_lock */
4252 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4253 {
4254         struct sockaddr *addr = p;
4255         struct bnx2x *bp = netdev_priv(dev);
4256         int rc = 0;
4257
4258         if (!is_valid_ether_addr(addr->sa_data)) {
4259                 BNX2X_ERR("Requested MAC address is not valid\n");
4260                 return -EINVAL;
4261         }
4262
4263         if (IS_MF_STORAGE_ONLY(bp)) {
4264                 BNX2X_ERR("Can't change address on STORAGE ONLY function\n");
4265                 return -EINVAL;
4266         }
4267
4268         if (netif_running(dev))  {
4269                 rc = bnx2x_set_eth_mac(bp, false);
4270                 if (rc)
4271                         return rc;
4272         }
4273
4274         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4275
4276         if (netif_running(dev))
4277                 rc = bnx2x_set_eth_mac(bp, true);
4278
4279         return rc;
4280 }
4281
4282 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4283 {
4284         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4285         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4286         u8 cos;
4287
4288         /* Common */
4289
4290         if (IS_FCOE_IDX(fp_index)) {
4291                 memset(sb, 0, sizeof(union host_hc_status_block));
4292                 fp->status_blk_mapping = 0;
4293         } else {
4294                 /* status blocks */
4295                 if (!CHIP_IS_E1x(bp))
4296                         BNX2X_PCI_FREE(sb->e2_sb,
4297                                        bnx2x_fp(bp, fp_index,
4298                                                 status_blk_mapping),
4299                                        sizeof(struct host_hc_status_block_e2));
4300                 else
4301                         BNX2X_PCI_FREE(sb->e1x_sb,
4302                                        bnx2x_fp(bp, fp_index,
4303                                                 status_blk_mapping),
4304                                        sizeof(struct host_hc_status_block_e1x));
4305         }
4306
4307         /* Rx */
4308         if (!skip_rx_queue(bp, fp_index)) {
4309                 bnx2x_free_rx_bds(fp);
4310
4311                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4312                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4313                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4314                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
4315                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
4316
4317                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4318                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
4319                                sizeof(struct eth_fast_path_rx_cqe) *
4320                                NUM_RCQ_BD);
4321
4322                 /* SGE ring */
4323                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4324                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4325                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
4326                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4327         }
4328
4329         /* Tx */
4330         if (!skip_tx_queue(bp, fp_index)) {
4331                 /* fastpath tx rings: tx_buf tx_desc */
4332                 for_each_cos_in_tx_queue(fp, cos) {
4333                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4334
4335                         DP(NETIF_MSG_IFDOWN,
4336                            "freeing tx memory of fp %d cos %d cid %d\n",
4337                            fp_index, cos, txdata->cid);
4338
4339                         BNX2X_FREE(txdata->tx_buf_ring);
4340                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
4341                                 txdata->tx_desc_mapping,
4342                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4343                 }
4344         }
4345         /* end of fastpath */
4346 }
4347
4348 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4349 {
4350         int i;
4351         for_each_cnic_queue(bp, i)
4352                 bnx2x_free_fp_mem_at(bp, i);
4353 }
4354
4355 void bnx2x_free_fp_mem(struct bnx2x *bp)
4356 {
4357         int i;
4358         for_each_eth_queue(bp, i)
4359                 bnx2x_free_fp_mem_at(bp, i);
4360 }
4361
4362 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4363 {
4364         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4365         if (!CHIP_IS_E1x(bp)) {
4366                 bnx2x_fp(bp, index, sb_index_values) =
4367                         (__le16 *)status_blk.e2_sb->sb.index_values;
4368                 bnx2x_fp(bp, index, sb_running_index) =
4369                         (__le16 *)status_blk.e2_sb->sb.running_index;
4370         } else {
4371                 bnx2x_fp(bp, index, sb_index_values) =
4372                         (__le16 *)status_blk.e1x_sb->sb.index_values;
4373                 bnx2x_fp(bp, index, sb_running_index) =
4374                         (__le16 *)status_blk.e1x_sb->sb.running_index;
4375         }
4376 }
4377
4378 /* Returns the number of actually allocated BDs */
4379 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4380                               int rx_ring_size)
4381 {
4382         struct bnx2x *bp = fp->bp;
4383         u16 ring_prod, cqe_ring_prod;
4384         int i, failure_cnt = 0;
4385
4386         fp->rx_comp_cons = 0;
4387         cqe_ring_prod = ring_prod = 0;
4388
4389         /* This routine is called only during fo init so
4390          * fp->eth_q_stats.rx_skb_alloc_failed = 0
4391          */
4392         for (i = 0; i < rx_ring_size; i++) {
4393                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4394                         failure_cnt++;
4395                         continue;
4396                 }
4397                 ring_prod = NEXT_RX_IDX(ring_prod);
4398                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4399                 WARN_ON(ring_prod <= (i - failure_cnt));
4400         }
4401
4402         if (failure_cnt)
4403                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4404                           i - failure_cnt, fp->index);
4405
4406         fp->rx_bd_prod = ring_prod;
4407         /* Limit the CQE producer by the CQE ring size */
4408         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4409                                cqe_ring_prod);
4410         fp->rx_pkt = fp->rx_calls = 0;
4411
4412         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4413
4414         return i - failure_cnt;
4415 }
4416
4417 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4418 {
4419         int i;
4420
4421         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4422                 struct eth_rx_cqe_next_page *nextpg;
4423
4424                 nextpg = (struct eth_rx_cqe_next_page *)
4425                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4426                 nextpg->addr_hi =
4427                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4428                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4429                 nextpg->addr_lo =
4430                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4431                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4432         }
4433 }
4434
4435 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4436 {
4437         union host_hc_status_block *sb;
4438         struct bnx2x_fastpath *fp = &bp->fp[index];
4439         int ring_size = 0;
4440         u8 cos;
4441         int rx_ring_size = 0;
4442
4443         if (!bp->rx_ring_size && IS_MF_STORAGE_ONLY(bp)) {
4444                 rx_ring_size = MIN_RX_SIZE_NONTPA;
4445                 bp->rx_ring_size = rx_ring_size;
4446         } else if (!bp->rx_ring_size) {
4447                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4448
4449                 if (CHIP_IS_E3(bp)) {
4450                         u32 cfg = SHMEM_RD(bp,
4451                                            dev_info.port_hw_config[BP_PORT(bp)].
4452                                            default_cfg);
4453
4454                         /* Decrease ring size for 1G functions */
4455                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4456                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
4457                                 rx_ring_size /= 10;
4458                 }
4459
4460                 /* allocate at least number of buffers required by FW */
4461                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4462                                      MIN_RX_SIZE_TPA, rx_ring_size);
4463
4464                 bp->rx_ring_size = rx_ring_size;
4465         } else /* if rx_ring_size specified - use it */
4466                 rx_ring_size = bp->rx_ring_size;
4467
4468         DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4469
4470         /* Common */
4471         sb = &bnx2x_fp(bp, index, status_blk);
4472
4473         if (!IS_FCOE_IDX(index)) {
4474                 /* status blocks */
4475                 if (!CHIP_IS_E1x(bp)) {
4476                         sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4477                                                     sizeof(struct host_hc_status_block_e2));
4478                         if (!sb->e2_sb)
4479                                 goto alloc_mem_err;
4480                 } else {
4481                         sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4482                                                      sizeof(struct host_hc_status_block_e1x));
4483                         if (!sb->e1x_sb)
4484                                 goto alloc_mem_err;
4485                 }
4486         }
4487
4488         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4489          * set shortcuts for it.
4490          */
4491         if (!IS_FCOE_IDX(index))
4492                 set_sb_shortcuts(bp, index);
4493
4494         /* Tx */
4495         if (!skip_tx_queue(bp, index)) {
4496                 /* fastpath tx rings: tx_buf tx_desc */
4497                 for_each_cos_in_tx_queue(fp, cos) {
4498                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4499
4500                         DP(NETIF_MSG_IFUP,
4501                            "allocating tx memory of fp %d cos %d\n",
4502                            index, cos);
4503
4504                         txdata->tx_buf_ring = kcalloc(NUM_TX_BD,
4505                                                       sizeof(struct sw_tx_bd),
4506                                                       GFP_KERNEL);
4507                         if (!txdata->tx_buf_ring)
4508                                 goto alloc_mem_err;
4509                         txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping,
4510                                                                sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4511                         if (!txdata->tx_desc_ring)
4512                                 goto alloc_mem_err;
4513                 }
4514         }
4515
4516         /* Rx */
4517         if (!skip_rx_queue(bp, index)) {
4518                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4519                 bnx2x_fp(bp, index, rx_buf_ring) =
4520                         kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL);
4521                 if (!bnx2x_fp(bp, index, rx_buf_ring))
4522                         goto alloc_mem_err;
4523                 bnx2x_fp(bp, index, rx_desc_ring) =
4524                         BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping),
4525                                         sizeof(struct eth_rx_bd) * NUM_RX_BD);
4526                 if (!bnx2x_fp(bp, index, rx_desc_ring))
4527                         goto alloc_mem_err;
4528
4529                 /* Seed all CQEs by 1s */
4530                 bnx2x_fp(bp, index, rx_comp_ring) =
4531                         BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping),
4532                                          sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD);
4533                 if (!bnx2x_fp(bp, index, rx_comp_ring))
4534                         goto alloc_mem_err;
4535
4536                 /* SGE ring */
4537                 bnx2x_fp(bp, index, rx_page_ring) =
4538                         kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page),
4539                                 GFP_KERNEL);
4540                 if (!bnx2x_fp(bp, index, rx_page_ring))
4541                         goto alloc_mem_err;
4542                 bnx2x_fp(bp, index, rx_sge_ring) =
4543                         BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping),
4544                                         BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4545                 if (!bnx2x_fp(bp, index, rx_sge_ring))
4546                         goto alloc_mem_err;
4547                 /* RX BD ring */
4548                 bnx2x_set_next_page_rx_bd(fp);
4549
4550                 /* CQ ring */
4551                 bnx2x_set_next_page_rx_cq(fp);
4552
4553                 /* BDs */
4554                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4555                 if (ring_size < rx_ring_size)
4556                         goto alloc_mem_err;
4557         }
4558
4559         return 0;
4560
4561 /* handles low memory cases */
4562 alloc_mem_err:
4563         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4564                                                 index, ring_size);
4565         /* FW will drop all packets if queue is not big enough,
4566          * In these cases we disable the queue
4567          * Min size is different for OOO, TPA and non-TPA queues
4568          */
4569         if (ring_size < (fp->mode == TPA_MODE_DISABLED ?
4570                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4571                         /* release memory allocated for this queue */
4572                         bnx2x_free_fp_mem_at(bp, index);
4573                         return -ENOMEM;
4574         }
4575         return 0;
4576 }
4577
4578 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4579 {
4580         if (!NO_FCOE(bp))
4581                 /* FCoE */
4582                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4583                         /* we will fail load process instead of mark
4584                          * NO_FCOE_FLAG
4585                          */
4586                         return -ENOMEM;
4587
4588         return 0;
4589 }
4590
4591 static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4592 {
4593         int i;
4594
4595         /* 1. Allocate FP for leading - fatal if error
4596          * 2. Allocate RSS - fix number of queues if error
4597          */
4598
4599         /* leading */
4600         if (bnx2x_alloc_fp_mem_at(bp, 0))
4601                 return -ENOMEM;
4602
4603         /* RSS */
4604         for_each_nondefault_eth_queue(bp, i)
4605                 if (bnx2x_alloc_fp_mem_at(bp, i))
4606                         break;
4607
4608         /* handle memory failures */
4609         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4610                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4611
4612                 WARN_ON(delta < 0);
4613                 bnx2x_shrink_eth_fp(bp, delta);
4614                 if (CNIC_SUPPORT(bp))
4615                         /* move non eth FPs next to last eth FP
4616                          * must be done in that order
4617                          * FCOE_IDX < FWD_IDX < OOO_IDX
4618                          */
4619
4620                         /* move FCoE fp even NO_FCOE_FLAG is on */
4621                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4622                 bp->num_ethernet_queues -= delta;
4623                 bp->num_queues = bp->num_ethernet_queues +
4624                                  bp->num_cnic_queues;
4625                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4626                           bp->num_queues + delta, bp->num_queues);
4627         }
4628
4629         return 0;
4630 }
4631
4632 void bnx2x_free_mem_bp(struct bnx2x *bp)
4633 {
4634         int i;
4635
4636         for (i = 0; i < bp->fp_array_size; i++)
4637                 kfree(bp->fp[i].tpa_info);
4638         kfree(bp->fp);
4639         kfree(bp->sp_objs);
4640         kfree(bp->fp_stats);
4641         kfree(bp->bnx2x_txq);
4642         kfree(bp->msix_table);
4643         kfree(bp->ilt);
4644 }
4645
4646 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4647 {
4648         struct bnx2x_fastpath *fp;
4649         struct msix_entry *tbl;
4650         struct bnx2x_ilt *ilt;
4651         int msix_table_size = 0;
4652         int fp_array_size, txq_array_size;
4653         int i;
4654
4655         /*
4656          * The biggest MSI-X table we might need is as a maximum number of fast
4657          * path IGU SBs plus default SB (for PF only).
4658          */
4659         msix_table_size = bp->igu_sb_cnt;
4660         if (IS_PF(bp))
4661                 msix_table_size++;
4662         BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4663
4664         /* fp array: RSS plus CNIC related L2 queues */
4665         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4666         bp->fp_array_size = fp_array_size;
4667         BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4668
4669         fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4670         if (!fp)
4671                 goto alloc_err;
4672         for (i = 0; i < bp->fp_array_size; i++) {
4673                 fp[i].tpa_info =
4674                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4675                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4676                 if (!(fp[i].tpa_info))
4677                         goto alloc_err;
4678         }
4679
4680         bp->fp = fp;
4681
4682         /* allocate sp objs */
4683         bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4684                               GFP_KERNEL);
4685         if (!bp->sp_objs)
4686                 goto alloc_err;
4687
4688         /* allocate fp_stats */
4689         bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4690                                GFP_KERNEL);
4691         if (!bp->fp_stats)
4692                 goto alloc_err;
4693
4694         /* Allocate memory for the transmission queues array */
4695         txq_array_size =
4696                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4697         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4698
4699         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4700                                 GFP_KERNEL);
4701         if (!bp->bnx2x_txq)
4702                 goto alloc_err;
4703
4704         /* msix table */
4705         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4706         if (!tbl)
4707                 goto alloc_err;
4708         bp->msix_table = tbl;
4709
4710         /* ilt */
4711         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4712         if (!ilt)
4713                 goto alloc_err;
4714         bp->ilt = ilt;
4715
4716         return 0;
4717 alloc_err:
4718         bnx2x_free_mem_bp(bp);
4719         return -ENOMEM;
4720 }
4721
4722 int bnx2x_reload_if_running(struct net_device *dev)
4723 {
4724         struct bnx2x *bp = netdev_priv(dev);
4725
4726         if (unlikely(!netif_running(dev)))
4727                 return 0;
4728
4729         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4730         return bnx2x_nic_load(bp, LOAD_NORMAL);
4731 }
4732
4733 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4734 {
4735         u32 sel_phy_idx = 0;
4736         if (bp->link_params.num_phys <= 1)
4737                 return INT_PHY;
4738
4739         if (bp->link_vars.link_up) {
4740                 sel_phy_idx = EXT_PHY1;
4741                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4742                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4743                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4744                         sel_phy_idx = EXT_PHY2;
4745         } else {
4746
4747                 switch (bnx2x_phy_selection(&bp->link_params)) {
4748                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4749                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4750                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4751                        sel_phy_idx = EXT_PHY1;
4752                        break;
4753                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4754                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4755                        sel_phy_idx = EXT_PHY2;
4756                        break;
4757                 }
4758         }
4759
4760         return sel_phy_idx;
4761 }
4762 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4763 {
4764         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4765         /*
4766          * The selected activated PHY is always after swapping (in case PHY
4767          * swapping is enabled). So when swapping is enabled, we need to reverse
4768          * the configuration
4769          */
4770
4771         if (bp->link_params.multi_phy_config &
4772             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4773                 if (sel_phy_idx == EXT_PHY1)
4774                         sel_phy_idx = EXT_PHY2;
4775                 else if (sel_phy_idx == EXT_PHY2)
4776                         sel_phy_idx = EXT_PHY1;
4777         }
4778         return LINK_CONFIG_IDX(sel_phy_idx);
4779 }
4780
4781 #ifdef NETDEV_FCOE_WWNN
4782 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4783 {
4784         struct bnx2x *bp = netdev_priv(dev);
4785         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4786
4787         switch (type) {
4788         case NETDEV_FCOE_WWNN:
4789                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4790                                 cp->fcoe_wwn_node_name_lo);
4791                 break;
4792         case NETDEV_FCOE_WWPN:
4793                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4794                                 cp->fcoe_wwn_port_name_lo);
4795                 break;
4796         default:
4797                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4798                 return -EINVAL;
4799         }
4800
4801         return 0;
4802 }
4803 #endif
4804
4805 /* called with rtnl_lock */
4806 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4807 {
4808         struct bnx2x *bp = netdev_priv(dev);
4809
4810         if (pci_num_vf(bp->pdev)) {
4811                 DP(BNX2X_MSG_IOV, "VFs are enabled, can not change MTU\n");
4812                 return -EPERM;
4813         }
4814
4815         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4816                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4817                 return -EAGAIN;
4818         }
4819
4820         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4821             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4822                 BNX2X_ERR("Can't support requested MTU size\n");
4823                 return -EINVAL;
4824         }
4825
4826         /* This does not race with packet allocation
4827          * because the actual alloc size is
4828          * only updated as part of load
4829          */
4830         dev->mtu = new_mtu;
4831
4832         return bnx2x_reload_if_running(dev);
4833 }
4834
4835 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4836                                      netdev_features_t features)
4837 {
4838         struct bnx2x *bp = netdev_priv(dev);
4839
4840         if (pci_num_vf(bp->pdev)) {
4841                 netdev_features_t changed = dev->features ^ features;
4842
4843                 /* Revert the requested changes in features if they
4844                  * would require internal reload of PF in bnx2x_set_features().
4845                  */
4846                 if (!(features & NETIF_F_RXCSUM) && !bp->disable_tpa) {
4847                         features &= ~NETIF_F_RXCSUM;
4848                         features |= dev->features & NETIF_F_RXCSUM;
4849                 }
4850
4851                 if (changed & NETIF_F_LOOPBACK) {
4852                         features &= ~NETIF_F_LOOPBACK;
4853                         features |= dev->features & NETIF_F_LOOPBACK;
4854                 }
4855         }
4856
4857         /* TPA requires Rx CSUM offloading */
4858         if (!(features & NETIF_F_RXCSUM)) {
4859                 features &= ~NETIF_F_LRO;
4860                 features &= ~NETIF_F_GRO;
4861         }
4862
4863         return features;
4864 }
4865
4866 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4867 {
4868         struct bnx2x *bp = netdev_priv(dev);
4869         netdev_features_t changes = features ^ dev->features;
4870         bool bnx2x_reload = false;
4871         int rc;
4872
4873         /* VFs or non SRIOV PFs should be able to change loopback feature */
4874         if (!pci_num_vf(bp->pdev)) {
4875                 if (features & NETIF_F_LOOPBACK) {
4876                         if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4877                                 bp->link_params.loopback_mode = LOOPBACK_BMAC;
4878                                 bnx2x_reload = true;
4879                         }
4880                 } else {
4881                         if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4882                                 bp->link_params.loopback_mode = LOOPBACK_NONE;
4883                                 bnx2x_reload = true;
4884                         }
4885                 }
4886         }
4887
4888         /* if GRO is changed while LRO is enabled, don't force a reload */
4889         if ((changes & NETIF_F_GRO) && (features & NETIF_F_LRO))
4890                 changes &= ~NETIF_F_GRO;
4891
4892         /* if GRO is changed while HW TPA is off, don't force a reload */
4893         if ((changes & NETIF_F_GRO) && bp->disable_tpa)
4894                 changes &= ~NETIF_F_GRO;
4895
4896         if (changes)
4897                 bnx2x_reload = true;
4898
4899         if (bnx2x_reload) {
4900                 if (bp->recovery_state == BNX2X_RECOVERY_DONE) {
4901                         dev->features = features;
4902                         rc = bnx2x_reload_if_running(dev);
4903                         return rc ? rc : 1;
4904                 }
4905                 /* else: bnx2x_nic_load() will be called at end of recovery */
4906         }
4907
4908         return 0;
4909 }
4910
4911 void bnx2x_tx_timeout(struct net_device *dev)
4912 {
4913         struct bnx2x *bp = netdev_priv(dev);
4914
4915 #ifdef BNX2X_STOP_ON_ERROR
4916         if (!bp->panic)
4917                 bnx2x_panic();
4918 #endif
4919
4920         /* This allows the netif to be shutdown gracefully before resetting */
4921         bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0);
4922 }
4923
4924 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4925 {
4926         struct net_device *dev = pci_get_drvdata(pdev);
4927         struct bnx2x *bp;
4928
4929         if (!dev) {
4930                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4931                 return -ENODEV;
4932         }
4933         bp = netdev_priv(dev);
4934
4935         rtnl_lock();
4936
4937         pci_save_state(pdev);
4938
4939         if (!netif_running(dev)) {
4940                 rtnl_unlock();
4941                 return 0;
4942         }
4943
4944         netif_device_detach(dev);
4945
4946         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4947
4948         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4949
4950         rtnl_unlock();
4951
4952         return 0;
4953 }
4954
4955 int bnx2x_resume(struct pci_dev *pdev)
4956 {
4957         struct net_device *dev = pci_get_drvdata(pdev);
4958         struct bnx2x *bp;
4959         int rc;
4960
4961         if (!dev) {
4962                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4963                 return -ENODEV;
4964         }
4965         bp = netdev_priv(dev);
4966
4967         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4968                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4969                 return -EAGAIN;
4970         }
4971
4972         rtnl_lock();
4973
4974         pci_restore_state(pdev);
4975
4976         if (!netif_running(dev)) {
4977                 rtnl_unlock();
4978                 return 0;
4979         }
4980
4981         bnx2x_set_power_state(bp, PCI_D0);
4982         netif_device_attach(dev);
4983
4984         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4985
4986         rtnl_unlock();
4987
4988         return rc;
4989 }
4990
4991 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4992                               u32 cid)
4993 {
4994         if (!cxt) {
4995                 BNX2X_ERR("bad context pointer %p\n", cxt);
4996                 return;
4997         }
4998
4999         /* ustorm cxt validation */
5000         cxt->ustorm_ag_context.cdu_usage =
5001                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5002                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
5003         /* xcontext validation */
5004         cxt->xstorm_ag_context.cdu_reserved =
5005                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5006                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
5007 }
5008
5009 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
5010                                     u8 fw_sb_id, u8 sb_index,
5011                                     u8 ticks)
5012 {
5013         u32 addr = BAR_CSTRORM_INTMEM +
5014                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
5015         REG_WR8(bp, addr, ticks);
5016         DP(NETIF_MSG_IFUP,
5017            "port %x fw_sb_id %d sb_index %d ticks %d\n",
5018            port, fw_sb_id, sb_index, ticks);
5019 }
5020
5021 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
5022                                     u16 fw_sb_id, u8 sb_index,
5023                                     u8 disable)
5024 {
5025         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
5026         u32 addr = BAR_CSTRORM_INTMEM +
5027                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
5028         u8 flags = REG_RD8(bp, addr);
5029         /* clear and set */
5030         flags &= ~HC_INDEX_DATA_HC_ENABLED;
5031         flags |= enable_flag;
5032         REG_WR8(bp, addr, flags);
5033         DP(NETIF_MSG_IFUP,
5034            "port %x fw_sb_id %d sb_index %d disable %d\n",
5035            port, fw_sb_id, sb_index, disable);
5036 }
5037
5038 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
5039                                     u8 sb_index, u8 disable, u16 usec)
5040 {
5041         int port = BP_PORT(bp);
5042         u8 ticks = usec / BNX2X_BTR;
5043
5044         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
5045
5046         disable = disable ? 1 : (usec ? 0 : 1);
5047         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
5048 }
5049
5050 void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag,
5051                             u32 verbose)
5052 {
5053         smp_mb__before_atomic();
5054         set_bit(flag, &bp->sp_rtnl_state);
5055         smp_mb__after_atomic();
5056         DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n",
5057            flag);
5058         schedule_delayed_work(&bp->sp_rtnl_task, 0);
5059 }
5060 EXPORT_SYMBOL(bnx2x_schedule_sp_rtnl);