drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  30 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  31 MODULE_AUTHOR("Emulex Corporation");
  32 MODULE_LICENSE("GPL");
  33
  34 static unsigned int num_vfs;
  35 module_param(num_vfs, uint, S_IRUGO);
  36 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  37
  38 static ushort rx_frag_size = 2048;
  39 module_param(rx_frag_size, ushort, S_IRUGO);
  40 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  41
  42 static const struct pci_device_id be_dev_ids[] = {
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  44         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  51         { 0 }
  52 };
  53 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  54 /* UE Status Low CSR */
  55 static const char * const ue_status_low_desc[] = {
  56         "CEV",
  57         "CTX",
  58         "DBUF",
  59         "ERX",
  60         "Host",
  61         "MPU",
  62         "NDMA",
  63         "PTC ",
  64         "RDMA ",
  65         "RXF ",
  66         "RXIPS ",
  67         "RXULP0 ",
  68         "RXULP1 ",
  69         "RXULP2 ",
  70         "TIM ",
  71         "TPOST ",
  72         "TPRE ",
  73         "TXIPS ",
  74         "TXULP0 ",
  75         "TXULP1 ",
  76         "UC ",
  77         "WDMA ",
  78         "TXULP2 ",
  79         "HOST1 ",
  80         "P0_OB_LINK ",
  81         "P1_OB_LINK ",
  82         "HOST_GPIO ",
  83         "MBOX ",
  84         "ERX2 ",
  85         "SPARE ",
  86         "JTAG ",
  87         "MPU_INTPEND "
  88 };
  89 /* UE Status High CSR */
  90 static const char * const ue_status_hi_desc[] = {
  91         "LPCMEMHOST",
  92         "MGMT_MAC",
  93         "PCS0ONLINE",
  94         "MPU_IRAM",
  95         "PCS1ONLINE",
  96         "PCTL0",
  97         "PCTL1",
  98         "PMEM",
  99         "RR",
 100         "TXPB",
 101         "RXPP",
 102         "XAUI",
 103         "TXP",
 104         "ARM",
 105         "IPC",
 106         "HOST2",
 107         "HOST3",
 108         "HOST4",
 109         "HOST5",
 110         "HOST6",
 111         "HOST7",
 112         "ECRC",
 113         "Poison TLP",
 114         "NETC",
 115         "PERIPH",
 116         "LLTXULP",
 117         "D2P",
 118         "RCON",
 119         "LDMA",
 120         "LLTXP",
 121         "LLTXPB",
 122         "Unknown"
 123 };
 124
 125
 126 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 127 {
 128         struct be_dma_mem *mem = &q->dma_mem;
 129         if (mem->va) {
 130                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 131                                   mem->dma);
 132                 mem->va = NULL;
 133         }
 134 }
 135
 136 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 137                           u16 len, u16 entry_size)
 138 {
 139         struct be_dma_mem *mem = &q->dma_mem;
 140
 141         memset(q, 0, sizeof(*q));
 142         q->len = len;
 143         q->entry_size = entry_size;
 144         mem->size = len * entry_size;
 145         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 146                                       GFP_KERNEL);
 147         if (!mem->va)
 148                 return -ENOMEM;
 149         return 0;
 150 }
 151
 152 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 153 {
 154         u32 reg, enabled;
 155
 156         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 157                               &reg);
 158         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 159
 160         if (!enabled && enable)
 161                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 162         else if (enabled && !enable)
 163                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 164         else
 165                 return;
 166
 167         pci_write_config_dword(adapter->pdev,
 168                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 169 }
 170
 171 static void be_intr_set(struct be_adapter *adapter, bool enable)
 172 {
 173         int status = 0;
 174
 175         /* On lancer interrupts can't be controlled via this register */
 176         if (lancer_chip(adapter))
 177                 return;
 178
 179         if (adapter->eeh_error)
 180                 return;
 181
 182         status = be_cmd_intr_set(adapter, enable);
 183         if (status)
 184                 be_reg_intr_set(adapter, enable);
 185 }
 186
 187 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 188 {
 189         u32 val = 0;
 190         val |= qid & DB_RQ_RING_ID_MASK;
 191         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 192
 193         wmb();
 194         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 195 }
 196
 197 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 198                           u16 posted)
 199 {
 200         u32 val = 0;
 201         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 202         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 203
 204         wmb();
 205         iowrite32(val, adapter->db + txo->db_offset);
 206 }
 207
 208 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 209                          bool arm, bool clear_int, u16 num_popped)
 210 {
 211         u32 val = 0;
 212         val |= qid & DB_EQ_RING_ID_MASK;
 213         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 214
 215         if (adapter->eeh_error)
 216                 return;
 217
 218         if (arm)
 219                 val |= 1 << DB_EQ_REARM_SHIFT;
 220         if (clear_int)
 221                 val |= 1 << DB_EQ_CLR_SHIFT;
 222         val |= 1 << DB_EQ_EVNT_SHIFT;
 223         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 224         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 225 }
 226
 227 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 228 {
 229         u32 val = 0;
 230         val |= qid & DB_CQ_RING_ID_MASK;
 231         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 232                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 233
 234         if (adapter->eeh_error)
 235                 return;
 236
 237         if (arm)
 238                 val |= 1 << DB_CQ_REARM_SHIFT;
 239         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 240         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 241 }
 242
 243 static int be_mac_addr_set(struct net_device *netdev, void *p)
 244 {
 245         struct be_adapter *adapter = netdev_priv(netdev);
 246         struct device *dev = &adapter->pdev->dev;
 247         struct sockaddr *addr = p;
 248         int status;
 249         u8 mac[ETH_ALEN];
 250         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 251
 252         if (!is_valid_ether_addr(addr->sa_data))
 253                 return -EADDRNOTAVAIL;
 254
 255         /* Proceed further only if, User provided MAC is different
 256          * from active MAC
 257          */
 258         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 259                 return 0;
 260
 261         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 262          * privilege or if PF did not provision the new MAC address.
 263          * On BE3, this cmd will always fail if the VF doesn't have the
 264          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 265          * the MAC for the VF.
 266          */
 267         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 268                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 269         if (!status) {
 270                 curr_pmac_id = adapter->pmac_id[0];
 271
 272                 /* Delete the old programmed MAC. This call may fail if the
 273                  * old MAC was already deleted by the PF driver.
 274                  */
 275                 if (adapter->pmac_id[0] != old_pmac_id)
 276                         be_cmd_pmac_del(adapter, adapter->if_handle,
 277                                         old_pmac_id, 0);
 278         }
 279
 280         /* Decide if the new MAC is successfully activated only after
 281          * querying the FW
 282          */
 283         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 284                                        adapter->if_handle, true, 0);
 285         if (status)
 286                 goto err;
 287
 288         /* The MAC change did not happen, either due to lack of privilege
 289          * or PF didn't pre-provision.
 290          */
 291         if (!ether_addr_equal(addr->sa_data, mac)) {
 292                 status = -EPERM;
 293                 goto err;
 294         }
 295
 296         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 297         dev_info(dev, "MAC address changed to %pM\n", mac);
 298         return 0;
 299 err:
 300         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 301         return status;
 302 }
 303
 304 /* BE2 supports only v0 cmd */
 305 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 306 {
 307         if (BE2_chip(adapter)) {
 308                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 309
 310                 return &cmd->hw_stats;
 311         } else if (BE3_chip(adapter)) {
 312                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 313
 314                 return &cmd->hw_stats;
 315         } else {
 316                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 317
 318                 return &cmd->hw_stats;
 319         }
 320 }
 321
 322 /* BE2 supports only v0 cmd */
 323 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 324 {
 325         if (BE2_chip(adapter)) {
 326                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 327
 328                 return &hw_stats->erx;
 329         } else if (BE3_chip(adapter)) {
 330                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 331
 332                 return &hw_stats->erx;
 333         } else {
 334                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 335
 336                 return &hw_stats->erx;
 337         }
 338 }
 339
 340 static void populate_be_v0_stats(struct be_adapter *adapter)
 341 {
 342         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 343         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 344         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 345         struct be_port_rxf_stats_v0 *port_stats =
 346                                         &rxf_stats->port[adapter->port_num];
 347         struct be_drv_stats *drvs = &adapter->drv_stats;
 348
 349         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 350         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 351         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 352         drvs->rx_control_frames = port_stats->rx_control_frames;
 353         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 354         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 355         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 356         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 357         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 358         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 359         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 360         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 361         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 362         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 363         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 364         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 365         drvs->rx_dropped_header_too_small =
 366                 port_stats->rx_dropped_header_too_small;
 367         drvs->rx_address_filtered =
 368                                         port_stats->rx_address_filtered +
 369                                         port_stats->rx_vlan_filtered;
 370         drvs->rx_alignment_symbol_errors =
 371                 port_stats->rx_alignment_symbol_errors;
 372
 373         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 374         drvs->tx_controlframes = port_stats->tx_controlframes;
 375
 376         if (adapter->port_num)
 377                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 378         else
 379                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 380         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 381         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 382         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 383         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 384         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 385         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 386         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 387 }
 388
 389 static void populate_be_v1_stats(struct be_adapter *adapter)
 390 {
 391         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 392         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 393         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 394         struct be_port_rxf_stats_v1 *port_stats =
 395                                         &rxf_stats->port[adapter->port_num];
 396         struct be_drv_stats *drvs = &adapter->drv_stats;
 397
 398         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 399         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 400         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 401         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 402         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 403         drvs->rx_control_frames = port_stats->rx_control_frames;
 404         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 405         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 406         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 407         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 408         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 409         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 410         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 411         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 412         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 413         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 414         drvs->rx_dropped_header_too_small =
 415                 port_stats->rx_dropped_header_too_small;
 416         drvs->rx_input_fifo_overflow_drop =
 417                 port_stats->rx_input_fifo_overflow_drop;
 418         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 419         drvs->rx_alignment_symbol_errors =
 420                 port_stats->rx_alignment_symbol_errors;
 421         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 422         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 423         drvs->tx_controlframes = port_stats->tx_controlframes;
 424         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 425         drvs->jabber_events = port_stats->jabber_events;
 426         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 427         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 428         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 429         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 430         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 431         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 432         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 433 }
 434
 435 static void populate_be_v2_stats(struct be_adapter *adapter)
 436 {
 437         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 438         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 439         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 440         struct be_port_rxf_stats_v2 *port_stats =
 441                                         &rxf_stats->port[adapter->port_num];
 442         struct be_drv_stats *drvs = &adapter->drv_stats;
 443
 444         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 445         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 446         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 447         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 448         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 449         drvs->rx_control_frames = port_stats->rx_control_frames;
 450         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 451         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 452         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 453         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 454         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 455         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 456         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 457         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 458         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 459         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 460         drvs->rx_dropped_header_too_small =
 461                 port_stats->rx_dropped_header_too_small;
 462         drvs->rx_input_fifo_overflow_drop =
 463                 port_stats->rx_input_fifo_overflow_drop;
 464         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 465         drvs->rx_alignment_symbol_errors =
 466                 port_stats->rx_alignment_symbol_errors;
 467         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 468         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 469         drvs->tx_controlframes = port_stats->tx_controlframes;
 470         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 471         drvs->jabber_events = port_stats->jabber_events;
 472         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 473         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 474         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 475         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 476         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 477         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 478         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 479         if (be_roce_supported(adapter)) {
 480                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 481                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 482                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 483                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 484                 drvs->roce_drops_payload_len =
 485                         port_stats->roce_drops_payload_len;
 486         }
 487 }
 488
 489 static void populate_lancer_stats(struct be_adapter *adapter)
 490 {
 491
 492         struct be_drv_stats *drvs = &adapter->drv_stats;
 493         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 494
 495         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 496         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 497         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 498         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 499         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 500         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 501         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 502         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 503         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 504         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 505         drvs->rx_dropped_tcp_length =
 506                                 pport_stats->rx_dropped_invalid_tcp_length;
 507         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 508         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 509         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 510         drvs->rx_dropped_header_too_small =
 511                                 pport_stats->rx_dropped_header_too_small;
 512         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 513         drvs->rx_address_filtered =
 514                                         pport_stats->rx_address_filtered +
 515                                         pport_stats->rx_vlan_filtered;
 516         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 517         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 518         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 519         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 520         drvs->jabber_events = pport_stats->rx_jabbers;
 521         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 522         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 523         drvs->rx_drops_too_many_frags =
 524                                 pport_stats->rx_drops_too_many_frags_lo;
 525 }
 526
 527 static void accumulate_16bit_val(u32 *acc, u16 val)
 528 {
 529 #define lo(x)                   (x & 0xFFFF)
 530 #define hi(x)                   (x & 0xFFFF0000)
 531         bool wrapped = val < lo(*acc);
 532         u32 newacc = hi(*acc) + val;
 533
 534         if (wrapped)
 535                 newacc += 65536;
 536         ACCESS_ONCE(*acc) = newacc;
 537 }
 538
 539 static void populate_erx_stats(struct be_adapter *adapter,
 540                                struct be_rx_obj *rxo, u32 erx_stat)
 541 {
 542         if (!BEx_chip(adapter))
 543                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 544         else
 545                 /* below erx HW counter can actually wrap around after
 546                  * 65535. Driver accumulates a 32-bit value
 547                  */
 548                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 549                                      (u16)erx_stat);
 550 }
 551
 552 void be_parse_stats(struct be_adapter *adapter)
 553 {
 554         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 555         struct be_rx_obj *rxo;
 556         int i;
 557         u32 erx_stat;
 558
 559         if (lancer_chip(adapter)) {
 560                 populate_lancer_stats(adapter);
 561         } else {
 562                 if (BE2_chip(adapter))
 563                         populate_be_v0_stats(adapter);
 564                 else if (BE3_chip(adapter))
 565                         /* for BE3 */
 566                         populate_be_v1_stats(adapter);
 567                 else
 568                         populate_be_v2_stats(adapter);
 569
 570                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 571                 for_all_rx_queues(adapter, rxo, i) {
 572                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 573                         populate_erx_stats(adapter, rxo, erx_stat);
 574                 }
 575         }
 576 }
 577
 578 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 579                                                 struct rtnl_link_stats64 *stats)
 580 {
 581         struct be_adapter *adapter = netdev_priv(netdev);
 582         struct be_drv_stats *drvs = &adapter->drv_stats;
 583         struct be_rx_obj *rxo;
 584         struct be_tx_obj *txo;
 585         u64 pkts, bytes;
 586         unsigned int start;
 587         int i;
 588
 589         for_all_rx_queues(adapter, rxo, i) {
 590                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 591                 do {
 592                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 593                         pkts = rx_stats(rxo)->rx_pkts;
 594                         bytes = rx_stats(rxo)->rx_bytes;
 595                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 596                 stats->rx_packets += pkts;
 597                 stats->rx_bytes += bytes;
 598                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 599                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 600                                         rx_stats(rxo)->rx_drops_no_frags;
 601         }
 602
 603         for_all_tx_queues(adapter, txo, i) {
 604                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 605                 do {
 606                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 607                         pkts = tx_stats(txo)->tx_pkts;
 608                         bytes = tx_stats(txo)->tx_bytes;
 609                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 610                 stats->tx_packets += pkts;
 611                 stats->tx_bytes += bytes;
 612         }
 613
 614         /* bad pkts received */
 615         stats->rx_errors = drvs->rx_crc_errors +
 616                 drvs->rx_alignment_symbol_errors +
 617                 drvs->rx_in_range_errors +
 618                 drvs->rx_out_range_errors +
 619                 drvs->rx_frame_too_long +
 620                 drvs->rx_dropped_too_small +
 621                 drvs->rx_dropped_too_short +
 622                 drvs->rx_dropped_header_too_small +
 623                 drvs->rx_dropped_tcp_length +
 624                 drvs->rx_dropped_runt;
 625
 626         /* detailed rx errors */
 627         stats->rx_length_errors = drvs->rx_in_range_errors +
 628                 drvs->rx_out_range_errors +
 629                 drvs->rx_frame_too_long;
 630
 631         stats->rx_crc_errors = drvs->rx_crc_errors;
 632
 633         /* frame alignment errors */
 634         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 635
 636         /* receiver fifo overrun */
 637         /* drops_no_pbuf is no per i/f, it's per BE card */
 638         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 639                                 drvs->rx_input_fifo_overflow_drop +
 640                                 drvs->rx_drops_no_pbuf;
 641         return stats;
 642 }
 643
 644 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 645 {
 646         struct net_device *netdev = adapter->netdev;
 647
 648         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 649                 netif_carrier_off(netdev);
 650                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 651         }
 652
 653         if (link_status)
 654                 netif_carrier_on(netdev);
 655         else
 656                 netif_carrier_off(netdev);
 657 }
 658
 659 static void be_tx_stats_update(struct be_tx_obj *txo,
 660                                u32 wrb_cnt, u32 copied, u32 gso_segs,
 661                                bool stopped)
 662 {
 663         struct be_tx_stats *stats = tx_stats(txo);
 664
 665         u64_stats_update_begin(&stats->sync);
 666         stats->tx_reqs++;
 667         stats->tx_wrbs += wrb_cnt;
 668         stats->tx_bytes += copied;
 669         stats->tx_pkts += (gso_segs ? gso_segs : 1);
 670         if (stopped)
 671                 stats->tx_stops++;
 672         u64_stats_update_end(&stats->sync);
 673 }
 674
 675 /* Determine number of WRB entries needed to xmit data in an skb */
 676 static u32 wrb_cnt_for_skb(struct be_adapter *adapter, struct sk_buff *skb,
 677                            bool *dummy)
 678 {
 679         int cnt = (skb->len > skb->data_len);
 680
 681         cnt += skb_shinfo(skb)->nr_frags;
 682
 683         /* to account for hdr wrb */
 684         cnt++;
 685         if (lancer_chip(adapter) || !(cnt & 1)) {
 686                 *dummy = false;
 687         } else {
 688                 /* add a dummy to make it an even num */
 689                 cnt++;
 690                 *dummy = true;
 691         }
 692         BUG_ON(cnt > BE_MAX_TX_FRAG_COUNT);
 693         return cnt;
 694 }
 695
 696 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 697 {
 698         wrb->frag_pa_hi = upper_32_bits(addr);
 699         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 700         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 701         wrb->rsvd0 = 0;
 702 }
 703
 704 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 705                                      struct sk_buff *skb)
 706 {
 707         u8 vlan_prio;
 708         u16 vlan_tag;
 709
 710         vlan_tag = vlan_tx_tag_get(skb);
 711         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 712         /* If vlan priority provided by OS is NOT in available bmap */
 713         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 714                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 715                                 adapter->recommended_prio;
 716
 717         return vlan_tag;
 718 }
 719
 720 /* Used only for IP tunnel packets */
 721 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 722 {
 723         return (inner_ip_hdr(skb)->version == 4) ?
 724                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 725 }
 726
 727 static u16 skb_ip_proto(struct sk_buff *skb)
 728 {
 729         return (ip_hdr(skb)->version == 4) ?
 730                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 731 }
 732
 733 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 734                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 735                          bool skip_hw_vlan)
 736 {
 737         u16 vlan_tag, proto;
 738
 739         memset(hdr, 0, sizeof(*hdr));
 740
 741         SET_TX_WRB_HDR_BITS(crc, hdr, 1);
 742
 743         if (skb_is_gso(skb)) {
 744                 SET_TX_WRB_HDR_BITS(lso, hdr, 1);
 745                 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size);
 746                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 747                         SET_TX_WRB_HDR_BITS(lso6, hdr, 1);
 748         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 749                 if (skb->encapsulation) {
 750                         SET_TX_WRB_HDR_BITS(ipcs, hdr, 1);
 751                         proto = skb_inner_ip_proto(skb);
 752                 } else {
 753                         proto = skb_ip_proto(skb);
 754                 }
 755                 if (proto == IPPROTO_TCP)
 756                         SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1);
 757                 else if (proto == IPPROTO_UDP)
 758                         SET_TX_WRB_HDR_BITS(udpcs, hdr, 1);
 759         }
 760
 761         if (vlan_tx_tag_present(skb)) {
 762                 SET_TX_WRB_HDR_BITS(vlan, hdr, 1);
 763                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 764                 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag);
 765         }
 766
 767         /* To skip HW VLAN tagging: evt = 1, compl = 0 */
 768         SET_TX_WRB_HDR_BITS(complete, hdr, !skip_hw_vlan);
 769         SET_TX_WRB_HDR_BITS(event, hdr, 1);
 770         SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt);
 771         SET_TX_WRB_HDR_BITS(len, hdr, len);
 772 }
 773
 774 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 775                           bool unmap_single)
 776 {
 777         dma_addr_t dma;
 778
 779         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 780
 781         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 782         if (wrb->frag_len) {
 783                 if (unmap_single)
 784                         dma_unmap_single(dev, dma, wrb->frag_len,
 785                                          DMA_TO_DEVICE);
 786                 else
 787                         dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE);
 788         }
 789 }
 790
 791 static int make_tx_wrbs(struct be_adapter *adapter, struct be_queue_info *txq,
 792                         struct sk_buff *skb, u32 wrb_cnt, bool dummy_wrb,
 793                         bool skip_hw_vlan)
 794 {
 795         dma_addr_t busaddr;
 796         int i, copied = 0;
 797         struct device *dev = &adapter->pdev->dev;
 798         struct sk_buff *first_skb = skb;
 799         struct be_eth_wrb *wrb;
 800         struct be_eth_hdr_wrb *hdr;
 801         bool map_single = false;
 802         u16 map_head;
 803
 804         hdr = queue_head_node(txq);
 805         queue_head_inc(txq);
 806         map_head = txq->head;
 807
 808         if (skb->len > skb->data_len) {
 809                 int len = skb_headlen(skb);
 810                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 811                 if (dma_mapping_error(dev, busaddr))
 812                         goto dma_err;
 813                 map_single = true;
 814                 wrb = queue_head_node(txq);
 815                 wrb_fill(wrb, busaddr, len);
 816                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 817                 queue_head_inc(txq);
 818                 copied += len;
 819         }
 820
 821         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 822                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 823                 busaddr = skb_frag_dma_map(dev, frag, 0,
 824                                            skb_frag_size(frag), DMA_TO_DEVICE);
 825                 if (dma_mapping_error(dev, busaddr))
 826                         goto dma_err;
 827                 wrb = queue_head_node(txq);
 828                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 829                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 830                 queue_head_inc(txq);
 831                 copied += skb_frag_size(frag);
 832         }
 833
 834         if (dummy_wrb) {
 835                 wrb = queue_head_node(txq);
 836                 wrb_fill(wrb, 0, 0);
 837                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 838                 queue_head_inc(txq);
 839         }
 840
 841         wrb_fill_hdr(adapter, hdr, first_skb, wrb_cnt, copied, skip_hw_vlan);
 842         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 843
 844         return copied;
 845 dma_err:
 846         txq->head = map_head;
 847         while (copied) {
 848                 wrb = queue_head_node(txq);
 849                 unmap_tx_frag(dev, wrb, map_single);
 850                 map_single = false;
 851                 copied -= wrb->frag_len;
 852                 adapter->drv_stats.dma_map_errors++;
 853                 queue_head_inc(txq);
 854         }
 855         return 0;
 856 }
 857
 858 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 859                                              struct sk_buff *skb,
 860                                              bool *skip_hw_vlan)
 861 {
 862         u16 vlan_tag = 0;
 863
 864         skb = skb_share_check(skb, GFP_ATOMIC);
 865         if (unlikely(!skb))
 866                 return skb;
 867
 868         if (vlan_tx_tag_present(skb))
 869                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 870
 871         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 872                 if (!vlan_tag)
 873                         vlan_tag = adapter->pvid;
 874                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 875                  * skip VLAN insertion
 876                  */
 877                 if (skip_hw_vlan)
 878                         *skip_hw_vlan = true;
 879         }
 880
 881         if (vlan_tag) {
 882                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 883                 if (unlikely(!skb))
 884                         return skb;
 885                 skb->vlan_tci = 0;
 886         }
 887
 888         /* Insert the outer VLAN, if any */
 889         if (adapter->qnq_vid) {
 890                 vlan_tag = adapter->qnq_vid;
 891                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 892                 if (unlikely(!skb))
 893                         return skb;
 894                 if (skip_hw_vlan)
 895                         *skip_hw_vlan = true;
 896         }
 897
 898         return skb;
 899 }
 900
 901 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 902 {
 903         struct ethhdr *eh = (struct ethhdr *)skb->data;
 904         u16 offset = ETH_HLEN;
 905
 906         if (eh->h_proto == htons(ETH_P_IPV6)) {
 907                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 908
 909                 offset += sizeof(struct ipv6hdr);
 910                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 911                     ip6h->nexthdr != NEXTHDR_UDP) {
 912                         struct ipv6_opt_hdr *ehdr =
 913                                 (struct ipv6_opt_hdr *) (skb->data + offset);
 914
 915                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 916                         if (ehdr->hdrlen == 0xff)
 917                                 return true;
 918                 }
 919         }
 920         return false;
 921 }
 922
 923 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 924 {
 925         return vlan_tx_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 926 }
 927
 928 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 929 {
 930         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 931 }
 932
 933 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 934                                                   struct sk_buff *skb,
 935                                                   bool *skip_hw_vlan)
 936 {
 937         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 938         unsigned int eth_hdr_len;
 939         struct iphdr *ip;
 940
 941         /* For padded packets, BE HW modifies tot_len field in IP header
 942          * incorrecly when VLAN tag is inserted by HW.
 943          * For padded packets, Lancer computes incorrect checksum.
 944          */
 945         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 946                                                 VLAN_ETH_HLEN : ETH_HLEN;
 947         if (skb->len <= 60 &&
 948             (lancer_chip(adapter) || vlan_tx_tag_present(skb)) &&
 949             is_ipv4_pkt(skb)) {
 950                 ip = (struct iphdr *)ip_hdr(skb);
 951                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 952         }
 953
 954         /* If vlan tag is already inlined in the packet, skip HW VLAN
 955          * tagging in pvid-tagging mode
 956          */
 957         if (be_pvid_tagging_enabled(adapter) &&
 958             veh->h_vlan_proto == htons(ETH_P_8021Q))
 959                 *skip_hw_vlan = true;
 960
 961         /* HW has a bug wherein it will calculate CSUM for VLAN
 962          * pkts even though it is disabled.
 963          * Manually insert VLAN in pkt.
 964          */
 965         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 966             vlan_tx_tag_present(skb)) {
 967                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 968                 if (unlikely(!skb))
 969                         goto err;
 970         }
 971
 972         /* HW may lockup when VLAN HW tagging is requested on
 973          * certain ipv6 packets. Drop such pkts if the HW workaround to
 974          * skip HW tagging is not enabled by FW.
 975          */
 976         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 977             (adapter->pvid || adapter->qnq_vid) &&
 978             !qnq_async_evt_rcvd(adapter)))
 979                 goto tx_drop;
 980
 981         /* Manual VLAN tag insertion to prevent:
 982          * ASIC lockup when the ASIC inserts VLAN tag into
 983          * certain ipv6 packets. Insert VLAN tags in driver,
 984          * and set event, completion, vlan bits accordingly
 985          * in the Tx WRB.
 986          */
 987         if (be_ipv6_tx_stall_chk(adapter, skb) &&
 988             be_vlan_tag_tx_chk(adapter, skb)) {
 989                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 990                 if (unlikely(!skb))
 991                         goto err;
 992         }
 993
 994         return skb;
 995 tx_drop:
 996         dev_kfree_skb_any(skb);
 997 err:
 998         return NULL;
 999 }
1000
1001 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1002                                            struct sk_buff *skb,
1003                                            bool *skip_hw_vlan)
1004 {
1005         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1006          * less may cause a transmit stall on that port. So the work-around is
1007          * to pad short packets (<= 32 bytes) to a 36-byte length.
1008          */
1009         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1010                 if (skb_padto(skb, 36))
1011                         return NULL;
1012                 skb->len = 36;
1013         }
1014
1015         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1016                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1017                 if (!skb)
1018                         return NULL;
1019         }
1020
1021         return skb;
1022 }
1023
1024 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1025 {
1026         struct be_adapter *adapter = netdev_priv(netdev);
1027         struct be_tx_obj *txo = &adapter->tx_obj[skb_get_queue_mapping(skb)];
1028         struct be_queue_info *txq = &txo->q;
1029         bool dummy_wrb, stopped = false;
1030         u32 wrb_cnt = 0, copied = 0;
1031         bool skip_hw_vlan = false;
1032         u32 start = txq->head;
1033
1034         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1035         if (!skb) {
1036                 tx_stats(txo)->tx_drv_drops++;
1037                 return NETDEV_TX_OK;
1038         }
1039
1040         wrb_cnt = wrb_cnt_for_skb(adapter, skb, &dummy_wrb);
1041
1042         copied = make_tx_wrbs(adapter, txq, skb, wrb_cnt, dummy_wrb,
1043                               skip_hw_vlan);
1044         if (copied) {
1045                 int gso_segs = skb_shinfo(skb)->gso_segs;
1046
1047                 /* record the sent skb in the sent_skb table */
1048                 BUG_ON(txo->sent_skb_list[start]);
1049                 txo->sent_skb_list[start] = skb;
1050
1051                 /* Ensure txq has space for the next skb; Else stop the queue
1052                  * *BEFORE* ringing the tx doorbell, so that we serialze the
1053                  * tx compls of the current transmit which'll wake up the queue
1054                  */
1055                 atomic_add(wrb_cnt, &txq->used);
1056                 if ((BE_MAX_TX_FRAG_COUNT + atomic_read(&txq->used)) >=
1057                                                                 txq->len) {
1058                         netif_stop_subqueue(netdev, skb_get_queue_mapping(skb));
1059                         stopped = true;
1060                 }
1061
1062                 be_txq_notify(adapter, txo, wrb_cnt);
1063
1064                 be_tx_stats_update(txo, wrb_cnt, copied, gso_segs, stopped);
1065         } else {
1066                 txq->head = start;
1067                 tx_stats(txo)->tx_drv_drops++;
1068                 dev_kfree_skb_any(skb);
1069         }
1070         return NETDEV_TX_OK;
1071 }
1072
1073 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1074 {
1075         struct be_adapter *adapter = netdev_priv(netdev);
1076         struct device *dev = &adapter->pdev->dev;
1077
1078         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1079                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1080                          BE_MIN_MTU, BE_MAX_MTU);
1081                 return -EINVAL;
1082         }
1083
1084         dev_info(dev, "MTU changed from %d to %d bytes\n",
1085                  netdev->mtu, new_mtu);
1086         netdev->mtu = new_mtu;
1087         return 0;
1088 }
1089
1090 /*
1091  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1092  * If the user configures more, place BE in vlan promiscuous mode.
1093  */
1094 static int be_vid_config(struct be_adapter *adapter)
1095 {
1096         u16 vids[BE_NUM_VLANS_SUPPORTED];
1097         u16 num = 0, i = 0;
1098         int status = 0;
1099
1100         /* No need to further configure vids if in promiscuous mode */
1101         if (adapter->promiscuous)
1102                 return 0;
1103
1104         if (adapter->vlans_added > be_max_vlans(adapter))
1105                 goto set_vlan_promisc;
1106
1107         /* Construct VLAN Table to give to HW */
1108         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1109                 vids[num++] = cpu_to_le16(i);
1110
1111         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1112         if (status) {
1113                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1114                 if (addl_status(status) ==
1115                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1116                         goto set_vlan_promisc;
1117                 dev_err(&adapter->pdev->dev,
1118                         "Setting HW VLAN filtering failed.\n");
1119         } else {
1120                 if (adapter->flags & BE_FLAGS_VLAN_PROMISC) {
1121                         /* hw VLAN filtering re-enabled. */
1122                         status = be_cmd_rx_filter(adapter,
1123                                                   BE_FLAGS_VLAN_PROMISC, OFF);
1124                         if (!status) {
1125                                 dev_info(&adapter->pdev->dev,
1126                                          "Disabling VLAN Promiscuous mode.\n");
1127                                 adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1128                         }
1129                 }
1130         }
1131
1132         return status;
1133
1134 set_vlan_promisc:
1135         if (adapter->flags & BE_FLAGS_VLAN_PROMISC)
1136                 return 0;
1137
1138         status = be_cmd_rx_filter(adapter, BE_FLAGS_VLAN_PROMISC, ON);
1139         if (!status) {
1140                 dev_info(&adapter->pdev->dev, "Enable VLAN Promiscuous mode\n");
1141                 adapter->flags |= BE_FLAGS_VLAN_PROMISC;
1142         } else
1143                 dev_err(&adapter->pdev->dev,
1144                         "Failed to enable VLAN Promiscuous mode.\n");
1145         return status;
1146 }
1147
1148 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1149 {
1150         struct be_adapter *adapter = netdev_priv(netdev);
1151         int status = 0;
1152
1153         /* Packets with VID 0 are always received by Lancer by default */
1154         if (lancer_chip(adapter) && vid == 0)
1155                 return status;
1156
1157         if (test_bit(vid, adapter->vids))
1158                 return status;
1159
1160         set_bit(vid, adapter->vids);
1161         adapter->vlans_added++;
1162
1163         status = be_vid_config(adapter);
1164         if (status) {
1165                 adapter->vlans_added--;
1166                 clear_bit(vid, adapter->vids);
1167         }
1168
1169         return status;
1170 }
1171
1172 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1173 {
1174         struct be_adapter *adapter = netdev_priv(netdev);
1175
1176         /* Packets with VID 0 are always received by Lancer by default */
1177         if (lancer_chip(adapter) && vid == 0)
1178                 return 0;
1179
1180         clear_bit(vid, adapter->vids);
1181         adapter->vlans_added--;
1182
1183         return be_vid_config(adapter);
1184 }
1185
1186 static void be_clear_promisc(struct be_adapter *adapter)
1187 {
1188         adapter->promiscuous = false;
1189         adapter->flags &= ~(BE_FLAGS_VLAN_PROMISC | BE_FLAGS_MCAST_PROMISC);
1190
1191         be_cmd_rx_filter(adapter, IFF_PROMISC, OFF);
1192 }
1193
1194 static void be_set_rx_mode(struct net_device *netdev)
1195 {
1196         struct be_adapter *adapter = netdev_priv(netdev);
1197         int status;
1198
1199         if (netdev->flags & IFF_PROMISC) {
1200                 be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1201                 adapter->promiscuous = true;
1202                 goto done;
1203         }
1204
1205         /* BE was previously in promiscuous mode; disable it */
1206         if (adapter->promiscuous) {
1207                 be_clear_promisc(adapter);
1208                 if (adapter->vlans_added)
1209                         be_vid_config(adapter);
1210         }
1211
1212         /* Enable multicast promisc if num configured exceeds what we support */
1213         if (netdev->flags & IFF_ALLMULTI ||
1214             netdev_mc_count(netdev) > be_max_mc(adapter))
1215                 goto set_mcast_promisc;
1216
1217         if (netdev_uc_count(netdev) != adapter->uc_macs) {
1218                 struct netdev_hw_addr *ha;
1219                 int i = 1; /* First slot is claimed by the Primary MAC */
1220
1221                 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) {
1222                         be_cmd_pmac_del(adapter, adapter->if_handle,
1223                                         adapter->pmac_id[i], 0);
1224                 }
1225
1226                 if (netdev_uc_count(netdev) > be_max_uc(adapter)) {
1227                         be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1228                         adapter->promiscuous = true;
1229                         goto done;
1230                 }
1231
1232                 netdev_for_each_uc_addr(ha, adapter->netdev) {
1233                         adapter->uc_macs++; /* First slot is for Primary MAC */
1234                         be_cmd_pmac_add(adapter, (u8 *)ha->addr,
1235                                         adapter->if_handle,
1236                                         &adapter->pmac_id[adapter->uc_macs], 0);
1237                 }
1238         }
1239
1240         status = be_cmd_rx_filter(adapter, IFF_MULTICAST, ON);
1241         if (!status) {
1242                 if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1243                         adapter->flags &= ~BE_FLAGS_MCAST_PROMISC;
1244                 goto done;
1245         }
1246
1247 set_mcast_promisc:
1248         if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1249                 return;
1250
1251         /* Set to MCAST promisc mode if setting MULTICAST address fails
1252          * or if num configured exceeds what we support
1253          */
1254         status = be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1255         if (!status)
1256                 adapter->flags |= BE_FLAGS_MCAST_PROMISC;
1257 done:
1258         return;
1259 }
1260
1261 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1262 {
1263         struct be_adapter *adapter = netdev_priv(netdev);
1264         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1265         int status;
1266
1267         if (!sriov_enabled(adapter))
1268                 return -EPERM;
1269
1270         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1271                 return -EINVAL;
1272
1273         /* Proceed further only if user provided MAC is different
1274          * from active MAC
1275          */
1276         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1277                 return 0;
1278
1279         if (BEx_chip(adapter)) {
1280                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1281                                 vf + 1);
1282
1283                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1284                                          &vf_cfg->pmac_id, vf + 1);
1285         } else {
1286                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1287                                         vf + 1);
1288         }
1289
1290         if (status) {
1291                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1292                         mac, vf, status);
1293                 return be_cmd_status(status);
1294         }
1295
1296         ether_addr_copy(vf_cfg->mac_addr, mac);
1297
1298         return 0;
1299 }
1300
1301 static int be_get_vf_config(struct net_device *netdev, int vf,
1302                             struct ifla_vf_info *vi)
1303 {
1304         struct be_adapter *adapter = netdev_priv(netdev);
1305         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1306
1307         if (!sriov_enabled(adapter))
1308                 return -EPERM;
1309
1310         if (vf >= adapter->num_vfs)
1311                 return -EINVAL;
1312
1313         vi->vf = vf;
1314         vi->max_tx_rate = vf_cfg->tx_rate;
1315         vi->min_tx_rate = 0;
1316         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1317         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1318         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1319         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1320
1321         return 0;
1322 }
1323
1324 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1325 {
1326         struct be_adapter *adapter = netdev_priv(netdev);
1327         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1328         int status = 0;
1329
1330         if (!sriov_enabled(adapter))
1331                 return -EPERM;
1332
1333         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1334                 return -EINVAL;
1335
1336         if (vlan || qos) {
1337                 vlan |= qos << VLAN_PRIO_SHIFT;
1338                 if (vf_cfg->vlan_tag != vlan)
1339                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1340                                                        vf_cfg->if_handle, 0);
1341         } else {
1342                 /* Reset Transparent Vlan Tagging. */
1343                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1344                                                vf + 1, vf_cfg->if_handle, 0);
1345         }
1346
1347         if (status) {
1348                 dev_err(&adapter->pdev->dev,
1349                         "VLAN %d config on VF %d failed : %#x\n", vlan,
1350                         vf, status);
1351                 return be_cmd_status(status);
1352         }
1353
1354         vf_cfg->vlan_tag = vlan;
1355
1356         return 0;
1357 }
1358
1359 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1360                              int min_tx_rate, int max_tx_rate)
1361 {
1362         struct be_adapter *adapter = netdev_priv(netdev);
1363         struct device *dev = &adapter->pdev->dev;
1364         int percent_rate, status = 0;
1365         u16 link_speed = 0;
1366         u8 link_status;
1367
1368         if (!sriov_enabled(adapter))
1369                 return -EPERM;
1370
1371         if (vf >= adapter->num_vfs)
1372                 return -EINVAL;
1373
1374         if (min_tx_rate)
1375                 return -EINVAL;
1376
1377         if (!max_tx_rate)
1378                 goto config_qos;
1379
1380         status = be_cmd_link_status_query(adapter, &link_speed,
1381                                           &link_status, 0);
1382         if (status)
1383                 goto err;
1384
1385         if (!link_status) {
1386                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1387                 status = -ENETDOWN;
1388                 goto err;
1389         }
1390
1391         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1392                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1393                         link_speed);
1394                 status = -EINVAL;
1395                 goto err;
1396         }
1397
1398         /* On Skyhawk the QOS setting must be done only as a % value */
1399         percent_rate = link_speed / 100;
1400         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1401                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1402                         percent_rate);
1403                 status = -EINVAL;
1404                 goto err;
1405         }
1406
1407 config_qos:
1408         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1409         if (status)
1410                 goto err;
1411
1412         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1413         return 0;
1414
1415 err:
1416         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1417                 max_tx_rate, vf);
1418         return be_cmd_status(status);
1419 }
1420 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1421                                 int link_state)
1422 {
1423         struct be_adapter *adapter = netdev_priv(netdev);
1424         int status;
1425
1426         if (!sriov_enabled(adapter))
1427                 return -EPERM;
1428
1429         if (vf >= adapter->num_vfs)
1430                 return -EINVAL;
1431
1432         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1433         if (status) {
1434                 dev_err(&adapter->pdev->dev,
1435                         "Link state change on VF %d failed: %#x\n", vf, status);
1436                 return be_cmd_status(status);
1437         }
1438
1439         adapter->vf_cfg[vf].plink_tracking = link_state;
1440
1441         return 0;
1442 }
1443
1444 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1445                           ulong now)
1446 {
1447         aic->rx_pkts_prev = rx_pkts;
1448         aic->tx_reqs_prev = tx_pkts;
1449         aic->jiffies = now;
1450 }
1451
1452 static void be_eqd_update(struct be_adapter *adapter)
1453 {
1454         struct be_set_eqd set_eqd[MAX_EVT_QS];
1455         int eqd, i, num = 0, start;
1456         struct be_aic_obj *aic;
1457         struct be_eq_obj *eqo;
1458         struct be_rx_obj *rxo;
1459         struct be_tx_obj *txo;
1460         u64 rx_pkts, tx_pkts;
1461         ulong now;
1462         u32 pps, delta;
1463
1464         for_all_evt_queues(adapter, eqo, i) {
1465                 aic = &adapter->aic_obj[eqo->idx];
1466                 if (!aic->enable) {
1467                         if (aic->jiffies)
1468                                 aic->jiffies = 0;
1469                         eqd = aic->et_eqd;
1470                         goto modify_eqd;
1471                 }
1472
1473                 rxo = &adapter->rx_obj[eqo->idx];
1474                 do {
1475                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1476                         rx_pkts = rxo->stats.rx_pkts;
1477                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1478
1479                 txo = &adapter->tx_obj[eqo->idx];
1480                 do {
1481                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1482                         tx_pkts = txo->stats.tx_reqs;
1483                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1484
1485
1486                 /* Skip, if wrapped around or first calculation */
1487                 now = jiffies;
1488                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1489                     rx_pkts < aic->rx_pkts_prev ||
1490                     tx_pkts < aic->tx_reqs_prev) {
1491                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1492                         continue;
1493                 }
1494
1495                 delta = jiffies_to_msecs(now - aic->jiffies);
1496                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1497                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1498                 eqd = (pps / 15000) << 2;
1499
1500                 if (eqd < 8)
1501                         eqd = 0;
1502                 eqd = min_t(u32, eqd, aic->max_eqd);
1503                 eqd = max_t(u32, eqd, aic->min_eqd);
1504
1505                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1506 modify_eqd:
1507                 if (eqd != aic->prev_eqd) {
1508                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1509                         set_eqd[num].eq_id = eqo->q.id;
1510                         aic->prev_eqd = eqd;
1511                         num++;
1512                 }
1513         }
1514
1515         if (num)
1516                 be_cmd_modify_eqd(adapter, set_eqd, num);
1517 }
1518
1519 static void be_rx_stats_update(struct be_rx_obj *rxo,
1520                                struct be_rx_compl_info *rxcp)
1521 {
1522         struct be_rx_stats *stats = rx_stats(rxo);
1523
1524         u64_stats_update_begin(&stats->sync);
1525         stats->rx_compl++;
1526         stats->rx_bytes += rxcp->pkt_size;
1527         stats->rx_pkts++;
1528         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1529                 stats->rx_mcast_pkts++;
1530         if (rxcp->err)
1531                 stats->rx_compl_err++;
1532         u64_stats_update_end(&stats->sync);
1533 }
1534
1535 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1536 {
1537         /* L4 checksum is not reliable for non TCP/UDP packets.
1538          * Also ignore ipcksm for ipv6 pkts
1539          */
1540         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1541                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1542 }
1543
1544 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1545 {
1546         struct be_adapter *adapter = rxo->adapter;
1547         struct be_rx_page_info *rx_page_info;
1548         struct be_queue_info *rxq = &rxo->q;
1549         u16 frag_idx = rxq->tail;
1550
1551         rx_page_info = &rxo->page_info_tbl[frag_idx];
1552         BUG_ON(!rx_page_info->page);
1553
1554         if (rx_page_info->last_frag) {
1555                 dma_unmap_page(&adapter->pdev->dev,
1556                                dma_unmap_addr(rx_page_info, bus),
1557                                adapter->big_page_size, DMA_FROM_DEVICE);
1558                 rx_page_info->last_frag = false;
1559         } else {
1560                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1561                                         dma_unmap_addr(rx_page_info, bus),
1562                                         rx_frag_size, DMA_FROM_DEVICE);
1563         }
1564
1565         queue_tail_inc(rxq);
1566         atomic_dec(&rxq->used);
1567         return rx_page_info;
1568 }
1569
1570 /* Throwaway the data in the Rx completion */
1571 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1572                                 struct be_rx_compl_info *rxcp)
1573 {
1574         struct be_rx_page_info *page_info;
1575         u16 i, num_rcvd = rxcp->num_rcvd;
1576
1577         for (i = 0; i < num_rcvd; i++) {
1578                 page_info = get_rx_page_info(rxo);
1579                 put_page(page_info->page);
1580                 memset(page_info, 0, sizeof(*page_info));
1581         }
1582 }
1583
1584 /*
1585  * skb_fill_rx_data forms a complete skb for an ether frame
1586  * indicated by rxcp.
1587  */
1588 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1589                              struct be_rx_compl_info *rxcp)
1590 {
1591         struct be_rx_page_info *page_info;
1592         u16 i, j;
1593         u16 hdr_len, curr_frag_len, remaining;
1594         u8 *start;
1595
1596         page_info = get_rx_page_info(rxo);
1597         start = page_address(page_info->page) + page_info->page_offset;
1598         prefetch(start);
1599
1600         /* Copy data in the first descriptor of this completion */
1601         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1602
1603         skb->len = curr_frag_len;
1604         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1605                 memcpy(skb->data, start, curr_frag_len);
1606                 /* Complete packet has now been moved to data */
1607                 put_page(page_info->page);
1608                 skb->data_len = 0;
1609                 skb->tail += curr_frag_len;
1610         } else {
1611                 hdr_len = ETH_HLEN;
1612                 memcpy(skb->data, start, hdr_len);
1613                 skb_shinfo(skb)->nr_frags = 1;
1614                 skb_frag_set_page(skb, 0, page_info->page);
1615                 skb_shinfo(skb)->frags[0].page_offset =
1616                                         page_info->page_offset + hdr_len;
1617                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1618                                   curr_frag_len - hdr_len);
1619                 skb->data_len = curr_frag_len - hdr_len;
1620                 skb->truesize += rx_frag_size;
1621                 skb->tail += hdr_len;
1622         }
1623         page_info->page = NULL;
1624
1625         if (rxcp->pkt_size <= rx_frag_size) {
1626                 BUG_ON(rxcp->num_rcvd != 1);
1627                 return;
1628         }
1629
1630         /* More frags present for this completion */
1631         remaining = rxcp->pkt_size - curr_frag_len;
1632         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1633                 page_info = get_rx_page_info(rxo);
1634                 curr_frag_len = min(remaining, rx_frag_size);
1635
1636                 /* Coalesce all frags from the same physical page in one slot */
1637                 if (page_info->page_offset == 0) {
1638                         /* Fresh page */
1639                         j++;
1640                         skb_frag_set_page(skb, j, page_info->page);
1641                         skb_shinfo(skb)->frags[j].page_offset =
1642                                                         page_info->page_offset;
1643                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1644                         skb_shinfo(skb)->nr_frags++;
1645                 } else {
1646                         put_page(page_info->page);
1647                 }
1648
1649                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1650                 skb->len += curr_frag_len;
1651                 skb->data_len += curr_frag_len;
1652                 skb->truesize += rx_frag_size;
1653                 remaining -= curr_frag_len;
1654                 page_info->page = NULL;
1655         }
1656         BUG_ON(j > MAX_SKB_FRAGS);
1657 }
1658
1659 /* Process the RX completion indicated by rxcp when GRO is disabled */
1660 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1661                                 struct be_rx_compl_info *rxcp)
1662 {
1663         struct be_adapter *adapter = rxo->adapter;
1664         struct net_device *netdev = adapter->netdev;
1665         struct sk_buff *skb;
1666
1667         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1668         if (unlikely(!skb)) {
1669                 rx_stats(rxo)->rx_drops_no_skbs++;
1670                 be_rx_compl_discard(rxo, rxcp);
1671                 return;
1672         }
1673
1674         skb_fill_rx_data(rxo, skb, rxcp);
1675
1676         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1677                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1678         else
1679                 skb_checksum_none_assert(skb);
1680
1681         skb->protocol = eth_type_trans(skb, netdev);
1682         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1683         if (netdev->features & NETIF_F_RXHASH)
1684                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1685
1686         skb->csum_level = rxcp->tunneled;
1687         skb_mark_napi_id(skb, napi);
1688
1689         if (rxcp->vlanf)
1690                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1691
1692         netif_receive_skb(skb);
1693 }
1694
1695 /* Process the RX completion indicated by rxcp when GRO is enabled */
1696 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1697                                     struct napi_struct *napi,
1698                                     struct be_rx_compl_info *rxcp)
1699 {
1700         struct be_adapter *adapter = rxo->adapter;
1701         struct be_rx_page_info *page_info;
1702         struct sk_buff *skb = NULL;
1703         u16 remaining, curr_frag_len;
1704         u16 i, j;
1705
1706         skb = napi_get_frags(napi);
1707         if (!skb) {
1708                 be_rx_compl_discard(rxo, rxcp);
1709                 return;
1710         }
1711
1712         remaining = rxcp->pkt_size;
1713         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1714                 page_info = get_rx_page_info(rxo);
1715
1716                 curr_frag_len = min(remaining, rx_frag_size);
1717
1718                 /* Coalesce all frags from the same physical page in one slot */
1719                 if (i == 0 || page_info->page_offset == 0) {
1720                         /* First frag or Fresh page */
1721                         j++;
1722                         skb_frag_set_page(skb, j, page_info->page);
1723                         skb_shinfo(skb)->frags[j].page_offset =
1724                                                         page_info->page_offset;
1725                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1726                 } else {
1727                         put_page(page_info->page);
1728                 }
1729                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1730                 skb->truesize += rx_frag_size;
1731                 remaining -= curr_frag_len;
1732                 memset(page_info, 0, sizeof(*page_info));
1733         }
1734         BUG_ON(j > MAX_SKB_FRAGS);
1735
1736         skb_shinfo(skb)->nr_frags = j + 1;
1737         skb->len = rxcp->pkt_size;
1738         skb->data_len = rxcp->pkt_size;
1739         skb->ip_summed = CHECKSUM_UNNECESSARY;
1740         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1741         if (adapter->netdev->features & NETIF_F_RXHASH)
1742                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1743
1744         skb->csum_level = rxcp->tunneled;
1745         skb_mark_napi_id(skb, napi);
1746
1747         if (rxcp->vlanf)
1748                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1749
1750         napi_gro_frags(napi);
1751 }
1752
1753 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1754                                  struct be_rx_compl_info *rxcp)
1755 {
1756         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1757         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1758         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1759         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1760         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1761         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1762         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1763         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1764         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1765         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1766         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1767         if (rxcp->vlanf) {
1768                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1769                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1770         }
1771         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1772         rxcp->tunneled =
1773                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1774 }
1775
1776 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1777                                  struct be_rx_compl_info *rxcp)
1778 {
1779         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1780         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1781         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1782         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1783         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1784         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1785         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1786         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1787         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1788         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1789         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1790         if (rxcp->vlanf) {
1791                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1792                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1793         }
1794         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1795         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1796 }
1797
1798 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1799 {
1800         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1801         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1802         struct be_adapter *adapter = rxo->adapter;
1803
1804         /* For checking the valid bit it is Ok to use either definition as the
1805          * valid bit is at the same position in both v0 and v1 Rx compl */
1806         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1807                 return NULL;
1808
1809         rmb();
1810         be_dws_le_to_cpu(compl, sizeof(*compl));
1811
1812         if (adapter->be3_native)
1813                 be_parse_rx_compl_v1(compl, rxcp);
1814         else
1815                 be_parse_rx_compl_v0(compl, rxcp);
1816
1817         if (rxcp->ip_frag)
1818                 rxcp->l4_csum = 0;
1819
1820         if (rxcp->vlanf) {
1821                 /* In QNQ modes, if qnq bit is not set, then the packet was
1822                  * tagged only with the transparent outer vlan-tag and must
1823                  * not be treated as a vlan packet by host
1824                  */
1825                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1826                         rxcp->vlanf = 0;
1827
1828                 if (!lancer_chip(adapter))
1829                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1830
1831                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1832                     !test_bit(rxcp->vlan_tag, adapter->vids))
1833                         rxcp->vlanf = 0;
1834         }
1835
1836         /* As the compl has been parsed, reset it; we wont touch it again */
1837         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1838
1839         queue_tail_inc(&rxo->cq);
1840         return rxcp;
1841 }
1842
1843 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1844 {
1845         u32 order = get_order(size);
1846
1847         if (order > 0)
1848                 gfp |= __GFP_COMP;
1849         return  alloc_pages(gfp, order);
1850 }
1851
1852 /*
1853  * Allocate a page, split it to fragments of size rx_frag_size and post as
1854  * receive buffers to BE
1855  */
1856 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp)
1857 {
1858         struct be_adapter *adapter = rxo->adapter;
1859         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1860         struct be_queue_info *rxq = &rxo->q;
1861         struct page *pagep = NULL;
1862         struct device *dev = &adapter->pdev->dev;
1863         struct be_eth_rx_d *rxd;
1864         u64 page_dmaaddr = 0, frag_dmaaddr;
1865         u32 posted, page_offset = 0;
1866
1867         page_info = &rxo->page_info_tbl[rxq->head];
1868         for (posted = 0; posted < MAX_RX_POST && !page_info->page; posted++) {
1869                 if (!pagep) {
1870                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1871                         if (unlikely(!pagep)) {
1872                                 rx_stats(rxo)->rx_post_fail++;
1873                                 break;
1874                         }
1875                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1876                                                     adapter->big_page_size,
1877                                                     DMA_FROM_DEVICE);
1878                         if (dma_mapping_error(dev, page_dmaaddr)) {
1879                                 put_page(pagep);
1880                                 pagep = NULL;
1881                                 adapter->drv_stats.dma_map_errors++;
1882                                 break;
1883                         }
1884                         page_offset = 0;
1885                 } else {
1886                         get_page(pagep);
1887                         page_offset += rx_frag_size;
1888                 }
1889                 page_info->page_offset = page_offset;
1890                 page_info->page = pagep;
1891
1892                 rxd = queue_head_node(rxq);
1893                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1894                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1895                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1896
1897                 /* Any space left in the current big page for another frag? */
1898                 if ((page_offset + rx_frag_size + rx_frag_size) >
1899                                         adapter->big_page_size) {
1900                         pagep = NULL;
1901                         page_info->last_frag = true;
1902                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1903                 } else {
1904                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
1905                 }
1906
1907                 prev_page_info = page_info;
1908                 queue_head_inc(rxq);
1909                 page_info = &rxo->page_info_tbl[rxq->head];
1910         }
1911
1912         /* Mark the last frag of a page when we break out of the above loop
1913          * with no more slots available in the RXQ
1914          */
1915         if (pagep) {
1916                 prev_page_info->last_frag = true;
1917                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
1918         }
1919
1920         if (posted) {
1921                 atomic_add(posted, &rxq->used);
1922                 if (rxo->rx_post_starved)
1923                         rxo->rx_post_starved = false;
1924                 be_rxq_notify(adapter, rxq->id, posted);
1925         } else if (atomic_read(&rxq->used) == 0) {
1926                 /* Let be_worker replenish when memory is available */
1927                 rxo->rx_post_starved = true;
1928         }
1929 }
1930
1931 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1932 {
1933         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1934
1935         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1936                 return NULL;
1937
1938         rmb();
1939         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1940
1941         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1942
1943         queue_tail_inc(tx_cq);
1944         return txcp;
1945 }
1946
1947 static u16 be_tx_compl_process(struct be_adapter *adapter,
1948                                struct be_tx_obj *txo, u16 last_index)
1949 {
1950         struct be_queue_info *txq = &txo->q;
1951         struct be_eth_wrb *wrb;
1952         struct sk_buff **sent_skbs = txo->sent_skb_list;
1953         struct sk_buff *sent_skb;
1954         u16 cur_index, num_wrbs = 1; /* account for hdr wrb */
1955         bool unmap_skb_hdr = true;
1956
1957         sent_skb = sent_skbs[txq->tail];
1958         BUG_ON(!sent_skb);
1959         sent_skbs[txq->tail] = NULL;
1960
1961         /* skip header wrb */
1962         queue_tail_inc(txq);
1963
1964         do {
1965                 cur_index = txq->tail;
1966                 wrb = queue_tail_node(txq);
1967                 unmap_tx_frag(&adapter->pdev->dev, wrb,
1968                               (unmap_skb_hdr && skb_headlen(sent_skb)));
1969                 unmap_skb_hdr = false;
1970
1971                 num_wrbs++;
1972                 queue_tail_inc(txq);
1973         } while (cur_index != last_index);
1974
1975         dev_consume_skb_any(sent_skb);
1976         return num_wrbs;
1977 }
1978
1979 /* Return the number of events in the event queue */
1980 static inline int events_get(struct be_eq_obj *eqo)
1981 {
1982         struct be_eq_entry *eqe;
1983         int num = 0;
1984
1985         do {
1986                 eqe = queue_tail_node(&eqo->q);
1987                 if (eqe->evt == 0)
1988                         break;
1989
1990                 rmb();
1991                 eqe->evt = 0;
1992                 num++;
1993                 queue_tail_inc(&eqo->q);
1994         } while (true);
1995
1996         return num;
1997 }
1998
1999 /* Leaves the EQ is disarmed state */
2000 static void be_eq_clean(struct be_eq_obj *eqo)
2001 {
2002         int num = events_get(eqo);
2003
2004         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2005 }
2006
2007 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2008 {
2009         struct be_rx_page_info *page_info;
2010         struct be_queue_info *rxq = &rxo->q;
2011         struct be_queue_info *rx_cq = &rxo->cq;
2012         struct be_rx_compl_info *rxcp;
2013         struct be_adapter *adapter = rxo->adapter;
2014         int flush_wait = 0;
2015
2016         /* Consume pending rx completions.
2017          * Wait for the flush completion (identified by zero num_rcvd)
2018          * to arrive. Notify CQ even when there are no more CQ entries
2019          * for HW to flush partially coalesced CQ entries.
2020          * In Lancer, there is no need to wait for flush compl.
2021          */
2022         for (;;) {
2023                 rxcp = be_rx_compl_get(rxo);
2024                 if (!rxcp) {
2025                         if (lancer_chip(adapter))
2026                                 break;
2027
2028                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2029                                 dev_warn(&adapter->pdev->dev,
2030                                          "did not receive flush compl\n");
2031                                 break;
2032                         }
2033                         be_cq_notify(adapter, rx_cq->id, true, 0);
2034                         mdelay(1);
2035                 } else {
2036                         be_rx_compl_discard(rxo, rxcp);
2037                         be_cq_notify(adapter, rx_cq->id, false, 1);
2038                         if (rxcp->num_rcvd == 0)
2039                                 break;
2040                 }
2041         }
2042
2043         /* After cleanup, leave the CQ in unarmed state */
2044         be_cq_notify(adapter, rx_cq->id, false, 0);
2045
2046         /* Then free posted rx buffers that were not used */
2047         while (atomic_read(&rxq->used) > 0) {
2048                 page_info = get_rx_page_info(rxo);
2049                 put_page(page_info->page);
2050                 memset(page_info, 0, sizeof(*page_info));
2051         }
2052         BUG_ON(atomic_read(&rxq->used));
2053         rxq->tail = rxq->head = 0;
2054 }
2055
2056 static void be_tx_compl_clean(struct be_adapter *adapter)
2057 {
2058         struct be_tx_obj *txo;
2059         struct be_queue_info *txq;
2060         struct be_eth_tx_compl *txcp;
2061         u16 end_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2062         struct sk_buff *sent_skb;
2063         bool dummy_wrb;
2064         int i, pending_txqs;
2065
2066         /* Stop polling for compls when HW has been silent for 10ms */
2067         do {
2068                 pending_txqs = adapter->num_tx_qs;
2069
2070                 for_all_tx_queues(adapter, txo, i) {
2071                         cmpl = 0;
2072                         num_wrbs = 0;
2073                         txq = &txo->q;
2074                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2075                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2076                                 num_wrbs += be_tx_compl_process(adapter, txo,
2077                                                                 end_idx);
2078                                 cmpl++;
2079                         }
2080                         if (cmpl) {
2081                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2082                                 atomic_sub(num_wrbs, &txq->used);
2083                                 timeo = 0;
2084                         }
2085                         if (atomic_read(&txq->used) == 0)
2086                                 pending_txqs--;
2087                 }
2088
2089                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2090                         break;
2091
2092                 mdelay(1);
2093         } while (true);
2094
2095         for_all_tx_queues(adapter, txo, i) {
2096                 txq = &txo->q;
2097                 if (atomic_read(&txq->used))
2098                         dev_err(&adapter->pdev->dev, "%d pending tx-compls\n",
2099                                 atomic_read(&txq->used));
2100
2101                 /* free posted tx for which compls will never arrive */
2102                 while (atomic_read(&txq->used)) {
2103                         sent_skb = txo->sent_skb_list[txq->tail];
2104                         end_idx = txq->tail;
2105                         num_wrbs = wrb_cnt_for_skb(adapter, sent_skb,
2106                                                    &dummy_wrb);
2107                         index_adv(&end_idx, num_wrbs - 1, txq->len);
2108                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2109                         atomic_sub(num_wrbs, &txq->used);
2110                 }
2111         }
2112 }
2113
2114 static void be_evt_queues_destroy(struct be_adapter *adapter)
2115 {
2116         struct be_eq_obj *eqo;
2117         int i;
2118
2119         for_all_evt_queues(adapter, eqo, i) {
2120                 if (eqo->q.created) {
2121                         be_eq_clean(eqo);
2122                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2123                         napi_hash_del(&eqo->napi);
2124                         netif_napi_del(&eqo->napi);
2125                 }
2126                 be_queue_free(adapter, &eqo->q);
2127         }
2128 }
2129
2130 static int be_evt_queues_create(struct be_adapter *adapter)
2131 {
2132         struct be_queue_info *eq;
2133         struct be_eq_obj *eqo;
2134         struct be_aic_obj *aic;
2135         int i, rc;
2136
2137         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2138                                     adapter->cfg_num_qs);
2139
2140         for_all_evt_queues(adapter, eqo, i) {
2141                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2142                                BE_NAPI_WEIGHT);
2143                 napi_hash_add(&eqo->napi);
2144                 aic = &adapter->aic_obj[i];
2145                 eqo->adapter = adapter;
2146                 eqo->idx = i;
2147                 aic->max_eqd = BE_MAX_EQD;
2148                 aic->enable = true;
2149
2150                 eq = &eqo->q;
2151                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2152                                     sizeof(struct be_eq_entry));
2153                 if (rc)
2154                         return rc;
2155
2156                 rc = be_cmd_eq_create(adapter, eqo);
2157                 if (rc)
2158                         return rc;
2159         }
2160         return 0;
2161 }
2162
2163 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2164 {
2165         struct be_queue_info *q;
2166
2167         q = &adapter->mcc_obj.q;
2168         if (q->created)
2169                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2170         be_queue_free(adapter, q);
2171
2172         q = &adapter->mcc_obj.cq;
2173         if (q->created)
2174                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2175         be_queue_free(adapter, q);
2176 }
2177
2178 /* Must be called only after TX qs are created as MCC shares TX EQ */
2179 static int be_mcc_queues_create(struct be_adapter *adapter)
2180 {
2181         struct be_queue_info *q, *cq;
2182
2183         cq = &adapter->mcc_obj.cq;
2184         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2185                            sizeof(struct be_mcc_compl)))
2186                 goto err;
2187
2188         /* Use the default EQ for MCC completions */
2189         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2190                 goto mcc_cq_free;
2191
2192         q = &adapter->mcc_obj.q;
2193         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2194                 goto mcc_cq_destroy;
2195
2196         if (be_cmd_mccq_create(adapter, q, cq))
2197                 goto mcc_q_free;
2198
2199         return 0;
2200
2201 mcc_q_free:
2202         be_queue_free(adapter, q);
2203 mcc_cq_destroy:
2204         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2205 mcc_cq_free:
2206         be_queue_free(adapter, cq);
2207 err:
2208         return -1;
2209 }
2210
2211 static void be_tx_queues_destroy(struct be_adapter *adapter)
2212 {
2213         struct be_queue_info *q;
2214         struct be_tx_obj *txo;
2215         u8 i;
2216
2217         for_all_tx_queues(adapter, txo, i) {
2218                 q = &txo->q;
2219                 if (q->created)
2220                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2221                 be_queue_free(adapter, q);
2222
2223                 q = &txo->cq;
2224                 if (q->created)
2225                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2226                 be_queue_free(adapter, q);
2227         }
2228 }
2229
2230 static int be_tx_qs_create(struct be_adapter *adapter)
2231 {
2232         struct be_queue_info *cq, *eq;
2233         struct be_tx_obj *txo;
2234         int status, i;
2235
2236         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2237
2238         for_all_tx_queues(adapter, txo, i) {
2239                 cq = &txo->cq;
2240                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2241                                         sizeof(struct be_eth_tx_compl));
2242                 if (status)
2243                         return status;
2244
2245                 u64_stats_init(&txo->stats.sync);
2246                 u64_stats_init(&txo->stats.sync_compl);
2247
2248                 /* If num_evt_qs is less than num_tx_qs, then more than
2249                  * one txq share an eq
2250                  */
2251                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2252                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2253                 if (status)
2254                         return status;
2255
2256                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2257                                         sizeof(struct be_eth_wrb));
2258                 if (status)
2259                         return status;
2260
2261                 status = be_cmd_txq_create(adapter, txo);
2262                 if (status)
2263                         return status;
2264         }
2265
2266         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2267                  adapter->num_tx_qs);
2268         return 0;
2269 }
2270
2271 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2272 {
2273         struct be_queue_info *q;
2274         struct be_rx_obj *rxo;
2275         int i;
2276
2277         for_all_rx_queues(adapter, rxo, i) {
2278                 q = &rxo->cq;
2279                 if (q->created)
2280                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2281                 be_queue_free(adapter, q);
2282         }
2283 }
2284
2285 static int be_rx_cqs_create(struct be_adapter *adapter)
2286 {
2287         struct be_queue_info *eq, *cq;
2288         struct be_rx_obj *rxo;
2289         int rc, i;
2290
2291         /* We can create as many RSS rings as there are EQs. */
2292         adapter->num_rx_qs = adapter->num_evt_qs;
2293
2294         /* We'll use RSS only if atleast 2 RSS rings are supported.
2295          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2296          */
2297         if (adapter->num_rx_qs > 1)
2298                 adapter->num_rx_qs++;
2299
2300         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2301         for_all_rx_queues(adapter, rxo, i) {
2302                 rxo->adapter = adapter;
2303                 cq = &rxo->cq;
2304                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2305                                     sizeof(struct be_eth_rx_compl));
2306                 if (rc)
2307                         return rc;
2308
2309                 u64_stats_init(&rxo->stats.sync);
2310                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2311                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2312                 if (rc)
2313                         return rc;
2314         }
2315
2316         dev_info(&adapter->pdev->dev,
2317                  "created %d RSS queue(s) and 1 default RX queue\n",
2318                  adapter->num_rx_qs - 1);
2319         return 0;
2320 }
2321
2322 static irqreturn_t be_intx(int irq, void *dev)
2323 {
2324         struct be_eq_obj *eqo = dev;
2325         struct be_adapter *adapter = eqo->adapter;
2326         int num_evts = 0;
2327
2328         /* IRQ is not expected when NAPI is scheduled as the EQ
2329          * will not be armed.
2330          * But, this can happen on Lancer INTx where it takes
2331          * a while to de-assert INTx or in BE2 where occasionaly
2332          * an interrupt may be raised even when EQ is unarmed.
2333          * If NAPI is already scheduled, then counting & notifying
2334          * events will orphan them.
2335          */
2336         if (napi_schedule_prep(&eqo->napi)) {
2337                 num_evts = events_get(eqo);
2338                 __napi_schedule(&eqo->napi);
2339                 if (num_evts)
2340                         eqo->spurious_intr = 0;
2341         }
2342         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2343
2344         /* Return IRQ_HANDLED only for the the first spurious intr
2345          * after a valid intr to stop the kernel from branding
2346          * this irq as a bad one!
2347          */
2348         if (num_evts || eqo->spurious_intr++ == 0)
2349                 return IRQ_HANDLED;
2350         else
2351                 return IRQ_NONE;
2352 }
2353
2354 static irqreturn_t be_msix(int irq, void *dev)
2355 {
2356         struct be_eq_obj *eqo = dev;
2357
2358         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2359         napi_schedule(&eqo->napi);
2360         return IRQ_HANDLED;
2361 }
2362
2363 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2364 {
2365         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2366 }
2367
2368 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2369                          int budget, int polling)
2370 {
2371         struct be_adapter *adapter = rxo->adapter;
2372         struct be_queue_info *rx_cq = &rxo->cq;
2373         struct be_rx_compl_info *rxcp;
2374         u32 work_done;
2375
2376         for (work_done = 0; work_done < budget; work_done++) {
2377                 rxcp = be_rx_compl_get(rxo);
2378                 if (!rxcp)
2379                         break;
2380
2381                 /* Is it a flush compl that has no data */
2382                 if (unlikely(rxcp->num_rcvd == 0))
2383                         goto loop_continue;
2384
2385                 /* Discard compl with partial DMA Lancer B0 */
2386                 if (unlikely(!rxcp->pkt_size)) {
2387                         be_rx_compl_discard(rxo, rxcp);
2388                         goto loop_continue;
2389                 }
2390
2391                 /* On BE drop pkts that arrive due to imperfect filtering in
2392                  * promiscuous mode on some skews
2393                  */
2394                 if (unlikely(rxcp->port != adapter->port_num &&
2395                              !lancer_chip(adapter))) {
2396                         be_rx_compl_discard(rxo, rxcp);
2397                         goto loop_continue;
2398                 }
2399
2400                 /* Don't do gro when we're busy_polling */
2401                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2402                         be_rx_compl_process_gro(rxo, napi, rxcp);
2403                 else
2404                         be_rx_compl_process(rxo, napi, rxcp);
2405
2406 loop_continue:
2407                 be_rx_stats_update(rxo, rxcp);
2408         }
2409
2410         if (work_done) {
2411                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2412
2413                 /* When an rx-obj gets into post_starved state, just
2414                  * let be_worker do the posting.
2415                  */
2416                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2417                     !rxo->rx_post_starved)
2418                         be_post_rx_frags(rxo, GFP_ATOMIC);
2419         }
2420
2421         return work_done;
2422 }
2423
2424 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2425 {
2426         switch (status) {
2427         case BE_TX_COMP_HDR_PARSE_ERR:
2428                 tx_stats(txo)->tx_hdr_parse_err++;
2429                 break;
2430         case BE_TX_COMP_NDMA_ERR:
2431                 tx_stats(txo)->tx_dma_err++;
2432                 break;
2433         case BE_TX_COMP_ACL_ERR:
2434                 tx_stats(txo)->tx_spoof_check_err++;
2435                 break;
2436         }
2437 }
2438
2439 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2440 {
2441         switch (status) {
2442         case LANCER_TX_COMP_LSO_ERR:
2443                 tx_stats(txo)->tx_tso_err++;
2444                 break;
2445         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2446         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2447                 tx_stats(txo)->tx_spoof_check_err++;
2448                 break;
2449         case LANCER_TX_COMP_QINQ_ERR:
2450                 tx_stats(txo)->tx_qinq_err++;
2451                 break;
2452         case LANCER_TX_COMP_PARITY_ERR:
2453                 tx_stats(txo)->tx_internal_parity_err++;
2454                 break;
2455         case LANCER_TX_COMP_DMA_ERR:
2456                 tx_stats(txo)->tx_dma_err++;
2457                 break;
2458         }
2459 }
2460
2461 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2462                           int idx)
2463 {
2464         struct be_eth_tx_compl *txcp;
2465         int num_wrbs = 0, work_done = 0;
2466         u32 compl_status;
2467         u16 last_idx;
2468
2469         while ((txcp = be_tx_compl_get(&txo->cq))) {
2470                 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2471                 num_wrbs += be_tx_compl_process(adapter, txo, last_idx);
2472                 work_done++;
2473
2474                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2475                 if (compl_status) {
2476                         if (lancer_chip(adapter))
2477                                 lancer_update_tx_err(txo, compl_status);
2478                         else
2479                                 be_update_tx_err(txo, compl_status);
2480                 }
2481         }
2482
2483         if (work_done) {
2484                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2485                 atomic_sub(num_wrbs, &txo->q.used);
2486
2487                 /* As Tx wrbs have been freed up, wake up netdev queue
2488                  * if it was stopped due to lack of tx wrbs.  */
2489                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2490                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2491                         netif_wake_subqueue(adapter->netdev, idx);
2492                 }
2493
2494                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2495                 tx_stats(txo)->tx_compl += work_done;
2496                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2497         }
2498 }
2499
2500 int be_poll(struct napi_struct *napi, int budget)
2501 {
2502         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2503         struct be_adapter *adapter = eqo->adapter;
2504         int max_work = 0, work, i, num_evts;
2505         struct be_rx_obj *rxo;
2506         struct be_tx_obj *txo;
2507
2508         num_evts = events_get(eqo);
2509
2510         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2511                 be_process_tx(adapter, txo, i);
2512
2513         if (be_lock_napi(eqo)) {
2514                 /* This loop will iterate twice for EQ0 in which
2515                  * completions of the last RXQ (default one) are also processed
2516                  * For other EQs the loop iterates only once
2517                  */
2518                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2519                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2520                         max_work = max(work, max_work);
2521                 }
2522                 be_unlock_napi(eqo);
2523         } else {
2524                 max_work = budget;
2525         }
2526
2527         if (is_mcc_eqo(eqo))
2528                 be_process_mcc(adapter);
2529
2530         if (max_work < budget) {
2531                 napi_complete(napi);
2532                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2533         } else {
2534                 /* As we'll continue in polling mode, count and clear events */
2535                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2536         }
2537         return max_work;
2538 }
2539
2540 #ifdef CONFIG_NET_RX_BUSY_POLL
2541 static int be_busy_poll(struct napi_struct *napi)
2542 {
2543         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2544         struct be_adapter *adapter = eqo->adapter;
2545         struct be_rx_obj *rxo;
2546         int i, work = 0;
2547
2548         if (!be_lock_busy_poll(eqo))
2549                 return LL_FLUSH_BUSY;
2550
2551         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2552                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2553                 if (work)
2554                         break;
2555         }
2556
2557         be_unlock_busy_poll(eqo);
2558         return work;
2559 }
2560 #endif
2561
2562 void be_detect_error(struct be_adapter *adapter)
2563 {
2564         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2565         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2566         u32 i;
2567         bool error_detected = false;
2568         struct device *dev = &adapter->pdev->dev;
2569         struct net_device *netdev = adapter->netdev;
2570
2571         if (be_hw_error(adapter))
2572                 return;
2573
2574         if (lancer_chip(adapter)) {
2575                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2576                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2577                         sliport_err1 = ioread32(adapter->db +
2578                                                 SLIPORT_ERROR1_OFFSET);
2579                         sliport_err2 = ioread32(adapter->db +
2580                                                 SLIPORT_ERROR2_OFFSET);
2581                         adapter->hw_error = true;
2582                         /* Do not log error messages if its a FW reset */
2583                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2584                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2585                                 dev_info(dev, "Firmware update in progress\n");
2586                         } else {
2587                                 error_detected = true;
2588                                 dev_err(dev, "Error detected in the card\n");
2589                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2590                                         sliport_status);
2591                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2592                                         sliport_err1);
2593                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2594                                         sliport_err2);
2595                         }
2596                 }
2597         } else {
2598                 pci_read_config_dword(adapter->pdev,
2599                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2600                 pci_read_config_dword(adapter->pdev,
2601                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2602                 pci_read_config_dword(adapter->pdev,
2603                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2604                 pci_read_config_dword(adapter->pdev,
2605                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2606
2607                 ue_lo = (ue_lo & ~ue_lo_mask);
2608                 ue_hi = (ue_hi & ~ue_hi_mask);
2609
2610                 /* On certain platforms BE hardware can indicate spurious UEs.
2611                  * Allow HW to stop working completely in case of a real UE.
2612                  * Hence not setting the hw_error for UE detection.
2613                  */
2614
2615                 if (ue_lo || ue_hi) {
2616                         error_detected = true;
2617                         dev_err(dev,
2618                                 "Unrecoverable Error detected in the adapter");
2619                         dev_err(dev, "Please reboot server to recover");
2620                         if (skyhawk_chip(adapter))
2621                                 adapter->hw_error = true;
2622                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2623                                 if (ue_lo & 1)
2624                                         dev_err(dev, "UE: %s bit set\n",
2625                                                 ue_status_low_desc[i]);
2626                         }
2627                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2628                                 if (ue_hi & 1)
2629                                         dev_err(dev, "UE: %s bit set\n",
2630                                                 ue_status_hi_desc[i]);
2631                         }
2632                 }
2633         }
2634         if (error_detected)
2635                 netif_carrier_off(netdev);
2636 }
2637
2638 static void be_msix_disable(struct be_adapter *adapter)
2639 {
2640         if (msix_enabled(adapter)) {
2641                 pci_disable_msix(adapter->pdev);
2642                 adapter->num_msix_vec = 0;
2643                 adapter->num_msix_roce_vec = 0;
2644         }
2645 }
2646
2647 static int be_msix_enable(struct be_adapter *adapter)
2648 {
2649         int i, num_vec;
2650         struct device *dev = &adapter->pdev->dev;
2651
2652         /* If RoCE is supported, program the max number of NIC vectors that
2653          * may be configured via set-channels, along with vectors needed for
2654          * RoCe. Else, just program the number we'll use initially.
2655          */
2656         if (be_roce_supported(adapter))
2657                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2658                                 2 * num_online_cpus());
2659         else
2660                 num_vec = adapter->cfg_num_qs;
2661
2662         for (i = 0; i < num_vec; i++)
2663                 adapter->msix_entries[i].entry = i;
2664
2665         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2666                                         MIN_MSIX_VECTORS, num_vec);
2667         if (num_vec < 0)
2668                 goto fail;
2669
2670         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2671                 adapter->num_msix_roce_vec = num_vec / 2;
2672                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2673                          adapter->num_msix_roce_vec);
2674         }
2675
2676         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2677
2678         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2679                  adapter->num_msix_vec);
2680         return 0;
2681
2682 fail:
2683         dev_warn(dev, "MSIx enable failed\n");
2684
2685         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2686         if (!be_physfn(adapter))
2687                 return num_vec;
2688         return 0;
2689 }
2690
2691 static inline int be_msix_vec_get(struct be_adapter *adapter,
2692                                   struct be_eq_obj *eqo)
2693 {
2694         return adapter->msix_entries[eqo->msix_idx].vector;
2695 }
2696
2697 static int be_msix_register(struct be_adapter *adapter)
2698 {
2699         struct net_device *netdev = adapter->netdev;
2700         struct be_eq_obj *eqo;
2701         int status, i, vec;
2702
2703         for_all_evt_queues(adapter, eqo, i) {
2704                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2705                 vec = be_msix_vec_get(adapter, eqo);
2706                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2707                 if (status)
2708                         goto err_msix;
2709         }
2710
2711         return 0;
2712 err_msix:
2713         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2714                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2715         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2716                  status);
2717         be_msix_disable(adapter);
2718         return status;
2719 }
2720
2721 static int be_irq_register(struct be_adapter *adapter)
2722 {
2723         struct net_device *netdev = adapter->netdev;
2724         int status;
2725
2726         if (msix_enabled(adapter)) {
2727                 status = be_msix_register(adapter);
2728                 if (status == 0)
2729                         goto done;
2730                 /* INTx is not supported for VF */
2731                 if (!be_physfn(adapter))
2732                         return status;
2733         }
2734
2735         /* INTx: only the first EQ is used */
2736         netdev->irq = adapter->pdev->irq;
2737         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2738                              &adapter->eq_obj[0]);
2739         if (status) {
2740                 dev_err(&adapter->pdev->dev,
2741                         "INTx request IRQ failed - err %d\n", status);
2742                 return status;
2743         }
2744 done:
2745         adapter->isr_registered = true;
2746         return 0;
2747 }
2748
2749 static void be_irq_unregister(struct be_adapter *adapter)
2750 {
2751         struct net_device *netdev = adapter->netdev;
2752         struct be_eq_obj *eqo;
2753         int i;
2754
2755         if (!adapter->isr_registered)
2756                 return;
2757
2758         /* INTx */
2759         if (!msix_enabled(adapter)) {
2760                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2761                 goto done;
2762         }
2763
2764         /* MSIx */
2765         for_all_evt_queues(adapter, eqo, i)
2766                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2767
2768 done:
2769         adapter->isr_registered = false;
2770 }
2771
2772 static void be_rx_qs_destroy(struct be_adapter *adapter)
2773 {
2774         struct be_queue_info *q;
2775         struct be_rx_obj *rxo;
2776         int i;
2777
2778         for_all_rx_queues(adapter, rxo, i) {
2779                 q = &rxo->q;
2780                 if (q->created) {
2781                         be_cmd_rxq_destroy(adapter, q);
2782                         be_rx_cq_clean(rxo);
2783                 }
2784                 be_queue_free(adapter, q);
2785         }
2786 }
2787
2788 static int be_close(struct net_device *netdev)
2789 {
2790         struct be_adapter *adapter = netdev_priv(netdev);
2791         struct be_eq_obj *eqo;
2792         int i;
2793
2794         /* This protection is needed as be_close() may be called even when the
2795          * adapter is in cleared state (after eeh perm failure)
2796          */
2797         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
2798                 return 0;
2799
2800         be_roce_dev_close(adapter);
2801
2802         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2803                 for_all_evt_queues(adapter, eqo, i) {
2804                         napi_disable(&eqo->napi);
2805                         be_disable_busy_poll(eqo);
2806                 }
2807                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2808         }
2809
2810         be_async_mcc_disable(adapter);
2811
2812         /* Wait for all pending tx completions to arrive so that
2813          * all tx skbs are freed.
2814          */
2815         netif_tx_disable(netdev);
2816         be_tx_compl_clean(adapter);
2817
2818         be_rx_qs_destroy(adapter);
2819
2820         for (i = 1; i < (adapter->uc_macs + 1); i++)
2821                 be_cmd_pmac_del(adapter, adapter->if_handle,
2822                                 adapter->pmac_id[i], 0);
2823         adapter->uc_macs = 0;
2824
2825         for_all_evt_queues(adapter, eqo, i) {
2826                 if (msix_enabled(adapter))
2827                         synchronize_irq(be_msix_vec_get(adapter, eqo));
2828                 else
2829                         synchronize_irq(netdev->irq);
2830                 be_eq_clean(eqo);
2831         }
2832
2833         be_irq_unregister(adapter);
2834
2835         return 0;
2836 }
2837
2838 static int be_rx_qs_create(struct be_adapter *adapter)
2839 {
2840         struct be_rx_obj *rxo;
2841         int rc, i, j;
2842         u8 rss_hkey[RSS_HASH_KEY_LEN];
2843         struct rss_info *rss = &adapter->rss_info;
2844
2845         for_all_rx_queues(adapter, rxo, i) {
2846                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
2847                                     sizeof(struct be_eth_rx_d));
2848                 if (rc)
2849                         return rc;
2850         }
2851
2852         /* The FW would like the default RXQ to be created first */
2853         rxo = default_rxo(adapter);
2854         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
2855                                adapter->if_handle, false, &rxo->rss_id);
2856         if (rc)
2857                 return rc;
2858
2859         for_all_rss_queues(adapter, rxo, i) {
2860                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
2861                                        rx_frag_size, adapter->if_handle,
2862                                        true, &rxo->rss_id);
2863                 if (rc)
2864                         return rc;
2865         }
2866
2867         if (be_multi_rxq(adapter)) {
2868                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
2869                         j += adapter->num_rx_qs - 1) {
2870                         for_all_rss_queues(adapter, rxo, i) {
2871                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
2872                                         break;
2873                                 rss->rsstable[j + i] = rxo->rss_id;
2874                                 rss->rss_queue[j + i] = i;
2875                         }
2876                 }
2877                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
2878                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
2879
2880                 if (!BEx_chip(adapter))
2881                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
2882                                 RSS_ENABLE_UDP_IPV6;
2883         } else {
2884                 /* Disable RSS, if only default RX Q is created */
2885                 rss->rss_flags = RSS_ENABLE_NONE;
2886         }
2887
2888         get_random_bytes(rss_hkey, RSS_HASH_KEY_LEN);
2889         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
2890                                128, rss_hkey);
2891         if (rc) {
2892                 rss->rss_flags = RSS_ENABLE_NONE;
2893                 return rc;
2894         }
2895
2896         memcpy(rss->rss_hkey, rss_hkey, RSS_HASH_KEY_LEN);
2897
2898         /* First time posting */
2899         for_all_rx_queues(adapter, rxo, i)
2900                 be_post_rx_frags(rxo, GFP_KERNEL);
2901         return 0;
2902 }
2903
2904 static int be_open(struct net_device *netdev)
2905 {
2906         struct be_adapter *adapter = netdev_priv(netdev);
2907         struct be_eq_obj *eqo;
2908         struct be_rx_obj *rxo;
2909         struct be_tx_obj *txo;
2910         u8 link_status;
2911         int status, i;
2912
2913         status = be_rx_qs_create(adapter);
2914         if (status)
2915                 goto err;
2916
2917         status = be_irq_register(adapter);
2918         if (status)
2919                 goto err;
2920
2921         for_all_rx_queues(adapter, rxo, i)
2922                 be_cq_notify(adapter, rxo->cq.id, true, 0);
2923
2924         for_all_tx_queues(adapter, txo, i)
2925                 be_cq_notify(adapter, txo->cq.id, true, 0);
2926
2927         be_async_mcc_enable(adapter);
2928
2929         for_all_evt_queues(adapter, eqo, i) {
2930                 napi_enable(&eqo->napi);
2931                 be_enable_busy_poll(eqo);
2932                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
2933         }
2934         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
2935
2936         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
2937         if (!status)
2938                 be_link_status_update(adapter, link_status);
2939
2940         netif_tx_start_all_queues(netdev);
2941         be_roce_dev_open(adapter);
2942
2943 #ifdef CONFIG_BE2NET_VXLAN
2944         if (skyhawk_chip(adapter))
2945                 vxlan_get_rx_port(netdev);
2946 #endif
2947
2948         return 0;
2949 err:
2950         be_close(adapter->netdev);
2951         return -EIO;
2952 }
2953
2954 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2955 {
2956         struct be_dma_mem cmd;
2957         int status = 0;
2958         u8 mac[ETH_ALEN];
2959
2960         memset(mac, 0, ETH_ALEN);
2961
2962         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2963         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
2964                                      GFP_KERNEL);
2965         if (!cmd.va)
2966                 return -ENOMEM;
2967
2968         if (enable) {
2969                 status = pci_write_config_dword(adapter->pdev,
2970                                                 PCICFG_PM_CONTROL_OFFSET,
2971                                                 PCICFG_PM_CONTROL_MASK);
2972                 if (status) {
2973                         dev_err(&adapter->pdev->dev,
2974                                 "Could not enable Wake-on-lan\n");
2975                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
2976                                           cmd.dma);
2977                         return status;
2978                 }
2979                 status = be_cmd_enable_magic_wol(adapter,
2980                                                  adapter->netdev->dev_addr,
2981                                                  &cmd);
2982                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
2983                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
2984         } else {
2985                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
2986                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
2987                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
2988         }
2989
2990         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
2991         return status;
2992 }
2993
2994 /*
2995  * Generate a seed MAC address from the PF MAC Address using jhash.
2996  * MAC Address for VFs are assigned incrementally starting from the seed.
2997  * These addresses are programmed in the ASIC by the PF and the VF driver
2998  * queries for the MAC address during its probe.
2999  */
3000 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3001 {
3002         u32 vf;
3003         int status = 0;
3004         u8 mac[ETH_ALEN];
3005         struct be_vf_cfg *vf_cfg;
3006
3007         be_vf_eth_addr_generate(adapter, mac);
3008
3009         for_all_vfs(adapter, vf_cfg, vf) {
3010                 if (BEx_chip(adapter))
3011                         status = be_cmd_pmac_add(adapter, mac,
3012                                                  vf_cfg->if_handle,
3013                                                  &vf_cfg->pmac_id, vf + 1);
3014                 else
3015                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3016                                                 vf + 1);
3017
3018                 if (status)
3019                         dev_err(&adapter->pdev->dev,
3020                                 "Mac address assignment failed for VF %d\n",
3021                                 vf);
3022                 else
3023                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3024
3025                 mac[5] += 1;
3026         }
3027         return status;
3028 }
3029
3030 static int be_vfs_mac_query(struct be_adapter *adapter)
3031 {
3032         int status, vf;
3033         u8 mac[ETH_ALEN];
3034         struct be_vf_cfg *vf_cfg;
3035
3036         for_all_vfs(adapter, vf_cfg, vf) {
3037                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3038                                                mac, vf_cfg->if_handle,
3039                                                false, vf+1);
3040                 if (status)
3041                         return status;
3042                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3043         }
3044         return 0;
3045 }
3046
3047 static void be_vf_clear(struct be_adapter *adapter)
3048 {
3049         struct be_vf_cfg *vf_cfg;
3050         u32 vf;
3051
3052         if (pci_vfs_assigned(adapter->pdev)) {
3053                 dev_warn(&adapter->pdev->dev,
3054                          "VFs are assigned to VMs: not disabling VFs\n");
3055                 goto done;
3056         }
3057
3058         pci_disable_sriov(adapter->pdev);
3059
3060         for_all_vfs(adapter, vf_cfg, vf) {
3061                 if (BEx_chip(adapter))
3062                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3063                                         vf_cfg->pmac_id, vf + 1);
3064                 else
3065                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3066                                        vf + 1);
3067
3068                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3069         }
3070 done:
3071         kfree(adapter->vf_cfg);
3072         adapter->num_vfs = 0;
3073         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3074 }
3075
3076 static void be_clear_queues(struct be_adapter *adapter)
3077 {
3078         be_mcc_queues_destroy(adapter);
3079         be_rx_cqs_destroy(adapter);
3080         be_tx_queues_destroy(adapter);
3081         be_evt_queues_destroy(adapter);
3082 }
3083
3084 static void be_cancel_worker(struct be_adapter *adapter)
3085 {
3086         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3087                 cancel_delayed_work_sync(&adapter->work);
3088                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3089         }
3090 }
3091
3092 static void be_mac_clear(struct be_adapter *adapter)
3093 {
3094         int i;
3095
3096         if (adapter->pmac_id) {
3097                 for (i = 0; i < (adapter->uc_macs + 1); i++)
3098                         be_cmd_pmac_del(adapter, adapter->if_handle,
3099                                         adapter->pmac_id[i], 0);
3100                 adapter->uc_macs = 0;
3101
3102                 kfree(adapter->pmac_id);
3103                 adapter->pmac_id = NULL;
3104         }
3105 }
3106
3107 #ifdef CONFIG_BE2NET_VXLAN
3108 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3109 {
3110         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3111                 be_cmd_manage_iface(adapter, adapter->if_handle,
3112                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3113
3114         if (adapter->vxlan_port)
3115                 be_cmd_set_vxlan_port(adapter, 0);
3116
3117         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3118         adapter->vxlan_port = 0;
3119 }
3120 #endif
3121
3122 static int be_clear(struct be_adapter *adapter)
3123 {
3124         be_cancel_worker(adapter);
3125
3126         if (sriov_enabled(adapter))
3127                 be_vf_clear(adapter);
3128
3129         /* Re-configure FW to distribute resources evenly across max-supported
3130          * number of VFs, only when VFs are not already enabled.
3131          */
3132         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3133                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3134                                         pci_sriov_get_totalvfs(adapter->pdev));
3135
3136 #ifdef CONFIG_BE2NET_VXLAN
3137         be_disable_vxlan_offloads(adapter);
3138 #endif
3139         /* delete the primary mac along with the uc-mac list */
3140         be_mac_clear(adapter);
3141
3142         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3143
3144         be_clear_queues(adapter);
3145
3146         be_msix_disable(adapter);
3147         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3148         return 0;
3149 }
3150
3151 static int be_vfs_if_create(struct be_adapter *adapter)
3152 {
3153         struct be_resources res = {0};
3154         struct be_vf_cfg *vf_cfg;
3155         u32 cap_flags, en_flags, vf;
3156         int status = 0;
3157
3158         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3159                     BE_IF_FLAGS_MULTICAST;
3160
3161         for_all_vfs(adapter, vf_cfg, vf) {
3162                 if (!BE3_chip(adapter)) {
3163                         status = be_cmd_get_profile_config(adapter, &res,
3164                                                            vf + 1);
3165                         if (!status)
3166                                 cap_flags = res.if_cap_flags;
3167                 }
3168
3169                 /* If a FW profile exists, then cap_flags are updated */
3170                 en_flags = cap_flags & (BE_IF_FLAGS_UNTAGGED |
3171                                         BE_IF_FLAGS_BROADCAST |
3172                                         BE_IF_FLAGS_MULTICAST);
3173                 status =
3174                     be_cmd_if_create(adapter, cap_flags, en_flags,
3175                                      &vf_cfg->if_handle, vf + 1);
3176                 if (status)
3177                         goto err;
3178         }
3179 err:
3180         return status;
3181 }
3182
3183 static int be_vf_setup_init(struct be_adapter *adapter)
3184 {
3185         struct be_vf_cfg *vf_cfg;
3186         int vf;
3187
3188         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3189                                   GFP_KERNEL);
3190         if (!adapter->vf_cfg)
3191                 return -ENOMEM;
3192
3193         for_all_vfs(adapter, vf_cfg, vf) {
3194                 vf_cfg->if_handle = -1;
3195                 vf_cfg->pmac_id = -1;
3196         }
3197         return 0;
3198 }
3199
3200 static int be_vf_setup(struct be_adapter *adapter)
3201 {
3202         struct device *dev = &adapter->pdev->dev;
3203         struct be_vf_cfg *vf_cfg;
3204         int status, old_vfs, vf;
3205         u32 privileges;
3206
3207         old_vfs = pci_num_vf(adapter->pdev);
3208
3209         status = be_vf_setup_init(adapter);
3210         if (status)
3211                 goto err;
3212
3213         if (old_vfs) {
3214                 for_all_vfs(adapter, vf_cfg, vf) {
3215                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3216                         if (status)
3217                                 goto err;
3218                 }
3219
3220                 status = be_vfs_mac_query(adapter);
3221                 if (status)
3222                         goto err;
3223         } else {
3224                 status = be_vfs_if_create(adapter);
3225                 if (status)
3226                         goto err;
3227
3228                 status = be_vf_eth_addr_config(adapter);
3229                 if (status)
3230                         goto err;
3231         }
3232
3233         for_all_vfs(adapter, vf_cfg, vf) {
3234                 /* Allow VFs to programs MAC/VLAN filters */
3235                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3236                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3237                         status = be_cmd_set_fn_privileges(adapter,
3238                                                           privileges |
3239                                                           BE_PRIV_FILTMGMT,
3240                                                           vf + 1);
3241                         if (!status)
3242                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3243                                          vf);
3244                 }
3245
3246                 /* Allow full available bandwidth */
3247                 if (!old_vfs)
3248                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3249
3250                 if (!old_vfs) {
3251                         be_cmd_enable_vf(adapter, vf + 1);
3252                         be_cmd_set_logical_link_config(adapter,
3253                                                        IFLA_VF_LINK_STATE_AUTO,
3254                                                        vf+1);
3255                 }
3256         }
3257
3258         if (!old_vfs) {
3259                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3260                 if (status) {
3261                         dev_err(dev, "SRIOV enable failed\n");
3262                         adapter->num_vfs = 0;
3263                         goto err;
3264                 }
3265         }
3266
3267         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3268         return 0;
3269 err:
3270         dev_err(dev, "VF setup failed\n");
3271         be_vf_clear(adapter);
3272         return status;
3273 }
3274
3275 /* Converting function_mode bits on BE3 to SH mc_type enums */
3276
3277 static u8 be_convert_mc_type(u32 function_mode)
3278 {
3279         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3280                 return vNIC1;
3281         else if (function_mode & QNQ_MODE)
3282                 return FLEX10;
3283         else if (function_mode & VNIC_MODE)
3284                 return vNIC2;
3285         else if (function_mode & UMC_ENABLED)
3286                 return UMC;
3287         else
3288                 return MC_NONE;
3289 }
3290
3291 /* On BE2/BE3 FW does not suggest the supported limits */
3292 static void BEx_get_resources(struct be_adapter *adapter,
3293                               struct be_resources *res)
3294 {
3295         bool use_sriov = adapter->num_vfs ? 1 : 0;
3296
3297         if (be_physfn(adapter))
3298                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3299         else
3300                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3301
3302         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3303
3304         if (be_is_mc(adapter)) {
3305                 /* Assuming that there are 4 channels per port,
3306                  * when multi-channel is enabled
3307                  */
3308                 if (be_is_qnq_mode(adapter))
3309                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3310                 else
3311                         /* In a non-qnq multichannel mode, the pvid
3312                          * takes up one vlan entry
3313                          */
3314                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3315         } else {
3316                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3317         }
3318
3319         res->max_mcast_mac = BE_MAX_MC;
3320
3321         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3322          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3323          *    *only* if it is RSS-capable.
3324          */
3325         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3326             !be_physfn(adapter) || (be_is_mc(adapter) &&
3327             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3328                 res->max_tx_qs = 1;
3329         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3330                 struct be_resources super_nic_res = {0};
3331
3332                 /* On a SuperNIC profile, the driver needs to use the
3333                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3334                  */
3335                 be_cmd_get_profile_config(adapter, &super_nic_res, 0);
3336                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3337                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3338         } else {
3339                 res->max_tx_qs = BE3_MAX_TX_QS;
3340         }
3341
3342         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3343             !use_sriov && be_physfn(adapter))
3344                 res->max_rss_qs = (adapter->be3_native) ?
3345                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3346         res->max_rx_qs = res->max_rss_qs + 1;
3347
3348         if (be_physfn(adapter))
3349                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3350                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3351         else
3352                 res->max_evt_qs = 1;
3353
3354         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3355         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3356                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3357 }
3358
3359 static void be_setup_init(struct be_adapter *adapter)
3360 {
3361         adapter->vlan_prio_bmap = 0xff;
3362         adapter->phy.link_speed = -1;
3363         adapter->if_handle = -1;
3364         adapter->be3_native = false;
3365         adapter->promiscuous = false;
3366         if (be_physfn(adapter))
3367                 adapter->cmd_privileges = MAX_PRIVILEGES;
3368         else
3369                 adapter->cmd_privileges = MIN_PRIVILEGES;
3370 }
3371
3372 static int be_get_sriov_config(struct be_adapter *adapter)
3373 {
3374         struct device *dev = &adapter->pdev->dev;
3375         struct be_resources res = {0};
3376         int max_vfs, old_vfs;
3377
3378         /* Some old versions of BE3 FW don't report max_vfs value */
3379         be_cmd_get_profile_config(adapter, &res, 0);
3380
3381         if (BE3_chip(adapter) && !res.max_vfs) {
3382                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3383                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3384         }
3385
3386         adapter->pool_res = res;
3387
3388         if (!be_max_vfs(adapter)) {
3389                 if (num_vfs)
3390                         dev_warn(dev, "device doesn't support SRIOV\n");
3391                 adapter->num_vfs = 0;
3392                 return 0;
3393         }
3394
3395         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3396
3397         /* validate num_vfs module param */
3398         old_vfs = pci_num_vf(adapter->pdev);
3399         if (old_vfs) {
3400                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3401                 if (old_vfs != num_vfs)
3402                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3403                 adapter->num_vfs = old_vfs;
3404         } else {
3405                 if (num_vfs > be_max_vfs(adapter)) {
3406                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3407                                  num_vfs);
3408                         dev_info(dev, "Limiting to %d VFs\n",
3409                                  be_max_vfs(adapter));
3410                 }
3411                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3412         }
3413
3414         return 0;
3415 }
3416
3417 static int be_get_resources(struct be_adapter *adapter)
3418 {
3419         struct device *dev = &adapter->pdev->dev;
3420         struct be_resources res = {0};
3421         int status;
3422
3423         if (BEx_chip(adapter)) {
3424                 BEx_get_resources(adapter, &res);
3425                 adapter->res = res;
3426         }
3427
3428         /* For Lancer, SH etc read per-function resource limits from FW.
3429          * GET_FUNC_CONFIG returns per function guaranteed limits.
3430          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3431          */
3432         if (!BEx_chip(adapter)) {
3433                 status = be_cmd_get_func_config(adapter, &res);
3434                 if (status)
3435                         return status;
3436
3437                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3438                 if (be_roce_supported(adapter))
3439                         res.max_evt_qs /= 2;
3440                 adapter->res = res;
3441         }
3442
3443         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3444                  be_max_txqs(adapter), be_max_rxqs(adapter),
3445                  be_max_rss(adapter), be_max_eqs(adapter),
3446                  be_max_vfs(adapter));
3447         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3448                  be_max_uc(adapter), be_max_mc(adapter),
3449                  be_max_vlans(adapter));
3450
3451         return 0;
3452 }
3453
3454 static void be_sriov_config(struct be_adapter *adapter)
3455 {
3456         struct device *dev = &adapter->pdev->dev;
3457         int status;
3458
3459         status = be_get_sriov_config(adapter);
3460         if (status) {
3461                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3462                 dev_err(dev, "SR-IOV cannot be enabled\n");
3463                 return;
3464         }
3465
3466         /* When the HW is in SRIOV capable configuration, the PF-pool
3467          * resources are equally distributed across the max-number of
3468          * VFs. The user may request only a subset of the max-vfs to be
3469          * enabled. Based on num_vfs, redistribute the resources across
3470          * num_vfs so that each VF will have access to more number of
3471          * resources. This facility is not available in BE3 FW.
3472          * Also, this is done by FW in Lancer chip.
3473          */
3474         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3475                 status = be_cmd_set_sriov_config(adapter,
3476                                                  adapter->pool_res,
3477                                                  adapter->num_vfs);
3478                 if (status)
3479                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3480         }
3481 }
3482
3483 static int be_get_config(struct be_adapter *adapter)
3484 {
3485         u16 profile_id;
3486         int status;
3487
3488         status = be_cmd_query_fw_cfg(adapter);
3489         if (status)
3490                 return status;
3491
3492          if (be_physfn(adapter)) {
3493                 status = be_cmd_get_active_profile(adapter, &profile_id);
3494                 if (!status)
3495                         dev_info(&adapter->pdev->dev,
3496                                  "Using profile 0x%x\n", profile_id);
3497         }
3498
3499         if (!BE2_chip(adapter) && be_physfn(adapter))
3500                 be_sriov_config(adapter);
3501
3502         status = be_get_resources(adapter);
3503         if (status)
3504                 return status;
3505
3506         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3507                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3508         if (!adapter->pmac_id)
3509                 return -ENOMEM;
3510
3511         /* Sanitize cfg_num_qs based on HW and platform limits */
3512         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3513
3514         return 0;
3515 }
3516
3517 static int be_mac_setup(struct be_adapter *adapter)
3518 {
3519         u8 mac[ETH_ALEN];
3520         int status;
3521
3522         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3523                 status = be_cmd_get_perm_mac(adapter, mac);
3524                 if (status)
3525                         return status;
3526
3527                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3528                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3529         } else {
3530                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3531                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3532         }
3533
3534         /* For BE3-R VFs, the PF programs the initial MAC address */
3535         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3536                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3537                                 &adapter->pmac_id[0], 0);
3538         return 0;
3539 }
3540
3541 static void be_schedule_worker(struct be_adapter *adapter)
3542 {
3543         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3544         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3545 }
3546
3547 static int be_setup_queues(struct be_adapter *adapter)
3548 {
3549         struct net_device *netdev = adapter->netdev;
3550         int status;
3551
3552         status = be_evt_queues_create(adapter);
3553         if (status)
3554                 goto err;
3555
3556         status = be_tx_qs_create(adapter);
3557         if (status)
3558                 goto err;
3559
3560         status = be_rx_cqs_create(adapter);
3561         if (status)
3562                 goto err;
3563
3564         status = be_mcc_queues_create(adapter);
3565         if (status)
3566                 goto err;
3567
3568         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3569         if (status)
3570                 goto err;
3571
3572         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3573         if (status)
3574                 goto err;
3575
3576         return 0;
3577 err:
3578         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3579         return status;
3580 }
3581
3582 int be_update_queues(struct be_adapter *adapter)
3583 {
3584         struct net_device *netdev = adapter->netdev;
3585         int status;
3586
3587         if (netif_running(netdev))
3588                 be_close(netdev);
3589
3590         be_cancel_worker(adapter);
3591
3592         /* If any vectors have been shared with RoCE we cannot re-program
3593          * the MSIx table.
3594          */
3595         if (!adapter->num_msix_roce_vec)
3596                 be_msix_disable(adapter);
3597
3598         be_clear_queues(adapter);
3599
3600         if (!msix_enabled(adapter)) {
3601                 status = be_msix_enable(adapter);
3602                 if (status)
3603                         return status;
3604         }
3605
3606         status = be_setup_queues(adapter);
3607         if (status)
3608                 return status;
3609
3610         be_schedule_worker(adapter);
3611
3612         if (netif_running(netdev))
3613                 status = be_open(netdev);
3614
3615         return status;
3616 }
3617
3618 static int be_setup(struct be_adapter *adapter)
3619 {
3620         struct device *dev = &adapter->pdev->dev;
3621         u32 tx_fc, rx_fc, en_flags;
3622         int status;
3623
3624         be_setup_init(adapter);
3625
3626         if (!lancer_chip(adapter))
3627                 be_cmd_req_native_mode(adapter);
3628
3629         status = be_get_config(adapter);
3630         if (status)
3631                 goto err;
3632
3633         status = be_msix_enable(adapter);
3634         if (status)
3635                 goto err;
3636
3637         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3638                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS;
3639         if (adapter->function_caps & BE_FUNCTION_CAPS_RSS)
3640                 en_flags |= BE_IF_FLAGS_RSS;
3641         en_flags = en_flags & be_if_cap_flags(adapter);
3642         status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
3643                                   &adapter->if_handle, 0);
3644         if (status)
3645                 goto err;
3646
3647         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3648         rtnl_lock();
3649         status = be_setup_queues(adapter);
3650         rtnl_unlock();
3651         if (status)
3652                 goto err;
3653
3654         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3655
3656         status = be_mac_setup(adapter);
3657         if (status)
3658                 goto err;
3659
3660         be_cmd_get_fw_ver(adapter);
3661         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3662
3663         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3664                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work.",
3665                         adapter->fw_ver);
3666                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3667         }
3668
3669         if (adapter->vlans_added)
3670                 be_vid_config(adapter);
3671
3672         be_set_rx_mode(adapter->netdev);
3673
3674         be_cmd_get_acpi_wol_cap(adapter);
3675
3676         be_cmd_get_flow_control(adapter, &tx_fc, &rx_fc);
3677
3678         if (rx_fc != adapter->rx_fc || tx_fc != adapter->tx_fc)
3679                 be_cmd_set_flow_control(adapter, adapter->tx_fc,
3680                                         adapter->rx_fc);
3681
3682         if (be_physfn(adapter))
3683                 be_cmd_set_logical_link_config(adapter,
3684                                                IFLA_VF_LINK_STATE_AUTO, 0);
3685
3686         if (adapter->num_vfs)
3687                 be_vf_setup(adapter);
3688
3689         status = be_cmd_get_phy_info(adapter);
3690         if (!status && be_pause_supported(adapter))
3691                 adapter->phy.fc_autoneg = 1;
3692
3693         be_schedule_worker(adapter);
3694         adapter->flags |= BE_FLAGS_SETUP_DONE;
3695         return 0;
3696 err:
3697         be_clear(adapter);
3698         return status;
3699 }
3700
3701 #ifdef CONFIG_NET_POLL_CONTROLLER
3702 static void be_netpoll(struct net_device *netdev)
3703 {
3704         struct be_adapter *adapter = netdev_priv(netdev);
3705         struct be_eq_obj *eqo;
3706         int i;
3707
3708         for_all_evt_queues(adapter, eqo, i) {
3709                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3710                 napi_schedule(&eqo->napi);
3711         }
3712
3713         return;
3714 }
3715 #endif
3716
3717 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3718
3719 static bool phy_flashing_required(struct be_adapter *adapter)
3720 {
3721         return (adapter->phy.phy_type == TN_8022 &&
3722                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3723 }
3724
3725 static bool is_comp_in_ufi(struct be_adapter *adapter,
3726                            struct flash_section_info *fsec, int type)
3727 {
3728         int i = 0, img_type = 0;
3729         struct flash_section_info_g2 *fsec_g2 = NULL;
3730
3731         if (BE2_chip(adapter))
3732                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3733
3734         for (i = 0; i < MAX_FLASH_COMP; i++) {
3735                 if (fsec_g2)
3736                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3737                 else
3738                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3739
3740                 if (img_type == type)
3741                         return true;
3742         }
3743         return false;
3744
3745 }
3746
3747 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3748                                                 int header_size,
3749                                                 const struct firmware *fw)
3750 {
3751         struct flash_section_info *fsec = NULL;
3752         const u8 *p = fw->data;
3753
3754         p += header_size;
3755         while (p < (fw->data + fw->size)) {
3756                 fsec = (struct flash_section_info *)p;
3757                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3758                         return fsec;
3759                 p += 32;
3760         }
3761         return NULL;
3762 }
3763
3764 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
3765                               u32 img_offset, u32 img_size, int hdr_size,
3766                               u16 img_optype, bool *crc_match)
3767 {
3768         u32 crc_offset;
3769         int status;
3770         u8 crc[4];
3771
3772         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_size - 4);
3773         if (status)
3774                 return status;
3775
3776         crc_offset = hdr_size + img_offset + img_size - 4;
3777
3778         /* Skip flashing, if crc of flashed region matches */
3779         if (!memcmp(crc, p + crc_offset, 4))
3780                 *crc_match = true;
3781         else
3782                 *crc_match = false;
3783
3784         return status;
3785 }
3786
3787 static int be_flash(struct be_adapter *adapter, const u8 *img,
3788                     struct be_dma_mem *flash_cmd, int optype, int img_size)
3789 {
3790         struct be_cmd_write_flashrom *req = flash_cmd->va;
3791         u32 total_bytes, flash_op, num_bytes;
3792         int status;
3793
3794         total_bytes = img_size;
3795         while (total_bytes) {
3796                 num_bytes = min_t(u32, 32*1024, total_bytes);
3797
3798                 total_bytes -= num_bytes;
3799
3800                 if (!total_bytes) {
3801                         if (optype == OPTYPE_PHY_FW)
3802                                 flash_op = FLASHROM_OPER_PHY_FLASH;
3803                         else
3804                                 flash_op = FLASHROM_OPER_FLASH;
3805                 } else {
3806                         if (optype == OPTYPE_PHY_FW)
3807                                 flash_op = FLASHROM_OPER_PHY_SAVE;
3808                         else
3809                                 flash_op = FLASHROM_OPER_SAVE;
3810                 }
3811
3812                 memcpy(req->data_buf, img, num_bytes);
3813                 img += num_bytes;
3814                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
3815                                                flash_op, num_bytes);
3816                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
3817                     optype == OPTYPE_PHY_FW)
3818                         break;
3819                 else if (status)
3820                         return status;
3821         }
3822         return 0;
3823 }
3824
3825 /* For BE2, BE3 and BE3-R */
3826 static int be_flash_BEx(struct be_adapter *adapter,
3827                         const struct firmware *fw,
3828                         struct be_dma_mem *flash_cmd, int num_of_images)
3829 {
3830         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
3831         struct device *dev = &adapter->pdev->dev;
3832         struct flash_section_info *fsec = NULL;
3833         int status, i, filehdr_size, num_comp;
3834         const struct flash_comp *pflashcomp;
3835         bool crc_match;
3836         const u8 *p;
3837
3838         struct flash_comp gen3_flash_types[] = {
3839                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
3840                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
3841                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
3842                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
3843                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
3844                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
3845                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
3846                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
3847                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
3848                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
3849                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
3850                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
3851                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
3852                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
3853                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
3854                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
3855                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
3856                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
3857                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
3858                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
3859         };
3860
3861         struct flash_comp gen2_flash_types[] = {
3862                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
3863                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
3864                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
3865                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
3866                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
3867                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
3868                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
3869                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
3870                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
3871                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
3872                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
3873                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
3874                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
3875                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
3876                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
3877                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
3878         };
3879
3880         if (BE3_chip(adapter)) {
3881                 pflashcomp = gen3_flash_types;
3882                 filehdr_size = sizeof(struct flash_file_hdr_g3);
3883                 num_comp = ARRAY_SIZE(gen3_flash_types);
3884         } else {
3885                 pflashcomp = gen2_flash_types;
3886                 filehdr_size = sizeof(struct flash_file_hdr_g2);
3887                 num_comp = ARRAY_SIZE(gen2_flash_types);
3888         }
3889
3890         /* Get flash section info*/
3891         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3892         if (!fsec) {
3893                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3894                 return -1;
3895         }
3896         for (i = 0; i < num_comp; i++) {
3897                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
3898                         continue;
3899
3900                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
3901                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
3902                         continue;
3903
3904                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
3905                     !phy_flashing_required(adapter))
3906                                 continue;
3907
3908                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
3909                         status = be_check_flash_crc(adapter, fw->data,
3910                                                     pflashcomp[i].offset,
3911                                                     pflashcomp[i].size,
3912                                                     filehdr_size +
3913                                                     img_hdrs_size,
3914                                                     OPTYPE_REDBOOT, &crc_match);
3915                         if (status) {
3916                                 dev_err(dev,
3917                                         "Could not get CRC for 0x%x region\n",
3918                                         pflashcomp[i].optype);
3919                                 continue;
3920                         }
3921
3922                         if (crc_match)
3923                                 continue;
3924                 }
3925
3926                 p = fw->data + filehdr_size + pflashcomp[i].offset +
3927                         img_hdrs_size;
3928                 if (p + pflashcomp[i].size > fw->data + fw->size)
3929                         return -1;
3930
3931                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
3932                                   pflashcomp[i].size);
3933                 if (status) {
3934                         dev_err(dev, "Flashing section type 0x%x failed\n",
3935                                 pflashcomp[i].img_type);
3936                         return status;
3937                 }
3938         }
3939         return 0;
3940 }
3941
3942 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
3943 {
3944         u32 img_type = le32_to_cpu(fsec_entry.type);
3945         u16 img_optype = le16_to_cpu(fsec_entry.optype);
3946
3947         if (img_optype != 0xFFFF)
3948                 return img_optype;
3949
3950         switch (img_type) {
3951         case IMAGE_FIRMWARE_iSCSI:
3952                 img_optype = OPTYPE_ISCSI_ACTIVE;
3953                 break;
3954         case IMAGE_BOOT_CODE:
3955                 img_optype = OPTYPE_REDBOOT;
3956                 break;
3957         case IMAGE_OPTION_ROM_ISCSI:
3958                 img_optype = OPTYPE_BIOS;
3959                 break;
3960         case IMAGE_OPTION_ROM_PXE:
3961                 img_optype = OPTYPE_PXE_BIOS;
3962                 break;
3963         case IMAGE_OPTION_ROM_FCoE:
3964                 img_optype = OPTYPE_FCOE_BIOS;
3965                 break;
3966         case IMAGE_FIRMWARE_BACKUP_iSCSI:
3967                 img_optype = OPTYPE_ISCSI_BACKUP;
3968                 break;
3969         case IMAGE_NCSI:
3970                 img_optype = OPTYPE_NCSI_FW;
3971                 break;
3972         case IMAGE_FLASHISM_JUMPVECTOR:
3973                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
3974                 break;
3975         case IMAGE_FIRMWARE_PHY:
3976                 img_optype = OPTYPE_SH_PHY_FW;
3977                 break;
3978         case IMAGE_REDBOOT_DIR:
3979                 img_optype = OPTYPE_REDBOOT_DIR;
3980                 break;
3981         case IMAGE_REDBOOT_CONFIG:
3982                 img_optype = OPTYPE_REDBOOT_CONFIG;
3983                 break;
3984         case IMAGE_UFI_DIR:
3985                 img_optype = OPTYPE_UFI_DIR;
3986                 break;
3987         default:
3988                 break;
3989         }
3990
3991         return img_optype;
3992 }
3993
3994 static int be_flash_skyhawk(struct be_adapter *adapter,
3995                             const struct firmware *fw,
3996                             struct be_dma_mem *flash_cmd, int num_of_images)
3997 {
3998         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
3999         struct device *dev = &adapter->pdev->dev;
4000         struct flash_section_info *fsec = NULL;
4001         u32 img_offset, img_size, img_type;
4002         int status, i, filehdr_size;
4003         bool crc_match, old_fw_img;
4004         u16 img_optype;
4005         const u8 *p;
4006
4007         filehdr_size = sizeof(struct flash_file_hdr_g3);
4008         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4009         if (!fsec) {
4010                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4011                 return -EINVAL;
4012         }
4013
4014         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4015                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4016                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4017                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4018                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4019                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4020
4021                 if (img_optype == 0xFFFF)
4022                         continue;
4023                 /* Don't bother verifying CRC if an old FW image is being
4024                  * flashed
4025                  */
4026                 if (old_fw_img)
4027                         goto flash;
4028
4029                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4030                                             img_size, filehdr_size +
4031                                             img_hdrs_size, img_optype,
4032                                             &crc_match);
4033                 /* The current FW image on the card does not recognize the new
4034                  * FLASH op_type. The FW download is partially complete.
4035                  * Reboot the server now to enable FW image to recognize the
4036                  * new FLASH op_type. To complete the remaining process,
4037                  * download the same FW again after the reboot.
4038                  */
4039                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4040                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4041                         dev_err(dev, "Flash incomplete. Reset the server\n");
4042                         dev_err(dev, "Download FW image again after reset\n");
4043                         return -EAGAIN;
4044                 } else if (status) {
4045                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4046                                 img_optype);
4047                         return -EFAULT;
4048                 }
4049
4050                 if (crc_match)
4051                         continue;
4052
4053 flash:
4054                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4055                 if (p + img_size > fw->data + fw->size)
4056                         return -1;
4057
4058                 status = be_flash(adapter, p, flash_cmd, img_optype, img_size);
4059                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4060                  * UFI_DIR region
4061                  */
4062                 if (old_fw_img &&
4063                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4064                      (img_optype == OPTYPE_UFI_DIR &&
4065                       base_status(status) == MCC_STATUS_FAILED))) {
4066                         continue;
4067                 } else if (status) {
4068                         dev_err(dev, "Flashing section type 0x%x failed\n",
4069                                 img_type);
4070                         return -EFAULT;
4071                 }
4072         }
4073         return 0;
4074 }
4075
4076 static int lancer_fw_download(struct be_adapter *adapter,
4077                               const struct firmware *fw)
4078 {
4079 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4080 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4081         struct device *dev = &adapter->pdev->dev;
4082         struct be_dma_mem flash_cmd;
4083         const u8 *data_ptr = NULL;
4084         u8 *dest_image_ptr = NULL;
4085         size_t image_size = 0;
4086         u32 chunk_size = 0;
4087         u32 data_written = 0;
4088         u32 offset = 0;
4089         int status = 0;
4090         u8 add_status = 0;
4091         u8 change_status;
4092
4093         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4094                 dev_err(dev, "FW image size should be multiple of 4\n");
4095                 return -EINVAL;
4096         }
4097
4098         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4099                                 + LANCER_FW_DOWNLOAD_CHUNK;
4100         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4101                                           &flash_cmd.dma, GFP_KERNEL);
4102         if (!flash_cmd.va)
4103                 return -ENOMEM;
4104
4105         dest_image_ptr = flash_cmd.va +
4106                                 sizeof(struct lancer_cmd_req_write_object);
4107         image_size = fw->size;
4108         data_ptr = fw->data;
4109
4110         while (image_size) {
4111                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4112
4113                 /* Copy the image chunk content. */
4114                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4115
4116                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4117                                                  chunk_size, offset,
4118                                                  LANCER_FW_DOWNLOAD_LOCATION,
4119                                                  &data_written, &change_status,
4120                                                  &add_status);
4121                 if (status)
4122                         break;
4123
4124                 offset += data_written;
4125                 data_ptr += data_written;
4126                 image_size -= data_written;
4127         }
4128
4129         if (!status) {
4130                 /* Commit the FW written */
4131                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4132                                                  0, offset,
4133                                                  LANCER_FW_DOWNLOAD_LOCATION,
4134                                                  &data_written, &change_status,
4135                                                  &add_status);
4136         }
4137
4138         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4139         if (status) {
4140                 dev_err(dev, "Firmware load error\n");
4141                 return be_cmd_status(status);
4142         }
4143
4144         dev_info(dev, "Firmware flashed successfully\n");
4145
4146         if (change_status == LANCER_FW_RESET_NEEDED) {
4147                 dev_info(dev, "Resetting adapter to activate new FW\n");
4148                 status = lancer_physdev_ctrl(adapter,
4149                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4150                 if (status) {
4151                         dev_err(dev, "Adapter busy, could not reset FW\n");
4152                         dev_err(dev, "Reboot server to activate new FW\n");
4153                 }
4154         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4155                 dev_info(dev, "Reboot server to activate new FW\n");
4156         }
4157
4158         return 0;
4159 }
4160
4161 #define UFI_TYPE2               2
4162 #define UFI_TYPE3               3
4163 #define UFI_TYPE3R              10
4164 #define UFI_TYPE4               4
4165 static int be_get_ufi_type(struct be_adapter *adapter,
4166                            struct flash_file_hdr_g3 *fhdr)
4167 {
4168         if (!fhdr)
4169                 goto be_get_ufi_exit;
4170
4171         if (skyhawk_chip(adapter) && fhdr->build[0] == '4')
4172                 return UFI_TYPE4;
4173         else if (BE3_chip(adapter) && fhdr->build[0] == '3') {
4174                 if (fhdr->asic_type_rev == 0x10)
4175                         return UFI_TYPE3R;
4176                 else
4177                         return UFI_TYPE3;
4178         } else if (BE2_chip(adapter) && fhdr->build[0] == '2')
4179                 return UFI_TYPE2;
4180
4181 be_get_ufi_exit:
4182         dev_err(&adapter->pdev->dev,
4183                 "UFI and Interface are not compatible for flashing\n");
4184         return -1;
4185 }
4186
4187 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4188 {
4189         struct flash_file_hdr_g3 *fhdr3;
4190         struct image_hdr *img_hdr_ptr = NULL;
4191         struct be_dma_mem flash_cmd;
4192         const u8 *p;
4193         int status = 0, i = 0, num_imgs = 0, ufi_type = 0;
4194
4195         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4196         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4197                                           &flash_cmd.dma, GFP_KERNEL);
4198         if (!flash_cmd.va) {
4199                 status = -ENOMEM;
4200                 goto be_fw_exit;
4201         }
4202
4203         p = fw->data;
4204         fhdr3 = (struct flash_file_hdr_g3 *)p;
4205
4206         ufi_type = be_get_ufi_type(adapter, fhdr3);
4207
4208         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4209         for (i = 0; i < num_imgs; i++) {
4210                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4211                                 (sizeof(struct flash_file_hdr_g3) +
4212                                  i * sizeof(struct image_hdr)));
4213                 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) {
4214                         switch (ufi_type) {
4215                         case UFI_TYPE4:
4216                                 status = be_flash_skyhawk(adapter, fw,
4217                                                           &flash_cmd, num_imgs);
4218                                 break;
4219                         case UFI_TYPE3R:
4220                                 status = be_flash_BEx(adapter, fw, &flash_cmd,
4221                                                       num_imgs);
4222                                 break;
4223                         case UFI_TYPE3:
4224                                 /* Do not flash this ufi on BE3-R cards */
4225                                 if (adapter->asic_rev < 0x10)
4226                                         status = be_flash_BEx(adapter, fw,
4227                                                               &flash_cmd,
4228                                                               num_imgs);
4229                                 else {
4230                                         status = -EINVAL;
4231                                         dev_err(&adapter->pdev->dev,
4232                                                 "Can't load BE3 UFI on BE3R\n");
4233                                 }
4234                         }
4235                 }
4236         }
4237
4238         if (ufi_type == UFI_TYPE2)
4239                 status = be_flash_BEx(adapter, fw, &flash_cmd, 0);
4240         else if (ufi_type == -1)
4241                 status = -EINVAL;
4242
4243         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4244                           flash_cmd.dma);
4245         if (status) {
4246                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
4247                 goto be_fw_exit;
4248         }
4249
4250         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4251
4252 be_fw_exit:
4253         return status;
4254 }
4255
4256 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4257 {
4258         const struct firmware *fw;
4259         int status;
4260
4261         if (!netif_running(adapter->netdev)) {
4262                 dev_err(&adapter->pdev->dev,
4263                         "Firmware load not allowed (interface is down)\n");
4264                 return -ENETDOWN;
4265         }
4266
4267         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4268         if (status)
4269                 goto fw_exit;
4270
4271         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4272
4273         if (lancer_chip(adapter))
4274                 status = lancer_fw_download(adapter, fw);
4275         else
4276                 status = be_fw_download(adapter, fw);
4277
4278         if (!status)
4279                 be_cmd_get_fw_ver(adapter);
4280
4281 fw_exit:
4282         release_firmware(fw);
4283         return status;
4284 }
4285
4286 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh)
4287 {
4288         struct be_adapter *adapter = netdev_priv(dev);
4289         struct nlattr *attr, *br_spec;
4290         int rem;
4291         int status = 0;
4292         u16 mode = 0;
4293
4294         if (!sriov_enabled(adapter))
4295                 return -EOPNOTSUPP;
4296
4297         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4298
4299         nla_for_each_nested(attr, br_spec, rem) {
4300                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4301                         continue;
4302
4303                 mode = nla_get_u16(attr);
4304                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4305                         return -EINVAL;
4306
4307                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4308                                                adapter->if_handle,
4309                                                mode == BRIDGE_MODE_VEPA ?
4310                                                PORT_FWD_TYPE_VEPA :
4311                                                PORT_FWD_TYPE_VEB);
4312                 if (status)
4313                         goto err;
4314
4315                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4316                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4317
4318                 return status;
4319         }
4320 err:
4321         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4322                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4323
4324         return status;
4325 }
4326
4327 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4328                                  struct net_device *dev, u32 filter_mask)
4329 {
4330         struct be_adapter *adapter = netdev_priv(dev);
4331         int status = 0;
4332         u8 hsw_mode;
4333
4334         if (!sriov_enabled(adapter))
4335                 return 0;
4336
4337         /* BE and Lancer chips support VEB mode only */
4338         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4339                 hsw_mode = PORT_FWD_TYPE_VEB;
4340         } else {
4341                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4342                                                adapter->if_handle, &hsw_mode);
4343                 if (status)
4344                         return 0;
4345         }
4346
4347         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4348                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4349                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB);
4350 }
4351
4352 #ifdef CONFIG_BE2NET_VXLAN
4353 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4354                               __be16 port)
4355 {
4356         struct be_adapter *adapter = netdev_priv(netdev);
4357         struct device *dev = &adapter->pdev->dev;
4358         int status;
4359
4360         if (lancer_chip(adapter) || BEx_chip(adapter))
4361                 return;
4362
4363         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4364                 dev_warn(dev, "Cannot add UDP port %d for VxLAN offloads\n",
4365                          be16_to_cpu(port));
4366                 dev_info(dev,
4367                          "Only one UDP port supported for VxLAN offloads\n");
4368                 return;
4369         }
4370
4371         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4372                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4373         if (status) {
4374                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4375                 goto err;
4376         }
4377
4378         status = be_cmd_set_vxlan_port(adapter, port);
4379         if (status) {
4380                 dev_warn(dev, "Failed to add VxLAN port\n");
4381                 goto err;
4382         }
4383         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4384         adapter->vxlan_port = port;
4385
4386         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4387                  be16_to_cpu(port));
4388         return;
4389 err:
4390         be_disable_vxlan_offloads(adapter);
4391         return;
4392 }
4393
4394 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4395                               __be16 port)
4396 {
4397         struct be_adapter *adapter = netdev_priv(netdev);
4398
4399         if (lancer_chip(adapter) || BEx_chip(adapter))
4400                 return;
4401
4402         if (adapter->vxlan_port != port)
4403                 return;
4404
4405         be_disable_vxlan_offloads(adapter);
4406
4407         dev_info(&adapter->pdev->dev,
4408                  "Disabled VxLAN offloads for UDP port %d\n",
4409                  be16_to_cpu(port));
4410 }
4411 #endif
4412
4413 static const struct net_device_ops be_netdev_ops = {
4414         .ndo_open               = be_open,
4415         .ndo_stop               = be_close,
4416         .ndo_start_xmit         = be_xmit,
4417         .ndo_set_rx_mode        = be_set_rx_mode,
4418         .ndo_set_mac_address    = be_mac_addr_set,
4419         .ndo_change_mtu         = be_change_mtu,
4420         .ndo_get_stats64        = be_get_stats64,
4421         .ndo_validate_addr      = eth_validate_addr,
4422         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4423         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4424         .ndo_set_vf_mac         = be_set_vf_mac,
4425         .ndo_set_vf_vlan        = be_set_vf_vlan,
4426         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4427         .ndo_get_vf_config      = be_get_vf_config,
4428         .ndo_set_vf_link_state  = be_set_vf_link_state,
4429 #ifdef CONFIG_NET_POLL_CONTROLLER
4430         .ndo_poll_controller    = be_netpoll,
4431 #endif
4432         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4433         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4434 #ifdef CONFIG_NET_RX_BUSY_POLL
4435         .ndo_busy_poll          = be_busy_poll,
4436 #endif
4437 #ifdef CONFIG_BE2NET_VXLAN
4438         .ndo_add_vxlan_port     = be_add_vxlan_port,
4439         .ndo_del_vxlan_port     = be_del_vxlan_port,
4440 #endif
4441 };
4442
4443 static void be_netdev_init(struct net_device *netdev)
4444 {
4445         struct be_adapter *adapter = netdev_priv(netdev);
4446
4447         if (skyhawk_chip(adapter)) {
4448                 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4449                                            NETIF_F_TSO | NETIF_F_TSO6 |
4450                                            NETIF_F_GSO_UDP_TUNNEL;
4451                 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4452         }
4453         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4454                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4455                 NETIF_F_HW_VLAN_CTAG_TX;
4456         if (be_multi_rxq(adapter))
4457                 netdev->hw_features |= NETIF_F_RXHASH;
4458
4459         netdev->features |= netdev->hw_features |
4460                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4461
4462         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4463                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4464
4465         netdev->priv_flags |= IFF_UNICAST_FLT;
4466
4467         netdev->flags |= IFF_MULTICAST;
4468
4469         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4470
4471         netdev->netdev_ops = &be_netdev_ops;
4472
4473         netdev->ethtool_ops = &be_ethtool_ops;
4474 }
4475
4476 static void be_unmap_pci_bars(struct be_adapter *adapter)
4477 {
4478         if (adapter->csr)
4479                 pci_iounmap(adapter->pdev, adapter->csr);
4480         if (adapter->db)
4481                 pci_iounmap(adapter->pdev, adapter->db);
4482 }
4483
4484 static int db_bar(struct be_adapter *adapter)
4485 {
4486         if (lancer_chip(adapter) || !be_physfn(adapter))
4487                 return 0;
4488         else
4489                 return 4;
4490 }
4491
4492 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4493 {
4494         if (skyhawk_chip(adapter)) {
4495                 adapter->roce_db.size = 4096;
4496                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4497                                                               db_bar(adapter));
4498                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4499                                                                db_bar(adapter));
4500         }
4501         return 0;
4502 }
4503
4504 static int be_map_pci_bars(struct be_adapter *adapter)
4505 {
4506         u8 __iomem *addr;
4507
4508         if (BEx_chip(adapter) && be_physfn(adapter)) {
4509                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4510                 if (!adapter->csr)
4511                         return -ENOMEM;
4512         }
4513
4514         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4515         if (!addr)
4516                 goto pci_map_err;
4517         adapter->db = addr;
4518
4519         be_roce_map_pci_bars(adapter);
4520         return 0;
4521
4522 pci_map_err:
4523         dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n");
4524         be_unmap_pci_bars(adapter);
4525         return -ENOMEM;
4526 }
4527
4528 static void be_ctrl_cleanup(struct be_adapter *adapter)
4529 {
4530         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4531
4532         be_unmap_pci_bars(adapter);
4533
4534         if (mem->va)
4535                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4536                                   mem->dma);
4537
4538         mem = &adapter->rx_filter;
4539         if (mem->va)
4540                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4541                                   mem->dma);
4542 }
4543
4544 static int be_ctrl_init(struct be_adapter *adapter)
4545 {
4546         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4547         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4548         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4549         u32 sli_intf;
4550         int status;
4551
4552         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4553         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4554                                  SLI_INTF_FAMILY_SHIFT;
4555         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4556
4557         status = be_map_pci_bars(adapter);
4558         if (status)
4559                 goto done;
4560
4561         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4562         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4563                                                 mbox_mem_alloc->size,
4564                                                 &mbox_mem_alloc->dma,
4565                                                 GFP_KERNEL);
4566         if (!mbox_mem_alloc->va) {
4567                 status = -ENOMEM;
4568                 goto unmap_pci_bars;
4569         }
4570         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4571         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4572         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4573         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4574
4575         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4576         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4577                                             rx_filter->size, &rx_filter->dma,
4578                                             GFP_KERNEL);
4579         if (!rx_filter->va) {
4580                 status = -ENOMEM;
4581                 goto free_mbox;
4582         }
4583
4584         mutex_init(&adapter->mbox_lock);
4585         spin_lock_init(&adapter->mcc_lock);
4586         spin_lock_init(&adapter->mcc_cq_lock);
4587
4588         init_completion(&adapter->et_cmd_compl);
4589         pci_save_state(adapter->pdev);
4590         return 0;
4591
4592 free_mbox:
4593         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4594                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4595
4596 unmap_pci_bars:
4597         be_unmap_pci_bars(adapter);
4598
4599 done:
4600         return status;
4601 }
4602
4603 static void be_stats_cleanup(struct be_adapter *adapter)
4604 {
4605         struct be_dma_mem *cmd = &adapter->stats_cmd;
4606
4607         if (cmd->va)
4608                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4609                                   cmd->va, cmd->dma);
4610 }
4611
4612 static int be_stats_init(struct be_adapter *adapter)
4613 {
4614         struct be_dma_mem *cmd = &adapter->stats_cmd;
4615
4616         if (lancer_chip(adapter))
4617                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4618         else if (BE2_chip(adapter))
4619                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4620         else if (BE3_chip(adapter))
4621                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4622         else
4623                 /* ALL non-BE ASICs */
4624                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4625
4626         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4627                                       GFP_KERNEL);
4628         if (!cmd->va)
4629                 return -ENOMEM;
4630         return 0;
4631 }
4632
4633 static void be_remove(struct pci_dev *pdev)
4634 {
4635         struct be_adapter *adapter = pci_get_drvdata(pdev);
4636
4637         if (!adapter)
4638                 return;
4639
4640         be_roce_dev_remove(adapter);
4641         be_intr_set(adapter, false);
4642
4643         cancel_delayed_work_sync(&adapter->func_recovery_work);
4644
4645         unregister_netdev(adapter->netdev);
4646
4647         be_clear(adapter);
4648
4649         /* tell fw we're done with firing cmds */
4650         be_cmd_fw_clean(adapter);
4651
4652         be_stats_cleanup(adapter);
4653
4654         be_ctrl_cleanup(adapter);
4655
4656         pci_disable_pcie_error_reporting(pdev);
4657
4658         pci_release_regions(pdev);
4659         pci_disable_device(pdev);
4660
4661         free_netdev(adapter->netdev);
4662 }
4663
4664 static int be_get_initial_config(struct be_adapter *adapter)
4665 {
4666         int status, level;
4667
4668         status = be_cmd_get_cntl_attributes(adapter);
4669         if (status)
4670                 return status;
4671
4672         /* Must be a power of 2 or else MODULO will BUG_ON */
4673         adapter->be_get_temp_freq = 64;
4674
4675         if (BEx_chip(adapter)) {
4676                 level = be_cmd_get_fw_log_level(adapter);
4677                 adapter->msg_enable =
4678                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4679         }
4680
4681         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
4682         return 0;
4683 }
4684
4685 static int lancer_recover_func(struct be_adapter *adapter)
4686 {
4687         struct device *dev = &adapter->pdev->dev;
4688         int status;
4689
4690         status = lancer_test_and_set_rdy_state(adapter);
4691         if (status)
4692                 goto err;
4693
4694         if (netif_running(adapter->netdev))
4695                 be_close(adapter->netdev);
4696
4697         be_clear(adapter);
4698
4699         be_clear_all_error(adapter);
4700
4701         status = be_setup(adapter);
4702         if (status)
4703                 goto err;
4704
4705         if (netif_running(adapter->netdev)) {
4706                 status = be_open(adapter->netdev);
4707                 if (status)
4708                         goto err;
4709         }
4710
4711         dev_err(dev, "Adapter recovery successful\n");
4712         return 0;
4713 err:
4714         if (status == -EAGAIN)
4715                 dev_err(dev, "Waiting for resource provisioning\n");
4716         else
4717                 dev_err(dev, "Adapter recovery failed\n");
4718
4719         return status;
4720 }
4721
4722 static void be_func_recovery_task(struct work_struct *work)
4723 {
4724         struct be_adapter *adapter =
4725                 container_of(work, struct be_adapter,  func_recovery_work.work);
4726         int status = 0;
4727
4728         be_detect_error(adapter);
4729
4730         if (adapter->hw_error && lancer_chip(adapter)) {
4731
4732                 rtnl_lock();
4733                 netif_device_detach(adapter->netdev);
4734                 rtnl_unlock();
4735
4736                 status = lancer_recover_func(adapter);
4737                 if (!status)
4738                         netif_device_attach(adapter->netdev);
4739         }
4740
4741         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
4742          * no need to attempt further recovery.
4743          */
4744         if (!status || status == -EAGAIN)
4745                 schedule_delayed_work(&adapter->func_recovery_work,
4746                                       msecs_to_jiffies(1000));
4747 }
4748
4749 static void be_worker(struct work_struct *work)
4750 {
4751         struct be_adapter *adapter =
4752                 container_of(work, struct be_adapter, work.work);
4753         struct be_rx_obj *rxo;
4754         int i;
4755
4756         /* when interrupts are not yet enabled, just reap any pending
4757         * mcc completions */
4758         if (!netif_running(adapter->netdev)) {
4759                 local_bh_disable();
4760                 be_process_mcc(adapter);
4761                 local_bh_enable();
4762                 goto reschedule;
4763         }
4764
4765         if (!adapter->stats_cmd_sent) {
4766                 if (lancer_chip(adapter))
4767                         lancer_cmd_get_pport_stats(adapter,
4768                                                 &adapter->stats_cmd);
4769                 else
4770                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
4771         }
4772
4773         if (be_physfn(adapter) &&
4774             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
4775                 be_cmd_get_die_temperature(adapter);
4776
4777         for_all_rx_queues(adapter, rxo, i) {
4778                 /* Replenish RX-queues starved due to memory
4779                  * allocation failures.
4780                  */
4781                 if (rxo->rx_post_starved)
4782                         be_post_rx_frags(rxo, GFP_KERNEL);
4783         }
4784
4785         be_eqd_update(adapter);
4786
4787 reschedule:
4788         adapter->work_counter++;
4789         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4790 }
4791
4792 /* If any VFs are already enabled don't FLR the PF */
4793 static bool be_reset_required(struct be_adapter *adapter)
4794 {
4795         return pci_num_vf(adapter->pdev) ? false : true;
4796 }
4797
4798 static char *mc_name(struct be_adapter *adapter)
4799 {
4800         char *str = ""; /* default */
4801
4802         switch (adapter->mc_type) {
4803         case UMC:
4804                 str = "UMC";
4805                 break;
4806         case FLEX10:
4807                 str = "FLEX10";
4808                 break;
4809         case vNIC1:
4810                 str = "vNIC-1";
4811                 break;
4812         case nPAR:
4813                 str = "nPAR";
4814                 break;
4815         case UFP:
4816                 str = "UFP";
4817                 break;
4818         case vNIC2:
4819                 str = "vNIC-2";
4820                 break;
4821         default:
4822                 str = "";
4823         }
4824
4825         return str;
4826 }
4827
4828 static inline char *func_name(struct be_adapter *adapter)
4829 {
4830         return be_physfn(adapter) ? "PF" : "VF";
4831 }
4832
4833 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
4834 {
4835         int status = 0;
4836         struct be_adapter *adapter;
4837         struct net_device *netdev;
4838         char port_name;
4839
4840         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
4841
4842         status = pci_enable_device(pdev);
4843         if (status)
4844                 goto do_none;
4845
4846         status = pci_request_regions(pdev, DRV_NAME);
4847         if (status)
4848                 goto disable_dev;
4849         pci_set_master(pdev);
4850
4851         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
4852         if (!netdev) {
4853                 status = -ENOMEM;
4854                 goto rel_reg;
4855         }
4856         adapter = netdev_priv(netdev);
4857         adapter->pdev = pdev;
4858         pci_set_drvdata(pdev, adapter);
4859         adapter->netdev = netdev;
4860         SET_NETDEV_DEV(netdev, &pdev->dev);
4861
4862         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4863         if (!status) {
4864                 netdev->features |= NETIF_F_HIGHDMA;
4865         } else {
4866                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4867                 if (status) {
4868                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
4869                         goto free_netdev;
4870                 }
4871         }
4872
4873         if (be_physfn(adapter)) {
4874                 status = pci_enable_pcie_error_reporting(pdev);
4875                 if (!status)
4876                         dev_info(&pdev->dev, "PCIe error reporting enabled\n");
4877         }
4878
4879         status = be_ctrl_init(adapter);
4880         if (status)
4881                 goto free_netdev;
4882
4883         /* sync up with fw's ready state */
4884         if (be_physfn(adapter)) {
4885                 status = be_fw_wait_ready(adapter);
4886                 if (status)
4887                         goto ctrl_clean;
4888         }
4889
4890         if (be_reset_required(adapter)) {
4891                 status = be_cmd_reset_function(adapter);
4892                 if (status)
4893                         goto ctrl_clean;
4894
4895                 /* Wait for interrupts to quiesce after an FLR */
4896                 msleep(100);
4897         }
4898
4899         /* Allow interrupts for other ULPs running on NIC function */
4900         be_intr_set(adapter, true);
4901
4902         /* tell fw we're ready to fire cmds */
4903         status = be_cmd_fw_init(adapter);
4904         if (status)
4905                 goto ctrl_clean;
4906
4907         status = be_stats_init(adapter);
4908         if (status)
4909                 goto ctrl_clean;
4910
4911         status = be_get_initial_config(adapter);
4912         if (status)
4913                 goto stats_clean;
4914
4915         INIT_DELAYED_WORK(&adapter->work, be_worker);
4916         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
4917         adapter->rx_fc = adapter->tx_fc = true;
4918
4919         status = be_setup(adapter);
4920         if (status)
4921                 goto stats_clean;
4922
4923         be_netdev_init(netdev);
4924         status = register_netdev(netdev);
4925         if (status != 0)
4926                 goto unsetup;
4927
4928         be_roce_dev_add(adapter);
4929
4930         schedule_delayed_work(&adapter->func_recovery_work,
4931                               msecs_to_jiffies(1000));
4932
4933         be_cmd_query_port_name(adapter, &port_name);
4934
4935         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
4936                  func_name(adapter), mc_name(adapter), port_name);
4937
4938         return 0;
4939
4940 unsetup:
4941         be_clear(adapter);
4942 stats_clean:
4943         be_stats_cleanup(adapter);
4944 ctrl_clean:
4945         be_ctrl_cleanup(adapter);
4946 free_netdev:
4947         free_netdev(netdev);
4948 rel_reg:
4949         pci_release_regions(pdev);
4950 disable_dev:
4951         pci_disable_device(pdev);
4952 do_none:
4953         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
4954         return status;
4955 }
4956
4957 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
4958 {
4959         struct be_adapter *adapter = pci_get_drvdata(pdev);
4960         struct net_device *netdev =  adapter->netdev;
4961
4962         if (adapter->wol_en)
4963                 be_setup_wol(adapter, true);
4964
4965         be_intr_set(adapter, false);
4966         cancel_delayed_work_sync(&adapter->func_recovery_work);
4967
4968         netif_device_detach(netdev);
4969         if (netif_running(netdev)) {
4970                 rtnl_lock();
4971                 be_close(netdev);
4972                 rtnl_unlock();
4973         }
4974         be_clear(adapter);
4975
4976         pci_save_state(pdev);
4977         pci_disable_device(pdev);
4978         pci_set_power_state(pdev, pci_choose_state(pdev, state));
4979         return 0;
4980 }
4981
4982 static int be_resume(struct pci_dev *pdev)
4983 {
4984         int status = 0;
4985         struct be_adapter *adapter = pci_get_drvdata(pdev);
4986         struct net_device *netdev =  adapter->netdev;
4987
4988         netif_device_detach(netdev);
4989
4990         status = pci_enable_device(pdev);
4991         if (status)
4992                 return status;
4993
4994         pci_set_power_state(pdev, PCI_D0);
4995         pci_restore_state(pdev);
4996
4997         status = be_fw_wait_ready(adapter);
4998         if (status)
4999                 return status;
5000
5001         be_intr_set(adapter, true);
5002         /* tell fw we're ready to fire cmds */
5003         status = be_cmd_fw_init(adapter);
5004         if (status)
5005                 return status;
5006
5007         be_setup(adapter);
5008         if (netif_running(netdev)) {
5009                 rtnl_lock();
5010                 be_open(netdev);
5011                 rtnl_unlock();
5012         }
5013
5014         schedule_delayed_work(&adapter->func_recovery_work,
5015                               msecs_to_jiffies(1000));
5016         netif_device_attach(netdev);
5017
5018         if (adapter->wol_en)
5019                 be_setup_wol(adapter, false);
5020
5021         return 0;
5022 }
5023
5024 /*
5025  * An FLR will stop BE from DMAing any data.
5026  */
5027 static void be_shutdown(struct pci_dev *pdev)
5028 {
5029         struct be_adapter *adapter = pci_get_drvdata(pdev);
5030
5031         if (!adapter)
5032                 return;
5033
5034         be_roce_dev_shutdown(adapter);
5035         cancel_delayed_work_sync(&adapter->work);
5036         cancel_delayed_work_sync(&adapter->func_recovery_work);
5037
5038         netif_device_detach(adapter->netdev);
5039
5040         be_cmd_reset_function(adapter);
5041
5042         pci_disable_device(pdev);
5043 }
5044
5045 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5046                                             pci_channel_state_t state)
5047 {
5048         struct be_adapter *adapter = pci_get_drvdata(pdev);
5049         struct net_device *netdev =  adapter->netdev;
5050
5051         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5052
5053         if (!adapter->eeh_error) {
5054                 adapter->eeh_error = true;
5055
5056                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5057
5058                 rtnl_lock();
5059                 netif_device_detach(netdev);
5060                 if (netif_running(netdev))
5061                         be_close(netdev);
5062                 rtnl_unlock();
5063
5064                 be_clear(adapter);
5065         }
5066
5067         if (state == pci_channel_io_perm_failure)
5068                 return PCI_ERS_RESULT_DISCONNECT;
5069
5070         pci_disable_device(pdev);
5071
5072         /* The error could cause the FW to trigger a flash debug dump.
5073          * Resetting the card while flash dump is in progress
5074          * can cause it not to recover; wait for it to finish.
5075          * Wait only for first function as it is needed only once per
5076          * adapter.
5077          */
5078         if (pdev->devfn == 0)
5079                 ssleep(30);
5080
5081         return PCI_ERS_RESULT_NEED_RESET;
5082 }
5083
5084 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5085 {
5086         struct be_adapter *adapter = pci_get_drvdata(pdev);
5087         int status;
5088
5089         dev_info(&adapter->pdev->dev, "EEH reset\n");
5090
5091         status = pci_enable_device(pdev);
5092         if (status)
5093                 return PCI_ERS_RESULT_DISCONNECT;
5094
5095         pci_set_master(pdev);
5096         pci_set_power_state(pdev, PCI_D0);
5097         pci_restore_state(pdev);
5098
5099         /* Check if card is ok and fw is ready */
5100         dev_info(&adapter->pdev->dev,
5101                  "Waiting for FW to be ready after EEH reset\n");
5102         status = be_fw_wait_ready(adapter);
5103         if (status)
5104                 return PCI_ERS_RESULT_DISCONNECT;
5105
5106         pci_cleanup_aer_uncorrect_error_status(pdev);
5107         be_clear_all_error(adapter);
5108         return PCI_ERS_RESULT_RECOVERED;
5109 }
5110
5111 static void be_eeh_resume(struct pci_dev *pdev)
5112 {
5113         int status = 0;
5114         struct be_adapter *adapter = pci_get_drvdata(pdev);
5115         struct net_device *netdev =  adapter->netdev;
5116
5117         dev_info(&adapter->pdev->dev, "EEH resume\n");
5118
5119         pci_save_state(pdev);
5120
5121         status = be_cmd_reset_function(adapter);
5122         if (status)
5123                 goto err;
5124
5125         /* On some BE3 FW versions, after a HW reset,
5126          * interrupts will remain disabled for each function.
5127          * So, explicitly enable interrupts
5128          */
5129         be_intr_set(adapter, true);
5130
5131         /* tell fw we're ready to fire cmds */
5132         status = be_cmd_fw_init(adapter);
5133         if (status)
5134                 goto err;
5135
5136         status = be_setup(adapter);
5137         if (status)
5138                 goto err;
5139
5140         if (netif_running(netdev)) {
5141                 status = be_open(netdev);
5142                 if (status)
5143                         goto err;
5144         }
5145
5146         schedule_delayed_work(&adapter->func_recovery_work,
5147                               msecs_to_jiffies(1000));
5148         netif_device_attach(netdev);
5149         return;
5150 err:
5151         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5152 }
5153
5154 static const struct pci_error_handlers be_eeh_handlers = {
5155         .error_detected = be_eeh_err_detected,
5156         .slot_reset = be_eeh_reset,
5157         .resume = be_eeh_resume,
5158 };
5159
5160 static struct pci_driver be_driver = {
5161         .name = DRV_NAME,
5162         .id_table = be_dev_ids,
5163         .probe = be_probe,
5164         .remove = be_remove,
5165         .suspend = be_suspend,
5166         .resume = be_resume,
5167         .shutdown = be_shutdown,
5168         .err_handler = &be_eeh_handlers
5169 };
5170
5171 static int __init be_init_module(void)
5172 {
5173         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5174             rx_frag_size != 2048) {
5175                 printk(KERN_WARNING DRV_NAME
5176                         " : Module param rx_frag_size must be 2048/4096/8192."
5177                         " Using 2048\n");
5178                 rx_frag_size = 2048;
5179         }
5180
5181         return pci_register_driver(&be_driver);
5182 }
5183 module_init(be_init_module);
5184
5185 static void __exit be_exit_module(void)
5186 {
5187         pci_unregister_driver(&be_driver);
5188 }
5189 module_exit(be_exit_module);