drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static int wide_ppm_offset;
  50 module_param(wide_ppm_offset, int, 0644);
  51 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  52
  53 static u32 crit_err_count;
  54 u32 int_mod_timer_init;
  55 u32 int_mod_cq_depth_256;
  56 u32 int_mod_cq_depth_128;
  57 u32 int_mod_cq_depth_32;
  58 u32 int_mod_cq_depth_24;
  59 u32 int_mod_cq_depth_16;
  60 u32 int_mod_cq_depth_4;
  61 u32 int_mod_cq_depth_1;
  62 static const u8 nes_max_critical_error_count = 100;
  63 #include "nes_cm.h"
  64
  65 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  66 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  67 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  68                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  69 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  70 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  71 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  72 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  73                                    struct nes_hw_aeqe *aeqe);
  74 static void process_critical_error(struct nes_device *nesdev);
  75 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  76 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  77 static void nes_terminate_timeout(unsigned long context);
  78 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  79
  80 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  81 static unsigned char *nes_iwarp_state_str[] = {
  82         "Non-Existant",
  83         "Idle",
  84         "RTS",
  85         "Closing",
  86         "RSVD1",
  87         "Terminate",
  88         "Error",
  89         "RSVD2",
  90 };
  91
  92 static unsigned char *nes_tcp_state_str[] = {
  93         "Non-Existant",
  94         "Closed",
  95         "Listen",
  96         "SYN Sent",
  97         "SYN Rcvd",
  98         "Established",
  99         "Close Wait",
 100         "FIN Wait 1",
 101         "Closing",
 102         "Last Ack",
 103         "FIN Wait 2",
 104         "Time Wait",
 105         "RSVD1",
 106         "RSVD2",
 107         "RSVD3",
 108         "RSVD4",
 109 };
 110 #endif
 111
 112
 113 /**
 114  * nes_nic_init_timer_defaults
 115  */
 116 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 117 {
 118         unsigned long flags;
 119         struct nes_adapter *nesadapter = nesdev->nesadapter;
 120         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 121
 122         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 123
 124         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 125         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 126         if (jumbomode) {
 127                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 128                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 129                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 130         } else {
 131                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 132                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 133                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 134         }
 135
 136         /* todo use netdev->mtu to set thresholds */
 137         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 138 }
 139
 140
 141 /**
 142  * nes_nic_init_timer
 143  */
 144 static void  nes_nic_init_timer(struct nes_device *nesdev)
 145 {
 146         unsigned long flags;
 147         struct nes_adapter *nesadapter = nesdev->nesadapter;
 148         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 149
 150         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 151
 152         if (shared_timer->timer_in_use_old == 0) {
 153                 nesdev->deepcq_count = 0;
 154                 shared_timer->timer_direction_upward = 0;
 155                 shared_timer->timer_direction_downward = 0;
 156                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 157                 shared_timer->timer_in_use_old = 0;
 158
 159         }
 160         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 161                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 162                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 163                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 164         }
 165         /* todo use netdev->mtu to set thresholds */
 166         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 167 }
 168
 169
 170 /**
 171  * nes_nic_tune_timer
 172  */
 173 static void nes_nic_tune_timer(struct nes_device *nesdev)
 174 {
 175         unsigned long flags;
 176         struct nes_adapter *nesadapter = nesdev->nesadapter;
 177         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 178         u16 cq_count = nesdev->currcq_count;
 179
 180         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 181
 182         if (shared_timer->cq_count_old <= cq_count)
 183                 shared_timer->cq_direction_downward = 0;
 184         else
 185                 shared_timer->cq_direction_downward++;
 186         shared_timer->cq_count_old = cq_count;
 187         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 188                 if (cq_count <= shared_timer->threshold_low &&
 189                     shared_timer->threshold_low > 4) {
 190                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 191                         shared_timer->cq_direction_downward=0;
 192                         nesdev->currcq_count = 0;
 193                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 194                         return;
 195                 }
 196         }
 197
 198         if (cq_count > 1) {
 199                 nesdev->deepcq_count += cq_count;
 200                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 201                         shared_timer->timer_direction_upward++;
 202                         shared_timer->timer_direction_downward = 0;
 203                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 204                         shared_timer->timer_direction_upward = 0;
 205                         shared_timer->timer_direction_downward = 0;
 206                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 207                         shared_timer->timer_direction_downward++;
 208                         shared_timer->timer_direction_upward = 0;
 209                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 210                         shared_timer->timer_in_use -= 2;
 211                         shared_timer->timer_direction_upward = 0;
 212                         shared_timer->timer_direction_downward++;
 213                 } else {
 214                         shared_timer->timer_in_use -= 4;
 215                         shared_timer->timer_direction_upward = 0;
 216                         shared_timer->timer_direction_downward++;
 217                 }
 218
 219                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 220                         shared_timer->timer_in_use += 3;
 221                         shared_timer->timer_direction_upward = 0;
 222                         shared_timer->timer_direction_downward = 0;
 223                 }
 224                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 225                         shared_timer->timer_in_use -= 4 ;
 226                         shared_timer->timer_direction_downward = 0;
 227                         shared_timer->timer_direction_upward = 0;
 228                 }
 229         }
 230
 231         /* boundary checking */
 232         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 233                 shared_timer->timer_in_use = shared_timer->threshold_high;
 234         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 235                 shared_timer->timer_in_use = shared_timer->threshold_low;
 236
 237         nesdev->currcq_count = 0;
 238
 239         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 240 }
 241
 242
 243 /**
 244  * nes_init_adapter - initialize adapter
 245  */
 246 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 247         struct nes_adapter *nesadapter = NULL;
 248         unsigned long num_pds;
 249         u32 u32temp;
 250         u32 port_count;
 251         u16 max_rq_wrs;
 252         u16 max_sq_wrs;
 253         u32 max_mr;
 254         u32 max_256pbl;
 255         u32 max_4kpbl;
 256         u32 max_qp;
 257         u32 max_irrq;
 258         u32 max_cq;
 259         u32 hte_index_mask;
 260         u32 adapter_size;
 261         u32 arp_table_size;
 262         u16 vendor_id;
 263         u16 device_id;
 264         u8  OneG_Mode;
 265         u8  func_index;
 266
 267         /* search the list of existing adapters */
 268         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 269                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 270                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 271                                 nesdev->pcidev->devfn,
 272                                 PCI_SLOT(nesadapter->devfn),
 273                                 nesadapter->bus_number,
 274                                 PCI_SLOT(nesdev->pcidev->devfn),
 275                                 nesdev->pcidev->bus->number );
 276                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 277                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 278                         nesadapter->ref_count++;
 279                         return nesadapter;
 280                 }
 281         }
 282
 283         /* no adapter found */
 284         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 285         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 286                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 287                                 hw_rev);
 288                 return NULL;
 289         }
 290
 291         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 292                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 293                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 294                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 295                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 296
 297         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 298
 299
 300         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 301                 return NULL;
 302
 303         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 304         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 305
 306         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 307         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 308                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 309                                 max_qp, u32temp);
 310                 max_qp = (u32)1 << (u32temp & 0x001f);
 311         }
 312
 313         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 314         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 315                         max_qp, hte_index_mask);
 316
 317         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 318
 319         max_irrq = 1 << (u32temp & 0x001f);
 320
 321         if (max_qp > max_irrq) {
 322                 max_qp = max_irrq;
 323                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 324                                 max_qp);
 325         }
 326
 327         /* there should be no reason to allocate more pds than qps */
 328         if (num_pds > max_qp)
 329                 num_pds = max_qp;
 330
 331         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 332         max_mr = (u32)8192 << (u32temp & 0x7);
 333
 334         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 335         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 336         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 337         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 338
 339         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 340         arp_table_size = 1 << u32temp;
 341
 342         adapter_size = (sizeof(struct nes_adapter) +
 343                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 347         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 348         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 349         adapter_size += sizeof(struct nes_qp **) * max_qp;
 350
 351         /* allocate a new adapter struct */
 352         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 353         if (nesadapter == NULL) {
 354                 return NULL;
 355         }
 356
 357         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 358                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 359
 360         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 361                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 362                 kfree(nesadapter);
 363                 return NULL;
 364         }
 365
 366         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 367                                 (nesadapter->mac_addr_low >> 24);
 368
 369         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 370                                  PCI_DEVICE_ID, &device_id);
 371         nesadapter->vendor_part_id = device_id;
 372
 373         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 374                                                         OneG_Mode)) {
 375                 kfree(nesadapter);
 376                 return NULL;
 377         }
 378         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 379
 380         memset(nesadapter->pft_mcast_map, 255,
 381                sizeof nesadapter->pft_mcast_map);
 382
 383         /* populate the new nesadapter */
 384         nesadapter->devfn = nesdev->pcidev->devfn;
 385         nesadapter->bus_number = nesdev->pcidev->bus->number;
 386         nesadapter->ref_count = 1;
 387         nesadapter->timer_int_req = 0xffff0000;
 388         nesadapter->OneG_Mode = OneG_Mode;
 389         nesadapter->doorbell_start = nesdev->doorbell_region;
 390
 391         /* nesadapter->tick_delta = clk_divisor; */
 392         nesadapter->hw_rev = hw_rev;
 393         nesadapter->port_count = port_count;
 394
 395         nesadapter->max_qp = max_qp;
 396         nesadapter->hte_index_mask = hte_index_mask;
 397         nesadapter->max_irrq = max_irrq;
 398         nesadapter->max_mr = max_mr;
 399         nesadapter->max_256pbl = max_256pbl - 1;
 400         nesadapter->max_4kpbl = max_4kpbl - 1;
 401         nesadapter->max_cq = max_cq;
 402         nesadapter->free_256pbl = max_256pbl - 1;
 403         nesadapter->free_4kpbl = max_4kpbl - 1;
 404         nesadapter->max_pd = num_pds;
 405         nesadapter->arp_table_size = arp_table_size;
 406
 407         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 408         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 409                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 410                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 411                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 412         } else {
 413                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 414                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 415                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 416                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 417         }
 418         /* Setup and enable the periodic timer */
 419         if (nesadapter->et_rx_coalesce_usecs_irq)
 420                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 421                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 422         else
 423                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 424
 425         nesadapter->base_pd = 1;
 426
 427         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 428                                        IB_DEVICE_MEM_WINDOW |
 429                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 430
 431         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 432                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 433         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 434         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 435         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 436         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 437         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 438
 439
 440         /* mark the usual suspect QPs and CQs as in use */
 441         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 442                 set_bit(u32temp, nesadapter->allocated_qps);
 443                 set_bit(u32temp, nesadapter->allocated_cqs);
 444         }
 445
 446         for (u32temp = 0; u32temp < 20; u32temp++)
 447                 set_bit(u32temp, nesadapter->allocated_pds);
 448         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 449
 450         max_rq_wrs = ((u32temp >> 8) & 3);
 451         switch (max_rq_wrs) {
 452                 case 0:
 453                         max_rq_wrs = 4;
 454                         break;
 455                 case 1:
 456                         max_rq_wrs = 16;
 457                         break;
 458                 case 2:
 459                         max_rq_wrs = 32;
 460                         break;
 461                 case 3:
 462                         max_rq_wrs = 512;
 463                         break;
 464         }
 465
 466         max_sq_wrs = (u32temp & 3);
 467         switch (max_sq_wrs) {
 468                 case 0:
 469                         max_sq_wrs = 4;
 470                         break;
 471                 case 1:
 472                         max_sq_wrs = 16;
 473                         break;
 474                 case 2:
 475                         max_sq_wrs = 32;
 476                         break;
 477                 case 3:
 478                         max_sq_wrs = 512;
 479                         break;
 480         }
 481         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 482         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 483
 484         nesadapter->max_sge = 4;
 485         nesadapter->max_cqe = 32766;
 486
 487         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 488                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 489                 kfree(nesadapter);
 490                 return NULL;
 491         }
 492
 493         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 494         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 495                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 496
 497         /* setup port configuration */
 498         if (nesadapter->port_count == 1) {
 499                 nesadapter->log_port = 0x00000000;
 500                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 501                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 502                 else
 503                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 504         } else {
 505                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 506                         nesadapter->log_port = 0x000000D8;
 507                 } else {
 508                         if (nesadapter->port_count == 2)
 509                                 nesadapter->log_port = 0x00000044;
 510                         else
 511                                 nesadapter->log_port = 0x000000e4;
 512                 }
 513                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 514         }
 515
 516         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 517                                                 nesadapter->log_port);
 518         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 519                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 520
 521         spin_lock_init(&nesadapter->resource_lock);
 522         spin_lock_init(&nesadapter->phy_lock);
 523         spin_lock_init(&nesadapter->pbl_lock);
 524         spin_lock_init(&nesadapter->periodic_timer_lock);
 525
 526         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 527         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 528         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 529         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 530
 531         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 532                 u32 pcs_control_status0, pcs_control_status1;
 533                 u32 reset_value;
 534                 u32 i = 0;
 535                 u32 int_cnt = 0;
 536                 u32 ext_cnt = 0;
 537                 unsigned long flags;
 538                 u32 j = 0;
 539
 540                 pcs_control_status0 = nes_read_indexed(nesdev,
 541                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 542                 pcs_control_status1 = nes_read_indexed(nesdev,
 543                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 544
 545                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 546                         pcs_control_status0 = nes_read_indexed(nesdev,
 547                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 548                         pcs_control_status1 = nes_read_indexed(nesdev,
 549                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 550                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 551                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 552                                 int_cnt++;
 553                         msleep(1);
 554                 }
 555                 if (int_cnt > 1) {
 556                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 557                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 558                         mh_detected++;
 559                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 560                         reset_value |= 0x0000003d;
 561                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 562
 563                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 564                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 565                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 566
 567                         pcs_control_status0 = nes_read_indexed(nesdev,
 568                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 569                         pcs_control_status1 = nes_read_indexed(nesdev,
 570                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 571
 572                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 573                                 pcs_control_status0 = nes_read_indexed(nesdev,
 574                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 575                                 pcs_control_status1 = nes_read_indexed(nesdev,
 576                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 577                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 578                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 579                                         if (++ext_cnt > int_cnt) {
 580                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 581                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 582                                                                 0x0000F088);
 583                                                 mh_detected++;
 584                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 585                                                 reset_value |= 0x0000003d;
 586                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 587
 588                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 589                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 590                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 591                                                 break;
 592                                         }
 593                                 }
 594                                 msleep(1);
 595                         }
 596                 }
 597         }
 598
 599         if (nesadapter->hw_rev == NE020_REV) {
 600                 init_timer(&nesadapter->mh_timer);
 601                 nesadapter->mh_timer.function = nes_mh_fix;
 602                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 603                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 604                 add_timer(&nesadapter->mh_timer);
 605         } else {
 606                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 607         }
 608
 609         init_timer(&nesadapter->lc_timer);
 610         nesadapter->lc_timer.function = nes_clc;
 611         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 612         nesadapter->lc_timer.data = (unsigned long)nesdev;
 613         add_timer(&nesadapter->lc_timer);
 614
 615         list_add_tail(&nesadapter->list, &nes_adapter_list);
 616
 617         for (func_index = 0; func_index < 8; func_index++) {
 618                 pci_bus_read_config_word(nesdev->pcidev->bus,
 619                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 620                                         func_index), 0, &vendor_id);
 621                 if (vendor_id == 0xffff)
 622                         break;
 623         }
 624         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 625                 func_index, pci_name(nesdev->pcidev));
 626         nesadapter->adapter_fcn_count = func_index;
 627
 628         return nesadapter;
 629 }
 630
 631
 632 /**
 633  * nes_reset_adapter_ne020
 634  */
 635 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 636 {
 637         u32 port_count;
 638         u32 u32temp;
 639         u32 i;
 640
 641         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 642         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 643         /* TODO: assuming that both SERDES are set the same for now */
 644         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 645         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 646                         u32temp, port_count);
 647         if (*OneG_Mode)
 648                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 649         u32temp &= 0xff00ffc0;
 650         switch (port_count) {
 651                 case 1:
 652                         u32temp |= 0x00ee0000;
 653                         break;
 654                 case 2:
 655                         u32temp |= 0x00cc0000;
 656                         break;
 657                 case 4:
 658                         u32temp |= 0x00000000;
 659                         break;
 660                 default:
 661                         return 0;
 662                         break;
 663         }
 664
 665         /* check and do full reset if needed */
 666         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 667                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 668                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 669
 670                 i = 0;
 671                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 672                         mdelay(1);
 673                 if (i > 10000) {
 674                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 675                         return 0;
 676                 }
 677
 678                 i = 0;
 679                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 680                         mdelay(1);
 681                 if (i > 10000) {
 682                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 683                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 684                         return 0;
 685                 }
 686         }
 687
 688         /* port reset */
 689         switch (port_count) {
 690                 case 1:
 691                         u32temp |= 0x00ee0010;
 692                         break;
 693                 case 2:
 694                         u32temp |= 0x00cc0030;
 695                         break;
 696                 case 4:
 697                         u32temp |= 0x00000030;
 698                         break;
 699         }
 700
 701         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 702         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 703
 704         i = 0;
 705         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 706                 mdelay(1);
 707         if (i > 10000) {
 708                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 709                 return 0;
 710         }
 711
 712         /* serdes 0 */
 713         i = 0;
 714         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 715                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 716                 mdelay(1);
 717         if (i > 5000) {
 718                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 719                 return 0;
 720         }
 721
 722         /* serdes 1 */
 723         if (port_count > 1) {
 724                 i = 0;
 725                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 726                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 727                         mdelay(1);
 728                 if (i > 5000) {
 729                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 730                         return 0;
 731                 }
 732         }
 733
 734         return port_count;
 735 }
 736
 737
 738 /**
 739  * nes_init_serdes
 740  */
 741 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 742                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 743 {
 744         int i;
 745         u32 u32temp;
 746         u32 sds;
 747
 748         if (hw_rev != NE020_REV) {
 749                 /* init serdes 0 */
 750                 if (wide_ppm_offset && (nesadapter->phy_type[0] == NES_PHY_TYPE_CX4))
 751                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 752                 else
 753                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 754
 755                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 756                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 757                         sds |= 0x00000100;
 758                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 759                 }
 760                 if (!OneG_Mode)
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 762
 763                 if (port_count < 2)
 764                         return 0;
 765
 766                 /* init serdes 1 */
 767                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 768                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 769
 770                 switch (nesadapter->phy_type[1]) {
 771                 case NES_PHY_TYPE_ARGUS:
 772                 case NES_PHY_TYPE_SFP_D:
 773                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 774                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 775                         break;
 776                 case NES_PHY_TYPE_CX4:
 777                         if (wide_ppm_offset)
 778                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 779                         break;
 780                 case NES_PHY_TYPE_PUMA_1G:
 781                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 782                         sds |= 0x000000100;
 783                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 784                 }
 785                 if (!OneG_Mode) {
 786                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 787                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 788                         sds &= 0xFFFFFFBF;
 789                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 790                 }
 791         } else {
 792                 /* init serdes 0 */
 793                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 794                 i = 0;
 795                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 796                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 797                         mdelay(1);
 798                 if (i > 5000) {
 799                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 800                         return 1;
 801                 }
 802                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 803                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 804                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 805                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 806                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 807                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 808                 if (OneG_Mode)
 809                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 810                 else
 811                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 812
 813                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 814                 if (port_count > 1) {
 815                         /* init serdes 1 */
 816                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 817                         i = 0;
 818                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 819                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 820                                 mdelay(1);
 821                         if (i > 5000) {
 822                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 823                                 /* return 1; */
 824                         }
 825                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 826                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 828                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 829                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 830                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 831                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 832                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 833                 }
 834         }
 835         return 0;
 836 }
 837
 838
 839 /**
 840  * nes_init_csr_ne020
 841  * Initialize registers for ne020 hardware
 842  */
 843 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 844 {
 845         u32 u32temp;
 846
 847         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 848
 849         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 850         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 851         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 852         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 853         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 854         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 855         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 856         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 857
 858         /* TODO: move these MAC register settings to NIC bringup */
 859         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 860         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 861         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 862         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 863         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 864         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 865         if (port_count > 1) {
 866                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 867                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 868                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 869                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 870                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 871                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 872                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 873         }
 874         if (port_count > 2) {
 875                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 876                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 877                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 878                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 879                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 880                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 881                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 882
 883                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 884                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 885                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 886                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 887                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 888                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 889                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 890         }
 891
 892         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 893         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 894         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 895                                                          0x00000001);
 896         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 897         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 898         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 899         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 900         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 901
 902         /* TODO: move this to code, get from EEPROM */
 903         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 904         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 905         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 906
 907         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 908         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 909
 910         if (hw_rev != NE020_REV) {
 911                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 912                 u32temp |= 0x80000000;
 913                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 914                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 915                 u32temp &= 0x7fffffff;
 916                 u32temp |= 0x7fff0010;
 917                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 918                 if (port_count > 1) {
 919                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 920                         u32temp &= 0x7fffffff;
 921                         u32temp |= 0x7fff0010;
 922                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 923                 }
 924         }
 925 }
 926
 927
 928 /**
 929  * nes_destroy_adapter - destroy the adapter structure
 930  */
 931 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 932 {
 933         struct nes_adapter *tmp_adapter;
 934
 935         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 936                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 937                                 tmp_adapter);
 938         }
 939
 940         nesadapter->ref_count--;
 941         if (!nesadapter->ref_count) {
 942                 if (nesadapter->hw_rev == NE020_REV) {
 943                         del_timer(&nesadapter->mh_timer);
 944                 }
 945                 del_timer(&nesadapter->lc_timer);
 946
 947                 list_del(&nesadapter->list);
 948                 kfree(nesadapter);
 949         }
 950 }
 951
 952
 953 /**
 954  * nes_init_cqp
 955  */
 956 int nes_init_cqp(struct nes_device *nesdev)
 957 {
 958         struct nes_adapter *nesadapter = nesdev->nesadapter;
 959         struct nes_hw_cqp_qp_context *cqp_qp_context;
 960         struct nes_hw_cqp_wqe *cqp_wqe;
 961         struct nes_hw_ceq *ceq;
 962         struct nes_hw_ceq *nic_ceq;
 963         struct nes_hw_aeq *aeq;
 964         void *vmem;
 965         dma_addr_t pmem;
 966         u32 count=0;
 967         u32 cqp_head;
 968         u64 u64temp;
 969         u32 u32temp;
 970
 971         /* allocate CQP memory */
 972         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 973         /* SQ is 512 byte aligned, others are 256 byte aligned */
 974         nesdev->cqp_mem_size = 512 +
 975                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 976                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 977                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 978                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 979                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 980                         sizeof(struct nes_hw_cqp_qp_context);
 981
 982         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 983                         &nesdev->cqp_pbase);
 984         if (!nesdev->cqp_vbase) {
 985                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 986                 return -ENOMEM;
 987         }
 988         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 989
 990         /* Allocate a twice the number of CQP requests as the SQ size */
 991         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 992                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 993         if (nesdev->nes_cqp_requests == NULL) {
 994                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 995                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 996                                 nesdev->cqp.sq_pbase);
 997                 return -ENOMEM;
 998         }
 999
1000         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1001                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1002
1003         spin_lock_init(&nesdev->cqp.lock);
1004         init_waitqueue_head(&nesdev->cqp.waitq);
1005
1006         /* Setup Various Structures */
1007         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1008                         ~(unsigned long)(512 - 1));
1009         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1010                         ~(unsigned long long)(512 - 1));
1011
1012         nesdev->cqp.sq_vbase = vmem;
1013         nesdev->cqp.sq_pbase = pmem;
1014         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1015         nesdev->cqp.sq_head = 0;
1016         nesdev->cqp.sq_tail = 0;
1017         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1018
1019         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1020         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1021
1022         nesdev->ccq.cq_vbase = vmem;
1023         nesdev->ccq.cq_pbase = pmem;
1024         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1025         nesdev->ccq.cq_head = 0;
1026         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1027         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1028
1029         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1030         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1031
1032         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1033         ceq = &nesadapter->ceq[nesdev->ceq_index];
1034         ceq->ceq_vbase = vmem;
1035         ceq->ceq_pbase = pmem;
1036         ceq->ceq_size = NES_CCEQ_SIZE;
1037         ceq->ceq_head = 0;
1038
1039         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1040         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1041
1042         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1043         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1044         nic_ceq->ceq_vbase = vmem;
1045         nic_ceq->ceq_pbase = pmem;
1046         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1047         nic_ceq->ceq_head = 0;
1048
1049         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1050         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1051
1052         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1053         aeq->aeq_vbase = vmem;
1054         aeq->aeq_pbase = pmem;
1055         aeq->aeq_size = nesadapter->max_qp;
1056         aeq->aeq_head = 0;
1057
1058         /* Setup QP Context */
1059         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1060         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1061
1062         cqp_qp_context = vmem;
1063         cqp_qp_context->context_words[0] =
1064                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1065         cqp_qp_context->context_words[1] = 0;
1066         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1067         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1068
1069
1070         /* Write the address to Create CQP */
1071         if ((sizeof(dma_addr_t) > 4)) {
1072                 nes_write_indexed(nesdev,
1073                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1074                                 ((u64)pmem) >> 32);
1075         } else {
1076                 nes_write_indexed(nesdev,
1077                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1078         }
1079         nes_write_indexed(nesdev,
1080                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1081                         (u32)pmem);
1082
1083         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1084         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1085
1086         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1087                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1088                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1089         }
1090
1091         /* Write Create CCQ WQE */
1092         cqp_head = nesdev->cqp.sq_head++;
1093         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1094         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1095         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1096                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1097                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1098         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1099                             (nesdev->ccq.cq_number |
1100                              ((u32)nesdev->ceq_index << 16)));
1101         u64temp = (u64)nesdev->ccq.cq_pbase;
1102         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1103         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1104         u64temp = (unsigned long)&nesdev->ccq;
1105         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1106                         cpu_to_le32((u32)(u64temp >> 1));
1107         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1108                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1109         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1110
1111         /* Write Create CEQ WQE */
1112         cqp_head = nesdev->cqp.sq_head++;
1113         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1114         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1115         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1116                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1117         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1118         u64temp = (u64)ceq->ceq_pbase;
1119         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1120
1121         /* Write Create AEQ WQE */
1122         cqp_head = nesdev->cqp.sq_head++;
1123         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1124         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1125         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1126                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1127         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1128         u64temp = (u64)aeq->aeq_pbase;
1129         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1130
1131         /* Write Create NIC CEQ WQE */
1132         cqp_head = nesdev->cqp.sq_head++;
1133         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1134         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1135         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1136                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1137         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1138         u64temp = (u64)nic_ceq->ceq_pbase;
1139         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1140
1141         /* Poll until CCQP done */
1142         count = 0;
1143         do {
1144                 if (count++ > 1000) {
1145                         printk(KERN_ERR PFX "Error creating CQP\n");
1146                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1147                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1148                         return -1;
1149                 }
1150                 udelay(10);
1151         } while (!(nes_read_indexed(nesdev,
1152                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1153
1154         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1155                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1156
1157         u32temp = 0x04800000;
1158         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1159
1160         /* wait for the CCQ, CEQ, and AEQ to get created */
1161         count = 0;
1162         do {
1163                 if (count++ > 1000) {
1164                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1165                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1166                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1167                         return -1;
1168                 }
1169                 udelay(10);
1170         } while (((nes_read_indexed(nesdev,
1171                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1172
1173         /* dump the QP status value */
1174         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1175                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1176
1177         nesdev->cqp.sq_tail++;
1178
1179         return 0;
1180 }
1181
1182
1183 /**
1184  * nes_destroy_cqp
1185  */
1186 int nes_destroy_cqp(struct nes_device *nesdev)
1187 {
1188         struct nes_hw_cqp_wqe *cqp_wqe;
1189         u32 count = 0;
1190         u32 cqp_head;
1191         unsigned long flags;
1192
1193         do {
1194                 if (count++ > 1000)
1195                         break;
1196                 udelay(10);
1197         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1198
1199         /* Reset CCQ */
1200         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1201                         nesdev->ccq.cq_number);
1202
1203         /* Disable device interrupts */
1204         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1205
1206         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1207
1208         /* Destroy the AEQ */
1209         cqp_head = nesdev->cqp.sq_head++;
1210         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1211         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1212         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1213                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1214         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1215
1216         /* Destroy the NIC CEQ */
1217         cqp_head = nesdev->cqp.sq_head++;
1218         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1219         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1220         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1221                         ((u32)nesdev->nic_ceq_index << 8));
1222
1223         /* Destroy the CEQ */
1224         cqp_head = nesdev->cqp.sq_head++;
1225         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1226         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1227         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1228                         (nesdev->ceq_index << 8));
1229
1230         /* Destroy the CCQ */
1231         cqp_head = nesdev->cqp.sq_head++;
1232         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1233         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1234         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1235         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1236                         ((u32)nesdev->ceq_index << 16));
1237
1238         /* Destroy CQP */
1239         cqp_head = nesdev->cqp.sq_head++;
1240         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1241         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1242         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1243                         NES_CQP_QP_TYPE_CQP);
1244         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1245
1246         barrier();
1247         /* Ring doorbell (5 WQEs) */
1248         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1249
1250         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1251
1252         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1253         count = 0;
1254         do {
1255                 if (count++ > 1000) {
1256                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1257                                         PCI_FUNC(nesdev->pcidev->devfn));
1258                         break;
1259                 }
1260                 udelay(10);
1261         } while (((nes_read_indexed(nesdev,
1262                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1263
1264         /* dump the QP status value */
1265         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1266                         PCI_FUNC(nesdev->pcidev->devfn),
1267                         nes_read_indexed(nesdev,
1268                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1269
1270         kfree(nesdev->nes_cqp_requests);
1271
1272         /* Free the control structures */
1273         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1274                         nesdev->cqp.sq_pbase);
1275
1276         return 0;
1277 }
1278
1279
1280 /**
1281  * nes_init_phy
1282  */
1283 int nes_init_phy(struct nes_device *nesdev)
1284 {
1285         struct nes_adapter *nesadapter = nesdev->nesadapter;
1286         u32 counter = 0;
1287         u32 sds;
1288         u32 mac_index = nesdev->mac_index;
1289         u32 tx_config = 0;
1290         u16 phy_data;
1291         u32 temp_phy_data = 0;
1292         u32 temp_phy_data2 = 0;
1293         u8  phy_type = nesadapter->phy_type[mac_index];
1294         u8  phy_index = nesadapter->phy_index[mac_index];
1295
1296         if ((nesadapter->OneG_Mode) &&
1297             (phy_type != NES_PHY_TYPE_PUMA_1G)) {
1298                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1299                 if (phy_type == NES_PHY_TYPE_1G) {
1300                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1301                         tx_config &= 0xFFFFFFE3;
1302                         tx_config |= 0x04;
1303                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1304                 }
1305
1306                 nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1307                 nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1308
1309                 /* Reset the PHY */
1310                 nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1311                 udelay(100);
1312                 counter = 0;
1313                 do {
1314                         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1315                         if (counter++ > 100)
1316                                 break;
1317                 } while (phy_data & 0x8000);
1318
1319                 /* Setting no phy loopback */
1320                 phy_data &= 0xbfff;
1321                 phy_data |= 0x1140;
1322                 nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1323                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1324                 nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1325                 nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1326
1327                 /* Setting the interrupt mask */
1328                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1329                 nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1330                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1331
1332                 /* turning on flow control */
1333                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1334                 nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1335                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1336
1337                 /* Clear Half duplex */
1338                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1339                 nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1340                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1341
1342                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1343                 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1344
1345                 return 0;
1346         }
1347
1348         if ((phy_type == NES_PHY_TYPE_IRIS) ||
1349             (phy_type == NES_PHY_TYPE_ARGUS) ||
1350             (phy_type == NES_PHY_TYPE_SFP_D)) {
1351                 /* setup 10G MDIO operation */
1352                 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1353                 tx_config &= 0xFFFFFFE3;
1354                 tx_config |= 0x15;
1355                 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1356         }
1357         if ((phy_type == NES_PHY_TYPE_ARGUS) ||
1358             (phy_type == NES_PHY_TYPE_SFP_D)) {
1359                 u32 first_time = 1;
1360
1361                 /* Check firmware heartbeat */
1362                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1363                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1364                 udelay(1500);
1365                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1366                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1367
1368                 if (temp_phy_data != temp_phy_data2) {
1369                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1370                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1371                         if ((temp_phy_data & 0xff) > 0x20)
1372                                 return 0;
1373                         printk(PFX "Reinitializing PHY\n");
1374                 }
1375
1376                 /* no heartbeat, configure the PHY */
1377                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1378                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1379                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1380                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1381                 if (phy_type == NES_PHY_TYPE_ARGUS) {
1382                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1383                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1384                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1385                 } else {
1386                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1387                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1388                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1389                 }
1390                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1391                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1392
1393                 /* setup LEDs */
1394                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1395                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1397
1398                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1399
1400                 /* Bring PHY out of reset */
1401                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1402
1403                 /* Check for heartbeat */
1404                 counter = 0;
1405                 mdelay(690);
1406                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1407                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1408                 do {
1409                         if (counter++ > 150) {
1410                                 printk(PFX "No PHY heartbeat\n");
1411                                 break;
1412                         }
1413                         mdelay(1);
1414                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1415                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1416                 } while ((temp_phy_data2 == temp_phy_data));
1417
1418                 /* wait for tracking */
1419                 counter = 0;
1420                 do {
1421                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1422                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1423                         if (counter++ > 300) {
1424                                 if (((temp_phy_data & 0xff) == 0x0) && first_time) {
1425                                         first_time = 0;
1426                                         counter = 0;
1427                                         /* reset AMCC PHY and try again */
1428                                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1429                                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1430                                         continue;
1431                                 } else {
1432                                         printk(PFX "PHY did not track\n");
1433                                         break;
1434                                 }
1435                         }
1436                         mdelay(10);
1437                 } while ((temp_phy_data & 0xff) < 0x30);
1438
1439                 /* setup signal integrity */
1440                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1441                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1442                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1443                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1444                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1445
1446                 /* reset serdes */
1447                 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1448                                        mac_index * 0x200);
1449                 sds |= 0x1;
1450                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1451                                   mac_index * 0x200, sds);
1452                 sds &= 0xfffffffe;
1453                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1454                                   mac_index * 0x200, sds);
1455
1456                 counter = 0;
1457                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1458                                 && (counter++ < 5000))
1459                         ;
1460         }
1461         return 0;
1462 }
1463
1464
1465 /**
1466  * nes_replenish_nic_rq
1467  */
1468 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1469 {
1470         unsigned long flags;
1471         dma_addr_t bus_address;
1472         struct sk_buff *skb;
1473         struct nes_hw_nic_rq_wqe *nic_rqe;
1474         struct nes_hw_nic *nesnic;
1475         struct nes_device *nesdev;
1476         u32 rx_wqes_posted = 0;
1477
1478         nesnic = &nesvnic->nic;
1479         nesdev = nesvnic->nesdev;
1480         spin_lock_irqsave(&nesnic->rq_lock, flags);
1481         if (nesnic->replenishing_rq !=0) {
1482                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1483                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1484                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1485                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1486                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1487                         add_timer(&nesvnic->rq_wqes_timer);
1488                 } else
1489                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1490                 return;
1491         }
1492         nesnic->replenishing_rq = 1;
1493         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1494         do {
1495                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1496                 if (skb) {
1497                         skb->dev = nesvnic->netdev;
1498
1499                         bus_address = pci_map_single(nesdev->pcidev,
1500                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1501
1502                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1503                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1504                                         cpu_to_le32(nesvnic->max_frame_size);
1505                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1506                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1507                                         cpu_to_le32((u32)bus_address);
1508                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1509                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1510                         nesnic->rx_skb[nesnic->rq_head] = skb;
1511                         nesnic->rq_head++;
1512                         nesnic->rq_head &= nesnic->rq_size - 1;
1513                         atomic_dec(&nesvnic->rx_skbs_needed);
1514                         barrier();
1515                         if (++rx_wqes_posted == 255) {
1516                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1517                                 rx_wqes_posted = 0;
1518                         }
1519                 } else {
1520                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1521                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1522                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1523                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1524                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1525                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1526                                 add_timer(&nesvnic->rq_wqes_timer);
1527                         } else
1528                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1529                         break;
1530                 }
1531         } while (atomic_read(&nesvnic->rx_skbs_needed));
1532         barrier();
1533         if (rx_wqes_posted)
1534                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1535         nesnic->replenishing_rq = 0;
1536 }
1537
1538
1539 /**
1540  * nes_rq_wqes_timeout
1541  */
1542 static void nes_rq_wqes_timeout(unsigned long parm)
1543 {
1544         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1545         printk("%s: Timer fired.\n", __func__);
1546         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1547         if (atomic_read(&nesvnic->rx_skbs_needed))
1548                 nes_replenish_nic_rq(nesvnic);
1549 }
1550
1551
1552 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1553                                void **tcph, u64 *hdr_flags, void *priv)
1554 {
1555         unsigned int ip_len;
1556         struct iphdr *iph;
1557         skb_reset_network_header(skb);
1558         iph = ip_hdr(skb);
1559         if (iph->protocol != IPPROTO_TCP)
1560                 return -1;
1561         ip_len = ip_hdrlen(skb);
1562         skb_set_transport_header(skb, ip_len);
1563         *tcph = tcp_hdr(skb);
1564
1565         *hdr_flags = LRO_IPV4 | LRO_TCP;
1566         *iphdr = iph;
1567         return 0;
1568 }
1569
1570
1571 /**
1572  * nes_init_nic_qp
1573  */
1574 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1575 {
1576         struct nes_hw_cqp_wqe *cqp_wqe;
1577         struct nes_hw_nic_sq_wqe *nic_sqe;
1578         struct nes_hw_nic_qp_context *nic_context;
1579         struct sk_buff *skb;
1580         struct nes_hw_nic_rq_wqe *nic_rqe;
1581         struct nes_vnic *nesvnic = netdev_priv(netdev);
1582         unsigned long flags;
1583         void *vmem;
1584         dma_addr_t pmem;
1585         u64 u64temp;
1586         int ret;
1587         u32 cqp_head;
1588         u32 counter;
1589         u32 wqe_count;
1590         u8 jumbomode=0;
1591
1592         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1593         nesvnic->nic_mem_size = 256 +
1594                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1595                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1596                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1597                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1598                         sizeof(struct nes_hw_nic_qp_context);
1599
1600         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1601                         &nesvnic->nic_pbase);
1602         if (!nesvnic->nic_vbase) {
1603                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1604                 return -ENOMEM;
1605         }
1606         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1607         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1608                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1609
1610         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1611                         ~(unsigned long)(256 - 1));
1612         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1613                         ~(unsigned long long)(256 - 1));
1614
1615         /* Setup the first Fragment buffers */
1616         nesvnic->nic.first_frag_vbase = vmem;
1617
1618         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1619                 nesvnic->nic.frag_paddr[counter] = pmem;
1620                 pmem += sizeof(struct nes_first_frag);
1621         }
1622
1623         /* setup the SQ */
1624         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1625
1626         nesvnic->nic.sq_vbase = (void *)vmem;
1627         nesvnic->nic.sq_pbase = pmem;
1628         nesvnic->nic.sq_head = 0;
1629         nesvnic->nic.sq_tail = 0;
1630         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1631         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1632                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1633                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1634                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1635                                 NES_NIC_SQ_WQE_COMPLETION);
1636                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1637                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1638                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1639                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1640                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1641                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1642         }
1643
1644         nesvnic->get_cqp_request = nes_get_cqp_request;
1645         nesvnic->post_cqp_request = nes_post_cqp_request;
1646         nesvnic->mcrq_mcast_filter = NULL;
1647
1648         spin_lock_init(&nesvnic->nic.rq_lock);
1649
1650         /* setup the RQ */
1651         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1652         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1653
1654
1655         nesvnic->nic.rq_vbase = vmem;
1656         nesvnic->nic.rq_pbase = pmem;
1657         nesvnic->nic.rq_head = 0;
1658         nesvnic->nic.rq_tail = 0;
1659         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1660
1661         /* setup the CQ */
1662         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1663         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1664
1665         if (nesdev->nesadapter->netdev_count > 2)
1666                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1667         else
1668                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1669
1670         nesvnic->nic_cq.cq_vbase = vmem;
1671         nesvnic->nic_cq.cq_pbase = pmem;
1672         nesvnic->nic_cq.cq_head = 0;
1673         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1674
1675         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1676
1677         /* Send CreateCQ request to CQP */
1678         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1679         cqp_head = nesdev->cqp.sq_head;
1680
1681         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1682         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1683
1684         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1685                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1686                         ((u32)nesvnic->nic_cq.cq_size << 16));
1687         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1688                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1689         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1690         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1691         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1692         u64temp = (unsigned long)&nesvnic->nic_cq;
1693         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1694         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1695                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1696         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1697         if (++cqp_head >= nesdev->cqp.sq_size)
1698                 cqp_head = 0;
1699         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1700         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1701
1702         /* Send CreateQP request to CQP */
1703         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1704         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1705                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1706                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1707         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1708                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1709                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1710         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1711                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1712         }
1713
1714         u64temp = (u64)nesvnic->nic.sq_pbase;
1715         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1716         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1717         u64temp = (u64)nesvnic->nic.rq_pbase;
1718         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1719         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1720
1721         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1722                         NES_CQP_QP_TYPE_NIC);
1723         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1724         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1725                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1726         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1727
1728         if (++cqp_head >= nesdev->cqp.sq_size)
1729                 cqp_head = 0;
1730         nesdev->cqp.sq_head = cqp_head;
1731
1732         barrier();
1733
1734         /* Ring doorbell (2 WQEs) */
1735         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1736
1737         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1738         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1739                         nesvnic->nic.qp_id);
1740
1741         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1742                         NES_EVENT_TIMEOUT);
1743         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1744                         nesvnic->nic.qp_id, ret);
1745         if (!ret) {
1746                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1747                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1748                                 nesvnic->nic_pbase);
1749                 return -EIO;
1750         }
1751
1752         /* Populate the RQ */
1753         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1754                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1755                 if (!skb) {
1756                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1757
1758                         nes_destroy_nic_qp(nesvnic);
1759                         return -ENOMEM;
1760                 }
1761
1762                 skb->dev = netdev;
1763
1764                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1765                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1766
1767                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1768                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1769                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1770                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1771                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1772                 nesvnic->nic.rx_skb[counter] = skb;
1773         }
1774
1775         wqe_count = NES_NIC_WQ_SIZE - 1;
1776         nesvnic->nic.rq_head = wqe_count;
1777         barrier();
1778         do {
1779                 counter = min(wqe_count, ((u32)255));
1780                 wqe_count -= counter;
1781                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1782         } while (wqe_count);
1783         init_timer(&nesvnic->rq_wqes_timer);
1784         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1785         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1786         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1787         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1788         {
1789                 nes_nic_init_timer(nesdev);
1790                 if (netdev->mtu > 1500)
1791                         jumbomode = 1;
1792                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1793         }
1794         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1795         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1796         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1797         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1798         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1799         nesvnic->lro_mgr.dev            = netdev;
1800         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1801         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1802         return 0;
1803 }
1804
1805
1806 /**
1807  * nes_destroy_nic_qp
1808  */
1809 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1810 {
1811         u64 u64temp;
1812         dma_addr_t bus_address;
1813         struct nes_device *nesdev = nesvnic->nesdev;
1814         struct nes_hw_cqp_wqe *cqp_wqe;
1815         struct nes_hw_nic_sq_wqe *nic_sqe;
1816         struct nes_hw_nic_rq_wqe *nic_rqe;
1817         __le16 *wqe_fragment_length;
1818         u16  wqe_fragment_index;
1819         u64 wqe_frag;
1820         u32 cqp_head;
1821         unsigned long flags;
1822         int ret;
1823
1824         /* Free remaining NIC receive buffers */
1825         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1826                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1827                 wqe_frag  = (u64)le32_to_cpu(
1828                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1829                 wqe_frag |= ((u64)le32_to_cpu(
1830                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1831                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1832                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1833                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1834                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1835         }
1836
1837         /* Free remaining NIC transmit buffers */
1838         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1839                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1840                 wqe_fragment_index = 1;
1841                 wqe_fragment_length = (__le16 *)
1842                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1843                 /* bump past the vlan tag */
1844                 wqe_fragment_length++;
1845                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1846                         u64temp = (u64)le32_to_cpu(
1847                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1848                                 wqe_fragment_index*2]);
1849                         u64temp += ((u64)le32_to_cpu(
1850                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1851                                 + wqe_fragment_index*2]))<<32;
1852                         bus_address = (dma_addr_t)u64temp;
1853                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1854                                         nesvnic->nic.first_frag_overflow)) {
1855                                 pci_unmap_single(nesdev->pcidev,
1856                                                 bus_address,
1857                                                 le16_to_cpu(wqe_fragment_length[
1858                                                         wqe_fragment_index++]),
1859                                                 PCI_DMA_TODEVICE);
1860                         }
1861                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1862                                 if (wqe_fragment_length[wqe_fragment_index]) {
1863                                         u64temp = le32_to_cpu(
1864                                                 nic_sqe->wqe_words[
1865                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1866                                                 wqe_fragment_index*2]);
1867                                         u64temp += ((u64)le32_to_cpu(
1868                                                 nic_sqe->wqe_words[
1869                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1870                                                 wqe_fragment_index*2]))<<32;
1871                                         bus_address = (dma_addr_t)u64temp;
1872                                         pci_unmap_page(nesdev->pcidev,
1873                                                         bus_address,
1874                                                         le16_to_cpu(
1875                                                         wqe_fragment_length[
1876                                                         wqe_fragment_index]),
1877                                                         PCI_DMA_TODEVICE);
1878                                 } else
1879                                         break;
1880                         }
1881                 }
1882                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1883                         dev_kfree_skb(
1884                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1885
1886                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1887                                         & (nesvnic->nic.sq_size - 1);
1888         }
1889
1890         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1891
1892         /* Destroy NIC QP */
1893         cqp_head = nesdev->cqp.sq_head;
1894         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1895         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1896
1897         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1898                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1899         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1900                 nesvnic->nic.qp_id);
1901
1902         if (++cqp_head >= nesdev->cqp.sq_size)
1903                 cqp_head = 0;
1904
1905         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1906
1907         /* Destroy NIC CQ */
1908         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1909         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1910                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1911         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1912                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1913
1914         if (++cqp_head >= nesdev->cqp.sq_size)
1915                 cqp_head = 0;
1916
1917         nesdev->cqp.sq_head = cqp_head;
1918         barrier();
1919
1920         /* Ring doorbell (2 WQEs) */
1921         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1922
1923         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1924         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1925                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1926                         cqp_head, nesdev->cqp.sq_head,
1927                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1928
1929         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1930                         NES_EVENT_TIMEOUT);
1931
1932         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1933                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1934                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1935         if (!ret) {
1936                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1937                                 nesvnic->nic.qp_id);
1938         }
1939
1940         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1941                         nesvnic->nic_pbase);
1942 }
1943
1944 /**
1945  * nes_napi_isr
1946  */
1947 int nes_napi_isr(struct nes_device *nesdev)
1948 {
1949         struct nes_adapter *nesadapter = nesdev->nesadapter;
1950         u32 int_stat;
1951
1952         if (nesdev->napi_isr_ran) {
1953                 /* interrupt status has already been read in ISR */
1954                 int_stat = nesdev->int_stat;
1955         } else {
1956                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1957                 nesdev->int_stat = int_stat;
1958                 nesdev->napi_isr_ran = 1;
1959         }
1960
1961         int_stat &= nesdev->int_req;
1962         /* iff NIC, process here, else wait for DPC */
1963         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1964                 nesdev->napi_isr_ran = 0;
1965                 nes_write32(nesdev->regs + NES_INT_STAT,
1966                         (int_stat &
1967                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1968
1969                 /* Process the CEQs */
1970                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1971
1972                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1973                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1974                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1975                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1976                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1977                                 /* Enable Periodic timer interrupts */
1978                                 nesdev->int_req |= NES_INT_TIMER;
1979                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1980                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1981                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1982                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1983                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1984                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1985                         }
1986
1987                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1988                         {
1989                                 nes_nic_init_timer(nesdev);
1990                         }
1991                         /* Enable interrupts, except CEQs */
1992                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1993                 } else {
1994                         /* Enable interrupts, make sure timer is off */
1995                         nesdev->int_req &= ~NES_INT_TIMER;
1996                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1997                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1998                 }
1999                 nesdev->deepcq_count = 0;
2000                 return 1;
2001         } else {
2002                 return 0;
2003         }
2004 }
2005
2006 static void process_critical_error(struct nes_device *nesdev)
2007 {
2008         u32 debug_error;
2009         u32 nes_idx_debug_error_masks0 = 0;
2010         u16 error_module = 0;
2011
2012         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2013         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2014                         (u16)debug_error);
2015         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2016                         0x01010000 | (debug_error & 0x0000ffff));
2017         if (crit_err_count++ > 10)
2018                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2019         error_module = (u16) (debug_error & 0x1F00) >> 8;
2020         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2021                         nes_max_critical_error_count) {
2022                 printk(KERN_ERR PFX "Masking off critical error for module "
2023                         "0x%02X\n", (u16)error_module);
2024                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2025                         NES_IDX_DEBUG_ERROR_MASKS0);
2026                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2027                         nes_idx_debug_error_masks0 | (1 << error_module));
2028         }
2029 }
2030 /**
2031  * nes_dpc
2032  */
2033 void nes_dpc(unsigned long param)
2034 {
2035         struct nes_device *nesdev = (struct nes_device *)param;
2036         struct nes_adapter *nesadapter = nesdev->nesadapter;
2037         u32 counter;
2038         u32 loop_counter = 0;
2039         u32 int_status_bit;
2040         u32 int_stat;
2041         u32 timer_stat;
2042         u32 temp_int_stat;
2043         u32 intf_int_stat;
2044         u32 processed_intf_int = 0;
2045         u16 processed_timer_int = 0;
2046         u16 completion_ints = 0;
2047         u16 timer_ints = 0;
2048
2049         /* nes_debug(NES_DBG_ISR, "\n"); */
2050
2051         do {
2052                 timer_stat = 0;
2053                 if (nesdev->napi_isr_ran) {
2054                         nesdev->napi_isr_ran = 0;
2055                         int_stat = nesdev->int_stat;
2056                 } else
2057                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2058                 if (processed_intf_int != 0)
2059                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2060                 else
2061                         int_stat &= nesdev->int_req;
2062                 if (processed_timer_int == 0) {
2063                         processed_timer_int = 1;
2064                         if (int_stat & NES_INT_TIMER) {
2065                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2066                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2067                                         int_stat &= ~NES_INT_TIMER;
2068                                 }
2069                         }
2070                 } else {
2071                         int_stat &= ~NES_INT_TIMER;
2072                 }
2073
2074                 if (int_stat) {
2075                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2076                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2077                                 /* Ack the interrupts */
2078                                 nes_write32(nesdev->regs+NES_INT_STAT,
2079                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2080                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2081                         }
2082
2083                         temp_int_stat = int_stat;
2084                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2085                                 if (int_stat & int_status_bit) {
2086                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2087                                         temp_int_stat &= ~int_status_bit;
2088                                         completion_ints = 1;
2089                                 }
2090                                 if (!(temp_int_stat & 0x0000ffff))
2091                                         break;
2092                                 int_status_bit <<= 1;
2093                         }
2094
2095                         /* Process the AEQ for this pci function */
2096                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2097                         if (int_stat & int_status_bit) {
2098                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2099                         }
2100
2101                         /* Process the MAC interrupt for this pci function */
2102                         int_status_bit = 1 << (24 + nesdev->mac_index);
2103                         if (int_stat & int_status_bit) {
2104                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2105                         }
2106
2107                         if (int_stat & NES_INT_TIMER) {
2108                                 if (timer_stat & nesdev->timer_int_req) {
2109                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2110                                                         (timer_stat & nesdev->timer_int_req) |
2111                                                         ~(nesdev->nesadapter->timer_int_req));
2112                                         timer_ints = 1;
2113                                 }
2114                         }
2115
2116                         if (int_stat & NES_INT_INTF) {
2117                                 processed_intf_int = 1;
2118                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2119                                 intf_int_stat &= nesdev->intf_int_req;
2120                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2121                                         process_critical_error(nesdev);
2122                                 }
2123                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2124                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2125                                         BUG();
2126                                 }
2127                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2128                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2129                                         BUG();
2130                                 }
2131                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2132                         }
2133
2134                         if (int_stat & NES_INT_TSW) {
2135                         }
2136                 }
2137                 /* Don't use the interface interrupt bit stay in loop */
2138                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2139                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2140         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2141
2142         if (timer_ints == 1) {
2143                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2144                         if (completion_ints == 0) {
2145                                 nesdev->timer_only_int_count++;
2146                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2147                                         nesdev->timer_only_int_count = 0;
2148                                         nesdev->int_req &= ~NES_INT_TIMER;
2149                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2150                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2151                                 } else {
2152                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2153                                 }
2154                         } else {
2155                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2156                                 {
2157                                         nes_nic_init_timer(nesdev);
2158                                 }
2159                                 nesdev->timer_only_int_count = 0;
2160                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2161                         }
2162                 } else {
2163                         nesdev->timer_only_int_count = 0;
2164                         nesdev->int_req &= ~NES_INT_TIMER;
2165                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2166                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2167                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2168                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2169                 }
2170         } else {
2171                 if ( (completion_ints == 1) &&
2172                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2173                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2174                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2175                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2176                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2177                         nesdev->timer_only_int_count = 0;
2178                         nesdev->int_req |= NES_INT_TIMER;
2179                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2180                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2181                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2182                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2183                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2184                 } else {
2185                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2186                 }
2187         }
2188         nesdev->deepcq_count = 0;
2189 }
2190
2191
2192 /**
2193  * nes_process_ceq
2194  */
2195 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2196 {
2197         u64 u64temp;
2198         struct nes_hw_cq *cq;
2199         u32 head;
2200         u32 ceq_size;
2201
2202         /* nes_debug(NES_DBG_CQ, "\n"); */
2203         head = ceq->ceq_head;
2204         ceq_size = ceq->ceq_size;
2205
2206         do {
2207                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2208                                 NES_CEQE_VALID) {
2209                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2210                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2211                         u64temp <<= 1;
2212                         cq = *((struct nes_hw_cq **)&u64temp);
2213                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2214                         barrier();
2215                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2216
2217                         /* call the event handler */
2218                         cq->ce_handler(nesdev, cq);
2219
2220                         if (++head >= ceq_size)
2221                                 head = 0;
2222                 } else {
2223                         break;
2224                 }
2225
2226         } while (1);
2227
2228         ceq->ceq_head = head;
2229 }
2230
2231
2232 /**
2233  * nes_process_aeq
2234  */
2235 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2236 {
2237         /* u64 u64temp; */
2238         u32 head;
2239         u32 aeq_size;
2240         u32 aeqe_misc;
2241         u32 aeqe_cq_id;
2242         struct nes_hw_aeqe volatile *aeqe;
2243
2244         head = aeq->aeq_head;
2245         aeq_size = aeq->aeq_size;
2246
2247         do {
2248                 aeqe = &aeq->aeq_vbase[head];
2249                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2250                         break;
2251                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2252                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2253                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2254                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2255                                 /* dealing with an accelerated QP related AE */
2256                                 /*
2257                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2258                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2259                                  */
2260                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2261                         } else {
2262                                 /* TODO: dealing with a CQP related AE */
2263                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2264                                                 (u16)(aeqe_misc >> 16));
2265                         }
2266                 }
2267
2268                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2269
2270                 if (++head >= aeq_size)
2271                         head = 0;
2272
2273                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2274         }
2275         while (1);
2276         aeq->aeq_head = head;
2277 }
2278
2279 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2280 {
2281         struct nes_adapter *nesadapter = nesdev->nesadapter;
2282         u32 reset_value;
2283         u32 i=0;
2284         u32 u32temp;
2285
2286         if (nesadapter->hw_rev == NE020_REV) {
2287                 return;
2288         }
2289         mh_detected++;
2290
2291         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2292
2293         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2294                 reset_value |= 0x0000001d;
2295         else
2296                 reset_value |= 0x0000002d;
2297
2298         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2299                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2300                         nesadapter->link_interrupt_count[0] = 0;
2301                         nesadapter->link_interrupt_count[1] = 0;
2302                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2303                         if (0x00000040 & u32temp)
2304                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2305                         else
2306                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2307
2308                         reset_value |= 0x0000003d;
2309                 }
2310                 nesadapter->link_interrupt_count[mac_index] = 0;
2311         }
2312
2313         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2314
2315         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2316                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2317
2318         if (0x0000003d == (reset_value & 0x0000003d)) {
2319                 u32 pcs_control_status0, pcs_control_status1;
2320
2321                 for (i = 0; i < 10; i++) {
2322                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2323                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2324                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2325                              && (pcs_control_status0 & 0x00100000))
2326                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2327                                 && (pcs_control_status1 & 0x00100000)))
2328                                 continue;
2329                         else
2330                                 break;
2331                 }
2332                 if (10 == i) {
2333                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2334                         if (0x00000040 & u32temp)
2335                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2336                         else
2337                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2338
2339                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2340
2341                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2342                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2343                 }
2344         }
2345 }
2346
2347 /**
2348  * nes_process_mac_intr
2349  */
2350 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2351 {
2352         unsigned long flags;
2353         u32 pcs_control_status;
2354         struct nes_adapter *nesadapter = nesdev->nesadapter;
2355         struct nes_vnic *nesvnic;
2356         u32 mac_status;
2357         u32 mac_index = nesdev->mac_index;
2358         u32 u32temp;
2359         u16 phy_data;
2360         u16 temp_phy_data;
2361         u32 pcs_val  = 0x0f0f0000;
2362         u32 pcs_mask = 0x0f1f0000;
2363         u32 cdr_ctrl;
2364
2365         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2366         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2367                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2368                 return;
2369         }
2370         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2371         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2372
2373         /* ack the MAC interrupt */
2374         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2375         /* Clear the interrupt */
2376         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2377
2378         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2379
2380         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2381                 nesdev->link_status_interrupts++;
2382                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2383                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2384                         nes_reset_link(nesdev, mac_index);
2385                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2386                 }
2387                 /* read the PHY interrupt status register */
2388                 if ((nesadapter->OneG_Mode) &&
2389                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2390                         do {
2391                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2392                                                 nesadapter->phy_index[mac_index], &phy_data);
2393                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2394                                                 nesadapter->phy_index[mac_index], phy_data);
2395                         } while (phy_data&0x8000);
2396
2397                         temp_phy_data = 0;
2398                         do {
2399                                 nes_read_1G_phy_reg(nesdev, 0x11,
2400                                                 nesadapter->phy_index[mac_index], &phy_data);
2401                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2402                                                 nesadapter->phy_index[mac_index], phy_data);
2403                                 if (temp_phy_data == phy_data)
2404                                         break;
2405                                 temp_phy_data = phy_data;
2406                         } while (1);
2407
2408                         nes_read_1G_phy_reg(nesdev, 0x1e,
2409                                         nesadapter->phy_index[mac_index], &phy_data);
2410                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2411                                         nesadapter->phy_index[mac_index], phy_data);
2412
2413                         nes_read_1G_phy_reg(nesdev, 1,
2414                                         nesadapter->phy_index[mac_index], &phy_data);
2415                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2416                                         nesadapter->phy_index[mac_index], phy_data);
2417
2418                         if (temp_phy_data & 0x1000) {
2419                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2420                                 phy_data = 4;
2421                         } else {
2422                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2423                         }
2424                 }
2425                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2426                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2427                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2428
2429                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2430                         switch (mac_index) {
2431                         case 1:
2432                         case 3:
2433                                 pcs_control_status = nes_read_indexed(nesdev,
2434                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2435                                 break;
2436                         default:
2437                                 pcs_control_status = nes_read_indexed(nesdev,
2438                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2439                                 break;
2440                         }
2441                 } else {
2442                         pcs_control_status = nes_read_indexed(nesdev,
2443                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2444                         pcs_control_status = nes_read_indexed(nesdev,
2445                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2446                 }
2447
2448                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2449                                 mac_index, pcs_control_status);
2450                 if ((nesadapter->OneG_Mode) &&
2451                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2452                         u32temp = 0x01010000;
2453                         if (nesadapter->port_count > 2) {
2454                                 u32temp |= 0x02020000;
2455                         }
2456                         if ((pcs_control_status & u32temp)!= u32temp) {
2457                                 phy_data = 0;
2458                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2459                         }
2460                 } else {
2461                         switch (nesadapter->phy_type[mac_index]) {
2462                         case NES_PHY_TYPE_IRIS:
2463                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2464                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2465                                 u32temp = 20;
2466                                 do {
2467                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2468                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2469                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2470                                                 break;
2471                                         temp_phy_data = phy_data;
2472                                 } while (1);
2473                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2474                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2475                                 break;
2476
2477                         case NES_PHY_TYPE_ARGUS:
2478                         case NES_PHY_TYPE_SFP_D:
2479                                 /* clear the alarms */
2480                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2481                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2482                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2483                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2484                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2485                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2486                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2487                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2488                                 /* check link status */
2489                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2490                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2491
2492                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2493                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2494                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2495                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2496
2497                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2498
2499                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2500                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2501                                 break;
2502
2503                         case NES_PHY_TYPE_PUMA_1G:
2504                                 if (mac_index < 2)
2505                                         pcs_val = pcs_mask = 0x01010000;
2506                                 else
2507                                         pcs_val = pcs_mask = 0x02020000;
2508                                 /* fall through */
2509                         default:
2510                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2511                                 break;
2512                         }
2513                 }
2514
2515                 if (phy_data & 0x0004) {
2516                         if (wide_ppm_offset &&
2517                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2518                             (nesadapter->hw_rev != NE020_REV)) {
2519                                 cdr_ctrl = nes_read_indexed(nesdev,
2520                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2521                                                             mac_index * 0x200);
2522                                 nes_write_indexed(nesdev,
2523                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2524                                                   mac_index * 0x200,
2525                                                   cdr_ctrl | 0x000F0000);
2526                         }
2527                         nesadapter->mac_link_down[mac_index] = 0;
2528                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2529                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2530                                                 nesvnic->linkup);
2531                                 if (nesvnic->linkup == 0) {
2532                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2533                                                         nesvnic->netdev->name, nesvnic->netdev);
2534                                         if (netif_queue_stopped(nesvnic->netdev))
2535                                                 netif_start_queue(nesvnic->netdev);
2536                                         nesvnic->linkup = 1;
2537                                         netif_carrier_on(nesvnic->netdev);
2538                                 }
2539                         }
2540                 } else {
2541                         if (wide_ppm_offset &&
2542                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2543                             (nesadapter->hw_rev != NE020_REV)) {
2544                                 cdr_ctrl = nes_read_indexed(nesdev,
2545                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2546                                                             mac_index * 0x200);
2547                                 nes_write_indexed(nesdev,
2548                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2549                                                   mac_index * 0x200,
2550                                                   cdr_ctrl & 0xFFF0FFFF);
2551                         }
2552                         nesadapter->mac_link_down[mac_index] = 1;
2553                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2554                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2555                                                 nesvnic->linkup);
2556                                 if (nesvnic->linkup == 1) {
2557                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2558                                                         nesvnic->netdev->name, nesvnic->netdev);
2559                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2560                                                 netif_stop_queue(nesvnic->netdev);
2561                                         nesvnic->linkup = 0;
2562                                         netif_carrier_off(nesvnic->netdev);
2563                                 }
2564                         }
2565                 }
2566         }
2567
2568         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2569 }
2570
2571
2572
2573 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2574 {
2575         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2576
2577         napi_schedule(&nesvnic->napi);
2578 }
2579
2580
2581 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2582 * getting out of nic_ce_handler
2583 */
2584 #define MAX_RQES_TO_PROCESS     384
2585
2586 /**
2587  * nes_nic_ce_handler
2588  */
2589 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2590 {
2591         u64 u64temp;
2592         dma_addr_t bus_address;
2593         struct nes_hw_nic *nesnic;
2594         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2595         struct nes_adapter *nesadapter = nesdev->nesadapter;
2596         struct nes_hw_nic_rq_wqe *nic_rqe;
2597         struct nes_hw_nic_sq_wqe *nic_sqe;
2598         struct sk_buff *skb;
2599         struct sk_buff *rx_skb;
2600         __le16 *wqe_fragment_length;
2601         u32 head;
2602         u32 cq_size;
2603         u32 rx_pkt_size;
2604         u32 cqe_count=0;
2605         u32 cqe_errv;
2606         u32 cqe_misc;
2607         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2608         u16 vlan_tag;
2609         u16 pkt_type;
2610         u16 rqes_processed = 0;
2611         u8 sq_cqes = 0;
2612         u8 nes_use_lro = 0;
2613
2614         head = cq->cq_head;
2615         cq_size = cq->cq_size;
2616         cq->cqes_pending = 1;
2617         if (nesvnic->netdev->features & NETIF_F_LRO)
2618                 nes_use_lro = 1;
2619         do {
2620                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2621                                 NES_NIC_CQE_VALID) {
2622                         nesnic = &nesvnic->nic;
2623                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2624                         if (cqe_misc & NES_NIC_CQE_SQ) {
2625                                 sq_cqes++;
2626                                 wqe_fragment_index = 1;
2627                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2628                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2629                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2630                                 /* bump past the vlan tag */
2631                                 wqe_fragment_length++;
2632                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2633                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2634                                                         wqe_fragment_index * 2]);
2635                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2636                                                         wqe_fragment_index * 2])) << 32;
2637                                         bus_address = (dma_addr_t)u64temp;
2638                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2639                                                 pci_unmap_single(nesdev->pcidev,
2640                                                                 bus_address,
2641                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2642                                                                 PCI_DMA_TODEVICE);
2643                                         }
2644                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2645                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2646                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2647                                                                                 wqe_fragment_index * 2]);
2648                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2649                                                                                 + wqe_fragment_index * 2])) <<32;
2650                                                         bus_address = (dma_addr_t)u64temp;
2651                                                         pci_unmap_page(nesdev->pcidev,
2652                                                                         bus_address,
2653                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2654                                                                         PCI_DMA_TODEVICE);
2655                                                 } else
2656                                                         break;
2657                                         }
2658                                 }
2659                                 if (skb)
2660                                         dev_kfree_skb_any(skb);
2661                                 nesnic->sq_tail++;
2662                                 nesnic->sq_tail &= nesnic->sq_size-1;
2663                                 if (sq_cqes > 128) {
2664                                         barrier();
2665                                 /* restart the queue if it had been stopped */
2666                                 if (netif_queue_stopped(nesvnic->netdev))
2667                                         netif_wake_queue(nesvnic->netdev);
2668                                         sq_cqes = 0;
2669                                 }
2670                         } else {
2671                                 rqes_processed ++;
2672
2673                                 cq->rx_cqes_completed++;
2674                                 cq->rx_pkts_indicated++;
2675                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2676                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2677                                 /* Get the skb */
2678                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2679                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2680                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2681                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2682                                 pci_unmap_single(nesdev->pcidev, bus_address,
2683                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2684                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2685                                 /* rx_skb->len = rx_pkt_size; */
2686                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2687                                 skb_put(rx_skb, rx_pkt_size);
2688                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2689                                 nesnic->rq_tail++;
2690                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2691
2692                                 atomic_inc(&nesvnic->rx_skbs_needed);
2693                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2694                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2695                                                         cq->cq_number | (cqe_count << 16));
2696                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2697                                         nesdev->currcq_count += cqe_count;
2698                                         cqe_count = 0;
2699                                         nes_replenish_nic_rq(nesvnic);
2700                                 }
2701                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2702                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2703                                 rx_skb->ip_summed = CHECKSUM_NONE;
2704
2705                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2706                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2707                                         if ((cqe_errv &
2708                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2709                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2710                                                 if (nesvnic->rx_checksum_disabled == 0) {
2711                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2712                                                 }
2713                                         } else
2714                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2715                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2716                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2717
2718                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2719                                         if ((cqe_errv &
2720                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2721                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2722                                                 if (nesvnic->rx_checksum_disabled == 0) {
2723                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2724                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2725                                                                   nesvnic->netdev->name); */
2726                                                 }
2727                                         } else
2728                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2729                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2730                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2731                                         }
2732                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2733                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2734
2735                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2736                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2737                                                 rx_skb = NULL;
2738                                 }
2739                                 if (rx_skb == NULL)
2740                                         goto skip_rx_indicate0;
2741
2742
2743                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2744                                     (nesvnic->vlan_grp != NULL)) {
2745                                         vlan_tag = (u16)(le32_to_cpu(
2746                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2747                                                         >> 16);
2748                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2749                                                         nesvnic->netdev->name, vlan_tag);
2750                                         if (nes_use_lro)
2751                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2752                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2753                                         else
2754                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2755                                 } else {
2756                                         if (nes_use_lro)
2757                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2758                                         else
2759                                                 nes_netif_rx(rx_skb);
2760                                 }
2761
2762 skip_rx_indicate0:
2763                                 ;
2764                                 /* nesvnic->netstats.rx_packets++; */
2765                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2766                         }
2767
2768                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2769                         /* Accounting... */
2770                         cqe_count++;
2771                         if (++head >= cq_size)
2772                                 head = 0;
2773                         if (cqe_count == 255) {
2774                                 /* Replenish Nic CQ */
2775                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2776                                                 cq->cq_number | (cqe_count << 16));
2777                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2778                                 nesdev->currcq_count += cqe_count;
2779                                 cqe_count = 0;
2780                         }
2781
2782                         if (cq->rx_cqes_completed >= nesvnic->budget)
2783                                 break;
2784                 } else {
2785                         cq->cqes_pending = 0;
2786                         break;
2787                 }
2788
2789         } while (1);
2790
2791         if (nes_use_lro)
2792                 lro_flush_all(&nesvnic->lro_mgr);
2793         if (sq_cqes) {
2794                 barrier();
2795                 /* restart the queue if it had been stopped */
2796                 if (netif_queue_stopped(nesvnic->netdev))
2797                         netif_wake_queue(nesvnic->netdev);
2798         }
2799         cq->cq_head = head;
2800         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2801                         cq->cq_number, cqe_count, cq->cq_head); */
2802         cq->cqe_allocs_pending = cqe_count;
2803         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2804         {
2805                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2806                 nesdev->currcq_count += cqe_count;
2807                 nes_nic_tune_timer(nesdev);
2808         }
2809         if (atomic_read(&nesvnic->rx_skbs_needed))
2810                 nes_replenish_nic_rq(nesvnic);
2811 }
2812
2813
2814 /**
2815  * nes_cqp_ce_handler
2816  */
2817 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2818 {
2819         u64 u64temp;
2820         unsigned long flags;
2821         struct nes_hw_cqp *cqp = NULL;
2822         struct nes_cqp_request *cqp_request;
2823         struct nes_hw_cqp_wqe *cqp_wqe;
2824         u32 head;
2825         u32 cq_size;
2826         u32 cqe_count=0;
2827         u32 error_code;
2828         /* u32 counter; */
2829
2830         head = cq->cq_head;
2831         cq_size = cq->cq_size;
2832
2833         do {
2834                 /* process the CQE */
2835                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2836                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2837
2838                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2839                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2840                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2841                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2842                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2843                         cqp = *((struct nes_hw_cqp **)&u64temp);
2844
2845                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2846                         if (error_code) {
2847                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2848                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2849                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2850                                                 (u16)(error_code >> 16),
2851                                                 (u16)error_code);
2852                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2853                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2854                         }
2855
2856                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2857                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2858                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2859                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2860                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2861                         if (cqp_request) {
2862                                 if (cqp_request->waiting) {
2863                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2864                                         cqp_request->major_code = (u16)(error_code >> 16);
2865                                         cqp_request->minor_code = (u16)error_code;
2866                                         barrier();
2867                                         cqp_request->request_done = 1;
2868                                         wake_up(&cqp_request->waitq);
2869                                         nes_put_cqp_request(nesdev, cqp_request);
2870                                 } else {
2871                                         if (cqp_request->callback)
2872                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2873                                         nes_free_cqp_request(nesdev, cqp_request);
2874                                 }
2875                         } else {
2876                                 wake_up(&nesdev->cqp.waitq);
2877                         }
2878
2879                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2880                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2881                         if (++cqp->sq_tail >= cqp->sq_size)
2882                                 cqp->sq_tail = 0;
2883
2884                         /* Accounting... */
2885                         cqe_count++;
2886                         if (++head >= cq_size)
2887                                 head = 0;
2888                 } else {
2889                         break;
2890                 }
2891         } while (1);
2892         cq->cq_head = head;
2893
2894         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2895         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2896                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2897                         (nesdev->cqp.sq_size - 1)) != 1)) {
2898                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2899                                 struct nes_cqp_request, list);
2900                 list_del_init(&cqp_request->list);
2901                 head = nesdev->cqp.sq_head++;
2902                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2903                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2904                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2905                 barrier();
2906                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2907                         cpu_to_le32((u32)((unsigned long)cqp_request));
2908                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2909                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2910                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2911                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2912                 /* Ring doorbell (1 WQEs) */
2913                 barrier();
2914                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2915         }
2916         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2917
2918         /* Arm the CCQ */
2919         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2920                         cq->cq_number);
2921         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2922 }
2923
2924
2925 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
2926 {
2927         u16 pkt_len;
2928
2929         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
2930                 /* skip over ethernet header */
2931                 pkt_len = be16_to_cpu(*(u16 *)(pkt + ETH_HLEN - 2));
2932                 pkt += ETH_HLEN;
2933
2934                 /* Skip over IP and TCP headers */
2935                 pkt += 4 * (pkt[0] & 0x0f);
2936                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
2937         }
2938         return pkt;
2939 }
2940
2941 /* Determine if incoming error pkt is rdma layer */
2942 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
2943 {
2944         u8 *pkt;
2945         u16 *mpa;
2946         u32 opcode = 0xffffffff;
2947
2948         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
2949                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
2950                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
2951                 opcode = be16_to_cpu(mpa[1]) & 0xf;
2952         }
2953
2954         return opcode;
2955 }
2956
2957 /* Build iWARP terminate header */
2958 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
2959 {
2960         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
2961         u16 ddp_seg_len;
2962         int copy_len = 0;
2963         u8 is_tagged = 0;
2964         u8 flush_code = 0;
2965         struct nes_terminate_hdr *termhdr;
2966
2967         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
2968         memset(termhdr, 0, 64);
2969
2970         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
2971
2972                 /* Use data from offending packet to fill in ddp & rdma hdrs */
2973                 pkt = locate_mpa(pkt, aeq_info);
2974                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
2975                 if (ddp_seg_len) {
2976                         copy_len = 2;
2977                         termhdr->hdrct = DDP_LEN_FLAG;
2978                         if (pkt[2] & 0x80) {
2979                                 is_tagged = 1;
2980                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
2981                                         copy_len += TERM_DDP_LEN_TAGGED;
2982                                         termhdr->hdrct |= DDP_HDR_FLAG;
2983                                 }
2984                         } else {
2985                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
2986                                         copy_len += TERM_DDP_LEN_UNTAGGED;
2987                                         termhdr->hdrct |= DDP_HDR_FLAG;
2988                                 }
2989
2990                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
2991                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
2992                                                 copy_len += TERM_RDMA_LEN;
2993                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
2994                                         }
2995                                 }
2996                         }
2997                 }
2998         }
2999
3000         switch (async_event_id) {
3001         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3002                 switch (iwarp_opcode(nesqp, aeq_info)) {
3003                 case IWARP_OPCODE_WRITE:
3004                         flush_code = IB_WC_LOC_PROT_ERR;
3005                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3006                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3007                         break;
3008                 default:
3009                         flush_code = IB_WC_REM_ACCESS_ERR;
3010                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3011                         termhdr->error_code = RDMAP_INV_STAG;
3012                 }
3013                 break;
3014         case NES_AEQE_AEID_AMP_INVALID_STAG:
3015                 flush_code = IB_WC_REM_ACCESS_ERR;
3016                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3017                 termhdr->error_code = RDMAP_INV_STAG;
3018                 break;
3019         case NES_AEQE_AEID_AMP_BAD_QP:
3020                 flush_code = IB_WC_LOC_QP_OP_ERR;
3021                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3022                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3023                 break;
3024         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3025         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3026                 switch (iwarp_opcode(nesqp, aeq_info)) {
3027                 case IWARP_OPCODE_SEND_INV:
3028                 case IWARP_OPCODE_SEND_SE_INV:
3029                         flush_code = IB_WC_REM_OP_ERR;
3030                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3031                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3032                         break;
3033                 default:
3034                         flush_code = IB_WC_REM_ACCESS_ERR;
3035                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3036                         termhdr->error_code = RDMAP_INV_STAG;
3037                 }
3038                 break;
3039         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3040                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3041                         flush_code = IB_WC_LOC_PROT_ERR;
3042                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3043                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3044                 } else {
3045                         flush_code = IB_WC_REM_ACCESS_ERR;
3046                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3047                         termhdr->error_code = RDMAP_INV_BOUNDS;
3048                 }
3049                 break;
3050         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3051         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3052         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3053                 flush_code = IB_WC_REM_ACCESS_ERR;
3054                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3055                 termhdr->error_code = RDMAP_ACCESS;
3056                 break;
3057         case NES_AEQE_AEID_AMP_TO_WRAP:
3058                 flush_code = IB_WC_REM_ACCESS_ERR;
3059                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3060                 termhdr->error_code = RDMAP_TO_WRAP;
3061                 break;
3062         case NES_AEQE_AEID_AMP_BAD_PD:
3063                 switch (iwarp_opcode(nesqp, aeq_info)) {
3064                 case IWARP_OPCODE_WRITE:
3065                         flush_code = IB_WC_LOC_PROT_ERR;
3066                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3067                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3068                         break;
3069                 case IWARP_OPCODE_SEND_INV:
3070                 case IWARP_OPCODE_SEND_SE_INV:
3071                         flush_code = IB_WC_REM_ACCESS_ERR;
3072                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3073                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3074                         break;
3075                 default:
3076                         flush_code = IB_WC_REM_ACCESS_ERR;
3077                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3078                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3079                 }
3080                 break;
3081         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3082                 flush_code = IB_WC_LOC_LEN_ERR;
3083                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3084                 termhdr->error_code = MPA_MARKER;
3085                 break;
3086         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3087                 flush_code = IB_WC_GENERAL_ERR;
3088                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3089                 termhdr->error_code = MPA_CRC;
3090                 break;
3091         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3092         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3093                 flush_code = IB_WC_LOC_LEN_ERR;
3094                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3095                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3096                 break;
3097         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3098         case NES_AEQE_AEID_DDP_NO_L_BIT:
3099                 flush_code = IB_WC_FATAL_ERR;
3100                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3101                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3102                 break;
3103         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3104         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3105                 flush_code = IB_WC_GENERAL_ERR;
3106                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3107                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3108                 break;
3109         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3110                 flush_code = IB_WC_LOC_LEN_ERR;
3111                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3112                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3113                 break;
3114         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3115                 flush_code = IB_WC_GENERAL_ERR;
3116                 if (is_tagged) {
3117                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3118                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3119                 } else {
3120                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3121                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3122                 }
3123                 break;
3124         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3125                 flush_code = IB_WC_GENERAL_ERR;
3126                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3127                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3128                 break;
3129         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3130                 flush_code = IB_WC_REM_OP_ERR;
3131                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3132                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3133                 break;
3134         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3135                 flush_code = IB_WC_GENERAL_ERR;
3136                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3137                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3138                 break;
3139         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3140                 flush_code = IB_WC_GENERAL_ERR;
3141                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3142                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3143                 break;
3144         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3145                 flush_code = IB_WC_LOC_QP_OP_ERR;
3146                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3147                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3148                 break;
3149         default:
3150                 flush_code = IB_WC_FATAL_ERR;
3151                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3152                 termhdr->error_code = RDMAP_UNSPECIFIED;
3153                 break;
3154         }
3155
3156         if (copy_len)
3157                 memcpy(termhdr + 1, pkt, copy_len);
3158
3159         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3160                 if (aeq_info & NES_AEQE_SQ)
3161                         nesqp->term_sq_flush_code = flush_code;
3162                 else
3163                         nesqp->term_rq_flush_code = flush_code;
3164         }
3165
3166         return sizeof(struct nes_terminate_hdr) + copy_len;
3167 }
3168
3169 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3170                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3171 {
3172         u64 context;
3173         unsigned long flags;
3174         u32 aeq_info;
3175         u16 async_event_id;
3176         u8 tcp_state;
3177         u8 iwarp_state;
3178         u32 termlen = 0;
3179         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3180                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3181         struct nes_adapter *nesadapter = nesdev->nesadapter;
3182
3183         if (nesqp->term_flags & NES_TERM_SENT)
3184                 return; /* Sanity check */
3185
3186         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3187         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3188         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3189         async_event_id = (u16)aeq_info;
3190
3191         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3192                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3193         if (!context) {
3194                 WARN_ON(!context);
3195                 return;
3196         }
3197
3198         nesqp = (struct nes_qp *)(unsigned long)context;
3199         spin_lock_irqsave(&nesqp->lock, flags);
3200         nesqp->hw_iwarp_state = iwarp_state;
3201         nesqp->hw_tcp_state = tcp_state;
3202         nesqp->last_aeq = async_event_id;
3203         nesqp->terminate_eventtype = eventtype;
3204         spin_unlock_irqrestore(&nesqp->lock, flags);
3205
3206         if (nesadapter->send_term_ok)
3207                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3208         else
3209                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3210
3211         nes_terminate_start_timer(nesqp);
3212         nesqp->term_flags |= NES_TERM_SENT;
3213         nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3214 }
3215
3216 static void nes_terminate_send_fin(struct nes_device *nesdev,
3217                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3218 {
3219         u32 aeq_info;
3220         u16 async_event_id;
3221         u8 tcp_state;
3222         u8 iwarp_state;
3223         unsigned long flags;
3224
3225         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3226         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3227         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3228         async_event_id = (u16)aeq_info;
3229
3230         spin_lock_irqsave(&nesqp->lock, flags);
3231         nesqp->hw_iwarp_state = iwarp_state;
3232         nesqp->hw_tcp_state = tcp_state;
3233         nesqp->last_aeq = async_event_id;
3234         spin_unlock_irqrestore(&nesqp->lock, flags);
3235
3236         /* Send the fin only */
3237         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3238                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3239 }
3240
3241 /* Cleanup after a terminate sent or received */
3242 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3243 {
3244         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3245         unsigned long flags;
3246         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3247         struct nes_device *nesdev = nesvnic->nesdev;
3248         u8 first_time = 0;
3249
3250         spin_lock_irqsave(&nesqp->lock, flags);
3251         if (nesqp->hte_added) {
3252                 nesqp->hte_added = 0;
3253                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3254         }
3255
3256         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3257         nesqp->term_flags |= NES_TERM_DONE;
3258         spin_unlock_irqrestore(&nesqp->lock, flags);
3259
3260         /* Make sure we go through this only once */
3261         if (first_time) {
3262                 if (timeout_occurred == 0)
3263                         del_timer(&nesqp->terminate_timer);
3264                 else
3265                         next_iwarp_state |= NES_CQP_QP_RESET;
3266
3267                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3268                 nes_cm_disconn(nesqp);
3269         }
3270 }
3271
3272 static void nes_terminate_received(struct nes_device *nesdev,
3273                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3274 {
3275         u32 aeq_info;
3276         u8 *pkt;
3277         u32 *mpa;
3278         u8 ddp_ctl;
3279         u8 rdma_ctl;
3280         u16 aeq_id = 0;
3281
3282         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3283         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3284                 /* Terminate is not a performance path so the silicon */
3285                 /* did not validate the frame - do it now */
3286                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3287                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3288                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3289                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3290                 if ((ddp_ctl & 0xc0) != 0x40)
3291                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3292                 else if ((ddp_ctl & 0x03) != 1)
3293                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3294                 else if (be32_to_cpu(mpa[2]) != 2)
3295                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3296                 else if (be32_to_cpu(mpa[3]) != 1)
3297                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3298                 else if (be32_to_cpu(mpa[4]) != 0)
3299                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3300                 else if ((rdma_ctl & 0xc0) != 0x40)
3301                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3302
3303                 if (aeq_id) {
3304                         /* Bad terminate recvd - send back a terminate */
3305                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3306                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3307                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3308                         return;
3309                 }
3310         }
3311
3312         nesqp->term_flags |= NES_TERM_RCVD;
3313         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3314         nes_terminate_start_timer(nesqp);
3315         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3316 }
3317
3318 /* Timeout routine in case terminate fails to complete */
3319 static void nes_terminate_timeout(unsigned long context)
3320 {
3321         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3322
3323         nes_terminate_done(nesqp, 1);
3324 }
3325
3326 /* Set a timer in case hw cannot complete the terminate sequence */
3327 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3328 {
3329         init_timer(&nesqp->terminate_timer);
3330         nesqp->terminate_timer.function = nes_terminate_timeout;
3331         nesqp->terminate_timer.expires = jiffies + HZ;
3332         nesqp->terminate_timer.data = (unsigned long)nesqp;
3333         add_timer(&nesqp->terminate_timer);
3334 }
3335
3336 /**
3337  * nes_process_iwarp_aeqe
3338  */
3339 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3340                                    struct nes_hw_aeqe *aeqe)
3341 {
3342         u64 context;
3343         unsigned long flags;
3344         struct nes_qp *nesqp;
3345         struct nes_hw_cq *hw_cq;
3346         struct nes_cq *nescq;
3347         int resource_allocated;
3348         struct nes_adapter *nesadapter = nesdev->nesadapter;
3349         u32 aeq_info;
3350         u32 next_iwarp_state = 0;
3351         u16 async_event_id;
3352         u8 tcp_state;
3353         u8 iwarp_state;
3354         int must_disconn = 1;
3355         int must_terminate = 0;
3356         struct ib_event ibevent;
3357
3358         nes_debug(NES_DBG_AEQ, "\n");
3359         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3360         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3361                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3362                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3363         } else {
3364                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3365                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3366                 BUG_ON(!context);
3367         }
3368
3369         async_event_id = (u16)aeq_info;
3370         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3371         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3372         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3373                         " Tcp state = %s, iWARP state = %s\n",
3374                         async_event_id,
3375                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3376                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3377
3378         switch (async_event_id) {
3379                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3380                         nesqp = (struct nes_qp *)(unsigned long)context;
3381
3382                         if (nesqp->term_flags)
3383                                 return; /* Ignore it, wait for close complete */
3384
3385                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3386                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3387                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3388                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3389                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3390                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3391                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3392                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3393                                                 async_event_id, nesqp->last_aeq, tcp_state);
3394                         }
3395
3396                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3397                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
3398                                 /* FIN Received but tcp state or IB state moved on,
3399                                                 should expect a close complete */
3400                                 return;
3401                         }
3402
3403                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3404                         nesqp = (struct nes_qp *)(unsigned long)context;
3405                         if (nesqp->term_flags) {
3406                                 nes_terminate_done(nesqp, 0);
3407                                 return;
3408                         }
3409
3410                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3411                 case NES_AEQE_AEID_RESET_SENT:
3412                         nesqp = (struct nes_qp *)(unsigned long)context;
3413                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
3414                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3415                         }
3416                         spin_lock_irqsave(&nesqp->lock, flags);
3417                         nesqp->hw_iwarp_state = iwarp_state;
3418                         nesqp->hw_tcp_state = tcp_state;
3419                         nesqp->last_aeq = async_event_id;
3420
3421                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
3422                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
3423                                 nesqp->hte_added = 0;
3424                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3425                         }
3426
3427                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
3428                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3429                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
3430                                 switch (nesqp->hw_iwarp_state) {
3431                                         case NES_AEQE_IWARP_STATE_RTS:
3432                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3433                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3434                                                 break;
3435                                         case NES_AEQE_IWARP_STATE_TERMINATE:
3436                                                 must_disconn = 0; /* terminate path takes care of disconn */
3437                                                 if (nesqp->term_flags == 0)
3438                                                         must_terminate = 1;
3439                                                 break;
3440                                 }
3441                         } else {
3442                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
3443                                         /* FIN Received but ib state not RTS,
3444                                                         close complete will be on its way */
3445                                         must_disconn = 0;
3446                                 }
3447                         }
3448                         spin_unlock_irqrestore(&nesqp->lock, flags);
3449
3450                         if (must_terminate)
3451                                 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3452                         else if (must_disconn) {
3453                                 if (next_iwarp_state) {
3454                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X\n",
3455                                                   nesqp->hwqp.qp_id, next_iwarp_state);
3456                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3457                                 }
3458                                 nes_cm_disconn(nesqp);
3459                         }
3460                         break;
3461
3462                 case NES_AEQE_AEID_TERMINATE_SENT:
3463                         nesqp = (struct nes_qp *)(unsigned long)context;
3464                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3465                         break;
3466
3467                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3468                         nesqp = (struct nes_qp *)(unsigned long)context;
3469                         nes_terminate_received(nesdev, nesqp, aeqe);
3470                         break;
3471
3472                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3473                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3474                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3475                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3476                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3477                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3478                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3479                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3480                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3481                 case NES_AEQE_AEID_AMP_TO_WRAP:
3482                         nesqp = (struct nes_qp *)(unsigned long)context;
3483                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3484                         break;
3485
3486                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3487                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3488                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3489                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3490                         nesqp = (struct nes_qp *)(unsigned long)context;
3491                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3492                                 aeq_info &= 0xffff0000;
3493                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3494                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3495                         }
3496
3497                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3498                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3499                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3500                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3501                 case NES_AEQE_AEID_AMP_BAD_QP:
3502                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3503                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3504                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3505                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3506                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3507                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3508                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3509                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3510                 case NES_AEQE_AEID_AMP_BAD_PD:
3511                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3512                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3513                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3514                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3515                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3516                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3517                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3518                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3519                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3520                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3521                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3522                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3523                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3524                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3525                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3526                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3527                 case NES_AEQE_AEID_BAD_CLOSE:
3528                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3529                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3530                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3531                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3532                         nesqp = (struct nes_qp *)(unsigned long)context;
3533                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3534                         break;
3535
3536                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3537                         context <<= 1;
3538                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3539                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3540                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3541                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3542                         if (resource_allocated) {
3543                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3544                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3545                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3546                                 if (hw_cq) {
3547                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3548                                         if (nescq->ibcq.event_handler) {
3549                                                 ibevent.device = nescq->ibcq.device;
3550                                                 ibevent.event = IB_EVENT_CQ_ERR;
3551                                                 ibevent.element.cq = &nescq->ibcq;
3552                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3553                                         }
3554                                 }
3555                         }
3556                         break;
3557
3558                 default:
3559                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3560                                         async_event_id);
3561                         break;
3562         }
3563
3564 }
3565
3566 /**
3567  * nes_iwarp_ce_handler
3568  */
3569 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3570 {
3571         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3572
3573         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3574                         nescq->hw_cq.cq_number); */
3575         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3576
3577         if (nescq->ibcq.comp_handler)
3578                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3579
3580         return;
3581 }
3582
3583
3584 /**
3585  * nes_manage_apbvt()
3586  */
3587 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3588                 u32 nic_index, u32 add_port)
3589 {
3590         struct nes_device *nesdev = nesvnic->nesdev;
3591         struct nes_hw_cqp_wqe *cqp_wqe;
3592         struct nes_cqp_request *cqp_request;
3593         int ret = 0;
3594         u16 major_code;
3595
3596         /* Send manage APBVT request to CQP */
3597         cqp_request = nes_get_cqp_request(nesdev);
3598         if (cqp_request == NULL) {
3599                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3600                 return -ENOMEM;
3601         }
3602         cqp_request->waiting = 1;
3603         cqp_wqe = &cqp_request->cqp_wqe;
3604
3605         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3606                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3607                         accel_local_port, accel_local_port, nic_index);
3608
3609         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3610         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3611                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3612         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3613                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3614
3615         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3616
3617         atomic_set(&cqp_request->refcount, 2);
3618         nes_post_cqp_request(nesdev, cqp_request);
3619
3620         if (add_port == NES_MANAGE_APBVT_ADD)
3621                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3622                                 NES_EVENT_TIMEOUT);
3623         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3624                         ret, cqp_request->major_code, cqp_request->minor_code);
3625         major_code = cqp_request->major_code;
3626
3627         nes_put_cqp_request(nesdev, cqp_request);
3628
3629         if (!ret)
3630                 return -ETIME;
3631         else if (major_code)
3632                 return -EIO;
3633         else
3634                 return 0;
3635 }
3636
3637
3638 /**
3639  * nes_manage_arp_cache
3640  */
3641 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3642                 u32 ip_addr, u32 action)
3643 {
3644         struct nes_hw_cqp_wqe *cqp_wqe;
3645         struct nes_vnic *nesvnic = netdev_priv(netdev);
3646         struct nes_device *nesdev;
3647         struct nes_cqp_request *cqp_request;
3648         int arp_index;
3649
3650         nesdev = nesvnic->nesdev;
3651         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3652         if (arp_index == -1) {
3653                 return;
3654         }
3655
3656         /* update the ARP entry */
3657         cqp_request = nes_get_cqp_request(nesdev);
3658         if (cqp_request == NULL) {
3659                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3660                 return;
3661         }
3662         cqp_request->waiting = 0;
3663         cqp_wqe = &cqp_request->cqp_wqe;
3664         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3665
3666         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3667                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3668         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3669                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3670         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3671
3672         if (action == NES_ARP_ADD) {
3673                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3674                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3675                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3676                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3677                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3678                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3679         } else {
3680                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3681                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3682         }
3683
3684         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3685                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3686
3687         atomic_set(&cqp_request->refcount, 1);
3688         nes_post_cqp_request(nesdev, cqp_request);
3689 }
3690
3691
3692 /**
3693  * flush_wqes
3694  */
3695 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3696                 u32 which_wq, u32 wait_completion)
3697 {
3698         struct nes_cqp_request *cqp_request;
3699         struct nes_hw_cqp_wqe *cqp_wqe;
3700         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3701         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3702         int ret;
3703
3704         cqp_request = nes_get_cqp_request(nesdev);
3705         if (cqp_request == NULL) {
3706                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3707                 return;
3708         }
3709         if (wait_completion) {
3710                 cqp_request->waiting = 1;
3711                 atomic_set(&cqp_request->refcount, 2);
3712         } else {
3713                 cqp_request->waiting = 0;
3714         }
3715         cqp_wqe = &cqp_request->cqp_wqe;
3716         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3717
3718         /* If wqe in error was identified, set code to be put into cqe */
3719         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3720                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3721                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3722                 nesqp->term_sq_flush_code = 0;
3723         }
3724
3725         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3726                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3727                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3728                 nesqp->term_rq_flush_code = 0;
3729         }
3730
3731         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3732                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3733                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3734         }
3735
3736         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3737                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3738         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3739
3740         nes_post_cqp_request(nesdev, cqp_request);
3741
3742         if (wait_completion) {
3743                 /* Wait for CQP */
3744                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3745                                 NES_EVENT_TIMEOUT);
3746                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3747                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3748                                 ret, cqp_request->major_code, cqp_request->minor_code);
3749                 nes_put_cqp_request(nesdev, cqp_request);
3750         }
3751 }