drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2011 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/slab.h>
  40
  41 #include "nes.h"
  42
  43 static int wide_ppm_offset;
  44 module_param(wide_ppm_offset, int, 0644);
  45 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  46
  47 static u32 crit_err_count;
  48 u32 int_mod_timer_init;
  49 u32 int_mod_cq_depth_256;
  50 u32 int_mod_cq_depth_128;
  51 u32 int_mod_cq_depth_32;
  52 u32 int_mod_cq_depth_24;
  53 u32 int_mod_cq_depth_16;
  54 u32 int_mod_cq_depth_4;
  55 u32 int_mod_cq_depth_1;
  56 static const u8 nes_max_critical_error_count = 100;
  57 #include "nes_cm.h"
  58
  59 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  60 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  61 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  62                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  63 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  64 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  65 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  66 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  67                                    struct nes_hw_aeqe *aeqe);
  68 static void process_critical_error(struct nes_device *nesdev);
  69 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  70 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  71 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  72
  73 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  74 static unsigned char *nes_iwarp_state_str[] = {
  75         "Non-Existent",
  76         "Idle",
  77         "RTS",
  78         "Closing",
  79         "RSVD1",
  80         "Terminate",
  81         "Error",
  82         "RSVD2",
  83 };
  84
  85 static unsigned char *nes_tcp_state_str[] = {
  86         "Non-Existent",
  87         "Closed",
  88         "Listen",
  89         "SYN Sent",
  90         "SYN Rcvd",
  91         "Established",
  92         "Close Wait",
  93         "FIN Wait 1",
  94         "Closing",
  95         "Last Ack",
  96         "FIN Wait 2",
  97         "Time Wait",
  98         "RSVD1",
  99         "RSVD2",
 100         "RSVD3",
 101         "RSVD4",
 102 };
 103 #endif
 104
 105 static inline void print_ip(struct nes_cm_node *cm_node)
 106 {
 107         unsigned char *rem_addr;
 108         if (cm_node) {
 109                 rem_addr = (unsigned char *)&cm_node->rem_addr;
 110                 printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
 111         }
 112 }
 113
 114 /**
 115  * nes_nic_init_timer_defaults
 116  */
 117 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 118 {
 119         unsigned long flags;
 120         struct nes_adapter *nesadapter = nesdev->nesadapter;
 121         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 122
 123         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 124
 125         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 126         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 127         if (jumbomode) {
 128                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 129                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 130                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 131         } else {
 132                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 133                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 134                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 135         }
 136
 137         /* todo use netdev->mtu to set thresholds */
 138         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 139 }
 140
 141
 142 /**
 143  * nes_nic_init_timer
 144  */
 145 static void  nes_nic_init_timer(struct nes_device *nesdev)
 146 {
 147         unsigned long flags;
 148         struct nes_adapter *nesadapter = nesdev->nesadapter;
 149         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 150
 151         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 152
 153         if (shared_timer->timer_in_use_old == 0) {
 154                 nesdev->deepcq_count = 0;
 155                 shared_timer->timer_direction_upward = 0;
 156                 shared_timer->timer_direction_downward = 0;
 157                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 158                 shared_timer->timer_in_use_old = 0;
 159
 160         }
 161         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 162                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 163                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 164                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 165         }
 166         /* todo use netdev->mtu to set thresholds */
 167         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 168 }
 169
 170
 171 /**
 172  * nes_nic_tune_timer
 173  */
 174 static void nes_nic_tune_timer(struct nes_device *nesdev)
 175 {
 176         unsigned long flags;
 177         struct nes_adapter *nesadapter = nesdev->nesadapter;
 178         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 179         u16 cq_count = nesdev->currcq_count;
 180
 181         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 182
 183         if (shared_timer->cq_count_old <= cq_count)
 184                 shared_timer->cq_direction_downward = 0;
 185         else
 186                 shared_timer->cq_direction_downward++;
 187         shared_timer->cq_count_old = cq_count;
 188         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 189                 if (cq_count <= shared_timer->threshold_low &&
 190                     shared_timer->threshold_low > 4) {
 191                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 192                         shared_timer->cq_direction_downward=0;
 193                         nesdev->currcq_count = 0;
 194                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 195                         return;
 196                 }
 197         }
 198
 199         if (cq_count > 1) {
 200                 nesdev->deepcq_count += cq_count;
 201                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 202                         shared_timer->timer_direction_upward++;
 203                         shared_timer->timer_direction_downward = 0;
 204                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 205                         shared_timer->timer_direction_upward = 0;
 206                         shared_timer->timer_direction_downward = 0;
 207                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 208                         shared_timer->timer_direction_downward++;
 209                         shared_timer->timer_direction_upward = 0;
 210                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 211                         shared_timer->timer_in_use -= 2;
 212                         shared_timer->timer_direction_upward = 0;
 213                         shared_timer->timer_direction_downward++;
 214                 } else {
 215                         shared_timer->timer_in_use -= 4;
 216                         shared_timer->timer_direction_upward = 0;
 217                         shared_timer->timer_direction_downward++;
 218                 }
 219
 220                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 221                         shared_timer->timer_in_use += 3;
 222                         shared_timer->timer_direction_upward = 0;
 223                         shared_timer->timer_direction_downward = 0;
 224                 }
 225                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 226                         shared_timer->timer_in_use -= 4 ;
 227                         shared_timer->timer_direction_downward = 0;
 228                         shared_timer->timer_direction_upward = 0;
 229                 }
 230         }
 231
 232         /* boundary checking */
 233         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 234                 shared_timer->timer_in_use = shared_timer->threshold_high;
 235         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 236                 shared_timer->timer_in_use = shared_timer->threshold_low;
 237
 238         nesdev->currcq_count = 0;
 239
 240         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 241 }
 242
 243
 244 /**
 245  * nes_init_adapter - initialize adapter
 246  */
 247 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 248         struct nes_adapter *nesadapter = NULL;
 249         unsigned long num_pds;
 250         u32 u32temp;
 251         u32 port_count;
 252         u16 max_rq_wrs;
 253         u16 max_sq_wrs;
 254         u32 max_mr;
 255         u32 max_256pbl;
 256         u32 max_4kpbl;
 257         u32 max_qp;
 258         u32 max_irrq;
 259         u32 max_cq;
 260         u32 hte_index_mask;
 261         u32 adapter_size;
 262         u32 arp_table_size;
 263         u16 vendor_id;
 264         u16 device_id;
 265         u8  OneG_Mode;
 266         u8  func_index;
 267
 268         /* search the list of existing adapters */
 269         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 270                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 271                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 272                                 nesdev->pcidev->devfn,
 273                                 PCI_SLOT(nesadapter->devfn),
 274                                 nesadapter->bus_number,
 275                                 PCI_SLOT(nesdev->pcidev->devfn),
 276                                 nesdev->pcidev->bus->number );
 277                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 278                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 279                         nesadapter->ref_count++;
 280                         return nesadapter;
 281                 }
 282         }
 283
 284         /* no adapter found */
 285         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 286         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 287                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 288                                 hw_rev);
 289                 return NULL;
 290         }
 291
 292         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 293                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 294                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 295                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 296                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 297
 298         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 299
 300
 301         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 302                 return NULL;
 303
 304         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 305         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 306
 307         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 308         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 309                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 310                                 max_qp, u32temp);
 311                 max_qp = (u32)1 << (u32temp & 0x001f);
 312         }
 313
 314         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 315         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 316                         max_qp, hte_index_mask);
 317
 318         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 319
 320         max_irrq = 1 << (u32temp & 0x001f);
 321
 322         if (max_qp > max_irrq) {
 323                 max_qp = max_irrq;
 324                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 325                                 max_qp);
 326         }
 327
 328         /* there should be no reason to allocate more pds than qps */
 329         if (num_pds > max_qp)
 330                 num_pds = max_qp;
 331
 332         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 333         max_mr = (u32)8192 << (u32temp & 0x7);
 334
 335         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 336         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 337         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 338         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 339
 340         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 341         arp_table_size = 1 << u32temp;
 342
 343         adapter_size = (sizeof(struct nes_adapter) +
 344                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 347         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 348         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 349         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 350         adapter_size += sizeof(struct nes_qp **) * max_qp;
 351
 352         /* allocate a new adapter struct */
 353         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 354         if (!nesadapter)
 355                 return NULL;
 356
 357         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 358                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 359
 360         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 361                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 362                 kfree(nesadapter);
 363                 return NULL;
 364         }
 365
 366         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 367                                 (nesadapter->mac_addr_low >> 24);
 368
 369         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 370                                  PCI_DEVICE_ID, &device_id);
 371         nesadapter->vendor_part_id = device_id;
 372
 373         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 374                                                         OneG_Mode)) {
 375                 kfree(nesadapter);
 376                 return NULL;
 377         }
 378         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 379
 380         memset(nesadapter->pft_mcast_map, 255,
 381                sizeof nesadapter->pft_mcast_map);
 382
 383         /* populate the new nesadapter */
 384         nesadapter->devfn = nesdev->pcidev->devfn;
 385         nesadapter->bus_number = nesdev->pcidev->bus->number;
 386         nesadapter->ref_count = 1;
 387         nesadapter->timer_int_req = 0xffff0000;
 388         nesadapter->OneG_Mode = OneG_Mode;
 389         nesadapter->doorbell_start = nesdev->doorbell_region;
 390
 391         /* nesadapter->tick_delta = clk_divisor; */
 392         nesadapter->hw_rev = hw_rev;
 393         nesadapter->port_count = port_count;
 394
 395         nesadapter->max_qp = max_qp;
 396         nesadapter->hte_index_mask = hte_index_mask;
 397         nesadapter->max_irrq = max_irrq;
 398         nesadapter->max_mr = max_mr;
 399         nesadapter->max_256pbl = max_256pbl - 1;
 400         nesadapter->max_4kpbl = max_4kpbl - 1;
 401         nesadapter->max_cq = max_cq;
 402         nesadapter->free_256pbl = max_256pbl - 1;
 403         nesadapter->free_4kpbl = max_4kpbl - 1;
 404         nesadapter->max_pd = num_pds;
 405         nesadapter->arp_table_size = arp_table_size;
 406
 407         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 408         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 409                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 410                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 411                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 412         } else {
 413                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 414                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 415                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 416                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 417         }
 418         /* Setup and enable the periodic timer */
 419         if (nesadapter->et_rx_coalesce_usecs_irq)
 420                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 421                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 422         else
 423                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 424
 425         nesadapter->base_pd = 1;
 426
 427         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 428                                        IB_DEVICE_MEM_WINDOW |
 429                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 430
 431         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 432                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 433         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 434         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 435         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 436         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 437         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 438
 439
 440         /* mark the usual suspect QPs, MR and CQs as in use */
 441         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 442                 set_bit(u32temp, nesadapter->allocated_qps);
 443                 set_bit(u32temp, nesadapter->allocated_cqs);
 444         }
 445         set_bit(0, nesadapter->allocated_mrs);
 446
 447         for (u32temp = 0; u32temp < 20; u32temp++)
 448                 set_bit(u32temp, nesadapter->allocated_pds);
 449         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 450
 451         max_rq_wrs = ((u32temp >> 8) & 3);
 452         switch (max_rq_wrs) {
 453                 case 0:
 454                         max_rq_wrs = 4;
 455                         break;
 456                 case 1:
 457                         max_rq_wrs = 16;
 458                         break;
 459                 case 2:
 460                         max_rq_wrs = 32;
 461                         break;
 462                 case 3:
 463                         max_rq_wrs = 512;
 464                         break;
 465         }
 466
 467         max_sq_wrs = (u32temp & 3);
 468         switch (max_sq_wrs) {
 469                 case 0:
 470                         max_sq_wrs = 4;
 471                         break;
 472                 case 1:
 473                         max_sq_wrs = 16;
 474                         break;
 475                 case 2:
 476                         max_sq_wrs = 32;
 477                         break;
 478                 case 3:
 479                         max_sq_wrs = 512;
 480                         break;
 481         }
 482         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 483         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 484
 485         nesadapter->max_sge = 4;
 486         nesadapter->max_cqe = 32766;
 487
 488         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 489                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 490                 kfree(nesadapter);
 491                 return NULL;
 492         }
 493
 494         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 495         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 496                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 497
 498         /* setup port configuration */
 499         if (nesadapter->port_count == 1) {
 500                 nesadapter->log_port = 0x00000000;
 501                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 502                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 503                 else
 504                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 505         } else {
 506                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 507                         nesadapter->log_port = 0x000000D8;
 508                 } else {
 509                         if (nesadapter->port_count == 2)
 510                                 nesadapter->log_port = 0x00000044;
 511                         else
 512                                 nesadapter->log_port = 0x000000e4;
 513                 }
 514                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 515         }
 516
 517         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 518                                                 nesadapter->log_port);
 519         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 520                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 521
 522         spin_lock_init(&nesadapter->resource_lock);
 523         spin_lock_init(&nesadapter->phy_lock);
 524         spin_lock_init(&nesadapter->pbl_lock);
 525         spin_lock_init(&nesadapter->periodic_timer_lock);
 526
 527         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 528         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 529         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 530         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 531
 532         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 533                 u32 pcs_control_status0, pcs_control_status1;
 534                 u32 reset_value;
 535                 u32 i = 0;
 536                 u32 int_cnt = 0;
 537                 u32 ext_cnt = 0;
 538                 unsigned long flags;
 539                 u32 j = 0;
 540
 541                 pcs_control_status0 = nes_read_indexed(nesdev,
 542                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 543                 pcs_control_status1 = nes_read_indexed(nesdev,
 544                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 545
 546                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 547                         pcs_control_status0 = nes_read_indexed(nesdev,
 548                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 549                         pcs_control_status1 = nes_read_indexed(nesdev,
 550                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 551                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 552                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 553                                 int_cnt++;
 554                         usleep_range(1000, 2000);
 555                 }
 556                 if (int_cnt > 1) {
 557                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 558                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 559                         mh_detected++;
 560                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 561                         reset_value |= 0x0000003d;
 562                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 563
 564                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 565                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 566                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 567
 568                         pcs_control_status0 = nes_read_indexed(nesdev,
 569                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 570                         pcs_control_status1 = nes_read_indexed(nesdev,
 571                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 572
 573                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 574                                 pcs_control_status0 = nes_read_indexed(nesdev,
 575                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 576                                 pcs_control_status1 = nes_read_indexed(nesdev,
 577                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 578                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 579                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 580                                         if (++ext_cnt > int_cnt) {
 581                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 582                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 583                                                                 0x0000F088);
 584                                                 mh_detected++;
 585                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 586                                                 reset_value |= 0x0000003d;
 587                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 588
 589                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 590                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 591                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 592                                                 break;
 593                                         }
 594                                 }
 595                                 usleep_range(1000, 2000);
 596                         }
 597                 }
 598         }
 599
 600         if (nesadapter->hw_rev == NE020_REV) {
 601                 init_timer(&nesadapter->mh_timer);
 602                 nesadapter->mh_timer.function = nes_mh_fix;
 603                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 604                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 605                 add_timer(&nesadapter->mh_timer);
 606         } else {
 607                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 608         }
 609
 610         init_timer(&nesadapter->lc_timer);
 611         nesadapter->lc_timer.function = nes_clc;
 612         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 613         nesadapter->lc_timer.data = (unsigned long)nesdev;
 614         add_timer(&nesadapter->lc_timer);
 615
 616         list_add_tail(&nesadapter->list, &nes_adapter_list);
 617
 618         for (func_index = 0; func_index < 8; func_index++) {
 619                 pci_bus_read_config_word(nesdev->pcidev->bus,
 620                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 621                                         func_index), 0, &vendor_id);
 622                 if (vendor_id == 0xffff)
 623                         break;
 624         }
 625         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 626                 func_index, pci_name(nesdev->pcidev));
 627         nesadapter->adapter_fcn_count = func_index;
 628
 629         return nesadapter;
 630 }
 631
 632
 633 /**
 634  * nes_reset_adapter_ne020
 635  */
 636 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 637 {
 638         u32 port_count;
 639         u32 u32temp;
 640         u32 i;
 641
 642         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 643         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 644         /* TODO: assuming that both SERDES are set the same for now */
 645         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 646         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 647                         u32temp, port_count);
 648         if (*OneG_Mode)
 649                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 650         u32temp &= 0xff00ffc0;
 651         switch (port_count) {
 652                 case 1:
 653                         u32temp |= 0x00ee0000;
 654                         break;
 655                 case 2:
 656                         u32temp |= 0x00cc0000;
 657                         break;
 658                 case 4:
 659                         u32temp |= 0x00000000;
 660                         break;
 661                 default:
 662                         return 0;
 663                         break;
 664         }
 665
 666         /* check and do full reset if needed */
 667         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 668                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 669                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 670
 671                 i = 0;
 672                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 673                         mdelay(1);
 674                 if (i > 10000) {
 675                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 676                         return 0;
 677                 }
 678
 679                 i = 0;
 680                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 681                         mdelay(1);
 682                 if (i > 10000) {
 683                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 684                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 685                         return 0;
 686                 }
 687         }
 688
 689         /* port reset */
 690         switch (port_count) {
 691                 case 1:
 692                         u32temp |= 0x00ee0010;
 693                         break;
 694                 case 2:
 695                         u32temp |= 0x00cc0030;
 696                         break;
 697                 case 4:
 698                         u32temp |= 0x00000030;
 699                         break;
 700         }
 701
 702         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 703         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 704
 705         i = 0;
 706         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 707                 mdelay(1);
 708         if (i > 10000) {
 709                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 710                 return 0;
 711         }
 712
 713         /* serdes 0 */
 714         i = 0;
 715         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 716                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 717                 mdelay(1);
 718         if (i > 5000) {
 719                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 720                 return 0;
 721         }
 722
 723         /* serdes 1 */
 724         if (port_count > 1) {
 725                 i = 0;
 726                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 727                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 728                         mdelay(1);
 729                 if (i > 5000) {
 730                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 731                         return 0;
 732                 }
 733         }
 734
 735         return port_count;
 736 }
 737
 738
 739 /**
 740  * nes_init_serdes
 741  */
 742 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 743                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 744 {
 745         int i;
 746         u32 u32temp;
 747         u32 sds;
 748
 749         if (hw_rev != NE020_REV) {
 750                 /* init serdes 0 */
 751                 switch (nesadapter->phy_type[0]) {
 752                 case NES_PHY_TYPE_CX4:
 753                         if (wide_ppm_offset)
 754                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 755                         else
 756                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 757                         break;
 758                 case NES_PHY_TYPE_KR:
 759                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 761                         break;
 762                 case NES_PHY_TYPE_PUMA_1G:
 763                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 764                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 765                         sds |= 0x00000100;
 766                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 767                         break;
 768                 default:
 769                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 770                         break;
 771                 }
 772
 773                 if (!OneG_Mode)
 774                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 775
 776                 if (port_count < 2)
 777                         return 0;
 778
 779                 /* init serdes 1 */
 780                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 781                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 782
 783                 switch (nesadapter->phy_type[1]) {
 784                 case NES_PHY_TYPE_ARGUS:
 785                 case NES_PHY_TYPE_SFP_D:
 786                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 787                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 788                         break;
 789                 case NES_PHY_TYPE_CX4:
 790                         if (wide_ppm_offset)
 791                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 792                         break;
 793                 case NES_PHY_TYPE_KR:
 794                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 795                         break;
 796                 case NES_PHY_TYPE_PUMA_1G:
 797                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 798                         sds |= 0x000000100;
 799                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 800                 }
 801                 if (!OneG_Mode) {
 802                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 803                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 804                         sds &= 0xFFFFFFBF;
 805                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 806                 }
 807         } else {
 808                 /* init serdes 0 */
 809                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 810                 i = 0;
 811                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 812                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 813                         mdelay(1);
 814                 if (i > 5000) {
 815                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 816                         return 1;
 817                 }
 818                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 819                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 820                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 821                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 822                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 823                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 824                 if (OneG_Mode)
 825                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 826                 else
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 828
 829                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 830                 if (port_count > 1) {
 831                         /* init serdes 1 */
 832                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 833                         i = 0;
 834                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 835                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 836                                 mdelay(1);
 837                         if (i > 5000) {
 838                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 839                                 /* return 1; */
 840                         }
 841                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 842                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 843                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 844                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 845                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 846                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 847                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 848                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 849                 }
 850         }
 851         return 0;
 852 }
 853
 854
 855 /**
 856  * nes_init_csr_ne020
 857  * Initialize registers for ne020 hardware
 858  */
 859 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 860 {
 861         u32 u32temp;
 862
 863         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 864
 865         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 866         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 867         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 868         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 869         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 870         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 871         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 872         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 873
 874         /* TODO: move these MAC register settings to NIC bringup */
 875         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 876         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 877         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 878         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 879         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 880         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 881         if (port_count > 1) {
 882                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 883                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 884                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 885                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 886                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 887                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 888                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 889         }
 890         if (port_count > 2) {
 891                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 892                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 893                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 894                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 895                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 896                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 897                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 898
 899                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 900                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 901                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 902                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 903                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 904                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 905                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 906         }
 907
 908         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 909         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 910         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 911                                                          0x00000001);
 912         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 913         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 914         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 915         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 916         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 917
 918         /* TODO: move this to code, get from EEPROM */
 919         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 920         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 921         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 922
 923         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 924         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 925
 926         if (hw_rev != NE020_REV) {
 927                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 928                 u32temp |= 0x80000000;
 929                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 930                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 931                 u32temp &= 0x7fffffff;
 932                 u32temp |= 0x7fff0010;
 933                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 934                 if (port_count > 1) {
 935                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 936                         u32temp &= 0x7fffffff;
 937                         u32temp |= 0x7fff0010;
 938                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 939                 }
 940         }
 941 }
 942
 943
 944 /**
 945  * nes_destroy_adapter - destroy the adapter structure
 946  */
 947 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 948 {
 949         struct nes_adapter *tmp_adapter;
 950
 951         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 952                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 953                                 tmp_adapter);
 954         }
 955
 956         nesadapter->ref_count--;
 957         if (!nesadapter->ref_count) {
 958                 if (nesadapter->hw_rev == NE020_REV) {
 959                         del_timer(&nesadapter->mh_timer);
 960                 }
 961                 del_timer(&nesadapter->lc_timer);
 962
 963                 list_del(&nesadapter->list);
 964                 kfree(nesadapter);
 965         }
 966 }
 967
 968
 969 /**
 970  * nes_init_cqp
 971  */
 972 int nes_init_cqp(struct nes_device *nesdev)
 973 {
 974         struct nes_adapter *nesadapter = nesdev->nesadapter;
 975         struct nes_hw_cqp_qp_context *cqp_qp_context;
 976         struct nes_hw_cqp_wqe *cqp_wqe;
 977         struct nes_hw_ceq *ceq;
 978         struct nes_hw_ceq *nic_ceq;
 979         struct nes_hw_aeq *aeq;
 980         void *vmem;
 981         dma_addr_t pmem;
 982         u32 count=0;
 983         u32 cqp_head;
 984         u64 u64temp;
 985         u32 u32temp;
 986
 987         /* allocate CQP memory */
 988         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 989         /* SQ is 512 byte aligned, others are 256 byte aligned */
 990         nesdev->cqp_mem_size = 512 +
 991                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 992                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 993                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 994                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 995                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 996                         sizeof(struct nes_hw_cqp_qp_context);
 997
 998         nesdev->cqp_vbase = pci_zalloc_consistent(nesdev->pcidev,
 999                                                   nesdev->cqp_mem_size,
1000                                                   &nesdev->cqp_pbase);
1001         if (!nesdev->cqp_vbase) {
1002                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1003                 return -ENOMEM;
1004         }
1005
1006         /* Allocate a twice the number of CQP requests as the SQ size */
1007         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1008                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1009         if (!nesdev->nes_cqp_requests) {
1010                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1011                                 nesdev->cqp.sq_pbase);
1012                 return -ENOMEM;
1013         }
1014
1015         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1016                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1017
1018         spin_lock_init(&nesdev->cqp.lock);
1019         init_waitqueue_head(&nesdev->cqp.waitq);
1020
1021         /* Setup Various Structures */
1022         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1023                         ~(unsigned long)(512 - 1));
1024         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1025                         ~(unsigned long long)(512 - 1));
1026
1027         nesdev->cqp.sq_vbase = vmem;
1028         nesdev->cqp.sq_pbase = pmem;
1029         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1030         nesdev->cqp.sq_head = 0;
1031         nesdev->cqp.sq_tail = 0;
1032         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1033
1034         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1035         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1036
1037         nesdev->ccq.cq_vbase = vmem;
1038         nesdev->ccq.cq_pbase = pmem;
1039         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1040         nesdev->ccq.cq_head = 0;
1041         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1042         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1043
1044         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1045         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1046
1047         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1048         ceq = &nesadapter->ceq[nesdev->ceq_index];
1049         ceq->ceq_vbase = vmem;
1050         ceq->ceq_pbase = pmem;
1051         ceq->ceq_size = NES_CCEQ_SIZE;
1052         ceq->ceq_head = 0;
1053
1054         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1055         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1056
1057         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1058         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1059         nic_ceq->ceq_vbase = vmem;
1060         nic_ceq->ceq_pbase = pmem;
1061         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1062         nic_ceq->ceq_head = 0;
1063
1064         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1065         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1066
1067         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1068         aeq->aeq_vbase = vmem;
1069         aeq->aeq_pbase = pmem;
1070         aeq->aeq_size = nesadapter->max_qp;
1071         aeq->aeq_head = 0;
1072
1073         /* Setup QP Context */
1074         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1075         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1076
1077         cqp_qp_context = vmem;
1078         cqp_qp_context->context_words[0] =
1079                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1080         cqp_qp_context->context_words[1] = 0;
1081         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1082         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1083
1084
1085         /* Write the address to Create CQP */
1086         if ((sizeof(dma_addr_t) > 4)) {
1087                 nes_write_indexed(nesdev,
1088                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1089                                 ((u64)pmem) >> 32);
1090         } else {
1091                 nes_write_indexed(nesdev,
1092                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1093         }
1094         nes_write_indexed(nesdev,
1095                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1096                         (u32)pmem);
1097
1098         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1099         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1100
1101         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1102                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1103                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1104         }
1105
1106         /* Write Create CCQ WQE */
1107         cqp_head = nesdev->cqp.sq_head++;
1108         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1109         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1110         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1111                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1112                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1113         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1114                             (nesdev->ccq.cq_number |
1115                              ((u32)nesdev->ceq_index << 16)));
1116         u64temp = (u64)nesdev->ccq.cq_pbase;
1117         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1118         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1119         u64temp = (unsigned long)&nesdev->ccq;
1120         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1121                         cpu_to_le32((u32)(u64temp >> 1));
1122         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1123                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1124         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1125
1126         /* Write Create CEQ WQE */
1127         cqp_head = nesdev->cqp.sq_head++;
1128         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1129         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1130         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1131                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1132         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1133         u64temp = (u64)ceq->ceq_pbase;
1134         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1135
1136         /* Write Create AEQ WQE */
1137         cqp_head = nesdev->cqp.sq_head++;
1138         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1139         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1140         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1141                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1142         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1143         u64temp = (u64)aeq->aeq_pbase;
1144         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1145
1146         /* Write Create NIC CEQ WQE */
1147         cqp_head = nesdev->cqp.sq_head++;
1148         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1149         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1150         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1151                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1152         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1153         u64temp = (u64)nic_ceq->ceq_pbase;
1154         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1155
1156         /* Poll until CCQP done */
1157         count = 0;
1158         do {
1159                 if (count++ > 1000) {
1160                         printk(KERN_ERR PFX "Error creating CQP\n");
1161                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1162                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1163                         return -1;
1164                 }
1165                 udelay(10);
1166         } while (!(nes_read_indexed(nesdev,
1167                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1168
1169         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1170                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1171
1172         u32temp = 0x04800000;
1173         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1174
1175         /* wait for the CCQ, CEQ, and AEQ to get created */
1176         count = 0;
1177         do {
1178                 if (count++ > 1000) {
1179                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1180                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1181                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1182                         return -1;
1183                 }
1184                 udelay(10);
1185         } while (((nes_read_indexed(nesdev,
1186                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1187
1188         /* dump the QP status value */
1189         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1190                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1191
1192         nesdev->cqp.sq_tail++;
1193
1194         return 0;
1195 }
1196
1197
1198 /**
1199  * nes_destroy_cqp
1200  */
1201 int nes_destroy_cqp(struct nes_device *nesdev)
1202 {
1203         struct nes_hw_cqp_wqe *cqp_wqe;
1204         u32 count = 0;
1205         u32 cqp_head;
1206         unsigned long flags;
1207
1208         do {
1209                 if (count++ > 1000)
1210                         break;
1211                 udelay(10);
1212         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1213
1214         /* Reset CCQ */
1215         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1216                         nesdev->ccq.cq_number);
1217
1218         /* Disable device interrupts */
1219         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1220
1221         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1222
1223         /* Destroy the AEQ */
1224         cqp_head = nesdev->cqp.sq_head++;
1225         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1226         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1227         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1228                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1229         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1230
1231         /* Destroy the NIC CEQ */
1232         cqp_head = nesdev->cqp.sq_head++;
1233         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1234         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1235         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1236                         ((u32)nesdev->nic_ceq_index << 8));
1237
1238         /* Destroy the CEQ */
1239         cqp_head = nesdev->cqp.sq_head++;
1240         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1241         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1242         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1243                         (nesdev->ceq_index << 8));
1244
1245         /* Destroy the CCQ */
1246         cqp_head = nesdev->cqp.sq_head++;
1247         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1248         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1249         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1250         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1251                         ((u32)nesdev->ceq_index << 16));
1252
1253         /* Destroy CQP */
1254         cqp_head = nesdev->cqp.sq_head++;
1255         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1256         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1257         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1258                         NES_CQP_QP_TYPE_CQP);
1259         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1260
1261         barrier();
1262         /* Ring doorbell (5 WQEs) */
1263         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1264
1265         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1266
1267         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1268         count = 0;
1269         do {
1270                 if (count++ > 1000) {
1271                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1272                                         PCI_FUNC(nesdev->pcidev->devfn));
1273                         break;
1274                 }
1275                 udelay(10);
1276         } while (((nes_read_indexed(nesdev,
1277                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1278
1279         /* dump the QP status value */
1280         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1281                         PCI_FUNC(nesdev->pcidev->devfn),
1282                         nes_read_indexed(nesdev,
1283                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1284
1285         kfree(nesdev->nes_cqp_requests);
1286
1287         /* Free the control structures */
1288         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1289                         nesdev->cqp.sq_pbase);
1290
1291         return 0;
1292 }
1293
1294
1295 /**
1296  * nes_init_1g_phy
1297  */
1298 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1299 {
1300         u32 counter = 0;
1301         u16 phy_data;
1302         int ret = 0;
1303
1304         nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1305         nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1306
1307         /* Reset the PHY */
1308         nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1309         udelay(100);
1310         counter = 0;
1311         do {
1312                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1313                 if (counter++ > 100) {
1314                         ret = -1;
1315                         break;
1316                 }
1317         } while (phy_data & 0x8000);
1318
1319         /* Setting no phy loopback */
1320         phy_data &= 0xbfff;
1321         phy_data |= 0x1140;
1322         nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1323         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1324         nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1325         nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1326
1327         /* Setting the interrupt mask */
1328         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1329         nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1330         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1331
1332         /* turning on flow control */
1333         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1334         nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1335         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1336
1337         /* Clear Half duplex */
1338         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1339         nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1340         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1341
1342         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1343         nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1344
1345         return ret;
1346 }
1347
1348
1349 /**
1350  * nes_init_2025_phy
1351  */
1352 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1353 {
1354         u32 temp_phy_data = 0;
1355         u32 temp_phy_data2 = 0;
1356         u32 counter = 0;
1357         u32 sds;
1358         u32 mac_index = nesdev->mac_index;
1359         int ret = 0;
1360         unsigned int first_attempt = 1;
1361
1362         /* Check firmware heartbeat */
1363         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1364         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1365         udelay(1500);
1366         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1367         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1368
1369         if (temp_phy_data != temp_phy_data2) {
1370                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1371                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1372                 if ((temp_phy_data & 0xff) > 0x20)
1373                         return 0;
1374                 printk(PFX "Reinitialize external PHY\n");
1375         }
1376
1377         /* no heartbeat, configure the PHY */
1378         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1379         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1380         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1381         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1382
1383         switch (phy_type) {
1384         case NES_PHY_TYPE_ARGUS:
1385                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1386                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1387                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1388                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1389                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1390                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1391                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1392
1393                 /* setup LEDs */
1394                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1395                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1397                 break;
1398
1399         case NES_PHY_TYPE_SFP_D:
1400                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1401                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1402                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1403                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1404                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1405                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1406                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1407
1408                 /* setup LEDs */
1409                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1410                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1412                 break;
1413
1414         case NES_PHY_TYPE_KR:
1415                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1416                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1417                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1418                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1420                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1421                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1422
1423                 /* setup LEDs */
1424                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1425                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1427
1428                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1429                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1430                 break;
1431         }
1432
1433         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1434
1435         /* Bring PHY out of reset */
1436         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1437
1438         /* Check for heartbeat */
1439         counter = 0;
1440         mdelay(690);
1441         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1442         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1443         do {
1444                 if (counter++ > 150) {
1445                         printk(PFX "No PHY heartbeat\n");
1446                         break;
1447                 }
1448                 mdelay(1);
1449                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1450                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1451         } while ((temp_phy_data2 == temp_phy_data));
1452
1453         /* wait for tracking */
1454         counter = 0;
1455         do {
1456                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1457                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1458                 if (counter++ > 300) {
1459                         if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1460                                 first_attempt = 0;
1461                                 counter = 0;
1462                                 /* reset AMCC PHY and try again */
1463                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1464                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1465                                 continue;
1466                         } else {
1467                                 ret = 1;
1468                                 break;
1469                         }
1470                 }
1471                 mdelay(10);
1472         } while ((temp_phy_data & 0xff) < 0x30);
1473
1474         /* setup signal integrity */
1475         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1476         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1477         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1478         if (phy_type == NES_PHY_TYPE_KR) {
1479                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1480         } else {
1481                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1482                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1483         }
1484
1485         /* reset serdes */
1486         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1487         sds |= 0x1;
1488         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1489         sds &= 0xfffffffe;
1490         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1491
1492         counter = 0;
1493         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1494                         && (counter++ < 5000))
1495                 ;
1496
1497         return ret;
1498 }
1499
1500
1501 /**
1502  * nes_init_phy
1503  */
1504 int nes_init_phy(struct nes_device *nesdev)
1505 {
1506         struct nes_adapter *nesadapter = nesdev->nesadapter;
1507         u32 mac_index = nesdev->mac_index;
1508         u32 tx_config = 0;
1509         unsigned long flags;
1510         u8  phy_type = nesadapter->phy_type[mac_index];
1511         u8  phy_index = nesadapter->phy_index[mac_index];
1512         int ret = 0;
1513
1514         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1515         if (phy_type == NES_PHY_TYPE_1G) {
1516                 /* setup 1G MDIO operation */
1517                 tx_config &= 0xFFFFFFE3;
1518                 tx_config |= 0x04;
1519         } else {
1520                 /* setup 10G MDIO operation */
1521                 tx_config &= 0xFFFFFFE3;
1522                 tx_config |= 0x1D;
1523         }
1524         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1525
1526         spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1527
1528         switch (phy_type) {
1529         case NES_PHY_TYPE_1G:
1530                 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1531                 break;
1532         case NES_PHY_TYPE_ARGUS:
1533         case NES_PHY_TYPE_SFP_D:
1534         case NES_PHY_TYPE_KR:
1535                 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1536                 break;
1537         }
1538
1539         spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1540
1541         return ret;
1542 }
1543
1544
1545 /**
1546  * nes_replenish_nic_rq
1547  */
1548 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1549 {
1550         unsigned long flags;
1551         dma_addr_t bus_address;
1552         struct sk_buff *skb;
1553         struct nes_hw_nic_rq_wqe *nic_rqe;
1554         struct nes_hw_nic *nesnic;
1555         struct nes_device *nesdev;
1556         struct nes_rskb_cb *cb;
1557         u32 rx_wqes_posted = 0;
1558
1559         nesnic = &nesvnic->nic;
1560         nesdev = nesvnic->nesdev;
1561         spin_lock_irqsave(&nesnic->rq_lock, flags);
1562         if (nesnic->replenishing_rq !=0) {
1563                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1564                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1565                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1566                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1567                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1568                         add_timer(&nesvnic->rq_wqes_timer);
1569                 } else
1570                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1571                 return;
1572         }
1573         nesnic->replenishing_rq = 1;
1574         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1575         do {
1576                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1577                 if (skb) {
1578                         skb->dev = nesvnic->netdev;
1579
1580                         bus_address = pci_map_single(nesdev->pcidev,
1581                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1582                         cb = (struct nes_rskb_cb *)&skb->cb[0];
1583                         cb->busaddr = bus_address;
1584                         cb->maplen = nesvnic->max_frame_size;
1585
1586                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1587                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1588                                         cpu_to_le32(nesvnic->max_frame_size);
1589                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1590                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1591                                         cpu_to_le32((u32)bus_address);
1592                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1593                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1594                         nesnic->rx_skb[nesnic->rq_head] = skb;
1595                         nesnic->rq_head++;
1596                         nesnic->rq_head &= nesnic->rq_size - 1;
1597                         atomic_dec(&nesvnic->rx_skbs_needed);
1598                         barrier();
1599                         if (++rx_wqes_posted == 255) {
1600                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1601                                 rx_wqes_posted = 0;
1602                         }
1603                 } else {
1604                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1605                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1606                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1607                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1608                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1609                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1610                                 add_timer(&nesvnic->rq_wqes_timer);
1611                         } else
1612                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1613                         break;
1614                 }
1615         } while (atomic_read(&nesvnic->rx_skbs_needed));
1616         barrier();
1617         if (rx_wqes_posted)
1618                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1619         nesnic->replenishing_rq = 0;
1620 }
1621
1622
1623 /**
1624  * nes_rq_wqes_timeout
1625  */
1626 static void nes_rq_wqes_timeout(unsigned long parm)
1627 {
1628         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1629         printk("%s: Timer fired.\n", __func__);
1630         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1631         if (atomic_read(&nesvnic->rx_skbs_needed))
1632                 nes_replenish_nic_rq(nesvnic);
1633 }
1634
1635
1636 /**
1637  * nes_init_nic_qp
1638  */
1639 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1640 {
1641         struct nes_hw_cqp_wqe *cqp_wqe;
1642         struct nes_hw_nic_sq_wqe *nic_sqe;
1643         struct nes_hw_nic_qp_context *nic_context;
1644         struct sk_buff *skb;
1645         struct nes_hw_nic_rq_wqe *nic_rqe;
1646         struct nes_vnic *nesvnic = netdev_priv(netdev);
1647         unsigned long flags;
1648         void *vmem;
1649         dma_addr_t pmem;
1650         u64 u64temp;
1651         int ret;
1652         u32 cqp_head;
1653         u32 counter;
1654         u32 wqe_count;
1655         struct nes_rskb_cb *cb;
1656         u8 jumbomode=0;
1657
1658         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1659         nesvnic->nic_mem_size = 256 +
1660                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1661                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1662                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1663                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1664                         sizeof(struct nes_hw_nic_qp_context);
1665
1666         nesvnic->nic_vbase = pci_zalloc_consistent(nesdev->pcidev,
1667                                                    nesvnic->nic_mem_size,
1668                                                    &nesvnic->nic_pbase);
1669         if (!nesvnic->nic_vbase) {
1670                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1671                 return -ENOMEM;
1672         }
1673         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1674                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1675
1676         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1677                         ~(unsigned long)(256 - 1));
1678         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1679                         ~(unsigned long long)(256 - 1));
1680
1681         /* Setup the first Fragment buffers */
1682         nesvnic->nic.first_frag_vbase = vmem;
1683
1684         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1685                 nesvnic->nic.frag_paddr[counter] = pmem;
1686                 pmem += sizeof(struct nes_first_frag);
1687         }
1688
1689         /* setup the SQ */
1690         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1691
1692         nesvnic->nic.sq_vbase = (void *)vmem;
1693         nesvnic->nic.sq_pbase = pmem;
1694         nesvnic->nic.sq_head = 0;
1695         nesvnic->nic.sq_tail = 0;
1696         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1697         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1698                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1699                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1700                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1701                                 NES_NIC_SQ_WQE_COMPLETION);
1702                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1703                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1704                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1705                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1706                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1707                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1708         }
1709
1710         nesvnic->get_cqp_request = nes_get_cqp_request;
1711         nesvnic->post_cqp_request = nes_post_cqp_request;
1712         nesvnic->mcrq_mcast_filter = NULL;
1713
1714         spin_lock_init(&nesvnic->nic.rq_lock);
1715
1716         /* setup the RQ */
1717         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1718         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1719
1720
1721         nesvnic->nic.rq_vbase = vmem;
1722         nesvnic->nic.rq_pbase = pmem;
1723         nesvnic->nic.rq_head = 0;
1724         nesvnic->nic.rq_tail = 0;
1725         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1726
1727         /* setup the CQ */
1728         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1729         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1730
1731         if (nesdev->nesadapter->netdev_count > 2)
1732                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1733         else
1734                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1735
1736         nesvnic->nic_cq.cq_vbase = vmem;
1737         nesvnic->nic_cq.cq_pbase = pmem;
1738         nesvnic->nic_cq.cq_head = 0;
1739         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1740
1741         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1742
1743         /* Send CreateCQ request to CQP */
1744         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1745         cqp_head = nesdev->cqp.sq_head;
1746
1747         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1748         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1749
1750         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1751                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1752                         ((u32)nesvnic->nic_cq.cq_size << 16));
1753         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1754                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1755         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1756         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1757         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1758         u64temp = (unsigned long)&nesvnic->nic_cq;
1759         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1760         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1761                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1762         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1763         if (++cqp_head >= nesdev->cqp.sq_size)
1764                 cqp_head = 0;
1765         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1766         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1767
1768         /* Send CreateQP request to CQP */
1769         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1770         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1771                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1772                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1773         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1774                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1775                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1776         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1777                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1778         }
1779
1780         u64temp = (u64)nesvnic->nic.sq_pbase;
1781         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1782         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1783         u64temp = (u64)nesvnic->nic.rq_pbase;
1784         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1785         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1786
1787         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1788                         NES_CQP_QP_TYPE_NIC);
1789         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1790         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1791                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1792         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1793
1794         if (++cqp_head >= nesdev->cqp.sq_size)
1795                 cqp_head = 0;
1796         nesdev->cqp.sq_head = cqp_head;
1797
1798         barrier();
1799
1800         /* Ring doorbell (2 WQEs) */
1801         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1802
1803         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1804         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1805                         nesvnic->nic.qp_id);
1806
1807         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1808                         NES_EVENT_TIMEOUT);
1809         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1810                         nesvnic->nic.qp_id, ret);
1811         if (!ret) {
1812                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1813                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1814                                 nesvnic->nic_pbase);
1815                 return -EIO;
1816         }
1817
1818         /* Populate the RQ */
1819         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1820                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1821                 if (!skb) {
1822                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1823
1824                         nes_destroy_nic_qp(nesvnic);
1825                         return -ENOMEM;
1826                 }
1827
1828                 skb->dev = netdev;
1829
1830                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1831                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1832                 cb = (struct nes_rskb_cb *)&skb->cb[0];
1833                 cb->busaddr = pmem;
1834                 cb->maplen = nesvnic->max_frame_size;
1835
1836                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1837                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1838                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1839                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1840                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1841                 nesvnic->nic.rx_skb[counter] = skb;
1842         }
1843
1844         wqe_count = NES_NIC_WQ_SIZE - 1;
1845         nesvnic->nic.rq_head = wqe_count;
1846         barrier();
1847         do {
1848                 counter = min(wqe_count, ((u32)255));
1849                 wqe_count -= counter;
1850                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1851         } while (wqe_count);
1852         setup_timer(&nesvnic->rq_wqes_timer, nes_rq_wqes_timeout,
1853                     (unsigned long)nesvnic);
1854         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1855         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1856         {
1857                 nes_nic_init_timer(nesdev);
1858                 if (netdev->mtu > 1500)
1859                         jumbomode = 1;
1860                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1861         }
1862         if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
1863                 (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
1864                         nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", netdev->name);
1865                         nes_destroy_nic_qp(nesvnic);
1866                 return -ENOMEM;
1867         }
1868
1869         return 0;
1870 }
1871
1872
1873 /**
1874  * nes_destroy_nic_qp
1875  */
1876 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1877 {
1878         u64 u64temp;
1879         dma_addr_t bus_address;
1880         struct nes_device *nesdev = nesvnic->nesdev;
1881         struct nes_hw_cqp_wqe *cqp_wqe;
1882         struct nes_hw_nic_sq_wqe *nic_sqe;
1883         __le16 *wqe_fragment_length;
1884         u16  wqe_fragment_index;
1885         u32 cqp_head;
1886         u32 wqm_cfg0;
1887         unsigned long flags;
1888         struct sk_buff *rx_skb;
1889         struct nes_rskb_cb *cb;
1890         int ret;
1891
1892         if (nesdev->nesadapter->allow_unaligned_fpdus)
1893                 nes_destroy_mgt(nesvnic);
1894
1895         /* clear wqe stall before destroying NIC QP */
1896         wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1897         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1898
1899         /* Free remaining NIC receive buffers */
1900         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1901                 rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
1902                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
1903                 pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
1904                         PCI_DMA_FROMDEVICE);
1905
1906                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1907                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1908         }
1909
1910         /* Free remaining NIC transmit buffers */
1911         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1912                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1913                 wqe_fragment_index = 1;
1914                 wqe_fragment_length = (__le16 *)
1915                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1916                 /* bump past the vlan tag */
1917                 wqe_fragment_length++;
1918                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1919                         u64temp = (u64)le32_to_cpu(
1920                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1921                                 wqe_fragment_index*2]);
1922                         u64temp += ((u64)le32_to_cpu(
1923                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1924                                 + wqe_fragment_index*2]))<<32;
1925                         bus_address = (dma_addr_t)u64temp;
1926                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1927                                         nesvnic->nic.first_frag_overflow)) {
1928                                 pci_unmap_single(nesdev->pcidev,
1929                                                 bus_address,
1930                                                 le16_to_cpu(wqe_fragment_length[
1931                                                         wqe_fragment_index++]),
1932                                                 PCI_DMA_TODEVICE);
1933                         }
1934                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1935                                 if (wqe_fragment_length[wqe_fragment_index]) {
1936                                         u64temp = le32_to_cpu(
1937                                                 nic_sqe->wqe_words[
1938                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1939                                                 wqe_fragment_index*2]);
1940                                         u64temp += ((u64)le32_to_cpu(
1941                                                 nic_sqe->wqe_words[
1942                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1943                                                 wqe_fragment_index*2]))<<32;
1944                                         bus_address = (dma_addr_t)u64temp;
1945                                         pci_unmap_page(nesdev->pcidev,
1946                                                         bus_address,
1947                                                         le16_to_cpu(
1948                                                         wqe_fragment_length[
1949                                                         wqe_fragment_index]),
1950                                                         PCI_DMA_TODEVICE);
1951                                 } else
1952                                         break;
1953                         }
1954                 }
1955                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1956                         dev_kfree_skb(
1957                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1958
1959                 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1960                                         & (nesvnic->nic.sq_size - 1);
1961         }
1962
1963         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1964
1965         /* Destroy NIC QP */
1966         cqp_head = nesdev->cqp.sq_head;
1967         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1968         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1969
1970         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1971                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1972         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1973                 nesvnic->nic.qp_id);
1974
1975         if (++cqp_head >= nesdev->cqp.sq_size)
1976                 cqp_head = 0;
1977
1978         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1979
1980         /* Destroy NIC CQ */
1981         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1982         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1983                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1984         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1985                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1986
1987         if (++cqp_head >= nesdev->cqp.sq_size)
1988                 cqp_head = 0;
1989
1990         nesdev->cqp.sq_head = cqp_head;
1991         barrier();
1992
1993         /* Ring doorbell (2 WQEs) */
1994         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1995
1996         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1997         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1998                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1999                         cqp_head, nesdev->cqp.sq_head,
2000                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2001
2002         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2003                         NES_EVENT_TIMEOUT);
2004
2005         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2006                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2007                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2008         if (!ret) {
2009                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2010                                 nesvnic->nic.qp_id);
2011         }
2012
2013         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2014                         nesvnic->nic_pbase);
2015
2016         /* restore old wqm_cfg0 value */
2017         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2018 }
2019
2020 /**
2021  * nes_napi_isr
2022  */
2023 int nes_napi_isr(struct nes_device *nesdev)
2024 {
2025         struct nes_adapter *nesadapter = nesdev->nesadapter;
2026         u32 int_stat;
2027
2028         if (nesdev->napi_isr_ran) {
2029                 /* interrupt status has already been read in ISR */
2030                 int_stat = nesdev->int_stat;
2031         } else {
2032                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2033                 nesdev->int_stat = int_stat;
2034                 nesdev->napi_isr_ran = 1;
2035         }
2036
2037         int_stat &= nesdev->int_req;
2038         /* iff NIC, process here, else wait for DPC */
2039         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2040                 nesdev->napi_isr_ran = 0;
2041                 nes_write32(nesdev->regs + NES_INT_STAT,
2042                         (int_stat &
2043                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2044
2045                 /* Process the CEQs */
2046                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2047
2048                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2049                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2050                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
2051                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2052                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2053                                 /* Enable Periodic timer interrupts */
2054                                 nesdev->int_req |= NES_INT_TIMER;
2055                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2056                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2057                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
2058                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
2059                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2060                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2061                         }
2062
2063                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2064                         {
2065                                 nes_nic_init_timer(nesdev);
2066                         }
2067                         /* Enable interrupts, except CEQs */
2068                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2069                 } else {
2070                         /* Enable interrupts, make sure timer is off */
2071                         nesdev->int_req &= ~NES_INT_TIMER;
2072                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2073                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2074                 }
2075                 nesdev->deepcq_count = 0;
2076                 return 1;
2077         } else {
2078                 return 0;
2079         }
2080 }
2081
2082 static void process_critical_error(struct nes_device *nesdev)
2083 {
2084         u32 debug_error;
2085         u32 nes_idx_debug_error_masks0 = 0;
2086         u16 error_module = 0;
2087
2088         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2089         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2090                         (u16)debug_error);
2091         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2092                         0x01010000 | (debug_error & 0x0000ffff));
2093         if (crit_err_count++ > 10)
2094                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2095         error_module = (u16) (debug_error & 0x1F00) >> 8;
2096         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2097                         nes_max_critical_error_count) {
2098                 printk(KERN_ERR PFX "Masking off critical error for module "
2099                         "0x%02X\n", (u16)error_module);
2100                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2101                         NES_IDX_DEBUG_ERROR_MASKS0);
2102                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2103                         nes_idx_debug_error_masks0 | (1 << error_module));
2104         }
2105 }
2106 /**
2107  * nes_dpc
2108  */
2109 void nes_dpc(unsigned long param)
2110 {
2111         struct nes_device *nesdev = (struct nes_device *)param;
2112         struct nes_adapter *nesadapter = nesdev->nesadapter;
2113         u32 counter;
2114         u32 loop_counter = 0;
2115         u32 int_status_bit;
2116         u32 int_stat;
2117         u32 timer_stat;
2118         u32 temp_int_stat;
2119         u32 intf_int_stat;
2120         u32 processed_intf_int = 0;
2121         u16 processed_timer_int = 0;
2122         u16 completion_ints = 0;
2123         u16 timer_ints = 0;
2124
2125         /* nes_debug(NES_DBG_ISR, "\n"); */
2126
2127         do {
2128                 timer_stat = 0;
2129                 if (nesdev->napi_isr_ran) {
2130                         nesdev->napi_isr_ran = 0;
2131                         int_stat = nesdev->int_stat;
2132                 } else
2133                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2134                 if (processed_intf_int != 0)
2135                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2136                 else
2137                         int_stat &= nesdev->int_req;
2138                 if (processed_timer_int == 0) {
2139                         processed_timer_int = 1;
2140                         if (int_stat & NES_INT_TIMER) {
2141                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2142                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2143                                         int_stat &= ~NES_INT_TIMER;
2144                                 }
2145                         }
2146                 } else {
2147                         int_stat &= ~NES_INT_TIMER;
2148                 }
2149
2150                 if (int_stat) {
2151                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2152                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2153                                 /* Ack the interrupts */
2154                                 nes_write32(nesdev->regs+NES_INT_STAT,
2155                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2156                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2157                         }
2158
2159                         temp_int_stat = int_stat;
2160                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2161                                 if (int_stat & int_status_bit) {
2162                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2163                                         temp_int_stat &= ~int_status_bit;
2164                                         completion_ints = 1;
2165                                 }
2166                                 if (!(temp_int_stat & 0x0000ffff))
2167                                         break;
2168                                 int_status_bit <<= 1;
2169                         }
2170
2171                         /* Process the AEQ for this pci function */
2172                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2173                         if (int_stat & int_status_bit) {
2174                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2175                         }
2176
2177                         /* Process the MAC interrupt for this pci function */
2178                         int_status_bit = 1 << (24 + nesdev->mac_index);
2179                         if (int_stat & int_status_bit) {
2180                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2181                         }
2182
2183                         if (int_stat & NES_INT_TIMER) {
2184                                 if (timer_stat & nesdev->timer_int_req) {
2185                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2186                                                         (timer_stat & nesdev->timer_int_req) |
2187                                                         ~(nesdev->nesadapter->timer_int_req));
2188                                         timer_ints = 1;
2189                                 }
2190                         }
2191
2192                         if (int_stat & NES_INT_INTF) {
2193                                 processed_intf_int = 1;
2194                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2195                                 intf_int_stat &= nesdev->intf_int_req;
2196                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2197                                         process_critical_error(nesdev);
2198                                 }
2199                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2200                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2201                                         BUG();
2202                                 }
2203                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2204                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2205                                         BUG();
2206                                 }
2207                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2208                         }
2209
2210                         if (int_stat & NES_INT_TSW) {
2211                         }
2212                 }
2213                 /* Don't use the interface interrupt bit stay in loop */
2214                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2215                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2216         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2217
2218         if (timer_ints == 1) {
2219                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2220                         if (completion_ints == 0) {
2221                                 nesdev->timer_only_int_count++;
2222                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2223                                         nesdev->timer_only_int_count = 0;
2224                                         nesdev->int_req &= ~NES_INT_TIMER;
2225                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2226                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2227                                 } else {
2228                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2229                                 }
2230                         } else {
2231                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2232                                 {
2233                                         nes_nic_init_timer(nesdev);
2234                                 }
2235                                 nesdev->timer_only_int_count = 0;
2236                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2237                         }
2238                 } else {
2239                         nesdev->timer_only_int_count = 0;
2240                         nesdev->int_req &= ~NES_INT_TIMER;
2241                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2242                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2243                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2244                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2245                 }
2246         } else {
2247                 if ( (completion_ints == 1) &&
2248                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2249                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2250                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2251                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2252                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2253                         nesdev->timer_only_int_count = 0;
2254                         nesdev->int_req |= NES_INT_TIMER;
2255                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2256                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2257                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2258                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2259                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2260                 } else {
2261                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2262                 }
2263         }
2264         nesdev->deepcq_count = 0;
2265 }
2266
2267
2268 /**
2269  * nes_process_ceq
2270  */
2271 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2272 {
2273         u64 u64temp;
2274         struct nes_hw_cq *cq;
2275         u32 head;
2276         u32 ceq_size;
2277
2278         /* nes_debug(NES_DBG_CQ, "\n"); */
2279         head = ceq->ceq_head;
2280         ceq_size = ceq->ceq_size;
2281
2282         do {
2283                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2284                                 NES_CEQE_VALID) {
2285                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2286                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2287                         u64temp <<= 1;
2288                         cq = *((struct nes_hw_cq **)&u64temp);
2289                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2290                         barrier();
2291                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2292
2293                         /* call the event handler */
2294                         cq->ce_handler(nesdev, cq);
2295
2296                         if (++head >= ceq_size)
2297                                 head = 0;
2298                 } else {
2299                         break;
2300                 }
2301
2302         } while (1);
2303
2304         ceq->ceq_head = head;
2305 }
2306
2307
2308 /**
2309  * nes_process_aeq
2310  */
2311 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2312 {
2313         /* u64 u64temp; */
2314         u32 head;
2315         u32 aeq_size;
2316         u32 aeqe_misc;
2317         u32 aeqe_cq_id;
2318         struct nes_hw_aeqe volatile *aeqe;
2319
2320         head = aeq->aeq_head;
2321         aeq_size = aeq->aeq_size;
2322
2323         do {
2324                 aeqe = &aeq->aeq_vbase[head];
2325                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2326                         break;
2327                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2328                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2329                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2330                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2331                                 /* dealing with an accelerated QP related AE */
2332                                 /*
2333                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2334                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2335                                  */
2336                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2337                         } else {
2338                                 /* TODO: dealing with a CQP related AE */
2339                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2340                                                 (u16)(aeqe_misc >> 16));
2341                         }
2342                 }
2343
2344                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2345
2346                 if (++head >= aeq_size)
2347                         head = 0;
2348
2349                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2350         }
2351         while (1);
2352         aeq->aeq_head = head;
2353 }
2354
2355 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2356 {
2357         struct nes_adapter *nesadapter = nesdev->nesadapter;
2358         u32 reset_value;
2359         u32 i=0;
2360         u32 u32temp;
2361
2362         if (nesadapter->hw_rev == NE020_REV) {
2363                 return;
2364         }
2365         mh_detected++;
2366
2367         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2368
2369         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2370                 reset_value |= 0x0000001d;
2371         else
2372                 reset_value |= 0x0000002d;
2373
2374         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2375                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2376                         nesadapter->link_interrupt_count[0] = 0;
2377                         nesadapter->link_interrupt_count[1] = 0;
2378                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2379                         if (0x00000040 & u32temp)
2380                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2381                         else
2382                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2383
2384                         reset_value |= 0x0000003d;
2385                 }
2386                 nesadapter->link_interrupt_count[mac_index] = 0;
2387         }
2388
2389         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2390
2391         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2392                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2393
2394         if (0x0000003d == (reset_value & 0x0000003d)) {
2395                 u32 pcs_control_status0, pcs_control_status1;
2396
2397                 for (i = 0; i < 10; i++) {
2398                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2399                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2400                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2401                              && (pcs_control_status0 & 0x00100000))
2402                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2403                                 && (pcs_control_status1 & 0x00100000)))
2404                                 continue;
2405                         else
2406                                 break;
2407                 }
2408                 if (10 == i) {
2409                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2410                         if (0x00000040 & u32temp)
2411                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2412                         else
2413                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2414
2415                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2416
2417                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2418                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2419                 }
2420         }
2421 }
2422
2423 /**
2424  * nes_process_mac_intr
2425  */
2426 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2427 {
2428         unsigned long flags;
2429         u32 pcs_control_status;
2430         struct nes_adapter *nesadapter = nesdev->nesadapter;
2431         struct nes_vnic *nesvnic;
2432         u32 mac_status;
2433         u32 mac_index = nesdev->mac_index;
2434         u32 u32temp;
2435         u16 phy_data;
2436         u16 temp_phy_data;
2437         u32 pcs_val  = 0x0f0f0000;
2438         u32 pcs_mask = 0x0f1f0000;
2439         u32 cdr_ctrl;
2440
2441         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2442         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2443                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2444                 return;
2445         }
2446         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2447
2448         /* ack the MAC interrupt */
2449         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2450         /* Clear the interrupt */
2451         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2452
2453         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2454
2455         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2456                 nesdev->link_status_interrupts++;
2457                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2458                         nes_reset_link(nesdev, mac_index);
2459
2460                 /* read the PHY interrupt status register */
2461                 if ((nesadapter->OneG_Mode) &&
2462                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2463                         do {
2464                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2465                                                 nesadapter->phy_index[mac_index], &phy_data);
2466                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2467                                                 nesadapter->phy_index[mac_index], phy_data);
2468                         } while (phy_data&0x8000);
2469
2470                         temp_phy_data = 0;
2471                         do {
2472                                 nes_read_1G_phy_reg(nesdev, 0x11,
2473                                                 nesadapter->phy_index[mac_index], &phy_data);
2474                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2475                                                 nesadapter->phy_index[mac_index], phy_data);
2476                                 if (temp_phy_data == phy_data)
2477                                         break;
2478                                 temp_phy_data = phy_data;
2479                         } while (1);
2480
2481                         nes_read_1G_phy_reg(nesdev, 0x1e,
2482                                         nesadapter->phy_index[mac_index], &phy_data);
2483                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2484                                         nesadapter->phy_index[mac_index], phy_data);
2485
2486                         nes_read_1G_phy_reg(nesdev, 1,
2487                                         nesadapter->phy_index[mac_index], &phy_data);
2488                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2489                                         nesadapter->phy_index[mac_index], phy_data);
2490
2491                         if (temp_phy_data & 0x1000) {
2492                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2493                                 phy_data = 4;
2494                         } else {
2495                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2496                         }
2497                 }
2498                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2499                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2500                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2501
2502                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2503                         switch (mac_index) {
2504                         case 1:
2505                         case 3:
2506                                 pcs_control_status = nes_read_indexed(nesdev,
2507                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2508                                 break;
2509                         default:
2510                                 pcs_control_status = nes_read_indexed(nesdev,
2511                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2512                                 break;
2513                         }
2514                 } else {
2515                         pcs_control_status = nes_read_indexed(nesdev,
2516                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2517                         pcs_control_status = nes_read_indexed(nesdev,
2518                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2519                 }
2520
2521                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2522                                 mac_index, pcs_control_status);
2523                 if ((nesadapter->OneG_Mode) &&
2524                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2525                         u32temp = 0x01010000;
2526                         if (nesadapter->port_count > 2) {
2527                                 u32temp |= 0x02020000;
2528                         }
2529                         if ((pcs_control_status & u32temp)!= u32temp) {
2530                                 phy_data = 0;
2531                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2532                         }
2533                 } else {
2534                         switch (nesadapter->phy_type[mac_index]) {
2535                         case NES_PHY_TYPE_ARGUS:
2536                         case NES_PHY_TYPE_SFP_D:
2537                         case NES_PHY_TYPE_KR:
2538                                 /* clear the alarms */
2539                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2540                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2541                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2542                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2543                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2544                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2545                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2546                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2547                                 /* check link status */
2548                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2549                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2550
2551                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2552                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2553                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2554                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2555
2556                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2557
2558                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2559                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2560                                 break;
2561
2562                         case NES_PHY_TYPE_PUMA_1G:
2563                                 if (mac_index < 2)
2564                                         pcs_val = pcs_mask = 0x01010000;
2565                                 else
2566                                         pcs_val = pcs_mask = 0x02020000;
2567                                 /* fall through */
2568                         default:
2569                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2570                                 break;
2571                         }
2572                 }
2573
2574                 if (phy_data & 0x0004) {
2575                         if (wide_ppm_offset &&
2576                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2577                             (nesadapter->hw_rev != NE020_REV)) {
2578                                 cdr_ctrl = nes_read_indexed(nesdev,
2579                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2580                                                             mac_index * 0x200);
2581                                 nes_write_indexed(nesdev,
2582                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2583                                                   mac_index * 0x200,
2584                                                   cdr_ctrl | 0x000F0000);
2585                         }
2586                         nesadapter->mac_link_down[mac_index] = 0;
2587                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2588                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2589                                                 nesvnic->linkup);
2590                                 if (nesvnic->linkup == 0) {
2591                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2592                                                         nesvnic->netdev->name, nesvnic->netdev);
2593                                         if (netif_queue_stopped(nesvnic->netdev))
2594                                                 netif_start_queue(nesvnic->netdev);
2595                                         nesvnic->linkup = 1;
2596                                         netif_carrier_on(nesvnic->netdev);
2597
2598                                         spin_lock(&nesvnic->port_ibevent_lock);
2599                                         if (nesvnic->of_device_registered) {
2600                                                 if (nesdev->iw_status == 0) {
2601                                                         nesdev->iw_status = 1;
2602                                                         nes_port_ibevent(nesvnic);
2603                                                 }
2604                                         }
2605                                         spin_unlock(&nesvnic->port_ibevent_lock);
2606                                 }
2607                         }
2608                 } else {
2609                         if (wide_ppm_offset &&
2610                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2611                             (nesadapter->hw_rev != NE020_REV)) {
2612                                 cdr_ctrl = nes_read_indexed(nesdev,
2613                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2614                                                             mac_index * 0x200);
2615                                 nes_write_indexed(nesdev,
2616                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2617                                                   mac_index * 0x200,
2618                                                   cdr_ctrl & 0xFFF0FFFF);
2619                         }
2620                         nesadapter->mac_link_down[mac_index] = 1;
2621                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2622                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2623                                                 nesvnic->linkup);
2624                                 if (nesvnic->linkup == 1) {
2625                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2626                                                         nesvnic->netdev->name, nesvnic->netdev);
2627                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2628                                                 netif_stop_queue(nesvnic->netdev);
2629                                         nesvnic->linkup = 0;
2630                                         netif_carrier_off(nesvnic->netdev);
2631
2632                                         spin_lock(&nesvnic->port_ibevent_lock);
2633                                         if (nesvnic->of_device_registered) {
2634                                                 if (nesdev->iw_status == 1) {
2635                                                         nesdev->iw_status = 0;
2636                                                         nes_port_ibevent(nesvnic);
2637                                                 }
2638                                         }
2639                                         spin_unlock(&nesvnic->port_ibevent_lock);
2640                                 }
2641                         }
2642                 }
2643                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2644                         nesdev->link_recheck = 1;
2645                         mod_delayed_work(system_wq, &nesdev->work,
2646                                          NES_LINK_RECHECK_DELAY);
2647                 }
2648         }
2649
2650         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2651
2652         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2653 }
2654
2655 void nes_recheck_link_status(struct work_struct *work)
2656 {
2657         unsigned long flags;
2658         struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2659         struct nes_adapter *nesadapter = nesdev->nesadapter;
2660         struct nes_vnic *nesvnic;
2661         u32 mac_index = nesdev->mac_index;
2662         u16 phy_data;
2663         u16 temp_phy_data;
2664
2665         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2666
2667         /* check link status */
2668         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2669         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2670
2671         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2672         nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2673         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2674         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2675
2676         phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2677
2678         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2679                 __func__, phy_data,
2680                 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2681
2682         if (phy_data & 0x0004) {
2683                 nesadapter->mac_link_down[mac_index] = 0;
2684                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2685                         if (nesvnic->linkup == 0) {
2686                                 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2687                                                 nesvnic->netdev->name, nesvnic->netdev);
2688                                 if (netif_queue_stopped(nesvnic->netdev))
2689                                         netif_start_queue(nesvnic->netdev);
2690                                 nesvnic->linkup = 1;
2691                                 netif_carrier_on(nesvnic->netdev);
2692
2693                                 spin_lock(&nesvnic->port_ibevent_lock);
2694                                 if (nesvnic->of_device_registered) {
2695                                         if (nesdev->iw_status == 0) {
2696                                                 nesdev->iw_status = 1;
2697                                                 nes_port_ibevent(nesvnic);
2698                                         }
2699                                 }
2700                                 spin_unlock(&nesvnic->port_ibevent_lock);
2701                         }
2702                 }
2703
2704         } else {
2705                 nesadapter->mac_link_down[mac_index] = 1;
2706                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2707                         if (nesvnic->linkup == 1) {
2708                                 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2709                                                 nesvnic->netdev->name, nesvnic->netdev);
2710                                 if (!(netif_queue_stopped(nesvnic->netdev)))
2711                                         netif_stop_queue(nesvnic->netdev);
2712                                 nesvnic->linkup = 0;
2713                                 netif_carrier_off(nesvnic->netdev);
2714
2715                                 spin_lock(&nesvnic->port_ibevent_lock);
2716                                 if (nesvnic->of_device_registered) {
2717                                         if (nesdev->iw_status == 1) {
2718                                                 nesdev->iw_status = 0;
2719                                                 nes_port_ibevent(nesvnic);
2720                                         }
2721                                 }
2722                                 spin_unlock(&nesvnic->port_ibevent_lock);
2723                         }
2724                 }
2725         }
2726         if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2727                 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2728         else
2729                 nesdev->link_recheck = 0;
2730
2731         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2732 }
2733
2734
2735 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2736 {
2737         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2738
2739         napi_schedule(&nesvnic->napi);
2740 }
2741
2742
2743 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2744 * getting out of nic_ce_handler
2745 */
2746 #define MAX_RQES_TO_PROCESS     384
2747
2748 /**
2749  * nes_nic_ce_handler
2750  */
2751 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2752 {
2753         u64 u64temp;
2754         dma_addr_t bus_address;
2755         struct nes_hw_nic *nesnic;
2756         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2757         struct nes_adapter *nesadapter = nesdev->nesadapter;
2758         struct nes_hw_nic_rq_wqe *nic_rqe;
2759         struct nes_hw_nic_sq_wqe *nic_sqe;
2760         struct sk_buff *skb;
2761         struct sk_buff *rx_skb;
2762         struct nes_rskb_cb *cb;
2763         __le16 *wqe_fragment_length;
2764         u32 head;
2765         u32 cq_size;
2766         u32 rx_pkt_size;
2767         u32 cqe_count=0;
2768         u32 cqe_errv;
2769         u32 cqe_misc;
2770         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2771         u16 vlan_tag;
2772         u16 pkt_type;
2773         u16 rqes_processed = 0;
2774         u8 sq_cqes = 0;
2775
2776         head = cq->cq_head;
2777         cq_size = cq->cq_size;
2778         cq->cqes_pending = 1;
2779         do {
2780                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2781                                 NES_NIC_CQE_VALID) {
2782                         nesnic = &nesvnic->nic;
2783                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2784                         if (cqe_misc & NES_NIC_CQE_SQ) {
2785                                 sq_cqes++;
2786                                 wqe_fragment_index = 1;
2787                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2788                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2789                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2790                                 /* bump past the vlan tag */
2791                                 wqe_fragment_length++;
2792                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2793                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2794                                                         wqe_fragment_index * 2]);
2795                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2796                                                         wqe_fragment_index * 2])) << 32;
2797                                         bus_address = (dma_addr_t)u64temp;
2798                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2799                                                 pci_unmap_single(nesdev->pcidev,
2800                                                                 bus_address,
2801                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2802                                                                 PCI_DMA_TODEVICE);
2803                                         }
2804                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2805                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2806                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2807                                                                                 wqe_fragment_index * 2]);
2808                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2809                                                                                 + wqe_fragment_index * 2])) <<32;
2810                                                         bus_address = (dma_addr_t)u64temp;
2811                                                         pci_unmap_page(nesdev->pcidev,
2812                                                                         bus_address,
2813                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2814                                                                         PCI_DMA_TODEVICE);
2815                                                 } else
2816                                                         break;
2817                                         }
2818                                 }
2819                                 if (skb)
2820                                         dev_kfree_skb_any(skb);
2821                                 nesnic->sq_tail++;
2822                                 nesnic->sq_tail &= nesnic->sq_size-1;
2823                                 if (sq_cqes > 128) {
2824                                         barrier();
2825                                         /* restart the queue if it had been stopped */
2826                                         if (netif_queue_stopped(nesvnic->netdev))
2827                                                 netif_wake_queue(nesvnic->netdev);
2828                                         sq_cqes = 0;
2829                                 }
2830                         } else {
2831                                 rqes_processed ++;
2832
2833                                 cq->rx_cqes_completed++;
2834                                 cq->rx_pkts_indicated++;
2835                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2836                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2837                                 /* Get the skb */
2838                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2839                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2840                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2841                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2842                                 pci_unmap_single(nesdev->pcidev, bus_address,
2843                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2844                                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
2845                                 cb->busaddr = 0;
2846                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2847                                 /* rx_skb->len = rx_pkt_size; */
2848                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2849                                 skb_put(rx_skb, rx_pkt_size);
2850                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2851                                 nesnic->rq_tail++;
2852                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2853
2854                                 atomic_inc(&nesvnic->rx_skbs_needed);
2855                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2856                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2857                                                         cq->cq_number | (cqe_count << 16));
2858                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2859                                         nesdev->currcq_count += cqe_count;
2860                                         cqe_count = 0;
2861                                         nes_replenish_nic_rq(nesvnic);
2862                                 }
2863                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2864                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2865                                 rx_skb->ip_summed = CHECKSUM_NONE;
2866
2867                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2868                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2869                                         if ((cqe_errv &
2870                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2871                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2872                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM)
2873                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2874                                         } else
2875                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2876                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2877                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2878
2879                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2880                                         if ((cqe_errv &
2881                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2882                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2883                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
2884                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2885                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2886                                                                   nesvnic->netdev->name); */
2887                                                 }
2888                                         } else
2889                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2890                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2891                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2892                                         }
2893                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2894                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2895
2896                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2897                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2898                                                 rx_skb = NULL;
2899                                 }
2900                                 if (rx_skb == NULL)
2901                                         goto skip_rx_indicate0;
2902
2903
2904                                 if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
2905                                         vlan_tag = (u16)(le32_to_cpu(
2906                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2907                                                         >> 16);
2908                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2909                                                         nesvnic->netdev->name, vlan_tag);
2910
2911                                         __vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag);
2912                                 }
2913                                 napi_gro_receive(&nesvnic->napi, rx_skb);
2914
2915 skip_rx_indicate0:
2916                                 ;
2917                                 /* nesvnic->netstats.rx_packets++; */
2918                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2919                         }
2920
2921                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2922                         /* Accounting... */
2923                         cqe_count++;
2924                         if (++head >= cq_size)
2925                                 head = 0;
2926                         if (cqe_count == 255) {
2927                                 /* Replenish Nic CQ */
2928                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2929                                                 cq->cq_number | (cqe_count << 16));
2930                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2931                                 nesdev->currcq_count += cqe_count;
2932                                 cqe_count = 0;
2933                         }
2934
2935                         if (cq->rx_cqes_completed >= nesvnic->budget)
2936                                 break;
2937                 } else {
2938                         cq->cqes_pending = 0;
2939                         break;
2940                 }
2941
2942         } while (1);
2943
2944         if (sq_cqes) {
2945                 barrier();
2946                 /* restart the queue if it had been stopped */
2947                 if (netif_queue_stopped(nesvnic->netdev))
2948                         netif_wake_queue(nesvnic->netdev);
2949         }
2950         cq->cq_head = head;
2951         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2952                         cq->cq_number, cqe_count, cq->cq_head); */
2953         cq->cqe_allocs_pending = cqe_count;
2954         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2955         {
2956                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2957                 nesdev->currcq_count += cqe_count;
2958                 nes_nic_tune_timer(nesdev);
2959         }
2960         if (atomic_read(&nesvnic->rx_skbs_needed))
2961                 nes_replenish_nic_rq(nesvnic);
2962 }
2963
2964
2965
2966 /**
2967  * nes_cqp_ce_handler
2968  */
2969 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2970 {
2971         u64 u64temp;
2972         unsigned long flags;
2973         struct nes_hw_cqp *cqp = NULL;
2974         struct nes_cqp_request *cqp_request;
2975         struct nes_hw_cqp_wqe *cqp_wqe;
2976         u32 head;
2977         u32 cq_size;
2978         u32 cqe_count=0;
2979         u32 error_code;
2980         u32 opcode;
2981         u32 ctx_index;
2982         /* u32 counter; */
2983
2984         head = cq->cq_head;
2985         cq_size = cq->cq_size;
2986
2987         do {
2988                 /* process the CQE */
2989                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2990                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2991
2992                 opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
2993                 if (opcode & NES_CQE_VALID) {
2994                         cqp = &nesdev->cqp;
2995
2996                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2997                         if (error_code) {
2998                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2999                                                 " Major/Minor codes = 0x%04X:%04X.\n",
3000                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
3001                                                 (u16)(error_code >> 16),
3002                                                 (u16)error_code);
3003                         }
3004
3005                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3006                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3007                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
3008                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3009
3010                         cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
3011                         if (cqp_request) {
3012                                 if (cqp_request->waiting) {
3013                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3014                                         cqp_request->major_code = (u16)(error_code >> 16);
3015                                         cqp_request->minor_code = (u16)error_code;
3016                                         barrier();
3017                                         cqp_request->request_done = 1;
3018                                         wake_up(&cqp_request->waitq);
3019                                         nes_put_cqp_request(nesdev, cqp_request);
3020                                 } else {
3021                                         if (cqp_request->callback)
3022                                                 cqp_request->cqp_callback(nesdev, cqp_request);
3023                                         nes_free_cqp_request(nesdev, cqp_request);
3024                                 }
3025                         } else {
3026                                 wake_up(&nesdev->cqp.waitq);
3027                         }
3028
3029                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3030                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3031                         if (++cqp->sq_tail >= cqp->sq_size)
3032                                 cqp->sq_tail = 0;
3033
3034                         /* Accounting... */
3035                         cqe_count++;
3036                         if (++head >= cq_size)
3037                                 head = 0;
3038                 } else {
3039                         break;
3040                 }
3041         } while (1);
3042         cq->cq_head = head;
3043
3044         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3045         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3046                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3047                         (nesdev->cqp.sq_size - 1)) != 1)) {
3048                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3049                                 struct nes_cqp_request, list);
3050                 list_del_init(&cqp_request->list);
3051                 head = nesdev->cqp.sq_head++;
3052                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3053                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3054                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3055                 barrier();
3056
3057                 opcode = le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX]);
3058                 if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
3059                         ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
3060                 else
3061                         ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
3062                 cqp_wqe->wqe_words[ctx_index] =
3063                         cpu_to_le32((u32)((unsigned long)cqp_request));
3064                 cqp_wqe->wqe_words[ctx_index + 1] =
3065                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3066                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3067                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3068                 /* Ring doorbell (1 WQEs) */
3069                 barrier();
3070                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3071         }
3072         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3073
3074         /* Arm the CCQ */
3075         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3076                         cq->cq_number);
3077         nes_read32(nesdev->regs+NES_CQE_ALLOC);
3078 }
3079
3080 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3081 {
3082         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3083                 /* skip over ethernet header */
3084                 pkt += ETH_HLEN;
3085
3086                 /* Skip over IP and TCP headers */
3087                 pkt += 4 * (pkt[0] & 0x0f);
3088                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3089         }
3090         return pkt;
3091 }
3092
3093 /* Determine if incoming error pkt is rdma layer */
3094 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3095 {
3096         u8 *pkt;
3097         u16 *mpa;
3098         u32 opcode = 0xffffffff;
3099
3100         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3101                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3102                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3103                 opcode = be16_to_cpu(mpa[1]) & 0xf;
3104         }
3105
3106         return opcode;
3107 }
3108
3109 /* Build iWARP terminate header */
3110 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3111 {
3112         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3113         u16 ddp_seg_len;
3114         int copy_len = 0;
3115         u8 is_tagged = 0;
3116         u8 flush_code = 0;
3117         struct nes_terminate_hdr *termhdr;
3118
3119         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3120         memset(termhdr, 0, 64);
3121
3122         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3123
3124                 /* Use data from offending packet to fill in ddp & rdma hdrs */
3125                 pkt = locate_mpa(pkt, aeq_info);
3126                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3127                 if (ddp_seg_len) {
3128                         copy_len = 2;
3129                         termhdr->hdrct = DDP_LEN_FLAG;
3130                         if (pkt[2] & 0x80) {
3131                                 is_tagged = 1;
3132                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3133                                         copy_len += TERM_DDP_LEN_TAGGED;
3134                                         termhdr->hdrct |= DDP_HDR_FLAG;
3135                                 }
3136                         } else {
3137                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3138                                         copy_len += TERM_DDP_LEN_UNTAGGED;
3139                                         termhdr->hdrct |= DDP_HDR_FLAG;
3140                                 }
3141
3142                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3143                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3144                                                 copy_len += TERM_RDMA_LEN;
3145                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
3146                                         }
3147                                 }
3148                         }
3149                 }
3150         }
3151
3152         switch (async_event_id) {
3153         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3154                 switch (iwarp_opcode(nesqp, aeq_info)) {
3155                 case IWARP_OPCODE_WRITE:
3156                         flush_code = IB_WC_LOC_PROT_ERR;
3157                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3158                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3159                         break;
3160                 default:
3161                         flush_code = IB_WC_REM_ACCESS_ERR;
3162                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3163                         termhdr->error_code = RDMAP_INV_STAG;
3164                 }
3165                 break;
3166         case NES_AEQE_AEID_AMP_INVALID_STAG:
3167                 flush_code = IB_WC_REM_ACCESS_ERR;
3168                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3169                 termhdr->error_code = RDMAP_INV_STAG;
3170                 break;
3171         case NES_AEQE_AEID_AMP_BAD_QP:
3172                 flush_code = IB_WC_LOC_QP_OP_ERR;
3173                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3174                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3175                 break;
3176         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3177         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3178                 switch (iwarp_opcode(nesqp, aeq_info)) {
3179                 case IWARP_OPCODE_SEND_INV:
3180                 case IWARP_OPCODE_SEND_SE_INV:
3181                         flush_code = IB_WC_REM_OP_ERR;
3182                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3183                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3184                         break;
3185                 default:
3186                         flush_code = IB_WC_REM_ACCESS_ERR;
3187                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3188                         termhdr->error_code = RDMAP_INV_STAG;
3189                 }
3190                 break;
3191         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3192                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3193                         flush_code = IB_WC_LOC_PROT_ERR;
3194                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3195                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3196                 } else {
3197                         flush_code = IB_WC_REM_ACCESS_ERR;
3198                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3199                         termhdr->error_code = RDMAP_INV_BOUNDS;
3200                 }
3201                 break;
3202         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3203         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3204         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3205                 flush_code = IB_WC_REM_ACCESS_ERR;
3206                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3207                 termhdr->error_code = RDMAP_ACCESS;
3208                 break;
3209         case NES_AEQE_AEID_AMP_TO_WRAP:
3210                 flush_code = IB_WC_REM_ACCESS_ERR;
3211                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3212                 termhdr->error_code = RDMAP_TO_WRAP;
3213                 break;
3214         case NES_AEQE_AEID_AMP_BAD_PD:
3215                 switch (iwarp_opcode(nesqp, aeq_info)) {
3216                 case IWARP_OPCODE_WRITE:
3217                         flush_code = IB_WC_LOC_PROT_ERR;
3218                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3219                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3220                         break;
3221                 case IWARP_OPCODE_SEND_INV:
3222                 case IWARP_OPCODE_SEND_SE_INV:
3223                         flush_code = IB_WC_REM_ACCESS_ERR;
3224                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3225                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3226                         break;
3227                 default:
3228                         flush_code = IB_WC_REM_ACCESS_ERR;
3229                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3230                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3231                 }
3232                 break;
3233         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3234                 flush_code = IB_WC_LOC_LEN_ERR;
3235                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3236                 termhdr->error_code = MPA_MARKER;
3237                 break;
3238         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3239                 flush_code = IB_WC_GENERAL_ERR;
3240                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3241                 termhdr->error_code = MPA_CRC;
3242                 break;
3243         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3244         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3245                 flush_code = IB_WC_LOC_LEN_ERR;
3246                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3247                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3248                 break;
3249         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3250         case NES_AEQE_AEID_DDP_NO_L_BIT:
3251                 flush_code = IB_WC_FATAL_ERR;
3252                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3253                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3254                 break;
3255         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3256         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3257                 flush_code = IB_WC_GENERAL_ERR;
3258                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3259                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3260                 break;
3261         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3262                 flush_code = IB_WC_LOC_LEN_ERR;
3263                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3264                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3265                 break;
3266         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3267                 flush_code = IB_WC_GENERAL_ERR;
3268                 if (is_tagged) {
3269                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3270                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3271                 } else {
3272                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3273                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3274                 }
3275                 break;
3276         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3277                 flush_code = IB_WC_GENERAL_ERR;
3278                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3279                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3280                 break;
3281         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3282                 flush_code = IB_WC_REM_OP_ERR;
3283                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3284                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3285                 break;
3286         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3287                 flush_code = IB_WC_GENERAL_ERR;
3288                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3289                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3290                 break;
3291         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3292                 flush_code = IB_WC_GENERAL_ERR;
3293                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3294                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3295                 break;
3296         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3297                 flush_code = IB_WC_LOC_QP_OP_ERR;
3298                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3299                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3300                 break;
3301         default:
3302                 flush_code = IB_WC_FATAL_ERR;
3303                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3304                 termhdr->error_code = RDMAP_UNSPECIFIED;
3305                 break;
3306         }
3307
3308         if (copy_len)
3309                 memcpy(termhdr + 1, pkt, copy_len);
3310
3311         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3312                 if (aeq_info & NES_AEQE_SQ)
3313                         nesqp->term_sq_flush_code = flush_code;
3314                 else
3315                         nesqp->term_rq_flush_code = flush_code;
3316         }
3317
3318         return sizeof(struct nes_terminate_hdr) + copy_len;
3319 }
3320
3321 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3322                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3323 {
3324         u64 context;
3325         unsigned long flags;
3326         u32 aeq_info;
3327         u16 async_event_id;
3328         u8 tcp_state;
3329         u8 iwarp_state;
3330         u32 termlen = 0;
3331         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3332                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3333         struct nes_adapter *nesadapter = nesdev->nesadapter;
3334
3335         if (nesqp->term_flags & NES_TERM_SENT)
3336                 return; /* Sanity check */
3337
3338         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3339         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3340         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3341         async_event_id = (u16)aeq_info;
3342
3343         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3344                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3345         if (!context) {
3346                 WARN_ON(!context);
3347                 return;
3348         }
3349
3350         nesqp = (struct nes_qp *)(unsigned long)context;
3351         spin_lock_irqsave(&nesqp->lock, flags);
3352         nesqp->hw_iwarp_state = iwarp_state;
3353         nesqp->hw_tcp_state = tcp_state;
3354         nesqp->last_aeq = async_event_id;
3355         nesqp->terminate_eventtype = eventtype;
3356         spin_unlock_irqrestore(&nesqp->lock, flags);
3357
3358         if (nesadapter->send_term_ok)
3359                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3360         else
3361                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3362
3363         if (!nesdev->iw_status)  {
3364                 nesqp->term_flags = NES_TERM_DONE;
3365                 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3366                 nes_cm_disconn(nesqp);
3367         } else {
3368                 nes_terminate_start_timer(nesqp);
3369                 nesqp->term_flags |= NES_TERM_SENT;
3370                 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3371         }
3372 }
3373
3374 static void nes_terminate_send_fin(struct nes_device *nesdev,
3375                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3376 {
3377         u32 aeq_info;
3378         u16 async_event_id;
3379         u8 tcp_state;
3380         u8 iwarp_state;
3381         unsigned long flags;
3382
3383         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3384         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3385         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3386         async_event_id = (u16)aeq_info;
3387
3388         spin_lock_irqsave(&nesqp->lock, flags);
3389         nesqp->hw_iwarp_state = iwarp_state;
3390         nesqp->hw_tcp_state = tcp_state;
3391         nesqp->last_aeq = async_event_id;
3392         spin_unlock_irqrestore(&nesqp->lock, flags);
3393
3394         /* Send the fin only */
3395         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3396                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3397 }
3398
3399 /* Cleanup after a terminate sent or received */
3400 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3401 {
3402         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3403         unsigned long flags;
3404         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3405         struct nes_device *nesdev = nesvnic->nesdev;
3406         u8 first_time = 0;
3407
3408         spin_lock_irqsave(&nesqp->lock, flags);
3409         if (nesqp->hte_added) {
3410                 nesqp->hte_added = 0;
3411                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3412         }
3413
3414         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3415         nesqp->term_flags |= NES_TERM_DONE;
3416         spin_unlock_irqrestore(&nesqp->lock, flags);
3417
3418         /* Make sure we go through this only once */
3419         if (first_time) {
3420                 if (timeout_occurred == 0)
3421                         del_timer(&nesqp->terminate_timer);
3422                 else
3423                         next_iwarp_state |= NES_CQP_QP_RESET;
3424
3425                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3426                 nes_cm_disconn(nesqp);
3427         }
3428 }
3429
3430 static void nes_terminate_received(struct nes_device *nesdev,
3431                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3432 {
3433         u32 aeq_info;
3434         u8 *pkt;
3435         u32 *mpa;
3436         u8 ddp_ctl;
3437         u8 rdma_ctl;
3438         u16 aeq_id = 0;
3439
3440         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3441         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3442                 /* Terminate is not a performance path so the silicon */
3443                 /* did not validate the frame - do it now */
3444                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3445                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3446                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3447                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3448                 if ((ddp_ctl & 0xc0) != 0x40)
3449                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3450                 else if ((ddp_ctl & 0x03) != 1)
3451                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3452                 else if (be32_to_cpu(mpa[2]) != 2)
3453                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3454                 else if (be32_to_cpu(mpa[3]) != 1)
3455                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3456                 else if (be32_to_cpu(mpa[4]) != 0)
3457                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3458                 else if ((rdma_ctl & 0xc0) != 0x40)
3459                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3460
3461                 if (aeq_id) {
3462                         /* Bad terminate recvd - send back a terminate */
3463                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3464                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3465                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3466                         return;
3467                 }
3468         }
3469
3470         nesqp->term_flags |= NES_TERM_RCVD;
3471         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3472         nes_terminate_start_timer(nesqp);
3473         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3474 }
3475
3476 /* Timeout routine in case terminate fails to complete */
3477 void nes_terminate_timeout(unsigned long context)
3478 {
3479         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3480
3481         nes_terminate_done(nesqp, 1);
3482 }
3483
3484 /* Set a timer in case hw cannot complete the terminate sequence */
3485 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3486 {
3487         mod_timer(&nesqp->terminate_timer, (jiffies + HZ));
3488 }
3489
3490 /**
3491  * nes_process_iwarp_aeqe
3492  */
3493 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3494                                    struct nes_hw_aeqe *aeqe)
3495 {
3496         u64 context;
3497         unsigned long flags;
3498         struct nes_qp *nesqp;
3499         struct nes_hw_cq *hw_cq;
3500         struct nes_cq *nescq;
3501         int resource_allocated;
3502         struct nes_adapter *nesadapter = nesdev->nesadapter;
3503         u32 aeq_info;
3504         u32 next_iwarp_state = 0;
3505         u32 aeqe_cq_id;
3506         u16 async_event_id;
3507         u8 tcp_state;
3508         u8 iwarp_state;
3509         struct ib_event ibevent;
3510
3511         nes_debug(NES_DBG_AEQ, "\n");
3512         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3513         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3514                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3515                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3516         } else {
3517                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3518                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3519                 BUG_ON(!context);
3520         }
3521
3522         /* context is nesqp unless async_event_id == CQ ERROR */
3523         nesqp = (struct nes_qp *)(unsigned long)context;
3524         async_event_id = (u16)aeq_info;
3525         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3526         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3527         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3528                         " Tcp state = %s, iWARP state = %s\n",
3529                         async_event_id,
3530                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3531                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3532
3533         aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3534         if (aeq_info & NES_AEQE_QP) {
3535                 if (!nes_is_resource_allocated(nesadapter,
3536                                 nesadapter->allocated_qps,
3537                                 aeqe_cq_id))
3538                         return;
3539         }
3540
3541         switch (async_event_id) {
3542                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3543                         if (nesqp->term_flags)
3544                                 return; /* Ignore it, wait for close complete */
3545
3546                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3547                                 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3548                                         (nesqp->ibqp_state == IB_QPS_RTS)) {
3549                                         spin_lock_irqsave(&nesqp->lock, flags);
3550                                         nesqp->hw_iwarp_state = iwarp_state;
3551                                         nesqp->hw_tcp_state = tcp_state;
3552                                         nesqp->last_aeq = async_event_id;
3553                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3554                                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3555                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3556                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3557                                         nes_cm_disconn(nesqp);
3558                                 }
3559                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3560                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3561                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3562                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3563                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3564                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3565                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3566                                                 async_event_id, nesqp->last_aeq, tcp_state);
3567                         }
3568                         break;
3569                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3570                         spin_lock_irqsave(&nesqp->lock, flags);
3571                         nesqp->hw_iwarp_state = iwarp_state;
3572                         nesqp->hw_tcp_state = tcp_state;
3573                         nesqp->last_aeq = async_event_id;
3574                         spin_unlock_irqrestore(&nesqp->lock, flags);
3575                         nes_cm_disconn(nesqp);
3576                         break;
3577
3578                 case NES_AEQE_AEID_RESET_SENT:
3579                         tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3580                         spin_lock_irqsave(&nesqp->lock, flags);
3581                         nesqp->hw_iwarp_state = iwarp_state;
3582                         nesqp->hw_tcp_state = tcp_state;
3583                         nesqp->last_aeq = async_event_id;
3584                         nesqp->hte_added = 0;
3585                         spin_unlock_irqrestore(&nesqp->lock, flags);
3586                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3587                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3588                         nes_cm_disconn(nesqp);
3589                         break;
3590
3591                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3592                         if (atomic_read(&nesqp->close_timer_started))
3593                                 return;
3594                         spin_lock_irqsave(&nesqp->lock, flags);
3595                         nesqp->hw_iwarp_state = iwarp_state;
3596                         nesqp->hw_tcp_state = tcp_state;
3597                         nesqp->last_aeq = async_event_id;
3598                         spin_unlock_irqrestore(&nesqp->lock, flags);
3599                         nes_cm_disconn(nesqp);
3600                         break;
3601
3602                 case NES_AEQE_AEID_TERMINATE_SENT:
3603                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3604                         break;
3605
3606                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3607                         nes_terminate_received(nesdev, nesqp, aeqe);
3608                         break;
3609
3610                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3611                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3612                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3613                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3614                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3615                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3616                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3617                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3618                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3619                 case NES_AEQE_AEID_AMP_TO_WRAP:
3620                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3621                                         nesqp->hwqp.qp_id, async_event_id);
3622                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3623                         break;
3624
3625                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3626                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3627                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3628                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3629                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3630                                 aeq_info &= 0xffff0000;
3631                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3632                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3633                         }
3634
3635                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3636                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3637                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3638                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3639                 case NES_AEQE_AEID_AMP_BAD_QP:
3640                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3641                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3642                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3643                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3644                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3645                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3646                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3647                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3648                 case NES_AEQE_AEID_AMP_BAD_PD:
3649                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3650                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3651                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3652                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3653                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3654                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3655                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3656                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3657                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3658                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3659                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3660                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3661                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3662                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3663                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3664                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3665                 case NES_AEQE_AEID_BAD_CLOSE:
3666                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3667                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3668                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3669                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3670                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3671                                         nesqp->hwqp.qp_id, async_event_id);
3672                         print_ip(nesqp->cm_node);
3673                         if (!atomic_read(&nesqp->close_timer_started))
3674                                 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3675                         break;
3676
3677                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3678                         context <<= 1;
3679                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3680                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3681                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3682                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3683                         if (resource_allocated) {
3684                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3685                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3686                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3687                                 if (hw_cq) {
3688                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3689                                         if (nescq->ibcq.event_handler) {
3690                                                 ibevent.device = nescq->ibcq.device;
3691                                                 ibevent.event = IB_EVENT_CQ_ERR;
3692                                                 ibevent.element.cq = &nescq->ibcq;
3693                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3694                                         }
3695                                 }
3696                         }
3697                         break;
3698
3699                 default:
3700                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3701                                         async_event_id);
3702                         break;
3703         }
3704
3705 }
3706
3707 /**
3708  * nes_iwarp_ce_handler
3709  */
3710 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3711 {
3712         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3713
3714         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3715                         nescq->hw_cq.cq_number); */
3716         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3717
3718         if (nescq->ibcq.comp_handler)
3719                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3720
3721         return;
3722 }
3723
3724
3725 /**
3726  * nes_manage_apbvt()
3727  */
3728 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3729                 u32 nic_index, u32 add_port)
3730 {
3731         struct nes_device *nesdev = nesvnic->nesdev;
3732         struct nes_hw_cqp_wqe *cqp_wqe;
3733         struct nes_cqp_request *cqp_request;
3734         int ret = 0;
3735         u16 major_code;
3736
3737         /* Send manage APBVT request to CQP */
3738         cqp_request = nes_get_cqp_request(nesdev);
3739         if (cqp_request == NULL) {
3740                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3741                 return -ENOMEM;
3742         }
3743         cqp_request->waiting = 1;
3744         cqp_wqe = &cqp_request->cqp_wqe;
3745
3746         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3747                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3748                         accel_local_port, accel_local_port, nic_index);
3749
3750         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3751         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3752                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3753         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3754                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3755
3756         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3757
3758         atomic_set(&cqp_request->refcount, 2);
3759         nes_post_cqp_request(nesdev, cqp_request);
3760
3761         if (add_port == NES_MANAGE_APBVT_ADD)
3762                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3763                                 NES_EVENT_TIMEOUT);
3764         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3765                         ret, cqp_request->major_code, cqp_request->minor_code);
3766         major_code = cqp_request->major_code;
3767
3768         nes_put_cqp_request(nesdev, cqp_request);
3769
3770         if (!ret)
3771                 return -ETIME;
3772         else if (major_code)
3773                 return -EIO;
3774         else
3775                 return 0;
3776 }
3777
3778
3779 /**
3780  * nes_manage_arp_cache
3781  */
3782 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3783                 u32 ip_addr, u32 action)
3784 {
3785         struct nes_hw_cqp_wqe *cqp_wqe;
3786         struct nes_vnic *nesvnic = netdev_priv(netdev);
3787         struct nes_device *nesdev;
3788         struct nes_cqp_request *cqp_request;
3789         int arp_index;
3790
3791         nesdev = nesvnic->nesdev;
3792         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3793         if (arp_index == -1) {
3794                 return;
3795         }
3796
3797         /* update the ARP entry */
3798         cqp_request = nes_get_cqp_request(nesdev);
3799         if (cqp_request == NULL) {
3800                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3801                 return;
3802         }
3803         cqp_request->waiting = 0;
3804         cqp_wqe = &cqp_request->cqp_wqe;
3805         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3806
3807         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3808                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3809         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3810                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3811         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3812
3813         if (action == NES_ARP_ADD) {
3814                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3815                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3816                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3817                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3818                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3819                                 (((u32)mac_addr[0]) << 8) | (u32)mac_addr[1]);
3820         } else {
3821                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3822                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3823         }
3824
3825         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3826                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3827
3828         atomic_set(&cqp_request->refcount, 1);
3829         nes_post_cqp_request(nesdev, cqp_request);
3830 }
3831
3832
3833 /**
3834  * flush_wqes
3835  */
3836 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3837                 u32 which_wq, u32 wait_completion)
3838 {
3839         struct nes_cqp_request *cqp_request;
3840         struct nes_hw_cqp_wqe *cqp_wqe;
3841         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3842         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3843         int ret;
3844
3845         cqp_request = nes_get_cqp_request(nesdev);
3846         if (cqp_request == NULL) {
3847                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3848                 return;
3849         }
3850         if (wait_completion) {
3851                 cqp_request->waiting = 1;
3852                 atomic_set(&cqp_request->refcount, 2);
3853         } else {
3854                 cqp_request->waiting = 0;
3855         }
3856         cqp_wqe = &cqp_request->cqp_wqe;
3857         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3858
3859         /* If wqe in error was identified, set code to be put into cqe */
3860         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3861                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3862                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3863                 nesqp->term_sq_flush_code = 0;
3864         }
3865
3866         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3867                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3868                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3869                 nesqp->term_rq_flush_code = 0;
3870         }
3871
3872         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3873                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3874                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3875         }
3876
3877         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3878                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3879         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3880
3881         nes_post_cqp_request(nesdev, cqp_request);
3882
3883         if (wait_completion) {
3884                 /* Wait for CQP */
3885                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3886                                 NES_EVENT_TIMEOUT);
3887                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3888                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3889                                 ret, cqp_request->major_code, cqp_request->minor_code);
3890                 nes_put_cqp_request(nesdev, cqp_request);
3891         }
3892 }