drivers/net/ethernet/cavium/liquidio/lio_main.c

   1 /**********************************************************************
   2 * Author: Cavium, Inc.
   3 *
   4 * Contact: support@cavium.com
   5 *          Please include "LiquidIO" in the subject.
   6 *
   7 * Copyright (c) 2003-2015 Cavium, Inc.
   8 *
   9 * This file is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License, Version 2, as
  11 * published by the Free Software Foundation.
  12 *
  13 * This file is distributed in the hope that it will be useful, but
  14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16 * NONINFRINGEMENT.  See the GNU General Public License for more
  17 * details.
  18 *
  19 * This file may also be available under a different license from Cavium.
  20 * Contact Cavium, Inc. for more information
  21 **********************************************************************/
  22 #include <linux/version.h>
  23 #include <linux/module.h>
  24 #include <linux/crc32.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/pci.h>
  27 #include <linux/pci_ids.h>
  28 #include <linux/ip.h>
  29 #include <net/ip.h>
  30 #include <linux/ipv6.h>
  31 #include <linux/net_tstamp.h>
  32 #include <linux/if_vlan.h>
  33 #include <linux/firmware.h>
  34 #include <linux/ethtool.h>
  35 #include <linux/ptp_clock_kernel.h>
  36 #include <linux/types.h>
  37 #include <linux/list.h>
  38 #include <linux/workqueue.h>
  39 #include <linux/interrupt.h>
  40 #include "octeon_config.h"
  41 #include "liquidio_common.h"
  42 #include "octeon_droq.h"
  43 #include "octeon_iq.h"
  44 #include "response_manager.h"
  45 #include "octeon_device.h"
  46 #include "octeon_nic.h"
  47 #include "octeon_main.h"
  48 #include "octeon_network.h"
  49 #include "cn66xx_regs.h"
  50 #include "cn66xx_device.h"
  51 #include "cn68xx_regs.h"
  52 #include "cn68xx_device.h"
  53 #include "liquidio_image.h"
  54
  55 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
  56 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
  57 MODULE_LICENSE("GPL");
  58 MODULE_VERSION(LIQUIDIO_VERSION);
  59 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX);
  60 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX);
  61 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX);
  62
  63 static int ddr_timeout = 10000;
  64 module_param(ddr_timeout, int, 0644);
  65 MODULE_PARM_DESC(ddr_timeout,
  66                  "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
  67
  68 static u32 console_bitmask;
  69 module_param(console_bitmask, int, 0644);
  70 MODULE_PARM_DESC(console_bitmask,
  71                  "Bitmask indicating which consoles have debug output redirected to syslog.");
  72
  73 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  74
  75 static int debug = -1;
  76 module_param(debug, int, 0644);
  77 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
  78
  79 static char fw_type[LIO_MAX_FW_TYPE_LEN];
  80 module_param_string(fw_type, fw_type, sizeof(fw_type), 0000);
  81 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\"");
  82
  83 static int conf_type;
  84 module_param(conf_type, int, 0);
  85 MODULE_PARM_DESC(conf_type, "select octeon configuration 0 default 1 ovs");
  86
  87 /* Bit mask values for lio->ifstate */
  88 #define   LIO_IFSTATE_DROQ_OPS             0x01
  89 #define   LIO_IFSTATE_REGISTERED           0x02
  90 #define   LIO_IFSTATE_RUNNING              0x04
  91 #define   LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08
  92
  93 /* Polling interval for determining when NIC application is alive */
  94 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
  95
  96 /* runtime link query interval */
  97 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
  98
  99 struct liquidio_if_cfg_context {
 100         int octeon_id;
 101
 102         wait_queue_head_t wc;
 103
 104         int cond;
 105 };
 106
 107 struct liquidio_if_cfg_resp {
 108         u64 rh;
 109         struct liquidio_if_cfg_info cfg_info;
 110         u64 status;
 111 };
 112
 113 struct oct_link_status_resp {
 114         u64 rh;
 115         struct oct_link_info link_info;
 116         u64 status;
 117 };
 118
 119 struct oct_timestamp_resp {
 120         u64 rh;
 121         u64 timestamp;
 122         u64 status;
 123 };
 124
 125 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
 126
 127 union tx_info {
 128         u64 u64;
 129         struct {
 130 #ifdef __BIG_ENDIAN_BITFIELD
 131                 u16 gso_size;
 132                 u16 gso_segs;
 133                 u32 reserved;
 134 #else
 135                 u32 reserved;
 136                 u16 gso_segs;
 137                 u16 gso_size;
 138 #endif
 139         } s;
 140 };
 141
 142 /** Octeon device properties to be used by the NIC module.
 143  * Each octeon device in the system will be represented
 144  * by this structure in the NIC module.
 145  */
 146
 147 #define OCTNIC_MAX_SG  (MAX_SKB_FRAGS)
 148
 149 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
 150 #define OCTNIC_GSO_MAX_SIZE (GSO_MAX_SIZE - OCTNIC_GSO_MAX_HEADER_SIZE)
 151
 152 /** Structure of a node in list of gather components maintained by
 153  * NIC driver for each network device.
 154  */
 155 struct octnic_gather {
 156         /** List manipulation. Next and prev pointers. */
 157         struct list_head list;
 158
 159         /** Size of the gather component at sg in bytes. */
 160         int sg_size;
 161
 162         /** Number of bytes that sg was adjusted to make it 8B-aligned. */
 163         int adjust;
 164
 165         /** Gather component that can accommodate max sized fragment list
 166          *  received from the IP layer.
 167          */
 168         struct octeon_sg_entry *sg;
 169 };
 170
 171 /** This structure is used by NIC driver to store information required
 172  * to free the sk_buff when the packet has been fetched by Octeon.
 173  * Bytes offset below assume worst-case of a 64-bit system.
 174  */
 175 struct octnet_buf_free_info {
 176         /** Bytes 1-8.  Pointer to network device private structure. */
 177         struct lio *lio;
 178
 179         /** Bytes 9-16.  Pointer to sk_buff. */
 180         struct sk_buff *skb;
 181
 182         /** Bytes 17-24.  Pointer to gather list. */
 183         struct octnic_gather *g;
 184
 185         /** Bytes 25-32. Physical address of skb->data or gather list. */
 186         u64 dptr;
 187
 188         /** Bytes 33-47. Piggybacked soft command, if any */
 189         struct octeon_soft_command *sc;
 190 };
 191
 192 struct handshake {
 193         struct completion init;
 194         struct completion started;
 195         struct pci_dev *pci_dev;
 196         int init_ok;
 197         int started_ok;
 198 };
 199
 200 struct octeon_device_priv {
 201         /** Tasklet structures for this device. */
 202         struct tasklet_struct droq_tasklet;
 203         unsigned long napi_mask;
 204 };
 205
 206 static int octeon_device_init(struct octeon_device *);
 207 static void liquidio_remove(struct pci_dev *pdev);
 208 static int liquidio_probe(struct pci_dev *pdev,
 209                           const struct pci_device_id *ent);
 210
 211 static struct handshake handshake[MAX_OCTEON_DEVICES];
 212 static struct completion first_stage;
 213
 214 static void octeon_droq_bh(unsigned long pdev)
 215 {
 216         int q_no;
 217         int reschedule = 0;
 218         struct octeon_device *oct = (struct octeon_device *)pdev;
 219         struct octeon_device_priv *oct_priv =
 220                 (struct octeon_device_priv *)oct->priv;
 221
 222         /* for (q_no = 0; q_no < oct->num_oqs; q_no++) { */
 223         for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES; q_no++) {
 224                 if (!(oct->io_qmask.oq & (1UL << q_no)))
 225                         continue;
 226                 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
 227                                                           MAX_PACKET_BUDGET);
 228         }
 229
 230         if (reschedule)
 231                 tasklet_schedule(&oct_priv->droq_tasklet);
 232 }
 233
 234 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 235 {
 236         struct octeon_device_priv *oct_priv =
 237                 (struct octeon_device_priv *)oct->priv;
 238         int retry = 100, pkt_cnt = 0, pending_pkts = 0;
 239         int i;
 240
 241         do {
 242                 pending_pkts = 0;
 243
 244                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) {
 245                         if (!(oct->io_qmask.oq & (1UL << i)))
 246                                 continue;
 247                         pkt_cnt += octeon_droq_check_hw_for_pkts(oct,
 248                                                                  oct->droq[i]);
 249                 }
 250                 if (pkt_cnt > 0) {
 251                         pending_pkts += pkt_cnt;
 252                         tasklet_schedule(&oct_priv->droq_tasklet);
 253                 }
 254                 pkt_cnt = 0;
 255                 schedule_timeout_uninterruptible(1);
 256
 257         } while (retry-- && pending_pkts);
 258
 259         return pkt_cnt;
 260 }
 261
 262 void octeon_report_tx_completion_to_bql(void *txq, unsigned int pkts_compl,
 263                                         unsigned int bytes_compl)
 264 {
 265         struct netdev_queue *netdev_queue = txq;
 266
 267         netdev_tx_completed_queue(netdev_queue, pkts_compl, bytes_compl);
 268 }
 269
 270 void octeon_update_tx_completion_counters(void *buf, int reqtype,
 271                                           unsigned int *pkts_compl,
 272                                           unsigned int *bytes_compl)
 273 {
 274         struct octnet_buf_free_info *finfo;
 275         struct sk_buff *skb = NULL;
 276         struct octeon_soft_command *sc;
 277
 278         switch (reqtype) {
 279         case REQTYPE_NORESP_NET:
 280         case REQTYPE_NORESP_NET_SG:
 281                 finfo = buf;
 282                 skb = finfo->skb;
 283                 break;
 284
 285         case REQTYPE_RESP_NET_SG:
 286         case REQTYPE_RESP_NET:
 287                 sc = buf;
 288                 skb = sc->callback_arg;
 289                 break;
 290
 291         default:
 292                 return;
 293         }
 294
 295         (*pkts_compl)++;
 296         *bytes_compl += skb->len;
 297 }
 298
 299 void octeon_report_sent_bytes_to_bql(void *buf, int reqtype)
 300 {
 301         struct octnet_buf_free_info *finfo;
 302         struct sk_buff *skb;
 303         struct octeon_soft_command *sc;
 304         struct netdev_queue *txq;
 305
 306         switch (reqtype) {
 307         case REQTYPE_NORESP_NET:
 308         case REQTYPE_NORESP_NET_SG:
 309                 finfo = buf;
 310                 skb = finfo->skb;
 311                 break;
 312
 313         case REQTYPE_RESP_NET_SG:
 314         case REQTYPE_RESP_NET:
 315                 sc = buf;
 316                 skb = sc->callback_arg;
 317                 break;
 318
 319         default:
 320                 return;
 321         }
 322
 323         txq = netdev_get_tx_queue(skb->dev, skb_get_queue_mapping(skb));
 324         netdev_tx_sent_queue(txq, skb->len);
 325 }
 326
 327 int octeon_console_debug_enabled(u32 console)
 328 {
 329         return (console_bitmask >> (console)) & 0x1;
 330 }
 331
 332 /**
 333  * \brief Forces all IO queues off on a given device
 334  * @param oct Pointer to Octeon device
 335  */
 336 static void force_io_queues_off(struct octeon_device *oct)
 337 {
 338         if ((oct->chip_id == OCTEON_CN66XX) ||
 339             (oct->chip_id == OCTEON_CN68XX)) {
 340                 /* Reset the Enable bits for Input Queues. */
 341                 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
 342
 343                 /* Reset the Enable bits for Output Queues. */
 344                 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
 345         }
 346 }
 347
 348 /**
 349  * \brief wait for all pending requests to complete
 350  * @param oct Pointer to Octeon device
 351  *
 352  * Called during shutdown sequence
 353  */
 354 static int wait_for_pending_requests(struct octeon_device *oct)
 355 {
 356         int i, pcount = 0;
 357
 358         for (i = 0; i < 100; i++) {
 359                 pcount =
 360                         atomic_read(&oct->response_list
 361                                 [OCTEON_ORDERED_SC_LIST].pending_req_count);
 362                 if (pcount)
 363                         schedule_timeout_uninterruptible(HZ / 10);
 364                  else
 365                         break;
 366         }
 367
 368         if (pcount)
 369                 return 1;
 370
 371         return 0;
 372 }
 373
 374 /**
 375  * \brief Cause device to go quiet so it can be safely removed/reset/etc
 376  * @param oct Pointer to Octeon device
 377  */
 378 static inline void pcierror_quiesce_device(struct octeon_device *oct)
 379 {
 380         int i;
 381
 382         /* Disable the input and output queues now. No more packets will
 383          * arrive from Octeon, but we should wait for all packet processing
 384          * to finish.
 385          */
 386         force_io_queues_off(oct);
 387
 388         /* To allow for in-flight requests */
 389         schedule_timeout_uninterruptible(100);
 390
 391         if (wait_for_pending_requests(oct))
 392                 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
 393
 394         /* Force all requests waiting to be fetched by OCTEON to complete. */
 395         for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) {
 396                 struct octeon_instr_queue *iq;
 397
 398                 if (!(oct->io_qmask.iq & (1UL << i)))
 399                         continue;
 400                 iq = oct->instr_queue[i];
 401
 402                 if (atomic_read(&iq->instr_pending)) {
 403                         spin_lock_bh(&iq->lock);
 404                         iq->fill_cnt = 0;
 405                         iq->octeon_read_index = iq->host_write_index;
 406                         iq->stats.instr_processed +=
 407                                 atomic_read(&iq->instr_pending);
 408                         lio_process_iq_request_list(oct, iq);
 409                         spin_unlock_bh(&iq->lock);
 410                 }
 411         }
 412
 413         /* Force all pending ordered list requests to time out. */
 414         lio_process_ordered_list(oct, 1);
 415
 416         /* We do not need to wait for output queue packets to be processed. */
 417 }
 418
 419 /**
 420  * \brief Cleanup PCI AER uncorrectable error status
 421  * @param dev Pointer to PCI device
 422  */
 423 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
 424 {
 425         int pos = 0x100;
 426         u32 status, mask;
 427
 428         pr_info("%s :\n", __func__);
 429
 430         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 431         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
 432         if (dev->error_state == pci_channel_io_normal)
 433                 status &= ~mask;        /* Clear corresponding nonfatal bits */
 434         else
 435                 status &= mask;         /* Clear corresponding fatal bits */
 436         pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
 437 }
 438
 439 /**
 440  * \brief Stop all PCI IO to a given device
 441  * @param dev Pointer to Octeon device
 442  */
 443 static void stop_pci_io(struct octeon_device *oct)
 444 {
 445         /* No more instructions will be forwarded. */
 446         atomic_set(&oct->status, OCT_DEV_IN_RESET);
 447
 448         pci_disable_device(oct->pci_dev);
 449
 450         /* Disable interrupts  */
 451         oct->fn_list.disable_interrupt(oct->chip);
 452
 453         pcierror_quiesce_device(oct);
 454
 455         /* Release the interrupt line */
 456         free_irq(oct->pci_dev->irq, oct);
 457
 458         if (oct->flags & LIO_FLAG_MSI_ENABLED)
 459                 pci_disable_msi(oct->pci_dev);
 460
 461         dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 462                 lio_get_state_string(&oct->status));
 463
 464         /* cn63xx_cleanup_aer_uncorrect_error_status(oct->pci_dev); */
 465         /* making it a common function for all OCTEON models */
 466         cleanup_aer_uncorrect_error_status(oct->pci_dev);
 467 }
 468
 469 /**
 470  * \brief called when PCI error is detected
 471  * @param pdev Pointer to PCI device
 472  * @param state The current pci connection state
 473  *
 474  * This function is called after a PCI bus error affecting
 475  * this device has been detected.
 476  */
 477 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
 478                                                      pci_channel_state_t state)
 479 {
 480         struct octeon_device *oct = pci_get_drvdata(pdev);
 481
 482         /* Non-correctable Non-fatal errors */
 483         if (state == pci_channel_io_normal) {
 484                 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
 485                 cleanup_aer_uncorrect_error_status(oct->pci_dev);
 486                 return PCI_ERS_RESULT_CAN_RECOVER;
 487         }
 488
 489         /* Non-correctable Fatal errors */
 490         dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
 491         stop_pci_io(oct);
 492
 493         /* Always return a DISCONNECT. There is no support for recovery but only
 494          * for a clean shutdown.
 495          */
 496         return PCI_ERS_RESULT_DISCONNECT;
 497 }
 498
 499 /**
 500  * \brief mmio handler
 501  * @param pdev Pointer to PCI device
 502  */
 503 static pci_ers_result_t liquidio_pcie_mmio_enabled(struct pci_dev *pdev)
 504 {
 505         /* We should never hit this since we never ask for a reset for a Fatal
 506          * Error. We always return DISCONNECT in io_error above.
 507          * But play safe and return RECOVERED for now.
 508          */
 509         return PCI_ERS_RESULT_RECOVERED;
 510 }
 511
 512 /**
 513  * \brief called after the pci bus has been reset.
 514  * @param pdev Pointer to PCI device
 515  *
 516  * Restart the card from scratch, as if from a cold-boot. Implementation
 517  * resembles the first-half of the octeon_resume routine.
 518  */
 519 static pci_ers_result_t liquidio_pcie_slot_reset(struct pci_dev *pdev)
 520 {
 521         /* We should never hit this since we never ask for a reset for a Fatal
 522          * Error. We always return DISCONNECT in io_error above.
 523          * But play safe and return RECOVERED for now.
 524          */
 525         return PCI_ERS_RESULT_RECOVERED;
 526 }
 527
 528 /**
 529  * \brief called when traffic can start flowing again.
 530  * @param pdev Pointer to PCI device
 531  *
 532  * This callback is called when the error recovery driver tells us that
 533  * its OK to resume normal operation. Implementation resembles the
 534  * second-half of the octeon_resume routine.
 535  */
 536 static void liquidio_pcie_resume(struct pci_dev *pdev)
 537 {
 538         /* Nothing to be done here. */
 539 }
 540
 541 #ifdef CONFIG_PM
 542 /**
 543  * \brief called when suspending
 544  * @param pdev Pointer to PCI device
 545  * @param state state to suspend to
 546  */
 547 static int liquidio_suspend(struct pci_dev *pdev, pm_message_t state)
 548 {
 549         return 0;
 550 }
 551
 552 /**
 553  * \brief called when resuming
 554  * @param pdev Pointer to PCI device
 555  */
 556 static int liquidio_resume(struct pci_dev *pdev)
 557 {
 558         return 0;
 559 }
 560 #endif
 561
 562 /* For PCI-E Advanced Error Recovery (AER) Interface */
 563 static const struct pci_error_handlers liquidio_err_handler = {
 564         .error_detected = liquidio_pcie_error_detected,
 565         .mmio_enabled   = liquidio_pcie_mmio_enabled,
 566         .slot_reset     = liquidio_pcie_slot_reset,
 567         .resume         = liquidio_pcie_resume,
 568 };
 569
 570 static const struct pci_device_id liquidio_pci_tbl[] = {
 571         {       /* 68xx */
 572                 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 573         },
 574         {       /* 66xx */
 575                 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 576         },
 577         {
 578                 0, 0, 0, 0, 0, 0, 0
 579         }
 580 };
 581 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
 582
 583 static struct pci_driver liquidio_pci_driver = {
 584         .name           = "LiquidIO",
 585         .id_table       = liquidio_pci_tbl,
 586         .probe          = liquidio_probe,
 587         .remove         = liquidio_remove,
 588         .err_handler    = &liquidio_err_handler,    /* For AER */
 589
 590 #ifdef CONFIG_PM
 591         .suspend        = liquidio_suspend,
 592         .resume         = liquidio_resume,
 593 #endif
 594
 595 };
 596
 597 /**
 598  * \brief register PCI driver
 599  */
 600 static int liquidio_init_pci(void)
 601 {
 602         return pci_register_driver(&liquidio_pci_driver);
 603 }
 604
 605 /**
 606  * \brief unregister PCI driver
 607  */
 608 static void liquidio_deinit_pci(void)
 609 {
 610         pci_unregister_driver(&liquidio_pci_driver);
 611 }
 612
 613 /**
 614  * \brief check interface state
 615  * @param lio per-network private data
 616  * @param state_flag flag state to check
 617  */
 618 static inline int ifstate_check(struct lio *lio, int state_flag)
 619 {
 620         return atomic_read(&lio->ifstate) & state_flag;
 621 }
 622
 623 /**
 624  * \brief set interface state
 625  * @param lio per-network private data
 626  * @param state_flag flag state to set
 627  */
 628 static inline void ifstate_set(struct lio *lio, int state_flag)
 629 {
 630         atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag));
 631 }
 632
 633 /**
 634  * \brief clear interface state
 635  * @param lio per-network private data
 636  * @param state_flag flag state to clear
 637  */
 638 static inline void ifstate_reset(struct lio *lio, int state_flag)
 639 {
 640         atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag)));
 641 }
 642
 643 /**
 644  * \brief Stop Tx queues
 645  * @param netdev network device
 646  */
 647 static inline void txqs_stop(struct net_device *netdev)
 648 {
 649         if (netif_is_multiqueue(netdev)) {
 650                 int i;
 651
 652                 for (i = 0; i < netdev->num_tx_queues; i++)
 653                         netif_stop_subqueue(netdev, i);
 654         } else {
 655                 netif_stop_queue(netdev);
 656         }
 657 }
 658
 659 /**
 660  * \brief Start Tx queues
 661  * @param netdev network device
 662  */
 663 static inline void txqs_start(struct net_device *netdev)
 664 {
 665         if (netif_is_multiqueue(netdev)) {
 666                 int i;
 667
 668                 for (i = 0; i < netdev->num_tx_queues; i++)
 669                         netif_start_subqueue(netdev, i);
 670         } else {
 671                 netif_start_queue(netdev);
 672         }
 673 }
 674
 675 /**
 676  * \brief Wake Tx queues
 677  * @param netdev network device
 678  */
 679 static inline void txqs_wake(struct net_device *netdev)
 680 {
 681         if (netif_is_multiqueue(netdev)) {
 682                 int i;
 683
 684                 for (i = 0; i < netdev->num_tx_queues; i++)
 685                         netif_wake_subqueue(netdev, i);
 686         } else {
 687                 netif_wake_queue(netdev);
 688         }
 689 }
 690
 691 /**
 692  * \brief Stop Tx queue
 693  * @param netdev network device
 694  */
 695 static void stop_txq(struct net_device *netdev)
 696 {
 697         txqs_stop(netdev);
 698 }
 699
 700 /**
 701  * \brief Start Tx queue
 702  * @param netdev network device
 703  */
 704 static void start_txq(struct net_device *netdev)
 705 {
 706         struct lio *lio = GET_LIO(netdev);
 707
 708         if (lio->linfo.link.s.status) {
 709                 txqs_start(netdev);
 710                 return;
 711         }
 712 }
 713
 714 /**
 715  * \brief Wake a queue
 716  * @param netdev network device
 717  * @param q which queue to wake
 718  */
 719 static inline void wake_q(struct net_device *netdev, int q)
 720 {
 721         if (netif_is_multiqueue(netdev))
 722                 netif_wake_subqueue(netdev, q);
 723         else
 724                 netif_wake_queue(netdev);
 725 }
 726
 727 /**
 728  * \brief Stop a queue
 729  * @param netdev network device
 730  * @param q which queue to stop
 731  */
 732 static inline void stop_q(struct net_device *netdev, int q)
 733 {
 734         if (netif_is_multiqueue(netdev))
 735                 netif_stop_subqueue(netdev, q);
 736         else
 737                 netif_stop_queue(netdev);
 738 }
 739
 740 /**
 741  * \brief Check Tx queue status, and take appropriate action
 742  * @param lio per-network private data
 743  * @returns 0 if full, number of queues woken up otherwise
 744  */
 745 static inline int check_txq_status(struct lio *lio)
 746 {
 747         int ret_val = 0;
 748
 749         if (netif_is_multiqueue(lio->netdev)) {
 750                 int numqs = lio->netdev->num_tx_queues;
 751                 int q, iq = 0;
 752
 753                 /* check each sub-queue state */
 754                 for (q = 0; q < numqs; q++) {
 755                         iq = lio->linfo.txpciq[q & (lio->linfo.num_txpciq - 1)];
 756                         if (octnet_iq_is_full(lio->oct_dev, iq))
 757                                 continue;
 758                         wake_q(lio->netdev, q);
 759                         ret_val++;
 760                 }
 761         } else {
 762                 if (octnet_iq_is_full(lio->oct_dev, lio->txq))
 763                         return 0;
 764                 wake_q(lio->netdev, lio->txq);
 765                 ret_val = 1;
 766         }
 767         return ret_val;
 768 }
 769
 770 /**
 771  * Remove the node at the head of the list. The list would be empty at
 772  * the end of this call if there are no more nodes in the list.
 773  */
 774 static inline struct list_head *list_delete_head(struct list_head *root)
 775 {
 776         struct list_head *node;
 777
 778         if ((root->prev == root) && (root->next == root))
 779                 node = NULL;
 780         else
 781                 node = root->next;
 782
 783         if (node)
 784                 list_del(node);
 785
 786         return node;
 787 }
 788
 789 /**
 790  * \brief Delete gather list
 791  * @param lio per-network private data
 792  */
 793 static void delete_glist(struct lio *lio)
 794 {
 795         struct octnic_gather *g;
 796
 797         do {
 798                 g = (struct octnic_gather *)
 799                     list_delete_head(&lio->glist);
 800                 if (g) {
 801                         if (g->sg)
 802                                 kfree((void *)((unsigned long)g->sg -
 803                                                 g->adjust));
 804                         kfree(g);
 805                 }
 806         } while (g);
 807 }
 808
 809 /**
 810  * \brief Setup gather list
 811  * @param lio per-network private data
 812  */
 813 static int setup_glist(struct lio *lio)
 814 {
 815         int i;
 816         struct octnic_gather *g;
 817
 818         INIT_LIST_HEAD(&lio->glist);
 819
 820         for (i = 0; i < lio->tx_qsize; i++) {
 821                 g = kzalloc(sizeof(*g), GFP_KERNEL);
 822                 if (!g)
 823                         break;
 824
 825                 g->sg_size =
 826                         ((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * OCT_SG_ENTRY_SIZE);
 827
 828                 g->sg = kmalloc(g->sg_size + 8, GFP_KERNEL);
 829                 if (!g->sg) {
 830                         kfree(g);
 831                         break;
 832                 }
 833
 834                 /* The gather component should be aligned on 64-bit boundary */
 835                 if (((unsigned long)g->sg) & 7) {
 836                         g->adjust = 8 - (((unsigned long)g->sg) & 7);
 837                         g->sg = (struct octeon_sg_entry *)
 838                                 ((unsigned long)g->sg + g->adjust);
 839                 }
 840                 list_add_tail(&g->list, &lio->glist);
 841         }
 842
 843         if (i == lio->tx_qsize)
 844                 return 0;
 845
 846         delete_glist(lio);
 847         return 1;
 848 }
 849
 850 /**
 851  * \brief Print link information
 852  * @param netdev network device
 853  */
 854 static void print_link_info(struct net_device *netdev)
 855 {
 856         struct lio *lio = GET_LIO(netdev);
 857
 858         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) {
 859                 struct oct_link_info *linfo = &lio->linfo;
 860
 861                 if (linfo->link.s.status) {
 862                         netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
 863                                    linfo->link.s.speed,
 864                                    (linfo->link.s.duplex) ? "Full" : "Half");
 865                 } else {
 866                         netif_info(lio, link, lio->netdev, "Link Down\n");
 867                 }
 868         }
 869 }
 870
 871 /**
 872  * \brief Update link status
 873  * @param netdev network device
 874  * @param ls link status structure
 875  *
 876  * Called on receipt of a link status response from the core application to
 877  * update each interface's link status.
 878  */
 879 static inline void update_link_status(struct net_device *netdev,
 880                                       union oct_link_status *ls)
 881 {
 882         struct lio *lio = GET_LIO(netdev);
 883
 884         if ((lio->intf_open) && (lio->linfo.link.u64 != ls->u64)) {
 885                 lio->linfo.link.u64 = ls->u64;
 886
 887                 print_link_info(netdev);
 888
 889                 if (lio->linfo.link.s.status) {
 890                         netif_carrier_on(netdev);
 891                         /* start_txq(netdev); */
 892                         txqs_wake(netdev);
 893                 } else {
 894                         netif_carrier_off(netdev);
 895                         stop_txq(netdev);
 896                 }
 897         }
 898 }
 899
 900 /**
 901  * \brief Droq packet processor sceduler
 902  * @param oct octeon device
 903  */
 904 static
 905 void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
 906 {
 907         struct octeon_device_priv *oct_priv =
 908                 (struct octeon_device_priv *)oct->priv;
 909         u64 oq_no;
 910         struct octeon_droq *droq;
 911
 912         if (oct->int_status & OCT_DEV_INTR_PKT_DATA) {
 913                 for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES; oq_no++) {
 914                         if (!(oct->droq_intr & (1 << oq_no)))
 915                                 continue;
 916
 917                         droq = oct->droq[oq_no];
 918
 919                         if (droq->ops.poll_mode) {
 920                                 droq->ops.napi_fn(droq);
 921                                 oct_priv->napi_mask |= (1 << oq_no);
 922                         } else {
 923                                 tasklet_schedule(&oct_priv->droq_tasklet);
 924                         }
 925                 }
 926         }
 927 }
 928
 929 /**
 930  * \brief Interrupt handler for octeon
 931  * @param irq unused
 932  * @param dev octeon device
 933  */
 934 static
 935 irqreturn_t liquidio_intr_handler(int irq __attribute__((unused)), void *dev)
 936 {
 937         struct octeon_device *oct = (struct octeon_device *)dev;
 938         irqreturn_t ret;
 939
 940         /* Disable our interrupts for the duration of ISR */
 941         oct->fn_list.disable_interrupt(oct->chip);
 942
 943         ret = oct->fn_list.process_interrupt_regs(oct);
 944
 945         if (ret == IRQ_HANDLED)
 946                 liquidio_schedule_droq_pkt_handlers(oct);
 947
 948         /* Re-enable our interrupts  */
 949         if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET))
 950                 oct->fn_list.enable_interrupt(oct->chip);
 951
 952         return ret;
 953 }
 954
 955 /**
 956  * \brief Setup interrupt for octeon device
 957  * @param oct octeon device
 958  *
 959  *  Enable interrupt in Octeon device as given in the PCI interrupt mask.
 960  */
 961 static int octeon_setup_interrupt(struct octeon_device *oct)
 962 {
 963         int irqret, err;
 964
 965         err = pci_enable_msi(oct->pci_dev);
 966         if (err)
 967                 dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
 968                          err);
 969         else
 970                 oct->flags |= LIO_FLAG_MSI_ENABLED;
 971
 972         irqret = request_irq(oct->pci_dev->irq, liquidio_intr_handler,
 973                              IRQF_SHARED, "octeon", oct);
 974         if (irqret) {
 975                 if (oct->flags & LIO_FLAG_MSI_ENABLED)
 976                         pci_disable_msi(oct->pci_dev);
 977                 dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
 978                         irqret);
 979                 return 1;
 980         }
 981
 982         return 0;
 983 }
 984
 985 /**
 986  * \brief PCI probe handler
 987  * @param pdev PCI device structure
 988  * @param ent unused
 989  */
 990 static int liquidio_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 991 {
 992         struct octeon_device *oct_dev = NULL;
 993         struct handshake *hs;
 994
 995         oct_dev = octeon_allocate_device(pdev->device,
 996                                          sizeof(struct octeon_device_priv));
 997         if (!oct_dev) {
 998                 dev_err(&pdev->dev, "Unable to allocate device\n");
 999                 return -ENOMEM;
1000         }
1001
1002         dev_info(&pdev->dev, "Initializing device %x:%x.\n",
1003                  (u32)pdev->vendor, (u32)pdev->device);
1004
1005         /* Assign octeon_device for this device to the private data area. */
1006         pci_set_drvdata(pdev, oct_dev);
1007
1008         /* set linux specific device pointer */
1009         oct_dev->pci_dev = (void *)pdev;
1010
1011         hs = &handshake[oct_dev->octeon_id];
1012         init_completion(&hs->init);
1013         init_completion(&hs->started);
1014         hs->pci_dev = pdev;
1015
1016         if (oct_dev->octeon_id == 0)
1017                 /* first LiquidIO NIC is detected */
1018                 complete(&first_stage);
1019
1020         if (octeon_device_init(oct_dev)) {
1021                 liquidio_remove(pdev);
1022                 return -ENOMEM;
1023         }
1024
1025         dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
1026
1027         return 0;
1028 }
1029
1030 /**
1031  *\brief Destroy resources associated with octeon device
1032  * @param pdev PCI device structure
1033  * @param ent unused
1034  */
1035 static void octeon_destroy_resources(struct octeon_device *oct)
1036 {
1037         int i;
1038         struct octeon_device_priv *oct_priv =
1039                 (struct octeon_device_priv *)oct->priv;
1040
1041         struct handshake *hs;
1042
1043         switch (atomic_read(&oct->status)) {
1044         case OCT_DEV_RUNNING:
1045         case OCT_DEV_CORE_OK:
1046
1047                 /* No more instructions will be forwarded. */
1048                 atomic_set(&oct->status, OCT_DEV_IN_RESET);
1049
1050                 oct->app_mode = CVM_DRV_INVALID_APP;
1051                 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
1052                         lio_get_state_string(&oct->status));
1053
1054                 schedule_timeout_uninterruptible(HZ / 10);
1055
1056                 /* fallthrough */
1057         case OCT_DEV_HOST_OK:
1058
1059                 /* fallthrough */
1060         case OCT_DEV_CONSOLE_INIT_DONE:
1061                 /* Remove any consoles */
1062                 octeon_remove_consoles(oct);
1063
1064                 /* fallthrough */
1065         case OCT_DEV_IO_QUEUES_DONE:
1066                 if (wait_for_pending_requests(oct))
1067                         dev_err(&oct->pci_dev->dev, "There were pending requests\n");
1068
1069                 if (lio_wait_for_instr_fetch(oct))
1070                         dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
1071
1072                 /* Disable the input and output queues now. No more packets will
1073                  * arrive from Octeon, but we should wait for all packet
1074                  * processing to finish.
1075                  */
1076                 oct->fn_list.disable_io_queues(oct);
1077
1078                 if (lio_wait_for_oq_pkts(oct))
1079                         dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1080
1081                 /* Disable interrupts  */
1082                 oct->fn_list.disable_interrupt(oct->chip);
1083
1084                 /* Release the interrupt line */
1085                 free_irq(oct->pci_dev->irq, oct);
1086
1087                 if (oct->flags & LIO_FLAG_MSI_ENABLED)
1088                         pci_disable_msi(oct->pci_dev);
1089
1090                 /* Soft reset the octeon device before exiting */
1091                 oct->fn_list.soft_reset(oct);
1092
1093                 /* Disable the device, releasing the PCI INT */
1094                 pci_disable_device(oct->pci_dev);
1095
1096                 /* fallthrough */
1097         case OCT_DEV_IN_RESET:
1098         case OCT_DEV_DROQ_INIT_DONE:
1099                 /*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/
1100                 mdelay(100);
1101                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) {
1102                         if (!(oct->io_qmask.oq & (1UL << i)))
1103                                 continue;
1104                         octeon_delete_droq(oct, i);
1105                 }
1106
1107                 /* Force any pending handshakes to complete */
1108                 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1109                         hs = &handshake[i];
1110
1111                         if (hs->pci_dev) {
1112                                 handshake[oct->octeon_id].init_ok = 0;
1113                                 complete(&handshake[oct->octeon_id].init);
1114                                 handshake[oct->octeon_id].started_ok = 0;
1115                                 complete(&handshake[oct->octeon_id].started);
1116                         }
1117                 }
1118
1119                 /* fallthrough */
1120         case OCT_DEV_RESP_LIST_INIT_DONE:
1121                 octeon_delete_response_list(oct);
1122
1123                 /* fallthrough */
1124         case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1125                 octeon_free_sc_buffer_pool(oct);
1126
1127                 /* fallthrough */
1128         case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1129                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) {
1130                         if (!(oct->io_qmask.iq & (1UL << i)))
1131                                 continue;
1132                         octeon_delete_instr_queue(oct, i);
1133                 }
1134
1135                 /* fallthrough */
1136         case OCT_DEV_DISPATCH_INIT_DONE:
1137                 octeon_delete_dispatch_list(oct);
1138                 cancel_delayed_work_sync(&oct->nic_poll_work.work);
1139
1140                 /* fallthrough */
1141         case OCT_DEV_PCI_MAP_DONE:
1142                 octeon_unmap_pci_barx(oct, 0);
1143                 octeon_unmap_pci_barx(oct, 1);
1144
1145                 /* fallthrough */
1146         case OCT_DEV_BEGIN_STATE:
1147                 /* Nothing to be done here either */
1148                 break;
1149         }                       /* end switch(oct->status) */
1150
1151         tasklet_kill(&oct_priv->droq_tasklet);
1152 }
1153
1154 /**
1155  * \brief Send Rx control command
1156  * @param lio per-network private data
1157  * @param start_stop whether to start or stop
1158  */
1159 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1160 {
1161         struct octnic_ctrl_pkt nctrl;
1162         struct octnic_ctrl_params nparams;
1163
1164         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1165
1166         nctrl.ncmd.s.cmd = OCTNET_CMD_RX_CTL;
1167         nctrl.ncmd.s.param1 = lio->linfo.ifidx;
1168         nctrl.ncmd.s.param2 = start_stop;
1169         nctrl.netpndev = (u64)lio->netdev;
1170
1171         nparams.resp_order = OCTEON_RESP_NORESPONSE;
1172
1173         if (octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams) < 0)
1174                 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1175 }
1176
1177 /**
1178  * \brief Destroy NIC device interface
1179  * @param oct octeon device
1180  * @param ifidx which interface to destroy
1181  *
1182  * Cleanup associated with each interface for an Octeon device  when NIC
1183  * module is being unloaded or if initialization fails during load.
1184  */
1185 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1186 {
1187         struct net_device *netdev = oct->props[ifidx].netdev;
1188         struct lio *lio;
1189
1190         if (!netdev) {
1191                 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1192                         __func__, ifidx);
1193                 return;
1194         }
1195
1196         lio = GET_LIO(netdev);
1197
1198         dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1199
1200         send_rx_ctrl_cmd(lio, 0);
1201
1202         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1203                 txqs_stop(netdev);
1204
1205         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1206                 unregister_netdev(netdev);
1207
1208         delete_glist(lio);
1209
1210         free_netdev(netdev);
1211
1212         oct->props[ifidx].netdev = NULL;
1213 }
1214
1215 /**
1216  * \brief Stop complete NIC functionality
1217  * @param oct octeon device
1218  */
1219 static int liquidio_stop_nic_module(struct octeon_device *oct)
1220 {
1221         int i, j;
1222         struct lio *lio;
1223
1224         dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1225         if (!oct->ifcount) {
1226                 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1227                 return 1;
1228         }
1229
1230         for (i = 0; i < oct->ifcount; i++) {
1231                 lio = GET_LIO(oct->props[i].netdev);
1232                 for (j = 0; j < lio->linfo.num_rxpciq; j++)
1233                         octeon_unregister_droq_ops(oct, lio->linfo.rxpciq[j]);
1234         }
1235
1236         for (i = 0; i < oct->ifcount; i++)
1237                 liquidio_destroy_nic_device(oct, i);
1238
1239         dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1240         return 0;
1241 }
1242
1243 /**
1244  * \brief Cleans up resources at unload time
1245  * @param pdev PCI device structure
1246  */
1247 static void liquidio_remove(struct pci_dev *pdev)
1248 {
1249         struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1250
1251         dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1252
1253         if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1254                 liquidio_stop_nic_module(oct_dev);
1255
1256         /* Reset the octeon device and cleanup all memory allocated for
1257          * the octeon device by driver.
1258          */
1259         octeon_destroy_resources(oct_dev);
1260
1261         dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1262
1263         /* This octeon device has been removed. Update the global
1264          * data structure to reflect this. Free the device structure.
1265          */
1266         octeon_free_device_mem(oct_dev);
1267 }
1268
1269 /**
1270  * \brief Identify the Octeon device and to map the BAR address space
1271  * @param oct octeon device
1272  */
1273 static int octeon_chip_specific_setup(struct octeon_device *oct)
1274 {
1275         u32 dev_id, rev_id;
1276         int ret = 1;
1277
1278         pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1279         pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1280         oct->rev_id = rev_id & 0xff;
1281
1282         switch (dev_id) {
1283         case OCTEON_CN68XX_PCIID:
1284                 oct->chip_id = OCTEON_CN68XX;
1285                 ret = lio_setup_cn68xx_octeon_device(oct);
1286                 break;
1287
1288         case OCTEON_CN66XX_PCIID:
1289                 oct->chip_id = OCTEON_CN66XX;
1290                 ret = lio_setup_cn66xx_octeon_device(oct);
1291                 break;
1292         default:
1293                 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1294                         dev_id);
1295         }
1296
1297         if (!ret)
1298                 dev_info(&oct->pci_dev->dev, "CN68XX PASS%d.%d %s\n",
1299                          OCTEON_MAJOR_REV(oct),
1300                          OCTEON_MINOR_REV(oct),
1301                          octeon_get_conf(oct)->card_name);
1302
1303         return ret;
1304 }
1305
1306 /**
1307  * \brief PCI initialization for each Octeon device.
1308  * @param oct octeon device
1309  */
1310 static int octeon_pci_os_setup(struct octeon_device *oct)
1311 {
1312         /* setup PCI stuff first */
1313         if (pci_enable_device(oct->pci_dev)) {
1314                 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1315                 return 1;
1316         }
1317
1318         if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1319                 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1320                 return 1;
1321         }
1322
1323         /* Enable PCI DMA Master. */
1324         pci_set_master(oct->pci_dev);
1325
1326         return 0;
1327 }
1328
1329 /**
1330  * \brief Check Tx queue state for a given network buffer
1331  * @param lio per-network private data
1332  * @param skb network buffer
1333  */
1334 static inline int check_txq_state(struct lio *lio, struct sk_buff *skb)
1335 {
1336         int q = 0, iq = 0;
1337
1338         if (netif_is_multiqueue(lio->netdev)) {
1339                 q = skb->queue_mapping;
1340                 iq = lio->linfo.txpciq[(q & (lio->linfo.num_txpciq - 1))];
1341         } else {
1342                 iq = lio->txq;
1343         }
1344
1345         if (octnet_iq_is_full(lio->oct_dev, iq))
1346                 return 0;
1347         wake_q(lio->netdev, q);
1348         return 1;
1349 }
1350
1351 /**
1352  * \brief Unmap and free network buffer
1353  * @param buf buffer
1354  */
1355 static void free_netbuf(void *buf)
1356 {
1357         struct sk_buff *skb;
1358         struct octnet_buf_free_info *finfo;
1359         struct lio *lio;
1360
1361         finfo = (struct octnet_buf_free_info *)buf;
1362         skb = finfo->skb;
1363         lio = finfo->lio;
1364
1365         dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1366                          DMA_TO_DEVICE);
1367
1368         check_txq_state(lio, skb);
1369
1370         recv_buffer_free((struct sk_buff *)skb);
1371 }
1372
1373 /**
1374  * \brief Unmap and free gather buffer
1375  * @param buf buffer
1376  */
1377 static void free_netsgbuf(void *buf)
1378 {
1379         struct octnet_buf_free_info *finfo;
1380         struct sk_buff *skb;
1381         struct lio *lio;
1382         struct octnic_gather *g;
1383         int i, frags;
1384
1385         finfo = (struct octnet_buf_free_info *)buf;
1386         skb = finfo->skb;
1387         lio = finfo->lio;
1388         g = finfo->g;
1389         frags = skb_shinfo(skb)->nr_frags;
1390
1391         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1392                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1393                          DMA_TO_DEVICE);
1394
1395         i = 1;
1396         while (frags--) {
1397                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1398
1399                 pci_unmap_page((lio->oct_dev)->pci_dev,
1400                                g->sg[(i >> 2)].ptr[(i & 3)],
1401                                frag->size, DMA_TO_DEVICE);
1402                 i++;
1403         }
1404
1405         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1406                          finfo->dptr, g->sg_size,
1407                          DMA_TO_DEVICE);
1408
1409         spin_lock(&lio->lock);
1410         list_add_tail(&g->list, &lio->glist);
1411         spin_unlock(&lio->lock);
1412
1413         check_txq_state(lio, skb);     /* mq support: sub-queue state check */
1414
1415         recv_buffer_free((struct sk_buff *)skb);
1416 }
1417
1418 /**
1419  * \brief Unmap and free gather buffer with response
1420  * @param buf buffer
1421  */
1422 static void free_netsgbuf_with_resp(void *buf)
1423 {
1424         struct octeon_soft_command *sc;
1425         struct octnet_buf_free_info *finfo;
1426         struct sk_buff *skb;
1427         struct lio *lio;
1428         struct octnic_gather *g;
1429         int i, frags;
1430
1431         sc = (struct octeon_soft_command *)buf;
1432         skb = (struct sk_buff *)sc->callback_arg;
1433         finfo = (struct octnet_buf_free_info *)&skb->cb;
1434
1435         lio = finfo->lio;
1436         g = finfo->g;
1437         frags = skb_shinfo(skb)->nr_frags;
1438
1439         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1440                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1441                          DMA_TO_DEVICE);
1442
1443         i = 1;
1444         while (frags--) {
1445                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1446
1447                 pci_unmap_page((lio->oct_dev)->pci_dev,
1448                                g->sg[(i >> 2)].ptr[(i & 3)],
1449                                frag->size, DMA_TO_DEVICE);
1450                 i++;
1451         }
1452
1453         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1454                          finfo->dptr, g->sg_size,
1455                          DMA_TO_DEVICE);
1456
1457         spin_lock(&lio->lock);
1458         list_add_tail(&g->list, &lio->glist);
1459         spin_unlock(&lio->lock);
1460
1461         /* Don't free the skb yet */
1462
1463         check_txq_state(lio, skb);
1464 }
1465
1466 /**
1467  * \brief Adjust ptp frequency
1468  * @param ptp PTP clock info
1469  * @param ppb how much to adjust by, in parts-per-billion
1470  */
1471 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
1472 {
1473         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1474         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1475         u64 comp, delta;
1476         unsigned long flags;
1477         bool neg_adj = false;
1478
1479         if (ppb < 0) {
1480                 neg_adj = true;
1481                 ppb = -ppb;
1482         }
1483
1484         /* The hardware adds the clock compensation value to the
1485          * PTP clock on every coprocessor clock cycle, so we
1486          * compute the delta in terms of coprocessor clocks.
1487          */
1488         delta = (u64)ppb << 32;
1489         do_div(delta, oct->coproc_clock_rate);
1490
1491         spin_lock_irqsave(&lio->ptp_lock, flags);
1492         comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1493         if (neg_adj)
1494                 comp -= delta;
1495         else
1496                 comp += delta;
1497         lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1498         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1499
1500         return 0;
1501 }
1502
1503 /**
1504  * \brief Adjust ptp time
1505  * @param ptp PTP clock info
1506  * @param delta how much to adjust by, in nanosecs
1507  */
1508 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1509 {
1510         unsigned long flags;
1511         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1512
1513         spin_lock_irqsave(&lio->ptp_lock, flags);
1514         lio->ptp_adjust += delta;
1515         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1516
1517         return 0;
1518 }
1519
1520 /**
1521  * \brief Get hardware clock time, including any adjustment
1522  * @param ptp PTP clock info
1523  * @param ts timespec
1524  */
1525 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1526                                 struct timespec64 *ts)
1527 {
1528         u64 ns;
1529         unsigned long flags;
1530         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1531         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1532
1533         spin_lock_irqsave(&lio->ptp_lock, flags);
1534         ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1535         ns += lio->ptp_adjust;
1536         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1537
1538         *ts = ns_to_timespec64(ns);
1539
1540         return 0;
1541 }
1542
1543 /**
1544  * \brief Set hardware clock time. Reset adjustment
1545  * @param ptp PTP clock info
1546  * @param ts timespec
1547  */
1548 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1549                                 const struct timespec64 *ts)
1550 {
1551         u64 ns;
1552         unsigned long flags;
1553         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1554         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1555
1556         ns = timespec_to_ns(ts);
1557
1558         spin_lock_irqsave(&lio->ptp_lock, flags);
1559         lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1560         lio->ptp_adjust = 0;
1561         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1562
1563         return 0;
1564 }
1565
1566 /**
1567  * \brief Check if PTP is enabled
1568  * @param ptp PTP clock info
1569  * @param rq request
1570  * @param on is it on
1571  */
1572 static int liquidio_ptp_enable(struct ptp_clock_info *ptp,
1573                                struct ptp_clock_request *rq, int on)
1574 {
1575         return -EOPNOTSUPP;
1576 }
1577
1578 /**
1579  * \brief Open PTP clock source
1580  * @param netdev network device
1581  */
1582 static void oct_ptp_open(struct net_device *netdev)
1583 {
1584         struct lio *lio = GET_LIO(netdev);
1585         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1586
1587         spin_lock_init(&lio->ptp_lock);
1588
1589         snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
1590         lio->ptp_info.owner = THIS_MODULE;
1591         lio->ptp_info.max_adj = 250000000;
1592         lio->ptp_info.n_alarm = 0;
1593         lio->ptp_info.n_ext_ts = 0;
1594         lio->ptp_info.n_per_out = 0;
1595         lio->ptp_info.pps = 0;
1596         lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
1597         lio->ptp_info.adjtime = liquidio_ptp_adjtime;
1598         lio->ptp_info.gettime64 = liquidio_ptp_gettime;
1599         lio->ptp_info.settime64 = liquidio_ptp_settime;
1600         lio->ptp_info.enable = liquidio_ptp_enable;
1601
1602         lio->ptp_adjust = 0;
1603
1604         lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
1605                                              &oct->pci_dev->dev);
1606
1607         if (IS_ERR(lio->ptp_clock))
1608                 lio->ptp_clock = NULL;
1609 }
1610
1611 /**
1612  * \brief Init PTP clock
1613  * @param oct octeon device
1614  */
1615 static void liquidio_ptp_init(struct octeon_device *oct)
1616 {
1617         u64 clock_comp, cfg;
1618
1619         clock_comp = (u64)NSEC_PER_SEC << 32;
1620         do_div(clock_comp, oct->coproc_clock_rate);
1621         lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1622
1623         /* Enable */
1624         cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
1625         lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
1626 }
1627
1628 /**
1629  * \brief Load firmware to device
1630  * @param oct octeon device
1631  *
1632  * Maps device to firmware filename, requests firmware, and downloads it
1633  */
1634 static int load_firmware(struct octeon_device *oct)
1635 {
1636         int ret = 0;
1637         const struct firmware *fw;
1638         char fw_name[LIO_MAX_FW_FILENAME_LEN];
1639         char *tmp_fw_type;
1640
1641         if (strncmp(fw_type, LIO_FW_NAME_TYPE_NONE,
1642                     sizeof(LIO_FW_NAME_TYPE_NONE)) == 0) {
1643                 dev_info(&oct->pci_dev->dev, "Skipping firmware load\n");
1644                 return ret;
1645         }
1646
1647         if (fw_type[0] == '\0')
1648                 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
1649         else
1650                 tmp_fw_type = fw_type;
1651
1652         sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
1653                 octeon_get_conf(oct)->card_name, tmp_fw_type,
1654                 LIO_FW_NAME_SUFFIX);
1655
1656         ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
1657         if (ret) {
1658                 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n.",
1659                         fw_name);
1660                 return ret;
1661         }
1662
1663         ret = octeon_download_firmware(oct, fw->data, fw->size);
1664
1665         release_firmware(fw);
1666
1667         return ret;
1668 }
1669
1670 /**
1671  * \brief Setup output queue
1672  * @param oct octeon device
1673  * @param q_no which queue
1674  * @param num_descs how many descriptors
1675  * @param desc_size size of each descriptor
1676  * @param app_ctx application context
1677  */
1678 static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
1679                              int desc_size, void *app_ctx)
1680 {
1681         int ret_val = 0;
1682
1683         dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no);
1684         /* droq creation and local register settings. */
1685         ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
1686         if (ret_val < 0)
1687                 return ret_val;
1688
1689         if (ret_val == 1) {
1690                 dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no);
1691                 return 0;
1692         }
1693         /* tasklet creation for the droq */
1694
1695         /* Enable the droq queues */
1696         octeon_set_droq_pkt_op(oct, q_no, 1);
1697
1698         /* Send Credit for Octeon Output queues. Credits are always
1699          * sent after the output queue is enabled.
1700          */
1701         writel(oct->droq[q_no]->max_count,
1702                oct->droq[q_no]->pkts_credit_reg);
1703
1704         return ret_val;
1705 }
1706
1707 /**
1708  * \brief Callback for getting interface configuration
1709  * @param status status of request
1710  * @param buf pointer to resp structure
1711  */
1712 static void if_cfg_callback(struct octeon_device *oct,
1713                             u32 status,
1714                             void *buf)
1715 {
1716         struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
1717         struct liquidio_if_cfg_resp *resp;
1718         struct liquidio_if_cfg_context *ctx;
1719
1720         resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
1721         ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
1722
1723         oct = lio_get_device(ctx->octeon_id);
1724         if (resp->status)
1725                 dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n",
1726                         CVM_CAST64(resp->status));
1727         ACCESS_ONCE(ctx->cond) = 1;
1728
1729         /* This barrier is required to be sure that the response has been
1730          * written fully before waking up the handler
1731          */
1732         wmb();
1733
1734         wake_up_interruptible(&ctx->wc);
1735 }
1736
1737 /**
1738  * \brief Select queue based on hash
1739  * @param dev Net device
1740  * @param skb sk_buff structure
1741  * @returns selected queue number
1742  */
1743 static u16 select_q(struct net_device *dev, struct sk_buff *skb,
1744                     void *accel_priv, select_queue_fallback_t fallback)
1745 {
1746         int qindex;
1747         struct lio *lio;
1748
1749         lio = GET_LIO(dev);
1750         /* select queue on chosen queue_mapping or core */
1751         qindex = skb_rx_queue_recorded(skb) ?
1752                  skb_get_rx_queue(skb) : smp_processor_id();
1753         return (u16)(qindex & (lio->linfo.num_txpciq - 1));
1754 }
1755
1756 /** Routine to push packets arriving on Octeon interface upto network layer.
1757  * @param oct_id   - octeon device id.
1758  * @param skbuff   - skbuff struct to be passed to network layer.
1759  * @param len      - size of total data received.
1760  * @param rh       - Control header associated with the packet
1761  * @param param    - additional control data with the packet
1762  */
1763 static void
1764 liquidio_push_packet(u32 octeon_id,
1765                      void *skbuff,
1766                      u32 len,
1767                      union octeon_rh *rh,
1768                      void *param)
1769 {
1770         struct napi_struct *napi = param;
1771         struct octeon_device *oct = lio_get_device(octeon_id);
1772         struct sk_buff *skb = (struct sk_buff *)skbuff;
1773         struct skb_shared_hwtstamps *shhwtstamps;
1774         u64 ns;
1775         struct net_device *netdev =
1776                 (struct net_device *)oct->props[rh->r_dh.link].netdev;
1777         struct octeon_droq *droq = container_of(param, struct octeon_droq,
1778                                                 napi);
1779         if (netdev) {
1780                 int packet_was_received;
1781                 struct lio *lio = GET_LIO(netdev);
1782
1783                 /* Do not proceed if the interface is not in RUNNING state. */
1784                 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
1785                         recv_buffer_free(skb);
1786                         droq->stats.rx_dropped++;
1787                         return;
1788                 }
1789
1790                 skb->dev = netdev;
1791
1792                 if (rh->r_dh.has_hwtstamp) {
1793                         /* timestamp is included from the hardware at the
1794                          * beginning of the packet.
1795                          */
1796                         if (ifstate_check(lio,
1797                                           LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
1798                                 /* Nanoseconds are in the first 64-bits
1799                                  * of the packet.
1800                                  */
1801                                 memcpy(&ns, (skb->data), sizeof(ns));
1802                                 shhwtstamps = skb_hwtstamps(skb);
1803                                 shhwtstamps->hwtstamp =
1804                                         ns_to_ktime(ns + lio->ptp_adjust);
1805                         }
1806                         skb_pull(skb, sizeof(ns));
1807                 }
1808
1809                 skb->protocol = eth_type_trans(skb, skb->dev);
1810
1811                 if ((netdev->features & NETIF_F_RXCSUM) &&
1812                     (rh->r_dh.csum_verified == CNNIC_CSUM_VERIFIED))
1813                         /* checksum has already been verified */
1814                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1815                 else
1816                         skb->ip_summed = CHECKSUM_NONE;
1817
1818                 packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP;
1819
1820                 if (packet_was_received) {
1821                         droq->stats.rx_bytes_received += len;
1822                         droq->stats.rx_pkts_received++;
1823                         netdev->last_rx = jiffies;
1824                 } else {
1825                         droq->stats.rx_dropped++;
1826                         netif_info(lio, rx_err, lio->netdev,
1827                                    "droq:%d  error rx_dropped:%llu\n",
1828                                    droq->q_no, droq->stats.rx_dropped);
1829                 }
1830
1831         } else {
1832                 recv_buffer_free(skb);
1833         }
1834 }
1835
1836 /**
1837  * \brief wrapper for calling napi_schedule
1838  * @param param parameters to pass to napi_schedule
1839  *
1840  * Used when scheduling on different CPUs
1841  */
1842 static void napi_schedule_wrapper(void *param)
1843 {
1844         struct napi_struct *napi = param;
1845
1846         napi_schedule(napi);
1847 }
1848
1849 /**
1850  * \brief callback when receive interrupt occurs and we are in NAPI mode
1851  * @param arg pointer to octeon output queue
1852  */
1853 static void liquidio_napi_drv_callback(void *arg)
1854 {
1855         struct octeon_droq *droq = arg;
1856         int this_cpu = smp_processor_id();
1857
1858         if (droq->cpu_id == this_cpu) {
1859                 napi_schedule(&droq->napi);
1860         } else {
1861                 struct call_single_data *csd = &droq->csd;
1862
1863                 csd->func = napi_schedule_wrapper;
1864                 csd->info = &droq->napi;
1865                 csd->flags = 0;
1866
1867                 smp_call_function_single_async(droq->cpu_id, csd);
1868         }
1869 }
1870
1871 /**
1872  * \brief Main NAPI poll function
1873  * @param droq octeon output queue
1874  * @param budget maximum number of items to process
1875  */
1876 static int liquidio_napi_do_rx(struct octeon_droq *droq, int budget)
1877 {
1878         int work_done;
1879         struct lio *lio = GET_LIO(droq->napi.dev);
1880         struct octeon_device *oct = lio->oct_dev;
1881
1882         work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
1883                                                  POLL_EVENT_PROCESS_PKTS,
1884                                                  budget);
1885         if (work_done < 0) {
1886                 netif_info(lio, rx_err, lio->netdev,
1887                            "Receive work_done < 0, rxq:%d\n", droq->q_no);
1888                 goto octnet_napi_finish;
1889         }
1890
1891         if (work_done > budget)
1892                 dev_err(&oct->pci_dev->dev, ">>>> %s work_done: %d budget: %d\n",
1893                         __func__, work_done, budget);
1894
1895         return work_done;
1896
1897 octnet_napi_finish:
1898         napi_complete(&droq->napi);
1899         octeon_process_droq_poll_cmd(oct, droq->q_no, POLL_EVENT_ENABLE_INTR,
1900                                      0);
1901         return 0;
1902 }
1903
1904 /**
1905  * \brief Entry point for NAPI polling
1906  * @param napi NAPI structure
1907  * @param budget maximum number of items to process
1908  */
1909 static int liquidio_napi_poll(struct napi_struct *napi, int budget)
1910 {
1911         struct octeon_droq *droq;
1912         int work_done;
1913
1914         droq = container_of(napi, struct octeon_droq, napi);
1915
1916         work_done = liquidio_napi_do_rx(droq, budget);
1917
1918         if (work_done < budget) {
1919                 napi_complete(napi);
1920                 octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no,
1921                                              POLL_EVENT_ENABLE_INTR, 0);
1922                 return 0;
1923         }
1924
1925         return work_done;
1926 }
1927
1928 /**
1929  * \brief Setup input and output queues
1930  * @param octeon_dev octeon device
1931  * @param net_device Net device
1932  *
1933  * Note: Queues are with respect to the octeon device. Thus
1934  * an input queue is for egress packets, and output queues
1935  * are for ingress packets.
1936  */
1937 static inline int setup_io_queues(struct octeon_device *octeon_dev,
1938                                   struct net_device *net_device)
1939 {
1940         static int first_time = 1;
1941         static struct octeon_droq_ops droq_ops;
1942         static int cpu_id;
1943         static int cpu_id_modulus;
1944         struct octeon_droq *droq;
1945         struct napi_struct *napi;
1946         int q, q_no, retval = 0;
1947         struct lio *lio;
1948         int num_tx_descs;
1949
1950         lio = GET_LIO(net_device);
1951         if (first_time) {
1952                 first_time = 0;
1953                 memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
1954
1955                 droq_ops.fptr = liquidio_push_packet;
1956
1957                 droq_ops.poll_mode = 1;
1958                 droq_ops.napi_fn = liquidio_napi_drv_callback;
1959                 cpu_id = 0;
1960                 cpu_id_modulus = num_present_cpus();
1961         }
1962
1963         /* set up DROQs. */
1964         for (q = 0; q < lio->linfo.num_rxpciq; q++) {
1965                 q_no = lio->linfo.rxpciq[q];
1966
1967                 retval = octeon_setup_droq(octeon_dev, q_no,
1968                                            CFG_GET_NUM_RX_DESCS_NIC_IF
1969                                                    (octeon_get_conf(octeon_dev),
1970                                                    lio->ifidx),
1971                                            CFG_GET_NUM_RX_BUF_SIZE_NIC_IF
1972                                                    (octeon_get_conf(octeon_dev),
1973                                                    lio->ifidx), NULL);
1974                 if (retval) {
1975                         dev_err(&octeon_dev->pci_dev->dev,
1976                                 " %s : Runtime DROQ(RxQ) creation failed.\n",
1977                                 __func__);
1978                         return 1;
1979                 }
1980
1981                 droq = octeon_dev->droq[q_no];
1982                 napi = &droq->napi;
1983                 netif_napi_add(net_device, napi, liquidio_napi_poll, 64);
1984
1985                 /* designate a CPU for this droq */
1986                 droq->cpu_id = cpu_id;
1987                 cpu_id++;
1988                 if (cpu_id >= cpu_id_modulus)
1989                         cpu_id = 0;
1990
1991                 octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
1992         }
1993
1994         /* set up IQs. */
1995         for (q = 0; q < lio->linfo.num_txpciq; q++) {
1996                 num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
1997                                                            (octeon_dev),
1998                                                            lio->ifidx);
1999                 retval = octeon_setup_iq(octeon_dev, lio->linfo.txpciq[q],
2000                                          num_tx_descs,
2001                                          netdev_get_tx_queue(net_device, q));
2002                 if (retval) {
2003                         dev_err(&octeon_dev->pci_dev->dev,
2004                                 " %s : Runtime IQ(TxQ) creation failed.\n",
2005                                 __func__);
2006                         return 1;
2007                 }
2008         }
2009
2010         return 0;
2011 }
2012
2013 /**
2014  * \brief Poll routine for checking transmit queue status
2015  * @param work work_struct data structure
2016  */
2017 static void octnet_poll_check_txq_status(struct work_struct *work)
2018 {
2019         struct cavium_wk *wk = (struct cavium_wk *)work;
2020         struct lio *lio = (struct lio *)wk->ctxptr;
2021
2022         if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
2023                 return;
2024
2025         check_txq_status(lio);
2026         queue_delayed_work(lio->txq_status_wq.wq,
2027                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2028 }
2029
2030 /**
2031  * \brief Sets up the txq poll check
2032  * @param netdev network device
2033  */
2034 static inline void setup_tx_poll_fn(struct net_device *netdev)
2035 {
2036         struct lio *lio = GET_LIO(netdev);
2037         struct octeon_device *oct = lio->oct_dev;
2038
2039         lio->txq_status_wq.wq = create_workqueue("txq-status");
2040         if (!lio->txq_status_wq.wq) {
2041                 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
2042                 return;
2043         }
2044         INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
2045                           octnet_poll_check_txq_status);
2046         lio->txq_status_wq.wk.ctxptr = lio;
2047         queue_delayed_work(lio->txq_status_wq.wq,
2048                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2049 }
2050
2051 /**
2052  * \brief Net device open for LiquidIO
2053  * @param netdev network device
2054  */
2055 static int liquidio_open(struct net_device *netdev)
2056 {
2057         struct lio *lio = GET_LIO(netdev);
2058         struct octeon_device *oct = lio->oct_dev;
2059         struct napi_struct *napi, *n;
2060
2061         list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
2062                 napi_enable(napi);
2063
2064         oct_ptp_open(netdev);
2065
2066         ifstate_set(lio, LIO_IFSTATE_RUNNING);
2067         setup_tx_poll_fn(netdev);
2068         start_txq(netdev);
2069
2070         netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
2071         try_module_get(THIS_MODULE);
2072
2073         /* tell Octeon to start forwarding packets to host */
2074         send_rx_ctrl_cmd(lio, 1);
2075
2076         /* Ready for link status updates */
2077         lio->intf_open = 1;
2078
2079         dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
2080                  netdev->name);
2081
2082         return 0;
2083 }
2084
2085 /**
2086  * \brief Net device stop for LiquidIO
2087  * @param netdev network device
2088  */
2089 static int liquidio_stop(struct net_device *netdev)
2090 {
2091         struct napi_struct *napi, *n;
2092         struct lio *lio = GET_LIO(netdev);
2093         struct octeon_device *oct = lio->oct_dev;
2094
2095         netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n");
2096         /* Inform that netif carrier is down */
2097         lio->intf_open = 0;
2098         lio->linfo.link.s.status = 0;
2099
2100         netif_carrier_off(netdev);
2101
2102         /* tell Octeon to stop forwarding packets to host */
2103         send_rx_ctrl_cmd(lio, 0);
2104
2105         cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
2106         flush_workqueue(lio->txq_status_wq.wq);
2107         destroy_workqueue(lio->txq_status_wq.wq);
2108
2109         if (lio->ptp_clock) {
2110                 ptp_clock_unregister(lio->ptp_clock);
2111                 lio->ptp_clock = NULL;
2112         }
2113
2114         ifstate_reset(lio, LIO_IFSTATE_RUNNING);
2115
2116         /* This is a hack that allows DHCP to continue working. */
2117         set_bit(__LINK_STATE_START, &lio->netdev->state);
2118
2119         list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
2120                 napi_disable(napi);
2121
2122         txqs_stop(netdev);
2123
2124         dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
2125         module_put(THIS_MODULE);
2126
2127         return 0;
2128 }
2129
2130 void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr)
2131 {
2132         struct octnic_ctrl_pkt *nctrl = (struct octnic_ctrl_pkt *)nctrl_ptr;
2133         struct net_device *netdev = (struct net_device *)nctrl->netpndev;
2134         struct lio *lio = GET_LIO(netdev);
2135         struct octeon_device *oct = lio->oct_dev;
2136
2137         switch (nctrl->ncmd.s.cmd) {
2138         case OCTNET_CMD_CHANGE_DEVFLAGS:
2139         case OCTNET_CMD_SET_MULTI_LIST:
2140                 break;
2141
2142         case OCTNET_CMD_CHANGE_MACADDR:
2143                 /* If command is successful, change the MACADDR. */
2144                 netif_info(lio, probe, lio->netdev, " MACAddr changed to 0x%llx\n",
2145                            CVM_CAST64(nctrl->udd[0]));
2146                 dev_info(&oct->pci_dev->dev, "%s MACAddr changed to 0x%llx\n",
2147                          netdev->name, CVM_CAST64(nctrl->udd[0]));
2148                 memcpy(netdev->dev_addr, ((u8 *)&nctrl->udd[0]) + 2, ETH_ALEN);
2149                 break;
2150
2151         case OCTNET_CMD_CHANGE_MTU:
2152                 /* If command is successful, change the MTU. */
2153                 netif_info(lio, probe, lio->netdev, " MTU Changed from %d to %d\n",
2154                            netdev->mtu, nctrl->ncmd.s.param2);
2155                 dev_info(&oct->pci_dev->dev, "%s MTU Changed from %d to %d\n",
2156                          netdev->name, netdev->mtu,
2157                          nctrl->ncmd.s.param2);
2158                 netdev->mtu = nctrl->ncmd.s.param2;
2159                 break;
2160
2161         case OCTNET_CMD_GPIO_ACCESS:
2162                 netif_info(lio, probe, lio->netdev, "LED Flashing visual identification\n");
2163
2164                 break;
2165
2166         case OCTNET_CMD_LRO_ENABLE:
2167                 dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name);
2168                 break;
2169
2170         case OCTNET_CMD_LRO_DISABLE:
2171                 dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n",
2172                          netdev->name);
2173                 break;
2174
2175         case OCTNET_CMD_VERBOSE_ENABLE:
2176                 dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name);
2177                 break;
2178
2179         case OCTNET_CMD_VERBOSE_DISABLE:
2180                 dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n",
2181                          netdev->name);
2182                 break;
2183
2184         case OCTNET_CMD_SET_SETTINGS:
2185                 dev_info(&oct->pci_dev->dev, "%s settings changed\n",
2186                          netdev->name);
2187
2188                 break;
2189
2190         default:
2191                 dev_err(&oct->pci_dev->dev, "%s Unknown cmd %d\n", __func__,
2192                         nctrl->ncmd.s.cmd);
2193         }
2194 }
2195
2196 /**
2197  * \brief Converts a mask based on net device flags
2198  * @param netdev network device
2199  *
2200  * This routine generates a octnet_ifflags mask from the net device flags
2201  * received from the OS.
2202  */
2203 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
2204 {
2205         enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
2206
2207         if (netdev->flags & IFF_PROMISC)
2208                 f |= OCTNET_IFFLAG_PROMISC;
2209
2210         if (netdev->flags & IFF_ALLMULTI)
2211                 f |= OCTNET_IFFLAG_ALLMULTI;
2212
2213         if (netdev->flags & IFF_MULTICAST) {
2214                 f |= OCTNET_IFFLAG_MULTICAST;
2215
2216                 /* Accept all multicast addresses if there are more than we
2217                  * can handle
2218                  */
2219                 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
2220                         f |= OCTNET_IFFLAG_ALLMULTI;
2221         }
2222
2223         if (netdev->flags & IFF_BROADCAST)
2224                 f |= OCTNET_IFFLAG_BROADCAST;
2225
2226         return f;
2227 }
2228
2229 /**
2230  * \brief Net device set_multicast_list
2231  * @param netdev network device
2232  */
2233 static void liquidio_set_mcast_list(struct net_device *netdev)
2234 {
2235         struct lio *lio = GET_LIO(netdev);
2236         struct octeon_device *oct = lio->oct_dev;
2237         struct octnic_ctrl_pkt nctrl;
2238         struct octnic_ctrl_params nparams;
2239         struct netdev_hw_addr *ha;
2240         u64 *mc;
2241         int ret, i;
2242         int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
2243
2244         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2245
2246         /* Create a ctrl pkt command to be sent to core app. */
2247         nctrl.ncmd.u64 = 0;
2248         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
2249         nctrl.ncmd.s.param1 = lio->linfo.ifidx;
2250         nctrl.ncmd.s.param2 = get_new_flags(netdev);
2251         nctrl.ncmd.s.param3 = mc_count;
2252         nctrl.ncmd.s.more = mc_count;
2253         nctrl.netpndev = (u64)netdev;
2254         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2255
2256         /* copy all the addresses into the udd */
2257         i = 0;
2258         mc = &nctrl.udd[0];
2259         netdev_for_each_mc_addr(ha, netdev) {
2260                 *mc = 0;
2261                 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
2262                 /* no need to swap bytes */
2263
2264                 if (++mc > &nctrl.udd[mc_count])
2265                         break;
2266         }
2267
2268         /* Apparently, any activity in this call from the kernel has to
2269          * be atomic. So we won't wait for response.
2270          */
2271         nctrl.wait_time = 0;
2272
2273         nparams.resp_order = OCTEON_RESP_NORESPONSE;
2274
2275         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
2276         if (ret < 0) {
2277                 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2278                         ret);
2279         }
2280 }
2281
2282 /**
2283  * \brief Net device set_mac_address
2284  * @param netdev network device
2285  */
2286 static int liquidio_set_mac(struct net_device *netdev, void *p)
2287 {
2288         int ret = 0;
2289         struct lio *lio = GET_LIO(netdev);
2290         struct octeon_device *oct = lio->oct_dev;
2291         struct sockaddr *addr = (struct sockaddr *)p;
2292         struct octnic_ctrl_pkt nctrl;
2293         struct octnic_ctrl_params nparams;
2294
2295         if ((!is_valid_ether_addr(addr->sa_data)) ||
2296             (ifstate_check(lio, LIO_IFSTATE_RUNNING)))
2297                 return -EADDRNOTAVAIL;
2298
2299         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2300
2301         nctrl.ncmd.u64 = 0;
2302         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2303         nctrl.ncmd.s.param1 = lio->linfo.ifidx;
2304         nctrl.ncmd.s.param2 = 0;
2305         nctrl.ncmd.s.more = 1;
2306         nctrl.netpndev = (u64)netdev;
2307         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2308         nctrl.wait_time = 100;
2309
2310         nctrl.udd[0] = 0;
2311         /* The MAC Address is presented in network byte order. */
2312         memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2313
2314         nparams.resp_order = OCTEON_RESP_ORDERED;
2315
2316         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
2317         if (ret < 0) {
2318                 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2319                 return -ENOMEM;
2320         }
2321         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2322         memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2323
2324         return 0;
2325 }
2326
2327 /**
2328  * \brief Net device get_stats
2329  * @param netdev network device
2330  */
2331 static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
2332 {
2333         struct lio *lio = GET_LIO(netdev);
2334         struct net_device_stats *stats = &netdev->stats;
2335         struct octeon_device *oct;
2336         u64 pkts = 0, drop = 0, bytes = 0;
2337         struct oct_droq_stats *oq_stats;
2338         struct oct_iq_stats *iq_stats;
2339         int i, iq_no, oq_no;
2340
2341         oct = lio->oct_dev;
2342
2343         for (i = 0; i < lio->linfo.num_txpciq; i++) {
2344                 iq_no = lio->linfo.txpciq[i];
2345                 iq_stats = &oct->instr_queue[iq_no]->stats;
2346                 pkts += iq_stats->tx_done;
2347                 drop += iq_stats->tx_dropped;
2348                 bytes += iq_stats->tx_tot_bytes;
2349         }
2350
2351         stats->tx_packets = pkts;
2352         stats->tx_bytes = bytes;
2353         stats->tx_dropped = drop;
2354
2355         pkts = 0;
2356         drop = 0;
2357         bytes = 0;
2358
2359         for (i = 0; i < lio->linfo.num_rxpciq; i++) {
2360                 oq_no = lio->linfo.rxpciq[i];
2361                 oq_stats = &oct->droq[oq_no]->stats;
2362                 pkts += oq_stats->rx_pkts_received;
2363                 drop += (oq_stats->rx_dropped +
2364                          oq_stats->dropped_nodispatch +
2365                          oq_stats->dropped_toomany +
2366                          oq_stats->dropped_nomem);
2367                 bytes += oq_stats->rx_bytes_received;
2368         }
2369
2370         stats->rx_bytes = bytes;
2371         stats->rx_packets = pkts;
2372         stats->rx_dropped = drop;
2373
2374         return stats;
2375 }
2376
2377 /**
2378  * \brief Net device change_mtu
2379  * @param netdev network device
2380  */
2381 static int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
2382 {
2383         struct lio *lio = GET_LIO(netdev);
2384         struct octeon_device *oct = lio->oct_dev;
2385         struct octnic_ctrl_pkt nctrl;
2386         struct octnic_ctrl_params nparams;
2387         int max_frm_size = new_mtu + OCTNET_FRM_HEADER_SIZE;
2388         int ret = 0;
2389
2390         /* Limit the MTU to make sure the ethernet packets are between 64 bytes
2391          * and 65535 bytes
2392          */
2393         if ((max_frm_size < OCTNET_MIN_FRM_SIZE) ||
2394             (max_frm_size > OCTNET_MAX_FRM_SIZE)) {
2395                 dev_err(&oct->pci_dev->dev, "Invalid MTU: %d\n", new_mtu);
2396                 dev_err(&oct->pci_dev->dev, "Valid range %d and %d\n",
2397                         (OCTNET_MIN_FRM_SIZE - OCTNET_FRM_HEADER_SIZE),
2398                         (OCTNET_MAX_FRM_SIZE - OCTNET_FRM_HEADER_SIZE));
2399                 return -EINVAL;
2400         }
2401
2402         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2403
2404         nctrl.ncmd.u64 = 0;
2405         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU;
2406         nctrl.ncmd.s.param1 = lio->linfo.ifidx;
2407         nctrl.ncmd.s.param2 = new_mtu;
2408         nctrl.wait_time = 100;
2409         nctrl.netpndev = (u64)netdev;
2410         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2411
2412         nparams.resp_order = OCTEON_RESP_ORDERED;
2413
2414         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
2415         if (ret < 0) {
2416                 dev_err(&oct->pci_dev->dev, "Failed to set MTU\n");
2417                 return -1;
2418         }
2419
2420         lio->mtu = new_mtu;
2421
2422         return 0;
2423 }
2424
2425 /**
2426  * \brief Handler for SIOCSHWTSTAMP ioctl
2427  * @param netdev network device
2428  * @param ifr interface request
2429  * @param cmd command
2430  */
2431 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2432 {
2433         struct hwtstamp_config conf;
2434         struct lio *lio = GET_LIO(netdev);
2435
2436         if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2437                 return -EFAULT;
2438
2439         if (conf.flags)
2440                 return -EINVAL;
2441
2442         switch (conf.tx_type) {
2443         case HWTSTAMP_TX_ON:
2444         case HWTSTAMP_TX_OFF:
2445                 break;
2446         default:
2447                 return -ERANGE;
2448         }
2449
2450         switch (conf.rx_filter) {
2451         case HWTSTAMP_FILTER_NONE:
2452                 break;
2453         case HWTSTAMP_FILTER_ALL:
2454         case HWTSTAMP_FILTER_SOME:
2455         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2456         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2457         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2458         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2459         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2460         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2461         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2462         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2463         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2464         case HWTSTAMP_FILTER_PTP_V2_EVENT:
2465         case HWTSTAMP_FILTER_PTP_V2_SYNC:
2466         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2467                 conf.rx_filter = HWTSTAMP_FILTER_ALL;
2468                 break;
2469         default:
2470                 return -ERANGE;
2471         }
2472
2473         if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2474                 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2475
2476         else
2477                 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2478
2479         return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2480 }
2481
2482 /**
2483  * \brief ioctl handler
2484  * @param netdev network device
2485  * @param ifr interface request
2486  * @param cmd command
2487  */
2488 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2489 {
2490         switch (cmd) {
2491         case SIOCSHWTSTAMP:
2492                 return hwtstamp_ioctl(netdev, ifr, cmd);
2493         default:
2494                 return -EOPNOTSUPP;
2495         }
2496 }
2497
2498 /**
2499  * \brief handle a Tx timestamp response
2500  * @param status response status
2501  * @param buf pointer to skb
2502  */
2503 static void handle_timestamp(struct octeon_device *oct,
2504                              u32 status,
2505                              void *buf)
2506 {
2507         struct octnet_buf_free_info *finfo;
2508         struct octeon_soft_command *sc;
2509         struct oct_timestamp_resp *resp;
2510         struct lio *lio;
2511         struct sk_buff *skb = (struct sk_buff *)buf;
2512
2513         finfo = (struct octnet_buf_free_info *)skb->cb;
2514         lio = finfo->lio;
2515         sc = finfo->sc;
2516         oct = lio->oct_dev;
2517         resp = (struct oct_timestamp_resp *)sc->virtrptr;
2518
2519         if (status != OCTEON_REQUEST_DONE) {
2520                 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2521                         CVM_CAST64(status));
2522                 resp->timestamp = 0;
2523         }
2524
2525         octeon_swap_8B_data(&resp->timestamp, 1);
2526
2527         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
2528                 struct skb_shared_hwtstamps ts;
2529                 u64 ns = resp->timestamp;
2530
2531                 netif_info(lio, tx_done, lio->netdev,
2532                            "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2533                            skb, (unsigned long long)ns);
2534                 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2535                 skb_tstamp_tx(skb, &ts);
2536         }
2537
2538         octeon_free_soft_command(oct, sc);
2539         recv_buffer_free(skb);
2540 }
2541
2542 /* \brief Send a data packet that will be timestamped
2543  * @param oct octeon device
2544  * @param ndata pointer to network data
2545  * @param finfo pointer to private network data
2546  */
2547 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2548                                          struct octnic_data_pkt *ndata,
2549                                          struct octnet_buf_free_info *finfo,
2550                                          int xmit_more)
2551 {
2552         int retval;
2553         struct octeon_soft_command *sc;
2554         struct octeon_instr_ih *ih;
2555         struct octeon_instr_rdp *rdp;
2556         struct lio *lio;
2557         int ring_doorbell;
2558
2559         lio = finfo->lio;
2560
2561         sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2562                                             sizeof(struct oct_timestamp_resp));
2563         finfo->sc = sc;
2564
2565         if (!sc) {
2566                 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2567                 return IQ_SEND_FAILED;
2568         }
2569
2570         if (ndata->reqtype == REQTYPE_NORESP_NET)
2571                 ndata->reqtype = REQTYPE_RESP_NET;
2572         else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2573                 ndata->reqtype = REQTYPE_RESP_NET_SG;
2574
2575         sc->callback = handle_timestamp;
2576         sc->callback_arg = finfo->skb;
2577         sc->iq_no = ndata->q_no;
2578
2579         ih = (struct octeon_instr_ih *)&sc->cmd.ih;
2580         rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp;
2581
2582         ring_doorbell = !xmit_more;
2583         retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
2584                                      sc, ih->dlengsz, ndata->reqtype);
2585
2586         if (retval) {
2587                 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2588                         retval);
2589                 octeon_free_soft_command(oct, sc);
2590         } else {
2591                 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2592         }
2593
2594         return retval;
2595 }
2596
2597 static inline int is_ipv4(struct sk_buff *skb)
2598 {
2599         return (skb->protocol == htons(ETH_P_IP)) &&
2600                (ip_hdr(skb)->version == 4);
2601 }
2602
2603 static inline int is_vlan(struct sk_buff *skb)
2604 {
2605         return skb->protocol == htons(ETH_P_8021Q);
2606 }
2607
2608 static inline int is_ip_fragmented(struct sk_buff *skb)
2609 {
2610         /* The Don't fragment and Reserved flag fields are ignored.
2611          * IP is fragmented if
2612          * -  the More fragments bit is set (indicating this IP is a fragment
2613          * with more to follow; the current offset could be 0 ).
2614          * -  ths offset field is non-zero.
2615          */
2616         return (ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)) ? 1 : 0;
2617 }
2618
2619 static inline int is_ipv6(struct sk_buff *skb)
2620 {
2621         return (skb->protocol == htons(ETH_P_IPV6)) &&
2622                (ipv6_hdr(skb)->version == 6);
2623 }
2624
2625 static inline int is_with_extn_hdr(struct sk_buff *skb)
2626 {
2627         return (ipv6_hdr(skb)->nexthdr != IPPROTO_TCP) &&
2628                (ipv6_hdr(skb)->nexthdr != IPPROTO_UDP);
2629 }
2630
2631 static inline int is_tcpudp(struct sk_buff *skb)
2632 {
2633         return (ip_hdr(skb)->protocol == IPPROTO_TCP) ||
2634                (ip_hdr(skb)->protocol == IPPROTO_UDP);
2635 }
2636
2637 static inline u32 get_ipv4_5tuple_tag(struct sk_buff *skb)
2638 {
2639         u32 tag;
2640         struct iphdr *iphdr = ip_hdr(skb);
2641
2642         tag = crc32(0, &iphdr->protocol, 1);
2643         tag = crc32(tag, (u8 *)&iphdr->saddr, 8);
2644         tag = crc32(tag, skb_transport_header(skb), 4);
2645         return tag;
2646 }
2647
2648 static inline u32 get_ipv6_5tuple_tag(struct sk_buff *skb)
2649 {
2650         u32 tag;
2651         struct ipv6hdr *ipv6hdr = ipv6_hdr(skb);
2652
2653         tag = crc32(0, &ipv6hdr->nexthdr, 1);
2654         tag = crc32(tag, (u8 *)&ipv6hdr->saddr, 32);
2655         tag = crc32(tag, skb_transport_header(skb), 4);
2656         return tag;
2657 }
2658
2659 /** \brief Transmit networks packets to the Octeon interface
2660  * @param skbuff   skbuff struct to be passed to network layer.
2661  * @param netdev    pointer to network device
2662  * @returns whether the packet was transmitted to the device okay or not
2663  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
2664  */
2665 static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2666 {
2667         struct lio *lio;
2668         struct octnet_buf_free_info *finfo;
2669         union octnic_cmd_setup cmdsetup;
2670         struct octnic_data_pkt ndata;
2671         struct octeon_device *oct;
2672         struct oct_iq_stats *stats;
2673         int cpu = 0, status = 0;
2674         int q_idx = 0, iq_no = 0;
2675         int xmit_more;
2676         u32 tag = 0;
2677
2678         lio = GET_LIO(netdev);
2679         oct = lio->oct_dev;
2680
2681         if (netif_is_multiqueue(netdev)) {
2682                 cpu = skb->queue_mapping;
2683                 q_idx = (cpu & (lio->linfo.num_txpciq - 1));
2684                 iq_no = lio->linfo.txpciq[q_idx];
2685         } else {
2686                 iq_no = lio->txq;
2687         }
2688
2689         stats = &oct->instr_queue[iq_no]->stats;
2690
2691         /* Check for all conditions in which the current packet cannot be
2692          * transmitted.
2693          */
2694         if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
2695             (!lio->linfo.link.s.status) ||
2696             (skb->len <= 0)) {
2697                 netif_info(lio, tx_err, lio->netdev,
2698                            "Transmit failed link_status : %d\n",
2699                            lio->linfo.link.s.status);
2700                 goto lio_xmit_failed;
2701         }
2702
2703         /* Use space in skb->cb to store info used to unmap and
2704          * free the buffers.
2705          */
2706         finfo = (struct octnet_buf_free_info *)skb->cb;
2707         finfo->lio = lio;
2708         finfo->skb = skb;
2709         finfo->sc = NULL;
2710
2711         /* Prepare the attributes for the data to be passed to OSI. */
2712         memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2713
2714         ndata.buf = (void *)finfo;
2715
2716         ndata.q_no = iq_no;
2717
2718         if (netif_is_multiqueue(netdev)) {
2719                 if (octnet_iq_is_full(oct, ndata.q_no)) {
2720                         /* defer sending if queue is full */
2721                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2722                                    ndata.q_no);
2723                         stats->tx_iq_busy++;
2724                         return NETDEV_TX_BUSY;
2725                 }
2726         } else {
2727                 if (octnet_iq_is_full(oct, lio->txq)) {
2728                         /* defer sending if queue is full */
2729                         stats->tx_iq_busy++;
2730                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2731                                    ndata.q_no);
2732                         return NETDEV_TX_BUSY;
2733                 }
2734         }
2735         /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
2736          *      lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no );
2737          */
2738
2739         ndata.datasize = skb->len;
2740
2741         cmdsetup.u64 = 0;
2742         cmdsetup.s.ifidx = lio->linfo.ifidx;
2743
2744         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2745                 if (is_ipv4(skb) && !is_ip_fragmented(skb) && is_tcpudp(skb)) {
2746                         tag = get_ipv4_5tuple_tag(skb);
2747
2748                         cmdsetup.s.cksum_offset = sizeof(struct ethhdr) + 1;
2749
2750                         if (ip_hdr(skb)->ihl > 5)
2751                                 cmdsetup.s.ipv4opts_ipv6exthdr =
2752                                                 OCT_PKT_PARAM_IPV4OPTS;
2753
2754                 } else if (is_ipv6(skb)) {
2755                         tag = get_ipv6_5tuple_tag(skb);
2756
2757                         cmdsetup.s.cksum_offset = sizeof(struct ethhdr) + 1;
2758
2759                         if (is_with_extn_hdr(skb))
2760                                 cmdsetup.s.ipv4opts_ipv6exthdr =
2761                                                 OCT_PKT_PARAM_IPV6EXTHDR;
2762
2763                 } else if (is_vlan(skb)) {
2764                         if (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto
2765                                 == htons(ETH_P_IP) &&
2766                                 !is_ip_fragmented(skb) && is_tcpudp(skb)) {
2767                                 tag = get_ipv4_5tuple_tag(skb);
2768
2769                                 cmdsetup.s.cksum_offset =
2770                                         sizeof(struct vlan_ethhdr) + 1;
2771
2772                                 if (ip_hdr(skb)->ihl > 5)
2773                                         cmdsetup.s.ipv4opts_ipv6exthdr =
2774                                                 OCT_PKT_PARAM_IPV4OPTS;
2775
2776                         } else if (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto
2777                                 == htons(ETH_P_IPV6)) {
2778                                 tag = get_ipv6_5tuple_tag(skb);
2779
2780                                 cmdsetup.s.cksum_offset =
2781                                         sizeof(struct vlan_ethhdr) + 1;
2782
2783                                 if (is_with_extn_hdr(skb))
2784                                         cmdsetup.s.ipv4opts_ipv6exthdr =
2785                                                 OCT_PKT_PARAM_IPV6EXTHDR;
2786                         }
2787                 }
2788         }
2789         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2790                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2791                 cmdsetup.s.timestamp = 1;
2792         }
2793
2794         if (skb_shinfo(skb)->nr_frags == 0) {
2795                 cmdsetup.s.u.datasize = skb->len;
2796                 octnet_prepare_pci_cmd(&ndata.cmd, &cmdsetup, tag);
2797                 /* Offload checksum calculation for TCP/UDP packets */
2798                 ndata.cmd.dptr = dma_map_single(&oct->pci_dev->dev,
2799                                                 skb->data,
2800                                                 skb->len,
2801                                                 DMA_TO_DEVICE);
2802                 if (dma_mapping_error(&oct->pci_dev->dev, ndata.cmd.dptr)) {
2803                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2804                                 __func__);
2805                         return NETDEV_TX_BUSY;
2806                 }
2807
2808                 finfo->dptr = ndata.cmd.dptr;
2809
2810                 ndata.reqtype = REQTYPE_NORESP_NET;
2811
2812         } else {
2813                 int i, frags;
2814                 struct skb_frag_struct *frag;
2815                 struct octnic_gather *g;
2816
2817                 spin_lock(&lio->lock);
2818                 g = (struct octnic_gather *)list_delete_head(&lio->glist);
2819                 spin_unlock(&lio->lock);
2820
2821                 if (!g) {
2822                         netif_info(lio, tx_err, lio->netdev,
2823                                    "Transmit scatter gather: glist null!\n");
2824                         goto lio_xmit_dma_failed;
2825                 }
2826
2827                 cmdsetup.s.gather = 1;
2828                 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
2829                 octnet_prepare_pci_cmd(&ndata.cmd, &cmdsetup, tag);
2830
2831                 memset(g->sg, 0, g->sg_size);
2832
2833                 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2834                                                  skb->data,
2835                                                  (skb->len - skb->data_len),
2836                                                  DMA_TO_DEVICE);
2837                 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2838                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2839                                 __func__);
2840                         return NETDEV_TX_BUSY;
2841                 }
2842                 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2843
2844                 frags = skb_shinfo(skb)->nr_frags;
2845                 i = 1;
2846                 while (frags--) {
2847                         frag = &skb_shinfo(skb)->frags[i - 1];
2848
2849                         g->sg[(i >> 2)].ptr[(i & 3)] =
2850                                 dma_map_page(&oct->pci_dev->dev,
2851                                              frag->page.p,
2852                                              frag->page_offset,
2853                                              frag->size,
2854                                              DMA_TO_DEVICE);
2855
2856                         add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3));
2857                         i++;
2858                 }
2859
2860                 ndata.cmd.dptr = dma_map_single(&oct->pci_dev->dev,
2861                                                 g->sg, g->sg_size,
2862                                                 DMA_TO_DEVICE);
2863                 if (dma_mapping_error(&oct->pci_dev->dev, ndata.cmd.dptr)) {
2864                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2865                                 __func__);
2866                         dma_unmap_single(&oct->pci_dev->dev, g->sg[0].ptr[0],
2867                                          skb->len - skb->data_len,
2868                                          DMA_TO_DEVICE);
2869                         return NETDEV_TX_BUSY;
2870                 }
2871
2872                 finfo->dptr = ndata.cmd.dptr;
2873                 finfo->g = g;
2874
2875                 ndata.reqtype = REQTYPE_NORESP_NET_SG;
2876         }
2877
2878         if (skb_shinfo(skb)->gso_size) {
2879                 struct octeon_instr_irh *irh =
2880                         (struct octeon_instr_irh *)&ndata.cmd.irh;
2881                 union tx_info *tx_info = (union tx_info *)&ndata.cmd.ossp[0];
2882
2883                 irh->len = 1;   /* to indicate that ossp[0] contains tx_info */
2884                 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2885                 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
2886         }
2887
2888         xmit_more = skb->xmit_more;
2889
2890         if (unlikely(cmdsetup.s.timestamp))
2891                 status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
2892         else
2893                 status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
2894         if (status == IQ_SEND_FAILED)
2895                 goto lio_xmit_dma_failed;
2896
2897         netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2898
2899         if (status == IQ_SEND_STOP)
2900                 stop_q(lio->netdev, q_idx);
2901
2902         netif_trans_update(netdev);
2903
2904         stats->tx_done++;
2905         stats->tx_tot_bytes += skb->len;
2906
2907         return NETDEV_TX_OK;
2908
2909 lio_xmit_dma_failed:
2910         dma_unmap_single(&oct->pci_dev->dev, ndata.cmd.dptr,
2911                          ndata.datasize, DMA_TO_DEVICE);
2912 lio_xmit_failed:
2913         stats->tx_dropped++;
2914         netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2915                    iq_no, stats->tx_dropped);
2916         recv_buffer_free(skb);
2917         return NETDEV_TX_OK;
2918 }
2919
2920 /** \brief Network device Tx timeout
2921  * @param netdev    pointer to network device
2922  */
2923 static void liquidio_tx_timeout(struct net_device *netdev)
2924 {
2925         struct lio *lio;
2926
2927         lio = GET_LIO(netdev);
2928
2929         netif_info(lio, tx_err, lio->netdev,
2930                    "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2931                    netdev->stats.tx_dropped);
2932         netif_trans_update(netdev);
2933         txqs_wake(netdev);
2934 }
2935
2936 int liquidio_set_feature(struct net_device *netdev, int cmd)
2937 {
2938         struct lio *lio = GET_LIO(netdev);
2939         struct octeon_device *oct = lio->oct_dev;
2940         struct octnic_ctrl_pkt nctrl;
2941         struct octnic_ctrl_params nparams;
2942         int ret = 0;
2943
2944         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2945
2946         nctrl.ncmd.u64 = 0;
2947         nctrl.ncmd.s.cmd = cmd;
2948         nctrl.ncmd.s.param1 = lio->linfo.ifidx;
2949         nctrl.ncmd.s.param2 = OCTNIC_LROIPV4 | OCTNIC_LROIPV6;
2950         nctrl.wait_time = 100;
2951         nctrl.netpndev = (u64)netdev;
2952         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2953
2954         nparams.resp_order = OCTEON_RESP_NORESPONSE;
2955
2956         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
2957         if (ret < 0) {
2958                 dev_err(&oct->pci_dev->dev, "Feature change failed in core (ret: 0x%x)\n",
2959                         ret);
2960         }
2961         return ret;
2962 }
2963
2964 /** \brief Net device fix features
2965  * @param netdev  pointer to network device
2966  * @param request features requested
2967  * @returns updated features list
2968  */
2969 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
2970                                                netdev_features_t request)
2971 {
2972         struct lio *lio = netdev_priv(netdev);
2973
2974         if ((request & NETIF_F_RXCSUM) &&
2975             !(lio->dev_capability & NETIF_F_RXCSUM))
2976                 request &= ~NETIF_F_RXCSUM;
2977
2978         if ((request & NETIF_F_HW_CSUM) &&
2979             !(lio->dev_capability & NETIF_F_HW_CSUM))
2980                 request &= ~NETIF_F_HW_CSUM;
2981
2982         if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
2983                 request &= ~NETIF_F_TSO;
2984
2985         if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
2986                 request &= ~NETIF_F_TSO6;
2987
2988         if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
2989                 request &= ~NETIF_F_LRO;
2990
2991         /*Disable LRO if RXCSUM is off */
2992         if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
2993             (lio->dev_capability & NETIF_F_LRO))
2994                 request &= ~NETIF_F_LRO;
2995
2996         return request;
2997 }
2998
2999 /** \brief Net device set features
3000  * @param netdev  pointer to network device
3001  * @param features features to enable/disable
3002  */
3003 static int liquidio_set_features(struct net_device *netdev,
3004                                  netdev_features_t features)
3005 {
3006         struct lio *lio = netdev_priv(netdev);
3007
3008         if (!((netdev->features ^ features) & NETIF_F_LRO))
3009                 return 0;
3010
3011         if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO))
3012                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE);
3013         else if (!(features & NETIF_F_LRO) &&
3014                  (lio->dev_capability & NETIF_F_LRO))
3015                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE);
3016
3017         return 0;
3018 }
3019
3020 static struct net_device_ops lionetdevops = {
3021         .ndo_open               = liquidio_open,
3022         .ndo_stop               = liquidio_stop,
3023         .ndo_start_xmit         = liquidio_xmit,
3024         .ndo_get_stats          = liquidio_get_stats,
3025         .ndo_set_mac_address    = liquidio_set_mac,
3026         .ndo_set_rx_mode        = liquidio_set_mcast_list,
3027         .ndo_tx_timeout         = liquidio_tx_timeout,
3028         .ndo_change_mtu         = liquidio_change_mtu,
3029         .ndo_do_ioctl           = liquidio_ioctl,
3030         .ndo_fix_features       = liquidio_fix_features,
3031         .ndo_set_features       = liquidio_set_features,
3032 };
3033
3034 /** \brief Entry point for the liquidio module
3035  */
3036 static int __init liquidio_init(void)
3037 {
3038         int i;
3039         struct handshake *hs;
3040
3041         init_completion(&first_stage);
3042
3043         octeon_init_device_list(conf_type);
3044
3045         if (liquidio_init_pci())
3046                 return -EINVAL;
3047
3048         wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3049
3050         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3051                 hs = &handshake[i];
3052                 if (hs->pci_dev) {
3053                         wait_for_completion(&hs->init);
3054                         if (!hs->init_ok) {
3055                                 /* init handshake failed */
3056                                 dev_err(&hs->pci_dev->dev,
3057                                         "Failed to init device\n");
3058                                 liquidio_deinit_pci();
3059                                 return -EIO;
3060                         }
3061                 }
3062         }
3063
3064         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3065                 hs = &handshake[i];
3066                 if (hs->pci_dev) {
3067                         wait_for_completion_timeout(&hs->started,
3068                                                     msecs_to_jiffies(30000));
3069                         if (!hs->started_ok) {
3070                                 /* starter handshake failed */
3071                                 dev_err(&hs->pci_dev->dev,
3072                                         "Firmware failed to start\n");
3073                                 liquidio_deinit_pci();
3074                                 return -EIO;
3075                         }
3076                 }
3077         }
3078
3079         return 0;
3080 }
3081
3082 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3083 {
3084         struct octeon_device *oct = (struct octeon_device *)buf;
3085         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3086         int ifidx = 0;
3087         union oct_link_status *ls;
3088         int i;
3089
3090         if ((recv_pkt->buffer_size[0] != sizeof(*ls)) ||
3091             (recv_pkt->rh.r_nic_info.ifidx > oct->ifcount)) {
3092                 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3093                         recv_pkt->buffer_size[0],
3094                         recv_pkt->rh.r_nic_info.ifidx);
3095                 goto nic_info_err;
3096         }
3097
3098         ifidx = recv_pkt->rh.r_nic_info.ifidx;
3099         ls = (union oct_link_status *)get_rbd(recv_pkt->buffer_ptr[0]);
3100
3101         octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3102
3103         update_link_status(oct->props[ifidx].netdev, ls);
3104
3105 nic_info_err:
3106         for (i = 0; i < recv_pkt->buffer_count; i++)
3107                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
3108         octeon_free_recv_info(recv_info);
3109         return 0;
3110 }
3111
3112 /**
3113  * \brief Setup network interfaces
3114  * @param octeon_dev  octeon device
3115  *
3116  * Called during init time for each device. It assumes the NIC
3117  * is already up and running.  The link information for each
3118  * interface is passed in link_info.
3119  */
3120 static int setup_nic_devices(struct octeon_device *octeon_dev)
3121 {
3122         struct lio *lio = NULL;
3123         struct net_device *netdev;
3124         u8 mac[6], i, j;
3125         struct octeon_soft_command *sc;
3126         struct liquidio_if_cfg_context *ctx;
3127         struct liquidio_if_cfg_resp *resp;
3128         struct octdev_props *props;
3129         int retval, num_iqueues, num_oqueues, q_no;
3130         u64 q_mask;
3131         int num_cpus = num_online_cpus();
3132         union oct_nic_if_cfg if_cfg;
3133         unsigned int base_queue;
3134         unsigned int gmx_port_id;
3135         u32 resp_size, ctx_size;
3136
3137         /* This is to handle link status changes */
3138         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3139                                     OPCODE_NIC_INFO,
3140                                     lio_nic_info, octeon_dev);
3141
3142         /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3143          * They are handled directly.
3144          */
3145         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3146                                         free_netbuf);
3147
3148         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3149                                         free_netsgbuf);
3150
3151         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3152                                         free_netsgbuf_with_resp);
3153
3154         for (i = 0; i < octeon_dev->ifcount; i++) {
3155                 resp_size = sizeof(struct liquidio_if_cfg_resp);
3156                 ctx_size = sizeof(struct liquidio_if_cfg_context);
3157                 sc = (struct octeon_soft_command *)
3158                         octeon_alloc_soft_command(octeon_dev, 0,
3159                                                   resp_size, ctx_size);
3160                 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3161                 ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
3162
3163                 num_iqueues =
3164                         CFG_GET_NUM_TXQS_NIC_IF(octeon_get_conf(octeon_dev), i);
3165                 num_oqueues =
3166                         CFG_GET_NUM_RXQS_NIC_IF(octeon_get_conf(octeon_dev), i);
3167                 base_queue =
3168                         CFG_GET_BASE_QUE_NIC_IF(octeon_get_conf(octeon_dev), i);
3169                 gmx_port_id =
3170                         CFG_GET_GMXID_NIC_IF(octeon_get_conf(octeon_dev), i);
3171                 if (num_iqueues > num_cpus)
3172                         num_iqueues = num_cpus;
3173                 if (num_oqueues > num_cpus)
3174                         num_oqueues = num_cpus;
3175                 dev_dbg(&octeon_dev->pci_dev->dev,
3176                         "requesting config for interface %d, iqs %d, oqs %d\n",
3177                         i, num_iqueues, num_oqueues);
3178                 ACCESS_ONCE(ctx->cond) = 0;
3179                 ctx->octeon_id = lio_get_device_id(octeon_dev);
3180                 init_waitqueue_head(&ctx->wc);
3181
3182                 if_cfg.u64 = 0;
3183                 if_cfg.s.num_iqueues = num_iqueues;
3184                 if_cfg.s.num_oqueues = num_oqueues;
3185                 if_cfg.s.base_queue = base_queue;
3186                 if_cfg.s.gmx_port_id = gmx_port_id;
3187                 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3188                                             OPCODE_NIC_IF_CFG, i,
3189                                             if_cfg.u64, 0);
3190
3191                 sc->callback = if_cfg_callback;
3192                 sc->callback_arg = sc;
3193                 sc->wait_time = 1000;
3194
3195                 retval = octeon_send_soft_command(octeon_dev, sc);
3196                 if (retval) {
3197                         dev_err(&octeon_dev->pci_dev->dev,
3198                                 "iq/oq config failed status: %x\n",
3199                                 retval);
3200                         /* Soft instr is freed by driver in case of failure. */
3201                         goto setup_nic_dev_fail;
3202                 }
3203
3204                 /* Sleep on a wait queue till the cond flag indicates that the
3205                  * response arrived or timed-out.
3206                  */
3207                 sleep_cond(&ctx->wc, &ctx->cond);
3208                 retval = resp->status;
3209                 if (retval) {
3210                         dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3211                         goto setup_nic_dev_fail;
3212                 }
3213
3214                 octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3215                                     (sizeof(struct liquidio_if_cfg_info)) >> 3);
3216
3217                 num_iqueues = hweight64(resp->cfg_info.iqmask);
3218                 num_oqueues = hweight64(resp->cfg_info.oqmask);
3219
3220                 if (!(num_iqueues) || !(num_oqueues)) {
3221                         dev_err(&octeon_dev->pci_dev->dev,
3222                                 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3223                                 resp->cfg_info.iqmask,
3224                                 resp->cfg_info.oqmask);
3225                         goto setup_nic_dev_fail;
3226                 }
3227                 dev_dbg(&octeon_dev->pci_dev->dev,
3228                         "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
3229                         i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3230                         num_iqueues, num_oqueues);
3231                 netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues);
3232
3233                 if (!netdev) {
3234                         dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3235                         goto setup_nic_dev_fail;
3236                 }
3237
3238                 props = &octeon_dev->props[i];
3239                 props->netdev = netdev;
3240
3241                 if (num_iqueues > 1)
3242                         lionetdevops.ndo_select_queue = select_q;
3243
3244                 /* Associate the routines that will handle different
3245                  * netdev tasks.
3246                  */
3247                 netdev->netdev_ops = &lionetdevops;
3248
3249                 lio = GET_LIO(netdev);
3250
3251                 memset(lio, 0, sizeof(struct lio));
3252
3253                 lio->linfo.ifidx = resp->cfg_info.ifidx;
3254                 lio->ifidx = resp->cfg_info.ifidx;
3255
3256                 lio->linfo.num_rxpciq = num_oqueues;
3257                 lio->linfo.num_txpciq = num_iqueues;
3258                 q_mask = resp->cfg_info.oqmask;
3259                 /* q_mask is 0-based and already verified mask is nonzero */
3260                 for (j = 0; j < num_oqueues; j++) {
3261                         q_no = __ffs64(q_mask);
3262                         q_mask &= (~(1UL << q_no));
3263                         lio->linfo.rxpciq[j] = q_no;
3264                 }
3265                 q_mask = resp->cfg_info.iqmask;
3266                 for (j = 0; j < num_iqueues; j++) {
3267                         q_no = __ffs64(q_mask);
3268                         q_mask &= (~(1UL << q_no));
3269                         lio->linfo.txpciq[j] = q_no;
3270                 }
3271                 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3272                 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3273                 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3274
3275                 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3276
3277                 lio->dev_capability = NETIF_F_HIGHDMA
3278                                       | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM
3279                                       | NETIF_F_SG | NETIF_F_RXCSUM
3280                                       | NETIF_F_TSO | NETIF_F_TSO6
3281                                       | NETIF_F_LRO;
3282                 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3283
3284                 netdev->features = lio->dev_capability;
3285                 netdev->vlan_features = lio->dev_capability;
3286
3287                 netdev->hw_features = lio->dev_capability;
3288
3289                 /* Point to the  properties for octeon device to which this
3290                  * interface belongs.
3291                  */
3292                 lio->oct_dev = octeon_dev;
3293                 lio->octprops = props;
3294                 lio->netdev = netdev;
3295                 spin_lock_init(&lio->lock);
3296
3297                 dev_dbg(&octeon_dev->pci_dev->dev,
3298                         "if%d gmx: %d hw_addr: 0x%llx\n", i,
3299                         lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3300
3301                 /* 64-bit swap required on LE machines */
3302                 octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3303                 for (j = 0; j < 6; j++)
3304                         mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3305
3306                 /* Copy MAC Address to OS network device structure */
3307
3308                 ether_addr_copy(netdev->dev_addr, mac);
3309
3310                 if (setup_io_queues(octeon_dev, netdev)) {
3311                         dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3312                         goto setup_nic_dev_fail;
3313                 }
3314
3315                 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3316
3317                 /* By default all interfaces on a single Octeon uses the same
3318                  * tx and rx queues
3319                  */
3320                 lio->txq = lio->linfo.txpciq[0];
3321                 lio->rxq = lio->linfo.rxpciq[0];
3322
3323                 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3324                 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3325
3326                 if (setup_glist(lio)) {
3327                         dev_err(&octeon_dev->pci_dev->dev,
3328                                 "Gather list allocation failed\n");
3329                         goto setup_nic_dev_fail;
3330                 }
3331
3332                 /* Register ethtool support */
3333                 liquidio_set_ethtool_ops(netdev);
3334
3335                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE);
3336
3337                 if ((debug != -1) && (debug & NETIF_MSG_HW))
3338                         liquidio_set_feature(netdev, OCTNET_CMD_VERBOSE_ENABLE);
3339
3340                 /* Register the network device with the OS */
3341                 if (register_netdev(netdev)) {
3342                         dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3343                         goto setup_nic_dev_fail;
3344                 }
3345
3346                 dev_dbg(&octeon_dev->pci_dev->dev,
3347                         "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3348                         i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3349                 netif_carrier_off(netdev);
3350
3351                 if (lio->linfo.link.s.status) {
3352                         netif_carrier_on(netdev);
3353                         start_txq(netdev);
3354                 } else {
3355                         netif_carrier_off(netdev);
3356                 }
3357
3358                 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3359
3360                 dev_dbg(&octeon_dev->pci_dev->dev,
3361                         "NIC ifidx:%d Setup successful\n", i);
3362
3363                 octeon_free_soft_command(octeon_dev, sc);
3364         }
3365
3366         return 0;
3367
3368 setup_nic_dev_fail:
3369
3370         octeon_free_soft_command(octeon_dev, sc);
3371
3372         while (i--) {
3373                 dev_err(&octeon_dev->pci_dev->dev,
3374                         "NIC ifidx:%d Setup failed\n", i);
3375                 liquidio_destroy_nic_device(octeon_dev, i);
3376         }
3377         return -ENODEV;
3378 }
3379
3380 /**
3381  * \brief initialize the NIC
3382  * @param oct octeon device
3383  *
3384  * This initialization routine is called once the Octeon device application is
3385  * up and running
3386  */
3387 static int liquidio_init_nic_module(struct octeon_device *oct)
3388 {
3389         struct oct_intrmod_cfg *intrmod_cfg;
3390         int retval = 0;
3391         int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
3392
3393         dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
3394
3395         /* only default iq and oq were initialized
3396          * initialize the rest as well
3397          */
3398         /* run port_config command for each port */
3399         oct->ifcount = num_nic_ports;
3400
3401         memset(oct->props, 0,
3402                sizeof(struct octdev_props) * num_nic_ports);
3403
3404         retval = setup_nic_devices(oct);
3405         if (retval) {
3406                 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
3407                 goto octnet_init_failure;
3408         }
3409
3410         liquidio_ptp_init(oct);
3411
3412         /* Initialize interrupt moderation params */
3413         intrmod_cfg = &((struct octeon_device *)oct)->intrmod;
3414         intrmod_cfg->intrmod_enable = 1;
3415         intrmod_cfg->intrmod_check_intrvl = LIO_INTRMOD_CHECK_INTERVAL;
3416         intrmod_cfg->intrmod_maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR;
3417         intrmod_cfg->intrmod_minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR;
3418         intrmod_cfg->intrmod_maxcnt_trigger = LIO_INTRMOD_MAXCNT_TRIGGER;
3419         intrmod_cfg->intrmod_maxtmr_trigger = LIO_INTRMOD_MAXTMR_TRIGGER;
3420         intrmod_cfg->intrmod_mintmr_trigger = LIO_INTRMOD_MINTMR_TRIGGER;
3421         intrmod_cfg->intrmod_mincnt_trigger = LIO_INTRMOD_MINCNT_TRIGGER;
3422
3423         dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
3424
3425         return retval;
3426
3427 octnet_init_failure:
3428
3429         oct->ifcount = 0;
3430
3431         return retval;
3432 }
3433
3434 /**
3435  * \brief starter callback that invokes the remaining initialization work after
3436  * the NIC is up and running.
3437  * @param octptr  work struct work_struct
3438  */
3439 static void nic_starter(struct work_struct *work)
3440 {
3441         struct octeon_device *oct;
3442         struct cavium_wk *wk = (struct cavium_wk *)work;
3443
3444         oct = (struct octeon_device *)wk->ctxptr;
3445
3446         if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
3447                 return;
3448
3449         /* If the status of the device is CORE_OK, the core
3450          * application has reported its application type. Call
3451          * any registered handlers now and move to the RUNNING
3452          * state.
3453          */
3454         if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
3455                 schedule_delayed_work(&oct->nic_poll_work.work,
3456                                       LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3457                 return;
3458         }
3459
3460         atomic_set(&oct->status, OCT_DEV_RUNNING);
3461
3462         if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
3463                 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
3464
3465                 if (liquidio_init_nic_module(oct))
3466                         dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
3467                 else
3468                         handshake[oct->octeon_id].started_ok = 1;
3469         } else {
3470                 dev_err(&oct->pci_dev->dev,
3471                         "Unexpected application running on NIC (%d). Check firmware.\n",
3472                         oct->app_mode);
3473         }
3474
3475         complete(&handshake[oct->octeon_id].started);
3476 }
3477
3478 /**
3479  * \brief Device initialization for each Octeon device that is probed
3480  * @param octeon_dev  octeon device
3481  */
3482 static int octeon_device_init(struct octeon_device *octeon_dev)
3483 {
3484         int j, ret;
3485         struct octeon_device_priv *oct_priv =
3486                 (struct octeon_device_priv *)octeon_dev->priv;
3487         atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
3488
3489         /* Enable access to the octeon device and make its DMA capability
3490          * known to the OS.
3491          */
3492         if (octeon_pci_os_setup(octeon_dev))
3493                 return 1;
3494
3495         /* Identify the Octeon type and map the BAR address space. */
3496         if (octeon_chip_specific_setup(octeon_dev)) {
3497                 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
3498                 return 1;
3499         }
3500
3501         atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
3502
3503         octeon_dev->app_mode = CVM_DRV_INVALID_APP;
3504
3505         /* Do a soft reset of the Octeon device. */
3506         if (octeon_dev->fn_list.soft_reset(octeon_dev))
3507                 return 1;
3508
3509         /* Initialize the dispatch mechanism used to push packets arriving on
3510          * Octeon Output queues.
3511          */
3512         if (octeon_init_dispatch_list(octeon_dev))
3513                 return 1;
3514
3515         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3516                                     OPCODE_NIC_CORE_DRV_ACTIVE,
3517                                     octeon_core_drv_init,
3518                                     octeon_dev);
3519
3520         INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
3521         octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
3522         schedule_delayed_work(&octeon_dev->nic_poll_work.work,
3523                               LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3524
3525         atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
3526
3527         octeon_set_io_queues_off(octeon_dev);
3528
3529         /*  Setup the data structures that manage this Octeon's Input queues. */
3530         if (octeon_setup_instr_queues(octeon_dev)) {
3531                 dev_err(&octeon_dev->pci_dev->dev,
3532                         "instruction queue initialization failed\n");
3533                 /* On error, release any previously allocated queues */
3534                 for (j = 0; j < octeon_dev->num_iqs; j++)
3535                         octeon_delete_instr_queue(octeon_dev, j);
3536                 return 1;
3537         }
3538         atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
3539
3540         /* Initialize soft command buffer pool
3541          */
3542         if (octeon_setup_sc_buffer_pool(octeon_dev)) {
3543                 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
3544                 return 1;
3545         }
3546         atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
3547
3548         /* Initialize lists to manage the requests of different types that
3549          * arrive from user & kernel applications for this octeon device.
3550          */
3551         if (octeon_setup_response_list(octeon_dev)) {
3552                 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
3553                 return 1;
3554         }
3555         atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
3556
3557         if (octeon_setup_output_queues(octeon_dev)) {
3558                 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
3559                 /* Release any previously allocated queues */
3560                 for (j = 0; j < octeon_dev->num_oqs; j++)
3561                         octeon_delete_droq(octeon_dev, j);
3562         }
3563
3564         atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
3565
3566         /* The input and output queue registers were setup earlier (the queues
3567          * were not enabled). Any additional registers that need to be
3568          * programmed should be done now.
3569          */
3570         ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
3571         if (ret) {
3572                 dev_err(&octeon_dev->pci_dev->dev,
3573                         "Failed to configure device registers\n");
3574                 return ret;
3575         }
3576
3577         /* Initialize the tasklet that handles output queue packet processing.*/
3578         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
3579         tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
3580                      (unsigned long)octeon_dev);
3581
3582         /* Setup the interrupt handler and record the INT SUM register address
3583          */
3584         octeon_setup_interrupt(octeon_dev);
3585
3586         /* Enable Octeon device interrupts */
3587         octeon_dev->fn_list.enable_interrupt(octeon_dev->chip);
3588
3589         /* Enable the input and output queues for this Octeon device */
3590         octeon_dev->fn_list.enable_io_queues(octeon_dev);
3591
3592         atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
3593
3594         dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
3595
3596         if (ddr_timeout == 0) {
3597                 dev_info(&octeon_dev->pci_dev->dev,
3598                          "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
3599         }
3600
3601         schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
3602
3603         /* Wait for the octeon to initialize DDR after the soft-reset. */
3604         ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
3605         if (ret) {
3606                 dev_err(&octeon_dev->pci_dev->dev,
3607                         "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
3608                         ret);
3609                 return 1;
3610         }
3611
3612         if (octeon_wait_for_bootloader(octeon_dev, 1000) != 0) {
3613                 dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
3614                 return 1;
3615         }
3616
3617         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
3618         ret = octeon_init_consoles(octeon_dev);
3619         if (ret) {
3620                 dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
3621                 return 1;
3622         }
3623         ret = octeon_add_console(octeon_dev, 0);
3624         if (ret) {
3625                 dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
3626                 return 1;
3627         }
3628
3629         atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
3630
3631         dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
3632         ret = load_firmware(octeon_dev);
3633         if (ret) {
3634                 dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
3635                 return 1;
3636         }
3637
3638         handshake[octeon_dev->octeon_id].init_ok = 1;
3639         complete(&handshake[octeon_dev->octeon_id].init);
3640
3641         atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
3642
3643         /* Send Credit for Octeon Output queues. Credits are always sent after
3644          * the output queue is enabled.
3645          */
3646         for (j = 0; j < octeon_dev->num_oqs; j++)
3647                 writel(octeon_dev->droq[j]->max_count,
3648                        octeon_dev->droq[j]->pkts_credit_reg);
3649
3650         /* Packets can start arriving on the output queues from this point. */
3651
3652         return 0;
3653 }
3654
3655 /**
3656  * \brief Exits the module
3657  */
3658 static void __exit liquidio_exit(void)
3659 {
3660         liquidio_deinit_pci();
3661
3662         pr_info("LiquidIO network module is now unloaded\n");
3663 }
3664
3665 module_init(liquidio_init);
3666 module_exit(liquidio_exit);