drivers/net/ethernet/stmicro/stmmac/stmmac_main.c

   1 /*******************************************************************************
   2   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
   3   ST Ethernet IPs are built around a Synopsys IP Core.
   4
   5         Copyright(C) 2007-2011 STMicroelectronics Ltd
   6
   7   This program is free software; you can redistribute it and/or modify it
   8   under the terms and conditions of the GNU General Public License,
   9   version 2, as published by the Free Software Foundation.
  10
  11   This program is distributed in the hope it will be useful, but WITHOUT
  12   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14   more details.
  15
  16   The full GNU General Public License is included in this distribution in
  17   the file called "COPYING".
  18
  19   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
  20
  21   Documentation available at:
  22         http://www.stlinux.com
  23   Support available at:
  24         https://bugzilla.stlinux.com/
  25 *******************************************************************************/
  26
  27 #include <linux/clk.h>
  28 #include <linux/kernel.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/ip.h>
  31 #include <linux/tcp.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/if_ether.h>
  35 #include <linux/crc32.h>
  36 #include <linux/mii.h>
  37 #include <linux/if.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/slab.h>
  41 #include <linux/prefetch.h>
  42 #include <linux/pinctrl/consumer.h>
  43 #ifdef CONFIG_DEBUG_FS
  44 #include <linux/debugfs.h>
  45 #include <linux/seq_file.h>
  46 #endif /* CONFIG_DEBUG_FS */
  47 #include <linux/net_tstamp.h>
  48 #include "stmmac_ptp.h"
  49 #include "stmmac.h"
  50 #include <linux/reset.h>
  51 #include <linux/of_mdio.h>
  52 #include "dwmac1000.h"
  53
  54 #define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x)
  55 #define TSO_MAX_BUFF_SIZE       (SZ_16K - 1)
  56
  57 /* Module parameters */
  58 #define TX_TIMEO        5000
  59 static int watchdog = TX_TIMEO;
  60 module_param(watchdog, int, S_IRUGO | S_IWUSR);
  61 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
  62
  63 static int debug = -1;
  64 module_param(debug, int, S_IRUGO | S_IWUSR);
  65 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
  66
  67 static int phyaddr = -1;
  68 module_param(phyaddr, int, S_IRUGO);
  69 MODULE_PARM_DESC(phyaddr, "Physical device address");
  70
  71 #define STMMAC_TX_THRESH        (DMA_TX_SIZE / 4)
  72 #define STMMAC_RX_THRESH        (DMA_RX_SIZE / 4)
  73
  74 static int flow_ctrl = FLOW_OFF;
  75 module_param(flow_ctrl, int, S_IRUGO | S_IWUSR);
  76 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
  77
  78 static int pause = PAUSE_TIME;
  79 module_param(pause, int, S_IRUGO | S_IWUSR);
  80 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
  81
  82 #define TC_DEFAULT 64
  83 static int tc = TC_DEFAULT;
  84 module_param(tc, int, S_IRUGO | S_IWUSR);
  85 MODULE_PARM_DESC(tc, "DMA threshold control value");
  86
  87 #define DEFAULT_BUFSIZE 1536
  88 static int buf_sz = DEFAULT_BUFSIZE;
  89 module_param(buf_sz, int, S_IRUGO | S_IWUSR);
  90 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
  91
  92 #define STMMAC_RX_COPYBREAK     256
  93
  94 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
  95                                       NETIF_MSG_LINK | NETIF_MSG_IFUP |
  96                                       NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
  97
  98 #define STMMAC_DEFAULT_LPI_TIMER        1000
  99 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 100 module_param(eee_timer, int, S_IRUGO | S_IWUSR);
 101 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
 102 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
 103
 104 /* By default the driver will use the ring mode to manage tx and rx descriptors,
 105  * but allow user to force to use the chain instead of the ring
 106  */
 107 static unsigned int chain_mode;
 108 module_param(chain_mode, int, S_IRUGO);
 109 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
 110
 111 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
 112
 113 #ifdef CONFIG_DEBUG_FS
 114 static int stmmac_init_fs(struct net_device *dev);
 115 static void stmmac_exit_fs(struct net_device *dev);
 116 #endif
 117
 118 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
 119
 120 /**
 121  * stmmac_verify_args - verify the driver parameters.
 122  * Description: it checks the driver parameters and set a default in case of
 123  * errors.
 124  */
 125 static void stmmac_verify_args(void)
 126 {
 127         if (unlikely(watchdog < 0))
 128                 watchdog = TX_TIMEO;
 129         if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
 130                 buf_sz = DEFAULT_BUFSIZE;
 131         if (unlikely(flow_ctrl > 1))
 132                 flow_ctrl = FLOW_AUTO;
 133         else if (likely(flow_ctrl < 0))
 134                 flow_ctrl = FLOW_OFF;
 135         if (unlikely((pause < 0) || (pause > 0xffff)))
 136                 pause = PAUSE_TIME;
 137         if (eee_timer < 0)
 138                 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 139 }
 140
 141 /**
 142  * stmmac_clk_csr_set - dynamically set the MDC clock
 143  * @priv: driver private structure
 144  * Description: this is to dynamically set the MDC clock according to the csr
 145  * clock input.
 146  * Note:
 147  *      If a specific clk_csr value is passed from the platform
 148  *      this means that the CSR Clock Range selection cannot be
 149  *      changed at run-time and it is fixed (as reported in the driver
 150  *      documentation). Viceversa the driver will try to set the MDC
 151  *      clock dynamically according to the actual clock input.
 152  */
 153 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
 154 {
 155         u32 clk_rate;
 156
 157         clk_rate = clk_get_rate(priv->plat->stmmac_clk);
 158
 159         /* Platform provided default clk_csr would be assumed valid
 160          * for all other cases except for the below mentioned ones.
 161          * For values higher than the IEEE 802.3 specified frequency
 162          * we can not estimate the proper divider as it is not known
 163          * the frequency of clk_csr_i. So we do not change the default
 164          * divider.
 165          */
 166         if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
 167                 if (clk_rate < CSR_F_35M)
 168                         priv->clk_csr = STMMAC_CSR_20_35M;
 169                 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
 170                         priv->clk_csr = STMMAC_CSR_35_60M;
 171                 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
 172                         priv->clk_csr = STMMAC_CSR_60_100M;
 173                 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
 174                         priv->clk_csr = STMMAC_CSR_100_150M;
 175                 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
 176                         priv->clk_csr = STMMAC_CSR_150_250M;
 177                 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
 178                         priv->clk_csr = STMMAC_CSR_250_300M;
 179         }
 180 }
 181
 182 static void print_pkt(unsigned char *buf, int len)
 183 {
 184         pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
 185         print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
 186 }
 187
 188 /**
 189  * stmmac_tx_avail - Get tx queue availability
 190  * @priv: driver private structure
 191  * @queue: TX queue index
 192  */
 193 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
 194 {
 195         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 196         u32 avail;
 197
 198         if (tx_q->dirty_tx > tx_q->cur_tx)
 199                 avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
 200         else
 201                 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
 202
 203         return avail;
 204 }
 205
 206 /**
 207  * stmmac_rx_dirty - Get RX queue dirty
 208  * @priv: driver private structure
 209  * @queue: RX queue index
 210  */
 211 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
 212 {
 213         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 214         u32 dirty;
 215
 216         if (rx_q->dirty_rx <= rx_q->cur_rx)
 217                 dirty = rx_q->cur_rx - rx_q->dirty_rx;
 218         else
 219                 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
 220
 221         return dirty;
 222 }
 223
 224 /**
 225  * stmmac_hw_fix_mac_speed - callback for speed selection
 226  * @priv: driver private structure
 227  * Description: on some platforms (e.g. ST), some HW system configuration
 228  * registers have to be set according to the link speed negotiated.
 229  */
 230 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv)
 231 {
 232         struct net_device *ndev = priv->dev;
 233         struct phy_device *phydev = ndev->phydev;
 234
 235         if (likely(priv->plat->fix_mac_speed))
 236                 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed);
 237 }
 238
 239 /**
 240  * stmmac_enable_eee_mode - check and enter in LPI mode
 241  * @priv: driver private structure
 242  * Description: this function is to verify and enter in LPI mode in case of
 243  * EEE.
 244  */
 245 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
 246 {
 247         u32 tx_cnt = priv->plat->tx_queues_to_use;
 248         u32 queue;
 249
 250         /* check if all TX queues have the work finished */
 251         for (queue = 0; queue < tx_cnt; queue++) {
 252                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 253
 254                 if (tx_q->dirty_tx != tx_q->cur_tx)
 255                         return; /* still unfinished work */
 256         }
 257
 258         /* Check and enter in LPI mode */
 259         if (!priv->tx_path_in_lpi_mode)
 260                 priv->hw->mac->set_eee_mode(priv->hw,
 261                                             priv->plat->en_tx_lpi_clockgating);
 262 }
 263
 264 /**
 265  * stmmac_disable_eee_mode - disable and exit from LPI mode
 266  * @priv: driver private structure
 267  * Description: this function is to exit and disable EEE in case of
 268  * LPI state is true. This is called by the xmit.
 269  */
 270 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
 271 {
 272         priv->hw->mac->reset_eee_mode(priv->hw);
 273         del_timer_sync(&priv->eee_ctrl_timer);
 274         priv->tx_path_in_lpi_mode = false;
 275 }
 276
 277 /**
 278  * stmmac_eee_ctrl_timer - EEE TX SW timer.
 279  * @arg : data hook
 280  * Description:
 281  *  if there is no data transfer and if we are not in LPI state,
 282  *  then MAC Transmitter can be moved to LPI state.
 283  */
 284 static void stmmac_eee_ctrl_timer(unsigned long arg)
 285 {
 286         struct stmmac_priv *priv = (struct stmmac_priv *)arg;
 287
 288         stmmac_enable_eee_mode(priv);
 289         mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
 290 }
 291
 292 /**
 293  * stmmac_eee_init - init EEE
 294  * @priv: driver private structure
 295  * Description:
 296  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
 297  *  can also manage EEE, this function enable the LPI state and start related
 298  *  timer.
 299  */
 300 bool stmmac_eee_init(struct stmmac_priv *priv)
 301 {
 302         struct net_device *ndev = priv->dev;
 303         unsigned long flags;
 304         bool ret = false;
 305
 306         /* Using PCS we cannot dial with the phy registers at this stage
 307          * so we do not support extra feature like EEE.
 308          */
 309         if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
 310             (priv->hw->pcs == STMMAC_PCS_TBI) ||
 311             (priv->hw->pcs == STMMAC_PCS_RTBI))
 312                 goto out;
 313
 314         /* MAC core supports the EEE feature. */
 315         if (priv->dma_cap.eee) {
 316                 int tx_lpi_timer = priv->tx_lpi_timer;
 317
 318                 /* Check if the PHY supports EEE */
 319                 if (phy_init_eee(ndev->phydev, 1)) {
 320                         /* To manage at run-time if the EEE cannot be supported
 321                          * anymore (for example because the lp caps have been
 322                          * changed).
 323                          * In that case the driver disable own timers.
 324                          */
 325                         spin_lock_irqsave(&priv->lock, flags);
 326                         if (priv->eee_active) {
 327                                 netdev_dbg(priv->dev, "disable EEE\n");
 328                                 del_timer_sync(&priv->eee_ctrl_timer);
 329                                 priv->hw->mac->set_eee_timer(priv->hw, 0,
 330                                                              tx_lpi_timer);
 331                         }
 332                         priv->eee_active = 0;
 333                         spin_unlock_irqrestore(&priv->lock, flags);
 334                         goto out;
 335                 }
 336                 /* Activate the EEE and start timers */
 337                 spin_lock_irqsave(&priv->lock, flags);
 338                 if (!priv->eee_active) {
 339                         priv->eee_active = 1;
 340                         setup_timer(&priv->eee_ctrl_timer,
 341                                     stmmac_eee_ctrl_timer,
 342                                     (unsigned long)priv);
 343                         mod_timer(&priv->eee_ctrl_timer,
 344                                   STMMAC_LPI_T(eee_timer));
 345
 346                         priv->hw->mac->set_eee_timer(priv->hw,
 347                                                      STMMAC_DEFAULT_LIT_LS,
 348                                                      tx_lpi_timer);
 349                 }
 350                 /* Set HW EEE according to the speed */
 351                 priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link);
 352
 353                 ret = true;
 354                 spin_unlock_irqrestore(&priv->lock, flags);
 355
 356                 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
 357         }
 358 out:
 359         return ret;
 360 }
 361
 362 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
 363  * @priv: driver private structure
 364  * @p : descriptor pointer
 365  * @skb : the socket buffer
 366  * Description :
 367  * This function will read timestamp from the descriptor & pass it to stack.
 368  * and also perform some sanity checks.
 369  */
 370 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
 371                                    struct dma_desc *p, struct sk_buff *skb)
 372 {
 373         struct skb_shared_hwtstamps shhwtstamp;
 374         u64 ns;
 375
 376         if (!priv->hwts_tx_en)
 377                 return;
 378
 379         /* exit if skb doesn't support hw tstamp */
 380         if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
 381                 return;
 382
 383         /* check tx tstamp status */
 384         if (!priv->hw->desc->get_tx_timestamp_status(p)) {
 385                 /* get the valid tstamp */
 386                 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 387
 388                 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 389                 shhwtstamp.hwtstamp = ns_to_ktime(ns);
 390
 391                 netdev_info(priv->dev, "get valid TX hw timestamp %llu\n", ns);
 392                 /* pass tstamp to stack */
 393                 skb_tstamp_tx(skb, &shhwtstamp);
 394         }
 395
 396         return;
 397 }
 398
 399 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
 400  * @priv: driver private structure
 401  * @p : descriptor pointer
 402  * @np : next descriptor pointer
 403  * @skb : the socket buffer
 404  * Description :
 405  * This function will read received packet's timestamp from the descriptor
 406  * and pass it to stack. It also perform some sanity checks.
 407  */
 408 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
 409                                    struct dma_desc *np, struct sk_buff *skb)
 410 {
 411         struct skb_shared_hwtstamps *shhwtstamp = NULL;
 412         u64 ns;
 413
 414         if (!priv->hwts_rx_en)
 415                 return;
 416
 417         /* Check if timestamp is available */
 418         if (!priv->hw->desc->get_rx_timestamp_status(p, priv->adv_ts)) {
 419                 /* For GMAC4, the valid timestamp is from CTX next desc. */
 420                 if (priv->plat->has_gmac4)
 421                         ns = priv->hw->desc->get_timestamp(np, priv->adv_ts);
 422                 else
 423                         ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 424
 425                 netdev_info(priv->dev, "get valid RX hw timestamp %llu\n", ns);
 426                 shhwtstamp = skb_hwtstamps(skb);
 427                 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 428                 shhwtstamp->hwtstamp = ns_to_ktime(ns);
 429         } else  {
 430                 netdev_err(priv->dev, "cannot get RX hw timestamp\n");
 431         }
 432 }
 433
 434 /**
 435  *  stmmac_hwtstamp_ioctl - control hardware timestamping.
 436  *  @dev: device pointer.
 437  *  @ifr: An IOCTL specific structure, that can contain a pointer to
 438  *  a proprietary structure used to pass information to the driver.
 439  *  Description:
 440  *  This function configures the MAC to enable/disable both outgoing(TX)
 441  *  and incoming(RX) packets time stamping based on user input.
 442  *  Return Value:
 443  *  0 on success and an appropriate -ve integer on failure.
 444  */
 445 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
 446 {
 447         struct stmmac_priv *priv = netdev_priv(dev);
 448         struct hwtstamp_config config;
 449         struct timespec64 now;
 450         u64 temp = 0;
 451         u32 ptp_v2 = 0;
 452         u32 tstamp_all = 0;
 453         u32 ptp_over_ipv4_udp = 0;
 454         u32 ptp_over_ipv6_udp = 0;
 455         u32 ptp_over_ethernet = 0;
 456         u32 snap_type_sel = 0;
 457         u32 ts_master_en = 0;
 458         u32 ts_event_en = 0;
 459         u32 value = 0;
 460         u32 sec_inc;
 461
 462         if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
 463                 netdev_alert(priv->dev, "No support for HW time stamping\n");
 464                 priv->hwts_tx_en = 0;
 465                 priv->hwts_rx_en = 0;
 466
 467                 return -EOPNOTSUPP;
 468         }
 469
 470         if (copy_from_user(&config, ifr->ifr_data,
 471                            sizeof(struct hwtstamp_config)))
 472                 return -EFAULT;
 473
 474         netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
 475                    __func__, config.flags, config.tx_type, config.rx_filter);
 476
 477         /* reserved for future extensions */
 478         if (config.flags)
 479                 return -EINVAL;
 480
 481         if (config.tx_type != HWTSTAMP_TX_OFF &&
 482             config.tx_type != HWTSTAMP_TX_ON)
 483                 return -ERANGE;
 484
 485         if (priv->adv_ts) {
 486                 switch (config.rx_filter) {
 487                 case HWTSTAMP_FILTER_NONE:
 488                         /* time stamp no incoming packet at all */
 489                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 490                         break;
 491
 492                 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 493                         /* PTP v1, UDP, any kind of event packet */
 494                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 495                         /* take time stamp for all event messages */
 496                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 497
 498                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 499                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 500                         break;
 501
 502                 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 503                         /* PTP v1, UDP, Sync packet */
 504                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
 505                         /* take time stamp for SYNC messages only */
 506                         ts_event_en = PTP_TCR_TSEVNTENA;
 507
 508                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 509                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 510                         break;
 511
 512                 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 513                         /* PTP v1, UDP, Delay_req packet */
 514                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
 515                         /* take time stamp for Delay_Req messages only */
 516                         ts_master_en = PTP_TCR_TSMSTRENA;
 517                         ts_event_en = PTP_TCR_TSEVNTENA;
 518
 519                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 520                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 521                         break;
 522
 523                 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 524                         /* PTP v2, UDP, any kind of event packet */
 525                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
 526                         ptp_v2 = PTP_TCR_TSVER2ENA;
 527                         /* take time stamp for all event messages */
 528                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 529
 530                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 531                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 532                         break;
 533
 534                 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 535                         /* PTP v2, UDP, Sync packet */
 536                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
 537                         ptp_v2 = PTP_TCR_TSVER2ENA;
 538                         /* take time stamp for SYNC messages only */
 539                         ts_event_en = PTP_TCR_TSEVNTENA;
 540
 541                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 542                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 543                         break;
 544
 545                 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 546                         /* PTP v2, UDP, Delay_req packet */
 547                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
 548                         ptp_v2 = PTP_TCR_TSVER2ENA;
 549                         /* take time stamp for Delay_Req messages only */
 550                         ts_master_en = PTP_TCR_TSMSTRENA;
 551                         ts_event_en = PTP_TCR_TSEVNTENA;
 552
 553                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 554                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 555                         break;
 556
 557                 case HWTSTAMP_FILTER_PTP_V2_EVENT:
 558                         /* PTP v2/802.AS1 any layer, any kind of event packet */
 559                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 560                         ptp_v2 = PTP_TCR_TSVER2ENA;
 561                         /* take time stamp for all event messages */
 562                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 563
 564                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 565                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 566                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 567                         break;
 568
 569                 case HWTSTAMP_FILTER_PTP_V2_SYNC:
 570                         /* PTP v2/802.AS1, any layer, Sync packet */
 571                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
 572                         ptp_v2 = PTP_TCR_TSVER2ENA;
 573                         /* take time stamp for SYNC messages only */
 574                         ts_event_en = PTP_TCR_TSEVNTENA;
 575
 576                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 577                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 578                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 579                         break;
 580
 581                 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 582                         /* PTP v2/802.AS1, any layer, Delay_req packet */
 583                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
 584                         ptp_v2 = PTP_TCR_TSVER2ENA;
 585                         /* take time stamp for Delay_Req messages only */
 586                         ts_master_en = PTP_TCR_TSMSTRENA;
 587                         ts_event_en = PTP_TCR_TSEVNTENA;
 588
 589                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 590                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 591                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 592                         break;
 593
 594                 case HWTSTAMP_FILTER_ALL:
 595                         /* time stamp any incoming packet */
 596                         config.rx_filter = HWTSTAMP_FILTER_ALL;
 597                         tstamp_all = PTP_TCR_TSENALL;
 598                         break;
 599
 600                 default:
 601                         return -ERANGE;
 602                 }
 603         } else {
 604                 switch (config.rx_filter) {
 605                 case HWTSTAMP_FILTER_NONE:
 606                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 607                         break;
 608                 default:
 609                         /* PTP v1, UDP, any kind of event packet */
 610                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 611                         break;
 612                 }
 613         }
 614         priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
 615         priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
 616
 617         if (!priv->hwts_tx_en && !priv->hwts_rx_en)
 618                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0);
 619         else {
 620                 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
 621                          tstamp_all | ptp_v2 | ptp_over_ethernet |
 622                          ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
 623                          ts_master_en | snap_type_sel);
 624                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value);
 625
 626                 /* program Sub Second Increment reg */
 627                 sec_inc = priv->hw->ptp->config_sub_second_increment(
 628                         priv->ptpaddr, priv->plat->clk_ptp_rate,
 629                         priv->plat->has_gmac4);
 630                 temp = div_u64(1000000000ULL, sec_inc);
 631
 632                 /* calculate default added value:
 633                  * formula is :
 634                  * addend = (2^32)/freq_div_ratio;
 635                  * where, freq_div_ratio = 1e9ns/sec_inc
 636                  */
 637                 temp = (u64)(temp << 32);
 638                 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
 639                 priv->hw->ptp->config_addend(priv->ptpaddr,
 640                                              priv->default_addend);
 641
 642                 /* initialize system time */
 643                 ktime_get_real_ts64(&now);
 644
 645                 /* lower 32 bits of tv_sec are safe until y2106 */
 646                 priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec,
 647                                             now.tv_nsec);
 648         }
 649
 650         return copy_to_user(ifr->ifr_data, &config,
 651                             sizeof(struct hwtstamp_config)) ? -EFAULT : 0;
 652 }
 653
 654 /**
 655  * stmmac_init_ptp - init PTP
 656  * @priv: driver private structure
 657  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
 658  * This is done by looking at the HW cap. register.
 659  * This function also registers the ptp driver.
 660  */
 661 static int stmmac_init_ptp(struct stmmac_priv *priv)
 662 {
 663         if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
 664                 return -EOPNOTSUPP;
 665
 666         priv->adv_ts = 0;
 667         /* Check if adv_ts can be enabled for dwmac 4.x core */
 668         if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp)
 669                 priv->adv_ts = 1;
 670         /* Dwmac 3.x core with extend_desc can support adv_ts */
 671         else if (priv->extend_desc && priv->dma_cap.atime_stamp)
 672                 priv->adv_ts = 1;
 673
 674         if (priv->dma_cap.time_stamp)
 675                 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
 676
 677         if (priv->adv_ts)
 678                 netdev_info(priv->dev,
 679                             "IEEE 1588-2008 Advanced Timestamp supported\n");
 680
 681         priv->hw->ptp = &stmmac_ptp;
 682         priv->hwts_tx_en = 0;
 683         priv->hwts_rx_en = 0;
 684
 685         stmmac_ptp_register(priv);
 686
 687         return 0;
 688 }
 689
 690 static void stmmac_release_ptp(struct stmmac_priv *priv)
 691 {
 692         if (priv->plat->clk_ptp_ref)
 693                 clk_disable_unprepare(priv->plat->clk_ptp_ref);
 694         stmmac_ptp_unregister(priv);
 695 }
 696
 697 /**
 698  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
 699  *  @priv: driver private structure
 700  *  Description: It is used for configuring the flow control in all queues
 701  */
 702 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
 703 {
 704         u32 tx_cnt = priv->plat->tx_queues_to_use;
 705
 706         priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl,
 707                                  priv->pause, tx_cnt);
 708 }
 709
 710 /**
 711  * stmmac_adjust_link - adjusts the link parameters
 712  * @dev: net device structure
 713  * Description: this is the helper called by the physical abstraction layer
 714  * drivers to communicate the phy link status. According the speed and duplex
 715  * this driver can invoke registered glue-logic as well.
 716  * It also invoke the eee initialization because it could happen when switch
 717  * on different networks (that are eee capable).
 718  */
 719 static void stmmac_adjust_link(struct net_device *dev)
 720 {
 721         struct stmmac_priv *priv = netdev_priv(dev);
 722         struct phy_device *phydev = dev->phydev;
 723         unsigned long flags;
 724         int new_state = 0;
 725
 726         if (!phydev)
 727                 return;
 728
 729         spin_lock_irqsave(&priv->lock, flags);
 730
 731         if (phydev->link) {
 732                 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
 733
 734                 /* Now we make sure that we can be in full duplex mode.
 735                  * If not, we operate in half-duplex mode. */
 736                 if (phydev->duplex != priv->oldduplex) {
 737                         new_state = 1;
 738                         if (!(phydev->duplex))
 739                                 ctrl &= ~priv->hw->link.duplex;
 740                         else
 741                                 ctrl |= priv->hw->link.duplex;
 742                         priv->oldduplex = phydev->duplex;
 743                 }
 744                 /* Flow Control operation */
 745                 if (phydev->pause)
 746                         stmmac_mac_flow_ctrl(priv, phydev->duplex);
 747
 748                 if (phydev->speed != priv->speed) {
 749                         new_state = 1;
 750                         switch (phydev->speed) {
 751                         case 1000:
 752                                 if (priv->plat->has_gmac ||
 753                                     priv->plat->has_gmac4)
 754                                         ctrl &= ~priv->hw->link.port;
 755                                 break;
 756                         case 100:
 757                                 if (priv->plat->has_gmac ||
 758                                     priv->plat->has_gmac4) {
 759                                         ctrl |= priv->hw->link.port;
 760                                         ctrl |= priv->hw->link.speed;
 761                                 } else {
 762                                         ctrl &= ~priv->hw->link.port;
 763                                 }
 764                                 break;
 765                         case 10:
 766                                 if (priv->plat->has_gmac ||
 767                                     priv->plat->has_gmac4) {
 768                                         ctrl |= priv->hw->link.port;
 769                                         ctrl &= ~(priv->hw->link.speed);
 770                                 } else {
 771                                         ctrl &= ~priv->hw->link.port;
 772                                 }
 773                                 break;
 774                         default:
 775                                 netif_warn(priv, link, priv->dev,
 776                                            "broken speed: %d\n", phydev->speed);
 777                                 phydev->speed = SPEED_UNKNOWN;
 778                                 break;
 779                         }
 780                         if (phydev->speed != SPEED_UNKNOWN)
 781                                 stmmac_hw_fix_mac_speed(priv);
 782                         priv->speed = phydev->speed;
 783                 }
 784
 785                 writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
 786
 787                 if (!priv->oldlink) {
 788                         new_state = 1;
 789                         priv->oldlink = 1;
 790                 }
 791         } else if (priv->oldlink) {
 792                 new_state = 1;
 793                 priv->oldlink = 0;
 794                 priv->speed = SPEED_UNKNOWN;
 795                 priv->oldduplex = DUPLEX_UNKNOWN;
 796         }
 797
 798         if (new_state && netif_msg_link(priv))
 799                 phy_print_status(phydev);
 800
 801         spin_unlock_irqrestore(&priv->lock, flags);
 802
 803         if (phydev->is_pseudo_fixed_link)
 804                 /* Stop PHY layer to call the hook to adjust the link in case
 805                  * of a switch is attached to the stmmac driver.
 806                  */
 807                 phydev->irq = PHY_IGNORE_INTERRUPT;
 808         else
 809                 /* At this stage, init the EEE if supported.
 810                  * Never called in case of fixed_link.
 811                  */
 812                 priv->eee_enabled = stmmac_eee_init(priv);
 813 }
 814
 815 /**
 816  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
 817  * @priv: driver private structure
 818  * Description: this is to verify if the HW supports the PCS.
 819  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
 820  * configured for the TBI, RTBI, or SGMII PHY interface.
 821  */
 822 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
 823 {
 824         int interface = priv->plat->interface;
 825
 826         if (priv->dma_cap.pcs) {
 827                 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
 828                     (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
 829                     (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
 830                     (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
 831                         netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
 832                         priv->hw->pcs = STMMAC_PCS_RGMII;
 833                 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
 834                         netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
 835                         priv->hw->pcs = STMMAC_PCS_SGMII;
 836                 }
 837         }
 838 }
 839
 840 /**
 841  * stmmac_init_phy - PHY initialization
 842  * @dev: net device structure
 843  * Description: it initializes the driver's PHY state, and attaches the PHY
 844  * to the mac driver.
 845  *  Return value:
 846  *  0 on success
 847  */
 848 static int stmmac_init_phy(struct net_device *dev)
 849 {
 850         struct stmmac_priv *priv = netdev_priv(dev);
 851         struct phy_device *phydev;
 852         char phy_id_fmt[MII_BUS_ID_SIZE + 3];
 853         char bus_id[MII_BUS_ID_SIZE];
 854         int interface = priv->plat->interface;
 855         int max_speed = priv->plat->max_speed;
 856         priv->oldlink = 0;
 857         priv->speed = SPEED_UNKNOWN;
 858         priv->oldduplex = DUPLEX_UNKNOWN;
 859
 860         if (priv->plat->phy_node) {
 861                 phydev = of_phy_connect(dev, priv->plat->phy_node,
 862                                         &stmmac_adjust_link, 0, interface);
 863         } else {
 864                 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x",
 865                          priv->plat->bus_id);
 866
 867                 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
 868                          priv->plat->phy_addr);
 869                 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__,
 870                            phy_id_fmt);
 871
 872                 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link,
 873                                      interface);
 874         }
 875
 876         if (IS_ERR_OR_NULL(phydev)) {
 877                 netdev_err(priv->dev, "Could not attach to PHY\n");
 878                 if (!phydev)
 879                         return -ENODEV;
 880
 881                 return PTR_ERR(phydev);
 882         }
 883
 884         /* Stop Advertising 1000BASE Capability if interface is not GMII */
 885         if ((interface == PHY_INTERFACE_MODE_MII) ||
 886             (interface == PHY_INTERFACE_MODE_RMII) ||
 887                 (max_speed < 1000 && max_speed > 0))
 888                 phydev->advertising &= ~(SUPPORTED_1000baseT_Half |
 889                                          SUPPORTED_1000baseT_Full);
 890
 891         /*
 892          * Broken HW is sometimes missing the pull-up resistor on the
 893          * MDIO line, which results in reads to non-existent devices returning
 894          * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent
 895          * device as well.
 896          * Note: phydev->phy_id is the result of reading the UID PHY registers.
 897          */
 898         if (!priv->plat->phy_node && phydev->phy_id == 0) {
 899                 phy_disconnect(phydev);
 900                 return -ENODEV;
 901         }
 902
 903         /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
 904          * subsequent PHY polling, make sure we force a link transition if
 905          * we have a UP/DOWN/UP transition
 906          */
 907         if (phydev->is_pseudo_fixed_link)
 908                 phydev->irq = PHY_POLL;
 909
 910         phy_attached_info(phydev);
 911         return 0;
 912 }
 913
 914 static void stmmac_display_rings(struct stmmac_priv *priv)
 915 {
 916         u32 rx_cnt = priv->plat->rx_queues_to_use;
 917         u32 tx_cnt = priv->plat->tx_queues_to_use;
 918         void *head_rx, *head_tx;
 919         u32 queue;
 920
 921         /* Display RX rings */
 922         for (queue = 0; queue < rx_cnt; queue++) {
 923                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 924
 925                 pr_info("\tRX Queue %d rings\n", queue);
 926
 927                 if (priv->extend_desc)
 928                         head_rx = (void *)rx_q->dma_erx;
 929                 else
 930                         head_rx = (void *)rx_q->dma_rx;
 931
 932                 /* Display Rx ring */
 933                 priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true);
 934         }
 935
 936         /* Display TX rings */
 937         for (queue = 0; queue < tx_cnt; queue++) {
 938                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 939
 940                 pr_info("\tTX Queue %d rings\n", queue);
 941
 942                 if (priv->extend_desc)
 943                         head_tx = (void *)tx_q->dma_etx;
 944                 else
 945                         head_tx = (void *)tx_q->dma_tx;
 946
 947                 /* Display Tx ring */
 948                 priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false);
 949         }
 950 }
 951
 952 static int stmmac_set_bfsize(int mtu, int bufsize)
 953 {
 954         int ret = bufsize;
 955
 956         if (mtu >= BUF_SIZE_4KiB)
 957                 ret = BUF_SIZE_8KiB;
 958         else if (mtu >= BUF_SIZE_2KiB)
 959                 ret = BUF_SIZE_4KiB;
 960         else if (mtu > DEFAULT_BUFSIZE)
 961                 ret = BUF_SIZE_2KiB;
 962         else
 963                 ret = DEFAULT_BUFSIZE;
 964
 965         return ret;
 966 }
 967
 968 /**
 969  * stmmac_clear_rx_descriptors - clear the descriptors of a RX queue
 970  * @priv: driver private structure
 971  * @queue: RX queue index
 972  * Description: this function is called to clear the RX descriptors
 973  * in case of both basic and extended descriptors are used.
 974  */
 975 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
 976 {
 977         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 978         u32 i = 0;
 979
 980         /* Clear the RX descriptors */
 981         for (i = 0; i < DMA_RX_SIZE; i++)
 982                 if (priv->extend_desc)
 983                         priv->hw->desc->init_rx_desc(&rx_q->dma_erx[i].basic,
 984                                                      priv->use_riwt, priv->mode,
 985                                                      (i == DMA_RX_SIZE - 1));
 986                 else
 987                         priv->hw->desc->init_rx_desc(&rx_q->dma_rx[i],
 988                                                      priv->use_riwt, priv->mode,
 989                                                      (i == DMA_RX_SIZE - 1));
 990 }
 991
 992 /**
 993  * stmmac_clear_tx_descriptors - clear the descriptors of a TX queue
 994  * @priv: driver private structure
 995  * @queue: TX queue index
 996  * Description: this function is called to clear the TX descriptors
 997  * in case of both basic and extended descriptors are used.
 998  */
 999 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1000 {
1001         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1002         u32 i = 0;
1003
1004         /* Clear the TX descriptors */
1005         for (i = 0; i < DMA_TX_SIZE; i++)
1006                 if (priv->extend_desc)
1007                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1008                                                      priv->mode,
1009                                                      (i == DMA_TX_SIZE - 1));
1010                 else
1011                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1012                                                      priv->mode,
1013                                                      (i == DMA_TX_SIZE - 1));
1014 }
1015
1016 /**
1017  * stmmac_clear_descriptors - clear descriptors
1018  * @priv: driver private structure
1019  * Description: this function is called to clear the tx and rx descriptors
1020  * in case of both basic and extended descriptors are used.
1021  */
1022 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1023 {
1024         u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1025         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1026         u32 queue;
1027
1028         for (queue = 0; queue < rx_queue_cnt; queue++)
1029                 stmmac_clear_rx_descriptors(priv, queue);
1030
1031         for (queue = 0; queue < tx_queue_cnt; queue++)
1032                 stmmac_clear_tx_descriptors(priv, queue);
1033 }
1034
1035 /**
1036  * stmmac_init_rx_buffers - init the RX descriptor buffer.
1037  * @priv: driver private structure
1038  * @p: descriptor pointer
1039  * @i: descriptor index
1040  * @flags: gfp flag.
1041  * @queue: RX queue index
1042  * Description: this function is called to allocate a receive buffer, perform
1043  * the DMA mapping and init the descriptor.
1044  */
1045 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1046                                   int i, gfp_t flags, u32 queue)
1047 {
1048         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1049         struct sk_buff *skb;
1050
1051         skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags);
1052         if (!skb) {
1053                 netdev_err(priv->dev,
1054                            "%s: Rx init fails; skb is NULL\n", __func__);
1055                 return -ENOMEM;
1056         }
1057         rx_q->rx_skbuff[i] = skb;
1058         rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
1059                                                 priv->dma_buf_sz,
1060                                                 DMA_FROM_DEVICE);
1061         if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) {
1062                 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__);
1063                 dev_kfree_skb_any(skb);
1064                 return -EINVAL;
1065         }
1066
1067         if (priv->synopsys_id >= DWMAC_CORE_4_00)
1068                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1069         else
1070                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1071
1072         if ((priv->hw->mode->init_desc3) &&
1073             (priv->dma_buf_sz == BUF_SIZE_16KiB))
1074                 priv->hw->mode->init_desc3(p);
1075
1076         return 0;
1077 }
1078
1079 /**
1080  * stmmac_free_rx_buffers - free RX buffers.
1081  * @priv: driver private structure
1082  * @queue: RX queue index
1083  * @i: buffer index
1084  */
1085 static void stmmac_free_rx_buffers(struct stmmac_priv *priv, u32 queue, int i)
1086 {
1087         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1088
1089         if (rx_q->rx_skbuff[i]) {
1090                 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i],
1091                                  priv->dma_buf_sz, DMA_FROM_DEVICE);
1092                 dev_kfree_skb_any(rx_q->rx_skbuff[i]);
1093         }
1094         rx_q->rx_skbuff[i] = NULL;
1095 }
1096
1097 /**
1098  * stmmac_free_tx_buffers - free RX buffers.
1099  * @priv: driver private structure
1100  * @queue: RX queue index
1101  * @i: buffer index
1102  */
1103 static void stmmac_free_tx_buffers(struct stmmac_priv *priv, u32 queue, u32 i)
1104 {
1105         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1106
1107         if (tx_q->tx_skbuff_dma[i].buf) {
1108                 if (tx_q->tx_skbuff_dma[i].map_as_page)
1109                         dma_unmap_page(priv->device,
1110                                        tx_q->tx_skbuff_dma[i].buf,
1111                                        tx_q->tx_skbuff_dma[i].len,
1112                                        DMA_TO_DEVICE);
1113                 else
1114                         dma_unmap_single(priv->device,
1115                                          tx_q->tx_skbuff_dma[i].buf,
1116                                          tx_q->tx_skbuff_dma[i].len,
1117                                          DMA_TO_DEVICE);
1118         }
1119
1120         if (tx_q->tx_skbuff[i]) {
1121                 dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1122                 tx_q->tx_skbuff[i] = NULL;
1123                 tx_q->tx_skbuff_dma[i].buf = 0;
1124                 tx_q->tx_skbuff_dma[i].map_as_page = false;
1125         }
1126 }
1127
1128 /**
1129  * init_tx_dma_desc_rings - init the TX descriptor rings
1130  * @dev: net device structure
1131  * Description: this function initializes the DMA TX descriptors
1132  * and allocates the socket buffers. It suppors the chained and ring
1133  * modes.
1134  */
1135 static int init_tx_dma_desc_rings(struct net_device *dev)
1136 {
1137         struct stmmac_priv *priv = netdev_priv(dev);
1138         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1139         u32 queue;
1140         int i = 0;
1141
1142         for (queue = 0; queue < tx_queue_cnt; queue++) {
1143                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1144
1145                 netif_dbg(priv, probe, priv->dev,
1146                           "(%s) dma_tx_phy=0x%08x\n", __func__,
1147                           (u32)tx_q->dma_tx_phy);
1148
1149                 /* Setup the chained descriptor addresses */
1150                 if (priv->mode == STMMAC_CHAIN_MODE) {
1151                         if (priv->extend_desc)
1152                                 priv->hw->mode->init(tx_q->dma_etx,
1153                                                      tx_q->dma_tx_phy,
1154                                                      DMA_TX_SIZE, 1);
1155                         else
1156                                 priv->hw->mode->init(tx_q->dma_tx,
1157                                                      tx_q->dma_tx_phy,
1158                                                      DMA_TX_SIZE, 0);
1159                 }
1160
1161                 for (i = 0; i < DMA_TX_SIZE; i++) {
1162                         struct dma_desc *p;
1163
1164                         if (priv->extend_desc)
1165                                 p = &((tx_q->dma_etx + i)->basic);
1166                         else
1167                                 p = tx_q->dma_tx + i;
1168
1169                         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1170                                 p->des0 = 0;
1171                                 p->des1 = 0;
1172                                 p->des2 = 0;
1173                                 p->des3 = 0;
1174                         } else {
1175                                 p->des2 = 0;
1176                         }
1177
1178                         tx_q->tx_skbuff_dma[i].buf = 0;
1179                         tx_q->tx_skbuff_dma[i].map_as_page = false;
1180                         tx_q->tx_skbuff_dma[i].len = 0;
1181                         tx_q->tx_skbuff_dma[i].last_segment = false;
1182                         tx_q->tx_skbuff[i] = NULL;
1183                 }
1184
1185                 tx_q->dirty_tx = 0;
1186                 tx_q->cur_tx = 0;
1187                 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1188         }
1189
1190         return 0;
1191 }
1192
1193 /**
1194  * init_rx_dma_desc_rings - init the RX descriptor rings
1195  * @dev: net device structure
1196  * @flags: gfp flag.
1197  * Description: this function initializes the DMA RX descriptors
1198  * and allocates the socket buffers. It suppors the chained and ring
1199  * modes.
1200  */
1201 static int init_rx_dma_desc_rings(struct net_device *dev, gfp_t flags)
1202 {
1203         struct stmmac_priv *priv = netdev_priv(dev);
1204         u32 rx_count = priv->plat->rx_queues_to_use;
1205         unsigned int bfsize = 0;
1206         int ret = -ENOMEM;
1207         u32 queue;
1208         int i;
1209
1210         if (priv->hw->mode->set_16kib_bfsize)
1211                 bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu);
1212
1213         if (bfsize < BUF_SIZE_16KiB)
1214                 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
1215
1216         priv->dma_buf_sz = bfsize;
1217
1218         /* RX INITIALIZATION */
1219         netif_dbg(priv, probe, priv->dev,
1220                   "SKB addresses:\nskb\t\tskb data\tdma data\n");
1221
1222         for (queue = 0; queue < rx_count; queue++) {
1223                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1224
1225                 netif_dbg(priv, probe, priv->dev,
1226                           "(%s) dma_rx_phy=0x%08x\n", __func__,
1227                           (u32)rx_q->dma_rx_phy);
1228
1229                 for (i = 0; i < DMA_RX_SIZE; i++) {
1230                         struct dma_desc *p;
1231
1232                         if (priv->extend_desc)
1233                                 p = &((rx_q->dma_erx + i)->basic);
1234                         else
1235                                 p = rx_q->dma_rx + i;
1236
1237                         ret = stmmac_init_rx_buffers(priv, p, i, flags, queue);
1238                         if (ret)
1239                                 goto err_init_rx_buffers;
1240
1241                         netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n",
1242                                   rx_q->rx_skbuff[i],
1243                                   rx_q->rx_skbuff[i]->data,
1244                                   (unsigned int)rx_q->rx_skbuff_dma[i]);
1245                 }
1246
1247                 rx_q->cur_rx = 0;
1248                 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1249
1250                 stmmac_clear_rx_descriptors(priv, queue);
1251
1252                 if (priv->mode == STMMAC_CHAIN_MODE) {
1253                         if (priv->extend_desc)
1254                                 priv->hw->mode->init(rx_q->dma_erx,
1255                                                      rx_q->dma_rx_phy,
1256                                                      DMA_RX_SIZE, 1);
1257                         else
1258                                 priv->hw->mode->init(rx_q->dma_rx,
1259                                                      rx_q->dma_rx_phy,
1260                                                      DMA_RX_SIZE, 0);
1261                 }
1262         }
1263
1264         buf_sz = bfsize;
1265
1266         return 0;
1267
1268 err_init_rx_buffers:
1269         while (queue-- >= 0) {
1270                 while (--i >= 0)
1271                         stmmac_free_rx_buffers(priv, queue, i);
1272
1273                 i = DMA_RX_SIZE;
1274         }
1275
1276         return ret;
1277 }
1278
1279 /**
1280  * init_dma_desc_rings - init the RX/TX descriptor rings
1281  * @dev: net device structure
1282  * @flags: gfp flag.
1283  * Description: this function initializes the DMA RX/TX descriptors
1284  * and allocates the socket buffers. It suppors the chained and ring
1285  * modes.
1286  */
1287 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1288 {
1289         struct stmmac_priv *priv = netdev_priv(dev);
1290         int ret = init_rx_dma_desc_rings(dev, flags);
1291
1292         if (ret)
1293                 return ret;
1294
1295         ret = init_tx_dma_desc_rings(dev);
1296
1297         if (netif_msg_hw(priv))
1298                 stmmac_display_rings(priv);
1299
1300         return ret;
1301 }
1302
1303 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1304 {
1305         int i;
1306
1307         for (i = 0; i < DMA_RX_SIZE; i++)
1308                 stmmac_free_rx_buffers(priv, queue, i);
1309 }
1310
1311 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1312 {
1313         int i;
1314
1315         for (i = 0; i < DMA_TX_SIZE; i++)
1316                 stmmac_free_tx_buffers(priv, queue, i);
1317 }
1318
1319 /**
1320  * free_rx_dma_desc_resources - free RX DMA resources
1321  * @priv: driver private structure
1322  */
1323 static void free_rx_dma_desc_resources(struct stmmac_priv *priv)
1324 {
1325         u32 rx_count = priv->plat->rx_queues_to_use;
1326         u32 queue = 0;
1327
1328         if (!priv->rx_queue)
1329                 return;
1330
1331         /* Free RX queue resources */
1332         for (queue = 0; queue < rx_count; queue++) {
1333                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1334
1335                 if (!rx_q)
1336                         break;
1337
1338                 /* Release the DMA RX socket buffers */
1339                 dma_free_rx_skbufs(priv, queue);
1340
1341                 kfree(rx_q->rx_skbuff);
1342
1343                 kfree(rx_q->rx_skbuff_dma);
1344
1345                 if (!priv->extend_desc)
1346                         dma_free_coherent(priv->device,
1347                                           DMA_RX_SIZE * sizeof(struct dma_desc),
1348                                           rx_q->dma_rx,
1349                                           rx_q->dma_rx_phy);
1350                 else
1351                         dma_free_coherent(priv->device, DMA_RX_SIZE *
1352                                           sizeof(struct dma_extended_desc),
1353                                           rx_q->dma_erx,
1354                                           rx_q->dma_rx_phy);
1355         }
1356
1357         kfree(priv->rx_queue);
1358 }
1359
1360 /**
1361  * free_tx_dma_desc_resources - free TX DMA resources
1362  * @priv: driver private structure
1363  */
1364 static void free_tx_dma_desc_resources(struct stmmac_priv *priv)
1365 {
1366         u32 tx_count = priv->plat->tx_queues_to_use;
1367         u32 queue = 0;
1368
1369         if (!priv->tx_queue)
1370                 return;
1371
1372         /* Free TX queue resources */
1373         for (queue = 0; queue < tx_count; queue++) {
1374                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1375
1376                 if (!tx_q)
1377                         break;
1378
1379                 /* Release the DMA TX socket buffers */
1380                 dma_free_tx_skbufs(priv, queue);
1381
1382                 kfree(tx_q->tx_skbuff);
1383
1384                 kfree(tx_q->tx_skbuff_dma);
1385
1386                 if (!priv->extend_desc)
1387                         dma_free_coherent(priv->device,
1388                                           DMA_TX_SIZE * sizeof(struct dma_desc),
1389                                           tx_q->dma_tx,
1390                                           tx_q->dma_tx_phy);
1391                 else
1392                         dma_free_coherent(priv->device, DMA_TX_SIZE *
1393                                           sizeof(struct dma_extended_desc),
1394                                           tx_q->dma_etx,
1395                                           tx_q->dma_tx_phy);
1396         }
1397
1398         kfree(priv->tx_queue);
1399 }
1400
1401 /**
1402  * free_dma_desc_resources - free All DMA resources
1403  * @priv: driver private structure
1404  */
1405 static void free_dma_desc_resources(struct stmmac_priv *priv)
1406 {
1407         free_rx_dma_desc_resources(priv);
1408         free_tx_dma_desc_resources(priv);
1409 }
1410
1411 /**
1412  * alloc_rx_dma_desc_resources - alloc RX resources.
1413  * @priv: private structure
1414  * Description: according to which descriptor can be used (extend or basic)
1415  * this function allocates the resources for RX paths. It pre-allocates the
1416  * RX socket buffer in order to allow zero-copy mechanism.
1417  */
1418 static int alloc_rx_dma_desc_resources(struct stmmac_priv *priv)
1419 {
1420         u32 rx_count = priv->plat->rx_queues_to_use;
1421         int ret = -ENOMEM;
1422         u32 queue = 0;
1423
1424         /* Allocate RX queues array */
1425         priv->rx_queue = kmalloc_array(rx_count,
1426                                        sizeof(struct stmmac_rx_queue),
1427                                        GFP_KERNEL);
1428         if (!priv->rx_queue) {
1429                 kfree(priv->rx_queue);
1430                 return -ENOMEM;
1431         }
1432
1433         /* RX queues buffers and DMA */
1434         for (queue = 0; queue < rx_count; queue++) {
1435                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1436
1437                 rx_q->queue_index = queue;
1438                 rx_q->priv_data = priv;
1439
1440                 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE,
1441                                                         sizeof(dma_addr_t),
1442                                                         GFP_KERNEL);
1443                 if (!rx_q->rx_skbuff_dma)
1444                         goto err_dma_buffers;
1445
1446                 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE,
1447                                                     sizeof(struct sk_buff *),
1448                                                     GFP_KERNEL);
1449                 if (!rx_q->rx_skbuff)
1450                         goto err_dma_buffers;
1451
1452                 if (priv->extend_desc) {
1453                         rx_q->dma_erx = dma_zalloc_coherent(priv->device,
1454                         (DMA_RX_SIZE * sizeof(struct dma_extended_desc)),
1455                         &rx_q->dma_rx_phy, GFP_KERNEL);
1456
1457                         if (!rx_q->dma_erx)
1458                                 goto err_dma_buffers;
1459                 } else {
1460                         rx_q->dma_rx = dma_zalloc_coherent(priv->device,
1461                         (DMA_RX_SIZE * sizeof(struct dma_desc)),
1462                         &rx_q->dma_rx_phy, GFP_KERNEL);
1463
1464                         if (!rx_q->dma_rx)
1465                                 goto err_dma_buffers;
1466                 }
1467         }
1468
1469         return 0;
1470
1471 err_dma_buffers:
1472         free_rx_dma_desc_resources(priv);
1473
1474         return ret;
1475 }
1476
1477 /**
1478  * alloc_tx_dma_desc_resources - alloc TX resources.
1479  * @priv: private structure
1480  * Description: according to which descriptor can be used (extend or basic)
1481  * this function allocates the resources for TX paths.
1482  */
1483 static int alloc_tx_dma_desc_resources(struct stmmac_priv *priv)
1484 {
1485         u32 tx_count = priv->plat->tx_queues_to_use;
1486         int ret = -ENOMEM;
1487         u32 queue = 0;
1488
1489         /* Allocate TX queues array */
1490         priv->tx_queue = kmalloc_array(tx_count,
1491                                        sizeof(struct stmmac_tx_queue),
1492                                        GFP_KERNEL);
1493         if (!priv->tx_queue)
1494                 return -ENOMEM;
1495
1496         /* TX queues buffers and DMA */
1497         for (queue = 0; queue < tx_count; queue++) {
1498                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1499
1500                 tx_q->queue_index = queue;
1501                 tx_q->priv_data = priv;
1502
1503                 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE,
1504                                           sizeof(struct stmmac_tx_info),
1505                                           GFP_KERNEL);
1506
1507                 if (!tx_q->tx_skbuff_dma)
1508                         goto err_dma_buffers;
1509
1510                 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE,
1511                                                     sizeof(struct sk_buff *),
1512                                                     GFP_KERNEL);
1513                 if (!tx_q->tx_skbuff)
1514                         goto err_dma_buffers;
1515
1516                 if (priv->extend_desc) {
1517                         tx_q->dma_etx =
1518                         dma_zalloc_coherent(priv->device,
1519                         (DMA_TX_SIZE * sizeof(struct dma_extended_desc)),
1520                         &tx_q->dma_tx_phy, GFP_KERNEL);
1521
1522                         if (!tx_q->dma_etx)
1523                                 goto err_dma_buffers;
1524                 } else {
1525                         tx_q->dma_tx =
1526                         dma_zalloc_coherent(priv->device,
1527                         (DMA_TX_SIZE * sizeof(struct dma_desc)),
1528                         &tx_q->dma_tx_phy, GFP_KERNEL);
1529
1530                         if (!tx_q->dma_tx)
1531                                 goto err_dma_buffers;
1532                 }
1533         }
1534
1535         return 0;
1536
1537 err_dma_buffers:
1538         free_tx_dma_desc_resources(priv);
1539
1540         return ret;
1541 }
1542
1543 /**
1544  * alloc_dma_desc_resources - alloc TX/RX resources.
1545  * @priv: private structure
1546  * Description: according to which descriptor can be used (extend or basic)
1547  * this function allocates the resources for TX and RX paths. In case of
1548  * reception, for example, it pre-allocated the RX socket buffer in order to
1549  * allow zero-copy mechanism.
1550  */
1551 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1552 {
1553         int ret = 0;
1554
1555         ret = alloc_tx_dma_desc_resources(priv);
1556         if (ret)
1557                 return ret;
1558
1559         ret = alloc_rx_dma_desc_resources(priv);
1560
1561         return ret;
1562 }
1563
1564 /**
1565  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
1566  *  @priv: driver private structure
1567  *  Description: It is used for enabling the rx queues in the MAC
1568  */
1569 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1570 {
1571         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1572         int queue;
1573         u8 mode;
1574
1575         for (queue = 0; queue < rx_queues_count; queue++) {
1576                 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1577                 priv->hw->mac->rx_queue_enable(priv->hw, mode, queue);
1578         }
1579 }
1580
1581 /**
1582  * stmmac_start_rx_dma - start RX DMA channel
1583  * @priv: driver private structure
1584  * @chan: RX channel index
1585  * Description:
1586  * This starts a RX DMA channel
1587  */
1588 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1589 {
1590         netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1591         priv->hw->dma->start_rx(priv->ioaddr, chan);
1592 }
1593
1594 /**
1595  * stmmac_start_tx_dma - start TX DMA channel
1596  * @priv: driver private structure
1597  * @chan: TX channel index
1598  * Description:
1599  * This starts a TX DMA channel
1600  */
1601 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1602 {
1603         netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1604         priv->hw->dma->start_tx(priv->ioaddr, chan);
1605 }
1606
1607 /**
1608  * stmmac_stop_rx_dma - stop RX DMA channel
1609  * @priv: driver private structure
1610  * @chan: RX channel index
1611  * Description:
1612  * This stops a RX DMA channel
1613  */
1614 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1615 {
1616         netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1617         priv->hw->dma->stop_rx(priv->ioaddr, chan);
1618 }
1619
1620 /**
1621  * stmmac_stop_tx_dma - stop TX DMA channel
1622  * @priv: driver private structure
1623  * @chan: TX channel index
1624  * Description:
1625  * This stops a TX DMA channel
1626  */
1627 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1628 {
1629         netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1630         priv->hw->dma->stop_tx(priv->ioaddr, chan);
1631 }
1632
1633 /**
1634  * stmmac_start_all_dma - start all RX and TX DMA channels
1635  * @priv: driver private structure
1636  * Description:
1637  * This starts all the RX and TX DMA channels
1638  */
1639 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1640 {
1641         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1642         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1643         u32 chan = 0;
1644
1645         for (chan = 0; chan < rx_channels_count; chan++)
1646                 stmmac_start_rx_dma(priv, chan);
1647
1648         for (chan = 0; chan < tx_channels_count; chan++)
1649                 stmmac_start_tx_dma(priv, chan);
1650 }
1651
1652 /**
1653  * stmmac_stop_all_dma - stop all RX and TX DMA channels
1654  * @priv: driver private structure
1655  * Description:
1656  * This stops the RX and TX DMA channels
1657  */
1658 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1659 {
1660         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1661         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1662         u32 chan = 0;
1663
1664         for (chan = 0; chan < rx_channels_count; chan++)
1665                 stmmac_stop_rx_dma(priv, chan);
1666
1667         for (chan = 0; chan < tx_channels_count; chan++)
1668                 stmmac_stop_tx_dma(priv, chan);
1669 }
1670
1671 /**
1672  *  stmmac_dma_operation_mode - HW DMA operation mode
1673  *  @priv: driver private structure
1674  *  Description: it is used for configuring the DMA operation mode register in
1675  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1676  */
1677 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1678 {
1679         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1680         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1681         int rxfifosz = priv->plat->rx_fifo_size;
1682         u32 txmode = 0;
1683         u32 rxmode = 0;
1684         u32 chan = 0;
1685
1686         if (rxfifosz == 0)
1687                 rxfifosz = priv->dma_cap.rx_fifo_size;
1688
1689         if (priv->plat->force_thresh_dma_mode) {
1690                 txmode = tc;
1691                 rxmode = tc;
1692         } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1693                 /*
1694                  * In case of GMAC, SF mode can be enabled
1695                  * to perform the TX COE in HW. This depends on:
1696                  * 1) TX COE if actually supported
1697                  * 2) There is no bugged Jumbo frame support
1698                  *    that needs to not insert csum in the TDES.
1699                  */
1700                 txmode = SF_DMA_MODE;
1701                 rxmode = SF_DMA_MODE;
1702                 priv->xstats.threshold = SF_DMA_MODE;
1703         } else {
1704                 txmode = tc;
1705                 rxmode = SF_DMA_MODE;
1706         }
1707
1708         /* configure all channels */
1709         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1710                 for (chan = 0; chan < rx_channels_count; chan++)
1711                         priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1712                                                    rxfifosz);
1713
1714                 for (chan = 0; chan < tx_channels_count; chan++)
1715                         priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan);
1716         } else {
1717                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1718                                         rxfifosz);
1719         }
1720 }
1721
1722 /**
1723  * stmmac_tx_clean - to manage the transmission completion
1724  * @priv: driver private structure
1725  * @queue: TX queue index
1726  * Description: it reclaims the transmit resources after transmission completes.
1727  */
1728 static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
1729 {
1730         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1731         unsigned int bytes_compl = 0, pkts_compl = 0;
1732         unsigned int entry = tx_q->dirty_tx;
1733
1734         netif_tx_lock(priv->dev);
1735
1736         priv->xstats.tx_clean++;
1737
1738         while (entry != tx_q->cur_tx) {
1739                 struct sk_buff *skb = tx_q->tx_skbuff[entry];
1740                 struct dma_desc *p;
1741                 int status;
1742
1743                 if (priv->extend_desc)
1744                         p = (struct dma_desc *)(tx_q->dma_etx + entry);
1745                 else
1746                         p = tx_q->dma_tx + entry;
1747
1748                 status = priv->hw->desc->tx_status(&priv->dev->stats,
1749                                                       &priv->xstats, p,
1750                                                       priv->ioaddr);
1751                 /* Check if the descriptor is owned by the DMA */
1752                 if (unlikely(status & tx_dma_own))
1753                         break;
1754
1755                 /* Just consider the last segment and ...*/
1756                 if (likely(!(status & tx_not_ls))) {
1757                         /* ... verify the status error condition */
1758                         if (unlikely(status & tx_err)) {
1759                                 priv->dev->stats.tx_errors++;
1760                         } else {
1761                                 priv->dev->stats.tx_packets++;
1762                                 priv->xstats.tx_pkt_n++;
1763                         }
1764                         stmmac_get_tx_hwtstamp(priv, p, skb);
1765                 }
1766
1767                 if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
1768                         if (tx_q->tx_skbuff_dma[entry].map_as_page)
1769                                 dma_unmap_page(priv->device,
1770                                                tx_q->tx_skbuff_dma[entry].buf,
1771                                                tx_q->tx_skbuff_dma[entry].len,
1772                                                DMA_TO_DEVICE);
1773                         else
1774                                 dma_unmap_single(priv->device,
1775                                                  tx_q->tx_skbuff_dma[entry].buf,
1776                                                  tx_q->tx_skbuff_dma[entry].len,
1777                                                  DMA_TO_DEVICE);
1778                         tx_q->tx_skbuff_dma[entry].buf = 0;
1779                         tx_q->tx_skbuff_dma[entry].len = 0;
1780                         tx_q->tx_skbuff_dma[entry].map_as_page = false;
1781                 }
1782
1783                 if (priv->hw->mode->clean_desc3)
1784                         priv->hw->mode->clean_desc3(tx_q, p);
1785
1786                 tx_q->tx_skbuff_dma[entry].last_segment = false;
1787                 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
1788
1789                 if (likely(skb != NULL)) {
1790                         pkts_compl++;
1791                         bytes_compl += skb->len;
1792                         dev_consume_skb_any(skb);
1793                         tx_q->tx_skbuff[entry] = NULL;
1794                 }
1795
1796                 priv->hw->desc->release_tx_desc(p, priv->mode);
1797
1798                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1799         }
1800         tx_q->dirty_tx = entry;
1801
1802         netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
1803                                   pkts_compl, bytes_compl);
1804
1805         if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
1806                                                                queue))) &&
1807             stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
1808                 netif_dbg(priv, tx_done, priv->dev,
1809                           "%s: restart transmit\n", __func__);
1810                 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1811         }
1812
1813         if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
1814                 stmmac_enable_eee_mode(priv);
1815                 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1816         }
1817         netif_tx_unlock(priv->dev);
1818 }
1819
1820 static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan)
1821 {
1822         priv->hw->dma->enable_dma_irq(priv->ioaddr, chan);
1823 }
1824
1825 static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan)
1826 {
1827         priv->hw->dma->disable_dma_irq(priv->ioaddr, chan);
1828 }
1829
1830 /**
1831  * stmmac_tx_err - to manage the tx error
1832  * @priv: driver private structure
1833  * @queue: queue index
1834  * Description: it cleans the descriptors and restarts the transmission
1835  * in case of transmission errors.
1836  */
1837 static void stmmac_tx_err(struct stmmac_priv *priv, u32 queue)
1838 {
1839         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1840         u32 chan = queue;
1841         int i;
1842
1843         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
1844
1845         stmmac_stop_tx_dma(priv, chan);
1846         dma_free_tx_skbufs(priv, queue);
1847         for (i = 0; i < DMA_TX_SIZE; i++)
1848                 if (priv->extend_desc)
1849                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1850                                                      priv->mode,
1851                                                      (i == DMA_TX_SIZE - 1));
1852                 else
1853                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1854                                                      priv->mode,
1855                                                      (i == DMA_TX_SIZE - 1));
1856         tx_q->dirty_tx = 0;
1857         tx_q->cur_tx = 0;
1858         netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1859         stmmac_start_tx_dma(priv, chan);
1860
1861         priv->dev->stats.tx_errors++;
1862         netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1863 }
1864
1865 /**
1866  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
1867  *  @priv: driver private structure
1868  *  @txmode: TX operating mode
1869  *  @rxmode: RX operating mode
1870  *  @chan: channel index
1871  *  Description: it is used for configuring of the DMA operation mode in
1872  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
1873  *  mode.
1874  */
1875 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
1876                                           u32 rxmode, u32 chan)
1877 {
1878         int rxfifosz = priv->plat->rx_fifo_size;
1879
1880         if (rxfifosz == 0)
1881                 rxfifosz = priv->dma_cap.rx_fifo_size;
1882
1883         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1884                 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1885                                            rxfifosz);
1886                 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan);
1887         } else {
1888                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1889                                         rxfifosz);
1890         }
1891 }
1892
1893 /**
1894  * stmmac_dma_interrupt - DMA ISR
1895  * @priv: driver private structure
1896  * Description: this is the DMA ISR. It is called by the main ISR.
1897  * It calls the dwmac dma routine and schedule poll method in case of some
1898  * work can be done.
1899  */
1900 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
1901 {
1902         u32 tx_channel_count = priv->plat->tx_queues_to_use;
1903         int status;
1904         u32 chan;
1905
1906         for (chan = 0; chan < tx_channel_count; chan++) {
1907                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
1908
1909                 status = priv->hw->dma->dma_interrupt(priv->ioaddr,
1910                                                       &priv->xstats, chan);
1911                 if (likely((status & handle_rx)) || (status & handle_tx)) {
1912                         if (likely(napi_schedule_prep(&rx_q->napi))) {
1913                                 stmmac_disable_dma_irq(priv, chan);
1914                                 __napi_schedule(&rx_q->napi);
1915                         }
1916                 }
1917
1918                 if (unlikely(status & tx_hard_error_bump_tc)) {
1919                         /* Try to bump up the dma threshold on this failure */
1920                         if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
1921                             (tc <= 256)) {
1922                                 tc += 64;
1923                                 if (priv->plat->force_thresh_dma_mode)
1924                                         stmmac_set_dma_operation_mode(priv,
1925                                                                       tc,
1926                                                                       tc,
1927                                                                       chan);
1928                                 else
1929                                         stmmac_set_dma_operation_mode(priv,
1930                                                                     tc,
1931                                                                     SF_DMA_MODE,
1932                                                                     chan);
1933                                 priv->xstats.threshold = tc;
1934                         }
1935                 } else if (unlikely(status == tx_hard_error)) {
1936                         stmmac_tx_err(priv, chan);
1937                 }
1938         }
1939 }
1940
1941 /**
1942  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
1943  * @priv: driver private structure
1944  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
1945  */
1946 static void stmmac_mmc_setup(struct stmmac_priv *priv)
1947 {
1948         unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
1949                             MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
1950
1951         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1952                 priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET;
1953                 priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET;
1954         } else {
1955                 priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET;
1956                 priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET;
1957         }
1958
1959         dwmac_mmc_intr_all_mask(priv->mmcaddr);
1960
1961         if (priv->dma_cap.rmon) {
1962                 dwmac_mmc_ctrl(priv->mmcaddr, mode);
1963                 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
1964         } else
1965                 netdev_info(priv->dev, "No MAC Management Counters available\n");
1966 }
1967
1968 /**
1969  * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors
1970  * @priv: driver private structure
1971  * Description: select the Enhanced/Alternate or Normal descriptors.
1972  * In case of Enhanced/Alternate, it checks if the extended descriptors are
1973  * supported by the HW capability register.
1974  */
1975 static void stmmac_selec_desc_mode(struct stmmac_priv *priv)
1976 {
1977         if (priv->plat->enh_desc) {
1978                 dev_info(priv->device, "Enhanced/Alternate descriptors\n");
1979
1980                 /* GMAC older than 3.50 has no extended descriptors */
1981                 if (priv->synopsys_id >= DWMAC_CORE_3_50) {
1982                         dev_info(priv->device, "Enabled extended descriptors\n");
1983                         priv->extend_desc = 1;
1984                 } else
1985                         dev_warn(priv->device, "Extended descriptors not supported\n");
1986
1987                 priv->hw->desc = &enh_desc_ops;
1988         } else {
1989                 dev_info(priv->device, "Normal descriptors\n");
1990                 priv->hw->desc = &ndesc_ops;
1991         }
1992 }
1993
1994 /**
1995  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
1996  * @priv: driver private structure
1997  * Description:
1998  *  new GMAC chip generations have a new register to indicate the
1999  *  presence of the optional feature/functions.
2000  *  This can be also used to override the value passed through the
2001  *  platform and necessary for old MAC10/100 and GMAC chips.
2002  */
2003 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2004 {
2005         u32 ret = 0;
2006
2007         if (priv->hw->dma->get_hw_feature) {
2008                 priv->hw->dma->get_hw_feature(priv->ioaddr,
2009                                               &priv->dma_cap);
2010                 ret = 1;
2011         }
2012
2013         return ret;
2014 }
2015
2016 /**
2017  * stmmac_check_ether_addr - check if the MAC addr is valid
2018  * @priv: driver private structure
2019  * Description:
2020  * it is to verify if the MAC address is valid, in case of failures it
2021  * generates a random MAC address
2022  */
2023 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2024 {
2025         if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2026                 priv->hw->mac->get_umac_addr(priv->hw,
2027                                              priv->dev->dev_addr, 0);
2028                 if (!is_valid_ether_addr(priv->dev->dev_addr))
2029                         eth_hw_addr_random(priv->dev);
2030                 netdev_info(priv->dev, "device MAC address %pM\n",
2031                             priv->dev->dev_addr);
2032         }
2033 }
2034
2035 /**
2036  * stmmac_init_dma_engine - DMA init.
2037  * @priv: driver private structure
2038  * Description:
2039  * It inits the DMA invoking the specific MAC/GMAC callback.
2040  * Some DMA parameters can be passed from the platform;
2041  * in case of these are not passed a default is kept for the MAC or GMAC.
2042  */
2043 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2044 {
2045         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2046         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2047         struct stmmac_rx_queue *rx_q;
2048         struct stmmac_tx_queue *tx_q;
2049         u32 dummy_dma_rx_phy = 0;
2050         u32 dummy_dma_tx_phy = 0;
2051         u32 chan = 0;
2052         int atds = 0;
2053         int ret = 0;
2054
2055         if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2056                 dev_err(priv->device, "Invalid DMA configuration\n");
2057                 return -EINVAL;
2058         }
2059
2060         if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2061                 atds = 1;
2062
2063         ret = priv->hw->dma->reset(priv->ioaddr);
2064         if (ret) {
2065                 dev_err(priv->device, "Failed to reset the dma\n");
2066                 return ret;
2067         }
2068
2069         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
2070                 /* DMA Configuration */
2071                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2072                                     dummy_dma_tx_phy, dummy_dma_rx_phy, atds);
2073
2074                 /* DMA RX Channel Configuration */
2075                 for (chan = 0; chan < rx_channels_count; chan++) {
2076                         rx_q = &priv->rx_queue[chan];
2077
2078                         priv->hw->dma->init_rx_chan(priv->ioaddr,
2079                                                     priv->plat->dma_cfg,
2080                                                     rx_q->dma_rx_phy, chan);
2081
2082                         rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2083                                     (DMA_RX_SIZE * sizeof(struct dma_desc));
2084                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
2085                                                        rx_q->rx_tail_addr,
2086                                                        chan);
2087                 }
2088
2089                 /* DMA TX Channel Configuration */
2090                 for (chan = 0; chan < tx_channels_count; chan++) {
2091                         tx_q = &priv->tx_queue[chan];
2092
2093                         priv->hw->dma->init_chan(priv->ioaddr,
2094                                                  priv->plat->dma_cfg,
2095                                                  chan);
2096
2097                         priv->hw->dma->init_tx_chan(priv->ioaddr,
2098                                                     priv->plat->dma_cfg,
2099                                                     tx_q->dma_tx_phy, chan);
2100
2101                         tx_q->tx_tail_addr = tx_q->dma_tx_phy +
2102                                     (DMA_TX_SIZE * sizeof(struct dma_desc));
2103                         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr,
2104                                                        tx_q->tx_tail_addr,
2105                                                        chan);
2106                 }
2107         } else {
2108                 rx_q = &priv->rx_queue[chan];
2109                 tx_q = &priv->tx_queue[chan];
2110
2111                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2112                                     tx_q->dma_tx_phy, rx_q->dma_rx_phy, atds);
2113         }
2114
2115         if (priv->plat->axi && priv->hw->dma->axi)
2116                 priv->hw->dma->axi(priv->ioaddr, priv->plat->axi);
2117
2118         return ret;
2119 }
2120
2121 /**
2122  * stmmac_tx_timer - mitigation sw timer for tx.
2123  * @data: data pointer
2124  * Description:
2125  * This is the timer handler to directly invoke the stmmac_tx_clean.
2126  */
2127 static void stmmac_tx_timer(unsigned long data)
2128 {
2129         struct stmmac_priv *priv = (struct stmmac_priv *)data;
2130         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2131         u32 queue;
2132
2133         /* let's scan all the tx queues */
2134         for (queue = 0; queue < tx_queues_count; queue++)
2135                 stmmac_tx_clean(priv, queue);
2136 }
2137
2138 /**
2139  * stmmac_stop_all_queues - Stop all queues
2140  * @priv: driver private structure
2141  */
2142 static void stmmac_stop_all_queues(struct stmmac_priv *priv)
2143 {
2144         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
2145         u32 queue;
2146
2147         for (queue = 0; queue < tx_queues_cnt; queue++)
2148                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
2149 }
2150
2151 /**
2152  * stmmac_start_all_queues - Start all queues
2153  * @priv: driver private structure
2154  */
2155 static void stmmac_start_all_queues(struct stmmac_priv *priv)
2156 {
2157         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
2158         u32 queue;
2159
2160         for (queue = 0; queue < tx_queues_cnt; queue++)
2161                 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
2162 }
2163
2164 /**
2165  * stmmac_disable_all_queues - Disable all queues
2166  * @priv: driver private structure
2167  */
2168 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
2169 {
2170         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
2171         u32 queue;
2172
2173         for (queue = 0; queue < rx_queues_cnt; queue++) {
2174                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
2175
2176                 napi_disable(&rx_q->napi);
2177         }
2178 }
2179
2180 /**
2181  * stmmac_enable_all_queues - Enable all queues
2182  * @priv: driver private structure
2183  */
2184 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
2185 {
2186         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
2187         u32 queue;
2188
2189         for (queue = 0; queue < rx_queues_cnt; queue++) {
2190                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
2191
2192                 napi_enable(&rx_q->napi);
2193         }
2194 }
2195
2196 /**
2197  * stmmac_init_tx_coalesce - init tx mitigation options.
2198  * @priv: driver private structure
2199  * Description:
2200  * This inits the transmit coalesce parameters: i.e. timer rate,
2201  * timer handler and default threshold used for enabling the
2202  * interrupt on completion bit.
2203  */
2204 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv)
2205 {
2206         priv->tx_coal_frames = STMMAC_TX_FRAMES;
2207         priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2208         init_timer(&priv->txtimer);
2209         priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer);
2210         priv->txtimer.data = (unsigned long)priv;
2211         priv->txtimer.function = stmmac_tx_timer;
2212         add_timer(&priv->txtimer);
2213 }
2214
2215 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2216 {
2217         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2218         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2219         u32 chan;
2220
2221         /* set TX ring length */
2222         if (priv->hw->dma->set_tx_ring_len) {
2223                 for (chan = 0; chan < tx_channels_count; chan++)
2224                         priv->hw->dma->set_tx_ring_len(priv->ioaddr,
2225                                                        (DMA_TX_SIZE - 1), chan);
2226         }
2227
2228         /* set RX ring length */
2229         if (priv->hw->dma->set_rx_ring_len) {
2230                 for (chan = 0; chan < rx_channels_count; chan++)
2231                         priv->hw->dma->set_rx_ring_len(priv->ioaddr,
2232                                                        (DMA_RX_SIZE - 1), chan);
2233         }
2234 }
2235
2236 /**
2237  *  stmmac_set_tx_queue_weight - Set TX queue weight
2238  *  @priv: driver private structure
2239  *  Description: It is used for setting TX queues weight
2240  */
2241 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2242 {
2243         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2244         u32 weight;
2245         u32 queue;
2246
2247         for (queue = 0; queue < tx_queues_count; queue++) {
2248                 weight = priv->plat->tx_queues_cfg[queue].weight;
2249                 priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue);
2250         }
2251 }
2252
2253 /**
2254  *  stmmac_configure_cbs - Configure CBS in TX queue
2255  *  @priv: driver private structure
2256  *  Description: It is used for configuring CBS in AVB TX queues
2257  */
2258 static void stmmac_configure_cbs(struct stmmac_priv *priv)
2259 {
2260         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2261         u32 mode_to_use;
2262         u32 queue;
2263
2264         for (queue = 0; queue < tx_queues_count; queue++) {
2265                 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2266                 if (mode_to_use == MTL_QUEUE_DCB)
2267                         continue;
2268
2269                 priv->hw->mac->config_cbs(priv->hw,
2270                                 priv->plat->tx_queues_cfg[queue].send_slope,
2271                                 priv->plat->tx_queues_cfg[queue].idle_slope,
2272                                 priv->plat->tx_queues_cfg[queue].high_credit,
2273                                 priv->plat->tx_queues_cfg[queue].low_credit,
2274                                 queue);
2275         }
2276 }
2277
2278 /**
2279  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2280  *  @priv: driver private structure
2281  *  Description: It is used for mapping RX queues to RX dma channels
2282  */
2283 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2284 {
2285         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2286         u32 queue;
2287         u32 chan;
2288
2289         for (queue = 0; queue < rx_queues_count; queue++) {
2290                 chan = priv->plat->rx_queues_cfg[queue].chan;
2291                 priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan);
2292         }
2293 }
2294
2295 /**
2296  *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
2297  *  @priv: driver private structure
2298  *  Description: It is used for configuring the RX Queue Priority
2299  */
2300 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
2301 {
2302         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2303         u32 queue;
2304         u32 prio;
2305
2306         for (queue = 0; queue < rx_queues_count; queue++) {
2307                 if (!priv->plat->rx_queues_cfg[queue].use_prio)
2308                         continue;
2309
2310                 prio = priv->plat->rx_queues_cfg[queue].prio;
2311                 priv->hw->mac->rx_queue_prio(priv->hw, prio, queue);
2312         }
2313 }
2314
2315 /**
2316  *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
2317  *  @priv: driver private structure
2318  *  Description: It is used for configuring the TX Queue Priority
2319  */
2320 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
2321 {
2322         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2323         u32 queue;
2324         u32 prio;
2325
2326         for (queue = 0; queue < tx_queues_count; queue++) {
2327                 if (!priv->plat->tx_queues_cfg[queue].use_prio)
2328                         continue;
2329
2330                 prio = priv->plat->tx_queues_cfg[queue].prio;
2331                 priv->hw->mac->tx_queue_prio(priv->hw, prio, queue);
2332         }
2333 }
2334
2335 /**
2336  *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
2337  *  @priv: driver private structure
2338  *  Description: It is used for configuring the RX queue routing
2339  */
2340 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
2341 {
2342         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2343         u32 queue;
2344         u8 packet;
2345
2346         for (queue = 0; queue < rx_queues_count; queue++) {
2347                 /* no specific packet type routing specified for the queue */
2348                 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
2349                         continue;
2350
2351                 packet = priv->plat->rx_queues_cfg[queue].pkt_route;
2352                 priv->hw->mac->rx_queue_prio(priv->hw, packet, queue);
2353         }
2354 }
2355
2356 /**
2357  *  stmmac_mtl_configuration - Configure MTL
2358  *  @priv: driver private structure
2359  *  Description: It is used for configurring MTL
2360  */
2361 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2362 {
2363         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2364         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2365
2366         if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight)
2367                 stmmac_set_tx_queue_weight(priv);
2368
2369         /* Configure MTL RX algorithms */
2370         if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms)
2371                 priv->hw->mac->prog_mtl_rx_algorithms(priv->hw,
2372                                                 priv->plat->rx_sched_algorithm);
2373
2374         /* Configure MTL TX algorithms */
2375         if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms)
2376                 priv->hw->mac->prog_mtl_tx_algorithms(priv->hw,
2377                                                 priv->plat->tx_sched_algorithm);
2378
2379         /* Configure CBS in AVB TX queues */
2380         if (tx_queues_count > 1 && priv->hw->mac->config_cbs)
2381                 stmmac_configure_cbs(priv);
2382
2383         /* Map RX MTL to DMA channels */
2384         if (rx_queues_count > 1 && priv->hw->mac->map_mtl_to_dma)
2385                 stmmac_rx_queue_dma_chan_map(priv);
2386
2387         /* Enable MAC RX Queues */
2388         if (priv->hw->mac->rx_queue_enable)
2389                 stmmac_mac_enable_rx_queues(priv);
2390
2391         /* Set RX priorities */
2392         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_prio)
2393                 stmmac_mac_config_rx_queues_prio(priv);
2394
2395         /* Set TX priorities */
2396         if (tx_queues_count > 1 && priv->hw->mac->tx_queue_prio)
2397                 stmmac_mac_config_tx_queues_prio(priv);
2398
2399         /* Set RX routing */
2400         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_routing)
2401                 stmmac_mac_config_rx_queues_routing(priv);
2402 }
2403
2404 /**
2405  * stmmac_hw_setup - setup mac in a usable state.
2406  *  @dev : pointer to the device structure.
2407  *  Description:
2408  *  this is the main function to setup the HW in a usable state because the
2409  *  dma engine is reset, the core registers are configured (e.g. AXI,
2410  *  Checksum features, timers). The DMA is ready to start receiving and
2411  *  transmitting.
2412  *  Return value:
2413  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2414  *  file on failure.
2415  */
2416 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2417 {
2418         struct stmmac_priv *priv = netdev_priv(dev);
2419         u32 rx_cnt = priv->plat->rx_queues_to_use;
2420         u32 tx_cnt = priv->plat->tx_queues_to_use;
2421         u32 chan;
2422         int ret;
2423
2424         /* DMA initialization and SW reset */
2425         ret = stmmac_init_dma_engine(priv);
2426         if (ret < 0) {
2427                 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2428                            __func__);
2429                 return ret;
2430         }
2431
2432         /* Copy the MAC addr into the HW  */
2433         priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0);
2434
2435         /* PS and related bits will be programmed according to the speed */
2436         if (priv->hw->pcs) {
2437                 int speed = priv->plat->mac_port_sel_speed;
2438
2439                 if ((speed == SPEED_10) || (speed == SPEED_100) ||
2440                     (speed == SPEED_1000)) {
2441                         priv->hw->ps = speed;
2442                 } else {
2443                         dev_warn(priv->device, "invalid port speed\n");
2444                         priv->hw->ps = 0;
2445                 }
2446         }
2447
2448         /* Initialize the MAC Core */
2449         priv->hw->mac->core_init(priv->hw, dev->mtu);
2450
2451         /* Initialize MTL*/
2452         if (priv->synopsys_id >= DWMAC_CORE_4_00)
2453                 stmmac_mtl_configuration(priv);
2454
2455         ret = priv->hw->mac->rx_ipc(priv->hw);
2456         if (!ret) {
2457                 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2458                 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2459                 priv->hw->rx_csum = 0;
2460         }
2461
2462         /* Enable the MAC Rx/Tx */
2463         priv->hw->mac->set_mac(priv->ioaddr, true);
2464
2465         /* Set the HW DMA mode and the COE */
2466         stmmac_dma_operation_mode(priv);
2467
2468         stmmac_mmc_setup(priv);
2469
2470         if (init_ptp) {
2471                 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2472                 if (ret < 0)
2473                         netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2474
2475                 ret = stmmac_init_ptp(priv);
2476                 if (ret == -EOPNOTSUPP)
2477                         netdev_warn(priv->dev, "PTP not supported by HW\n");
2478                 else if (ret)
2479                         netdev_warn(priv->dev, "PTP init failed\n");
2480         }
2481
2482 #ifdef CONFIG_DEBUG_FS
2483         ret = stmmac_init_fs(dev);
2484         if (ret < 0)
2485                 netdev_warn(priv->dev, "%s: failed debugFS registration\n",
2486                             __func__);
2487 #endif
2488         /* Start the ball rolling... */
2489         stmmac_start_all_dma(priv);
2490
2491         priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2492
2493         if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) {
2494                 priv->rx_riwt = MAX_DMA_RIWT;
2495                 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt);
2496         }
2497
2498         if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane)
2499                 priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0);
2500
2501         /* set TX and RX rings length */
2502         stmmac_set_rings_length(priv);
2503
2504         /* Enable TSO */
2505         if (priv->tso) {
2506                 for (chan = 0; chan < tx_cnt; chan++)
2507                         priv->hw->dma->enable_tso(priv->ioaddr, 1, chan);
2508         }
2509
2510         return 0;
2511 }
2512
2513 static void stmmac_hw_teardown(struct net_device *dev)
2514 {
2515         struct stmmac_priv *priv = netdev_priv(dev);
2516
2517         clk_disable_unprepare(priv->plat->clk_ptp_ref);
2518 }
2519
2520 /**
2521  *  stmmac_open - open entry point of the driver
2522  *  @dev : pointer to the device structure.
2523  *  Description:
2524  *  This function is the open entry point of the driver.
2525  *  Return value:
2526  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2527  *  file on failure.
2528  */
2529 static int stmmac_open(struct net_device *dev)
2530 {
2531         struct stmmac_priv *priv = netdev_priv(dev);
2532         int ret;
2533
2534         stmmac_check_ether_addr(priv);
2535
2536         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
2537             priv->hw->pcs != STMMAC_PCS_TBI &&
2538             priv->hw->pcs != STMMAC_PCS_RTBI) {
2539                 ret = stmmac_init_phy(dev);
2540                 if (ret) {
2541                         netdev_err(priv->dev,
2542                                    "%s: Cannot attach to PHY (error: %d)\n",
2543                                    __func__, ret);
2544                         return ret;
2545                 }
2546         }
2547
2548         /* Extra statistics */
2549         memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2550         priv->xstats.threshold = tc;
2551
2552         priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2553
2554         ret = stmmac_hw_setup(dev, true);
2555         if (ret < 0) {
2556                 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2557                 goto init_error;
2558         }
2559
2560         stmmac_init_tx_coalesce(priv);
2561
2562         if (dev->phydev)
2563                 phy_start(dev->phydev);
2564
2565         /* Request the IRQ lines */
2566         ret = request_irq(dev->irq, stmmac_interrupt,
2567                           IRQF_SHARED, dev->name, dev);
2568         if (unlikely(ret < 0)) {
2569                 netdev_err(priv->dev,
2570                            "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2571                            __func__, dev->irq, ret);
2572                 goto irq_error;
2573         }
2574
2575         /* Request the Wake IRQ in case of another line is used for WoL */
2576         if (priv->wol_irq != dev->irq) {
2577                 ret = request_irq(priv->wol_irq, stmmac_interrupt,
2578                                   IRQF_SHARED, dev->name, dev);
2579                 if (unlikely(ret < 0)) {
2580                         netdev_err(priv->dev,
2581                                    "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2582                                    __func__, priv->wol_irq, ret);
2583                         goto wolirq_error;
2584                 }
2585         }
2586
2587         /* Request the IRQ lines */
2588         if (priv->lpi_irq > 0) {
2589                 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2590                                   dev->name, dev);
2591                 if (unlikely(ret < 0)) {
2592                         netdev_err(priv->dev,
2593                                    "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2594                                    __func__, priv->lpi_irq, ret);
2595                         goto lpiirq_error;
2596                 }
2597         }
2598
2599         stmmac_enable_all_queues(priv);
2600         stmmac_start_all_queues(priv);
2601
2602         return 0;
2603
2604 lpiirq_error:
2605         if (priv->wol_irq != dev->irq)
2606                 free_irq(priv->wol_irq, dev);
2607 wolirq_error:
2608         free_irq(dev->irq, dev);
2609 irq_error:
2610         if (dev->phydev)
2611                 phy_stop(dev->phydev);
2612
2613         del_timer_sync(&priv->txtimer);
2614         stmmac_hw_teardown(dev);
2615 init_error:
2616         free_dma_desc_resources(priv);
2617
2618         if (dev->phydev)
2619                 phy_disconnect(dev->phydev);
2620
2621         return ret;
2622 }
2623
2624 /**
2625  *  stmmac_release - close entry point of the driver
2626  *  @dev : device pointer.
2627  *  Description:
2628  *  This is the stop entry point of the driver.
2629  */
2630 static int stmmac_release(struct net_device *dev)
2631 {
2632         struct stmmac_priv *priv = netdev_priv(dev);
2633
2634         if (priv->eee_enabled)
2635                 del_timer_sync(&priv->eee_ctrl_timer);
2636
2637         /* Stop and disconnect the PHY */
2638         if (dev->phydev) {
2639                 phy_stop(dev->phydev);
2640                 phy_disconnect(dev->phydev);
2641         }
2642
2643         stmmac_stop_all_queues(priv);
2644
2645         stmmac_disable_all_queues(priv);
2646
2647         del_timer_sync(&priv->txtimer);
2648
2649         /* Free the IRQ lines */
2650         free_irq(dev->irq, dev);
2651         if (priv->wol_irq != dev->irq)
2652                 free_irq(priv->wol_irq, dev);
2653         if (priv->lpi_irq > 0)
2654                 free_irq(priv->lpi_irq, dev);
2655
2656         /* Stop TX/RX DMA and clear the descriptors */
2657         stmmac_stop_all_dma(priv);
2658
2659         /* Release and free the Rx/Tx resources */
2660         free_dma_desc_resources(priv);
2661
2662         /* Disable the MAC Rx/Tx */
2663         priv->hw->mac->set_mac(priv->ioaddr, false);
2664
2665         netif_carrier_off(dev);
2666
2667 #ifdef CONFIG_DEBUG_FS
2668         stmmac_exit_fs(dev);
2669 #endif
2670
2671         stmmac_release_ptp(priv);
2672
2673         return 0;
2674 }
2675
2676 /**
2677  *  stmmac_tso_allocator - close entry point of the driver
2678  *  @priv: driver private structure
2679  *  @des: buffer start address
2680  *  @total_len: total length to fill in descriptors
2681  *  @last_segmant: condition for the last descriptor
2682  *  @queue: TX queue index
2683  *  Description:
2684  *  This function fills descriptor and request new descriptors according to
2685  *  buffer length to fill
2686  */
2687 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des,
2688                                  int total_len, bool last_segment, u32 queue)
2689 {
2690         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2691         struct dma_desc *desc;
2692         u32 buff_size;
2693         int tmp_len;
2694
2695         tmp_len = total_len;
2696
2697         while (tmp_len > 0) {
2698                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2699                 desc = tx_q->dma_tx + tx_q->cur_tx;
2700
2701                 desc->des0 = cpu_to_le32(des + (total_len - tmp_len));
2702                 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2703                             TSO_MAX_BUFF_SIZE : tmp_len;
2704
2705                 priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size,
2706                         0, 1,
2707                         (last_segment) && (buff_size < TSO_MAX_BUFF_SIZE),
2708                         0, 0);
2709
2710                 tmp_len -= TSO_MAX_BUFF_SIZE;
2711         }
2712 }
2713
2714 /**
2715  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
2716  *  @skb : the socket buffer
2717  *  @dev : device pointer
2718  *  Description: this is the transmit function that is called on TSO frames
2719  *  (support available on GMAC4 and newer chips).
2720  *  Diagram below show the ring programming in case of TSO frames:
2721  *
2722  *  First Descriptor
2723  *   --------
2724  *   | DES0 |---> buffer1 = L2/L3/L4 header
2725  *   | DES1 |---> TCP Payload (can continue on next descr...)
2726  *   | DES2 |---> buffer 1 and 2 len
2727  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
2728  *   --------
2729  *      |
2730  *     ...
2731  *      |
2732  *   --------
2733  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
2734  *   | DES1 | --|
2735  *   | DES2 | --> buffer 1 and 2 len
2736  *   | DES3 |
2737  *   --------
2738  *
2739  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
2740  */
2741 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
2742 {
2743         struct dma_desc *desc, *first, *mss_desc = NULL;
2744         struct stmmac_priv *priv = netdev_priv(dev);
2745         u32 queue = skb_get_queue_mapping(skb);
2746         int nfrags = skb_shinfo(skb)->nr_frags;
2747         unsigned int first_entry, des;
2748         struct stmmac_tx_queue *tx_q;
2749         int tmp_pay_len = 0;
2750         u32 pay_len, mss;
2751         u8 proto_hdr_len;
2752         int i;
2753
2754         tx_q = &priv->tx_queue[queue];
2755
2756         /* Compute header lengths */
2757         proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2758
2759         /* Desc availability based on threshold should be enough safe */
2760         if (unlikely(stmmac_tx_avail(priv, queue) <
2761                 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2762                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2763                         netif_tx_stop_queue(netdev_get_tx_queue(dev, queue));
2764                         /* This is a hard error, log it. */
2765                         netdev_err(priv->dev,
2766                                    "%s: Tx Ring full when queue awake\n",
2767                                    __func__);
2768                 }
2769                 return NETDEV_TX_BUSY;
2770         }
2771
2772         pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
2773
2774         mss = skb_shinfo(skb)->gso_size;
2775
2776         /* set new MSS value if needed */
2777         if (mss != priv->mss) {
2778                 mss_desc = tx_q->dma_tx + tx_q->cur_tx;
2779                 priv->hw->desc->set_mss(mss_desc, mss);
2780                 priv->mss = mss;
2781                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2782         }
2783
2784         if (netif_msg_tx_queued(priv)) {
2785                 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
2786                         __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss);
2787                 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
2788                         skb->data_len);
2789         }
2790
2791         first_entry = tx_q->cur_tx;
2792
2793         desc = tx_q->dma_tx + first_entry;
2794         first = desc;
2795
2796         /* first descriptor: fill Headers on Buf1 */
2797         des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
2798                              DMA_TO_DEVICE);
2799         if (dma_mapping_error(priv->device, des))
2800                 goto dma_map_err;
2801
2802         tx_q->tx_skbuff_dma[first_entry].buf = des;
2803         tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
2804         tx_q->tx_skbuff[first_entry] = skb;
2805
2806         first->des0 = cpu_to_le32(des);
2807
2808         /* Fill start of payload in buff2 of first descriptor */
2809         if (pay_len)
2810                 first->des1 = cpu_to_le32(des + proto_hdr_len);
2811
2812         /* If needed take extra descriptors to fill the remaining payload */
2813         tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
2814
2815         stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
2816
2817         /* Prepare fragments */
2818         for (i = 0; i < nfrags; i++) {
2819                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2820
2821                 des = skb_frag_dma_map(priv->device, frag, 0,
2822                                        skb_frag_size(frag),
2823                                        DMA_TO_DEVICE);
2824                 if (dma_mapping_error(priv->device, des))
2825                         goto dma_map_err;
2826
2827                 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
2828                                      (i == nfrags - 1), queue);
2829
2830                 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
2831                 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
2832                 tx_q->tx_skbuff[tx_q->cur_tx] = NULL;
2833                 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
2834         }
2835
2836         tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
2837
2838         tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2839
2840         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
2841                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
2842                           __func__);
2843                 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue));
2844         }
2845
2846         dev->stats.tx_bytes += skb->len;
2847         priv->xstats.tx_tso_frames++;
2848         priv->xstats.tx_tso_nfrags += nfrags;
2849
2850         /* Manage tx mitigation */
2851         priv->tx_count_frames += nfrags + 1;
2852         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
2853                 mod_timer(&priv->txtimer,
2854                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
2855         } else {
2856                 priv->tx_count_frames = 0;
2857                 priv->hw->desc->set_tx_ic(desc);
2858                 priv->xstats.tx_set_ic_bit++;
2859         }
2860
2861         if (!priv->hwts_tx_en)
2862                 skb_tx_timestamp(skb);
2863
2864         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2865                      priv->hwts_tx_en)) {
2866                 /* declare that device is doing timestamping */
2867                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2868                 priv->hw->desc->enable_tx_timestamp(first);
2869         }
2870
2871         /* Complete the first descriptor before granting the DMA */
2872         priv->hw->desc->prepare_tso_tx_desc(first, 1,
2873                         proto_hdr_len,
2874                         pay_len,
2875                         1, tx_q->tx_skbuff_dma[first_entry].last_segment,
2876                         tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len));
2877
2878         /* If context desc is used to change MSS */
2879         if (mss_desc)
2880                 priv->hw->desc->set_tx_owner(mss_desc);
2881
2882         /* The own bit must be the latest setting done when prepare the
2883          * descriptor and then barrier is needed to make sure that
2884          * all is coherent before granting the DMA engine.
2885          */
2886         dma_wmb();
2887
2888         if (netif_msg_pktdata(priv)) {
2889                 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
2890                         __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
2891                         tx_q->cur_tx, first, nfrags);
2892
2893                 priv->hw->desc->display_ring((void *)tx_q->dma_tx, DMA_TX_SIZE,
2894                                              0);
2895
2896                 pr_info(">>> frame to be transmitted: ");
2897                 print_pkt(skb->data, skb_headlen(skb));
2898         }
2899
2900         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
2901
2902         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
2903                                        queue);
2904
2905         return NETDEV_TX_OK;
2906
2907 dma_map_err:
2908         dev_err(priv->device, "Tx dma map failed\n");
2909         dev_kfree_skb(skb);
2910         priv->dev->stats.tx_dropped++;
2911         return NETDEV_TX_OK;
2912 }
2913
2914 /**
2915  *  stmmac_xmit - Tx entry point of the driver
2916  *  @skb : the socket buffer
2917  *  @dev : device pointer
2918  *  Description : this is the tx entry point of the driver.
2919  *  It programs the chain or the ring and supports oversized frames
2920  *  and SG feature.
2921  */
2922 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
2923 {
2924         struct stmmac_priv *priv = netdev_priv(dev);
2925         unsigned int nopaged_len = skb_headlen(skb);
2926         int i, csum_insertion = 0, is_jumbo = 0;
2927         u32 queue = skb_get_queue_mapping(skb);
2928         int nfrags = skb_shinfo(skb)->nr_frags;
2929         unsigned int entry, first_entry;
2930         struct dma_desc *desc, *first;
2931         struct stmmac_tx_queue *tx_q;
2932         unsigned int enh_desc;
2933         unsigned int des;
2934
2935         tx_q = &priv->tx_queue[queue];
2936
2937         /* Manage oversized TCP frames for GMAC4 device */
2938         if (skb_is_gso(skb) && priv->tso) {
2939                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2940                         return stmmac_tso_xmit(skb, dev);
2941         }
2942
2943         if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
2944                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2945                         netif_tx_stop_queue(netdev_get_tx_queue(dev, queue));
2946                         /* This is a hard error, log it. */
2947                         netdev_err(priv->dev,
2948                                    "%s: Tx Ring full when queue awake\n",
2949                                    __func__);
2950                 }
2951                 return NETDEV_TX_BUSY;
2952         }
2953
2954         if (priv->tx_path_in_lpi_mode)
2955                 stmmac_disable_eee_mode(priv);
2956
2957         entry = tx_q->cur_tx;
2958         first_entry = entry;
2959
2960         csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
2961
2962         if (likely(priv->extend_desc))
2963                 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
2964         else
2965                 desc = tx_q->dma_tx + entry;
2966
2967         first = desc;
2968
2969         tx_q->tx_skbuff[first_entry] = skb;
2970
2971         enh_desc = priv->plat->enh_desc;
2972         /* To program the descriptors according to the size of the frame */
2973         if (enh_desc)
2974                 is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc);
2975
2976         if (unlikely(is_jumbo) && likely(priv->synopsys_id <
2977                                          DWMAC_CORE_4_00)) {
2978                 entry = priv->hw->mode->jumbo_frm(tx_q, skb, csum_insertion);
2979                 if (unlikely(entry < 0))
2980                         goto dma_map_err;
2981         }
2982
2983         for (i = 0; i < nfrags; i++) {
2984                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2985                 int len = skb_frag_size(frag);
2986                 bool last_segment = (i == (nfrags - 1));
2987
2988                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
2989
2990                 if (likely(priv->extend_desc))
2991                         desc = (struct dma_desc *)(tx_q->dma_etx + entry);
2992                 else
2993                         desc = tx_q->dma_tx + entry;
2994
2995                 des = skb_frag_dma_map(priv->device, frag, 0, len,
2996                                        DMA_TO_DEVICE);
2997                 if (dma_mapping_error(priv->device, des))
2998                         goto dma_map_err; /* should reuse desc w/o issues */
2999
3000                 tx_q->tx_skbuff[entry] = NULL;
3001
3002                 tx_q->tx_skbuff_dma[entry].buf = des;
3003                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3004                         desc->des0 = cpu_to_le32(des);
3005                 else
3006                         desc->des2 = cpu_to_le32(des);
3007
3008                 tx_q->tx_skbuff_dma[entry].map_as_page = true;
3009                 tx_q->tx_skbuff_dma[entry].len = len;
3010                 tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
3011
3012                 /* Prepare the descriptor and set the own bit too */
3013                 priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion,
3014                                                 priv->mode, 1, last_segment);
3015         }
3016
3017         entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3018
3019         tx_q->cur_tx = entry;
3020
3021         if (netif_msg_pktdata(priv)) {
3022                 void *tx_head;
3023
3024                 netdev_dbg(priv->dev,
3025                            "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
3026                            __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3027                            entry, first, nfrags);
3028
3029                 if (priv->extend_desc)
3030                         tx_head = (void *)tx_q->dma_etx;
3031                 else
3032                         tx_head = (void *)tx_q->dma_tx;
3033
3034                 priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false);
3035
3036                 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
3037                 print_pkt(skb->data, skb->len);
3038         }
3039
3040         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3041                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3042                           __func__);
3043                 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue));
3044         }
3045
3046         dev->stats.tx_bytes += skb->len;
3047
3048         /* According to the coalesce parameter the IC bit for the latest
3049          * segment is reset and the timer re-started to clean the tx status.
3050          * This approach takes care about the fragments: desc is the first
3051          * element in case of no SG.
3052          */
3053         priv->tx_count_frames += nfrags + 1;
3054         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
3055                 mod_timer(&priv->txtimer,
3056                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
3057         } else {
3058                 priv->tx_count_frames = 0;
3059                 priv->hw->desc->set_tx_ic(desc);
3060                 priv->xstats.tx_set_ic_bit++;
3061         }
3062
3063         if (!priv->hwts_tx_en)
3064                 skb_tx_timestamp(skb);
3065
3066         /* Ready to fill the first descriptor and set the OWN bit w/o any
3067          * problems because all the descriptors are actually ready to be
3068          * passed to the DMA engine.
3069          */
3070         if (likely(!is_jumbo)) {
3071                 bool last_segment = (nfrags == 0);
3072
3073                 des = dma_map_single(priv->device, skb->data,
3074                                      nopaged_len, DMA_TO_DEVICE);
3075                 if (dma_mapping_error(priv->device, des))
3076                         goto dma_map_err;
3077
3078                 tx_q->tx_skbuff_dma[first_entry].buf = des;
3079                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3080                         first->des0 = cpu_to_le32(des);
3081                 else
3082                         first->des2 = cpu_to_le32(des);
3083
3084                 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3085                 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3086
3087                 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3088                              priv->hwts_tx_en)) {
3089                         /* declare that device is doing timestamping */
3090                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3091                         priv->hw->desc->enable_tx_timestamp(first);
3092                 }
3093
3094                 /* Prepare the first descriptor setting the OWN bit too */
3095                 priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len,
3096                                                 csum_insertion, priv->mode, 1,
3097                                                 last_segment);
3098
3099                 /* The own bit must be the latest setting done when prepare the
3100                  * descriptor and then barrier is needed to make sure that
3101                  * all is coherent before granting the DMA engine.
3102                  */
3103                 dma_wmb();
3104         }
3105
3106         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3107
3108         if (priv->synopsys_id < DWMAC_CORE_4_00)
3109                 priv->hw->dma->enable_dma_transmission(priv->ioaddr);
3110         else
3111                 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
3112                                                queue);
3113
3114         return NETDEV_TX_OK;
3115
3116 dma_map_err:
3117         netdev_err(priv->dev, "Tx DMA map failed\n");
3118         dev_kfree_skb(skb);
3119         priv->dev->stats.tx_dropped++;
3120         return NETDEV_TX_OK;
3121 }
3122
3123 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3124 {
3125         struct ethhdr *ehdr;
3126         u16 vlanid;
3127
3128         if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) ==
3129             NETIF_F_HW_VLAN_CTAG_RX &&
3130             !__vlan_get_tag(skb, &vlanid)) {
3131                 /* pop the vlan tag */
3132                 ehdr = (struct ethhdr *)skb->data;
3133                 memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2);
3134                 skb_pull(skb, VLAN_HLEN);
3135                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid);
3136         }
3137 }
3138
3139
3140 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
3141 {
3142         if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
3143                 return 0;
3144
3145         return 1;
3146 }
3147
3148 /**
3149  * stmmac_rx_refill - refill used skb preallocated buffers
3150  * @priv: driver private structure
3151  * @queue: RX queue index
3152  * Description : this is to reallocate the skb for the reception process
3153  * that is based on zero-copy.
3154  */
3155 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3156 {
3157         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3158         int dirty = stmmac_rx_dirty(priv, queue);
3159         unsigned int entry = rx_q->dirty_rx;
3160         int bfsize = priv->dma_buf_sz;
3161
3162         while (dirty-- > 0) {
3163                 struct dma_desc *p;
3164
3165                 if (priv->extend_desc)
3166                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3167                 else
3168                         p = rx_q->dma_rx + entry;
3169
3170                 if (!rx_q->rx_skbuff[entry]) {
3171                         struct sk_buff *skb;
3172
3173                         skb = netdev_alloc_skb_ip_align(priv->dev, bfsize);
3174                         if (unlikely(!skb)) {
3175                                 /* so for a while no zero-copy! */
3176                                 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH;
3177                                 if (unlikely(net_ratelimit()))
3178                                         dev_err(priv->device,
3179                                                 "fail to alloc skb entry %d\n",
3180                                                 entry);
3181                                 break;
3182                         }
3183
3184                         rx_q->rx_skbuff[entry] = skb;
3185                         rx_q->rx_skbuff_dma[entry] =
3186                             dma_map_single(priv->device, skb->data, bfsize,
3187                                            DMA_FROM_DEVICE);
3188                         if (dma_mapping_error(priv->device,
3189                                               rx_q->rx_skbuff_dma[entry])) {
3190                                 netdev_err(priv->dev, "Rx DMA map failed\n");
3191                                 dev_kfree_skb(skb);
3192                                 break;
3193                         }
3194
3195                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) {
3196                                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3197                                 p->des1 = 0;
3198                         } else {
3199                                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3200                         }
3201                         if (priv->hw->mode->refill_desc3)
3202                                 priv->hw->mode->refill_desc3(rx_q, p);
3203
3204                         if (rx_q->rx_zeroc_thresh > 0)
3205                                 rx_q->rx_zeroc_thresh--;
3206
3207                         netif_dbg(priv, rx_status, priv->dev,
3208                                   "refill entry #%d\n", entry);
3209                 }
3210                 dma_wmb();
3211
3212                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3213                         priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0);
3214                 else
3215                         priv->hw->desc->set_rx_owner(p);
3216
3217                 dma_wmb();
3218
3219                 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
3220         }
3221         rx_q->dirty_rx = entry;
3222 }
3223
3224 /**
3225  * stmmac_rx - manage the receive process
3226  * @priv: driver private structure
3227  * @limit: napi bugget.
3228  * Description :  this the function called by the napi poll method.
3229  * It gets all the frames inside the ring.
3230  */
3231 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3232 {
3233         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3234         unsigned int entry = rx_q->cur_rx;
3235         int coe = priv->hw->rx_csum;
3236         unsigned int next_entry;
3237         unsigned int count = 0;
3238
3239         if (netif_msg_rx_status(priv)) {
3240                 void *rx_head;
3241
3242                 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3243                 if (priv->extend_desc)
3244                         rx_head = (void *)rx_q->dma_erx;
3245                 else
3246                         rx_head = (void *)rx_q->dma_rx;
3247
3248                 priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true);
3249         }
3250         while (count < limit) {
3251                 int status;
3252                 struct dma_desc *p;
3253                 struct dma_desc *np;
3254
3255                 if (priv->extend_desc)
3256                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3257                 else
3258                         p = rx_q->dma_rx + entry;
3259
3260                 /* read the status of the incoming frame */
3261                 status = priv->hw->desc->rx_status(&priv->dev->stats,
3262                                                    &priv->xstats, p);
3263                 /* check if managed by the DMA otherwise go ahead */
3264                 if (unlikely(status & dma_own))
3265                         break;
3266
3267                 count++;
3268
3269                 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
3270                 next_entry = rx_q->cur_rx;
3271
3272                 if (priv->extend_desc)
3273                         np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3274                 else
3275                         np = rx_q->dma_rx + next_entry;
3276
3277                 prefetch(np);
3278
3279                 if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status))
3280                         priv->hw->desc->rx_extended_status(&priv->dev->stats,
3281                                                            &priv->xstats,
3282                                                            rx_q->dma_erx +
3283                                                            entry);
3284                 if (unlikely(status == discard_frame)) {
3285                         priv->dev->stats.rx_errors++;
3286                         if (priv->hwts_rx_en && !priv->extend_desc) {
3287                                 /* DESC2 & DESC3 will be overwritten by device
3288                                  * with timestamp value, hence reinitialize
3289                                  * them in stmmac_rx_refill() function so that
3290                                  * device can reuse it.
3291                                  */
3292                                 rx_q->rx_skbuff[entry] = NULL;
3293                                 dma_unmap_single(priv->device,
3294                                                  rx_q->rx_skbuff_dma[entry],
3295                                                  priv->dma_buf_sz,
3296                                                  DMA_FROM_DEVICE);
3297                         }
3298                 } else {
3299                         struct sk_buff *skb;
3300                         int frame_len;
3301                         unsigned int des;
3302
3303                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3304                                 des = le32_to_cpu(p->des0);
3305                         else
3306                                 des = le32_to_cpu(p->des2);
3307
3308                         frame_len = priv->hw->desc->get_rx_frame_len(p, coe);
3309
3310                         /*  If frame length is greater than skb buffer size
3311                          *  (preallocated during init) then the packet is
3312                          *  ignored
3313                          */
3314                         if (frame_len > priv->dma_buf_sz) {
3315                                 netdev_err(priv->dev,
3316                                            "len %d larger than size (%d)\n",
3317                                            frame_len, priv->dma_buf_sz);
3318                                 priv->dev->stats.rx_length_errors++;
3319                                 break;
3320                         }
3321
3322                         /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3323                          * Type frames (LLC/LLC-SNAP)
3324                          */
3325                         if (unlikely(status != llc_snap))
3326                                 frame_len -= ETH_FCS_LEN;
3327
3328                         if (netif_msg_rx_status(priv)) {
3329                                 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n",
3330                                            p, entry, des);
3331                                 if (frame_len > ETH_FRAME_LEN)
3332                                         netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
3333                                                    frame_len, status);
3334                         }
3335
3336                         /* The zero-copy is always used for all the sizes
3337                          * in case of GMAC4 because it needs
3338                          * to refill the used descriptors, always.
3339                          */
3340                         if (unlikely(!priv->plat->has_gmac4 &&
3341                                      ((frame_len < priv->rx_copybreak) ||
3342                                      stmmac_rx_threshold_count(rx_q)))) {
3343                                 skb = netdev_alloc_skb_ip_align(priv->dev,
3344                                                                 frame_len);
3345                                 if (unlikely(!skb)) {
3346                                         if (net_ratelimit())
3347                                                 dev_warn(priv->device,
3348                                                          "packet dropped\n");
3349                                         priv->dev->stats.rx_dropped++;
3350                                         break;
3351                                 }
3352
3353                                 dma_sync_single_for_cpu(priv->device,
3354                                                         rx_q->rx_skbuff_dma
3355                                                         [entry], frame_len,
3356                                                         DMA_FROM_DEVICE);
3357                                 skb_copy_to_linear_data(skb,
3358                                                         rx_q->
3359                                                         rx_skbuff[entry]->data,
3360                                                         frame_len);
3361
3362                                 skb_put(skb, frame_len);
3363                                 dma_sync_single_for_device(priv->device,
3364                                                            rx_q->rx_skbuff_dma
3365                                                            [entry], frame_len,
3366                                                            DMA_FROM_DEVICE);
3367                         } else {
3368                                 skb = rx_q->rx_skbuff[entry];
3369                                 if (unlikely(!skb)) {
3370                                         netdev_err(priv->dev,
3371                                                    "%s: Inconsistent Rx chain\n",
3372                                                    priv->dev->name);
3373                                         priv->dev->stats.rx_dropped++;
3374                                         break;
3375                                 }
3376                                 prefetch(skb->data - NET_IP_ALIGN);
3377                                 rx_q->rx_skbuff[entry] = NULL;
3378                                 rx_q->rx_zeroc_thresh++;
3379
3380                                 skb_put(skb, frame_len);
3381                                 dma_unmap_single(priv->device,
3382                                                  rx_q->rx_skbuff_dma[entry],
3383                                                  priv->dma_buf_sz,
3384                                                  DMA_FROM_DEVICE);
3385                         }
3386
3387                         if (netif_msg_pktdata(priv)) {
3388                                 netdev_dbg(priv->dev, "frame received (%dbytes)",
3389                                            frame_len);
3390                                 print_pkt(skb->data, frame_len);
3391                         }
3392
3393                         stmmac_get_rx_hwtstamp(priv, p, np, skb);
3394
3395                         stmmac_rx_vlan(priv->dev, skb);
3396
3397                         skb->protocol = eth_type_trans(skb, priv->dev);
3398
3399                         if (unlikely(!coe))
3400                                 skb_checksum_none_assert(skb);
3401                         else
3402                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3403
3404                         napi_gro_receive(&rx_q->napi, skb);
3405
3406                         priv->dev->stats.rx_packets++;
3407                         priv->dev->stats.rx_bytes += frame_len;
3408                 }
3409                 entry = next_entry;
3410         }
3411
3412         stmmac_rx_refill(priv, queue);
3413
3414         priv->xstats.rx_pkt_n += count;
3415
3416         return count;
3417 }
3418
3419 /**
3420  *  stmmac_poll - stmmac poll method (NAPI)
3421  *  @napi : pointer to the napi structure.
3422  *  @budget : maximum number of packets that the current CPU can receive from
3423  *            all interfaces.
3424  *  Description :
3425  *  To look at the incoming frames and clear the tx resources.
3426  */
3427 static int stmmac_poll(struct napi_struct *napi, int budget)
3428 {
3429         struct stmmac_rx_queue *rx_q =
3430                 container_of(napi, struct stmmac_rx_queue, napi);
3431         struct stmmac_priv *priv = rx_q->priv_data;
3432         u32 tx_count = priv->dma_cap.number_tx_queues;
3433         u32 chan = rx_q->queue_index;
3434         u32 work_done = 0;
3435         u32 queue = 0;
3436
3437         priv->xstats.napi_poll++;
3438         /* check all the queues */
3439         for (queue = 0; queue < tx_count; queue++)
3440                 stmmac_tx_clean(priv, queue);
3441
3442         /* Process RX packets from this queue */
3443         work_done = stmmac_rx(priv, budget, rx_q->queue_index);
3444
3445         if (work_done < budget) {
3446                 napi_complete_done(napi, work_done);
3447                 stmmac_enable_dma_irq(priv, chan);
3448         }
3449         return work_done;
3450 }
3451
3452 /**
3453  *  stmmac_tx_timeout
3454  *  @dev : Pointer to net device structure
3455  *  Description: this function is called when a packet transmission fails to
3456  *   complete within a reasonable time. The driver will mark the error in the
3457  *   netdev structure and arrange for the device to be reset to a sane state
3458  *   in order to transmit a new packet.
3459  */
3460 static void stmmac_tx_timeout(struct net_device *dev)
3461 {
3462         struct stmmac_priv *priv = netdev_priv(dev);
3463         u32 tx_count = priv->plat->tx_queues_to_use;
3464         u32 chan;
3465
3466         /* Clear Tx resources and restart transmitting again */
3467         for (chan = 0; chan < tx_count; chan++)
3468                 stmmac_tx_err(priv, chan);
3469 }
3470
3471 /**
3472  *  stmmac_set_rx_mode - entry point for multicast addressing
3473  *  @dev : pointer to the device structure
3474  *  Description:
3475  *  This function is a driver entry point which gets called by the kernel
3476  *  whenever multicast addresses must be enabled/disabled.
3477  *  Return value:
3478  *  void.
3479  */
3480 static void stmmac_set_rx_mode(struct net_device *dev)
3481 {
3482         struct stmmac_priv *priv = netdev_priv(dev);
3483
3484         priv->hw->mac->set_filter(priv->hw, dev);
3485 }
3486
3487 /**
3488  *  stmmac_change_mtu - entry point to change MTU size for the device.
3489  *  @dev : device pointer.
3490  *  @new_mtu : the new MTU size for the device.
3491  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
3492  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
3493  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
3494  *  Return value:
3495  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3496  *  file on failure.
3497  */
3498 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3499 {
3500         struct stmmac_priv *priv = netdev_priv(dev);
3501
3502         if (netif_running(dev)) {
3503                 netdev_err(priv->dev, "must be stopped to change its MTU\n");
3504                 return -EBUSY;
3505         }
3506
3507         dev->mtu = new_mtu;
3508
3509         netdev_update_features(dev);
3510
3511         return 0;
3512 }
3513
3514 static netdev_features_t stmmac_fix_features(struct net_device *dev,
3515                                              netdev_features_t features)
3516 {
3517         struct stmmac_priv *priv = netdev_priv(dev);
3518
3519         if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3520                 features &= ~NETIF_F_RXCSUM;
3521
3522         if (!priv->plat->tx_coe)
3523                 features &= ~NETIF_F_CSUM_MASK;
3524
3525         /* Some GMAC devices have a bugged Jumbo frame support that
3526          * needs to have the Tx COE disabled for oversized frames
3527          * (due to limited buffer sizes). In this case we disable
3528          * the TX csum insertion in the TDES and not use SF.
3529          */
3530         if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3531                 features &= ~NETIF_F_CSUM_MASK;
3532
3533         /* Disable tso if asked by ethtool */
3534         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3535                 if (features & NETIF_F_TSO)
3536                         priv->tso = true;
3537                 else
3538                         priv->tso = false;
3539         }
3540
3541         return features;
3542 }
3543
3544 static int stmmac_set_features(struct net_device *netdev,
3545                                netdev_features_t features)
3546 {
3547         struct stmmac_priv *priv = netdev_priv(netdev);
3548
3549         /* Keep the COE Type in case of csum is supporting */
3550         if (features & NETIF_F_RXCSUM)
3551                 priv->hw->rx_csum = priv->plat->rx_coe;
3552         else
3553                 priv->hw->rx_csum = 0;
3554         /* No check needed because rx_coe has been set before and it will be
3555          * fixed in case of issue.
3556          */
3557         priv->hw->mac->rx_ipc(priv->hw);
3558
3559         return 0;
3560 }
3561
3562 /**
3563  *  stmmac_interrupt - main ISR
3564  *  @irq: interrupt number.
3565  *  @dev_id: to pass the net device pointer.
3566  *  Description: this is the main driver interrupt service routine.
3567  *  It can call:
3568  *  o DMA service routine (to manage incoming frame reception and transmission
3569  *    status)
3570  *  o Core interrupts to manage: remote wake-up, management counter, LPI
3571  *    interrupts.
3572  */
3573 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
3574 {
3575         struct net_device *dev = (struct net_device *)dev_id;
3576         struct stmmac_priv *priv = netdev_priv(dev);
3577         u32 rx_cnt = priv->plat->rx_queues_to_use;
3578         u32 tx_cnt = priv->plat->tx_queues_to_use;
3579         u32 queues_count;
3580         u32 queue;
3581
3582         queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
3583
3584         if (priv->irq_wake)
3585                 pm_wakeup_event(priv->device, 0);
3586
3587         if (unlikely(!dev)) {
3588                 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
3589                 return IRQ_NONE;
3590         }
3591
3592         /* To handle GMAC own interrupts */
3593         if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) {
3594                 int status = priv->hw->mac->host_irq_status(priv->hw,
3595                                                             &priv->xstats);
3596
3597                 if (unlikely(status)) {
3598                         /* For LPI we need to save the tx status */
3599                         if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
3600                                 priv->tx_path_in_lpi_mode = true;
3601                         if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
3602                                 priv->tx_path_in_lpi_mode = false;
3603                 }
3604
3605                 if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3606                         for (queue = 0; queue < queues_count; queue++) {
3607                                 struct stmmac_rx_queue *rx_q =
3608                                 &priv->rx_queue[queue];
3609
3610                                 status |=
3611                                 priv->hw->mac->host_mtl_irq_status(priv->hw,
3612                                                                    queue);
3613
3614                                 if (status & CORE_IRQ_MTL_RX_OVERFLOW &&
3615                                     priv->hw->dma->set_rx_tail_ptr)
3616                                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
3617                                                                 rx_q->rx_tail_addr,
3618                                                                 queue);
3619                         }
3620                 }
3621
3622                 /* PCS link status */
3623                 if (priv->hw->pcs) {
3624                         if (priv->xstats.pcs_link)
3625                                 netif_carrier_on(dev);
3626                         else
3627                                 netif_carrier_off(dev);
3628                 }
3629         }
3630
3631         /* To handle DMA interrupts */
3632         stmmac_dma_interrupt(priv);
3633
3634         return IRQ_HANDLED;
3635 }
3636
3637 #ifdef CONFIG_NET_POLL_CONTROLLER
3638 /* Polling receive - used by NETCONSOLE and other diagnostic tools
3639  * to allow network I/O with interrupts disabled.
3640  */
3641 static void stmmac_poll_controller(struct net_device *dev)
3642 {
3643         disable_irq(dev->irq);
3644         stmmac_interrupt(dev->irq, dev);
3645         enable_irq(dev->irq);
3646 }
3647 #endif
3648
3649 /**
3650  *  stmmac_ioctl - Entry point for the Ioctl
3651  *  @dev: Device pointer.
3652  *  @rq: An IOCTL specefic structure, that can contain a pointer to
3653  *  a proprietary structure used to pass information to the driver.
3654  *  @cmd: IOCTL command
3655  *  Description:
3656  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
3657  */
3658 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3659 {
3660         int ret = -EOPNOTSUPP;
3661
3662         if (!netif_running(dev))
3663                 return -EINVAL;
3664
3665         switch (cmd) {
3666         case SIOCGMIIPHY:
3667         case SIOCGMIIREG:
3668         case SIOCSMIIREG:
3669                 if (!dev->phydev)
3670                         return -EINVAL;
3671                 ret = phy_mii_ioctl(dev->phydev, rq, cmd);
3672                 break;
3673         case SIOCSHWTSTAMP:
3674                 ret = stmmac_hwtstamp_ioctl(dev, rq);
3675                 break;
3676         default:
3677                 break;
3678         }
3679
3680         return ret;
3681 }
3682
3683 #ifdef CONFIG_DEBUG_FS
3684 static struct dentry *stmmac_fs_dir;
3685
3686 static void sysfs_display_ring(void *head, int size, int extend_desc,
3687                                struct seq_file *seq)
3688 {
3689         int i;
3690         struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
3691         struct dma_desc *p = (struct dma_desc *)head;
3692
3693         for (i = 0; i < size; i++) {
3694                 if (extend_desc) {
3695                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3696                                    i, (unsigned int)virt_to_phys(ep),
3697                                    le32_to_cpu(ep->basic.des0),
3698                                    le32_to_cpu(ep->basic.des1),
3699                                    le32_to_cpu(ep->basic.des2),
3700                                    le32_to_cpu(ep->basic.des3));
3701                         ep++;
3702                 } else {
3703                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3704                                    i, (unsigned int)virt_to_phys(ep),
3705                                    le32_to_cpu(p->des0), le32_to_cpu(p->des1),
3706                                    le32_to_cpu(p->des2), le32_to_cpu(p->des3));
3707                         p++;
3708                 }
3709                 seq_printf(seq, "\n");
3710         }
3711 }
3712
3713 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v)
3714 {
3715         struct net_device *dev = seq->private;
3716         struct stmmac_priv *priv = netdev_priv(dev);
3717         u32 rx_count = priv->plat->rx_queues_to_use;
3718         u32 tx_count = priv->plat->tx_queues_to_use;
3719         u32 queue;
3720
3721         for (queue = 0; queue < rx_count; queue++) {
3722                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3723
3724                 seq_printf(seq, "RX Queue %d:\n", queue);
3725
3726                 if (priv->extend_desc) {
3727                         seq_printf(seq, "Extended descriptor ring:\n");
3728                         sysfs_display_ring((void *)rx_q->dma_erx,
3729                                            DMA_RX_SIZE, 1, seq);
3730                 } else {
3731                         seq_printf(seq, "Descriptor ring:\n");
3732                         sysfs_display_ring((void *)rx_q->dma_rx,
3733                                            DMA_RX_SIZE, 0, seq);
3734                 }
3735         }
3736
3737         for (queue = 0; queue < tx_count; queue++) {
3738                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
3739
3740                 seq_printf(seq, "TX Queue %d:\n", queue);
3741
3742                 if (priv->extend_desc) {
3743                         seq_printf(seq, "Extended descriptor ring:\n");
3744                         sysfs_display_ring((void *)tx_q->dma_etx,
3745                                            DMA_TX_SIZE, 1, seq);
3746                 } else {
3747                         seq_printf(seq, "Descriptor ring:\n");
3748                         sysfs_display_ring((void *)tx_q->dma_tx,
3749                                            DMA_TX_SIZE, 0, seq);
3750                 }
3751         }
3752
3753         return 0;
3754 }
3755
3756 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file)
3757 {
3758         return single_open(file, stmmac_sysfs_ring_read, inode->i_private);
3759 }
3760
3761 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */
3762
3763 static const struct file_operations stmmac_rings_status_fops = {
3764         .owner = THIS_MODULE,
3765         .open = stmmac_sysfs_ring_open,
3766         .read = seq_read,
3767         .llseek = seq_lseek,
3768         .release = single_release,
3769 };
3770
3771 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v)
3772 {
3773         struct net_device *dev = seq->private;
3774         struct stmmac_priv *priv = netdev_priv(dev);
3775
3776         if (!priv->hw_cap_support) {
3777                 seq_printf(seq, "DMA HW features not supported\n");
3778                 return 0;
3779         }
3780
3781         seq_printf(seq, "==============================\n");
3782         seq_printf(seq, "\tDMA HW features\n");
3783         seq_printf(seq, "==============================\n");
3784
3785         seq_printf(seq, "\t10/100 Mbps: %s\n",
3786                    (priv->dma_cap.mbps_10_100) ? "Y" : "N");
3787         seq_printf(seq, "\t1000 Mbps: %s\n",
3788                    (priv->dma_cap.mbps_1000) ? "Y" : "N");
3789         seq_printf(seq, "\tHalf duplex: %s\n",
3790                    (priv->dma_cap.half_duplex) ? "Y" : "N");
3791         seq_printf(seq, "\tHash Filter: %s\n",
3792                    (priv->dma_cap.hash_filter) ? "Y" : "N");
3793         seq_printf(seq, "\tMultiple MAC address registers: %s\n",
3794                    (priv->dma_cap.multi_addr) ? "Y" : "N");
3795         seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
3796                    (priv->dma_cap.pcs) ? "Y" : "N");
3797         seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
3798                    (priv->dma_cap.sma_mdio) ? "Y" : "N");
3799         seq_printf(seq, "\tPMT Remote wake up: %s\n",
3800                    (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
3801         seq_printf(seq, "\tPMT Magic Frame: %s\n",
3802                    (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
3803         seq_printf(seq, "\tRMON module: %s\n",
3804                    (priv->dma_cap.rmon) ? "Y" : "N");
3805         seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
3806                    (priv->dma_cap.time_stamp) ? "Y" : "N");
3807         seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
3808                    (priv->dma_cap.atime_stamp) ? "Y" : "N");
3809         seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
3810                    (priv->dma_cap.eee) ? "Y" : "N");
3811         seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
3812         seq_printf(seq, "\tChecksum Offload in TX: %s\n",
3813                    (priv->dma_cap.tx_coe) ? "Y" : "N");
3814         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3815                 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
3816                            (priv->dma_cap.rx_coe) ? "Y" : "N");
3817         } else {
3818                 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
3819                            (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
3820                 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
3821                            (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
3822         }
3823         seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
3824                    (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
3825         seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
3826                    priv->dma_cap.number_rx_channel);
3827         seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
3828                    priv->dma_cap.number_tx_channel);
3829         seq_printf(seq, "\tEnhanced descriptors: %s\n",
3830                    (priv->dma_cap.enh_desc) ? "Y" : "N");
3831
3832         return 0;
3833 }
3834
3835 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file)
3836 {
3837         return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private);
3838 }
3839
3840 static const struct file_operations stmmac_dma_cap_fops = {
3841         .owner = THIS_MODULE,
3842         .open = stmmac_sysfs_dma_cap_open,
3843         .read = seq_read,
3844         .llseek = seq_lseek,
3845         .release = single_release,
3846 };
3847
3848 static int stmmac_init_fs(struct net_device *dev)
3849 {
3850         struct stmmac_priv *priv = netdev_priv(dev);
3851
3852         /* Create per netdev entries */
3853         priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
3854
3855         if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) {
3856                 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n");
3857
3858                 return -ENOMEM;
3859         }
3860
3861         /* Entry to report DMA RX/TX rings */
3862         priv->dbgfs_rings_status =
3863                 debugfs_create_file("descriptors_status", S_IRUGO,
3864                                     priv->dbgfs_dir, dev,
3865                                     &stmmac_rings_status_fops);
3866
3867         if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) {
3868                 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n");
3869                 debugfs_remove_recursive(priv->dbgfs_dir);
3870
3871                 return -ENOMEM;
3872         }
3873
3874         /* Entry to report the DMA HW features */
3875         priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", S_IRUGO,
3876                                             priv->dbgfs_dir,
3877                                             dev, &stmmac_dma_cap_fops);
3878
3879         if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) {
3880                 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n");
3881                 debugfs_remove_recursive(priv->dbgfs_dir);
3882
3883                 return -ENOMEM;
3884         }
3885
3886         return 0;
3887 }
3888
3889 static void stmmac_exit_fs(struct net_device *dev)
3890 {
3891         struct stmmac_priv *priv = netdev_priv(dev);
3892
3893         debugfs_remove_recursive(priv->dbgfs_dir);
3894 }
3895 #endif /* CONFIG_DEBUG_FS */
3896
3897 static const struct net_device_ops stmmac_netdev_ops = {
3898         .ndo_open = stmmac_open,
3899         .ndo_start_xmit = stmmac_xmit,
3900         .ndo_stop = stmmac_release,
3901         .ndo_change_mtu = stmmac_change_mtu,
3902         .ndo_fix_features = stmmac_fix_features,
3903         .ndo_set_features = stmmac_set_features,
3904         .ndo_set_rx_mode = stmmac_set_rx_mode,
3905         .ndo_tx_timeout = stmmac_tx_timeout,
3906         .ndo_do_ioctl = stmmac_ioctl,
3907 #ifdef CONFIG_NET_POLL_CONTROLLER
3908         .ndo_poll_controller = stmmac_poll_controller,
3909 #endif
3910         .ndo_set_mac_address = eth_mac_addr,
3911 };
3912
3913 /**
3914  *  stmmac_hw_init - Init the MAC device
3915  *  @priv: driver private structure
3916  *  Description: this function is to configure the MAC device according to
3917  *  some platform parameters or the HW capability register. It prepares the
3918  *  driver to use either ring or chain modes and to setup either enhanced or
3919  *  normal descriptors.
3920  */
3921 static int stmmac_hw_init(struct stmmac_priv *priv)
3922 {
3923         struct mac_device_info *mac;
3924
3925         /* Identify the MAC HW device */
3926         if (priv->plat->has_gmac) {
3927                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
3928                 mac = dwmac1000_setup(priv->ioaddr,
3929                                       priv->plat->multicast_filter_bins,
3930                                       priv->plat->unicast_filter_entries,
3931                                       &priv->synopsys_id);
3932         } else if (priv->plat->has_gmac4) {
3933                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
3934                 mac = dwmac4_setup(priv->ioaddr,
3935                                    priv->plat->multicast_filter_bins,
3936                                    priv->plat->unicast_filter_entries,
3937                                    &priv->synopsys_id);
3938         } else {
3939                 mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id);
3940         }
3941         if (!mac)
3942                 return -ENOMEM;
3943
3944         priv->hw = mac;
3945
3946         /* To use the chained or ring mode */
3947         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3948                 priv->hw->mode = &dwmac4_ring_mode_ops;
3949         } else {
3950                 if (chain_mode) {
3951                         priv->hw->mode = &chain_mode_ops;
3952                         dev_info(priv->device, "Chain mode enabled\n");
3953                         priv->mode = STMMAC_CHAIN_MODE;
3954                 } else {
3955                         priv->hw->mode = &ring_mode_ops;
3956                         dev_info(priv->device, "Ring mode enabled\n");
3957                         priv->mode = STMMAC_RING_MODE;
3958                 }
3959         }
3960
3961         /* Get the HW capability (new GMAC newer than 3.50a) */
3962         priv->hw_cap_support = stmmac_get_hw_features(priv);
3963         if (priv->hw_cap_support) {
3964                 dev_info(priv->device, "DMA HW capability register supported\n");
3965
3966                 /* We can override some gmac/dma configuration fields: e.g.
3967                  * enh_desc, tx_coe (e.g. that are passed through the
3968                  * platform) with the values from the HW capability
3969                  * register (if supported).
3970                  */
3971                 priv->plat->enh_desc = priv->dma_cap.enh_desc;
3972                 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
3973                 priv->hw->pmt = priv->plat->pmt;
3974
3975                 /* TXCOE doesn't work in thresh DMA mode */
3976                 if (priv->plat->force_thresh_dma_mode)
3977                         priv->plat->tx_coe = 0;
3978                 else
3979                         priv->plat->tx_coe = priv->dma_cap.tx_coe;
3980
3981                 /* In case of GMAC4 rx_coe is from HW cap register. */
3982                 priv->plat->rx_coe = priv->dma_cap.rx_coe;
3983
3984                 if (priv->dma_cap.rx_coe_type2)
3985                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
3986                 else if (priv->dma_cap.rx_coe_type1)
3987                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
3988
3989         } else {
3990                 dev_info(priv->device, "No HW DMA feature register supported\n");
3991         }
3992
3993         /* To use alternate (extended), normal or GMAC4 descriptor structures */
3994         if (priv->synopsys_id >= DWMAC_CORE_4_00)
3995                 priv->hw->desc = &dwmac4_desc_ops;
3996         else
3997                 stmmac_selec_desc_mode(priv);
3998
3999         if (priv->plat->rx_coe) {
4000                 priv->hw->rx_csum = priv->plat->rx_coe;
4001                 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
4002                 if (priv->synopsys_id < DWMAC_CORE_4_00)
4003                         dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
4004         }
4005         if (priv->plat->tx_coe)
4006                 dev_info(priv->device, "TX Checksum insertion supported\n");
4007
4008         if (priv->plat->pmt) {
4009                 dev_info(priv->device, "Wake-Up On Lan supported\n");
4010                 device_set_wakeup_capable(priv->device, 1);
4011         }
4012
4013         if (priv->dma_cap.tsoen)
4014                 dev_info(priv->device, "TSO supported\n");
4015
4016         return 0;
4017 }
4018
4019 /**
4020  * stmmac_dvr_probe
4021  * @device: device pointer
4022  * @plat_dat: platform data pointer
4023  * @res: stmmac resource pointer
4024  * Description: this is the main probe function used to
4025  * call the alloc_etherdev, allocate the priv structure.
4026  * Return:
4027  * returns 0 on success, otherwise errno.
4028  */
4029 int stmmac_dvr_probe(struct device *device,
4030                      struct plat_stmmacenet_data *plat_dat,
4031                      struct stmmac_resources *res)
4032 {
4033         struct net_device *ndev = NULL;
4034         struct stmmac_priv *priv;
4035         int ret = 0;
4036         u32 queue;
4037
4038         ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv),
4039                                   MTL_MAX_TX_QUEUES,
4040                                   MTL_MAX_RX_QUEUES);
4041         if (!ndev)
4042                 return -ENOMEM;
4043
4044         SET_NETDEV_DEV(ndev, device);
4045
4046         priv = netdev_priv(ndev);
4047         priv->device = device;
4048         priv->dev = ndev;
4049
4050         stmmac_set_ethtool_ops(ndev);
4051         priv->pause = pause;
4052         priv->plat = plat_dat;
4053         priv->ioaddr = res->addr;
4054         priv->dev->base_addr = (unsigned long)res->addr;
4055
4056         priv->dev->irq = res->irq;
4057         priv->wol_irq = res->wol_irq;
4058         priv->lpi_irq = res->lpi_irq;
4059
4060         if (res->mac)
4061                 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
4062
4063         dev_set_drvdata(device, priv->dev);
4064
4065         /* Verify driver arguments */
4066         stmmac_verify_args();
4067
4068         /* Override with kernel parameters if supplied XXX CRS XXX
4069          * this needs to have multiple instances
4070          */
4071         if ((phyaddr >= 0) && (phyaddr <= 31))
4072                 priv->plat->phy_addr = phyaddr;
4073
4074         if (priv->plat->stmmac_rst)
4075                 reset_control_deassert(priv->plat->stmmac_rst);
4076
4077         /* Init MAC and get the capabilities */
4078         ret = stmmac_hw_init(priv);
4079         if (ret)
4080                 goto error_hw_init;
4081
4082         /* Configure real RX and TX queues */
4083         netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use);
4084         netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
4085
4086         priv->dma_buf_sz = STMMAC_ALIGN(buf_sz);
4087
4088         ndev->netdev_ops = &stmmac_netdev_ops;
4089
4090         ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4091                             NETIF_F_RXCSUM;
4092
4093         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4094                 ndev->hw_features |= NETIF_F_TSO;
4095                 priv->tso = true;
4096                 dev_info(priv->device, "TSO feature enabled\n");
4097         }
4098         ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4099         ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4100 #ifdef STMMAC_VLAN_TAG_USED
4101         /* Both mac100 and gmac support receive VLAN tag detection */
4102         ndev->features |= NETIF_F_HW_VLAN_CTAG_RX;
4103 #endif
4104         priv->msg_enable = netif_msg_init(debug, default_msg_level);
4105
4106         /* MTU range: 46 - hw-specific max */
4107         ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
4108         if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
4109                 ndev->max_mtu = JUMBO_LEN;
4110         else
4111                 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
4112         /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
4113          * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
4114          */
4115         if ((priv->plat->maxmtu < ndev->max_mtu) &&
4116             (priv->plat->maxmtu >= ndev->min_mtu))
4117                 ndev->max_mtu = priv->plat->maxmtu;
4118         else if (priv->plat->maxmtu < ndev->min_mtu)
4119                 dev_warn(priv->device,
4120                          "%s: warning: maxmtu having invalid value (%d)\n",
4121                          __func__, priv->plat->maxmtu);
4122
4123         if (flow_ctrl)
4124                 priv->flow_ctrl = FLOW_AUTO;    /* RX/TX pause on */
4125
4126         /* Rx Watchdog is available in the COREs newer than the 3.40.
4127          * In some case, for example on bugged HW this feature
4128          * has to be disable and this can be done by passing the
4129          * riwt_off field from the platform.
4130          */
4131         if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) {
4132                 priv->use_riwt = 1;
4133                 dev_info(priv->device,
4134                          "Enable RX Mitigation via HW Watchdog Timer\n");
4135         }
4136
4137         ret = alloc_dma_desc_resources(priv);
4138         if (ret < 0) {
4139                 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
4140                            __func__);
4141                 goto init_dma_error;
4142         }
4143
4144         ret = init_dma_desc_rings(priv->dev, GFP_KERNEL);
4145         if (ret < 0) {
4146                 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
4147                            __func__);
4148                 goto init_dma_error;
4149         }
4150
4151         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4152                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4153
4154                 netif_napi_add(ndev, &rx_q->napi, stmmac_poll,
4155                                (64 * priv->plat->rx_queues_to_use));
4156         }
4157
4158         spin_lock_init(&priv->lock);
4159
4160         /* If a specific clk_csr value is passed from the platform
4161          * this means that the CSR Clock Range selection cannot be
4162          * changed at run-time and it is fixed. Viceversa the driver'll try to
4163          * set the MDC clock dynamically according to the csr actual
4164          * clock input.
4165          */
4166         if (!priv->plat->clk_csr)
4167                 stmmac_clk_csr_set(priv);
4168         else
4169                 priv->clk_csr = priv->plat->clk_csr;
4170
4171         stmmac_check_pcs_mode(priv);
4172
4173         if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
4174             priv->hw->pcs != STMMAC_PCS_TBI &&
4175             priv->hw->pcs != STMMAC_PCS_RTBI) {
4176                 /* MDIO bus Registration */
4177                 ret = stmmac_mdio_register(ndev);
4178                 if (ret < 0) {
4179                         dev_err(priv->device,
4180                                 "%s: MDIO bus (id: %d) registration failed",
4181                                 __func__, priv->plat->bus_id);
4182                         goto error_mdio_register;
4183                 }
4184         }
4185
4186         ret = register_netdev(ndev);
4187         if (ret) {
4188                 dev_err(priv->device, "%s: ERROR %i registering the device\n",
4189                         __func__, ret);
4190                 goto error_netdev_register;
4191         }
4192
4193         return ret;
4194
4195 error_netdev_register:
4196         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4197             priv->hw->pcs != STMMAC_PCS_TBI &&
4198             priv->hw->pcs != STMMAC_PCS_RTBI)
4199                 stmmac_mdio_unregister(ndev);
4200 error_mdio_register:
4201         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4202                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4203
4204                 netif_napi_del(&rx_q->napi);
4205         }
4206 init_dma_error:
4207         free_dma_desc_resources(priv);
4208 error_hw_init:
4209         free_netdev(ndev);
4210
4211         return ret;
4212 }
4213 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
4214
4215 /**
4216  * stmmac_dvr_remove
4217  * @dev: device pointer
4218  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
4219  * changes the link status, releases the DMA descriptor rings.
4220  */
4221 int stmmac_dvr_remove(struct device *dev)
4222 {
4223         struct net_device *ndev = dev_get_drvdata(dev);
4224         struct stmmac_priv *priv = netdev_priv(ndev);
4225
4226         netdev_info(priv->dev, "%s: removing driver", __func__);
4227
4228         stmmac_stop_all_dma(priv);
4229
4230         priv->hw->mac->set_mac(priv->ioaddr, false);
4231         netif_carrier_off(ndev);
4232         unregister_netdev(ndev);
4233         if (priv->plat->stmmac_rst)
4234                 reset_control_assert(priv->plat->stmmac_rst);
4235         clk_disable_unprepare(priv->plat->pclk);
4236         clk_disable_unprepare(priv->plat->stmmac_clk);
4237         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4238             priv->hw->pcs != STMMAC_PCS_TBI &&
4239             priv->hw->pcs != STMMAC_PCS_RTBI)
4240                 stmmac_mdio_unregister(ndev);
4241         free_netdev(ndev);
4242
4243         return 0;
4244 }
4245 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
4246
4247 /**
4248  * stmmac_suspend - suspend callback
4249  * @dev: device pointer
4250  * Description: this is the function to suspend the device and it is called
4251  * by the platform driver to stop the network queue, release the resources,
4252  * program the PMT register (for WoL), clean and release driver resources.
4253  */
4254 int stmmac_suspend(struct device *dev)
4255 {
4256         struct net_device *ndev = dev_get_drvdata(dev);
4257         struct stmmac_priv *priv = netdev_priv(ndev);
4258         unsigned long flags;
4259
4260         if (!ndev || !netif_running(ndev))
4261                 return 0;
4262
4263         if (ndev->phydev)
4264                 phy_stop(ndev->phydev);
4265
4266         spin_lock_irqsave(&priv->lock, flags);
4267
4268         netif_device_detach(ndev);
4269         stmmac_stop_all_queues(priv);
4270
4271         stmmac_disable_all_queues(priv);
4272
4273         /* Stop TX/RX DMA */
4274         stmmac_stop_all_dma(priv);
4275
4276         /* Enable Power down mode by programming the PMT regs */
4277         if (device_may_wakeup(priv->device)) {
4278                 priv->hw->mac->pmt(priv->hw, priv->wolopts);
4279                 priv->irq_wake = 1;
4280         } else {
4281                 priv->hw->mac->set_mac(priv->ioaddr, false);
4282                 pinctrl_pm_select_sleep_state(priv->device);
4283                 /* Disable clock in case of PWM is off */
4284                 clk_disable(priv->plat->pclk);
4285                 clk_disable(priv->plat->stmmac_clk);
4286         }
4287         spin_unlock_irqrestore(&priv->lock, flags);
4288
4289         priv->oldlink = 0;
4290         priv->speed = SPEED_UNKNOWN;
4291         priv->oldduplex = DUPLEX_UNKNOWN;
4292         return 0;
4293 }
4294 EXPORT_SYMBOL_GPL(stmmac_suspend);
4295
4296 /**
4297  * stmmac_reset_queues_param - reset queue parameters
4298  * @dev: device pointer
4299  */
4300 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
4301 {
4302         u32 rx_cnt = priv->plat->rx_queues_to_use;
4303         u32 tx_cnt = priv->plat->tx_queues_to_use;
4304         u32 queue;
4305
4306         for (queue = 0; queue < rx_cnt; queue++) {
4307                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4308
4309                 rx_q->cur_rx = 0;
4310                 rx_q->dirty_rx = 0;
4311         }
4312
4313         for (queue = 0; queue < tx_cnt; queue++) {
4314                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4315
4316                 tx_q->cur_tx = 0;
4317                 tx_q->dirty_tx = 0;
4318         }
4319 }
4320
4321 /**
4322  * stmmac_resume - resume callback
4323  * @dev: device pointer
4324  * Description: when resume this function is invoked to setup the DMA and CORE
4325  * in a usable state.
4326  */
4327 int stmmac_resume(struct device *dev)
4328 {
4329         struct net_device *ndev = dev_get_drvdata(dev);
4330         struct stmmac_priv *priv = netdev_priv(ndev);
4331         unsigned long flags;
4332
4333         if (!netif_running(ndev))
4334                 return 0;
4335
4336         /* Power Down bit, into the PM register, is cleared
4337          * automatically as soon as a magic packet or a Wake-up frame
4338          * is received. Anyway, it's better to manually clear
4339          * this bit because it can generate problems while resuming
4340          * from another devices (e.g. serial console).
4341          */
4342         if (device_may_wakeup(priv->device)) {
4343                 spin_lock_irqsave(&priv->lock, flags);
4344                 priv->hw->mac->pmt(priv->hw, 0);
4345                 spin_unlock_irqrestore(&priv->lock, flags);
4346                 priv->irq_wake = 0;
4347         } else {
4348                 pinctrl_pm_select_default_state(priv->device);
4349                 /* enable the clk previously disabled */
4350                 clk_enable(priv->plat->stmmac_clk);
4351                 clk_enable(priv->plat->pclk);
4352                 /* reset the phy so that it's ready */
4353                 if (priv->mii)
4354                         stmmac_mdio_reset(priv->mii);
4355         }
4356
4357         netif_device_attach(ndev);
4358
4359         spin_lock_irqsave(&priv->lock, flags);
4360
4361         stmmac_reset_queues_param(priv);
4362
4363         /* reset private mss value to force mss context settings at
4364          * next tso xmit (only used for gmac4).
4365          */
4366         priv->mss = 0;
4367
4368         stmmac_clear_descriptors(priv);
4369
4370         stmmac_hw_setup(ndev, false);
4371         stmmac_init_tx_coalesce(priv);
4372         stmmac_set_rx_mode(ndev);
4373
4374         stmmac_enable_all_queues(priv);
4375
4376         stmmac_start_all_queues(priv);
4377
4378         spin_unlock_irqrestore(&priv->lock, flags);
4379
4380         if (ndev->phydev)
4381                 phy_start(ndev->phydev);
4382
4383         return 0;
4384 }
4385 EXPORT_SYMBOL_GPL(stmmac_resume);
4386
4387 #ifndef MODULE
4388 static int __init stmmac_cmdline_opt(char *str)
4389 {
4390         char *opt;
4391
4392         if (!str || !*str)
4393                 return -EINVAL;
4394         while ((opt = strsep(&str, ",")) != NULL) {
4395                 if (!strncmp(opt, "debug:", 6)) {
4396                         if (kstrtoint(opt + 6, 0, &debug))
4397                                 goto err;
4398                 } else if (!strncmp(opt, "phyaddr:", 8)) {
4399                         if (kstrtoint(opt + 8, 0, &phyaddr))
4400                                 goto err;
4401                 } else if (!strncmp(opt, "buf_sz:", 7)) {
4402                         if (kstrtoint(opt + 7, 0, &buf_sz))
4403                                 goto err;
4404                 } else if (!strncmp(opt, "tc:", 3)) {
4405                         if (kstrtoint(opt + 3, 0, &tc))
4406                                 goto err;
4407                 } else if (!strncmp(opt, "watchdog:", 9)) {
4408                         if (kstrtoint(opt + 9, 0, &watchdog))
4409                                 goto err;
4410                 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
4411                         if (kstrtoint(opt + 10, 0, &flow_ctrl))
4412                                 goto err;
4413                 } else if (!strncmp(opt, "pause:", 6)) {
4414                         if (kstrtoint(opt + 6, 0, &pause))
4415                                 goto err;
4416                 } else if (!strncmp(opt, "eee_timer:", 10)) {
4417                         if (kstrtoint(opt + 10, 0, &eee_timer))
4418                                 goto err;
4419                 } else if (!strncmp(opt, "chain_mode:", 11)) {
4420                         if (kstrtoint(opt + 11, 0, &chain_mode))
4421                                 goto err;
4422                 }
4423         }
4424         return 0;
4425
4426 err:
4427         pr_err("%s: ERROR broken module parameter conversion", __func__);
4428         return -EINVAL;
4429 }
4430
4431 __setup("stmmaceth=", stmmac_cmdline_opt);
4432 #endif /* MODULE */
4433
4434 static int __init stmmac_init(void)
4435 {
4436 #ifdef CONFIG_DEBUG_FS
4437         /* Create debugfs main directory if it doesn't exist yet */
4438         if (!stmmac_fs_dir) {
4439                 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
4440
4441                 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) {
4442                         pr_err("ERROR %s, debugfs create directory failed\n",
4443                                STMMAC_RESOURCE_NAME);
4444
4445                         return -ENOMEM;
4446                 }
4447         }
4448 #endif
4449
4450         return 0;
4451 }
4452
4453 static void __exit stmmac_exit(void)
4454 {
4455 #ifdef CONFIG_DEBUG_FS
4456         debugfs_remove_recursive(stmmac_fs_dir);
4457 #endif
4458 }
4459
4460 module_init(stmmac_init)
4461 module_exit(stmmac_exit)
4462
4463 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
4464 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
4465 MODULE_LICENSE("GPL");