drivers/net/ethernet/stmicro/stmmac/stmmac_main.c

   1 /*******************************************************************************
   2   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
   3   ST Ethernet IPs are built around a Synopsys IP Core.
   4
   5         Copyright(C) 2007-2011 STMicroelectronics Ltd
   6
   7   This program is free software; you can redistribute it and/or modify it
   8   under the terms and conditions of the GNU General Public License,
   9   version 2, as published by the Free Software Foundation.
  10
  11   This program is distributed in the hope it will be useful, but WITHOUT
  12   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14   more details.
  15
  16   The full GNU General Public License is included in this distribution in
  17   the file called "COPYING".
  18
  19   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
  20
  21   Documentation available at:
  22         http://www.stlinux.com
  23   Support available at:
  24         https://bugzilla.stlinux.com/
  25 *******************************************************************************/
  26
  27 #include <linux/clk.h>
  28 #include <linux/kernel.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/ip.h>
  31 #include <linux/tcp.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/if_ether.h>
  35 #include <linux/crc32.h>
  36 #include <linux/mii.h>
  37 #include <linux/if.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/slab.h>
  41 #include <linux/prefetch.h>
  42 #include <linux/pinctrl/consumer.h>
  43 #ifdef CONFIG_DEBUG_FS
  44 #include <linux/debugfs.h>
  45 #include <linux/seq_file.h>
  46 #endif /* CONFIG_DEBUG_FS */
  47 #include <linux/net_tstamp.h>
  48 #include "stmmac_ptp.h"
  49 #include "stmmac.h"
  50 #include <linux/reset.h>
  51 #include <linux/of_mdio.h>
  52 #include "dwmac1000.h"
  53
  54 #define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x)
  55 #define TSO_MAX_BUFF_SIZE       (SZ_16K - 1)
  56
  57 /* Module parameters */
  58 #define TX_TIMEO        5000
  59 static int watchdog = TX_TIMEO;
  60 module_param(watchdog, int, S_IRUGO | S_IWUSR);
  61 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
  62
  63 static int debug = -1;
  64 module_param(debug, int, S_IRUGO | S_IWUSR);
  65 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
  66
  67 static int phyaddr = -1;
  68 module_param(phyaddr, int, S_IRUGO);
  69 MODULE_PARM_DESC(phyaddr, "Physical device address");
  70
  71 #define STMMAC_TX_THRESH        (DMA_TX_SIZE / 4)
  72 #define STMMAC_RX_THRESH        (DMA_RX_SIZE / 4)
  73
  74 static int flow_ctrl = FLOW_OFF;
  75 module_param(flow_ctrl, int, S_IRUGO | S_IWUSR);
  76 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
  77
  78 static int pause = PAUSE_TIME;
  79 module_param(pause, int, S_IRUGO | S_IWUSR);
  80 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
  81
  82 #define TC_DEFAULT 64
  83 static int tc = TC_DEFAULT;
  84 module_param(tc, int, S_IRUGO | S_IWUSR);
  85 MODULE_PARM_DESC(tc, "DMA threshold control value");
  86
  87 #define DEFAULT_BUFSIZE 1536
  88 static int buf_sz = DEFAULT_BUFSIZE;
  89 module_param(buf_sz, int, S_IRUGO | S_IWUSR);
  90 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
  91
  92 #define STMMAC_RX_COPYBREAK     256
  93
  94 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
  95                                       NETIF_MSG_LINK | NETIF_MSG_IFUP |
  96                                       NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
  97
  98 #define STMMAC_DEFAULT_LPI_TIMER        1000
  99 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 100 module_param(eee_timer, int, S_IRUGO | S_IWUSR);
 101 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
 102 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
 103
 104 /* By default the driver will use the ring mode to manage tx and rx descriptors,
 105  * but allow user to force to use the chain instead of the ring
 106  */
 107 static unsigned int chain_mode;
 108 module_param(chain_mode, int, S_IRUGO);
 109 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
 110
 111 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
 112
 113 #ifdef CONFIG_DEBUG_FS
 114 static int stmmac_init_fs(struct net_device *dev);
 115 static void stmmac_exit_fs(struct net_device *dev);
 116 #endif
 117
 118 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
 119
 120 /**
 121  * stmmac_verify_args - verify the driver parameters.
 122  * Description: it checks the driver parameters and set a default in case of
 123  * errors.
 124  */
 125 static void stmmac_verify_args(void)
 126 {
 127         if (unlikely(watchdog < 0))
 128                 watchdog = TX_TIMEO;
 129         if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
 130                 buf_sz = DEFAULT_BUFSIZE;
 131         if (unlikely(flow_ctrl > 1))
 132                 flow_ctrl = FLOW_AUTO;
 133         else if (likely(flow_ctrl < 0))
 134                 flow_ctrl = FLOW_OFF;
 135         if (unlikely((pause < 0) || (pause > 0xffff)))
 136                 pause = PAUSE_TIME;
 137         if (eee_timer < 0)
 138                 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 139 }
 140
 141 /**
 142  * stmmac_disable_all_queues - Disable all queues
 143  * @priv: driver private structure
 144  */
 145 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
 146 {
 147         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 148         u32 queue;
 149
 150         for (queue = 0; queue < rx_queues_cnt; queue++) {
 151                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 152
 153                 napi_disable(&rx_q->napi);
 154         }
 155 }
 156
 157 /**
 158  * stmmac_enable_all_queues - Enable all queues
 159  * @priv: driver private structure
 160  */
 161 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
 162 {
 163         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 164         u32 queue;
 165
 166         for (queue = 0; queue < rx_queues_cnt; queue++) {
 167                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 168
 169                 napi_enable(&rx_q->napi);
 170         }
 171 }
 172
 173 /**
 174  * stmmac_stop_all_queues - Stop all queues
 175  * @priv: driver private structure
 176  */
 177 static void stmmac_stop_all_queues(struct stmmac_priv *priv)
 178 {
 179         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 180         u32 queue;
 181
 182         for (queue = 0; queue < tx_queues_cnt; queue++)
 183                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
 184 }
 185
 186 /**
 187  * stmmac_start_all_queues - Start all queues
 188  * @priv: driver private structure
 189  */
 190 static void stmmac_start_all_queues(struct stmmac_priv *priv)
 191 {
 192         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 193         u32 queue;
 194
 195         for (queue = 0; queue < tx_queues_cnt; queue++)
 196                 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
 197 }
 198
 199 /**
 200  * stmmac_clk_csr_set - dynamically set the MDC clock
 201  * @priv: driver private structure
 202  * Description: this is to dynamically set the MDC clock according to the csr
 203  * clock input.
 204  * Note:
 205  *      If a specific clk_csr value is passed from the platform
 206  *      this means that the CSR Clock Range selection cannot be
 207  *      changed at run-time and it is fixed (as reported in the driver
 208  *      documentation). Viceversa the driver will try to set the MDC
 209  *      clock dynamically according to the actual clock input.
 210  */
 211 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
 212 {
 213         u32 clk_rate;
 214
 215         clk_rate = clk_get_rate(priv->plat->stmmac_clk);
 216
 217         /* Platform provided default clk_csr would be assumed valid
 218          * for all other cases except for the below mentioned ones.
 219          * For values higher than the IEEE 802.3 specified frequency
 220          * we can not estimate the proper divider as it is not known
 221          * the frequency of clk_csr_i. So we do not change the default
 222          * divider.
 223          */
 224         if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
 225                 if (clk_rate < CSR_F_35M)
 226                         priv->clk_csr = STMMAC_CSR_20_35M;
 227                 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
 228                         priv->clk_csr = STMMAC_CSR_35_60M;
 229                 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
 230                         priv->clk_csr = STMMAC_CSR_60_100M;
 231                 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
 232                         priv->clk_csr = STMMAC_CSR_100_150M;
 233                 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
 234                         priv->clk_csr = STMMAC_CSR_150_250M;
 235                 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
 236                         priv->clk_csr = STMMAC_CSR_250_300M;
 237         }
 238
 239         if (priv->plat->has_sun8i) {
 240                 if (clk_rate > 160000000)
 241                         priv->clk_csr = 0x03;
 242                 else if (clk_rate > 80000000)
 243                         priv->clk_csr = 0x02;
 244                 else if (clk_rate > 40000000)
 245                         priv->clk_csr = 0x01;
 246                 else
 247                         priv->clk_csr = 0;
 248         }
 249 }
 250
 251 static void print_pkt(unsigned char *buf, int len)
 252 {
 253         pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
 254         print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
 255 }
 256
 257 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
 258 {
 259         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 260         u32 avail;
 261
 262         if (tx_q->dirty_tx > tx_q->cur_tx)
 263                 avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
 264         else
 265                 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
 266
 267         return avail;
 268 }
 269
 270 /**
 271  * stmmac_rx_dirty - Get RX queue dirty
 272  * @priv: driver private structure
 273  * @queue: RX queue index
 274  */
 275 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
 276 {
 277         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 278         u32 dirty;
 279
 280         if (rx_q->dirty_rx <= rx_q->cur_rx)
 281                 dirty = rx_q->cur_rx - rx_q->dirty_rx;
 282         else
 283                 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
 284
 285         return dirty;
 286 }
 287
 288 /**
 289  * stmmac_hw_fix_mac_speed - callback for speed selection
 290  * @priv: driver private structure
 291  * Description: on some platforms (e.g. ST), some HW system configuration
 292  * registers have to be set according to the link speed negotiated.
 293  */
 294 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv)
 295 {
 296         struct net_device *ndev = priv->dev;
 297         struct phy_device *phydev = ndev->phydev;
 298
 299         if (likely(priv->plat->fix_mac_speed))
 300                 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed);
 301 }
 302
 303 /**
 304  * stmmac_enable_eee_mode - check and enter in LPI mode
 305  * @priv: driver private structure
 306  * Description: this function is to verify and enter in LPI mode in case of
 307  * EEE.
 308  */
 309 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
 310 {
 311         u32 tx_cnt = priv->plat->tx_queues_to_use;
 312         u32 queue;
 313
 314         /* check if all TX queues have the work finished */
 315         for (queue = 0; queue < tx_cnt; queue++) {
 316                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 317
 318                 if (tx_q->dirty_tx != tx_q->cur_tx)
 319                         return; /* still unfinished work */
 320         }
 321
 322         /* Check and enter in LPI mode */
 323         if (!priv->tx_path_in_lpi_mode)
 324                 priv->hw->mac->set_eee_mode(priv->hw,
 325                                             priv->plat->en_tx_lpi_clockgating);
 326 }
 327
 328 /**
 329  * stmmac_disable_eee_mode - disable and exit from LPI mode
 330  * @priv: driver private structure
 331  * Description: this function is to exit and disable EEE in case of
 332  * LPI state is true. This is called by the xmit.
 333  */
 334 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
 335 {
 336         priv->hw->mac->reset_eee_mode(priv->hw);
 337         del_timer_sync(&priv->eee_ctrl_timer);
 338         priv->tx_path_in_lpi_mode = false;
 339 }
 340
 341 /**
 342  * stmmac_eee_ctrl_timer - EEE TX SW timer.
 343  * @arg : data hook
 344  * Description:
 345  *  if there is no data transfer and if we are not in LPI state,
 346  *  then MAC Transmitter can be moved to LPI state.
 347  */
 348 static void stmmac_eee_ctrl_timer(unsigned long arg)
 349 {
 350         struct stmmac_priv *priv = (struct stmmac_priv *)arg;
 351
 352         stmmac_enable_eee_mode(priv);
 353         mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
 354 }
 355
 356 /**
 357  * stmmac_eee_init - init EEE
 358  * @priv: driver private structure
 359  * Description:
 360  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
 361  *  can also manage EEE, this function enable the LPI state and start related
 362  *  timer.
 363  */
 364 bool stmmac_eee_init(struct stmmac_priv *priv)
 365 {
 366         struct net_device *ndev = priv->dev;
 367         unsigned long flags;
 368         bool ret = false;
 369
 370         /* Using PCS we cannot dial with the phy registers at this stage
 371          * so we do not support extra feature like EEE.
 372          */
 373         if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
 374             (priv->hw->pcs == STMMAC_PCS_TBI) ||
 375             (priv->hw->pcs == STMMAC_PCS_RTBI))
 376                 goto out;
 377
 378         /* MAC core supports the EEE feature. */
 379         if (priv->dma_cap.eee) {
 380                 int tx_lpi_timer = priv->tx_lpi_timer;
 381
 382                 /* Check if the PHY supports EEE */
 383                 if (phy_init_eee(ndev->phydev, 1)) {
 384                         /* To manage at run-time if the EEE cannot be supported
 385                          * anymore (for example because the lp caps have been
 386                          * changed).
 387                          * In that case the driver disable own timers.
 388                          */
 389                         spin_lock_irqsave(&priv->lock, flags);
 390                         if (priv->eee_active) {
 391                                 netdev_dbg(priv->dev, "disable EEE\n");
 392                                 del_timer_sync(&priv->eee_ctrl_timer);
 393                                 priv->hw->mac->set_eee_timer(priv->hw, 0,
 394                                                              tx_lpi_timer);
 395                         }
 396                         priv->eee_active = 0;
 397                         spin_unlock_irqrestore(&priv->lock, flags);
 398                         goto out;
 399                 }
 400                 /* Activate the EEE and start timers */
 401                 spin_lock_irqsave(&priv->lock, flags);
 402                 if (!priv->eee_active) {
 403                         priv->eee_active = 1;
 404                         setup_timer(&priv->eee_ctrl_timer,
 405                                     stmmac_eee_ctrl_timer,
 406                                     (unsigned long)priv);
 407                         mod_timer(&priv->eee_ctrl_timer,
 408                                   STMMAC_LPI_T(eee_timer));
 409
 410                         priv->hw->mac->set_eee_timer(priv->hw,
 411                                                      STMMAC_DEFAULT_LIT_LS,
 412                                                      tx_lpi_timer);
 413                 }
 414                 /* Set HW EEE according to the speed */
 415                 priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link);
 416
 417                 ret = true;
 418                 spin_unlock_irqrestore(&priv->lock, flags);
 419
 420                 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
 421         }
 422 out:
 423         return ret;
 424 }
 425
 426 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
 427  * @priv: driver private structure
 428  * @p : descriptor pointer
 429  * @skb : the socket buffer
 430  * Description :
 431  * This function will read timestamp from the descriptor & pass it to stack.
 432  * and also perform some sanity checks.
 433  */
 434 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
 435                                    struct dma_desc *p, struct sk_buff *skb)
 436 {
 437         struct skb_shared_hwtstamps shhwtstamp;
 438         u64 ns;
 439
 440         if (!priv->hwts_tx_en)
 441                 return;
 442
 443         /* exit if skb doesn't support hw tstamp */
 444         if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
 445                 return;
 446
 447         /* check tx tstamp status */
 448         if (priv->hw->desc->get_tx_timestamp_status(p)) {
 449                 /* get the valid tstamp */
 450                 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 451
 452                 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 453                 shhwtstamp.hwtstamp = ns_to_ktime(ns);
 454
 455                 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
 456                 /* pass tstamp to stack */
 457                 skb_tstamp_tx(skb, &shhwtstamp);
 458         }
 459
 460         return;
 461 }
 462
 463 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
 464  * @priv: driver private structure
 465  * @p : descriptor pointer
 466  * @np : next descriptor pointer
 467  * @skb : the socket buffer
 468  * Description :
 469  * This function will read received packet's timestamp from the descriptor
 470  * and pass it to stack. It also perform some sanity checks.
 471  */
 472 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
 473                                    struct dma_desc *np, struct sk_buff *skb)
 474 {
 475         struct skb_shared_hwtstamps *shhwtstamp = NULL;
 476         u64 ns;
 477
 478         if (!priv->hwts_rx_en)
 479                 return;
 480
 481         /* Check if timestamp is available */
 482         if (priv->hw->desc->get_rx_timestamp_status(p, priv->adv_ts)) {
 483                 /* For GMAC4, the valid timestamp is from CTX next desc. */
 484                 if (priv->plat->has_gmac4)
 485                         ns = priv->hw->desc->get_timestamp(np, priv->adv_ts);
 486                 else
 487                         ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 488
 489                 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
 490                 shhwtstamp = skb_hwtstamps(skb);
 491                 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 492                 shhwtstamp->hwtstamp = ns_to_ktime(ns);
 493         } else  {
 494                 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
 495         }
 496 }
 497
 498 /**
 499  *  stmmac_hwtstamp_ioctl - control hardware timestamping.
 500  *  @dev: device pointer.
 501  *  @ifr: An IOCTL specific structure, that can contain a pointer to
 502  *  a proprietary structure used to pass information to the driver.
 503  *  Description:
 504  *  This function configures the MAC to enable/disable both outgoing(TX)
 505  *  and incoming(RX) packets time stamping based on user input.
 506  *  Return Value:
 507  *  0 on success and an appropriate -ve integer on failure.
 508  */
 509 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
 510 {
 511         struct stmmac_priv *priv = netdev_priv(dev);
 512         struct hwtstamp_config config;
 513         struct timespec64 now;
 514         u64 temp = 0;
 515         u32 ptp_v2 = 0;
 516         u32 tstamp_all = 0;
 517         u32 ptp_over_ipv4_udp = 0;
 518         u32 ptp_over_ipv6_udp = 0;
 519         u32 ptp_over_ethernet = 0;
 520         u32 snap_type_sel = 0;
 521         u32 ts_master_en = 0;
 522         u32 ts_event_en = 0;
 523         u32 value = 0;
 524         u32 sec_inc;
 525
 526         if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
 527                 netdev_alert(priv->dev, "No support for HW time stamping\n");
 528                 priv->hwts_tx_en = 0;
 529                 priv->hwts_rx_en = 0;
 530
 531                 return -EOPNOTSUPP;
 532         }
 533
 534         if (copy_from_user(&config, ifr->ifr_data,
 535                            sizeof(struct hwtstamp_config)))
 536                 return -EFAULT;
 537
 538         netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
 539                    __func__, config.flags, config.tx_type, config.rx_filter);
 540
 541         /* reserved for future extensions */
 542         if (config.flags)
 543                 return -EINVAL;
 544
 545         if (config.tx_type != HWTSTAMP_TX_OFF &&
 546             config.tx_type != HWTSTAMP_TX_ON)
 547                 return -ERANGE;
 548
 549         if (priv->adv_ts) {
 550                 switch (config.rx_filter) {
 551                 case HWTSTAMP_FILTER_NONE:
 552                         /* time stamp no incoming packet at all */
 553                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 554                         break;
 555
 556                 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 557                         /* PTP v1, UDP, any kind of event packet */
 558                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 559                         /* take time stamp for all event messages */
 560                         if (priv->plat->has_gmac4)
 561                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 562                         else
 563                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 564
 565                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 566                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 567                         break;
 568
 569                 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 570                         /* PTP v1, UDP, Sync packet */
 571                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
 572                         /* take time stamp for SYNC messages only */
 573                         ts_event_en = PTP_TCR_TSEVNTENA;
 574
 575                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 576                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 577                         break;
 578
 579                 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 580                         /* PTP v1, UDP, Delay_req packet */
 581                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
 582                         /* take time stamp for Delay_Req messages only */
 583                         ts_master_en = PTP_TCR_TSMSTRENA;
 584                         ts_event_en = PTP_TCR_TSEVNTENA;
 585
 586                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 587                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 588                         break;
 589
 590                 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 591                         /* PTP v2, UDP, any kind of event packet */
 592                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
 593                         ptp_v2 = PTP_TCR_TSVER2ENA;
 594                         /* take time stamp for all event messages */
 595                         if (priv->plat->has_gmac4)
 596                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 597                         else
 598                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 599
 600                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 601                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 602                         break;
 603
 604                 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 605                         /* PTP v2, UDP, Sync packet */
 606                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
 607                         ptp_v2 = PTP_TCR_TSVER2ENA;
 608                         /* take time stamp for SYNC messages only */
 609                         ts_event_en = PTP_TCR_TSEVNTENA;
 610
 611                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 612                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 613                         break;
 614
 615                 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 616                         /* PTP v2, UDP, Delay_req packet */
 617                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
 618                         ptp_v2 = PTP_TCR_TSVER2ENA;
 619                         /* take time stamp for Delay_Req messages only */
 620                         ts_master_en = PTP_TCR_TSMSTRENA;
 621                         ts_event_en = PTP_TCR_TSEVNTENA;
 622
 623                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 624                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 625                         break;
 626
 627                 case HWTSTAMP_FILTER_PTP_V2_EVENT:
 628                         /* PTP v2/802.AS1 any layer, any kind of event packet */
 629                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 630                         ptp_v2 = PTP_TCR_TSVER2ENA;
 631                         /* take time stamp for all event messages */
 632                         if (priv->plat->has_gmac4)
 633                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 634                         else
 635                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 636
 637                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 638                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 639                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 640                         break;
 641
 642                 case HWTSTAMP_FILTER_PTP_V2_SYNC:
 643                         /* PTP v2/802.AS1, any layer, Sync packet */
 644                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
 645                         ptp_v2 = PTP_TCR_TSVER2ENA;
 646                         /* take time stamp for SYNC messages only */
 647                         ts_event_en = PTP_TCR_TSEVNTENA;
 648
 649                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 650                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 651                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 652                         break;
 653
 654                 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 655                         /* PTP v2/802.AS1, any layer, Delay_req packet */
 656                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
 657                         ptp_v2 = PTP_TCR_TSVER2ENA;
 658                         /* take time stamp for Delay_Req messages only */
 659                         ts_master_en = PTP_TCR_TSMSTRENA;
 660                         ts_event_en = PTP_TCR_TSEVNTENA;
 661
 662                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 663                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 664                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 665                         break;
 666
 667                 case HWTSTAMP_FILTER_NTP_ALL:
 668                 case HWTSTAMP_FILTER_ALL:
 669                         /* time stamp any incoming packet */
 670                         config.rx_filter = HWTSTAMP_FILTER_ALL;
 671                         tstamp_all = PTP_TCR_TSENALL;
 672                         break;
 673
 674                 default:
 675                         return -ERANGE;
 676                 }
 677         } else {
 678                 switch (config.rx_filter) {
 679                 case HWTSTAMP_FILTER_NONE:
 680                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 681                         break;
 682                 default:
 683                         /* PTP v1, UDP, any kind of event packet */
 684                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 685                         break;
 686                 }
 687         }
 688         priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
 689         priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
 690
 691         if (!priv->hwts_tx_en && !priv->hwts_rx_en)
 692                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0);
 693         else {
 694                 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
 695                          tstamp_all | ptp_v2 | ptp_over_ethernet |
 696                          ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
 697                          ts_master_en | snap_type_sel);
 698                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value);
 699
 700                 /* program Sub Second Increment reg */
 701                 sec_inc = priv->hw->ptp->config_sub_second_increment(
 702                         priv->ptpaddr, priv->plat->clk_ptp_rate,
 703                         priv->plat->has_gmac4);
 704                 temp = div_u64(1000000000ULL, sec_inc);
 705
 706                 /* calculate default added value:
 707                  * formula is :
 708                  * addend = (2^32)/freq_div_ratio;
 709                  * where, freq_div_ratio = 1e9ns/sec_inc
 710                  */
 711                 temp = (u64)(temp << 32);
 712                 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
 713                 priv->hw->ptp->config_addend(priv->ptpaddr,
 714                                              priv->default_addend);
 715
 716                 /* initialize system time */
 717                 ktime_get_real_ts64(&now);
 718
 719                 /* lower 32 bits of tv_sec are safe until y2106 */
 720                 priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec,
 721                                             now.tv_nsec);
 722         }
 723
 724         return copy_to_user(ifr->ifr_data, &config,
 725                             sizeof(struct hwtstamp_config)) ? -EFAULT : 0;
 726 }
 727
 728 /**
 729  * stmmac_init_ptp - init PTP
 730  * @priv: driver private structure
 731  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
 732  * This is done by looking at the HW cap. register.
 733  * This function also registers the ptp driver.
 734  */
 735 static int stmmac_init_ptp(struct stmmac_priv *priv)
 736 {
 737         if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
 738                 return -EOPNOTSUPP;
 739
 740         priv->adv_ts = 0;
 741         /* Check if adv_ts can be enabled for dwmac 4.x core */
 742         if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp)
 743                 priv->adv_ts = 1;
 744         /* Dwmac 3.x core with extend_desc can support adv_ts */
 745         else if (priv->extend_desc && priv->dma_cap.atime_stamp)
 746                 priv->adv_ts = 1;
 747
 748         if (priv->dma_cap.time_stamp)
 749                 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
 750
 751         if (priv->adv_ts)
 752                 netdev_info(priv->dev,
 753                             "IEEE 1588-2008 Advanced Timestamp supported\n");
 754
 755         priv->hw->ptp = &stmmac_ptp;
 756         priv->hwts_tx_en = 0;
 757         priv->hwts_rx_en = 0;
 758
 759         stmmac_ptp_register(priv);
 760
 761         return 0;
 762 }
 763
 764 static void stmmac_release_ptp(struct stmmac_priv *priv)
 765 {
 766         if (priv->plat->clk_ptp_ref)
 767                 clk_disable_unprepare(priv->plat->clk_ptp_ref);
 768         stmmac_ptp_unregister(priv);
 769 }
 770
 771 /**
 772  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
 773  *  @priv: driver private structure
 774  *  Description: It is used for configuring the flow control in all queues
 775  */
 776 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
 777 {
 778         u32 tx_cnt = priv->plat->tx_queues_to_use;
 779
 780         priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl,
 781                                  priv->pause, tx_cnt);
 782 }
 783
 784 /**
 785  * stmmac_adjust_link - adjusts the link parameters
 786  * @dev: net device structure
 787  * Description: this is the helper called by the physical abstraction layer
 788  * drivers to communicate the phy link status. According the speed and duplex
 789  * this driver can invoke registered glue-logic as well.
 790  * It also invoke the eee initialization because it could happen when switch
 791  * on different networks (that are eee capable).
 792  */
 793 static void stmmac_adjust_link(struct net_device *dev)
 794 {
 795         struct stmmac_priv *priv = netdev_priv(dev);
 796         struct phy_device *phydev = dev->phydev;
 797         unsigned long flags;
 798         bool new_state = false;
 799
 800         if (!phydev)
 801                 return;
 802
 803         spin_lock_irqsave(&priv->lock, flags);
 804
 805         if (phydev->link) {
 806                 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
 807
 808                 /* Now we make sure that we can be in full duplex mode.
 809                  * If not, we operate in half-duplex mode. */
 810                 if (phydev->duplex != priv->oldduplex) {
 811                         new_state = true;
 812                         if (!phydev->duplex)
 813                                 ctrl &= ~priv->hw->link.duplex;
 814                         else
 815                                 ctrl |= priv->hw->link.duplex;
 816                         priv->oldduplex = phydev->duplex;
 817                 }
 818                 /* Flow Control operation */
 819                 if (phydev->pause)
 820                         stmmac_mac_flow_ctrl(priv, phydev->duplex);
 821
 822                 if (phydev->speed != priv->speed) {
 823                         new_state = true;
 824                         ctrl &= ~priv->hw->link.speed_mask;
 825                         switch (phydev->speed) {
 826                         case SPEED_1000:
 827                                 ctrl |= priv->hw->link.speed1000;
 828                                 break;
 829                         case SPEED_100:
 830                                 ctrl |= priv->hw->link.speed100;
 831                                 break;
 832                         case SPEED_10:
 833                                 ctrl |= priv->hw->link.speed10;
 834                                 break;
 835                         default:
 836                                 netif_warn(priv, link, priv->dev,
 837                                            "broken speed: %d\n", phydev->speed);
 838                                 phydev->speed = SPEED_UNKNOWN;
 839                                 break;
 840                         }
 841                         if (phydev->speed != SPEED_UNKNOWN)
 842                                 stmmac_hw_fix_mac_speed(priv);
 843                         priv->speed = phydev->speed;
 844                 }
 845
 846                 writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
 847
 848                 if (!priv->oldlink) {
 849                         new_state = true;
 850                         priv->oldlink = true;
 851                 }
 852         } else if (priv->oldlink) {
 853                 new_state = true;
 854                 priv->oldlink = false;
 855                 priv->speed = SPEED_UNKNOWN;
 856                 priv->oldduplex = DUPLEX_UNKNOWN;
 857         }
 858
 859         if (new_state && netif_msg_link(priv))
 860                 phy_print_status(phydev);
 861
 862         spin_unlock_irqrestore(&priv->lock, flags);
 863
 864         if (phydev->is_pseudo_fixed_link)
 865                 /* Stop PHY layer to call the hook to adjust the link in case
 866                  * of a switch is attached to the stmmac driver.
 867                  */
 868                 phydev->irq = PHY_IGNORE_INTERRUPT;
 869         else
 870                 /* At this stage, init the EEE if supported.
 871                  * Never called in case of fixed_link.
 872                  */
 873                 priv->eee_enabled = stmmac_eee_init(priv);
 874 }
 875
 876 /**
 877  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
 878  * @priv: driver private structure
 879  * Description: this is to verify if the HW supports the PCS.
 880  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
 881  * configured for the TBI, RTBI, or SGMII PHY interface.
 882  */
 883 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
 884 {
 885         int interface = priv->plat->interface;
 886
 887         if (priv->dma_cap.pcs) {
 888                 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
 889                     (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
 890                     (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
 891                     (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
 892                         netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
 893                         priv->hw->pcs = STMMAC_PCS_RGMII;
 894                 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
 895                         netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
 896                         priv->hw->pcs = STMMAC_PCS_SGMII;
 897                 }
 898         }
 899 }
 900
 901 /**
 902  * stmmac_init_phy - PHY initialization
 903  * @dev: net device structure
 904  * Description: it initializes the driver's PHY state, and attaches the PHY
 905  * to the mac driver.
 906  *  Return value:
 907  *  0 on success
 908  */
 909 static int stmmac_init_phy(struct net_device *dev)
 910 {
 911         struct stmmac_priv *priv = netdev_priv(dev);
 912         struct phy_device *phydev;
 913         char phy_id_fmt[MII_BUS_ID_SIZE + 3];
 914         char bus_id[MII_BUS_ID_SIZE];
 915         int interface = priv->plat->interface;
 916         int max_speed = priv->plat->max_speed;
 917         priv->oldlink = false;
 918         priv->speed = SPEED_UNKNOWN;
 919         priv->oldduplex = DUPLEX_UNKNOWN;
 920
 921         if (priv->plat->phy_node) {
 922                 phydev = of_phy_connect(dev, priv->plat->phy_node,
 923                                         &stmmac_adjust_link, 0, interface);
 924         } else {
 925                 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x",
 926                          priv->plat->bus_id);
 927
 928                 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
 929                          priv->plat->phy_addr);
 930                 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__,
 931                            phy_id_fmt);
 932
 933                 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link,
 934                                      interface);
 935         }
 936
 937         if (IS_ERR_OR_NULL(phydev)) {
 938                 netdev_err(priv->dev, "Could not attach to PHY\n");
 939                 if (!phydev)
 940                         return -ENODEV;
 941
 942                 return PTR_ERR(phydev);
 943         }
 944
 945         /* Stop Advertising 1000BASE Capability if interface is not GMII */
 946         if ((interface == PHY_INTERFACE_MODE_MII) ||
 947             (interface == PHY_INTERFACE_MODE_RMII) ||
 948                 (max_speed < 1000 && max_speed > 0))
 949                 phydev->advertising &= ~(SUPPORTED_1000baseT_Half |
 950                                          SUPPORTED_1000baseT_Full);
 951
 952         /*
 953          * Broken HW is sometimes missing the pull-up resistor on the
 954          * MDIO line, which results in reads to non-existent devices returning
 955          * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent
 956          * device as well.
 957          * Note: phydev->phy_id is the result of reading the UID PHY registers.
 958          */
 959         if (!priv->plat->phy_node && phydev->phy_id == 0) {
 960                 phy_disconnect(phydev);
 961                 return -ENODEV;
 962         }
 963
 964         /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
 965          * subsequent PHY polling, make sure we force a link transition if
 966          * we have a UP/DOWN/UP transition
 967          */
 968         if (phydev->is_pseudo_fixed_link)
 969                 phydev->irq = PHY_POLL;
 970
 971         phy_attached_info(phydev);
 972         return 0;
 973 }
 974
 975 static void stmmac_display_rx_rings(struct stmmac_priv *priv)
 976 {
 977         u32 rx_cnt = priv->plat->rx_queues_to_use;
 978         void *head_rx;
 979         u32 queue;
 980
 981         /* Display RX rings */
 982         for (queue = 0; queue < rx_cnt; queue++) {
 983                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 984
 985                 pr_info("\tRX Queue %u rings\n", queue);
 986
 987                 if (priv->extend_desc)
 988                         head_rx = (void *)rx_q->dma_erx;
 989                 else
 990                         head_rx = (void *)rx_q->dma_rx;
 991
 992                 /* Display RX ring */
 993                 priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true);
 994         }
 995 }
 996
 997 static void stmmac_display_tx_rings(struct stmmac_priv *priv)
 998 {
 999         u32 tx_cnt = priv->plat->tx_queues_to_use;
1000         void *head_tx;
1001         u32 queue;
1002
1003         /* Display TX rings */
1004         for (queue = 0; queue < tx_cnt; queue++) {
1005                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1006
1007                 pr_info("\tTX Queue %d rings\n", queue);
1008
1009                 if (priv->extend_desc)
1010                         head_tx = (void *)tx_q->dma_etx;
1011                 else
1012                         head_tx = (void *)tx_q->dma_tx;
1013
1014                 priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false);
1015         }
1016 }
1017
1018 static void stmmac_display_rings(struct stmmac_priv *priv)
1019 {
1020         /* Display RX ring */
1021         stmmac_display_rx_rings(priv);
1022
1023         /* Display TX ring */
1024         stmmac_display_tx_rings(priv);
1025 }
1026
1027 static int stmmac_set_bfsize(int mtu, int bufsize)
1028 {
1029         int ret = bufsize;
1030
1031         if (mtu >= BUF_SIZE_4KiB)
1032                 ret = BUF_SIZE_8KiB;
1033         else if (mtu >= BUF_SIZE_2KiB)
1034                 ret = BUF_SIZE_4KiB;
1035         else if (mtu > DEFAULT_BUFSIZE)
1036                 ret = BUF_SIZE_2KiB;
1037         else
1038                 ret = DEFAULT_BUFSIZE;
1039
1040         return ret;
1041 }
1042
1043 /**
1044  * stmmac_clear_rx_descriptors - clear RX descriptors
1045  * @priv: driver private structure
1046  * @queue: RX queue index
1047  * Description: this function is called to clear the RX descriptors
1048  * in case of both basic and extended descriptors are used.
1049  */
1050 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1051 {
1052         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1053         int i;
1054
1055         /* Clear the RX descriptors */
1056         for (i = 0; i < DMA_RX_SIZE; i++)
1057                 if (priv->extend_desc)
1058                         priv->hw->desc->init_rx_desc(&rx_q->dma_erx[i].basic,
1059                                                      priv->use_riwt, priv->mode,
1060                                                      (i == DMA_RX_SIZE - 1));
1061                 else
1062                         priv->hw->desc->init_rx_desc(&rx_q->dma_rx[i],
1063                                                      priv->use_riwt, priv->mode,
1064                                                      (i == DMA_RX_SIZE - 1));
1065 }
1066
1067 /**
1068  * stmmac_clear_tx_descriptors - clear tx descriptors
1069  * @priv: driver private structure
1070  * @queue: TX queue index.
1071  * Description: this function is called to clear the TX descriptors
1072  * in case of both basic and extended descriptors are used.
1073  */
1074 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1075 {
1076         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1077         int i;
1078
1079         /* Clear the TX descriptors */
1080         for (i = 0; i < DMA_TX_SIZE; i++)
1081                 if (priv->extend_desc)
1082                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1083                                                      priv->mode,
1084                                                      (i == DMA_TX_SIZE - 1));
1085                 else
1086                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1087                                                      priv->mode,
1088                                                      (i == DMA_TX_SIZE - 1));
1089 }
1090
1091 /**
1092  * stmmac_clear_descriptors - clear descriptors
1093  * @priv: driver private structure
1094  * Description: this function is called to clear the TX and RX descriptors
1095  * in case of both basic and extended descriptors are used.
1096  */
1097 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1098 {
1099         u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1100         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1101         u32 queue;
1102
1103         /* Clear the RX descriptors */
1104         for (queue = 0; queue < rx_queue_cnt; queue++)
1105                 stmmac_clear_rx_descriptors(priv, queue);
1106
1107         /* Clear the TX descriptors */
1108         for (queue = 0; queue < tx_queue_cnt; queue++)
1109                 stmmac_clear_tx_descriptors(priv, queue);
1110 }
1111
1112 /**
1113  * stmmac_init_rx_buffers - init the RX descriptor buffer.
1114  * @priv: driver private structure
1115  * @p: descriptor pointer
1116  * @i: descriptor index
1117  * @flags: gfp flag
1118  * @queue: RX queue index
1119  * Description: this function is called to allocate a receive buffer, perform
1120  * the DMA mapping and init the descriptor.
1121  */
1122 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1123                                   int i, gfp_t flags, u32 queue)
1124 {
1125         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1126         struct sk_buff *skb;
1127
1128         skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags);
1129         if (!skb) {
1130                 netdev_err(priv->dev,
1131                            "%s: Rx init fails; skb is NULL\n", __func__);
1132                 return -ENOMEM;
1133         }
1134         rx_q->rx_skbuff[i] = skb;
1135         rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
1136                                                 priv->dma_buf_sz,
1137                                                 DMA_FROM_DEVICE);
1138         if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) {
1139                 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__);
1140                 dev_kfree_skb_any(skb);
1141                 return -EINVAL;
1142         }
1143
1144         if (priv->synopsys_id >= DWMAC_CORE_4_00)
1145                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1146         else
1147                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1148
1149         if ((priv->hw->mode->init_desc3) &&
1150             (priv->dma_buf_sz == BUF_SIZE_16KiB))
1151                 priv->hw->mode->init_desc3(p);
1152
1153         return 0;
1154 }
1155
1156 /**
1157  * stmmac_free_rx_buffer - free RX dma buffers
1158  * @priv: private structure
1159  * @queue: RX queue index
1160  * @i: buffer index.
1161  */
1162 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1163 {
1164         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1165
1166         if (rx_q->rx_skbuff[i]) {
1167                 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i],
1168                                  priv->dma_buf_sz, DMA_FROM_DEVICE);
1169                 dev_kfree_skb_any(rx_q->rx_skbuff[i]);
1170         }
1171         rx_q->rx_skbuff[i] = NULL;
1172 }
1173
1174 /**
1175  * stmmac_free_tx_buffer - free RX dma buffers
1176  * @priv: private structure
1177  * @queue: RX queue index
1178  * @i: buffer index.
1179  */
1180 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1181 {
1182         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1183
1184         if (tx_q->tx_skbuff_dma[i].buf) {
1185                 if (tx_q->tx_skbuff_dma[i].map_as_page)
1186                         dma_unmap_page(priv->device,
1187                                        tx_q->tx_skbuff_dma[i].buf,
1188                                        tx_q->tx_skbuff_dma[i].len,
1189                                        DMA_TO_DEVICE);
1190                 else
1191                         dma_unmap_single(priv->device,
1192                                          tx_q->tx_skbuff_dma[i].buf,
1193                                          tx_q->tx_skbuff_dma[i].len,
1194                                          DMA_TO_DEVICE);
1195         }
1196
1197         if (tx_q->tx_skbuff[i]) {
1198                 dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1199                 tx_q->tx_skbuff[i] = NULL;
1200                 tx_q->tx_skbuff_dma[i].buf = 0;
1201                 tx_q->tx_skbuff_dma[i].map_as_page = false;
1202         }
1203 }
1204
1205 /**
1206  * init_dma_rx_desc_rings - init the RX descriptor rings
1207  * @dev: net device structure
1208  * @flags: gfp flag.
1209  * Description: this function initializes the DMA RX descriptors
1210  * and allocates the socket buffers. It supports the chained and ring
1211  * modes.
1212  */
1213 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1214 {
1215         struct stmmac_priv *priv = netdev_priv(dev);
1216         u32 rx_count = priv->plat->rx_queues_to_use;
1217         unsigned int bfsize = 0;
1218         int ret = -ENOMEM;
1219         int queue;
1220         int i;
1221
1222         if (priv->hw->mode->set_16kib_bfsize)
1223                 bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu);
1224
1225         if (bfsize < BUF_SIZE_16KiB)
1226                 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
1227
1228         priv->dma_buf_sz = bfsize;
1229
1230         /* RX INITIALIZATION */
1231         netif_dbg(priv, probe, priv->dev,
1232                   "SKB addresses:\nskb\t\tskb data\tdma data\n");
1233
1234         for (queue = 0; queue < rx_count; queue++) {
1235                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1236
1237                 netif_dbg(priv, probe, priv->dev,
1238                           "(%s) dma_rx_phy=0x%08x\n", __func__,
1239                           (u32)rx_q->dma_rx_phy);
1240
1241                 for (i = 0; i < DMA_RX_SIZE; i++) {
1242                         struct dma_desc *p;
1243
1244                         if (priv->extend_desc)
1245                                 p = &((rx_q->dma_erx + i)->basic);
1246                         else
1247                                 p = rx_q->dma_rx + i;
1248
1249                         ret = stmmac_init_rx_buffers(priv, p, i, flags,
1250                                                      queue);
1251                         if (ret)
1252                                 goto err_init_rx_buffers;
1253
1254                         netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n",
1255                                   rx_q->rx_skbuff[i], rx_q->rx_skbuff[i]->data,
1256                                   (unsigned int)rx_q->rx_skbuff_dma[i]);
1257                 }
1258
1259                 rx_q->cur_rx = 0;
1260                 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1261
1262                 stmmac_clear_rx_descriptors(priv, queue);
1263
1264                 /* Setup the chained descriptor addresses */
1265                 if (priv->mode == STMMAC_CHAIN_MODE) {
1266                         if (priv->extend_desc)
1267                                 priv->hw->mode->init(rx_q->dma_erx,
1268                                                      rx_q->dma_rx_phy,
1269                                                      DMA_RX_SIZE, 1);
1270                         else
1271                                 priv->hw->mode->init(rx_q->dma_rx,
1272                                                      rx_q->dma_rx_phy,
1273                                                      DMA_RX_SIZE, 0);
1274                 }
1275         }
1276
1277         buf_sz = bfsize;
1278
1279         return 0;
1280
1281 err_init_rx_buffers:
1282         while (queue >= 0) {
1283                 while (--i >= 0)
1284                         stmmac_free_rx_buffer(priv, queue, i);
1285
1286                 if (queue == 0)
1287                         break;
1288
1289                 i = DMA_RX_SIZE;
1290                 queue--;
1291         }
1292
1293         return ret;
1294 }
1295
1296 /**
1297  * init_dma_tx_desc_rings - init the TX descriptor rings
1298  * @dev: net device structure.
1299  * Description: this function initializes the DMA TX descriptors
1300  * and allocates the socket buffers. It supports the chained and ring
1301  * modes.
1302  */
1303 static int init_dma_tx_desc_rings(struct net_device *dev)
1304 {
1305         struct stmmac_priv *priv = netdev_priv(dev);
1306         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1307         u32 queue;
1308         int i;
1309
1310         for (queue = 0; queue < tx_queue_cnt; queue++) {
1311                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1312
1313                 netif_dbg(priv, probe, priv->dev,
1314                           "(%s) dma_tx_phy=0x%08x\n", __func__,
1315                          (u32)tx_q->dma_tx_phy);
1316
1317                 /* Setup the chained descriptor addresses */
1318                 if (priv->mode == STMMAC_CHAIN_MODE) {
1319                         if (priv->extend_desc)
1320                                 priv->hw->mode->init(tx_q->dma_etx,
1321                                                      tx_q->dma_tx_phy,
1322                                                      DMA_TX_SIZE, 1);
1323                         else
1324                                 priv->hw->mode->init(tx_q->dma_tx,
1325                                                      tx_q->dma_tx_phy,
1326                                                      DMA_TX_SIZE, 0);
1327                 }
1328
1329                 for (i = 0; i < DMA_TX_SIZE; i++) {
1330                         struct dma_desc *p;
1331                         if (priv->extend_desc)
1332                                 p = &((tx_q->dma_etx + i)->basic);
1333                         else
1334                                 p = tx_q->dma_tx + i;
1335
1336                         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1337                                 p->des0 = 0;
1338                                 p->des1 = 0;
1339                                 p->des2 = 0;
1340                                 p->des3 = 0;
1341                         } else {
1342                                 p->des2 = 0;
1343                         }
1344
1345                         tx_q->tx_skbuff_dma[i].buf = 0;
1346                         tx_q->tx_skbuff_dma[i].map_as_page = false;
1347                         tx_q->tx_skbuff_dma[i].len = 0;
1348                         tx_q->tx_skbuff_dma[i].last_segment = false;
1349                         tx_q->tx_skbuff[i] = NULL;
1350                 }
1351
1352                 tx_q->dirty_tx = 0;
1353                 tx_q->cur_tx = 0;
1354
1355                 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1356         }
1357
1358         return 0;
1359 }
1360
1361 /**
1362  * init_dma_desc_rings - init the RX/TX descriptor rings
1363  * @dev: net device structure
1364  * @flags: gfp flag.
1365  * Description: this function initializes the DMA RX/TX descriptors
1366  * and allocates the socket buffers. It supports the chained and ring
1367  * modes.
1368  */
1369 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1370 {
1371         struct stmmac_priv *priv = netdev_priv(dev);
1372         int ret;
1373
1374         ret = init_dma_rx_desc_rings(dev, flags);
1375         if (ret)
1376                 return ret;
1377
1378         ret = init_dma_tx_desc_rings(dev);
1379
1380         stmmac_clear_descriptors(priv);
1381
1382         if (netif_msg_hw(priv))
1383                 stmmac_display_rings(priv);
1384
1385         return ret;
1386 }
1387
1388 /**
1389  * dma_free_rx_skbufs - free RX dma buffers
1390  * @priv: private structure
1391  * @queue: RX queue index
1392  */
1393 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1394 {
1395         int i;
1396
1397         for (i = 0; i < DMA_RX_SIZE; i++)
1398                 stmmac_free_rx_buffer(priv, queue, i);
1399 }
1400
1401 /**
1402  * dma_free_tx_skbufs - free TX dma buffers
1403  * @priv: private structure
1404  * @queue: TX queue index
1405  */
1406 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1407 {
1408         int i;
1409
1410         for (i = 0; i < DMA_TX_SIZE; i++)
1411                 stmmac_free_tx_buffer(priv, queue, i);
1412 }
1413
1414 /**
1415  * free_dma_rx_desc_resources - free RX dma desc resources
1416  * @priv: private structure
1417  */
1418 static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1419 {
1420         u32 rx_count = priv->plat->rx_queues_to_use;
1421         u32 queue;
1422
1423         /* Free RX queue resources */
1424         for (queue = 0; queue < rx_count; queue++) {
1425                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1426
1427                 /* Release the DMA RX socket buffers */
1428                 dma_free_rx_skbufs(priv, queue);
1429
1430                 /* Free DMA regions of consistent memory previously allocated */
1431                 if (!priv->extend_desc)
1432                         dma_free_coherent(priv->device,
1433                                           DMA_RX_SIZE * sizeof(struct dma_desc),
1434                                           rx_q->dma_rx, rx_q->dma_rx_phy);
1435                 else
1436                         dma_free_coherent(priv->device, DMA_RX_SIZE *
1437                                           sizeof(struct dma_extended_desc),
1438                                           rx_q->dma_erx, rx_q->dma_rx_phy);
1439
1440                 kfree(rx_q->rx_skbuff_dma);
1441                 kfree(rx_q->rx_skbuff);
1442         }
1443 }
1444
1445 /**
1446  * free_dma_tx_desc_resources - free TX dma desc resources
1447  * @priv: private structure
1448  */
1449 static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1450 {
1451         u32 tx_count = priv->plat->tx_queues_to_use;
1452         u32 queue;
1453
1454         /* Free TX queue resources */
1455         for (queue = 0; queue < tx_count; queue++) {
1456                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1457
1458                 /* Release the DMA TX socket buffers */
1459                 dma_free_tx_skbufs(priv, queue);
1460
1461                 /* Free DMA regions of consistent memory previously allocated */
1462                 if (!priv->extend_desc)
1463                         dma_free_coherent(priv->device,
1464                                           DMA_TX_SIZE * sizeof(struct dma_desc),
1465                                           tx_q->dma_tx, tx_q->dma_tx_phy);
1466                 else
1467                         dma_free_coherent(priv->device, DMA_TX_SIZE *
1468                                           sizeof(struct dma_extended_desc),
1469                                           tx_q->dma_etx, tx_q->dma_tx_phy);
1470
1471                 kfree(tx_q->tx_skbuff_dma);
1472                 kfree(tx_q->tx_skbuff);
1473         }
1474 }
1475
1476 /**
1477  * alloc_dma_rx_desc_resources - alloc RX resources.
1478  * @priv: private structure
1479  * Description: according to which descriptor can be used (extend or basic)
1480  * this function allocates the resources for TX and RX paths. In case of
1481  * reception, for example, it pre-allocated the RX socket buffer in order to
1482  * allow zero-copy mechanism.
1483  */
1484 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
1485 {
1486         u32 rx_count = priv->plat->rx_queues_to_use;
1487         int ret = -ENOMEM;
1488         u32 queue;
1489
1490         /* RX queues buffers and DMA */
1491         for (queue = 0; queue < rx_count; queue++) {
1492                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1493
1494                 rx_q->queue_index = queue;
1495                 rx_q->priv_data = priv;
1496
1497                 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE,
1498                                                     sizeof(dma_addr_t),
1499                                                     GFP_KERNEL);
1500                 if (!rx_q->rx_skbuff_dma)
1501                         goto err_dma;
1502
1503                 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE,
1504                                                 sizeof(struct sk_buff *),
1505                                                 GFP_KERNEL);
1506                 if (!rx_q->rx_skbuff)
1507                         goto err_dma;
1508
1509                 if (priv->extend_desc) {
1510                         rx_q->dma_erx = dma_zalloc_coherent(priv->device,
1511                                                             DMA_RX_SIZE *
1512                                                             sizeof(struct
1513                                                             dma_extended_desc),
1514                                                             &rx_q->dma_rx_phy,
1515                                                             GFP_KERNEL);
1516                         if (!rx_q->dma_erx)
1517                                 goto err_dma;
1518
1519                 } else {
1520                         rx_q->dma_rx = dma_zalloc_coherent(priv->device,
1521                                                            DMA_RX_SIZE *
1522                                                            sizeof(struct
1523                                                            dma_desc),
1524                                                            &rx_q->dma_rx_phy,
1525                                                            GFP_KERNEL);
1526                         if (!rx_q->dma_rx)
1527                                 goto err_dma;
1528                 }
1529         }
1530
1531         return 0;
1532
1533 err_dma:
1534         free_dma_rx_desc_resources(priv);
1535
1536         return ret;
1537 }
1538
1539 /**
1540  * alloc_dma_tx_desc_resources - alloc TX resources.
1541  * @priv: private structure
1542  * Description: according to which descriptor can be used (extend or basic)
1543  * this function allocates the resources for TX and RX paths. In case of
1544  * reception, for example, it pre-allocated the RX socket buffer in order to
1545  * allow zero-copy mechanism.
1546  */
1547 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
1548 {
1549         u32 tx_count = priv->plat->tx_queues_to_use;
1550         int ret = -ENOMEM;
1551         u32 queue;
1552
1553         /* TX queues buffers and DMA */
1554         for (queue = 0; queue < tx_count; queue++) {
1555                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1556
1557                 tx_q->queue_index = queue;
1558                 tx_q->priv_data = priv;
1559
1560                 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE,
1561                                                     sizeof(*tx_q->tx_skbuff_dma),
1562                                                     GFP_KERNEL);
1563                 if (!tx_q->tx_skbuff_dma)
1564                         goto err_dma;
1565
1566                 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE,
1567                                                 sizeof(struct sk_buff *),
1568                                                 GFP_KERNEL);
1569                 if (!tx_q->tx_skbuff)
1570                         goto err_dma;
1571
1572                 if (priv->extend_desc) {
1573                         tx_q->dma_etx = dma_zalloc_coherent(priv->device,
1574                                                             DMA_TX_SIZE *
1575                                                             sizeof(struct
1576                                                             dma_extended_desc),
1577                                                             &tx_q->dma_tx_phy,
1578                                                             GFP_KERNEL);
1579                         if (!tx_q->dma_etx)
1580                                 goto err_dma;
1581                 } else {
1582                         tx_q->dma_tx = dma_zalloc_coherent(priv->device,
1583                                                            DMA_TX_SIZE *
1584                                                            sizeof(struct
1585                                                                   dma_desc),
1586                                                            &tx_q->dma_tx_phy,
1587                                                            GFP_KERNEL);
1588                         if (!tx_q->dma_tx)
1589                                 goto err_dma;
1590                 }
1591         }
1592
1593         return 0;
1594
1595 err_dma:
1596         free_dma_tx_desc_resources(priv);
1597
1598         return ret;
1599 }
1600
1601 /**
1602  * alloc_dma_desc_resources - alloc TX/RX resources.
1603  * @priv: private structure
1604  * Description: according to which descriptor can be used (extend or basic)
1605  * this function allocates the resources for TX and RX paths. In case of
1606  * reception, for example, it pre-allocated the RX socket buffer in order to
1607  * allow zero-copy mechanism.
1608  */
1609 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1610 {
1611         /* RX Allocation */
1612         int ret = alloc_dma_rx_desc_resources(priv);
1613
1614         if (ret)
1615                 return ret;
1616
1617         ret = alloc_dma_tx_desc_resources(priv);
1618
1619         return ret;
1620 }
1621
1622 /**
1623  * free_dma_desc_resources - free dma desc resources
1624  * @priv: private structure
1625  */
1626 static void free_dma_desc_resources(struct stmmac_priv *priv)
1627 {
1628         /* Release the DMA RX socket buffers */
1629         free_dma_rx_desc_resources(priv);
1630
1631         /* Release the DMA TX socket buffers */
1632         free_dma_tx_desc_resources(priv);
1633 }
1634
1635 /**
1636  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
1637  *  @priv: driver private structure
1638  *  Description: It is used for enabling the rx queues in the MAC
1639  */
1640 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1641 {
1642         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1643         int queue;
1644         u8 mode;
1645
1646         for (queue = 0; queue < rx_queues_count; queue++) {
1647                 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1648                 priv->hw->mac->rx_queue_enable(priv->hw, mode, queue);
1649         }
1650 }
1651
1652 /**
1653  * stmmac_start_rx_dma - start RX DMA channel
1654  * @priv: driver private structure
1655  * @chan: RX channel index
1656  * Description:
1657  * This starts a RX DMA channel
1658  */
1659 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1660 {
1661         netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1662         priv->hw->dma->start_rx(priv->ioaddr, chan);
1663 }
1664
1665 /**
1666  * stmmac_start_tx_dma - start TX DMA channel
1667  * @priv: driver private structure
1668  * @chan: TX channel index
1669  * Description:
1670  * This starts a TX DMA channel
1671  */
1672 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1673 {
1674         netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1675         priv->hw->dma->start_tx(priv->ioaddr, chan);
1676 }
1677
1678 /**
1679  * stmmac_stop_rx_dma - stop RX DMA channel
1680  * @priv: driver private structure
1681  * @chan: RX channel index
1682  * Description:
1683  * This stops a RX DMA channel
1684  */
1685 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1686 {
1687         netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1688         priv->hw->dma->stop_rx(priv->ioaddr, chan);
1689 }
1690
1691 /**
1692  * stmmac_stop_tx_dma - stop TX DMA channel
1693  * @priv: driver private structure
1694  * @chan: TX channel index
1695  * Description:
1696  * This stops a TX DMA channel
1697  */
1698 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1699 {
1700         netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1701         priv->hw->dma->stop_tx(priv->ioaddr, chan);
1702 }
1703
1704 /**
1705  * stmmac_start_all_dma - start all RX and TX DMA channels
1706  * @priv: driver private structure
1707  * Description:
1708  * This starts all the RX and TX DMA channels
1709  */
1710 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1711 {
1712         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1713         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1714         u32 chan = 0;
1715
1716         for (chan = 0; chan < rx_channels_count; chan++)
1717                 stmmac_start_rx_dma(priv, chan);
1718
1719         for (chan = 0; chan < tx_channels_count; chan++)
1720                 stmmac_start_tx_dma(priv, chan);
1721 }
1722
1723 /**
1724  * stmmac_stop_all_dma - stop all RX and TX DMA channels
1725  * @priv: driver private structure
1726  * Description:
1727  * This stops the RX and TX DMA channels
1728  */
1729 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1730 {
1731         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1732         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1733         u32 chan = 0;
1734
1735         for (chan = 0; chan < rx_channels_count; chan++)
1736                 stmmac_stop_rx_dma(priv, chan);
1737
1738         for (chan = 0; chan < tx_channels_count; chan++)
1739                 stmmac_stop_tx_dma(priv, chan);
1740 }
1741
1742 /**
1743  *  stmmac_dma_operation_mode - HW DMA operation mode
1744  *  @priv: driver private structure
1745  *  Description: it is used for configuring the DMA operation mode register in
1746  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1747  */
1748 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1749 {
1750         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1751         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1752         int rxfifosz = priv->plat->rx_fifo_size;
1753         u32 txmode = 0;
1754         u32 rxmode = 0;
1755         u32 chan = 0;
1756
1757         if (rxfifosz == 0)
1758                 rxfifosz = priv->dma_cap.rx_fifo_size;
1759
1760         if (priv->plat->force_thresh_dma_mode) {
1761                 txmode = tc;
1762                 rxmode = tc;
1763         } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1764                 /*
1765                  * In case of GMAC, SF mode can be enabled
1766                  * to perform the TX COE in HW. This depends on:
1767                  * 1) TX COE if actually supported
1768                  * 2) There is no bugged Jumbo frame support
1769                  *    that needs to not insert csum in the TDES.
1770                  */
1771                 txmode = SF_DMA_MODE;
1772                 rxmode = SF_DMA_MODE;
1773                 priv->xstats.threshold = SF_DMA_MODE;
1774         } else {
1775                 txmode = tc;
1776                 rxmode = SF_DMA_MODE;
1777         }
1778
1779         /* configure all channels */
1780         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1781                 for (chan = 0; chan < rx_channels_count; chan++)
1782                         priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1783                                                    rxfifosz);
1784
1785                 for (chan = 0; chan < tx_channels_count; chan++)
1786                         priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan);
1787         } else {
1788                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1789                                         rxfifosz);
1790         }
1791 }
1792
1793 /**
1794  * stmmac_tx_clean - to manage the transmission completion
1795  * @priv: driver private structure
1796  * @queue: TX queue index
1797  * Description: it reclaims the transmit resources after transmission completes.
1798  */
1799 static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
1800 {
1801         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1802         unsigned int bytes_compl = 0, pkts_compl = 0;
1803         unsigned int entry = tx_q->dirty_tx;
1804
1805         netif_tx_lock(priv->dev);
1806
1807         priv->xstats.tx_clean++;
1808
1809         while (entry != tx_q->cur_tx) {
1810                 struct sk_buff *skb = tx_q->tx_skbuff[entry];
1811                 struct dma_desc *p;
1812                 int status;
1813
1814                 if (priv->extend_desc)
1815                         p = (struct dma_desc *)(tx_q->dma_etx + entry);
1816                 else
1817                         p = tx_q->dma_tx + entry;
1818
1819                 status = priv->hw->desc->tx_status(&priv->dev->stats,
1820                                                       &priv->xstats, p,
1821                                                       priv->ioaddr);
1822                 /* Check if the descriptor is owned by the DMA */
1823                 if (unlikely(status & tx_dma_own))
1824                         break;
1825
1826                 /* Just consider the last segment and ...*/
1827                 if (likely(!(status & tx_not_ls))) {
1828                         /* ... verify the status error condition */
1829                         if (unlikely(status & tx_err)) {
1830                                 priv->dev->stats.tx_errors++;
1831                         } else {
1832                                 priv->dev->stats.tx_packets++;
1833                                 priv->xstats.tx_pkt_n++;
1834                         }
1835                         stmmac_get_tx_hwtstamp(priv, p, skb);
1836                 }
1837
1838                 if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
1839                         if (tx_q->tx_skbuff_dma[entry].map_as_page)
1840                                 dma_unmap_page(priv->device,
1841                                                tx_q->tx_skbuff_dma[entry].buf,
1842                                                tx_q->tx_skbuff_dma[entry].len,
1843                                                DMA_TO_DEVICE);
1844                         else
1845                                 dma_unmap_single(priv->device,
1846                                                  tx_q->tx_skbuff_dma[entry].buf,
1847                                                  tx_q->tx_skbuff_dma[entry].len,
1848                                                  DMA_TO_DEVICE);
1849                         tx_q->tx_skbuff_dma[entry].buf = 0;
1850                         tx_q->tx_skbuff_dma[entry].len = 0;
1851                         tx_q->tx_skbuff_dma[entry].map_as_page = false;
1852                 }
1853
1854                 if (priv->hw->mode->clean_desc3)
1855                         priv->hw->mode->clean_desc3(tx_q, p);
1856
1857                 tx_q->tx_skbuff_dma[entry].last_segment = false;
1858                 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
1859
1860                 if (likely(skb != NULL)) {
1861                         pkts_compl++;
1862                         bytes_compl += skb->len;
1863                         dev_consume_skb_any(skb);
1864                         tx_q->tx_skbuff[entry] = NULL;
1865                 }
1866
1867                 priv->hw->desc->release_tx_desc(p, priv->mode);
1868
1869                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1870         }
1871         tx_q->dirty_tx = entry;
1872
1873         netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
1874                                   pkts_compl, bytes_compl);
1875
1876         if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
1877                                                                 queue))) &&
1878             stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
1879
1880                 netif_dbg(priv, tx_done, priv->dev,
1881                           "%s: restart transmit\n", __func__);
1882                 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1883         }
1884
1885         if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
1886                 stmmac_enable_eee_mode(priv);
1887                 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1888         }
1889         netif_tx_unlock(priv->dev);
1890 }
1891
1892 static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan)
1893 {
1894         priv->hw->dma->enable_dma_irq(priv->ioaddr, chan);
1895 }
1896
1897 static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan)
1898 {
1899         priv->hw->dma->disable_dma_irq(priv->ioaddr, chan);
1900 }
1901
1902 /**
1903  * stmmac_tx_err - to manage the tx error
1904  * @priv: driver private structure
1905  * @chan: channel index
1906  * Description: it cleans the descriptors and restarts the transmission
1907  * in case of transmission errors.
1908  */
1909 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
1910 {
1911         struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
1912         int i;
1913
1914         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
1915
1916         stmmac_stop_tx_dma(priv, chan);
1917         dma_free_tx_skbufs(priv, chan);
1918         for (i = 0; i < DMA_TX_SIZE; i++)
1919                 if (priv->extend_desc)
1920                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1921                                                      priv->mode,
1922                                                      (i == DMA_TX_SIZE - 1));
1923                 else
1924                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1925                                                      priv->mode,
1926                                                      (i == DMA_TX_SIZE - 1));
1927         tx_q->dirty_tx = 0;
1928         tx_q->cur_tx = 0;
1929         netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
1930         stmmac_start_tx_dma(priv, chan);
1931
1932         priv->dev->stats.tx_errors++;
1933         netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
1934 }
1935
1936 /**
1937  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
1938  *  @priv: driver private structure
1939  *  @txmode: TX operating mode
1940  *  @rxmode: RX operating mode
1941  *  @chan: channel index
1942  *  Description: it is used for configuring of the DMA operation mode in
1943  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
1944  *  mode.
1945  */
1946 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
1947                                           u32 rxmode, u32 chan)
1948 {
1949         int rxfifosz = priv->plat->rx_fifo_size;
1950
1951         if (rxfifosz == 0)
1952                 rxfifosz = priv->dma_cap.rx_fifo_size;
1953
1954         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1955                 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1956                                            rxfifosz);
1957                 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan);
1958         } else {
1959                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1960                                         rxfifosz);
1961         }
1962 }
1963
1964 /**
1965  * stmmac_dma_interrupt - DMA ISR
1966  * @priv: driver private structure
1967  * Description: this is the DMA ISR. It is called by the main ISR.
1968  * It calls the dwmac dma routine and schedule poll method in case of some
1969  * work can be done.
1970  */
1971 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
1972 {
1973         u32 tx_channel_count = priv->plat->tx_queues_to_use;
1974         int status;
1975         u32 chan;
1976
1977         for (chan = 0; chan < tx_channel_count; chan++) {
1978                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
1979
1980                 status = priv->hw->dma->dma_interrupt(priv->ioaddr,
1981                                                       &priv->xstats, chan);
1982                 if (likely((status & handle_rx)) || (status & handle_tx)) {
1983                         if (likely(napi_schedule_prep(&rx_q->napi))) {
1984                                 stmmac_disable_dma_irq(priv, chan);
1985                                 __napi_schedule(&rx_q->napi);
1986                         }
1987                 }
1988
1989                 if (unlikely(status & tx_hard_error_bump_tc)) {
1990                         /* Try to bump up the dma threshold on this failure */
1991                         if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
1992                             (tc <= 256)) {
1993                                 tc += 64;
1994                                 if (priv->plat->force_thresh_dma_mode)
1995                                         stmmac_set_dma_operation_mode(priv,
1996                                                                       tc,
1997                                                                       tc,
1998                                                                       chan);
1999                                 else
2000                                         stmmac_set_dma_operation_mode(priv,
2001                                                                     tc,
2002                                                                     SF_DMA_MODE,
2003                                                                     chan);
2004                                 priv->xstats.threshold = tc;
2005                         }
2006                 } else if (unlikely(status == tx_hard_error)) {
2007                         stmmac_tx_err(priv, chan);
2008                 }
2009         }
2010 }
2011
2012 /**
2013  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
2014  * @priv: driver private structure
2015  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
2016  */
2017 static void stmmac_mmc_setup(struct stmmac_priv *priv)
2018 {
2019         unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2020                             MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2021
2022         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
2023                 priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET;
2024                 priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET;
2025         } else {
2026                 priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET;
2027                 priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET;
2028         }
2029
2030         dwmac_mmc_intr_all_mask(priv->mmcaddr);
2031
2032         if (priv->dma_cap.rmon) {
2033                 dwmac_mmc_ctrl(priv->mmcaddr, mode);
2034                 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2035         } else
2036                 netdev_info(priv->dev, "No MAC Management Counters available\n");
2037 }
2038
2039 /**
2040  * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors
2041  * @priv: driver private structure
2042  * Description: select the Enhanced/Alternate or Normal descriptors.
2043  * In case of Enhanced/Alternate, it checks if the extended descriptors are
2044  * supported by the HW capability register.
2045  */
2046 static void stmmac_selec_desc_mode(struct stmmac_priv *priv)
2047 {
2048         if (priv->plat->enh_desc) {
2049                 dev_info(priv->device, "Enhanced/Alternate descriptors\n");
2050
2051                 /* GMAC older than 3.50 has no extended descriptors */
2052                 if (priv->synopsys_id >= DWMAC_CORE_3_50) {
2053                         dev_info(priv->device, "Enabled extended descriptors\n");
2054                         priv->extend_desc = 1;
2055                 } else
2056                         dev_warn(priv->device, "Extended descriptors not supported\n");
2057
2058                 priv->hw->desc = &enh_desc_ops;
2059         } else {
2060                 dev_info(priv->device, "Normal descriptors\n");
2061                 priv->hw->desc = &ndesc_ops;
2062         }
2063 }
2064
2065 /**
2066  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2067  * @priv: driver private structure
2068  * Description:
2069  *  new GMAC chip generations have a new register to indicate the
2070  *  presence of the optional feature/functions.
2071  *  This can be also used to override the value passed through the
2072  *  platform and necessary for old MAC10/100 and GMAC chips.
2073  */
2074 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2075 {
2076         u32 ret = 0;
2077
2078         if (priv->hw->dma->get_hw_feature) {
2079                 priv->hw->dma->get_hw_feature(priv->ioaddr,
2080                                               &priv->dma_cap);
2081                 ret = 1;
2082         }
2083
2084         return ret;
2085 }
2086
2087 /**
2088  * stmmac_check_ether_addr - check if the MAC addr is valid
2089  * @priv: driver private structure
2090  * Description:
2091  * it is to verify if the MAC address is valid, in case of failures it
2092  * generates a random MAC address
2093  */
2094 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2095 {
2096         if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2097                 priv->hw->mac->get_umac_addr(priv->hw,
2098                                              priv->dev->dev_addr, 0);
2099                 if (!is_valid_ether_addr(priv->dev->dev_addr))
2100                         eth_hw_addr_random(priv->dev);
2101                 netdev_info(priv->dev, "device MAC address %pM\n",
2102                             priv->dev->dev_addr);
2103         }
2104 }
2105
2106 /**
2107  * stmmac_init_dma_engine - DMA init.
2108  * @priv: driver private structure
2109  * Description:
2110  * It inits the DMA invoking the specific MAC/GMAC callback.
2111  * Some DMA parameters can be passed from the platform;
2112  * in case of these are not passed a default is kept for the MAC or GMAC.
2113  */
2114 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2115 {
2116         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2117         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2118         struct stmmac_rx_queue *rx_q;
2119         struct stmmac_tx_queue *tx_q;
2120         u32 dummy_dma_rx_phy = 0;
2121         u32 dummy_dma_tx_phy = 0;
2122         u32 chan = 0;
2123         int atds = 0;
2124         int ret = 0;
2125
2126         if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2127                 dev_err(priv->device, "Invalid DMA configuration\n");
2128                 return -EINVAL;
2129         }
2130
2131         if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2132                 atds = 1;
2133
2134         ret = priv->hw->dma->reset(priv->ioaddr);
2135         if (ret) {
2136                 dev_err(priv->device, "Failed to reset the dma\n");
2137                 return ret;
2138         }
2139
2140         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
2141                 /* DMA Configuration */
2142                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2143                                     dummy_dma_tx_phy, dummy_dma_rx_phy, atds);
2144
2145                 /* DMA RX Channel Configuration */
2146                 for (chan = 0; chan < rx_channels_count; chan++) {
2147                         rx_q = &priv->rx_queue[chan];
2148
2149                         priv->hw->dma->init_rx_chan(priv->ioaddr,
2150                                                     priv->plat->dma_cfg,
2151                                                     rx_q->dma_rx_phy, chan);
2152
2153                         rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2154                                     (DMA_RX_SIZE * sizeof(struct dma_desc));
2155                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
2156                                                        rx_q->rx_tail_addr,
2157                                                        chan);
2158                 }
2159
2160                 /* DMA TX Channel Configuration */
2161                 for (chan = 0; chan < tx_channels_count; chan++) {
2162                         tx_q = &priv->tx_queue[chan];
2163
2164                         priv->hw->dma->init_chan(priv->ioaddr,
2165                                                  priv->plat->dma_cfg,
2166                                                  chan);
2167
2168                         priv->hw->dma->init_tx_chan(priv->ioaddr,
2169                                                     priv->plat->dma_cfg,
2170                                                     tx_q->dma_tx_phy, chan);
2171
2172                         tx_q->tx_tail_addr = tx_q->dma_tx_phy +
2173                                     (DMA_TX_SIZE * sizeof(struct dma_desc));
2174                         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr,
2175                                                        tx_q->tx_tail_addr,
2176                                                        chan);
2177                 }
2178         } else {
2179                 rx_q = &priv->rx_queue[chan];
2180                 tx_q = &priv->tx_queue[chan];
2181                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2182                                     tx_q->dma_tx_phy, rx_q->dma_rx_phy, atds);
2183         }
2184
2185         if (priv->plat->axi && priv->hw->dma->axi)
2186                 priv->hw->dma->axi(priv->ioaddr, priv->plat->axi);
2187
2188         return ret;
2189 }
2190
2191 /**
2192  * stmmac_tx_timer - mitigation sw timer for tx.
2193  * @data: data pointer
2194  * Description:
2195  * This is the timer handler to directly invoke the stmmac_tx_clean.
2196  */
2197 static void stmmac_tx_timer(unsigned long data)
2198 {
2199         struct stmmac_priv *priv = (struct stmmac_priv *)data;
2200         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2201         u32 queue;
2202
2203         /* let's scan all the tx queues */
2204         for (queue = 0; queue < tx_queues_count; queue++)
2205                 stmmac_tx_clean(priv, queue);
2206 }
2207
2208 /**
2209  * stmmac_init_tx_coalesce - init tx mitigation options.
2210  * @priv: driver private structure
2211  * Description:
2212  * This inits the transmit coalesce parameters: i.e. timer rate,
2213  * timer handler and default threshold used for enabling the
2214  * interrupt on completion bit.
2215  */
2216 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv)
2217 {
2218         priv->tx_coal_frames = STMMAC_TX_FRAMES;
2219         priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2220         init_timer(&priv->txtimer);
2221         priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer);
2222         priv->txtimer.data = (unsigned long)priv;
2223         priv->txtimer.function = stmmac_tx_timer;
2224         add_timer(&priv->txtimer);
2225 }
2226
2227 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2228 {
2229         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2230         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2231         u32 chan;
2232
2233         /* set TX ring length */
2234         if (priv->hw->dma->set_tx_ring_len) {
2235                 for (chan = 0; chan < tx_channels_count; chan++)
2236                         priv->hw->dma->set_tx_ring_len(priv->ioaddr,
2237                                                        (DMA_TX_SIZE - 1), chan);
2238         }
2239
2240         /* set RX ring length */
2241         if (priv->hw->dma->set_rx_ring_len) {
2242                 for (chan = 0; chan < rx_channels_count; chan++)
2243                         priv->hw->dma->set_rx_ring_len(priv->ioaddr,
2244                                                        (DMA_RX_SIZE - 1), chan);
2245         }
2246 }
2247
2248 /**
2249  *  stmmac_set_tx_queue_weight - Set TX queue weight
2250  *  @priv: driver private structure
2251  *  Description: It is used for setting TX queues weight
2252  */
2253 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2254 {
2255         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2256         u32 weight;
2257         u32 queue;
2258
2259         for (queue = 0; queue < tx_queues_count; queue++) {
2260                 weight = priv->plat->tx_queues_cfg[queue].weight;
2261                 priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue);
2262         }
2263 }
2264
2265 /**
2266  *  stmmac_configure_cbs - Configure CBS in TX queue
2267  *  @priv: driver private structure
2268  *  Description: It is used for configuring CBS in AVB TX queues
2269  */
2270 static void stmmac_configure_cbs(struct stmmac_priv *priv)
2271 {
2272         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2273         u32 mode_to_use;
2274         u32 queue;
2275
2276         /* queue 0 is reserved for legacy traffic */
2277         for (queue = 1; queue < tx_queues_count; queue++) {
2278                 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2279                 if (mode_to_use == MTL_QUEUE_DCB)
2280                         continue;
2281
2282                 priv->hw->mac->config_cbs(priv->hw,
2283                                 priv->plat->tx_queues_cfg[queue].send_slope,
2284                                 priv->plat->tx_queues_cfg[queue].idle_slope,
2285                                 priv->plat->tx_queues_cfg[queue].high_credit,
2286                                 priv->plat->tx_queues_cfg[queue].low_credit,
2287                                 queue);
2288         }
2289 }
2290
2291 /**
2292  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2293  *  @priv: driver private structure
2294  *  Description: It is used for mapping RX queues to RX dma channels
2295  */
2296 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2297 {
2298         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2299         u32 queue;
2300         u32 chan;
2301
2302         for (queue = 0; queue < rx_queues_count; queue++) {
2303                 chan = priv->plat->rx_queues_cfg[queue].chan;
2304                 priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan);
2305         }
2306 }
2307
2308 /**
2309  *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
2310  *  @priv: driver private structure
2311  *  Description: It is used for configuring the RX Queue Priority
2312  */
2313 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
2314 {
2315         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2316         u32 queue;
2317         u32 prio;
2318
2319         for (queue = 0; queue < rx_queues_count; queue++) {
2320                 if (!priv->plat->rx_queues_cfg[queue].use_prio)
2321                         continue;
2322
2323                 prio = priv->plat->rx_queues_cfg[queue].prio;
2324                 priv->hw->mac->rx_queue_prio(priv->hw, prio, queue);
2325         }
2326 }
2327
2328 /**
2329  *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
2330  *  @priv: driver private structure
2331  *  Description: It is used for configuring the TX Queue Priority
2332  */
2333 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
2334 {
2335         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2336         u32 queue;
2337         u32 prio;
2338
2339         for (queue = 0; queue < tx_queues_count; queue++) {
2340                 if (!priv->plat->tx_queues_cfg[queue].use_prio)
2341                         continue;
2342
2343                 prio = priv->plat->tx_queues_cfg[queue].prio;
2344                 priv->hw->mac->tx_queue_prio(priv->hw, prio, queue);
2345         }
2346 }
2347
2348 /**
2349  *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
2350  *  @priv: driver private structure
2351  *  Description: It is used for configuring the RX queue routing
2352  */
2353 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
2354 {
2355         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2356         u32 queue;
2357         u8 packet;
2358
2359         for (queue = 0; queue < rx_queues_count; queue++) {
2360                 /* no specific packet type routing specified for the queue */
2361                 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
2362                         continue;
2363
2364                 packet = priv->plat->rx_queues_cfg[queue].pkt_route;
2365                 priv->hw->mac->rx_queue_prio(priv->hw, packet, queue);
2366         }
2367 }
2368
2369 /**
2370  *  stmmac_mtl_configuration - Configure MTL
2371  *  @priv: driver private structure
2372  *  Description: It is used for configurring MTL
2373  */
2374 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2375 {
2376         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2377         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2378
2379         if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight)
2380                 stmmac_set_tx_queue_weight(priv);
2381
2382         /* Configure MTL RX algorithms */
2383         if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms)
2384                 priv->hw->mac->prog_mtl_rx_algorithms(priv->hw,
2385                                                 priv->plat->rx_sched_algorithm);
2386
2387         /* Configure MTL TX algorithms */
2388         if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms)
2389                 priv->hw->mac->prog_mtl_tx_algorithms(priv->hw,
2390                                                 priv->plat->tx_sched_algorithm);
2391
2392         /* Configure CBS in AVB TX queues */
2393         if (tx_queues_count > 1 && priv->hw->mac->config_cbs)
2394                 stmmac_configure_cbs(priv);
2395
2396         /* Map RX MTL to DMA channels */
2397         if (priv->hw->mac->map_mtl_to_dma)
2398                 stmmac_rx_queue_dma_chan_map(priv);
2399
2400         /* Enable MAC RX Queues */
2401         if (priv->hw->mac->rx_queue_enable)
2402                 stmmac_mac_enable_rx_queues(priv);
2403
2404         /* Set RX priorities */
2405         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_prio)
2406                 stmmac_mac_config_rx_queues_prio(priv);
2407
2408         /* Set TX priorities */
2409         if (tx_queues_count > 1 && priv->hw->mac->tx_queue_prio)
2410                 stmmac_mac_config_tx_queues_prio(priv);
2411
2412         /* Set RX routing */
2413         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_routing)
2414                 stmmac_mac_config_rx_queues_routing(priv);
2415 }
2416
2417 /**
2418  * stmmac_hw_setup - setup mac in a usable state.
2419  *  @dev : pointer to the device structure.
2420  *  Description:
2421  *  this is the main function to setup the HW in a usable state because the
2422  *  dma engine is reset, the core registers are configured (e.g. AXI,
2423  *  Checksum features, timers). The DMA is ready to start receiving and
2424  *  transmitting.
2425  *  Return value:
2426  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2427  *  file on failure.
2428  */
2429 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2430 {
2431         struct stmmac_priv *priv = netdev_priv(dev);
2432         u32 rx_cnt = priv->plat->rx_queues_to_use;
2433         u32 tx_cnt = priv->plat->tx_queues_to_use;
2434         u32 chan;
2435         int ret;
2436
2437         /* DMA initialization and SW reset */
2438         ret = stmmac_init_dma_engine(priv);
2439         if (ret < 0) {
2440                 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2441                            __func__);
2442                 return ret;
2443         }
2444
2445         /* Copy the MAC addr into the HW  */
2446         priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0);
2447
2448         /* PS and related bits will be programmed according to the speed */
2449         if (priv->hw->pcs) {
2450                 int speed = priv->plat->mac_port_sel_speed;
2451
2452                 if ((speed == SPEED_10) || (speed == SPEED_100) ||
2453                     (speed == SPEED_1000)) {
2454                         priv->hw->ps = speed;
2455                 } else {
2456                         dev_warn(priv->device, "invalid port speed\n");
2457                         priv->hw->ps = 0;
2458                 }
2459         }
2460
2461         /* Initialize the MAC Core */
2462         priv->hw->mac->core_init(priv->hw, dev->mtu);
2463
2464         /* Initialize MTL*/
2465         if (priv->synopsys_id >= DWMAC_CORE_4_00)
2466                 stmmac_mtl_configuration(priv);
2467
2468         ret = priv->hw->mac->rx_ipc(priv->hw);
2469         if (!ret) {
2470                 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2471                 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2472                 priv->hw->rx_csum = 0;
2473         }
2474
2475         /* Enable the MAC Rx/Tx */
2476         priv->hw->mac->set_mac(priv->ioaddr, true);
2477
2478         /* Set the HW DMA mode and the COE */
2479         stmmac_dma_operation_mode(priv);
2480
2481         stmmac_mmc_setup(priv);
2482
2483         if (init_ptp) {
2484                 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2485                 if (ret < 0)
2486                         netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2487
2488                 ret = stmmac_init_ptp(priv);
2489                 if (ret == -EOPNOTSUPP)
2490                         netdev_warn(priv->dev, "PTP not supported by HW\n");
2491                 else if (ret)
2492                         netdev_warn(priv->dev, "PTP init failed\n");
2493         }
2494
2495 #ifdef CONFIG_DEBUG_FS
2496         ret = stmmac_init_fs(dev);
2497         if (ret < 0)
2498                 netdev_warn(priv->dev, "%s: failed debugFS registration\n",
2499                             __func__);
2500 #endif
2501         /* Start the ball rolling... */
2502         stmmac_start_all_dma(priv);
2503
2504         priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2505
2506         if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) {
2507                 priv->rx_riwt = MAX_DMA_RIWT;
2508                 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt);
2509         }
2510
2511         if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane)
2512                 priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0);
2513
2514         /* set TX and RX rings length */
2515         stmmac_set_rings_length(priv);
2516
2517         /* Enable TSO */
2518         if (priv->tso) {
2519                 for (chan = 0; chan < tx_cnt; chan++)
2520                         priv->hw->dma->enable_tso(priv->ioaddr, 1, chan);
2521         }
2522
2523         return 0;
2524 }
2525
2526 static void stmmac_hw_teardown(struct net_device *dev)
2527 {
2528         struct stmmac_priv *priv = netdev_priv(dev);
2529
2530         clk_disable_unprepare(priv->plat->clk_ptp_ref);
2531 }
2532
2533 /**
2534  *  stmmac_open - open entry point of the driver
2535  *  @dev : pointer to the device structure.
2536  *  Description:
2537  *  This function is the open entry point of the driver.
2538  *  Return value:
2539  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2540  *  file on failure.
2541  */
2542 static int stmmac_open(struct net_device *dev)
2543 {
2544         struct stmmac_priv *priv = netdev_priv(dev);
2545         int ret;
2546
2547         stmmac_check_ether_addr(priv);
2548
2549         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
2550             priv->hw->pcs != STMMAC_PCS_TBI &&
2551             priv->hw->pcs != STMMAC_PCS_RTBI) {
2552                 ret = stmmac_init_phy(dev);
2553                 if (ret) {
2554                         netdev_err(priv->dev,
2555                                    "%s: Cannot attach to PHY (error: %d)\n",
2556                                    __func__, ret);
2557                         return ret;
2558                 }
2559         }
2560
2561         /* Extra statistics */
2562         memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2563         priv->xstats.threshold = tc;
2564
2565         priv->dma_buf_sz = STMMAC_ALIGN(buf_sz);
2566         priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2567
2568         ret = alloc_dma_desc_resources(priv);
2569         if (ret < 0) {
2570                 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
2571                            __func__);
2572                 goto dma_desc_error;
2573         }
2574
2575         ret = init_dma_desc_rings(dev, GFP_KERNEL);
2576         if (ret < 0) {
2577                 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
2578                            __func__);
2579                 goto init_error;
2580         }
2581
2582         ret = stmmac_hw_setup(dev, true);
2583         if (ret < 0) {
2584                 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2585                 goto init_error;
2586         }
2587
2588         stmmac_init_tx_coalesce(priv);
2589
2590         if (dev->phydev)
2591                 phy_start(dev->phydev);
2592
2593         /* Request the IRQ lines */
2594         ret = request_irq(dev->irq, stmmac_interrupt,
2595                           IRQF_SHARED, dev->name, dev);
2596         if (unlikely(ret < 0)) {
2597                 netdev_err(priv->dev,
2598                            "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2599                            __func__, dev->irq, ret);
2600                 goto irq_error;
2601         }
2602
2603         /* Request the Wake IRQ in case of another line is used for WoL */
2604         if (priv->wol_irq != dev->irq) {
2605                 ret = request_irq(priv->wol_irq, stmmac_interrupt,
2606                                   IRQF_SHARED, dev->name, dev);
2607                 if (unlikely(ret < 0)) {
2608                         netdev_err(priv->dev,
2609                                    "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2610                                    __func__, priv->wol_irq, ret);
2611                         goto wolirq_error;
2612                 }
2613         }
2614
2615         /* Request the IRQ lines */
2616         if (priv->lpi_irq > 0) {
2617                 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2618                                   dev->name, dev);
2619                 if (unlikely(ret < 0)) {
2620                         netdev_err(priv->dev,
2621                                    "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2622                                    __func__, priv->lpi_irq, ret);
2623                         goto lpiirq_error;
2624                 }
2625         }
2626
2627         stmmac_enable_all_queues(priv);
2628         stmmac_start_all_queues(priv);
2629
2630         return 0;
2631
2632 lpiirq_error:
2633         if (priv->wol_irq != dev->irq)
2634                 free_irq(priv->wol_irq, dev);
2635 wolirq_error:
2636         free_irq(dev->irq, dev);
2637 irq_error:
2638         if (dev->phydev)
2639                 phy_stop(dev->phydev);
2640
2641         del_timer_sync(&priv->txtimer);
2642         stmmac_hw_teardown(dev);
2643 init_error:
2644         free_dma_desc_resources(priv);
2645 dma_desc_error:
2646         if (dev->phydev)
2647                 phy_disconnect(dev->phydev);
2648
2649         return ret;
2650 }
2651
2652 /**
2653  *  stmmac_release - close entry point of the driver
2654  *  @dev : device pointer.
2655  *  Description:
2656  *  This is the stop entry point of the driver.
2657  */
2658 static int stmmac_release(struct net_device *dev)
2659 {
2660         struct stmmac_priv *priv = netdev_priv(dev);
2661
2662         if (priv->eee_enabled)
2663                 del_timer_sync(&priv->eee_ctrl_timer);
2664
2665         /* Stop and disconnect the PHY */
2666         if (dev->phydev) {
2667                 phy_stop(dev->phydev);
2668                 phy_disconnect(dev->phydev);
2669         }
2670
2671         stmmac_stop_all_queues(priv);
2672
2673         stmmac_disable_all_queues(priv);
2674
2675         del_timer_sync(&priv->txtimer);
2676
2677         /* Free the IRQ lines */
2678         free_irq(dev->irq, dev);
2679         if (priv->wol_irq != dev->irq)
2680                 free_irq(priv->wol_irq, dev);
2681         if (priv->lpi_irq > 0)
2682                 free_irq(priv->lpi_irq, dev);
2683
2684         /* Stop TX/RX DMA and clear the descriptors */
2685         stmmac_stop_all_dma(priv);
2686
2687         /* Release and free the Rx/Tx resources */
2688         free_dma_desc_resources(priv);
2689
2690         /* Disable the MAC Rx/Tx */
2691         priv->hw->mac->set_mac(priv->ioaddr, false);
2692
2693         netif_carrier_off(dev);
2694
2695 #ifdef CONFIG_DEBUG_FS
2696         stmmac_exit_fs(dev);
2697 #endif
2698
2699         stmmac_release_ptp(priv);
2700
2701         return 0;
2702 }
2703
2704 /**
2705  *  stmmac_tso_allocator - close entry point of the driver
2706  *  @priv: driver private structure
2707  *  @des: buffer start address
2708  *  @total_len: total length to fill in descriptors
2709  *  @last_segmant: condition for the last descriptor
2710  *  @queue: TX queue index
2711  *  Description:
2712  *  This function fills descriptor and request new descriptors according to
2713  *  buffer length to fill
2714  */
2715 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des,
2716                                  int total_len, bool last_segment, u32 queue)
2717 {
2718         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2719         struct dma_desc *desc;
2720         u32 buff_size;
2721         int tmp_len;
2722
2723         tmp_len = total_len;
2724
2725         while (tmp_len > 0) {
2726                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2727                 desc = tx_q->dma_tx + tx_q->cur_tx;
2728
2729                 desc->des0 = cpu_to_le32(des + (total_len - tmp_len));
2730                 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2731                             TSO_MAX_BUFF_SIZE : tmp_len;
2732
2733                 priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size,
2734                         0, 1,
2735                         (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
2736                         0, 0);
2737
2738                 tmp_len -= TSO_MAX_BUFF_SIZE;
2739         }
2740 }
2741
2742 /**
2743  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
2744  *  @skb : the socket buffer
2745  *  @dev : device pointer
2746  *  Description: this is the transmit function that is called on TSO frames
2747  *  (support available on GMAC4 and newer chips).
2748  *  Diagram below show the ring programming in case of TSO frames:
2749  *
2750  *  First Descriptor
2751  *   --------
2752  *   | DES0 |---> buffer1 = L2/L3/L4 header
2753  *   | DES1 |---> TCP Payload (can continue on next descr...)
2754  *   | DES2 |---> buffer 1 and 2 len
2755  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
2756  *   --------
2757  *      |
2758  *     ...
2759  *      |
2760  *   --------
2761  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
2762  *   | DES1 | --|
2763  *   | DES2 | --> buffer 1 and 2 len
2764  *   | DES3 |
2765  *   --------
2766  *
2767  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
2768  */
2769 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
2770 {
2771         struct dma_desc *desc, *first, *mss_desc = NULL;
2772         struct stmmac_priv *priv = netdev_priv(dev);
2773         int nfrags = skb_shinfo(skb)->nr_frags;
2774         u32 queue = skb_get_queue_mapping(skb);
2775         unsigned int first_entry, des;
2776         struct stmmac_tx_queue *tx_q;
2777         int tmp_pay_len = 0;
2778         u32 pay_len, mss;
2779         u8 proto_hdr_len;
2780         int i;
2781
2782         tx_q = &priv->tx_queue[queue];
2783
2784         /* Compute header lengths */
2785         proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2786
2787         /* Desc availability based on threshold should be enough safe */
2788         if (unlikely(stmmac_tx_avail(priv, queue) <
2789                 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2790                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2791                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
2792                                                                 queue));
2793                         /* This is a hard error, log it. */
2794                         netdev_err(priv->dev,
2795                                    "%s: Tx Ring full when queue awake\n",
2796                                    __func__);
2797                 }
2798                 return NETDEV_TX_BUSY;
2799         }
2800
2801         pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
2802
2803         mss = skb_shinfo(skb)->gso_size;
2804
2805         /* set new MSS value if needed */
2806         if (mss != priv->mss) {
2807                 mss_desc = tx_q->dma_tx + tx_q->cur_tx;
2808                 priv->hw->desc->set_mss(mss_desc, mss);
2809                 priv->mss = mss;
2810                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2811         }
2812
2813         if (netif_msg_tx_queued(priv)) {
2814                 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
2815                         __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss);
2816                 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
2817                         skb->data_len);
2818         }
2819
2820         first_entry = tx_q->cur_tx;
2821
2822         desc = tx_q->dma_tx + first_entry;
2823         first = desc;
2824
2825         /* first descriptor: fill Headers on Buf1 */
2826         des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
2827                              DMA_TO_DEVICE);
2828         if (dma_mapping_error(priv->device, des))
2829                 goto dma_map_err;
2830
2831         tx_q->tx_skbuff_dma[first_entry].buf = des;
2832         tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
2833
2834         first->des0 = cpu_to_le32(des);
2835
2836         /* Fill start of payload in buff2 of first descriptor */
2837         if (pay_len)
2838                 first->des1 = cpu_to_le32(des + proto_hdr_len);
2839
2840         /* If needed take extra descriptors to fill the remaining payload */
2841         tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
2842
2843         stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
2844
2845         /* Prepare fragments */
2846         for (i = 0; i < nfrags; i++) {
2847                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2848
2849                 des = skb_frag_dma_map(priv->device, frag, 0,
2850                                        skb_frag_size(frag),
2851                                        DMA_TO_DEVICE);
2852                 if (dma_mapping_error(priv->device, des))
2853                         goto dma_map_err;
2854
2855                 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
2856                                      (i == nfrags - 1), queue);
2857
2858                 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
2859                 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
2860                 tx_q->tx_skbuff[tx_q->cur_tx] = NULL;
2861                 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
2862         }
2863
2864         tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
2865
2866         /* Only the last descriptor gets to point to the skb. */
2867         tx_q->tx_skbuff[tx_q->cur_tx] = skb;
2868
2869         /* We've used all descriptors we need for this skb, however,
2870          * advance cur_tx so that it references a fresh descriptor.
2871          * ndo_start_xmit will fill this descriptor the next time it's
2872          * called and stmmac_tx_clean may clean up to this descriptor.
2873          */
2874         tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2875
2876         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
2877                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
2878                           __func__);
2879                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
2880         }
2881
2882         dev->stats.tx_bytes += skb->len;
2883         priv->xstats.tx_tso_frames++;
2884         priv->xstats.tx_tso_nfrags += nfrags;
2885
2886         /* Manage tx mitigation */
2887         priv->tx_count_frames += nfrags + 1;
2888         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
2889                 mod_timer(&priv->txtimer,
2890                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
2891         } else {
2892                 priv->tx_count_frames = 0;
2893                 priv->hw->desc->set_tx_ic(desc);
2894                 priv->xstats.tx_set_ic_bit++;
2895         }
2896
2897         skb_tx_timestamp(skb);
2898
2899         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2900                      priv->hwts_tx_en)) {
2901                 /* declare that device is doing timestamping */
2902                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2903                 priv->hw->desc->enable_tx_timestamp(first);
2904         }
2905
2906         /* Complete the first descriptor before granting the DMA */
2907         priv->hw->desc->prepare_tso_tx_desc(first, 1,
2908                         proto_hdr_len,
2909                         pay_len,
2910                         1, tx_q->tx_skbuff_dma[first_entry].last_segment,
2911                         tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len));
2912
2913         /* If context desc is used to change MSS */
2914         if (mss_desc)
2915                 priv->hw->desc->set_tx_owner(mss_desc);
2916
2917         /* The own bit must be the latest setting done when prepare the
2918          * descriptor and then barrier is needed to make sure that
2919          * all is coherent before granting the DMA engine.
2920          */
2921         dma_wmb();
2922
2923         if (netif_msg_pktdata(priv)) {
2924                 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
2925                         __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
2926                         tx_q->cur_tx, first, nfrags);
2927
2928                 priv->hw->desc->display_ring((void *)tx_q->dma_tx, DMA_TX_SIZE,
2929                                              0);
2930
2931                 pr_info(">>> frame to be transmitted: ");
2932                 print_pkt(skb->data, skb_headlen(skb));
2933         }
2934
2935         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
2936
2937         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
2938                                        queue);
2939
2940         return NETDEV_TX_OK;
2941
2942 dma_map_err:
2943         dev_err(priv->device, "Tx dma map failed\n");
2944         dev_kfree_skb(skb);
2945         priv->dev->stats.tx_dropped++;
2946         return NETDEV_TX_OK;
2947 }
2948
2949 /**
2950  *  stmmac_xmit - Tx entry point of the driver
2951  *  @skb : the socket buffer
2952  *  @dev : device pointer
2953  *  Description : this is the tx entry point of the driver.
2954  *  It programs the chain or the ring and supports oversized frames
2955  *  and SG feature.
2956  */
2957 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
2958 {
2959         struct stmmac_priv *priv = netdev_priv(dev);
2960         unsigned int nopaged_len = skb_headlen(skb);
2961         int i, csum_insertion = 0, is_jumbo = 0;
2962         u32 queue = skb_get_queue_mapping(skb);
2963         int nfrags = skb_shinfo(skb)->nr_frags;
2964         int entry;
2965         unsigned int first_entry;
2966         struct dma_desc *desc, *first;
2967         struct stmmac_tx_queue *tx_q;
2968         unsigned int enh_desc;
2969         unsigned int des;
2970
2971         tx_q = &priv->tx_queue[queue];
2972
2973         /* Manage oversized TCP frames for GMAC4 device */
2974         if (skb_is_gso(skb) && priv->tso) {
2975                 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
2976                         return stmmac_tso_xmit(skb, dev);
2977         }
2978
2979         if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
2980                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2981                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
2982                                                                 queue));
2983                         /* This is a hard error, log it. */
2984                         netdev_err(priv->dev,
2985                                    "%s: Tx Ring full when queue awake\n",
2986                                    __func__);
2987                 }
2988                 return NETDEV_TX_BUSY;
2989         }
2990
2991         if (priv->tx_path_in_lpi_mode)
2992                 stmmac_disable_eee_mode(priv);
2993
2994         entry = tx_q->cur_tx;
2995         first_entry = entry;
2996
2997         csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
2998
2999         if (likely(priv->extend_desc))
3000                 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3001         else
3002                 desc = tx_q->dma_tx + entry;
3003
3004         first = desc;
3005
3006         enh_desc = priv->plat->enh_desc;
3007         /* To program the descriptors according to the size of the frame */
3008         if (enh_desc)
3009                 is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc);
3010
3011         if (unlikely(is_jumbo) && likely(priv->synopsys_id <
3012                                          DWMAC_CORE_4_00)) {
3013                 entry = priv->hw->mode->jumbo_frm(tx_q, skb, csum_insertion);
3014                 if (unlikely(entry < 0))
3015                         goto dma_map_err;
3016         }
3017
3018         for (i = 0; i < nfrags; i++) {
3019                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3020                 int len = skb_frag_size(frag);
3021                 bool last_segment = (i == (nfrags - 1));
3022
3023                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3024
3025                 if (likely(priv->extend_desc))
3026                         desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3027                 else
3028                         desc = tx_q->dma_tx + entry;
3029
3030                 des = skb_frag_dma_map(priv->device, frag, 0, len,
3031                                        DMA_TO_DEVICE);
3032                 if (dma_mapping_error(priv->device, des))
3033                         goto dma_map_err; /* should reuse desc w/o issues */
3034
3035                 tx_q->tx_skbuff[entry] = NULL;
3036
3037                 tx_q->tx_skbuff_dma[entry].buf = des;
3038                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3039                         desc->des0 = cpu_to_le32(des);
3040                 else
3041                         desc->des2 = cpu_to_le32(des);
3042
3043                 tx_q->tx_skbuff_dma[entry].map_as_page = true;
3044                 tx_q->tx_skbuff_dma[entry].len = len;
3045                 tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
3046
3047                 /* Prepare the descriptor and set the own bit too */
3048                 priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion,
3049                                                 priv->mode, 1, last_segment,
3050                                                 skb->len);
3051         }
3052
3053         /* Only the last descriptor gets to point to the skb. */
3054         tx_q->tx_skbuff[entry] = skb;
3055
3056         /* We've used all descriptors we need for this skb, however,
3057          * advance cur_tx so that it references a fresh descriptor.
3058          * ndo_start_xmit will fill this descriptor the next time it's
3059          * called and stmmac_tx_clean may clean up to this descriptor.
3060          */
3061         entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3062         tx_q->cur_tx = entry;
3063
3064         if (netif_msg_pktdata(priv)) {
3065                 void *tx_head;
3066
3067                 netdev_dbg(priv->dev,
3068                            "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
3069                            __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3070                            entry, first, nfrags);
3071
3072                 if (priv->extend_desc)
3073                         tx_head = (void *)tx_q->dma_etx;
3074                 else
3075                         tx_head = (void *)tx_q->dma_tx;
3076
3077                 priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false);
3078
3079                 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
3080                 print_pkt(skb->data, skb->len);
3081         }
3082
3083         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3084                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3085                           __func__);
3086                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3087         }
3088
3089         dev->stats.tx_bytes += skb->len;
3090
3091         /* According to the coalesce parameter the IC bit for the latest
3092          * segment is reset and the timer re-started to clean the tx status.
3093          * This approach takes care about the fragments: desc is the first
3094          * element in case of no SG.
3095          */
3096         priv->tx_count_frames += nfrags + 1;
3097         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
3098                 mod_timer(&priv->txtimer,
3099                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
3100         } else {
3101                 priv->tx_count_frames = 0;
3102                 priv->hw->desc->set_tx_ic(desc);
3103                 priv->xstats.tx_set_ic_bit++;
3104         }
3105
3106         skb_tx_timestamp(skb);
3107
3108         /* Ready to fill the first descriptor and set the OWN bit w/o any
3109          * problems because all the descriptors are actually ready to be
3110          * passed to the DMA engine.
3111          */
3112         if (likely(!is_jumbo)) {
3113                 bool last_segment = (nfrags == 0);
3114
3115                 des = dma_map_single(priv->device, skb->data,
3116                                      nopaged_len, DMA_TO_DEVICE);
3117                 if (dma_mapping_error(priv->device, des))
3118                         goto dma_map_err;
3119
3120                 tx_q->tx_skbuff_dma[first_entry].buf = des;
3121                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3122                         first->des0 = cpu_to_le32(des);
3123                 else
3124                         first->des2 = cpu_to_le32(des);
3125
3126                 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3127                 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3128
3129                 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3130                              priv->hwts_tx_en)) {
3131                         /* declare that device is doing timestamping */
3132                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3133                         priv->hw->desc->enable_tx_timestamp(first);
3134                 }
3135
3136                 /* Prepare the first descriptor setting the OWN bit too */
3137                 priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len,
3138                                                 csum_insertion, priv->mode, 1,
3139                                                 last_segment, skb->len);
3140
3141                 /* The own bit must be the latest setting done when prepare the
3142                  * descriptor and then barrier is needed to make sure that
3143                  * all is coherent before granting the DMA engine.
3144                  */
3145                 dma_wmb();
3146         }
3147
3148         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3149
3150         if (priv->synopsys_id < DWMAC_CORE_4_00)
3151                 priv->hw->dma->enable_dma_transmission(priv->ioaddr);
3152         else
3153                 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
3154                                                queue);
3155
3156         return NETDEV_TX_OK;
3157
3158 dma_map_err:
3159         netdev_err(priv->dev, "Tx DMA map failed\n");
3160         dev_kfree_skb(skb);
3161         priv->dev->stats.tx_dropped++;
3162         return NETDEV_TX_OK;
3163 }
3164
3165 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3166 {
3167         struct ethhdr *ehdr;
3168         u16 vlanid;
3169
3170         if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) ==
3171             NETIF_F_HW_VLAN_CTAG_RX &&
3172             !__vlan_get_tag(skb, &vlanid)) {
3173                 /* pop the vlan tag */
3174                 ehdr = (struct ethhdr *)skb->data;
3175                 memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2);
3176                 skb_pull(skb, VLAN_HLEN);
3177                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid);
3178         }
3179 }
3180
3181
3182 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
3183 {
3184         if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
3185                 return 0;
3186
3187         return 1;
3188 }
3189
3190 /**
3191  * stmmac_rx_refill - refill used skb preallocated buffers
3192  * @priv: driver private structure
3193  * @queue: RX queue index
3194  * Description : this is to reallocate the skb for the reception process
3195  * that is based on zero-copy.
3196  */
3197 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3198 {
3199         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3200         int dirty = stmmac_rx_dirty(priv, queue);
3201         unsigned int entry = rx_q->dirty_rx;
3202
3203         int bfsize = priv->dma_buf_sz;
3204
3205         while (dirty-- > 0) {
3206                 struct dma_desc *p;
3207
3208                 if (priv->extend_desc)
3209                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3210                 else
3211                         p = rx_q->dma_rx + entry;
3212
3213                 if (likely(!rx_q->rx_skbuff[entry])) {
3214                         struct sk_buff *skb;
3215
3216                         skb = netdev_alloc_skb_ip_align(priv->dev, bfsize);
3217                         if (unlikely(!skb)) {
3218                                 /* so for a while no zero-copy! */
3219                                 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH;
3220                                 if (unlikely(net_ratelimit()))
3221                                         dev_err(priv->device,
3222                                                 "fail to alloc skb entry %d\n",
3223                                                 entry);
3224                                 break;
3225                         }
3226
3227                         rx_q->rx_skbuff[entry] = skb;
3228                         rx_q->rx_skbuff_dma[entry] =
3229                             dma_map_single(priv->device, skb->data, bfsize,
3230                                            DMA_FROM_DEVICE);
3231                         if (dma_mapping_error(priv->device,
3232                                               rx_q->rx_skbuff_dma[entry])) {
3233                                 netdev_err(priv->dev, "Rx DMA map failed\n");
3234                                 dev_kfree_skb(skb);
3235                                 break;
3236                         }
3237
3238                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) {
3239                                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3240                                 p->des1 = 0;
3241                         } else {
3242                                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3243                         }
3244                         if (priv->hw->mode->refill_desc3)
3245                                 priv->hw->mode->refill_desc3(rx_q, p);
3246
3247                         if (rx_q->rx_zeroc_thresh > 0)
3248                                 rx_q->rx_zeroc_thresh--;
3249
3250                         netif_dbg(priv, rx_status, priv->dev,
3251                                   "refill entry #%d\n", entry);
3252                 }
3253                 dma_wmb();
3254
3255                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3256                         priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0);
3257                 else
3258                         priv->hw->desc->set_rx_owner(p);
3259
3260                 dma_wmb();
3261
3262                 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
3263         }
3264         rx_q->dirty_rx = entry;
3265 }
3266
3267 /**
3268  * stmmac_rx - manage the receive process
3269  * @priv: driver private structure
3270  * @limit: napi bugget
3271  * @queue: RX queue index.
3272  * Description :  this the function called by the napi poll method.
3273  * It gets all the frames inside the ring.
3274  */
3275 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3276 {
3277         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3278         unsigned int entry = rx_q->cur_rx;
3279         int coe = priv->hw->rx_csum;
3280         unsigned int next_entry;
3281         unsigned int count = 0;
3282
3283         if (netif_msg_rx_status(priv)) {
3284                 void *rx_head;
3285
3286                 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3287                 if (priv->extend_desc)
3288                         rx_head = (void *)rx_q->dma_erx;
3289                 else
3290                         rx_head = (void *)rx_q->dma_rx;
3291
3292                 priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true);
3293         }
3294         while (count < limit) {
3295                 int status;
3296                 struct dma_desc *p;
3297                 struct dma_desc *np;
3298
3299                 if (priv->extend_desc)
3300                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3301                 else
3302                         p = rx_q->dma_rx + entry;
3303
3304                 /* read the status of the incoming frame */
3305                 status = priv->hw->desc->rx_status(&priv->dev->stats,
3306                                                    &priv->xstats, p);
3307                 /* check if managed by the DMA otherwise go ahead */
3308                 if (unlikely(status & dma_own))
3309                         break;
3310
3311                 count++;
3312
3313                 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
3314                 next_entry = rx_q->cur_rx;
3315
3316                 if (priv->extend_desc)
3317                         np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3318                 else
3319                         np = rx_q->dma_rx + next_entry;
3320
3321                 prefetch(np);
3322
3323                 if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status))
3324                         priv->hw->desc->rx_extended_status(&priv->dev->stats,
3325                                                            &priv->xstats,
3326                                                            rx_q->dma_erx +
3327                                                            entry);
3328                 if (unlikely(status == discard_frame)) {
3329                         priv->dev->stats.rx_errors++;
3330                         if (priv->hwts_rx_en && !priv->extend_desc) {
3331                                 /* DESC2 & DESC3 will be overwritten by device
3332                                  * with timestamp value, hence reinitialize
3333                                  * them in stmmac_rx_refill() function so that
3334                                  * device can reuse it.
3335                                  */
3336                                 rx_q->rx_skbuff[entry] = NULL;
3337                                 dma_unmap_single(priv->device,
3338                                                  rx_q->rx_skbuff_dma[entry],
3339                                                  priv->dma_buf_sz,
3340                                                  DMA_FROM_DEVICE);
3341                         }
3342                 } else {
3343                         struct sk_buff *skb;
3344                         int frame_len;
3345                         unsigned int des;
3346
3347                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3348                                 des = le32_to_cpu(p->des0);
3349                         else
3350                                 des = le32_to_cpu(p->des2);
3351
3352                         frame_len = priv->hw->desc->get_rx_frame_len(p, coe);
3353
3354                         /*  If frame length is greater than skb buffer size
3355                          *  (preallocated during init) then the packet is
3356                          *  ignored
3357                          */
3358                         if (frame_len > priv->dma_buf_sz) {
3359                                 netdev_err(priv->dev,
3360                                            "len %d larger than size (%d)\n",
3361                                            frame_len, priv->dma_buf_sz);
3362                                 priv->dev->stats.rx_length_errors++;
3363                                 break;
3364                         }
3365
3366                         /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3367                          * Type frames (LLC/LLC-SNAP)
3368                          */
3369                         if (unlikely(status != llc_snap))
3370                                 frame_len -= ETH_FCS_LEN;
3371
3372                         if (netif_msg_rx_status(priv)) {
3373                                 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n",
3374                                            p, entry, des);
3375                                 if (frame_len > ETH_FRAME_LEN)
3376                                         netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
3377                                                    frame_len, status);
3378                         }
3379
3380                         /* The zero-copy is always used for all the sizes
3381                          * in case of GMAC4 because it needs
3382                          * to refill the used descriptors, always.
3383                          */
3384                         if (unlikely(!priv->plat->has_gmac4 &&
3385                                      ((frame_len < priv->rx_copybreak) ||
3386                                      stmmac_rx_threshold_count(rx_q)))) {
3387                                 skb = netdev_alloc_skb_ip_align(priv->dev,
3388                                                                 frame_len);
3389                                 if (unlikely(!skb)) {
3390                                         if (net_ratelimit())
3391                                                 dev_warn(priv->device,
3392                                                          "packet dropped\n");
3393                                         priv->dev->stats.rx_dropped++;
3394                                         break;
3395                                 }
3396
3397                                 dma_sync_single_for_cpu(priv->device,
3398                                                         rx_q->rx_skbuff_dma
3399                                                         [entry], frame_len,
3400                                                         DMA_FROM_DEVICE);
3401                                 skb_copy_to_linear_data(skb,
3402                                                         rx_q->
3403                                                         rx_skbuff[entry]->data,
3404                                                         frame_len);
3405
3406                                 skb_put(skb, frame_len);
3407                                 dma_sync_single_for_device(priv->device,
3408                                                            rx_q->rx_skbuff_dma
3409                                                            [entry], frame_len,
3410                                                            DMA_FROM_DEVICE);
3411                         } else {
3412                                 skb = rx_q->rx_skbuff[entry];
3413                                 if (unlikely(!skb)) {
3414                                         netdev_err(priv->dev,
3415                                                    "%s: Inconsistent Rx chain\n",
3416                                                    priv->dev->name);
3417                                         priv->dev->stats.rx_dropped++;
3418                                         break;
3419                                 }
3420                                 prefetch(skb->data - NET_IP_ALIGN);
3421                                 rx_q->rx_skbuff[entry] = NULL;
3422                                 rx_q->rx_zeroc_thresh++;
3423
3424                                 skb_put(skb, frame_len);
3425                                 dma_unmap_single(priv->device,
3426                                                  rx_q->rx_skbuff_dma[entry],
3427                                                  priv->dma_buf_sz,
3428                                                  DMA_FROM_DEVICE);
3429                         }
3430
3431                         if (netif_msg_pktdata(priv)) {
3432                                 netdev_dbg(priv->dev, "frame received (%dbytes)",
3433                                            frame_len);
3434                                 print_pkt(skb->data, frame_len);
3435                         }
3436
3437                         stmmac_get_rx_hwtstamp(priv, p, np, skb);
3438
3439                         stmmac_rx_vlan(priv->dev, skb);
3440
3441                         skb->protocol = eth_type_trans(skb, priv->dev);
3442
3443                         if (unlikely(!coe))
3444                                 skb_checksum_none_assert(skb);
3445                         else
3446                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3447
3448                         napi_gro_receive(&rx_q->napi, skb);
3449
3450                         priv->dev->stats.rx_packets++;
3451                         priv->dev->stats.rx_bytes += frame_len;
3452                 }
3453                 entry = next_entry;
3454         }
3455
3456         stmmac_rx_refill(priv, queue);
3457
3458         priv->xstats.rx_pkt_n += count;
3459
3460         return count;
3461 }
3462
3463 /**
3464  *  stmmac_poll - stmmac poll method (NAPI)
3465  *  @napi : pointer to the napi structure.
3466  *  @budget : maximum number of packets that the current CPU can receive from
3467  *            all interfaces.
3468  *  Description :
3469  *  To look at the incoming frames and clear the tx resources.
3470  */
3471 static int stmmac_poll(struct napi_struct *napi, int budget)
3472 {
3473         struct stmmac_rx_queue *rx_q =
3474                 container_of(napi, struct stmmac_rx_queue, napi);
3475         struct stmmac_priv *priv = rx_q->priv_data;
3476         u32 tx_count = priv->plat->tx_queues_to_use;
3477         u32 chan = rx_q->queue_index;
3478         int work_done = 0;
3479         u32 queue;
3480
3481         priv->xstats.napi_poll++;
3482
3483         /* check all the queues */
3484         for (queue = 0; queue < tx_count; queue++)
3485                 stmmac_tx_clean(priv, queue);
3486
3487         work_done = stmmac_rx(priv, budget, rx_q->queue_index);
3488         if (work_done < budget) {
3489                 napi_complete_done(napi, work_done);
3490                 stmmac_enable_dma_irq(priv, chan);
3491         }
3492         return work_done;
3493 }
3494
3495 /**
3496  *  stmmac_tx_timeout
3497  *  @dev : Pointer to net device structure
3498  *  Description: this function is called when a packet transmission fails to
3499  *   complete within a reasonable time. The driver will mark the error in the
3500  *   netdev structure and arrange for the device to be reset to a sane state
3501  *   in order to transmit a new packet.
3502  */
3503 static void stmmac_tx_timeout(struct net_device *dev)
3504 {
3505         struct stmmac_priv *priv = netdev_priv(dev);
3506         u32 tx_count = priv->plat->tx_queues_to_use;
3507         u32 chan;
3508
3509         /* Clear Tx resources and restart transmitting again */
3510         for (chan = 0; chan < tx_count; chan++)
3511                 stmmac_tx_err(priv, chan);
3512 }
3513
3514 /**
3515  *  stmmac_set_rx_mode - entry point for multicast addressing
3516  *  @dev : pointer to the device structure
3517  *  Description:
3518  *  This function is a driver entry point which gets called by the kernel
3519  *  whenever multicast addresses must be enabled/disabled.
3520  *  Return value:
3521  *  void.
3522  */
3523 static void stmmac_set_rx_mode(struct net_device *dev)
3524 {
3525         struct stmmac_priv *priv = netdev_priv(dev);
3526
3527         priv->hw->mac->set_filter(priv->hw, dev);
3528 }
3529
3530 /**
3531  *  stmmac_change_mtu - entry point to change MTU size for the device.
3532  *  @dev : device pointer.
3533  *  @new_mtu : the new MTU size for the device.
3534  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
3535  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
3536  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
3537  *  Return value:
3538  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3539  *  file on failure.
3540  */
3541 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3542 {
3543         struct stmmac_priv *priv = netdev_priv(dev);
3544
3545         if (netif_running(dev)) {
3546                 netdev_err(priv->dev, "must be stopped to change its MTU\n");
3547                 return -EBUSY;
3548         }
3549
3550         dev->mtu = new_mtu;
3551
3552         netdev_update_features(dev);
3553
3554         return 0;
3555 }
3556
3557 static netdev_features_t stmmac_fix_features(struct net_device *dev,
3558                                              netdev_features_t features)
3559 {
3560         struct stmmac_priv *priv = netdev_priv(dev);
3561
3562         if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3563                 features &= ~NETIF_F_RXCSUM;
3564
3565         if (!priv->plat->tx_coe)
3566                 features &= ~NETIF_F_CSUM_MASK;
3567
3568         /* Some GMAC devices have a bugged Jumbo frame support that
3569          * needs to have the Tx COE disabled for oversized frames
3570          * (due to limited buffer sizes). In this case we disable
3571          * the TX csum insertion in the TDES and not use SF.
3572          */
3573         if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3574                 features &= ~NETIF_F_CSUM_MASK;
3575
3576         /* Disable tso if asked by ethtool */
3577         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3578                 if (features & NETIF_F_TSO)
3579                         priv->tso = true;
3580                 else
3581                         priv->tso = false;
3582         }
3583
3584         return features;
3585 }
3586
3587 static int stmmac_set_features(struct net_device *netdev,
3588                                netdev_features_t features)
3589 {
3590         struct stmmac_priv *priv = netdev_priv(netdev);
3591
3592         /* Keep the COE Type in case of csum is supporting */
3593         if (features & NETIF_F_RXCSUM)
3594                 priv->hw->rx_csum = priv->plat->rx_coe;
3595         else
3596                 priv->hw->rx_csum = 0;
3597         /* No check needed because rx_coe has been set before and it will be
3598          * fixed in case of issue.
3599          */
3600         priv->hw->mac->rx_ipc(priv->hw);
3601
3602         return 0;
3603 }
3604
3605 /**
3606  *  stmmac_interrupt - main ISR
3607  *  @irq: interrupt number.
3608  *  @dev_id: to pass the net device pointer.
3609  *  Description: this is the main driver interrupt service routine.
3610  *  It can call:
3611  *  o DMA service routine (to manage incoming frame reception and transmission
3612  *    status)
3613  *  o Core interrupts to manage: remote wake-up, management counter, LPI
3614  *    interrupts.
3615  */
3616 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
3617 {
3618         struct net_device *dev = (struct net_device *)dev_id;
3619         struct stmmac_priv *priv = netdev_priv(dev);
3620         u32 rx_cnt = priv->plat->rx_queues_to_use;
3621         u32 tx_cnt = priv->plat->tx_queues_to_use;
3622         u32 queues_count;
3623         u32 queue;
3624
3625         queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
3626
3627         if (priv->irq_wake)
3628                 pm_wakeup_event(priv->device, 0);
3629
3630         if (unlikely(!dev)) {
3631                 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
3632                 return IRQ_NONE;
3633         }
3634
3635         /* To handle GMAC own interrupts */
3636         if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) {
3637                 int status = priv->hw->mac->host_irq_status(priv->hw,
3638                                                             &priv->xstats);
3639
3640                 if (unlikely(status)) {
3641                         /* For LPI we need to save the tx status */
3642                         if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
3643                                 priv->tx_path_in_lpi_mode = true;
3644                         if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
3645                                 priv->tx_path_in_lpi_mode = false;
3646                 }
3647
3648                 if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3649                         for (queue = 0; queue < queues_count; queue++) {
3650                                 struct stmmac_rx_queue *rx_q =
3651                                 &priv->rx_queue[queue];
3652
3653                                 status |=
3654                                 priv->hw->mac->host_mtl_irq_status(priv->hw,
3655                                                                    queue);
3656
3657                                 if (status & CORE_IRQ_MTL_RX_OVERFLOW &&
3658                                     priv->hw->dma->set_rx_tail_ptr)
3659                                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
3660                                                                 rx_q->rx_tail_addr,
3661                                                                 queue);
3662                         }
3663                 }
3664
3665                 /* PCS link status */
3666                 if (priv->hw->pcs) {
3667                         if (priv->xstats.pcs_link)
3668                                 netif_carrier_on(dev);
3669                         else
3670                                 netif_carrier_off(dev);
3671                 }
3672         }
3673
3674         /* To handle DMA interrupts */
3675         stmmac_dma_interrupt(priv);
3676
3677         return IRQ_HANDLED;
3678 }
3679
3680 #ifdef CONFIG_NET_POLL_CONTROLLER
3681 /* Polling receive - used by NETCONSOLE and other diagnostic tools
3682  * to allow network I/O with interrupts disabled.
3683  */
3684 static void stmmac_poll_controller(struct net_device *dev)
3685 {
3686         disable_irq(dev->irq);
3687         stmmac_interrupt(dev->irq, dev);
3688         enable_irq(dev->irq);
3689 }
3690 #endif
3691
3692 /**
3693  *  stmmac_ioctl - Entry point for the Ioctl
3694  *  @dev: Device pointer.
3695  *  @rq: An IOCTL specefic structure, that can contain a pointer to
3696  *  a proprietary structure used to pass information to the driver.
3697  *  @cmd: IOCTL command
3698  *  Description:
3699  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
3700  */
3701 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3702 {
3703         int ret = -EOPNOTSUPP;
3704
3705         if (!netif_running(dev))
3706                 return -EINVAL;
3707
3708         switch (cmd) {
3709         case SIOCGMIIPHY:
3710         case SIOCGMIIREG:
3711         case SIOCSMIIREG:
3712                 if (!dev->phydev)
3713                         return -EINVAL;
3714                 ret = phy_mii_ioctl(dev->phydev, rq, cmd);
3715                 break;
3716         case SIOCSHWTSTAMP:
3717                 ret = stmmac_hwtstamp_ioctl(dev, rq);
3718                 break;
3719         default:
3720                 break;
3721         }
3722
3723         return ret;
3724 }
3725
3726 #ifdef CONFIG_DEBUG_FS
3727 static struct dentry *stmmac_fs_dir;
3728
3729 static void sysfs_display_ring(void *head, int size, int extend_desc,
3730                                struct seq_file *seq)
3731 {
3732         int i;
3733         struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
3734         struct dma_desc *p = (struct dma_desc *)head;
3735
3736         for (i = 0; i < size; i++) {
3737                 if (extend_desc) {
3738                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3739                                    i, (unsigned int)virt_to_phys(ep),
3740                                    le32_to_cpu(ep->basic.des0),
3741                                    le32_to_cpu(ep->basic.des1),
3742                                    le32_to_cpu(ep->basic.des2),
3743                                    le32_to_cpu(ep->basic.des3));
3744                         ep++;
3745                 } else {
3746                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3747                                    i, (unsigned int)virt_to_phys(p),
3748                                    le32_to_cpu(p->des0), le32_to_cpu(p->des1),
3749                                    le32_to_cpu(p->des2), le32_to_cpu(p->des3));
3750                         p++;
3751                 }
3752                 seq_printf(seq, "\n");
3753         }
3754 }
3755
3756 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v)
3757 {
3758         struct net_device *dev = seq->private;
3759         struct stmmac_priv *priv = netdev_priv(dev);
3760         u32 rx_count = priv->plat->rx_queues_to_use;
3761         u32 tx_count = priv->plat->tx_queues_to_use;
3762         u32 queue;
3763
3764         for (queue = 0; queue < rx_count; queue++) {
3765                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3766
3767                 seq_printf(seq, "RX Queue %d:\n", queue);
3768
3769                 if (priv->extend_desc) {
3770                         seq_printf(seq, "Extended descriptor ring:\n");
3771                         sysfs_display_ring((void *)rx_q->dma_erx,
3772                                            DMA_RX_SIZE, 1, seq);
3773                 } else {
3774                         seq_printf(seq, "Descriptor ring:\n");
3775                         sysfs_display_ring((void *)rx_q->dma_rx,
3776                                            DMA_RX_SIZE, 0, seq);
3777                 }
3778         }
3779
3780         for (queue = 0; queue < tx_count; queue++) {
3781                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
3782
3783                 seq_printf(seq, "TX Queue %d:\n", queue);
3784
3785                 if (priv->extend_desc) {
3786                         seq_printf(seq, "Extended descriptor ring:\n");
3787                         sysfs_display_ring((void *)tx_q->dma_etx,
3788                                            DMA_TX_SIZE, 1, seq);
3789                 } else {
3790                         seq_printf(seq, "Descriptor ring:\n");
3791                         sysfs_display_ring((void *)tx_q->dma_tx,
3792                                            DMA_TX_SIZE, 0, seq);
3793                 }
3794         }
3795
3796         return 0;
3797 }
3798
3799 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file)
3800 {
3801         return single_open(file, stmmac_sysfs_ring_read, inode->i_private);
3802 }
3803
3804 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */
3805
3806 static const struct file_operations stmmac_rings_status_fops = {
3807         .owner = THIS_MODULE,
3808         .open = stmmac_sysfs_ring_open,
3809         .read = seq_read,
3810         .llseek = seq_lseek,
3811         .release = single_release,
3812 };
3813
3814 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v)
3815 {
3816         struct net_device *dev = seq->private;
3817         struct stmmac_priv *priv = netdev_priv(dev);
3818
3819         if (!priv->hw_cap_support) {
3820                 seq_printf(seq, "DMA HW features not supported\n");
3821                 return 0;
3822         }
3823
3824         seq_printf(seq, "==============================\n");
3825         seq_printf(seq, "\tDMA HW features\n");
3826         seq_printf(seq, "==============================\n");
3827
3828         seq_printf(seq, "\t10/100 Mbps: %s\n",
3829                    (priv->dma_cap.mbps_10_100) ? "Y" : "N");
3830         seq_printf(seq, "\t1000 Mbps: %s\n",
3831                    (priv->dma_cap.mbps_1000) ? "Y" : "N");
3832         seq_printf(seq, "\tHalf duplex: %s\n",
3833                    (priv->dma_cap.half_duplex) ? "Y" : "N");
3834         seq_printf(seq, "\tHash Filter: %s\n",
3835                    (priv->dma_cap.hash_filter) ? "Y" : "N");
3836         seq_printf(seq, "\tMultiple MAC address registers: %s\n",
3837                    (priv->dma_cap.multi_addr) ? "Y" : "N");
3838         seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
3839                    (priv->dma_cap.pcs) ? "Y" : "N");
3840         seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
3841                    (priv->dma_cap.sma_mdio) ? "Y" : "N");
3842         seq_printf(seq, "\tPMT Remote wake up: %s\n",
3843                    (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
3844         seq_printf(seq, "\tPMT Magic Frame: %s\n",
3845                    (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
3846         seq_printf(seq, "\tRMON module: %s\n",
3847                    (priv->dma_cap.rmon) ? "Y" : "N");
3848         seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
3849                    (priv->dma_cap.time_stamp) ? "Y" : "N");
3850         seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
3851                    (priv->dma_cap.atime_stamp) ? "Y" : "N");
3852         seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
3853                    (priv->dma_cap.eee) ? "Y" : "N");
3854         seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
3855         seq_printf(seq, "\tChecksum Offload in TX: %s\n",
3856                    (priv->dma_cap.tx_coe) ? "Y" : "N");
3857         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3858                 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
3859                            (priv->dma_cap.rx_coe) ? "Y" : "N");
3860         } else {
3861                 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
3862                            (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
3863                 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
3864                            (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
3865         }
3866         seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
3867                    (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
3868         seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
3869                    priv->dma_cap.number_rx_channel);
3870         seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
3871                    priv->dma_cap.number_tx_channel);
3872         seq_printf(seq, "\tEnhanced descriptors: %s\n",
3873                    (priv->dma_cap.enh_desc) ? "Y" : "N");
3874
3875         return 0;
3876 }
3877
3878 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file)
3879 {
3880         return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private);
3881 }
3882
3883 static const struct file_operations stmmac_dma_cap_fops = {
3884         .owner = THIS_MODULE,
3885         .open = stmmac_sysfs_dma_cap_open,
3886         .read = seq_read,
3887         .llseek = seq_lseek,
3888         .release = single_release,
3889 };
3890
3891 static int stmmac_init_fs(struct net_device *dev)
3892 {
3893         struct stmmac_priv *priv = netdev_priv(dev);
3894
3895         /* Create per netdev entries */
3896         priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
3897
3898         if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) {
3899                 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n");
3900
3901                 return -ENOMEM;
3902         }
3903
3904         /* Entry to report DMA RX/TX rings */
3905         priv->dbgfs_rings_status =
3906                 debugfs_create_file("descriptors_status", S_IRUGO,
3907                                     priv->dbgfs_dir, dev,
3908                                     &stmmac_rings_status_fops);
3909
3910         if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) {
3911                 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n");
3912                 debugfs_remove_recursive(priv->dbgfs_dir);
3913
3914                 return -ENOMEM;
3915         }
3916
3917         /* Entry to report the DMA HW features */
3918         priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", S_IRUGO,
3919                                             priv->dbgfs_dir,
3920                                             dev, &stmmac_dma_cap_fops);
3921
3922         if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) {
3923                 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n");
3924                 debugfs_remove_recursive(priv->dbgfs_dir);
3925
3926                 return -ENOMEM;
3927         }
3928
3929         return 0;
3930 }
3931
3932 static void stmmac_exit_fs(struct net_device *dev)
3933 {
3934         struct stmmac_priv *priv = netdev_priv(dev);
3935
3936         debugfs_remove_recursive(priv->dbgfs_dir);
3937 }
3938 #endif /* CONFIG_DEBUG_FS */
3939
3940 static const struct net_device_ops stmmac_netdev_ops = {
3941         .ndo_open = stmmac_open,
3942         .ndo_start_xmit = stmmac_xmit,
3943         .ndo_stop = stmmac_release,
3944         .ndo_change_mtu = stmmac_change_mtu,
3945         .ndo_fix_features = stmmac_fix_features,
3946         .ndo_set_features = stmmac_set_features,
3947         .ndo_set_rx_mode = stmmac_set_rx_mode,
3948         .ndo_tx_timeout = stmmac_tx_timeout,
3949         .ndo_do_ioctl = stmmac_ioctl,
3950 #ifdef CONFIG_NET_POLL_CONTROLLER
3951         .ndo_poll_controller = stmmac_poll_controller,
3952 #endif
3953         .ndo_set_mac_address = eth_mac_addr,
3954 };
3955
3956 /**
3957  *  stmmac_hw_init - Init the MAC device
3958  *  @priv: driver private structure
3959  *  Description: this function is to configure the MAC device according to
3960  *  some platform parameters or the HW capability register. It prepares the
3961  *  driver to use either ring or chain modes and to setup either enhanced or
3962  *  normal descriptors.
3963  */
3964 static int stmmac_hw_init(struct stmmac_priv *priv)
3965 {
3966         struct mac_device_info *mac;
3967
3968         /* Identify the MAC HW device */
3969         if (priv->plat->setup) {
3970                 mac = priv->plat->setup(priv);
3971         } else if (priv->plat->has_gmac) {
3972                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
3973                 mac = dwmac1000_setup(priv->ioaddr,
3974                                       priv->plat->multicast_filter_bins,
3975                                       priv->plat->unicast_filter_entries,
3976                                       &priv->synopsys_id);
3977         } else if (priv->plat->has_gmac4) {
3978                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
3979                 mac = dwmac4_setup(priv->ioaddr,
3980                                    priv->plat->multicast_filter_bins,
3981                                    priv->plat->unicast_filter_entries,
3982                                    &priv->synopsys_id);
3983         } else {
3984                 mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id);
3985         }
3986         if (!mac)
3987                 return -ENOMEM;
3988
3989         priv->hw = mac;
3990
3991         /* dwmac-sun8i only work in chain mode */
3992         if (priv->plat->has_sun8i)
3993                 chain_mode = 1;
3994
3995         /* To use the chained or ring mode */
3996         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3997                 priv->hw->mode = &dwmac4_ring_mode_ops;
3998         } else {
3999                 if (chain_mode) {
4000                         priv->hw->mode = &chain_mode_ops;
4001                         dev_info(priv->device, "Chain mode enabled\n");
4002                         priv->mode = STMMAC_CHAIN_MODE;
4003                 } else {
4004                         priv->hw->mode = &ring_mode_ops;
4005                         dev_info(priv->device, "Ring mode enabled\n");
4006                         priv->mode = STMMAC_RING_MODE;
4007                 }
4008         }
4009
4010         /* Get the HW capability (new GMAC newer than 3.50a) */
4011         priv->hw_cap_support = stmmac_get_hw_features(priv);
4012         if (priv->hw_cap_support) {
4013                 dev_info(priv->device, "DMA HW capability register supported\n");
4014
4015                 /* We can override some gmac/dma configuration fields: e.g.
4016                  * enh_desc, tx_coe (e.g. that are passed through the
4017                  * platform) with the values from the HW capability
4018                  * register (if supported).
4019                  */
4020                 priv->plat->enh_desc = priv->dma_cap.enh_desc;
4021                 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
4022                 priv->hw->pmt = priv->plat->pmt;
4023
4024                 /* TXCOE doesn't work in thresh DMA mode */
4025                 if (priv->plat->force_thresh_dma_mode)
4026                         priv->plat->tx_coe = 0;
4027                 else
4028                         priv->plat->tx_coe = priv->dma_cap.tx_coe;
4029
4030                 /* In case of GMAC4 rx_coe is from HW cap register. */
4031                 priv->plat->rx_coe = priv->dma_cap.rx_coe;
4032
4033                 if (priv->dma_cap.rx_coe_type2)
4034                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
4035                 else if (priv->dma_cap.rx_coe_type1)
4036                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
4037
4038         } else {
4039                 dev_info(priv->device, "No HW DMA feature register supported\n");
4040         }
4041
4042         /* To use alternate (extended), normal or GMAC4 descriptor structures */
4043         if (priv->synopsys_id >= DWMAC_CORE_4_00)
4044                 priv->hw->desc = &dwmac4_desc_ops;
4045         else
4046                 stmmac_selec_desc_mode(priv);
4047
4048         if (priv->plat->rx_coe) {
4049                 priv->hw->rx_csum = priv->plat->rx_coe;
4050                 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
4051                 if (priv->synopsys_id < DWMAC_CORE_4_00)
4052                         dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
4053         }
4054         if (priv->plat->tx_coe)
4055                 dev_info(priv->device, "TX Checksum insertion supported\n");
4056
4057         if (priv->plat->pmt) {
4058                 dev_info(priv->device, "Wake-Up On Lan supported\n");
4059                 device_set_wakeup_capable(priv->device, 1);
4060         }
4061
4062         if (priv->dma_cap.tsoen)
4063                 dev_info(priv->device, "TSO supported\n");
4064
4065         return 0;
4066 }
4067
4068 /**
4069  * stmmac_dvr_probe
4070  * @device: device pointer
4071  * @plat_dat: platform data pointer
4072  * @res: stmmac resource pointer
4073  * Description: this is the main probe function used to
4074  * call the alloc_etherdev, allocate the priv structure.
4075  * Return:
4076  * returns 0 on success, otherwise errno.
4077  */
4078 int stmmac_dvr_probe(struct device *device,
4079                      struct plat_stmmacenet_data *plat_dat,
4080                      struct stmmac_resources *res)
4081 {
4082         struct net_device *ndev = NULL;
4083         struct stmmac_priv *priv;
4084         int ret = 0;
4085         u32 queue;
4086
4087         ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv),
4088                                   MTL_MAX_TX_QUEUES,
4089                                   MTL_MAX_RX_QUEUES);
4090         if (!ndev)
4091                 return -ENOMEM;
4092
4093         SET_NETDEV_DEV(ndev, device);
4094
4095         priv = netdev_priv(ndev);
4096         priv->device = device;
4097         priv->dev = ndev;
4098
4099         stmmac_set_ethtool_ops(ndev);
4100         priv->pause = pause;
4101         priv->plat = plat_dat;
4102         priv->ioaddr = res->addr;
4103         priv->dev->base_addr = (unsigned long)res->addr;
4104
4105         priv->dev->irq = res->irq;
4106         priv->wol_irq = res->wol_irq;
4107         priv->lpi_irq = res->lpi_irq;
4108
4109         if (res->mac)
4110                 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
4111
4112         dev_set_drvdata(device, priv->dev);
4113
4114         /* Verify driver arguments */
4115         stmmac_verify_args();
4116
4117         /* Override with kernel parameters if supplied XXX CRS XXX
4118          * this needs to have multiple instances
4119          */
4120         if ((phyaddr >= 0) && (phyaddr <= 31))
4121                 priv->plat->phy_addr = phyaddr;
4122
4123         if (priv->plat->stmmac_rst) {
4124                 ret = reset_control_assert(priv->plat->stmmac_rst);
4125                 reset_control_deassert(priv->plat->stmmac_rst);
4126                 /* Some reset controllers have only reset callback instead of
4127                  * assert + deassert callbacks pair.
4128                  */
4129                 if (ret == -ENOTSUPP)
4130                         reset_control_reset(priv->plat->stmmac_rst);
4131         }
4132
4133         /* Init MAC and get the capabilities */
4134         ret = stmmac_hw_init(priv);
4135         if (ret)
4136                 goto error_hw_init;
4137
4138         /* Configure real RX and TX queues */
4139         netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use);
4140         netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
4141
4142         ndev->netdev_ops = &stmmac_netdev_ops;
4143
4144         ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4145                             NETIF_F_RXCSUM;
4146
4147         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4148                 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
4149                 priv->tso = true;
4150                 dev_info(priv->device, "TSO feature enabled\n");
4151         }
4152         ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4153         ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4154 #ifdef STMMAC_VLAN_TAG_USED
4155         /* Both mac100 and gmac support receive VLAN tag detection */
4156         ndev->features |= NETIF_F_HW_VLAN_CTAG_RX;
4157 #endif
4158         priv->msg_enable = netif_msg_init(debug, default_msg_level);
4159
4160         /* MTU range: 46 - hw-specific max */
4161         ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
4162         if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
4163                 ndev->max_mtu = JUMBO_LEN;
4164         else
4165                 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
4166         /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
4167          * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
4168          */
4169         if ((priv->plat->maxmtu < ndev->max_mtu) &&
4170             (priv->plat->maxmtu >= ndev->min_mtu))
4171                 ndev->max_mtu = priv->plat->maxmtu;
4172         else if (priv->plat->maxmtu < ndev->min_mtu)
4173                 dev_warn(priv->device,
4174                          "%s: warning: maxmtu having invalid value (%d)\n",
4175                          __func__, priv->plat->maxmtu);
4176
4177         if (flow_ctrl)
4178                 priv->flow_ctrl = FLOW_AUTO;    /* RX/TX pause on */
4179
4180         /* Rx Watchdog is available in the COREs newer than the 3.40.
4181          * In some case, for example on bugged HW this feature
4182          * has to be disable and this can be done by passing the
4183          * riwt_off field from the platform.
4184          */
4185         if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) {
4186                 priv->use_riwt = 1;
4187                 dev_info(priv->device,
4188                          "Enable RX Mitigation via HW Watchdog Timer\n");
4189         }
4190
4191         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4192                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4193
4194                 netif_napi_add(ndev, &rx_q->napi, stmmac_poll,
4195                                (8 * priv->plat->rx_queues_to_use));
4196         }
4197
4198         spin_lock_init(&priv->lock);
4199
4200         /* If a specific clk_csr value is passed from the platform
4201          * this means that the CSR Clock Range selection cannot be
4202          * changed at run-time and it is fixed. Viceversa the driver'll try to
4203          * set the MDC clock dynamically according to the csr actual
4204          * clock input.
4205          */
4206         if (!priv->plat->clk_csr)
4207                 stmmac_clk_csr_set(priv);
4208         else
4209                 priv->clk_csr = priv->plat->clk_csr;
4210
4211         stmmac_check_pcs_mode(priv);
4212
4213         if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
4214             priv->hw->pcs != STMMAC_PCS_TBI &&
4215             priv->hw->pcs != STMMAC_PCS_RTBI) {
4216                 /* MDIO bus Registration */
4217                 ret = stmmac_mdio_register(ndev);
4218                 if (ret < 0) {
4219                         dev_err(priv->device,
4220                                 "%s: MDIO bus (id: %d) registration failed",
4221                                 __func__, priv->plat->bus_id);
4222                         goto error_mdio_register;
4223                 }
4224         }
4225
4226         ret = register_netdev(ndev);
4227         if (ret) {
4228                 dev_err(priv->device, "%s: ERROR %i registering the device\n",
4229                         __func__, ret);
4230                 goto error_netdev_register;
4231         }
4232
4233         return ret;
4234
4235 error_netdev_register:
4236         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4237             priv->hw->pcs != STMMAC_PCS_TBI &&
4238             priv->hw->pcs != STMMAC_PCS_RTBI)
4239                 stmmac_mdio_unregister(ndev);
4240 error_mdio_register:
4241         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4242                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4243
4244                 netif_napi_del(&rx_q->napi);
4245         }
4246 error_hw_init:
4247         free_netdev(ndev);
4248
4249         return ret;
4250 }
4251 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
4252
4253 /**
4254  * stmmac_dvr_remove
4255  * @dev: device pointer
4256  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
4257  * changes the link status, releases the DMA descriptor rings.
4258  */
4259 int stmmac_dvr_remove(struct device *dev)
4260 {
4261         struct net_device *ndev = dev_get_drvdata(dev);
4262         struct stmmac_priv *priv = netdev_priv(ndev);
4263
4264         netdev_info(priv->dev, "%s: removing driver", __func__);
4265
4266         stmmac_stop_all_dma(priv);
4267
4268         priv->hw->mac->set_mac(priv->ioaddr, false);
4269         netif_carrier_off(ndev);
4270         unregister_netdev(ndev);
4271         if (priv->plat->stmmac_rst)
4272                 reset_control_assert(priv->plat->stmmac_rst);
4273         clk_disable_unprepare(priv->plat->pclk);
4274         clk_disable_unprepare(priv->plat->stmmac_clk);
4275         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4276             priv->hw->pcs != STMMAC_PCS_TBI &&
4277             priv->hw->pcs != STMMAC_PCS_RTBI)
4278                 stmmac_mdio_unregister(ndev);
4279         free_netdev(ndev);
4280
4281         return 0;
4282 }
4283 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
4284
4285 /**
4286  * stmmac_suspend - suspend callback
4287  * @dev: device pointer
4288  * Description: this is the function to suspend the device and it is called
4289  * by the platform driver to stop the network queue, release the resources,
4290  * program the PMT register (for WoL), clean and release driver resources.
4291  */
4292 int stmmac_suspend(struct device *dev)
4293 {
4294         struct net_device *ndev = dev_get_drvdata(dev);
4295         struct stmmac_priv *priv = netdev_priv(ndev);
4296         unsigned long flags;
4297
4298         if (!ndev || !netif_running(ndev))
4299                 return 0;
4300
4301         if (ndev->phydev)
4302                 phy_stop(ndev->phydev);
4303
4304         spin_lock_irqsave(&priv->lock, flags);
4305
4306         netif_device_detach(ndev);
4307         stmmac_stop_all_queues(priv);
4308
4309         stmmac_disable_all_queues(priv);
4310
4311         /* Stop TX/RX DMA */
4312         stmmac_stop_all_dma(priv);
4313
4314         /* Enable Power down mode by programming the PMT regs */
4315         if (device_may_wakeup(priv->device)) {
4316                 priv->hw->mac->pmt(priv->hw, priv->wolopts);
4317                 priv->irq_wake = 1;
4318         } else {
4319                 priv->hw->mac->set_mac(priv->ioaddr, false);
4320                 pinctrl_pm_select_sleep_state(priv->device);
4321                 /* Disable clock in case of PWM is off */
4322                 clk_disable(priv->plat->pclk);
4323                 clk_disable(priv->plat->stmmac_clk);
4324         }
4325         spin_unlock_irqrestore(&priv->lock, flags);
4326
4327         priv->oldlink = false;
4328         priv->speed = SPEED_UNKNOWN;
4329         priv->oldduplex = DUPLEX_UNKNOWN;
4330         return 0;
4331 }
4332 EXPORT_SYMBOL_GPL(stmmac_suspend);
4333
4334 /**
4335  * stmmac_reset_queues_param - reset queue parameters
4336  * @dev: device pointer
4337  */
4338 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
4339 {
4340         u32 rx_cnt = priv->plat->rx_queues_to_use;
4341         u32 tx_cnt = priv->plat->tx_queues_to_use;
4342         u32 queue;
4343
4344         for (queue = 0; queue < rx_cnt; queue++) {
4345                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4346
4347                 rx_q->cur_rx = 0;
4348                 rx_q->dirty_rx = 0;
4349         }
4350
4351         for (queue = 0; queue < tx_cnt; queue++) {
4352                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4353
4354                 tx_q->cur_tx = 0;
4355                 tx_q->dirty_tx = 0;
4356         }
4357 }
4358
4359 /**
4360  * stmmac_resume - resume callback
4361  * @dev: device pointer
4362  * Description: when resume this function is invoked to setup the DMA and CORE
4363  * in a usable state.
4364  */
4365 int stmmac_resume(struct device *dev)
4366 {
4367         struct net_device *ndev = dev_get_drvdata(dev);
4368         struct stmmac_priv *priv = netdev_priv(ndev);
4369         unsigned long flags;
4370
4371         if (!netif_running(ndev))
4372                 return 0;
4373
4374         /* Power Down bit, into the PM register, is cleared
4375          * automatically as soon as a magic packet or a Wake-up frame
4376          * is received. Anyway, it's better to manually clear
4377          * this bit because it can generate problems while resuming
4378          * from another devices (e.g. serial console).
4379          */
4380         if (device_may_wakeup(priv->device)) {
4381                 spin_lock_irqsave(&priv->lock, flags);
4382                 priv->hw->mac->pmt(priv->hw, 0);
4383                 spin_unlock_irqrestore(&priv->lock, flags);
4384                 priv->irq_wake = 0;
4385         } else {
4386                 pinctrl_pm_select_default_state(priv->device);
4387                 /* enable the clk previously disabled */
4388                 clk_enable(priv->plat->stmmac_clk);
4389                 clk_enable(priv->plat->pclk);
4390                 /* reset the phy so that it's ready */
4391                 if (priv->mii)
4392                         stmmac_mdio_reset(priv->mii);
4393         }
4394
4395         netif_device_attach(ndev);
4396
4397         spin_lock_irqsave(&priv->lock, flags);
4398
4399         stmmac_reset_queues_param(priv);
4400
4401         /* reset private mss value to force mss context settings at
4402          * next tso xmit (only used for gmac4).
4403          */
4404         priv->mss = 0;
4405
4406         stmmac_clear_descriptors(priv);
4407
4408         stmmac_hw_setup(ndev, false);
4409         stmmac_init_tx_coalesce(priv);
4410         stmmac_set_rx_mode(ndev);
4411
4412         stmmac_enable_all_queues(priv);
4413
4414         stmmac_start_all_queues(priv);
4415
4416         spin_unlock_irqrestore(&priv->lock, flags);
4417
4418         if (ndev->phydev)
4419                 phy_start(ndev->phydev);
4420
4421         return 0;
4422 }
4423 EXPORT_SYMBOL_GPL(stmmac_resume);
4424
4425 #ifndef MODULE
4426 static int __init stmmac_cmdline_opt(char *str)
4427 {
4428         char *opt;
4429
4430         if (!str || !*str)
4431                 return -EINVAL;
4432         while ((opt = strsep(&str, ",")) != NULL) {
4433                 if (!strncmp(opt, "debug:", 6)) {
4434                         if (kstrtoint(opt + 6, 0, &debug))
4435                                 goto err;
4436                 } else if (!strncmp(opt, "phyaddr:", 8)) {
4437                         if (kstrtoint(opt + 8, 0, &phyaddr))
4438                                 goto err;
4439                 } else if (!strncmp(opt, "buf_sz:", 7)) {
4440                         if (kstrtoint(opt + 7, 0, &buf_sz))
4441                                 goto err;
4442                 } else if (!strncmp(opt, "tc:", 3)) {
4443                         if (kstrtoint(opt + 3, 0, &tc))
4444                                 goto err;
4445                 } else if (!strncmp(opt, "watchdog:", 9)) {
4446                         if (kstrtoint(opt + 9, 0, &watchdog))
4447                                 goto err;
4448                 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
4449                         if (kstrtoint(opt + 10, 0, &flow_ctrl))
4450                                 goto err;
4451                 } else if (!strncmp(opt, "pause:", 6)) {
4452                         if (kstrtoint(opt + 6, 0, &pause))
4453                                 goto err;
4454                 } else if (!strncmp(opt, "eee_timer:", 10)) {
4455                         if (kstrtoint(opt + 10, 0, &eee_timer))
4456                                 goto err;
4457                 } else if (!strncmp(opt, "chain_mode:", 11)) {
4458                         if (kstrtoint(opt + 11, 0, &chain_mode))
4459                                 goto err;
4460                 }
4461         }
4462         return 0;
4463
4464 err:
4465         pr_err("%s: ERROR broken module parameter conversion", __func__);
4466         return -EINVAL;
4467 }
4468
4469 __setup("stmmaceth=", stmmac_cmdline_opt);
4470 #endif /* MODULE */
4471
4472 static int __init stmmac_init(void)
4473 {
4474 #ifdef CONFIG_DEBUG_FS
4475         /* Create debugfs main directory if it doesn't exist yet */
4476         if (!stmmac_fs_dir) {
4477                 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
4478
4479                 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) {
4480                         pr_err("ERROR %s, debugfs create directory failed\n",
4481                                STMMAC_RESOURCE_NAME);
4482
4483                         return -ENOMEM;
4484                 }
4485         }
4486 #endif
4487
4488         return 0;
4489 }
4490
4491 static void __exit stmmac_exit(void)
4492 {
4493 #ifdef CONFIG_DEBUG_FS
4494         debugfs_remove_recursive(stmmac_fs_dir);
4495 #endif
4496 }
4497
4498 module_init(stmmac_init)
4499 module_exit(stmmac_exit)
4500
4501 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
4502 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
4503 MODULE_LICENSE("GPL");