drivers/net/ethernet/broadcom/genet/bcmgenet.c

   1 /*
   2  * Broadcom GENET (Gigabit Ethernet) controller driver
   3  *
   4  * Copyright (c) 2014 Broadcom Corporation
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #define pr_fmt(fmt)                             "bcmgenet: " fmt
  12
  13 #include <linux/kernel.h>
  14 #include <linux/module.h>
  15 #include <linux/sched.h>
  16 #include <linux/types.h>
  17 #include <linux/fcntl.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/string.h>
  20 #include <linux/if_ether.h>
  21 #include <linux/init.h>
  22 #include <linux/errno.h>
  23 #include <linux/delay.h>
  24 #include <linux/platform_device.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/pm.h>
  27 #include <linux/clk.h>
  28 #include <linux/of.h>
  29 #include <linux/of_address.h>
  30 #include <linux/of_irq.h>
  31 #include <linux/of_net.h>
  32 #include <linux/of_platform.h>
  33 #include <net/arp.h>
  34
  35 #include <linux/mii.h>
  36 #include <linux/ethtool.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/inetdevice.h>
  39 #include <linux/etherdevice.h>
  40 #include <linux/skbuff.h>
  41 #include <linux/in.h>
  42 #include <linux/ip.h>
  43 #include <linux/ipv6.h>
  44 #include <linux/phy.h>
  45 #include <linux/platform_data/bcmgenet.h>
  46
  47 #include <asm/unaligned.h>
  48
  49 #include "bcmgenet.h"
  50
  51 /* Maximum number of hardware queues, downsized if needed */
  52 #define GENET_MAX_MQ_CNT        4
  53
  54 /* Default highest priority queue for multi queue support */
  55 #define GENET_Q0_PRIORITY       0
  56
  57 #define GENET_Q16_RX_BD_CNT     \
  58         (TOTAL_DESC - priv->hw_params->rx_queues * priv->hw_params->rx_bds_per_q)
  59 #define GENET_Q16_TX_BD_CNT     \
  60         (TOTAL_DESC - priv->hw_params->tx_queues * priv->hw_params->tx_bds_per_q)
  61
  62 #define RX_BUF_LENGTH           2048
  63 #define SKB_ALIGNMENT           32
  64
  65 /* Tx/Rx DMA register offset, skip 256 descriptors */
  66 #define WORDS_PER_BD(p)         (p->hw_params->words_per_bd)
  67 #define DMA_DESC_SIZE           (WORDS_PER_BD(priv) * sizeof(u32))
  68
  69 #define GENET_TDMA_REG_OFF      (priv->hw_params->tdma_offset + \
  70                                 TOTAL_DESC * DMA_DESC_SIZE)
  71
  72 #define GENET_RDMA_REG_OFF      (priv->hw_params->rdma_offset + \
  73                                 TOTAL_DESC * DMA_DESC_SIZE)
  74
  75 static inline void dmadesc_set_length_status(struct bcmgenet_priv *priv,
  76                                              void __iomem *d, u32 value)
  77 {
  78         __raw_writel(value, d + DMA_DESC_LENGTH_STATUS);
  79 }
  80
  81 static inline u32 dmadesc_get_length_status(struct bcmgenet_priv *priv,
  82                                             void __iomem *d)
  83 {
  84         return __raw_readl(d + DMA_DESC_LENGTH_STATUS);
  85 }
  86
  87 static inline void dmadesc_set_addr(struct bcmgenet_priv *priv,
  88                                     void __iomem *d,
  89                                     dma_addr_t addr)
  90 {
  91         __raw_writel(lower_32_bits(addr), d + DMA_DESC_ADDRESS_LO);
  92
  93         /* Register writes to GISB bus can take couple hundred nanoseconds
  94          * and are done for each packet, save these expensive writes unless
  95          * the platform is explicitly configured for 64-bits/LPAE.
  96          */
  97 #ifdef CONFIG_PHYS_ADDR_T_64BIT
  98         if (priv->hw_params->flags & GENET_HAS_40BITS)
  99                 __raw_writel(upper_32_bits(addr), d + DMA_DESC_ADDRESS_HI);
 100 #endif
 101 }
 102
 103 /* Combined address + length/status setter */
 104 static inline void dmadesc_set(struct bcmgenet_priv *priv,
 105                                void __iomem *d, dma_addr_t addr, u32 val)
 106 {
 107         dmadesc_set_length_status(priv, d, val);
 108         dmadesc_set_addr(priv, d, addr);
 109 }
 110
 111 static inline dma_addr_t dmadesc_get_addr(struct bcmgenet_priv *priv,
 112                                           void __iomem *d)
 113 {
 114         dma_addr_t addr;
 115
 116         addr = __raw_readl(d + DMA_DESC_ADDRESS_LO);
 117
 118         /* Register writes to GISB bus can take couple hundred nanoseconds
 119          * and are done for each packet, save these expensive writes unless
 120          * the platform is explicitly configured for 64-bits/LPAE.
 121          */
 122 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 123         if (priv->hw_params->flags & GENET_HAS_40BITS)
 124                 addr |= (u64)__raw_readl(d + DMA_DESC_ADDRESS_HI) << 32;
 125 #endif
 126         return addr;
 127 }
 128
 129 #define GENET_VER_FMT   "%1d.%1d EPHY: 0x%04x"
 130
 131 #define GENET_MSG_DEFAULT       (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
 132                                 NETIF_MSG_LINK)
 133
 134 static inline u32 bcmgenet_rbuf_ctrl_get(struct bcmgenet_priv *priv)
 135 {
 136         if (GENET_IS_V1(priv))
 137                 return bcmgenet_rbuf_readl(priv, RBUF_FLUSH_CTRL_V1);
 138         else
 139                 return bcmgenet_sys_readl(priv, SYS_RBUF_FLUSH_CTRL);
 140 }
 141
 142 static inline void bcmgenet_rbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
 143 {
 144         if (GENET_IS_V1(priv))
 145                 bcmgenet_rbuf_writel(priv, val, RBUF_FLUSH_CTRL_V1);
 146         else
 147                 bcmgenet_sys_writel(priv, val, SYS_RBUF_FLUSH_CTRL);
 148 }
 149
 150 /* These macros are defined to deal with register map change
 151  * between GENET1.1 and GENET2. Only those currently being used
 152  * by driver are defined.
 153  */
 154 static inline u32 bcmgenet_tbuf_ctrl_get(struct bcmgenet_priv *priv)
 155 {
 156         if (GENET_IS_V1(priv))
 157                 return bcmgenet_rbuf_readl(priv, TBUF_CTRL_V1);
 158         else
 159                 return __raw_readl(priv->base +
 160                                 priv->hw_params->tbuf_offset + TBUF_CTRL);
 161 }
 162
 163 static inline void bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
 164 {
 165         if (GENET_IS_V1(priv))
 166                 bcmgenet_rbuf_writel(priv, val, TBUF_CTRL_V1);
 167         else
 168                 __raw_writel(val, priv->base +
 169                                 priv->hw_params->tbuf_offset + TBUF_CTRL);
 170 }
 171
 172 static inline u32 bcmgenet_bp_mc_get(struct bcmgenet_priv *priv)
 173 {
 174         if (GENET_IS_V1(priv))
 175                 return bcmgenet_rbuf_readl(priv, TBUF_BP_MC_V1);
 176         else
 177                 return __raw_readl(priv->base +
 178                                 priv->hw_params->tbuf_offset + TBUF_BP_MC);
 179 }
 180
 181 static inline void bcmgenet_bp_mc_set(struct bcmgenet_priv *priv, u32 val)
 182 {
 183         if (GENET_IS_V1(priv))
 184                 bcmgenet_rbuf_writel(priv, val, TBUF_BP_MC_V1);
 185         else
 186                 __raw_writel(val, priv->base +
 187                                 priv->hw_params->tbuf_offset + TBUF_BP_MC);
 188 }
 189
 190 /* RX/TX DMA register accessors */
 191 enum dma_reg {
 192         DMA_RING_CFG = 0,
 193         DMA_CTRL,
 194         DMA_STATUS,
 195         DMA_SCB_BURST_SIZE,
 196         DMA_ARB_CTRL,
 197         DMA_PRIORITY_0,
 198         DMA_PRIORITY_1,
 199         DMA_PRIORITY_2,
 200         DMA_INDEX2RING_0,
 201         DMA_INDEX2RING_1,
 202         DMA_INDEX2RING_2,
 203         DMA_INDEX2RING_3,
 204         DMA_INDEX2RING_4,
 205         DMA_INDEX2RING_5,
 206         DMA_INDEX2RING_6,
 207         DMA_INDEX2RING_7,
 208 };
 209
 210 static const u8 bcmgenet_dma_regs_v3plus[] = {
 211         [DMA_RING_CFG]          = 0x00,
 212         [DMA_CTRL]              = 0x04,
 213         [DMA_STATUS]            = 0x08,
 214         [DMA_SCB_BURST_SIZE]    = 0x0C,
 215         [DMA_ARB_CTRL]          = 0x2C,
 216         [DMA_PRIORITY_0]        = 0x30,
 217         [DMA_PRIORITY_1]        = 0x34,
 218         [DMA_PRIORITY_2]        = 0x38,
 219         [DMA_INDEX2RING_0]      = 0x70,
 220         [DMA_INDEX2RING_1]      = 0x74,
 221         [DMA_INDEX2RING_2]      = 0x78,
 222         [DMA_INDEX2RING_3]      = 0x7C,
 223         [DMA_INDEX2RING_4]      = 0x80,
 224         [DMA_INDEX2RING_5]      = 0x84,
 225         [DMA_INDEX2RING_6]      = 0x88,
 226         [DMA_INDEX2RING_7]      = 0x8C,
 227 };
 228
 229 static const u8 bcmgenet_dma_regs_v2[] = {
 230         [DMA_RING_CFG]          = 0x00,
 231         [DMA_CTRL]              = 0x04,
 232         [DMA_STATUS]            = 0x08,
 233         [DMA_SCB_BURST_SIZE]    = 0x0C,
 234         [DMA_ARB_CTRL]          = 0x30,
 235         [DMA_PRIORITY_0]        = 0x34,
 236         [DMA_PRIORITY_1]        = 0x38,
 237         [DMA_PRIORITY_2]        = 0x3C,
 238 };
 239
 240 static const u8 bcmgenet_dma_regs_v1[] = {
 241         [DMA_CTRL]              = 0x00,
 242         [DMA_STATUS]            = 0x04,
 243         [DMA_SCB_BURST_SIZE]    = 0x0C,
 244         [DMA_ARB_CTRL]          = 0x30,
 245         [DMA_PRIORITY_0]        = 0x34,
 246         [DMA_PRIORITY_1]        = 0x38,
 247         [DMA_PRIORITY_2]        = 0x3C,
 248 };
 249
 250 /* Set at runtime once bcmgenet version is known */
 251 static const u8 *bcmgenet_dma_regs;
 252
 253 static inline struct bcmgenet_priv *dev_to_priv(struct device *dev)
 254 {
 255         return netdev_priv(dev_get_drvdata(dev));
 256 }
 257
 258 static inline u32 bcmgenet_tdma_readl(struct bcmgenet_priv *priv,
 259                                       enum dma_reg r)
 260 {
 261         return __raw_readl(priv->base + GENET_TDMA_REG_OFF +
 262                         DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 263 }
 264
 265 static inline void bcmgenet_tdma_writel(struct bcmgenet_priv *priv,
 266                                         u32 val, enum dma_reg r)
 267 {
 268         __raw_writel(val, priv->base + GENET_TDMA_REG_OFF +
 269                         DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 270 }
 271
 272 static inline u32 bcmgenet_rdma_readl(struct bcmgenet_priv *priv,
 273                                       enum dma_reg r)
 274 {
 275         return __raw_readl(priv->base + GENET_RDMA_REG_OFF +
 276                         DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 277 }
 278
 279 static inline void bcmgenet_rdma_writel(struct bcmgenet_priv *priv,
 280                                         u32 val, enum dma_reg r)
 281 {
 282         __raw_writel(val, priv->base + GENET_RDMA_REG_OFF +
 283                         DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 284 }
 285
 286 /* RDMA/TDMA ring registers and accessors
 287  * we merge the common fields and just prefix with T/D the registers
 288  * having different meaning depending on the direction
 289  */
 290 enum dma_ring_reg {
 291         TDMA_READ_PTR = 0,
 292         RDMA_WRITE_PTR = TDMA_READ_PTR,
 293         TDMA_READ_PTR_HI,
 294         RDMA_WRITE_PTR_HI = TDMA_READ_PTR_HI,
 295         TDMA_CONS_INDEX,
 296         RDMA_PROD_INDEX = TDMA_CONS_INDEX,
 297         TDMA_PROD_INDEX,
 298         RDMA_CONS_INDEX = TDMA_PROD_INDEX,
 299         DMA_RING_BUF_SIZE,
 300         DMA_START_ADDR,
 301         DMA_START_ADDR_HI,
 302         DMA_END_ADDR,
 303         DMA_END_ADDR_HI,
 304         DMA_MBUF_DONE_THRESH,
 305         TDMA_FLOW_PERIOD,
 306         RDMA_XON_XOFF_THRESH = TDMA_FLOW_PERIOD,
 307         TDMA_WRITE_PTR,
 308         RDMA_READ_PTR = TDMA_WRITE_PTR,
 309         TDMA_WRITE_PTR_HI,
 310         RDMA_READ_PTR_HI = TDMA_WRITE_PTR_HI
 311 };
 312
 313 /* GENET v4 supports 40-bits pointer addressing
 314  * for obvious reasons the LO and HI word parts
 315  * are contiguous, but this offsets the other
 316  * registers.
 317  */
 318 static const u8 genet_dma_ring_regs_v4[] = {
 319         [TDMA_READ_PTR]                 = 0x00,
 320         [TDMA_READ_PTR_HI]              = 0x04,
 321         [TDMA_CONS_INDEX]               = 0x08,
 322         [TDMA_PROD_INDEX]               = 0x0C,
 323         [DMA_RING_BUF_SIZE]             = 0x10,
 324         [DMA_START_ADDR]                = 0x14,
 325         [DMA_START_ADDR_HI]             = 0x18,
 326         [DMA_END_ADDR]                  = 0x1C,
 327         [DMA_END_ADDR_HI]               = 0x20,
 328         [DMA_MBUF_DONE_THRESH]          = 0x24,
 329         [TDMA_FLOW_PERIOD]              = 0x28,
 330         [TDMA_WRITE_PTR]                = 0x2C,
 331         [TDMA_WRITE_PTR_HI]             = 0x30,
 332 };
 333
 334 static const u8 genet_dma_ring_regs_v123[] = {
 335         [TDMA_READ_PTR]                 = 0x00,
 336         [TDMA_CONS_INDEX]               = 0x04,
 337         [TDMA_PROD_INDEX]               = 0x08,
 338         [DMA_RING_BUF_SIZE]             = 0x0C,
 339         [DMA_START_ADDR]                = 0x10,
 340         [DMA_END_ADDR]                  = 0x14,
 341         [DMA_MBUF_DONE_THRESH]          = 0x18,
 342         [TDMA_FLOW_PERIOD]              = 0x1C,
 343         [TDMA_WRITE_PTR]                = 0x20,
 344 };
 345
 346 /* Set at runtime once GENET version is known */
 347 static const u8 *genet_dma_ring_regs;
 348
 349 static inline u32 bcmgenet_tdma_ring_readl(struct bcmgenet_priv *priv,
 350                                            unsigned int ring,
 351                                            enum dma_ring_reg r)
 352 {
 353         return __raw_readl(priv->base + GENET_TDMA_REG_OFF +
 354                         (DMA_RING_SIZE * ring) +
 355                         genet_dma_ring_regs[r]);
 356 }
 357
 358 static inline void bcmgenet_tdma_ring_writel(struct bcmgenet_priv *priv,
 359                                              unsigned int ring, u32 val,
 360                                              enum dma_ring_reg r)
 361 {
 362         __raw_writel(val, priv->base + GENET_TDMA_REG_OFF +
 363                         (DMA_RING_SIZE * ring) +
 364                         genet_dma_ring_regs[r]);
 365 }
 366
 367 static inline u32 bcmgenet_rdma_ring_readl(struct bcmgenet_priv *priv,
 368                                            unsigned int ring,
 369                                            enum dma_ring_reg r)
 370 {
 371         return __raw_readl(priv->base + GENET_RDMA_REG_OFF +
 372                         (DMA_RING_SIZE * ring) +
 373                         genet_dma_ring_regs[r]);
 374 }
 375
 376 static inline void bcmgenet_rdma_ring_writel(struct bcmgenet_priv *priv,
 377                                              unsigned int ring, u32 val,
 378                                              enum dma_ring_reg r)
 379 {
 380         __raw_writel(val, priv->base + GENET_RDMA_REG_OFF +
 381                         (DMA_RING_SIZE * ring) +
 382                         genet_dma_ring_regs[r]);
 383 }
 384
 385 static int bcmgenet_get_settings(struct net_device *dev,
 386                                  struct ethtool_cmd *cmd)
 387 {
 388         struct bcmgenet_priv *priv = netdev_priv(dev);
 389
 390         if (!netif_running(dev))
 391                 return -EINVAL;
 392
 393         if (!priv->phydev)
 394                 return -ENODEV;
 395
 396         return phy_ethtool_gset(priv->phydev, cmd);
 397 }
 398
 399 static int bcmgenet_set_settings(struct net_device *dev,
 400                                  struct ethtool_cmd *cmd)
 401 {
 402         struct bcmgenet_priv *priv = netdev_priv(dev);
 403
 404         if (!netif_running(dev))
 405                 return -EINVAL;
 406
 407         if (!priv->phydev)
 408                 return -ENODEV;
 409
 410         return phy_ethtool_sset(priv->phydev, cmd);
 411 }
 412
 413 static int bcmgenet_set_rx_csum(struct net_device *dev,
 414                                 netdev_features_t wanted)
 415 {
 416         struct bcmgenet_priv *priv = netdev_priv(dev);
 417         u32 rbuf_chk_ctrl;
 418         bool rx_csum_en;
 419
 420         rx_csum_en = !!(wanted & NETIF_F_RXCSUM);
 421
 422         rbuf_chk_ctrl = bcmgenet_rbuf_readl(priv, RBUF_CHK_CTRL);
 423
 424         /* enable rx checksumming */
 425         if (rx_csum_en)
 426                 rbuf_chk_ctrl |= RBUF_RXCHK_EN;
 427         else
 428                 rbuf_chk_ctrl &= ~RBUF_RXCHK_EN;
 429         priv->desc_rxchk_en = rx_csum_en;
 430
 431         /* If UniMAC forwards CRC, we need to skip over it to get
 432          * a valid CHK bit to be set in the per-packet status word
 433         */
 434         if (rx_csum_en && priv->crc_fwd_en)
 435                 rbuf_chk_ctrl |= RBUF_SKIP_FCS;
 436         else
 437                 rbuf_chk_ctrl &= ~RBUF_SKIP_FCS;
 438
 439         bcmgenet_rbuf_writel(priv, rbuf_chk_ctrl, RBUF_CHK_CTRL);
 440
 441         return 0;
 442 }
 443
 444 static int bcmgenet_set_tx_csum(struct net_device *dev,
 445                                 netdev_features_t wanted)
 446 {
 447         struct bcmgenet_priv *priv = netdev_priv(dev);
 448         bool desc_64b_en;
 449         u32 tbuf_ctrl, rbuf_ctrl;
 450
 451         tbuf_ctrl = bcmgenet_tbuf_ctrl_get(priv);
 452         rbuf_ctrl = bcmgenet_rbuf_readl(priv, RBUF_CTRL);
 453
 454         desc_64b_en = !!(wanted & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM));
 455
 456         /* enable 64 bytes descriptor in both directions (RBUF and TBUF) */
 457         if (desc_64b_en) {
 458                 tbuf_ctrl |= RBUF_64B_EN;
 459                 rbuf_ctrl |= RBUF_64B_EN;
 460         } else {
 461                 tbuf_ctrl &= ~RBUF_64B_EN;
 462                 rbuf_ctrl &= ~RBUF_64B_EN;
 463         }
 464         priv->desc_64b_en = desc_64b_en;
 465
 466         bcmgenet_tbuf_ctrl_set(priv, tbuf_ctrl);
 467         bcmgenet_rbuf_writel(priv, rbuf_ctrl, RBUF_CTRL);
 468
 469         return 0;
 470 }
 471
 472 static int bcmgenet_set_features(struct net_device *dev,
 473                                  netdev_features_t features)
 474 {
 475         netdev_features_t changed = features ^ dev->features;
 476         netdev_features_t wanted = dev->wanted_features;
 477         int ret = 0;
 478
 479         if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))
 480                 ret = bcmgenet_set_tx_csum(dev, wanted);
 481         if (changed & (NETIF_F_RXCSUM))
 482                 ret = bcmgenet_set_rx_csum(dev, wanted);
 483
 484         return ret;
 485 }
 486
 487 static u32 bcmgenet_get_msglevel(struct net_device *dev)
 488 {
 489         struct bcmgenet_priv *priv = netdev_priv(dev);
 490
 491         return priv->msg_enable;
 492 }
 493
 494 static void bcmgenet_set_msglevel(struct net_device *dev, u32 level)
 495 {
 496         struct bcmgenet_priv *priv = netdev_priv(dev);
 497
 498         priv->msg_enable = level;
 499 }
 500
 501 static int bcmgenet_get_coalesce(struct net_device *dev,
 502                                  struct ethtool_coalesce *ec)
 503 {
 504         struct bcmgenet_priv *priv = netdev_priv(dev);
 505
 506         ec->tx_max_coalesced_frames =
 507                 bcmgenet_tdma_ring_readl(priv, DESC_INDEX,
 508                                          DMA_MBUF_DONE_THRESH);
 509
 510         return 0;
 511 }
 512
 513 static int bcmgenet_set_coalesce(struct net_device *dev,
 514                                  struct ethtool_coalesce *ec)
 515 {
 516         struct bcmgenet_priv *priv = netdev_priv(dev);
 517         unsigned int i;
 518
 519         if (ec->tx_max_coalesced_frames > DMA_INTR_THRESHOLD_MASK ||
 520             ec->tx_max_coalesced_frames == 0)
 521                 return -EINVAL;
 522
 523         /* GENET TDMA hardware does not support a configurable timeout, but will
 524          * always generate an interrupt either after MBDONE packets have been
 525          * transmitted, or when the ring is emtpy.
 526          */
 527         if (ec->tx_coalesce_usecs || ec->tx_coalesce_usecs_high ||
 528             ec->tx_coalesce_usecs_irq || ec->tx_coalesce_usecs_high ||
 529             ec->tx_coalesce_usecs_low)
 530                 return -EOPNOTSUPP;
 531
 532         /* Program all TX queues with the same values, as there is no
 533          * ethtool knob to do coalescing on a per-queue basis
 534          */
 535         for (i = 0; i < priv->hw_params->tx_queues; i++)
 536                 bcmgenet_tdma_ring_writel(priv, i,
 537                                           ec->tx_max_coalesced_frames,
 538                                           DMA_MBUF_DONE_THRESH);
 539         bcmgenet_tdma_ring_writel(priv, DESC_INDEX,
 540                                   ec->tx_max_coalesced_frames,
 541                                   DMA_MBUF_DONE_THRESH);
 542
 543         return 0;
 544 }
 545
 546 /* standard ethtool support functions. */
 547 enum bcmgenet_stat_type {
 548         BCMGENET_STAT_NETDEV = -1,
 549         BCMGENET_STAT_MIB_RX,
 550         BCMGENET_STAT_MIB_TX,
 551         BCMGENET_STAT_RUNT,
 552         BCMGENET_STAT_MISC,
 553         BCMGENET_STAT_SOFT,
 554 };
 555
 556 struct bcmgenet_stats {
 557         char stat_string[ETH_GSTRING_LEN];
 558         int stat_sizeof;
 559         int stat_offset;
 560         enum bcmgenet_stat_type type;
 561         /* reg offset from UMAC base for misc counters */
 562         u16 reg_offset;
 563 };
 564
 565 #define STAT_NETDEV(m) { \
 566         .stat_string = __stringify(m), \
 567         .stat_sizeof = sizeof(((struct net_device_stats *)0)->m), \
 568         .stat_offset = offsetof(struct net_device_stats, m), \
 569         .type = BCMGENET_STAT_NETDEV, \
 570 }
 571
 572 #define STAT_GENET_MIB(str, m, _type) { \
 573         .stat_string = str, \
 574         .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
 575         .stat_offset = offsetof(struct bcmgenet_priv, m), \
 576         .type = _type, \
 577 }
 578
 579 #define STAT_GENET_MIB_RX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_RX)
 580 #define STAT_GENET_MIB_TX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_TX)
 581 #define STAT_GENET_RUNT(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_RUNT)
 582 #define STAT_GENET_SOFT_MIB(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_SOFT)
 583
 584 #define STAT_GENET_MISC(str, m, offset) { \
 585         .stat_string = str, \
 586         .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
 587         .stat_offset = offsetof(struct bcmgenet_priv, m), \
 588         .type = BCMGENET_STAT_MISC, \
 589         .reg_offset = offset, \
 590 }
 591
 592
 593 /* There is a 0xC gap between the end of RX and beginning of TX stats and then
 594  * between the end of TX stats and the beginning of the RX RUNT
 595  */
 596 #define BCMGENET_STAT_OFFSET    0xc
 597
 598 /* Hardware counters must be kept in sync because the order/offset
 599  * is important here (order in structure declaration = order in hardware)
 600  */
 601 static const struct bcmgenet_stats bcmgenet_gstrings_stats[] = {
 602         /* general stats */
 603         STAT_NETDEV(rx_packets),
 604         STAT_NETDEV(tx_packets),
 605         STAT_NETDEV(rx_bytes),
 606         STAT_NETDEV(tx_bytes),
 607         STAT_NETDEV(rx_errors),
 608         STAT_NETDEV(tx_errors),
 609         STAT_NETDEV(rx_dropped),
 610         STAT_NETDEV(tx_dropped),
 611         STAT_NETDEV(multicast),
 612         /* UniMAC RSV counters */
 613         STAT_GENET_MIB_RX("rx_64_octets", mib.rx.pkt_cnt.cnt_64),
 614         STAT_GENET_MIB_RX("rx_65_127_oct", mib.rx.pkt_cnt.cnt_127),
 615         STAT_GENET_MIB_RX("rx_128_255_oct", mib.rx.pkt_cnt.cnt_255),
 616         STAT_GENET_MIB_RX("rx_256_511_oct", mib.rx.pkt_cnt.cnt_511),
 617         STAT_GENET_MIB_RX("rx_512_1023_oct", mib.rx.pkt_cnt.cnt_1023),
 618         STAT_GENET_MIB_RX("rx_1024_1518_oct", mib.rx.pkt_cnt.cnt_1518),
 619         STAT_GENET_MIB_RX("rx_vlan_1519_1522_oct", mib.rx.pkt_cnt.cnt_mgv),
 620         STAT_GENET_MIB_RX("rx_1522_2047_oct", mib.rx.pkt_cnt.cnt_2047),
 621         STAT_GENET_MIB_RX("rx_2048_4095_oct", mib.rx.pkt_cnt.cnt_4095),
 622         STAT_GENET_MIB_RX("rx_4096_9216_oct", mib.rx.pkt_cnt.cnt_9216),
 623         STAT_GENET_MIB_RX("rx_pkts", mib.rx.pkt),
 624         STAT_GENET_MIB_RX("rx_bytes", mib.rx.bytes),
 625         STAT_GENET_MIB_RX("rx_multicast", mib.rx.mca),
 626         STAT_GENET_MIB_RX("rx_broadcast", mib.rx.bca),
 627         STAT_GENET_MIB_RX("rx_fcs", mib.rx.fcs),
 628         STAT_GENET_MIB_RX("rx_control", mib.rx.cf),
 629         STAT_GENET_MIB_RX("rx_pause", mib.rx.pf),
 630         STAT_GENET_MIB_RX("rx_unknown", mib.rx.uo),
 631         STAT_GENET_MIB_RX("rx_align", mib.rx.aln),
 632         STAT_GENET_MIB_RX("rx_outrange", mib.rx.flr),
 633         STAT_GENET_MIB_RX("rx_code", mib.rx.cde),
 634         STAT_GENET_MIB_RX("rx_carrier", mib.rx.fcr),
 635         STAT_GENET_MIB_RX("rx_oversize", mib.rx.ovr),
 636         STAT_GENET_MIB_RX("rx_jabber", mib.rx.jbr),
 637         STAT_GENET_MIB_RX("rx_mtu_err", mib.rx.mtue),
 638         STAT_GENET_MIB_RX("rx_good_pkts", mib.rx.pok),
 639         STAT_GENET_MIB_RX("rx_unicast", mib.rx.uc),
 640         STAT_GENET_MIB_RX("rx_ppp", mib.rx.ppp),
 641         STAT_GENET_MIB_RX("rx_crc", mib.rx.rcrc),
 642         /* UniMAC TSV counters */
 643         STAT_GENET_MIB_TX("tx_64_octets", mib.tx.pkt_cnt.cnt_64),
 644         STAT_GENET_MIB_TX("tx_65_127_oct", mib.tx.pkt_cnt.cnt_127),
 645         STAT_GENET_MIB_TX("tx_128_255_oct", mib.tx.pkt_cnt.cnt_255),
 646         STAT_GENET_MIB_TX("tx_256_511_oct", mib.tx.pkt_cnt.cnt_511),
 647         STAT_GENET_MIB_TX("tx_512_1023_oct", mib.tx.pkt_cnt.cnt_1023),
 648         STAT_GENET_MIB_TX("tx_1024_1518_oct", mib.tx.pkt_cnt.cnt_1518),
 649         STAT_GENET_MIB_TX("tx_vlan_1519_1522_oct", mib.tx.pkt_cnt.cnt_mgv),
 650         STAT_GENET_MIB_TX("tx_1522_2047_oct", mib.tx.pkt_cnt.cnt_2047),
 651         STAT_GENET_MIB_TX("tx_2048_4095_oct", mib.tx.pkt_cnt.cnt_4095),
 652         STAT_GENET_MIB_TX("tx_4096_9216_oct", mib.tx.pkt_cnt.cnt_9216),
 653         STAT_GENET_MIB_TX("tx_pkts", mib.tx.pkts),
 654         STAT_GENET_MIB_TX("tx_multicast", mib.tx.mca),
 655         STAT_GENET_MIB_TX("tx_broadcast", mib.tx.bca),
 656         STAT_GENET_MIB_TX("tx_pause", mib.tx.pf),
 657         STAT_GENET_MIB_TX("tx_control", mib.tx.cf),
 658         STAT_GENET_MIB_TX("tx_fcs_err", mib.tx.fcs),
 659         STAT_GENET_MIB_TX("tx_oversize", mib.tx.ovr),
 660         STAT_GENET_MIB_TX("tx_defer", mib.tx.drf),
 661         STAT_GENET_MIB_TX("tx_excess_defer", mib.tx.edf),
 662         STAT_GENET_MIB_TX("tx_single_col", mib.tx.scl),
 663         STAT_GENET_MIB_TX("tx_multi_col", mib.tx.mcl),
 664         STAT_GENET_MIB_TX("tx_late_col", mib.tx.lcl),
 665         STAT_GENET_MIB_TX("tx_excess_col", mib.tx.ecl),
 666         STAT_GENET_MIB_TX("tx_frags", mib.tx.frg),
 667         STAT_GENET_MIB_TX("tx_total_col", mib.tx.ncl),
 668         STAT_GENET_MIB_TX("tx_jabber", mib.tx.jbr),
 669         STAT_GENET_MIB_TX("tx_bytes", mib.tx.bytes),
 670         STAT_GENET_MIB_TX("tx_good_pkts", mib.tx.pok),
 671         STAT_GENET_MIB_TX("tx_unicast", mib.tx.uc),
 672         /* UniMAC RUNT counters */
 673         STAT_GENET_RUNT("rx_runt_pkts", mib.rx_runt_cnt),
 674         STAT_GENET_RUNT("rx_runt_valid_fcs", mib.rx_runt_fcs),
 675         STAT_GENET_RUNT("rx_runt_inval_fcs_align", mib.rx_runt_fcs_align),
 676         STAT_GENET_RUNT("rx_runt_bytes", mib.rx_runt_bytes),
 677         /* Misc UniMAC counters */
 678         STAT_GENET_MISC("rbuf_ovflow_cnt", mib.rbuf_ovflow_cnt,
 679                         UMAC_RBUF_OVFL_CNT),
 680         STAT_GENET_MISC("rbuf_err_cnt", mib.rbuf_err_cnt, UMAC_RBUF_ERR_CNT),
 681         STAT_GENET_MISC("mdf_err_cnt", mib.mdf_err_cnt, UMAC_MDF_ERR_CNT),
 682         STAT_GENET_SOFT_MIB("alloc_rx_buff_failed", mib.alloc_rx_buff_failed),
 683         STAT_GENET_SOFT_MIB("rx_dma_failed", mib.rx_dma_failed),
 684         STAT_GENET_SOFT_MIB("tx_dma_failed", mib.tx_dma_failed),
 685 };
 686
 687 #define BCMGENET_STATS_LEN      ARRAY_SIZE(bcmgenet_gstrings_stats)
 688
 689 static void bcmgenet_get_drvinfo(struct net_device *dev,
 690                                  struct ethtool_drvinfo *info)
 691 {
 692         strlcpy(info->driver, "bcmgenet", sizeof(info->driver));
 693         strlcpy(info->version, "v2.0", sizeof(info->version));
 694         info->n_stats = BCMGENET_STATS_LEN;
 695 }
 696
 697 static int bcmgenet_get_sset_count(struct net_device *dev, int string_set)
 698 {
 699         switch (string_set) {
 700         case ETH_SS_STATS:
 701                 return BCMGENET_STATS_LEN;
 702         default:
 703                 return -EOPNOTSUPP;
 704         }
 705 }
 706
 707 static void bcmgenet_get_strings(struct net_device *dev, u32 stringset,
 708                                  u8 *data)
 709 {
 710         int i;
 711
 712         switch (stringset) {
 713         case ETH_SS_STATS:
 714                 for (i = 0; i < BCMGENET_STATS_LEN; i++) {
 715                         memcpy(data + i * ETH_GSTRING_LEN,
 716                                bcmgenet_gstrings_stats[i].stat_string,
 717                                ETH_GSTRING_LEN);
 718                 }
 719                 break;
 720         }
 721 }
 722
 723 static void bcmgenet_update_mib_counters(struct bcmgenet_priv *priv)
 724 {
 725         int i, j = 0;
 726
 727         for (i = 0; i < BCMGENET_STATS_LEN; i++) {
 728                 const struct bcmgenet_stats *s;
 729                 u8 offset = 0;
 730                 u32 val = 0;
 731                 char *p;
 732
 733                 s = &bcmgenet_gstrings_stats[i];
 734                 switch (s->type) {
 735                 case BCMGENET_STAT_NETDEV:
 736                 case BCMGENET_STAT_SOFT:
 737                         continue;
 738                 case BCMGENET_STAT_MIB_RX:
 739                 case BCMGENET_STAT_MIB_TX:
 740                 case BCMGENET_STAT_RUNT:
 741                         if (s->type != BCMGENET_STAT_MIB_RX)
 742                                 offset = BCMGENET_STAT_OFFSET;
 743                         val = bcmgenet_umac_readl(priv,
 744                                                   UMAC_MIB_START + j + offset);
 745                         break;
 746                 case BCMGENET_STAT_MISC:
 747                         val = bcmgenet_umac_readl(priv, s->reg_offset);
 748                         /* clear if overflowed */
 749                         if (val == ~0)
 750                                 bcmgenet_umac_writel(priv, 0, s->reg_offset);
 751                         break;
 752                 }
 753
 754                 j += s->stat_sizeof;
 755                 p = (char *)priv + s->stat_offset;
 756                 *(u32 *)p = val;
 757         }
 758 }
 759
 760 static void bcmgenet_get_ethtool_stats(struct net_device *dev,
 761                                        struct ethtool_stats *stats,
 762                                        u64 *data)
 763 {
 764         struct bcmgenet_priv *priv = netdev_priv(dev);
 765         int i;
 766
 767         if (netif_running(dev))
 768                 bcmgenet_update_mib_counters(priv);
 769
 770         for (i = 0; i < BCMGENET_STATS_LEN; i++) {
 771                 const struct bcmgenet_stats *s;
 772                 char *p;
 773
 774                 s = &bcmgenet_gstrings_stats[i];
 775                 if (s->type == BCMGENET_STAT_NETDEV)
 776                         p = (char *)&dev->stats;
 777                 else
 778                         p = (char *)priv;
 779                 p += s->stat_offset;
 780                 data[i] = *(u32 *)p;
 781         }
 782 }
 783
 784 static void bcmgenet_eee_enable_set(struct net_device *dev, bool enable)
 785 {
 786         struct bcmgenet_priv *priv = netdev_priv(dev);
 787         u32 off = priv->hw_params->tbuf_offset + TBUF_ENERGY_CTRL;
 788         u32 reg;
 789
 790         if (enable && !priv->clk_eee_enabled) {
 791                 clk_prepare_enable(priv->clk_eee);
 792                 priv->clk_eee_enabled = true;
 793         }
 794
 795         reg = bcmgenet_umac_readl(priv, UMAC_EEE_CTRL);
 796         if (enable)
 797                 reg |= EEE_EN;
 798         else
 799                 reg &= ~EEE_EN;
 800         bcmgenet_umac_writel(priv, reg, UMAC_EEE_CTRL);
 801
 802         /* Enable EEE and switch to a 27Mhz clock automatically */
 803         reg = __raw_readl(priv->base + off);
 804         if (enable)
 805                 reg |= TBUF_EEE_EN | TBUF_PM_EN;
 806         else
 807                 reg &= ~(TBUF_EEE_EN | TBUF_PM_EN);
 808         __raw_writel(reg, priv->base + off);
 809
 810         /* Do the same for thing for RBUF */
 811         reg = bcmgenet_rbuf_readl(priv, RBUF_ENERGY_CTRL);
 812         if (enable)
 813                 reg |= RBUF_EEE_EN | RBUF_PM_EN;
 814         else
 815                 reg &= ~(RBUF_EEE_EN | RBUF_PM_EN);
 816         bcmgenet_rbuf_writel(priv, reg, RBUF_ENERGY_CTRL);
 817
 818         if (!enable && priv->clk_eee_enabled) {
 819                 clk_disable_unprepare(priv->clk_eee);
 820                 priv->clk_eee_enabled = false;
 821         }
 822
 823         priv->eee.eee_enabled = enable;
 824         priv->eee.eee_active = enable;
 825 }
 826
 827 static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_eee *e)
 828 {
 829         struct bcmgenet_priv *priv = netdev_priv(dev);
 830         struct ethtool_eee *p = &priv->eee;
 831
 832         if (GENET_IS_V1(priv))
 833                 return -EOPNOTSUPP;
 834
 835         e->eee_enabled = p->eee_enabled;
 836         e->eee_active = p->eee_active;
 837         e->tx_lpi_timer = bcmgenet_umac_readl(priv, UMAC_EEE_LPI_TIMER);
 838
 839         return phy_ethtool_get_eee(priv->phydev, e);
 840 }
 841
 842 static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_eee *e)
 843 {
 844         struct bcmgenet_priv *priv = netdev_priv(dev);
 845         struct ethtool_eee *p = &priv->eee;
 846         int ret = 0;
 847
 848         if (GENET_IS_V1(priv))
 849                 return -EOPNOTSUPP;
 850
 851         p->eee_enabled = e->eee_enabled;
 852
 853         if (!p->eee_enabled) {
 854                 bcmgenet_eee_enable_set(dev, false);
 855         } else {
 856                 ret = phy_init_eee(priv->phydev, 0);
 857                 if (ret) {
 858                         netif_err(priv, hw, dev, "EEE initialization failed\n");
 859                         return ret;
 860                 }
 861
 862                 bcmgenet_umac_writel(priv, e->tx_lpi_timer, UMAC_EEE_LPI_TIMER);
 863                 bcmgenet_eee_enable_set(dev, true);
 864         }
 865
 866         return phy_ethtool_set_eee(priv->phydev, e);
 867 }
 868
 869 static int bcmgenet_nway_reset(struct net_device *dev)
 870 {
 871         struct bcmgenet_priv *priv = netdev_priv(dev);
 872
 873         return genphy_restart_aneg(priv->phydev);
 874 }
 875
 876 /* standard ethtool support functions. */
 877 static struct ethtool_ops bcmgenet_ethtool_ops = {
 878         .get_strings            = bcmgenet_get_strings,
 879         .get_sset_count         = bcmgenet_get_sset_count,
 880         .get_ethtool_stats      = bcmgenet_get_ethtool_stats,
 881         .get_settings           = bcmgenet_get_settings,
 882         .set_settings           = bcmgenet_set_settings,
 883         .get_drvinfo            = bcmgenet_get_drvinfo,
 884         .get_link               = ethtool_op_get_link,
 885         .get_msglevel           = bcmgenet_get_msglevel,
 886         .set_msglevel           = bcmgenet_set_msglevel,
 887         .get_wol                = bcmgenet_get_wol,
 888         .set_wol                = bcmgenet_set_wol,
 889         .get_eee                = bcmgenet_get_eee,
 890         .set_eee                = bcmgenet_set_eee,
 891         .nway_reset             = bcmgenet_nway_reset,
 892         .get_coalesce           = bcmgenet_get_coalesce,
 893         .set_coalesce           = bcmgenet_set_coalesce,
 894 };
 895
 896 /* Power down the unimac, based on mode. */
 897 static int bcmgenet_power_down(struct bcmgenet_priv *priv,
 898                                 enum bcmgenet_power_mode mode)
 899 {
 900         int ret = 0;
 901         u32 reg;
 902
 903         switch (mode) {
 904         case GENET_POWER_CABLE_SENSE:
 905                 phy_detach(priv->phydev);
 906                 break;
 907
 908         case GENET_POWER_WOL_MAGIC:
 909                 ret = bcmgenet_wol_power_down_cfg(priv, mode);
 910                 break;
 911
 912         case GENET_POWER_PASSIVE:
 913                 /* Power down LED */
 914                 if (priv->hw_params->flags & GENET_HAS_EXT) {
 915                         reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
 916                         reg |= (EXT_PWR_DOWN_PHY |
 917                                 EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS);
 918                         bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
 919
 920                         bcmgenet_phy_power_set(priv->dev, false);
 921                 }
 922                 break;
 923         default:
 924                 break;
 925         }
 926
 927         return 0;
 928 }
 929
 930 static void bcmgenet_power_up(struct bcmgenet_priv *priv,
 931                               enum bcmgenet_power_mode mode)
 932 {
 933         u32 reg;
 934
 935         if (!(priv->hw_params->flags & GENET_HAS_EXT))
 936                 return;
 937
 938         reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
 939
 940         switch (mode) {
 941         case GENET_POWER_PASSIVE:
 942                 reg &= ~(EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_PHY |
 943                                 EXT_PWR_DOWN_BIAS);
 944                 /* fallthrough */
 945         case GENET_POWER_CABLE_SENSE:
 946                 /* enable APD */
 947                 reg |= EXT_PWR_DN_EN_LD;
 948                 break;
 949         case GENET_POWER_WOL_MAGIC:
 950                 bcmgenet_wol_power_up_cfg(priv, mode);
 951                 return;
 952         default:
 953                 break;
 954         }
 955
 956         bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
 957         if (mode == GENET_POWER_PASSIVE)
 958                 bcmgenet_phy_power_set(priv->dev, true);
 959 }
 960
 961 /* ioctl handle special commands that are not present in ethtool. */
 962 static int bcmgenet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 963 {
 964         struct bcmgenet_priv *priv = netdev_priv(dev);
 965         int val = 0;
 966
 967         if (!netif_running(dev))
 968                 return -EINVAL;
 969
 970         switch (cmd) {
 971         case SIOCGMIIPHY:
 972         case SIOCGMIIREG:
 973         case SIOCSMIIREG:
 974                 if (!priv->phydev)
 975                         val = -ENODEV;
 976                 else
 977                         val = phy_mii_ioctl(priv->phydev, rq, cmd);
 978                 break;
 979
 980         default:
 981                 val = -EINVAL;
 982                 break;
 983         }
 984
 985         return val;
 986 }
 987
 988 static struct enet_cb *bcmgenet_get_txcb(struct bcmgenet_priv *priv,
 989                                          struct bcmgenet_tx_ring *ring)
 990 {
 991         struct enet_cb *tx_cb_ptr;
 992
 993         tx_cb_ptr = ring->cbs;
 994         tx_cb_ptr += ring->write_ptr - ring->cb_ptr;
 995
 996         /* Advancing local write pointer */
 997         if (ring->write_ptr == ring->end_ptr)
 998                 ring->write_ptr = ring->cb_ptr;
 999         else
1000                 ring->write_ptr++;
1001
1002         return tx_cb_ptr;
1003 }
1004
1005 /* Simple helper to free a control block's resources */
1006 static void bcmgenet_free_cb(struct enet_cb *cb)
1007 {
1008         dev_kfree_skb_any(cb->skb);
1009         cb->skb = NULL;
1010         dma_unmap_addr_set(cb, dma_addr, 0);
1011 }
1012
1013 static inline void bcmgenet_rx_ring16_int_disable(struct bcmgenet_rx_ring *ring)
1014 {
1015         bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_RXDMA_DONE,
1016                                  INTRL2_CPU_MASK_SET);
1017 }
1018
1019 static inline void bcmgenet_rx_ring16_int_enable(struct bcmgenet_rx_ring *ring)
1020 {
1021         bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_RXDMA_DONE,
1022                                  INTRL2_CPU_MASK_CLEAR);
1023 }
1024
1025 static inline void bcmgenet_rx_ring_int_disable(struct bcmgenet_rx_ring *ring)
1026 {
1027         bcmgenet_intrl2_1_writel(ring->priv,
1028                                  1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
1029                                  INTRL2_CPU_MASK_SET);
1030 }
1031
1032 static inline void bcmgenet_rx_ring_int_enable(struct bcmgenet_rx_ring *ring)
1033 {
1034         bcmgenet_intrl2_1_writel(ring->priv,
1035                                  1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
1036                                  INTRL2_CPU_MASK_CLEAR);
1037 }
1038
1039 static inline void bcmgenet_tx_ring16_int_disable(struct bcmgenet_tx_ring *ring)
1040 {
1041         bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_TXDMA_DONE,
1042                                  INTRL2_CPU_MASK_SET);
1043 }
1044
1045 static inline void bcmgenet_tx_ring16_int_enable(struct bcmgenet_tx_ring *ring)
1046 {
1047         bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_TXDMA_DONE,
1048                                  INTRL2_CPU_MASK_CLEAR);
1049 }
1050
1051 static inline void bcmgenet_tx_ring_int_enable(struct bcmgenet_tx_ring *ring)
1052 {
1053         bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,
1054                                  INTRL2_CPU_MASK_CLEAR);
1055 }
1056
1057 static inline void bcmgenet_tx_ring_int_disable(struct bcmgenet_tx_ring *ring)
1058 {
1059         bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,
1060                                  INTRL2_CPU_MASK_SET);
1061 }
1062
1063 /* Unlocked version of the reclaim routine */
1064 static unsigned int __bcmgenet_tx_reclaim(struct net_device *dev,
1065                                           struct bcmgenet_tx_ring *ring)
1066 {
1067         struct bcmgenet_priv *priv = netdev_priv(dev);
1068         struct enet_cb *tx_cb_ptr;
1069         struct netdev_queue *txq;
1070         unsigned int pkts_compl = 0;
1071         unsigned int c_index;
1072         unsigned int txbds_ready;
1073         unsigned int txbds_processed = 0;
1074
1075         /* Compute how many buffers are transmitted since last xmit call */
1076         c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX);
1077         c_index &= DMA_C_INDEX_MASK;
1078
1079         if (likely(c_index >= ring->c_index))
1080                 txbds_ready = c_index - ring->c_index;
1081         else
1082                 txbds_ready = (DMA_C_INDEX_MASK + 1) - ring->c_index + c_index;
1083
1084         netif_dbg(priv, tx_done, dev,
1085                   "%s ring=%d old_c_index=%u c_index=%u txbds_ready=%u\n",
1086                   __func__, ring->index, ring->c_index, c_index, txbds_ready);
1087
1088         /* Reclaim transmitted buffers */
1089         while (txbds_processed < txbds_ready) {
1090                 tx_cb_ptr = &priv->tx_cbs[ring->clean_ptr];
1091                 if (tx_cb_ptr->skb) {
1092                         pkts_compl++;
1093                         dev->stats.tx_packets++;
1094                         dev->stats.tx_bytes += tx_cb_ptr->skb->len;
1095                         dma_unmap_single(&dev->dev,
1096                                          dma_unmap_addr(tx_cb_ptr, dma_addr),
1097                                          tx_cb_ptr->skb->len,
1098                                          DMA_TO_DEVICE);
1099                         bcmgenet_free_cb(tx_cb_ptr);
1100                 } else if (dma_unmap_addr(tx_cb_ptr, dma_addr)) {
1101                         dev->stats.tx_bytes +=
1102                                 dma_unmap_len(tx_cb_ptr, dma_len);
1103                         dma_unmap_page(&dev->dev,
1104                                        dma_unmap_addr(tx_cb_ptr, dma_addr),
1105                                        dma_unmap_len(tx_cb_ptr, dma_len),
1106                                        DMA_TO_DEVICE);
1107                         dma_unmap_addr_set(tx_cb_ptr, dma_addr, 0);
1108                 }
1109
1110                 txbds_processed++;
1111                 if (likely(ring->clean_ptr < ring->end_ptr))
1112                         ring->clean_ptr++;
1113                 else
1114                         ring->clean_ptr = ring->cb_ptr;
1115         }
1116
1117         ring->free_bds += txbds_processed;
1118         ring->c_index = (ring->c_index + txbds_processed) & DMA_C_INDEX_MASK;
1119
1120         if (ring->free_bds > (MAX_SKB_FRAGS + 1)) {
1121                 txq = netdev_get_tx_queue(dev, ring->queue);
1122                 if (netif_tx_queue_stopped(txq))
1123                         netif_tx_wake_queue(txq);
1124         }
1125
1126         return pkts_compl;
1127 }
1128
1129 static unsigned int bcmgenet_tx_reclaim(struct net_device *dev,
1130                                 struct bcmgenet_tx_ring *ring)
1131 {
1132         unsigned int released;
1133         unsigned long flags;
1134
1135         spin_lock_irqsave(&ring->lock, flags);
1136         released = __bcmgenet_tx_reclaim(dev, ring);
1137         spin_unlock_irqrestore(&ring->lock, flags);
1138
1139         return released;
1140 }
1141
1142 static int bcmgenet_tx_poll(struct napi_struct *napi, int budget)
1143 {
1144         struct bcmgenet_tx_ring *ring =
1145                 container_of(napi, struct bcmgenet_tx_ring, napi);
1146         unsigned int work_done = 0;
1147
1148         work_done = bcmgenet_tx_reclaim(ring->priv->dev, ring);
1149
1150         if (work_done == 0) {
1151                 napi_complete(napi);
1152                 ring->int_enable(ring);
1153
1154                 return 0;
1155         }
1156
1157         return budget;
1158 }
1159
1160 static void bcmgenet_tx_reclaim_all(struct net_device *dev)
1161 {
1162         struct bcmgenet_priv *priv = netdev_priv(dev);
1163         int i;
1164
1165         if (netif_is_multiqueue(dev)) {
1166                 for (i = 0; i < priv->hw_params->tx_queues; i++)
1167                         bcmgenet_tx_reclaim(dev, &priv->tx_rings[i]);
1168         }
1169
1170         bcmgenet_tx_reclaim(dev, &priv->tx_rings[DESC_INDEX]);
1171 }
1172
1173 /* Transmits a single SKB (either head of a fragment or a single SKB)
1174  * caller must hold priv->lock
1175  */
1176 static int bcmgenet_xmit_single(struct net_device *dev,
1177                                 struct sk_buff *skb,
1178                                 u16 dma_desc_flags,
1179                                 struct bcmgenet_tx_ring *ring)
1180 {
1181         struct bcmgenet_priv *priv = netdev_priv(dev);
1182         struct device *kdev = &priv->pdev->dev;
1183         struct enet_cb *tx_cb_ptr;
1184         unsigned int skb_len;
1185         dma_addr_t mapping;
1186         u32 length_status;
1187         int ret;
1188
1189         tx_cb_ptr = bcmgenet_get_txcb(priv, ring);
1190
1191         if (unlikely(!tx_cb_ptr))
1192                 BUG();
1193
1194         tx_cb_ptr->skb = skb;
1195
1196         skb_len = skb_headlen(skb) < ETH_ZLEN ? ETH_ZLEN : skb_headlen(skb);
1197
1198         mapping = dma_map_single(kdev, skb->data, skb_len, DMA_TO_DEVICE);
1199         ret = dma_mapping_error(kdev, mapping);
1200         if (ret) {
1201                 priv->mib.tx_dma_failed++;
1202                 netif_err(priv, tx_err, dev, "Tx DMA map failed\n");
1203                 dev_kfree_skb(skb);
1204                 return ret;
1205         }
1206
1207         dma_unmap_addr_set(tx_cb_ptr, dma_addr, mapping);
1208         dma_unmap_len_set(tx_cb_ptr, dma_len, skb->len);
1209         length_status = (skb_len << DMA_BUFLENGTH_SHIFT) | dma_desc_flags |
1210                         (priv->hw_params->qtag_mask << DMA_TX_QTAG_SHIFT) |
1211                         DMA_TX_APPEND_CRC;
1212
1213         if (skb->ip_summed == CHECKSUM_PARTIAL)
1214                 length_status |= DMA_TX_DO_CSUM;
1215
1216         dmadesc_set(priv, tx_cb_ptr->bd_addr, mapping, length_status);
1217
1218         return 0;
1219 }
1220
1221 /* Transmit a SKB fragment */
1222 static int bcmgenet_xmit_frag(struct net_device *dev,
1223                               skb_frag_t *frag,
1224                               u16 dma_desc_flags,
1225                               struct bcmgenet_tx_ring *ring)
1226 {
1227         struct bcmgenet_priv *priv = netdev_priv(dev);
1228         struct device *kdev = &priv->pdev->dev;
1229         struct enet_cb *tx_cb_ptr;
1230         dma_addr_t mapping;
1231         int ret;
1232
1233         tx_cb_ptr = bcmgenet_get_txcb(priv, ring);
1234
1235         if (unlikely(!tx_cb_ptr))
1236                 BUG();
1237         tx_cb_ptr->skb = NULL;
1238
1239         mapping = skb_frag_dma_map(kdev, frag, 0,
1240                                    skb_frag_size(frag), DMA_TO_DEVICE);
1241         ret = dma_mapping_error(kdev, mapping);
1242         if (ret) {
1243                 priv->mib.tx_dma_failed++;
1244                 netif_err(priv, tx_err, dev, "%s: Tx DMA map failed\n",
1245                           __func__);
1246                 return ret;
1247         }
1248
1249         dma_unmap_addr_set(tx_cb_ptr, dma_addr, mapping);
1250         dma_unmap_len_set(tx_cb_ptr, dma_len, frag->size);
1251
1252         dmadesc_set(priv, tx_cb_ptr->bd_addr, mapping,
1253                     (frag->size << DMA_BUFLENGTH_SHIFT) | dma_desc_flags |
1254                     (priv->hw_params->qtag_mask << DMA_TX_QTAG_SHIFT));
1255
1256         return 0;
1257 }
1258
1259 /* Reallocate the SKB to put enough headroom in front of it and insert
1260  * the transmit checksum offsets in the descriptors
1261  */
1262 static struct sk_buff *bcmgenet_put_tx_csum(struct net_device *dev,
1263                                             struct sk_buff *skb)
1264 {
1265         struct status_64 *status = NULL;
1266         struct sk_buff *new_skb;
1267         u16 offset;
1268         u8 ip_proto;
1269         u16 ip_ver;
1270         u32 tx_csum_info;
1271
1272         if (unlikely(skb_headroom(skb) < sizeof(*status))) {
1273                 /* If 64 byte status block enabled, must make sure skb has
1274                  * enough headroom for us to insert 64B status block.
1275                  */
1276                 new_skb = skb_realloc_headroom(skb, sizeof(*status));
1277                 dev_kfree_skb(skb);
1278                 if (!new_skb) {
1279                         dev->stats.tx_dropped++;
1280                         return NULL;
1281                 }
1282                 skb = new_skb;
1283         }
1284
1285         skb_push(skb, sizeof(*status));
1286         status = (struct status_64 *)skb->data;
1287
1288         if (skb->ip_summed  == CHECKSUM_PARTIAL) {
1289                 ip_ver = htons(skb->protocol);
1290                 switch (ip_ver) {
1291                 case ETH_P_IP:
1292                         ip_proto = ip_hdr(skb)->protocol;
1293                         break;
1294                 case ETH_P_IPV6:
1295                         ip_proto = ipv6_hdr(skb)->nexthdr;
1296                         break;
1297                 default:
1298                         return skb;
1299                 }
1300
1301                 offset = skb_checksum_start_offset(skb) - sizeof(*status);
1302                 tx_csum_info = (offset << STATUS_TX_CSUM_START_SHIFT) |
1303                                 (offset + skb->csum_offset);
1304
1305                 /* Set the length valid bit for TCP and UDP and just set
1306                  * the special UDP flag for IPv4, else just set to 0.
1307                  */
1308                 if (ip_proto == IPPROTO_TCP || ip_proto == IPPROTO_UDP) {
1309                         tx_csum_info |= STATUS_TX_CSUM_LV;
1310                         if (ip_proto == IPPROTO_UDP && ip_ver == ETH_P_IP)
1311                                 tx_csum_info |= STATUS_TX_CSUM_PROTO_UDP;
1312                 } else {
1313                         tx_csum_info = 0;
1314                 }
1315
1316                 status->tx_csum_info = tx_csum_info;
1317         }
1318
1319         return skb;
1320 }
1321
1322 static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev)
1323 {
1324         struct bcmgenet_priv *priv = netdev_priv(dev);
1325         struct bcmgenet_tx_ring *ring = NULL;
1326         struct netdev_queue *txq;
1327         unsigned long flags = 0;
1328         int nr_frags, index;
1329         u16 dma_desc_flags;
1330         int ret;
1331         int i;
1332
1333         index = skb_get_queue_mapping(skb);
1334         /* Mapping strategy:
1335          * queue_mapping = 0, unclassified, packet xmited through ring16
1336          * queue_mapping = 1, goes to ring 0. (highest priority queue
1337          * queue_mapping = 2, goes to ring 1.
1338          * queue_mapping = 3, goes to ring 2.
1339          * queue_mapping = 4, goes to ring 3.
1340          */
1341         if (index == 0)
1342                 index = DESC_INDEX;
1343         else
1344                 index -= 1;
1345
1346         nr_frags = skb_shinfo(skb)->nr_frags;
1347         ring = &priv->tx_rings[index];
1348         txq = netdev_get_tx_queue(dev, ring->queue);
1349
1350         spin_lock_irqsave(&ring->lock, flags);
1351         if (ring->free_bds <= nr_frags + 1) {
1352                 netif_tx_stop_queue(txq);
1353                 netdev_err(dev, "%s: tx ring %d full when queue %d awake\n",
1354                            __func__, index, ring->queue);
1355                 ret = NETDEV_TX_BUSY;
1356                 goto out;
1357         }
1358
1359         if (skb_padto(skb, ETH_ZLEN)) {
1360                 ret = NETDEV_TX_OK;
1361                 goto out;
1362         }
1363
1364         /* set the SKB transmit checksum */
1365         if (priv->desc_64b_en) {
1366                 skb = bcmgenet_put_tx_csum(dev, skb);
1367                 if (!skb) {
1368                         ret = NETDEV_TX_OK;
1369                         goto out;
1370                 }
1371         }
1372
1373         dma_desc_flags = DMA_SOP;
1374         if (nr_frags == 0)
1375                 dma_desc_flags |= DMA_EOP;
1376
1377         /* Transmit single SKB or head of fragment list */
1378         ret = bcmgenet_xmit_single(dev, skb, dma_desc_flags, ring);
1379         if (ret) {
1380                 ret = NETDEV_TX_OK;
1381                 goto out;
1382         }
1383
1384         /* xmit fragment */
1385         for (i = 0; i < nr_frags; i++) {
1386                 ret = bcmgenet_xmit_frag(dev,
1387                                          &skb_shinfo(skb)->frags[i],
1388                                          (i == nr_frags - 1) ? DMA_EOP : 0,
1389                                          ring);
1390                 if (ret) {
1391                         ret = NETDEV_TX_OK;
1392                         goto out;
1393                 }
1394         }
1395
1396         skb_tx_timestamp(skb);
1397
1398         /* Decrement total BD count and advance our write pointer */
1399         ring->free_bds -= nr_frags + 1;
1400         ring->prod_index += nr_frags + 1;
1401         ring->prod_index &= DMA_P_INDEX_MASK;
1402
1403         if (ring->free_bds <= (MAX_SKB_FRAGS + 1))
1404                 netif_tx_stop_queue(txq);
1405
1406         if (!skb->xmit_more || netif_xmit_stopped(txq))
1407                 /* Packets are ready, update producer index */
1408                 bcmgenet_tdma_ring_writel(priv, ring->index,
1409                                           ring->prod_index, TDMA_PROD_INDEX);
1410 out:
1411         spin_unlock_irqrestore(&ring->lock, flags);
1412
1413         return ret;
1414 }
1415
1416 static struct sk_buff *bcmgenet_rx_refill(struct bcmgenet_priv *priv,
1417                                           struct enet_cb *cb)
1418 {
1419         struct device *kdev = &priv->pdev->dev;
1420         struct sk_buff *skb;
1421         struct sk_buff *rx_skb;
1422         dma_addr_t mapping;
1423
1424         /* Allocate a new Rx skb */
1425         skb = netdev_alloc_skb(priv->dev, priv->rx_buf_len + SKB_ALIGNMENT);
1426         if (!skb) {
1427                 priv->mib.alloc_rx_buff_failed++;
1428                 netif_err(priv, rx_err, priv->dev,
1429                           "%s: Rx skb allocation failed\n", __func__);
1430                 return NULL;
1431         }
1432
1433         /* DMA-map the new Rx skb */
1434         mapping = dma_map_single(kdev, skb->data, priv->rx_buf_len,
1435                                  DMA_FROM_DEVICE);
1436         if (dma_mapping_error(kdev, mapping)) {
1437                 priv->mib.rx_dma_failed++;
1438                 dev_kfree_skb_any(skb);
1439                 netif_err(priv, rx_err, priv->dev,
1440                           "%s: Rx skb DMA mapping failed\n", __func__);
1441                 return NULL;
1442         }
1443
1444         /* Grab the current Rx skb from the ring and DMA-unmap it */
1445         rx_skb = cb->skb;
1446         if (likely(rx_skb))
1447                 dma_unmap_single(kdev, dma_unmap_addr(cb, dma_addr),
1448                                  priv->rx_buf_len, DMA_FROM_DEVICE);
1449
1450         /* Put the new Rx skb on the ring */
1451         cb->skb = skb;
1452         dma_unmap_addr_set(cb, dma_addr, mapping);
1453         dmadesc_set_addr(priv, cb->bd_addr, mapping);
1454
1455         /* Return the current Rx skb to caller */
1456         return rx_skb;
1457 }
1458
1459 /* bcmgenet_desc_rx - descriptor based rx process.
1460  * this could be called from bottom half, or from NAPI polling method.
1461  */
1462 static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,
1463                                      unsigned int budget)
1464 {
1465         struct bcmgenet_priv *priv = ring->priv;
1466         struct net_device *dev = priv->dev;
1467         struct enet_cb *cb;
1468         struct sk_buff *skb;
1469         u32 dma_length_status;
1470         unsigned long dma_flag;
1471         int len;
1472         unsigned int rxpktprocessed = 0, rxpkttoprocess;
1473         unsigned int p_index;
1474         unsigned int discards;
1475         unsigned int chksum_ok = 0;
1476
1477         p_index = bcmgenet_rdma_ring_readl(priv, ring->index, RDMA_PROD_INDEX);
1478
1479         discards = (p_index >> DMA_P_INDEX_DISCARD_CNT_SHIFT) &
1480                    DMA_P_INDEX_DISCARD_CNT_MASK;
1481         if (discards > ring->old_discards) {
1482                 discards = discards - ring->old_discards;
1483                 dev->stats.rx_missed_errors += discards;
1484                 dev->stats.rx_errors += discards;
1485                 ring->old_discards += discards;
1486
1487                 /* Clear HW register when we reach 75% of maximum 0xFFFF */
1488                 if (ring->old_discards >= 0xC000) {
1489                         ring->old_discards = 0;
1490                         bcmgenet_rdma_ring_writel(priv, ring->index, 0,
1491                                                   RDMA_PROD_INDEX);
1492                 }
1493         }
1494
1495         p_index &= DMA_P_INDEX_MASK;
1496
1497         if (likely(p_index >= ring->c_index))
1498                 rxpkttoprocess = p_index - ring->c_index;
1499         else
1500                 rxpkttoprocess = (DMA_C_INDEX_MASK + 1) - ring->c_index +
1501                                  p_index;
1502
1503         netif_dbg(priv, rx_status, dev,
1504                   "RDMA: rxpkttoprocess=%d\n", rxpkttoprocess);
1505
1506         while ((rxpktprocessed < rxpkttoprocess) &&
1507                (rxpktprocessed < budget)) {
1508                 cb = &priv->rx_cbs[ring->read_ptr];
1509                 skb = bcmgenet_rx_refill(priv, cb);
1510
1511                 if (unlikely(!skb)) {
1512                         dev->stats.rx_dropped++;
1513                         goto next;
1514                 }
1515
1516                 if (!priv->desc_64b_en) {
1517                         dma_length_status =
1518                                 dmadesc_get_length_status(priv, cb->bd_addr);
1519                 } else {
1520                         struct status_64 *status;
1521
1522                         status = (struct status_64 *)skb->data;
1523                         dma_length_status = status->length_status;
1524                 }
1525
1526                 /* DMA flags and length are still valid no matter how
1527                  * we got the Receive Status Vector (64B RSB or register)
1528                  */
1529                 dma_flag = dma_length_status & 0xffff;
1530                 len = dma_length_status >> DMA_BUFLENGTH_SHIFT;
1531
1532                 netif_dbg(priv, rx_status, dev,
1533                           "%s:p_ind=%d c_ind=%d read_ptr=%d len_stat=0x%08x\n",
1534                           __func__, p_index, ring->c_index,
1535                           ring->read_ptr, dma_length_status);
1536
1537                 if (unlikely(!(dma_flag & DMA_EOP) || !(dma_flag & DMA_SOP))) {
1538                         netif_err(priv, rx_status, dev,
1539                                   "dropping fragmented packet!\n");
1540                         dev->stats.rx_errors++;
1541                         dev_kfree_skb_any(skb);
1542                         goto next;
1543                 }
1544
1545                 /* report errors */
1546                 if (unlikely(dma_flag & (DMA_RX_CRC_ERROR |
1547                                                 DMA_RX_OV |
1548                                                 DMA_RX_NO |
1549                                                 DMA_RX_LG |
1550                                                 DMA_RX_RXER))) {
1551                         netif_err(priv, rx_status, dev, "dma_flag=0x%x\n",
1552                                   (unsigned int)dma_flag);
1553                         if (dma_flag & DMA_RX_CRC_ERROR)
1554                                 dev->stats.rx_crc_errors++;
1555                         if (dma_flag & DMA_RX_OV)
1556                                 dev->stats.rx_over_errors++;
1557                         if (dma_flag & DMA_RX_NO)
1558                                 dev->stats.rx_frame_errors++;
1559                         if (dma_flag & DMA_RX_LG)
1560                                 dev->stats.rx_length_errors++;
1561                         dev->stats.rx_errors++;
1562                         dev_kfree_skb_any(skb);
1563                         goto next;
1564                 } /* error packet */
1565
1566                 chksum_ok = (dma_flag & priv->dma_rx_chk_bit) &&
1567                              priv->desc_rxchk_en;
1568
1569                 skb_put(skb, len);
1570                 if (priv->desc_64b_en) {
1571                         skb_pull(skb, 64);
1572                         len -= 64;
1573                 }
1574
1575                 if (likely(chksum_ok))
1576                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1577
1578                 /* remove hardware 2bytes added for IP alignment */
1579                 skb_pull(skb, 2);
1580                 len -= 2;
1581
1582                 if (priv->crc_fwd_en) {
1583                         skb_trim(skb, len - ETH_FCS_LEN);
1584                         len -= ETH_FCS_LEN;
1585                 }
1586
1587                 /*Finish setting up the received SKB and send it to the kernel*/
1588                 skb->protocol = eth_type_trans(skb, priv->dev);
1589                 dev->stats.rx_packets++;
1590                 dev->stats.rx_bytes += len;
1591                 if (dma_flag & DMA_RX_MULT)
1592                         dev->stats.multicast++;
1593
1594                 /* Notify kernel */
1595                 napi_gro_receive(&ring->napi, skb);
1596                 netif_dbg(priv, rx_status, dev, "pushed up to kernel\n");
1597
1598 next:
1599                 rxpktprocessed++;
1600                 if (likely(ring->read_ptr < ring->end_ptr))
1601                         ring->read_ptr++;
1602                 else
1603                         ring->read_ptr = ring->cb_ptr;
1604
1605                 ring->c_index = (ring->c_index + 1) & DMA_C_INDEX_MASK;
1606                 bcmgenet_rdma_ring_writel(priv, ring->index, ring->c_index, RDMA_CONS_INDEX);
1607         }
1608
1609         return rxpktprocessed;
1610 }
1611
1612 /* Rx NAPI polling method */
1613 static int bcmgenet_rx_poll(struct napi_struct *napi, int budget)
1614 {
1615         struct bcmgenet_rx_ring *ring = container_of(napi,
1616                         struct bcmgenet_rx_ring, napi);
1617         unsigned int work_done;
1618
1619         work_done = bcmgenet_desc_rx(ring, budget);
1620
1621         if (work_done < budget) {
1622                 napi_complete(napi);
1623                 ring->int_enable(ring);
1624         }
1625
1626         return work_done;
1627 }
1628
1629 /* Assign skb to RX DMA descriptor. */
1630 static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv,
1631                                      struct bcmgenet_rx_ring *ring)
1632 {
1633         struct enet_cb *cb;
1634         struct sk_buff *skb;
1635         int i;
1636
1637         netif_dbg(priv, hw, priv->dev, "%s\n", __func__);
1638
1639         /* loop here for each buffer needing assign */
1640         for (i = 0; i < ring->size; i++) {
1641                 cb = ring->cbs + i;
1642                 skb = bcmgenet_rx_refill(priv, cb);
1643                 if (skb)
1644                         dev_kfree_skb_any(skb);
1645                 if (!cb->skb)
1646                         return -ENOMEM;
1647         }
1648
1649         return 0;
1650 }
1651
1652 static void bcmgenet_free_rx_buffers(struct bcmgenet_priv *priv)
1653 {
1654         struct enet_cb *cb;
1655         int i;
1656
1657         for (i = 0; i < priv->num_rx_bds; i++) {
1658                 cb = &priv->rx_cbs[i];
1659
1660                 if (dma_unmap_addr(cb, dma_addr)) {
1661                         dma_unmap_single(&priv->dev->dev,
1662                                          dma_unmap_addr(cb, dma_addr),
1663                                          priv->rx_buf_len, DMA_FROM_DEVICE);
1664                         dma_unmap_addr_set(cb, dma_addr, 0);
1665                 }
1666
1667                 if (cb->skb)
1668                         bcmgenet_free_cb(cb);
1669         }
1670 }
1671
1672 static void umac_enable_set(struct bcmgenet_priv *priv, u32 mask, bool enable)
1673 {
1674         u32 reg;
1675
1676         reg = bcmgenet_umac_readl(priv, UMAC_CMD);
1677         if (enable)
1678                 reg |= mask;
1679         else
1680                 reg &= ~mask;
1681         bcmgenet_umac_writel(priv, reg, UMAC_CMD);
1682
1683         /* UniMAC stops on a packet boundary, wait for a full-size packet
1684          * to be processed
1685          */
1686         if (enable == 0)
1687                 usleep_range(1000, 2000);
1688 }
1689
1690 static int reset_umac(struct bcmgenet_priv *priv)
1691 {
1692         struct device *kdev = &priv->pdev->dev;
1693         unsigned int timeout = 0;
1694         u32 reg;
1695
1696         /* 7358a0/7552a0: bad default in RBUF_FLUSH_CTRL.umac_sw_rst */
1697         bcmgenet_rbuf_ctrl_set(priv, 0);
1698         udelay(10);
1699
1700         /* disable MAC while updating its registers */
1701         bcmgenet_umac_writel(priv, 0, UMAC_CMD);
1702
1703         /* issue soft reset, wait for it to complete */
1704         bcmgenet_umac_writel(priv, CMD_SW_RESET, UMAC_CMD);
1705         while (timeout++ < 1000) {
1706                 reg = bcmgenet_umac_readl(priv, UMAC_CMD);
1707                 if (!(reg & CMD_SW_RESET))
1708                         return 0;
1709
1710                 udelay(1);
1711         }
1712
1713         if (timeout == 1000) {
1714                 dev_err(kdev,
1715                         "timeout waiting for MAC to come out of reset\n");
1716                 return -ETIMEDOUT;
1717         }
1718
1719         return 0;
1720 }
1721
1722 static void bcmgenet_intr_disable(struct bcmgenet_priv *priv)
1723 {
1724         /* Mask all interrupts.*/
1725         bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
1726         bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
1727         bcmgenet_intrl2_0_writel(priv, 0, INTRL2_CPU_MASK_CLEAR);
1728         bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
1729         bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
1730         bcmgenet_intrl2_1_writel(priv, 0, INTRL2_CPU_MASK_CLEAR);
1731 }
1732
1733 static int init_umac(struct bcmgenet_priv *priv)
1734 {
1735         struct device *kdev = &priv->pdev->dev;
1736         int ret;
1737         u32 reg;
1738         u32 int0_enable = 0;
1739         u32 int1_enable = 0;
1740         int i;
1741
1742         dev_dbg(&priv->pdev->dev, "bcmgenet: init_umac\n");
1743
1744         ret = reset_umac(priv);
1745         if (ret)
1746                 return ret;
1747
1748         bcmgenet_umac_writel(priv, 0, UMAC_CMD);
1749         /* clear tx/rx counter */
1750         bcmgenet_umac_writel(priv,
1751                              MIB_RESET_RX | MIB_RESET_TX | MIB_RESET_RUNT,
1752                              UMAC_MIB_CTRL);
1753         bcmgenet_umac_writel(priv, 0, UMAC_MIB_CTRL);
1754
1755         bcmgenet_umac_writel(priv, ENET_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN);
1756
1757         /* init rx registers, enable ip header optimization */
1758         reg = bcmgenet_rbuf_readl(priv, RBUF_CTRL);
1759         reg |= RBUF_ALIGN_2B;
1760         bcmgenet_rbuf_writel(priv, reg, RBUF_CTRL);
1761
1762         if (!GENET_IS_V1(priv) && !GENET_IS_V2(priv))
1763                 bcmgenet_rbuf_writel(priv, 1, RBUF_TBUF_SIZE_CTRL);
1764
1765         bcmgenet_intr_disable(priv);
1766
1767         /* Enable Rx default queue 16 interrupts */
1768         int0_enable |= UMAC_IRQ_RXDMA_DONE;
1769
1770         /* Enable Tx default queue 16 interrupts */
1771         int0_enable |= UMAC_IRQ_TXDMA_DONE;
1772
1773         /* Monitor cable plug/unplugged event for internal PHY */
1774         if (priv->internal_phy) {
1775                 int0_enable |= UMAC_IRQ_LINK_EVENT;
1776         } else if (priv->ext_phy) {
1777                 int0_enable |= UMAC_IRQ_LINK_EVENT;
1778         } else if (priv->phy_interface == PHY_INTERFACE_MODE_MOCA) {
1779                 if (priv->hw_params->flags & GENET_HAS_MOCA_LINK_DET)
1780                         int0_enable |= UMAC_IRQ_LINK_EVENT;
1781
1782                 reg = bcmgenet_bp_mc_get(priv);
1783                 reg |= BIT(priv->hw_params->bp_in_en_shift);
1784
1785                 /* bp_mask: back pressure mask */
1786                 if (netif_is_multiqueue(priv->dev))
1787                         reg |= priv->hw_params->bp_in_mask;
1788                 else
1789                         reg &= ~priv->hw_params->bp_in_mask;
1790                 bcmgenet_bp_mc_set(priv, reg);
1791         }
1792
1793         /* Enable MDIO interrupts on GENET v3+ */
1794         if (priv->hw_params->flags & GENET_HAS_MDIO_INTR)
1795                 int0_enable |= (UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR);
1796
1797         /* Enable Rx priority queue interrupts */
1798         for (i = 0; i < priv->hw_params->rx_queues; ++i)
1799                 int1_enable |= (1 << (UMAC_IRQ1_RX_INTR_SHIFT + i));
1800
1801         /* Enable Tx priority queue interrupts */
1802         for (i = 0; i < priv->hw_params->tx_queues; ++i)
1803                 int1_enable |= (1 << i);
1804
1805         bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);
1806         bcmgenet_intrl2_1_writel(priv, int1_enable, INTRL2_CPU_MASK_CLEAR);
1807
1808         /* Enable rx/tx engine.*/
1809         dev_dbg(kdev, "done init umac\n");
1810
1811         return 0;
1812 }
1813
1814 /* Initialize a Tx ring along with corresponding hardware registers */
1815 static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv,
1816                                   unsigned int index, unsigned int size,
1817                                   unsigned int start_ptr, unsigned int end_ptr)
1818 {
1819         struct bcmgenet_tx_ring *ring = &priv->tx_rings[index];
1820         u32 words_per_bd = WORDS_PER_BD(priv);
1821         u32 flow_period_val = 0;
1822
1823         spin_lock_init(&ring->lock);
1824         ring->priv = priv;
1825         ring->index = index;
1826         if (index == DESC_INDEX) {
1827                 ring->queue = 0;
1828                 ring->int_enable = bcmgenet_tx_ring16_int_enable;
1829                 ring->int_disable = bcmgenet_tx_ring16_int_disable;
1830         } else {
1831                 ring->queue = index + 1;
1832                 ring->int_enable = bcmgenet_tx_ring_int_enable;
1833                 ring->int_disable = bcmgenet_tx_ring_int_disable;
1834         }
1835         ring->cbs = priv->tx_cbs + start_ptr;
1836         ring->size = size;
1837         ring->clean_ptr = start_ptr;
1838         ring->c_index = 0;
1839         ring->free_bds = size;
1840         ring->write_ptr = start_ptr;
1841         ring->cb_ptr = start_ptr;
1842         ring->end_ptr = end_ptr - 1;
1843         ring->prod_index = 0;
1844
1845         /* Set flow period for ring != 16 */
1846         if (index != DESC_INDEX)
1847                 flow_period_val = ENET_MAX_MTU_SIZE << 16;
1848
1849         bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_PROD_INDEX);
1850         bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_CONS_INDEX);
1851         bcmgenet_tdma_ring_writel(priv, index, 1, DMA_MBUF_DONE_THRESH);
1852         /* Disable rate control for now */
1853         bcmgenet_tdma_ring_writel(priv, index, flow_period_val,
1854                                   TDMA_FLOW_PERIOD);
1855         bcmgenet_tdma_ring_writel(priv, index,
1856                                   ((size << DMA_RING_SIZE_SHIFT) |
1857                                    RX_BUF_LENGTH), DMA_RING_BUF_SIZE);
1858
1859         /* Set start and end address, read and write pointers */
1860         bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
1861                                   DMA_START_ADDR);
1862         bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
1863                                   TDMA_READ_PTR);
1864         bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
1865                                   TDMA_WRITE_PTR);
1866         bcmgenet_tdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
1867                                   DMA_END_ADDR);
1868 }
1869
1870 /* Initialize a RDMA ring */
1871 static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv,
1872                                  unsigned int index, unsigned int size,
1873                                  unsigned int start_ptr, unsigned int end_ptr)
1874 {
1875         struct bcmgenet_rx_ring *ring = &priv->rx_rings[index];
1876         u32 words_per_bd = WORDS_PER_BD(priv);
1877         int ret;
1878
1879         ring->priv = priv;
1880         ring->index = index;
1881         if (index == DESC_INDEX) {
1882                 ring->int_enable = bcmgenet_rx_ring16_int_enable;
1883                 ring->int_disable = bcmgenet_rx_ring16_int_disable;
1884         } else {
1885                 ring->int_enable = bcmgenet_rx_ring_int_enable;
1886                 ring->int_disable = bcmgenet_rx_ring_int_disable;
1887         }
1888         ring->cbs = priv->rx_cbs + start_ptr;
1889         ring->size = size;
1890         ring->c_index = 0;
1891         ring->read_ptr = start_ptr;
1892         ring->cb_ptr = start_ptr;
1893         ring->end_ptr = end_ptr - 1;
1894
1895         ret = bcmgenet_alloc_rx_buffers(priv, ring);
1896         if (ret)
1897                 return ret;
1898
1899         bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_PROD_INDEX);
1900         bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_CONS_INDEX);
1901         bcmgenet_rdma_ring_writel(priv, index, 1, DMA_MBUF_DONE_THRESH);
1902         bcmgenet_rdma_ring_writel(priv, index,
1903                                   ((size << DMA_RING_SIZE_SHIFT) |
1904                                    RX_BUF_LENGTH), DMA_RING_BUF_SIZE);
1905         bcmgenet_rdma_ring_writel(priv, index,
1906                                   (DMA_FC_THRESH_LO <<
1907                                    DMA_XOFF_THRESHOLD_SHIFT) |
1908                                    DMA_FC_THRESH_HI, RDMA_XON_XOFF_THRESH);
1909
1910         /* Set start and end address, read and write pointers */
1911         bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
1912                                   DMA_START_ADDR);
1913         bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
1914                                   RDMA_READ_PTR);
1915         bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
1916                                   RDMA_WRITE_PTR);
1917         bcmgenet_rdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
1918                                   DMA_END_ADDR);
1919
1920         return ret;
1921 }
1922
1923 static void bcmgenet_init_tx_napi(struct bcmgenet_priv *priv)
1924 {
1925         unsigned int i;
1926         struct bcmgenet_tx_ring *ring;
1927
1928         for (i = 0; i < priv->hw_params->tx_queues; ++i) {
1929                 ring = &priv->tx_rings[i];
1930                 netif_napi_add(priv->dev, &ring->napi, bcmgenet_tx_poll, 64);
1931         }
1932
1933         ring = &priv->tx_rings[DESC_INDEX];
1934         netif_napi_add(priv->dev, &ring->napi, bcmgenet_tx_poll, 64);
1935 }
1936
1937 static void bcmgenet_enable_tx_napi(struct bcmgenet_priv *priv)
1938 {
1939         unsigned int i;
1940         struct bcmgenet_tx_ring *ring;
1941
1942         for (i = 0; i < priv->hw_params->tx_queues; ++i) {
1943                 ring = &priv->tx_rings[i];
1944                 napi_enable(&ring->napi);
1945         }
1946
1947         ring = &priv->tx_rings[DESC_INDEX];
1948         napi_enable(&ring->napi);
1949 }
1950
1951 static void bcmgenet_disable_tx_napi(struct bcmgenet_priv *priv)
1952 {
1953         unsigned int i;
1954         struct bcmgenet_tx_ring *ring;
1955
1956         for (i = 0; i < priv->hw_params->tx_queues; ++i) {
1957                 ring = &priv->tx_rings[i];
1958                 napi_disable(&ring->napi);
1959         }
1960
1961         ring = &priv->tx_rings[DESC_INDEX];
1962         napi_disable(&ring->napi);
1963 }
1964
1965 static void bcmgenet_fini_tx_napi(struct bcmgenet_priv *priv)
1966 {
1967         unsigned int i;
1968         struct bcmgenet_tx_ring *ring;
1969
1970         for (i = 0; i < priv->hw_params->tx_queues; ++i) {
1971                 ring = &priv->tx_rings[i];
1972                 netif_napi_del(&ring->napi);
1973         }
1974
1975         ring = &priv->tx_rings[DESC_INDEX];
1976         netif_napi_del(&ring->napi);
1977 }
1978
1979 /* Initialize Tx queues
1980  *
1981  * Queues 0-3 are priority-based, each one has 32 descriptors,
1982  * with queue 0 being the highest priority queue.
1983  *
1984  * Queue 16 is the default Tx queue with
1985  * GENET_Q16_TX_BD_CNT = 256 - 4 * 32 = 128 descriptors.
1986  *
1987  * The transmit control block pool is then partitioned as follows:
1988  * - Tx queue 0 uses tx_cbs[0..31]
1989  * - Tx queue 1 uses tx_cbs[32..63]
1990  * - Tx queue 2 uses tx_cbs[64..95]
1991  * - Tx queue 3 uses tx_cbs[96..127]
1992  * - Tx queue 16 uses tx_cbs[128..255]
1993  */
1994 static void bcmgenet_init_tx_queues(struct net_device *dev)
1995 {
1996         struct bcmgenet_priv *priv = netdev_priv(dev);
1997         u32 i, dma_enable;
1998         u32 dma_ctrl, ring_cfg;
1999         u32 dma_priority[3] = {0, 0, 0};
2000
2001         dma_ctrl = bcmgenet_tdma_readl(priv, DMA_CTRL);
2002         dma_enable = dma_ctrl & DMA_EN;
2003         dma_ctrl &= ~DMA_EN;
2004         bcmgenet_tdma_writel(priv, dma_ctrl, DMA_CTRL);
2005
2006         dma_ctrl = 0;
2007         ring_cfg = 0;
2008
2009         /* Enable strict priority arbiter mode */
2010         bcmgenet_tdma_writel(priv, DMA_ARBITER_SP, DMA_ARB_CTRL);
2011
2012         /* Initialize Tx priority queues */
2013         for (i = 0; i < priv->hw_params->tx_queues; i++) {
2014                 bcmgenet_init_tx_ring(priv, i, priv->hw_params->tx_bds_per_q,
2015                                       i * priv->hw_params->tx_bds_per_q,
2016                                       (i + 1) * priv->hw_params->tx_bds_per_q);
2017                 ring_cfg |= (1 << i);
2018                 dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
2019                 dma_priority[DMA_PRIO_REG_INDEX(i)] |=
2020                         ((GENET_Q0_PRIORITY + i) << DMA_PRIO_REG_SHIFT(i));
2021         }
2022
2023         /* Initialize Tx default queue 16 */
2024         bcmgenet_init_tx_ring(priv, DESC_INDEX, GENET_Q16_TX_BD_CNT,
2025                               priv->hw_params->tx_queues *
2026                               priv->hw_params->tx_bds_per_q,
2027                               TOTAL_DESC);
2028         ring_cfg |= (1 << DESC_INDEX);
2029         dma_ctrl |= (1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT));
2030         dma_priority[DMA_PRIO_REG_INDEX(DESC_INDEX)] |=
2031                 ((GENET_Q0_PRIORITY + priv->hw_params->tx_queues) <<
2032                  DMA_PRIO_REG_SHIFT(DESC_INDEX));
2033
2034         /* Set Tx queue priorities */
2035         bcmgenet_tdma_writel(priv, dma_priority[0], DMA_PRIORITY_0);
2036         bcmgenet_tdma_writel(priv, dma_priority[1], DMA_PRIORITY_1);
2037         bcmgenet_tdma_writel(priv, dma_priority[2], DMA_PRIORITY_2);
2038
2039         /* Initialize Tx NAPI */
2040         bcmgenet_init_tx_napi(priv);
2041
2042         /* Enable Tx queues */
2043         bcmgenet_tdma_writel(priv, ring_cfg, DMA_RING_CFG);
2044
2045         /* Enable Tx DMA */
2046         if (dma_enable)
2047                 dma_ctrl |= DMA_EN;
2048         bcmgenet_tdma_writel(priv, dma_ctrl, DMA_CTRL);
2049 }
2050
2051 static void bcmgenet_init_rx_napi(struct bcmgenet_priv *priv)
2052 {
2053         unsigned int i;
2054         struct bcmgenet_rx_ring *ring;
2055
2056         for (i = 0; i < priv->hw_params->rx_queues; ++i) {
2057                 ring = &priv->rx_rings[i];
2058                 netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll, 64);
2059         }
2060
2061         ring = &priv->rx_rings[DESC_INDEX];
2062         netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll, 64);
2063 }
2064
2065 static void bcmgenet_enable_rx_napi(struct bcmgenet_priv *priv)
2066 {
2067         unsigned int i;
2068         struct bcmgenet_rx_ring *ring;
2069
2070         for (i = 0; i < priv->hw_params->rx_queues; ++i) {
2071                 ring = &priv->rx_rings[i];
2072                 napi_enable(&ring->napi);
2073         }
2074
2075         ring = &priv->rx_rings[DESC_INDEX];
2076         napi_enable(&ring->napi);
2077 }
2078
2079 static void bcmgenet_disable_rx_napi(struct bcmgenet_priv *priv)
2080 {
2081         unsigned int i;
2082         struct bcmgenet_rx_ring *ring;
2083
2084         for (i = 0; i < priv->hw_params->rx_queues; ++i) {
2085                 ring = &priv->rx_rings[i];
2086                 napi_disable(&ring->napi);
2087         }
2088
2089         ring = &priv->rx_rings[DESC_INDEX];
2090         napi_disable(&ring->napi);
2091 }
2092
2093 static void bcmgenet_fini_rx_napi(struct bcmgenet_priv *priv)
2094 {
2095         unsigned int i;
2096         struct bcmgenet_rx_ring *ring;
2097
2098         for (i = 0; i < priv->hw_params->rx_queues; ++i) {
2099                 ring = &priv->rx_rings[i];
2100                 netif_napi_del(&ring->napi);
2101         }
2102
2103         ring = &priv->rx_rings[DESC_INDEX];
2104         netif_napi_del(&ring->napi);
2105 }
2106
2107 /* Initialize Rx queues
2108  *
2109  * Queues 0-15 are priority queues. Hardware Filtering Block (HFB) can be
2110  * used to direct traffic to these queues.
2111  *
2112  * Queue 16 is the default Rx queue with GENET_Q16_RX_BD_CNT descriptors.
2113  */
2114 static int bcmgenet_init_rx_queues(struct net_device *dev)
2115 {
2116         struct bcmgenet_priv *priv = netdev_priv(dev);
2117         u32 i;
2118         u32 dma_enable;
2119         u32 dma_ctrl;
2120         u32 ring_cfg;
2121         int ret;
2122
2123         dma_ctrl = bcmgenet_rdma_readl(priv, DMA_CTRL);
2124         dma_enable = dma_ctrl & DMA_EN;
2125         dma_ctrl &= ~DMA_EN;
2126         bcmgenet_rdma_writel(priv, dma_ctrl, DMA_CTRL);
2127
2128         dma_ctrl = 0;
2129         ring_cfg = 0;
2130
2131         /* Initialize Rx priority queues */
2132         for (i = 0; i < priv->hw_params->rx_queues; i++) {
2133                 ret = bcmgenet_init_rx_ring(priv, i,
2134                                             priv->hw_params->rx_bds_per_q,
2135                                             i * priv->hw_params->rx_bds_per_q,
2136                                             (i + 1) *
2137                                             priv->hw_params->rx_bds_per_q);
2138                 if (ret)
2139                         return ret;
2140
2141                 ring_cfg |= (1 << i);
2142                 dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
2143         }
2144
2145         /* Initialize Rx default queue 16 */
2146         ret = bcmgenet_init_rx_ring(priv, DESC_INDEX, GENET_Q16_RX_BD_CNT,
2147                                     priv->hw_params->rx_queues *
2148                                     priv->hw_params->rx_bds_per_q,
2149                                     TOTAL_DESC);
2150         if (ret)
2151                 return ret;
2152
2153         ring_cfg |= (1 << DESC_INDEX);
2154         dma_ctrl |= (1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT));
2155
2156         /* Initialize Rx NAPI */
2157         bcmgenet_init_rx_napi(priv);
2158
2159         /* Enable rings */
2160         bcmgenet_rdma_writel(priv, ring_cfg, DMA_RING_CFG);
2161
2162         /* Configure ring as descriptor ring and re-enable DMA if enabled */
2163         if (dma_enable)
2164                 dma_ctrl |= DMA_EN;
2165         bcmgenet_rdma_writel(priv, dma_ctrl, DMA_CTRL);
2166
2167         return 0;
2168 }
2169
2170 static int bcmgenet_dma_teardown(struct bcmgenet_priv *priv)
2171 {
2172         int ret = 0;
2173         int timeout = 0;
2174         u32 reg;
2175         u32 dma_ctrl;
2176         int i;
2177
2178         /* Disable TDMA to stop add more frames in TX DMA */
2179         reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2180         reg &= ~DMA_EN;
2181         bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2182
2183         /* Check TDMA status register to confirm TDMA is disabled */
2184         while (timeout++ < DMA_TIMEOUT_VAL) {
2185                 reg = bcmgenet_tdma_readl(priv, DMA_STATUS);
2186                 if (reg & DMA_DISABLED)
2187                         break;
2188
2189                 udelay(1);
2190         }
2191
2192         if (timeout == DMA_TIMEOUT_VAL) {
2193                 netdev_warn(priv->dev, "Timed out while disabling TX DMA\n");
2194                 ret = -ETIMEDOUT;
2195         }
2196
2197         /* Wait 10ms for packet drain in both tx and rx dma */
2198         usleep_range(10000, 20000);
2199
2200         /* Disable RDMA */
2201         reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2202         reg &= ~DMA_EN;
2203         bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2204
2205         timeout = 0;
2206         /* Check RDMA status register to confirm RDMA is disabled */
2207         while (timeout++ < DMA_TIMEOUT_VAL) {
2208                 reg = bcmgenet_rdma_readl(priv, DMA_STATUS);
2209                 if (reg & DMA_DISABLED)
2210                         break;
2211
2212                 udelay(1);
2213         }
2214
2215         if (timeout == DMA_TIMEOUT_VAL) {
2216                 netdev_warn(priv->dev, "Timed out while disabling RX DMA\n");
2217                 ret = -ETIMEDOUT;
2218         }
2219
2220         dma_ctrl = 0;
2221         for (i = 0; i < priv->hw_params->rx_queues; i++)
2222                 dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
2223         reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2224         reg &= ~dma_ctrl;
2225         bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2226
2227         dma_ctrl = 0;
2228         for (i = 0; i < priv->hw_params->tx_queues; i++)
2229                 dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
2230         reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2231         reg &= ~dma_ctrl;
2232         bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2233
2234         return ret;
2235 }
2236
2237 static void bcmgenet_fini_dma(struct bcmgenet_priv *priv)
2238 {
2239         int i;
2240
2241         bcmgenet_fini_rx_napi(priv);
2242         bcmgenet_fini_tx_napi(priv);
2243
2244         /* disable DMA */
2245         bcmgenet_dma_teardown(priv);
2246
2247         for (i = 0; i < priv->num_tx_bds; i++) {
2248                 if (priv->tx_cbs[i].skb != NULL) {
2249                         dev_kfree_skb(priv->tx_cbs[i].skb);
2250                         priv->tx_cbs[i].skb = NULL;
2251                 }
2252         }
2253
2254         bcmgenet_free_rx_buffers(priv);
2255         kfree(priv->rx_cbs);
2256         kfree(priv->tx_cbs);
2257 }
2258
2259 /* init_edma: Initialize DMA control register */
2260 static int bcmgenet_init_dma(struct bcmgenet_priv *priv)
2261 {
2262         int ret;
2263         unsigned int i;
2264         struct enet_cb *cb;
2265
2266         netif_dbg(priv, hw, priv->dev, "%s\n", __func__);
2267
2268         /* Initialize common Rx ring structures */
2269         priv->rx_bds = priv->base + priv->hw_params->rdma_offset;
2270         priv->num_rx_bds = TOTAL_DESC;
2271         priv->rx_cbs = kcalloc(priv->num_rx_bds, sizeof(struct enet_cb),
2272                                GFP_KERNEL);
2273         if (!priv->rx_cbs)
2274                 return -ENOMEM;
2275
2276         for (i = 0; i < priv->num_rx_bds; i++) {
2277                 cb = priv->rx_cbs + i;
2278                 cb->bd_addr = priv->rx_bds + i * DMA_DESC_SIZE;
2279         }
2280
2281         /* Initialize common TX ring structures */
2282         priv->tx_bds = priv->base + priv->hw_params->tdma_offset;
2283         priv->num_tx_bds = TOTAL_DESC;
2284         priv->tx_cbs = kcalloc(priv->num_tx_bds, sizeof(struct enet_cb),
2285                                GFP_KERNEL);
2286         if (!priv->tx_cbs) {
2287                 kfree(priv->rx_cbs);
2288                 return -ENOMEM;
2289         }
2290
2291         for (i = 0; i < priv->num_tx_bds; i++) {
2292                 cb = priv->tx_cbs + i;
2293                 cb->bd_addr = priv->tx_bds + i * DMA_DESC_SIZE;
2294         }
2295
2296         /* Init rDma */
2297         bcmgenet_rdma_writel(priv, DMA_MAX_BURST_LENGTH, DMA_SCB_BURST_SIZE);
2298
2299         /* Initialize Rx queues */
2300         ret = bcmgenet_init_rx_queues(priv->dev);
2301         if (ret) {
2302                 netdev_err(priv->dev, "failed to initialize Rx queues\n");
2303                 bcmgenet_free_rx_buffers(priv);
2304                 kfree(priv->rx_cbs);
2305                 kfree(priv->tx_cbs);
2306                 return ret;
2307         }
2308
2309         /* Init tDma */
2310         bcmgenet_tdma_writel(priv, DMA_MAX_BURST_LENGTH, DMA_SCB_BURST_SIZE);
2311
2312         /* Initialize Tx queues */
2313         bcmgenet_init_tx_queues(priv->dev);
2314
2315         return 0;
2316 }
2317
2318 /* Interrupt bottom half */
2319 static void bcmgenet_irq_task(struct work_struct *work)
2320 {
2321         struct bcmgenet_priv *priv = container_of(
2322                         work, struct bcmgenet_priv, bcmgenet_irq_work);
2323
2324         netif_dbg(priv, intr, priv->dev, "%s\n", __func__);
2325
2326         if (priv->irq0_stat & UMAC_IRQ_MPD_R) {
2327                 priv->irq0_stat &= ~UMAC_IRQ_MPD_R;
2328                 netif_dbg(priv, wol, priv->dev,
2329                           "magic packet detected, waking up\n");
2330                 bcmgenet_power_up(priv, GENET_POWER_WOL_MAGIC);
2331         }
2332
2333         /* Link UP/DOWN event */
2334         if ((priv->hw_params->flags & GENET_HAS_MDIO_INTR) &&
2335             (priv->irq0_stat & UMAC_IRQ_LINK_EVENT)) {
2336                 phy_mac_interrupt(priv->phydev,
2337                                   !!(priv->irq0_stat & UMAC_IRQ_LINK_UP));
2338                 priv->irq0_stat &= ~UMAC_IRQ_LINK_EVENT;
2339         }
2340 }
2341
2342 /* bcmgenet_isr1: handle Rx and Tx priority queues */
2343 static irqreturn_t bcmgenet_isr1(int irq, void *dev_id)
2344 {
2345         struct bcmgenet_priv *priv = dev_id;
2346         struct bcmgenet_rx_ring *rx_ring;
2347         struct bcmgenet_tx_ring *tx_ring;
2348         unsigned int index;
2349
2350         /* Save irq status for bottom-half processing. */
2351         priv->irq1_stat =
2352                 bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_STAT) &
2353                 ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
2354
2355         /* clear interrupts */
2356         bcmgenet_intrl2_1_writel(priv, priv->irq1_stat, INTRL2_CPU_CLEAR);
2357
2358         netif_dbg(priv, intr, priv->dev,
2359                   "%s: IRQ=0x%x\n", __func__, priv->irq1_stat);
2360
2361         /* Check Rx priority queue interrupts */
2362         for (index = 0; index < priv->hw_params->rx_queues; index++) {
2363                 if (!(priv->irq1_stat & BIT(UMAC_IRQ1_RX_INTR_SHIFT + index)))
2364                         continue;
2365
2366                 rx_ring = &priv->rx_rings[index];
2367
2368                 if (likely(napi_schedule_prep(&rx_ring->napi))) {
2369                         rx_ring->int_disable(rx_ring);
2370                         __napi_schedule(&rx_ring->napi);
2371                 }
2372         }
2373
2374         /* Check Tx priority queue interrupts */
2375         for (index = 0; index < priv->hw_params->tx_queues; index++) {
2376                 if (!(priv->irq1_stat & BIT(index)))
2377                         continue;
2378
2379                 tx_ring = &priv->tx_rings[index];
2380
2381                 if (likely(napi_schedule_prep(&tx_ring->napi))) {
2382                         tx_ring->int_disable(tx_ring);
2383                         __napi_schedule(&tx_ring->napi);
2384                 }
2385         }
2386
2387         return IRQ_HANDLED;
2388 }
2389
2390 /* bcmgenet_isr0: handle Rx and Tx default queues + other stuff */
2391 static irqreturn_t bcmgenet_isr0(int irq, void *dev_id)
2392 {
2393         struct bcmgenet_priv *priv = dev_id;
2394         struct bcmgenet_rx_ring *rx_ring;
2395         struct bcmgenet_tx_ring *tx_ring;
2396
2397         /* Save irq status for bottom-half processing. */
2398         priv->irq0_stat =
2399                 bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT) &
2400                 ~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);
2401
2402         /* clear interrupts */
2403         bcmgenet_intrl2_0_writel(priv, priv->irq0_stat, INTRL2_CPU_CLEAR);
2404
2405         netif_dbg(priv, intr, priv->dev,
2406                   "IRQ=0x%x\n", priv->irq0_stat);
2407
2408         if (priv->irq0_stat & UMAC_IRQ_RXDMA_DONE) {
2409                 rx_ring = &priv->rx_rings[DESC_INDEX];
2410
2411                 if (likely(napi_schedule_prep(&rx_ring->napi))) {
2412                         rx_ring->int_disable(rx_ring);
2413                         __napi_schedule(&rx_ring->napi);
2414                 }
2415         }
2416
2417         if (priv->irq0_stat & UMAC_IRQ_TXDMA_DONE) {
2418                 tx_ring = &priv->tx_rings[DESC_INDEX];
2419
2420                 if (likely(napi_schedule_prep(&tx_ring->napi))) {
2421                         tx_ring->int_disable(tx_ring);
2422                         __napi_schedule(&tx_ring->napi);
2423                 }
2424         }
2425
2426         if (priv->irq0_stat & (UMAC_IRQ_PHY_DET_R |
2427                                 UMAC_IRQ_PHY_DET_F |
2428                                 UMAC_IRQ_LINK_EVENT |
2429                                 UMAC_IRQ_HFB_SM |
2430                                 UMAC_IRQ_HFB_MM |
2431                                 UMAC_IRQ_MPD_R)) {
2432                 /* all other interested interrupts handled in bottom half */
2433                 schedule_work(&priv->bcmgenet_irq_work);
2434         }
2435
2436         if ((priv->hw_params->flags & GENET_HAS_MDIO_INTR) &&
2437             priv->irq0_stat & (UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR)) {
2438                 priv->irq0_stat &= ~(UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR);
2439                 wake_up(&priv->wq);
2440         }
2441
2442         return IRQ_HANDLED;
2443 }
2444
2445 static irqreturn_t bcmgenet_wol_isr(int irq, void *dev_id)
2446 {
2447         struct bcmgenet_priv *priv = dev_id;
2448
2449         pm_wakeup_event(&priv->pdev->dev, 0);
2450
2451         return IRQ_HANDLED;
2452 }
2453
2454 #ifdef CONFIG_NET_POLL_CONTROLLER
2455 static void bcmgenet_poll_controller(struct net_device *dev)
2456 {
2457         struct bcmgenet_priv *priv = netdev_priv(dev);
2458
2459         /* Invoke the main RX/TX interrupt handler */
2460         disable_irq(priv->irq0);
2461         bcmgenet_isr0(priv->irq0, priv);
2462         enable_irq(priv->irq0);
2463
2464         /* And the interrupt handler for RX/TX priority queues */
2465         disable_irq(priv->irq1);
2466         bcmgenet_isr1(priv->irq1, priv);
2467         enable_irq(priv->irq1);
2468 }
2469 #endif
2470
2471 static void bcmgenet_umac_reset(struct bcmgenet_priv *priv)
2472 {
2473         u32 reg;
2474
2475         reg = bcmgenet_rbuf_ctrl_get(priv);
2476         reg |= BIT(1);
2477         bcmgenet_rbuf_ctrl_set(priv, reg);
2478         udelay(10);
2479
2480         reg &= ~BIT(1);
2481         bcmgenet_rbuf_ctrl_set(priv, reg);
2482         udelay(10);
2483 }
2484
2485 static void bcmgenet_set_hw_addr(struct bcmgenet_priv *priv,
2486                                  unsigned char *addr)
2487 {
2488         bcmgenet_umac_writel(priv, (addr[0] << 24) | (addr[1] << 16) |
2489                         (addr[2] << 8) | addr[3], UMAC_MAC0);
2490         bcmgenet_umac_writel(priv, (addr[4] << 8) | addr[5], UMAC_MAC1);
2491 }
2492
2493 /* Returns a reusable dma control register value */
2494 static u32 bcmgenet_dma_disable(struct bcmgenet_priv *priv)
2495 {
2496         u32 reg;
2497         u32 dma_ctrl;
2498
2499         /* disable DMA */
2500         dma_ctrl = 1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT) | DMA_EN;
2501         reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2502         reg &= ~dma_ctrl;
2503         bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2504
2505         reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2506         reg &= ~dma_ctrl;
2507         bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2508
2509         bcmgenet_umac_writel(priv, 1, UMAC_TX_FLUSH);
2510         udelay(10);
2511         bcmgenet_umac_writel(priv, 0, UMAC_TX_FLUSH);
2512
2513         return dma_ctrl;
2514 }
2515
2516 static void bcmgenet_enable_dma(struct bcmgenet_priv *priv, u32 dma_ctrl)
2517 {
2518         u32 reg;
2519
2520         reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2521         reg |= dma_ctrl;
2522         bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2523
2524         reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2525         reg |= dma_ctrl;
2526         bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2527 }
2528
2529 static bool bcmgenet_hfb_is_filter_enabled(struct bcmgenet_priv *priv,
2530                                            u32 f_index)
2531 {
2532         u32 offset;
2533         u32 reg;
2534
2535         offset = HFB_FLT_ENABLE_V3PLUS + (f_index < 32) * sizeof(u32);
2536         reg = bcmgenet_hfb_reg_readl(priv, offset);
2537         return !!(reg & (1 << (f_index % 32)));
2538 }
2539
2540 static void bcmgenet_hfb_enable_filter(struct bcmgenet_priv *priv, u32 f_index)
2541 {
2542         u32 offset;
2543         u32 reg;
2544
2545         offset = HFB_FLT_ENABLE_V3PLUS + (f_index < 32) * sizeof(u32);
2546         reg = bcmgenet_hfb_reg_readl(priv, offset);
2547         reg |= (1 << (f_index % 32));
2548         bcmgenet_hfb_reg_writel(priv, reg, offset);
2549 }
2550
2551 static void bcmgenet_hfb_set_filter_rx_queue_mapping(struct bcmgenet_priv *priv,
2552                                                      u32 f_index, u32 rx_queue)
2553 {
2554         u32 offset;
2555         u32 reg;
2556
2557         offset = f_index / 8;
2558         reg = bcmgenet_rdma_readl(priv, DMA_INDEX2RING_0 + offset);
2559         reg &= ~(0xF << (4 * (f_index % 8)));
2560         reg |= ((rx_queue & 0xF) << (4 * (f_index % 8)));
2561         bcmgenet_rdma_writel(priv, reg, DMA_INDEX2RING_0 + offset);
2562 }
2563
2564 static void bcmgenet_hfb_set_filter_length(struct bcmgenet_priv *priv,
2565                                            u32 f_index, u32 f_length)
2566 {
2567         u32 offset;
2568         u32 reg;
2569
2570         offset = HFB_FLT_LEN_V3PLUS +
2571                  ((priv->hw_params->hfb_filter_cnt - 1 - f_index) / 4) *
2572                  sizeof(u32);
2573         reg = bcmgenet_hfb_reg_readl(priv, offset);
2574         reg &= ~(0xFF << (8 * (f_index % 4)));
2575         reg |= ((f_length & 0xFF) << (8 * (f_index % 4)));
2576         bcmgenet_hfb_reg_writel(priv, reg, offset);
2577 }
2578
2579 static int bcmgenet_hfb_find_unused_filter(struct bcmgenet_priv *priv)
2580 {
2581         u32 f_index;
2582
2583         for (f_index = 0; f_index < priv->hw_params->hfb_filter_cnt; f_index++)
2584                 if (!bcmgenet_hfb_is_filter_enabled(priv, f_index))
2585                         return f_index;
2586
2587         return -ENOMEM;
2588 }
2589
2590 /* bcmgenet_hfb_add_filter
2591  *
2592  * Add new filter to Hardware Filter Block to match and direct Rx traffic to
2593  * desired Rx queue.
2594  *
2595  * f_data is an array of unsigned 32-bit integers where each 32-bit integer
2596  * provides filter data for 2 bytes (4 nibbles) of Rx frame:
2597  *
2598  * bits 31:20 - unused
2599  * bit  19    - nibble 0 match enable
2600  * bit  18    - nibble 1 match enable
2601  * bit  17    - nibble 2 match enable
2602  * bit  16    - nibble 3 match enable
2603  * bits 15:12 - nibble 0 data
2604  * bits 11:8  - nibble 1 data
2605  * bits 7:4   - nibble 2 data
2606  * bits 3:0   - nibble 3 data
2607  *
2608  * Example:
2609  * In order to match:
2610  * - Ethernet frame type = 0x0800 (IP)
2611  * - IP version field = 4
2612  * - IP protocol field = 0x11 (UDP)
2613  *
2614  * The following filter is needed:
2615  * u32 hfb_filter_ipv4_udp[] = {
2616  *   Rx frame offset 0x00: 0x00000000, 0x00000000, 0x00000000, 0x00000000,
2617  *   Rx frame offset 0x08: 0x00000000, 0x00000000, 0x000F0800, 0x00084000,
2618  *   Rx frame offset 0x10: 0x00000000, 0x00000000, 0x00000000, 0x00030011,
2619  * };
2620  *
2621  * To add the filter to HFB and direct the traffic to Rx queue 0, call:
2622  * bcmgenet_hfb_add_filter(priv, hfb_filter_ipv4_udp,
2623  *                         ARRAY_SIZE(hfb_filter_ipv4_udp), 0);
2624  */
2625 int bcmgenet_hfb_add_filter(struct bcmgenet_priv *priv, u32 *f_data,
2626                             u32 f_length, u32 rx_queue)
2627 {
2628         int f_index;
2629         u32 i;
2630
2631         f_index = bcmgenet_hfb_find_unused_filter(priv);
2632         if (f_index < 0)
2633                 return -ENOMEM;
2634
2635         if (f_length > priv->hw_params->hfb_filter_size)
2636                 return -EINVAL;
2637
2638         for (i = 0; i < f_length; i++)
2639                 bcmgenet_hfb_writel(priv, f_data[i],
2640                         (f_index * priv->hw_params->hfb_filter_size + i) *
2641                         sizeof(u32));
2642
2643         bcmgenet_hfb_set_filter_length(priv, f_index, 2 * f_length);
2644         bcmgenet_hfb_set_filter_rx_queue_mapping(priv, f_index, rx_queue);
2645         bcmgenet_hfb_enable_filter(priv, f_index);
2646         bcmgenet_hfb_reg_writel(priv, 0x1, HFB_CTRL);
2647
2648         return 0;
2649 }
2650
2651 /* bcmgenet_hfb_clear
2652  *
2653  * Clear Hardware Filter Block and disable all filtering.
2654  */
2655 static void bcmgenet_hfb_clear(struct bcmgenet_priv *priv)
2656 {
2657         u32 i;
2658
2659         bcmgenet_hfb_reg_writel(priv, 0x0, HFB_CTRL);
2660         bcmgenet_hfb_reg_writel(priv, 0x0, HFB_FLT_ENABLE_V3PLUS);
2661         bcmgenet_hfb_reg_writel(priv, 0x0, HFB_FLT_ENABLE_V3PLUS + 4);
2662
2663         for (i = DMA_INDEX2RING_0; i <= DMA_INDEX2RING_7; i++)
2664                 bcmgenet_rdma_writel(priv, 0x0, i);
2665
2666         for (i = 0; i < (priv->hw_params->hfb_filter_cnt / 4); i++)
2667                 bcmgenet_hfb_reg_writel(priv, 0x0,
2668                                         HFB_FLT_LEN_V3PLUS + i * sizeof(u32));
2669
2670         for (i = 0; i < priv->hw_params->hfb_filter_cnt *
2671                         priv->hw_params->hfb_filter_size; i++)
2672                 bcmgenet_hfb_writel(priv, 0x0, i * sizeof(u32));
2673 }
2674
2675 static void bcmgenet_hfb_init(struct bcmgenet_priv *priv)
2676 {
2677         if (GENET_IS_V1(priv) || GENET_IS_V2(priv))
2678                 return;
2679
2680         bcmgenet_hfb_clear(priv);
2681 }
2682
2683 static void bcmgenet_netif_start(struct net_device *dev)
2684 {
2685         struct bcmgenet_priv *priv = netdev_priv(dev);
2686
2687         /* Start the network engine */
2688         bcmgenet_enable_rx_napi(priv);
2689         bcmgenet_enable_tx_napi(priv);
2690
2691         umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, true);
2692
2693         netif_tx_start_all_queues(dev);
2694
2695         phy_start(priv->phydev);
2696 }
2697
2698 static int bcmgenet_open(struct net_device *dev)
2699 {
2700         struct bcmgenet_priv *priv = netdev_priv(dev);
2701         unsigned long dma_ctrl;
2702         u32 reg;
2703         int ret;
2704
2705         netif_dbg(priv, ifup, dev, "bcmgenet_open\n");
2706
2707         /* Turn on the clock */
2708         clk_prepare_enable(priv->clk);
2709
2710         /* If this is an internal GPHY, power it back on now, before UniMAC is
2711          * brought out of reset as absolutely no UniMAC activity is allowed
2712          */
2713         if (priv->internal_phy)
2714                 bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
2715
2716         /* take MAC out of reset */
2717         bcmgenet_umac_reset(priv);
2718
2719         ret = init_umac(priv);
2720         if (ret)
2721                 goto err_clk_disable;
2722
2723         /* disable ethernet MAC while updating its registers */
2724         umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, false);
2725
2726         /* Make sure we reflect the value of CRC_CMD_FWD */
2727         reg = bcmgenet_umac_readl(priv, UMAC_CMD);
2728         priv->crc_fwd_en = !!(reg & CMD_CRC_FWD);
2729
2730         bcmgenet_set_hw_addr(priv, dev->dev_addr);
2731
2732         if (priv->internal_phy) {
2733                 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
2734                 reg |= EXT_ENERGY_DET_MASK;
2735                 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
2736         }
2737
2738         /* Disable RX/TX DMA and flush TX queues */
2739         dma_ctrl = bcmgenet_dma_disable(priv);
2740
2741         /* Reinitialize TDMA and RDMA and SW housekeeping */
2742         ret = bcmgenet_init_dma(priv);
2743         if (ret) {
2744                 netdev_err(dev, "failed to initialize DMA\n");
2745                 goto err_clk_disable;
2746         }
2747
2748         /* Always enable ring 16 - descriptor ring */
2749         bcmgenet_enable_dma(priv, dma_ctrl);
2750
2751         /* HFB init */
2752         bcmgenet_hfb_init(priv);
2753
2754         ret = request_irq(priv->irq0, bcmgenet_isr0, IRQF_SHARED,
2755                           dev->name, priv);
2756         if (ret < 0) {
2757                 netdev_err(dev, "can't request IRQ %d\n", priv->irq0);
2758                 goto err_fini_dma;
2759         }
2760
2761         ret = request_irq(priv->irq1, bcmgenet_isr1, IRQF_SHARED,
2762                           dev->name, priv);
2763         if (ret < 0) {
2764                 netdev_err(dev, "can't request IRQ %d\n", priv->irq1);
2765                 goto err_irq0;
2766         }
2767
2768         ret = bcmgenet_mii_probe(dev);
2769         if (ret) {
2770                 netdev_err(dev, "failed to connect to PHY\n");
2771                 goto err_irq1;
2772         }
2773
2774         bcmgenet_netif_start(dev);
2775
2776         return 0;
2777
2778 err_irq1:
2779         free_irq(priv->irq1, priv);
2780 err_irq0:
2781         free_irq(priv->irq0, priv);
2782 err_fini_dma:
2783         bcmgenet_fini_dma(priv);
2784 err_clk_disable:
2785         clk_disable_unprepare(priv->clk);
2786         return ret;
2787 }
2788
2789 static void bcmgenet_netif_stop(struct net_device *dev)
2790 {
2791         struct bcmgenet_priv *priv = netdev_priv(dev);
2792
2793         netif_tx_stop_all_queues(dev);
2794         phy_stop(priv->phydev);
2795         bcmgenet_intr_disable(priv);
2796         bcmgenet_disable_rx_napi(priv);
2797         bcmgenet_disable_tx_napi(priv);
2798
2799         /* Wait for pending work items to complete. Since interrupts are
2800          * disabled no new work will be scheduled.
2801          */
2802         cancel_work_sync(&priv->bcmgenet_irq_work);
2803
2804         priv->old_link = -1;
2805         priv->old_speed = -1;
2806         priv->old_duplex = -1;
2807         priv->old_pause = -1;
2808 }
2809
2810 static int bcmgenet_close(struct net_device *dev)
2811 {
2812         struct bcmgenet_priv *priv = netdev_priv(dev);
2813         int ret;
2814
2815         netif_dbg(priv, ifdown, dev, "bcmgenet_close\n");
2816
2817         bcmgenet_netif_stop(dev);
2818
2819         /* Really kill the PHY state machine and disconnect from it */
2820         phy_disconnect(priv->phydev);
2821
2822         /* Disable MAC receive */
2823         umac_enable_set(priv, CMD_RX_EN, false);
2824
2825         ret = bcmgenet_dma_teardown(priv);
2826         if (ret)
2827                 return ret;
2828
2829         /* Disable MAC transmit. TX DMA disabled have to done before this */
2830         umac_enable_set(priv, CMD_TX_EN, false);
2831
2832         /* tx reclaim */
2833         bcmgenet_tx_reclaim_all(dev);
2834         bcmgenet_fini_dma(priv);
2835
2836         free_irq(priv->irq0, priv);
2837         free_irq(priv->irq1, priv);
2838
2839         if (priv->internal_phy)
2840                 ret = bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
2841
2842         clk_disable_unprepare(priv->clk);
2843
2844         return ret;
2845 }
2846
2847 static void bcmgenet_dump_tx_queue(struct bcmgenet_tx_ring *ring)
2848 {
2849         struct bcmgenet_priv *priv = ring->priv;
2850         u32 p_index, c_index, intsts, intmsk;
2851         struct netdev_queue *txq;
2852         unsigned int free_bds;
2853         unsigned long flags;
2854         bool txq_stopped;
2855
2856         if (!netif_msg_tx_err(priv))
2857                 return;
2858
2859         txq = netdev_get_tx_queue(priv->dev, ring->queue);
2860
2861         spin_lock_irqsave(&ring->lock, flags);
2862         if (ring->index == DESC_INDEX) {
2863                 intsts = ~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);
2864                 intmsk = UMAC_IRQ_TXDMA_DONE | UMAC_IRQ_TXDMA_MBDONE;
2865         } else {
2866                 intsts = ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
2867                 intmsk = 1 << ring->index;
2868         }
2869         c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX);
2870         p_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_PROD_INDEX);
2871         txq_stopped = netif_tx_queue_stopped(txq);
2872         free_bds = ring->free_bds;
2873         spin_unlock_irqrestore(&ring->lock, flags);
2874
2875         netif_err(priv, tx_err, priv->dev, "Ring %d queue %d status summary\n"
2876                   "TX queue status: %s, interrupts: %s\n"
2877                   "(sw)free_bds: %d (sw)size: %d\n"
2878                   "(sw)p_index: %d (hw)p_index: %d\n"
2879                   "(sw)c_index: %d (hw)c_index: %d\n"
2880                   "(sw)clean_p: %d (sw)write_p: %d\n"
2881                   "(sw)cb_ptr: %d (sw)end_ptr: %d\n",
2882                   ring->index, ring->queue,
2883                   txq_stopped ? "stopped" : "active",
2884                   intsts & intmsk ? "enabled" : "disabled",
2885                   free_bds, ring->size,
2886                   ring->prod_index, p_index & DMA_P_INDEX_MASK,
2887                   ring->c_index, c_index & DMA_C_INDEX_MASK,
2888                   ring->clean_ptr, ring->write_ptr,
2889                   ring->cb_ptr, ring->end_ptr);
2890 }
2891
2892 static void bcmgenet_timeout(struct net_device *dev)
2893 {
2894         struct bcmgenet_priv *priv = netdev_priv(dev);
2895         u32 int0_enable = 0;
2896         u32 int1_enable = 0;
2897         unsigned int q;
2898
2899         netif_dbg(priv, tx_err, dev, "bcmgenet_timeout\n");
2900
2901         for (q = 0; q < priv->hw_params->tx_queues; q++)
2902                 bcmgenet_dump_tx_queue(&priv->tx_rings[q]);
2903         bcmgenet_dump_tx_queue(&priv->tx_rings[DESC_INDEX]);
2904
2905         bcmgenet_tx_reclaim_all(dev);
2906
2907         for (q = 0; q < priv->hw_params->tx_queues; q++)
2908                 int1_enable |= (1 << q);
2909
2910         int0_enable = UMAC_IRQ_TXDMA_DONE;
2911
2912         /* Re-enable TX interrupts if disabled */
2913         bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);
2914         bcmgenet_intrl2_1_writel(priv, int1_enable, INTRL2_CPU_MASK_CLEAR);
2915
2916         dev->trans_start = jiffies;
2917
2918         dev->stats.tx_errors++;
2919
2920         netif_tx_wake_all_queues(dev);
2921 }
2922
2923 #define MAX_MC_COUNT    16
2924
2925 static inline void bcmgenet_set_mdf_addr(struct bcmgenet_priv *priv,
2926                                          unsigned char *addr,
2927                                          int *i,
2928                                          int *mc)
2929 {
2930         u32 reg;
2931
2932         bcmgenet_umac_writel(priv, addr[0] << 8 | addr[1],
2933                              UMAC_MDF_ADDR + (*i * 4));
2934         bcmgenet_umac_writel(priv, addr[2] << 24 | addr[3] << 16 |
2935                              addr[4] << 8 | addr[5],
2936                              UMAC_MDF_ADDR + ((*i + 1) * 4));
2937         reg = bcmgenet_umac_readl(priv, UMAC_MDF_CTRL);
2938         reg |= (1 << (MAX_MC_COUNT - *mc));
2939         bcmgenet_umac_writel(priv, reg, UMAC_MDF_CTRL);
2940         *i += 2;
2941         (*mc)++;
2942 }
2943
2944 static void bcmgenet_set_rx_mode(struct net_device *dev)
2945 {
2946         struct bcmgenet_priv *priv = netdev_priv(dev);
2947         struct netdev_hw_addr *ha;
2948         int i, mc;
2949         u32 reg;
2950
2951         netif_dbg(priv, hw, dev, "%s: %08X\n", __func__, dev->flags);
2952
2953         /* Promiscuous mode */
2954         reg = bcmgenet_umac_readl(priv, UMAC_CMD);
2955         if (dev->flags & IFF_PROMISC) {
2956                 reg |= CMD_PROMISC;
2957                 bcmgenet_umac_writel(priv, reg, UMAC_CMD);
2958                 bcmgenet_umac_writel(priv, 0, UMAC_MDF_CTRL);
2959                 return;
2960         } else {
2961                 reg &= ~CMD_PROMISC;
2962                 bcmgenet_umac_writel(priv, reg, UMAC_CMD);
2963         }
2964
2965         /* UniMac doesn't support ALLMULTI */
2966         if (dev->flags & IFF_ALLMULTI) {
2967                 netdev_warn(dev, "ALLMULTI is not supported\n");
2968                 return;
2969         }
2970
2971         /* update MDF filter */
2972         i = 0;
2973         mc = 0;
2974         /* Broadcast */
2975         bcmgenet_set_mdf_addr(priv, dev->broadcast, &i, &mc);
2976         /* my own address.*/
2977         bcmgenet_set_mdf_addr(priv, dev->dev_addr, &i, &mc);
2978         /* Unicast list*/
2979         if (netdev_uc_count(dev) > (MAX_MC_COUNT - mc))
2980                 return;
2981
2982         if (!netdev_uc_empty(dev))
2983                 netdev_for_each_uc_addr(ha, dev)
2984                         bcmgenet_set_mdf_addr(priv, ha->addr, &i, &mc);
2985         /* Multicast */
2986         if (netdev_mc_empty(dev) || netdev_mc_count(dev) >= (MAX_MC_COUNT - mc))
2987                 return;
2988
2989         netdev_for_each_mc_addr(ha, dev)
2990                 bcmgenet_set_mdf_addr(priv, ha->addr, &i, &mc);
2991 }
2992
2993 /* Set the hardware MAC address. */
2994 static int bcmgenet_set_mac_addr(struct net_device *dev, void *p)
2995 {
2996         struct sockaddr *addr = p;
2997
2998         /* Setting the MAC address at the hardware level is not possible
2999          * without disabling the UniMAC RX/TX enable bits.
3000          */
3001         if (netif_running(dev))
3002                 return -EBUSY;
3003
3004         ether_addr_copy(dev->dev_addr, addr->sa_data);
3005
3006         return 0;
3007 }
3008
3009 static const struct net_device_ops bcmgenet_netdev_ops = {
3010         .ndo_open               = bcmgenet_open,
3011         .ndo_stop               = bcmgenet_close,
3012         .ndo_start_xmit         = bcmgenet_xmit,
3013         .ndo_tx_timeout         = bcmgenet_timeout,
3014         .ndo_set_rx_mode        = bcmgenet_set_rx_mode,
3015         .ndo_set_mac_address    = bcmgenet_set_mac_addr,
3016         .ndo_do_ioctl           = bcmgenet_ioctl,
3017         .ndo_set_features       = bcmgenet_set_features,
3018 #ifdef CONFIG_NET_POLL_CONTROLLER
3019         .ndo_poll_controller    = bcmgenet_poll_controller,
3020 #endif
3021 };
3022
3023 /* Array of GENET hardware parameters/characteristics */
3024 static struct bcmgenet_hw_params bcmgenet_hw_params[] = {
3025         [GENET_V1] = {
3026                 .tx_queues = 0,
3027                 .tx_bds_per_q = 0,
3028                 .rx_queues = 0,
3029                 .rx_bds_per_q = 0,
3030                 .bp_in_en_shift = 16,
3031                 .bp_in_mask = 0xffff,
3032                 .hfb_filter_cnt = 16,
3033                 .qtag_mask = 0x1F,
3034                 .hfb_offset = 0x1000,
3035                 .rdma_offset = 0x2000,
3036                 .tdma_offset = 0x3000,
3037                 .words_per_bd = 2,
3038         },
3039         [GENET_V2] = {
3040                 .tx_queues = 4,
3041                 .tx_bds_per_q = 32,
3042                 .rx_queues = 0,
3043                 .rx_bds_per_q = 0,
3044                 .bp_in_en_shift = 16,
3045                 .bp_in_mask = 0xffff,
3046                 .hfb_filter_cnt = 16,
3047                 .qtag_mask = 0x1F,
3048                 .tbuf_offset = 0x0600,
3049                 .hfb_offset = 0x1000,
3050                 .hfb_reg_offset = 0x2000,
3051                 .rdma_offset = 0x3000,
3052                 .tdma_offset = 0x4000,
3053                 .words_per_bd = 2,
3054                 .flags = GENET_HAS_EXT,
3055         },
3056         [GENET_V3] = {
3057                 .tx_queues = 4,
3058                 .tx_bds_per_q = 32,
3059                 .rx_queues = 0,
3060                 .rx_bds_per_q = 0,
3061                 .bp_in_en_shift = 17,
3062                 .bp_in_mask = 0x1ffff,
3063                 .hfb_filter_cnt = 48,
3064                 .hfb_filter_size = 128,
3065                 .qtag_mask = 0x3F,
3066                 .tbuf_offset = 0x0600,
3067                 .hfb_offset = 0x8000,
3068                 .hfb_reg_offset = 0xfc00,
3069                 .rdma_offset = 0x10000,
3070                 .tdma_offset = 0x11000,
3071                 .words_per_bd = 2,
3072                 .flags = GENET_HAS_EXT | GENET_HAS_MDIO_INTR |
3073                          GENET_HAS_MOCA_LINK_DET,
3074         },
3075         [GENET_V4] = {
3076                 .tx_queues = 4,
3077                 .tx_bds_per_q = 32,
3078                 .rx_queues = 0,
3079                 .rx_bds_per_q = 0,
3080                 .bp_in_en_shift = 17,
3081                 .bp_in_mask = 0x1ffff,
3082                 .hfb_filter_cnt = 48,
3083                 .hfb_filter_size = 128,
3084                 .qtag_mask = 0x3F,
3085                 .tbuf_offset = 0x0600,
3086                 .hfb_offset = 0x8000,
3087                 .hfb_reg_offset = 0xfc00,
3088                 .rdma_offset = 0x2000,
3089                 .tdma_offset = 0x4000,
3090                 .words_per_bd = 3,
3091                 .flags = GENET_HAS_40BITS | GENET_HAS_EXT |
3092                          GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET,
3093         },
3094 };
3095
3096 /* Infer hardware parameters from the detected GENET version */
3097 static void bcmgenet_set_hw_params(struct bcmgenet_priv *priv)
3098 {
3099         struct bcmgenet_hw_params *params;
3100         u32 reg;
3101         u8 major;
3102         u16 gphy_rev;
3103
3104         if (GENET_IS_V4(priv)) {
3105                 bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
3106                 genet_dma_ring_regs = genet_dma_ring_regs_v4;
3107                 priv->dma_rx_chk_bit = DMA_RX_CHK_V3PLUS;
3108                 priv->version = GENET_V4;
3109         } else if (GENET_IS_V3(priv)) {
3110                 bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
3111                 genet_dma_ring_regs = genet_dma_ring_regs_v123;
3112                 priv->dma_rx_chk_bit = DMA_RX_CHK_V3PLUS;
3113                 priv->version = GENET_V3;
3114         } else if (GENET_IS_V2(priv)) {
3115                 bcmgenet_dma_regs = bcmgenet_dma_regs_v2;
3116                 genet_dma_ring_regs = genet_dma_ring_regs_v123;
3117                 priv->dma_rx_chk_bit = DMA_RX_CHK_V12;
3118                 priv->version = GENET_V2;
3119         } else if (GENET_IS_V1(priv)) {
3120                 bcmgenet_dma_regs = bcmgenet_dma_regs_v1;
3121                 genet_dma_ring_regs = genet_dma_ring_regs_v123;
3122                 priv->dma_rx_chk_bit = DMA_RX_CHK_V12;
3123                 priv->version = GENET_V1;
3124         }
3125
3126         /* enum genet_version starts at 1 */
3127         priv->hw_params = &bcmgenet_hw_params[priv->version];
3128         params = priv->hw_params;
3129
3130         /* Read GENET HW version */
3131         reg = bcmgenet_sys_readl(priv, SYS_REV_CTRL);
3132         major = (reg >> 24 & 0x0f);
3133         if (major == 5)
3134                 major = 4;
3135         else if (major == 0)
3136                 major = 1;
3137         if (major != priv->version) {
3138                 dev_err(&priv->pdev->dev,
3139                         "GENET version mismatch, got: %d, configured for: %d\n",
3140                         major, priv->version);
3141         }
3142
3143         /* Print the GENET core version */
3144         dev_info(&priv->pdev->dev, "GENET " GENET_VER_FMT,
3145                  major, (reg >> 16) & 0x0f, reg & 0xffff);
3146
3147         /* Store the integrated PHY revision for the MDIO probing function
3148          * to pass this information to the PHY driver. The PHY driver expects
3149          * to find the PHY major revision in bits 15:8 while the GENET register
3150          * stores that information in bits 7:0, account for that.
3151          *
3152          * On newer chips, starting with PHY revision G0, a new scheme is
3153          * deployed similar to the Starfighter 2 switch with GPHY major
3154          * revision in bits 15:8 and patch level in bits 7:0. Major revision 0
3155          * is reserved as well as special value 0x01ff, we have a small
3156          * heuristic to check for the new GPHY revision and re-arrange things
3157          * so the GPHY driver is happy.
3158          */
3159         gphy_rev = reg & 0xffff;
3160
3161         /* This is the good old scheme, just GPHY major, no minor nor patch */
3162         if ((gphy_rev & 0xf0) != 0)
3163                 priv->gphy_rev = gphy_rev << 8;
3164
3165         /* This is the new scheme, GPHY major rolls over with 0x10 = rev G0 */
3166         else if ((gphy_rev & 0xff00) != 0)
3167                 priv->gphy_rev = gphy_rev;
3168
3169         /* This is reserved so should require special treatment */
3170         else if (gphy_rev == 0 || gphy_rev == 0x01ff) {
3171                 pr_warn("Invalid GPHY revision detected: 0x%04x\n", gphy_rev);
3172                 return;
3173         }
3174
3175 #ifdef CONFIG_PHYS_ADDR_T_64BIT
3176         if (!(params->flags & GENET_HAS_40BITS))
3177                 pr_warn("GENET does not support 40-bits PA\n");
3178 #endif
3179
3180         pr_debug("Configuration for version: %d\n"
3181                 "TXq: %1d, TXqBDs: %1d, RXq: %1d, RXqBDs: %1d\n"
3182                 "BP << en: %2d, BP msk: 0x%05x\n"
3183                 "HFB count: %2d, QTAQ msk: 0x%05x\n"
3184                 "TBUF: 0x%04x, HFB: 0x%04x, HFBreg: 0x%04x\n"
3185                 "RDMA: 0x%05x, TDMA: 0x%05x\n"
3186                 "Words/BD: %d\n",
3187                 priv->version,
3188                 params->tx_queues, params->tx_bds_per_q,
3189                 params->rx_queues, params->rx_bds_per_q,
3190                 params->bp_in_en_shift, params->bp_in_mask,
3191                 params->hfb_filter_cnt, params->qtag_mask,
3192                 params->tbuf_offset, params->hfb_offset,
3193                 params->hfb_reg_offset,
3194                 params->rdma_offset, params->tdma_offset,
3195                 params->words_per_bd);
3196 }
3197
3198 static const struct of_device_id bcmgenet_match[] = {
3199         { .compatible = "brcm,genet-v1", .data = (void *)GENET_V1 },
3200         { .compatible = "brcm,genet-v2", .data = (void *)GENET_V2 },
3201         { .compatible = "brcm,genet-v3", .data = (void *)GENET_V3 },
3202         { .compatible = "brcm,genet-v4", .data = (void *)GENET_V4 },
3203         { },
3204 };
3205
3206 static int bcmgenet_probe(struct platform_device *pdev)
3207 {
3208         struct bcmgenet_platform_data *pd = pdev->dev.platform_data;
3209         struct device_node *dn = pdev->dev.of_node;
3210         const struct of_device_id *of_id = NULL;
3211         struct bcmgenet_priv *priv;
3212         struct net_device *dev;
3213         const void *macaddr;
3214         struct resource *r;
3215         int err = -EIO;
3216
3217         /* Up to GENET_MAX_MQ_CNT + 1 TX queues and RX queues */
3218         dev = alloc_etherdev_mqs(sizeof(*priv), GENET_MAX_MQ_CNT + 1,
3219                                  GENET_MAX_MQ_CNT + 1);
3220         if (!dev) {
3221                 dev_err(&pdev->dev, "can't allocate net device\n");
3222                 return -ENOMEM;
3223         }
3224
3225         if (dn) {
3226                 of_id = of_match_node(bcmgenet_match, dn);
3227                 if (!of_id)
3228                         return -EINVAL;
3229         }
3230
3231         priv = netdev_priv(dev);
3232         priv->irq0 = platform_get_irq(pdev, 0);
3233         priv->irq1 = platform_get_irq(pdev, 1);
3234         priv->wol_irq = platform_get_irq(pdev, 2);
3235         if (!priv->irq0 || !priv->irq1) {
3236                 dev_err(&pdev->dev, "can't find IRQs\n");
3237                 err = -EINVAL;
3238                 goto err;
3239         }
3240
3241         if (dn) {
3242                 macaddr = of_get_mac_address(dn);
3243                 if (!macaddr) {
3244                         dev_err(&pdev->dev, "can't find MAC address\n");
3245                         err = -EINVAL;
3246                         goto err;
3247                 }
3248         } else {
3249                 macaddr = pd->mac_address;
3250         }
3251
3252         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3253         priv->base = devm_ioremap_resource(&pdev->dev, r);
3254         if (IS_ERR(priv->base)) {
3255                 err = PTR_ERR(priv->base);
3256                 goto err;
3257         }
3258
3259         SET_NETDEV_DEV(dev, &pdev->dev);
3260         dev_set_drvdata(&pdev->dev, dev);
3261         ether_addr_copy(dev->dev_addr, macaddr);
3262         dev->watchdog_timeo = 2 * HZ;
3263         dev->ethtool_ops = &bcmgenet_ethtool_ops;
3264         dev->netdev_ops = &bcmgenet_netdev_ops;
3265
3266         priv->msg_enable = netif_msg_init(-1, GENET_MSG_DEFAULT);
3267
3268         /* Set hardware features */
3269         dev->hw_features |= NETIF_F_SG | NETIF_F_IP_CSUM |
3270                 NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM;
3271
3272         /* Request the WOL interrupt and advertise suspend if available */
3273         priv->wol_irq_disabled = true;
3274         err = devm_request_irq(&pdev->dev, priv->wol_irq, bcmgenet_wol_isr, 0,
3275                                dev->name, priv);
3276         if (!err)
3277                 device_set_wakeup_capable(&pdev->dev, 1);
3278
3279         /* Set the needed headroom to account for any possible
3280          * features enabling/disabling at runtime
3281          */
3282         dev->needed_headroom += 64;
3283
3284         netdev_boot_setup_check(dev);
3285
3286         priv->dev = dev;
3287         priv->pdev = pdev;
3288         if (of_id)
3289                 priv->version = (enum bcmgenet_version)of_id->data;
3290         else
3291                 priv->version = pd->genet_version;
3292
3293         priv->clk = devm_clk_get(&priv->pdev->dev, "enet");
3294         if (IS_ERR(priv->clk)) {
3295                 dev_warn(&priv->pdev->dev, "failed to get enet clock\n");
3296                 priv->clk = NULL;
3297         }
3298
3299         clk_prepare_enable(priv->clk);
3300
3301         bcmgenet_set_hw_params(priv);
3302
3303         /* Mii wait queue */
3304         init_waitqueue_head(&priv->wq);
3305         /* Always use RX_BUF_LENGTH (2KB) buffer for all chips */
3306         priv->rx_buf_len = RX_BUF_LENGTH;
3307         INIT_WORK(&priv->bcmgenet_irq_work, bcmgenet_irq_task);
3308
3309         priv->clk_wol = devm_clk_get(&priv->pdev->dev, "enet-wol");
3310         if (IS_ERR(priv->clk_wol)) {
3311                 dev_warn(&priv->pdev->dev, "failed to get enet-wol clock\n");
3312                 priv->clk_wol = NULL;
3313         }
3314
3315         priv->clk_eee = devm_clk_get(&priv->pdev->dev, "enet-eee");
3316         if (IS_ERR(priv->clk_eee)) {
3317                 dev_warn(&priv->pdev->dev, "failed to get enet-eee clock\n");
3318                 priv->clk_eee = NULL;
3319         }
3320
3321         err = reset_umac(priv);
3322         if (err)
3323                 goto err_clk_disable;
3324
3325         err = bcmgenet_mii_init(dev);
3326         if (err)
3327                 goto err_clk_disable;
3328
3329         /* setup number of real queues  + 1 (GENET_V1 has 0 hardware queues
3330          * just the ring 16 descriptor based TX
3331          */
3332         netif_set_real_num_tx_queues(priv->dev, priv->hw_params->tx_queues + 1);
3333         netif_set_real_num_rx_queues(priv->dev, priv->hw_params->rx_queues + 1);
3334
3335         /* libphy will determine the link state */
3336         netif_carrier_off(dev);
3337
3338         /* Turn off the main clock, WOL clock is handled separately */
3339         clk_disable_unprepare(priv->clk);
3340
3341         err = register_netdev(dev);
3342         if (err)
3343                 goto err;
3344
3345         return err;
3346
3347 err_clk_disable:
3348         clk_disable_unprepare(priv->clk);
3349 err:
3350         free_netdev(dev);
3351         return err;
3352 }
3353
3354 static int bcmgenet_remove(struct platform_device *pdev)
3355 {
3356         struct bcmgenet_priv *priv = dev_to_priv(&pdev->dev);
3357
3358         dev_set_drvdata(&pdev->dev, NULL);
3359         unregister_netdev(priv->dev);
3360         bcmgenet_mii_exit(priv->dev);
3361         free_netdev(priv->dev);
3362
3363         return 0;
3364 }
3365
3366 #ifdef CONFIG_PM_SLEEP
3367 static int bcmgenet_suspend(struct device *d)
3368 {
3369         struct net_device *dev = dev_get_drvdata(d);
3370         struct bcmgenet_priv *priv = netdev_priv(dev);
3371         int ret;
3372
3373         if (!netif_running(dev))
3374                 return 0;
3375
3376         bcmgenet_netif_stop(dev);
3377
3378         phy_suspend(priv->phydev);
3379
3380         netif_device_detach(dev);
3381
3382         /* Disable MAC receive */
3383         umac_enable_set(priv, CMD_RX_EN, false);
3384
3385         ret = bcmgenet_dma_teardown(priv);
3386         if (ret)
3387                 return ret;
3388
3389         /* Disable MAC transmit. TX DMA disabled have to done before this */
3390         umac_enable_set(priv, CMD_TX_EN, false);
3391
3392         /* tx reclaim */
3393         bcmgenet_tx_reclaim_all(dev);
3394         bcmgenet_fini_dma(priv);
3395
3396         /* Prepare the device for Wake-on-LAN and switch to the slow clock */
3397         if (device_may_wakeup(d) && priv->wolopts) {
3398                 ret = bcmgenet_power_down(priv, GENET_POWER_WOL_MAGIC);
3399                 clk_prepare_enable(priv->clk_wol);
3400         } else if (priv->internal_phy) {
3401                 ret = bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
3402         }
3403
3404         /* Turn off the clocks */
3405         clk_disable_unprepare(priv->clk);
3406
3407         return ret;
3408 }
3409
3410 static int bcmgenet_resume(struct device *d)
3411 {
3412         struct net_device *dev = dev_get_drvdata(d);
3413         struct bcmgenet_priv *priv = netdev_priv(dev);
3414         unsigned long dma_ctrl;
3415         int ret;
3416         u32 reg;
3417
3418         if (!netif_running(dev))
3419                 return 0;
3420
3421         /* Turn on the clock */
3422         ret = clk_prepare_enable(priv->clk);
3423         if (ret)
3424                 return ret;
3425
3426         /* If this is an internal GPHY, power it back on now, before UniMAC is
3427          * brought out of reset as absolutely no UniMAC activity is allowed
3428          */
3429         if (priv->internal_phy)
3430                 bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
3431
3432         bcmgenet_umac_reset(priv);
3433
3434         ret = init_umac(priv);
3435         if (ret)
3436                 goto out_clk_disable;
3437
3438         /* From WOL-enabled suspend, switch to regular clock */
3439         if (priv->wolopts)
3440                 clk_disable_unprepare(priv->clk_wol);
3441
3442         phy_init_hw(priv->phydev);
3443         /* Speed settings must be restored */
3444         bcmgenet_mii_config(priv->dev);
3445
3446         /* disable ethernet MAC while updating its registers */
3447         umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, false);
3448
3449         bcmgenet_set_hw_addr(priv, dev->dev_addr);
3450
3451         if (priv->internal_phy) {
3452                 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
3453                 reg |= EXT_ENERGY_DET_MASK;
3454                 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
3455         }
3456
3457         if (priv->wolopts)
3458                 bcmgenet_power_up(priv, GENET_POWER_WOL_MAGIC);
3459
3460         /* Disable RX/TX DMA and flush TX queues */
3461         dma_ctrl = bcmgenet_dma_disable(priv);
3462
3463         /* Reinitialize TDMA and RDMA and SW housekeeping */
3464         ret = bcmgenet_init_dma(priv);
3465         if (ret) {
3466                 netdev_err(dev, "failed to initialize DMA\n");
3467                 goto out_clk_disable;
3468         }
3469
3470         /* Always enable ring 16 - descriptor ring */
3471         bcmgenet_enable_dma(priv, dma_ctrl);
3472
3473         netif_device_attach(dev);
3474
3475         phy_resume(priv->phydev);
3476
3477         if (priv->eee.eee_enabled)
3478                 bcmgenet_eee_enable_set(dev, true);
3479
3480         bcmgenet_netif_start(dev);
3481
3482         return 0;
3483
3484 out_clk_disable:
3485         clk_disable_unprepare(priv->clk);
3486         return ret;
3487 }
3488 #endif /* CONFIG_PM_SLEEP */
3489
3490 static SIMPLE_DEV_PM_OPS(bcmgenet_pm_ops, bcmgenet_suspend, bcmgenet_resume);
3491
3492 static struct platform_driver bcmgenet_driver = {
3493         .probe  = bcmgenet_probe,
3494         .remove = bcmgenet_remove,
3495         .driver = {
3496                 .name   = "bcmgenet",
3497                 .of_match_table = bcmgenet_match,
3498                 .pm     = &bcmgenet_pm_ops,
3499         },
3500 };
3501 module_platform_driver(bcmgenet_driver);
3502
3503 MODULE_AUTHOR("Broadcom Corporation");
3504 MODULE_DESCRIPTION("Broadcom GENET Ethernet controller driver");
3505 MODULE_ALIAS("platform:bcmgenet");
3506 MODULE_LICENSE("GPL");