drivers/net/ethernet/apm/xgene/xgene_enet_main.c

   1 /* Applied Micro X-Gene SoC Ethernet Driver
   2  *
   3  * Copyright (c) 2014, Applied Micro Circuits Corporation
   4  * Authors: Iyappan Subramanian <isubramanian@apm.com>
   5  *          Ravi Patel <rapatel@apm.com>
   6  *          Keyur Chudgar <kchudgar@apm.com>
   7  *
   8  * This program is free software; you can redistribute  it and/or modify it
   9  * under  the terms of  the GNU General  Public License as published by the
  10  * Free Software Foundation;  either version 2 of the  License, or (at your
  11  * option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20  */
  21
  22 #include <linux/gpio.h>
  23 #include "xgene_enet_main.h"
  24 #include "xgene_enet_hw.h"
  25 #include "xgene_enet_sgmac.h"
  26 #include "xgene_enet_xgmac.h"
  27
  28 #define RES_ENET_CSR    0
  29 #define RES_RING_CSR    1
  30 #define RES_RING_CMD    2
  31
  32 static const struct of_device_id xgene_enet_of_match[];
  33 static const struct acpi_device_id xgene_enet_acpi_match[];
  34
  35 static void xgene_enet_init_bufpool(struct xgene_enet_desc_ring *buf_pool)
  36 {
  37         struct xgene_enet_raw_desc16 *raw_desc;
  38         int i;
  39
  40         if (!buf_pool)
  41                 return;
  42
  43         for (i = 0; i < buf_pool->slots; i++) {
  44                 raw_desc = &buf_pool->raw_desc16[i];
  45
  46                 /* Hardware expects descriptor in little endian format */
  47                 raw_desc->m0 = cpu_to_le64(i |
  48                                 SET_VAL(FPQNUM, buf_pool->dst_ring_num) |
  49                                 SET_VAL(STASH, 3));
  50         }
  51 }
  52
  53 static u16 xgene_enet_get_data_len(u64 bufdatalen)
  54 {
  55         u16 hw_len, mask;
  56
  57         hw_len = GET_VAL(BUFDATALEN, bufdatalen);
  58
  59         if (unlikely(hw_len == 0x7800)) {
  60                 return 0;
  61         } else if (!(hw_len & BIT(14))) {
  62                 mask = GENMASK(13, 0);
  63                 return (hw_len & mask) ? (hw_len & mask) : SIZE_16K;
  64         } else if (!(hw_len & GENMASK(13, 12))) {
  65                 mask = GENMASK(11, 0);
  66                 return (hw_len & mask) ? (hw_len & mask) : SIZE_4K;
  67         } else {
  68                 mask = GENMASK(11, 0);
  69                 return (hw_len & mask) ? (hw_len & mask) : SIZE_2K;
  70         }
  71 }
  72
  73 static u16 xgene_enet_set_data_len(u32 size)
  74 {
  75         u16 hw_len;
  76
  77         hw_len =  (size == SIZE_4K) ? BIT(14) : 0;
  78
  79         return hw_len;
  80 }
  81
  82 static int xgene_enet_refill_pagepool(struct xgene_enet_desc_ring *buf_pool,
  83                                       u32 nbuf)
  84 {
  85         struct xgene_enet_raw_desc16 *raw_desc;
  86         struct xgene_enet_pdata *pdata;
  87         struct net_device *ndev;
  88         dma_addr_t dma_addr;
  89         struct device *dev;
  90         struct page *page;
  91         u32 slots, tail;
  92         u16 hw_len;
  93         int i;
  94
  95         if (unlikely(!buf_pool))
  96                 return 0;
  97
  98         ndev = buf_pool->ndev;
  99         pdata = netdev_priv(ndev);
 100         dev = ndev_to_dev(ndev);
 101         slots = buf_pool->slots - 1;
 102         tail = buf_pool->tail;
 103
 104         for (i = 0; i < nbuf; i++) {
 105                 raw_desc = &buf_pool->raw_desc16[tail];
 106
 107                 page = dev_alloc_page();
 108                 if (unlikely(!page))
 109                         return -ENOMEM;
 110
 111                 dma_addr = dma_map_page(dev, page, 0,
 112                                         PAGE_SIZE, DMA_FROM_DEVICE);
 113                 if (unlikely(dma_mapping_error(dev, dma_addr))) {
 114                         put_page(page);
 115                         return -ENOMEM;
 116                 }
 117
 118                 hw_len = xgene_enet_set_data_len(PAGE_SIZE);
 119                 raw_desc->m1 = cpu_to_le64(SET_VAL(DATAADDR, dma_addr) |
 120                                            SET_VAL(BUFDATALEN, hw_len) |
 121                                            SET_BIT(COHERENT));
 122
 123                 buf_pool->frag_page[tail] = page;
 124                 tail = (tail + 1) & slots;
 125         }
 126
 127         pdata->ring_ops->wr_cmd(buf_pool, nbuf);
 128         buf_pool->tail = tail;
 129
 130         return 0;
 131 }
 132
 133 static int xgene_enet_refill_bufpool(struct xgene_enet_desc_ring *buf_pool,
 134                                      u32 nbuf)
 135 {
 136         struct sk_buff *skb;
 137         struct xgene_enet_raw_desc16 *raw_desc;
 138         struct xgene_enet_pdata *pdata;
 139         struct net_device *ndev;
 140         struct device *dev;
 141         dma_addr_t dma_addr;
 142         u32 tail = buf_pool->tail;
 143         u32 slots = buf_pool->slots - 1;
 144         u16 bufdatalen, len;
 145         int i;
 146
 147         ndev = buf_pool->ndev;
 148         dev = ndev_to_dev(buf_pool->ndev);
 149         pdata = netdev_priv(ndev);
 150
 151         bufdatalen = BUF_LEN_CODE_2K | (SKB_BUFFER_SIZE & GENMASK(11, 0));
 152         len = XGENE_ENET_STD_MTU;
 153
 154         for (i = 0; i < nbuf; i++) {
 155                 raw_desc = &buf_pool->raw_desc16[tail];
 156
 157                 skb = netdev_alloc_skb_ip_align(ndev, len);
 158                 if (unlikely(!skb))
 159                         return -ENOMEM;
 160
 161                 dma_addr = dma_map_single(dev, skb->data, len, DMA_FROM_DEVICE);
 162                 if (dma_mapping_error(dev, dma_addr)) {
 163                         netdev_err(ndev, "DMA mapping error\n");
 164                         dev_kfree_skb_any(skb);
 165                         return -EINVAL;
 166                 }
 167
 168                 buf_pool->rx_skb[tail] = skb;
 169
 170                 raw_desc->m1 = cpu_to_le64(SET_VAL(DATAADDR, dma_addr) |
 171                                            SET_VAL(BUFDATALEN, bufdatalen) |
 172                                            SET_BIT(COHERENT));
 173                 tail = (tail + 1) & slots;
 174         }
 175
 176         pdata->ring_ops->wr_cmd(buf_pool, nbuf);
 177         buf_pool->tail = tail;
 178
 179         return 0;
 180 }
 181
 182 static u8 xgene_enet_hdr_len(const void *data)
 183 {
 184         const struct ethhdr *eth = data;
 185
 186         return (eth->h_proto == htons(ETH_P_8021Q)) ? VLAN_ETH_HLEN : ETH_HLEN;
 187 }
 188
 189 static void xgene_enet_delete_bufpool(struct xgene_enet_desc_ring *buf_pool)
 190 {
 191         struct device *dev = ndev_to_dev(buf_pool->ndev);
 192         struct xgene_enet_raw_desc16 *raw_desc;
 193         dma_addr_t dma_addr;
 194         int i;
 195
 196         /* Free up the buffers held by hardware */
 197         for (i = 0; i < buf_pool->slots; i++) {
 198                 if (buf_pool->rx_skb[i]) {
 199                         dev_kfree_skb_any(buf_pool->rx_skb[i]);
 200
 201                         raw_desc = &buf_pool->raw_desc16[i];
 202                         dma_addr = GET_VAL(DATAADDR, le64_to_cpu(raw_desc->m1));
 203                         dma_unmap_single(dev, dma_addr, XGENE_ENET_MAX_MTU,
 204                                          DMA_FROM_DEVICE);
 205                 }
 206         }
 207 }
 208
 209 static void xgene_enet_delete_pagepool(struct xgene_enet_desc_ring *buf_pool)
 210 {
 211         struct device *dev = ndev_to_dev(buf_pool->ndev);
 212         dma_addr_t dma_addr;
 213         struct page *page;
 214         int i;
 215
 216         /* Free up the buffers held by hardware */
 217         for (i = 0; i < buf_pool->slots; i++) {
 218                 page = buf_pool->frag_page[i];
 219                 if (page) {
 220                         dma_addr = buf_pool->frag_dma_addr[i];
 221                         dma_unmap_page(dev, dma_addr, PAGE_SIZE,
 222                                        DMA_FROM_DEVICE);
 223                         put_page(page);
 224                 }
 225         }
 226 }
 227
 228 static irqreturn_t xgene_enet_rx_irq(const int irq, void *data)
 229 {
 230         struct xgene_enet_desc_ring *rx_ring = data;
 231
 232         if (napi_schedule_prep(&rx_ring->napi)) {
 233                 disable_irq_nosync(irq);
 234                 __napi_schedule(&rx_ring->napi);
 235         }
 236
 237         return IRQ_HANDLED;
 238 }
 239
 240 static int xgene_enet_tx_completion(struct xgene_enet_desc_ring *cp_ring,
 241                                     struct xgene_enet_raw_desc *raw_desc)
 242 {
 243         struct xgene_enet_pdata *pdata = netdev_priv(cp_ring->ndev);
 244         struct sk_buff *skb;
 245         struct device *dev;
 246         skb_frag_t *frag;
 247         dma_addr_t *frag_dma_addr;
 248         u16 skb_index;
 249         u8 mss_index;
 250         u8 status;
 251         int i;
 252
 253         skb_index = GET_VAL(USERINFO, le64_to_cpu(raw_desc->m0));
 254         skb = cp_ring->cp_skb[skb_index];
 255         frag_dma_addr = &cp_ring->frag_dma_addr[skb_index * MAX_SKB_FRAGS];
 256
 257         dev = ndev_to_dev(cp_ring->ndev);
 258         dma_unmap_single(dev, GET_VAL(DATAADDR, le64_to_cpu(raw_desc->m1)),
 259                          skb_headlen(skb),
 260                          DMA_TO_DEVICE);
 261
 262         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 263                 frag = &skb_shinfo(skb)->frags[i];
 264                 dma_unmap_page(dev, frag_dma_addr[i], skb_frag_size(frag),
 265                                DMA_TO_DEVICE);
 266         }
 267
 268         if (GET_BIT(ET, le64_to_cpu(raw_desc->m3))) {
 269                 mss_index = GET_VAL(MSS, le64_to_cpu(raw_desc->m3));
 270                 spin_lock(&pdata->mss_lock);
 271                 pdata->mss_refcnt[mss_index]--;
 272                 spin_unlock(&pdata->mss_lock);
 273         }
 274
 275         /* Checking for error */
 276         status = GET_VAL(LERR, le64_to_cpu(raw_desc->m0));
 277         if (unlikely(status > 2)) {
 278                 cp_ring->tx_dropped++;
 279                 cp_ring->tx_errors++;
 280         }
 281
 282         if (likely(skb)) {
 283                 dev_kfree_skb_any(skb);
 284         } else {
 285                 netdev_err(cp_ring->ndev, "completion skb is NULL\n");
 286         }
 287
 288         return 0;
 289 }
 290
 291 static int xgene_enet_setup_mss(struct net_device *ndev, u32 mss)
 292 {
 293         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 294         int mss_index = -EBUSY;
 295         int i;
 296
 297         spin_lock(&pdata->mss_lock);
 298
 299         /* Reuse the slot if MSS matches */
 300         for (i = 0; mss_index < 0 && i < NUM_MSS_REG; i++) {
 301                 if (pdata->mss[i] == mss) {
 302                         pdata->mss_refcnt[i]++;
 303                         mss_index = i;
 304                 }
 305         }
 306
 307         /* Overwrite the slot with ref_count = 0 */
 308         for (i = 0; mss_index < 0 && i < NUM_MSS_REG; i++) {
 309                 if (!pdata->mss_refcnt[i]) {
 310                         pdata->mss_refcnt[i]++;
 311                         pdata->mac_ops->set_mss(pdata, mss, i);
 312                         pdata->mss[i] = mss;
 313                         mss_index = i;
 314                 }
 315         }
 316
 317         spin_unlock(&pdata->mss_lock);
 318
 319         return mss_index;
 320 }
 321
 322 static int xgene_enet_work_msg(struct sk_buff *skb, u64 *hopinfo)
 323 {
 324         struct net_device *ndev = skb->dev;
 325         struct iphdr *iph;
 326         u8 l3hlen = 0, l4hlen = 0;
 327         u8 ethhdr, proto = 0, csum_enable = 0;
 328         u32 hdr_len, mss = 0;
 329         u32 i, len, nr_frags;
 330         int mss_index;
 331
 332         ethhdr = xgene_enet_hdr_len(skb->data);
 333
 334         if (unlikely(skb->protocol != htons(ETH_P_IP)) &&
 335             unlikely(skb->protocol != htons(ETH_P_8021Q)))
 336                 goto out;
 337
 338         if (unlikely(!(skb->dev->features & NETIF_F_IP_CSUM)))
 339                 goto out;
 340
 341         iph = ip_hdr(skb);
 342         if (unlikely(ip_is_fragment(iph)))
 343                 goto out;
 344
 345         if (likely(iph->protocol == IPPROTO_TCP)) {
 346                 l4hlen = tcp_hdrlen(skb) >> 2;
 347                 csum_enable = 1;
 348                 proto = TSO_IPPROTO_TCP;
 349                 if (ndev->features & NETIF_F_TSO) {
 350                         hdr_len = ethhdr + ip_hdrlen(skb) + tcp_hdrlen(skb);
 351                         mss = skb_shinfo(skb)->gso_size;
 352
 353                         if (skb_is_nonlinear(skb)) {
 354                                 len = skb_headlen(skb);
 355                                 nr_frags = skb_shinfo(skb)->nr_frags;
 356
 357                                 for (i = 0; i < 2 && i < nr_frags; i++)
 358                                         len += skb_shinfo(skb)->frags[i].size;
 359
 360                                 /* HW requires header must reside in 3 buffer */
 361                                 if (unlikely(hdr_len > len)) {
 362                                         if (skb_linearize(skb))
 363                                                 return 0;
 364                                 }
 365                         }
 366
 367                         if (!mss || ((skb->len - hdr_len) <= mss))
 368                                 goto out;
 369
 370                         mss_index = xgene_enet_setup_mss(ndev, mss);
 371                         if (unlikely(mss_index < 0))
 372                                 return -EBUSY;
 373
 374                         *hopinfo |= SET_BIT(ET) | SET_VAL(MSS, mss_index);
 375                 }
 376         } else if (iph->protocol == IPPROTO_UDP) {
 377                 l4hlen = UDP_HDR_SIZE;
 378                 csum_enable = 1;
 379         }
 380 out:
 381         l3hlen = ip_hdrlen(skb) >> 2;
 382         *hopinfo |= SET_VAL(TCPHDR, l4hlen) |
 383                     SET_VAL(IPHDR, l3hlen) |
 384                     SET_VAL(ETHHDR, ethhdr) |
 385                     SET_VAL(EC, csum_enable) |
 386                     SET_VAL(IS, proto) |
 387                     SET_BIT(IC) |
 388                     SET_BIT(TYPE_ETH_WORK_MESSAGE);
 389
 390         return 0;
 391 }
 392
 393 static u16 xgene_enet_encode_len(u16 len)
 394 {
 395         return (len == BUFLEN_16K) ? 0 : len;
 396 }
 397
 398 static void xgene_set_addr_len(__le64 *desc, u32 idx, dma_addr_t addr, u32 len)
 399 {
 400         desc[idx ^ 1] = cpu_to_le64(SET_VAL(DATAADDR, addr) |
 401                                     SET_VAL(BUFDATALEN, len));
 402 }
 403
 404 static __le64 *xgene_enet_get_exp_bufs(struct xgene_enet_desc_ring *ring)
 405 {
 406         __le64 *exp_bufs;
 407
 408         exp_bufs = &ring->exp_bufs[ring->exp_buf_tail * MAX_EXP_BUFFS];
 409         memset(exp_bufs, 0, sizeof(__le64) * MAX_EXP_BUFFS);
 410         ring->exp_buf_tail = (ring->exp_buf_tail + 1) & ((ring->slots / 2) - 1);
 411
 412         return exp_bufs;
 413 }
 414
 415 static dma_addr_t *xgene_get_frag_dma_array(struct xgene_enet_desc_ring *ring)
 416 {
 417         return &ring->cp_ring->frag_dma_addr[ring->tail * MAX_SKB_FRAGS];
 418 }
 419
 420 static int xgene_enet_setup_tx_desc(struct xgene_enet_desc_ring *tx_ring,
 421                                     struct sk_buff *skb)
 422 {
 423         struct device *dev = ndev_to_dev(tx_ring->ndev);
 424         struct xgene_enet_pdata *pdata = netdev_priv(tx_ring->ndev);
 425         struct xgene_enet_raw_desc *raw_desc;
 426         __le64 *exp_desc = NULL, *exp_bufs = NULL;
 427         dma_addr_t dma_addr, pbuf_addr, *frag_dma_addr;
 428         skb_frag_t *frag;
 429         u16 tail = tx_ring->tail;
 430         u64 hopinfo = 0;
 431         u32 len, hw_len;
 432         u8 ll = 0, nv = 0, idx = 0;
 433         bool split = false;
 434         u32 size, offset, ell_bytes = 0;
 435         u32 i, fidx, nr_frags, count = 1;
 436         int ret;
 437
 438         raw_desc = &tx_ring->raw_desc[tail];
 439         tail = (tail + 1) & (tx_ring->slots - 1);
 440         memset(raw_desc, 0, sizeof(struct xgene_enet_raw_desc));
 441
 442         ret = xgene_enet_work_msg(skb, &hopinfo);
 443         if (ret)
 444                 return ret;
 445
 446         raw_desc->m3 = cpu_to_le64(SET_VAL(HENQNUM, tx_ring->dst_ring_num) |
 447                                    hopinfo);
 448
 449         len = skb_headlen(skb);
 450         hw_len = xgene_enet_encode_len(len);
 451
 452         dma_addr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 453         if (dma_mapping_error(dev, dma_addr)) {
 454                 netdev_err(tx_ring->ndev, "DMA mapping error\n");
 455                 return -EINVAL;
 456         }
 457
 458         /* Hardware expects descriptor in little endian format */
 459         raw_desc->m1 = cpu_to_le64(SET_VAL(DATAADDR, dma_addr) |
 460                                    SET_VAL(BUFDATALEN, hw_len) |
 461                                    SET_BIT(COHERENT));
 462
 463         if (!skb_is_nonlinear(skb))
 464                 goto out;
 465
 466         /* scatter gather */
 467         nv = 1;
 468         exp_desc = (void *)&tx_ring->raw_desc[tail];
 469         tail = (tail + 1) & (tx_ring->slots - 1);
 470         memset(exp_desc, 0, sizeof(struct xgene_enet_raw_desc));
 471
 472         nr_frags = skb_shinfo(skb)->nr_frags;
 473         for (i = nr_frags; i < 4 ; i++)
 474                 exp_desc[i ^ 1] = cpu_to_le64(LAST_BUFFER);
 475
 476         frag_dma_addr = xgene_get_frag_dma_array(tx_ring);
 477
 478         for (i = 0, fidx = 0; split || (fidx < nr_frags); i++) {
 479                 if (!split) {
 480                         frag = &skb_shinfo(skb)->frags[fidx];
 481                         size = skb_frag_size(frag);
 482                         offset = 0;
 483
 484                         pbuf_addr = skb_frag_dma_map(dev, frag, 0, size,
 485                                                      DMA_TO_DEVICE);
 486                         if (dma_mapping_error(dev, pbuf_addr))
 487                                 return -EINVAL;
 488
 489                         frag_dma_addr[fidx] = pbuf_addr;
 490                         fidx++;
 491
 492                         if (size > BUFLEN_16K)
 493                                 split = true;
 494                 }
 495
 496                 if (size > BUFLEN_16K) {
 497                         len = BUFLEN_16K;
 498                         size -= BUFLEN_16K;
 499                 } else {
 500                         len = size;
 501                         split = false;
 502                 }
 503
 504                 dma_addr = pbuf_addr + offset;
 505                 hw_len = xgene_enet_encode_len(len);
 506
 507                 switch (i) {
 508                 case 0:
 509                 case 1:
 510                 case 2:
 511                         xgene_set_addr_len(exp_desc, i, dma_addr, hw_len);
 512                         break;
 513                 case 3:
 514                         if (split || (fidx != nr_frags)) {
 515                                 exp_bufs = xgene_enet_get_exp_bufs(tx_ring);
 516                                 xgene_set_addr_len(exp_bufs, idx, dma_addr,
 517                                                    hw_len);
 518                                 idx++;
 519                                 ell_bytes += len;
 520                         } else {
 521                                 xgene_set_addr_len(exp_desc, i, dma_addr,
 522                                                    hw_len);
 523                         }
 524                         break;
 525                 default:
 526                         xgene_set_addr_len(exp_bufs, idx, dma_addr, hw_len);
 527                         idx++;
 528                         ell_bytes += len;
 529                         break;
 530                 }
 531
 532                 if (split)
 533                         offset += BUFLEN_16K;
 534         }
 535         count++;
 536
 537         if (idx) {
 538                 ll = 1;
 539                 dma_addr = dma_map_single(dev, exp_bufs,
 540                                           sizeof(u64) * MAX_EXP_BUFFS,
 541                                           DMA_TO_DEVICE);
 542                 if (dma_mapping_error(dev, dma_addr)) {
 543                         dev_kfree_skb_any(skb);
 544                         return -EINVAL;
 545                 }
 546                 i = ell_bytes >> LL_BYTES_LSB_LEN;
 547                 exp_desc[2] = cpu_to_le64(SET_VAL(DATAADDR, dma_addr) |
 548                                           SET_VAL(LL_BYTES_MSB, i) |
 549                                           SET_VAL(LL_LEN, idx));
 550                 raw_desc->m2 = cpu_to_le64(SET_VAL(LL_BYTES_LSB, ell_bytes));
 551         }
 552
 553 out:
 554         raw_desc->m0 = cpu_to_le64(SET_VAL(LL, ll) | SET_VAL(NV, nv) |
 555                                    SET_VAL(USERINFO, tx_ring->tail));
 556         tx_ring->cp_ring->cp_skb[tx_ring->tail] = skb;
 557         pdata->tx_level[tx_ring->cp_ring->index] += count;
 558         tx_ring->tail = tail;
 559
 560         return count;
 561 }
 562
 563 static netdev_tx_t xgene_enet_start_xmit(struct sk_buff *skb,
 564                                          struct net_device *ndev)
 565 {
 566         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 567         struct xgene_enet_desc_ring *tx_ring;
 568         int index = skb->queue_mapping;
 569         u32 tx_level = pdata->tx_level[index];
 570         int count;
 571
 572         tx_ring = pdata->tx_ring[index];
 573         if (tx_level < pdata->txc_level[index])
 574                 tx_level += ((typeof(pdata->tx_level[index]))~0U);
 575
 576         if ((tx_level - pdata->txc_level[index]) > pdata->tx_qcnt_hi) {
 577                 netif_stop_subqueue(ndev, index);
 578                 return NETDEV_TX_BUSY;
 579         }
 580
 581         if (skb_padto(skb, XGENE_MIN_ENET_FRAME_SIZE))
 582                 return NETDEV_TX_OK;
 583
 584         count = xgene_enet_setup_tx_desc(tx_ring, skb);
 585         if (count == -EBUSY)
 586                 return NETDEV_TX_BUSY;
 587
 588         if (count <= 0) {
 589                 dev_kfree_skb_any(skb);
 590                 return NETDEV_TX_OK;
 591         }
 592
 593         skb_tx_timestamp(skb);
 594
 595         tx_ring->tx_packets++;
 596         tx_ring->tx_bytes += skb->len;
 597
 598         pdata->ring_ops->wr_cmd(tx_ring, count);
 599         return NETDEV_TX_OK;
 600 }
 601
 602 static void xgene_enet_rx_csum(struct sk_buff *skb)
 603 {
 604         struct net_device *ndev = skb->dev;
 605         struct iphdr *iph = ip_hdr(skb);
 606
 607         if (!(ndev->features & NETIF_F_RXCSUM))
 608                 return;
 609
 610         if (skb->protocol != htons(ETH_P_IP))
 611                 return;
 612
 613         if (ip_is_fragment(iph))
 614                 return;
 615
 616         if (iph->protocol != IPPROTO_TCP && iph->protocol != IPPROTO_UDP)
 617                 return;
 618
 619         skb->ip_summed = CHECKSUM_UNNECESSARY;
 620 }
 621
 622 static void xgene_enet_free_pagepool(struct xgene_enet_desc_ring *buf_pool,
 623                                      struct xgene_enet_raw_desc *raw_desc,
 624                                      struct xgene_enet_raw_desc *exp_desc)
 625 {
 626         __le64 *desc = (void *)exp_desc;
 627         dma_addr_t dma_addr;
 628         struct device *dev;
 629         struct page *page;
 630         u16 slots, head;
 631         u32 frag_size;
 632         int i;
 633
 634         if (!buf_pool || !raw_desc || !exp_desc ||
 635             (!GET_VAL(NV, le64_to_cpu(raw_desc->m0))))
 636                 return;
 637
 638         dev = ndev_to_dev(buf_pool->ndev);
 639         slots = buf_pool->slots - 1;
 640         head = buf_pool->head;
 641
 642         for (i = 0; i < 4; i++) {
 643                 frag_size = xgene_enet_get_data_len(le64_to_cpu(desc[i ^ 1]));
 644                 if (!frag_size)
 645                         break;
 646
 647                 dma_addr = GET_VAL(DATAADDR, le64_to_cpu(desc[i ^ 1]));
 648                 dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_FROM_DEVICE);
 649
 650                 page = buf_pool->frag_page[head];
 651                 put_page(page);
 652
 653                 buf_pool->frag_page[head] = NULL;
 654                 head = (head + 1) & slots;
 655         }
 656         buf_pool->head = head;
 657 }
 658
 659 /* Errata 10GE_10 and ENET_15 - Fix duplicated HW statistic counters */
 660 static bool xgene_enet_errata_10GE_10(struct sk_buff *skb, u32 len, u8 status)
 661 {
 662         if (status == INGRESS_CRC &&
 663             len >= (ETHER_STD_PACKET + 1) &&
 664             len <= (ETHER_STD_PACKET + 4) &&
 665             skb->protocol == htons(ETH_P_8021Q))
 666                 return true;
 667
 668         return false;
 669 }
 670
 671 /* Errata 10GE_8 and ENET_11 - allow packet with length <=64B */
 672 static bool xgene_enet_errata_10GE_8(struct sk_buff *skb, u32 len, u8 status)
 673 {
 674         if (status == INGRESS_PKT_LEN && len == ETHER_MIN_PACKET) {
 675                 if (ntohs(eth_hdr(skb)->h_proto) < 46)
 676                         return true;
 677         }
 678
 679         return false;
 680 }
 681
 682 static int xgene_enet_rx_frame(struct xgene_enet_desc_ring *rx_ring,
 683                                struct xgene_enet_raw_desc *raw_desc,
 684                                struct xgene_enet_raw_desc *exp_desc)
 685 {
 686         struct xgene_enet_desc_ring *buf_pool, *page_pool;
 687         u32 datalen, frag_size, skb_index;
 688         struct xgene_enet_pdata *pdata;
 689         struct net_device *ndev;
 690         dma_addr_t dma_addr;
 691         struct sk_buff *skb;
 692         struct device *dev;
 693         struct page *page;
 694         u16 slots, head;
 695         int i, ret = 0;
 696         __le64 *desc;
 697         u8 status;
 698         bool nv;
 699
 700         ndev = rx_ring->ndev;
 701         pdata = netdev_priv(ndev);
 702         dev = ndev_to_dev(rx_ring->ndev);
 703         buf_pool = rx_ring->buf_pool;
 704         page_pool = rx_ring->page_pool;
 705
 706         dma_unmap_single(dev, GET_VAL(DATAADDR, le64_to_cpu(raw_desc->m1)),
 707                          XGENE_ENET_STD_MTU, DMA_FROM_DEVICE);
 708         skb_index = GET_VAL(USERINFO, le64_to_cpu(raw_desc->m0));
 709         skb = buf_pool->rx_skb[skb_index];
 710         buf_pool->rx_skb[skb_index] = NULL;
 711
 712         datalen = xgene_enet_get_data_len(le64_to_cpu(raw_desc->m1));
 713         skb_put(skb, datalen);
 714         prefetch(skb->data - NET_IP_ALIGN);
 715         skb->protocol = eth_type_trans(skb, ndev);
 716
 717         /* checking for error */
 718         status = (GET_VAL(ELERR, le64_to_cpu(raw_desc->m0)) << LERR_LEN) |
 719                   GET_VAL(LERR, le64_to_cpu(raw_desc->m0));
 720         if (unlikely(status)) {
 721                 if (xgene_enet_errata_10GE_8(skb, datalen, status)) {
 722                         pdata->false_rflr++;
 723                 } else if (xgene_enet_errata_10GE_10(skb, datalen, status)) {
 724                         pdata->vlan_rjbr++;
 725                 } else {
 726                         dev_kfree_skb_any(skb);
 727                         xgene_enet_free_pagepool(page_pool, raw_desc, exp_desc);
 728                         xgene_enet_parse_error(rx_ring, status);
 729                         rx_ring->rx_dropped++;
 730                         goto out;
 731                 }
 732         }
 733
 734         nv = GET_VAL(NV, le64_to_cpu(raw_desc->m0));
 735         if (!nv) {
 736                 /* strip off CRC as HW isn't doing this */
 737                 datalen -= 4;
 738                 goto skip_jumbo;
 739         }
 740
 741         slots = page_pool->slots - 1;
 742         head = page_pool->head;
 743         desc = (void *)exp_desc;
 744
 745         for (i = 0; i < 4; i++) {
 746                 frag_size = xgene_enet_get_data_len(le64_to_cpu(desc[i ^ 1]));
 747                 if (!frag_size)
 748                         break;
 749
 750                 dma_addr = GET_VAL(DATAADDR, le64_to_cpu(desc[i ^ 1]));
 751                 dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_FROM_DEVICE);
 752
 753                 page = page_pool->frag_page[head];
 754                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 0,
 755                                 frag_size, PAGE_SIZE);
 756
 757                 datalen += frag_size;
 758
 759                 page_pool->frag_page[head] = NULL;
 760                 head = (head + 1) & slots;
 761         }
 762
 763         page_pool->head = head;
 764         rx_ring->npagepool -= skb_shinfo(skb)->nr_frags;
 765
 766 skip_jumbo:
 767         skb_checksum_none_assert(skb);
 768         xgene_enet_rx_csum(skb);
 769
 770         rx_ring->rx_packets++;
 771         rx_ring->rx_bytes += datalen;
 772         napi_gro_receive(&rx_ring->napi, skb);
 773
 774 out:
 775         if (rx_ring->npagepool <= 0) {
 776                 ret = xgene_enet_refill_pagepool(page_pool, NUM_NXTBUFPOOL);
 777                 rx_ring->npagepool = NUM_NXTBUFPOOL;
 778                 if (ret)
 779                         return ret;
 780         }
 781
 782         if (--rx_ring->nbufpool == 0) {
 783                 ret = xgene_enet_refill_bufpool(buf_pool, NUM_BUFPOOL);
 784                 rx_ring->nbufpool = NUM_BUFPOOL;
 785         }
 786
 787         return ret;
 788 }
 789
 790 static bool is_rx_desc(struct xgene_enet_raw_desc *raw_desc)
 791 {
 792         return GET_VAL(FPQNUM, le64_to_cpu(raw_desc->m0)) ? true : false;
 793 }
 794
 795 static int xgene_enet_process_ring(struct xgene_enet_desc_ring *ring,
 796                                    int budget)
 797 {
 798         struct net_device *ndev = ring->ndev;
 799         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 800         struct xgene_enet_raw_desc *raw_desc, *exp_desc;
 801         u16 head = ring->head;
 802         u16 slots = ring->slots - 1;
 803         int ret, desc_count, count = 0, processed = 0;
 804         bool is_completion;
 805
 806         do {
 807                 raw_desc = &ring->raw_desc[head];
 808                 desc_count = 0;
 809                 is_completion = false;
 810                 exp_desc = NULL;
 811                 if (unlikely(xgene_enet_is_desc_slot_empty(raw_desc)))
 812                         break;
 813
 814                 /* read fpqnum field after dataaddr field */
 815                 dma_rmb();
 816                 if (GET_BIT(NV, le64_to_cpu(raw_desc->m0))) {
 817                         head = (head + 1) & slots;
 818                         exp_desc = &ring->raw_desc[head];
 819
 820                         if (unlikely(xgene_enet_is_desc_slot_empty(exp_desc))) {
 821                                 head = (head - 1) & slots;
 822                                 break;
 823                         }
 824                         dma_rmb();
 825                         count++;
 826                         desc_count++;
 827                 }
 828                 if (is_rx_desc(raw_desc)) {
 829                         ret = xgene_enet_rx_frame(ring, raw_desc, exp_desc);
 830                 } else {
 831                         ret = xgene_enet_tx_completion(ring, raw_desc);
 832                         is_completion = true;
 833                 }
 834                 xgene_enet_mark_desc_slot_empty(raw_desc);
 835                 if (exp_desc)
 836                         xgene_enet_mark_desc_slot_empty(exp_desc);
 837
 838                 head = (head + 1) & slots;
 839                 count++;
 840                 desc_count++;
 841                 processed++;
 842                 if (is_completion)
 843                         pdata->txc_level[ring->index] += desc_count;
 844
 845                 if (ret)
 846                         break;
 847         } while (--budget);
 848
 849         if (likely(count)) {
 850                 pdata->ring_ops->wr_cmd(ring, -count);
 851                 ring->head = head;
 852
 853                 if (__netif_subqueue_stopped(ndev, ring->index))
 854                         netif_start_subqueue(ndev, ring->index);
 855         }
 856
 857         return processed;
 858 }
 859
 860 static int xgene_enet_napi(struct napi_struct *napi, const int budget)
 861 {
 862         struct xgene_enet_desc_ring *ring;
 863         int processed;
 864
 865         ring = container_of(napi, struct xgene_enet_desc_ring, napi);
 866         processed = xgene_enet_process_ring(ring, budget);
 867
 868         if (processed != budget) {
 869                 napi_complete_done(napi, processed);
 870                 enable_irq(ring->irq);
 871         }
 872
 873         return processed;
 874 }
 875
 876 static void xgene_enet_timeout(struct net_device *ndev)
 877 {
 878         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 879         struct netdev_queue *txq;
 880         int i;
 881
 882         pdata->mac_ops->reset(pdata);
 883
 884         for (i = 0; i < pdata->txq_cnt; i++) {
 885                 txq = netdev_get_tx_queue(ndev, i);
 886                 txq->trans_start = jiffies;
 887                 netif_tx_start_queue(txq);
 888         }
 889 }
 890
 891 static void xgene_enet_set_irq_name(struct net_device *ndev)
 892 {
 893         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 894         struct xgene_enet_desc_ring *ring;
 895         int i;
 896
 897         for (i = 0; i < pdata->rxq_cnt; i++) {
 898                 ring = pdata->rx_ring[i];
 899                 if (!pdata->cq_cnt) {
 900                         snprintf(ring->irq_name, IRQ_ID_SIZE, "%s-rx-txc",
 901                                  ndev->name);
 902                 } else {
 903                         snprintf(ring->irq_name, IRQ_ID_SIZE, "%s-rx-%d",
 904                                  ndev->name, i);
 905                 }
 906         }
 907
 908         for (i = 0; i < pdata->cq_cnt; i++) {
 909                 ring = pdata->tx_ring[i]->cp_ring;
 910                 snprintf(ring->irq_name, IRQ_ID_SIZE, "%s-txc-%d",
 911                          ndev->name, i);
 912         }
 913 }
 914
 915 static int xgene_enet_register_irq(struct net_device *ndev)
 916 {
 917         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 918         struct device *dev = ndev_to_dev(ndev);
 919         struct xgene_enet_desc_ring *ring;
 920         int ret = 0, i;
 921
 922         xgene_enet_set_irq_name(ndev);
 923         for (i = 0; i < pdata->rxq_cnt; i++) {
 924                 ring = pdata->rx_ring[i];
 925                 irq_set_status_flags(ring->irq, IRQ_DISABLE_UNLAZY);
 926                 ret = devm_request_irq(dev, ring->irq, xgene_enet_rx_irq,
 927                                        0, ring->irq_name, ring);
 928                 if (ret) {
 929                         netdev_err(ndev, "Failed to request irq %s\n",
 930                                    ring->irq_name);
 931                 }
 932         }
 933
 934         for (i = 0; i < pdata->cq_cnt; i++) {
 935                 ring = pdata->tx_ring[i]->cp_ring;
 936                 irq_set_status_flags(ring->irq, IRQ_DISABLE_UNLAZY);
 937                 ret = devm_request_irq(dev, ring->irq, xgene_enet_rx_irq,
 938                                        0, ring->irq_name, ring);
 939                 if (ret) {
 940                         netdev_err(ndev, "Failed to request irq %s\n",
 941                                    ring->irq_name);
 942                 }
 943         }
 944
 945         return ret;
 946 }
 947
 948 static void xgene_enet_free_irq(struct net_device *ndev)
 949 {
 950         struct xgene_enet_pdata *pdata;
 951         struct xgene_enet_desc_ring *ring;
 952         struct device *dev;
 953         int i;
 954
 955         pdata = netdev_priv(ndev);
 956         dev = ndev_to_dev(ndev);
 957
 958         for (i = 0; i < pdata->rxq_cnt; i++) {
 959                 ring = pdata->rx_ring[i];
 960                 irq_clear_status_flags(ring->irq, IRQ_DISABLE_UNLAZY);
 961                 devm_free_irq(dev, ring->irq, ring);
 962         }
 963
 964         for (i = 0; i < pdata->cq_cnt; i++) {
 965                 ring = pdata->tx_ring[i]->cp_ring;
 966                 irq_clear_status_flags(ring->irq, IRQ_DISABLE_UNLAZY);
 967                 devm_free_irq(dev, ring->irq, ring);
 968         }
 969 }
 970
 971 static void xgene_enet_napi_enable(struct xgene_enet_pdata *pdata)
 972 {
 973         struct napi_struct *napi;
 974         int i;
 975
 976         for (i = 0; i < pdata->rxq_cnt; i++) {
 977                 napi = &pdata->rx_ring[i]->napi;
 978                 napi_enable(napi);
 979         }
 980
 981         for (i = 0; i < pdata->cq_cnt; i++) {
 982                 napi = &pdata->tx_ring[i]->cp_ring->napi;
 983                 napi_enable(napi);
 984         }
 985 }
 986
 987 static void xgene_enet_napi_disable(struct xgene_enet_pdata *pdata)
 988 {
 989         struct napi_struct *napi;
 990         int i;
 991
 992         for (i = 0; i < pdata->rxq_cnt; i++) {
 993                 napi = &pdata->rx_ring[i]->napi;
 994                 napi_disable(napi);
 995         }
 996
 997         for (i = 0; i < pdata->cq_cnt; i++) {
 998                 napi = &pdata->tx_ring[i]->cp_ring->napi;
 999                 napi_disable(napi);
1000         }
1001 }
1002
1003 static int xgene_enet_open(struct net_device *ndev)
1004 {
1005         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1006         const struct xgene_mac_ops *mac_ops = pdata->mac_ops;
1007         int ret;
1008
1009         ret = netif_set_real_num_tx_queues(ndev, pdata->txq_cnt);
1010         if (ret)
1011                 return ret;
1012
1013         ret = netif_set_real_num_rx_queues(ndev, pdata->rxq_cnt);
1014         if (ret)
1015                 return ret;
1016
1017         xgene_enet_napi_enable(pdata);
1018         ret = xgene_enet_register_irq(ndev);
1019         if (ret)
1020                 return ret;
1021
1022         if (ndev->phydev) {
1023                 phy_start(ndev->phydev);
1024         } else {
1025                 schedule_delayed_work(&pdata->link_work, PHY_POLL_LINK_OFF);
1026                 netif_carrier_off(ndev);
1027         }
1028
1029         mac_ops->tx_enable(pdata);
1030         mac_ops->rx_enable(pdata);
1031         netif_tx_start_all_queues(ndev);
1032
1033         return ret;
1034 }
1035
1036 static int xgene_enet_close(struct net_device *ndev)
1037 {
1038         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1039         const struct xgene_mac_ops *mac_ops = pdata->mac_ops;
1040         int i;
1041
1042         netif_tx_stop_all_queues(ndev);
1043         mac_ops->tx_disable(pdata);
1044         mac_ops->rx_disable(pdata);
1045
1046         if (ndev->phydev)
1047                 phy_stop(ndev->phydev);
1048         else
1049                 cancel_delayed_work_sync(&pdata->link_work);
1050
1051         xgene_enet_free_irq(ndev);
1052         xgene_enet_napi_disable(pdata);
1053         for (i = 0; i < pdata->rxq_cnt; i++)
1054                 xgene_enet_process_ring(pdata->rx_ring[i], -1);
1055
1056         return 0;
1057 }
1058 static void xgene_enet_delete_ring(struct xgene_enet_desc_ring *ring)
1059 {
1060         struct xgene_enet_pdata *pdata;
1061         struct device *dev;
1062
1063         pdata = netdev_priv(ring->ndev);
1064         dev = ndev_to_dev(ring->ndev);
1065
1066         pdata->ring_ops->clear(ring);
1067         dmam_free_coherent(dev, ring->size, ring->desc_addr, ring->dma);
1068 }
1069
1070 static void xgene_enet_delete_desc_rings(struct xgene_enet_pdata *pdata)
1071 {
1072         struct xgene_enet_desc_ring *buf_pool, *page_pool;
1073         struct xgene_enet_desc_ring *ring;
1074         int i;
1075
1076         for (i = 0; i < pdata->txq_cnt; i++) {
1077                 ring = pdata->tx_ring[i];
1078                 if (ring) {
1079                         xgene_enet_delete_ring(ring);
1080                         pdata->port_ops->clear(pdata, ring);
1081                         if (pdata->cq_cnt)
1082                                 xgene_enet_delete_ring(ring->cp_ring);
1083                         pdata->tx_ring[i] = NULL;
1084                 }
1085
1086         }
1087
1088         for (i = 0; i < pdata->rxq_cnt; i++) {
1089                 ring = pdata->rx_ring[i];
1090                 if (ring) {
1091                         page_pool = ring->page_pool;
1092                         if (page_pool) {
1093                                 xgene_enet_delete_pagepool(page_pool);
1094                                 xgene_enet_delete_ring(page_pool);
1095                                 pdata->port_ops->clear(pdata, page_pool);
1096                         }
1097
1098                         buf_pool = ring->buf_pool;
1099                         xgene_enet_delete_bufpool(buf_pool);
1100                         xgene_enet_delete_ring(buf_pool);
1101                         pdata->port_ops->clear(pdata, buf_pool);
1102
1103                         xgene_enet_delete_ring(ring);
1104                         pdata->rx_ring[i] = NULL;
1105                 }
1106
1107         }
1108 }
1109
1110 static int xgene_enet_get_ring_size(struct device *dev,
1111                                     enum xgene_enet_ring_cfgsize cfgsize)
1112 {
1113         int size = -EINVAL;
1114
1115         switch (cfgsize) {
1116         case RING_CFGSIZE_512B:
1117                 size = 0x200;
1118                 break;
1119         case RING_CFGSIZE_2KB:
1120                 size = 0x800;
1121                 break;
1122         case RING_CFGSIZE_16KB:
1123                 size = 0x4000;
1124                 break;
1125         case RING_CFGSIZE_64KB:
1126                 size = 0x10000;
1127                 break;
1128         case RING_CFGSIZE_512KB:
1129                 size = 0x80000;
1130                 break;
1131         default:
1132                 dev_err(dev, "Unsupported cfg ring size %d\n", cfgsize);
1133                 break;
1134         }
1135
1136         return size;
1137 }
1138
1139 static void xgene_enet_free_desc_ring(struct xgene_enet_desc_ring *ring)
1140 {
1141         struct xgene_enet_pdata *pdata;
1142         struct device *dev;
1143
1144         if (!ring)
1145                 return;
1146
1147         dev = ndev_to_dev(ring->ndev);
1148         pdata = netdev_priv(ring->ndev);
1149
1150         if (ring->desc_addr) {
1151                 pdata->ring_ops->clear(ring);
1152                 dmam_free_coherent(dev, ring->size, ring->desc_addr, ring->dma);
1153         }
1154         devm_kfree(dev, ring);
1155 }
1156
1157 static void xgene_enet_free_desc_rings(struct xgene_enet_pdata *pdata)
1158 {
1159         struct xgene_enet_desc_ring *page_pool;
1160         struct device *dev = &pdata->pdev->dev;
1161         struct xgene_enet_desc_ring *ring;
1162         void *p;
1163         int i;
1164
1165         for (i = 0; i < pdata->txq_cnt; i++) {
1166                 ring = pdata->tx_ring[i];
1167                 if (ring) {
1168                         if (ring->cp_ring && ring->cp_ring->cp_skb)
1169                                 devm_kfree(dev, ring->cp_ring->cp_skb);
1170
1171                         if (ring->cp_ring && pdata->cq_cnt)
1172                                 xgene_enet_free_desc_ring(ring->cp_ring);
1173
1174                         xgene_enet_free_desc_ring(ring);
1175                 }
1176
1177         }
1178
1179         for (i = 0; i < pdata->rxq_cnt; i++) {
1180                 ring = pdata->rx_ring[i];
1181                 if (ring) {
1182                         if (ring->buf_pool) {
1183                                 if (ring->buf_pool->rx_skb)
1184                                         devm_kfree(dev, ring->buf_pool->rx_skb);
1185
1186                                 xgene_enet_free_desc_ring(ring->buf_pool);
1187                         }
1188
1189                         page_pool = ring->page_pool;
1190                         if (page_pool) {
1191                                 p = page_pool->frag_page;
1192                                 if (p)
1193                                         devm_kfree(dev, p);
1194
1195                                 p = page_pool->frag_dma_addr;
1196                                 if (p)
1197                                         devm_kfree(dev, p);
1198                         }
1199
1200                         xgene_enet_free_desc_ring(ring);
1201                 }
1202         }
1203 }
1204
1205 static bool is_irq_mbox_required(struct xgene_enet_pdata *pdata,
1206                                  struct xgene_enet_desc_ring *ring)
1207 {
1208         if ((pdata->enet_id == XGENE_ENET2) &&
1209             (xgene_enet_ring_owner(ring->id) == RING_OWNER_CPU)) {
1210                 return true;
1211         }
1212
1213         return false;
1214 }
1215
1216 static void __iomem *xgene_enet_ring_cmd_base(struct xgene_enet_pdata *pdata,
1217                                               struct xgene_enet_desc_ring *ring)
1218 {
1219         u8 num_ring_id_shift = pdata->ring_ops->num_ring_id_shift;
1220
1221         return pdata->ring_cmd_addr + (ring->num << num_ring_id_shift);
1222 }
1223
1224 static struct xgene_enet_desc_ring *xgene_enet_create_desc_ring(
1225                         struct net_device *ndev, u32 ring_num,
1226                         enum xgene_enet_ring_cfgsize cfgsize, u32 ring_id)
1227 {
1228         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1229         struct device *dev = ndev_to_dev(ndev);
1230         struct xgene_enet_desc_ring *ring;
1231         void *irq_mbox_addr;
1232         int size;
1233
1234         size = xgene_enet_get_ring_size(dev, cfgsize);
1235         if (size < 0)
1236                 return NULL;
1237
1238         ring = devm_kzalloc(dev, sizeof(struct xgene_enet_desc_ring),
1239                             GFP_KERNEL);
1240         if (!ring)
1241                 return NULL;
1242
1243         ring->ndev = ndev;
1244         ring->num = ring_num;
1245         ring->cfgsize = cfgsize;
1246         ring->id = ring_id;
1247
1248         ring->desc_addr = dmam_alloc_coherent(dev, size, &ring->dma,
1249                                               GFP_KERNEL | __GFP_ZERO);
1250         if (!ring->desc_addr) {
1251                 devm_kfree(dev, ring);
1252                 return NULL;
1253         }
1254         ring->size = size;
1255
1256         if (is_irq_mbox_required(pdata, ring)) {
1257                 irq_mbox_addr = dmam_alloc_coherent(dev, INTR_MBOX_SIZE,
1258                                                     &ring->irq_mbox_dma,
1259                                                     GFP_KERNEL | __GFP_ZERO);
1260                 if (!irq_mbox_addr) {
1261                         dmam_free_coherent(dev, size, ring->desc_addr,
1262                                            ring->dma);
1263                         devm_kfree(dev, ring);
1264                         return NULL;
1265                 }
1266                 ring->irq_mbox_addr = irq_mbox_addr;
1267         }
1268
1269         ring->cmd_base = xgene_enet_ring_cmd_base(pdata, ring);
1270         ring->cmd = ring->cmd_base + INC_DEC_CMD_ADDR;
1271         ring = pdata->ring_ops->setup(ring);
1272         netdev_dbg(ndev, "ring info: num=%d  size=%d  id=%d  slots=%d\n",
1273                    ring->num, ring->size, ring->id, ring->slots);
1274
1275         return ring;
1276 }
1277
1278 static u16 xgene_enet_get_ring_id(enum xgene_ring_owner owner, u8 bufnum)
1279 {
1280         return (owner << 6) | (bufnum & GENMASK(5, 0));
1281 }
1282
1283 static enum xgene_ring_owner xgene_derive_ring_owner(struct xgene_enet_pdata *p)
1284 {
1285         enum xgene_ring_owner owner;
1286
1287         if (p->enet_id == XGENE_ENET1) {
1288                 switch (p->phy_mode) {
1289                 case PHY_INTERFACE_MODE_SGMII:
1290                         owner = RING_OWNER_ETH0;
1291                         break;
1292                 default:
1293                         owner = (!p->port_id) ? RING_OWNER_ETH0 :
1294                                                 RING_OWNER_ETH1;
1295                         break;
1296                 }
1297         } else {
1298                 owner = (!p->port_id) ? RING_OWNER_ETH0 : RING_OWNER_ETH1;
1299         }
1300
1301         return owner;
1302 }
1303
1304 static u8 xgene_start_cpu_bufnum(struct xgene_enet_pdata *pdata)
1305 {
1306         struct device *dev = &pdata->pdev->dev;
1307         u32 cpu_bufnum;
1308         int ret;
1309
1310         ret = device_property_read_u32(dev, "channel", &cpu_bufnum);
1311
1312         return (!ret) ? cpu_bufnum : pdata->cpu_bufnum;
1313 }
1314
1315 static int xgene_enet_create_desc_rings(struct net_device *ndev)
1316 {
1317         struct xgene_enet_desc_ring *rx_ring, *tx_ring, *cp_ring;
1318         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1319         struct xgene_enet_desc_ring *page_pool = NULL;
1320         struct xgene_enet_desc_ring *buf_pool = NULL;
1321         struct device *dev = ndev_to_dev(ndev);
1322         u8 eth_bufnum = pdata->eth_bufnum;
1323         u8 bp_bufnum = pdata->bp_bufnum;
1324         u16 ring_num = pdata->ring_num;
1325         enum xgene_ring_owner owner;
1326         dma_addr_t dma_exp_bufs;
1327         u16 ring_id, slots;
1328         __le64 *exp_bufs;
1329         int i, ret, size;
1330         u8 cpu_bufnum;
1331
1332         cpu_bufnum = xgene_start_cpu_bufnum(pdata);
1333
1334         for (i = 0; i < pdata->rxq_cnt; i++) {
1335                 /* allocate rx descriptor ring */
1336                 owner = xgene_derive_ring_owner(pdata);
1337                 ring_id = xgene_enet_get_ring_id(RING_OWNER_CPU, cpu_bufnum++);
1338                 rx_ring = xgene_enet_create_desc_ring(ndev, ring_num++,
1339                                                       RING_CFGSIZE_16KB,
1340                                                       ring_id);
1341                 if (!rx_ring) {
1342                         ret = -ENOMEM;
1343                         goto err;
1344                 }
1345
1346                 /* allocate buffer pool for receiving packets */
1347                 owner = xgene_derive_ring_owner(pdata);
1348                 ring_id = xgene_enet_get_ring_id(owner, bp_bufnum++);
1349                 buf_pool = xgene_enet_create_desc_ring(ndev, ring_num++,
1350                                                        RING_CFGSIZE_16KB,
1351                                                        ring_id);
1352                 if (!buf_pool) {
1353                         ret = -ENOMEM;
1354                         goto err;
1355                 }
1356
1357                 rx_ring->nbufpool = NUM_BUFPOOL;
1358                 rx_ring->npagepool = NUM_NXTBUFPOOL;
1359                 rx_ring->irq = pdata->irqs[i];
1360                 buf_pool->rx_skb = devm_kcalloc(dev, buf_pool->slots,
1361                                                 sizeof(struct sk_buff *),
1362                                                 GFP_KERNEL);
1363                 if (!buf_pool->rx_skb) {
1364                         ret = -ENOMEM;
1365                         goto err;
1366                 }
1367
1368                 buf_pool->dst_ring_num = xgene_enet_dst_ring_num(buf_pool);
1369                 rx_ring->buf_pool = buf_pool;
1370                 pdata->rx_ring[i] = rx_ring;
1371
1372                 if ((pdata->enet_id == XGENE_ENET1 &&  pdata->rxq_cnt > 4) ||
1373                     (pdata->enet_id == XGENE_ENET2 &&  pdata->rxq_cnt > 16)) {
1374                         break;
1375                 }
1376
1377                 /* allocate next buffer pool for jumbo packets */
1378                 owner = xgene_derive_ring_owner(pdata);
1379                 ring_id = xgene_enet_get_ring_id(owner, bp_bufnum++);
1380                 page_pool = xgene_enet_create_desc_ring(ndev, ring_num++,
1381                                                         RING_CFGSIZE_16KB,
1382                                                         ring_id);
1383                 if (!page_pool) {
1384                         ret = -ENOMEM;
1385                         goto err;
1386                 }
1387
1388                 slots = page_pool->slots;
1389                 page_pool->frag_page = devm_kcalloc(dev, slots,
1390                                                     sizeof(struct page *),
1391                                                     GFP_KERNEL);
1392                 if (!page_pool->frag_page) {
1393                         ret = -ENOMEM;
1394                         goto err;
1395                 }
1396
1397                 page_pool->frag_dma_addr = devm_kcalloc(dev, slots,
1398                                                         sizeof(dma_addr_t),
1399                                                         GFP_KERNEL);
1400                 if (!page_pool->frag_dma_addr) {
1401                         ret = -ENOMEM;
1402                         goto err;
1403                 }
1404
1405                 page_pool->dst_ring_num = xgene_enet_dst_ring_num(page_pool);
1406                 rx_ring->page_pool = page_pool;
1407         }
1408
1409         for (i = 0; i < pdata->txq_cnt; i++) {
1410                 /* allocate tx descriptor ring */
1411                 owner = xgene_derive_ring_owner(pdata);
1412                 ring_id = xgene_enet_get_ring_id(owner, eth_bufnum++);
1413                 tx_ring = xgene_enet_create_desc_ring(ndev, ring_num++,
1414                                                       RING_CFGSIZE_16KB,
1415                                                       ring_id);
1416                 if (!tx_ring) {
1417                         ret = -ENOMEM;
1418                         goto err;
1419                 }
1420
1421                 size = (tx_ring->slots / 2) * sizeof(__le64) * MAX_EXP_BUFFS;
1422                 exp_bufs = dmam_alloc_coherent(dev, size, &dma_exp_bufs,
1423                                                GFP_KERNEL | __GFP_ZERO);
1424                 if (!exp_bufs) {
1425                         ret = -ENOMEM;
1426                         goto err;
1427                 }
1428                 tx_ring->exp_bufs = exp_bufs;
1429
1430                 pdata->tx_ring[i] = tx_ring;
1431
1432                 if (!pdata->cq_cnt) {
1433                         cp_ring = pdata->rx_ring[i];
1434                 } else {
1435                         /* allocate tx completion descriptor ring */
1436                         ring_id = xgene_enet_get_ring_id(RING_OWNER_CPU,
1437                                                          cpu_bufnum++);
1438                         cp_ring = xgene_enet_create_desc_ring(ndev, ring_num++,
1439                                                               RING_CFGSIZE_16KB,
1440                                                               ring_id);
1441                         if (!cp_ring) {
1442                                 ret = -ENOMEM;
1443                                 goto err;
1444                         }
1445
1446                         cp_ring->irq = pdata->irqs[pdata->rxq_cnt + i];
1447                         cp_ring->index = i;
1448                 }
1449
1450                 cp_ring->cp_skb = devm_kcalloc(dev, tx_ring->slots,
1451                                                sizeof(struct sk_buff *),
1452                                                GFP_KERNEL);
1453                 if (!cp_ring->cp_skb) {
1454                         ret = -ENOMEM;
1455                         goto err;
1456                 }
1457
1458                 size = sizeof(dma_addr_t) * MAX_SKB_FRAGS;
1459                 cp_ring->frag_dma_addr = devm_kcalloc(dev, tx_ring->slots,
1460                                                       size, GFP_KERNEL);
1461                 if (!cp_ring->frag_dma_addr) {
1462                         devm_kfree(dev, cp_ring->cp_skb);
1463                         ret = -ENOMEM;
1464                         goto err;
1465                 }
1466
1467                 tx_ring->cp_ring = cp_ring;
1468                 tx_ring->dst_ring_num = xgene_enet_dst_ring_num(cp_ring);
1469         }
1470
1471         if (pdata->ring_ops->coalesce)
1472                 pdata->ring_ops->coalesce(pdata->tx_ring[0]);
1473         pdata->tx_qcnt_hi = pdata->tx_ring[0]->slots - 128;
1474
1475         return 0;
1476
1477 err:
1478         xgene_enet_free_desc_rings(pdata);
1479         return ret;
1480 }
1481
1482 static void xgene_enet_get_stats64(
1483                         struct net_device *ndev,
1484                         struct rtnl_link_stats64 *stats)
1485 {
1486         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1487         struct xgene_enet_desc_ring *ring;
1488         int i;
1489
1490         for (i = 0; i < pdata->txq_cnt; i++) {
1491                 ring = pdata->tx_ring[i];
1492                 if (ring) {
1493                         stats->tx_packets += ring->tx_packets;
1494                         stats->tx_bytes += ring->tx_bytes;
1495                         stats->tx_dropped += ring->tx_dropped;
1496                         stats->tx_errors += ring->tx_errors;
1497                 }
1498         }
1499
1500         for (i = 0; i < pdata->rxq_cnt; i++) {
1501                 ring = pdata->rx_ring[i];
1502                 if (ring) {
1503                         stats->rx_packets += ring->rx_packets;
1504                         stats->rx_bytes += ring->rx_bytes;
1505                         stats->rx_dropped += ring->rx_dropped;
1506                         stats->rx_errors += ring->rx_errors +
1507                                 ring->rx_length_errors +
1508                                 ring->rx_crc_errors +
1509                                 ring->rx_frame_errors +
1510                                 ring->rx_fifo_errors;
1511                         stats->rx_length_errors += ring->rx_length_errors;
1512                         stats->rx_crc_errors += ring->rx_crc_errors;
1513                         stats->rx_frame_errors += ring->rx_frame_errors;
1514                         stats->rx_fifo_errors += ring->rx_fifo_errors;
1515                 }
1516         }
1517 }
1518
1519 static int xgene_enet_set_mac_address(struct net_device *ndev, void *addr)
1520 {
1521         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1522         int ret;
1523
1524         ret = eth_mac_addr(ndev, addr);
1525         if (ret)
1526                 return ret;
1527         pdata->mac_ops->set_mac_addr(pdata);
1528
1529         return ret;
1530 }
1531
1532 static int xgene_change_mtu(struct net_device *ndev, int new_mtu)
1533 {
1534         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1535         int frame_size;
1536
1537         if (!netif_running(ndev))
1538                 return 0;
1539
1540         frame_size = (new_mtu > ETH_DATA_LEN) ? (new_mtu + 18) : 0x600;
1541
1542         xgene_enet_close(ndev);
1543         ndev->mtu = new_mtu;
1544         pdata->mac_ops->set_framesize(pdata, frame_size);
1545         xgene_enet_open(ndev);
1546
1547         return 0;
1548 }
1549
1550 static const struct net_device_ops xgene_ndev_ops = {
1551         .ndo_open = xgene_enet_open,
1552         .ndo_stop = xgene_enet_close,
1553         .ndo_start_xmit = xgene_enet_start_xmit,
1554         .ndo_tx_timeout = xgene_enet_timeout,
1555         .ndo_get_stats64 = xgene_enet_get_stats64,
1556         .ndo_change_mtu = xgene_change_mtu,
1557         .ndo_set_mac_address = xgene_enet_set_mac_address,
1558 };
1559
1560 #ifdef CONFIG_ACPI
1561 static void xgene_get_port_id_acpi(struct device *dev,
1562                                   struct xgene_enet_pdata *pdata)
1563 {
1564         acpi_status status;
1565         u64 temp;
1566
1567         status = acpi_evaluate_integer(ACPI_HANDLE(dev), "_SUN", NULL, &temp);
1568         if (ACPI_FAILURE(status)) {
1569                 pdata->port_id = 0;
1570         } else {
1571                 pdata->port_id = temp;
1572         }
1573
1574         return;
1575 }
1576 #endif
1577
1578 static void xgene_get_port_id_dt(struct device *dev, struct xgene_enet_pdata *pdata)
1579 {
1580         u32 id = 0;
1581
1582         of_property_read_u32(dev->of_node, "port-id", &id);
1583
1584         pdata->port_id = id & BIT(0);
1585
1586         return;
1587 }
1588
1589 static int xgene_get_tx_delay(struct xgene_enet_pdata *pdata)
1590 {
1591         struct device *dev = &pdata->pdev->dev;
1592         int delay, ret;
1593
1594         ret = of_property_read_u32(dev->of_node, "tx-delay", &delay);
1595         if (ret) {
1596                 pdata->tx_delay = 4;
1597                 return 0;
1598         }
1599
1600         if (delay < 0 || delay > 7) {
1601                 dev_err(dev, "Invalid tx-delay specified\n");
1602                 return -EINVAL;
1603         }
1604
1605         pdata->tx_delay = delay;
1606
1607         return 0;
1608 }
1609
1610 static int xgene_get_rx_delay(struct xgene_enet_pdata *pdata)
1611 {
1612         struct device *dev = &pdata->pdev->dev;
1613         int delay, ret;
1614
1615         ret = of_property_read_u32(dev->of_node, "rx-delay", &delay);
1616         if (ret) {
1617                 pdata->rx_delay = 2;
1618                 return 0;
1619         }
1620
1621         if (delay < 0 || delay > 7) {
1622                 dev_err(dev, "Invalid rx-delay specified\n");
1623                 return -EINVAL;
1624         }
1625
1626         pdata->rx_delay = delay;
1627
1628         return 0;
1629 }
1630
1631 static int xgene_enet_get_irqs(struct xgene_enet_pdata *pdata)
1632 {
1633         struct platform_device *pdev = pdata->pdev;
1634         struct device *dev = &pdev->dev;
1635         int i, ret, max_irqs;
1636
1637         if (phy_interface_mode_is_rgmii(pdata->phy_mode))
1638                 max_irqs = 1;
1639         else if (pdata->phy_mode == PHY_INTERFACE_MODE_SGMII)
1640                 max_irqs = 2;
1641         else
1642                 max_irqs = XGENE_MAX_ENET_IRQ;
1643
1644         for (i = 0; i < max_irqs; i++) {
1645                 ret = platform_get_irq(pdev, i);
1646                 if (ret <= 0) {
1647                         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
1648                                 max_irqs = i;
1649                                 pdata->rxq_cnt = max_irqs / 2;
1650                                 pdata->txq_cnt = max_irqs / 2;
1651                                 pdata->cq_cnt = max_irqs / 2;
1652                                 break;
1653                         }
1654                         dev_err(dev, "Unable to get ENET IRQ\n");
1655                         ret = ret ? : -ENXIO;
1656                         return ret;
1657                 }
1658                 pdata->irqs[i] = ret;
1659         }
1660
1661         return 0;
1662 }
1663
1664 static int xgene_enet_check_phy_handle(struct xgene_enet_pdata *pdata)
1665 {
1666         int ret;
1667
1668         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII)
1669                 return 0;
1670
1671         if (!IS_ENABLED(CONFIG_MDIO_XGENE))
1672                 return 0;
1673
1674         ret = xgene_enet_phy_connect(pdata->ndev);
1675         if (!ret)
1676                 pdata->mdio_driver = true;
1677
1678         return 0;
1679 }
1680
1681 static void xgene_enet_gpiod_get(struct xgene_enet_pdata *pdata)
1682 {
1683         struct device *dev = &pdata->pdev->dev;
1684
1685         pdata->sfp_gpio_en = false;
1686         if (pdata->phy_mode != PHY_INTERFACE_MODE_XGMII ||
1687             (!device_property_present(dev, "sfp-gpios") &&
1688              !device_property_present(dev, "rxlos-gpios")))
1689                 return;
1690
1691         pdata->sfp_gpio_en = true;
1692         pdata->sfp_rdy = gpiod_get(dev, "rxlos", GPIOD_IN);
1693         if (IS_ERR(pdata->sfp_rdy))
1694                 pdata->sfp_rdy = gpiod_get(dev, "sfp", GPIOD_IN);
1695 }
1696
1697 static int xgene_enet_get_resources(struct xgene_enet_pdata *pdata)
1698 {
1699         struct platform_device *pdev;
1700         struct net_device *ndev;
1701         struct device *dev;
1702         struct resource *res;
1703         void __iomem *base_addr;
1704         u32 offset;
1705         int ret = 0;
1706
1707         pdev = pdata->pdev;
1708         dev = &pdev->dev;
1709         ndev = pdata->ndev;
1710
1711         res = platform_get_resource(pdev, IORESOURCE_MEM, RES_ENET_CSR);
1712         if (!res) {
1713                 dev_err(dev, "Resource enet_csr not defined\n");
1714                 return -ENODEV;
1715         }
1716         pdata->base_addr = devm_ioremap(dev, res->start, resource_size(res));
1717         if (!pdata->base_addr) {
1718                 dev_err(dev, "Unable to retrieve ENET Port CSR region\n");
1719                 return -ENOMEM;
1720         }
1721
1722         res = platform_get_resource(pdev, IORESOURCE_MEM, RES_RING_CSR);
1723         if (!res) {
1724                 dev_err(dev, "Resource ring_csr not defined\n");
1725                 return -ENODEV;
1726         }
1727         pdata->ring_csr_addr = devm_ioremap(dev, res->start,
1728                                                         resource_size(res));
1729         if (!pdata->ring_csr_addr) {
1730                 dev_err(dev, "Unable to retrieve ENET Ring CSR region\n");
1731                 return -ENOMEM;
1732         }
1733
1734         res = platform_get_resource(pdev, IORESOURCE_MEM, RES_RING_CMD);
1735         if (!res) {
1736                 dev_err(dev, "Resource ring_cmd not defined\n");
1737                 return -ENODEV;
1738         }
1739         pdata->ring_cmd_addr = devm_ioremap(dev, res->start,
1740                                                         resource_size(res));
1741         if (!pdata->ring_cmd_addr) {
1742                 dev_err(dev, "Unable to retrieve ENET Ring command region\n");
1743                 return -ENOMEM;
1744         }
1745
1746         if (dev->of_node)
1747                 xgene_get_port_id_dt(dev, pdata);
1748 #ifdef CONFIG_ACPI
1749         else
1750                 xgene_get_port_id_acpi(dev, pdata);
1751 #endif
1752
1753         if (!device_get_mac_address(dev, ndev->dev_addr, ETH_ALEN))
1754                 eth_hw_addr_random(ndev);
1755
1756         memcpy(ndev->perm_addr, ndev->dev_addr, ndev->addr_len);
1757
1758         pdata->phy_mode = device_get_phy_mode(dev);
1759         if (pdata->phy_mode < 0) {
1760                 dev_err(dev, "Unable to get phy-connection-type\n");
1761                 return pdata->phy_mode;
1762         }
1763         if (!phy_interface_mode_is_rgmii(pdata->phy_mode) &&
1764             pdata->phy_mode != PHY_INTERFACE_MODE_SGMII &&
1765             pdata->phy_mode != PHY_INTERFACE_MODE_XGMII) {
1766                 dev_err(dev, "Incorrect phy-connection-type specified\n");
1767                 return -ENODEV;
1768         }
1769
1770         ret = xgene_get_tx_delay(pdata);
1771         if (ret)
1772                 return ret;
1773
1774         ret = xgene_get_rx_delay(pdata);
1775         if (ret)
1776                 return ret;
1777
1778         ret = xgene_enet_get_irqs(pdata);
1779         if (ret)
1780                 return ret;
1781
1782         ret = xgene_enet_check_phy_handle(pdata);
1783         if (ret)
1784                 return ret;
1785
1786         xgene_enet_gpiod_get(pdata);
1787
1788         pdata->clk = devm_clk_get(&pdev->dev, NULL);
1789         if (IS_ERR(pdata->clk)) {
1790                 if (pdata->phy_mode != PHY_INTERFACE_MODE_SGMII) {
1791                         /* Abort if the clock is defined but couldn't be
1792                          * retrived. Always abort if the clock is missing on
1793                          * DT system as the driver can't cope with this case.
1794                          */
1795                         if (PTR_ERR(pdata->clk) != -ENOENT || dev->of_node)
1796                                 return PTR_ERR(pdata->clk);
1797                         /* Firmware may have set up the clock already. */
1798                         dev_info(dev, "clocks have been setup already\n");
1799                 }
1800         }
1801
1802         if (pdata->phy_mode != PHY_INTERFACE_MODE_XGMII)
1803                 base_addr = pdata->base_addr - (pdata->port_id * MAC_OFFSET);
1804         else
1805                 base_addr = pdata->base_addr;
1806         pdata->eth_csr_addr = base_addr + BLOCK_ETH_CSR_OFFSET;
1807         pdata->cle.base = base_addr + BLOCK_ETH_CLE_CSR_OFFSET;
1808         pdata->eth_ring_if_addr = base_addr + BLOCK_ETH_RING_IF_OFFSET;
1809         pdata->eth_diag_csr_addr = base_addr + BLOCK_ETH_DIAG_CSR_OFFSET;
1810         if (phy_interface_mode_is_rgmii(pdata->phy_mode) ||
1811             pdata->phy_mode == PHY_INTERFACE_MODE_SGMII) {
1812                 pdata->mcx_mac_addr = pdata->base_addr + BLOCK_ETH_MAC_OFFSET;
1813                 pdata->mcx_stats_addr =
1814                         pdata->base_addr + BLOCK_ETH_STATS_OFFSET;
1815                 offset = (pdata->enet_id == XGENE_ENET1) ?
1816                           BLOCK_ETH_MAC_CSR_OFFSET :
1817                           X2_BLOCK_ETH_MAC_CSR_OFFSET;
1818                 pdata->mcx_mac_csr_addr = base_addr + offset;
1819         } else {
1820                 pdata->mcx_mac_addr = base_addr + BLOCK_AXG_MAC_OFFSET;
1821                 pdata->mcx_stats_addr = base_addr + BLOCK_AXG_STATS_OFFSET;
1822                 pdata->mcx_mac_csr_addr = base_addr + BLOCK_AXG_MAC_CSR_OFFSET;
1823                 pdata->pcs_addr = base_addr + BLOCK_PCS_OFFSET;
1824         }
1825         pdata->rx_buff_cnt = NUM_PKT_BUF;
1826
1827         return 0;
1828 }
1829
1830 static int xgene_enet_init_hw(struct xgene_enet_pdata *pdata)
1831 {
1832         struct xgene_enet_cle *enet_cle = &pdata->cle;
1833         struct xgene_enet_desc_ring *page_pool;
1834         struct net_device *ndev = pdata->ndev;
1835         struct xgene_enet_desc_ring *buf_pool;
1836         u16 dst_ring_num, ring_id;
1837         int i, ret;
1838         u32 count;
1839
1840         ret = pdata->port_ops->reset(pdata);
1841         if (ret)
1842                 return ret;
1843
1844         ret = xgene_enet_create_desc_rings(ndev);
1845         if (ret) {
1846                 netdev_err(ndev, "Error in ring configuration\n");
1847                 return ret;
1848         }
1849
1850         /* setup buffer pool */
1851         for (i = 0; i < pdata->rxq_cnt; i++) {
1852                 buf_pool = pdata->rx_ring[i]->buf_pool;
1853                 xgene_enet_init_bufpool(buf_pool);
1854                 page_pool = pdata->rx_ring[i]->page_pool;
1855                 xgene_enet_init_bufpool(page_pool);
1856
1857                 count = pdata->rx_buff_cnt;
1858                 ret = xgene_enet_refill_bufpool(buf_pool, count);
1859                 if (ret)
1860                         goto err;
1861
1862                 ret = xgene_enet_refill_pagepool(page_pool, count);
1863                 if (ret)
1864                         goto err;
1865
1866         }
1867
1868         dst_ring_num = xgene_enet_dst_ring_num(pdata->rx_ring[0]);
1869         buf_pool = pdata->rx_ring[0]->buf_pool;
1870         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
1871                 /* Initialize and Enable  PreClassifier Tree */
1872                 enet_cle->max_nodes = 512;
1873                 enet_cle->max_dbptrs = 1024;
1874                 enet_cle->parsers = 3;
1875                 enet_cle->active_parser = PARSER_ALL;
1876                 enet_cle->ptree.start_node = 0;
1877                 enet_cle->ptree.start_dbptr = 0;
1878                 enet_cle->jump_bytes = 8;
1879                 ret = pdata->cle_ops->cle_init(pdata);
1880                 if (ret) {
1881                         netdev_err(ndev, "Preclass Tree init error\n");
1882                         goto err;
1883                 }
1884
1885         } else {
1886                 dst_ring_num = xgene_enet_dst_ring_num(pdata->rx_ring[0]);
1887                 buf_pool = pdata->rx_ring[0]->buf_pool;
1888                 page_pool = pdata->rx_ring[0]->page_pool;
1889                 ring_id = (page_pool) ? page_pool->id : 0;
1890                 pdata->port_ops->cle_bypass(pdata, dst_ring_num,
1891                                             buf_pool->id, ring_id);
1892         }
1893
1894         ndev->max_mtu = XGENE_ENET_MAX_MTU;
1895         pdata->phy_speed = SPEED_UNKNOWN;
1896         pdata->mac_ops->init(pdata);
1897
1898         return ret;
1899
1900 err:
1901         xgene_enet_delete_desc_rings(pdata);
1902         return ret;
1903 }
1904
1905 static void xgene_enet_setup_ops(struct xgene_enet_pdata *pdata)
1906 {
1907         switch (pdata->phy_mode) {
1908         case PHY_INTERFACE_MODE_RGMII:
1909         case PHY_INTERFACE_MODE_RGMII_ID:
1910         case PHY_INTERFACE_MODE_RGMII_RXID:
1911         case PHY_INTERFACE_MODE_RGMII_TXID:
1912                 pdata->mac_ops = &xgene_gmac_ops;
1913                 pdata->port_ops = &xgene_gport_ops;
1914                 pdata->rm = RM3;
1915                 pdata->rxq_cnt = 1;
1916                 pdata->txq_cnt = 1;
1917                 pdata->cq_cnt = 0;
1918                 break;
1919         case PHY_INTERFACE_MODE_SGMII:
1920                 pdata->mac_ops = &xgene_sgmac_ops;
1921                 pdata->port_ops = &xgene_sgport_ops;
1922                 pdata->rm = RM1;
1923                 pdata->rxq_cnt = 1;
1924                 pdata->txq_cnt = 1;
1925                 pdata->cq_cnt = 1;
1926                 break;
1927         default:
1928                 pdata->mac_ops = &xgene_xgmac_ops;
1929                 pdata->port_ops = &xgene_xgport_ops;
1930                 pdata->cle_ops = &xgene_cle3in_ops;
1931                 pdata->rm = RM0;
1932                 if (!pdata->rxq_cnt) {
1933                         pdata->rxq_cnt = XGENE_NUM_RX_RING;
1934                         pdata->txq_cnt = XGENE_NUM_TX_RING;
1935                         pdata->cq_cnt = XGENE_NUM_TXC_RING;
1936                 }
1937                 break;
1938         }
1939
1940         if (pdata->enet_id == XGENE_ENET1) {
1941                 switch (pdata->port_id) {
1942                 case 0:
1943                         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
1944                                 pdata->cpu_bufnum = X2_START_CPU_BUFNUM_0;
1945                                 pdata->eth_bufnum = X2_START_ETH_BUFNUM_0;
1946                                 pdata->bp_bufnum = X2_START_BP_BUFNUM_0;
1947                                 pdata->ring_num = START_RING_NUM_0;
1948                         } else {
1949                                 pdata->cpu_bufnum = START_CPU_BUFNUM_0;
1950                                 pdata->eth_bufnum = START_ETH_BUFNUM_0;
1951                                 pdata->bp_bufnum = START_BP_BUFNUM_0;
1952                                 pdata->ring_num = START_RING_NUM_0;
1953                         }
1954                         break;
1955                 case 1:
1956                         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
1957                                 pdata->cpu_bufnum = XG_START_CPU_BUFNUM_1;
1958                                 pdata->eth_bufnum = XG_START_ETH_BUFNUM_1;
1959                                 pdata->bp_bufnum = XG_START_BP_BUFNUM_1;
1960                                 pdata->ring_num = XG_START_RING_NUM_1;
1961                         } else {
1962                                 pdata->cpu_bufnum = START_CPU_BUFNUM_1;
1963                                 pdata->eth_bufnum = START_ETH_BUFNUM_1;
1964                                 pdata->bp_bufnum = START_BP_BUFNUM_1;
1965                                 pdata->ring_num = START_RING_NUM_1;
1966                         }
1967                         break;
1968                 default:
1969                         break;
1970                 }
1971                 pdata->ring_ops = &xgene_ring1_ops;
1972         } else {
1973                 switch (pdata->port_id) {
1974                 case 0:
1975                         pdata->cpu_bufnum = X2_START_CPU_BUFNUM_0;
1976                         pdata->eth_bufnum = X2_START_ETH_BUFNUM_0;
1977                         pdata->bp_bufnum = X2_START_BP_BUFNUM_0;
1978                         pdata->ring_num = X2_START_RING_NUM_0;
1979                         break;
1980                 case 1:
1981                         pdata->cpu_bufnum = X2_START_CPU_BUFNUM_1;
1982                         pdata->eth_bufnum = X2_START_ETH_BUFNUM_1;
1983                         pdata->bp_bufnum = X2_START_BP_BUFNUM_1;
1984                         pdata->ring_num = X2_START_RING_NUM_1;
1985                         break;
1986                 default:
1987                         break;
1988                 }
1989                 pdata->rm = RM0;
1990                 pdata->ring_ops = &xgene_ring2_ops;
1991         }
1992 }
1993
1994 static void xgene_enet_napi_add(struct xgene_enet_pdata *pdata)
1995 {
1996         struct napi_struct *napi;
1997         int i;
1998
1999         for (i = 0; i < pdata->rxq_cnt; i++) {
2000                 napi = &pdata->rx_ring[i]->napi;
2001                 netif_napi_add(pdata->ndev, napi, xgene_enet_napi,
2002                                NAPI_POLL_WEIGHT);
2003         }
2004
2005         for (i = 0; i < pdata->cq_cnt; i++) {
2006                 napi = &pdata->tx_ring[i]->cp_ring->napi;
2007                 netif_napi_add(pdata->ndev, napi, xgene_enet_napi,
2008                                NAPI_POLL_WEIGHT);
2009         }
2010 }
2011
2012 #ifdef CONFIG_ACPI
2013 static const struct acpi_device_id xgene_enet_acpi_match[] = {
2014         { "APMC0D05", XGENE_ENET1},
2015         { "APMC0D30", XGENE_ENET1},
2016         { "APMC0D31", XGENE_ENET1},
2017         { "APMC0D3F", XGENE_ENET1},
2018         { "APMC0D26", XGENE_ENET2},
2019         { "APMC0D25", XGENE_ENET2},
2020         { }
2021 };
2022 MODULE_DEVICE_TABLE(acpi, xgene_enet_acpi_match);
2023 #endif
2024
2025 static const struct of_device_id xgene_enet_of_match[] = {
2026         {.compatible = "apm,xgene-enet",    .data = (void *)XGENE_ENET1},
2027         {.compatible = "apm,xgene1-sgenet", .data = (void *)XGENE_ENET1},
2028         {.compatible = "apm,xgene1-xgenet", .data = (void *)XGENE_ENET1},
2029         {.compatible = "apm,xgene2-sgenet", .data = (void *)XGENE_ENET2},
2030         {.compatible = "apm,xgene2-xgenet", .data = (void *)XGENE_ENET2},
2031         {},
2032 };
2033
2034 MODULE_DEVICE_TABLE(of, xgene_enet_of_match);
2035
2036 static int xgene_enet_probe(struct platform_device *pdev)
2037 {
2038         struct net_device *ndev;
2039         struct xgene_enet_pdata *pdata;
2040         struct device *dev = &pdev->dev;
2041         void (*link_state)(struct work_struct *);
2042         const struct of_device_id *of_id;
2043         int ret;
2044
2045         ndev = alloc_etherdev_mqs(sizeof(struct xgene_enet_pdata),
2046                                   XGENE_NUM_RX_RING, XGENE_NUM_TX_RING);
2047         if (!ndev)
2048                 return -ENOMEM;
2049
2050         pdata = netdev_priv(ndev);
2051
2052         pdata->pdev = pdev;
2053         pdata->ndev = ndev;
2054         SET_NETDEV_DEV(ndev, dev);
2055         platform_set_drvdata(pdev, pdata);
2056         ndev->netdev_ops = &xgene_ndev_ops;
2057         xgene_enet_set_ethtool_ops(ndev);
2058         ndev->features |= NETIF_F_IP_CSUM |
2059                           NETIF_F_GSO |
2060                           NETIF_F_GRO |
2061                           NETIF_F_SG;
2062
2063         of_id = of_match_device(xgene_enet_of_match, &pdev->dev);
2064         if (of_id) {
2065                 pdata->enet_id = (enum xgene_enet_id)of_id->data;
2066         }
2067 #ifdef CONFIG_ACPI
2068         else {
2069                 const struct acpi_device_id *acpi_id;
2070
2071                 acpi_id = acpi_match_device(xgene_enet_acpi_match, &pdev->dev);
2072                 if (acpi_id)
2073                         pdata->enet_id = (enum xgene_enet_id) acpi_id->driver_data;
2074         }
2075 #endif
2076         if (!pdata->enet_id) {
2077                 ret = -ENODEV;
2078                 goto err;
2079         }
2080
2081         ret = xgene_enet_get_resources(pdata);
2082         if (ret)
2083                 goto err;
2084
2085         xgene_enet_setup_ops(pdata);
2086         spin_lock_init(&pdata->mac_lock);
2087
2088         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
2089                 ndev->features |= NETIF_F_TSO | NETIF_F_RXCSUM;
2090                 spin_lock_init(&pdata->mss_lock);
2091         }
2092         ndev->hw_features = ndev->features;
2093
2094         ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(64));
2095         if (ret) {
2096                 netdev_err(ndev, "No usable DMA configuration\n");
2097                 goto err;
2098         }
2099
2100         ret = xgene_enet_init_hw(pdata);
2101         if (ret)
2102                 goto err;
2103
2104         link_state = pdata->mac_ops->link_state;
2105         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
2106                 INIT_DELAYED_WORK(&pdata->link_work, link_state);
2107         } else if (!pdata->mdio_driver) {
2108                 if (phy_interface_mode_is_rgmii(pdata->phy_mode))
2109                         ret = xgene_enet_mdio_config(pdata);
2110                 else
2111                         INIT_DELAYED_WORK(&pdata->link_work, link_state);
2112
2113                 if (ret)
2114                         goto err1;
2115         }
2116
2117         spin_lock_init(&pdata->stats_lock);
2118         ret = xgene_extd_stats_init(pdata);
2119         if (ret)
2120                 goto err2;
2121
2122         xgene_enet_napi_add(pdata);
2123         ret = register_netdev(ndev);
2124         if (ret) {
2125                 netdev_err(ndev, "Failed to register netdev\n");
2126                 goto err2;
2127         }
2128
2129         return 0;
2130
2131 err2:
2132         /*
2133          * If necessary, free_netdev() will call netif_napi_del() and undo
2134          * the effects of xgene_enet_napi_add()'s calls to netif_napi_add().
2135          */
2136
2137         if (pdata->mdio_driver)
2138                 xgene_enet_phy_disconnect(pdata);
2139         else if (phy_interface_mode_is_rgmii(pdata->phy_mode))
2140                 xgene_enet_mdio_remove(pdata);
2141 err1:
2142         xgene_enet_delete_desc_rings(pdata);
2143 err:
2144         free_netdev(ndev);
2145         return ret;
2146 }
2147
2148 static int xgene_enet_remove(struct platform_device *pdev)
2149 {
2150         struct xgene_enet_pdata *pdata;
2151         struct net_device *ndev;
2152
2153         pdata = platform_get_drvdata(pdev);
2154         ndev = pdata->ndev;
2155
2156         rtnl_lock();
2157         if (netif_running(ndev))
2158                 dev_close(ndev);
2159         rtnl_unlock();
2160
2161         if (pdata->mdio_driver)
2162                 xgene_enet_phy_disconnect(pdata);
2163         else if (phy_interface_mode_is_rgmii(pdata->phy_mode))
2164                 xgene_enet_mdio_remove(pdata);
2165
2166         unregister_netdev(ndev);
2167         xgene_enet_delete_desc_rings(pdata);
2168         pdata->port_ops->shutdown(pdata);
2169         free_netdev(ndev);
2170
2171         return 0;
2172 }
2173
2174 static void xgene_enet_shutdown(struct platform_device *pdev)
2175 {
2176         struct xgene_enet_pdata *pdata;
2177
2178         pdata = platform_get_drvdata(pdev);
2179         if (!pdata)
2180                 return;
2181
2182         if (!pdata->ndev)
2183                 return;
2184
2185         xgene_enet_remove(pdev);
2186 }
2187
2188 static struct platform_driver xgene_enet_driver = {
2189         .driver = {
2190                    .name = "xgene-enet",
2191                    .of_match_table = of_match_ptr(xgene_enet_of_match),
2192                    .acpi_match_table = ACPI_PTR(xgene_enet_acpi_match),
2193         },
2194         .probe = xgene_enet_probe,
2195         .remove = xgene_enet_remove,
2196         .shutdown = xgene_enet_shutdown,
2197 };
2198
2199 module_platform_driver(xgene_enet_driver);
2200
2201 MODULE_DESCRIPTION("APM X-Gene SoC Ethernet driver");
2202 MODULE_VERSION(XGENE_DRV_VERSION);
2203 MODULE_AUTHOR("Iyappan Subramanian <isubramanian@apm.com>");
2204 MODULE_AUTHOR("Keyur Chudgar <kchudgar@apm.com>");
2205 MODULE_LICENSE("GPL");