/* * aQuantia Corporation Network Driver * Copyright (C) 2017 aQuantia Corporation. All rights reserved * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. */ #include "atl_ring.h" #include #include #include #include #include #include #include #include "atl_trace.h" #define atl_update_ring_stat(ring, stat, delta) \ do { \ struct atl_desc_ring *_ring = (ring); \ \ u64_stats_update_begin(&_ring->syncp); \ _ring->stats.stat += (delta); \ u64_stats_update_end(&_ring->syncp); \ } while (0) static inline uint32_t fetch_tx_head(struct atl_desc_ring *ring) { #ifdef ATL_TX_HEAD_WB //XXX #else return atl_read(ring_hw(ring), ATL_TX_RING_HEAD(ring)); #endif } static int tx_full(struct atl_desc_ring *ring, int needed) { struct atl_nic *nic = ring->qvec->nic; if (likely(ring_space(ring) >= needed)) return 0; netif_stop_subqueue(ring->qvec->nic->ndev, ring->qvec->idx); atl_nic_dbg("Stopping tx queue\n"); smp_mb(); // Check if another CPU freed some space if (likely(ring_space(ring) < needed)) return -EAGAIN; netif_start_subqueue(ring->qvec->nic->ndev, ring->qvec->idx); atl_nic_dbg("Restarting tx queue in %s...\n", __func__); atl_update_ring_stat(ring, tx.tx_restart, 1); return 0; } static void atl_txbuf_free(struct atl_txbuf *txbuf, struct device *dev, uint32_t idx) { if (txbuf->skb) { if (dma_unmap_len(txbuf, len)) { dma_unmap_single(dev, dma_unmap_addr(txbuf, daddr), dma_unmap_len(txbuf, len), DMA_TO_DEVICE); trace_atl_dma_unmap_head(-1, idx, dma_unmap_addr(txbuf, daddr), dma_unmap_len(txbuf, len), txbuf->skb); } dev_kfree_skb_any(txbuf->skb); } else if (dma_unmap_len(txbuf, len)) { dma_unmap_page(dev, dma_unmap_addr(txbuf, daddr), dma_unmap_len(txbuf, len), DMA_TO_DEVICE); trace_atl_dma_unmap_frag(-1, idx, dma_unmap_addr(txbuf, daddr), dma_unmap_len(txbuf, len), txbuf->skb); } txbuf->last = -1; txbuf->skb = NULL; dma_unmap_len_set(txbuf, len, 0); } static inline struct netdev_queue *atl_txq(struct atl_desc_ring *ring) { return netdev_get_tx_queue(ring->qvec->nic->ndev, ring->qvec->idx); } static unsigned int atl_tx_free_low = MAX_SKB_FRAGS + 4; module_param_named(tx_free_low, atl_tx_free_low, uint, 0644); static unsigned int atl_tx_free_high = MAX_SKB_FRAGS * 3; module_param_named(tx_free_high, atl_tx_free_high, uint, 0644); static inline int skb_xmit_more(struct sk_buff *skb) { return skb->xmit_more; } static netdev_tx_t atl_map_xmit_skb(struct sk_buff *skb, struct atl_desc_ring *ring, struct atl_txbuf *first_buf) { int idx = ring->tail; struct device *dev = ring->qvec->dev; struct atl_tx_desc *desc = &ring->desc.tx; struct skb_frag_struct *frag; /* Header's DMA mapping must be stored in the txbuf that has * ->skb set, even if it corresponds to the context * descriptor and not the first data descriptor */ struct atl_txbuf *txbuf = first_buf; unsigned int len = skb_headlen(skb); unsigned int frags = skb_shinfo(skb)->nr_frags; dma_addr_t daddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE); trace_atl_dma_map_head(-1, idx, daddr, len, skb, skb->data); for (frag = &skb_shinfo(skb)->frags[0];; frag++) { if (dma_mapping_error(dev, daddr)) goto err_dma; dma_unmap_len_set(txbuf, len, len); dma_unmap_addr_set(txbuf, daddr, daddr); desc->daddr = cpu_to_le64(daddr); while (len > ATL_DATA_PER_TXD) { desc->len = cpu_to_le16(ATL_DATA_PER_TXD); WRITE_ONCE(ring->hw.descs[idx].tx, *desc); bump_ptr(idx, ring, 1); daddr += ATL_DATA_PER_TXD; len -= ATL_DATA_PER_TXD; desc->daddr = cpu_to_le64(daddr); } desc->len = cpu_to_le16(len); if (!frags) break; WRITE_ONCE(ring->hw.descs[idx].tx, *desc); bump_ptr(idx, ring, 1); txbuf = &ring->txbufs[idx]; len = skb_frag_size(frag); daddr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE); trace_atl_dma_map_frag(frag - &skb_shinfo(skb)->frags[0], idx, daddr, len, skb, skb_frag_address(frag)); frags--; } //Last descriptor desc->eop = 1; #if defined(ATL_TX_DESC_WB) || defined(ATL_TX_HEAD_WB) desc->cmd |= tx_desc_cmd_wb; #endif WRITE_ONCE(ring->hw.descs[idx].tx, *desc); first_buf->last = idx; bump_ptr(idx, ring, 1); ring->txbufs[idx].last = -1; ring->tail = idx; /* Stop queue if no space for another packet */ tx_full(ring, atl_tx_free_low); /* Delay bumping the HW tail if another packet is pending and * there's space for it. */ if (skb_xmit_more(skb) && !netif_xmit_stopped(atl_txq(ring))) return NETDEV_TX_OK; wmb(); atl_write(ring_hw(ring), ATL_TX_RING_TAIL(ring), ring->tail); return NETDEV_TX_OK; err_dma: dev_err(dev, "atl_map_skb failed\n"); for (;;) { atl_txbuf_free(txbuf, dev, idx); if (txbuf == first_buf) break; bump_ptr(idx, ring, -1); txbuf = &ring->txbufs[idx]; } ring->tail = idx; atl_update_ring_stat(ring, tx.dma_map_failed, 1); return -EFAULT; } static uint32_t atl_insert_context(struct atl_txbuf *txbuf, struct atl_desc_ring *ring, unsigned int *len) { struct sk_buff *skb = txbuf->skb; struct atl_tx_ctx *ctx; unsigned int hdr_len; uint32_t tx_cmd = 0; int mss; DECLARE_SCRATCH_DESC(scratch); ctx = &DESC_PTR(ring, ring->tail, scratch)->ctx; memset(ctx, 0, sizeof(*ctx)); txbuf->bytes = *len; txbuf->packets = 1; mss = skb_shinfo(skb)->gso_size; if (mss && (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { tx_cmd |= tx_desc_cmd_lso | tx_desc_cmd_l4cs; ctx->mss_len = mss; ctx->cmd = ctx_cmd_tcp; ctx->l2_len = skb_network_offset(skb); if (skb_is_gso_v6(skb)) ctx->cmd |= ctx_cmd_ipv6; ctx->l3_len = skb_transport_offset(skb) - ctx->l2_len; ctx->l4_len = tcp_hdrlen(skb); hdr_len = ctx->l2_len + ctx->l3_len + ctx->l4_len; *len -= hdr_len; txbuf->packets = skb_shinfo(skb)->gso_segs; txbuf->bytes += (txbuf->packets - 1) * hdr_len; } if (skb_vlan_tag_present(skb)) { tx_cmd |= tx_desc_cmd_vlan; ctx->vlan_tag = skb_vlan_tag_get(skb); } if (tx_cmd) { ctx->type = tx_desc_type_context; ctx->idx = 0; COMMIT_DESC(ring, ring->tail, scratch); bump_tail(ring, 1); } return tx_cmd; } netdev_tx_t atl_start_xmit(struct sk_buff *skb, struct net_device *ndev) { struct atl_nic *nic = netdev_priv(ndev); struct atl_desc_ring *ring = &nic->qvecs[skb->queue_mapping].tx; unsigned int len = skb->len; struct atl_tx_desc *desc; struct atl_txbuf *txbuf; uint32_t cmd_from_ctx; if (tx_full(ring, skb_shinfo(skb)->nr_frags + 4)) { atl_update_ring_stat(ring, tx.tx_busy, 1); return NETDEV_TX_BUSY; } txbuf = &ring->txbufs[ring->tail]; txbuf->skb = skb; cmd_from_ctx = atl_insert_context(txbuf, ring, &len); /* use ring->desc unconditionally as it will serve as a * template for all descriptors */ desc = &ring->desc.tx; memset(desc, 0, sizeof(*desc)); desc->cmd = cmd_from_ctx; desc->cmd |= tx_desc_cmd_fcs; desc->ct_en = !!cmd_from_ctx; desc->type = tx_desc_type_desc; desc->pay_len = len; if (skb->ip_summed == CHECKSUM_PARTIAL) { uint8_t l4_proto = 0; switch (skb->protocol) { case htons(ETH_P_IP): desc->cmd |= tx_desc_cmd_ipv4cs; l4_proto = ip_hdr(skb)->protocol; break; case htons(ETH_P_IPV6): l4_proto = ipv6_hdr(skb)->nexthdr; break; } switch (l4_proto) { case IPPROTO_TCP: case IPPROTO_UDP: desc->cmd |= tx_desc_cmd_l4cs; break; } } return atl_map_xmit_skb(skb, ring, txbuf); } static unsigned int atl_tx_clean_budget = 256; module_param_named(tx_clean_budget, atl_tx_clean_budget, uint, 0644); // Returns true if all work done static bool atl_clean_tx(struct atl_desc_ring *ring) { struct atl_nic *nic = ring->qvec->nic; struct device *dev = ring->qvec->dev; uint32_t first = READ_ONCE(ring->head); #ifndef ATL_TX_DESC_WB uint32_t done = atl_get_tx_head(ring); #endif uint32_t budget = atl_tx_clean_budget; unsigned int bytes = 0, packets = 0; struct atl_tx_desc *last_desc; atl_nic_dbg("descs in ring: %d\n", ring_occupied(ring)); do { struct atl_txbuf *txbuf = &ring->txbufs[first]; struct sk_buff *skb = txbuf->skb; uint32_t last = txbuf->last; if (last == -1) break; #ifdef ATL_TX_DESC_WB last_desc = &ring->hw.descs[last].tx; if (!last_desc->dd) break; #else if ((first <= last && done >= first && done <= last) || ((first > last) && (done >= first || done <= last))) break; #endif bump_ptr(last, ring, 1); napi_consume_skb(txbuf->skb, budget); trace_atl_dma_unmap_head(-1, first, dma_unmap_addr(txbuf, daddr), dma_unmap_len(txbuf, len), skb); txbuf->skb = NULL; txbuf->last = -1; dma_unmap_single(dev, dma_unmap_addr(txbuf, daddr), dma_unmap_len(txbuf, len), DMA_TO_DEVICE); dma_unmap_len_set(txbuf, len, 0); bytes += txbuf->bytes; packets += txbuf->packets; for (bump_ptr(first, ring, 1); first != last; bump_ptr(first, ring, 1)) { txbuf = &ring->txbufs[first]; if (dma_unmap_len(txbuf, len)) { dma_unmap_page(dev, dma_unmap_addr(txbuf, daddr), dma_unmap_len(txbuf, len), DMA_TO_DEVICE); trace_atl_dma_unmap_frag(-1, first, dma_unmap_addr(txbuf, daddr), dma_unmap_len(txbuf, len), skb); dma_unmap_len_set(txbuf, len, 0); } } } while (--budget); u64_stats_update_begin(&ring->syncp); ring->stats.tx.bytes += bytes; ring->stats.tx.packets += packets; u64_stats_update_end(&ring->syncp); WRITE_ONCE(ring->head, first); if (ring_space(ring) > atl_tx_free_high) { struct net_device *ndev = nic->ndev; smp_mb(); if (__netif_subqueue_stopped(ndev, ring->qvec->idx) && test_bit(ATL_ST_UP, &nic->state)) { atl_nic_dbg("restarting tx queue\n"); netif_wake_subqueue(ndev, ring->qvec->idx); atl_update_ring_stat(ring, tx.tx_restart, 1); } } return !!budget; } static bool atl_rx_checksum(struct sk_buff *skb, struct atl_rx_desc_wb *desc, struct atl_desc_ring *ring) { struct atl_nic *nic = ring->qvec->nic; struct net_device *ndev = nic->ndev; int csum_ok = 1, recheck = 0; skb_checksum_none_assert(skb); if (desc->rx_stat & atl_rx_stat_mac_err) { atl_update_ring_stat(ring, rx.mac_err, 1); atl_nic_dbg("rx MAC err: rx_stat %d pkt_type %d len %d\n", desc->rx_stat, desc->pkt_type, desc->pkt_len); goto drop; } if (!(ndev->features & NETIF_F_RXCSUM)) return true; switch (desc->pkt_type & atl_rx_pkt_type_l3_msk) { case atl_rx_pkt_type_ipv4: csum_ok &= !(desc->rx_stat & atl_rx_stat_ipv4_err); /* Fallthrough */ case atl_rx_pkt_type_ipv6: break; default: return true; } switch (desc->pkt_type & atl_rx_pkt_type_l4_msk) { case atl_rx_pkt_type_tcp: case atl_rx_pkt_type_udp: recheck = desc->pkt_len <= 60; csum_ok &= !(desc->rx_stat & atl_rx_stat_l4_err); break; default: return true; } if (csum_ok) { skb->ip_summed = CHECKSUM_UNNECESSARY; return true; } else if (recheck) return true; atl_update_ring_stat(ring, rx.csum_err, 1); atl_nic_dbg("bad rx checksum: rx_stat %d pkt_type %d len %d\n", desc->rx_stat, desc->pkt_type, desc->pkt_len); if (ndev->features & NETIF_F_RXALL) return true; drop: dev_kfree_skb_any(skb); return false; } static void atl_rx_hash(struct sk_buff *skb, struct atl_rx_desc_wb *desc, struct net_device *ndev) { uint8_t rss_type = desc->rss_type; if (!(ndev->features & NETIF_F_RXHASH) || rss_type < 2 || rss_type > 7) return; skb_set_hash(skb, le32_to_cpu(desc->rss_hash), (rss_type > 3 && rss_type < 8) ? PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); } static bool atl_rx_packet(struct sk_buff *skb, struct atl_rx_desc_wb *desc, struct atl_desc_ring *ring) { struct net_device *ndev = ring->qvec->nic->ndev; struct napi_struct *napi = &ring->qvec->napi; if (!atl_rx_checksum(skb, desc, ring)) return false; if (!skb_is_nonlinear(skb) && eth_skb_pad(skb)) return false; if (ndev->features & NETIF_F_HW_VLAN_CTAG_RX && desc->rx_estat & atl_rx_estat_vlan_stripped) { __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), le16_to_cpu(desc->vlan_tag)); } atl_rx_hash(skb, desc, ndev); skb_record_rx_queue(skb, ring->qvec->idx); skb->protocol = eth_type_trans(skb, ndev); if (skb->pkt_type == PACKET_MULTICAST) atl_update_ring_stat(ring, rx.multicast, 1); napi_gro_receive(napi, skb); return true; } unsigned int atl_rx_linear; module_param_named(rx_linear, atl_rx_linear, uint, 0444); /* DMA mappings of buffer pages are accounted via struct * atl_rxpage. Being mapped counts as a single additional reference * for the target page. */ static int atl_get_page(struct atl_pgref *pgref, unsigned int order, struct device *dev) { struct atl_rxpage *rxpage; struct page *page; dma_addr_t daddr; int ret = -ENOMEM; rxpage = kmalloc(sizeof(*rxpage), GFP_ATOMIC | __GFP_NOWARN); if (unlikely(!rxpage)) return ret; page = dev_alloc_pages(order); if (unlikely(!page)) goto free_rxpage; daddr = dma_map_page(dev, page, 0, PAGE_SIZE << order, DMA_FROM_DEVICE); trace_atl_dma_map_rxbuf(-1, -1, daddr, PAGE_SIZE << order, NULL, page_to_virt(page)); if (unlikely(dma_mapping_error(dev, daddr))) goto free_page; rxpage->page = page; rxpage->daddr = daddr; rxpage->order = order; rxpage->mapcount = 1; pgref->rxpage = rxpage; pgref->pg_off = 0; return 0; free_page: __free_pages(page, order); free_rxpage: kfree(rxpage); return ret; } static int atl_get_pages(struct atl_rxbuf *rxbuf, struct atl_desc_ring *ring) { int ret; struct device *dev = ring->qvec->dev; if (likely((rxbuf->head.rxpage || atl_rx_linear) && rxbuf->data.rxpage)) return 0; if (!rxbuf->head.rxpage && !atl_rx_linear) { ret = atl_get_page(&rxbuf->head, ATL_RX_HEAD_ORDER, dev); if (ret) { atl_update_ring_stat(ring, rx.alloc_head_page_failed, 1); return ret; } atl_update_ring_stat(ring, rx.alloc_head_page, 1); } if (!rxbuf->data.rxpage) { ret = atl_get_page(&rxbuf->data, ATL_RX_DATA_ORDER, dev); if (ret) { atl_update_ring_stat(ring, rx.alloc_data_page_failed, 1); return ret; } atl_update_ring_stat(ring, rx.alloc_data_page, 1); } return 0; } static inline void atl_fill_rx_desc(struct atl_desc_ring *ring, struct atl_rxbuf *rxbuf) { struct atl_rx_desc *desc; DECLARE_SCRATCH_DESC(scratch); desc = &DESC_PTR(ring, ring->tail, scratch)->rx; desc->daddr = atl_buf_daddr(&rxbuf->data) + (atl_rx_linear ? ATL_RX_HEADROOM : 0); /* Assigning haddr clears dd as bufs are cacheline-aligned * and ATL_RX_HEADROOM is even */ desc->haddr = atl_rx_linear ? 0 : atl_buf_daddr(&rxbuf->head) + ATL_RX_HEADROOM; trace_atl_fill_rx_desc(ring->tail, desc); COMMIT_DESC(ring, ring->tail, scratch); } static int atl_fill_rx(struct atl_desc_ring *ring, uint32_t count) { int ret = 0; while (count) { struct atl_rxbuf *rxbuf = &ring->rxbufs[ring->tail]; ret = atl_get_pages(rxbuf, ring); if (ret) break; atl_fill_rx_desc(ring, rxbuf); bump_tail(ring, 1); count--; } /* If tail ptr passed the next_to_recycle ptr, clamp the * latter to the former. */ if (ring->next_to_recycle < ring->head ? ring->next_to_recycle < ring->tail && ring->tail < ring->head : ring->tail > ring->next_to_recycle || ring->tail < ring->head) ring->next_to_recycle = ring->tail; wmb(); atl_write(ring_hw(ring), ATL_RX_RING_TAIL(ring), ring->tail); return ret; } static inline void atl_get_rxpage(struct atl_pgref *pgref) { pgref->rxpage->mapcount++; } static inline void __atl_free_rxpage(struct atl_rxpage *rxpage, struct device *dev) { unsigned int len = PAGE_SIZE << rxpage->order; dma_unmap_page(dev, rxpage->daddr, len, DMA_FROM_DEVICE); trace_atl_dma_unmap_rxbuf(-1, -1, rxpage->daddr, len, NULL); /* Drop the ref for dma mapping. */ __free_pages(rxpage->page, rxpage->order); kfree(rxpage); } static inline void atl_put_rxpage(struct atl_pgref *pgref, struct device *dev) { struct atl_rxpage *rxpage = pgref->rxpage; if (!rxpage) return; if (--rxpage->mapcount) return; __atl_free_rxpage(rxpage, dev); pgref->rxpage = 0; } static bool atl_recycle_or_put_page(struct atl_pgref *pgref, unsigned int buf_len, struct device *dev) { unsigned int order = pgref->rxpage->order; unsigned int size = PAGE_SIZE << order; struct page *page = pgref->rxpage->page; if (!page_is_pfmemalloc(page) && pgref->pg_off + buf_len < size) return true; atl_put_rxpage(pgref, dev); return false; } static void atl_maybe_recycle_rxbuf(struct atl_desc_ring *ring, struct atl_rxbuf *rxbuf) { int reused = 0; struct atl_pgref *head = &rxbuf->head, *data = &rxbuf->data; struct atl_rxbuf *new = &ring->rxbufs[ring->next_to_recycle]; unsigned int data_len = ATL_RX_BUF_SIZE + (atl_rx_linear ? ATL_RX_HDR_OVRHD : 0); if (!atl_rx_linear && atl_recycle_or_put_page(head, ATL_RX_HDR_SIZE + ATL_RX_HDR_OVRHD, ring->qvec->dev)) { new->head = *head; reused = 1; atl_update_ring_stat(ring, rx.reused_head_page, 1); } head->rxpage = 0; if (atl_recycle_or_put_page(data, data_len, ring->qvec->dev)) { new->data = *data; reused = 1; atl_update_ring_stat(ring, rx.reused_data_page, 1); } data->rxpage = 0; if (reused) bump_ptr(ring->next_to_recycle, ring, 1); } static unsigned int atl_data_len(struct atl_rx_desc_wb *wb) { unsigned int len = le16_to_cpu(wb->pkt_len); if (!wb->eop) return ATL_RX_BUF_SIZE; if (!wb->rsc_cnt && wb->sph) len -= wb->hdr_len; len &= ATL_RX_BUF_SIZE - 1; return len ?: ATL_RX_BUF_SIZE; } static void atl_sync_range(struct atl_desc_ring *ring, struct atl_pgref *pgref, unsigned int offt, unsigned int len) { dma_addr_t daddr = pgref->rxpage->daddr; unsigned int pg_off = pgref->pg_off + offt; dma_sync_single_range_for_cpu(ring->qvec->dev, daddr, pg_off, len, DMA_FROM_DEVICE); trace_atl_sync_rx_range(-1, daddr, pg_off, len); } static struct sk_buff *atl_init_skb(struct atl_desc_ring *ring, struct atl_rxbuf *rxbuf, struct atl_rx_desc_wb *wb) { struct sk_buff *skb; unsigned int hdr_len, alloc, tailroom, len; unsigned int data_len = atl_data_len(wb); void *hdr; struct atl_pgref *pgref; struct atl_nic *nic = ring->qvec->nic; if (atl_rx_linear) { if (!wb->eop) { atl_nic_err("Multi-frag packet in linear mode\n"); atl_update_ring_stat(ring, rx.linear_dropped, 1); return (void *)-1l; } hdr_len = len = data_len; tailroom = 0; pgref = &rxbuf->data; } else { hdr_len = wb->hdr_len; if (hdr_len == 0) { atl_nic_err("Header parse error\n"); return (void *)-1l; } /* If entire packet fits into ATL_RX_HDR_SIZE, reserve * enough space to pull the data part into skb head * and make it linear, otherwise allocate space for * hdr_len only */ len = (wb->sph ? hdr_len : 0) + data_len; if (!wb->eop || len > ATL_RX_HDR_SIZE) len = hdr_len; /* reserve space for potential __pskb_pull_tail() */ tailroom = min(ATL_RX_TAILROOM, ATL_RX_HDR_SIZE - len); pgref = &rxbuf->head; } if (atl_rx_linear || (wb->sph && (wb->eop || !wb->rsc_cnt))) atl_sync_range(ring, pgref, ATL_RX_HEADROOM, hdr_len); alloc = len + tailroom + ATL_RX_HEADROOM; alloc += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); alloc = SKB_DATA_ALIGN(alloc); hdr = atl_buf_vaddr(pgref); skb = build_skb(hdr, alloc); if (unlikely(!skb)) { atl_update_ring_stat(ring, rx.alloc_skb_failed, 1); return NULL; } if (wb->rsc_cnt && !wb->eop) { struct atl_cb *atl_cb = ATL_CB(skb); /* First frag of a multi-frag RSC packet. Either head or * data buffer, depending on whether the header was * split off by HW, might still be accessed by * RSC. Delay processing till EOP. */ if (wb->sph) { atl_cb->pgref = *pgref; atl_cb->head = true; /* Safe to sync the data buf. !wb->eop * implies the data buffer is completely filled. */ atl_sync_range(ring, &rxbuf->data, 0, ATL_RX_BUF_SIZE); } else { atl_cb->pgref = rxbuf->data; atl_cb->head = false; /* No need to sync head fragment as nothing * was DMA'd into it */ } atl_get_rxpage(&atl_cb->pgref); } pgref->pg_off += alloc; page_ref_inc(pgref->rxpage->page); if (!atl_rx_linear && !wb->sph) { atl_nic_dbg("Header not split despite non-zero hdr_len (%d)\n", hdr_len); /* Make skb head empty -- will copy the real header * from the data buffer later */ hdr_len = 0; } skb_reserve(skb, ATL_RX_HEADROOM); skb_put(skb, hdr_len); return skb; } static inline void atl_skb_put_data(struct sk_buff *skb, void *data, unsigned int len) { memcpy(skb_tail_pointer(skb), data, len); skb->tail += len; skb->len += len; } static struct sk_buff *atl_process_rx_frag(struct atl_desc_ring *ring, struct atl_rxbuf *rxbuf, struct atl_rx_desc_wb *wb) { bool first_frag = false; bool hdr_split = !!wb->sph; unsigned int hdr_len, data_len, aligned_data_len; unsigned int data_offt = 0, to_pull = 0; struct sk_buff *skb = rxbuf->skb; struct atl_cb *atl_cb; struct atl_pgref *headref = &rxbuf->head, *dataref = &rxbuf->data; struct device *dev = ring->qvec->dev; if (!skb) { /* First buffer of a packet */ skb = atl_init_skb(ring, rxbuf, wb); first_frag = true; } else rxbuf->skb = NULL; if (unlikely(!skb || skb == (void *)-1l)) return skb; hdr_len = wb->hdr_len; data_len = atl_data_len(wb); if (atl_rx_linear) { /* Linear skb mode. The entire packet was DMA'd into * the data buffer and skb has already been built * around it and dataref's pg_off has been increased * in atl_init_skb() */ atl_maybe_recycle_rxbuf(ring, rxbuf); return skb; } /* Align the start of the next buffer in the page. This also * serves as truesize increment when the paged frag is added * to skb. */ aligned_data_len = ALIGN(data_len, L1_CACHE_BYTES); if (first_frag && !hdr_split) /* Header was not split off, so skip over it * when adding the paged frag */ data_offt = hdr_len; if (!first_frag || wb->eop || !wb->rsc_cnt) { atl_sync_range(ring, dataref, 0, data_len); /* If header was not split off by HW, remember to pull * it into the skb head later. The rest of the data * buffer might need to be pulled too for small * packets, so delay the actual copying till later */ if (first_frag && !hdr_split) to_pull = hdr_len; } /* If the entire packet fits within ATL_RX_HDR_SIZE bytes, * pull it into the skb head. This handles the header not * having been split by HW case correctly too, as * skb_headlen() will be zero in that case and data_len will * hold the whole packet length. */ if (first_frag && skb_headlen(skb) + data_len <= ATL_RX_HDR_SIZE) { to_pull = data_len; /* Recycle the data buffer as we're copying the * contents to skb head. */ aligned_data_len = 0; } else { /* Add the data buffer to paged frag list, skipping * the un-split header if any -- it will be copied to * skb head later. */ skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, dataref->rxpage->page, dataref->pg_off + data_offt, data_len - data_offt, aligned_data_len); page_ref_inc(dataref->rxpage->page); } if (to_pull) atl_skb_put_data(skb, atl_buf_vaddr(dataref), to_pull); /* Update the data buf's pg_off to point to free * space. Header buf's offset was updated in atl_init_skb() * for first frag of the packet only. */ dataref->pg_off += aligned_data_len; atl_maybe_recycle_rxbuf(ring, rxbuf); if (first_frag || !wb->eop || !wb->rsc_cnt) return skb; /* The last descriptor of RSC packet is done, unmap the head * fragment. */ atl_cb = ATL_CB(skb); headref = &atl_cb->pgref; if (unlikely(!headref->rxpage)) return skb; if (likely(atl_cb->head)) { atl_sync_range(ring, headref, ATL_RX_HEADROOM, hdr_len); atl_put_rxpage(headref, dev); } else { atl_sync_range(ring, headref, 0, ATL_RX_BUF_SIZE); /* Data buf's sync being delayed implies header was * not split off by HW. Fix that now. */ atl_skb_put_data(skb, atl_buf_vaddr(headref), hdr_len); atl_put_rxpage(headref, dev); } return skb; } unsigned int atl_rx_refill_batch = 16; module_param_named(rx_refill_batch, atl_rx_refill_batch, uint, 0644); static int atl_clean_rx(struct atl_desc_ring *ring, int budget) { unsigned int packets = 0; unsigned int bytes = 0; struct sk_buff *skb; while (packets < budget) { uint32_t space = ring_space(ring); struct atl_rx_desc_wb *wb; struct atl_rxbuf *rxbuf; unsigned int len; DECLARE_SCRATCH_DESC(scratch); if (space >= atl_rx_refill_batch) atl_fill_rx(ring, space); rxbuf = &ring->rxbufs[ring->head]; wb = &DESC_PTR(ring, ring->head, scratch)->wb; FETCH_DESC(ring, ring->head, scratch); if (!wb->dd) break; DESC_RMB(); skb = atl_process_rx_frag(ring, rxbuf, wb); /* Treat allocation errors as transient and retry later */ if (!skb) { struct atl_nic *nic = ring->qvec->nic; atl_nic_err("failed to alloc skb for RX packet\n"); break; } if (skb == (void *)-1l) atl_maybe_recycle_rxbuf(ring, rxbuf); bump_head(ring, 1); if (!wb->eop) { uint32_t next = wb->rsc_cnt ? le16_to_cpu(wb->next_desp) : ring->head; /* If atl_process_rx_flags() returned any * other error this propagates the error to * the next descriptor of the packet, * preventing it from being treated as a start * of a new packet later. */ ring->rxbufs[next].skb = skb; atl_update_ring_stat(ring, rx.non_eop_descs, 1); continue; } if (skb == (void *)-1l) continue; len = skb->len; if (atl_rx_packet(skb, wb, ring)) { packets++; bytes += len; } } u64_stats_update_begin(&ring->syncp); ring->stats.rx.bytes += bytes; ring->stats.rx.packets += packets; u64_stats_update_end(&ring->syncp); return packets; } unsigned int atl_min_intr_delay = 10; module_param_named(min_intr_delay, atl_min_intr_delay, uint, 0644); static void atl_set_intr_throttle(struct atl_queue_vec *qvec) { struct atl_hw *hw = &qvec->nic->hw; atl_write(hw, ATL_INTR_THRTL(atl_qvec_intr(qvec)), 1 << 0x1f | ((atl_min_intr_delay / 2) & 0x1ff) << 0x10); } static int atl_poll(struct napi_struct *napi, int budget) { struct atl_queue_vec *qvec; struct atl_nic *nic; bool clean_done; int rx_cleaned; qvec = container_of(napi, struct atl_queue_vec, napi); nic = qvec->nic; clean_done = atl_clean_tx(&qvec->tx); rx_cleaned = atl_clean_rx(&qvec->rx, budget); clean_done &= (rx_cleaned < budget); if (!clean_done) return budget; napi_complete_done(napi, rx_cleaned); atl_intr_enable(&nic->hw, BIT(atl_qvec_intr(qvec))); /* atl_set_intr_throttle(&nic->hw, qvec->idx); */ return rx_cleaned; } /* XXX NOTE: only checked on device probe for now */ static int enable_msi = 1; module_param_named(msi, enable_msi, int, 0444); static int atl_config_interrupts(struct atl_nic *nic) { struct atl_hw *hw = &nic->hw; unsigned int flags; int ret; struct irq_affinity iaff = { .pre_vectors = ATL_NUM_NON_RING_IRQS, .post_vectors = 0, }; if (enable_msi) { flags = PCI_IRQ_MSIX | PCI_IRQ_MSI | PCI_IRQ_AFFINITY; ret = pci_alloc_irq_vectors_affinity(hw->pdev, ATL_NUM_NON_RING_IRQS + 1, ATL_NUM_NON_RING_IRQS + nic->requested_nvecs, flags, &iaff); /* pci_alloc_irq_vectors() never allocates less * than min_vectors */ if (ret > 0) { ret -= ATL_NUM_NON_RING_IRQS; nic->nvecs = ret; nic->flags |= ATL_FL_MULTIPLE_VECTORS; return ret; } } atl_nic_warn("Couldn't allocate MSI-X / MSI vectors, falling back to legacy interrupts\n"); ret = pci_alloc_irq_vectors(hw->pdev, 1, 1, PCI_IRQ_LEGACY); if (ret < 0) { atl_nic_err("Couldn't allocate legacy IRQ\n"); return ret; } nic->nvecs = 1; nic->flags &= ~ATL_FL_MULTIPLE_VECTORS; return 1; } irqreturn_t atl_ring_irq(int irq, void *priv) { struct napi_struct *napi = priv; napi_schedule_irqoff(napi); return IRQ_HANDLED; } void atl_clear_datapath(struct atl_nic *nic) { int i; struct atl_queue_vec *qvecs = nic->qvecs; /* If atl_reconfigure() have failed previously, * atl_clear_datapath() can be called again on * pci_ops->remove(), without an intervening * atl_setup_datapath(). */ if (!test_and_clear_bit(ATL_ST_CONFIGURED, &nic->state)) return; #ifdef ATL_COMPAT_PCI_ALLOC_IRQ_VECTORS_AFFINITY for (i = 0; i < nic->nvecs; i++) { int vector = pci_irq_vector(nic->hw.pdev, i + ATL_NUM_NON_RING_IRQS); irq_set_affinity_hint(vector, NULL); } #endif pci_free_irq_vectors(nic->hw.pdev); if (!qvecs) return; for (i = 0; i < nic->nvecs; i++) netif_napi_del(&qvecs[i].napi); kfree(qvecs); nic->qvecs = NULL; } int atl_setup_datapath(struct atl_nic *nic) { int nvecs, i, ret; struct atl_queue_vec *qvec; nvecs = atl_config_interrupts(nic); if (nvecs < 0) return nvecs; qvec = kcalloc(nvecs, sizeof(*qvec), GFP_KERNEL); if (!qvec) { atl_nic_err("Couldn't alloc qvecs\n"); ret = -ENOMEM; goto exit_free; } nic->qvecs = qvec; for (i = 0; i < nvecs; i++, qvec++) { qvec->nic = nic; qvec->idx = i; qvec->dev = &nic->hw.pdev->dev; qvec->rx.hw.reg_base = ATL_RX_RING(i); qvec->rx.qvec = qvec; qvec->rx.hw.size = nic->requested_rx_size; qvec->tx.hw.reg_base = ATL_TX_RING(i); qvec->tx.qvec = qvec; qvec->tx.hw.size = nic->requested_tx_size; u64_stats_init(&qvec->rx.syncp); u64_stats_init(&qvec->tx.syncp); netif_napi_add(nic->ndev, &qvec->napi, atl_poll, 64); } atl_compat_calc_affinities(nic); nic->max_mtu = atl_rx_linear ? ATL_MAX_RX_LINEAR_MTU : ATL_MAX_MTU; set_bit(ATL_ST_CONFIGURED, &nic->state); return 0; exit_free: atl_clear_datapath(nic); return ret; } static inline void atl_free_rxpage(struct atl_pgref *pgref, struct device *dev) { struct atl_rxpage *rxpage = pgref->rxpage; if (!rxpage) return; /* Unmap, dropping the ref for being mapped */ __atl_free_rxpage(rxpage, dev); pgref->rxpage = 0; } /* Releases any skbs that may have been queued on ring positions yet * to be processes by poll. The buffers are kept to be re-used after * resume / thaw. */ static void atl_clear_rx_bufs(struct atl_desc_ring *ring) { unsigned int bufs = ring_occupied(ring); struct device *dev = ring->qvec->dev; while (bufs) { struct atl_rxbuf *rxbuf = &ring->rxbufs[ring->head]; struct sk_buff *skb = rxbuf->skb; if (skb) { struct atl_pgref *pgref = &ATL_CB(skb)->pgref; atl_put_rxpage(pgref, dev); dev_kfree_skb_any(skb); rxbuf->skb = NULL; } bump_head(ring, 1); bufs--; } } static void atl_free_rx_bufs(struct atl_desc_ring *ring) { struct device *dev = ring->qvec->dev; struct atl_rxbuf *rxbuf; if (!ring->rxbufs) return; for (rxbuf = ring->rxbufs; rxbuf < &ring->rxbufs[ring->hw.size]; rxbuf++) { atl_free_rxpage(&rxbuf->head, dev); atl_free_rxpage(&rxbuf->data, dev); } } static void atl_free_tx_bufs(struct atl_desc_ring *ring) { unsigned int bufs = ring_occupied(ring); if (!ring->txbufs) return; while (bufs) { struct atl_txbuf *txbuf; bump_tail(ring, -1); txbuf = &ring->txbufs[ring->tail]; atl_txbuf_free(txbuf, ring->qvec->dev, ring->tail); bufs--; } } static void atl_free_ring(struct atl_desc_ring *ring) { if (ring->bufs) { vfree(ring->bufs); ring->bufs = 0; } atl_free_descs(ring->qvec->nic, &ring->hw); } static int atl_alloc_ring(struct atl_desc_ring *ring, size_t buf_size, char *type) { int ret; struct atl_nic *nic = ring->qvec->nic; int idx = ring->qvec->idx; ret = atl_alloc_descs(nic, &ring->hw); if (ret) { atl_nic_err("Couldn't alloc %s[%d] descriptors\n", type, idx); return ret; } ring->bufs = vzalloc(ring->hw.size * buf_size); if (!ring->bufs) { atl_nic_err("Couldn't alloc %s[%d] %sbufs\n", type, idx, type); ret = -ENOMEM; goto free; } ring->head = ring->tail = atl_read(&nic->hw, ATL_RING_HEAD(ring)) & 0x1fff; return 0; free: atl_free_ring(ring); return ret; } static int atl_alloc_qvec_intr(struct atl_queue_vec *qvec) { struct atl_nic *nic = qvec->nic; int vector; int ret; snprintf(qvec->name, sizeof(qvec->name), "%s-ring-%d", nic->ndev->name, qvec->idx); if (!(nic->flags & ATL_FL_MULTIPLE_VECTORS)) return 0; vector = pci_irq_vector(nic->hw.pdev, atl_qvec_intr(qvec)); ret = request_irq(vector, atl_ring_irq, 0, qvec->name, &qvec->napi); if (ret) { atl_nic_err("request MSI ring vector failed: %d\n", -ret); return ret; } atl_compat_set_affinity(vector, qvec); return 0; } static void atl_free_qvec_intr(struct atl_queue_vec *qvec) { int vector = pci_irq_vector(qvec->nic->hw.pdev, atl_qvec_intr(qvec)); if (!(qvec->nic->flags & ATL_FL_MULTIPLE_VECTORS)) return; atl_compat_set_affinity(vector, NULL); free_irq(vector, &qvec->napi); } static int atl_alloc_qvec(struct atl_queue_vec *qvec) { struct atl_txbuf *txbuf; int count = qvec->tx.hw.size; int ret; ret = atl_alloc_qvec_intr(qvec); if (ret) return ret; ret = atl_alloc_ring(&qvec->tx, sizeof(struct atl_txbuf), "tx"); if (ret) goto free_irq; ret = atl_alloc_ring(&qvec->rx, sizeof(struct atl_rxbuf), "rx"); if (ret) goto free_tx; for (txbuf = qvec->tx.txbufs; count; count--) (txbuf++)->last = -1; return 0; free_tx: atl_free_ring(&qvec->tx); free_irq: atl_free_qvec_intr(qvec); return ret; } static void atl_free_qvec(struct atl_queue_vec *qvec) { struct atl_desc_ring *rx = &qvec->rx; struct atl_desc_ring *tx = &qvec->tx; atl_free_rx_bufs(rx); atl_free_ring(rx); atl_free_ring(tx); atl_free_qvec_intr(qvec); } int atl_alloc_rings(struct atl_nic *nic) { struct atl_queue_vec *qvec; int ret; atl_for_each_qvec(nic, qvec) { ret = atl_alloc_qvec(qvec); if (ret) goto free; } return 0; free: while(--qvec >= &nic->qvecs[0]) atl_free_qvec(qvec); return ret; } void atl_free_rings(struct atl_nic *nic) { struct atl_queue_vec *qvec; atl_for_each_qvec(nic, qvec) atl_free_qvec(qvec); } static unsigned int atl_rx_mod_hyst = 10, atl_tx_mod_hyst = 10; module_param_named(rx_mod_hyst, atl_rx_mod_hyst, uint, 0644); module_param_named(tx_mod_hyst, atl_tx_mod_hyst, uint, 0644); static void atl_set_intr_mod_qvec(struct atl_queue_vec *qvec) { struct atl_nic *nic = qvec->nic; struct atl_hw *hw = &nic->hw; unsigned int min, max; int idx = qvec->idx; min = nic->rx_intr_delay - atl_min_intr_delay; max = min + atl_rx_mod_hyst; atl_write(hw, ATL_RX_INTR_MOD_CTRL(idx), (max / 2) << 0x10 | (min / 2) << 8 | 2); min = nic->tx_intr_delay - atl_min_intr_delay; max = min + atl_tx_mod_hyst; atl_write(hw, ATL_TX_INTR_MOD_CTRL(idx), (max / 2) << 0x10 | (min / 2) << 8 | 2); } void atl_set_intr_mod(struct atl_nic *nic) { struct atl_queue_vec *qvec; atl_for_each_qvec(nic, qvec) atl_set_intr_mod_qvec(qvec); } static void atl_start_rx_ring(struct atl_desc_ring *ring) { struct atl_hw *hw = &ring->qvec->nic->hw; int idx = ring->qvec->idx; unsigned int rx_ctl; atl_write(hw, ATL_RING_BASE_LSW(ring), ring->hw.daddr); atl_write(hw, ATL_RING_BASE_MSW(ring), ring->hw.daddr >> 32); atl_write(hw, ATL_RX_RING_TAIL(ring), ring->tail); atl_write(hw, ATL_RX_RING_BUF_SIZE(ring), (ATL_RX_HDR_SIZE / 64) << 8 | ATL_RX_BUF_SIZE / 1024); atl_write(hw, ATL_RX_RING_THRESH(ring), 8 << 0x10 | 24 << 0x18); /* LRO */ atl_write_bits(hw, ATL_RX_LRO_PKT_LIM(idx), (idx & 7) * 4, 2, 3); /* Enable ring | VLAN offload | header split in non-linear mode */ rx_ctl = BIT(31) | BIT(29) | ring->hw.size | (atl_rx_linear ? 0 : BIT(28)); atl_write(hw, ATL_RX_RING_CTL(ring), rx_ctl); } static void atl_start_tx_ring(struct atl_desc_ring *ring) { struct atl_nic *nic = ring->qvec->nic; struct atl_hw *hw = &nic->hw; atl_write(hw, ATL_RING_BASE_LSW(ring), ring->hw.daddr); atl_write(hw, ATL_RING_BASE_MSW(ring), ring->hw.daddr >> 32); /* Enable TSO on all active Tx rings */ atl_write(hw, ATL_TX_LSO_CTRL, BIT(nic->nvecs) - 1); atl_write(hw, ATL_TX_RING_TAIL(ring), ring->tail); atl_write(hw, ATL_TX_RING_THRESH(ring), 8 << 8 | 8 << 0x10 | 24 << 0x18); atl_write(hw, ATL_TX_RING_CTL(ring), BIT(31) | ring->hw.size); } static int atl_start_qvec(struct atl_queue_vec *qvec) { struct atl_desc_ring *rx = &qvec->rx; struct atl_desc_ring *tx = &qvec->tx; struct atl_hw *hw = &qvec->nic->hw; int intr = atl_qvec_intr(qvec); struct atl_rxbuf *rxbuf; int ret; rx->head = rx->tail = atl_read(hw, ATL_RING_HEAD(rx)) & 0x1fff; tx->head = tx->tail = atl_read(hw, ATL_RING_HEAD(tx)) & 0x1fff; ret = atl_fill_rx(rx, ring_space(rx)); if (ret) return ret; rx->next_to_recycle = rx->tail; /* rxbuf at ->next_to_recycle is always kept empty so that * atl_maybe_recycle_rxbuf() always have a spot to recyle into * without overwriting a pgref to an already allocated page, * leaking memory. It's also the guard element in the ring * that keeps ->tail from overrunning ->head. If it's nonempty * on ring init (e.g. after a sleep-wake cycle) just release * the pages. */ rxbuf = &rx->rxbufs[rx->next_to_recycle]; atl_put_rxpage(&rxbuf->head, qvec->dev); atl_put_rxpage(&rxbuf->data, qvec->dev); /* Map ring interrups into corresponding cause bit*/ atl_set_intr_bits(hw, qvec->idx, intr, intr); atl_set_intr_throttle(qvec); napi_enable(&qvec->napi); atl_set_intr_mod_qvec(qvec); atl_intr_enable(hw, BIT(atl_qvec_intr(qvec))); atl_start_tx_ring(tx); atl_start_rx_ring(rx); return 0; } static void atl_stop_qvec(struct atl_queue_vec *qvec) { struct atl_desc_ring *rx = &qvec->rx; struct atl_desc_ring *tx = &qvec->tx; struct atl_hw *hw = &qvec->nic->hw; /* Disable and reset rings */ atl_write(hw, ATL_RING_CTL(rx), BIT(25)); atl_write(hw, ATL_RING_CTL(tx), BIT(25)); udelay(10); atl_write(hw, ATL_RING_CTL(rx), 0); atl_write(hw, ATL_RING_CTL(tx), 0); atl_intr_disable(hw, BIT(atl_qvec_intr(qvec))); napi_disable(&qvec->napi); atl_clear_rx_bufs(rx); atl_free_tx_bufs(tx); } static void atl_set_lro(struct atl_nic *nic) { struct atl_hw *hw = &nic->hw; uint32_t val = nic->ndev->features & NETIF_F_LRO ? BIT(nic->nvecs) - 1 : 0; atl_write_bits(hw, ATL_RX_LRO_CTRL1, 0, nic->nvecs, val); atl_write_bits(hw, ATL_INTR_RSC_EN, 0, nic->nvecs, val); } int atl_start_rings(struct atl_nic *nic) { struct atl_hw *hw = &nic->hw; uint32_t mask; struct atl_queue_vec *qvec; int ret; mask = BIT(nic->nvecs + ATL_NUM_NON_RING_IRQS) - BIT(ATL_NUM_NON_RING_IRQS); /* Enable auto-masking of ring interrupts on intr generation */ atl_set_bits(hw, ATL_INTR_AUTO_MASK, mask); /* Enable status auto-clear on intr generation */ atl_set_bits(hw, ATL_INTR_AUTO_CLEAR, mask); atl_set_lro(nic); atl_set_rss_tbl(hw); atl_for_each_qvec(nic, qvec) { ret = atl_start_qvec(qvec); if (ret) goto stop; } return 0; stop: while (--qvec >= &nic->qvecs[0]) atl_stop_qvec(qvec); return ret; } void atl_stop_rings(struct atl_nic *nic) { struct atl_queue_vec *qvec; struct atl_hw *hw = &nic->hw; atl_for_each_qvec(nic, qvec) atl_stop_qvec(qvec); atl_write_bit(hw, 0x5a00, 0, 1); udelay(10); atl_write_bit(hw, 0x5a00, 0, 0); } int atl_set_features(struct net_device *ndev, netdev_features_t features) { netdev_features_t changed = ndev->features ^ features; ndev->features = features; if (changed & NETIF_F_LRO) atl_set_lro(netdev_priv(ndev)); return 0; } void atl_get_ring_stats(struct atl_desc_ring *ring, struct atl_ring_stats *stats) { unsigned int start; do { start = u64_stats_fetch_begin_irq(&ring->syncp); memcpy(stats, &ring->stats, sizeof(*stats)); } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); } #define atl_add_stats(_dst, _src) \ do { \ int i; \ uint64_t *dst = (uint64_t *)(&(_dst)); \ uint64_t *src = (uint64_t *)(&(_src)); \ \ for (i = 0; i < sizeof(_dst) / sizeof(uint64_t); i++) \ dst[i] += src[i]; \ } while (0) void atl_update_global_stats(struct atl_nic *nic) { int i; struct atl_ring_stats stats; memset(&stats, 0, sizeof(stats)); atl_update_eth_stats(nic); spin_lock(&nic->stats_lock); memset(&nic->stats.rx, 0, sizeof(nic->stats.rx)); memset(&nic->stats.tx, 0, sizeof(nic->stats.tx)); for (i = 0; i < nic->nvecs; i++) { atl_get_ring_stats(&nic->qvecs[i].rx, &stats); atl_add_stats(nic->stats.rx, stats.rx); atl_get_ring_stats(&nic->qvecs[i].tx, &stats); atl_add_stats(nic->stats.tx, stats.tx); } spin_unlock(&nic->stats_lock); } void atl_get_stats64(struct net_device *ndev, struct rtnl_link_stats64 *nstats) { struct atl_nic *nic = netdev_priv(ndev); struct atl_global_stats *stats = &nic->stats; atl_update_global_stats(nic); nstats->rx_bytes = stats->rx.bytes; nstats->rx_packets = stats->rx.packets; nstats->tx_bytes = stats->tx.bytes; nstats->tx_packets = stats->tx.packets; nstats->rx_crc_errors = stats->rx.csum_err; nstats->rx_frame_errors = stats->rx.mac_err; nstats->rx_errors = nstats->rx_crc_errors + nstats->rx_frame_errors; nstats->multicast = stats->rx.multicast; nstats->tx_aborted_errors = stats->tx.dma_map_failed; nstats->tx_errors = nstats->tx_aborted_errors; }