Back when the libeth Rx core was initially written, devmem was a draft
and netmem_ref didn't exist in the mainline. Now that it's here, make
libeth MP-agnostic before introducing any new code or any new library
users.
When it's known that the created PP/FQ is for header buffers, use faster
"unsafe" underscored netmem <--> virt accessors as netmem_is_net_iov()
is always false in that case, but consumes some cycles (bit test +
true branch).
Misc: replace explicit EXPORT_SYMBOL_NS_GPL("NS") with
DEFAULT_SYMBOL_NAMESPACE.

Reviewed-by: Mina Almasry <almasrymina@google.com>
Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
---
 include/net/libeth/rx.h                       | 22 +++++++------
 drivers/net/ethernet/intel/iavf/iavf_txrx.c   | 14 ++++----
 .../ethernet/intel/idpf/idpf_singleq_txrx.c   |  2 +-
 drivers/net/ethernet/intel/idpf/idpf_txrx.c   | 33 +++++++++++--------
 drivers/net/ethernet/intel/libeth/rx.c        | 20 ++++++-----
 5 files changed, 51 insertions(+), 40 deletions(-)

diff --git a/include/net/libeth/rx.h b/include/net/libeth/rx.h
index ab05024be518..7d5dc58984b1 100644
--- a/include/net/libeth/rx.h
+++ b/include/net/libeth/rx.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (C) 2024 Intel Corporation */
+/* Copyright (C) 2024-2025 Intel Corporation */
 
 #ifndef __LIBETH_RX_H
 #define __LIBETH_RX_H
@@ -31,7 +31,7 @@
 
 /**
  * struct libeth_fqe - structure representing an Rx buffer (fill queue element)
- * @page: page holding the buffer
+ * @netmem: network memory reference holding the buffer
  * @offset: offset from the page start (to the headroom)
  * @truesize: total space occupied by the buffer (w/ headroom and tailroom)
  *
@@ -40,7 +40,7 @@
  * former, @offset is always 0 and @truesize is always ```PAGE_SIZE```.
  */
 struct libeth_fqe {
-	struct page		*page;
+	netmem_ref		netmem;
 	u32			offset;
 	u32			truesize;
 } __aligned_largest;
@@ -102,15 +102,16 @@ static inline dma_addr_t libeth_rx_alloc(const struct libeth_fq_fp *fq, u32 i)
 	struct libeth_fqe *buf = &fq->fqes[i];
 
 	buf->truesize = fq->truesize;
-	buf->page = page_pool_dev_alloc(fq->pp, &buf->offset, &buf->truesize);
-	if (unlikely(!buf->page))
+	buf->netmem = page_pool_dev_alloc_netmem(fq->pp, &buf->offset,
+						 &buf->truesize);
+	if (unlikely(!buf->netmem))
 		return DMA_MAPPING_ERROR;
 
-	return page_pool_get_dma_addr(buf->page) + buf->offset +
+	return page_pool_get_dma_addr_netmem(buf->netmem) + buf->offset +
 	       fq->pp->p.offset;
 }
 
-void libeth_rx_recycle_slow(struct page *page);
+void libeth_rx_recycle_slow(netmem_ref netmem);
 
 /**
  * libeth_rx_sync_for_cpu - synchronize or recycle buffer post DMA
@@ -126,18 +127,19 @@ void libeth_rx_recycle_slow(struct page *page);
 static inline bool libeth_rx_sync_for_cpu(const struct libeth_fqe *fqe,
 					  u32 len)
 {
-	struct page *page = fqe->page;
+	netmem_ref netmem = fqe->netmem;
 
 	/* Very rare, but possible case. The most common reason:
 	 * the last fragment contained FCS only, which was then
 	 * stripped by the HW.
 	 */
 	if (unlikely(!len)) {
-		libeth_rx_recycle_slow(page);
+		libeth_rx_recycle_slow(netmem);
 		return false;
 	}
 
-	page_pool_dma_sync_for_cpu(page->pp, page, fqe->offset, len);
+	page_pool_dma_sync_netmem_for_cpu(netmem_get_pp(netmem), netmem,
+					  fqe->offset, len);
 
 	return true;
 }
diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
index 422312b8b54a..35d353d38129 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
@@ -723,7 +723,7 @@ static void iavf_clean_rx_ring(struct iavf_ring *rx_ring)
 	for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) {
 		const struct libeth_fqe *rx_fqes = &rx_ring->rx_fqes[i];
 
-		page_pool_put_full_page(rx_ring->pp, rx_fqes->page, false);
+		libeth_rx_recycle_slow(rx_fqes->netmem);
 
 		if (unlikely(++i == rx_ring->count))
 			i = 0;
@@ -1197,10 +1197,11 @@ static void iavf_add_rx_frag(struct sk_buff *skb,
 			     const struct libeth_fqe *rx_buffer,
 			     unsigned int size)
 {
-	u32 hr = rx_buffer->page->pp->p.offset;
+	u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset;
 
-	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
-			rx_buffer->offset + hr, size, rx_buffer->truesize);
+	skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags,
+			       rx_buffer->netmem, rx_buffer->offset + hr,
+			       size, rx_buffer->truesize);
 }
 
 /**
@@ -1214,12 +1215,13 @@ static void iavf_add_rx_frag(struct sk_buff *skb,
 static struct sk_buff *iavf_build_skb(const struct libeth_fqe *rx_buffer,
 				      unsigned int size)
 {
-	u32 hr = rx_buffer->page->pp->p.offset;
+	struct page *buf_page = __netmem_to_page(rx_buffer->netmem);
+	u32 hr = buf_page->pp->p.offset;
 	struct sk_buff *skb;
 	void *va;
 
 	/* prefetch first cache line of first page */
-	va = page_address(rx_buffer->page) + rx_buffer->offset;
+	va = page_address(buf_page) + rx_buffer->offset;
 	net_prefetch(va + hr);
 
 	/* build an skb around the page buffer */
diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
index eae1b6f474e6..aeb2ca5f5a0a 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
@@ -1009,7 +1009,7 @@ static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget)
 			break;
 
 skip_data:
-		rx_buf->page = NULL;
+		rx_buf->netmem = 0;
 
 		IDPF_SINGLEQ_BUMP_RING_IDX(rx_q, ntc);
 		cleaned_count++;
diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c
index bdf52cef3891..6254806c2072 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c
@@ -382,12 +382,12 @@ static int idpf_tx_desc_alloc_all(struct idpf_vport *vport)
  */
 static void idpf_rx_page_rel(struct libeth_fqe *rx_buf)
 {
-	if (unlikely(!rx_buf->page))
+	if (unlikely(!rx_buf->netmem))
 		return;
 
-	page_pool_put_full_page(rx_buf->page->pp, rx_buf->page, false);
+	libeth_rx_recycle_slow(rx_buf->netmem);
 
-	rx_buf->page = NULL;
+	rx_buf->netmem = 0;
 	rx_buf->offset = 0;
 }
 
@@ -3096,10 +3096,10 @@ idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb,
 void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb,
 		      unsigned int size)
 {
-	u32 hr = rx_buf->page->pp->p.offset;
+	u32 hr = netmem_get_pp(rx_buf->netmem)->p.offset;
 
-	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buf->page,
-			rx_buf->offset + hr, size, rx_buf->truesize);
+	skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, rx_buf->netmem,
+			       rx_buf->offset + hr, size, rx_buf->truesize);
 }
 
 /**
@@ -3122,16 +3122,20 @@ static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr,
 			     struct libeth_fqe *buf, u32 data_len)
 {
 	u32 copy = data_len <= L1_CACHE_BYTES ? data_len : ETH_HLEN;
+	struct page *hdr_page, *buf_page;
 	const void *src;
 	void *dst;
 
-	if (!libeth_rx_sync_for_cpu(buf, copy))
+	if (unlikely(netmem_is_net_iov(buf->netmem)) ||
+	    !libeth_rx_sync_for_cpu(buf, copy))
 		return 0;
 
-	dst = page_address(hdr->page) + hdr->offset + hdr->page->pp->p.offset;
-	src = page_address(buf->page) + buf->offset + buf->page->pp->p.offset;
-	memcpy(dst, src, LARGEST_ALIGN(copy));
+	hdr_page = __netmem_to_page(hdr->netmem);
+	buf_page = __netmem_to_page(buf->netmem);
+	dst = page_address(hdr_page) + hdr->offset + hdr_page->pp->p.offset;
+	src = page_address(buf_page) + buf->offset + buf_page->pp->p.offset;
 
+	memcpy(dst, src, LARGEST_ALIGN(copy));
 	buf->offset += copy;
 
 	return copy;
@@ -3147,11 +3151,12 @@ static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr,
  */
 struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size)
 {
-	u32 hr = buf->page->pp->p.offset;
+	struct page *buf_page = __netmem_to_page(buf->netmem);
+	u32 hr = buf_page->pp->p.offset;
 	struct sk_buff *skb;
 	void *va;
 
-	va = page_address(buf->page) + buf->offset;
+	va = page_address(buf_page) + buf->offset;
 	prefetch(va + hr);
 
 	skb = napi_build_skb(va, buf->truesize);
@@ -3302,7 +3307,7 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget)
 			u64_stats_update_end(&rxq->stats_sync);
 		}
 
-		hdr->page = NULL;
+		hdr->netmem = 0;
 
 payload:
 		if (!libeth_rx_sync_for_cpu(rx_buf, pkt_len))
@@ -3318,7 +3323,7 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget)
 			break;
 
 skip_data:
-		rx_buf->page = NULL;
+		rx_buf->netmem = 0;
 
 		idpf_rx_post_buf_refill(refillq, buf_id);
 		IDPF_RX_BUMP_NTC(rxq, ntc);
diff --git a/drivers/net/ethernet/intel/libeth/rx.c b/drivers/net/ethernet/intel/libeth/rx.c
index 66d1d23b8ad2..aa5d878181f7 100644
--- a/drivers/net/ethernet/intel/libeth/rx.c
+++ b/drivers/net/ethernet/intel/libeth/rx.c
@@ -1,5 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (C) 2024 Intel Corporation */
+/* Copyright (C) 2024-2025 Intel Corporation */
+
+#define DEFAULT_SYMBOL_NAMESPACE	"LIBETH"
 
 #include <net/libeth/rx.h>
 
@@ -186,7 +188,7 @@ int libeth_rx_fq_create(struct libeth_fq *fq, struct napi_struct *napi)
 
 	return -ENOMEM;
 }
-EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_create, "LIBETH");
+EXPORT_SYMBOL_GPL(libeth_rx_fq_create);
 
 /**
  * libeth_rx_fq_destroy - destroy a &page_pool created by libeth
@@ -197,19 +199,19 @@ void libeth_rx_fq_destroy(struct libeth_fq *fq)
 	kvfree(fq->fqes);
 	page_pool_destroy(fq->pp);
 }
-EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_destroy, "LIBETH");
+EXPORT_SYMBOL_GPL(libeth_rx_fq_destroy);
 
 /**
- * libeth_rx_recycle_slow - recycle a libeth page from the NAPI context
- * @page: page to recycle
+ * libeth_rx_recycle_slow - recycle libeth netmem
+ * @netmem: network memory to recycle
  *
  * To be used on exceptions or rare cases not requiring fast inline recycling.
  */
-void libeth_rx_recycle_slow(struct page *page)
+void __cold libeth_rx_recycle_slow(netmem_ref netmem)
 {
-	page_pool_recycle_direct(page->pp, page);
+	page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false);
 }
-EXPORT_SYMBOL_NS_GPL(libeth_rx_recycle_slow, "LIBETH");
+EXPORT_SYMBOL_GPL(libeth_rx_recycle_slow);
 
 /* Converting abstract packet type numbers into a software structure with
  * the packet parameters to do O(1) lookup on Rx.
@@ -251,7 +253,7 @@ void libeth_rx_pt_gen_hash_type(struct libeth_rx_pt *pt)
 	pt->hash_type |= libeth_rx_pt_xdp_iprot[pt->inner_prot];
 	pt->hash_type |= libeth_rx_pt_xdp_pl[pt->payload_layer];
 }
-EXPORT_SYMBOL_NS_GPL(libeth_rx_pt_gen_hash_type, "LIBETH");
+EXPORT_SYMBOL_GPL(libeth_rx_pt_gen_hash_type);
 
 /* Module */
 
-- 
2.49.0

> -----Original Message-----
> From: Intel-wired-lan <intel-wired-lan-bounces@osuosl.org> On Behalf Of
> Alexander Lobakin
> Sent: Tuesday, April 15, 2025 7:28 PM
> To: intel-wired-lan@lists.osuosl.org
> Cc: Lobakin, Aleksander <aleksander.lobakin@intel.com>; Kubiak, Michal
> <michal.kubiak@intel.com>; Fijalkowski, Maciej
> <maciej.fijalkowski@intel.com>; Nguyen, Anthony L
> <anthony.l.nguyen@intel.com>; Kitszel, Przemyslaw
> <przemyslaw.kitszel@intel.com>; Andrew Lunn <andrew+netdev@lunn.ch>;
> David S. Miller <davem@davemloft.net>; Dumazet, Eric
> <edumazet@google.com>; Jakub Kicinski <kuba@kernel.org>; Paolo Abeni
> <pabeni@redhat.com>; Alexei Starovoitov <ast@kernel.org>; Daniel
> Borkmann <daniel@iogearbox.net>; Jesper Dangaard Brouer
> <hawk@kernel.org>; John Fastabend <john.fastabend@gmail.com>; Simon
> Horman <horms@kernel.org>; bpf@vger.kernel.org; netdev@vger.kernel.org;
> linux-kernel@vger.kernel.org; Mina Almasry <almasrymina@google.com>
> Subject: [Intel-wired-lan] [PATCH iwl-next 01/16] libeth: convert to netmem
> 
> Back when the libeth Rx core was initially written, devmem was a draft and
> netmem_ref didn't exist in the mainline. Now that it's here, make libeth MP-
> agnostic before introducing any new code or any new library users.
> When it's known that the created PP/FQ is for header buffers, use faster
> "unsafe" underscored netmem <--> virt accessors as netmem_is_net_iov() is
> always false in that case, but consumes some cycles (bit test + true branch).
> Misc: replace explicit EXPORT_SYMBOL_NS_GPL("NS") with
> DEFAULT_SYMBOL_NAMESPACE.
> 
> Reviewed-by: Mina Almasry <almasrymina@google.com>
> Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
> ---
>  include/net/libeth/rx.h                       | 22 +++++++------
>  drivers/net/ethernet/intel/iavf/iavf_txrx.c   | 14 ++++----
>  .../ethernet/intel/idpf/idpf_singleq_txrx.c   |  2 +-
>  drivers/net/ethernet/intel/idpf/idpf_txrx.c   | 33 +++++++++++--------
>  drivers/net/ethernet/intel/libeth/rx.c        | 20 ++++++-----
>  5 files changed, 51 insertions(+), 40 deletions(-)
> 
> diff --git a/include/net/libeth/rx.h b/include/net/libeth/rx.h index
> ab05024be518..7d5dc58984b1 100644
> --- a/include/net/libeth/rx.h
> +++ b/include/net/libeth/rx.h
> @@ -1,5 +1,5 @@
>  /* SPDX-License-Identifier: GPL-2.0-only */
> -/* Copyright (C) 2024 Intel Corporation */
> +/* Copyright (C) 2024-2025 Intel Corporation */
> 
>  #ifndef __LIBETH_RX_H
>  #define __LIBETH_RX_H
> @@ -31,7 +31,7 @@
> 
>  /**
>   * struct libeth_fqe - structure representing an Rx buffer (fill queue element)
> - * @page: page holding the buffer
> + * @netmem: network memory reference holding the buffer
>   * @offset: offset from the page start (to the headroom)
>   * @truesize: total space occupied by the buffer (w/ headroom and tailroom)
>   *
> @@ -40,7 +40,7 @@
>   * former, @offset is always 0 and @truesize is always ```PAGE_SIZE```.
>   */
>  struct libeth_fqe {
> -	struct page		*page;
> +	netmem_ref		netmem;
>  	u32			offset;
>  	u32			truesize;
>  } __aligned_largest;
> @@ -102,15 +102,16 @@ static inline dma_addr_t libeth_rx_alloc(const
> struct libeth_fq_fp *fq, u32 i)
>  	struct libeth_fqe *buf = &fq->fqes[i];
> 
>  	buf->truesize = fq->truesize;
> -	buf->page = page_pool_dev_alloc(fq->pp, &buf->offset, &buf-
> >truesize);
> -	if (unlikely(!buf->page))
> +	buf->netmem = page_pool_dev_alloc_netmem(fq->pp, &buf->offset,
> +						 &buf->truesize);
> +	if (unlikely(!buf->netmem))
>  		return DMA_MAPPING_ERROR;
> 
> -	return page_pool_get_dma_addr(buf->page) + buf->offset +
> +	return page_pool_get_dma_addr_netmem(buf->netmem) + buf-
> >offset +
>  	       fq->pp->p.offset;
>  }
> 
> -void libeth_rx_recycle_slow(struct page *page);
> +void libeth_rx_recycle_slow(netmem_ref netmem);
> 
>  /**
>   * libeth_rx_sync_for_cpu - synchronize or recycle buffer post DMA @@ -
> 126,18 +127,19 @@ void libeth_rx_recycle_slow(struct page *page);  static
> inline bool libeth_rx_sync_for_cpu(const struct libeth_fqe *fqe,
>  					  u32 len)
>  {
> -	struct page *page = fqe->page;
> +	netmem_ref netmem = fqe->netmem;
> 
>  	/* Very rare, but possible case. The most common reason:
>  	 * the last fragment contained FCS only, which was then
>  	 * stripped by the HW.
>  	 */
>  	if (unlikely(!len)) {
> -		libeth_rx_recycle_slow(page);
> +		libeth_rx_recycle_slow(netmem);
>  		return false;
>  	}
> 
> -	page_pool_dma_sync_for_cpu(page->pp, page, fqe->offset, len);
> +	page_pool_dma_sync_netmem_for_cpu(netmem_get_pp(netmem),
> netmem,
> +					  fqe->offset, len);
> 
>  	return true;
>  }
> diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c
> b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
> index 422312b8b54a..35d353d38129 100644
> --- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c
> +++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
> @@ -723,7 +723,7 @@ static void iavf_clean_rx_ring(struct iavf_ring
> *rx_ring)
>  	for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) {
>  		const struct libeth_fqe *rx_fqes = &rx_ring->rx_fqes[i];
> 
> -		page_pool_put_full_page(rx_ring->pp, rx_fqes->page, false);
> +		libeth_rx_recycle_slow(rx_fqes->netmem);
> 
>  		if (unlikely(++i == rx_ring->count))
>  			i = 0;
> @@ -1197,10 +1197,11 @@ static void iavf_add_rx_frag(struct sk_buff *skb,
>  			     const struct libeth_fqe *rx_buffer,
>  			     unsigned int size)
>  {
> -	u32 hr = rx_buffer->page->pp->p.offset;
> +	u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset;
> 
> -	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
> -			rx_buffer->offset + hr, size, rx_buffer->truesize);
> +	skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags,
> +			       rx_buffer->netmem, rx_buffer->offset + hr,
> +			       size, rx_buffer->truesize);
>  }
> 
>  /**
> @@ -1214,12 +1215,13 @@ static void iavf_add_rx_frag(struct sk_buff *skb,
> static struct sk_buff *iavf_build_skb(const struct libeth_fqe *rx_buffer,
>  				      unsigned int size)
>  {
> -	u32 hr = rx_buffer->page->pp->p.offset;
> +	struct page *buf_page = __netmem_to_page(rx_buffer->netmem);
> +	u32 hr = buf_page->pp->p.offset;
>  	struct sk_buff *skb;
>  	void *va;
> 
>  	/* prefetch first cache line of first page */
> -	va = page_address(rx_buffer->page) + rx_buffer->offset;
> +	va = page_address(buf_page) + rx_buffer->offset;
>  	net_prefetch(va + hr);
> 
>  	/* build an skb around the page buffer */ diff --git
> a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
> b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
> index eae1b6f474e6..aeb2ca5f5a0a 100644
> --- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
> +++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
> @@ -1009,7 +1009,7 @@ static int idpf_rx_singleq_clean(struct
> idpf_rx_queue *rx_q, int budget)
>  			break;
> 
>  skip_data:
> -		rx_buf->page = NULL;
> +		rx_buf->netmem = 0;
> 
>  		IDPF_SINGLEQ_BUMP_RING_IDX(rx_q, ntc);
>  		cleaned_count++;
> diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c
> b/drivers/net/ethernet/intel/idpf/idpf_txrx.c
> index bdf52cef3891..6254806c2072 100644
> --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c
> +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c
> @@ -382,12 +382,12 @@ static int idpf_tx_desc_alloc_all(struct idpf_vport
> *vport)
>   */
>  static void idpf_rx_page_rel(struct libeth_fqe *rx_buf)  {
> -	if (unlikely(!rx_buf->page))
> +	if (unlikely(!rx_buf->netmem))
>  		return;
> 
> -	page_pool_put_full_page(rx_buf->page->pp, rx_buf->page, false);
> +	libeth_rx_recycle_slow(rx_buf->netmem);
> 
> -	rx_buf->page = NULL;
> +	rx_buf->netmem = 0;
>  	rx_buf->offset = 0;
>  }
> 
> @@ -3096,10 +3096,10 @@ idpf_rx_process_skb_fields(struct
> idpf_rx_queue *rxq, struct sk_buff *skb,  void idpf_rx_add_frag(struct
> idpf_rx_buf *rx_buf, struct sk_buff *skb,
>  		      unsigned int size)
>  {
> -	u32 hr = rx_buf->page->pp->p.offset;
> +	u32 hr = netmem_get_pp(rx_buf->netmem)->p.offset;
> 
> -	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buf->page,
> -			rx_buf->offset + hr, size, rx_buf->truesize);
> +	skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, rx_buf-
> >netmem,
> +			       rx_buf->offset + hr, size, rx_buf->truesize);
>  }
> 
>  /**
> @@ -3122,16 +3122,20 @@ static u32 idpf_rx_hsplit_wa(const struct
> libeth_fqe *hdr,
>  			     struct libeth_fqe *buf, u32 data_len)  {
>  	u32 copy = data_len <= L1_CACHE_BYTES ? data_len : ETH_HLEN;
> +	struct page *hdr_page, *buf_page;
>  	const void *src;
>  	void *dst;
> 
> -	if (!libeth_rx_sync_for_cpu(buf, copy))
> +	if (unlikely(netmem_is_net_iov(buf->netmem)) ||
> +	    !libeth_rx_sync_for_cpu(buf, copy))
>  		return 0;
> 
> -	dst = page_address(hdr->page) + hdr->offset + hdr->page->pp-
> >p.offset;
> -	src = page_address(buf->page) + buf->offset + buf->page->pp-
> >p.offset;
> -	memcpy(dst, src, LARGEST_ALIGN(copy));
> +	hdr_page = __netmem_to_page(hdr->netmem);
> +	buf_page = __netmem_to_page(buf->netmem);
> +	dst = page_address(hdr_page) + hdr->offset + hdr_page->pp-
> >p.offset;
> +	src = page_address(buf_page) + buf->offset + buf_page->pp->p.offset;
> 
> +	memcpy(dst, src, LARGEST_ALIGN(copy));
Can you avoid 'unstable' API __netmem_to_page()  usage? For example:

- dst = page_address(hdr->page) + hdr->offset + hdr->page->pp->p.offset;
- src = page_address(buf->page) + buf->offset + buf->page->pp->p.offset;
- memcpy(dst, src, LARGEST_ALIGN(copy));
+ dst = netmem_address(hdr->netmem) + hdr->offset;
+ src = netmem_address(buf->netmem) + buf->offset;
+ memcpy(dst, src, LARGEST_ALIGN(copy));

>  	buf->offset += copy;
> 
>  	return copy;
> @@ -3147,11 +3151,12 @@ static u32 idpf_rx_hsplit_wa(const struct
> libeth_fqe *hdr,
>   */
>  struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size)  {
> -	u32 hr = buf->page->pp->p.offset;
> +	struct page *buf_page = __netmem_to_page(buf->netmem);
> +	u32 hr = buf_page->pp->p.offset;
>  	struct sk_buff *skb;
>  	void *va;
> 
> -	va = page_address(buf->page) + buf->offset;
> +	va = page_address(buf_page) + buf->offset;
>  	prefetch(va + hr);
> 
>  	skb = napi_build_skb(va, buf->truesize); @@ -3302,7 +3307,7 @@
> static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget)
>  			u64_stats_update_end(&rxq->stats_sync);
>  		}
> 
> -		hdr->page = NULL;
> +		hdr->netmem = 0;
> 
>  payload:
>  		if (!libeth_rx_sync_for_cpu(rx_buf, pkt_len)) @@ -3318,7
> +3323,7 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int
> budget)
>  			break;
> 
>  skip_data:
> -		rx_buf->page = NULL;
> +		rx_buf->netmem = 0;
> 
>  		idpf_rx_post_buf_refill(refillq, buf_id);
>  		IDPF_RX_BUMP_NTC(rxq, ntc);
> diff --git a/drivers/net/ethernet/intel/libeth/rx.c
> b/drivers/net/ethernet/intel/libeth/rx.c
> index 66d1d23b8ad2..aa5d878181f7 100644
> --- a/drivers/net/ethernet/intel/libeth/rx.c
> +++ b/drivers/net/ethernet/intel/libeth/rx.c
> @@ -1,5 +1,7 @@
>  // SPDX-License-Identifier: GPL-2.0-only
> -/* Copyright (C) 2024 Intel Corporation */
> +/* Copyright (C) 2024-2025 Intel Corporation */
> +
> +#define DEFAULT_SYMBOL_NAMESPACE	"LIBETH"
> 
>  #include <net/libeth/rx.h>
> 
> @@ -186,7 +188,7 @@ int libeth_rx_fq_create(struct libeth_fq *fq, struct
> napi_struct *napi)
> 
>  	return -ENOMEM;
>  }
> -EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_create, "LIBETH");
> +EXPORT_SYMBOL_GPL(libeth_rx_fq_create);
> 
>  /**
>   * libeth_rx_fq_destroy - destroy a &page_pool created by libeth @@ -197,19
> +199,19 @@ void libeth_rx_fq_destroy(struct libeth_fq *fq)
>  	kvfree(fq->fqes);
>  	page_pool_destroy(fq->pp);
>  }
> -EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_destroy, "LIBETH");
> +EXPORT_SYMBOL_GPL(libeth_rx_fq_destroy);
> 
>  /**
> - * libeth_rx_recycle_slow - recycle a libeth page from the NAPI context
> - * @page: page to recycle
> + * libeth_rx_recycle_slow - recycle libeth netmem
> + * @netmem: network memory to recycle
>   *
>   * To be used on exceptions or rare cases not requiring fast inline recycling.
>   */
> -void libeth_rx_recycle_slow(struct page *page)
> +void __cold libeth_rx_recycle_slow(netmem_ref netmem)
>  {
> -	page_pool_recycle_direct(page->pp, page);
> +	page_pool_put_full_netmem(netmem_get_pp(netmem), netmem,
> false);
>  }
> -EXPORT_SYMBOL_NS_GPL(libeth_rx_recycle_slow, "LIBETH");
> +EXPORT_SYMBOL_GPL(libeth_rx_recycle_slow);
> 
>  /* Converting abstract packet type numbers into a software structure with
>   * the packet parameters to do O(1) lookup on Rx.
> @@ -251,7 +253,7 @@ void libeth_rx_pt_gen_hash_type(struct libeth_rx_pt
> *pt)
>  	pt->hash_type |= libeth_rx_pt_xdp_iprot[pt->inner_prot];
>  	pt->hash_type |= libeth_rx_pt_xdp_pl[pt->payload_layer];
>  }
> -EXPORT_SYMBOL_NS_GPL(libeth_rx_pt_gen_hash_type, "LIBETH");
> +EXPORT_SYMBOL_GPL(libeth_rx_pt_gen_hash_type);
> 
>  /* Module */
> 
> --
> 2.49.0

From: Loktionov, Aleksandr <aleksandr.loktionov@intel.com>
Date: Wed, 16 Apr 2025 08:54:39 +0200

> 
> 
>> -----Original Message-----
>> From: Intel-wired-lan <intel-wired-lan-bounces@osuosl.org> On Behalf Of
>> Alexander Lobakin
>> Sent: Tuesday, April 15, 2025 7:28 PM
>> To: intel-wired-lan@lists.osuosl.org
>> Cc: Lobakin, Aleksander <aleksander.lobakin@intel.com>; Kubiak, Michal
>> <michal.kubiak@intel.com>; Fijalkowski, Maciej
>> <maciej.fijalkowski@intel.com>; Nguyen, Anthony L
>> <anthony.l.nguyen@intel.com>; Kitszel, Przemyslaw
>> <przemyslaw.kitszel@intel.com>; Andrew Lunn <andrew+netdev@lunn.ch>;
>> David S. Miller <davem@davemloft.net>; Dumazet, Eric
>> <edumazet@google.com>; Jakub Kicinski <kuba@kernel.org>; Paolo Abeni
>> <pabeni@redhat.com>; Alexei Starovoitov <ast@kernel.org>; Daniel
>> Borkmann <daniel@iogearbox.net>; Jesper Dangaard Brouer
>> <hawk@kernel.org>; John Fastabend <john.fastabend@gmail.com>; Simon
>> Horman <horms@kernel.org>; bpf@vger.kernel.org; netdev@vger.kernel.org;
>> linux-kernel@vger.kernel.org; Mina Almasry <almasrymina@google.com>
>> Subject: [Intel-wired-lan] [PATCH iwl-next 01/16] libeth: convert to netmem

[...]

>> +	memcpy(dst, src, LARGEST_ALIGN(copy));
> Can you avoid 'unstable' API __netmem_to_page()  usage? For example:
> 
> - dst = page_address(hdr->page) + hdr->offset + hdr->page->pp->p.offset;
> - src = page_address(buf->page) + buf->offset + buf->page->pp->p.offset;
> - memcpy(dst, src, LARGEST_ALIGN(copy));
> + dst = netmem_address(hdr->netmem) + hdr->offset;
> + src = netmem_address(buf->netmem) + buf->offset;
> + memcpy(dst, src, LARGEST_ALIGN(copy));

Those underscored versions were introduced by me to avoid redundant
overhead when it's known the buffer is backed by a page -- for cases
like this one, so your propo makes no sense to me.

> 
>>  	buf->offset += copy;
>>
>>  	return copy;

Thanks,
Olek