The NPA Aura pool that is dedicated for 1st pass inline IPsec flows
raises an interrupt when the buffers of that aura_id drop below a
threshold value.
Add the following changes to handle this interrupt
- Increase the number of MSIX vectors requested for the PF/VF to
include NPA vector.
- Create a workqueue (refill_npa_inline_ipsecq) to allocate and
refill buffers to the pool.
- When the interrupt is raised, schedule the workqueue entry,
cn10k_ipsec_npa_refill_inb_ipsecq(), where the current count of
consumed buffers is determined via NPA_LF_AURA_OP_CNT and then
replenished.
Signed-off-by: Tanmay Jagdale <tanmay@marvell.com>
---
.../marvell/octeontx2/nic/cn10k_ipsec.c | 102 +++++++++++++++++-
.../marvell/octeontx2/nic/cn10k_ipsec.h | 1 +
.../ethernet/marvell/octeontx2/nic/otx2_pf.c | 4 +
.../ethernet/marvell/octeontx2/nic/otx2_vf.c | 4 +
4 files changed, 110 insertions(+), 1 deletion(-)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c
index b88c1b4c5839..365327ab9079 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c
@@ -519,10 +519,77 @@ static int cn10k_ipsec_setup_nix_rx_hw_resources(struct otx2_nic *pfvf)
return err;
}
+static void cn10k_ipsec_npa_refill_inb_ipsecq(struct work_struct *work)
+{
+ struct cn10k_ipsec *ipsec = container_of(work, struct cn10k_ipsec,
+ refill_npa_inline_ipsecq);
+ struct otx2_nic *pfvf = container_of(ipsec, struct otx2_nic, ipsec);
+ struct otx2_pool *pool = NULL;
+ struct otx2_qset *qset = NULL;
+ u64 val, *ptr, op_int = 0, count;
+ int err, pool_id, idx;
+ dma_addr_t bufptr;
+
+ qset = &pfvf->qset;
+
+ val = otx2_read64(pfvf, NPA_LF_QINTX_INT(0));
+ if (!(val & 1))
+ return;
+
+ ptr = otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_INT);
+ val = otx2_atomic64_add(((u64)pfvf->ipsec.inb_ipsec_pool << 44), ptr);
+
+ /* Error interrupt bits */
+ if (val & 0xff)
+ op_int = (val & 0xff);
+
+ /* Refill buffers on a Threshold interrupt */
+ if (val & (1 << 16)) {
+ /* Get the current number of buffers consumed */
+ ptr = otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_CNT);
+ count = otx2_atomic64_add(((u64)pfvf->ipsec.inb_ipsec_pool << 44), ptr);
+ count &= GENMASK_ULL(35, 0);
+
+ /* Refill */
+ pool_id = pfvf->ipsec.inb_ipsec_pool;
+ pool = &pfvf->qset.pool[pool_id];
+
+ for (idx = 0; idx < count; idx++) {
+ err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, idx);
+ if (err) {
+ netdev_err(pfvf->netdev,
+ "Insufficient memory for IPsec pool buffers\n");
+ break;
+ }
+ pfvf->hw_ops->aura_freeptr(pfvf, pool_id,
+ bufptr + OTX2_HEAD_ROOM);
+ }
+
+ op_int |= (1 << 16);
+ }
+
+ /* Clear/ACK Interrupt */
+ if (op_int)
+ otx2_write64(pfvf, NPA_LF_AURA_OP_INT,
+ ((u64)pfvf->ipsec.inb_ipsec_pool << 44) | op_int);
+}
+
+static irqreturn_t cn10k_ipsec_npa_inb_ipsecq_intr_handler(int irq, void *data)
+{
+ struct otx2_nic *pf = data;
+
+ schedule_work(&pf->ipsec.refill_npa_inline_ipsecq);
+
+ return IRQ_HANDLED;
+}
+
static int cn10k_inb_cpt_init(struct net_device *netdev)
{
struct otx2_nic *pfvf = netdev_priv(netdev);
- int ret = 0;
+ int ret = 0, vec;
+ char *irq_name;
+ void *ptr;
+ u64 val;
ret = cn10k_ipsec_setup_nix_rx_hw_resources(pfvf);
if (ret) {
@@ -530,6 +597,34 @@ static int cn10k_inb_cpt_init(struct net_device *netdev)
return ret;
}
+ /* Work entry for refilling the NPA queue for ingress inline IPSec */
+ INIT_WORK(&pfvf->ipsec.refill_npa_inline_ipsecq,
+ cn10k_ipsec_npa_refill_inb_ipsecq);
+
+ /* Register NPA interrupt */
+ vec = pfvf->hw.npa_msixoff;
+ irq_name = &pfvf->hw.irq_name[vec * NAME_SIZE];
+ snprintf(irq_name, NAME_SIZE, "%s-npa-qint", pfvf->netdev->name);
+
+ ret = request_irq(pci_irq_vector(pfvf->pdev, vec),
+ cn10k_ipsec_npa_inb_ipsecq_intr_handler, 0,
+ irq_name, pfvf);
+ if (ret) {
+ dev_err(pfvf->dev,
+ "RVUPF%d: IRQ registration failed for NPA QINT%d\n",
+ rvu_get_pf(pfvf->pcifunc), 0);
+ return ret;
+ }
+
+ /* Enable NPA threshold interrupt */
+ ptr = otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_INT);
+ val = BIT_ULL(43) | BIT_ULL(17);
+ otx2_write64(pfvf, NPA_LF_AURA_OP_INT,
+ ((u64)pfvf->ipsec.inb_ipsec_pool << 44) | val);
+
+ /* Enable interrupt */
+ otx2_write64(pfvf, NPA_LF_QINTX_ENA_W1S(0), BIT_ULL(0));
+
return ret;
}
@@ -1028,6 +1123,8 @@ EXPORT_SYMBOL(cn10k_ipsec_init);
void cn10k_ipsec_clean(struct otx2_nic *pf)
{
+ int vec;
+
if (!is_dev_support_ipsec_offload(pf->pdev))
return;
@@ -1043,6 +1140,9 @@ void cn10k_ipsec_clean(struct otx2_nic *pf)
/* Free Ingress SA table */
qmem_free(pf->dev, pf->ipsec.inb_sa);
+
+ vec = pci_irq_vector(pf->pdev, pf->hw.npa_msixoff);
+ free_irq(vec, pf);
}
EXPORT_SYMBOL(cn10k_ipsec_clean);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.h b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.h
index 5b7b8f3db913..30d5812d52ad 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.h
@@ -117,6 +117,7 @@ struct cn10k_ipsec {
struct qmem *inb_sa;
struct list_head inb_sw_ctx_list;
DECLARE_BITMAP(inb_sa_table, CN10K_IPSEC_INB_MAX_SA);
+ struct work_struct refill_npa_inline_ipsecq;
};
/* CN10K IPSEC Security Association (SA) */
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
index 8f1c17fa5a0b..0ffc56efcc23 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
@@ -2909,6 +2909,10 @@ int otx2_realloc_msix_vectors(struct otx2_nic *pf)
num_vec = hw->nix_msixoff;
num_vec += NIX_LF_CINT_VEC_START + hw->max_queues;
+ /* Update number of vectors to include NPA */
+ if (hw->nix_msixoff < hw->npa_msixoff)
+ num_vec = hw->npa_msixoff + 1;
+
otx2_disable_mbox_intr(pf);
pci_free_irq_vectors(hw->pdev);
err = pci_alloc_irq_vectors(hw->pdev, num_vec, num_vec, PCI_IRQ_MSIX);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
index fb4da816d218..0b0f8a94ca41 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
@@ -521,6 +521,10 @@ static int otx2vf_realloc_msix_vectors(struct otx2_nic *vf)
num_vec = hw->nix_msixoff;
num_vec += NIX_LF_CINT_VEC_START + hw->max_queues;
+ /* Update number of vectors to include NPA */
+ if (hw->nix_msixoff < hw->npa_msixoff)
+ num_vec = hw->npa_msixoff + 1;
+
otx2vf_disable_mbox_intr(vf);
pci_free_irq_vectors(hw->pdev);
err = pci_alloc_irq_vectors(hw->pdev, num_vec, num_vec, PCI_IRQ_MSIX);
--
2.43.0On Fri, May 02, 2025 at 06:49:52PM +0530, Tanmay Jagdale wrote:
> The NPA Aura pool that is dedicated for 1st pass inline IPsec flows
> raises an interrupt when the buffers of that aura_id drop below a
> threshold value.
>
> Add the following changes to handle this interrupt
> - Increase the number of MSIX vectors requested for the PF/VF to
> include NPA vector.
> - Create a workqueue (refill_npa_inline_ipsecq) to allocate and
> refill buffers to the pool.
> - When the interrupt is raised, schedule the workqueue entry,
> cn10k_ipsec_npa_refill_inb_ipsecq(), where the current count of
> consumed buffers is determined via NPA_LF_AURA_OP_CNT and then
> replenished.
>
> Signed-off-by: Tanmay Jagdale <tanmay@marvell.com>
...
> diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c
> index b88c1b4c5839..365327ab9079 100644
> --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c
> +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c
> @@ -519,10 +519,77 @@ static int cn10k_ipsec_setup_nix_rx_hw_resources(struct otx2_nic *pfvf)
> return err;
> }
>
> +static void cn10k_ipsec_npa_refill_inb_ipsecq(struct work_struct *work)
> +{
> + struct cn10k_ipsec *ipsec = container_of(work, struct cn10k_ipsec,
> + refill_npa_inline_ipsecq);
> + struct otx2_nic *pfvf = container_of(ipsec, struct otx2_nic, ipsec);
> + struct otx2_pool *pool = NULL;
> + struct otx2_qset *qset = NULL;
> + u64 val, *ptr, op_int = 0, count;
> + int err, pool_id, idx;
> + dma_addr_t bufptr;
> +
> + qset = &pfvf->qset;
> +
> + val = otx2_read64(pfvf, NPA_LF_QINTX_INT(0));
> + if (!(val & 1))
> + return;
> +
> + ptr = otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_INT);
Sparse complains about __iomem annotations around here:
.../cn10k_ipsec.c:539:13: warning: incorrect type in assignment (different address spaces)
.../cn10k_ipsec.c:539:13: expected unsigned long long [usertype] *ptr
.../cn10k_ipsec.c:539:13: got void [noderef] __iomem *
.../cn10k_ipsec.c:549:21: warning: incorrect type in assignment (different address spaces)
.../cn10k_ipsec.c:549:21: expected unsigned long long [usertype] *ptr
.../cn10k_ipsec.c:549:21: got void [noderef] __iomem *
.../cn10k_ipsec.c:620:13: warning: incorrect type in assignment (different address spaces)
.../cn10k_ipsec.c:620:13: expected void *ptr
.../cn10k_ipsec.c:620:13: got void [noderef] __iomem *
> + val = otx2_atomic64_add(((u64)pfvf->ipsec.inb_ipsec_pool << 44), ptr);
> +
> + /* Error interrupt bits */
> + if (val & 0xff)
> + op_int = (val & 0xff);
> +
> + /* Refill buffers on a Threshold interrupt */
> + if (val & (1 << 16)) {
> + /* Get the current number of buffers consumed */
> + ptr = otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_CNT);
> + count = otx2_atomic64_add(((u64)pfvf->ipsec.inb_ipsec_pool << 44), ptr);
> + count &= GENMASK_ULL(35, 0);
> +
> + /* Refill */
> + pool_id = pfvf->ipsec.inb_ipsec_pool;
> + pool = &pfvf->qset.pool[pool_id];
> +
> + for (idx = 0; idx < count; idx++) {
> + err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, idx);
> + if (err) {
> + netdev_err(pfvf->netdev,
> + "Insufficient memory for IPsec pool buffers\n");
> + break;
> + }
> + pfvf->hw_ops->aura_freeptr(pfvf, pool_id,
> + bufptr + OTX2_HEAD_ROOM);
> + }
> +
> + op_int |= (1 << 16);
> + }
> +
> + /* Clear/ACK Interrupt */
> + if (op_int)
> + otx2_write64(pfvf, NPA_LF_AURA_OP_INT,
> + ((u64)pfvf->ipsec.inb_ipsec_pool << 44) | op_int);
> +}
...
Hi Tanmay,
kernel test robot noticed the following build warnings:
[auto build test WARNING on net-next/main]
url: https://github.com/intel-lab-lkp/linux/commits/Tanmay-Jagdale/crypto-octeontx2-Share-engine-group-info-with-AF-driver/20250502-213203
base: net-next/main
patch link: https://lore.kernel.org/r/20250502132005.611698-12-tanmay%40marvell.com
patch subject: [net-next PATCH v1 11/15] octeontx2-pf: ipsec: Handle NPA threshold interrupt
config: x86_64-allyesconfig (https://download.01.org/0day-ci/archive/20250507/202505071904.TWc5095k-lkp@intel.com/config)
compiler: clang version 20.1.2 (https://github.com/llvm/llvm-project 58df0ef89dd64126512e4ee27b4ac3fd8ddf6247)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250507/202505071904.TWc5095k-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202505071904.TWc5095k-lkp@intel.com/
All warnings (new ones prefixed by >>):
drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c:488:6: warning: variable 'pool' is used uninitialized whenever 'if' condition is true [-Wsometimes-uninitialized]
488 | if (err)
| ^~~
drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c:512:23: note: uninitialized use occurs here
512 | qmem_free(pfvf->dev, pool->stack);
| ^~~~
drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c:488:2: note: remove the 'if' if its condition is always false
488 | if (err)
| ^~~~~~~~
489 | goto pool_fail;
| ~~~~~~~~~~~~~~
drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c:466:24: note: initialize the variable 'pool' to silence this warning
466 | struct otx2_pool *pool;
| ^
| = NULL
>> drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c:528:20: warning: variable 'qset' set but not used [-Wunused-but-set-variable]
528 | struct otx2_qset *qset = NULL;
| ^
>> drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c:591:8: warning: variable 'ptr' set but not used [-Wunused-but-set-variable]
591 | void *ptr;
| ^
3 warnings generated.
vim +/qset +528 drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c
521
522 static void cn10k_ipsec_npa_refill_inb_ipsecq(struct work_struct *work)
523 {
524 struct cn10k_ipsec *ipsec = container_of(work, struct cn10k_ipsec,
525 refill_npa_inline_ipsecq);
526 struct otx2_nic *pfvf = container_of(ipsec, struct otx2_nic, ipsec);
527 struct otx2_pool *pool = NULL;
> 528 struct otx2_qset *qset = NULL;
529 u64 val, *ptr, op_int = 0, count;
530 int err, pool_id, idx;
531 dma_addr_t bufptr;
532
533 qset = &pfvf->qset;
534
535 val = otx2_read64(pfvf, NPA_LF_QINTX_INT(0));
536 if (!(val & 1))
537 return;
538
539 ptr = otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_INT);
540 val = otx2_atomic64_add(((u64)pfvf->ipsec.inb_ipsec_pool << 44), ptr);
541
542 /* Error interrupt bits */
543 if (val & 0xff)
544 op_int = (val & 0xff);
545
546 /* Refill buffers on a Threshold interrupt */
547 if (val & (1 << 16)) {
548 /* Get the current number of buffers consumed */
549 ptr = otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_CNT);
550 count = otx2_atomic64_add(((u64)pfvf->ipsec.inb_ipsec_pool << 44), ptr);
551 count &= GENMASK_ULL(35, 0);
552
553 /* Refill */
554 pool_id = pfvf->ipsec.inb_ipsec_pool;
555 pool = &pfvf->qset.pool[pool_id];
556
557 for (idx = 0; idx < count; idx++) {
558 err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, idx);
559 if (err) {
560 netdev_err(pfvf->netdev,
561 "Insufficient memory for IPsec pool buffers\n");
562 break;
563 }
564 pfvf->hw_ops->aura_freeptr(pfvf, pool_id,
565 bufptr + OTX2_HEAD_ROOM);
566 }
567
568 op_int |= (1 << 16);
569 }
570
571 /* Clear/ACK Interrupt */
572 if (op_int)
573 otx2_write64(pfvf, NPA_LF_AURA_OP_INT,
574 ((u64)pfvf->ipsec.inb_ipsec_pool << 44) | op_int);
575 }
576
577 static irqreturn_t cn10k_ipsec_npa_inb_ipsecq_intr_handler(int irq, void *data)
578 {
579 struct otx2_nic *pf = data;
580
581 schedule_work(&pf->ipsec.refill_npa_inline_ipsecq);
582
583 return IRQ_HANDLED;
584 }
585
586 static int cn10k_inb_cpt_init(struct net_device *netdev)
587 {
588 struct otx2_nic *pfvf = netdev_priv(netdev);
589 int ret = 0, vec;
590 char *irq_name;
> 591 void *ptr;
592 u64 val;
593
594 ret = cn10k_ipsec_setup_nix_rx_hw_resources(pfvf);
595 if (ret) {
596 netdev_err(netdev, "Failed to setup NIX HW resources for IPsec\n");
597 return ret;
598 }
599
600 /* Work entry for refilling the NPA queue for ingress inline IPSec */
601 INIT_WORK(&pfvf->ipsec.refill_npa_inline_ipsecq,
602 cn10k_ipsec_npa_refill_inb_ipsecq);
603
604 /* Register NPA interrupt */
605 vec = pfvf->hw.npa_msixoff;
606 irq_name = &pfvf->hw.irq_name[vec * NAME_SIZE];
607 snprintf(irq_name, NAME_SIZE, "%s-npa-qint", pfvf->netdev->name);
608
609 ret = request_irq(pci_irq_vector(pfvf->pdev, vec),
610 cn10k_ipsec_npa_inb_ipsecq_intr_handler, 0,
611 irq_name, pfvf);
612 if (ret) {
613 dev_err(pfvf->dev,
614 "RVUPF%d: IRQ registration failed for NPA QINT%d\n",
615 rvu_get_pf(pfvf->pcifunc), 0);
616 return ret;
617 }
618
619 /* Enable NPA threshold interrupt */
620 ptr = otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_INT);
621 val = BIT_ULL(43) | BIT_ULL(17);
622 otx2_write64(pfvf, NPA_LF_AURA_OP_INT,
623 ((u64)pfvf->ipsec.inb_ipsec_pool << 44) | val);
624
625 /* Enable interrupt */
626 otx2_write64(pfvf, NPA_LF_QINTX_ENA_W1S(0), BIT_ULL(0));
627
628 return ret;
629 }
630
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
© 2016 - 2026 Red Hat, Inc.