drivers/net/ethernet/mellanox/mlx5/core/eq.c | 41 +++++++++++++++++++ .../net/ethernet/mellanox/mlx5/core/lib/eq.h | 1 + 2 files changed, 42 insertions(+)
Interrupt lost scenario has been observed in multiple issues during IRQ
migration due to cpu scaling activity. This further led to the presence of
unhandled EQE's causing corresponding Mellanox transmission queues to
become full and get timedout. This patch overcomes this situation by
polling the EQ associated with the IRQ which undergoes migration, to
recover any unhandled EQE's and keep the transmission uninterrupted from
the corresponding queue.
Signed-off-by: Praveen Kumar Kannoju <praveen.kannoju@oracle.com>
---
drivers/net/ethernet/mellanox/mlx5/core/eq.c | 41 +++++++++++++++++++
.../net/ethernet/mellanox/mlx5/core/lib/eq.h | 1 +
2 files changed, 42 insertions(+)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 25499da177bc..4f0653305f46 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -22,6 +22,10 @@
#include "devlink.h"
#include "en_accel/ipsec.h"
+unsigned int mlx5_reap_eq_irq_aff_change;
+module_param(mlx5_reap_eq_irq_aff_change, int, 0644);
+MODULE_PARM_DESC(mlx5_reap_eq_irq_aff_change, "mlx5_reap_eq_irq_aff_change: 0 = Disable MLX5 EQ Reap upon IRQ affinity change, \
+ 1 = Enable MLX5 EQ Reap upon IRQ affinity change. Default=0");
enum {
MLX5_EQE_OWNER_INIT_VAL = 0x1,
};
@@ -951,10 +955,36 @@ static int alloc_rmap(struct mlx5_core_dev *mdev) { return 0; }
static void free_rmap(struct mlx5_core_dev *mdev) {}
#endif
+void mlx5_eq_reap_irq_notify(struct irq_affinity_notify *notify, const cpumask_t *mask)
+{
+ u32 eqe_count;
+ struct mlx5_eq_comp *eq = container_of(notify, struct mlx5_eq_comp, notify);
+
+ if (mlx5_reap_eq_irq_aff_change) {
+ mlx5_core_warn(eq->core.dev, "irqn = 0x%x migration notified, EQ 0x%x: Cons = 0x%x\n",
+ eq->core.irqn, eq->core.eqn, eq->core.cons_index);
+
+ while (!rtnl_trylock())
+ msleep(20);
+
+ eqe_count = mlx5_eq_poll_irq_disabled(eq);
+ if (eqe_count)
+ mlx5_core_warn(eq->core.dev, "Recovered %d eqes on EQ 0x%x\n",
+ eqe_count, eq->core.eqn);
+ rtnl_unlock();
+ }
+}
+
+void mlx5_eq_reap_irq_release(struct kref *ref) {}
+
static void destroy_comp_eq(struct mlx5_core_dev *dev, struct mlx5_eq_comp *eq, u16 vecidx)
{
struct mlx5_eq_table *table = dev->priv.eq_table;
+ if (irq_set_affinity_notifier(eq->core.irqn, NULL))
+ mlx5_core_warn(dev, "failed to unset EQ 0x%x to irq 0x%x affinty\n",
+ eq->core.eqn, eq->core.irqn);
+
xa_erase(&table->comp_eqs, vecidx);
mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
if (destroy_unmap_eq(dev, &eq->core))
@@ -990,6 +1020,7 @@ static int create_comp_eq(struct mlx5_core_dev *dev, u16 vecidx)
struct mlx5_irq *irq;
int nent;
int err;
+ int ret;
lockdep_assert_held(&table->comp_lock);
if (table->curr_comp_eqs == table->max_comp_eqs) {
@@ -1036,6 +1067,16 @@ static int create_comp_eq(struct mlx5_core_dev *dev, u16 vecidx)
if (err)
goto disable_eq;
+ eq->notify.notify = mlx5_eq_reap_irq_notify;
+ eq->notify.release = mlx5_eq_reap_irq_release;
+ ret = irq_set_affinity_notifier(eq->core.irqn, &eq->notify);
+ if (ret) {
+ mlx5_core_warn(dev, "mlx5_eq_reap_irq_nofifier: EQ 0x%x irqn = 0x%x irq_set_affinity_notifier failed: %d\n",
+ eq->core.eqn, eq->core.irqn, ret);
+ }
+ mlx5_core_dbg(dev, "mlx5_eq_reap_irq_nofifier: EQ 0x%x irqn = 0x%x irq_set_affinity_notifier set.\n",
+ eq->core.eqn, eq->core.irqn);
+
table->curr_comp_eqs++;
return eq->core.eqn;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h
index b1edc71ffc6d..669bacb9e390 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h
@@ -46,6 +46,7 @@ struct mlx5_eq_comp {
struct notifier_block irq_nb;
struct mlx5_eq_tasklet tasklet_ctx;
struct list_head list;
+ struct irq_affinity_notify notify;
};
static inline u32 eq_get_size(struct mlx5_eq *eq)
--
2.43.7Hi Praveen,
kernel test robot noticed the following build warnings:
[auto build test WARNING on net-next/main]
[also build test WARNING on net/main linus/master v7.0-rc2 next-20260304]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Praveen-Kumar-Kannoju/net-mlx5-poll-mlx5-eq-during-irq-migration/20260305-003505
base: net-next/main
patch link: https://lore.kernel.org/r/20260304161704.910564-1-praveen.kannoju%40oracle.com
patch subject: [PATCH] net/mlx5: poll mlx5 eq during irq migration
config: loongarch-randconfig-r121-20260305 (https://download.01.org/0day-ci/archive/20260305/202603051910.7oo8wCfc-lkp@intel.com/config)
compiler: loongarch64-linux-gcc (GCC) 15.2.0
sparse: v0.6.5-rc1
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260305/202603051910.7oo8wCfc-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202603051910.7oo8wCfc-lkp@intel.com/
sparse warnings: (new ones prefixed by >>)
>> drivers/net/ethernet/mellanox/mlx5/core/eq.c:25:14: sparse: sparse: symbol 'mlx5_reap_eq_irq_aff_change' was not declared. Should it be static?
>> drivers/net/ethernet/mellanox/mlx5/core/eq.c:958:6: sparse: sparse: symbol 'mlx5_eq_reap_irq_notify' was not declared. Should it be static?
>> drivers/net/ethernet/mellanox/mlx5/core/eq.c:978:6: sparse: sparse: symbol 'mlx5_eq_reap_irq_release' was not declared. Should it be static?
vim +/mlx5_reap_eq_irq_aff_change +25 drivers/net/ethernet/mellanox/mlx5/core/eq.c
24
> 25 unsigned int mlx5_reap_eq_irq_aff_change;
26 module_param(mlx5_reap_eq_irq_aff_change, int, 0644);
27 MODULE_PARM_DESC(mlx5_reap_eq_irq_aff_change, "mlx5_reap_eq_irq_aff_change: 0 = Disable MLX5 EQ Reap upon IRQ affinity change, \
28 1 = Enable MLX5 EQ Reap upon IRQ affinity change. Default=0");
29 enum {
30 MLX5_EQE_OWNER_INIT_VAL = 0x1,
31 };
32
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi Praveen,
kernel test robot noticed the following build warnings:
[auto build test WARNING on net-next/main]
[also build test WARNING on net/main linus/master v7.0-rc2 next-20260304]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Praveen-Kumar-Kannoju/net-mlx5-poll-mlx5-eq-during-irq-migration/20260305-003505
base: net-next/main
patch link: https://lore.kernel.org/r/20260304161704.910564-1-praveen.kannoju%40oracle.com
patch subject: [PATCH] net/mlx5: poll mlx5 eq during irq migration
config: x86_64-rhel-9.4 (https://download.01.org/0day-ci/archive/20260305/202603051743.ceus9qzu-lkp@intel.com/config)
compiler: gcc-14 (Debian 14.2.0-19) 14.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260305/202603051743.ceus9qzu-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202603051743.ceus9qzu-lkp@intel.com/
All warnings (new ones prefixed by >>):
>> drivers/net/ethernet/mellanox/mlx5/core/eq.c:958:6: warning: no previous prototype for 'mlx5_eq_reap_irq_notify' [-Wmissing-prototypes]
958 | void mlx5_eq_reap_irq_notify(struct irq_affinity_notify *notify, const cpumask_t *mask)
| ^~~~~~~~~~~~~~~~~~~~~~~
>> drivers/net/ethernet/mellanox/mlx5/core/eq.c:978:6: warning: no previous prototype for 'mlx5_eq_reap_irq_release' [-Wmissing-prototypes]
978 | void mlx5_eq_reap_irq_release(struct kref *ref) {}
| ^~~~~~~~~~~~~~~~~~~~~~~~
vim +/mlx5_eq_reap_irq_notify +958 drivers/net/ethernet/mellanox/mlx5/core/eq.c
957
> 958 void mlx5_eq_reap_irq_notify(struct irq_affinity_notify *notify, const cpumask_t *mask)
959 {
960 u32 eqe_count;
961 struct mlx5_eq_comp *eq = container_of(notify, struct mlx5_eq_comp, notify);
962
963 if (mlx5_reap_eq_irq_aff_change) {
964 mlx5_core_warn(eq->core.dev, "irqn = 0x%x migration notified, EQ 0x%x: Cons = 0x%x\n",
965 eq->core.irqn, eq->core.eqn, eq->core.cons_index);
966
967 while (!rtnl_trylock())
968 msleep(20);
969
970 eqe_count = mlx5_eq_poll_irq_disabled(eq);
971 if (eqe_count)
972 mlx5_core_warn(eq->core.dev, "Recovered %d eqes on EQ 0x%x\n",
973 eqe_count, eq->core.eqn);
974 rtnl_unlock();
975 }
976 }
977
> 978 void mlx5_eq_reap_irq_release(struct kref *ref) {}
979
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi Praveen,
kernel test robot noticed the following build warnings:
[auto build test WARNING on net-next/main]
[also build test WARNING on net/main linus/master v7.0-rc2 next-20260304]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Praveen-Kumar-Kannoju/net-mlx5-poll-mlx5-eq-during-irq-migration/20260305-003505
base: net-next/main
patch link: https://lore.kernel.org/r/20260304161704.910564-1-praveen.kannoju%40oracle.com
patch subject: [PATCH] net/mlx5: poll mlx5 eq during irq migration
config: x86_64-buildonly-randconfig-002-20260305 (https://download.01.org/0day-ci/archive/20260305/202603051647.fykhqQ3H-lkp@intel.com/config)
compiler: clang version 20.1.8 (https://github.com/llvm/llvm-project 87f0227cb60147a26a1eeb4fb06e3b505e9c7261)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260305/202603051647.fykhqQ3H-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202603051647.fykhqQ3H-lkp@intel.com/
All warnings (new ones prefixed by >>):
>> drivers/net/ethernet/mellanox/mlx5/core/eq.c:958:6: warning: no previous prototype for function 'mlx5_eq_reap_irq_notify' [-Wmissing-prototypes]
958 | void mlx5_eq_reap_irq_notify(struct irq_affinity_notify *notify, const cpumask_t *mask)
| ^
drivers/net/ethernet/mellanox/mlx5/core/eq.c:958:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
958 | void mlx5_eq_reap_irq_notify(struct irq_affinity_notify *notify, const cpumask_t *mask)
| ^
| static
>> drivers/net/ethernet/mellanox/mlx5/core/eq.c:978:6: warning: no previous prototype for function 'mlx5_eq_reap_irq_release' [-Wmissing-prototypes]
978 | void mlx5_eq_reap_irq_release(struct kref *ref) {}
| ^
drivers/net/ethernet/mellanox/mlx5/core/eq.c:978:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
978 | void mlx5_eq_reap_irq_release(struct kref *ref) {}
| ^
| static
2 warnings generated.
vim +/mlx5_eq_reap_irq_notify +958 drivers/net/ethernet/mellanox/mlx5/core/eq.c
957
> 958 void mlx5_eq_reap_irq_notify(struct irq_affinity_notify *notify, const cpumask_t *mask)
959 {
960 u32 eqe_count;
961 struct mlx5_eq_comp *eq = container_of(notify, struct mlx5_eq_comp, notify);
962
963 if (mlx5_reap_eq_irq_aff_change) {
964 mlx5_core_warn(eq->core.dev, "irqn = 0x%x migration notified, EQ 0x%x: Cons = 0x%x\n",
965 eq->core.irqn, eq->core.eqn, eq->core.cons_index);
966
967 while (!rtnl_trylock())
968 msleep(20);
969
970 eqe_count = mlx5_eq_poll_irq_disabled(eq);
971 if (eqe_count)
972 mlx5_core_warn(eq->core.dev, "Recovered %d eqes on EQ 0x%x\n",
973 eqe_count, eq->core.eqn);
974 rtnl_unlock();
975 }
976 }
977
> 978 void mlx5_eq_reap_irq_release(struct kref *ref) {}
979
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi Praveen,
kernel test robot noticed the following build warnings:
[auto build test WARNING on net-next/main]
[also build test WARNING on net/main linus/master v6.16-rc1 next-20260304]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Praveen-Kumar-Kannoju/net-mlx5-poll-mlx5-eq-during-irq-migration/20260305-003505
base: net-next/main
patch link: https://lore.kernel.org/r/20260304161704.910564-1-praveen.kannoju%40oracle.com
patch subject: [PATCH] net/mlx5: poll mlx5 eq during irq migration
config: x86_64-rhel-9.4-ltp (https://download.01.org/0day-ci/archive/20260305/202603050528.5JWnahEr-lkp@intel.com/config)
compiler: gcc-14 (Debian 14.2.0-19) 14.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260305/202603050528.5JWnahEr-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202603050528.5JWnahEr-lkp@intel.com/
All warnings (new ones prefixed by >>):
>> drivers/net/ethernet/mellanox/mlx5/core/eq.c:958:6: warning: no previous prototype for 'mlx5_eq_reap_irq_notify' [-Wmissing-prototypes]
958 | void mlx5_eq_reap_irq_notify(struct irq_affinity_notify *notify, const cpumask_t *mask)
| ^~~~~~~~~~~~~~~~~~~~~~~
>> drivers/net/ethernet/mellanox/mlx5/core/eq.c:978:6: warning: no previous prototype for 'mlx5_eq_reap_irq_release' [-Wmissing-prototypes]
978 | void mlx5_eq_reap_irq_release(struct kref *ref) {}
| ^~~~~~~~~~~~~~~~~~~~~~~~
vim +/mlx5_eq_reap_irq_notify +958 drivers/net/ethernet/mellanox/mlx5/core/eq.c
957
> 958 void mlx5_eq_reap_irq_notify(struct irq_affinity_notify *notify, const cpumask_t *mask)
959 {
960 u32 eqe_count;
961 struct mlx5_eq_comp *eq = container_of(notify, struct mlx5_eq_comp, notify);
962
963 if (mlx5_reap_eq_irq_aff_change) {
964 mlx5_core_warn(eq->core.dev, "irqn = 0x%x migration notified, EQ 0x%x: Cons = 0x%x\n",
965 eq->core.irqn, eq->core.eqn, eq->core.cons_index);
966
967 while (!rtnl_trylock())
968 msleep(20);
969
970 eqe_count = mlx5_eq_poll_irq_disabled(eq);
971 if (eqe_count)
972 mlx5_core_warn(eq->core.dev, "Recovered %d eqes on EQ 0x%x\n",
973 eqe_count, eq->core.eqn);
974 rtnl_unlock();
975 }
976 }
977
> 978 void mlx5_eq_reap_irq_release(struct kref *ref) {}
979
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
On Wed, Mar 04, 2026 at 04:17:04PM +0000, Praveen Kumar Kannoju wrote: > Interrupt lost scenario has been observed in multiple issues during IRQ > migration due to cpu scaling activity. This further led to the presence of > unhandled EQE's causing corresponding Mellanox transmission queues to > become full and get timedout. This patch overcomes this situation by > polling the EQ associated with the IRQ which undergoes migration, to > recover any unhandled EQE's and keep the transmission uninterrupted from > the corresponding queue. What? This does not seem like something we should do like this. IRQ migration is not supposed to loose interrupts, this seems like a IRQ layer bug to me. If it is buggy and loosing interrupts it should probably inject a spurious interrupt around these events so all devices can be enjoy the bug fix. Basically you need to explain with alot more detail why the IRQ was lost, not just some hand wavey "migration something something".. BTW there are known bugs in things like qemu that can loose interrupts around changes to the MSI (and worse than that too), but I thought they were all fixed now? Jason
© 2016 - 2026 Red Hat, Inc.