From nobody Fri Dec 19 17:37:45 2025 Received: from szxga04-in.huawei.com (szxga04-in.huawei.com [45.249.212.190]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CDDC81E520E; Sat, 10 May 2025 08:12:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=45.249.212.190 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864775; cv=none; b=H2XdhZffGALpZ1HGlJ49aHZwaGMXpPCnvj/VWe1GRpjfuUeJRCa65Ntg+2hYQijNsUQakBWo5DdvSE9s0GdP74HmEgv/eIJgp40c1iFQ0R2hirnKCSHkVquBbUw7OywRXmTdcyJ0BitXY88JhdSIbe35roLOtBHXiNgAxe4qjIU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864775; c=relaxed/simple; bh=A5+ITZPFqeQAd/lavYspgGlWi9KYaC3OCIwzNzoVjfQ=; h=From:To:CC:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=nk8oeVcezDQM3R4Ls+lXIB/QEOQRd49cHUkSweKaor0J3FkOg8BNfiRNgDB08lCCCnFSbaBU84VXBQ13QqngVRiqjUl0NGCcfYK5P1ILhRgqEdiDgOJtuiiCCKBreQ7JJupIwFSJfZfbI1Oq32CQHBGtFqGs8+1qW6l9aQ8q9x4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com; spf=pass smtp.mailfrom=huawei.com; arc=none smtp.client-ip=45.249.212.190 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=huawei.com Received: from mail.maildlp.com (unknown [172.19.163.44]) by szxga04-in.huawei.com (SkyGuard) with ESMTP id 4Zvdmx1S0Yz2CdG9; Sat, 10 May 2025 16:09:05 +0800 (CST) Received: from kwepemg500006.china.huawei.com (unknown [7.202.181.43]) by mail.maildlp.com (Postfix) with ESMTPS id 208AC14022E; Sat, 10 May 2025 16:12:42 +0800 (CST) Received: from huawei.com (10.50.165.33) by kwepemg500006.china.huawei.com (7.202.181.43) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.1544.11; Sat, 10 May 2025 16:12:41 +0800 From: Longfang Liu To: , , , CC: , , , Subject: [PATCH v8 1/6] hisi_acc_vfio_pci: fix XQE dma address error Date: Sat, 10 May 2025 16:11:50 +0800 Message-ID: <20250510081155.55840-2-liulongfang@huawei.com> X-Mailer: git-send-email 2.24.0 In-Reply-To: <20250510081155.55840-1-liulongfang@huawei.com> References: <20250510081155.55840-1-liulongfang@huawei.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-ClientProxiedBy: dggems705-chm.china.huawei.com (10.3.19.182) To kwepemg500006.china.huawei.com (7.202.181.43) Content-Type: text/plain; charset="utf-8" The dma addresses of EQE and AEQE are wrong after migration and results in guest kernel-mode encryption services failure. Comparing the definition of hardware registers, we found that there was an error when the data read from the register was combined into an address. Therefore, the address combination sequence needs to be corrected. Even after fixing the above problem, we still have an issue where the Guest from an old kernel can get migrated to new kernel and may result in wrong data. In order to ensure that the address is correct after migration, if an old magic number is detected, the dma address needs to be updated. Fixes: b0eed085903e ("hisi_acc_vfio_pci: Add support for VFIO live migratio= n") Signed-off-by: Longfang Liu Reviewed-by: Shameer Kolothum --- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 41 ++++++++++++++++--- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.h | 14 ++++++- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/= pci/hisilicon/hisi_acc_vfio_pci.c index 451c639299eb..304dbdfa0e95 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -350,6 +350,32 @@ static int vf_qm_func_stop(struct hisi_qm *qm) return hisi_qm_mb(qm, QM_MB_CMD_PAUSE_QM, 0, 0, 0); } =20 +static int vf_qm_version_check(struct acc_vf_data *vf_data, struct device = *dev) +{ + switch (vf_data->acc_magic) { + case ACC_DEV_MAGIC_V2: + if (vf_data->major_ver !=3D ACC_DRV_MAJOR_VER) { + dev_info(dev, "migration driver version<%u.%u> not match!\n", + vf_data->major_ver, vf_data->minor_ver); + return -EINVAL; + } + break; + case ACC_DEV_MAGIC_V1: + /* Correct dma address */ + vf_data->eqe_dma =3D vf_data->qm_eqc_dw[QM_XQC_ADDR_HIGH]; + vf_data->eqe_dma <<=3D QM_XQC_ADDR_OFFSET; + vf_data->eqe_dma |=3D vf_data->qm_eqc_dw[QM_XQC_ADDR_LOW]; + vf_data->aeqe_dma =3D vf_data->qm_aeqc_dw[QM_XQC_ADDR_HIGH]; + vf_data->aeqe_dma <<=3D QM_XQC_ADDR_OFFSET; + vf_data->aeqe_dma |=3D vf_data->qm_aeqc_dw[QM_XQC_ADDR_LOW]; + break; + default: + return -EINVAL; + } + + return 0; +} + static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, struct hisi_acc_vf_migration_file *migf) { @@ -363,7 +389,8 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_de= vice *hisi_acc_vdev, if (migf->total_length < QM_MATCH_SIZE || hisi_acc_vdev->match_done) return 0; =20 - if (vf_data->acc_magic !=3D ACC_DEV_MAGIC) { + ret =3D vf_qm_version_check(vf_data, dev); + if (ret) { dev_err(dev, "failed to match ACC_DEV_MAGIC\n"); return -EINVAL; } @@ -418,7 +445,9 @@ static int vf_qm_get_match_data(struct hisi_acc_vf_core= _device *hisi_acc_vdev, int vf_id =3D hisi_acc_vdev->vf_id; int ret; =20 - vf_data->acc_magic =3D ACC_DEV_MAGIC; + vf_data->acc_magic =3D ACC_DEV_MAGIC_V2; + vf_data->major_ver =3D ACC_DRV_MAJOR_VER; + vf_data->minor_ver =3D ACC_DRV_MINOR_VER; /* Save device id */ vf_data->dev_id =3D hisi_acc_vdev->vf_dev->device; =20 @@ -496,12 +525,12 @@ static int vf_qm_read_data(struct hisi_qm *vf_qm, str= uct acc_vf_data *vf_data) return -EINVAL; =20 /* Every reg is 32 bit, the dma address is 64 bit. */ - vf_data->eqe_dma =3D vf_data->qm_eqc_dw[1]; + vf_data->eqe_dma =3D vf_data->qm_eqc_dw[QM_XQC_ADDR_HIGH]; vf_data->eqe_dma <<=3D QM_XQC_ADDR_OFFSET; - vf_data->eqe_dma |=3D vf_data->qm_eqc_dw[0]; - vf_data->aeqe_dma =3D vf_data->qm_aeqc_dw[1]; + vf_data->eqe_dma |=3D vf_data->qm_eqc_dw[QM_XQC_ADDR_LOW]; + vf_data->aeqe_dma =3D vf_data->qm_aeqc_dw[QM_XQC_ADDR_HIGH]; vf_data->aeqe_dma <<=3D QM_XQC_ADDR_OFFSET; - vf_data->aeqe_dma |=3D vf_data->qm_aeqc_dw[0]; + vf_data->aeqe_dma |=3D vf_data->qm_aeqc_dw[QM_XQC_ADDR_LOW]; =20 /* Through SQC_BT/CQC_BT to get sqc and cqc address */ ret =3D qm_get_sqc(vf_qm, &vf_data->sqc_dma); diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h b/drivers/vfio/= pci/hisilicon/hisi_acc_vfio_pci.h index 245d7537b2bc..91002ceeebc1 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h @@ -39,6 +39,9 @@ #define QM_REG_ADDR_OFFSET 0x0004 =20 #define QM_XQC_ADDR_OFFSET 32U +#define QM_XQC_ADDR_LOW 0x1 +#define QM_XQC_ADDR_HIGH 0x2 + #define QM_VF_AEQ_INT_MASK 0x0004 #define QM_VF_EQ_INT_MASK 0x000c #define QM_IFC_INT_SOURCE_V 0x0020 @@ -50,10 +53,15 @@ #define QM_EQC_DW0 0X8000 #define QM_AEQC_DW0 0X8020 =20 +#define ACC_DRV_MAJOR_VER 1 +#define ACC_DRV_MINOR_VER 0 + +#define ACC_DEV_MAGIC_V1 0XCDCDCDCDFEEDAACC +#define ACC_DEV_MAGIC_V2 0xAACCFEEDDECADEDE + struct acc_vf_data { #define QM_MATCH_SIZE offsetofend(struct acc_vf_data, qm_rsv_state) /* QM match information */ -#define ACC_DEV_MAGIC 0XCDCDCDCDFEEDAACC u64 acc_magic; u32 qp_num; u32 dev_id; @@ -61,7 +69,9 @@ struct acc_vf_data { u32 qp_base; u32 vf_qm_state; /* QM reserved match information */ - u32 qm_rsv_state[3]; + u16 major_ver; + u16 minor_ver; + u32 qm_rsv_state[2]; =20 /* QM RW regs */ u32 aeq_int_mask; --=20 2.24.0 From nobody Fri Dec 19 17:37:45 2025 Received: from szxga02-in.huawei.com (szxga02-in.huawei.com [45.249.212.188]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 10D0F1DFDA1; Sat, 10 May 2025 08:13:16 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=45.249.212.188 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864803; cv=none; b=KRHh6SvC7qc5pBgpeG7WpjLKVigBRRnbxPxzWH3xhjkVPnVIcMlCDyqmNwvM3ZdP/JI5XjV9jcfPrj6l9PbhDHCKdAFpjq0E4wEsPjrx/yrQYOPu+BvTE7FDgpVjXLMCnrpuNKvMZImXEtfQ7WgSOl4qtWCSmqBhHnTSszQQbQE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864803; c=relaxed/simple; bh=S467IL2al7IcCPUj4hfHZzYZN2jJjDv3J69qQJP9vog=; h=From:To:CC:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=FrAinkZXCjDqc7K02Z8BsRXgz1BLh3xzEy5fZSBD9mXZehcmrsQXQNNMTHTcYKvTikBS8Tehp2S8WrcjefhIX1hPBOAV9unRac7RSiT1sQ/UWyTZqnhJ/tJEqeqRx6LP6ctG2VLd492wgYYlQajSW94EGZvBWmLQuMHjrT7OllY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com; spf=pass smtp.mailfrom=huawei.com; arc=none smtp.client-ip=45.249.212.188 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=huawei.com Received: from mail.maildlp.com (unknown [172.19.162.254]) by szxga02-in.huawei.com (SkyGuard) with ESMTP id 4ZvdrC2kBFzQwNw; Sat, 10 May 2025 16:11:55 +0800 (CST) Received: from kwepemg500006.china.huawei.com (unknown [7.202.181.43]) by mail.maildlp.com (Postfix) with ESMTPS id 1E70F180489; Sat, 10 May 2025 16:13:13 +0800 (CST) Received: from huawei.com (10.50.165.33) by kwepemg500006.china.huawei.com (7.202.181.43) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.1544.11; Sat, 10 May 2025 16:13:12 +0800 From: Longfang Liu To: , , , CC: , , , Subject: [PATCH v8 2/6] hisi_acc_vfio_pci: add eq and aeq interruption restore Date: Sat, 10 May 2025 16:11:51 +0800 Message-ID: <20250510081155.55840-3-liulongfang@huawei.com> X-Mailer: git-send-email 2.24.0 In-Reply-To: <20250510081155.55840-1-liulongfang@huawei.com> References: <20250510081155.55840-1-liulongfang@huawei.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-ClientProxiedBy: dggems706-chm.china.huawei.com (10.3.19.183) To kwepemg500006.china.huawei.com (7.202.181.43) Content-Type: text/plain; charset="utf-8" In order to ensure that the task packets of the accelerator device are not lost during the migration process, it is necessary to send an EQ and AEQ command to the device after the live migration is completed and to update the completion position of the task queue. Let the device recheck the completed tasks data and if there are uncollected packets, device resend a task completion interrupt to the software. Fixes: b0eed085903e ("hisi_acc_vfio_pci: Add support for VFIO live migratio= n") Signed-off-by: Longfang Liu Reviewed-by: Shameer Kolothum --- drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/= pci/hisilicon/hisi_acc_vfio_pci.c index 304dbdfa0e95..80217aea5475 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -470,6 +470,19 @@ static int vf_qm_get_match_data(struct hisi_acc_vf_cor= e_device *hisi_acc_vdev, return 0; } =20 +static void vf_qm_xeqc_save(struct hisi_qm *qm, + struct hisi_acc_vf_migration_file *migf) +{ + struct acc_vf_data *vf_data =3D &migf->vf_data; + u16 eq_head, aeq_head; + + eq_head =3D vf_data->qm_eqc_dw[0] & 0xFFFF; + qm_db(qm, 0, QM_DOORBELL_CMD_EQ, eq_head, 0); + + aeq_head =3D vf_data->qm_aeqc_dw[0] & 0xFFFF; + qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, aeq_head, 0); +} + static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, struct hisi_acc_vf_migration_file *migf) { @@ -578,6 +591,9 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_dev= ice *hisi_acc_vdev, return -EINVAL; =20 migf->total_length =3D sizeof(struct acc_vf_data); + /* Save eqc and aeqc interrupt information */ + vf_qm_xeqc_save(vf_qm, migf); + return 0; } =20 --=20 2.24.0 From nobody Fri Dec 19 17:37:45 2025 Received: from szxga08-in.huawei.com (szxga08-in.huawei.com [45.249.212.255]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 940561E2853; Sat, 10 May 2025 08:13:52 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=45.249.212.255 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864836; cv=none; b=nBXfscf72TY1KGRK+BdI1L2byJ2mxMVsHNDYmwV0L3KfaD4KmHt4wq2LGhDxl+5ukbFw9fx/RdDIRX0DT+uFO3//0ABesdDC6465zO32pV/ZQ2+ggWfP+cqTfBXpnz8iOwZUOuOhETpMQsaXn0IxgT+hxV2VRvdyF633kxc8tKg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864836; c=relaxed/simple; bh=3IozYO/eR/kQuw7+sST29tdLq1X0HD/pbb+i53spYM0=; h=From:To:CC:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=PHLsIoPI3N9Nqn5EaJfRk5XBOOqH9E0FB0/A8c/Zf+jKORu4AazmJqkjRUV8Iwh8N1jINcuIC1J9VPpeMKri2YIjpJ8gmQDGRNRqYq8v6W4sR5g4uBUVv8+Rr4AL6U7qBdgT+RpIjV1oTXeGVbpO7QHmiYc4GjE0vdFwQuNchnU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com; spf=pass smtp.mailfrom=huawei.com; arc=none smtp.client-ip=45.249.212.255 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=huawei.com Received: from mail.maildlp.com (unknown [172.19.163.48]) by szxga08-in.huawei.com (SkyGuard) with ESMTP id 4Zvdrl2CM2z1d176; Sat, 10 May 2025 16:12:23 +0800 (CST) Received: from kwepemg500006.china.huawei.com (unknown [7.202.181.43]) by mail.maildlp.com (Postfix) with ESMTPS id 1EA50180087; Sat, 10 May 2025 16:13:44 +0800 (CST) Received: from huawei.com (10.50.165.33) by kwepemg500006.china.huawei.com (7.202.181.43) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.1544.11; Sat, 10 May 2025 16:13:43 +0800 From: Longfang Liu To: , , , CC: , , , Subject: [PATCH v8 3/6] hisi_acc_vfio_pci: bugfix cache write-back issue Date: Sat, 10 May 2025 16:11:52 +0800 Message-ID: <20250510081155.55840-4-liulongfang@huawei.com> X-Mailer: git-send-email 2.24.0 In-Reply-To: <20250510081155.55840-1-liulongfang@huawei.com> References: <20250510081155.55840-1-liulongfang@huawei.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-ClientProxiedBy: dggems705-chm.china.huawei.com (10.3.19.182) To kwepemg500006.china.huawei.com (7.202.181.43) Content-Type: text/plain; charset="utf-8" At present, cache write-back is placed in the device data copy stage after stopping the device operation. Writing back to the cache at this stage will cause the data obtained by the cache to be written back to be empty. In order to ensure that the cache data is written back successfully, the data needs to be written back into the stop device stage. Fixes: b0eed085903e ("hisi_acc_vfio_pci: Add support for VFIO live migratio= n") Signed-off-by: Longfang Liu Reviewed-by: Shameer Kolothum --- drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/= pci/hisilicon/hisi_acc_vfio_pci.c index 80217aea5475..d96446f499ed 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -566,7 +566,6 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_dev= ice *hisi_acc_vdev, { struct acc_vf_data *vf_data =3D &migf->vf_data; struct hisi_qm *vf_qm =3D &hisi_acc_vdev->vf_qm; - struct device *dev =3D &vf_qm->pdev->dev; int ret; =20 if (unlikely(qm_wait_dev_not_ready(vf_qm))) { @@ -580,12 +579,6 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_de= vice *hisi_acc_vdev, vf_data->vf_qm_state =3D QM_READY; hisi_acc_vdev->vf_qm_state =3D vf_data->vf_qm_state; =20 - ret =3D vf_qm_cache_wb(vf_qm); - if (ret) { - dev_err(dev, "failed to writeback QM Cache!\n"); - return ret; - } - ret =3D vf_qm_read_data(vf_qm, vf_data); if (ret) return -EINVAL; @@ -1012,6 +1005,13 @@ static int hisi_acc_vf_stop_device(struct hisi_acc_v= f_core_device *hisi_acc_vdev dev_err(dev, "failed to check QM INT state!\n"); return ret; } + + ret =3D vf_qm_cache_wb(vf_qm); + if (ret) { + dev_err(dev, "failed to writeback QM cache!\n"); + return ret; + } + return 0; } =20 --=20 2.24.0 From nobody Fri Dec 19 17:37:45 2025 Received: from szxga07-in.huawei.com (szxga07-in.huawei.com [45.249.212.35]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 3A15A1E5202; Sat, 10 May 2025 08:14:17 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=45.249.212.35 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864860; cv=none; b=hjeU9IECSIEhK7iNub3Z+15TGnf3XiElAmeJJ4AFaknQ1uuf9mufiMKtFEtDO0+oha42kxvA1Xl1lRi3MKCeOgqrs+NwT47V3SjNJJtpeV9OTsPSwjtusaRn0MuSKmzfEDCqkzQqZNCBVlFpSf5gML6lskqJoaXqUxjG7LkMsnw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864860; c=relaxed/simple; bh=RE1GKSJYC4+voiVb9yP2Gn4VIMyTKSBLMoJcSC5qnAM=; h=From:To:CC:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=AeKBNltxcGfg8KI9kvGURrcmtDwDGAKzZ7Bh/OKCtrTfbItivQMsl1AZt1kDVFgtvXFY+piKlviXPJLDSX66cugFHmm1ADXQiT0lLC5shxKdge53aR/JAgpMZl9Ah0uu8w8nb4o2ROTbjaTEjHhuFqu/J7IhvJmlcmrn7wvyZf0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com; spf=pass smtp.mailfrom=huawei.com; arc=none smtp.client-ip=45.249.212.35 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=huawei.com Received: from mail.maildlp.com (unknown [172.19.88.214]) by szxga07-in.huawei.com (SkyGuard) with ESMTP id 4Zvdt80WzJzsSjX; Sat, 10 May 2025 16:13:36 +0800 (CST) Received: from kwepemg500006.china.huawei.com (unknown [7.202.181.43]) by mail.maildlp.com (Postfix) with ESMTPS id EAD901A0171; Sat, 10 May 2025 16:14:14 +0800 (CST) Received: from huawei.com (10.50.165.33) by kwepemg500006.china.huawei.com (7.202.181.43) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.1544.11; Sat, 10 May 2025 16:14:14 +0800 From: Longfang Liu To: , , , CC: , , , Subject: [PATCH v8 4/6] hisi_acc_vfio_pci: bugfix the problem of uninstalling driver Date: Sat, 10 May 2025 16:11:53 +0800 Message-ID: <20250510081155.55840-5-liulongfang@huawei.com> X-Mailer: git-send-email 2.24.0 In-Reply-To: <20250510081155.55840-1-liulongfang@huawei.com> References: <20250510081155.55840-1-liulongfang@huawei.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-ClientProxiedBy: dggems701-chm.china.huawei.com (10.3.19.178) To kwepemg500006.china.huawei.com (7.202.181.43) Content-Type: text/plain; charset="utf-8" In a live migration scenario. If the number of VFs at the destination is greater than the source, the recovery operation will fail and qemu will not be able to complete the process and exit after shutting down the device FD. This will cause the driver to be unable to be unloaded normally due to abnormal reference counting of the live migration driver caused by the abnormal closing operation of fd. Therefore, make sure the migration file descriptor references are always released when the device is closed. Fixes: b0eed085903e ("hisi_acc_vfio_pci: Add support for VFIO live migratio= n") Signed-off-by: Longfang Liu Reviewed-by: Shameer Kolothum --- drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/= pci/hisilicon/hisi_acc_vfio_pci.c index d96446f499ed..cadc82419dca 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -1508,6 +1508,7 @@ static void hisi_acc_vfio_pci_close_device(struct vfi= o_device *core_vdev) struct hisi_acc_vf_core_device *hisi_acc_vdev =3D hisi_acc_get_vf_dev(cor= e_vdev); struct hisi_qm *vf_qm =3D &hisi_acc_vdev->vf_qm; =20 + hisi_acc_vf_disable_fds(hisi_acc_vdev); mutex_lock(&hisi_acc_vdev->open_mutex); hisi_acc_vdev->dev_opened =3D false; iounmap(vf_qm->io_base); --=20 2.24.0 From nobody Fri Dec 19 17:37:45 2025 Received: from szxga01-in.huawei.com (szxga01-in.huawei.com [45.249.212.187]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6CD4E1E9B0B; Sat, 10 May 2025 08:15:03 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=45.249.212.187 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864905; cv=none; b=O2u1Zlqpn78Wn5yuGm8xKitenB0uCHc4NvwxhCRemtmvMEKoRUM6Ss9PlgYOFRjBnxffMiI7acXZ6iz2HOEfvs9Xf01skPnUrBrhb/kVCgStfRZzm7e+FoDs7pwZkzcRc1XBEGRV7sBfdvrEtCZyV16Elb8usqqgwiTNoH+O72o= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864905; c=relaxed/simple; bh=dkYYHw3JXtqYb0l7FvQRydGnN1GI1HNQqbs7a0Dklrg=; h=From:To:CC:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=VYS80kTsyHeGF99o3aNEU5GO4yU4lt1QKwFI1Id0RjSVr4lC8KL1h+HHjBaMp0xijBQPD24yv2UYXYGkj00gv0hKtOHf1YpbNaqHsDb+o+wNGMqom19+EBerR2N7g1shr/6j1w1hw1PeyYshQh5oDgXCebnLdVAKGBofVMJlDpI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com; spf=pass smtp.mailfrom=huawei.com; arc=none smtp.client-ip=45.249.212.187 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=huawei.com Received: from mail.maildlp.com (unknown [172.19.162.254]) by szxga01-in.huawei.com (SkyGuard) with ESMTP id 4Zvdsw6bbQz13LV3; Sat, 10 May 2025 16:13:24 +0800 (CST) Received: from kwepemg500006.china.huawei.com (unknown [7.202.181.43]) by mail.maildlp.com (Postfix) with ESMTPS id D64E9180489; Sat, 10 May 2025 16:14:45 +0800 (CST) Received: from huawei.com (10.50.165.33) by kwepemg500006.china.huawei.com (7.202.181.43) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.1544.11; Sat, 10 May 2025 16:14:45 +0800 From: Longfang Liu To: , , , CC: , , , Subject: [PATCH v8 5/6] hisi_acc_vfio_pci: bugfix live migration function without VF device driver Date: Sat, 10 May 2025 16:11:54 +0800 Message-ID: <20250510081155.55840-6-liulongfang@huawei.com> X-Mailer: git-send-email 2.24.0 In-Reply-To: <20250510081155.55840-1-liulongfang@huawei.com> References: <20250510081155.55840-1-liulongfang@huawei.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-ClientProxiedBy: dggems706-chm.china.huawei.com (10.3.19.183) To kwepemg500006.china.huawei.com (7.202.181.43) Content-Type: text/plain; charset="utf-8" If the VF device driver is not loaded in the Guest OS and we attempt to perform device data migration, the address of the migrated data will be NULL. The live migration recovery operation on the destination side will access a null address value, which will cause access errors. Therefore, live migration of VMs without added VF device drivers does not require device data migration. In addition, when the queue address data obtained by the destination is empty, device queue recovery processing will not be performed. Fixes: b0eed085903e ("hisi_acc_vfio_pci: Add support for VFIO live migratio= n") Signed-off-by: Longfang Liu Reviewed-by: Shameer Kolothum --- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/= pci/hisilicon/hisi_acc_vfio_pci.c index cadc82419dca..d12a350440d3 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -426,13 +426,6 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_d= evice *hisi_acc_vdev, return -EINVAL; } =20 - ret =3D qm_write_regs(vf_qm, QM_VF_STATE, &vf_data->vf_qm_state, 1); - if (ret) { - dev_err(dev, "failed to write QM_VF_STATE\n"); - return ret; - } - - hisi_acc_vdev->vf_qm_state =3D vf_data->vf_qm_state; hisi_acc_vdev->match_done =3D true; return 0; } @@ -498,6 +491,20 @@ static int vf_qm_load_data(struct hisi_acc_vf_core_dev= ice *hisi_acc_vdev, if (migf->total_length < sizeof(struct acc_vf_data)) return -EINVAL; =20 + if (!vf_data->eqe_dma || !vf_data->aeqe_dma || + !vf_data->sqc_dma || !vf_data->cqc_dma) { + dev_info(dev, "resume dma addr is NULL!\n"); + hisi_acc_vdev->vf_qm_state =3D QM_NOT_READY; + return 0; + } + + ret =3D qm_write_regs(qm, QM_VF_STATE, &vf_data->vf_qm_state, 1); + if (ret) { + dev_err(dev, "failed to write QM_VF_STATE\n"); + return -EINVAL; + } + hisi_acc_vdev->vf_qm_state =3D vf_data->vf_qm_state; + qm->eqe_dma =3D vf_data->eqe_dma; qm->aeqe_dma =3D vf_data->aeqe_dma; qm->sqc_dma =3D vf_data->sqc_dma; @@ -1531,6 +1538,7 @@ static int hisi_acc_vfio_pci_migrn_init_dev(struct vf= io_device *core_vdev) hisi_acc_vdev->vf_id =3D pci_iov_vf_id(pdev) + 1; hisi_acc_vdev->pf_qm =3D pf_qm; hisi_acc_vdev->vf_dev =3D pdev; + hisi_acc_vdev->vf_qm_state =3D QM_NOT_READY; mutex_init(&hisi_acc_vdev->state_mutex); mutex_init(&hisi_acc_vdev->open_mutex); =20 --=20 2.24.0 From nobody Fri Dec 19 17:37:45 2025 Received: from szxga06-in.huawei.com (szxga06-in.huawei.com [45.249.212.32]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4F3AF1E2853; Sat, 10 May 2025 08:15:24 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=45.249.212.32 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864928; cv=none; b=YDVut6/AlivRMFbTncLU3qjSR6nGoSHUF+phJuPILvdtru92v8Y/Y+AlO1h+QepqPeD1Yz7tKErNyIxebvG6CB85sgJZE7OTkNXt8ABtzLg7KDDpanzXHkJr4j07g2S8cDM5qe6ltCjXTAyhdoUS1EDuCKyEOJ/y0HWmihoCsaQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746864928; c=relaxed/simple; bh=VXGH4gHfoq5zBVBDrUGxQPKyMbZKpfTr9hgObAwt+ig=; h=From:To:CC:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=lRQbbWaJ8xR228qw1ngbrnZfDqAtfUmYrzrXWmI0Mz6vGWwxYW8sPTFQZByU/6bPQ2RPQ/v3snZkFR1VEeMvvKKysKYrv9oKETfzim9IymqGjkicXaqfd5maZz8Yr/yzdHE0kPa6DpVCxWiwPogp8XrEnxPL4C2/si+qLBuKv5M= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com; spf=pass smtp.mailfrom=huawei.com; arc=none smtp.client-ip=45.249.212.32 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=huawei.com Received: from mail.maildlp.com (unknown [172.19.88.163]) by szxga06-in.huawei.com (SkyGuard) with ESMTP id 4Zvdwz2NPCz27hF3; Sat, 10 May 2025 16:16:03 +0800 (CST) Received: from kwepemg500006.china.huawei.com (unknown [7.202.181.43]) by mail.maildlp.com (Postfix) with ESMTPS id C95CA1800B1; Sat, 10 May 2025 16:15:16 +0800 (CST) Received: from huawei.com (10.50.165.33) by kwepemg500006.china.huawei.com (7.202.181.43) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.1544.11; Sat, 10 May 2025 16:15:16 +0800 From: Longfang Liu To: , , , CC: , , , Subject: [PATCH v8 6/6] hisi_acc_vfio_pci: update function return values. Date: Sat, 10 May 2025 16:11:55 +0800 Message-ID: <20250510081155.55840-7-liulongfang@huawei.com> X-Mailer: git-send-email 2.24.0 In-Reply-To: <20250510081155.55840-1-liulongfang@huawei.com> References: <20250510081155.55840-1-liulongfang@huawei.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-ClientProxiedBy: dggems705-chm.china.huawei.com (10.3.19.182) To kwepemg500006.china.huawei.com (7.202.181.43) Content-Type: text/plain; charset="utf-8" In this driver file, many functions call sub-functions and use ret to store the error code of the sub-functions. However, instead of directly returning ret to the caller, they use a converted error code, which prevents the end-user from clearly understanding the root cause of the error. Therefore, the code needs to be modified to directly return the error code from the sub-functions. Signed-off-by: Longfang Liu Reviewed-by: Shameer Kolothum --- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/= pci/hisilicon/hisi_acc_vfio_pci.c index d12a350440d3..2149f49aeec7 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -190,9 +190,10 @@ static int qm_set_regs(struct hisi_qm *qm, struct acc_= vf_data *vf_data) int ret; =20 /* Check VF state */ - if (unlikely(hisi_qm_wait_mb_ready(qm))) { + ret =3D hisi_qm_wait_mb_ready(qm); + if (unlikely(ret)) { dev_err(&qm->pdev->dev, "QM device is not ready to write\n"); - return -EBUSY; + return ret; } =20 ret =3D qm_write_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1); @@ -325,13 +326,15 @@ static void qm_dev_cmd_init(struct hisi_qm *qm) static int vf_qm_cache_wb(struct hisi_qm *qm) { unsigned int val; + int ret; =20 writel(0x1, qm->io_base + QM_CACHE_WB_START); - if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, + ret =3D readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, val, val & BIT(0), MB_POLL_PERIOD_US, - MB_POLL_TIMEOUT_US)) { + MB_POLL_TIMEOUT_US); + if (ret) { dev_err(&qm->pdev->dev, "vf QM writeback sqc cache fail\n"); - return -EINVAL; + return ret; } =20 return 0; @@ -392,7 +395,7 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_de= vice *hisi_acc_vdev, ret =3D vf_qm_version_check(vf_data, dev); if (ret) { dev_err(dev, "failed to match ACC_DEV_MAGIC\n"); - return -EINVAL; + return ret; } =20 if (vf_data->dev_id !=3D hisi_acc_vdev->vf_dev->device) { @@ -404,7 +407,7 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_de= vice *hisi_acc_vdev, ret =3D qm_get_vft(vf_qm, &vf_qm->qp_base); if (ret <=3D 0) { dev_err(dev, "failed to get vft qp nums\n"); - return -EINVAL; + return ret; } =20 if (ret !=3D vf_data->qp_num) { @@ -501,7 +504,7 @@ static int vf_qm_load_data(struct hisi_acc_vf_core_devi= ce *hisi_acc_vdev, ret =3D qm_write_regs(qm, QM_VF_STATE, &vf_data->vf_qm_state, 1); if (ret) { dev_err(dev, "failed to write QM_VF_STATE\n"); - return -EINVAL; + return ret; } hisi_acc_vdev->vf_qm_state =3D vf_data->vf_qm_state; =20 @@ -542,7 +545,7 @@ static int vf_qm_read_data(struct hisi_qm *vf_qm, struc= t acc_vf_data *vf_data) =20 ret =3D qm_get_regs(vf_qm, vf_data); if (ret) - return -EINVAL; + return ret; =20 /* Every reg is 32 bit, the dma address is 64 bit. */ vf_data->eqe_dma =3D vf_data->qm_eqc_dw[QM_XQC_ADDR_HIGH]; @@ -556,13 +559,13 @@ static int vf_qm_read_data(struct hisi_qm *vf_qm, str= uct acc_vf_data *vf_data) ret =3D qm_get_sqc(vf_qm, &vf_data->sqc_dma); if (ret) { dev_err(dev, "failed to read SQC addr!\n"); - return -EINVAL; + return ret; } =20 ret =3D qm_get_cqc(vf_qm, &vf_data->cqc_dma); if (ret) { dev_err(dev, "failed to read CQC addr!\n"); - return -EINVAL; + return ret; } =20 return 0; @@ -588,7 +591,7 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_dev= ice *hisi_acc_vdev, =20 ret =3D vf_qm_read_data(vf_qm, vf_data); if (ret) - return -EINVAL; + return ret; =20 migf->total_length =3D sizeof(struct acc_vf_data); /* Save eqc and aeqc interrupt information */ @@ -1379,7 +1382,7 @@ static int hisi_acc_vf_debug_check(struct seq_file *s= eq, struct vfio_device *vde ret =3D qm_wait_dev_not_ready(vf_qm); if (ret) { seq_puts(seq, "VF device not ready!\n"); - return -EBUSY; + return ret; } =20 return 0; --=20 2.24.0