[v3] vfio/nvgrace-gpu: Support huge PFNMAP and wait for GPU ready post reset

[PATCH v3 5/7] vfio: move barmap to a separate function and export

Posted by ankita@nvidia.com 1 week, 3 days ago

From: Ankit Agrawal <ankita@nvidia.com>

Move the code to map the BAR to a separate function.

This would be reused by the nvgrace-gpu module.

Signed-off-by: Ankit Agrawal <ankita@nvidia.com>
---
 drivers/vfio/pci/vfio_pci_core.c | 38 ++++++++++++++++++++++----------
 include/linux/vfio_pci_core.h    |  1 +
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 29dcf78905a6..d1ff1c0aa727 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -1717,6 +1717,29 @@ static const struct vm_operations_struct vfio_pci_mmap_ops = {
 #endif
 };
 
+int vfio_pci_core_barmap(struct vfio_pci_core_device *vdev, unsigned int index)
+{
+	struct pci_dev *pdev = vdev->pdev;
+	int ret;
+
+	if (vdev->barmap[index])
+		return 0;
+
+	ret = pci_request_selected_regions(pdev,
+					   1 << index, "vfio-pci");
+	if (ret)
+		return ret;
+
+	vdev->barmap[index] = pci_iomap(pdev, index, 0);
+	if (!vdev->barmap[index]) {
+		pci_release_selected_regions(pdev, 1 << index);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(vfio_pci_core_barmap);
+
 int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
 {
 	struct vfio_pci_core_device *vdev =
@@ -1761,18 +1784,9 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma
 	 * Even though we don't make use of the barmap for the mmap,
 	 * we need to request the region and the barmap tracks that.
 	 */
-	if (!vdev->barmap[index]) {
-		ret = pci_request_selected_regions(pdev,
-						   1 << index, "vfio-pci");
-		if (ret)
-			return ret;
-
-		vdev->barmap[index] = pci_iomap(pdev, index, 0);
-		if (!vdev->barmap[index]) {
-			pci_release_selected_regions(pdev, 1 << index);
-			return -ENOMEM;
-		}
-	}
+	ret = vfio_pci_core_barmap(vdev, index);
+	if (ret)
+		return ret;
 
 	vma->vm_private_data = vdev;
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index a097a66485b4..75f04d613e0c 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -121,6 +121,7 @@ ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *bu
 		size_t count, loff_t *ppos);
 vm_fault_t vfio_pci_map_pfn(struct vm_fault *vmf, unsigned long pfn,
 			    unsigned int order);
+int vfio_pci_core_barmap(struct vfio_pci_core_device *vdev, unsigned int index);
 int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma);
 void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count);
 int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf);
-- 
2.34.1

Re: [PATCH v3 5/7] vfio: move barmap to a separate function and export

Posted by Alex Williamson 1 week, 3 days ago

On Fri, 21 Nov 2025 14:11:39 +0000
<ankita@nvidia.com> wrote:

> From: Ankit Agrawal <ankita@nvidia.com>
> 
> Move the code to map the BAR to a separate function.
> 
> This would be reused by the nvgrace-gpu module.
> 
> Signed-off-by: Ankit Agrawal <ankita@nvidia.com>
> ---
>  drivers/vfio/pci/vfio_pci_core.c | 38 ++++++++++++++++++++++----------
>  include/linux/vfio_pci_core.h    |  1 +
>  2 files changed, 27 insertions(+), 12 deletions(-)
> 
> diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
> index 29dcf78905a6..d1ff1c0aa727 100644
> --- a/drivers/vfio/pci/vfio_pci_core.c
> +++ b/drivers/vfio/pci/vfio_pci_core.c
> @@ -1717,6 +1717,29 @@ static const struct vm_operations_struct vfio_pci_mmap_ops = {
>  #endif
>  };
>  
> +int vfio_pci_core_barmap(struct vfio_pci_core_device *vdev, unsigned int index)
> +{
> +	struct pci_dev *pdev = vdev->pdev;
> +	int ret;
> +
> +	if (vdev->barmap[index])
> +		return 0;
> +
> +	ret = pci_request_selected_regions(pdev,
> +					   1 << index, "vfio-pci");
> +	if (ret)
> +		return ret;
> +
> +	vdev->barmap[index] = pci_iomap(pdev, index, 0);
> +	if (!vdev->barmap[index]) {
> +		pci_release_selected_regions(pdev, 1 << index);
> +		return -ENOMEM;
> +	}
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(vfio_pci_core_barmap);

Looks a lot like vfio_pci_core_setup_barmap() ;)

Thanks,
Alex


> +
>  int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
>  {
>  	struct vfio_pci_core_device *vdev =
> @@ -1761,18 +1784,9 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma
>  	 * Even though we don't make use of the barmap for the mmap,
>  	 * we need to request the region and the barmap tracks that.
>  	 */
> -	if (!vdev->barmap[index]) {
> -		ret = pci_request_selected_regions(pdev,
> -						   1 << index, "vfio-pci");
> -		if (ret)
> -			return ret;
> -
> -		vdev->barmap[index] = pci_iomap(pdev, index, 0);
> -		if (!vdev->barmap[index]) {
> -			pci_release_selected_regions(pdev, 1 << index);
> -			return -ENOMEM;
> -		}
> -	}
> +	ret = vfio_pci_core_barmap(vdev, index);
> +	if (ret)
> +		return ret;
>  
>  	vma->vm_private_data = vdev;
>  	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
> diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
> index a097a66485b4..75f04d613e0c 100644
> --- a/include/linux/vfio_pci_core.h
> +++ b/include/linux/vfio_pci_core.h
> @@ -121,6 +121,7 @@ ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *bu
>  		size_t count, loff_t *ppos);
>  vm_fault_t vfio_pci_map_pfn(struct vm_fault *vmf, unsigned long pfn,
>  			    unsigned int order);
> +int vfio_pci_core_barmap(struct vfio_pci_core_device *vdev, unsigned int index);
>  int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma);
>  void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count);
>  int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf);

Re: [PATCH v3 5/7] vfio: move barmap to a separate function and export

Posted by Donald Dutile 1 week, 3 days ago

On 11/21/25 11:39 AM, Alex Williamson wrote:
> On Fri, 21 Nov 2025 14:11:39 +0000
> <ankita@nvidia.com> wrote:
> 
>> From: Ankit Agrawal <ankita@nvidia.com>
>>
>> Move the code to map the BAR to a separate function.
>>
>> This would be reused by the nvgrace-gpu module.
>>
>> Signed-off-by: Ankit Agrawal <ankita@nvidia.com>
>> ---
>>   drivers/vfio/pci/vfio_pci_core.c | 38 ++++++++++++++++++++++----------
>>   include/linux/vfio_pci_core.h    |  1 +
>>   2 files changed, 27 insertions(+), 12 deletions(-)
>>
>> diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
>> index 29dcf78905a6..d1ff1c0aa727 100644
>> --- a/drivers/vfio/pci/vfio_pci_core.c
>> +++ b/drivers/vfio/pci/vfio_pci_core.c
>> @@ -1717,6 +1717,29 @@ static const struct vm_operations_struct vfio_pci_mmap_ops = {
>>   #endif
>>   };
>>   
>> +int vfio_pci_core_barmap(struct vfio_pci_core_device *vdev, unsigned int index)
>> +{
>> +	struct pci_dev *pdev = vdev->pdev;
>> +	int ret;
>> +
>> +	if (vdev->barmap[index])
>> +		return 0;
>> +
>> +	ret = pci_request_selected_regions(pdev,
>> +					   1 << index, "vfio-pci");
>> +	if (ret)
>> +		return ret;
>> +
>> +	vdev->barmap[index] = pci_iomap(pdev, index, 0);
>> +	if (!vdev->barmap[index]) {
>> +		pci_release_selected_regions(pdev, 1 << index);
>> +		return -ENOMEM;
>> +	}
>> +
>> +	return 0;
>> +}
>> +EXPORT_SYMBOL_GPL(vfio_pci_core_barmap);
> 
> Looks a lot like vfio_pci_core_setup_barmap() ;)
> 
> Thanks,
> Alex
> 
+1 ... and below ...
> 
>> +
>>   int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
>>   {
>>   	struct vfio_pci_core_device *vdev =
>> @@ -1761,18 +1784,9 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma
>>   	 * Even though we don't make use of the barmap for the mmap,
>>   	 * we need to request the region and the barmap tracks that.
>>   	 */
>> -	if (!vdev->barmap[index]) {
>> -		ret = pci_request_selected_regions(pdev,
>> -						   1 << index, "vfio-pci");
>> -		if (ret)
>> -			return ret;
>> -
>> -		vdev->barmap[index] = pci_iomap(pdev, index, 0);
>> -		if (!vdev->barmap[index]) {
>> -			pci_release_selected_regions(pdev, 1 << index);
>> -			return -ENOMEM;
>> -		}
>> -	}
>> +	ret = vfio_pci_core_barmap(vdev, index);
>> +	if (ret)
>> +		return ret;
>>   
so, vfio_pci_core_mmap() should be calling vfio_pci_core_setup_barmap() vs. what it does above currently?

--Don

>>   	vma->vm_private_data = vdev;
>>   	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
>> diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
>> index a097a66485b4..75f04d613e0c 100644
>> --- a/include/linux/vfio_pci_core.h
>> +++ b/include/linux/vfio_pci_core.h
>> @@ -121,6 +121,7 @@ ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *bu
>>   		size_t count, loff_t *ppos);
>>   vm_fault_t vfio_pci_map_pfn(struct vm_fault *vmf, unsigned long pfn,
>>   			    unsigned int order);
>> +int vfio_pci_core_barmap(struct vfio_pci_core_device *vdev, unsigned int index);
>>   int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma);
>>   void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count);
>>   int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf);
> 
>

Re: [PATCH v3 5/7] vfio: move barmap to a separate function and export

Posted by Alex Williamson 1 week, 3 days ago

On Fri, 21 Nov 2025 15:58:19 -0500
Donald Dutile <ddutile@redhat.com> wrote:

> On 11/21/25 11:39 AM, Alex Williamson wrote:
> > On Fri, 21 Nov 2025 14:11:39 +0000
> > <ankita@nvidia.com> wrote:  
> >> +
> >>   int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
> >>   {
> >>   	struct vfio_pci_core_device *vdev =
> >> @@ -1761,18 +1784,9 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma
> >>   	 * Even though we don't make use of the barmap for the mmap,
> >>   	 * we need to request the region and the barmap tracks that.
> >>   	 */
> >> -	if (!vdev->barmap[index]) {
> >> -		ret = pci_request_selected_regions(pdev,
> >> -						   1 << index, "vfio-pci");
> >> -		if (ret)
> >> -			return ret;
> >> -
> >> -		vdev->barmap[index] = pci_iomap(pdev, index, 0);
> >> -		if (!vdev->barmap[index]) {
> >> -			pci_release_selected_regions(pdev, 1 << index);
> >> -			return -ENOMEM;
> >> -		}
> >> -	}
> >> +	ret = vfio_pci_core_barmap(vdev, index);
> >> +	if (ret)
> >> +		return ret;
> >>     
> so, vfio_pci_core_mmap() should be calling vfio_pci_core_setup_barmap() vs. what it does above currently?

Yeah, it probably should have happened in 8bccc5b80678 ("vfio/pci:
Expose vfio_pci_core_setup_barmap()") when the static function in
vfio_pci_rdwr.c was exported.  Better late than never.  Thanks,

Alex

[PATCH v3 1/7] vfio/nvgrace-gpu: Use faults to map device memory
[PATCH v3 2/7] vfio: export function to map the VMA
[PATCH v3 3/7] vfio/nvgrace-gpu: Add support for huge pfnmap
[PATCH v3 4/7] vfio: export vfio_find_cap_start
[PATCH v3 5/7] vfio: move barmap to a separate function and export
[PATCH v3 6/7] vfio/nvgrace-gpu: split the code to wait for GPU ready
[PATCH v3 7/7] vfio/nvgrace-gpu: wait for the GPU mem to be ready