[v2] reduce system memory requirement for hibernation

[PATCH v2 1/5] drm/ttm: add ttm_device_prepare_hibernation() api

Posted by Samuel Zhang 3 months ago

This new api is used for hibernation to move GTT BOs to shmem after
VRAM eviction. shmem will be flushed to swap disk later to reduce
the system memory usage for hibernation.

Signed-off-by: Samuel Zhang <guoqing.zhang@amd.com>
---
 drivers/gpu/drm/ttm/ttm_device.c | 29 +++++++++++++++++++++++++++++
 include/drm/ttm/ttm_device.h     |  1 +
 2 files changed, 30 insertions(+)

diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
index 02e797fd1891..19ab35ffeead 100644
--- a/drivers/gpu/drm/ttm/ttm_device.c
+++ b/drivers/gpu/drm/ttm/ttm_device.c
@@ -123,6 +123,35 @@ static int ttm_global_init(void)
 	return ret;
 }
 
+/**
+ * move GTT BOs to shmem for hibernation.
+ *
+ * returns 0 on success, negative on failure.
+ */
+int ttm_device_prepare_hibernation(void)
+{
+	struct ttm_operation_ctx ctx = {
+		.interruptible = false,
+		.no_wait_gpu = false,
+		.force_alloc = true
+	};
+	struct ttm_global *glob = &ttm_glob;
+	struct ttm_device *bdev;
+	int ret = 0;
+
+	mutex_lock(&ttm_global_mutex);
+	list_for_each_entry(bdev, &glob->device_list, device_list) {
+		do {
+			ret = ttm_device_swapout(bdev, &ctx, GFP_KERNEL);
+		} while (ret > 0);
+		if (ret < 0)
+			break;
+	}
+	mutex_unlock(&ttm_global_mutex);
+	return ret;
+}
+EXPORT_SYMBOL(ttm_device_prepare_hibernation);
+
 /*
  * A buffer object shrink method that tries to swap out the first
  * buffer object on the global::swap_lru list.
diff --git a/include/drm/ttm/ttm_device.h b/include/drm/ttm/ttm_device.h
index 39b8636b1845..b45498b398dd 100644
--- a/include/drm/ttm/ttm_device.h
+++ b/include/drm/ttm/ttm_device.h
@@ -272,6 +272,7 @@ struct ttm_device {
 int ttm_global_swapout(struct ttm_operation_ctx *ctx, gfp_t gfp_flags);
 int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx,
 		       gfp_t gfp_flags);
+int ttm_device_prepare_hibernation(void);
 
 static inline struct ttm_resource_manager *
 ttm_manager_type(struct ttm_device *bdev, int mem_type)
-- 
2.43.5

Re: [PATCH v2 1/5] drm/ttm: add ttm_device_prepare_hibernation() api

Posted by Christian König 3 months ago

On 04.07.25 12:12, Samuel Zhang wrote:
> This new api is used for hibernation to move GTT BOs to shmem after
> VRAM eviction. shmem will be flushed to swap disk later to reduce
> the system memory usage for hibernation.
> 
> Signed-off-by: Samuel Zhang <guoqing.zhang@amd.com>
> ---
>  drivers/gpu/drm/ttm/ttm_device.c | 29 +++++++++++++++++++++++++++++
>  include/drm/ttm/ttm_device.h     |  1 +
>  2 files changed, 30 insertions(+)
> 
> diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
> index 02e797fd1891..19ab35ffeead 100644
> --- a/drivers/gpu/drm/ttm/ttm_device.c
> +++ b/drivers/gpu/drm/ttm/ttm_device.c
> @@ -123,6 +123,35 @@ static int ttm_global_init(void)
>  	return ret;
>  }
>  
> +/**
> + * move GTT BOs to shmem for hibernation.
> + *
> + * returns 0 on success, negative on failure.
> + */
> +int ttm_device_prepare_hibernation(void)

This needs the device as argument.

> +{
> +	struct ttm_operation_ctx ctx = {
> +		.interruptible = false,
> +		.no_wait_gpu = false,
> +		.force_alloc = true
> +	};
> +	struct ttm_global *glob = &ttm_glob;
> +	struct ttm_device *bdev;
> +	int ret = 0;
> +
> +	mutex_lock(&ttm_global_mutex);
> +	list_for_each_entry(bdev, &glob->device_list, device_list) {
> +		do {
> +			ret = ttm_device_swapout(bdev, &ctx, GFP_KERNEL);
> +		} while (ret > 0);
> +		if (ret < 0)
> +			break;
> +	}

In other words call ttm_device_swapout() in a loop for a specific device and not for all devices.

Regards,
Christian.

> +	mutex_unlock(&ttm_global_mutex);
> +	return ret;
> +}
> +EXPORT_SYMBOL(ttm_device_prepare_hibernation);
> +
>  /*
>   * A buffer object shrink method that tries to swap out the first
>   * buffer object on the global::swap_lru list.
> diff --git a/include/drm/ttm/ttm_device.h b/include/drm/ttm/ttm_device.h
> index 39b8636b1845..b45498b398dd 100644
> --- a/include/drm/ttm/ttm_device.h
> +++ b/include/drm/ttm/ttm_device.h
> @@ -272,6 +272,7 @@ struct ttm_device {
>  int ttm_global_swapout(struct ttm_operation_ctx *ctx, gfp_t gfp_flags);
>  int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx,
>  		       gfp_t gfp_flags);
> +int ttm_device_prepare_hibernation(void);
>  
>  static inline struct ttm_resource_manager *
>  ttm_manager_type(struct ttm_device *bdev, int mem_type)

Re: [PATCH v2 1/5] drm/ttm: add ttm_device_prepare_hibernation() api

Posted by Mario Limonciello 3 months ago

On 7/4/2025 6:12 AM, Samuel Zhang wrote:
> This new api is used for hibernation to move GTT BOs to shmem after
> VRAM eviction. shmem will be flushed to swap disk later to reduce
> the system memory usage for hibernation.
> 
> Signed-off-by: Samuel Zhang <guoqing.zhang@amd.com>
> ---
>   drivers/gpu/drm/ttm/ttm_device.c | 29 +++++++++++++++++++++++++++++
>   include/drm/ttm/ttm_device.h     |  1 +
>   2 files changed, 30 insertions(+)
> 
> diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
> index 02e797fd1891..19ab35ffeead 100644
> --- a/drivers/gpu/drm/ttm/ttm_device.c
> +++ b/drivers/gpu/drm/ttm/ttm_device.c
> @@ -123,6 +123,35 @@ static int ttm_global_init(void)
>   	return ret;
>   }
>   
> +/**
> + * move GTT BOs to shmem for hibernation.
> + *
> + * returns 0 on success, negative on failure.
> + */
> +int ttm_device_prepare_hibernation(void)
> +{
> +	struct ttm_operation_ctx ctx = {
> +		.interruptible = false,
> +		.no_wait_gpu = false,
> +		.force_alloc = true
> +	};
> +	struct ttm_global *glob = &ttm_glob;
> +	struct ttm_device *bdev;
> +	int ret = 0;
> +
> +	mutex_lock(&ttm_global_mutex);
> +	list_for_each_entry(bdev, &glob->device_list, device_list) {
> +		do {
> +			ret = ttm_device_swapout(bdev, &ctx, GFP_KERNEL);
> +		} while (ret > 0);
> +		if (ret < 0)
> +			break;
> +	}
> +	mutex_unlock(&ttm_global_mutex);
> +	return ret;

I'd personally rather see scoped guard here so you can return 
immediately and the guard will clean up but up to Christian what he thinks.

int ret;

scoped_guard(mutex, &ttm_global_mutex) {
	list_for_each_entry(bdev, &glob->device_list, device_list) {
		do {
			ret = ttm_device_swapout(bdev, &ctx, GFP_KERNEL);
		} while (ret > 0);
		if (ret)
			return ret;
}

return 0;

> +}
> +EXPORT_SYMBOL(ttm_device_prepare_hibernation);
> +
>   /*
>    * A buffer object shrink method that tries to swap out the first
>    * buffer object on the global::swap_lru list.
> diff --git a/include/drm/ttm/ttm_device.h b/include/drm/ttm/ttm_device.h
> index 39b8636b1845..b45498b398dd 100644
> --- a/include/drm/ttm/ttm_device.h
> +++ b/include/drm/ttm/ttm_device.h
> @@ -272,6 +272,7 @@ struct ttm_device {
>   int ttm_global_swapout(struct ttm_operation_ctx *ctx, gfp_t gfp_flags);
>   int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx,
>   		       gfp_t gfp_flags);
> +int ttm_device_prepare_hibernation(void);
>   
>   static inline struct ttm_resource_manager *
>   ttm_manager_type(struct ttm_device *bdev, int mem_type)

Re: [PATCH v2 1/5] drm/ttm: add ttm_device_prepare_hibernation() api

Posted by Matthew Brost 3 months ago

On Sun, Jul 06, 2025 at 04:44:27PM -0400, Mario Limonciello wrote:
> On 7/4/2025 6:12 AM, Samuel Zhang wrote:
> > This new api is used for hibernation to move GTT BOs to shmem after
> > VRAM eviction. shmem will be flushed to swap disk later to reduce
> > the system memory usage for hibernation.
> > 
> > Signed-off-by: Samuel Zhang <guoqing.zhang@amd.com>
> > ---
> >   drivers/gpu/drm/ttm/ttm_device.c | 29 +++++++++++++++++++++++++++++
> >   include/drm/ttm/ttm_device.h     |  1 +
> >   2 files changed, 30 insertions(+)
> > 
> > diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
> > index 02e797fd1891..19ab35ffeead 100644
> > --- a/drivers/gpu/drm/ttm/ttm_device.c
> > +++ b/drivers/gpu/drm/ttm/ttm_device.c
> > @@ -123,6 +123,35 @@ static int ttm_global_init(void)
> >   	return ret;
> >   }
> > +/**
> > + * move GTT BOs to shmem for hibernation.
> > + *
> > + * returns 0 on success, negative on failure.
> > + */
> > +int ttm_device_prepare_hibernation(void)
> > +{
> > +	struct ttm_operation_ctx ctx = {
> > +		.interruptible = false,
> > +		.no_wait_gpu = false,
> > +		.force_alloc = true
> > +	};
> > +	struct ttm_global *glob = &ttm_glob;
> > +	struct ttm_device *bdev;
> > +	int ret = 0;
> > +
> > +	mutex_lock(&ttm_global_mutex);
> > +	list_for_each_entry(bdev, &glob->device_list, device_list) {
> > +		do {
> > +			ret = ttm_device_swapout(bdev, &ctx, GFP_KERNEL);
> > +		} while (ret > 0);
> > +		if (ret < 0)
> > +			break;
> > +	}
> > +	mutex_unlock(&ttm_global_mutex);
> > +	return ret;
> 
> I'd personally rather see scoped guard here so you can return immediately
> and the guard will clean up but up to Christian what he thinks.
> 
> int ret;
> 
> scoped_guard(mutex, &ttm_global_mutex) {

guard(mutex)(&ttm_global_mutex) would be more apporiate for this as the
scope of the guard is the entire function.

Matt

> 	list_for_each_entry(bdev, &glob->device_list, device_list) {
> 		do {
> 			ret = ttm_device_swapout(bdev, &ctx, GFP_KERNEL);
> 		} while (ret > 0);
> 		if (ret)
> 			return ret;
> }
> 
> return 0;
> 
> > +}
> > +EXPORT_SYMBOL(ttm_device_prepare_hibernation);
> > +
> >   /*
> >    * A buffer object shrink method that tries to swap out the first
> >    * buffer object on the global::swap_lru list.
> > diff --git a/include/drm/ttm/ttm_device.h b/include/drm/ttm/ttm_device.h
> > index 39b8636b1845..b45498b398dd 100644
> > --- a/include/drm/ttm/ttm_device.h
> > +++ b/include/drm/ttm/ttm_device.h
> > @@ -272,6 +272,7 @@ struct ttm_device {
> >   int ttm_global_swapout(struct ttm_operation_ctx *ctx, gfp_t gfp_flags);
> >   int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx,
> >   		       gfp_t gfp_flags);
> > +int ttm_device_prepare_hibernation(void);
> >   static inline struct ttm_resource_manager *
> >   ttm_manager_type(struct ttm_device *bdev, int mem_type)
>