From nobody Mon Feb 9 02:27:18 2026 Received: from madrid.collaboradmins.com (madrid.collaboradmins.com [46.235.227.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4CD523C498 for ; Fri, 5 Jan 2024 18:47:34 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=collabora.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=collabora.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=collabora.com header.i=@collabora.com header.b="fEQPp3pm" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=collabora.com; s=mail; t=1704480452; bh=aX78HPzWZ5Oplw/Z9BqINCxZ65QTKKO+E85eRzWNe7o=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=fEQPp3pmDG+iVYgJIHsvF2gCGoCgY5eTAg8EJmsHXStLWn22zl361EWtYYAMHWN0O v7AoWXW5gn7FQHdeujiNDZlGo9xCXPdiEsvwbz7yzhQRJzTJKHpiWKYy2kKyNO45gL iLBD3pJU8/jATothCgc5M/L+SYC72gW6bUprIJhH3PUq98JLvxUgg+H3davLbFNXaU 54SDKYwrHQwcYJktMmp7AZtlm9qH/seJlZOVkS8mhaJIlkThhLqp9htWmmtAJa3/l8 TJsNcfN7H4vTbQdFbdcBYBxWw82mtGQvWBeBwSYZAZrjn31NpzgUp6tQCacxUJpF21 xg7/Cke6XNETg== Received: from workpc.. (cola.collaboradmins.com [195.201.22.229]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) (Authenticated sender: dmitry.osipenko) by madrid.collaboradmins.com (Postfix) with ESMTPSA id 928B3378204D; Fri, 5 Jan 2024 18:47:31 +0000 (UTC) From: Dmitry Osipenko To: David Airlie , Gerd Hoffmann , Gurchetan Singh , Chia-I Wu , Daniel Vetter , Maarten Lankhorst , Maxime Ripard , Thomas Zimmermann , =?UTF-8?q?Christian=20K=C3=B6nig?= , Qiang Yu , Steven Price , Boris Brezillon , Emma Anholt , Melissa Wen Cc: dri-devel@lists.freedesktop.org, linux-kernel@vger.kernel.org, kernel@collabora.com, virtualization@lists.linux-foundation.org Subject: [PATCH v19 30/30] drm/panfrost: Switch to generic memory shrinker Date: Fri, 5 Jan 2024 21:46:24 +0300 Message-ID: <20240105184624.508603-31-dmitry.osipenko@collabora.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20240105184624.508603-1-dmitry.osipenko@collabora.com> References: <20240105184624.508603-1-dmitry.osipenko@collabora.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Replace Panfrost's custom memory shrinker with a common drm-shmem memory shrinker. Co-developed-by: Boris Brezillon Signed-off-by: Boris Brezillon Signed-off-by: Dmitry Osipenko --- drivers/gpu/drm/drm_gem_shmem_helper.c | 4 +- drivers/gpu/drm/panfrost/Makefile | 1 - drivers/gpu/drm/panfrost/panfrost_device.h | 4 - drivers/gpu/drm/panfrost/panfrost_drv.c | 29 ++-- drivers/gpu/drm/panfrost/panfrost_gem.c | 60 ++++---- drivers/gpu/drm/panfrost/panfrost_gem.h | 9 -- .../gpu/drm/panfrost/panfrost_gem_shrinker.c | 140 ------------------ drivers/gpu/drm/panfrost/panfrost_job.c | 18 ++- drivers/gpu/drm/panfrost/panfrost_mmu.c | 24 ++- include/drm/drm_gem_shmem_helper.h | 7 - 10 files changed, 83 insertions(+), 213 deletions(-) delete mode 100644 drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_g= em_shmem_helper.c index 7d2fe12bd793..56e88378079b 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -89,8 +89,6 @@ __drm_gem_shmem_create(struct drm_device *dev, size_t siz= e, bool private) if (ret) goto err_release; =20 - INIT_LIST_HEAD(&shmem->madv_list); - if (!private) { /* * Our buffers are kept pinned, so allocating them @@ -619,6 +617,8 @@ void drm_gem_shmem_purge_locked(struct drm_gem_shmem_ob= ject *shmem) { struct drm_gem_object *obj =3D &shmem->base; =20 + drm_WARN_ON_ONCE(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); + drm_gem_shmem_shrinker_put_pages_locked(shmem); drm_gem_free_mmap_offset(obj); =20 diff --git a/drivers/gpu/drm/panfrost/Makefile b/drivers/gpu/drm/panfrost/M= akefile index 2c01c1e7523e..f2cb1ab0a32d 100644 --- a/drivers/gpu/drm/panfrost/Makefile +++ b/drivers/gpu/drm/panfrost/Makefile @@ -5,7 +5,6 @@ panfrost-y :=3D \ panfrost_device.o \ panfrost_devfreq.o \ panfrost_gem.o \ - panfrost_gem_shrinker.o \ panfrost_gpu.o \ panfrost_job.o \ panfrost_mmu.o \ diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/p= anfrost/panfrost_device.h index 62f7e3527385..cea6df9cd650 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.h +++ b/drivers/gpu/drm/panfrost/panfrost_device.h @@ -140,10 +140,6 @@ struct panfrost_device { atomic_t pending; } reset; =20 - struct mutex shrinker_lock; - struct list_head shrinker_list; - struct shrinker *shrinker; - struct panfrost_devfreq pfdevfreq; =20 struct { diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panf= rost/panfrost_drv.c index a15d62f19afb..5c730d15a24d 100644 --- a/drivers/gpu/drm/panfrost/panfrost_drv.c +++ b/drivers/gpu/drm/panfrost/panfrost_drv.c @@ -171,7 +171,6 @@ panfrost_lookup_bos(struct drm_device *dev, break; } =20 - atomic_inc(&bo->gpu_usecount); job->mappings[i] =3D mapping; } =20 @@ -397,7 +396,6 @@ static int panfrost_ioctl_madvise(struct drm_device *de= v, void *data, { struct panfrost_file_priv *priv =3D file_priv->driver_priv; struct drm_panfrost_madvise *args =3D data; - struct panfrost_device *pfdev =3D dev->dev_private; struct drm_gem_object *gem_obj; struct panfrost_gem_object *bo; int ret =3D 0; @@ -410,11 +408,15 @@ static int panfrost_ioctl_madvise(struct drm_device *= dev, void *data, =20 bo =3D to_panfrost_bo(gem_obj); =20 + if (bo->is_heap) { + args->retained =3D 1; + goto out_put_object; + } + ret =3D dma_resv_lock_interruptible(bo->base.base.resv, NULL); if (ret) goto out_put_object; =20 - mutex_lock(&pfdev->shrinker_lock); mutex_lock(&bo->mappings.lock); if (args->madv =3D=3D PANFROST_MADV_DONTNEED) { struct panfrost_gem_mapping *first; @@ -440,17 +442,8 @@ static int panfrost_ioctl_madvise(struct drm_device *d= ev, void *data, =20 args->retained =3D drm_gem_shmem_madvise_locked(&bo->base, args->madv); =20 - if (args->retained) { - if (args->madv =3D=3D PANFROST_MADV_DONTNEED) - list_move_tail(&bo->base.madv_list, - &pfdev->shrinker_list); - else if (args->madv =3D=3D PANFROST_MADV_WILLNEED) - list_del_init(&bo->base.madv_list); - } - out_unlock_mappings: mutex_unlock(&bo->mappings.lock); - mutex_unlock(&pfdev->shrinker_lock); dma_resv_unlock(bo->base.base.resv); out_put_object: drm_gem_object_put(gem_obj); @@ -635,9 +628,6 @@ static int panfrost_probe(struct platform_device *pdev) ddev->dev_private =3D pfdev; pfdev->ddev =3D ddev; =20 - mutex_init(&pfdev->shrinker_lock); - INIT_LIST_HEAD(&pfdev->shrinker_list); - err =3D panfrost_device_init(pfdev); if (err) { if (err !=3D -EPROBE_DEFER) @@ -659,13 +649,13 @@ static int panfrost_probe(struct platform_device *pde= v) if (err < 0) goto err_out1; =20 - err =3D panfrost_gem_shrinker_init(ddev); - if (err) - goto err_out2; + err =3D drmm_gem_shmem_init(ddev); + if (err < 0) + goto err_unregister_dev; =20 return 0; =20 -err_out2: +err_unregister_dev: drm_dev_unregister(ddev); err_out1: pm_runtime_disable(pfdev->dev); @@ -682,7 +672,6 @@ static void panfrost_remove(struct platform_device *pde= v) struct drm_device *ddev =3D pfdev->ddev; =20 drm_dev_unregister(ddev); - panfrost_gem_shrinker_cleanup(ddev); =20 pm_runtime_get_sync(pfdev->dev); pm_runtime_disable(pfdev->dev); diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panf= rost/panfrost_gem.c index 8c26b7e41b95..05eb5a89c4ed 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem.c +++ b/drivers/gpu/drm/panfrost/panfrost_gem.c @@ -17,17 +17,6 @@ static void panfrost_gem_free_object(struct drm_gem_object *obj) { struct panfrost_gem_object *bo =3D to_panfrost_bo(obj); - struct panfrost_device *pfdev =3D obj->dev->dev_private; - - /* - * Make sure the BO is no longer inserted in the shrinker list before - * taking care of the destruction itself. If we don't do that we have a - * race condition between this function and what's done in - * panfrost_gem_shrinker_scan(). - */ - mutex_lock(&pfdev->shrinker_lock); - list_del_init(&bo->base.madv_list); - mutex_unlock(&pfdev->shrinker_lock); =20 /* * If we still have mappings attached to the BO, there's a problem in @@ -57,26 +46,23 @@ panfrost_gem_mapping_get(struct panfrost_gem_object *bo, return mapping; } =20 -static void -panfrost_gem_teardown_mapping(struct panfrost_gem_mapping *mapping) +static void panfrost_gem_mapping_release(struct kref *kref) { + struct panfrost_gem_mapping *mapping =3D + container_of(kref, struct panfrost_gem_mapping, refcount); + struct panfrost_gem_object *bo =3D mapping->obj; + struct panfrost_device *pfdev =3D bo->base.base.dev->dev_private; + + /* Shrinker may purge the mapping at the same time. */ + dma_resv_lock(mapping->obj->base.base.resv, NULL); if (mapping->active) panfrost_mmu_unmap(mapping); + dma_resv_unlock(mapping->obj->base.base.resv); =20 spin_lock(&mapping->mmu->mm_lock); if (drm_mm_node_allocated(&mapping->mmnode)) drm_mm_remove_node(&mapping->mmnode); spin_unlock(&mapping->mmu->mm_lock); -} - -static void panfrost_gem_mapping_release(struct kref *kref) -{ - struct panfrost_gem_mapping *mapping =3D - container_of(kref, struct panfrost_gem_mapping, refcount); - struct panfrost_gem_object *bo =3D mapping->obj; - struct panfrost_device *pfdev =3D bo->base.base.dev->dev_private; - - panfrost_gem_teardown_mapping(mapping); =20 /* On heap BOs, release the sgts created in the fault handler path. */ if (bo->sgts) { @@ -117,12 +103,14 @@ void panfrost_gem_mapping_put(struct panfrost_gem_map= ping *mapping) kref_put(&mapping->refcount, panfrost_gem_mapping_release); } =20 -void panfrost_gem_teardown_mappings_locked(struct panfrost_gem_object *bo) +void panfrost_gem_evict_mappings_locked(struct panfrost_gem_object *bo) { struct panfrost_gem_mapping *mapping; =20 - list_for_each_entry(mapping, &bo->mappings.list, node) - panfrost_gem_teardown_mapping(mapping); + list_for_each_entry(mapping, &bo->mappings.list, node) { + if (mapping->active) + panfrost_mmu_unmap(mapping); + } } =20 int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_pr= iv) @@ -251,6 +239,25 @@ static size_t panfrost_gem_rss(struct drm_gem_object *= obj) return 0; } =20 +static int panfrost_shmem_evict(struct drm_gem_object *obj) +{ + struct panfrost_gem_object *bo =3D to_panfrost_bo(obj); + + if (!drm_gem_shmem_is_purgeable(&bo->base)) + return -EBUSY; + + if (!mutex_trylock(&bo->mappings.lock)) + return -EBUSY; + + panfrost_gem_evict_mappings_locked(bo); + + drm_gem_shmem_purge_locked(&bo->base); + + mutex_unlock(&bo->mappings.lock); + + return 0; +} + static const struct drm_gem_object_funcs panfrost_gem_funcs =3D { .free =3D panfrost_gem_free_object, .open =3D panfrost_gem_open, @@ -265,6 +272,7 @@ static const struct drm_gem_object_funcs panfrost_gem_f= uncs =3D { .status =3D panfrost_gem_status, .rss =3D panfrost_gem_rss, .vm_ops =3D &drm_gem_shmem_vm_ops, + .evict =3D panfrost_shmem_evict, }; =20 /** diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.h b/drivers/gpu/drm/panf= rost/panfrost_gem.h index 7516b7ecf7fe..8ddc2d310d29 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem.h +++ b/drivers/gpu/drm/panfrost/panfrost_gem.h @@ -30,12 +30,6 @@ struct panfrost_gem_object { struct mutex lock; } mappings; =20 - /* - * Count the number of jobs referencing this BO so we don't let the - * shrinker reclaim this object prematurely. - */ - atomic_t gpu_usecount; - /* * Object chunk size currently mapped onto physical memory */ @@ -86,7 +80,4 @@ panfrost_gem_mapping_get(struct panfrost_gem_object *bo, void panfrost_gem_mapping_put(struct panfrost_gem_mapping *mapping); void panfrost_gem_teardown_mappings_locked(struct panfrost_gem_object *bo); =20 -int panfrost_gem_shrinker_init(struct drm_device *dev); -void panfrost_gem_shrinker_cleanup(struct drm_device *dev); - #endif /* __PANFROST_GEM_H__ */ diff --git a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c b/drivers/gpu= /drm/panfrost/panfrost_gem_shrinker.c deleted file mode 100644 index 7b4deba803ed..000000000000 --- a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2019 Arm Ltd. - * - * Based on msm_gem_freedreno.c: - * Copyright (C) 2016 Red Hat - * Author: Rob Clark - */ - -#include - -#include -#include - -#include "panfrost_device.h" -#include "panfrost_gem.h" -#include "panfrost_mmu.h" - -static bool panfrost_gem_shmem_is_purgeable(struct drm_gem_shmem_object *s= hmem) -{ - return (shmem->madv > 0) && - !refcount_read(&shmem->pages_pin_count) && shmem->sgt && - !shmem->base.dma_buf && !shmem->base.import_attach; -} - -static unsigned long -panfrost_gem_shrinker_count(struct shrinker *shrinker, struct shrink_contr= ol *sc) -{ - struct panfrost_device *pfdev =3D shrinker->private_data; - struct drm_gem_shmem_object *shmem; - unsigned long count =3D 0; - - if (!mutex_trylock(&pfdev->shrinker_lock)) - return 0; - - list_for_each_entry(shmem, &pfdev->shrinker_list, madv_list) { - if (panfrost_gem_shmem_is_purgeable(shmem)) - count +=3D shmem->base.size >> PAGE_SHIFT; - } - - mutex_unlock(&pfdev->shrinker_lock); - - return count; -} - -static bool panfrost_gem_purge(struct drm_gem_object *obj) -{ - struct drm_gem_shmem_object *shmem =3D to_drm_gem_shmem_obj(obj); - struct panfrost_gem_object *bo =3D to_panfrost_bo(obj); - bool ret =3D false; - - if (atomic_read(&bo->gpu_usecount)) - return false; - - if (!mutex_trylock(&bo->mappings.lock)) - return false; - - if (!dma_resv_trylock(shmem->base.resv)) - goto unlock_mappings; - - /* BO might have become unpurgeable if the last pages_use_count ref - * was dropped, but the BO hasn't been destroyed yet. - */ - if (!panfrost_gem_shmem_is_purgeable(shmem)) - goto unlock_mappings; - - panfrost_gem_teardown_mappings_locked(bo); - drm_gem_shmem_purge_locked(&bo->base); - ret =3D true; - - dma_resv_unlock(shmem->base.resv); - -unlock_mappings: - mutex_unlock(&bo->mappings.lock); - return ret; -} - -static unsigned long -panfrost_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_contro= l *sc) -{ - struct panfrost_device *pfdev =3D shrinker->private_data; - struct drm_gem_shmem_object *shmem, *tmp; - unsigned long freed =3D 0; - - if (!mutex_trylock(&pfdev->shrinker_lock)) - return SHRINK_STOP; - - list_for_each_entry_safe(shmem, tmp, &pfdev->shrinker_list, madv_list) { - if (freed >=3D sc->nr_to_scan) - break; - if (panfrost_gem_shmem_is_purgeable(shmem) && - panfrost_gem_purge(&shmem->base)) { - freed +=3D shmem->base.size >> PAGE_SHIFT; - list_del_init(&shmem->madv_list); - } - } - - mutex_unlock(&pfdev->shrinker_lock); - - if (freed > 0) - pr_info_ratelimited("Purging %lu bytes\n", freed << PAGE_SHIFT); - - return freed; -} - -/** - * panfrost_gem_shrinker_init - Initialize panfrost shrinker - * @dev: DRM device - * - * This function registers and sets up the panfrost shrinker. - */ -int panfrost_gem_shrinker_init(struct drm_device *dev) -{ - struct panfrost_device *pfdev =3D dev->dev_private; - - pfdev->shrinker =3D shrinker_alloc(0, "drm-panfrost"); - if (!pfdev->shrinker) - return -ENOMEM; - - pfdev->shrinker->count_objects =3D panfrost_gem_shrinker_count; - pfdev->shrinker->scan_objects =3D panfrost_gem_shrinker_scan; - pfdev->shrinker->private_data =3D pfdev; - - shrinker_register(pfdev->shrinker); - - return 0; -} - -/** - * panfrost_gem_shrinker_cleanup - Clean up panfrost shrinker - * @dev: DRM device - * - * This function unregisters the panfrost shrinker. - */ -void panfrost_gem_shrinker_cleanup(struct drm_device *dev) -{ - struct panfrost_device *pfdev =3D dev->dev_private; - - if (pfdev->shrinker) - shrinker_free(pfdev->shrinker); -} diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panf= rost/panfrost_job.c index 0c2dbf6ef2a5..9e26cb013191 100644 --- a/drivers/gpu/drm/panfrost/panfrost_job.c +++ b/drivers/gpu/drm/panfrost/panfrost_job.c @@ -289,6 +289,19 @@ static void panfrost_attach_object_fences(struct drm_g= em_object **bos, dma_resv_add_fence(bos[i]->resv, fence, DMA_RESV_USAGE_WRITE); } =20 +static int panfrost_objects_prepare(struct drm_gem_object **bos, int bo_co= unt) +{ + struct panfrost_gem_object *bo; + int ret =3D 0; + + while (!ret && bo_count--) { + bo =3D to_panfrost_bo(bos[bo_count]); + ret =3D bo->base.madv !=3D PANFROST_MADV_WILLNEED ? -EINVAL : 0; + } + + return ret; +} + int panfrost_job_push(struct panfrost_job *job) { struct panfrost_device *pfdev =3D job->pfdev; @@ -300,6 +313,10 @@ int panfrost_job_push(struct panfrost_job *job) if (ret) return ret; =20 + ret =3D panfrost_objects_prepare(job->bos, job->bo_count); + if (ret) + goto unlock; + mutex_lock(&pfdev->sched_lock); drm_sched_job_arm(&job->base); =20 @@ -341,7 +358,6 @@ static void panfrost_job_cleanup(struct kref *ref) if (!job->mappings[i]) break; =20 - atomic_dec(&job->mappings[i]->obj->gpu_usecount); panfrost_gem_mapping_put(job->mappings[i]); } kvfree(job->mappings); diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panf= rost/panfrost_mmu.c index 4a0b4bf03f1a..22e18f7986e7 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -328,6 +328,7 @@ int panfrost_mmu_map(struct panfrost_gem_mapping *mappi= ng) struct panfrost_device *pfdev =3D to_panfrost_device(obj->dev); struct sg_table *sgt; int prot =3D IOMMU_READ | IOMMU_WRITE; + int ret =3D 0; =20 if (WARN_ON(mapping->active)) return 0; @@ -335,15 +336,32 @@ int panfrost_mmu_map(struct panfrost_gem_mapping *map= ping) if (bo->noexec) prot |=3D IOMMU_NOEXEC; =20 + if (!obj->import_attach) { + /* + * Don't allow shrinker to move pages while pages are mapped. + * It's fine to move pages afterwards because shrinker will + * take care of unmapping pages during eviction. + */ + ret =3D drm_gem_shmem_pin(shmem); + if (ret) + return ret; + } + sgt =3D drm_gem_shmem_get_pages_sgt(shmem); - if (WARN_ON(IS_ERR(sgt))) - return PTR_ERR(sgt); + if (WARN_ON(IS_ERR(sgt))) { + ret =3D PTR_ERR(sgt); + goto unpin; + } =20 mmu_map_sg(pfdev, mapping->mmu, mapping->mmnode.start << PAGE_SHIFT, prot, sgt); mapping->active =3D true; =20 - return 0; +unpin: + if (!obj->import_attach) + drm_gem_shmem_unpin(shmem); + + return ret; } =20 void panfrost_mmu_unmap(struct panfrost_gem_mapping *mapping) diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem= _helper.h index 167f00f089de..9c6bb00260fc 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -61,13 +61,6 @@ struct drm_gem_shmem_object { */ int madv; =20 - /** - * @madv_list: List entry for madvise tracking - * - * Typically used by drivers to track purgeable objects - */ - struct list_head madv_list; - /** * @sgt: Scatter/gather table for imported PRIME buffers */ --=20 2.43.0