From nobody Fri Dec 19 03:03:12 2025 Received: from mblankhorst.nl (lankhorst.se [141.105.120.124]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BB266197545; Thu, 27 Jun 2024 15:57:52 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=141.105.120.124 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1719503874; cv=none; b=nMnyN9OsKbsMLdt4FZgkofTEHuM2FWyQNa41pCRD6dD5tXA3+xE3xrIjZGvinuMlOJqbs46CdKyKEvGjhqX5GPyr/ATWRo39oyDzkJ/KiNRU6qZBvUiEEGkl4PN1ur7UHxqSM0dfLAnUveBugQtsm8jxt6zYwGR6QNXTb+9YOqc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1719503874; c=relaxed/simple; bh=Rns6XUiZ0ltiJEVjrsqm3d4+k2Dfm45L33s43TEe3Es=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=SMTfzp7/yB9bn2t3t7kWvlFUVUeYxJl9WR3IrFhZHTlwpsrSqf4+kk/LbRBjFsQ62u/40cErjME4Ut0z+kMJ3MwC4pi9d71fDhNn9Vz+dwfa37vDNM86ODyS3FlxTLdfoIr0XMpbL4kLbfT7DA5ozuWn5mfgmm5x8Z6DRW1hNSQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=fail (p=none dis=none) header.from=linux.intel.com; spf=none smtp.mailfrom=mblankhorst.nl; arc=none smtp.client-ip=141.105.120.124 Authentication-Results: smtp.subspace.kernel.org; dmarc=fail (p=none dis=none) header.from=linux.intel.com Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=mblankhorst.nl From: Maarten Lankhorst To: intel-xe@lists.freedesktop.org, linux-kernel@vger.kernel.org, dri-devel@lists.freedesktop.org, Tejun Heo , Zefan Li , Johannes Weiner , Andrew Morton , Christian Koenig , Huang Rui , Maarten Lankhorst , Maxime Ripard , Thomas Zimmermann , David Airlie , Daniel Vetter Cc: Friedrich Vock , cgroups@vger.kernel.org, linux-mm@kvack.org Subject: [RFC PATCH 3/6] drm/ttm: Handle cgroup based eviction in TTM Date: Thu, 27 Jun 2024 17:47:22 +0200 Message-ID: <20240627154754.74828-4-maarten.lankhorst@linux.intel.com> X-Mailer: git-send-email 2.45.2 In-Reply-To: <20240627154754.74828-1-maarten.lankhorst@linux.intel.com> References: <20240627154754.74828-1-maarten.lankhorst@linux.intel.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" cgroup resource allocation has to be handled in TTM, so -EAGAIN from cgroups can be converted into -ENOSPC, and the limitcg can be properly evicted in ttm code. When hitting a resource limit through -EAGAIN, the cgroup for which the limit is hit is also returned. This allows eviction to delete only from cgroups which are a subgroup of the current cgroup. The returned CSS is used to determine if eviction is valuable for a given resource, and allows TTM to only target specific resources to lower memory usage. Signed-off-by: Maarten Lankhorst Co-developed-by: Friedrich Vock --- drivers/gpu/drm/ttm/tests/ttm_bo_test.c | 18 ++++----- drivers/gpu/drm/ttm/tests/ttm_resource_test.c | 2 +- drivers/gpu/drm/ttm/ttm_bo.c | 38 ++++++++++++++++--- drivers/gpu/drm/ttm/ttm_resource.c | 28 ++++++++++++-- include/drm/ttm/ttm_bo.h | 3 +- include/drm/ttm/ttm_resource.h | 16 +++++++- 6 files changed, 84 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c b/drivers/gpu/drm/ttm/= tests/ttm_bo_test.c index 1f8a4f8adc92..e2adc336dda8 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c @@ -252,13 +252,13 @@ static void ttm_bo_unreserve_basic(struct kunit *test) bo =3D ttm_bo_kunit_init(test, test->priv, BO_SIZE); bo->priority =3D bo_prio; =20 - err =3D ttm_resource_alloc(bo, place, &res1); + err =3D ttm_resource_alloc(bo, place, &res1, NULL); KUNIT_ASSERT_EQ(test, err, 0); =20 bo->resource =3D res1; =20 /* Add a dummy resource to populate LRU */ - ttm_resource_alloc(bo, place, &res2); + ttm_resource_alloc(bo, place, &res2, NULL); =20 dma_resv_lock(bo->base.resv, NULL); ttm_bo_unreserve(bo); @@ -294,12 +294,12 @@ static void ttm_bo_unreserve_pinned(struct kunit *tes= t) dma_resv_lock(bo->base.resv, NULL); ttm_bo_pin(bo); =20 - err =3D ttm_resource_alloc(bo, place, &res1); + err =3D ttm_resource_alloc(bo, place, &res1, NULL); KUNIT_ASSERT_EQ(test, err, 0); bo->resource =3D res1; =20 /* Add a dummy resource to the pinned list */ - err =3D ttm_resource_alloc(bo, place, &res2); + err =3D ttm_resource_alloc(bo, place, &res2, NULL); KUNIT_ASSERT_EQ(test, err, 0); KUNIT_ASSERT_EQ(test, list_is_last(&res2->lru, &priv->ttm_dev->pinned), 1); @@ -343,7 +343,7 @@ static void ttm_bo_unreserve_bulk(struct kunit *test) ttm_bo_set_bulk_move(bo1, &lru_bulk_move); dma_resv_unlock(bo1->base.resv); =20 - err =3D ttm_resource_alloc(bo1, place, &res1); + err =3D ttm_resource_alloc(bo1, place, &res1, NULL); KUNIT_ASSERT_EQ(test, err, 0); bo1->resource =3D res1; =20 @@ -351,7 +351,7 @@ static void ttm_bo_unreserve_bulk(struct kunit *test) ttm_bo_set_bulk_move(bo2, &lru_bulk_move); dma_resv_unlock(bo2->base.resv); =20 - err =3D ttm_resource_alloc(bo2, place, &res2); + err =3D ttm_resource_alloc(bo2, place, &res2, NULL); KUNIT_ASSERT_EQ(test, err, 0); bo2->resource =3D res2; =20 @@ -387,7 +387,7 @@ static void ttm_bo_put_basic(struct kunit *test) bo =3D ttm_bo_kunit_init(test, test->priv, BO_SIZE); bo->type =3D ttm_bo_type_device; =20 - err =3D ttm_resource_alloc(bo, place, &res); + err =3D ttm_resource_alloc(bo, place, &res, NULL); KUNIT_ASSERT_EQ(test, err, 0); bo->resource =3D res; =20 @@ -504,7 +504,7 @@ static void ttm_bo_pin_unpin_resource(struct kunit *tes= t) =20 bo =3D ttm_bo_kunit_init(test, test->priv, BO_SIZE); =20 - err =3D ttm_resource_alloc(bo, place, &res); + err =3D ttm_resource_alloc(bo, place, &res, NULL); KUNIT_ASSERT_EQ(test, err, 0); bo->resource =3D res; =20 @@ -555,7 +555,7 @@ static void ttm_bo_multiple_pin_one_unpin(struct kunit = *test) =20 bo =3D ttm_bo_kunit_init(test, test->priv, BO_SIZE); =20 - err =3D ttm_resource_alloc(bo, place, &res); + err =3D ttm_resource_alloc(bo, place, &res, NULL); KUNIT_ASSERT_EQ(test, err, 0); bo->resource =3D res; =20 diff --git a/drivers/gpu/drm/ttm/tests/ttm_resource_test.c b/drivers/gpu/dr= m/ttm/tests/ttm_resource_test.c index 029e1f094bb0..c7d3d86ff98b 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_resource_test.c +++ b/drivers/gpu/drm/ttm/tests/ttm_resource_test.c @@ -302,7 +302,7 @@ static void ttm_sys_man_free_basic(struct kunit *test) res =3D kunit_kzalloc(test, sizeof(*res), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, res); =20 - ttm_resource_alloc(bo, place, &res); + ttm_resource_alloc(bo, place, &res, NULL, NULL); =20 man =3D ttm_manager_type(priv->devs->ttm_dev, mem_type); man->func->free(man, res); diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 6396dece0db1..6ca92b64f2fe 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -42,6 +42,7 @@ #include #include #include +#include #include =20 #include "ttm_module.h" @@ -594,18 +595,24 @@ int ttm_mem_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, const struct ttm_place *place, struct ttm_operation_ctx *ctx, - struct ww_acquire_ctx *ticket) + struct ww_acquire_ctx *ticket, + struct drmcgroup_pool_state *limitcss) { struct ttm_buffer_object *bo =3D NULL, *busy_bo =3D NULL; struct ttm_resource_cursor cursor; struct ttm_resource *res; bool locked =3D false; int ret; + bool try_low =3D false, hit_low =3D false; =20 spin_lock(&bdev->lru_lock); +retry: ttm_resource_manager_for_each_res(man, &cursor, res) { bool busy; =20 + if (!drmcs_evict_valuable(limitcss, man->cgdev, man->cgidx, res->css, tr= y_low, &hit_low)) + continue; + if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place, &locked, &busy)) { if (busy && !busy_bo && ticket !=3D @@ -623,13 +630,25 @@ int ttm_mem_evict_first(struct ttm_device *bdev, } =20 if (!bo) { + if (!ticket && !try_low && hit_low) + goto hit_low; + if (busy_bo && !ttm_bo_get_unless_zero(busy_bo)) busy_bo =3D NULL; + + if (!busy_bo && !try_low && hit_low) + goto hit_low; + spin_unlock(&bdev->lru_lock); ret =3D ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); if (busy_bo) ttm_bo_put(busy_bo); return ret; + +hit_low: + busy_bo =3D NULL; + try_low =3D true; + goto retry; } =20 if (bo->deleted) { @@ -769,14 +788,19 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_ob= ject *bo, continue; =20 do { - ret =3D ttm_resource_alloc(bo, place, res); - if (unlikely(ret && ret !=3D -ENOSPC)) + struct drmcgroup_pool_state *limitcss =3D NULL; + + ret =3D ttm_resource_alloc(bo, place, res, force_space ? &limitcss : NU= LL); + if (unlikely(ret && ret !=3D -ENOSPC && ret !=3D -EAGAIN)) { + drmcs_pool_put(limitcss); return ret; + } if (likely(!ret) || !force_space) break; =20 ret =3D ttm_mem_evict_first(bdev, man, place, ctx, - ticket); + ticket, limitcss); + drmcs_pool_put(limitcss); if (unlikely(ret =3D=3D -EBUSY)) break; if (unlikely(ret)) @@ -1162,7 +1186,7 @@ int ttm_bo_swapout(struct ttm_buffer_object *bo, stru= ct ttm_operation_ctx *ctx, =20 memset(&hop, 0, sizeof(hop)); place.mem_type =3D TTM_PL_SYSTEM; - ret =3D ttm_resource_alloc(bo, &place, &evict_mem); + ret =3D ttm_resource_alloc(bo, &place, &evict_mem, NULL); if (unlikely(ret)) goto out; =20 @@ -1201,7 +1225,9 @@ int ttm_bo_swapout(struct ttm_buffer_object *bo, stru= ct ttm_operation_ctx *ctx, if (locked) dma_resv_unlock(bo->base.resv); ttm_bo_put(bo); - return ret =3D=3D -EBUSY ? -ENOSPC : ret; + if (ret =3D=3D -EAGAIN || ret =3D=3D -EBUSY) + return -ENOSPC; + return ret; } =20 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_r= esource.c index 4a66b851b67d..1a8312afe247 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -26,6 +26,7 @@ #include #include #include +#include =20 #include #include @@ -229,15 +230,31 @@ EXPORT_SYMBOL(ttm_resource_fini); =20 int ttm_resource_alloc(struct ttm_buffer_object *bo, const struct ttm_place *place, - struct ttm_resource **res_ptr) + struct ttm_resource **res_ptr, + struct drmcgroup_pool_state **limitcs) { struct ttm_resource_manager *man =3D ttm_manager_type(bo->bdev, place->mem_type); + struct drmcgroup_pool_state *drmcs =3D NULL; int ret; =20 + if (man->cgdev) { + ret =3D drmcg_try_charge(&drmcs, limitcs, + man->cgdev, man->cgidx, + bo->base.size); + if (ret) + return ret; + } + ret =3D man->func->alloc(man, bo, place, res_ptr); - if (ret) + if (ret) { + if (drmcs) + drmcg_uncharge(drmcs, man->cgdev, man->cgidx, + bo->base.size); return ret; + } + + (*res_ptr)->css =3D drmcs; =20 spin_lock(&bo->bdev->lru_lock); ttm_resource_add_bulk_move(*res_ptr, bo); @@ -249,6 +266,7 @@ EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc); void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource *= *res) { struct ttm_resource_manager *man; + struct drmcgroup_pool_state *css; =20 if (!*res) return; @@ -256,9 +274,13 @@ void ttm_resource_free(struct ttm_buffer_object *bo, s= truct ttm_resource **res) spin_lock(&bo->bdev->lru_lock); ttm_resource_del_bulk_move(*res, bo); spin_unlock(&bo->bdev->lru_lock); + + css =3D (*res)->css; man =3D ttm_manager_type(bo->bdev, (*res)->mem_type); man->func->free(man, *res); *res =3D NULL; + if (man->cgdev) + drmcg_uncharge(css, man->cgdev, man->cgidx, bo->base.size); } EXPORT_SYMBOL(ttm_resource_free); =20 @@ -401,7 +423,7 @@ int ttm_resource_manager_evict_all(struct ttm_device *b= dev, while (!list_empty(&man->lru[i])) { spin_unlock(&bdev->lru_lock); ret =3D ttm_mem_evict_first(bdev, man, NULL, &ctx, - NULL); + NULL, NULL); if (ret) return ret; spin_lock(&bdev->lru_lock); diff --git a/include/drm/ttm/ttm_bo.h b/include/drm/ttm/ttm_bo.h index 6ccf96c91f3a..709a30b6bfe6 100644 --- a/include/drm/ttm/ttm_bo.h +++ b/include/drm/ttm/ttm_bo.h @@ -386,7 +386,8 @@ int ttm_mem_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, const struct ttm_place *place, struct ttm_operation_ctx *ctx, - struct ww_acquire_ctx *ticket); + struct ww_acquire_ctx *ticket, + struct drmcgroup_pool_state *limitcg); vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, struct vm_fault *vmf); vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h index 69769355139f..0574ebb226cb 100644 --- a/include/drm/ttm/ttm_resource.h +++ b/include/drm/ttm/ttm_resource.h @@ -38,6 +38,7 @@ #define TTM_MAX_BO_PRIORITY 4U #define TTM_NUM_MEM_TYPES 8 =20 +struct drmcgroup_device; struct ttm_device; struct ttm_resource_manager; struct ttm_resource; @@ -174,6 +175,15 @@ struct ttm_resource_manager { * bdev->lru_lock. */ uint64_t usage; + + /** + * @cgdev: drmcgroup_device used for memory accounting, if not NULL. + */ + struct drmcgroup_device *cgdev; + /** + * @cgidx: Resource index used by this resource manager for cgroup accoun= ting + */ + u32 cgidx; }; =20 /** @@ -202,6 +212,7 @@ struct ttm_bus_placement { * @placement: Placement flags. * @bus: Placement on io bus accessible to the CPU * @bo: weak reference to the BO, protected by ttm_device::lru_lock + * @css: cgroup state this resource is charged to * * Structure indicating the placement and space resources used by a * buffer object. @@ -214,6 +225,8 @@ struct ttm_resource { struct ttm_bus_placement bus; struct ttm_buffer_object *bo; =20 + struct drmcgroup_pool_state *css; + /** * @lru: Least recently used list, see &ttm_resource_manager.lru */ @@ -362,7 +375,8 @@ void ttm_resource_fini(struct ttm_resource_manager *man, =20 int ttm_resource_alloc(struct ttm_buffer_object *bo, const struct ttm_place *place, - struct ttm_resource **res); + struct ttm_resource **res, + struct drmcgroup_pool_state **limitcs); void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource *= *res); bool ttm_resource_intersects(struct ttm_device *bdev, struct ttm_resource *res, --=20 2.45.2