From nobody Fri Dec 19 00:38:04 2025 Received: from out-175.mta0.migadu.com (out-175.mta0.migadu.com [91.218.175.175]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7524C883B for ; Thu, 28 Dec 2023 09:46:36 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; dmarc=fail (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. From: Chengming Zhou Date: Thu, 28 Dec 2023 09:45:46 +0000 Subject: [PATCH v5 5/5] mm/zswap: change per-cpu mutex and buffer to per-acomp_ctx Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Message-Id: <20231213-zswap-dstmem-v5-5-9382162bbf05@bytedance.com> References: <20231213-zswap-dstmem-v5-0-9382162bbf05@bytedance.com> In-Reply-To: <20231213-zswap-dstmem-v5-0-9382162bbf05@bytedance.com> To: Barry Song <21cnbao@gmail.com>, Yosry Ahmed , Nhat Pham , Andrew Morton , Dan Streetman , Vitaly Wool , Johannes Weiner , Chris Li , Seth Jennings Cc: Yosry Ahmed , Nhat Pham , Chris Li , linux-mm@kvack.org, linux-kernel@vger.kernel.org, Chengming Zhou X-Developer-Signature: v=1; a=ed25519-sha256; t=1703756775; l=7667; i=zhouchengming@bytedance.com; s=20231204; h=from:subject:message-id; bh=4WwCjcTM4cvOikkhRW0eIrRaRjoJsRX6DvdiNXR+aoM=; b=zjkyeZ8GJvLmdKBjLPG2tkPu01IHk9nTR5X4BWfcrNSIJNqRsEvvUkf2037UBEjkA7VN2xRdg U2WvnDymn6ZCVI0TllIFnqKc5NdiqSdYpssVP3etAh5tKo+MGJxgduq X-Developer-Key: i=zhouchengming@bytedance.com; a=ed25519; pk=xFTmRtMG3vELGJBUiml7OYNdM393WOMv0iWWeQEVVdA= X-Migadu-Flow: FLOW_OUT First of all, we need to rename acomp_ctx->dstmem field to buffer, since we are now using for purposes other than compression. Then we change per-cpu mutex and buffer to per-acomp_ctx, since them belong to the acomp_ctx and are necessary parts when used in the compress/decompress contexts. So we can remove the old per-cpu mutex and dstmem. Acked-by: Chris Li (Google) Reviewed-by: Nhat Pham Signed-off-by: Chengming Zhou --- include/linux/cpuhotplug.h | 1 - mm/zswap.c | 104 ++++++++++++++---------------------------= ---- 2 files changed, 33 insertions(+), 72 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index efc0c0b07efb..c3e06e21766a 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -124,7 +124,6 @@ enum cpuhp_state { CPUHP_ARM_BL_PREPARE, CPUHP_TRACE_RB_PREPARE, CPUHP_MM_ZS_PREPARE, - CPUHP_MM_ZSWP_MEM_PREPARE, CPUHP_MM_ZSWP_POOL_PREPARE, CPUHP_KVM_PPC_BOOK3S_PREPARE, CPUHP_ZCOMP_PREPARE, diff --git a/mm/zswap.c b/mm/zswap.c index e756b2af838b..7c18755c6e1c 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -166,8 +166,8 @@ struct crypto_acomp_ctx { struct crypto_acomp *acomp; struct acomp_req *req; struct crypto_wait wait; - u8 *dstmem; - struct mutex *mutex; + u8 *buffer; + struct mutex mutex; }; =20 /* @@ -694,63 +694,26 @@ static void zswap_alloc_shrinker(struct zswap_pool *p= ool) /********************************* * per-cpu code **********************************/ -static DEFINE_PER_CPU(u8 *, zswap_dstmem); -/* - * If users dynamically change the zpool type and compressor at runtime, i= .e. - * zswap is running, zswap can have more than one zpool on one cpu, but th= ey - * are sharing dtsmem. So we need this mutex to be per-cpu. - */ -static DEFINE_PER_CPU(struct mutex *, zswap_mutex); - -static int zswap_dstmem_prepare(unsigned int cpu) -{ - struct mutex *mutex; - u8 *dst; - - dst =3D kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu)); - if (!dst) - return -ENOMEM; - - mutex =3D kmalloc_node(sizeof(*mutex), GFP_KERNEL, cpu_to_node(cpu)); - if (!mutex) { - kfree(dst); - return -ENOMEM; - } - - mutex_init(mutex); - per_cpu(zswap_dstmem, cpu) =3D dst; - per_cpu(zswap_mutex, cpu) =3D mutex; - return 0; -} - -static int zswap_dstmem_dead(unsigned int cpu) -{ - struct mutex *mutex; - u8 *dst; - - mutex =3D per_cpu(zswap_mutex, cpu); - kfree(mutex); - per_cpu(zswap_mutex, cpu) =3D NULL; - - dst =3D per_cpu(zswap_dstmem, cpu); - kfree(dst); - per_cpu(zswap_dstmem, cpu) =3D NULL; - - return 0; -} - static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *nod= e) { struct zswap_pool *pool =3D hlist_entry(node, struct zswap_pool, node); struct crypto_acomp_ctx *acomp_ctx =3D per_cpu_ptr(pool->acomp_ctx, cpu); struct crypto_acomp *acomp; struct acomp_req *req; + int ret; + + mutex_init(&acomp_ctx->mutex); + + acomp_ctx->buffer =3D kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node= (cpu)); + if (!acomp_ctx->buffer) + return -ENOMEM; =20 acomp =3D crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu)); if (IS_ERR(acomp)) { pr_err("could not alloc crypto acomp %s : %ld\n", pool->tfm_name, PTR_ERR(acomp)); - return PTR_ERR(acomp); + ret =3D PTR_ERR(acomp); + goto acomp_fail; } acomp_ctx->acomp =3D acomp; =20 @@ -758,8 +721,8 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, str= uct hlist_node *node) if (!req) { pr_err("could not alloc crypto acomp_request %s\n", pool->tfm_name); - crypto_free_acomp(acomp_ctx->acomp); - return -ENOMEM; + ret =3D -ENOMEM; + goto req_fail; } acomp_ctx->req =3D req; =20 @@ -772,10 +735,13 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, s= truct hlist_node *node) acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &acomp_ctx->wait); =20 - acomp_ctx->mutex =3D per_cpu(zswap_mutex, cpu); - acomp_ctx->dstmem =3D per_cpu(zswap_dstmem, cpu); - return 0; + +req_fail: + crypto_free_acomp(acomp_ctx->acomp); +acomp_fail: + kfree(acomp_ctx->buffer); + return ret; } =20 static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node) @@ -788,6 +754,7 @@ static int zswap_cpu_comp_dead(unsigned int cpu, struct= hlist_node *node) acomp_request_free(acomp_ctx->req); if (!IS_ERR_OR_NULL(acomp_ctx->acomp)) crypto_free_acomp(acomp_ctx->acomp); + kfree(acomp_ctx->buffer); } =20 return 0; @@ -1400,12 +1367,12 @@ static void __zswap_load(struct zswap_entry *entry,= struct page *page) u8 *src; =20 acomp_ctx =3D raw_cpu_ptr(entry->pool->acomp_ctx); - mutex_lock(acomp_ctx->mutex); + mutex_lock(&acomp_ctx->mutex); =20 src =3D zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO); if (!zpool_can_sleep_mapped(zpool)) { - memcpy(acomp_ctx->dstmem, src, entry->length); - src =3D acomp_ctx->dstmem; + memcpy(acomp_ctx->buffer, src, entry->length); + src =3D acomp_ctx->buffer; zpool_unmap_handle(zpool, entry->handle); } =20 @@ -1415,7 +1382,7 @@ static void __zswap_load(struct zswap_entry *entry, s= truct page *page) acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, = PAGE_SIZE); BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ct= x->wait)); BUG_ON(acomp_ctx->req->dlen !=3D PAGE_SIZE); - mutex_unlock(acomp_ctx->mutex); + mutex_unlock(&acomp_ctx->mutex); =20 if (zpool_can_sleep_mapped(zpool)) zpool_unmap_handle(zpool, entry->handle); @@ -1636,13 +1603,17 @@ bool zswap_store(struct folio *folio) /* compress */ acomp_ctx =3D raw_cpu_ptr(entry->pool->acomp_ctx); =20 - mutex_lock(acomp_ctx->mutex); + mutex_lock(&acomp_ctx->mutex); =20 - dst =3D acomp_ctx->dstmem; + dst =3D acomp_ctx->buffer; sg_init_table(&input, 1); sg_set_page(&input, page, PAGE_SIZE, 0); =20 - /* zswap_dstmem is of size (PAGE_SIZE * 2). Reflect same in sg_list */ + /* + * We need PAGE_SIZE * 2 here since there maybe over-compression case, + * and hardware-accelerators may won't check the dst buffer size, so + * giving the dst buffer with enough length to avoid buffer overflow. + */ sg_init_one(&output, dst, PAGE_SIZE * 2); acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen= ); /* @@ -1682,7 +1653,7 @@ bool zswap_store(struct folio *folio) buf =3D zpool_map_handle(zpool, handle, ZPOOL_MM_WO); memcpy(buf, dst, dlen); zpool_unmap_handle(zpool, handle); - mutex_unlock(acomp_ctx->mutex); + mutex_unlock(&acomp_ctx->mutex); =20 /* populate entry */ entry->swpentry =3D swp_entry(type, offset); @@ -1725,7 +1696,7 @@ bool zswap_store(struct folio *folio) return true; =20 put_dstmem: - mutex_unlock(acomp_ctx->mutex); + mutex_unlock(&acomp_ctx->mutex); put_pool: zswap_pool_put(entry->pool); freepage: @@ -1900,13 +1871,6 @@ static int zswap_setup(void) goto cache_fail; } =20 - ret =3D cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare", - zswap_dstmem_prepare, zswap_dstmem_dead); - if (ret) { - pr_err("dstmem alloc failed\n"); - goto dstmem_fail; - } - ret =3D cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE, "mm/zswap_pool:prepare", zswap_cpu_comp_prepare, @@ -1938,8 +1902,6 @@ static int zswap_setup(void) if (pool) zswap_pool_destroy(pool); hp_fail: - cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE); -dstmem_fail: kmem_cache_destroy(zswap_entry_cache); cache_fail: /* if built-in, we aren't unloaded on failure; don't allow use */ --=20 b4 0.10.1