From nobody Tue Jun 23 11:10:23 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id D19FDC433F5 for ; Mon, 7 Mar 2022 07:45:38 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S235928AbiCGHqb (ORCPT ); Mon, 7 Mar 2022 02:46:31 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:49840 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S235910AbiCGHqZ (ORCPT ); Mon, 7 Mar 2022 02:46:25 -0500 Received: from out199-9.us.a.mail.aliyun.com (out199-9.us.a.mail.aliyun.com [47.90.199.9]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 35BC75F8C5 for ; Sun, 6 Mar 2022 23:45:30 -0800 (PST) X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R811e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=e01e04395;MF=dtcccc@linux.alibaba.com;NM=1;PH=DS;RN=7;SR=0;TI=SMTPD_---0V6SREei_1646639125; Received: from localhost.localdomain(mailfrom:dtcccc@linux.alibaba.com fp:SMTPD_---0V6SREei_1646639125) by smtp.aliyun-inc.com(127.0.0.1); Mon, 07 Mar 2022 15:45:26 +0800 From: Tianchen Ding To: Alexander Potapenko , Marco Elver , Dmitry Vyukov , Andrew Morton Cc: kasan-dev@googlegroups.com, linux-mm@kvack.org, linux-kernel@vger.kernel.org Subject: [PATCH v3 1/2] kfence: Allow re-enabling KFENCE after system startup Date: Mon, 7 Mar 2022 15:45:15 +0800 Message-Id: <20220307074516.6920-2-dtcccc@linux.alibaba.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20220307074516.6920-1-dtcccc@linux.alibaba.com> References: <20220307074516.6920-1-dtcccc@linux.alibaba.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" If once KFENCE is disabled by: echo 0 > /sys/module/kfence/parameters/sample_interval KFENCE could never be re-enabled until next rebooting. Allow re-enabling it by writing a positive num to sample_interval. Signed-off-by: Tianchen Ding Reviewed-by: Marco Elver --- mm/kfence/core.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 13128fa13062..caa4e84c8b79 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -38,14 +38,17 @@ #define KFENCE_WARN_ON(cond) = \ ({ \ const bool __cond =3D WARN_ON(cond); \ - if (unlikely(__cond)) \ + if (unlikely(__cond)) { \ WRITE_ONCE(kfence_enabled, false); \ + disabled_by_warn =3D true; \ + } \ __cond; \ }) =20 /* =3D=3D=3D Data =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D */ =20 static bool kfence_enabled __read_mostly; +static bool disabled_by_warn __read_mostly; =20 unsigned long kfence_sample_interval __read_mostly =3D CONFIG_KFENCE_SAMPL= E_INTERVAL; EXPORT_SYMBOL_GPL(kfence_sample_interval); /* Export for test modules. */ @@ -55,6 +58,7 @@ EXPORT_SYMBOL_GPL(kfence_sample_interval); /* Export for = test modules. */ #endif #define MODULE_PARAM_PREFIX "kfence." =20 +static int kfence_enable_late(void); static int param_set_sample_interval(const char *val, const struct kernel_= param *kp) { unsigned long num; @@ -65,10 +69,11 @@ static int param_set_sample_interval(const char *val, c= onst struct kernel_param =20 if (!num) /* Using 0 to indicate KFENCE is disabled. */ WRITE_ONCE(kfence_enabled, false); - else if (!READ_ONCE(kfence_enabled) && system_state !=3D SYSTEM_BOOTING) - return -EINVAL; /* Cannot (re-)enable KFENCE on-the-fly. */ =20 *((unsigned long *)kp->arg) =3D num; + + if (num && !READ_ONCE(kfence_enabled) && system_state !=3D SYSTEM_BOOTING) + return disabled_by_warn ? -EINVAL : kfence_enable_late(); return 0; } =20 @@ -787,6 +792,16 @@ void __init kfence_init(void) (void *)(__kfence_pool + KFENCE_POOL_SIZE)); } =20 +static int kfence_enable_late(void) +{ + if (!__kfence_pool) + return -EINVAL; + + WRITE_ONCE(kfence_enabled, true); + queue_delayed_work(system_unbound_wq, &kfence_timer, 0); + return 0; +} + void kfence_shutdown_cache(struct kmem_cache *s) { unsigned long flags; --=20 2.27.0 From nobody Tue Jun 23 11:10:23 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 31A13C433EF for ; Mon, 7 Mar 2022 07:45:36 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S235923AbiCGHq2 (ORCPT ); Mon, 7 Mar 2022 02:46:28 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:49832 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231518AbiCGHqY (ORCPT ); Mon, 7 Mar 2022 02:46:24 -0500 Received: from out30-43.freemail.mail.aliyun.com (out30-43.freemail.mail.aliyun.com [115.124.30.43]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id D99C35F8C0 for ; Sun, 6 Mar 2022 23:45:30 -0800 (PST) X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R161e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=e01e04426;MF=dtcccc@linux.alibaba.com;NM=1;PH=DS;RN=7;SR=0;TI=SMTPD_---0V6SREfU_1646639126; Received: from localhost.localdomain(mailfrom:dtcccc@linux.alibaba.com fp:SMTPD_---0V6SREfU_1646639126) by smtp.aliyun-inc.com(127.0.0.1); Mon, 07 Mar 2022 15:45:27 +0800 From: Tianchen Ding To: Alexander Potapenko , Marco Elver , Dmitry Vyukov , Andrew Morton Cc: kasan-dev@googlegroups.com, linux-mm@kvack.org, linux-kernel@vger.kernel.org Subject: [PATCH v3 2/2] kfence: Alloc kfence_pool after system startup Date: Mon, 7 Mar 2022 15:45:16 +0800 Message-Id: <20220307074516.6920-3-dtcccc@linux.alibaba.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20220307074516.6920-1-dtcccc@linux.alibaba.com> References: <20220307074516.6920-1-dtcccc@linux.alibaba.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Allow enabling KFENCE after system startup by allocating its pool via the page allocator. This provides the flexibility to enable KFENCE even if it wasn't enabled at boot time. Signed-off-by: Tianchen Ding Reviewed-by: Marco Elver Tested-by: Marco Elver Tested-by: Peng Liu --- mm/kfence/core.c | 111 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 90 insertions(+), 21 deletions(-) diff --git a/mm/kfence/core.c b/mm/kfence/core.c index caa4e84c8b79..f126b53b9b85 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -96,7 +96,7 @@ static unsigned long kfence_skip_covered_thresh __read_mo= stly =3D 75; module_param_named(skip_covered_thresh, kfence_skip_covered_thresh, ulong,= 0644); =20 /* The pool of pages used for guard pages and objects. */ -char *__kfence_pool __ro_after_init; +char *__kfence_pool __read_mostly; EXPORT_SYMBOL(__kfence_pool); /* Export for test modules. */ =20 /* @@ -537,17 +537,19 @@ static void rcu_guarded_free(struct rcu_head *h) kfence_guarded_free((void *)meta->addr, meta, false); } =20 -static bool __init kfence_init_pool(void) +/* + * Initialization of the KFENCE pool after its allocation. + * Returns 0 on success; otherwise returns the address up to + * which partial initialization succeeded. + */ +static unsigned long kfence_init_pool(void) { unsigned long addr =3D (unsigned long)__kfence_pool; struct page *pages; int i; =20 - if (!__kfence_pool) - return false; - if (!arch_kfence_init_pool()) - goto err; + return addr; =20 pages =3D virt_to_page(addr); =20 @@ -565,7 +567,7 @@ static bool __init kfence_init_pool(void) =20 /* Verify we do not have a compound head page. */ if (WARN_ON(compound_head(&pages[i]) !=3D &pages[i])) - goto err; + return addr; =20 __SetPageSlab(&pages[i]); } @@ -578,7 +580,7 @@ static bool __init kfence_init_pool(void) */ for (i =3D 0; i < 2; i++) { if (unlikely(!kfence_protect(addr))) - goto err; + return addr; =20 addr +=3D PAGE_SIZE; } @@ -595,7 +597,7 @@ static bool __init kfence_init_pool(void) =20 /* Protect the right redzone. */ if (unlikely(!kfence_protect(addr + PAGE_SIZE))) - goto err; + return addr; =20 addr +=3D 2 * PAGE_SIZE; } @@ -608,9 +610,21 @@ static bool __init kfence_init_pool(void) */ kmemleak_free(__kfence_pool); =20 - return true; + return 0; +} + +static bool __init kfence_init_pool_early(void) +{ + unsigned long addr; + + if (!__kfence_pool) + return false; + + addr =3D kfence_init_pool(); + + if (!addr) + return true; =20 -err: /* * Only release unprotected pages, and do not try to go back and change * page attributes due to risk of failing to do so as well. If changing @@ -623,6 +637,26 @@ static bool __init kfence_init_pool(void) return false; } =20 +static bool kfence_init_pool_late(void) +{ + unsigned long addr, free_size; + + addr =3D kfence_init_pool(); + + if (!addr) + return true; + + /* Same as above. */ + free_size =3D KFENCE_POOL_SIZE - (addr - (unsigned long)__kfence_pool); +#ifdef CONFIG_CONTIG_ALLOC + free_contig_range(page_to_pfn(virt_to_page(addr)), free_size / PAGE_SIZE); +#else + free_pages_exact((void *)addr, free_size); +#endif + __kfence_pool =3D NULL; + return false; +} + /* =3D=3D=3D DebugFS Interface =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D */ =20 static int stats_show(struct seq_file *seq, void *v) @@ -771,31 +805,66 @@ void __init kfence_alloc_pool(void) pr_err("failed to allocate pool\n"); } =20 +static void kfence_init_enable(void) +{ + if (!IS_ENABLED(CONFIG_KFENCE_STATIC_KEYS)) + static_branch_enable(&kfence_allocation_key); + WRITE_ONCE(kfence_enabled, true); + queue_delayed_work(system_unbound_wq, &kfence_timer, 0); + pr_info("initialized - using %lu bytes for %d objects at 0x%p-0x%p\n", KF= ENCE_POOL_SIZE, + CONFIG_KFENCE_NUM_OBJECTS, (void *)__kfence_pool, + (void *)(__kfence_pool + KFENCE_POOL_SIZE)); +} + void __init kfence_init(void) { + stack_hash_seed =3D (u32)random_get_entropy(); + /* Setting kfence_sample_interval to 0 on boot disables KFENCE. */ if (!kfence_sample_interval) return; =20 - stack_hash_seed =3D (u32)random_get_entropy(); - if (!kfence_init_pool()) { + if (!kfence_init_pool_early()) { pr_err("%s failed\n", __func__); return; } =20 - if (!IS_ENABLED(CONFIG_KFENCE_STATIC_KEYS)) - static_branch_enable(&kfence_allocation_key); - WRITE_ONCE(kfence_enabled, true); - queue_delayed_work(system_unbound_wq, &kfence_timer, 0); - pr_info("initialized - using %lu bytes for %d objects at 0x%p-0x%p\n", KF= ENCE_POOL_SIZE, - CONFIG_KFENCE_NUM_OBJECTS, (void *)__kfence_pool, - (void *)(__kfence_pool + KFENCE_POOL_SIZE)); + kfence_init_enable(); +} + +static int kfence_init_late(void) +{ + const unsigned long nr_pages =3D KFENCE_POOL_SIZE / PAGE_SIZE; +#ifdef CONFIG_CONTIG_ALLOC + struct page *pages; + + pages =3D alloc_contig_pages(nr_pages, GFP_KERNEL, first_online_node, NUL= L); + if (!pages) + return -ENOMEM; + __kfence_pool =3D page_to_virt(pages); +#else + if (nr_pages > MAX_ORDER_NR_PAGES) { + pr_warn("KFENCE_NUM_OBJECTS too large for buddy allocator\n"); + return -EINVAL; + } + __kfence_pool =3D alloc_pages_exact(KFENCE_POOL_SIZE, GFP_KERNEL); + if (!__kfence_pool) + return -ENOMEM; +#endif + + if (!kfence_init_pool_late()) { + pr_err("%s failed\n", __func__); + return -EBUSY; + } + + kfence_init_enable(); + return 0; } =20 static int kfence_enable_late(void) { if (!__kfence_pool) - return -EINVAL; + return kfence_init_late(); =20 WRITE_ONCE(kfence_enabled, true); queue_delayed_work(system_unbound_wq, &kfence_timer, 0); --=20 2.27.0