From nobody Wed Dec 17 21:01:18 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 2B011C61D9D for ; Wed, 22 Nov 2023 01:08:31 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S235018AbjKVBI0 (ORCPT ); Tue, 21 Nov 2023 20:08:26 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:58830 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S234995AbjKVBIW (ORCPT ); Tue, 21 Nov 2023 20:08:22 -0500 Received: from mail-pg1-x536.google.com (mail-pg1-x536.google.com [IPv6:2607:f8b0:4864:20::536]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E425D199 for ; Tue, 21 Nov 2023 17:08:18 -0800 (PST) Received: by mail-pg1-x536.google.com with SMTP id 41be03b00d2f7-5ab94fc098cso3802649a12.1 for ; Tue, 21 Nov 2023 17:08:18 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sifive.com; s=google; t=1700615298; x=1701220098; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=/6X+mOMZoGJuf/KhFjronMx0sFEsDU8bO3mA0YdbfIg=; b=H3U84WlFWMabFJAnu3tQEA+F0LLai5ajuAUMjCUF0bKHxyPQRdJ+QKJPp5+1oqdG91 +ZDHnKGt+FlNuhfmnVsYoTM7hLRUvq/FRGGeD8I3jgv2nYqdFw85QIaDik6dzXUqdITS 2vekBoz6xau8XcW63DKWbPkuV4V+XjvxHlQq+ZxtaUK3Gz/C7xQh3MzJpnbI0n2WTaEX zPeSgjpKxj9RTmhpZ6wE6RrxCx8VMok8ar+TNuRxvyjBOcPXlsvzezEHqrkoQ7T/kbB/ 7xZZFV5YNqV5COo+t/rDGl6haTjm2P4erF30M+umkCusAHsbG+GZnwtDD6KfK/ek/V6+ N54g== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1700615298; x=1701220098; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=/6X+mOMZoGJuf/KhFjronMx0sFEsDU8bO3mA0YdbfIg=; b=V2gN6eTvUSpkg8SnnbljjHB7/hj2OEk4RR9clhKEn3IZR6FkkR+hygDyr3lobSKXGm N8N1OOfwEhF0emqstXdieDqk9fmkLhsfwMmb2M42+QTBbigGfPvlX+FW8QuwPteGq7Up 0dLA9PAA2pMTQHEUaJDQSn6TPMwRcB5bGtthBJng9iGHSC/3dyHlThB/ZJgTRLJ2XC5y ChqTOOm4sAcL23iInxJImwUxKWR73TXVZ8sy3rOSTDb1MytEW4K7Zi6hf0UttL8Aq1Kz V6dSKEMukEgnCJ4E9pc0OsanlaSOgJkTjRCYOo9gMv+L9ehjlrN/qnd6V+91XwGCGMlj k9GA== X-Gm-Message-State: AOJu0YxtsAtnCCmVbsLeuASPBj/GliWYk+Y0PhWmHrLPXYMaWlzi7/CM rxHKbnQQeW3DJKJfo5WEHthJSA== X-Google-Smtp-Source: AGHT+IF0MPSUCqiTsnl1eQc+NOEkZdfylg4Cinzvui2qig7vBCcLaky4/GlXwa1fL7VF+JjfOGRcVA== X-Received: by 2002:a05:6a20:431a:b0:18a:da5a:3825 with SMTP id h26-20020a056a20431a00b0018ada5a3825mr948843pzk.34.1700615298368; Tue, 21 Nov 2023 17:08:18 -0800 (PST) Received: from sw06.internal.sifive.com ([4.53.31.132]) by smtp.gmail.com with ESMTPSA id bn5-20020a056a00324500b006be047268d5sm8713961pfb.174.2023.11.21.17.08.17 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 21 Nov 2023 17:08:18 -0800 (PST) From: Samuel Holland To: Palmer Dabbelt , linux-riscv@lists.infradead.org Cc: linux-kernel@vger.kernel.org, linux-mm@kvack.org, Alexandre Ghiti , Samuel Holland Subject: [PATCH v3 1/8] riscv: mm: Combine the SMP and UP TLB flush code Date: Tue, 21 Nov 2023 17:07:12 -0800 Message-ID: <20231122010815.3545294-2-samuel.holland@sifive.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231122010815.3545294-1-samuel.holland@sifive.com> References: <20231122010815.3545294-1-samuel.holland@sifive.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" In SMP configurations, all TLB flushing narrower than flush_tlb_all() goes through __flush_tlb_range(). Do the same in UP configurations. This allows UP configurations to take advantage of recent improvements to the code in tlbflush.c, such as support for huge pages and flushing multiple-page ranges. Signed-off-by: Samuel Holland --- (no changes since v2) Changes in v2: - Move the SMP/UP merge earlier in the series to avoid build issues - Make a copy of __flush_tlb_range() instead of adding ifdefs inside - local_flush_tlb_all() is the only function used on !MMU (smpboot.c) arch/riscv/include/asm/tlbflush.h | 33 +++++++------------------------ arch/riscv/mm/Makefile | 5 +---- arch/riscv/mm/tlbflush.c | 13 ++++++++++++ 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlb= flush.h index 8f3418c5f172..317a1811aa51 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -27,13 +27,12 @@ static inline void local_flush_tlb_page(unsigned long a= ddr) { ALT_FLUSH_TLB_PAGE(__asm__ __volatile__ ("sfence.vma %0" : : "r" (addr) := "memory")); } -#else /* CONFIG_MMU */ -#define local_flush_tlb_all() do { } while (0) -#define local_flush_tlb_page(addr) do { } while (0) -#endif /* CONFIG_MMU */ =20 -#if defined(CONFIG_SMP) && defined(CONFIG_MMU) +#ifdef CONFIG_SMP void flush_tlb_all(void); +#else +#define flush_tlb_all() local_flush_tlb_all() +#endif void flush_tlb_mm(struct mm_struct *mm); void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned int page_size); @@ -46,26 +45,8 @@ void flush_tlb_kernel_range(unsigned long start, unsigne= d long end); void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); #endif -#else /* CONFIG_SMP && CONFIG_MMU */ - -#define flush_tlb_all() local_flush_tlb_all() -#define flush_tlb_page(vma, addr) local_flush_tlb_page(addr) - -static inline void flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ - local_flush_tlb_all(); -} - -/* Flush a range of kernel pages */ -static inline void flush_tlb_kernel_range(unsigned long start, - unsigned long end) -{ - local_flush_tlb_all(); -} - -#define flush_tlb_mm(mm) flush_tlb_all() -#define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() -#endif /* !CONFIG_SMP || !CONFIG_MMU */ +#else /* CONFIG_MMU */ +#define local_flush_tlb_all() do { } while (0) +#endif /* CONFIG_MMU */ =20 #endif /* _ASM_RISCV_TLBFLUSH_H */ diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index 3a4dfc8babcf..96e65c571ce8 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -13,15 +13,12 @@ endif KCOV_INSTRUMENT_init.o :=3D n =20 obj-y +=3D init.o -obj-$(CONFIG_MMU) +=3D extable.o fault.o pageattr.o +obj-$(CONFIG_MMU) +=3D extable.o fault.o pageattr.o tlbflush.o obj-y +=3D cacheflush.o obj-y +=3D context.o obj-y +=3D pgtable.o obj-y +=3D pmem.o =20 -ifeq ($(CONFIG_MMU),y) -obj-$(CONFIG_SMP) +=3D tlbflush.o -endif obj-$(CONFIG_HUGETLB_PAGE) +=3D hugetlbpage.o obj-$(CONFIG_PTDUMP_CORE) +=3D ptdump.o obj-$(CONFIG_KASAN) +=3D kasan_init.o diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index e6659d7368b3..22d7ed5abf8e 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -66,6 +66,7 @@ static inline void local_flush_tlb_range_asid(unsigned lo= ng start, local_flush_tlb_range_threshold_asid(start, size, stride, asid); } =20 +#ifdef CONFIG_SMP static void __ipi_flush_tlb_all(void *info) { local_flush_tlb_all(); @@ -138,6 +139,18 @@ static void __flush_tlb_range(struct mm_struct *mm, un= signed long start, if (mm) put_cpu(); } +#else +static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, + unsigned long size, unsigned long stride) +{ + unsigned long asid =3D FLUSH_TLB_NO_ASID; + + if (mm && static_branch_unlikely(&use_asid_allocator)) + asid =3D atomic_long_read(&mm->context.id) & asid_mask; + + local_flush_tlb_range_asid(start, size, stride, asid); +} +#endif =20 void flush_tlb_mm(struct mm_struct *mm) { --=20 2.42.0 From nobody Wed Dec 17 21:01:18 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id DBB39C61D85 for ; Wed, 22 Nov 2023 01:08:30 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S235025AbjKVBI2 (ORCPT ); Tue, 21 Nov 2023 20:08:28 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:58840 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231307AbjKVBIY (ORCPT ); Tue, 21 Nov 2023 20:08:24 -0500 Received: from mail-oi1-x232.google.com (mail-oi1-x232.google.com [IPv6:2607:f8b0:4864:20::232]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 71B65199 for ; Tue, 21 Nov 2023 17:08:20 -0800 (PST) Received: by mail-oi1-x232.google.com with SMTP id 5614622812f47-3b83d2981b5so205144b6e.0 for ; Tue, 21 Nov 2023 17:08:20 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sifive.com; s=google; t=1700615300; x=1701220100; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=4E3lnavwhWkbsPt/8jollKuN1Wp7270MVQq6Nf32IZI=; b=De9XE7y20ckTyRWrQa0Hbzaoo+nq6WHS5nTYBCqRAFQ86M9Y+UUqDZrtGW6qbGvbam GQWyR45fGKunNpZlHr3OK5+tRLH2Sr7H8s2+WIT3q3iMucD9E6OqkLvNchgsIy8EzEpU ffS+dsPylvPmEkh0dL/gYjCpb24upaSF/yXYwp6u0AGj+YvWJUZOil2jaSrELWKk7bSy FDinXNnPm0RgrV2BjpZ79xUgYKv3AVqBipGKjxI4gVROvr0uZE+YdrqZTCwSoQhucuSr hjYLt/wsZxfDzl0yVo1/8FSPz6KqXmz6xp6fpxXlBt4XGveYUNIbAaajxXPaSJ5YwDAR I2Ew== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1700615300; x=1701220100; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=4E3lnavwhWkbsPt/8jollKuN1Wp7270MVQq6Nf32IZI=; b=LdW4YfR4DuuUWXpLcgdY1QI2O7O+w4G5v2th7+9fwgu48iTWCtmeBTro2u1UIAzEY3 odaWC7rNvH4nwlm71mY9bN25aU0zPeufmHW7RvCaxfa/TT3oZiUzg8+srLsxcJEarTW2 opQosRjspPmKHITfKa7VQ+0elCrpgW/eIAmgDejb7iTIltC9+7uKSKIKfA1+LT50w+u7 xc5z/H+nIdL/uR6C+G+vCzlAXF5HD6P+dSTH68pJfhM6DqwgKUf11nwFxmPtqtsg/mee +k6xI8YcBtSa5YSvTmISJK3kovQ45CT6mqRTTyan/yh3359Silw3w9Ku+D5cQToaUK4r /jeQ== X-Gm-Message-State: AOJu0YzIQylPPv1fCN9gCuZdecdp5jZ1vH5H6o+C0UF2IoBIVnPtR2dO aGuxlVtNKxVOnzKK0f8DFzEiRA== X-Google-Smtp-Source: AGHT+IFhS1aXruOMSeKbF/kr742mxFr/4ERJTn5q1RSxs6qFscJwCL7KnIKBT8DwBL+aJBuJSXtucQ== X-Received: by 2002:a05:6871:e805:b0:1d6:8b7f:ce89 with SMTP id qd5-20020a056871e80500b001d68b7fce89mr1213468oac.51.1700615299719; Tue, 21 Nov 2023 17:08:19 -0800 (PST) Received: from sw06.internal.sifive.com ([4.53.31.132]) by smtp.gmail.com with ESMTPSA id bn5-20020a056a00324500b006be047268d5sm8713961pfb.174.2023.11.21.17.08.19 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 21 Nov 2023 17:08:19 -0800 (PST) From: Samuel Holland To: Palmer Dabbelt , linux-riscv@lists.infradead.org Cc: linux-kernel@vger.kernel.org, linux-mm@kvack.org, Alexandre Ghiti , Samuel Holland Subject: [PATCH v3 2/8] riscv: Apply SiFive CIP-1200 workaround to single-ASID sfence.vma Date: Tue, 21 Nov 2023 17:07:13 -0800 Message-ID: <20231122010815.3545294-3-samuel.holland@sifive.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231122010815.3545294-1-samuel.holland@sifive.com> References: <20231122010815.3545294-1-samuel.holland@sifive.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" commit 3f1e782998cd ("riscv: add ASID-based tlbflushing methods") added calls to the sfence.vma instruction with rs2 !=3D x0. These single-ASID instruction variants are also affected by SiFive errata CIP-1200. Until now, the errata workaround was not needed for the single-ASID sfence.vma variants, because they were only used when the ASID allocator was enabled, and the affected SiFive platforms do not support multiple ASIDs. However, we are going to start using those sfence.vma variants regardless of ASID support, so now we need alternatives covering them. Signed-off-by: Samuel Holland --- (no changes since v2) Changes in v2: - Rebase on Alexandre's "riscv: tlb flush improvements" series v5 arch/riscv/include/asm/errata_list.h | 12 +++++++++++- arch/riscv/include/asm/tlbflush.h | 19 ++++++++++++++++++- arch/riscv/mm/tlbflush.c | 23 ----------------------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/arch/riscv/include/asm/errata_list.h b/arch/riscv/include/asm/= errata_list.h index 83ed25e43553..6781460ae564 100644 --- a/arch/riscv/include/asm/errata_list.h +++ b/arch/riscv/include/asm/errata_list.h @@ -44,11 +44,21 @@ ALTERNATIVE(__stringify(RISCV_PTR do_page_fault), \ CONFIG_ERRATA_SIFIVE_CIP_453) #else /* !__ASSEMBLY__ */ =20 -#define ALT_FLUSH_TLB_PAGE(x) \ +#define ALT_SFENCE_VMA_ASID(asid) \ +asm(ALTERNATIVE("sfence.vma x0, %0", "sfence.vma", SIFIVE_VENDOR_ID, \ + ERRATA_SIFIVE_CIP_1200, CONFIG_ERRATA_SIFIVE_CIP_1200) \ + : : "r" (asid) : "memory") + +#define ALT_SFENCE_VMA_ADDR(addr) \ asm(ALTERNATIVE("sfence.vma %0", "sfence.vma", SIFIVE_VENDOR_ID, \ ERRATA_SIFIVE_CIP_1200, CONFIG_ERRATA_SIFIVE_CIP_1200) \ : : "r" (addr) : "memory") =20 +#define ALT_SFENCE_VMA_ADDR_ASID(addr, asid) \ +asm(ALTERNATIVE("sfence.vma %0, %1", "sfence.vma", SIFIVE_VENDOR_ID, \ + ERRATA_SIFIVE_CIP_1200, CONFIG_ERRATA_SIFIVE_CIP_1200) \ + : : "r" (addr), "r" (asid) : "memory") + /* * _val is marked as "will be overwritten", so need to set it to 0 * in the default case. diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlb= flush.h index 317a1811aa51..e529a643be17 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -22,10 +22,27 @@ static inline void local_flush_tlb_all(void) __asm__ __volatile__ ("sfence.vma" : : : "memory"); } =20 +static inline void local_flush_tlb_all_asid(unsigned long asid) +{ + if (asid !=3D FLUSH_TLB_NO_ASID) + ALT_SFENCE_VMA_ASID(asid); + else + local_flush_tlb_all(); +} + /* Flush one page from local TLB */ static inline void local_flush_tlb_page(unsigned long addr) { - ALT_FLUSH_TLB_PAGE(__asm__ __volatile__ ("sfence.vma %0" : : "r" (addr) := "memory")); + ALT_SFENCE_VMA_ADDR(addr); +} + +static inline void local_flush_tlb_page_asid(unsigned long addr, + unsigned long asid) +{ + if (asid !=3D FLUSH_TLB_NO_ASID) + ALT_SFENCE_VMA_ADDR_ASID(addr, asid); + else + local_flush_tlb_page(addr); } =20 #ifdef CONFIG_SMP diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 22d7ed5abf8e..0feccb8932d2 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -7,29 +7,6 @@ #include #include =20 -static inline void local_flush_tlb_all_asid(unsigned long asid) -{ - if (asid !=3D FLUSH_TLB_NO_ASID) - __asm__ __volatile__ ("sfence.vma x0, %0" - : - : "r" (asid) - : "memory"); - else - local_flush_tlb_all(); -} - -static inline void local_flush_tlb_page_asid(unsigned long addr, - unsigned long asid) -{ - if (asid !=3D FLUSH_TLB_NO_ASID) - __asm__ __volatile__ ("sfence.vma %0, %1" - : - : "r" (addr), "r" (asid) - : "memory"); - else - local_flush_tlb_page(addr); -} - /* * Flush entire TLB if number of entries to be flushed is greater * than the threshold below. --=20 2.42.0 From nobody Wed Dec 17 21:01:18 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0DB7CC61D9B for ; Wed, 22 Nov 2023 01:08:31 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S235033AbjKVBIc (ORCPT ); Tue, 21 Nov 2023 20:08:32 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:58864 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S235014AbjKVBIZ (ORCPT ); Tue, 21 Nov 2023 20:08:25 -0500 Received: from mail-pf1-x431.google.com (mail-pf1-x431.google.com [IPv6:2607:f8b0:4864:20::431]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E7C9C199 for ; Tue, 21 Nov 2023 17:08:21 -0800 (PST) Received: by mail-pf1-x431.google.com with SMTP id d2e1a72fcca58-6b709048f32so5533362b3a.0 for ; Tue, 21 Nov 2023 17:08:21 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sifive.com; s=google; t=1700615301; x=1701220101; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=VvpOAl9p0Aiam4sGuchXKDW+3q/tf43lzqxId4c/moI=; b=FrLswurMHZq4M5NffbQkhqAXMCarQlZX+nqEuPodr1N58lvjRdBIrLp5f0NLvCp6xx IFs+P7iSw8zPRaRSwwWT/rB01WDRzMU9rbjMAA9N6X2yw2rrRYFnANHXz+a1YohZ5eQd 2YYI3xWMVrAmEA5xWAy4Lv+aWvpssZhVxbi23NbRgpU05bnC4PNJdFHlGUXxaVoL7zIH Ve6TvQKvZHYpDGGSfQWFOvwvX9qFe/NNKRv1mamhsAT//kluZjFBiUjlyoxymtR3LLn1 FGozJ3baqyN/Y4KLeN34HsPlyo/eYfJHZzOB2/4AHNrYwyOj2DRnoPK+KOy7HnKDYbYF kjFg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1700615301; x=1701220101; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=VvpOAl9p0Aiam4sGuchXKDW+3q/tf43lzqxId4c/moI=; b=c8pwhu+Q5IRq+wNxUhQw+MDPY+uV73/r+H3pbRzqAr4ivRWhyDGFXmdi9LsMvoaOZ7 wEd1TT0odd0uDKzaH4CRGiEqquT6Jc84t3y+ySMkZxqeWmBoMj84U5scK2YP1WWFSPkX hn4+IDzzYqayLnNkh2l/GQwTPnfjuq+FlkLW+efIPe5b3RUBlHoAj/p6RQb8li6rbd/Z USzyXrOBZlmv6hAlz/tRiIYvVeve2s68w0hpux0lrrPKoBrqYYB3nhQGZGciDbPugjEe WfuhQguFGBJxWoVJgt0EOaydbpmCCKNju/EFJlXvd+2Ixh3u5ATMaHDzwvOd9mdnPT9g 5l6w== X-Gm-Message-State: AOJu0YxaSdqd2qrzm3kKMfwBBcPxG12nx870OR64T2HiFK1Zg7lSkDHQ V58FxJo5ng/puRUmFo5kZtVqTg== X-Google-Smtp-Source: AGHT+IF9d9fQIS/asTDDunhMGjFlgWHJ5Uekq1PrtWCEzSD057jeC559ik7d8wrAC6IaVqDr4/Nfcw== X-Received: by 2002:a05:6a00:3927:b0:6c6:aaa4:5d68 with SMTP id fh39-20020a056a00392700b006c6aaa45d68mr871292pfb.27.1700615301365; Tue, 21 Nov 2023 17:08:21 -0800 (PST) Received: from sw06.internal.sifive.com ([4.53.31.132]) by smtp.gmail.com with ESMTPSA id bn5-20020a056a00324500b006be047268d5sm8713961pfb.174.2023.11.21.17.08.20 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 21 Nov 2023 17:08:20 -0800 (PST) From: Samuel Holland To: Palmer Dabbelt , linux-riscv@lists.infradead.org Cc: linux-kernel@vger.kernel.org, linux-mm@kvack.org, Alexandre Ghiti , Samuel Holland Subject: [PATCH v3 3/8] riscv: Avoid TLB flush loops when affected by SiFive CIP-1200 Date: Tue, 21 Nov 2023 17:07:14 -0800 Message-ID: <20231122010815.3545294-4-samuel.holland@sifive.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231122010815.3545294-1-samuel.holland@sifive.com> References: <20231122010815.3545294-1-samuel.holland@sifive.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Since implementations affected by SiFive errata CIP-1200 always use the global variant of the sfence.vma instruction, they only need to execute the instruction once. The range-based loop only hurts performance. Signed-off-by: Samuel Holland --- Changes in v3: - New patch for v3 arch/riscv/errata/sifive/errata.c | 3 +++ arch/riscv/include/asm/tlbflush.h | 2 ++ arch/riscv/mm/tlbflush.c | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/riscv/errata/sifive/errata.c b/arch/riscv/errata/sifive/e= rrata.c index 3d9a32d791f7..00e011d78866 100644 --- a/arch/riscv/errata/sifive/errata.c +++ b/arch/riscv/errata/sifive/errata.c @@ -42,6 +42,9 @@ static bool errata_cip_1200_check_func(unsigned long arc= h_id, unsigned long imp return false; if ((impid & 0xffffff) > 0x200630 || impid =3D=3D 0x1200626) return false; + + tlb_flush_all_threshold =3D 0; + return true; } =20 diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlb= flush.h index e529a643be17..3b393f765805 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -62,6 +62,8 @@ void flush_tlb_kernel_range(unsigned long start, unsigned= long end); void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); #endif + +extern unsigned long tlb_flush_all_threshold; #else /* CONFIG_MMU */ #define local_flush_tlb_all() do { } while (0) #endif /* CONFIG_MMU */ diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 0feccb8932d2..27b3744b5673 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -11,7 +11,7 @@ * Flush entire TLB if number of entries to be flushed is greater * than the threshold below. */ -static unsigned long tlb_flush_all_threshold __read_mostly =3D 64; +unsigned long tlb_flush_all_threshold __read_mostly =3D 64; =20 static void local_flush_tlb_range_threshold_asid(unsigned long start, unsigned long size, --=20 2.42.0 From nobody Wed Dec 17 21:01:18 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id EC85AC61D9C for ; Wed, 22 Nov 2023 01:08:32 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S235055AbjKVBIe (ORCPT ); Tue, 21 Nov 2023 20:08:34 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:58882 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S234995AbjKVBI0 (ORCPT ); Tue, 21 Nov 2023 20:08:26 -0500 Received: from mail-pf1-x42c.google.com (mail-pf1-x42c.google.com [IPv6:2607:f8b0:4864:20::42c]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 14AB21BB for ; Tue, 21 Nov 2023 17:08:23 -0800 (PST) Received: by mail-pf1-x42c.google.com with SMTP id d2e1a72fcca58-6c4eb5fda3cso6286216b3a.2 for ; Tue, 21 Nov 2023 17:08:23 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sifive.com; s=google; t=1700615302; x=1701220102; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=qsEPuyPCg+yq6nUonOiNDjvoYO+XNuK3/GzQIQmVNGs=; b=BAKUfur8M7wCgnKW9L4xD1mGcMUV0e2ccjSzZUOcnyPB5E9shcA1ErtjBfaazbsTxF A+9OBFxE94rx9wuawd4q0n2jQx+xygm6O6+FGYS+8Q806faI6F7aERHzOVtUNOpP4AuA b8b/s8VFL5miRongxWE5+54N27vkMf1ObEcSPrRtdxtMLoZ940kRShUzYpA8qKjPcNZC aViqcrreE05Asrm4rRRDl4lAbsYIOZBk0oHF+TFZp5yPJTgviFxaATH/NZXkdJ5+0uuz eW0ct+rlkxG1Hmxku9QZ06tAZMFynBR51c4mZXCrLu4wgJ2YNjGz+BqAYo+gayLMMm0h GO2w== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1700615302; x=1701220102; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=qsEPuyPCg+yq6nUonOiNDjvoYO+XNuK3/GzQIQmVNGs=; b=oqRn7rNI6wWEpbeESEpfXSYlQuhjOFlluFBpWa6zeihYZpPfz0bYTu4AvThtaTpDrb pXbU2pXVlBiqWNkDMKnwkxuVzgTaF5Gmi1/TJue21+PS/lLkOe1K57o5n36RuB/3+VZ9 id9tvkw+3oMwgbcAq3NZnvH164ELvfL/znoykaEzOi6SFnfh0lpUE32zpBFdaIZaKJ+t l69Lp3kCEQyUQBCqtViX6jmeUbAc6mGK9IqmoUNZ72DDbuWskCt3h8nf0QrssDvcLopd eZZLu267Kw36yphiYLOgO9GGV5gFDPTaywogmP1+765Mu5WDC+PLiCeNTRJ8LMp7QGCn Byfg== X-Gm-Message-State: AOJu0YwKTOgyj/C0z1rwLTrGHL3qNtqz2bhjrfiRU2kqi/IngZaqVwE+ ThLZA1fkVkySTsl0sMvK/6BYmw== X-Google-Smtp-Source: AGHT+IF7ribPLGqSLRhKn+k/5RoFarRyyKm41YCW2vHm9TG9JHEkBRqiRle7Sp7LVXO54oVTBXbYlQ== X-Received: by 2002:a05:6a20:c6cd:b0:188:75c:e69a with SMTP id gw13-20020a056a20c6cd00b00188075ce69amr779809pzb.4.1700615302563; Tue, 21 Nov 2023 17:08:22 -0800 (PST) Received: from sw06.internal.sifive.com ([4.53.31.132]) by smtp.gmail.com with ESMTPSA id bn5-20020a056a00324500b006be047268d5sm8713961pfb.174.2023.11.21.17.08.21 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 21 Nov 2023 17:08:22 -0800 (PST) From: Samuel Holland To: Palmer Dabbelt , linux-riscv@lists.infradead.org Cc: linux-kernel@vger.kernel.org, linux-mm@kvack.org, Alexandre Ghiti , Samuel Holland Subject: [PATCH v3 4/8] riscv: mm: Introduce cntx2asid/cntx2version helper macros Date: Tue, 21 Nov 2023 17:07:15 -0800 Message-ID: <20231122010815.3545294-5-samuel.holland@sifive.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231122010815.3545294-1-samuel.holland@sifive.com> References: <20231122010815.3545294-1-samuel.holland@sifive.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" When using the ASID allocator, the MM context ID contains two values: the ASID in the lower bits, and the allocator version number in the remaining bits. Use macros to make this separation more obvious. Signed-off-by: Samuel Holland --- (no changes since v1) arch/riscv/include/asm/mmu.h | 3 +++ arch/riscv/mm/context.c | 12 ++++++------ arch/riscv/mm/tlbflush.c | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/arch/riscv/include/asm/mmu.h b/arch/riscv/include/asm/mmu.h index 355504b37f8e..a550fbf770be 100644 --- a/arch/riscv/include/asm/mmu.h +++ b/arch/riscv/include/asm/mmu.h @@ -26,6 +26,9 @@ typedef struct { #endif } mm_context_t; =20 +#define cntx2asid(cntx) ((cntx) & asid_mask) +#define cntx2version(cntx) ((cntx) & ~asid_mask) + void __init create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, pgprot_t prot); #endif /* __ASSEMBLY__ */ diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index 217fd4de6134..43d005f63253 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -81,7 +81,7 @@ static void __flush_context(void) if (cntx =3D=3D 0) cntx =3D per_cpu(reserved_context, i); =20 - __set_bit(cntx & asid_mask, context_asid_map); + __set_bit(cntx2asid(cntx), context_asid_map); per_cpu(reserved_context, i) =3D cntx; } =20 @@ -102,7 +102,7 @@ static unsigned long __new_context(struct mm_struct *mm) lockdep_assert_held(&context_lock); =20 if (cntx !=3D 0) { - unsigned long newcntx =3D ver | (cntx & asid_mask); + unsigned long newcntx =3D ver | cntx2asid(cntx); =20 /* * If our current CONTEXT was active during a rollover, we @@ -115,7 +115,7 @@ static unsigned long __new_context(struct mm_struct *mm) * We had a valid CONTEXT in a previous life, so try to * re-use it if possible. */ - if (!__test_and_set_bit(cntx & asid_mask, context_asid_map)) + if (!__test_and_set_bit(cntx2asid(cntx), context_asid_map)) return newcntx; } =20 @@ -168,7 +168,7 @@ static void set_mm_asid(struct mm_struct *mm, unsigned = int cpu) */ old_active_cntx =3D atomic_long_read(&per_cpu(active_context, cpu)); if (old_active_cntx && - ((cntx & ~asid_mask) =3D=3D atomic_long_read(¤t_version)) && + (cntx2version(cntx) =3D=3D atomic_long_read(¤t_version)) && atomic_long_cmpxchg_relaxed(&per_cpu(active_context, cpu), old_active_cntx, cntx)) goto switch_mm_fast; @@ -177,7 +177,7 @@ static void set_mm_asid(struct mm_struct *mm, unsigned = int cpu) =20 /* Check that our ASID belongs to the current_version. */ cntx =3D atomic_long_read(&mm->context.id); - if ((cntx & ~asid_mask) !=3D atomic_long_read(¤t_version)) { + if (cntx2version(cntx) !=3D atomic_long_read(¤t_version)) { cntx =3D __new_context(mm); atomic_long_set(&mm->context.id, cntx); } @@ -191,7 +191,7 @@ static void set_mm_asid(struct mm_struct *mm, unsigned = int cpu) =20 switch_mm_fast: csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | - ((cntx & asid_mask) << SATP_ASID_SHIFT) | + (cntx2asid(cntx) << SATP_ASID_SHIFT) | satp_mode); =20 if (need_flush_tlb) diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 27b3744b5673..23409d70440f 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -91,7 +91,7 @@ static void __flush_tlb_range(struct mm_struct *mm, unsig= ned long start, broadcast =3D cpumask_any_but(cmask, cpuid) < nr_cpu_ids; =20 if (static_branch_unlikely(&use_asid_allocator)) - asid =3D atomic_long_read(&mm->context.id) & asid_mask; + asid =3D cntx2asid(atomic_long_read(&mm->context.id)); } else { cmask =3D cpu_online_mask; broadcast =3D true; @@ -123,7 +123,7 @@ static void __flush_tlb_range(struct mm_struct *mm, uns= igned long start, unsigned long asid =3D FLUSH_TLB_NO_ASID; =20 if (mm && static_branch_unlikely(&use_asid_allocator)) - asid =3D atomic_long_read(&mm->context.id) & asid_mask; + asid =3D cntx2asid(atomic_long_read(&mm->context.id)); =20 local_flush_tlb_range_asid(start, size, stride, asid); } --=20 2.42.0 From nobody Wed Dec 17 21:01:18 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id ED2B9C61D97 for ; Wed, 22 Nov 2023 01:08:34 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1343492AbjKVBIg (ORCPT ); Tue, 21 Nov 2023 20:08:36 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:58916 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S235024AbjKVBI2 (ORCPT ); Tue, 21 Nov 2023 20:08:28 -0500 Received: from mail-il1-x135.google.com (mail-il1-x135.google.com [IPv6:2607:f8b0:4864:20::135]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 7BFEC1A3 for ; Tue, 21 Nov 2023 17:08:24 -0800 (PST) Received: by mail-il1-x135.google.com with SMTP id e9e14a558f8ab-35b07f701c4so7176355ab.2 for ; Tue, 21 Nov 2023 17:08:24 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sifive.com; s=google; t=1700615304; x=1701220104; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=b39ZSDVnn3/TpDDYcLNRgBYhWVyekPNc8OuxKxdgheo=; b=jGFhxv+q81USzJS8Lm5q+YQPv1bGi5N1k4DKcQFe1lg7H+HKyRO3zQkQb/wmFGjTDl wJA69QBg9P3oCQNwL0amVuWmJOm+m241n9H/bJfX1jjDppKnOC3g2D4pvmQkocb4UzED 36WIyys7pwiwgwn7mM3F0SCa7BGBs+/58jxJi/xJIA69OQXTuREk8dWz7KiVtXJtaqlF rYvBA7Zn5Y8I7ASNFFVrQwoaZUF2Cg4cAi+oQNtgCHYIDmOVpsasAvnXIsl/djFPtQOa QaIicCaDoCgNiLst0qMBn3/88hdVNJ19CygLRZhiYQdCDkdmrAYwyZYPdScUumoeDr6l xJ1w== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1700615304; x=1701220104; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=b39ZSDVnn3/TpDDYcLNRgBYhWVyekPNc8OuxKxdgheo=; b=fWWhyVan4DVFQ45e5IAYDvnppPvaZPCn3IlQ1w7NlCxlquyHzOI0u5/QQ/Po3PIN1a fWlRz/kXmC2w9MiFczQcMcbIdPNppRHNP91Dp7MimRgXCkhsCL40xZRFyywAOth/XV8K jB7pmsRgC3B6QxHhQP6uyF54rd5fnfWxQpERNyoQIi5t3najYCQ8LchmjRxplDxwG1El NcCgmXQAvkvcnt4X6WpYCYliQVDeh7ZCmEQQz/cs1FcWsfhkuBo+0zFHlgwfF0+H6tg1 pddblWsyT9zB+nI7F+D8OeeWs7FHco4OL54x9jL8f+tLT8zDRMjBJ0vqO/ysaLaazVW1 WbVw== X-Gm-Message-State: AOJu0YzZ06kFEx7PiMsQFZnoZWlkaV++59MTo/OTJMIknHl7ug4YaFZO ltwTX9FSlPWgqyPePPt6tA3sdMX+jdMOiYmfnhY= X-Google-Smtp-Source: AGHT+IGi2JpVuCgk8T+HgcOfCC/VHmmgf8iSnDwMNXS0Bc2add3cL906GsPfWT4iX9QyNn2xttBJig== X-Received: by 2002:a05:6e02:b48:b0:35b:2b49:d816 with SMTP id f8-20020a056e020b4800b0035b2b49d816mr721861ilu.13.1700615303918; Tue, 21 Nov 2023 17:08:23 -0800 (PST) Received: from sw06.internal.sifive.com ([4.53.31.132]) by smtp.gmail.com with ESMTPSA id bn5-20020a056a00324500b006be047268d5sm8713961pfb.174.2023.11.21.17.08.23 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 21 Nov 2023 17:08:23 -0800 (PST) From: Samuel Holland To: Palmer Dabbelt , linux-riscv@lists.infradead.org Cc: linux-kernel@vger.kernel.org, linux-mm@kvack.org, Alexandre Ghiti , Samuel Holland Subject: [PATCH v3 5/8] riscv: mm: Use a fixed layout for the MM context ID Date: Tue, 21 Nov 2023 17:07:16 -0800 Message-ID: <20231122010815.3545294-6-samuel.holland@sifive.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231122010815.3545294-1-samuel.holland@sifive.com> References: <20231122010815.3545294-1-samuel.holland@sifive.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Currently, the size of the ASID field in the MM context ID dynamically depends on the number of hardware-supported ASID bits. This requires reading a global variable to extract either field from the context ID. Instead, allocate the maximum possible number of bits to the ASID field, so the layout of the context ID is known at compile-time. Signed-off-by: Samuel Holland --- (no changes since v1) arch/riscv/include/asm/mmu.h | 4 ++-- arch/riscv/include/asm/tlbflush.h | 2 -- arch/riscv/mm/context.c | 6 ++---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/arch/riscv/include/asm/mmu.h b/arch/riscv/include/asm/mmu.h index a550fbf770be..dc0273f7905f 100644 --- a/arch/riscv/include/asm/mmu.h +++ b/arch/riscv/include/asm/mmu.h @@ -26,8 +26,8 @@ typedef struct { #endif } mm_context_t; =20 -#define cntx2asid(cntx) ((cntx) & asid_mask) -#define cntx2version(cntx) ((cntx) & ~asid_mask) +#define cntx2asid(cntx) ((cntx) & SATP_ASID_MASK) +#define cntx2version(cntx) ((cntx) & ~SATP_ASID_MASK) =20 void __init create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, pgprot_t prot); diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlb= flush.h index 3b393f765805..4448d907f2c9 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -15,8 +15,6 @@ #define FLUSH_TLB_NO_ASID ((unsigned long)-1) =20 #ifdef CONFIG_MMU -extern unsigned long asid_mask; - static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory"); diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index 43d005f63253..b5170ac1b742 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -22,7 +22,6 @@ DEFINE_STATIC_KEY_FALSE(use_asid_allocator); =20 static unsigned long asid_bits; static unsigned long num_asids; -unsigned long asid_mask; =20 static atomic_long_t current_version; =20 @@ -128,7 +127,7 @@ static unsigned long __new_context(struct mm_struct *mm) goto set_asid; =20 /* We're out of ASIDs, so increment current_version */ - ver =3D atomic_long_add_return_relaxed(num_asids, ¤t_version); + ver =3D atomic_long_add_return_relaxed(BIT(SATP_ASID_BITS), ¤t_vers= ion); =20 /* Flush everything */ __flush_context(); @@ -247,7 +246,6 @@ static int __init asids_init(void) /* Pre-compute ASID details */ if (asid_bits) { num_asids =3D 1 << asid_bits; - asid_mask =3D num_asids - 1; } =20 /* @@ -255,7 +253,7 @@ static int __init asids_init(void) * at-least twice more than CPUs */ if (num_asids > (2 * num_possible_cpus())) { - atomic_long_set(¤t_version, num_asids); + atomic_long_set(¤t_version, BIT(SATP_ASID_BITS)); =20 context_asid_map =3D bitmap_zalloc(num_asids, GFP_KERNEL); if (!context_asid_map) --=20 2.42.0 From nobody Wed Dec 17 21:01:18 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4CBF1C61D85 for ; Wed, 22 Nov 2023 01:08:41 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S235061AbjKVBIm (ORCPT ); Tue, 21 Nov 2023 20:08:42 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:58946 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S235031AbjKVBIa (ORCPT ); Tue, 21 Nov 2023 20:08:30 -0500 Received: from mail-ot1-x333.google.com (mail-ot1-x333.google.com [IPv6:2607:f8b0:4864:20::333]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id C0439D47 for ; Tue, 21 Nov 2023 17:08:25 -0800 (PST) Received: by mail-ot1-x333.google.com with SMTP id 46e09a7af769-6ce2eaf7c2bso3883743a34.0 for ; Tue, 21 Nov 2023 17:08:25 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sifive.com; s=google; t=1700615305; x=1701220105; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=O8uQuAx5HbeGrqX4Ip/qQY7nv6lrxw4qacCJFbpKbko=; b=nXz2QW8m13PAMP28eJBbdowxNhob8TF/eJCZS1R2xO0VquNmNSxQv5NPcDTr5i1TPy 4WHRT7idnD58/grgstzYoKrqbP96c9gZWrvEgO7oxDYeAjSEo+MYOcxj9OFlvWXliAYm dIyH0yBKAEhngtl5MEGBTwWc1kbH2XtSs/59IlPV7AmfSU30NVkbkBtBNJ46wxCkuEo9 UkYn54jCqiPkVeYiVv5jzLLjH8vi0CI/qnnr+Bs+f2nYdO4YlpbWc55TQ1+x47Ainhpm 1RlqdtJyurK8DQsuO6t2ETeDGom3rSzlQPYYwIz4v5VU14Fn24eqAK1I+NTivu+BD8KR KsWA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1700615305; x=1701220105; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=O8uQuAx5HbeGrqX4Ip/qQY7nv6lrxw4qacCJFbpKbko=; b=OXsADn7ndgpc6XsEQmTrduJTWYIEWV+H7fcwpf0EnoRsPQklrw0yZQxoBWwrHhjfb2 84bI+AJt25mnJ/nBATcawlNLrPx629Q9AhyYiQcNrq3peM8yfXFiF8/eXCyC6+frbtJP XATYclOHhTOc3z3ISip5YrIaxU9U6vtSBXRD3egZ7xDT8dk1wJWxI0Dig61DELKHTZie epPzfUgSpMtkOmpAQCK6n4NoKr97HwsU+5XewPkSvJqO5vyczTeoMUk1puJKCjhe6ayP FqUnWvyis4uzWtiGRUjhsZvcg6yAm+xt/R6eH6eNTrRwhEcWyCj2b8bb3NAWN9yYYo5n YDvw== X-Gm-Message-State: AOJu0YwpZLzFf8gfwyZE6WYuN/bqOEr+BjXy9DraowP5riyxRzYvtlII b0/FeKBFZ4qJ3Bh4iElrhZq2AQ== X-Google-Smtp-Source: AGHT+IHzY35P/bmGNlxsboL33Yxa5yN9S+7CxJ9PS0wbWCSnN9yRamcO2jnn4/Cd8sidyrAKu3XrJw== X-Received: by 2002:a05:6870:9b0c:b0:1f5:b5ca:435e with SMTP id hq12-20020a0568709b0c00b001f5b5ca435emr1104182oab.52.1700615304951; Tue, 21 Nov 2023 17:08:24 -0800 (PST) Received: from sw06.internal.sifive.com ([4.53.31.132]) by smtp.gmail.com with ESMTPSA id bn5-20020a056a00324500b006be047268d5sm8713961pfb.174.2023.11.21.17.08.24 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 21 Nov 2023 17:08:24 -0800 (PST) From: Samuel Holland To: Palmer Dabbelt , linux-riscv@lists.infradead.org Cc: linux-kernel@vger.kernel.org, linux-mm@kvack.org, Alexandre Ghiti , Samuel Holland Subject: [PATCH v3 6/8] riscv: mm: Make asid_bits a local variable Date: Tue, 21 Nov 2023 17:07:17 -0800 Message-ID: <20231122010815.3545294-7-samuel.holland@sifive.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231122010815.3545294-1-samuel.holland@sifive.com> References: <20231122010815.3545294-1-samuel.holland@sifive.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" This variable is only used inside asids_init(). Signed-off-by: Samuel Holland --- (no changes since v1) arch/riscv/mm/context.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index b5170ac1b742..43a8bc2d5af4 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -20,7 +20,6 @@ =20 DEFINE_STATIC_KEY_FALSE(use_asid_allocator); =20 -static unsigned long asid_bits; static unsigned long num_asids; =20 static atomic_long_t current_version; @@ -226,7 +225,7 @@ static inline void set_mm(struct mm_struct *prev, =20 static int __init asids_init(void) { - unsigned long old; + unsigned long asid_bits, old; =20 /* Figure-out number of ASID bits in HW */ old =3D csr_read(CSR_SATP); --=20 2.42.0 From nobody Wed Dec 17 21:01:18 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 924F4C61D9C for ; Wed, 22 Nov 2023 01:08:37 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1343497AbjKVBIj (ORCPT ); Tue, 21 Nov 2023 20:08:39 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:43376 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S235040AbjKVBId (ORCPT ); Tue, 21 Nov 2023 20:08:33 -0500 Received: from mail-pf1-x430.google.com (mail-pf1-x430.google.com [IPv6:2607:f8b0:4864:20::430]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 769FFD49 for ; Tue, 21 Nov 2023 17:08:26 -0800 (PST) Received: by mail-pf1-x430.google.com with SMTP id d2e1a72fcca58-6c115026985so6307211b3a.1 for ; Tue, 21 Nov 2023 17:08:26 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sifive.com; s=google; t=1700615306; x=1701220106; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=MROdnZxFJAn7brM6ye/jIKPJcvVCbRqlgsjUSxtP0PY=; b=HQj+34PuOvMXL7ecc/kUppXI1cRtaZlO8glozZvsZ6jQc5lf6H79mfMraLq8jhQn82 A984J/GFHnRxzJComBteltfgeESCtWYnIxRiG5+BMtsYt0OcjpKbHoi821Eh0Z7vmPFx zDXjYHpYmjSkQFQnnCHDB8wuVQqKl9/fiHtW7enbaJlkYsO0QCJfGO8iqmBIbG4WxV+n 7zSLZhoV1frwMRdatBxyBQsx8Zyes2gY9AdYAKaocvjID3j6VjNuA5cpmwywaeiWaYPV OeKKMj5uMcrSapv80IoASjSRJuUtWcCMeUXVQTV1YxZx6IlGWY48abFVth1jmnXEz6T3 2w5A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1700615306; x=1701220106; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=MROdnZxFJAn7brM6ye/jIKPJcvVCbRqlgsjUSxtP0PY=; b=sA6fGDHdbkpAS01mDaxllQrPqpWdsiP7/puD/MRVyaPxgIFLxkL51ibptxPoJ99vQm b74x1hmdHlaJ0hdCMPaVLXtjMJBe9Fkfpfz7TZ77+XUTW0emyF+ZN+uZMFod02+U1fM2 otteYSPB2dtJkcT696X44OK3XHdPor3sBDZ8aLRJEJp0uUNw14SU2TkA3afqGog56S3U Vxub3JlhIKo5nDWM0dsOusXiyhTg1BhzHYQnXaDVrOYiTiI7vNA6PM5xehITESB7zdPm 0vp/47h09FetMv0t3UIe69vXlWc0d+2tFnU4MVj+bBPMtoOSZjv8iYmbD7inBFEb1ltp dzCg== X-Gm-Message-State: AOJu0Yw2IItFvXBQOKMvi70H28/cQN4EK8mUOEiSXXKetz60t0P/2J/n r6il5zwkZuJLs2Fc4g3RhWcvHhw8+jQPZnqizIY= X-Google-Smtp-Source: AGHT+IFVZu+XEimHfZXOsjvtj11LrkIBAYyBlYwN/UfpJ7aqLGYZgxb4tCN+mMPanboANiNPoJtK3w== X-Received: by 2002:a05:6a00:b54:b0:6cb:a1fe:5217 with SMTP id p20-20020a056a000b5400b006cba1fe5217mr1041980pfo.16.1700615306009; Tue, 21 Nov 2023 17:08:26 -0800 (PST) Received: from sw06.internal.sifive.com ([4.53.31.132]) by smtp.gmail.com with ESMTPSA id bn5-20020a056a00324500b006be047268d5sm8713961pfb.174.2023.11.21.17.08.25 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 21 Nov 2023 17:08:25 -0800 (PST) From: Samuel Holland To: Palmer Dabbelt , linux-riscv@lists.infradead.org Cc: linux-kernel@vger.kernel.org, linux-mm@kvack.org, Alexandre Ghiti , Samuel Holland Subject: [PATCH v3 7/8] riscv: mm: Preserve global TLB entries when switching contexts Date: Tue, 21 Nov 2023 17:07:18 -0800 Message-ID: <20231122010815.3545294-8-samuel.holland@sifive.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231122010815.3545294-1-samuel.holland@sifive.com> References: <20231122010815.3545294-1-samuel.holland@sifive.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" If the CPU does not support multiple ASIDs, all MM contexts use ASID 0. In this case, it is still beneficial to flush the TLB by ASID, as the single-ASID variant of the sfence.vma instruction preserves TLB entries for global (kernel) pages. This optimization is recommended by the RISC-V privileged specification: If the implementation does not provide ASIDs, or software chooses to always use ASID 0, then after every satp write, software should execute SFENCE.VMA with rs1=3Dx0. In the common case that no global translations have been modified, rs2 should be set to a register other than x0 but which contains the value zero, so that global translations are not flushed. It is not possible to apply this optimization when using the ASID allocator, because that code must flush the TLB for all ASIDs at once when incrementing the version number. Signed-off-by: Samuel Holland --- (no changes since v1) arch/riscv/mm/context.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index 43a8bc2d5af4..3ca9b653df7d 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -200,7 +200,7 @@ static void set_mm_noasid(struct mm_struct *mm) { /* Switch the page table and blindly nuke entire local TLB */ csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | satp_mode); - local_flush_tlb_all(); + local_flush_tlb_all_asid(0); } =20 static inline void set_mm(struct mm_struct *prev, --=20 2.42.0 From nobody Wed Dec 17 21:01:18 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5C0C8C61D85 for ; Wed, 22 Nov 2023 01:08:45 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S235076AbjKVBIp (ORCPT ); Tue, 21 Nov 2023 20:08:45 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:43388 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S235053AbjKVBId (ORCPT ); Tue, 21 Nov 2023 20:08:33 -0500 Received: from mail-oi1-x22b.google.com (mail-oi1-x22b.google.com [IPv6:2607:f8b0:4864:20::22b]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id B1662199 for ; Tue, 21 Nov 2023 17:08:27 -0800 (PST) Received: by mail-oi1-x22b.google.com with SMTP id 5614622812f47-3b58d96a3bbso3637913b6e.1 for ; Tue, 21 Nov 2023 17:08:27 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sifive.com; s=google; t=1700615307; x=1701220107; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=Wj2ICW1Fptg7hPvDVbuVBoYZZibiQ7SyAcM7RK2jyBY=; b=eq1SR9abgzTitUEVa1WXCZZRtNJamTF6j/hVuHV/kxEuo/1w6ja5oRlG65TCxU//PU WVcCuHAHdjowyegBpF+RMRaM61V336FgQzWVo0zNkx/6aZZoWqpg8rXAoJCEl6YIZj+F UucYvk0idUl5uZtNlCUnwJnm1HCkWLWkEV9+0keo0nqVgO/Bt6o8Lc6uFxg/xr5A/khk kjF8EkV38tZrBl9d/cu5/e2/ZT4BCOcAAueoaxZS/L3JTeV1LLntPjn1stHlE2ARTPVq wKLUfoXYHYA1PKYkAIV7/CvDSczNWgI7JcUlK0FR7jelPOr5KfGWNwBJ3zz3IOV9z3tq PIHA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1700615307; x=1701220107; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=Wj2ICW1Fptg7hPvDVbuVBoYZZibiQ7SyAcM7RK2jyBY=; b=wUcK0WZDIyA9PTjTqWzDOdLoWOoJ/BtzscBbHW9b+CE2hIzHJV8LPbIfbdu+QPV5zA L/jLEPqlLgGSiNsxJFHSDgk6ggnJwU/sPOej6wxFmeP28kGkp5P+NsgTsfA89fVbE44a oDnaiKgvkL5J/tZlvhtIRk+5Jr5NxAW9hiJlk0KR3wc+al/ukiBSo7GAzc7uGw5zBAM6 MpWb9xbIJGUOQBbve55HL5Bgb1DKqJroqBOz/wifIy05wyLCcN4OG1THZpWSZ4TnnIVm lrANTQTudZ8Sjo9qkWPHLa2SOmNRUfI8VbYDUfdpTyRY6lSvCt1PEUW6xXHMNBlNkOCU MOsA== X-Gm-Message-State: AOJu0YxjM5NEqpBvYQMA0eG+ovOqygcIJDshArNgywP6PqT/FQ9uCauK 7p6H0Q97Hmmpubo3qw0X+arjOw== X-Google-Smtp-Source: AGHT+IE4b2JAkpIlmqFLR7/QsgdBYQzfOQCASKlXtamU7cDqbXpB+e41Rju/TG/nF3J4VXpKvwiJIw== X-Received: by 2002:a05:6808:699:b0:3b7:673:8705 with SMTP id k25-20020a056808069900b003b706738705mr885196oig.18.1700615307054; Tue, 21 Nov 2023 17:08:27 -0800 (PST) Received: from sw06.internal.sifive.com ([4.53.31.132]) by smtp.gmail.com with ESMTPSA id bn5-20020a056a00324500b006be047268d5sm8713961pfb.174.2023.11.21.17.08.26 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 21 Nov 2023 17:08:26 -0800 (PST) From: Samuel Holland To: Palmer Dabbelt , linux-riscv@lists.infradead.org Cc: linux-kernel@vger.kernel.org, linux-mm@kvack.org, Alexandre Ghiti , Samuel Holland Subject: [PATCH v3 8/8] riscv: mm: Always use ASID to flush MM contexts Date: Tue, 21 Nov 2023 17:07:19 -0800 Message-ID: <20231122010815.3545294-9-samuel.holland@sifive.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231122010815.3545294-1-samuel.holland@sifive.com> References: <20231122010815.3545294-1-samuel.holland@sifive.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Even if multiple ASIDs are not supported, using the single-ASID variant of the sfence.vma instruction preserves TLB entries for global (kernel) pages. So it is always more efficient to use the single-ASID code path. Signed-off-by: Samuel Holland --- (no changes since v2) Changes in v2: - Update both copies of __flush_tlb_range() arch/riscv/include/asm/mmu_context.h | 2 -- arch/riscv/mm/context.c | 3 +-- arch/riscv/mm/tlbflush.c | 5 ++--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/= mmu_context.h index 7030837adc1a..b0659413a080 100644 --- a/arch/riscv/include/asm/mmu_context.h +++ b/arch/riscv/include/asm/mmu_context.h @@ -33,8 +33,6 @@ static inline int init_new_context(struct task_struct *ts= k, return 0; } =20 -DECLARE_STATIC_KEY_FALSE(use_asid_allocator); - #include =20 #endif /* _ASM_RISCV_MMU_CONTEXT_H */ diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index 3ca9b653df7d..20057085ab8a 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -18,8 +18,7 @@ =20 #ifdef CONFIG_MMU =20 -DEFINE_STATIC_KEY_FALSE(use_asid_allocator); - +static DEFINE_STATIC_KEY_FALSE(use_asid_allocator); static unsigned long num_asids; =20 static atomic_long_t current_version; diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 23409d70440f..d6619be10341 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -90,8 +90,7 @@ static void __flush_tlb_range(struct mm_struct *mm, unsig= ned long start, /* check if the tlbflush needs to be sent to other CPUs */ broadcast =3D cpumask_any_but(cmask, cpuid) < nr_cpu_ids; =20 - if (static_branch_unlikely(&use_asid_allocator)) - asid =3D cntx2asid(atomic_long_read(&mm->context.id)); + asid =3D cntx2asid(atomic_long_read(&mm->context.id)); } else { cmask =3D cpu_online_mask; broadcast =3D true; @@ -122,7 +121,7 @@ static void __flush_tlb_range(struct mm_struct *mm, uns= igned long start, { unsigned long asid =3D FLUSH_TLB_NO_ASID; =20 - if (mm && static_branch_unlikely(&use_asid_allocator)) + if (mm) asid =3D cntx2asid(atomic_long_read(&mm->context.id)); =20 local_flush_tlb_range_asid(start, size, stride, asid); --=20 2.42.0