We can use the PR_{S,G}ET_MEMORY_CONSISTENCY_MODEL prctl calls to change
the memory consistency model at run-time if we have Ssdtso.
This patch registers RISCV_WMO and RISCV_TSO as valid arguments
for these prctl calls and implements the glue code to switch
between these.

Signed-off-by: Christoph Müllner <christoph.muellner@vrull.eu>
---
 .../mm/dynamic-memory-consistency-model.rst   | 12 +++-
 arch/riscv/include/asm/processor.h            |  7 ++
 arch/riscv/kernel/Makefile                    |  1 +
 arch/riscv/kernel/dtso.c                      | 67 +++++++++++++++++++
 include/uapi/linux/prctl.h                    |  2 +
 5 files changed, 88 insertions(+), 1 deletion(-)
 create mode 100644 arch/riscv/kernel/dtso.c

diff --git a/Documentation/mm/dynamic-memory-consistency-model.rst b/Documentation/mm/dynamic-memory-consistency-model.rst
index 1fce855a1fad..c8188c174e27 100644
--- a/Documentation/mm/dynamic-memory-consistency-model.rst
+++ b/Documentation/mm/dynamic-memory-consistency-model.rst
@@ -73,4 +73,14 @@ Supported memory consistency models
 This section defines the memory consistency models which are supported
 by the prctl interface.
 
-<none>
+RISC-V
+------
+
+RISC-V uses RVWMO (RISC-V weak memory ordering) as default memory consistency
+model. TSO (total store ordering) is another specified model and provides
+additional ordering guarantees. Switching user-mode processes from RVWMO to TSO
+is possible when the Ssdtso extension is available.
+
+* :c:macro:`PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO`: RISC-V weak memory ordering (default).
+
+* :c:macro:`PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO`: RISC-V total store ordering.
diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
index a8509cc31ab2..05e05fddc94d 100644
--- a/arch/riscv/include/asm/processor.h
+++ b/arch/riscv/include/asm/processor.h
@@ -184,6 +184,13 @@ extern int set_unalign_ctl(struct task_struct *tsk, unsigned int val);
 #define GET_UNALIGN_CTL(tsk, addr)	get_unalign_ctl((tsk), (addr))
 #define SET_UNALIGN_CTL(tsk, val)	set_unalign_ctl((tsk), (val))
 
+#ifdef CONFIG_RISCV_ISA_SSDTSO
+extern int dtso_set_memory_consistency_model(unsigned long arg);
+extern int dtso_get_memory_consistency_model(void);
+#define SET_MEMORY_CONSISTENCY_MODEL(arg)	dtso_set_memory_consistency_model(arg)
+#define GET_MEMORY_CONSISTENCY_MODEL()		dtso_get_memory_consistency_model()
+#endif /* CONIG_RISCV_ISA_SSDTSO */
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* _ASM_RISCV_PROCESSOR_H */
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index f71910718053..85f7291da498 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -65,6 +65,7 @@ obj-$(CONFIG_RISCV_MISALIGNED)	+= traps_misaligned.o
 obj-$(CONFIG_FPU)		+= fpu.o
 obj-$(CONFIG_RISCV_ISA_V)	+= vector.o
 obj-$(CONFIG_RISCV_ISA_V)	+= kernel_mode_vector.o
+obj-$(CONFIG_RISCV_ISA_SSDTSO)	+= dtso.o
 obj-$(CONFIG_SMP)		+= smpboot.o
 obj-$(CONFIG_SMP)		+= smp.o
 obj-$(CONFIG_SMP)		+= cpu_ops.o
diff --git a/arch/riscv/kernel/dtso.c b/arch/riscv/kernel/dtso.c
new file mode 100644
index 000000000000..591d5f9de0f5
--- /dev/null
+++ b/arch/riscv/kernel/dtso.c
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (c) 2024 Christoph Muellner <christoph.muellner@vrull.eu>
+ */
+
+#include <linux/cpu.h>
+#include <linux/smp.h>
+#include <linux/prctl.h>
+
+#include <asm/cpu.h>
+#include <asm/dtso.h>
+
+#include <trace/events/ipi.h>
+
+int dtso_set_memory_consistency_model(unsigned long arg)
+{
+	int cpu;
+	unsigned long cur_model = get_memory_consistency_model(current);
+	unsigned long new_model;
+
+	switch (arg) {
+	case PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO:
+		new_model = RISCV_MEMORY_CONSISTENCY_MODEL_WMO;
+		break;
+	case PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO:
+		new_model = RISCV_MEMORY_CONSISTENCY_MODEL_TSO;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* No change requested. */
+	if (cur_model == new_model)
+		return 0;
+
+	/* Enabling TSO only works if DTSO is available. */
+	if (new_model == PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO && !has_dtso())
+		return -EINVAL;
+
+	/* Switching TSO->WMO is not allowed. */
+	if (new_model == RISCV_MEMORY_CONSISTENCY_MODEL_WMO)
+		return -EINVAL;
+
+	/* Set the new model in the task struct. */
+	set_memory_consitency_model(current, new_model);
+
+	/*
+	 * We need to reschedule all threads of the current process.
+	 * Let's do this by rescheduling all CPUs.
+	 * This is stricter than necessary, but since this call is
+	 * not expected to happen frequently the impact is low.
+	 */
+	for_each_cpu(cpu, cpu_online_mask)
+		smp_send_reschedule(cpu);
+
+	return 0;
+}
+
+int dtso_get_memory_consistency_model(void)
+{
+	unsigned long cur_model = get_memory_consistency_model(current);
+
+	if (cur_model == RISCV_MEMORY_CONSISTENCY_MODEL_TSO)
+		return PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO;
+
+	return PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO;
+}
diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
index 579662731eaa..20264bdc3092 100644
--- a/include/uapi/linux/prctl.h
+++ b/include/uapi/linux/prctl.h
@@ -308,5 +308,7 @@ struct prctl_mm_map {
 
 #define PR_SET_MEMORY_CONSISTENCY_MODEL		71
 #define PR_GET_MEMORY_CONSISTENCY_MODEL		72
+# define PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO	1
+# define PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO	2
 
 #endif /* _LINUX_PRCTL_H */
-- 
2.43.0

Hi Mullner,

Thanks for this work, although it has already lasted for about 2 years.

On 9/2/2024 14:40, Christoph Müllner wrote:
> We can use the PR_{S,G}ET_MEMORY_CONSISTENCY_MODEL prctl calls to change
> the memory consistency model at run-time if we have Ssdtso.
> This patch registers RISCV_WMO and RISCV_TSO as valid arguments
> for these prctl calls and implements the glue code to switch
> between these.
> 
> Signed-off-by: Christoph Müllner <christoph.muellner@vrull.eu>
> ---
>   .../mm/dynamic-memory-consistency-model.rst   | 12 +++-
>   arch/riscv/include/asm/processor.h            |  7 ++
>   arch/riscv/kernel/Makefile                    |  1 +
>   arch/riscv/kernel/dtso.c                      | 67 +++++++++++++++++++
>   include/uapi/linux/prctl.h                    |  2 +
>   5 files changed, 88 insertions(+), 1 deletion(-)
>   create mode 100644 arch/riscv/kernel/dtso.c
> 
> diff --git a/Documentation/mm/dynamic-memory-consistency-model.rst b/Documentation/mm/dynamic-memory-consistency-model.rst
> index 1fce855a1fad..c8188c174e27 100644
> --- a/Documentation/mm/dynamic-memory-consistency-model.rst
> +++ b/Documentation/mm/dynamic-memory-consistency-model.rst
> @@ -73,4 +73,14 @@ Supported memory consistency models
>   This section defines the memory consistency models which are supported
>   by the prctl interface.
>   
> -<none>
> +RISC-V
> +------
> +
> +RISC-V uses RVWMO (RISC-V weak memory ordering) as default memory consistency
> +model. TSO (total store ordering) is another specified model and provides
> +additional ordering guarantees. Switching user-mode processes from RVWMO to TSO
> +is possible when the Ssdtso extension is available.
> +
> +* :c:macro:`PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO`: RISC-V weak memory ordering (default).
> +
> +* :c:macro:`PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO`: RISC-V total store ordering.
> diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
> index a8509cc31ab2..05e05fddc94d 100644
> --- a/arch/riscv/include/asm/processor.h
> +++ b/arch/riscv/include/asm/processor.h
> @@ -184,6 +184,13 @@ extern int set_unalign_ctl(struct task_struct *tsk, unsigned int val);
>   #define GET_UNALIGN_CTL(tsk, addr)	get_unalign_ctl((tsk), (addr))
>   #define SET_UNALIGN_CTL(tsk, val)	set_unalign_ctl((tsk), (val))
>   
> +#ifdef CONFIG_RISCV_ISA_SSDTSO
> +extern int dtso_set_memory_consistency_model(unsigned long arg);
> +extern int dtso_get_memory_consistency_model(void);
> +#define SET_MEMORY_CONSISTENCY_MODEL(arg)	dtso_set_memory_consistency_model(arg)
> +#define GET_MEMORY_CONSISTENCY_MODEL()		dtso_get_memory_consistency_model()
> +#endif /* CONIG_RISCV_ISA_SSDTSO */
> +
>   #endif /* __ASSEMBLY__ */
>   
>   #endif /* _ASM_RISCV_PROCESSOR_H */
> diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
> index f71910718053..85f7291da498 100644
> --- a/arch/riscv/kernel/Makefile
> +++ b/arch/riscv/kernel/Makefile
> @@ -65,6 +65,7 @@ obj-$(CONFIG_RISCV_MISALIGNED)	+= traps_misaligned.o
>   obj-$(CONFIG_FPU)		+= fpu.o
>   obj-$(CONFIG_RISCV_ISA_V)	+= vector.o
>   obj-$(CONFIG_RISCV_ISA_V)	+= kernel_mode_vector.o
> +obj-$(CONFIG_RISCV_ISA_SSDTSO)	+= dtso.o
>   obj-$(CONFIG_SMP)		+= smpboot.o
>   obj-$(CONFIG_SMP)		+= smp.o
>   obj-$(CONFIG_SMP)		+= cpu_ops.o
> diff --git a/arch/riscv/kernel/dtso.c b/arch/riscv/kernel/dtso.c
> new file mode 100644
> index 000000000000..591d5f9de0f5
> --- /dev/null
> +++ b/arch/riscv/kernel/dtso.c
> @@ -0,0 +1,67 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + * Copyright (c) 2024 Christoph Muellner <christoph.muellner@vrull.eu>
> + */
> +
> +#include <linux/cpu.h>
> +#include <linux/smp.h>
> +#include <linux/prctl.h>
> +
> +#include <asm/cpu.h>
> +#include <asm/dtso.h>
> +
> +#include <trace/events/ipi.h>
> +
> +int dtso_set_memory_consistency_model(unsigned long arg)
> +{
> +	int cpu;
> +	unsigned long cur_model = get_memory_consistency_model(current);
> +	unsigned long new_model;
> +
> +	switch (arg) {
> +	case PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO:
> +		new_model = RISCV_MEMORY_CONSISTENCY_MODEL_WMO;
> +		break;
> +	case PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO:
> +		new_model = RISCV_MEMORY_CONSISTENCY_MODEL_TSO;
> +		break;
> +	default:
> +		return -EINVAL;
> +	}
> +
> +	/* No change requested. */
> +	if (cur_model == new_model)
> +		return 0;
> +
> +	/* Enabling TSO only works if DTSO is available. */
> +	if (new_model == PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO && !has_dtso())
> +		return -EINVAL;
> +
> +	/* Switching TSO->WMO is not allowed. */
> +	if (new_model == RISCV_MEMORY_CONSISTENCY_MODEL_WMO)
> +		return -EINVAL;
> +
> +	/* Set the new model in the task struct. */
> +	set_memory_consitency_model(current, new_model);
> +
> +	/*
> +	 * We need to reschedule all threads of the current process.
> +	 * Let's do this by rescheduling all CPUs.
> +	 * This is stricter than necessary, but since this call is
> +	 * not expected to happen frequently the impact is low.
> +	 */
> +	for_each_cpu(cpu, cpu_online_mask)
> +		smp_send_reschedule(cpu);
> +
> +	return 0;
> +}
> +
> +int dtso_get_memory_consistency_model(void)
> +{
> +	unsigned long cur_model = get_memory_consistency_model(current);
> +
> +	if (cur_model == RISCV_MEMORY_CONSISTENCY_MODEL_TSO)
> +		return PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO;
> +
> +	return PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO;
> +}
> diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
> index 579662731eaa..20264bdc3092 100644
> --- a/include/uapi/linux/prctl.h
> +++ b/include/uapi/linux/prctl.h
> @@ -308,5 +308,7 @@ struct prctl_mm_map {
>   
>   #define PR_SET_MEMORY_CONSISTENCY_MODEL		71
>   #define PR_GET_MEMORY_CONSISTENCY_MODEL		72
> +# define PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO	1
> +# define PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO	2

Should we replace "PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO" with 
"PR_MEMORY_CONSISTENCY_MODEL_TSO", so that it can share the same key as 
Apple's TSO implementation [1]? RISC-V Ssdtso would make such prctl more 
likely to be accepted.

[1] https://lore.kernel.org/lkml/20240411-tso-v1-0-754f11abfbff@marcan.st/

Thanks,
Yangyu Chen

>   
>   #endif /* _LINUX_PRCTL_H */