From nobody Wed Feb 11 23:44:39 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 833AFC7EE22 for ; Wed, 3 May 2023 09:07:37 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229948AbjECJHe (ORCPT ); Wed, 3 May 2023 05:07:34 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45290 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229705AbjECJHX (ORCPT ); Wed, 3 May 2023 05:07:23 -0400 Received: from mail-wr1-x434.google.com (mail-wr1-x434.google.com [IPv6:2a00:1450:4864:20::434]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id D7E55420B for ; Wed, 3 May 2023 02:07:18 -0700 (PDT) Received: by mail-wr1-x434.google.com with SMTP id ffacd0b85a97d-30639daee76so1168136f8f.1 for ; Wed, 03 May 2023 02:07:18 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104837; x=1685696837; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=eyybpekGpHGVvvaVewFrSrZRoEZZ/l3dGyzewmhKKwg=; b=Hpob7q1r0p4V0caXE2crEE/51FttP5ZNpxWPBnl62DekIdsJcpMiZqQ+U+0jmvfFhs BuAsztkxherqbyie+iWYnyQMMPYONBuj/oH4n3bIkTFCW6YxZ1Qzi8QEu4wrtQeU4R3m 7pVC6wXg8s6dc6fWLdUSxUdlxle+M8WZmsl6xw4ADtWTPKY3Cr+OeVC1eL/tZD3HUO9D q8HSTwHe/phJrkywr0Cj2YrKfBXdA8V3FvuTAJaBlWK+ou8a65EmklXozWcnoVqQl1Nt SaOHMzoSKIqGjhbX5y3I97bbfDs7YJsYXztcWuw71U7QjlCUZXS0UIkgwyYoGaNAdEA8 IrdQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104837; x=1685696837; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=eyybpekGpHGVvvaVewFrSrZRoEZZ/l3dGyzewmhKKwg=; b=ae9HYl3ufYHoY72dA88M1IrcDC1jQwLvM0/ONfyK36jPp7mMnZ43mOcCc1mwU+yyDk +ylJmpr26U6awHm6PTe82n/NlRO3G6p9nYIoDdTwc+lwIa9hxTywi3YAISoWYJjZrJW1 5ljkTVECL9iTkjonjEf8pJ4JD5a66NCdVbaks3t8v7B+JdqNDFZqf6mDuEblxZPD/VEn 6xmUCvITQWNAlV6x+DcuyH8Tx4X/Ezh3CaDHn4NxchlPwMoPWLvgJgFTAc41vQjFLg9i yvXoev/WW9RjJgh4k40dGEnbZRJUPvCGMkjzH6zL1f2lYvZ8zvQAbx+DKZBuQSQ1P0Bd zLVg== X-Gm-Message-State: AC+VfDxiuJHovpPNMpMV66Y6lDQaCTYjy4q0RKkA7DET16yRGEArd1dh Fyneh3udDQjeWRA8UdtTv1HhXw== X-Google-Smtp-Source: ACHHUZ4c7pdGYmFDE+EObHEdf29jnpXn/rtwlDJWATkEBCQ9LPR+y1yLX/HtD1fPIL1dXO2oFBe9Lg== X-Received: by 2002:a5d:54d1:0:b0:306:2ff6:5cbf with SMTP id x17-20020a5d54d1000000b003062ff65cbfmr6873320wrv.24.1683104837081; Wed, 03 May 2023 02:07:17 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id e2-20020a056000120200b00306281cfa59sm9741741wrx.47.2023.05.03.02.07.16 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:16 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 01/11] rust: add radix tree abstraction Date: Wed, 3 May 2023 11:06:58 +0200 Message-Id: <20230503090708.2524310-2-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Andreas Hindborg Add abstractions for the C radix_tree. This abstraction allows Rust code to= use the radix_tree as a map from `u64` keys to `ForeignOwnable` values. Signed-off-by: Andreas Hindborg --- rust/bindings/bindings_helper.h | 2 + rust/bindings/lib.rs | 1 + rust/helpers.c | 22 +++++ rust/kernel/lib.rs | 1 + rust/kernel/radix_tree.rs | 156 ++++++++++++++++++++++++++++++++ 5 files changed, 182 insertions(+) create mode 100644 rust/kernel/radix_tree.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helpe= r.h index 50e7a76d5455..52834962b94d 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -10,7 +10,9 @@ #include #include #include +#include =20 /* `bindgen` gets confused at certain things. */ const gfp_t BINDINGS_GFP_KERNEL =3D GFP_KERNEL; +const gfp_t BINDINGS_GFP_ATOMIC =3D GFP_ATOMIC; const gfp_t BINDINGS___GFP_ZERO =3D __GFP_ZERO; diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 7b246454e009..62f36a9eb1f4 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -51,4 +51,5 @@ mod bindings_helper { pub use bindings_raw::*; =20 pub const GFP_KERNEL: gfp_t =3D BINDINGS_GFP_KERNEL; +pub const GFP_ATOMIC: gfp_t =3D BINDINGS_GFP_ATOMIC; pub const __GFP_ZERO: gfp_t =3D BINDINGS___GFP_ZERO; diff --git a/rust/helpers.c b/rust/helpers.c index 81e80261d597..5dd5e325b7cc 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -26,6 +26,7 @@ #include #include #include +#include =20 __noreturn void rust_helper_BUG(void) { @@ -128,6 +129,27 @@ void rust_helper_put_task_struct(struct task_struct *t) } EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); =20 +void rust_helper_init_radix_tree(struct xarray *tree, gfp_t gfp_mask) +{ + INIT_RADIX_TREE(tree, gfp_mask); +} +EXPORT_SYMBOL_GPL(rust_helper_init_radix_tree); + +void **rust_helper_radix_tree_iter_init(struct radix_tree_iter *iter, + unsigned long start) +{ + return radix_tree_iter_init(iter, start); +} +EXPORT_SYMBOL_GPL(rust_helper_radix_tree_iter_init); + +void **rust_helper_radix_tree_next_slot(void **slot, + struct radix_tree_iter *iter, + unsigned flags) +{ + return radix_tree_next_slot(slot, iter, flags); +} +EXPORT_SYMBOL_GPL(rust_helper_radix_tree_next_slot); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` ty= pe * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 676995d4e460..a85cb6aae8d6 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -40,6 +40,7 @@ pub mod init; pub mod ioctl; pub mod prelude; pub mod print; +pub mod radix_tree; mod static_assert; #[doc(hidden)] pub mod std_vendor; diff --git a/rust/kernel/radix_tree.rs b/rust/kernel/radix_tree.rs new file mode 100644 index 000000000000..f659ab8b017c --- /dev/null +++ b/rust/kernel/radix_tree.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! RadixTree abstraction. +//! +//! C header: [`include/linux/radix_tree.h`](../../include/linux/radix_tre= e.h) + +use crate::error::to_result; +use crate::error::Result; +use crate::types::ForeignOwnable; +use crate::types::Opaque; +use crate::types::ScopeGuard; +use alloc::boxed::Box; +use core::marker::PhantomData; +use core::pin::Pin; + +type Key =3D u64; + +/// A map of `u64` to `ForeignOwnable` +/// +/// # Invariants +/// +/// - `tree` always points to a valid and initialized `struct radix_tree`. +/// - Pointers stored in the tree are created by a call to `ForignOwnable:= :into_foreign()` +pub struct RadixTree { + tree: Pin>>, + _marker: PhantomData, +} + +impl RadixTree { + /// Create a new radix tree + /// + /// Note: This function allocates memory with `GFP_ATOMIC`. + pub fn new() -> Result { + let tree =3D Pin::from(Box::try_new(Opaque::uninit())?); + + // SAFETY: `tree` points to allocated but not initialized memory. = This + // call will initialize the memory. + unsafe { bindings::init_radix_tree(tree.get(), bindings::GFP_ATOMI= C) }; + + Ok(Self { + tree, + _marker: PhantomData, + }) + } + + /// Try to insert a value into the tree + pub fn try_insert(&mut self, key: Key, value: V) -> Result<()> { + // SAFETY: `self.tree` points to a valid and initialized `struct r= adix_tree` + let ret =3D + unsafe { bindings::radix_tree_insert(self.tree.get(), key, val= ue.into_foreign() as _) }; + to_result(ret) + } + + /// Search for `key` in the map. Returns a reference to the associated + /// value if found. + pub fn get(&self, key: Key) -> Option> { + // SAFETY: `self.tree` points to a valid and initialized `struct r= adix_tree` + let item =3D + core::ptr::NonNull::new(unsafe { bindings::radix_tree_lookup(s= elf.tree.get(), key) })?; + + // SAFETY: `item` was created by a call to + // `ForeignOwnable::into_foreign()`. As `get_mut()` and `remove()`= takes + // a `&mut self`, no mutable borrows for `item` can exist and + // `ForeignOwnable::from_foreign()` cannot be called until this bo= rrow + // is dropped. + Some(unsafe { V::borrow(item.as_ptr()) }) + } + + /// Search for `key` in the map. Return a mutable reference to the + /// associated value if found. + pub fn get_mut(&mut self, key: Key) -> Option> { + let item =3D + core::ptr::NonNull::new(unsafe { bindings::radix_tree_lookup(s= elf.tree.get(), key) })?; + + // SAFETY: `item` was created by a call to + // `ForeignOwnable::into_foreign()`. As `get()` takes a `&self` and + // `remove()` takes a `&mut self`, no borrows for `item` can exist= and + // `ForeignOwnable::from_foreign()` cannot be called until this bo= rrow + // is dropped. + Some(MutBorrow { + guard: unsafe { V::borrow_mut(item.as_ptr()) }, + _marker: core::marker::PhantomData, + }) + } + + /// Search for `key` in the map. If `key` is found, the key and value = is + /// removed from the map and the value is returned. + pub fn remove(&mut self, key: Key) -> Option { + // SAFETY: `self.tree` points to a valid and initialized `struct r= adix_tree` + let item =3D + core::ptr::NonNull::new(unsafe { bindings::radix_tree_delete(s= elf.tree.get(), key) })?; + + // SAFETY: `item` was created by a call to + // `ForeignOwnable::into_foreign()` and no borrows to `item` can e= xist + // because this function takes a `&mut self`. + Some(unsafe { ForeignOwnable::from_foreign(item.as_ptr()) }) + } +} + +impl Drop for RadixTree { + fn drop(&mut self) { + let mut iter =3D bindings::radix_tree_iter { + index: 0, + next_index: 0, + tags: 0, + node: core::ptr::null_mut(), + }; + + // SAFETY: Iter is valid as we allocated it on the stack above + let mut slot =3D unsafe { bindings::radix_tree_iter_init(&mut iter= , 0) }; + loop { + if slot.is_null() { + // SAFETY: Both `self.tree` and `iter` are valid + slot =3D unsafe { bindings::radix_tree_next_chunk(self.tre= e.get(), &mut iter, 0) }; + } + + if slot.is_null() { + break; + } + + // SAFETY: `self.tree` is valid and iter is managed by + // `radix_tree_next_chunk()` and `radix_tree_next_slot()` + let item =3D unsafe { bindings::radix_tree_delete(self.tree.ge= t(), iter.index) }; + assert!(!item.is_null()); + + // SAFETY: All items in the tree are created by a call to + // `ForeignOwnable::into_foreign()`. + let _ =3D unsafe { V::from_foreign(item) }; + + // SAFETY: `self.tree` is valid and iter is managed by + // `radix_tree_next_chunk()` and `radix_tree_next_slot()`. Slo= t is + // not null. + slot =3D unsafe { bindings::radix_tree_next_slot(slot, &mut it= er, 0) }; + } + } +} + +/// A mutable borrow of an object owned by a `RadixTree` +pub struct MutBorrow<'a, V: ForeignOwnable> { + guard: ScopeGuard, + _marker: core::marker::PhantomData<&'a mut V>, +} + +impl<'a, V: ForeignOwnable> core::ops::Deref for MutBorrow<'a, V> { + type Target =3D ScopeGuard; + + fn deref(&self) -> &Self::Target { + &self.guard + } +} + +impl<'a, V: ForeignOwnable> core::ops::DerefMut for MutBorrow<'a, V> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.guard + } +} --=20 2.40.0 From nobody Wed Feb 11 23:44:39 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0EDD0C7EE22 for ; Wed, 3 May 2023 09:07:33 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229941AbjECJH3 (ORCPT ); Wed, 3 May 2023 05:07:29 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45316 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229764AbjECJHZ (ORCPT ); Wed, 3 May 2023 05:07:25 -0400 Received: from mail-wr1-x432.google.com (mail-wr1-x432.google.com [IPv6:2a00:1450:4864:20::432]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id DF9764691 for ; Wed, 3 May 2023 02:07:19 -0700 (PDT) Received: by mail-wr1-x432.google.com with SMTP id ffacd0b85a97d-3063afa2372so1356511f8f.0 for ; Wed, 03 May 2023 02:07:19 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104838; x=1685696838; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=s2OF8cEuaV6eZXvZ8+TxAkEcAWTLOcEvP8M1z8Cu3eg=; b=gxwZbpfudAGlkc9suPmtflL1jblQr4MK2hN1E3gaOGf/Plu3em+uZBGDw0rNUr7SUW EFB1a9htiLdSYQkjbn0JwIhIuQzJvNeRrzMlV0eXOLXXzIMN08VTuR5LYMKTn7m05OvH 9LHtC/oRDhdH3ACVUabMlgXW9Ewt8dD9XDI4Ez+21QfYADeNMEfiUmJzzu0XcckSKJuW Nj3Eh8mnhKzN9/T17uqbvi142/KhX1PBU6Qk/kU/ro0xuE5e8kN3AfwwdKqZvdZVSRvp 4qO3v7fHN7bd8dDrEFaz1iXrwav7RAETaSSc1oLICAvTlYivVQel8jdUiGT3J45aYlDO jWSQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104838; x=1685696838; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=s2OF8cEuaV6eZXvZ8+TxAkEcAWTLOcEvP8M1z8Cu3eg=; b=B3djbIyFvFI1xcdYRleM7my6CV2w9AGmi0Js/kbqc5yaZdghRmV013oVBgH/RGSrX/ uwutFZUtCJjbSCtD4QXxRaAmofOZgkgEWbDc2MkQ0jEjFkpCfwzIIgGd+5fTvhO8566O rO0eWgAKDtOCgXZgUq2bUTSLSXrqfT/T1vu7NBg7PvqsOdC15ne+wRVXusIFCJzQoxA6 a+UrziC21rfJwkOro1mllKR1bLKhA1ab9yoRqRzmsPiV+Bueffv5nf4HAMX24tmQQ+A7 CahJyDikTuXNJVfAybXwCxIJ5lcmZQGu+AhscxDX+DzwHsv6qbd7WMXnMifsHYl6GXsH sjAA== X-Gm-Message-State: AC+VfDzqkwFbW16BVS1rN61ShpWfUt0Dr8AOWAwCCE/aYvXbMNeCywjB Zixf9suOGvFwXGS8nQJHSRVwyg== X-Google-Smtp-Source: ACHHUZ7yrDB5gpNUkKE0X5q8MEbvZmkfVFN8rXKsdmGE0Vy4CSxFk+RB6D8+DyE/dyNCU1ItHJSB8g== X-Received: by 2002:a05:6000:1145:b0:306:2cf5:79dc with SMTP id d5-20020a056000114500b003062cf579dcmr6255554wrx.35.1683104838080; Wed, 03 May 2023 02:07:18 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id t15-20020adfe44f000000b002f00793bd7asm33035982wrm.27.2023.05.03.02.07.17 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:17 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 02/11] rust: add `pages` module for handling page allocation Date: Wed, 3 May 2023 11:06:59 +0200 Message-Id: <20230503090708.2524310-3-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Andreas Hindborg This patch adds support for working with pages of order 0. Support for pages with higher order is deferred. Page allocation flags are fixed in this patc= h. Future work might allow the user to specify allocation flags. This patch is a heavily modified version of code available in the rust tree= [1], primarily adding support for multiple page mapping strategies. [1] https://github.com/rust-for-Linux/linux/tree/bc22545f38d74473cfef3e9fd6= 5432733435b79f/rust/kernel/pages.rs Signed-off-by: Andreas Hindborg --- rust/helpers.c | 31 +++++ rust/kernel/lib.rs | 6 + rust/kernel/pages.rs | 284 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 321 insertions(+) create mode 100644 rust/kernel/pages.rs diff --git a/rust/helpers.c b/rust/helpers.c index 5dd5e325b7cc..9bd9d95da951 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -27,6 +27,7 @@ #include #include #include +#include =20 __noreturn void rust_helper_BUG(void) { @@ -150,6 +151,36 @@ void **rust_helper_radix_tree_next_slot(void **slot, } EXPORT_SYMBOL_GPL(rust_helper_radix_tree_next_slot); =20 +void *rust_helper_kmap(struct page *page) +{ + return kmap(page); +} +EXPORT_SYMBOL_GPL(rust_helper_kmap); + +void rust_helper_kunmap(struct page *page) +{ + return kunmap(page); +} +EXPORT_SYMBOL_GPL(rust_helper_kunmap); + +void *rust_helper_kmap_atomic(struct page *page) +{ + return kmap_atomic(page); +} +EXPORT_SYMBOL_GPL(rust_helper_kmap_atomic); + +void rust_helper_kunmap_atomic(void *address) +{ + kunmap_atomic(address); +} +EXPORT_SYMBOL_GPL(rust_helper_kunmap_atomic); + +struct page *rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order) +{ + return alloc_pages(gfp_mask, order); +} +EXPORT_SYMBOL_GPL(rust_helper_alloc_pages); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` ty= pe * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index a85cb6aae8d6..8bef6686504b 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -38,6 +38,7 @@ mod build_assert; pub mod error; pub mod init; pub mod ioctl; +pub mod pages; pub mod prelude; pub mod print; pub mod radix_tree; @@ -57,6 +58,11 @@ pub use uapi; #[doc(hidden)] pub use build_error::build_error; =20 +/// Page size defined in terms of the `PAGE_SHIFT` macro from C. +/// +/// [`PAGE_SHIFT`]: ../../../include/asm-generic/page.h +pub const PAGE_SIZE: u32 =3D 1 << bindings::PAGE_SHIFT; + /// Prefix to appear before log messages printed from within the `kernel` = crate. const __LOG_PREFIX: &[u8] =3D b"rust_kernel\0"; =20 diff --git a/rust/kernel/pages.rs b/rust/kernel/pages.rs new file mode 100644 index 000000000000..ed51b053dd5d --- /dev/null +++ b/rust/kernel/pages.rs @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel page allocation and management. +//! +//! This module currently provides limited support. It supports pages of o= rder 0 +//! for most operations. Page allocation flags are fixed. + +use crate::{bindings, error::code::*, error::Result, PAGE_SIZE}; +use core::{marker::PhantomData, ptr}; + +/// A set of physical pages. +/// +/// `Pages` holds a reference to a set of pages of order `ORDER`. Having t= he order as a generic +/// const allows the struct to have the same size as a pointer. +/// +/// # Invariants +/// +/// The pointer `Pages::pages` is valid and points to 2^ORDER pages. +pub struct Pages { + pub(crate) pages: *mut bindings::page, +} + +impl Pages { + /// Allocates a new set of contiguous pages. + pub fn new() -> Result { + let pages =3D unsafe { + bindings::alloc_pages( + bindings::GFP_KERNEL | bindings::__GFP_ZERO | bindings::__= _GFP_HIGHMEM, + ORDER, + ) + }; + if pages.is_null() { + return Err(ENOMEM); + } + // INVARIANTS: We checked that the allocation above succeeded. + // SAFETY: We allocated pages above + Ok(unsafe { Self::from_raw(pages) }) + } + + /// Create a `Pages` from a raw `struct page` pointer + /// + /// # Safety + /// + /// Caller must own the pages pointed to by `ptr` as these will be fre= ed + /// when the returned `Pages` is dropped. + pub unsafe fn from_raw(ptr: *mut bindings::page) -> Self { + Self { pages: ptr } + } +} + +impl Pages<0> { + #[inline(always)] + fn check_offset_and_map( + &self, + offset: usize, + len: usize, + ) -> Result> + where + Pages<0>: MappingActions, + { + let end =3D offset.checked_add(len).ok_or(EINVAL)?; + if end as u32 > PAGE_SIZE { + return Err(EINVAL); + } + + let mapping =3D >::map(self); + + Ok(mapping) + } + + #[inline(always)] + unsafe fn read_internal( + &self, + dest: *mut u8, + offset: usize, + len: usize, + ) -> Result + where + Pages<0>: MappingActions, + { + let mapping =3D self.check_offset_and_map::(offset, len)?; + + unsafe { ptr::copy_nonoverlapping((mapping.ptr as *mut u8).add(off= set), dest, len) }; + Ok(()) + } + + /// Maps the pages and reads from them into the given buffer. + /// + /// # Safety + /// + /// Callers must ensure that the destination buffer is valid for the g= iven + /// length. Additionally, if the raw buffer is intended to be recast, = they + /// must ensure that the data can be safely cast; + /// [`crate::io_buffer::ReadableFromBytes`] has more details about it. + /// `dest` may not point to the source page. + #[inline(always)] + pub unsafe fn read(&self, dest: *mut u8, offset: usize, len: usize) ->= Result { + unsafe { self.read_internal::(dest, offset, len= ) } + } + + /// Maps the pages and reads from them into the given buffer. The page= is + /// mapped atomically. + /// + /// # Safety + /// + /// Callers must ensure that the destination buffer is valid for the g= iven + /// length. Additionally, if the raw buffer is intended to be recast, = they + /// must ensure that the data can be safely cast; + /// [`crate::io_buffer::ReadableFromBytes`] has more details about it. + /// `dest` may not point to the source page. + #[inline(always)] + pub unsafe fn read_atomic(&self, dest: *mut u8, offset: usize, len: us= ize) -> Result { + unsafe { self.read_internal::(dest, offset, len= ) } + } + + #[inline(always)] + unsafe fn write_internal( + &self, + src: *const u8, + offset: usize, + len: usize, + ) -> Result + where + Pages<0>: MappingActions, + { + let mapping =3D self.check_offset_and_map::(offset, len)?; + + unsafe { ptr::copy_nonoverlapping(src, (mapping.ptr as *mut u8).ad= d(offset), len) }; + Ok(()) + } + + /// Maps the pages and writes into them from the given buffer. + /// + /// # Safety + /// + /// Callers must ensure that the buffer is valid for the given length. + /// Additionally, if the page is (or will be) mapped by userspace, the= y must + /// ensure that no kernel data is leaked through padding if it was cas= t from + /// another type; [`crate::io_buffer::WritableToBytes`] has more detai= ls + /// about it. `src` must not point to the destination page. + #[inline(always)] + pub unsafe fn write(&self, src: *const u8, offset: usize, len: usize) = -> Result { + unsafe { self.write_internal::(src, offset, len= ) } + } + + /// Maps the pages and writes into them from the given buffer. The pag= e is + /// mapped atomically. + /// + /// # Safety + /// + /// Callers must ensure that the buffer is valid for the given length. + /// Additionally, if the page is (or will be) mapped by userspace, the= y must + /// ensure that no kernel data is leaked through padding if it was cas= t from + /// another type; [`crate::io_buffer::WritableToBytes`] has more detai= ls + /// about it. `src` must not point to the destination page. + #[inline(always)] + pub unsafe fn write_atomic(&self, src: *const u8, offset: usize, len: = usize) -> Result { + unsafe { self.write_internal::(src, offset, len= ) } + } + + /// Maps the page at index 0. + #[inline(always)] + pub fn kmap(&self) -> PageMapping<'_, NormalMappingInfo> { + let ptr =3D unsafe { bindings::kmap(self.pages) }; + + PageMapping { + page: self.pages, + ptr, + _phantom: PhantomData, + _phantom2: PhantomData, + } + } + + /// Atomically Maps the page at index 0. + #[inline(always)] + pub fn kmap_atomic(&self) -> PageMapping<'_, AtomicMappingInfo> { + let ptr =3D unsafe { bindings::kmap_atomic(self.pages) }; + + PageMapping { + page: self.pages, + ptr, + _phantom: PhantomData, + _phantom2: PhantomData, + } + } +} + +impl Drop for Pages { + fn drop(&mut self) { + // SAFETY: By the type invariants, we know the pages are allocated= with the given order. + unsafe { bindings::__free_pages(self.pages, ORDER) }; + } +} + +/// Specifies the type of page mapping +pub trait MappingInfo {} + +/// Encapsulates methods to map and unmap pages +pub trait MappingActions +where + Pages<0>: MappingActions, +{ + /// Map a page into the kernel address scpace + fn map(pages: &Pages<0>) -> PageMapping<'_, I>; + + /// Unmap a page specified by `mapping` + /// + /// # Safety + /// + /// Must only be called by `PageMapping::drop()`. + unsafe fn unmap(mapping: &PageMapping<'_, I>); +} + +/// A type state indicating that pages were mapped atomically +pub struct AtomicMappingInfo; +impl MappingInfo for AtomicMappingInfo {} + +/// A type state indicating that pages were not mapped atomically +pub struct NormalMappingInfo; +impl MappingInfo for NormalMappingInfo {} + +impl MappingActions for Pages<0> { + #[inline(always)] + fn map(pages: &Pages<0>) -> PageMapping<'_, AtomicMappingInfo> { + pages.kmap_atomic() + } + + #[inline(always)] + unsafe fn unmap(mapping: &PageMapping<'_, AtomicMappingInfo>) { + // SAFETY: An instance of `PageMapping` is created only when `kmap= ` succeeded for the given + // page, so it is safe to unmap it here. + unsafe { bindings::kunmap_atomic(mapping.ptr) }; + } +} + +impl MappingActions for Pages<0> { + #[inline(always)] + fn map(pages: &Pages<0>) -> PageMapping<'_, NormalMappingInfo> { + pages.kmap() + } + + #[inline(always)] + unsafe fn unmap(mapping: &PageMapping<'_, NormalMappingInfo>) { + // SAFETY: An instance of `PageMapping` is created only when `kmap= ` succeeded for the given + // page, so it is safe to unmap it here. + unsafe { bindings::kunmap(mapping.page) }; + } +} + +/// An owned page mapping. When this struct is dropped, the page is unmapp= ed. +pub struct PageMapping<'a, I: MappingInfo> +where + Pages<0>: MappingActions, +{ + page: *mut bindings::page, + ptr: *mut core::ffi::c_void, + _phantom: PhantomData<&'a i32>, + _phantom2: PhantomData, +} + +impl<'a, I: MappingInfo> PageMapping<'a, I> +where + Pages<0>: MappingActions, +{ + /// Return a pointer to the wrapped `struct page` + #[inline(always)] + pub fn get_ptr(&self) -> *mut core::ffi::c_void { + self.ptr + } +} + +// Because we do not have Drop specialization, we have to do this dance. L= ife +// would be much more simple if we could have `impl Drop for PageMapping<'= _, +// Atomic>` and `impl Drop for PageMapping<'_, NotAtomic>` +impl Drop for PageMapping<'_, I> +where + Pages<0>: MappingActions, +{ + #[inline(always)] + fn drop(&mut self) { + // SAFETY: We are OK to call this because we are `PageMapping::dro= p()` + unsafe { as MappingActions>::unmap(self) } + } +} --=20 2.40.0 From nobody Wed Feb 11 23:44:39 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 468FBC77B7F for ; Wed, 3 May 2023 09:07:48 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229524AbjECJHp (ORCPT ); Wed, 3 May 2023 05:07:45 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45390 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229907AbjECJH1 (ORCPT ); Wed, 3 May 2023 05:07:27 -0400 Received: from mail-wm1-x32f.google.com (mail-wm1-x32f.google.com [IPv6:2a00:1450:4864:20::32f]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 1A70149EC for ; Wed, 3 May 2023 02:07:21 -0700 (PDT) Received: by mail-wm1-x32f.google.com with SMTP id 5b1f17b1804b1-3f178da21b5so31755925e9.3 for ; Wed, 03 May 2023 02:07:20 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104839; x=1685696839; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=oTm6JqVuE+nMnHE8FtSNpIwMZXEBmyO5CLHkBrzqYiM=; b=y8ejhwlgPFRLA58nqaLi6rcO3cTM2Bfu5R3HefR0o7cV9HTZoOxGUPasuvY46ZCwRh Sieo0Nw1Hzb4wY/iGDsNeHNIVg922qi3pacnA1K1Se735gJ7tsTrShXrb9Q3Tud15qCH pSi0ybeIHv49VuB2OI3rl+J+JTSb+9Q6ggrpcJFtEdi5YM3IsSttMHyzuBP0ZuAFksoN vicg6hsE+vaLEKi9gFV/LC2GyxdFyAnbn301c0wlUyeORPMq2Ykn/8QtIJHJeargKiXo g6ZYJxIn2Ojk9efoIi4W30BLPTSKqbSu7FqbbaUdRjKDx6DSjir7twQZkzXbwUjeKM5j +eEg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104839; x=1685696839; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=oTm6JqVuE+nMnHE8FtSNpIwMZXEBmyO5CLHkBrzqYiM=; b=ARgkB/7wsNnBfz8kp/2VCBhWoa4VU8Ed7NhzzWXJECFz8XggMQPfTIRIuWLlyEYlkl EGKBnDUvqd18jFHbQo3Z9DvOimHjUd9cXsP+L5PBB/QREOIA9qg7gbO7sF8vJtPXCrL1 Vxh7e36XAZ6kYRvgZTb4+I701eUFfXI8/hMEDS4LatItkKblGCKucp08Z+DrlQ2SbMca t2Th6fHQ86sxBKzfgfnJAfjqCq1sSinjPSNsJHveg/7SQdm9vNNe3MuRCV5Xs21Q4IMi 7uK6waXtgcf2sSTq91jxnjBY98Z1TF+b5EPvAwCEhpeZHZmRjbDiJJnK4mHwm21JgiZl 5Wgw== X-Gm-Message-State: AC+VfDz8yJE8aheqOJXSHWKiSfLGu6LuVN33p/ZGeV09yStMglDida7C hvyKRiWisvvk6jdxgL/CBONYWg== X-Google-Smtp-Source: ACHHUZ7NYkY5wjU5kOgvBlYNkrutaPEK/czR704t4GtHEI0ibjC9b/NRGmiwBmTmJwdWADfymkFv2Q== X-Received: by 2002:a7b:c383:0:b0:3f3:3a81:334 with SMTP id s3-20020a7bc383000000b003f33a810334mr6690102wmj.29.1683104839156; Wed, 03 May 2023 02:07:19 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id s7-20020a1cf207000000b003f1733feb3dsm1305041wmc.0.2023.05.03.02.07.18 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:18 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 03/11] rust: block: introduce `kernel::block::mq` module Date: Wed, 3 May 2023 11:07:00 +0200 Message-Id: <20230503090708.2524310-4-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Andreas Hindborg Add initial abstractions for working with blk-mq. This patch is a maintained, refactored subset of code originally published = by Wedson Almeida Filho [1]. [1] https://github.com/wedsonaf/linux/tree/f2cfd2fe0e2ca4e90994f96afe268bbd= 4382a891/rust/kernel/blk/mq.rs Cc: Wedson Almeida Filho Signed-off-by: Andreas Hindborg --- rust/bindings/bindings_helper.h | 2 + rust/helpers.c | 22 +++ rust/kernel/block.rs | 5 + rust/kernel/block/mq.rs | 15 ++ rust/kernel/block/mq/gen_disk.rs | 133 +++++++++++++++ rust/kernel/block/mq/operations.rs | 260 +++++++++++++++++++++++++++++ rust/kernel/block/mq/raw_writer.rs | 30 ++++ rust/kernel/block/mq/request.rs | 71 ++++++++ rust/kernel/block/mq/tag_set.rs | 92 ++++++++++ rust/kernel/error.rs | 4 + rust/kernel/lib.rs | 1 + 11 files changed, 635 insertions(+) create mode 100644 rust/kernel/block.rs create mode 100644 rust/kernel/block/mq.rs create mode 100644 rust/kernel/block/mq/gen_disk.rs create mode 100644 rust/kernel/block/mq/operations.rs create mode 100644 rust/kernel/block/mq/raw_writer.rs create mode 100644 rust/kernel/block/mq/request.rs create mode 100644 rust/kernel/block/mq/tag_set.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helpe= r.h index 52834962b94d..86c07eeb1ba1 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -11,6 +11,8 @@ #include #include #include +#include +#include =20 /* `bindgen` gets confused at certain things. */ const gfp_t BINDINGS_GFP_KERNEL =3D GFP_KERNEL; diff --git a/rust/helpers.c b/rust/helpers.c index 9bd9d95da951..a59341084774 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -18,6 +18,7 @@ * accidentally exposed. */ =20 +#include #include #include #include @@ -28,6 +29,8 @@ #include #include #include +#include +#include =20 __noreturn void rust_helper_BUG(void) { @@ -130,6 +133,25 @@ void rust_helper_put_task_struct(struct task_struct *t) } EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); =20 +struct bio_vec rust_helper_req_bvec(struct request *rq) +{ + return req_bvec(rq); +} +EXPORT_SYMBOL_GPL(rust_helper_req_bvec); + +void *rust_helper_blk_mq_rq_to_pdu(struct request *rq) +{ + return blk_mq_rq_to_pdu(rq); +} +EXPORT_SYMBOL_GPL(rust_helper_blk_mq_rq_to_pdu); + +void rust_helper_bio_advance_iter_single(const struct bio *bio, + struct bvec_iter *iter, + unsigned int bytes) { + bio_advance_iter_single(bio, iter, bytes); +} +EXPORT_SYMBOL_GPL(rust_helper_bio_advance_iter_single); + void rust_helper_init_radix_tree(struct xarray *tree, gfp_t gfp_mask) { INIT_RADIX_TREE(tree, gfp_mask); diff --git a/rust/kernel/block.rs b/rust/kernel/block.rs new file mode 100644 index 000000000000..4c93317a568a --- /dev/null +++ b/rust/kernel/block.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for working with the block layer + +pub mod mq; diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs new file mode 100644 index 000000000000..5b40f6a73c0f --- /dev/null +++ b/rust/kernel/block/mq.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides types for implementing drivers that interface the +//! blk-mq subsystem + +mod gen_disk; +mod operations; +mod raw_writer; +mod request; +mod tag_set; + +pub use gen_disk::GenDisk; +pub use operations::Operations; +pub use request::Request; +pub use tag_set::TagSet; diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_di= sk.rs new file mode 100644 index 000000000000..50496af15bbf --- /dev/null +++ b/rust/kernel/block/mq/gen_disk.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GenDisk abstraction +//! +//! C header: [`include/linux/blkdev.h`](../../include/linux/blkdev.h) +//! C header: [`include/linux/blk_mq.h`](../../include/linux/blk_mq.h) + +use crate::block::mq::{raw_writer::RawWriter, Operations, TagSet}; +use crate::{ + bindings, error::from_err_ptr, error::Result, sync::Arc, types::Foreig= nOwnable, + types::ScopeGuard, +}; +use core::fmt::{self, Write}; + +/// A generic block device +/// +/// # Invariants +/// +/// - `gendisk` must always point to an initialized and valid `struct gen= disk`. +pub struct GenDisk { + _tagset: Arc>, + gendisk: *mut bindings::gendisk, +} + +// SAFETY: `GenDisk` is an owned pointer to a `struct gendisk` and an `Arc= ` to a +// `TagSet` It is safe to send this to other threads as long as T is Send. +unsafe impl Send for GenDisk {} + +impl GenDisk { + /// Try to create a new `GenDisk` + pub fn try_new(tagset: Arc>, queue_data: T::QueueData) -> Re= sult { + let data =3D queue_data.into_foreign(); + let recover_data =3D ScopeGuard::new(|| { + // SAFETY: T::QueueData was created by the call to `into_forei= gn()` above + unsafe { T::QueueData::from_foreign(data) }; + }); + + let lock_class_key =3D crate::sync::LockClassKey::new(); + + // SAFETY: `tagset.raw_tag_set()` points to a valid and initialize= d tag set + let gendisk =3D from_err_ptr(unsafe { + bindings::__blk_mq_alloc_disk(tagset.raw_tag_set(), data as _,= lock_class_key.as_ptr()) + })?; + + const TABLE: bindings::block_device_operations =3D bindings::block= _device_operations { + submit_bio: None, + open: None, + release: None, + ioctl: None, + compat_ioctl: None, + check_events: None, + unlock_native_capacity: None, + getgeo: None, + set_read_only: None, + swap_slot_free_notify: None, + report_zones: None, + devnode: None, + alternative_gpt_sector: None, + get_unique_id: None, + owner: core::ptr::null_mut(), + pr_ops: core::ptr::null_mut(), + free_disk: None, + poll_bio: None, + }; + + // SAFETY: gendisk is a valid pointer as we initialized it above + unsafe { (*gendisk).fops =3D &TABLE }; + + recover_data.dismiss(); + Ok(Self { + _tagset: tagset, + gendisk, + }) + } + + /// Set the name of the device + pub fn set_name(&self, args: fmt::Arguments<'_>) -> Result { + let mut raw_writer =3D RawWriter::from_array(unsafe { &mut (*self.= gendisk).disk_name }); + raw_writer.write_fmt(args)?; + raw_writer.write_char('\0')?; + Ok(()) + } + + /// Register the device with the kernel. When this function return, the + /// device is accessible from VFS. The kernel may issue reads to the d= evice + /// during registration to discover partition infomation. + pub fn add(&self) -> Result { + crate::error::to_result(unsafe { + bindings::device_add_disk(core::ptr::null_mut(), self.gendisk,= core::ptr::null_mut()) + }) + } + + /// Call to tell the block layer the capcacity of the device + pub fn set_capacity(&self, sectors: u64) { + unsafe { bindings::set_capacity(self.gendisk, sectors) }; + } + + /// Set the logical block size of the device + pub fn set_queue_logical_block_size(&self, size: u32) { + unsafe { bindings::blk_queue_logical_block_size((*self.gendisk).qu= eue, size) }; + } + + /// Set the physical block size of the device + pub fn set_queue_physical_block_size(&self, size: u32) { + unsafe { bindings::blk_queue_physical_block_size((*self.gendisk).q= ueue, size) }; + } + + /// Set the rotational media attribute for the device + pub fn set_rotational(&self, rotational: bool) { + if !rotational { + unsafe { + bindings::blk_queue_flag_set(bindings::QUEUE_FLAG_NONROT, = (*self.gendisk).queue) + }; + } else { + unsafe { + bindings::blk_queue_flag_clear(bindings::QUEUE_FLAG_NONROT= , (*self.gendisk).queue) + }; + } + } +} + +impl Drop for GenDisk { + fn drop(&mut self) { + let queue_data =3D unsafe { (*(*self.gendisk).queue).queuedata }; + + unsafe { bindings::del_gendisk(self.gendisk) }; + + // SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` w= ith a + // call to `ForeignOwnable::into_pointer()` to create `queuedata`. + // `ForeignOwnable::from_foreign()` is only called here. + let _queue_data =3D unsafe { T::QueueData::from_foreign(queue_data= ) }; + } +} diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/oper= ations.rs new file mode 100644 index 000000000000..fb1ab707d1f0 --- /dev/null +++ b/rust/kernel/block/mq/operations.rs @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides an interface for blk-mq drivers to implement. +//! +//! C header: [`include/linux/blk-mq.h`](../../include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::{tag_set::TagSetRef, Request}, + error::{from_result, Result}, + types::ForeignOwnable, +}; +use core::{marker::PhantomData, pin::Pin}; + +/// Implement this trait to interface blk-mq as block devices +#[macros::vtable] +pub trait Operations: Sized { + /// Data associated with a request. This data is located next to the r= equest + /// structure. + type RequestData; + + /// Data associated with the `struct request_queue` that is allocated = for + /// the `GenDisk` associated with this `Operations` implementation. + type QueueData: ForeignOwnable; + + /// Data associated with a dispatch queue. This is stored as a pointer= in + /// `struct blk_mq_hw_ctx`. + type HwData: ForeignOwnable; + + /// Data associated with a tag set. This is stored as a pointer in `st= ruct + /// blk_mq_tag_set`. + type TagSetData: ForeignOwnable; + + /// Called by the kernel to allocate a new `RequestData`. The structur= e will + /// eventually be pinned, so defer initialization to `init_request_dat= a()` + fn new_request_data( + _tagset_data: ::Borrowed<'_>, + ) -> Result; + + /// Called by the kernel to initialize a previously allocated `Request= Data` + fn init_request_data( + _tagset_data: ::Borrowed<'_>, + _data: Pin<&mut Self::RequestData>, + ) -> Result { + Ok(()) + } + + /// Called by the kernel to queue a request with the driver. If `is_la= st` is + /// `false`, the driver is allowed to defer commiting the request. + fn queue_rq( + hw_data: ::Borrowed<'_>, + queue_data: ::Borrowed<'_>, + rq: &Request, + is_last: bool, + ) -> Result; + + /// Called by the kernel to indicate that queued requests should be su= bmitted + fn commit_rqs( + hw_data: ::Borrowed<'_>, + queue_data: ::Borrowed<'_>, + ); + + /// Called by the kernel when the request is completed + fn complete(_rq: &Request); + + /// Called by the kernel to allocate and initialize a driver specific = hardware context data + fn init_hctx( + tagset_data: ::Borrowed<'_>, + hctx_idx: u32, + ) -> Result; + + /// Called by the kernel to poll the device for completed requests. On= ly used for poll queues. + fn poll(_hw_data: ::Borrowed<'_>) -> i= 32 { + unreachable!() + } + + /// Called by the kernel to map submission queues to CPU cores. + fn map_queues(_tag_set: &TagSetRef) { + unreachable!() + } + + // There is no need for exit_request() because `drop` will be called. +} + +pub(crate) struct OperationsVtable(PhantomData); + +impl OperationsVtable { + // # Safety + // + // The caller of this function must ensure that `hctx` and `bd` are va= lid + // and initialized. The pointees must outlive this function. Further + // `hctx->driver_data` must be a pointer created by a call to + // `Self::init_hctx_callback()` and the pointee must outlive this func= tion. + // This function must not be called with a `hctx` for which + // `Self::exit_hctx_callback()` has been called. + unsafe extern "C" fn queue_rq_callback( + hctx: *mut bindings::blk_mq_hw_ctx, + bd: *const bindings::blk_mq_queue_data, + ) -> bindings::blk_status_t { + // SAFETY: `bd` is valid as required by the safety requirement for= this function. + let rq =3D unsafe { (*bd).rq }; + + // SAFETY: The safety requirement for this function ensure that + // `(*hctx).driver_data` was returned by a call to + // `Self::init_hctx_callback()`. That function uses + // `PointerWrapper::into_pointer()` to create `driver_data`. Furth= er, + // the returned value does not outlive this function and + // `from_foreign()` is not called until `Self::exit_hctx_callback(= )` is + // called. By the safety requirement of this function and contract= with + // the `blk-mq` API, `queue_rq_callback()` will not be called afte= r that + // point. + let hw_data =3D unsafe { T::HwData::borrow((*hctx).driver_data) }; + + // SAFETY: `hctx` is valid as required by this function. + let queue_data =3D unsafe { (*(*hctx).queue).queuedata }; + + // SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` w= ith a + // call to `ForeignOwnable::into_pointer()` to create `queuedata`. + // `ForeignOwnable::from_foreign()` is only called when the tagset= is + // dropped, which happens after we are dropped. + let queue_data =3D unsafe { T::QueueData::borrow(queue_data) }; + + // SAFETY: `bd` is valid as required by the safety requirement for= this function. + let ret =3D T::queue_rq( + hw_data, + queue_data, + &unsafe { Request::from_ptr(rq) }, + unsafe { (*bd).last }, + ); + if let Err(e) =3D ret { + e.to_blk_status() + } else { + bindings::BLK_STS_OK as _ + } + } + + unsafe extern "C" fn commit_rqs_callback(hctx: *mut bindings::blk_mq_h= w_ctx) { + let hw_data =3D unsafe { T::HwData::borrow((*hctx).driver_data) }; + + // SAFETY: `hctx` is valid as required by this function. + let queue_data =3D unsafe { (*(*hctx).queue).queuedata }; + + // SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` w= ith a + // call to `ForeignOwnable::into_pointer()` to create `queuedata`. + // `ForeignOwnable::from_foreign()` is only called when the tagset= is + // dropped, which happens after we are dropped. + let queue_data =3D unsafe { T::QueueData::borrow(queue_data) }; + T::commit_rqs(hw_data, queue_data) + } + + unsafe extern "C" fn complete_callback(rq: *mut bindings::request) { + T::complete(&unsafe { Request::from_ptr(rq) }); + } + + unsafe extern "C" fn poll_callback( + hctx: *mut bindings::blk_mq_hw_ctx, + _iob: *mut bindings::io_comp_batch, + ) -> core::ffi::c_int { + let hw_data =3D unsafe { T::HwData::borrow((*hctx).driver_data) }; + T::poll(hw_data) + } + + unsafe extern "C" fn init_hctx_callback( + hctx: *mut bindings::blk_mq_hw_ctx, + tagset_data: *mut core::ffi::c_void, + hctx_idx: core::ffi::c_uint, + ) -> core::ffi::c_int { + from_result(|| { + let tagset_data =3D unsafe { T::TagSetData::borrow(tagset_data= ) }; + let data =3D T::init_hctx(tagset_data, hctx_idx)?; + unsafe { (*hctx).driver_data =3D data.into_foreign() as _ }; + Ok(0) + }) + } + + unsafe extern "C" fn exit_hctx_callback( + hctx: *mut bindings::blk_mq_hw_ctx, + _hctx_idx: core::ffi::c_uint, + ) { + let ptr =3D unsafe { (*hctx).driver_data }; + unsafe { T::HwData::from_foreign(ptr) }; + } + + unsafe extern "C" fn init_request_callback( + set: *mut bindings::blk_mq_tag_set, + rq: *mut bindings::request, + _hctx_idx: core::ffi::c_uint, + _numa_node: core::ffi::c_uint, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: The tagset invariants guarantee that all requests a= re allocated with extra memory + // for the request data. + let pdu =3D unsafe { bindings::blk_mq_rq_to_pdu(rq) } as *mut = T::RequestData; + let tagset_data =3D unsafe { T::TagSetData::borrow((*set).driv= er_data) }; + + let v =3D T::new_request_data(tagset_data)?; + + // SAFETY: `pdu` memory is valid, as it was allocated by the c= aller. + unsafe { pdu.write(v) }; + + let tagset_data =3D unsafe { T::TagSetData::borrow((*set).driv= er_data) }; + // SAFETY: `pdu` memory is valid and properly initialised. + T::init_request_data(tagset_data, unsafe { Pin::new_unchecked(= &mut *pdu) })?; + + Ok(0) + }) + } + + unsafe extern "C" fn exit_request_callback( + _set: *mut bindings::blk_mq_tag_set, + rq: *mut bindings::request, + _hctx_idx: core::ffi::c_uint, + ) { + // SAFETY: The tagset invariants guarantee that all requests are a= llocated with extra memory + // for the request data. + let pdu =3D unsafe { bindings::blk_mq_rq_to_pdu(rq) } as *mut T::R= equestData; + + // SAFETY: `pdu` is valid for read and write and is properly initi= alised. + unsafe { core::ptr::drop_in_place(pdu) }; + } + + unsafe extern "C" fn map_queues_callback(tag_set_ptr: *mut bindings::b= lk_mq_tag_set) { + let tag_set =3D unsafe { TagSetRef::from_ptr(tag_set_ptr) }; + T::map_queues(&tag_set); + } + + const VTABLE: bindings::blk_mq_ops =3D bindings::blk_mq_ops { + queue_rq: Some(Self::queue_rq_callback), + queue_rqs: None, + commit_rqs: Some(Self::commit_rqs_callback), + get_budget: None, + put_budget: None, + set_rq_budget_token: None, + get_rq_budget_token: None, + timeout: None, + poll: if T::HAS_POLL { + Some(Self::poll_callback) + } else { + None + }, + complete: Some(Self::complete_callback), + init_hctx: Some(Self::init_hctx_callback), + exit_hctx: Some(Self::exit_hctx_callback), + init_request: Some(Self::init_request_callback), + exit_request: Some(Self::exit_request_callback), + cleanup_rq: None, + busy: None, + map_queues: if T::HAS_MAP_QUEUES { + Some(Self::map_queues_callback) + } else { + None + }, + #[cfg(CONFIG_BLK_DEBUG_FS)] + show_rq: None, + }; + + pub(crate) const unsafe fn build() -> &'static bindings::blk_mq_ops { + &Self::VTABLE + } +} diff --git a/rust/kernel/block/mq/raw_writer.rs b/rust/kernel/block/mq/raw_= writer.rs new file mode 100644 index 000000000000..25c16ee0b1f7 --- /dev/null +++ b/rust/kernel/block/mq/raw_writer.rs @@ -0,0 +1,30 @@ +use core::fmt::{self, Write}; + +pub(crate) struct RawWriter { + ptr: *mut u8, + len: usize, +} + +impl RawWriter { + unsafe fn new(ptr: *mut u8, len: usize) -> Self { + Self { ptr, len } + } + + pub(crate) fn from_array(a: &mut [core::ffi::c_char; N= ]) -> Self { + unsafe { Self::new(&mut a[0] as *mut _ as _, N) } + } +} + +impl Write for RawWriter { + fn write_str(&mut self, s: &str) -> fmt::Result { + let bytes =3D s.as_bytes(); + let len =3D bytes.len(); + if len > self.len { + return Err(fmt::Error); + } + unsafe { core::ptr::copy_nonoverlapping(&bytes[0], self.ptr, len) = }; + self.ptr =3D unsafe { self.ptr.add(len) }; + self.len -=3D len; + Ok(()) + } +} diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request= .rs new file mode 100644 index 000000000000..e95ae3fd71ad --- /dev/null +++ b/rust/kernel/block/mq/request.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides a wrapper for the C `struct request` type. +//! +//! C header: [`include/linux/blk-mq.h`](../../include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::Operations, + error::{Error, Result}, +}; +use core::marker::PhantomData; + +/// A wrapper around a blk-mq `struct request`. This represents an IO requ= est. +pub struct Request { + ptr: *mut bindings::request, + _p: PhantomData, +} + +impl Request { + pub(crate) unsafe fn from_ptr(ptr: *mut bindings::request) -> Self { + Self { + ptr, + _p: PhantomData, + } + } + + /// Get the command identifier for the request + pub fn command(&self) -> u32 { + unsafe { (*self.ptr).cmd_flags & ((1 << bindings::REQ_OP_BITS) - 1= ) } + } + + /// Call this to indicate to the kernel that the request has been issu= ed by the driver + pub fn start(&self) { + unsafe { bindings::blk_mq_start_request(self.ptr) }; + } + + /// Call this to indicate to the kernel that the request has been comp= leted without errors + // TODO: Consume rq so that we can't use it after ending it? + pub fn end_ok(&self) { + unsafe { bindings::blk_mq_end_request(self.ptr, bindings::BLK_STS_= OK as _) }; + } + + /// Call this to indicate to the kernel that the request completed wit= h an error + pub fn end_err(&self, err: Error) { + unsafe { bindings::blk_mq_end_request(self.ptr, err.to_blk_status(= )) }; + } + + /// Call this to indicate that the request completed with the status i= ndicated by `status` + pub fn end(&self, status: Result) { + if let Err(e) =3D status { + self.end_err(e); + } else { + self.end_ok(); + } + } + + /// Call this to schedule defered completion of the request + // TODO: Consume rq so that we can't use it after completing it? + pub fn complete(&self) { + if !unsafe { bindings::blk_mq_complete_request_remote(self.ptr) } { + T::complete(&unsafe { Self::from_ptr(self.ptr) }); + } + } + + /// Get the target sector for the request + #[inline(always)] + pub fn sector(&self) -> usize { + unsafe { (*self.ptr).__sector as usize } + } +} diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set= .rs new file mode 100644 index 000000000000..d122db7f6d0e --- /dev/null +++ b/rust/kernel/block/mq/tag_set.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides the `TagSet` struct to wrap the C `struct blk_mq_= tag_set`. +//! +//! C header: [`include/linux/blk-mq.h`](../../include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::{operations::OperationsVtable, Operations}, + error::{Error, Result}, + sync::Arc, + types::ForeignOwnable, +}; +use core::{cell::UnsafeCell, convert::TryInto, marker::PhantomData}; + +/// A wrapper for the C `struct blk_mq_tag_set` +pub struct TagSet { + inner: UnsafeCell, + _p: PhantomData, +} + +impl TagSet { + /// Try to create a new tag set + pub fn try_new( + nr_hw_queues: u32, + tagset_data: T::TagSetData, + num_tags: u32, + num_maps: u32, + ) -> Result> { + let tagset =3D Arc::try_new(Self { + inner: UnsafeCell::new(bindings::blk_mq_tag_set::default()), + _p: PhantomData, + })?; + + // SAFETY: We just allocated `tagset`, we know this is the only re= ference to it. + let inner =3D unsafe { &mut *tagset.inner.get() }; + + inner.ops =3D unsafe { OperationsVtable::::build() }; + inner.nr_hw_queues =3D nr_hw_queues; + inner.timeout =3D 0; // 0 means default which is 30 * HZ in C + inner.numa_node =3D bindings::NUMA_NO_NODE; + inner.queue_depth =3D num_tags; + inner.cmd_size =3D core::mem::size_of::().try_into= ()?; + inner.flags =3D bindings::BLK_MQ_F_SHOULD_MERGE; + inner.driver_data =3D tagset_data.into_foreign() as _; + inner.nr_maps =3D num_maps; + + // SAFETY: `inner` points to valid and initialised memory. + let ret =3D unsafe { bindings::blk_mq_alloc_tag_set(inner) }; + if ret < 0 { + // SAFETY: We created `driver_data` above with `into_foreign` + unsafe { T::TagSetData::from_foreign(inner.driver_data) }; + return Err(Error::from_errno(ret)); + } + + Ok(tagset) + } + + /// Return the pointer to the wrapped `struct blk_mq_tag_set` + pub(crate) fn raw_tag_set(&self) -> *mut bindings::blk_mq_tag_set { + self.inner.get() + } +} + +impl Drop for TagSet { + fn drop(&mut self) { + let tagset_data =3D unsafe { (*self.inner.get()).driver_data }; + + // SAFETY: `inner` is valid and has been properly initialised duri= ng construction. + unsafe { bindings::blk_mq_free_tag_set(self.inner.get()) }; + + // SAFETY: `tagset_data` was created by a call to + // `ForeignOwnable::into_foreign` in `TagSet::try_new()` + unsafe { T::TagSetData::from_foreign(tagset_data) }; + } +} + +/// A tag set reference. Used to control lifetime and prevent drop of TagS= et references passed to +/// `Operations::map_queues()` +pub struct TagSetRef { + ptr: *mut bindings::blk_mq_tag_set, +} + +impl TagSetRef { + pub(crate) unsafe fn from_ptr(tagset: *mut bindings::blk_mq_tag_set) -= > Self { + Self { ptr: tagset } + } + + pub fn ptr(&self) -> *mut bindings::blk_mq_tag_set { + self.ptr + } +} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 5f4114b30b94..421fef677321 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -107,6 +107,10 @@ impl Error { self.0 } =20 + pub(crate) fn to_blk_status(self) -> bindings::blk_status_t { + unsafe { bindings::errno_to_blk_status(self.0) } + } + /// Returns the error encoded as a pointer. #[allow(dead_code)] pub(crate) fn to_ptr(self) -> *mut T { diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 8bef6686504b..cd798d12d97c 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -34,6 +34,7 @@ extern crate self as kernel; #[cfg(not(test))] #[cfg(not(testlib))] mod allocator; +pub mod block; mod build_assert; pub mod error; pub mod init; --=20 2.40.0 From nobody Wed Feb 11 23:44:39 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id A3AFCC7EE22 for ; Wed, 3 May 2023 09:07:42 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229974AbjECJHk (ORCPT ); Wed, 3 May 2023 05:07:40 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45366 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229887AbjECJH0 (ORCPT ); Wed, 3 May 2023 05:07:26 -0400 Received: from mail-wr1-x435.google.com (mail-wr1-x435.google.com [IPv6:2a00:1450:4864:20::435]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id BE53B4C08 for ; Wed, 3 May 2023 02:07:21 -0700 (PDT) Received: by mail-wr1-x435.google.com with SMTP id ffacd0b85a97d-3063433fa66so1615333f8f.3 for ; Wed, 03 May 2023 02:07:21 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104840; x=1685696840; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=8Hg9EF+CTDGGAsWt0N5WyI6rk7QsJ8yExmPcN8MX0qQ=; b=d+lWf2ETGwZ6l0kF5yoMkucYwDf2kDuSHvqzafoL6LSaWDyChZFeb/y3Ask+uZL3ub Crm1FWg7HSXUUHQu4aZsOctpyLqbz0tvhgPgilALIxNAKTgLG9YZ5O7UUoSFUznFtAd7 8OtJqUv7m3XxiwrPol077Wc7De84xWLwlJ9zH1FgVk7LrKu7kj7KMolcbNdrGquUbXHA gx1o+RI6ewKReUGP7O57duA5S2sDgZdoARg8MtGBlJBT45+yVJtwMrDiwgD1CJt2xVPz ns0renNS1R/Xu68zuPNBQGUcpB38HrRkV/8/C94FuyxzQvT6lrnKR7AKdmaNVmf1WOts FeSw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104840; x=1685696840; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=8Hg9EF+CTDGGAsWt0N5WyI6rk7QsJ8yExmPcN8MX0qQ=; b=Vpl/kCTWml/uHg5/3waZ0shiVT9tqUMg97FXQQSSIPKBEBAzzu/8I+FScCzXAO5AJp bgLDjwtWPXst9nyU7GsDev67/DNUxfJYSH2vkkSI4sks3pMOh+rcxs+tBL3nt1UA4EXU V1+eOmeitXvMfTeLiVdtn7+4ml7iO8TJjD39nzqrvBvJgVwHyu84566SjeOd48/GaDRc c+wKOMXwMbULanwFH4TTYFtpB2302Dn1AsPINfrP4sU4dxKDOKr+jidzIBGspKPhKCrP RBmmyGoJonU8b9ZvyPhuBL71XL4jfUwTr1sNEABYzP3W5D7hqGiQpx310Glc1WFNOd+P BivA== X-Gm-Message-State: AC+VfDzqb0m+w/YPRS/qZ8uXbwBdStDwm+l7ns9GWDxwRh5OiC1LkVFG jigK8G6/HGj15huqtcUeKwn7ig== X-Google-Smtp-Source: ACHHUZ6T2y7vjejoAr+gXXz7fkMQ4wlDgEasl/1jBzVe1iLoMPS+zptIdaYP9DejgMdUNDfXGrvBNg== X-Received: by 2002:adf:e484:0:b0:306:320b:5dbd with SMTP id i4-20020adfe484000000b00306320b5dbdmr5824254wrm.71.1683104840210; Wed, 03 May 2023 02:07:20 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id v12-20020adfe28c000000b0030497b3224bsm20046305wri.64.2023.05.03.02.07.19 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:19 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 04/11] rust: block: introduce `kernel::block::bio` module Date: Wed, 3 May 2023 11:07:01 +0200 Message-Id: <20230503090708.2524310-5-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Andreas Hindborg Add abstractions for working with `struct bio`. Signed-off-by: Andreas Hindborg --- rust/kernel/block.rs | 1 + rust/kernel/block/bio.rs | 93 ++++++++++++++++ rust/kernel/block/bio/vec.rs | 181 ++++++++++++++++++++++++++++++++ rust/kernel/block/mq/request.rs | 16 +++ 4 files changed, 291 insertions(+) create mode 100644 rust/kernel/block/bio.rs create mode 100644 rust/kernel/block/bio/vec.rs diff --git a/rust/kernel/block.rs b/rust/kernel/block.rs index 4c93317a568a..1797859551fd 100644 --- a/rust/kernel/block.rs +++ b/rust/kernel/block.rs @@ -2,4 +2,5 @@ =20 //! Types for working with the block layer =20 +pub mod bio; pub mod mq; diff --git a/rust/kernel/block/bio.rs b/rust/kernel/block/bio.rs new file mode 100644 index 000000000000..6e93e4420105 --- /dev/null +++ b/rust/kernel/block/bio.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for working with the bio layer. +//! +//! C header: [`include/linux/blk_types.h`](../../include/linux/blk_types.= h) + +use core::fmt; +use core::ptr::NonNull; + +mod vec; + +pub use vec::BioSegmentIterator; +pub use vec::Segment; + +/// A wrapper around a `struct bio` pointer +/// +/// # Invariants +/// +/// First field must alwyas be a valid pointer to a valid `struct bio`. +pub struct Bio<'a>( + NonNull, + core::marker::PhantomData<&'a ()>, +); + +impl<'a> Bio<'a> { + /// Returns an iterator over segments in this `Bio`. Does not consider + /// segments of other bios in this bio chain. + #[inline(always)] + pub fn segment_iter(&'a self) -> BioSegmentIterator<'a> { + BioSegmentIterator::new(self) + } + + /// Get a pointer to the `bio_vec` off this bio + #[inline(always)] + fn io_vec(&self) -> *const bindings::bio_vec { + // SAFETY: By type invariant, get_raw() returns a valid pointer to= a + // valid `struct bio` + unsafe { (*self.get_raw()).bi_io_vec } + } + + /// Return a copy of the `bvec_iter` for this `Bio` + #[inline(always)] + fn iter(&self) -> bindings::bvec_iter { + // SAFETY: self.0 is always a valid pointer + unsafe { (*self.get_raw()).bi_iter } + } + + /// Get the next `Bio` in the chain + #[inline(always)] + fn next(&self) -> Option> { + // SAFETY: self.0 is always a valid pointer + let next =3D unsafe { (*self.get_raw()).bi_next }; + Some(Self(NonNull::new(next)?, core::marker::PhantomData)) + } + + /// Return the raw pointer of the wrapped `struct bio` + #[inline(always)] + fn get_raw(&self) -> *const bindings::bio { + self.0.as_ptr() + } + + /// Create an instance of `Bio` from a raw pointer. Does check that the + /// pointer is not null. + #[inline(always)] + pub(crate) unsafe fn from_raw(ptr: *mut bindings::bio) -> Option> { + Some(Self(NonNull::new(ptr)?, core::marker::PhantomData)) + } +} + +impl core::fmt::Display for Bio<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Bio {:?}", self.0.as_ptr()) + } +} + +/// An iterator over `Bio` +pub struct BioIterator<'a> { + pub(crate) bio: Option>, +} + +impl<'a> core::iter::Iterator for BioIterator<'a> { + type Item =3D Bio<'a>; + + #[inline(always)] + fn next(&mut self) -> Option> { + if let Some(current) =3D self.bio.take() { + self.bio =3D current.next(); + Some(current) + } else { + None + } + } +} diff --git a/rust/kernel/block/bio/vec.rs b/rust/kernel/block/bio/vec.rs new file mode 100644 index 000000000000..acd328a6fe54 --- /dev/null +++ b/rust/kernel/block/bio/vec.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for working with `struct bio_vec` IO vectors +//! +//! C header: [`include/linux/bvec.h`](../../include/linux/bvec.h) + +use super::Bio; +use crate::error::Result; +use crate::pages::Pages; +use core::fmt; +use core::mem::ManuallyDrop; + +#[inline(always)] +fn mp_bvec_iter_offset(bvec: *const bindings::bio_vec, iter: &bindings::bv= ec_iter) -> u32 { + (unsafe { (*bvec_iter_bvec(bvec, iter)).bv_offset }) + iter.bi_bvec_do= ne +} + +#[inline(always)] +fn mp_bvec_iter_page( + bvec: *const bindings::bio_vec, + iter: &bindings::bvec_iter, +) -> *mut bindings::page { + unsafe { (*bvec_iter_bvec(bvec, iter)).bv_page } +} + +#[inline(always)] +fn mp_bvec_iter_page_idx(bvec: *const bindings::bio_vec, iter: &bindings::= bvec_iter) -> usize { + (mp_bvec_iter_offset(bvec, iter) / crate::PAGE_SIZE) as usize +} + +#[inline(always)] +fn mp_bvec_iter_len(bvec: *const bindings::bio_vec, iter: &bindings::bvec_= iter) -> u32 { + iter.bi_size + .min(unsafe { (*bvec_iter_bvec(bvec, iter)).bv_len } - iter.bi_bve= c_done) +} + +#[inline(always)] +fn bvec_iter_bvec( + bvec: *const bindings::bio_vec, + iter: &bindings::bvec_iter, +) -> *const bindings::bio_vec { + unsafe { bvec.add(iter.bi_idx as usize) } +} + +#[inline(always)] +fn bvec_iter_page( + bvec: *const bindings::bio_vec, + iter: &bindings::bvec_iter, +) -> *mut bindings::page { + unsafe { mp_bvec_iter_page(bvec, iter).add(mp_bvec_iter_page_idx(bvec,= iter)) } +} + +#[inline(always)] +fn bvec_iter_len(bvec: *const bindings::bio_vec, iter: &bindings::bvec_ite= r) -> u32 { + mp_bvec_iter_len(bvec, iter).min(crate::PAGE_SIZE - bvec_iter_offset(b= vec, iter)) +} + +#[inline(always)] +fn bvec_iter_offset(bvec: *const bindings::bio_vec, iter: &bindings::bvec_= iter) -> u32 { + mp_bvec_iter_offset(bvec, iter) % crate::PAGE_SIZE +} + +/// A wrapper around a `strutct bio_vec` - a contiguous range of physical = memory addresses +/// +/// # Invariants +/// +/// `bio_vec` must always be initialized and valid +pub struct Segment<'a> { + bio_vec: bindings::bio_vec, + _marker: core::marker::PhantomData<&'a ()>, +} + +impl Segment<'_> { + /// Get he lenght of the segment in bytes + #[inline(always)] + pub fn len(&self) -> usize { + self.bio_vec.bv_len as usize + } + + /// Returns true if the length of the segment is 0 + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len() =3D=3D 0 + } + + /// Get the offset field of the `bio_vec` + #[inline(always)] + pub fn offset(&self) -> usize { + self.bio_vec.bv_offset as usize + } + + /// Copy data of this segment into `page`. + #[inline(always)] + pub fn copy_to_page_atomic(&self, page: &mut Pages<0>) -> Result { + // SAFETY: self.bio_vec is valid and thus bv_page must be a valid + // pointer to a `struct page`. We do not own the page, but we prev= ent + // drop by wrapping the `Pages` in `ManuallyDrop`. + let our_page =3D ManuallyDrop::new(unsafe { Pages::<0>::from_raw(s= elf.bio_vec.bv_page) }); + let our_map =3D our_page.kmap_atomic(); + + // TODO: Checck offset is within page - what guarantees does `bio_= vec` provide? + let ptr =3D unsafe { (our_map.get_ptr() as *const u8).add(self.off= set()) }; + + unsafe { page.write_atomic(ptr, self.offset(), self.len()) } + } + + /// Copy data from `page` into this segment + #[inline(always)] + pub fn copy_from_page_atomic(&mut self, page: &Pages<0>) -> Result { + // SAFETY: self.bio_vec is valid and thus bv_page must be a valid + // pointer to a `struct page`. We do not own the page, but we prev= ent + // drop by wrapping the `Pages` in `ManuallyDrop`. + let our_page =3D ManuallyDrop::new(unsafe { Pages::<0>::from_raw(s= elf.bio_vec.bv_page) }); + let our_map =3D our_page.kmap_atomic(); + + // TODO: Checck offset is within page + let ptr =3D unsafe { (our_map.get_ptr() as *mut u8).add(self.offse= t()) }; + + unsafe { page.read_atomic(ptr, self.offset(), self.len()) } + } +} + +impl core::fmt::Display for Segment<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "Segment {:?} len: {}", + self.bio_vec.bv_page, self.bio_vec.bv_len + ) + } +} + +/// An iterator over `Segment` +pub struct BioSegmentIterator<'a> { + bio: &'a Bio<'a>, + iter: bindings::bvec_iter, +} + +impl<'a> BioSegmentIterator<'a> { + #[inline(always)] + pub(crate) fn new(bio: &'a Bio<'a>) -> BioSegmentIterator<'_> { + Self { + bio, + iter: bio.iter(), + } + } +} + +impl<'a> core::iter::Iterator for BioSegmentIterator<'a> { + type Item =3D Segment<'a>; + + #[inline(always)] + fn next(&mut self) -> Option { + if self.iter.bi_size =3D=3D 0 { + return None; + } + + // Macro + // bio_vec =3D bio_iter_iovec(bio, self.iter) + // bio_vec =3D bvec_iter_bvec(bio.bi_io_vec, self.iter); + let bio_vec_ret =3D bindings::bio_vec { + bv_page: bvec_iter_page(self.bio.io_vec(), &self.iter), + bv_len: bvec_iter_len(self.bio.io_vec(), &self.iter), + bv_offset: bvec_iter_offset(self.bio.io_vec(), &self.iter), + }; + + // Static C function + unsafe { + bindings::bio_advance_iter_single( + self.bio.get_raw(), + &mut self.iter as *mut bindings::bvec_iter, + bio_vec_ret.bv_len, + ) + }; + + Some(Segment { + bio_vec: bio_vec_ret, + _marker: core::marker::PhantomData, + }) + } +} diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request= .rs index e95ae3fd71ad..ccb1033b64b6 100644 --- a/rust/kernel/block/mq/request.rs +++ b/rust/kernel/block/mq/request.rs @@ -11,6 +11,9 @@ use crate::{ }; use core::marker::PhantomData; =20 +use crate::block::bio::Bio; +use crate::block::bio::BioIterator; + /// A wrapper around a blk-mq `struct request`. This represents an IO requ= est. pub struct Request { ptr: *mut bindings::request, @@ -63,6 +66,19 @@ impl Request { } } =20 + /// Get a wrapper for the first Bio in this request + #[inline(always)] + pub fn bio(&self) -> Option> { + let ptr =3D unsafe { (*self.ptr).bio }; + unsafe { Bio::from_raw(ptr) } + } + + /// Get an iterator over all bio structurs in this request + #[inline(always)] + pub fn bio_iter(&self) -> BioIterator<'_> { + BioIterator { bio: self.bio() } + } + /// Get the target sector for the request #[inline(always)] pub fn sector(&self) -> usize { --=20 2.40.0 From nobody Wed Feb 11 23:44:39 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id C7925C77B7F for ; Wed, 3 May 2023 09:07:54 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229995AbjECJHx (ORCPT ); Wed, 3 May 2023 05:07:53 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45818 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229911AbjECJHh (ORCPT ); Wed, 3 May 2023 05:07:37 -0400 Received: from mail-wr1-x42f.google.com (mail-wr1-x42f.google.com [IPv6:2a00:1450:4864:20::42f]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 6D3201FE3 for ; Wed, 3 May 2023 02:07:23 -0700 (PDT) Received: by mail-wr1-x42f.google.com with SMTP id ffacd0b85a97d-2fc3f1d6f8cso2976472f8f.3 for ; Wed, 03 May 2023 02:07:23 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104841; x=1685696841; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=gSA28PAasqbQ6gihh6v56FFurfwvVr3++GQNYiXLBOM=; b=vQsE9+nWwjoc0CqWeJtb/ZfNGHy0rUvS1Y6KMIM/8XIALq5wndo94M0A9dNDG/sgx3 TntjBxyhGd3sPHq81BA5d+rCrZqPK+YW4IthsZPI1CtTP6bLLkexUwvgl31L4ic67Qf3 3a/12tbFt4/FvWOnftXK8qByjLSFlbpkmQMkf5C8I3hc6vlYpAnFzNFcDxdS2MRw4IEx wPbbwhJsNepi1C2Pis2b+onQe5ZIJJr4481QrdlPAHTxbbNDusKuXuoh7FCB5nas1ipJ GJZUZOXkGrlNsq2sAq9Ubuo5revBkkMADIqKLVPTnZWg7liUy4NgrQWnKD4vuzI/VlLI s34Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104841; x=1685696841; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=gSA28PAasqbQ6gihh6v56FFurfwvVr3++GQNYiXLBOM=; b=gubZbuZWBV2BcI1ZfV/2i/NNXAjhxlKHPxxydid1/9yvDoxvQ23FhL8kLh9xqpjmqh +evqKAjDStriPfD2YQyIAocLq96s4Lu2mMZmaEJolo1l4/stN9juV482uWMix+3sEIAC wdsZhTQ/Lx/zduXdjXs1QuiUHBEU0Gad079WtN3/gqwocQANouajj6PL26f9nCsi7ecK 2BQaNgISUaW76Vzaig3qYd+phGxEAi4LyqVZPhpLrNac4tmHXB5J3b3BBzi03NVxOUML 5MSCsGIAS6XgbMaAV04y9/dAQpFOzYR+w5fjo4jroRGIoiwylBq/kL4vXlkSBZ8hKAyY r1XA== X-Gm-Message-State: AC+VfDwezXsJCCFxbU0Z9vwV1r9J7jmKgS2NbWruyO3nFDdOHZWKT8Oi 9qad3/yNRQVLfGq0C3iT0j2Fhg== X-Google-Smtp-Source: ACHHUZ7Ksbe7Ooyt5nBUxHytKU58iAJRK1L03foLc8Hcc6DMHl/WjWoe8IHgUJ0g8IzAAH00aNw/ZQ== X-Received: by 2002:adf:f68c:0:b0:2f5:6430:35c with SMTP id v12-20020adff68c000000b002f56430035cmr13820637wrp.26.1683104841285; Wed, 03 May 2023 02:07:21 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id q11-20020a5d574b000000b003049d7b9f4csm19166645wrw.32.2023.05.03.02.07.20 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:21 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 05/11] RUST: add `module_params` macro Date: Wed, 3 May 2023 11:07:02 +0200 Message-Id: <20230503090708.2524310-6-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Andreas Hindborg This patch includes changes required for Rust kernel modules to utilize mod= ule parameters. Imported with minor changes from `rust` branch [1], see original code for attributions. [1] https://github.com/Rust-for-Linux/linux/tree/bc22545f38d74473cfef3e9fd6= 5432733435b79f Cc: adamrk --- rust/kernel/lib.rs | 3 + rust/kernel/module_param.rs | 501 ++++++++++++++++++++++++++++++++++++ rust/macros/helpers.rs | 46 +++- rust/macros/module.rs | 402 +++++++++++++++++++++++++++-- 4 files changed, 920 insertions(+), 32 deletions(-) create mode 100644 rust/kernel/module_param.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index cd798d12d97c..a0bd0b0e2aef 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -22,6 +22,7 @@ #![feature(pin_macro)] #![feature(receiver_trait)] #![feature(unsize)] +#![feature(const_mut_refs)] =20 // Ensure conditional compilation based on the kernel configuration works; // otherwise we may silently break things like initcall handling. @@ -39,6 +40,8 @@ mod build_assert; pub mod error; pub mod init; pub mod ioctl; +#[doc(hidden)] +pub mod module_param; pub mod pages; pub mod prelude; pub mod print; diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs new file mode 100644 index 000000000000..539decbdcd48 --- /dev/null +++ b/rust/kernel/module_param.rs @@ -0,0 +1,501 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for module parameters. +//! +//! C header: [`include/linux/moduleparam.h`](../../../include/linux/modul= eparam.h) + +use crate::error::{code::*, from_result}; +use crate::str::{CStr, Formatter}; +use core::fmt::Write; + +/// Types that can be used for module parameters. +/// +/// Note that displaying the type in `sysfs` will fail if +/// [`alloc::string::ToString::to_string`] (as implemented through the +/// [`core::fmt::Display`] trait) writes more than [`PAGE_SIZE`] +/// bytes (including an additional null terminator). +/// +/// [`PAGE_SIZE`]: `crate::PAGE_SIZE` +pub trait ModuleParam: core::fmt::Display + core::marker::Sized { + /// The `ModuleParam` will be used by the kernel module through this t= ype. + /// + /// This may differ from `Self` if, for example, `Self` needs to track + /// ownership without exposing it or allocate extra space for other po= ssible + /// parameter values. See [`StringParam`] or [`ArrayParam`] for exampl= es. + type Value: ?Sized; + + /// Whether the parameter is allowed to be set without an argument. + /// + /// Setting this to `true` allows the parameter to be passed without an + /// argument (e.g. just `module.param` instead of `module.param=3Dfoo`= ). + const NOARG_ALLOWED: bool; + + /// Convert a parameter argument into the parameter value. + /// + /// `None` should be returned when parsing of the argument fails. + /// `arg =3D=3D None` indicates that the parameter was passed without = an + /// argument. If `NOARG_ALLOWED` is set to `false` then `arg` is guara= nteed + /// to always be `Some(_)`. + /// + /// Parameters passed at boot time will be set before [`kmalloc`] is + /// available (even if the module is loaded at a later time). However,= in + /// this case, the argument buffer will be valid for the entire lifeti= me of + /// the kernel. So implementations of this method which need to alloca= te + /// should first check that the allocator is available (with + /// [`crate::bindings::slab_is_available`]) and when it is not availab= le + /// provide an alternative implementation which doesn't allocate. In c= ases + /// where the allocator is not available it is safe to save references= to + /// `arg` in `Self`, but in other cases a copy should be made. + /// + /// [`kmalloc`]: ../../../include/linux/slab.h + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option; + + /// Get the current value of the parameter for use in the kernel modul= e. + /// + /// This function should not be used directly. Instead use the wrapper + /// `read` which will be generated by [`macros::module`]. + fn value(&self) -> &Self::Value; + + /// Set the module parameter from a string. + /// + /// Used to set the parameter value when loading the module or when set + /// through `sysfs`. + /// + /// # Safety + /// + /// If `val` is non-null then it must point to a valid null-terminated + /// string. The `arg` field of `param` must be an instance of `Self`. + unsafe extern "C" fn set_param( + val: *const core::ffi::c_char, + param: *const crate::bindings::kernel_param, + ) -> core::ffi::c_int { + let arg =3D if val.is_null() { + None + } else { + Some(unsafe { CStr::from_char_ptr(val).as_bytes() }) + }; + match Self::try_from_param_arg(arg) { + Some(new_value) =3D> { + let old_value =3D unsafe { (*param).__bindgen_anon_1.arg a= s *mut Self }; + let _ =3D unsafe { core::ptr::replace(old_value, new_value= ) }; + 0 + } + None =3D> EINVAL.to_errno(), + } + } + + /// Write a string representation of the current parameter value to `b= uf`. + /// + /// Used for displaying the current parameter value in `sysfs`. + /// + /// # Safety + /// + /// `buf` must be a buffer of length at least `kernel::PAGE_SIZE` that= is + /// writeable. The `arg` field of `param` must be an instance of `Self= `. + unsafe extern "C" fn get_param( + buf: *mut core::ffi::c_char, + param: *const crate::bindings::kernel_param, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: The C contracts guarantees that the buffer is at le= ast `PAGE_SIZE` bytes. + let mut f =3D unsafe { Formatter::from_buffer(buf.cast(), crat= e::PAGE_SIZE as usize) }; + unsafe { write!(f, "{}\0", *((*param).__bindgen_anon_1.arg as = *mut Self)) }?; + Ok(f.bytes_written().try_into()?) + }) + } + + /// Drop the parameter. + /// + /// Called when unloading a module. + /// + /// # Safety + /// + /// The `arg` field of `param` must be an instance of `Self`. + unsafe extern "C" fn free(arg: *mut core::ffi::c_void) { + unsafe { core::ptr::drop_in_place(arg as *mut Self) }; + } +} + +/// Trait for parsing integers. +/// +/// Strings beginning with `0x`, `0o`, or `0b` are parsed as hex, octal, or +/// binary respectively. Strings beginning with `0` otherwise are parsed as +/// octal. Anything else is parsed as decimal. A leading `+` or `-` is also +/// permitted. Any string parsed by [`kstrtol()`] or [`kstrtoul()`] will be +/// successfully parsed. +/// +/// [`kstrtol()`]: https://www.kernel.org/doc/html/latest/core-api/kernel-= api.html#c.kstrtol +/// [`kstrtoul()`]: https://www.kernel.org/doc/html/latest/core-api/kernel= -api.html#c.kstrtoul +trait ParseInt: Sized { + fn from_str_radix(src: &str, radix: u32) -> Result; + fn checked_neg(self) -> Option; + + fn from_str_unsigned(src: &str) -> Result { + let (radix, digits) =3D if let Some(n) =3D src.strip_prefix("0x") { + (16, n) + } else if let Some(n) =3D src.strip_prefix("0X") { + (16, n) + } else if let Some(n) =3D src.strip_prefix("0o") { + (8, n) + } else if let Some(n) =3D src.strip_prefix("0O") { + (8, n) + } else if let Some(n) =3D src.strip_prefix("0b") { + (2, n) + } else if let Some(n) =3D src.strip_prefix("0B") { + (2, n) + } else if src.starts_with('0') { + (8, src) + } else { + (10, src) + }; + Self::from_str_radix(digits, radix) + } + + fn from_str(src: &str) -> Option { + match src.bytes().next() { + None =3D> None, + Some(b'-') =3D> Self::from_str_unsigned(&src[1..]).ok()?.check= ed_neg(), + Some(b'+') =3D> Some(Self::from_str_unsigned(&src[1..]).ok()?), + Some(_) =3D> Some(Self::from_str_unsigned(src).ok()?), + } + } +} + +macro_rules! impl_parse_int { + ($ty:ident) =3D> { + impl ParseInt for $ty { + fn from_str_radix(src: &str, radix: u32) -> Result { + $ty::from_str_radix(src, radix) + } + + fn checked_neg(self) -> Option { + self.checked_neg() + } + } + }; +} + +impl_parse_int!(i8); +impl_parse_int!(u8); +impl_parse_int!(i16); +impl_parse_int!(u16); +impl_parse_int!(i32); +impl_parse_int!(u32); +impl_parse_int!(i64); +impl_parse_int!(u64); +impl_parse_int!(isize); +impl_parse_int!(usize); + +macro_rules! impl_module_param { + ($ty:ident) =3D> { + impl ModuleParam for $ty { + type Value =3D $ty; + + const NOARG_ALLOWED: bool =3D false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + let bytes =3D arg?; + let utf8 =3D core::str::from_utf8(bytes).ok()?; + <$ty as crate::module_param::ParseInt>::from_str(utf8) + } + + #[inline(always)] + fn value(&self) -> &Self::Value { + self + } + } + }; +} + +#[doc(hidden)] +#[macro_export] +/// Generate a static [`kernel_param_ops`](../../../include/linux/modulepa= ram.h) struct. +/// +/// # Examples +/// +/// ```ignore +/// make_param_ops!( +/// /// Documentation for new param ops. +/// PARAM_OPS_MYTYPE, // Name for the static. +/// MyType // A type which implements [`ModuleParam`]. +/// ); +/// ``` +macro_rules! make_param_ops { + ($ops:ident, $ty:ty) =3D> { + $crate::make_param_ops!( + #[doc=3D""] + $ops, + $ty + ); + }; + ($(#[$meta:meta])* $ops:ident, $ty:ty) =3D> { + $(#[$meta])* + /// + /// Static [`kernel_param_ops`](../../../include/linux/moduleparam= .h) + /// struct generated by [`make_param_ops`]. + pub static $ops: $crate::bindings::kernel_param_ops =3D $crate::bi= ndings::kernel_param_ops { + flags: if <$ty as $crate::module_param::ModuleParam>::NOARG_AL= LOWED { + $crate::bindings::KERNEL_PARAM_OPS_FL_NOARG + } else { + 0 + }, + set: Some(<$ty as $crate::module_param::ModuleParam>::set_para= m), + get: Some(<$ty as $crate::module_param::ModuleParam>::get_para= m), + free: Some(<$ty as $crate::module_param::ModuleParam>::free), + }; + }; +} + +impl_module_param!(i8); +impl_module_param!(u8); +impl_module_param!(i16); +impl_module_param!(u16); +impl_module_param!(i32); +impl_module_param!(u32); +impl_module_param!(i64); +impl_module_param!(u64); +impl_module_param!(isize); +impl_module_param!(usize); + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`i8`]. + PARAM_OPS_I8, + i8 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`u8`]. + PARAM_OPS_U8, + u8 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`i16`]. + PARAM_OPS_I16, + i16 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`u16`]. + PARAM_OPS_U16, + u16 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`i32`]. + PARAM_OPS_I32, + i32 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`u32`]. + PARAM_OPS_U32, + u32 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`i64`]. + PARAM_OPS_I64, + i64 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`u64`]. + PARAM_OPS_U64, + u64 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`isize`]. + PARAM_OPS_ISIZE, + isize +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`usize`]. + PARAM_OPS_USIZE, + usize +); + +impl ModuleParam for bool { + type Value =3D bool; + + const NOARG_ALLOWED: bool =3D true; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + match arg { + None =3D> Some(true), + Some(b"y") | Some(b"Y") | Some(b"1") | Some(b"true") =3D> Some= (true), + Some(b"n") | Some(b"N") | Some(b"0") | Some(b"false") =3D> Som= e(false), + _ =3D> None, + } + } + + #[inline(always)] + fn value(&self) -> &Self::Value { + self + } +} + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`bool`]. + PARAM_OPS_BOOL, + bool +); + +/// An array of at __most__ `N` values. +/// +/// # Invariant +/// +/// The first `self.used` elements of `self.values` are initialized. +pub struct ArrayParam { + values: [core::mem::MaybeUninit; N], + used: usize, +} + +impl ArrayParam { + fn values(&self) -> &[T] { + // SAFETY: The invariant maintained by `ArrayParam` allows us to c= ast + // the first `self.used` elements to `T`. + unsafe { + &*(&self.values[0..self.used] as *const [core::mem::MaybeUnini= t] as *const [T]) + } + } +} + +impl ArrayParam { + const fn new() -> Self { + // INVARIANT: The first `self.used` elements of `self.values` are + // initialized. + ArrayParam { + values: [core::mem::MaybeUninit::uninit(); N], + used: 0, + } + } + + const fn push(&mut self, val: T) { + if self.used < N { + // INVARIANT: The first `self.used` elements of `self.values` = are + // initialized. + self.values[self.used] =3D core::mem::MaybeUninit::new(val); + self.used +=3D 1; + } + } + + /// Create an instance of `ArrayParam` initialized with `vals`. + /// + /// This function is only meant to be used in the [`module::module`] m= acro. + pub const fn create(vals: &[T]) -> Self { + let mut result =3D ArrayParam::new(); + let mut i =3D 0; + while i < vals.len() { + result.push(vals[i]); + i +=3D 1; + } + result + } +} + +impl core::fmt::Display for ArrayPa= ram { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + for val in self.values() { + write!(f, "{},", val)?; + } + Ok(()) + } +} + +impl ModulePar= am + for ArrayParam +{ + type Value =3D [T]; + + const NOARG_ALLOWED: bool =3D false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + arg.and_then(|args| { + let mut result =3D Self::new(); + for arg in args.split(|b| *b =3D=3D b',') { + result.push(T::try_from_param_arg(Some(arg))?); + } + Some(result) + }) + } + + fn value(&self) -> &Self::Value { + self.values() + } +} + +/// A C-style string parameter. +/// +/// The Rust version of the [`charp`] parameter. This type is meant to be +/// used by the [`macros::module`] macro, not handled directly. Instead us= e the +/// `read` method generated by that macro. +/// +/// [`charp`]: ../../../include/linux/moduleparam.h +pub enum StringParam { + /// A borrowed parameter value. + /// + /// Either the default value (which is static in the module) or borrow= ed + /// from the original argument buffer used to set the value. + Ref(&'static [u8]), + + /// A value that was allocated when the parameter was set. + /// + /// The value needs to be freed when the parameter is reset or the mod= ule is + /// unloaded. + Owned(alloc::vec::Vec), +} + +impl StringParam { + fn bytes(&self) -> &[u8] { + match self { + StringParam::Ref(bytes) =3D> *bytes, + StringParam::Owned(vec) =3D> &vec[..], + } + } +} + +impl core::fmt::Display for StringParam { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let bytes =3D self.bytes(); + match core::str::from_utf8(bytes) { + Ok(utf8) =3D> write!(f, "{}", utf8), + Err(_) =3D> write!(f, "{:?}", bytes), + } + } +} + +impl ModuleParam for StringParam { + type Value =3D [u8]; + + const NOARG_ALLOWED: bool =3D false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + // SAFETY: It is always safe to call [`slab_is_available`](../../.= ./include/linux/slab.h). + let slab_available =3D unsafe { crate::bindings::slab_is_available= () }; + arg.and_then(|arg| { + if slab_available { + let mut vec =3D alloc::vec::Vec::new(); + vec.try_extend_from_slice(arg).ok()?; + Some(StringParam::Owned(vec)) + } else { + Some(StringParam::Ref(arg)) + } + }) + } + + fn value(&self) -> &Self::Value { + self.bytes() + } +} + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux= /moduleparam.h) + /// for [`StringParam`]. + PARAM_OPS_STR, + StringParam +); diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs index b2bdd4d8c958..8b5f9bc414d7 100644 --- a/rust/macros/helpers.rs +++ b/rust/macros/helpers.rs @@ -18,6 +18,16 @@ pub(crate) fn try_literal(it: &mut token_stream::IntoIte= r) -> Option { } } =20 +pub(crate) fn try_byte_string(it: &mut token_stream::IntoIter) -> Option { + try_literal(it).and_then(|byte_string| { + if byte_string.starts_with("b\"") && byte_string.ends_with('\"') { + Some(byte_string[2..byte_string.len() - 1].to_string()) + } else { + None + } + }) +} + pub(crate) fn try_string(it: &mut token_stream::IntoIter) -> Option { try_literal(it).and_then(|string| { if string.starts_with('\"') && string.ends_with('\"') { @@ -46,14 +56,8 @@ pub(crate) fn expect_punct(it: &mut token_stream::IntoIt= er) -> char { } } =20 -pub(crate) fn expect_string(it: &mut token_stream::IntoIter) -> String { - try_string(it).expect("Expected string") -} - -pub(crate) fn expect_string_ascii(it: &mut token_stream::IntoIter) -> Stri= ng { - let string =3D try_string(it).expect("Expected string"); - assert!(string.is_ascii(), "Expected ASCII string"); - string +pub(crate) fn expect_literal(it: &mut token_stream::IntoIter) -> String { + try_literal(it).expect("Expected Literal") } =20 pub(crate) fn expect_group(it: &mut token_stream::IntoIter) -> Group { @@ -64,8 +68,34 @@ pub(crate) fn expect_group(it: &mut token_stream::IntoIt= er) -> Group { } } =20 +pub(crate) fn expect_string(it: &mut token_stream::IntoIter) -> String { + try_string(it).expect("Expected string") +} + +pub(crate) fn expect_string_ascii(it: &mut token_stream::IntoIter) -> Stri= ng { + let string =3D try_string(it).expect("Expected string"); + assert!(string.is_ascii(), "Expected ASCII string"); + string +} + pub(crate) fn expect_end(it: &mut token_stream::IntoIter) { if it.next().is_some() { panic!("Expected end"); } } + +pub(crate) fn get_literal(it: &mut token_stream::IntoIter, expected_name: = &str) -> String { + assert_eq!(expect_ident(it), expected_name); + assert_eq!(expect_punct(it), ':'); + let literal =3D expect_literal(it); + assert_eq!(expect_punct(it), ','); + literal +} + +pub(crate) fn get_string(it: &mut token_stream::IntoIter, expected_name: &= str) -> String { + assert_eq!(expect_ident(it), expected_name); + assert_eq!(expect_punct(it), ':'); + let string =3D expect_string(it); + assert_eq!(expect_punct(it), ','); + string +} diff --git a/rust/macros/module.rs b/rust/macros/module.rs index fb1244f8c2e6..172631e7fb05 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -1,25 +1,40 @@ // SPDX-License-Identifier: GPL-2.0 =20 -use crate::helpers::*; -use proc_macro::{token_stream, Delimiter, Literal, TokenStream, TokenTree}; +use proc_macro::{token_stream, Delimiter, Group, Literal, TokenStream, Tok= enTree}; use std::fmt::Write; =20 -fn expect_string_array(it: &mut token_stream::IntoIter) -> Vec { - let group =3D expect_group(it); - assert_eq!(group.delimiter(), Delimiter::Bracket); - let mut values =3D Vec::new(); - let mut it =3D group.stream().into_iter(); - - while let Some(val) =3D try_string(&mut it) { - assert!(val.is_ascii(), "Expected ASCII string"); - values.push(val); - match it.next() { - Some(TokenTree::Punct(punct)) =3D> assert_eq!(punct.as_char(),= ','), - None =3D> break, - _ =3D> panic!("Expected ',' or end of array"), +use crate::helpers::*; + +#[derive(Clone, PartialEq)] +enum ParamType { + Ident(String), + Array { vals: String, max_length: usize }, +} + +fn expect_array_fields(it: &mut token_stream::IntoIter) -> ParamType { + assert_eq!(expect_punct(it), '<'); + let vals =3D expect_ident(it); + assert_eq!(expect_punct(it), ','); + let max_length_str =3D expect_literal(it); + let max_length =3D max_length_str + .parse::() + .expect("Expected usize length"); + assert_eq!(expect_punct(it), '>'); + ParamType::Array { vals, max_length } +} + +fn expect_type(it: &mut token_stream::IntoIter) -> ParamType { + if let TokenTree::Ident(ident) =3D it + .next() + .expect("Reached end of token stream for param type") + { + match ident.to_string().as_ref() { + "ArrayParam" =3D> expect_array_fields(it), + _ =3D> ParamType::Ident(ident.to_string()), } + } else { + panic!("Expected Param Type") } - values } =20 struct ModInfoBuilder<'a> { @@ -87,6 +102,113 @@ impl<'a> ModInfoBuilder<'a> { self.emit_only_builtin(field, content); self.emit_only_loadable(field, content); } + + fn emit_param(&mut self, field: &str, param: &str, content: &str) { + let content =3D format!("{param}:{content}", param =3D param, cont= ent =3D content); + self.emit(field, &content); + } +} + +fn permissions_are_readonly(perms: &str) -> bool { + let (radix, digits) =3D if let Some(n) =3D perms.strip_prefix("0x") { + (16, n) + } else if let Some(n) =3D perms.strip_prefix("0o") { + (8, n) + } else if let Some(n) =3D perms.strip_prefix("0b") { + (2, n) + } else { + (10, perms) + }; + match u32::from_str_radix(digits, radix) { + Ok(perms) =3D> perms & 0o222 =3D=3D 0, + Err(_) =3D> false, + } +} + +fn param_ops_path(param_type: &str) -> &'static str { + match param_type { + "bool" =3D> "kernel::module_param::PARAM_OPS_BOOL", + "i8" =3D> "kernel::module_param::PARAM_OPS_I8", + "u8" =3D> "kernel::module_param::PARAM_OPS_U8", + "i16" =3D> "kernel::module_param::PARAM_OPS_I16", + "u16" =3D> "kernel::module_param::PARAM_OPS_U16", + "i32" =3D> "kernel::module_param::PARAM_OPS_I32", + "u32" =3D> "kernel::module_param::PARAM_OPS_U32", + "i64" =3D> "kernel::module_param::PARAM_OPS_I64", + "u64" =3D> "kernel::module_param::PARAM_OPS_U64", + "isize" =3D> "kernel::module_param::PARAM_OPS_ISIZE", + "usize" =3D> "kernel::module_param::PARAM_OPS_USIZE", + "str" =3D> "kernel::module_param::PARAM_OPS_STR", + t =3D> panic!("Unrecognized type {}", t), + } +} + +#[allow(clippy::type_complexity)] +fn try_simple_param_val( + param_type: &str, +) -> Box Option> { + match param_type { + "bool" =3D> Box::new(try_ident), + "str" =3D> Box::new(|param_it| { + try_byte_string(param_it) + .map(|s| format!("kernel::module_param::StringParam::Ref(b= \"{}\")", s)) + }), + _ =3D> Box::new(try_literal), + } +} + +fn get_default(param_type: &ParamType, param_it: &mut token_stream::IntoIt= er) -> String { + let try_param_val =3D match param_type { + ParamType::Ident(ref param_type) + | ParamType::Array { + vals: ref param_type, + max_length: _, + } =3D> try_simple_param_val(param_type), + }; + assert_eq!(expect_ident(param_it), "default"); + assert_eq!(expect_punct(param_it), ':'); + let default =3D match param_type { + ParamType::Ident(_) =3D> try_param_val(param_it).expect("Expected = default param value"), + ParamType::Array { + vals: _, + max_length: _, + } =3D> { + let group =3D expect_group(param_it); + assert_eq!(group.delimiter(), Delimiter::Bracket); + let mut default_vals =3D Vec::new(); + let mut it =3D group.stream().into_iter(); + + while let Some(default_val) =3D try_param_val(&mut it) { + default_vals.push(default_val); + match it.next() { + Some(TokenTree::Punct(punct)) =3D> assert_eq!(punct.as= _char(), ','), + None =3D> break, + _ =3D> panic!("Expected ',' or end of array default va= lues"), + } + } + + let mut default_array =3D "kernel::module_param::ArrayParam::c= reate(&[".to_string(); + default_array.push_str( + &default_vals + .iter() + .map(|val| val.to_string()) + .collect::>() + .join(","), + ); + default_array.push_str("])"); + default_array + } + }; + assert_eq!(expect_punct(param_it), ','); + default +} + +fn generated_array_ops_name(vals: &str, max_length: usize) -> String { + format!( + "__generated_array_ops_{vals}_{max_length}", + vals =3D vals, + max_length =3D max_length + ) } =20 #[derive(Debug, Default)] @@ -96,15 +218,24 @@ struct ModuleInfo { name: String, author: Option, description: Option, - alias: Option>, + alias: Option, + params: Option, } =20 impl ModuleInfo { fn parse(it: &mut token_stream::IntoIter) -> Self { let mut info =3D ModuleInfo::default(); =20 - const EXPECTED_KEYS: &[&str] =3D - &["type", "name", "author", "description", "license", "alias"]; + const EXPECTED_KEYS: &[&str] =3D &[ + "type", + "name", + "author", + "description", + "license", + "alias", + "alias_rtnl_link", + "params", + ]; const REQUIRED_KEYS: &[&str] =3D &["type", "name", "license"]; let mut seen_keys =3D Vec::new(); =20 @@ -130,7 +261,11 @@ impl ModuleInfo { "author" =3D> info.author =3D Some(expect_string(it)), "description" =3D> info.description =3D Some(expect_string= (it)), "license" =3D> info.license =3D expect_string_ascii(it), - "alias" =3D> info.alias =3D Some(expect_string_array(it)), + "alias" =3D> info.alias =3D Some(expect_string_ascii(it)), + "alias_rtnl_link" =3D> { + info.alias =3D Some(format!("rtnl-link-{}", expect_str= ing_ascii(it))) + } + "params" =3D> info.params =3D Some(expect_group(it)), _ =3D> panic!( "Unknown key \"{}\". Valid keys are: {:?}.", key, EXPECTED_KEYS @@ -181,10 +316,8 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { modinfo.emit("description", &description); } modinfo.emit("license", &info.license); - if let Some(aliases) =3D info.alias { - for alias in aliases { - modinfo.emit("alias", &alias); - } + if let Some(alias) =3D info.alias { + modinfo.emit("alias", &alias); } =20 // Built-in modules also export the `file` modinfo string. @@ -192,6 +325,195 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { std::env::var("RUST_MODFILE").expect("Unable to fetch RUST_MODFILE= environmental variable"); modinfo.emit_only_builtin("file", &file); =20 + let mut array_types_to_generate =3D Vec::new(); + if let Some(params) =3D info.params { + assert_eq!(params.delimiter(), Delimiter::Brace); + + let mut it =3D params.stream().into_iter(); + + loop { + let param_name =3D match it.next() { + Some(TokenTree::Ident(ident)) =3D> ident.to_string(), + Some(_) =3D> panic!("Expected Ident or end"), + None =3D> break, + }; + + assert_eq!(expect_punct(&mut it), ':'); + let param_type =3D expect_type(&mut it); + let group =3D expect_group(&mut it); + assert_eq!(expect_punct(&mut it), ','); + + assert_eq!(group.delimiter(), Delimiter::Brace); + + let mut param_it =3D group.stream().into_iter(); + let param_default =3D get_default(¶m_type, &mut param_it); + let param_permissions =3D get_literal(&mut param_it, "permissi= ons"); + let param_description =3D get_string(&mut param_it, "descripti= on"); + expect_end(&mut param_it); + + // TODO: More primitive types. + // TODO: Other kinds: unsafes, etc. + let (param_kernel_type, ops): (String, _) =3D match param_type= { + ParamType::Ident(ref param_type) =3D> ( + param_type.to_string(), + param_ops_path(param_type).to_string(), + ), + ParamType::Array { + ref vals, + max_length, + } =3D> { + array_types_to_generate.push((vals.clone(), max_length= )); + ( + format!("__rust_array_param_{}_{}", vals, max_leng= th), + generated_array_ops_name(vals, max_length), + ) + } + }; + + modinfo.emit_param("parmtype", ¶m_name, ¶m_kernel_type= ); + modinfo.emit_param("parm", ¶m_name, ¶m_description); + let param_type_internal =3D match param_type { + ParamType::Ident(ref param_type) =3D> match param_type.as_= ref() { + "str" =3D> "kernel::module_param::StringParam".to_stri= ng(), + other =3D> other.to_string(), + }, + ParamType::Array { + ref vals, + max_length, + } =3D> format!( + "kernel::module_param::ArrayParam<{vals}, {max_length}= >", + vals =3D vals, + max_length =3D max_length + ), + }; + let read_func =3D if permissions_are_readonly(¶m_permissio= ns) { + format!( + " + fn read(&self) + -> &<{param_type_internal} as kernel::module_p= aram::ModuleParam>::Value {{ + // SAFETY: Parameters do not need to be locked= because they are + // read only or sysfs is not enabled. + unsafe {{ + <{param_type_internal} as kernel::module_p= aram::ModuleParam>::value( + &__{name}_{param_name}_value + ) + }} + }} + ", + name =3D info.name, + param_name =3D param_name, + param_type_internal =3D param_type_internal, + ) + } else { + format!( + " + fn read<'lck>(&self, lock: &'lck kernel::KParamGua= rd) + -> &'lck <{param_type_internal} as kernel::mod= ule_param::ModuleParam>::Value {{ + // SAFETY: Parameters are locked by `KParamGua= rd`. + unsafe {{ + <{param_type_internal} as kernel::module_p= aram::ModuleParam>::value( + &__{name}_{param_name}_value + ) + }} + }} + ", + name =3D info.name, + param_name =3D param_name, + param_type_internal =3D param_type_internal, + ) + }; + let kparam =3D format!( + " + kernel::bindings::kernel_param__bindgen_ty_1 {{ + arg: unsafe {{ &__{name}_{param_name}_value }} + as *const _ as *mut core::ffi::c_void, + }}, + ", + name =3D info.name, + param_name =3D param_name, + ); + write!( + modinfo.buffer, + " + static mut __{name}_{param_name}_value: {param_type_intern= al} =3D {param_default}; + + struct __{name}_{param_name}; + + impl __{name}_{param_name} {{ {read_func} }} + + const {param_name}: __{name}_{param_name} =3D __{name}_{pa= ram_name}; + + // Note: the C macro that generates the static structs for= the `__param` section + // asks for them to be `aligned(sizeof(void *))`. However,= that was put in place + // in 2003 in commit 38d5b085d2a0 (\"[PATCH] Fix over-alig= nment problem on x86-64\") + // to undo GCC over-alignment of static structs of >32 byt= es. It seems that is + // not the case anymore, so we simplify to a transparent r= epresentation here + // in the expectation that it is not needed anymore. + // TODO: Revisit this to confirm the above comment and rem= ove it if it happened. + #[repr(transparent)] + struct __{name}_{param_name}_RacyKernelParam(kernel::bindi= ngs::kernel_param); + + unsafe impl Sync for __{name}_{param_name}_RacyKernelParam= {{ + }} + + #[cfg(not(MODULE))] + const __{name}_{param_name}_name: *const core::ffi::c_char= =3D + b\"{name}.{param_name}\\0\" as *const _ as *const core= ::ffi::c_char; + + #[cfg(MODULE)] + const __{name}_{param_name}_name: *const core::ffi::c_char= =3D + b\"{param_name}\\0\" as *const _ as *const core::ffi::= c_char; + + #[link_section =3D \"__param\"] + #[used] + static __{name}_{param_name}_struct: __{name}_{param_name}= _RacyKernelParam =3D + __{name}_{param_name}_RacyKernelParam(kernel::bindings= ::kernel_param {{ + name: __{name}_{param_name}_name, + // SAFETY: `__this_module` is constructed by the k= ernel at load time + // and will not be freed until the module is unloa= ded. + #[cfg(MODULE)] + mod_: unsafe {{ &kernel::bindings::__this_module a= s *const _ as *mut _ }}, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: unsafe {{ &{ops} }} as *const kernel::binding= s::kernel_param_ops, + perm: {permissions}, + level: -1, + flags: 0, + __bindgen_anon_1: {kparam} + }}); + ", + name =3D info.name, + param_type_internal =3D param_type_internal, + read_func =3D read_func, + param_default =3D param_default, + param_name =3D param_name, + ops =3D ops, + permissions =3D param_permissions, + kparam =3D kparam, + ) + .unwrap(); + } + } + + let mut generated_array_types =3D String::new(); + + for (vals, max_length) in array_types_to_generate { + let ops_name =3D generated_array_ops_name(&vals, max_length); + write!( + generated_array_types, + " + kernel::make_param_ops!( + {ops_name}, + kernel::module_param::ArrayParam<{vals}, {{ {max_lengt= h} }}> + ); + ", + ops_name =3D ops_name, + vals =3D vals, + max_length =3D max_length, + ) + .unwrap(); + } + format!( " /// The module name. @@ -291,12 +613,44 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { }} =20 {modinfo} + + {generated_array_types} ", type_ =3D info.type_, name =3D info.name, modinfo =3D modinfo.buffer, + generated_array_types =3D generated_array_types, initcall_section =3D ".initcall6.init" ) .parse() .expect("Error parsing formatted string into token stream.") } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_permissions_are_readonly() { + assert!(permissions_are_readonly("0b000000000")); + assert!(permissions_are_readonly("0o000")); + assert!(permissions_are_readonly("000")); + assert!(permissions_are_readonly("0x000")); + + assert!(!permissions_are_readonly("0b111111111")); + assert!(!permissions_are_readonly("0o777")); + assert!(!permissions_are_readonly("511")); + assert!(!permissions_are_readonly("0x1ff")); + + assert!(permissions_are_readonly("0o014")); + assert!(permissions_are_readonly("0o015")); + + assert!(!permissions_are_readonly("0o214")); + assert!(!permissions_are_readonly("0o024")); + assert!(!permissions_are_readonly("0o012")); + + assert!(!permissions_are_readonly("0o315")); + assert!(!permissions_are_readonly("0o065")); + assert!(!permissions_are_readonly("0o017")); + } +} --=20 2.40.0 From nobody Wed Feb 11 23:44:39 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id A5CC7C7EE22 for ; Wed, 3 May 2023 09:07:50 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229823AbjECJHt (ORCPT ); Wed, 3 May 2023 05:07:49 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45468 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229946AbjECJHb (ORCPT ); Wed, 3 May 2023 05:07:31 -0400 Received: from mail-wm1-x32e.google.com (mail-wm1-x32e.google.com [IPv6:2a00:1450:4864:20::32e]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 3D6714C21 for ; Wed, 3 May 2023 02:07:24 -0700 (PDT) Received: by mail-wm1-x32e.google.com with SMTP id 5b1f17b1804b1-3f1728c2a57so48890795e9.0 for ; Wed, 03 May 2023 02:07:23 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104842; x=1685696842; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=5cH5GZPThZX1LLdPnjlMEvQzzaSHQbdfYQa3aafCtFw=; b=Hgz+JsybFVjncw15HasJvie2Dl4VadDcEYJtGAk4kb/RQguZRxQO+WUSHbBzCchQfK 4fR/4HHzFj7feyZ/Ey1XOlfTNiuid/oK5BCfCTwEDhAS24mdDiPOWkT4Xnon05qvrKsI 9DROCY4yYimdZfsorqum1Kgn1FKSyFiupk9F01cUByEOnAhbB/Zk8PwTZ8IQQh3Ysld8 EQ/BMmUi7pFY1YFHuqo9s0suNHftuZROm2snpKuc0maQI6BsY8e6/z20fLheOG/FOaSj ck2bzsszlaJfYe3wOeCcoVvxNqXt9pxyYKS7bIBfx+nUkWsDba1tg8Zs6sA0UfeBQnJ9 LT3w== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104842; x=1685696842; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=5cH5GZPThZX1LLdPnjlMEvQzzaSHQbdfYQa3aafCtFw=; b=XzCKfBOjEIIdFies9ongBrzgc+OQUsgM5g89t0OkBPHcr85XYayK5Lf5Zl2KMD1CSL ejx2FKftDXv09cSIA3poYOGorc3iR+71R3KbY6u4JTaaXAJ3JT/BW53YTZ6oWynUml6G +ZYOxslrG2qKkAcqXmd5KhGA7VKrIQm6H94RY8xHUqdPPF3m/g85DBk1mlwbUqts364b acdpdCo87Wme/e3X93d4G3dxTCSSXLoLGQOUtN9LX2p5eM/NGd31fhZAEMLYBjx/+fp6 1WAXYv2ewDzL+h9Y4rQ2sRphUEiWnVSYDRHDtBNNMTbqnydRbdsNk47odvOJAQKuD+H0 uStg== X-Gm-Message-State: AC+VfDzIy8zBVxcQcZRH9KimAJdrRgdTa8JPliQUpP2ZQbyPoxpTjW9h ZchWg7IvVtG9esIxO8BREp1a+g== X-Google-Smtp-Source: ACHHUZ4EBxDh+2FdUjZNnlPIs7Z43merWOWOdSZYBzuMbtFHhqsjGSgkNu/agPTqqGu/gJ3ZmsuWWw== X-Received: by 2002:a7b:cb85:0:b0:3f1:7136:dd45 with SMTP id m5-20020a7bcb85000000b003f17136dd45mr14187268wmi.30.1683104842308; Wed, 03 May 2023 02:07:22 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id n15-20020a1c720f000000b003f339b2d06fsm1271659wmc.4.2023.05.03.02.07.21 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:22 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 06/11] rust: apply cache line padding for `SpinLock` Date: Wed, 3 May 2023 11:07:03 +0200 Message-Id: <20230503090708.2524310-7-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Andreas Hindborg The kernel `struct spinlock` is 4 bytes on x86 when lockdep is not enabled.= The structure is not padded to fit a cache line. The effect of this for `SpinLo= ck` is that the lock variable and the value protected by the lock will share a = cache line, depending on the alignment requirements of the protected value. Align= ing the lock variable and the protected value to a cache line yields a 20% performance increase for the Rust null block driver for sequential reads to memory backed devices at 6 concurrent readers. Signed-off-by: Andreas Hindborg --- rust/kernel/cache_padded.rs | 33 +++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 2 ++ rust/kernel/sync/lock.rs | 9 ++++++--- rust/kernel/sync/lock/spinlock.rs | 13 ++++++++---- 4 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 rust/kernel/cache_padded.rs diff --git a/rust/kernel/cache_padded.rs b/rust/kernel/cache_padded.rs new file mode 100644 index 000000000000..758678e71f50 --- /dev/null +++ b/rust/kernel/cache_padded.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 + +#[repr(align(64))] +pub struct CachePadded { + value: T, +} + +unsafe impl Send for CachePadded {} +unsafe impl Sync for CachePadded {} + +impl CachePadded { + /// Pads and aligns a value to 64 bytes. + #[inline(always)] + pub(crate) const fn new(t: T) -> CachePadded { + CachePadded:: { value: t } + } +} + +impl core::ops::Deref for CachePadded { + type Target =3D T; + + #[inline(always)] + fn deref(&self) -> &T { + &self.value + } +} + +impl core::ops::DerefMut for CachePadded { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + &mut self.value + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index a0bd0b0e2aef..426e2dea0da6 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -37,6 +37,7 @@ extern crate self as kernel; mod allocator; pub mod block; mod build_assert; +mod cache_padded; pub mod error; pub mod init; pub mod ioctl; @@ -56,6 +57,7 @@ pub mod types; =20 #[doc(hidden)] pub use bindings; +pub(crate) use cache_padded::CachePadded; pub use macros; pub use uapi; =20 diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index a2216325632d..1c584b1df30d 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -6,7 +6,9 @@ //! spinlocks, raw spinlocks) to be provided with minimal effort. =20 use super::LockClassKey; -use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque, t= ypes::ScopeGuard}; +use crate::{ + bindings, init::PinInit, pin_init, str::CStr, types::Opaque, types::Sc= opeGuard, CachePadded, +}; use core::{cell::UnsafeCell, marker::PhantomData, marker::PhantomPinned}; use macros::pin_data; =20 @@ -87,7 +89,7 @@ pub struct Lock { _pin: PhantomPinned, =20 /// The data protected by the lock. - pub(crate) data: UnsafeCell, + pub(crate) data: CachePadded>, } =20 // SAFETY: `Lock` can be transferred across thread boundaries iff the data= it protects can. @@ -102,7 +104,7 @@ impl Lock { #[allow(clippy::new_ret_no_self)] pub fn new(t: T, name: &'static CStr, key: &'static LockClassKey) -> i= mpl PinInit { pin_init!(Self { - data: UnsafeCell::new(t), + data: CachePadded::new(UnsafeCell::new(t)), _pin: PhantomPinned, // SAFETY: `slot` is valid while the closure is called and bot= h `name` and `key` have // static lifetimes so they live indefinitely. @@ -115,6 +117,7 @@ impl Lock { =20 impl Lock { /// Acquires the lock and gives the caller access to the data protecte= d by it. + #[inline(always)] pub fn lock(&self) -> Guard<'_, T, B> { // SAFETY: The constructor of the type calls `init`, so the existe= nce of the object proves // that `init` was called. diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index 979b56464a4e..e39142a8148c 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,7 +4,10 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. =20 +use core::ops::DerefMut; + use crate::bindings; +use crate::CachePadded; =20 /// Creates a [`SpinLock`] initialiser with the given name and a newly-cre= ated lock class. /// @@ -90,7 +93,7 @@ pub struct SpinLockBackend; // SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclus= ion. `relock` uses the // default implementation that always calls the same locking method. unsafe impl super::Backend for SpinLockBackend { - type State =3D bindings::spinlock_t; + type State =3D CachePadded; type GuardState =3D (); =20 unsafe fn init( @@ -100,18 +103,20 @@ unsafe impl super::Backend for SpinLockBackend { ) { // SAFETY: The safety requirements ensure that `ptr` is valid for = writes, and `name` and // `key` are valid for read indefinitely. - unsafe { bindings::__spin_lock_init(ptr, name, key) } + unsafe { bindings::__spin_lock_init((&mut *ptr).deref_mut(), name,= key) } } =20 + #[inline(always)] unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { // SAFETY: The safety requirements of this function ensure that `p= tr` points to valid // memory, and that it has been initialised before. - unsafe { bindings::spin_lock(ptr) } + unsafe { bindings::spin_lock((&mut *ptr).deref_mut()) } } =20 + #[inline(always)] unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardStat= e) { // SAFETY: The safety requirements of this function ensure that `p= tr` is valid and that the // caller is the owner of the mutex. - unsafe { bindings::spin_unlock(ptr) } + unsafe { bindings::spin_unlock((&mut *ptr).deref_mut()) } } } --=20 2.40.0 From nobody Wed Feb 11 23:44:39 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 6A4A0C77B75 for ; Wed, 3 May 2023 09:08:00 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229558AbjECJH6 (ORCPT ); Wed, 3 May 2023 05:07:58 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45870 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229952AbjECJHi (ORCPT ); Wed, 3 May 2023 05:07:38 -0400 Received: from mail-wr1-x42c.google.com (mail-wr1-x42c.google.com [IPv6:2a00:1450:4864:20::42c]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 2B1CA4EC5 for ; Wed, 3 May 2023 02:07:24 -0700 (PDT) Received: by mail-wr1-x42c.google.com with SMTP id ffacd0b85a97d-3062678861fso2733867f8f.0 for ; Wed, 03 May 2023 02:07:24 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104843; x=1685696843; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=BtKpWEX99/tXFIUB1b2EGEDNutHLi8kX/M6s4oIe5M4=; b=CxSTo+18rMVa2ypTkZTy0wLUDTZw1QYq8jpoxdVhk/p84nZH5UHbiZq1swvBWxc3Xl r2kO5WRrEPvmzDQeUiP6rVG+UuPHdE+apzClMIY04+UBt5rRJASMvfHzn8yHzRZItfCa eICcSstADFeX/SdIHIB5B8+VFI4XL8BgekNIxDTO4LdHoRLMZqNr73unzM6ADYmaIyeq tJ6dsNUjxn+lkY8wiCDlibjPjlC8Y01rsjloOfV2tBlRm8G3dYPzQyejyyH9hyMJxKY2 Ga0zGhnlQF0t9qD5/DaEQMXU1PD13S+IC5523kVFJfpei0LQl8U9b9kNdjDOMCoLgVnw vA1A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104843; x=1685696843; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=BtKpWEX99/tXFIUB1b2EGEDNutHLi8kX/M6s4oIe5M4=; b=HgxF/rR/rSlO4oFvIvCF4Gd6Q1zH9LhkTcsZBNR02i/tG6Kd/Vb0V14iwHSK1ZGXRg 5SFE0N6vLXz3Zti83Z1JEqixweBITWfcG9vmNDmaNAFWeFaDGgUpF2J4Bd4ZEWjwt45v 8TOpiYzEns7b/NrkRwtMObKKg4EejfJNwKoy5x89AxTD5mv4qRmi1PZQU0ujr0Ds/VGE PxgJgnMC3Ktr8Ns2s4TwNIRLlMEz9Rco6FHb9iYyrU/gMYAgrMbTMk/GO8c5lPCavaOc sMrYqXKLMMcKXy2lFcxOXIHyDdzVS0Gd/fV957GhattQhoTzs8cFDbzj8s4Kfb+F25eQ 9zqg== X-Gm-Message-State: AC+VfDzGYsXv8AApPFulAkRgkHSEtReysUOJwIeNatWS9iPI8TSz0RSQ EgLgD8SwJbvzhR510LmdV2pxKg== X-Google-Smtp-Source: ACHHUZ7KbXS+IwQVsR80XNjrwtE1ZUCnYctnByWWLdgm7ZrK387pgkawXMC7JKEwhn48tufrkT8hcQ== X-Received: by 2002:a5d:4577:0:b0:306:2b5a:d8db with SMTP id a23-20020a5d4577000000b003062b5ad8dbmr6680255wrc.23.1683104843273; Wed, 03 May 2023 02:07:23 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id x9-20020a5d60c9000000b002fbdb797483sm32900298wrt.49.2023.05.03.02.07.22 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:23 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Wedson Almeida Filho , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 07/11] rust: lock: add support for `Lock::lock_irqsave` Date: Wed, 3 May 2023 11:07:04 +0200 Message-Id: <20230503090708.2524310-8-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Wedson Almeida Filho This allows locks like spinlocks and raw spinlocks to expose a `lock_irqsave` variant in Rust that corresponds to the C version. Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho --- rust/kernel/sync/lock.rs | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 1c584b1df30d..bb21af8a8377 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -72,6 +72,44 @@ pub unsafe trait Backend { } } =20 +/// The "backend" of a lock that supports the irq-save variant. +/// +/// # Safety +/// +/// The same requirements wrt mutual exclusion in [`Backend`] apply for ac= quiring the lock via +/// [`IrqSaveBackend::lock_irqsave`]. +/// +/// Additionally, when [`IrqSaveBackend::lock_irqsave`] is used to acquire= the lock, implementers +/// must disable interrupts on lock, and restore interrupt state on unlock= . Implementers may use +/// [`Backend::GuardState`] to store state needed to keep track of the int= errupt state. +pub unsafe trait IrqSaveBackend: Backend { + /// Acquires the lock, making the caller its owner. + /// + /// Before acquiring the lock, it disables interrupts, and returns the= previous interrupt state + /// as its guard state so that the guard can restore it when it is dro= pped. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously cal= led. + #[must_use] + unsafe fn lock_irqsave(ptr: *mut Self::State) -> Self::GuardState; +} + +impl Lock { + /// Acquires the lock and gives the caller access to the data protecte= d by it. + /// + /// Before acquiring the lock, it disables interrupts. When the guard = is dropped, the interrupt + /// state (either enabled or disabled) is restored to its state before + /// [`lock_irqsave`](Self::lock_irqsave) was called. + pub fn lock_irqsave(&self) -> Guard<'_, T, B> { + // SAFETY: The constructor of the type calls `init`, so the existe= nce of the object proves + // that `init` was called. + let state =3D unsafe { B::lock_irqsave(self.state.get()) }; + // SAFETY: The lock was just acquired. + unsafe { Guard::new(self, state) } + } +} + /// A mutual exclusion primitive. /// /// Exposes one of the kernel locking primitives. Which one is exposed dep= ends on the lock backend --=20 2.40.0 From nobody Wed Feb 11 23:44:39 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 6D1ADC77B7F for ; Wed, 3 May 2023 09:08:09 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230033AbjECJIH (ORCPT ); Wed, 3 May 2023 05:08:07 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45922 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229707AbjECJHj (ORCPT ); Wed, 3 May 2023 05:07:39 -0400 Received: from mail-wr1-x436.google.com (mail-wr1-x436.google.com [IPv6:2a00:1450:4864:20::436]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 98CAF46B7 for ; Wed, 3 May 2023 02:07:26 -0700 (PDT) Received: by mail-wr1-x436.google.com with SMTP id ffacd0b85a97d-305f0491e62so4324421f8f.3 for ; Wed, 03 May 2023 02:07:26 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104844; x=1685696844; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=pfKdjWdresPJSdXc5oUQnKkLhNP+u1b3Z6UA6/bvZEs=; b=zxf3V0pixeyAdrD64XjVTcoz5GBZP1Ez2NPEYfg4bJ+8OpHIgbs5IPYWoyRFF/oRls 1HiTMTqiw5oPzKcjXk85WtG2qWppkjvU02IYtCNv/P6lilDjZ3I/tCqie+hxdHS9fI72 TFUlh6Wb66UftjAJVkN8zXebQeRzUQEjndgJGChQSPHdwiL4EDhLk2wZR45z/mLS3qsS 0hle1vi+M/A5q3ANxKiVAd1H1NxXbl8yRYYvbayFD9yVys1oohANbByOMWC+mJ8fu1Aq h3If9GnckODucwV34nms5ONfQlshxGTXJ2YdkMkaVW/TPTSFD4UmrIVnUA6QvhXfqCL+ 1Ffw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104844; x=1685696844; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=pfKdjWdresPJSdXc5oUQnKkLhNP+u1b3Z6UA6/bvZEs=; b=lftjJpWP1QyAbXWOngXxkaxwWypbOFLENowjk5U4EZaxG0arc7GNpK0/tMgVish332 pAfoshzNNVuE+aXaEoyJZV8OIKSURQsBUHinDuHHJvRR4jtZG3RSzHu4Pz2Ax4o9QHMi zm8ehaJIZ6ckwBdvQ6VC8/EcPDsZiOBvLWESm+dInbsOCdkR0EHp0yi+b+xhvA22cBtS TP2SXP0holCznkb7oqe7HRQh1UhkkIkVWYlSgF9Tm05E9RwRfIt599JfPQNmXjjsOHyD RVxcec/l6RNr3ct8Lu6bRjdeuL3itb3GOE8q/IcE0jK8sMTjAB1owis/5V1NVgV1M9XP BN+Q== X-Gm-Message-State: AC+VfDxzegZtZjWk2lrL40Gk4uoiMWXq0u8yEw3SrItsGS3K2hsXsZLZ 314OSSOUjW5bLr5eR6D2fgzRvA== X-Google-Smtp-Source: ACHHUZ5UM0xcnar/k/FZ+nJlonCpodkAJIUEXzZR9u3QQyFGMIWvcy7zXUt0eX7nCk//Wv22bv3Duw== X-Received: by 2002:adf:ee8d:0:b0:306:42e2:5ec3 with SMTP id b13-20020adfee8d000000b0030642e25ec3mr912726wro.6.1683104844271; Wed, 03 May 2023 02:07:24 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id k11-20020a7bc40b000000b003f173a2b2f6sm1248839wmi.12.2023.05.03.02.07.23 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:24 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Wedson Almeida Filho , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 08/11] rust: lock: implement `IrqSaveBackend` for `SpinLock` Date: Wed, 3 May 2023 11:07:05 +0200 Message-Id: <20230503090708.2524310-9-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Wedson Almeida Filho This allows Rust code to use the `lock_irqsave` variant of spinlocks. Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Will Deacon Cc: Waiman Long Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho --- rust/helpers.c | 16 ++++++++++++ rust/kernel/sync/lock/spinlock.rs | 41 +++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/rust/helpers.c b/rust/helpers.c index a59341084774..7f5b95f652e1 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -203,6 +203,22 @@ struct page *rust_helper_alloc_pages(gfp_t gfp_mask, u= nsigned int order) } EXPORT_SYMBOL_GPL(rust_helper_alloc_pages); =20 +unsigned long rust_helper_spin_lock_irqsave(spinlock_t *lock) +{ + unsigned long flags; + + spin_lock_irqsave(lock, flags); + + return flags; +} +EXPORT_SYMBOL_GPL(rust_helper_spin_lock_irqsave); + +void rust_helper_spin_unlock_irqrestore(spinlock_t *lock, unsigned long fl= ags) +{ + spin_unlock_irqrestore(lock, flags); +} +EXPORT_SYMBOL_GPL(rust_helper_spin_unlock_irqrestore); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` ty= pe * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index e39142a8148c..50b8775bb49d 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -64,6 +64,8 @@ macro_rules! new_spinlock { /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); +/// assert_eq!(e.d.lock_irqsave().a, 20); +/// assert_eq!(e.d.lock_irqsave().b, 30); /// ``` /// /// The following example shows how to use interior mutability to modify t= he contents of a struct @@ -81,6 +83,12 @@ macro_rules! new_spinlock { /// let mut guard =3D m.lock(); /// guard.a +=3D 10; /// guard.b +=3D 20; +/// +/// fn example2(m: &SpinLock) { +/// let mut guard =3D m.lock_irqsave(); +/// guard.a +=3D 10; +/// guard.b +=3D 20; +/// } /// } /// ``` /// @@ -94,7 +102,7 @@ pub struct SpinLockBackend; // default implementation that always calls the same locking method. unsafe impl super::Backend for SpinLockBackend { type State =3D CachePadded; - type GuardState =3D (); + type GuardState =3D Option; =20 unsafe fn init( ptr: *mut Self::State, @@ -110,13 +118,32 @@ unsafe impl super::Backend for SpinLockBackend { unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { // SAFETY: The safety requirements of this function ensure that `p= tr` points to valid // memory, and that it has been initialised before. - unsafe { bindings::spin_lock((&mut *ptr).deref_mut()) } + unsafe { bindings::spin_lock((&mut *ptr).deref_mut()) }; + None } =20 - #[inline(always)] - unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardStat= e) { - // SAFETY: The safety requirements of this function ensure that `p= tr` is valid and that the - // caller is the owner of the mutex. - unsafe { bindings::spin_unlock((&mut *ptr).deref_mut()) } + unsafe fn unlock(ptr: *mut Self::State, guard_state: &Self::GuardState= ) { + match guard_state { + // SAFETY: The safety requirements of this function ensure tha= t `ptr` is valid and that + // the caller is the owner of the mutex. + Some(flags) =3D> unsafe { + bindings::spin_unlock_irqrestore((&mut *ptr).deref_mut(), = *flags) + }, + // SAFETY: The safety requirements of this function ensure tha= t `ptr` is valid and that + // the caller is the owner of the mutex. + None =3D> unsafe { bindings::spin_unlock((&mut *ptr).deref_mut= ()) }, + } + } +} + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclus= ion. We use the `irqsave` +// variant of the C lock acquisition functions to disable interrupts and r= etrieve the original +// interrupt state, and the `irqrestore` variant of the lock release funct= ions to restore the state +// in `unlock` -- we use the guard context to determine which method was u= sed to acquire the lock. +unsafe impl super::IrqSaveBackend for SpinLockBackend { + unsafe fn lock_irqsave(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `p= tr` points to valid + // memory, and that it has been initialised before. + Some(unsafe { bindings::spin_lock_irqsave((&mut *ptr).deref_mut())= }) } } --=20 2.40.0 From nobody Wed Feb 11 23:44:39 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4B8D2C7EE22 for ; Wed, 3 May 2023 09:08:04 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230028AbjECJIC (ORCPT ); Wed, 3 May 2023 05:08:02 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45904 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229967AbjECJHj (ORCPT ); Wed, 3 May 2023 05:07:39 -0400 Received: from mail-wm1-x329.google.com (mail-wm1-x329.google.com [IPv6:2a00:1450:4864:20::329]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 129D44EDE for ; Wed, 3 May 2023 02:07:26 -0700 (PDT) Received: by mail-wm1-x329.google.com with SMTP id 5b1f17b1804b1-3f178da21afso32606765e9.1 for ; Wed, 03 May 2023 02:07:26 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104845; x=1685696845; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=n2aCenBwPL0i3vTawCb2L8YinJvZAslW/YJEjXhFLa8=; b=AkqTOFPVsQoS7ielM6GeSVWnFtz9yMESQff8kVfpSdZiy3bdaaWLpuG1t40bS0s7on TDgZIlo0iiLY/gG5xlQwmXkLkeWaeeMQho78fmcfwXr9vbm6f7BCJ5iI1WrR7q3Hq/B8 eriq0v/fAaOJFcSS6bHc2AuiUNCaQsbgtGUGWsqZWxzyvj4YwzKANe2QUZrEgauRlAi0 NXz0GQAAphVn7VtICawZK7usZ2YbQdSVM0vxmLmJsR+gx1YoAZPAl1wyrseet8M/HpaE 0YWtzJTugKmoNajsIAjcEDi5kVkiG0HUiPkLga9N2S03hYveh/RedYTApIvgbetEAu5S 8+zA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104845; x=1685696845; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=n2aCenBwPL0i3vTawCb2L8YinJvZAslW/YJEjXhFLa8=; b=kfBZDrWq7HlkliKvl2FDXG3wUxJYacyF4LOG4bjyExpABgEUOzgjAIuVTvpO32xNIm 3sZUcrRH5fDoPJYQ8K0Q2KUgN1VkJpLb/9ZqFFuqMXpiLvI3EoXiP1twQdHLALpnmFdx fCCPVGGGid8dTECwRM0Az5C0GNAq27CY/wzUS8dshI6rFqacKBX7A/lG7J+B/n7JqtsY lJa/XUuQg1zjrXvle17CIsrLltSDFePz+hOj0iGRsabT6EMa/5Ecckh18h6yl8OgyuGJ C9EucUXwNSj+tIvaSRjhfOHwJgcGWDTAeOQg6/dLY6Rcw6oUyR0aDkRotzV0h600nEZ0 AOgw== X-Gm-Message-State: AC+VfDz5p7xqiqAVBvRf3cqp1JuM7TTRgb7nRX93BVfCBBg7ohDStEvO TEeFhVA+C35R+fM+xnKjzzWeww== X-Google-Smtp-Source: ACHHUZ6TOXRdg3DPcncxdnXc/lCXM7ZZOYLzK+Nmqw1ltFhclc2DCjmvwo383Z/Dy4ERb34WRGJ4yg== X-Received: by 2002:a1c:7916:0:b0:3f1:952c:3c70 with SMTP id l22-20020a1c7916000000b003f1952c3c70mr13719294wme.40.1683104845321; Wed, 03 May 2023 02:07:25 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id g10-20020a7bc4ca000000b003f046ad52efsm1235624wmk.31.2023.05.03.02.07.24 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:25 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 09/11] RUST: implement `ForeignOwnable` for `Pin` Date: Wed, 3 May 2023 11:07:06 +0200 Message-Id: <20230503090708.2524310-10-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Andreas Hindborg Implement `ForeignOwnable for Pin where T: ForeignOwnable + Deref`. Imported from rust tree [1] [1] https://github.com/Rust-for-Linux/linux/tree/bc22545f38d74473cfef3e9fd6= 5432733435b79f Cc: Wedson Almeida Filho --- rust/kernel/types.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 29db59d6119a..98e71e96a7fc 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -9,6 +9,7 @@ use core::{ marker::PhantomData, mem::MaybeUninit, ops::{Deref, DerefMut}, + pin::Pin, ptr::NonNull, }; =20 @@ -100,6 +101,29 @@ impl ForeignOwnable for () { unsafe fn from_foreign(_: *const core::ffi::c_void) -> Self {} } =20 +impl ForeignOwnable for Pin { + type Borrowed<'a> =3D T::Borrowed<'a>; + + fn into_foreign(self) -> *const core::ffi::c_void { + // SAFETY: We continue to treat the pointer as pinned by returning= just a pointer to it to + // the caller. + let inner =3D unsafe { Pin::into_inner_unchecked(self) }; + inner.into_foreign() + } + + unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> Self::Borrowed<= 'a> { + // SAFETY: The safety requirements for this function are the same = as the ones for + // `T::borrow`. + unsafe { T::borrow(ptr) } + } + + unsafe fn from_foreign(p: *const core::ffi::c_void) -> Self { + // SAFETY: The object was originally pinned. + // The passed pointer comes from a previous call to `T::into_forei= gn`. + unsafe { Pin::new_unchecked(T::from_foreign(p)) } + } +} + /// Runs a cleanup function/closure when dropped. /// /// The [`ScopeGuard::dismiss`] function prevents the cleanup function fro= m running. --=20 2.40.0 From nobody Wed Feb 11 23:44:39 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 54459C7EE2A for ; Wed, 3 May 2023 09:08:12 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229803AbjECJIK (ORCPT ); Wed, 3 May 2023 05:08:10 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45952 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229973AbjECJHj (ORCPT ); Wed, 3 May 2023 05:07:39 -0400 Received: from mail-wm1-x330.google.com (mail-wm1-x330.google.com [IPv6:2a00:1450:4864:20::330]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id CF87649EC for ; Wed, 3 May 2023 02:07:28 -0700 (PDT) Received: by mail-wm1-x330.google.com with SMTP id 5b1f17b1804b1-3f1950f5628so47382935e9.3 for ; Wed, 03 May 2023 02:07:28 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104847; x=1685696847; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=NatBW5eIyTK1anrfiMCu2himhC60W7yfvOzay2vGAv8=; b=p4BdIUQKmjerjPm5oD73Ay9h/299qjSSXcgv4Bb+Qmp2g4owzjxFGY/dLkga6OOGno Nr2jHy7hW1KNXt2jFPKbJW4XgnrPwqb0NM4WY1MM+UiU7g1J3uHlVIYp9v2p258ltsrF t7OVqRC2Ti6RK+iAoEbzM+tW8O5ETUmXvIXQ0Nxl6UW1vz8hYm/rqZHDMKt3tb7YvkKb iSqk4PNYf+RSBTYroInYiqq9Vr6T8hmJUsoBUh2dCy7hB3MQazTr8tkOvGxRyX7V7C39 56UYzcYTSvClGccgLv6Ha/V3GAC/brvbNBumsb2Up+/YyAQ1fBFju2Cj4nuQOlnNUGET azOg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104847; x=1685696847; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=NatBW5eIyTK1anrfiMCu2himhC60W7yfvOzay2vGAv8=; b=C4UcpfjM5uxFCfZN3I632bYW8Gl07NlmDWC95eNCbQFYfnFEq4UL0ByrWDOutpaQBJ mNjIeJaHjewu7ZiG3tK/plz05e7r2CkONu0Y5PtXRVv4QGgiY2gzH4HK6/xxnsysFqai cZSFTIEjgm8z3edbrYKobZZzF2AY0VPJieIrGzO1tACM0qOpavaAVTPZBWDV3L/lJXQq BgafV208QLdpFARatAPhwmgCIJZjnCNz8j+SVPLEcIv1jyErUaKXQ9nGpS+2uo3aIYRc juMJyW148d7XYjOHRl6S7+9OKc3s8IBqu4zA5En+3xhquziG+eHedIoNzB4IaLyKVXSg Waaw== X-Gm-Message-State: AC+VfDweEeE7XnGh1MwxSwwJMhWOTzHzQjIbrru00CmVsNjcMN+nr2Hl eiMd0qNGbMFGRE1E9VTpnS1Mow== X-Google-Smtp-Source: ACHHUZ4V+4yXzD+7jMqjGJI2nu+9smDJfF7r7MsYfyR2VAHsFx3pXLD9Y8eoC2R23SPbbiOVCzgrKw== X-Received: by 2002:a1c:4b0c:0:b0:3ee:b3bf:5f7c with SMTP id y12-20020a1c4b0c000000b003eeb3bf5f7cmr14680443wma.23.1683104846390; Wed, 03 May 2023 02:07:26 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id x2-20020a05600c21c200b003f182c11667sm1233908wmj.39.2023.05.03.02.07.25 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:26 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 10/11] rust: add null block driver Date: Wed, 3 May 2023 11:07:07 +0200 Message-Id: <20230503090708.2524310-11-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Andreas Hindborg The driver is implemented entirely using safe Rust. This initial version supports direct completion, read and write requests, blk-mq and optional me= mory backing. Signed-off-by: Andreas Hindborg --- drivers/block/Kconfig | 4 + drivers/block/Makefile | 4 + drivers/block/rnull-helpers.c | 60 ++++++++++++ drivers/block/rnull.rs | 177 ++++++++++++++++++++++++++++++++++ scripts/Makefile.build | 2 +- 5 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 drivers/block/rnull-helpers.c create mode 100644 drivers/block/rnull.rs diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index f79f20430ef7..644ef1bc7574 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -354,6 +354,10 @@ config VIRTIO_BLK This is the virtual block driver for virtio. It can be used with QEMU based VMMs (like KVM or Xen). Say Y or M. =20 +config BLK_DEV_RS_NULL + tristate "Rust null block driver" + depends on RUST + config BLK_DEV_RBD tristate "Rados block device (RBD)" depends on INET && BLOCK diff --git a/drivers/block/Makefile b/drivers/block/Makefile index 101612cba303..cebbeece4bc2 100644 --- a/drivers/block/Makefile +++ b/drivers/block/Makefile @@ -9,6 +9,10 @@ # needed for trace events ccflags-y +=3D -I$(src) =20 +obj-$(CONFIG_BLK_DEV_RS_NULL) +=3D rnull_mod.o +rnull_mod-y :=3D rnull-helpers.o rnull.o +LLVM_LINK_FIX_drivers/block/rnull_mod.o :=3D 1 + obj-$(CONFIG_MAC_FLOPPY) +=3D swim3.o obj-$(CONFIG_BLK_DEV_SWIM) +=3D swim_mod.o obj-$(CONFIG_BLK_DEV_FD) +=3D floppy.o diff --git a/drivers/block/rnull-helpers.c b/drivers/block/rnull-helpers.c new file mode 100644 index 000000000000..826f2773ed93 --- /dev/null +++ b/drivers/block/rnull-helpers.c @@ -0,0 +1,60 @@ + +#include + +__attribute__((always_inline)) +void rust_helper_bio_advance_iter_single(const struct bio *bio, + struct bvec_iter *iter, unsigne= d int bytes) +{ + bio_advance_iter_single(bio, iter, bytes); +} + +__attribute__((always_inline)) void *rust_helper_kmap(struct page *page) +{ + return kmap(page); +} + +__attribute__((always_inline)) void rust_helper_kunmap(struct page *page) +{ + return kunmap(page); +} + +__attribute__((always_inline)) void *rust_helper_kmap_atomic(struct page *= page) +{ + return kmap_atomic(page); +} + +__attribute__((always_inline)) void rust_helper_kunmap_atomic(void* addres= s) +{ + kunmap_atomic(address); +} + +__attribute__((always_inline)) struct page * +rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order) +{ + return alloc_pages(gfp_mask, order); +} + +__attribute__((always_inline)) void rust_helper_spin_lock_irq(spinlock_t *= lock) +{ + spin_lock_irq(lock); +} + +__attribute__((always_inline)) void +rust_helper_spin_unlock_irq(spinlock_t *lock) +{ + spin_unlock_irq(lock); +} +__attribute__((always_inline)) unsigned long +rust_helper_spin_lock_irqsave(spinlock_t *lock) +{ + unsigned long flags; + + spin_lock_irqsave(lock, flags); + + return flags; +} +__attribute__((always_inline)) void +rust_helper_spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) +{ + spin_unlock_irqrestore(lock, flags); +} diff --git a/drivers/block/rnull.rs b/drivers/block/rnull.rs new file mode 100644 index 000000000000..d95025664a60 --- /dev/null +++ b/drivers/block/rnull.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This is a null block driver. It currently supports optional memory bac= king, +//! blk-mq interface and direct completion. The driver is configured at mo= dule +//! load time by parameters `memory_backed` and `capacity_mib`. + +use kernel::{ + bindings, + block::{ + bio::Segment, + mq::{self, GenDisk, Operations, TagSet}, + }, + error::Result, + macros::vtable, + new_mutex, new_spinlock, + pages::Pages, + pr_info, + prelude::*, + radix_tree::RadixTree, + sync::{Arc, Mutex, SpinLock}, + types::ForeignOwnable, +}; + +module! { + type: NullBlkModule, + name: "rs_null_blk", + author: "Andreas Hindborg", + license: "GPL v2", + params: { + memory_backed: bool { + default: true, + permissions: 0, + description: "Use memory backing", + }, + capacity_mib: u64 { + default: 4096, + permissions: 0, + description: "Device capacity in MiB", + }, + }, +} + +struct NullBlkModule { + _disk: Pin>>>, +} + +fn add_disk(tagset: Arc>) -> Result> { + let tree =3D RadixTree::new()?; + let queue_data =3D Box::pin_init(new_spinlock!(tree, "rnullb:mem"))?; + + let disk =3D GenDisk::try_new(tagset, queue_data)?; + disk.set_name(format_args!("rnullb{}", 0))?; + disk.set_capacity(*capacity_mib.read() << 11); + disk.set_queue_logical_block_size(4096); + disk.set_queue_physical_block_size(4096); + disk.set_rotational(false); + Ok(disk) +} + +impl kernel::Module for NullBlkModule { + fn init(_module: &'static ThisModule) -> Result { + pr_info!("Rust null_blk loaded\n"); + // Major device number? + let tagset =3D TagSet::try_new(1, (), 256, 1)?; + let disk =3D Box::pin_init(new_mutex!(add_disk(tagset)?, "nullb:di= sk"))?; + + disk.lock().add()?; + + Ok(Self { _disk: disk }) + } +} + +impl Drop for NullBlkModule { + fn drop(&mut self) { + pr_info!("Dropping rnullb\n"); + } +} + +struct NullBlkDevice; +type Tree =3D kernel::radix_tree::RadixTree>>; +type Data =3D Pin>>; + +impl NullBlkDevice { + #[inline(always)] + fn write(tree: &mut Tree, sector: usize, segment: &Segment<'_>) -> Res= ult { + let idx =3D sector >> 3; // TODO: PAGE_SECTOR_SHIFT + let mut page =3D if let Some(page) =3D tree.get_mut(idx as u64) { + page + } else { + tree.try_insert(idx as u64, Box::try_new(Pages::new()?)?)?; + tree.get_mut(idx as u64).unwrap() + }; + + segment.copy_to_page_atomic(&mut page)?; + + Ok(()) + } + + #[inline(always)] + fn read(tree: &mut Tree, sector: usize, segment: &mut Segment<'_>) -> = Result { + let idx =3D sector >> 3; // TODO: PAGE_SECTOR_SHIFT + if let Some(page) =3D tree.get(idx as u64) { + segment.copy_from_page_atomic(page)?; + } + + Ok(()) + } + + #[inline(never)] + fn transfer( + command: bindings::req_op, + tree: &mut Tree, + sector: usize, + segment: &mut Segment<'_>, + ) -> Result { + match command { + bindings::req_op_REQ_OP_WRITE =3D> Self::write(tree, sector, s= egment)?, + bindings::req_op_REQ_OP_READ =3D> Self::read(tree, sector, seg= ment)?, + _ =3D> (), + } + Ok(()) + } +} + +#[vtable] +impl Operations for NullBlkDevice { + type RequestData =3D (); + type QueueData =3D Data; + type HwData =3D (); + type TagSetData =3D (); + + fn new_request_data( + _tagset_data: ::Borrowed<'_>, + ) -> Result { + Ok(()) + } + + #[inline(always)] + fn queue_rq( + _hw_data: ::Borrowed<'_>, + queue_data: ::Borrowed<'_>, + rq: &mq::Request, + _is_last: bool, + ) -> Result { + rq.start(); + if *memory_backed.read() { + let mut tree =3D queue_data.lock_irqsave(); + + let mut sector =3D rq.sector(); + for bio in rq.bio_iter() { + for mut segment in bio.segment_iter() { + let _ =3D Self::transfer(rq.command(), &mut tree, sect= or, &mut segment); + sector +=3D segment.len() >> 9; // TODO: SECTOR_SHIFT + } + } + } + rq.end_ok(); + Ok(()) + } + + fn commit_rqs( + _hw_data: ::Borrowed<'_>, + _queue_data: ::Borrowed<'_>, + ) { + } + + fn complete(_rq: &mq::Request) { + //rq.end_ok(); + } + + fn init_hctx( + _tagset_data: ::Borrowed<'_>, + _hctx_idx: u32, + ) -> Result { + Ok(()) + } +} diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 9f94fc83f086..94127fc3cf77 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -277,7 +277,7 @@ $(obj)/%.lst: $(src)/%.c FORCE # Compile Rust sources (.rs) # ------------------------------------------------------------------------= --- =20 -rust_allowed_features :=3D core_ffi_c,explicit_generic_args_with_impl_trai= t,new_uninit,pin_macro +rust_allowed_features :=3D core_ffi_c,explicit_generic_args_with_impl_trai= t,new_uninit,pin_macro,allocator_api =20 rust_common_cmd =3D \ RUST_MODFILE=3D$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \ --=20 2.40.0 From nobody Wed Feb 11 23:44:39 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id ED675C77B78 for ; Wed, 3 May 2023 09:08:24 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229519AbjECJIX (ORCPT ); Wed, 3 May 2023 05:08:23 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:46322 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S230004AbjECJHy (ORCPT ); Wed, 3 May 2023 05:07:54 -0400 Received: from mail-wm1-x330.google.com (mail-wm1-x330.google.com [IPv6:2a00:1450:4864:20::330]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id EF5574ED7 for ; Wed, 3 May 2023 02:07:29 -0700 (PDT) Received: by mail-wm1-x330.google.com with SMTP id 5b1f17b1804b1-3f1950f5676so50210595e9.3 for ; Wed, 03 May 2023 02:07:29 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104848; x=1685696848; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=oHrN32lBaAC78cOIMsVaenJPrzKJjknoApgieLS2EVM=; b=oA2PVI7kXpD5OIbFur9s7uaOC1Nh/L72pwK/S62AuuQQaQp4WS6rCh6e1Qiu311wQM i+tmijmYwryRpAB+BQXDD6we65meeLaoR7RiUgcCodh6T1WdeG3rXHU3aZ7RyUOXMl3G TpBB2cGgmPdRWJt4X4inUYYm8i6bJcsAq9WphlZqTjKnuQu0J21VXA/d2UN7hiTp+25Y oBFKxT3L7/ViA4VeKCx5UI0YHrZqbgbOXKdIaytdI7oCcrlH1MOqviwi9jouVUUIxAd6 bxkaj0/ygyyrL74KdQ1C3LsOjbhlUrt5uX0Z8rNBRK89XnyS3Fh7uD9pLe/0c96ZDAgH tviw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104848; x=1685696848; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=oHrN32lBaAC78cOIMsVaenJPrzKJjknoApgieLS2EVM=; b=KzphisuLEcRclMB6+xTWlDPMwybULwS6iq9yyPiCalxDSE25y27Y2IT2J2r4Q0LJyo hRkiEshUifdaEBUbf1nW62rkGr1TLi7SHxMhL/KHamlIvrLx74iPcJM9znxRpESgDtMc YJ1lG7OOMnH8mdU7FqsOxgBEpkZDJaHLOXQdLHQRQcn1jSIax7t/V8Z6VsBW8QkkGj4L BzU4IIjoLXmksLoFgWJUA4xufTecsjeRAdaqBqlv2y8D6mkp3FoqTXEi8NyoI7NF1Tqb 9fLeg6S+QqhnZsSJVZme6bIhRnWf0i1yJZk27/wYV8k8GB1MQOxdgJUlt1MEE+hfORof b58g== X-Gm-Message-State: AC+VfDx1GS4VOJh7UImD9N7t8aFbF97J87RR7tvf7Ajz9VVhfaC3/Qj/ 8J6q3ZXndDZ7COG4eUCEgFaGXA== X-Google-Smtp-Source: ACHHUZ7P6M/jU8n50FGEjgTSgEYAcUdKtRM+n+M8XLe9UPXqN9jUX6WcfyARZKzQxoFqI3CdjYYuxQ== X-Received: by 2002:a05:6000:11c5:b0:2fb:600e:55bd with SMTP id i5-20020a05600011c500b002fb600e55bdmr15479168wrx.39.1683104848425; Wed, 03 May 2023 02:07:28 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id e22-20020a5d5956000000b003012030a0c6sm33106775wri.18.2023.05.03.02.07.27 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:28 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 11/11] rust: inline a number of short functions Date: Wed, 3 May 2023 11:07:08 +0200 Message-Id: <20230503090708.2524310-12-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Andreas Hindborg The rust compiler will not inline functions that live in vmlinux when building modules. Add inline directives to these short functions to ensure = that they are inlined when building modules. Signed-off-by: Andreas Hindborg --- rust/kernel/sync/lock.rs | 2 ++ rust/kernel/sync/lock/mutex.rs | 2 ++ rust/kernel/sync/lock/spinlock.rs | 2 ++ rust/kernel/types.rs | 6 ++++++ 4 files changed, 12 insertions(+) diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index bb21af8a8377..4bfc2f5d9841 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -101,6 +101,7 @@ impl Lock { /// Before acquiring the lock, it disables interrupts. When the guard = is dropped, the interrupt /// state (either enabled or disabled) is restored to its state before /// [`lock_irqsave`](Self::lock_irqsave) was called. + #[inline(always)] pub fn lock_irqsave(&self) -> Guard<'_, T, B> { // SAFETY: The constructor of the type calls `init`, so the existe= nce of the object proves // that `init` was called. @@ -210,6 +211,7 @@ impl core::ops::DerefMut for Gua= rd<'_, T, B> { } =20 impl Drop for Guard<'_, T, B> { + #[inline(always)] fn drop(&mut self) { // SAFETY: The caller owns the lock, so it is safe to unlock it. unsafe { B::unlock(self.lock.state.get(), &self.state) }; diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 923472f04af4..5e8096811b98 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -104,12 +104,14 @@ unsafe impl super::Backend for MutexBackend { unsafe { bindings::__mutex_init(ptr, name, key) } } =20 + #[inline(always)] unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { // SAFETY: The safety requirements of this function ensure that `p= tr` points to valid // memory, and that it has been initialised before. unsafe { bindings::mutex_lock(ptr) }; } =20 + #[inline(always)] unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardStat= e) { // SAFETY: The safety requirements of this function ensure that `p= tr` is valid and that the // caller is the owner of the mutex. diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index 50b8775bb49d..23a973dab85c 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -122,6 +122,7 @@ unsafe impl super::Backend for SpinLockBackend { None } =20 + #[inline(always)] unsafe fn unlock(ptr: *mut Self::State, guard_state: &Self::GuardState= ) { match guard_state { // SAFETY: The safety requirements of this function ensure tha= t `ptr` is valid and that @@ -141,6 +142,7 @@ unsafe impl super::Backend for SpinLockBackend { // interrupt state, and the `irqrestore` variant of the lock release funct= ions to restore the state // in `unlock` -- we use the guard context to determine which method was u= sed to acquire the lock. unsafe impl super::IrqSaveBackend for SpinLockBackend { + #[inline(always)] unsafe fn lock_irqsave(ptr: *mut Self::State) -> Self::GuardState { // SAFETY: The safety requirements of this function ensure that `p= tr` points to valid // memory, and that it has been initialised before. diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 98e71e96a7fc..7be1f64bbde9 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -70,10 +70,12 @@ pub trait ForeignOwnable: Sized { impl ForeignOwnable for Box { type Borrowed<'a> =3D &'a T; =20 + #[inline(always)] fn into_foreign(self) -> *const core::ffi::c_void { Box::into_raw(self) as _ } =20 + #[inline(always)] unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> &'a T { // SAFETY: The safety requirements for this function ensure that t= he object is still alive, // so it is safe to dereference the raw pointer. @@ -82,6 +84,7 @@ impl ForeignOwnable for Box { unsafe { &*ptr.cast() } } =20 + #[inline(always)] unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { // SAFETY: The safety requirements of this function ensure that `p= tr` comes from a previous // call to `Self::into_foreign`. @@ -92,12 +95,15 @@ impl ForeignOwnable for Box { impl ForeignOwnable for () { type Borrowed<'a> =3D (); =20 + #[inline(always)] fn into_foreign(self) -> *const core::ffi::c_void { core::ptr::NonNull::dangling().as_ptr() } =20 + #[inline(always)] unsafe fn borrow<'a>(_: *const core::ffi::c_void) -> Self::Borrowed<'a= > {} =20 + #[inline(always)] unsafe fn from_foreign(_: *const core::ffi::c_void) -> Self {} } =20 --=20 2.40.0