From nobody Sun May 24 19:33:53 2026 Received: from mail-wr1-f73.google.com (mail-wr1-f73.google.com [209.85.221.73]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 02265322522 for ; Sat, 23 May 2026 07:31:11 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.221.73 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779521473; cv=none; b=g8HsWSldBx3JZEa2mMzupknAhR/mzDw3a0/gF1uptReUEUXMLV3YO6MdLaGgveTVRbjWaRoR59dSl8fQ0Jzguq0CfXTIi1Y0NKetIRQCQAWavXnutkYRuP2de2uhpJtW0Jq4sizYET41AMfhZsE+Hv7rhH4c4rgmkEA8ayA1rG0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779521473; c=relaxed/simple; bh=tyhQG6r+bBgYPrhfWVw7lrzJDNjbNm777mEFaeqoR9c=; h=Date:In-Reply-To:Mime-Version:References:Message-ID:Subject:From: To:Cc:Content-Type; b=pL+6qVMsy2XCKVvX0/nroE04L/vYaoGEQxdJc8oT+/e8rPKoBIeH3FLwBhKTq6lYY08IJXi+walGqZ5I3NqKYGtAp0PwJndVHJizuemDSKSkVdaiWVrWQel+0sqIDgH8/0YSEw0ZsUn5Vxqk9FO3WDPVXb1/HcBB7gI/g/Ssxuk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b=V8+zHTvb; arc=none smtp.client-ip=209.85.221.73 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b="V8+zHTvb" Received: by mail-wr1-f73.google.com with SMTP id ffacd0b85a97d-44d79da8cf7so6357491f8f.2 for ; Sat, 23 May 2026 00:31:11 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20251104; t=1779521470; x=1780126270; darn=vger.kernel.org; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:from:to:cc:subject:date:message-id:reply-to; bh=Id6aSpScy4pFl4oZ61cb6pWcP62ojgwGrghEMv9dsLE=; b=V8+zHTvbq6GdEGTQ9mlN2d1rWQ905CSmQuQKF74jj0Y2TcoGWHvSJkc4gdTMn+Kzq7 XvB/630Y2Z7e2b3blkmbA1nU+RKx94aAPrbf9URagWWDmRdz+ZHWJ+m5PWrKc8Dm5jWu JBey3JtHSjHQuafHMmGkF1A9QRVwiexA/8Kh+05R2xWy3RlNxo0mFoWZeHO37XvjZLna h6e79HJCEnDM29aHYCqbgxwN27IMMgWZaSfq5N8NM9pu1I2rYUirR+ztEdWjrI4920nl S65EOujd6qJ3jv1dxF/wJAdPhYpeO8sA+E85AijgbhEPWJ8Xm+9d2vbXUzLurvzBR9UF qtaA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20251104; t=1779521470; x=1780126270; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=Id6aSpScy4pFl4oZ61cb6pWcP62ojgwGrghEMv9dsLE=; b=efQfXf4BPjnegWw1D6Lae0WfgXHGnALwddAbMop+rCr0i05JTGzLn9+eRBC866zFSL RnfN0sdmpm2cqyAUFghT2yFMPS5ht2iZEd0O5UhwRDe0oLZ4vgFaSLOMptIWV3TpScNM KE2hkwu9Lu13V0JSCemrcBlHLNzQBxUX/it4C9G6S4BojxDEd0KQpiP9nijKb9ruXWEo BQxdVWuyFUAAQLSXCJ6MWJjQx9afQBVUCwpaKPqAhriFy2F8OUPEvAlQ3jOPLfwgSu8l XPuQWspQ3TM/Wf6vsnTvaHI7Juikcy3G47plpLfdY1nuCOE7+lBLJb/JqCdMiR2iYsJk TgbA== X-Forwarded-Encrypted: i=1; AFNElJ93rzU+6bva3mJf62e90Ke98fR9iOZzTE4Rswlnk4Efw1nmQ1IBmmmQZFiyNEC9sNNNOJMk0XNih0K2hV0=@vger.kernel.org X-Gm-Message-State: AOJu0YyhCsvm7iAvQoVZsjTQCLqrRJQfaPOQ+3lgragWa68HO6+EH2Lz C5wlamhdNd3DoOWK9O17LuIGEMkfG+wDGNR8npTZv65SIMvu6MDDGXYJrEPFWG9FYmhaayBOHsa tTJ4bLdSwFz9+B3O1Og== X-Received: from wmte15.prod.google.com ([2002:a05:600c:8b2f:b0:486:fe68:2045]) (user=aliceryhl job=prod-delivery.src-stubby-dispatcher) by 2002:a05:600c:4510:b0:48f:d5d7:df63 with SMTP id 5b1f17b1804b1-490428dfd36mr99366585e9.27.1779521470046; Sat, 23 May 2026 00:31:10 -0700 (PDT) Date: Sat, 23 May 2026 07:30:59 +0000 In-Reply-To: <20260523-upgrade-poll-v4-0-f5b4c747eac2@google.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 References: <20260523-upgrade-poll-v4-0-f5b4c747eac2@google.com> X-Developer-Key: i=aliceryhl@google.com; a=openpgp; fpr=49F6C1FAA74960F43A5B86A1EE7A392FDE96209F X-Developer-Signature: v=1; a=openpgp-sha256; l=3877; i=aliceryhl@google.com; h=from:subject:message-id; bh=tyhQG6r+bBgYPrhfWVw7lrzJDNjbNm777mEFaeqoR9c=; b=owEBbQKS/ZANAwAKAQRYvu5YxjlGAcsmYgBqEVe6bHJnNgMPAxm+EAe7NpXfylUzA44aWIVef Xl7yHKroqCJAjMEAAEKAB0WIQSDkqKUTWQHCvFIvbIEWL7uWMY5RgUCahFXugAKCRAEWL7uWMY5 RvrVD/95qLsXw74/ELP3qn9OVixefPlxg0gMZKyBMEL3ZN5KKL6yqYahZuljsr/0tAGnsE7bqDm S2NvqHbfeXxUFx99ynzbp1Qojx6JVFgwokGZWkzy0opOU7lCSpwIfgZKaJ+IRJZU5JDx2wsVpP9 SDRVomYe/ukKGEXv/JK2NO2I7b2AdY8G4CTlFXE8tpTvFDWcVO6YjMaaU9EXrpwAidhiaAc3wyh EGeZpnCXDv2BsLOp2uIGBYN5ReD8++8S2RCPhcN86YkBCEYvoJLLkQbRxc1rCAp4EGqaFEdS9CL /oSthZ3NCl8R+Qc9okdXIoPBzwzVtP6cVo5L1XiMKQC9s5bxFvnK94dvOpB5vK4wdZHg/lOyF7z oyxdwI1wQaAXTcwWOI7RzuzY6e8+voVZ+N534cCRnkHtFOifacWmCOboPmzLozVLw3onLbliSTX mErlBiUhEEoCnL0adE1+h4jyhIdAZaOMYuTVQE8OrtuHH1VI/wpxxqehDk541Al+9zmFMI63D8L jg3p5fM+MFgUSUWruTJNnlyKAPomd8ucBS/7wFN0ORDKvq6e/Teydyk3edjM55lYcgeO5sOsDNe Y4+y1mKNTN+S/x+zGkYwIIpQVqrig1WYIfzYL38PAnJYYRhIKK2eyKpbwrvAqGWg/8pb+9w+g4g MeoJZ4IVIsuHz6g== X-Mailer: b4 0.14.3 Message-ID: <20260523-upgrade-poll-v4-1-f5b4c747eac2@google.com> Subject: [PATCH v4 1/2] rust: poll: use kfree_rcu() for PollCondVar From: Alice Ryhl To: Greg Kroah-Hartman , Carlos Llamas , Christian Brauner , Boqun Feng Cc: "Paul E. McKenney" , Alexander Viro , Jan Kara , Miguel Ojeda , Boqun Feng , "=?utf-8?q?Bj=C3=B6rn_Roy_Baron?=" , Benno Lossin , Andreas Hindborg , Trevor Gross , Danilo Krummrich , Gary Guo , linux-fsdevel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-kernel@vger.kernel.org, Alice Ryhl Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Rust Binder currently uses PollCondVar, but it calls synchronize_rcu() in the destructor, which we would like to avoid. Add a variation of PollCondVar that kfree_rcu() instead. One could avoid the `rcu` field and allocate the rcu_head on drop using a fallback to synchronize_rcu() on ENOMEM. However, I'd prefer to avoid the potential for synchronize_rcu(), and Binder will only use this for a small fraction of processes, so even if it changes which kmalloc bucket it falls into, the extra memory is not a problem. Signed-off-by: Alice Ryhl --- rust/kernel/sync/poll.rs | 73 ++++++++++++++++++++++++++++++++++++++++++++= +++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index 0ec985d560c8..684dfa242b1a 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -5,12 +5,18 @@ //! Utilities for working with `struct poll_table`. =20 use crate::{ + alloc::AllocError, bindings, fs::File, prelude::*, sync::{CondVar, LockClassKey}, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, // }; -use core::{marker::PhantomData, ops::Deref}; =20 /// Creates a [`PollCondVar`] initialiser with the given name and a newly-= created lock class. #[macro_export] @@ -66,6 +72,7 @@ pub fn register_wait(&self, file: &File, cv: &PollCondVar= ) { /// /// [`CondVar`]: crate::sync::CondVar #[pin_data(PinnedDrop)] +#[repr(transparent)] pub struct PollCondVar { #[pin] inner: CondVar, @@ -104,3 +111,67 @@ fn drop(self: Pin<&mut Self>) { unsafe { bindings::synchronize_rcu() }; } } + +/// A [`KBox`] that uses `kfree_rcu`. +/// +/// [`KBox`]: PollCondVar +pub struct PollCondVarBox { + inner: ManuallyDrop>>, +} + +#[pin_data] +#[repr(C)] +struct PollCondVarBoxInner { + #[pin] + inner: PollCondVar, + rcu: Opaque, +} + +// SAFETY: PollCondVar is Send +unsafe impl Send for PollCondVarBoxInner {} +// SAFETY: PollCondVar is Sync +unsafe impl Sync for PollCondVarBoxInner {} + +impl PollCondVarBox { + /// Constructs a new boxed [`PollCondVar`]. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Re= sult { + let b =3D KBox::pin_init( + pin_init!(PollCondVarBoxInner { + inner <- PollCondVar::new(name, key), + rcu: Opaque::uninit(), + }), + GFP_KERNEL, + ) + .map_err(|_| AllocError)?; + + Ok(PollCondVarBox { + inner: ManuallyDrop::new(b), + }) + } +} + +impl Deref for PollCondVarBox { + type Target =3D PollCondVar; + fn deref(&self) -> &PollCondVar { + &self.inner.inner + } +} + +impl Drop for PollCondVarBox { + #[inline] + fn drop(&mut self) { + // SAFETY: ManuallyDrop::take ok because not already taken. + let boxed =3D unsafe { ManuallyDrop::take(&mut self.inner) }; + + // SAFETY: The code below frees the box without calling the actual= destructor of the type, + // but it's okay because it re-implements the destructor using `kf= ree_rcu()` in place of + // `synchronize_rcu()`. + let ptr =3D KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxe= d) }); + + // SAFETY: The pointer points at a valid `wait_queue_head`. + unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queu= e_head.get()) }; + + // SAFETY: This was allocated using `KBox::pin_init`, so it can be= freed with `kvfree`. + unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::()) }; + } +} --=20 2.54.0.746.g67dd491aae-goog From nobody Sun May 24 19:33:53 2026 Received: from mail-wm1-f74.google.com (mail-wm1-f74.google.com [209.85.128.74]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7464634844C for ; Sat, 23 May 2026 07:31:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.128.74 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779521475; cv=none; b=njhe04x4UqxTW8ti/Er6o4Rx8Aojmr/vEJpph3Xsoiq/886BHRyQLFF1tC7J/U4tZWdH9tGaoMiILoWlBf5eQeGjz6pk/2NoTkn9mcMHIe3kFgt76aCcv9ipo80WAxyaY5nxtRmmqnpP1vDofWjTtd6nyd6EIE9i32kc/paic1w= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779521475; c=relaxed/simple; bh=yFY7UOU204U56FFouEe4824rGLY/b4dnz9zmw97BTnA=; h=Date:In-Reply-To:Mime-Version:References:Message-ID:Subject:From: To:Cc:Content-Type; b=XCZcpSYpVIM6a7NAF7Oa2X9k2RyH1QFWf9TzxyDcNjf757OvNIfRGIj5jOekk5uGwDeNepstxCdmgfwvComf9CMa20bBnZEr7d9QDDBscQB+VlqmHOB6WvLvthK/eiCpjUkWmaRQKtcUThNGiiUnN5t5C0C8t/uAh+3XvqerHV0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b=lk0GhpYB; arc=none smtp.client-ip=209.85.128.74 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b="lk0GhpYB" Received: by mail-wm1-f74.google.com with SMTP id 5b1f17b1804b1-48fe40b61a3so51898485e9.3 for ; Sat, 23 May 2026 00:31:13 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20251104; t=1779521472; x=1780126272; darn=vger.kernel.org; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:from:to:cc:subject:date:message-id:reply-to; bh=xPMNyVxj4vwISQchDGotmmLEW6H9drbn28Eb6PwFhzQ=; b=lk0GhpYBKU+DDFCft8aJYKxA9inIzs4C8XDOy+w+KIlVPLBx0ROnIWq/ln3iAIuVpR S3PaBmsiXzTBugPigB/jAvHz5Jo56wbizSfsHpdufUMuj+G9uQXXVpuCFTJJM13G1Xwt pYXiZGuAML42+pwnBUnxrv0EURHqGpW0au8yUN+zjprJH4b8/diIVbaa5a8vjkW2Gd+V Fk4tZR4+ygGAYEH7xhHaYpZg/5Vq3e2egDGBawUsyLHYFoXh/enYzYb5IQ7GbmcFSx/B 6W8FbRwdZmrxgKWSR9PaUUt787XTDBPIVmHWTiQ+77UhEw1PhPdSdBH4MYoj1RriTLSi Uwhw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20251104; t=1779521472; x=1780126272; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=xPMNyVxj4vwISQchDGotmmLEW6H9drbn28Eb6PwFhzQ=; b=R4ynZVGITMpnHqao9NSVfU7D2UGo0nqR4kQHT51WHni/8jB5wVXOzcOVFO+EX6MrMt 2Zm4QUWTX9ghGhlVbyZMv3jsGlMY0patYPK9qxtCnylvUSHiYF3hmqN2Tb/fr1RSjEkO pGHXRHB3Nq75ZOQUKAnlrP52XO/3qJFRI3y/MbT/NT6HYRzEpLYLiD7XYRS6GaK1DZeQ CbYi+sJ/gYUpBnA4/hJbzLQAIyYISt+MJWorvd+ukTEuwuRG0sJnzPWQJWGtxK7+xWvt 9updRNz2JgHqzHCWSA+WrWOpoI8T+C4sFbWsKdFwO0mHmXS5XGs61wBwiU/myXKzpfhX 7EYg== X-Forwarded-Encrypted: i=1; AFNElJ8UiClOHBLE7BHvgH9zIuMIz4QWgkU5laqIZHsT7xbT7EvtJx+p2oLrpR4tfsLRHg933fqRbYkdfFUAwMs=@vger.kernel.org X-Gm-Message-State: AOJu0YzEKb1OuZrnUiKLihWRCNUgAiiWJkAhBfmnB0OnPiIkX3vw97Cx b5B16mZNH8TBZcq2qClt0Vffsz4hlAeF5sG5C8D7Hplj/4LrczvW6XwRD7NHmMlrHTM8e2bgij8 qBIGXLOmBik/+yUSEJQ== X-Received: from wmbet15.prod.google.com ([2002:a05:600c:818f:b0:490:3dc3:e5bb]) (user=aliceryhl job=prod-delivery.src-stubby-dispatcher) by 2002:a05:600c:a402:b0:48f:e6de:1cba with SMTP id 5b1f17b1804b1-490426c5b3emr86380125e9.17.1779521471795; Sat, 23 May 2026 00:31:11 -0700 (PDT) Date: Sat, 23 May 2026 07:31:00 +0000 In-Reply-To: <20260523-upgrade-poll-v4-0-f5b4c747eac2@google.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 References: <20260523-upgrade-poll-v4-0-f5b4c747eac2@google.com> X-Developer-Key: i=aliceryhl@google.com; a=openpgp; fpr=49F6C1FAA74960F43A5B86A1EE7A392FDE96209F X-Developer-Signature: v=1; a=openpgp-sha256; l=18003; i=aliceryhl@google.com; h=from:subject:message-id; bh=yFY7UOU204U56FFouEe4824rGLY/b4dnz9zmw97BTnA=; b=owEBbQKS/ZANAwAKAQRYvu5YxjlGAcsmYgBqEVe66TuRjeZf9MT3ej4PCWEaGa9KhbQJSVves ATN/ZYsJ/KJAjMEAAEKAB0WIQSDkqKUTWQHCvFIvbIEWL7uWMY5RgUCahFXugAKCRAEWL7uWMY5 Ri6cD/9cOgKS4ddRZ9YhzSd76bR1laJ2jgl7B3dh0Zo5TzSpdTN9nngN4UqqPKr3vXxlrwNxmvs diJiH4JEu5nmAn4xk46HkNjcvcdFnIsfx1lkyTOEefmx0qC48/ZSTNEg2TNs6DDSKFYIgIexJGH ZhOQ8BGiFVJ7YjHwaHc0MSZu5DrjLMCbWc9PNmDSV1XheIxrv3AvO505KipZseo/J2lILD3GA82 NX4vH48eIEiWV4r1ngA4zxyp/JlriQcNu+381vpd44oPXSJTyUrv6b36C2nIYpqW/2P1QyVINyJ /Rheqh4eX3d0hU6OG2/lgLYDgQmk4kGtYYEX9SwOR61gB89Om27xIIrxI4D4R/ThxHcsc04o/Qh PWZHXrF1QLPLFRoOtJFFDRaF1AZMQlSPTSPJOicMC08jkKizU1lLfEXwBOP/8DgNdS+U5wd8snA WTxF/IWVEaZ4DP+Gq1loPEU0YhaWcQk75marhr2GBn9Vsr67Cu+HcjirAj/qepK4dVXeQGZvyYn sS6LqxXpt5KfqvaAL0X/T1o+8XDqcObblWP406z/RaAMO2ytTsYkIqSNoMpBOJADLgGusf9YVVX nnbEaEHz3m2M8BouOdbrWdA+yAuSXfQGK/Ay1xUW9wdtn12ZIHmhqt6cZuo3zeVqiRR0LkM1128 ktXzH0vY7foUwog== X-Mailer: b4 0.14.3 Message-ID: <20260523-upgrade-poll-v4-2-f5b4c747eac2@google.com> Subject: [PATCH v4 2/2] rust_binder: move (e)poll wait queue to Process From: Alice Ryhl To: Greg Kroah-Hartman , Carlos Llamas , Christian Brauner , Boqun Feng Cc: "Paul E. McKenney" , Alexander Viro , Jan Kara , Miguel Ojeda , Boqun Feng , "=?utf-8?q?Bj=C3=B6rn_Roy_Baron?=" , Benno Lossin , Andreas Hindborg , Trevor Gross , Danilo Krummrich , Gary Guo , linux-fsdevel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-kernel@vger.kernel.org, Alice Ryhl Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Most processes do not use Rust Binder with epoll, so avoid paying the synchronize_rcu() cost in drop for those that don't need it. For those that do, we also manage to replace synchronize_rcu() with kfree_rcu(), though we introduce an extra allocation. In case the last ref to an Arc is dropped outside of deferred_release(), this also ensures that synchronize_rcu() is not called in destructor of Arc in other places. Theoretically that could lead to jank by making other syscalls slow, which would be problematic. Signed-off-by: Alice Ryhl --- drivers/android/binder/node.rs | 4 +- drivers/android/binder/process.rs | 74 +++++++++++++++++++++++++------= -- drivers/android/binder/thread.rs | 78 +++++++++++++++++--------------= ---- drivers/android/binder/transaction.rs | 6 ++- 4 files changed, 102 insertions(+), 60 deletions(-) diff --git a/drivers/android/binder/node.rs b/drivers/android/binder/node.rs index 69f757ff7461..865be0fe72c3 100644 --- a/drivers/android/binder/node.rs +++ b/drivers/android/binder/node.rs @@ -536,7 +536,7 @@ pub(crate) fn submit_oneway( inner.oneway_todo.push_back(transaction); } else { inner.has_oneway_transaction =3D true; - guard.push_work(transaction)?; + guard.push_work(&self.owner, transaction)?; } Ok(()) } @@ -568,7 +568,7 @@ pub(crate) fn pending_oneway_finished(&self) { let transaction =3D inner.oneway_todo.pop_front(); inner.has_oneway_transaction =3D transaction.is_some(); if let Some(transaction) =3D transaction { - match guard.push_work(transaction) { + match guard.push_work(&self.owner, transaction) { Ok(()) =3D> {} Err((_err, work)) =3D> { // Process is dead. diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/pro= cess.rs index 820cbd541435..6a1669fa8a78 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -30,7 +30,15 @@ sync::{ aref::ARef, lock::{spinlock::SpinLockBackend, Guard}, - Arc, ArcBorrow, CondVar, CondVarTimeoutResult, Mutex, SpinLock, Un= iqueArc, + poll::PollCondVarBox, + Arc, + ArcBorrow, + CondVar, + CondVarTimeoutResult, + Mutex, + SetOnce, + SpinLock, + UniqueArc, // }, task::Task, uaccess::{UserSlice, UserSliceReader}, @@ -172,21 +180,26 @@ fn new() -> Self { /// taken while holding the inner process lock. pub(crate) fn push_work( &mut self, + proc: &Process, work: DLArc, ) -> Result<(), (BinderError, DLArc)> { + let sync =3D work.should_sync_wakeup(); + // Try to find a ready thread to which to push the work. if let Some(thread) =3D self.ready_threads.pop_front() { // Push to thread while holding state lock. This prevents the = thread from giving up // (for example, because of a signal) when we're about to deli= ver work. - match thread.push_work(work) { + match thread.push_work_inner(work, sync) { PushWorkRes::Ok =3D> Ok(()), + PushWorkRes::OkNotifyPoll =3D> { + proc.notify_poll(sync); + Ok(()) + } PushWorkRes::FailedDead(work) =3D> Err((BinderError::new_d= ead(), work)), } } else if self.is_dead { Err((BinderError::new_dead(), work)) } else { - let sync =3D work.should_sync_wakeup(); - // Didn't find a thread waiting for proc work; this can happen // in two scenarios: // 1. All threads are busy handling transactions @@ -194,17 +207,12 @@ pub(crate) fn push_work( // the kernel driver soon and pick up this work. // 2. Threads are using the (e)poll interface, in which case // they may be blocked on the waitqueue without having been - // added to waiting_threads. For this case, we just iterate - // over all threads not handling transaction work, and - // wake them all up. We wake all because we don't know whet= her - // a thread that called into (e)poll is handling non-binder - // work currently. + // added to waiting_threads. For this case, we wake it up + // directly. self.work.push_back(work); =20 // Wake up polling threads, if any. - for thread in self.threads.values() { - thread.notify_if_poll_ready(sync); - } + proc.notify_poll(sync); =20 Ok(()) } @@ -227,11 +235,11 @@ pub(crate) fn update_node_refcount( =20 // If we decided that we need to push work, push either to the pro= cess or to a thread if // one is specified. - if let Some(node) =3D push { + if let Some(pnode) =3D push { if let Some(thread) =3D othread { - thread.push_work_deferred(node); + thread.push_work_deferred(pnode); } else { - let _ =3D self.push_work(node); + let _ =3D self.push_work(&node.owner, pnode); // Nothing to do: `push_work` may fail if the process is d= ead, but that's ok as in // that case, it doesn't care about the notification. } @@ -457,6 +465,12 @@ pub(crate) struct Process { #[pin] node_refs: Mutex, =20 + // Synchronizes `register_wait` calls to the `PollCondVarBox`. + // + // The `PollCondVarBox` is not stored here because synchronization is + // done for `register_wait` only. Wakeups do not take this lock. + poll: SetOnce, + // Work node for deferred work item. #[pin] defer_work: Work, @@ -516,6 +530,7 @@ fn new(ctx: Arc, cred: ARef) -> Re= sult> { defer_work <- kernel::new_work!("Process::defer_work"), links <- ListLinks::new(), stats: BinderStats::new(), + poll: SetOnce::new(), }), GFP_KERNEL, )?; @@ -715,7 +730,7 @@ fn get_current_thread(self: ArcBorrow<'_, Self>) -> Res= ult> { =20 pub(crate) fn push_work(&self, work: DLArc) -> Bind= erResult { // If push_work fails, drop the work item outside the lock. - let res =3D self.inner.lock().push_work(work); + let res =3D self.inner.lock().push_work(self, work); match res { Ok(()) =3D> Ok(()), Err((err, work)) =3D> { @@ -994,7 +1009,7 @@ pub(crate) fn inc_ref_done(&self, reader: &mut UserSli= ceReader, strong: bool) -> if let Ok(Some(node)) =3D inner.get_existing_node(ptr, cookie) { if let Some(node) =3D node.inc_ref_done_locked(strong, &mut in= ner) { // This only fails if the process is dead. - let _ =3D inner.push_work(node); + let _ =3D inner.push_work(self, node); } } Ok(()) @@ -1505,6 +1520,15 @@ pub(crate) fn ioctl_freeze(&self, info: &BinderFreez= eInfo) -> Result { } } } + + pub(crate) fn notify_poll(&self, sync: bool) { + if let Some(poll) =3D self.poll.as_ref() { + if sync { + poll.notify_sync(); + } + poll.notify_all(); + } + } } =20 fn get_frozen_status(data: UserSlice) -> Result { @@ -1696,7 +1720,21 @@ pub(crate) fn poll( table: PollTable<'_>, ) -> Result { let thread =3D this.get_current_thread()?; - let (from_proc, mut mask) =3D thread.poll(file, table); + { + let poll =3D loop { + if let Some(poll) =3D this.poll.as_ref() { + break poll; + } + + let poll =3D PollCondVarBox::new(c"Process::poll", kernel:= :static_lock_class!())?; + // Reuse our existing mutex to synchronize callers initial= izing. + let _guard =3D this.node_refs.lock(); + this.poll.populate(poll); + }; + + table.register_wait(file, poll); + } + let (from_proc, mut mask) =3D thread.poll()?; if mask =3D=3D 0 && from_proc && !this.inner.lock().work.is_empty(= ) { mask |=3D bindings::POLLIN; } diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thre= ad.rs index 97d5f31e8fe3..57016971c065 100644 --- a/drivers/android/binder/thread.rs +++ b/drivers/android/binder/thread.rs @@ -9,15 +9,14 @@ =20 use kernel::{ bindings, - fs::{File, LocalFile}, + fs::LocalFile, list::{AtomicTracker, List, ListArc, ListLinks, TryNewListArc}, prelude::*, security, seq_file::SeqFile, seq_print, sync::atomic::{ordering::Relaxed, Atomic}, - sync::poll::{PollCondVar, PollTable}, - sync::{aref::ARef, Arc, SpinLock}, + sync::{aref::ARef, Arc, CondVar, SpinLock}, task::Task, uaccess::{UserPtr, UserSlice, UserSliceReader}, uapi, @@ -225,8 +224,10 @@ fn claim_next(&mut self, size: usize) -> Result= { } } =20 +#[must_use] pub(crate) enum PushWorkRes { Ok, + OkNotifyPoll, FailedDead(DLArc), } =20 @@ -234,6 +235,7 @@ impl PushWorkRes { fn is_ok(&self) -> bool { match self { PushWorkRes::Ok =3D> true, + PushWorkRes::OkNotifyPoll =3D> true, PushWorkRes::FailedDead(_) =3D> false, } } @@ -310,27 +312,32 @@ fn pop_work(&mut self) -> Option> { =20 fn push_work(&mut self, work: DLArc) -> PushWorkRes= { if self.is_dead { - PushWorkRes::FailedDead(work) + return PushWorkRes::FailedDead(work); + } + self.work_list.push_back(work); + self.process_work_list =3D true; + if self.looper_flags & LOOPER_POLL !=3D 0 { + PushWorkRes::OkNotifyPoll } else { - self.work_list.push_back(work); - self.process_work_list =3D true; PushWorkRes::Ok } } =20 - fn push_reply_work(&mut self, code: u32) { + fn push_reply_work(&mut self, code: u32) -> PushWorkRes { if let Ok(work) =3D ListArc::try_from_arc(self.reply_work.clone())= { work.set_error_code(code); - self.push_work(work); + self.push_work(work) } else { pr_warn!("Thread reply work is already in use."); + PushWorkRes::Ok } } =20 fn push_return_work(&mut self, reply: u32) { if let Ok(work) =3D ListArc::try_from_arc(self.return_work.clone()= ) { work.set_error_code(reply); - self.push_work(work); + // Not notifying: Reply to current thread. + let _ =3D self.push_work(work); } else { pr_warn!("Thread return work is already in use."); } @@ -422,7 +429,7 @@ pub(crate) struct Thread { #[pin] inner: SpinLock, #[pin] - work_condvar: PollCondVar, + work_condvar: CondVar, /// Used to insert this thread into the process' `ready_threads` list. /// /// INVARIANT: May never be used for any other list than the `self.pro= cess.ready_threads`. @@ -453,7 +460,7 @@ pub(crate) fn new(id: i32, process: Arc) -> Re= sult> { process, task: ARef::from(&**kernel::current!()), inner <- kernel::new_spinlock!(inner, "Thread::inner"), - work_condvar <- kernel::new_poll_condvar!("Thread::work_co= ndvar"), + work_condvar <- kernel::new_condvar!("Thread::work_condvar= "), links <- ListLinks::new(), links_track <- AtomicTracker::new(), }), @@ -617,7 +624,14 @@ fn get_work(self: &Arc, wait: bool) -> Result) -> Push= WorkRes { let sync =3D work.should_sync_wakeup(); + self.push_work_inner(work, sync) + } =20 + pub(crate) fn push_work_inner( + &self, + work: DLArc, + sync: bool, + ) -> PushWorkRes { let res =3D self.inner.lock().push_work(work); =20 if res.is_ok() { @@ -636,7 +650,8 @@ pub(crate) fn push_work(&self, work: DLArc) -> PushWorkRes { pub(crate) fn push_work_if_looper(&self, work: DLArc) -> BinderResult { let mut inner =3D self.inner.lock(); if inner.is_looper() && !inner.is_dead { - inner.push_work(work); + // Not notifying: Reply to current thread. + let _ =3D inner.push_work(work); Ok(()) } else { drop(inner); @@ -1142,7 +1157,7 @@ fn deliver_single_reply( transaction.set_outstanding(&mut self.process.inner.lock()); } =20 - { + let ret =3D { let mut inner =3D self.inner.lock(); if !inner.pop_transaction_replied(transaction) { return false; @@ -1153,15 +1168,16 @@ fn deliver_single_reply( } =20 match reply { - Ok(work) =3D> { - inner.push_work(work); - } + Ok(work) =3D> inner.push_work(work), Err(code) =3D> inner.push_reply_work(code), } - } + }; =20 // Notify the thread now that we've released the inner lock. self.work_condvar.notify_sync(); + if matches!(ret, PushWorkRes::OkNotifyPoll) { + self.process.notify_poll(true); + } false } =20 @@ -1319,7 +1335,8 @@ fn reply_inner(self: &Arc, info: &mut Transacti= onInfo) -> BinderResult { let process =3D orig.from.process.clone(); let allow_fds =3D orig.flags & TF_ACCEPT_FDS !=3D 0; let reply =3D Transaction::new_reply(self, process, info, allo= w_fds)?; - self.inner.lock().push_work(completion); + // Not notifying: Reply to current thread. + let _ =3D self.inner.lock().push_work(completion); orig.from.deliver_reply(Ok(reply), &orig); Ok(()) })() @@ -1351,7 +1368,8 @@ fn oneway_transaction_inner(self: &Arc, info: &= mut TransactionInfo) -> Bin }; let list_completion =3D DTRWrap::arc_try_new(DeliverCode::new(code= ))?; let completion =3D list_completion.clone_arc(); - self.inner.lock().push_work(list_completion); + // Not notifying: Reply to current thread. + let _ =3D self.inner.lock().push_work(list_completion); match transaction.submit(info) { Ok(()) =3D> Ok(()), Err(err) =3D> { @@ -1551,10 +1569,9 @@ pub(crate) fn write_read(self: &Arc, data: Use= rSlice, wait: bool) -> Resul ret } =20 - pub(crate) fn poll(&self, file: &File, table: PollTable<'_>) -> (bool,= u32) { - table.register_wait(file, &self.work_condvar); + pub(crate) fn poll(&self) -> Result<(bool, u32)> { let mut inner =3D self.inner.lock(); - (inner.should_use_process_work_queue(), inner.poll()) + Ok((inner.should_use_process_work_queue(), inner.poll())) } =20 /// Make the call to `get_work` or `get_work_local` return immediately= , if any. @@ -1571,26 +1588,9 @@ pub(crate) fn exit_looper(&self) { } } =20 - pub(crate) fn notify_if_poll_ready(&self, sync: bool) { - // Determine if we need to notify. This requires the lock. - let inner =3D self.inner.lock(); - let notify =3D inner.looper_flags & LOOPER_POLL !=3D 0 && inner.sh= ould_use_process_work_queue(); - drop(inner); - - // Now that the lock is no longer held, notify the waiters if we h= ave to. - if notify { - if sync { - self.work_condvar.notify_sync(); - } else { - self.work_condvar.notify_one(); - } - } - } - pub(crate) fn release(self: &Arc) { self.inner.lock().is_dead =3D true; =20 - //self.work_condvar.clear(); self.unwind_transaction_stack(); =20 // Cancel all pending work items. diff --git a/drivers/android/binder/transaction.rs b/drivers/android/binder= /transaction.rs index 47d5e4d88b07..475f7f7c29fb 100644 --- a/drivers/android/binder/transaction.rs +++ b/drivers/android/binder/transaction.rs @@ -322,11 +322,15 @@ pub(crate) fn submit(self: DLArc, info: &mut Tr= ansactionInfo) -> BinderRes crate::trace::trace_transaction(false, &self, Some(&thread.tas= k)); match thread.push_work(self) { PushWorkRes::Ok =3D> Ok(()), + PushWorkRes::OkNotifyPoll =3D> { + process.notify_poll(true); + Ok(()) + } PushWorkRes::FailedDead(me) =3D> Err((BinderError::new_dea= d(), me)), } } else { crate::trace::trace_transaction(false, &self, None); - process_inner.push_work(self) + process_inner.push_work(&process, self) }; drop(process_inner); =20 --=20 2.54.0.746.g67dd491aae-goog