From nobody Sat May 30 12:34:35 2026 Received: from mail-wm1-f73.google.com (mail-wm1-f73.google.com [209.85.128.73]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 94CB4340281 for ; Fri, 8 May 2026 06:56:26 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.128.73 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1778223390; cv=none; b=Vd62NeMttMH9TAIs3QVG+qjrLMjqoplTA8USlRmuKD+Zj/X1BAETCFvqwuRTwyK/N6YlYecTmsGvCNNOuF6aYYSSNwnBRaFa4OjesIrdxPvjI1d7/orEqTvxp+ou75z2gHqPtFWgCWJewFhdRRZq1bqIjyuN+l0132YwR+4muj4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1778223390; c=relaxed/simple; bh=CLtHpbaS1mKle8E88YQIhhMZXFuxKBPqK83vNenbugs=; h=Date:In-Reply-To:Mime-Version:References:Message-ID:Subject:From: To:Cc:Content-Type; b=ufSY/OS4w/WKwqCe6iN/09twvWCYxG06KHYjSw7CS/sSWOBfKPPBxF1OtqwiFUF9IZqOcbWqzY7UI/ywjTSCi9Qd6J8P/Dmor8E15hSWDYiniTm/Adqc6cu0YyozHEEBKfxa97sHAy6n9zbs22RbrBgVI0JPS0G+CEfLFlMk8PA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b=ScwoWPlW; arc=none smtp.client-ip=209.85.128.73 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b="ScwoWPlW" Received: by mail-wm1-f73.google.com with SMTP id 5b1f17b1804b1-48d121738feso13976245e9.2 for ; Thu, 07 May 2026 23:56:25 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20251104; t=1778223384; x=1778828184; darn=vger.kernel.org; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:from:to:cc:subject:date:message-id:reply-to; bh=gwR5zUblvdpSOLiS9fu4K+/lNRVrhgwROb1hUxq0umc=; b=ScwoWPlWjWp2AwDH+knW6mKO/88bWlvFG12uvon34qZ4d6ZkzYlprUB6kBlwyRn93J wm3lJgic8puwNHrpBFcGpsm6lAAvX1xn0xPMeH5aixkWpYxsabxeThkvW4TpEqsgwA5I XR6gAiD3pUWkOt0mOsaC4fGpk3nw+QZ7JmiYFOsFvnYCy383kYHmJ/mimmsyeA75OEMd izDppb3AILTQhIHOv4d4vyoM7jwIeLS8k8ZvqYa2P1ElGcD+gAWO6PYS/ZswNWBJSNny FCyrJZjLxxsehrHVweu6AxmLLlyaEoPmMfADzZmL8Vx3iQicGaWKPJAoFw8qR44HaXa4 f+JA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20251104; t=1778223384; x=1778828184; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=gwR5zUblvdpSOLiS9fu4K+/lNRVrhgwROb1hUxq0umc=; b=bEicvoNxCPnq2pJzc4hw+KcRQDPx98ytQPuyXiCxtsugxtm2RSPejSG6a8vuWCi2QK 04lZMGBKpKCJ+RceLyCzw7LFT0R/AkaP5G4cymRkFVJrJKxUrxn+I82tIPbzctmut+FX Z6TjwCSby6+EBftzBSYhfem0mwogvXrlC8RliNvbBYjV2tSZcpzv7HJb/Q85KkXdtDX6 8FxFKTEFq+KlavGDSliUm75mgfC0OqSiAyHA17Bt/IAORLFKopN2mjUw8f0xRxa8tEkO Ij/l+BTINqKz/IllKVHkCEhKKbR4efrILdaOn4fJlKYIjJB8jNo4xgMJfnqigQB1TZPU qRxQ== X-Forwarded-Encrypted: i=1; AFNElJ8iHLQc/+8U32xUaJAPkiMfPrdKtPAQPtjT7KaUNSn7HeptrbWNxykWjFsrapbxHEcLnup7+TYe4B2BNac=@vger.kernel.org X-Gm-Message-State: AOJu0YxI7+8FnaQsVPCRNW46978UDQEUkZ9SPnPvgKCq18X0BVhmlkSA jT9AAvy8MaqkLKnMS2lSSzjoso/rZKYql7DUm82XFrmAul0vQ1UkrAlF1LoW6GmELPANAEYTgoz lqqoItTruLlp7tJ37qg== X-Received: from wmco5.prod.google.com ([2002:a05:600c:a305:b0:48a:6051:b5af]) (user=aliceryhl job=prod-delivery.src-stubby-dispatcher) by 2002:a05:600c:871b:b0:483:709e:f238 with SMTP id 5b1f17b1804b1-48e51f4652fmr182535775e9.29.1778223384122; Thu, 07 May 2026 23:56:24 -0700 (PDT) Date: Fri, 08 May 2026 06:55:11 +0000 In-Reply-To: <20260508-upgrade-poll-v3-0-0c619fe846e8@google.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 References: <20260508-upgrade-poll-v3-0-0c619fe846e8@google.com> X-Developer-Key: i=aliceryhl@google.com; a=openpgp; fpr=49F6C1FAA74960F43A5B86A1EE7A392FDE96209F X-Developer-Signature: v=1; a=openpgp-sha256; l=3877; i=aliceryhl@google.com; h=from:subject:message-id; bh=CLtHpbaS1mKle8E88YQIhhMZXFuxKBPqK83vNenbugs=; b=owEBbQKS/ZANAwAKAQRYvu5YxjlGAcsmYgBp/YkVE+D4eBTnrKBD7RpNEkaLJYcGJN9FVMPVA tpjeihBSDmJAjMEAAEKAB0WIQSDkqKUTWQHCvFIvbIEWL7uWMY5RgUCaf2JFQAKCRAEWL7uWMY5 RjLnD/4qe7AUfbmAav2TJeIzh0lJeGXw+I6OP8Tq2TCTVjiPv3qlWfj8oCy1my4vgtAbTdqaODG WZsvmVoE5jwKnjjsMSHqpyHtX1WwduCAb+PlEdEplDn5Z9e0bXM2nXoJ82uPMU3daSgvSoUJein 8mtr4xdHo05M2flIbnkxtI6emRljXVo48raQ1A6wkxgJ0Y/m8HGDSAZjUBo7Wo8HZ7EkAxSJxFL JOMcp+14P11ZiQdcVH/yIZ82trhm5x6zdDvFXyAy6duRH6SrcQ3D2Ersf/kQTUXGZDMwKVQjKfi y6dk2AoALrQbVpwMAF+8eXpLKsq39iO1wbjgJCr0VBM/3ZNrLWxuS3doO3Rynfv2YdTNttNutK0 8FNBS5rq+CWglBg6kJfabbMGtVQuaCot+qET7tg7pH8+zMuta3OVXoOfhhv/QMSZzbJKqdcrv+D zn95oZTU4TfSrMnwdLO7y5QjMIEAmAg1yP3abXMjN5GxCUvs7P+otDBePN2RAfH9BNMxHdDOQrO dp9f+JekhMyD+SzUASjvJJJVC5FlBPTkOz5tUx1nd167Fv3c4C9UhTSazmsjrci7572lzndzRgF xluguRFQIJ/Y9r22t3uugsBXAhJIr5g96tPgTOqMDSesq0HiMBFcPeaG0DHKIitkvDSSE0gO6qA yR3x5CXSjKhhGZQ== X-Mailer: b4 0.14.3 Message-ID: <20260508-upgrade-poll-v3-1-0c619fe846e8@google.com> Subject: [PATCH v3 1/2] rust: poll: use kfree_rcu() for PollCondVar From: Alice Ryhl To: Greg Kroah-Hartman , Carlos Llamas , Christian Brauner , Boqun Feng Cc: "Paul E. McKenney" , Alexander Viro , Jan Kara , Miguel Ojeda , Boqun Feng , "=?utf-8?q?Bj=C3=B6rn_Roy_Baron?=" , Benno Lossin , Andreas Hindborg , Trevor Gross , Danilo Krummrich , Gary Guo , linux-fsdevel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-kernel@vger.kernel.org, Alice Ryhl Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Rust Binder currently uses PollCondVar, but it calls synchronize_rcu() in the destructor, which we would like to avoid. Add a variation of PollCondVar that kfree_rcu() instead. One could avoid the `rcu` field and allocate the rcu_head on drop using a fallback to synchronize_rcu() on ENOMEM. However, I'd prefer to avoid the potential for synchronize_rcu(), and Binder will only use this for a small fraction of processes, so even if it changes which kmalloc bucket it falls into, the extra memory is not a problem. Signed-off-by: Alice Ryhl --- rust/kernel/sync/poll.rs | 73 ++++++++++++++++++++++++++++++++++++++++++++= +++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index 0ec985d560c8..684dfa242b1a 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -5,12 +5,18 @@ //! Utilities for working with `struct poll_table`. =20 use crate::{ + alloc::AllocError, bindings, fs::File, prelude::*, sync::{CondVar, LockClassKey}, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, // }; -use core::{marker::PhantomData, ops::Deref}; =20 /// Creates a [`PollCondVar`] initialiser with the given name and a newly-= created lock class. #[macro_export] @@ -66,6 +72,7 @@ pub fn register_wait(&self, file: &File, cv: &PollCondVar= ) { /// /// [`CondVar`]: crate::sync::CondVar #[pin_data(PinnedDrop)] +#[repr(transparent)] pub struct PollCondVar { #[pin] inner: CondVar, @@ -104,3 +111,67 @@ fn drop(self: Pin<&mut Self>) { unsafe { bindings::synchronize_rcu() }; } } + +/// A [`KBox`] that uses `kfree_rcu`. +/// +/// [`KBox`]: PollCondVar +pub struct PollCondVarBox { + inner: ManuallyDrop>>, +} + +#[pin_data] +#[repr(C)] +struct PollCondVarBoxInner { + #[pin] + inner: PollCondVar, + rcu: Opaque, +} + +// SAFETY: PollCondVar is Send +unsafe impl Send for PollCondVarBoxInner {} +// SAFETY: PollCondVar is Sync +unsafe impl Sync for PollCondVarBoxInner {} + +impl PollCondVarBox { + /// Constructs a new boxed [`PollCondVar`]. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Re= sult { + let b =3D KBox::pin_init( + pin_init!(PollCondVarBoxInner { + inner <- PollCondVar::new(name, key), + rcu: Opaque::uninit(), + }), + GFP_KERNEL, + ) + .map_err(|_| AllocError)?; + + Ok(PollCondVarBox { + inner: ManuallyDrop::new(b), + }) + } +} + +impl Deref for PollCondVarBox { + type Target =3D PollCondVar; + fn deref(&self) -> &PollCondVar { + &self.inner.inner + } +} + +impl Drop for PollCondVarBox { + #[inline] + fn drop(&mut self) { + // SAFETY: ManuallyDrop::take ok because not already taken. + let boxed =3D unsafe { ManuallyDrop::take(&mut self.inner) }; + + // SAFETY: The code below frees the box without calling the actual= destructor of the type, + // but it's okay because it re-implements the destructor using `kf= ree_rcu()` in place of + // `synchronize_rcu()`. + let ptr =3D KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxe= d) }); + + // SAFETY: The pointer points at a valid `wait_queue_head`. + unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queu= e_head.get()) }; + + // SAFETY: This was allocated using `KBox::pin_init`, so it can be= freed with `kvfree`. + unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::()) }; + } +} --=20 2.54.0.563.g4f69b47b94-goog From nobody Sat May 30 12:34:35 2026 Received: from mail-wr1-f74.google.com (mail-wr1-f74.google.com [209.85.221.74]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BCE3634F46F for ; Fri, 8 May 2026 06:56:28 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.221.74 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1778223393; cv=none; b=CYJuqbaeZMquY5GasUDQ001xV25bug6ZP10xlq75vVloPvw4DgjeY0By7QmFZ67qICx7d6DnkfphTct8Ck3paXCGyOqXKCd2KBwT7CwP+WCe0RVxTwpOW/bkw73frWKfCu03gotMhfq1QKhBgUEwXtKjqCunlzN8ZC8ooock9R4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1778223393; c=relaxed/simple; bh=7ggYT7Cf6g063n4K/hPOh+tKS9dcGxoYdeOPk9iTix8=; h=Date:In-Reply-To:Mime-Version:References:Message-ID:Subject:From: To:Cc:Content-Type; b=apMddpa/OHWoaFw6XidT5mkNq+n1r/gUXIxTHiO2br28ZW3L2bnjPklVhvEaTuQEmD3UgK0UCHdZr2qAVi4N7IuwBQNpY7YsEJf1+qDfIDWi0Zgpcudq0wRc74lhLu59Cvr0HgUSm6Yb1w2+uFD/Vhy7gu6UeE8HAiKXSlnQuDY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b=fm9GDUCK; arc=none smtp.client-ip=209.85.221.74 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b="fm9GDUCK" Received: by mail-wr1-f74.google.com with SMTP id ffacd0b85a97d-44d79da8cf7so1428649f8f.2 for ; Thu, 07 May 2026 23:56:28 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20251104; t=1778223386; x=1778828186; darn=vger.kernel.org; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:from:to:cc:subject:date:message-id:reply-to; bh=XyW8IIOCNsLiUtVIwDCGziMFg169iWbV1XdBe620o5g=; b=fm9GDUCKSjlNKgZxE8JTlmk+3wSTSuZJoU8jPBDNP/KrFvCOXPs4HprfKhLwDkiG5/ qufaptpafN+wtcLzAPq/4kw7+YpHWakPZ8H4hJt6jvbfu2RA/Er8TPaF5OUrgilRzucH 7zAVCCcOJEL53gHYsVwnwW4Z371OScdEd8L/ksGyDqIy0My8FN7ODj+zr4gU1YqP05Pd /Rw1ulfZddkSR0hXvCIg2Ui8JyI/yi7pAPNuojwaRbuC+emffxbOHpIwRTmKuUQ00R1G 6V2+SAkPO6nXd29rzpkYu99HzTphbAF9vGfoyDQGqwSdixA8DCMgqNHxPjV6IcS+yZDr GdoQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20251104; t=1778223386; x=1778828186; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=XyW8IIOCNsLiUtVIwDCGziMFg169iWbV1XdBe620o5g=; b=rvdanqSzAAizN7pYHZ4RvmYVc21Zj6HXNTUWnHdTPulLvwTe06zsRtUXDHYxB+gMjB rrICYt4BhdLeqCY1p429LTILQEr5Zh9tfojtvs7DW8h9hEG0G55fFa95YV+NE49dUEdq wJPPHhxKQ6Ckti67NWSU5lEuT9JyIpvlKtCWewUXXMhZMoKgs39VyoOKzX8+Dok6CC0R 8JEKWpPJFV9THCOnoPwTl710iAWxJzPMzOE9cAEXdDDhV9fjPAUeQCzYVt7sGi6wqWnT TQ7YbmvatzcleP498ztARtVJ681i34L4jCDw/PGS3mddeQQCFB7L5qE4Ze9DeppI3XCt ndjA== X-Forwarded-Encrypted: i=1; AFNElJ+QhLzikd7sNdelOT7zTOBPnURaV6093QiLQbi63qz3qdD12vakmsq2q0Z5jp8c0aCX9pNvJ8QrHoh44pA=@vger.kernel.org X-Gm-Message-State: AOJu0Ywi9GxZbR0mN2Lq9rfR+beoX9L90hQ/cTlgXgUKQpMdk56MRwlk iuIxJV5h2nBDpiBlckvqfhGtDcxo/PdEMwei0gO1vxSqk8rme9xnJzHCF0+zYmWSmDUz9v8EFd4 1hcLqGm66xGk9pBiPfA== X-Received: from wrua3.prod.google.com ([2002:a5d:4d43:0:b0:449:dcfe:7e41]) (user=aliceryhl job=prod-delivery.src-stubby-dispatcher) by 2002:a05:6000:4381:b0:441:247a:e98e with SMTP id ffacd0b85a97d-4515ce1c738mr17900362f8f.24.1778223385682; Thu, 07 May 2026 23:56:25 -0700 (PDT) Date: Fri, 08 May 2026 06:55:12 +0000 In-Reply-To: <20260508-upgrade-poll-v3-0-0c619fe846e8@google.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 References: <20260508-upgrade-poll-v3-0-0c619fe846e8@google.com> X-Developer-Key: i=aliceryhl@google.com; a=openpgp; fpr=49F6C1FAA74960F43A5B86A1EE7A392FDE96209F X-Developer-Signature: v=1; a=openpgp-sha256; l=14834; i=aliceryhl@google.com; h=from:subject:message-id; bh=7ggYT7Cf6g063n4K/hPOh+tKS9dcGxoYdeOPk9iTix8=; b=owEBbQKS/ZANAwAKAQRYvu5YxjlGAcsmYgBp/YkVF9kjl9PXk0VnSKuyAEtWQ6InwiiCbMlBi yDcDHR+3z2JAjMEAAEKAB0WIQSDkqKUTWQHCvFIvbIEWL7uWMY5RgUCaf2JFQAKCRAEWL7uWMY5 RuTOEACIlvXZMK8/IQEOLwBYKrbLUXzotxq90F7esS1PGcaoefsnQJnLqyVzr362IGOnPlKhG61 aF8IURyNu0G2pN9IB7BCf51cHiOVS2bgMaIBfoB5xmAR/AEvXfU6/b+bQVD5oSB5Aq1PthabEvB PTX1RKJJckPyY2jNIICdPdHRK8NbE1/J0+qK6JARWQFgVW3J/zgM5CSCxboYzjyYVedP5QlRzgK RCwufetdXuT/JpeJDzfiw6jv0xlY6/oWqV3X3B1MM9IuzAyKw/g2aHlb89K4KnaUqFO2OubHyHT u3t+KNeZ+xIT6JYuqLSzW9/wlm+4k17P1y6LLee1VlztPxhop6UL7jKazP+0UM6sIP50kgiLtPw M74imUn2M/nU0ixr5l4L+O8D3VEjHwOq0dPLQFMtAT4XAaH48BfX1wwCw8aeepneppaPbTtm5qH rt9vBKgSWHTv0pys+yv8HcnREts9tA9Stf/TtZDlT/OVBsPLBN8uOlvcm/1+uPAqTj9yjWbA8B4 QiKGYmQP2jGL2rRGROpnD1fNDi4Ld3yj5v7NId1qWrgw+mS8UgSVRXS51R+PQ5m3E2QW6slhQ39 NkSidGredGE1ypCB8gOiCF5IKgQUAInyIptA9lItSZFkkYat1a2WS4uFRkMzjuHa9mdZmuhJRv9 33jzrwGhe5pDwiQ== X-Mailer: b4 0.14.3 Message-ID: <20260508-upgrade-poll-v3-2-0c619fe846e8@google.com> Subject: [PATCH v3 2/2] rust_binder: move (e)poll wait queue to Process From: Alice Ryhl To: Greg Kroah-Hartman , Carlos Llamas , Christian Brauner , Boqun Feng Cc: "Paul E. McKenney" , Alexander Viro , Jan Kara , Miguel Ojeda , Boqun Feng , "=?utf-8?q?Bj=C3=B6rn_Roy_Baron?=" , Benno Lossin , Andreas Hindborg , Trevor Gross , Danilo Krummrich , Gary Guo , linux-fsdevel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-kernel@vger.kernel.org, Alice Ryhl Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Most processes do not use Rust Binder with epoll, so avoid paying the synchronize_rcu() cost in drop for those that don't need it. For those that do, we also manage to replace synchronize_rcu() with kfree_rcu(), though we introduce an extra allocation. In case the last ref to an Arc is dropped outside of deferred_release(), this also ensures that synchronize_rcu() is not called in destructor of Arc in other places. Theoretically that could lead to jank by making other syscalls slow, which would be problematic. Signed-off-by: Alice Ryhl --- drivers/android/binder/process.rs | 51 +++++++++++++++++------ drivers/android/binder/thread.rs | 78 +++++++++++++++++--------------= ---- drivers/android/binder/transaction.rs | 4 ++ 3 files changed, 82 insertions(+), 51 deletions(-) diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/pro= cess.rs index 820cbd541435..29fdfc964685 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -30,6 +30,7 @@ sync::{ aref::ARef, lock::{spinlock::SpinLockBackend, Guard}, + poll::PollCondVarBox, Arc, ArcBorrow, CondVar, CondVarTimeoutResult, Mutex, SpinLock, Un= iqueArc, }, task::Task, @@ -135,6 +136,7 @@ pub(crate) struct ProcessInner { pub(crate) binderfs_file: Option, /// Check for oneway spam oneway_spam_detection_enabled: bool, + poll: Option, } =20 impl ProcessInner { @@ -158,6 +160,7 @@ fn new() -> Self { async_recv: false, binderfs_file: None, oneway_spam_detection_enabled: false, + poll: None, } } =20 @@ -174,19 +177,23 @@ pub(crate) fn push_work( &mut self, work: DLArc, ) -> Result<(), (BinderError, DLArc)> { + let sync =3D work.should_sync_wakeup(); + // Try to find a ready thread to which to push the work. if let Some(thread) =3D self.ready_threads.pop_front() { // Push to thread while holding state lock. This prevents the = thread from giving up // (for example, because of a signal) when we're about to deli= ver work. - match thread.push_work(work) { + match thread.push_work_inner(work, sync) { PushWorkRes::Ok =3D> Ok(()), + PushWorkRes::OkNotifyPoll =3D> { + self.notify_poll(sync); + Ok(()) + } PushWorkRes::FailedDead(work) =3D> Err((BinderError::new_d= ead(), work)), } } else if self.is_dead { Err((BinderError::new_dead(), work)) } else { - let sync =3D work.should_sync_wakeup(); - // Didn't find a thread waiting for proc work; this can happen // in two scenarios: // 1. All threads are busy handling transactions @@ -194,22 +201,26 @@ pub(crate) fn push_work( // the kernel driver soon and pick up this work. // 2. Threads are using the (e)poll interface, in which case // they may be blocked on the waitqueue without having been - // added to waiting_threads. For this case, we just iterate - // over all threads not handling transaction work, and - // wake them all up. We wake all because we don't know whet= her - // a thread that called into (e)poll is handling non-binder - // work currently. + // added to waiting_threads. For this case, we wake it up + // directly. self.work.push_back(work); =20 // Wake up polling threads, if any. - for thread in self.threads.values() { - thread.notify_if_poll_ready(sync); - } + self.notify_poll(sync); =20 Ok(()) } } =20 + pub(crate) fn notify_poll(&self, sync: bool) { + if let Some(poll) =3D &self.poll { + if sync { + poll.notify_sync(); + } + poll.notify_all(); + } + } + pub(crate) fn remove_node(&mut self, ptr: u64) { self.nodes.remove(&ptr); } @@ -1322,11 +1333,15 @@ fn deferred_flush(&self) { =20 fn deferred_release(self: Arc) { let is_manager =3D { + // These variables contain values to be dropped after releasin= g spinlock. + let _poll; + let mut inner =3D self.inner.lock(); inner.is_dead =3D true; inner.is_frozen =3D IsFrozen::No; inner.sync_recv =3D false; inner.async_recv =3D false; + _poll =3D inner.poll.take(); inner.is_manager }; =20 @@ -1696,7 +1711,19 @@ pub(crate) fn poll( table: PollTable<'_>, ) -> Result { let thread =3D this.get_current_thread()?; - let (from_proc, mut mask) =3D thread.poll(file, table); + { + let mut inner =3D this.inner.lock(); + let poll =3D if let Some(poll) =3D &inner.poll { + &**poll + } else { + drop(inner); + let poll =3D PollCondVarBox::new(c"Process::poll", kernel:= :static_lock_class!())?; + inner =3D this.inner.lock(); + &**inner.poll.get_or_insert(poll) + }; + table.register_wait(file, poll); + } + let (from_proc, mut mask) =3D thread.poll()?; if mask =3D=3D 0 && from_proc && !this.inner.lock().work.is_empty(= ) { mask |=3D bindings::POLLIN; } diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thre= ad.rs index 97d5f31e8fe3..425c50298706 100644 --- a/drivers/android/binder/thread.rs +++ b/drivers/android/binder/thread.rs @@ -9,15 +9,14 @@ =20 use kernel::{ bindings, - fs::{File, LocalFile}, + fs::LocalFile, list::{AtomicTracker, List, ListArc, ListLinks, TryNewListArc}, prelude::*, security, seq_file::SeqFile, seq_print, sync::atomic::{ordering::Relaxed, Atomic}, - sync::poll::{PollCondVar, PollTable}, - sync::{aref::ARef, Arc, SpinLock}, + sync::{aref::ARef, Arc, CondVar, SpinLock}, task::Task, uaccess::{UserPtr, UserSlice, UserSliceReader}, uapi, @@ -225,8 +224,10 @@ fn claim_next(&mut self, size: usize) -> Result= { } } =20 +#[must_use] pub(crate) enum PushWorkRes { Ok, + OkNotifyPoll, FailedDead(DLArc), } =20 @@ -234,6 +235,7 @@ impl PushWorkRes { fn is_ok(&self) -> bool { match self { PushWorkRes::Ok =3D> true, + PushWorkRes::OkNotifyPoll =3D> true, PushWorkRes::FailedDead(_) =3D> false, } } @@ -310,27 +312,32 @@ fn pop_work(&mut self) -> Option> { =20 fn push_work(&mut self, work: DLArc) -> PushWorkRes= { if self.is_dead { - PushWorkRes::FailedDead(work) + return PushWorkRes::FailedDead(work); + } + self.work_list.push_back(work); + self.process_work_list =3D true; + if self.looper_flags & LOOPER_POLL !=3D 0 { + PushWorkRes::OkNotifyPoll } else { - self.work_list.push_back(work); - self.process_work_list =3D true; PushWorkRes::Ok } } =20 - fn push_reply_work(&mut self, code: u32) { + fn push_reply_work(&mut self, code: u32) -> PushWorkRes { if let Ok(work) =3D ListArc::try_from_arc(self.reply_work.clone())= { work.set_error_code(code); - self.push_work(work); + self.push_work(work) } else { pr_warn!("Thread reply work is already in use."); + PushWorkRes::Ok } } =20 fn push_return_work(&mut self, reply: u32) { if let Ok(work) =3D ListArc::try_from_arc(self.return_work.clone()= ) { work.set_error_code(reply); - self.push_work(work); + // Not notifying: Reply to current thread. + let _ =3D self.push_work(work); } else { pr_warn!("Thread return work is already in use."); } @@ -422,7 +429,7 @@ pub(crate) struct Thread { #[pin] inner: SpinLock, #[pin] - work_condvar: PollCondVar, + work_condvar: CondVar, /// Used to insert this thread into the process' `ready_threads` list. /// /// INVARIANT: May never be used for any other list than the `self.pro= cess.ready_threads`. @@ -453,7 +460,7 @@ pub(crate) fn new(id: i32, process: Arc) -> Re= sult> { process, task: ARef::from(&**kernel::current!()), inner <- kernel::new_spinlock!(inner, "Thread::inner"), - work_condvar <- kernel::new_poll_condvar!("Thread::work_co= ndvar"), + work_condvar <- kernel::new_condvar!("Thread::work_condvar= "), links <- ListLinks::new(), links_track <- AtomicTracker::new(), }), @@ -617,7 +624,14 @@ fn get_work(self: &Arc, wait: bool) -> Result) -> Push= WorkRes { let sync =3D work.should_sync_wakeup(); + self.push_work_inner(work, sync) + } =20 + pub(crate) fn push_work_inner( + &self, + work: DLArc, + sync: bool, + ) -> PushWorkRes { let res =3D self.inner.lock().push_work(work); =20 if res.is_ok() { @@ -636,7 +650,8 @@ pub(crate) fn push_work(&self, work: DLArc) -> PushWorkRes { pub(crate) fn push_work_if_looper(&self, work: DLArc) -> BinderResult { let mut inner =3D self.inner.lock(); if inner.is_looper() && !inner.is_dead { - inner.push_work(work); + // Not notifying: Reply to current thread. + let _ =3D inner.push_work(work); Ok(()) } else { drop(inner); @@ -1142,7 +1157,7 @@ fn deliver_single_reply( transaction.set_outstanding(&mut self.process.inner.lock()); } =20 - { + let ret =3D { let mut inner =3D self.inner.lock(); if !inner.pop_transaction_replied(transaction) { return false; @@ -1153,15 +1168,16 @@ fn deliver_single_reply( } =20 match reply { - Ok(work) =3D> { - inner.push_work(work); - } + Ok(work) =3D> inner.push_work(work), Err(code) =3D> inner.push_reply_work(code), } - } + }; =20 // Notify the thread now that we've released the inner lock. self.work_condvar.notify_sync(); + if matches!(ret, PushWorkRes::OkNotifyPoll) { + self.process.inner.lock().notify_poll(true); + } false } =20 @@ -1319,7 +1335,8 @@ fn reply_inner(self: &Arc, info: &mut Transacti= onInfo) -> BinderResult { let process =3D orig.from.process.clone(); let allow_fds =3D orig.flags & TF_ACCEPT_FDS !=3D 0; let reply =3D Transaction::new_reply(self, process, info, allo= w_fds)?; - self.inner.lock().push_work(completion); + // Not notifying: Reply to current thread. + let _ =3D self.inner.lock().push_work(completion); orig.from.deliver_reply(Ok(reply), &orig); Ok(()) })() @@ -1351,7 +1368,8 @@ fn oneway_transaction_inner(self: &Arc, info: &= mut TransactionInfo) -> Bin }; let list_completion =3D DTRWrap::arc_try_new(DeliverCode::new(code= ))?; let completion =3D list_completion.clone_arc(); - self.inner.lock().push_work(list_completion); + // Not notifying: Reply to current thread. + let _ =3D self.inner.lock().push_work(list_completion); match transaction.submit(info) { Ok(()) =3D> Ok(()), Err(err) =3D> { @@ -1551,10 +1569,9 @@ pub(crate) fn write_read(self: &Arc, data: Use= rSlice, wait: bool) -> Resul ret } =20 - pub(crate) fn poll(&self, file: &File, table: PollTable<'_>) -> (bool,= u32) { - table.register_wait(file, &self.work_condvar); + pub(crate) fn poll(&self) -> Result<(bool, u32)> { let mut inner =3D self.inner.lock(); - (inner.should_use_process_work_queue(), inner.poll()) + Ok((inner.should_use_process_work_queue(), inner.poll())) } =20 /// Make the call to `get_work` or `get_work_local` return immediately= , if any. @@ -1571,26 +1588,9 @@ pub(crate) fn exit_looper(&self) { } } =20 - pub(crate) fn notify_if_poll_ready(&self, sync: bool) { - // Determine if we need to notify. This requires the lock. - let inner =3D self.inner.lock(); - let notify =3D inner.looper_flags & LOOPER_POLL !=3D 0 && inner.sh= ould_use_process_work_queue(); - drop(inner); - - // Now that the lock is no longer held, notify the waiters if we h= ave to. - if notify { - if sync { - self.work_condvar.notify_sync(); - } else { - self.work_condvar.notify_one(); - } - } - } - pub(crate) fn release(self: &Arc) { self.inner.lock().is_dead =3D true; =20 - //self.work_condvar.clear(); self.unwind_transaction_stack(); =20 // Cancel all pending work items. diff --git a/drivers/android/binder/transaction.rs b/drivers/android/binder= /transaction.rs index 47d5e4d88b07..3c9620da3d25 100644 --- a/drivers/android/binder/transaction.rs +++ b/drivers/android/binder/transaction.rs @@ -322,6 +322,10 @@ pub(crate) fn submit(self: DLArc, info: &mut Tra= nsactionInfo) -> BinderRes crate::trace::trace_transaction(false, &self, Some(&thread.tas= k)); match thread.push_work(self) { PushWorkRes::Ok =3D> Ok(()), + PushWorkRes::OkNotifyPoll =3D> { + process_inner.notify_poll(true); + Ok(()) + } PushWorkRes::FailedDead(me) =3D> Err((BinderError::new_dea= d(), me)), } } else { --=20 2.54.0.563.g4f69b47b94-goog