From nobody Fri Jun 12 07:28:03 2026 Received: from mail-lf1-f74.google.com (mail-lf1-f74.google.com [209.85.167.74]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 29DEA3BCD04 for ; Thu, 11 Jun 2026 09:29:19 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.167.74 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1781170161; cv=none; b=WeJjlhwoM6UT+/FPbw/AKqFjVOXssqWChOavdHiNWPNUjI/511xui5d5TsMEJ7eO0S824aXmlWOmo+LsiSeWybGXUNrLtEb0JfhF+tKIk3f8Hq12HmnU4P0Md2f0CNRmzbyskRT6PztJxLmDW6T4vHJoZrnlm4KdKlZAOLOFstM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1781170161; c=relaxed/simple; bh=Qi5DAQgATfLRz7Bnn3G/2iiJAjf5fznjFr5yRi6S/6c=; h=Date:In-Reply-To:Mime-Version:References:Message-ID:Subject:From: To:Cc:Content-Type; b=R3cjtC3Jj+FNWT9kT31aV7G07s0uY3QgF81+6zxVaiAAsxBOF03sFS1zqDmgg6SRczh5PBMh+tjC523HN0kJ73qHVDCp7GBj3fJc8y1HWOWskTJgYgBWYP+42XaijQh88BKY6v5TJbSd30Aq7EbN54kbWIH3eIUll/sC4ppUSog= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b=UHv2PY1l; arc=none smtp.client-ip=209.85.167.74 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b="UHv2PY1l" Received: by mail-lf1-f74.google.com with SMTP id 2adb3069b0e04-5aa607bbfe2so4573689e87.3 for ; Thu, 11 Jun 2026 02:29:18 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20251104; t=1781170157; x=1781774957; darn=vger.kernel.org; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:from:to:cc:subject:date:message-id:reply-to; bh=tF/awWsMTV8tnH567MOX5eq9NoKU3JJoN57UzqYjc+s=; b=UHv2PY1lJOWFGV4SwHxCItjD15yDlH2M4QP8DvUfucwFl8Xuikd8q8J1hb8Fxo3d1t G+I04m25OLAEjCoeb7XTeAm8HH0O953AAkfpemxvPXihKF+WxfvUQxurJhRCthIrMNIs q5F863skf1GBH11bwjNCgF9T7knnIMUo3h4MKYQAjEN+X8FK9NBP2MvCtMkc5OqphMxr 9+iBeGe8qzoRP0bWIcGHptvgr3oVvpiIU8bPbQlldzxie72XwwzlQpqi2EzMcK8fsXZx saAWRrmYSkoRRVLg7YR746QF/fJ/F4mEWsAY2LCiYA1tGoqaLllQhAWwG7lyhFn2Lu7n VIUQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20251104; t=1781170157; x=1781774957; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=tF/awWsMTV8tnH567MOX5eq9NoKU3JJoN57UzqYjc+s=; b=EY5K9Eyan7nN89ESX0IVfvkrYBVgyoUsWjuUQKUA3b1OTsU4/l5lUeLjhsKoFREtE8 HocskH7D1tVwPQse6RN9DSkz18P4P2uxYphodx8F2D7RTzRv5Qxd8R5Z8vv6sYmMaonU /LvMYSBVTq96w4grhk9Lx9sbH1hFIs63crMvs8aKM2V3vr2fjMl0IEECxj7TYgiwGbjs sL9QWPJBfbz+diiJAQprSIwrjL6qxl8OiqxLZ97Xmi20JWQrghRzE+XOW+ZVKfPeU5vV p1n7LIeHxDWnUSdoKOnhbTX5su6cNDX6JUovdE8YIRuDS0mkAz3ZHL5GzMjpRSMsYiVJ FVeA== X-Forwarded-Encrypted: i=1; AFNElJ9QwvIWTwUNoiBeDnyUlQeMkOGdAjx2XycerI3Dy3RCgOojXgn4oAZSc8tH0T7Z1hz9OzATNkpv3eVSN8k=@vger.kernel.org X-Gm-Message-State: AOJu0Yz3ci7F5Tgiat0bFZ7UCTQEVCadzWyIi5D+ULKVU2DTqDZtKjw/ /biQPH/V2q3ZUkWmtkiy0voPPgtfGWioeBhzTs6VRgAWhA/7LJDAFfF7osZPevHPhCkSmC8fcnx 55ZZlnOLqeSbIefX65A== X-Received: from lfgy11.prod.google.com ([2002:ac2:55ab:0:b0:5aa:6526:560d]) (user=aliceryhl job=prod-delivery.src-stubby-dispatcher) by 2002:ac2:4f0c:0:b0:5aa:659b:2a95 with SMTP id 2adb3069b0e04-5ad27fbfc16mr684231e87.36.1781170157095; Thu, 11 Jun 2026 02:29:17 -0700 (PDT) Date: Thu, 11 Jun 2026 09:29:00 +0000 In-Reply-To: <20260611-upgrade-poll-v5-0-497a11c06828@google.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 References: <20260611-upgrade-poll-v5-0-497a11c06828@google.com> X-Developer-Key: i=aliceryhl@google.com; a=openpgp; fpr=49F6C1FAA74960F43A5B86A1EE7A392FDE96209F X-Developer-Signature: v=1; a=openpgp-sha256; l=3878; i=aliceryhl@google.com; h=from:subject:message-id; bh=Qi5DAQgATfLRz7Bnn3G/2iiJAjf5fznjFr5yRi6S/6c=; b=owEBbQKS/ZANAwAKAQRYvu5YxjlGAcsmYgBqKn/pOwzsWAIqWxAYuIbRBEORuNHtM1nvq3sSS DpJcr2OGCmJAjMEAAEKAB0WIQSDkqKUTWQHCvFIvbIEWL7uWMY5RgUCaip/6QAKCRAEWL7uWMY5 RgEHD/43JtOfQhqZf0qn88EkxissaAcYHd/Tfm7JdUSLqfLb4UZSmYMLp+wGf5eZDSGewq7VQX6 CWGBlmtBbvlmzwts5AUeBlKIc4FKfRTuPrQS1EQnQC78VH9FPrqJz+FkjrYNY/U94IZC/hT/9ye oWEX1R3JKBvEoOjWhz7w3csjF+rM/3OLNNcqsMMe5JysPD9CZ1HT5Kz3mW7rhJpilTKfG4f9bPU m6u2DYHqGfd1fM7OGD0kBuM/GFvgSGJk24mE2Ylceyy0JnNK0ndLl81g2cR17+8GySIr3ZiGnHs /mUroOqgpXUdGnMzGLBiaRnQYun2Su26wuC+tUGqqof9kQTnLXmuPj0m/uwHzoFNV6N0f7sgCG3 XUSmI0QeYe1IvZf8smFIt6geTv2ns9zIe23ibGabdpmEMYbz7rCMqWnf4B+v6nSMtFysF6CBYD/ BgmW1r188PiRUZTBojqu7uXVWXDW9D25VEjyFKUi41zuKmKUOOVdDT5Dg238LZsd6a7/9Y5qX2o R0sol5gdlL0CtMO35RbCMr4fdwXaFYhQF9RzQ2cPf4BpR7b8PGh3bUgpgu98tQfLJLjEhpE1LUs Ef00WItBO95UPWO2SPslhMsPB8OtpW+iiFlcwqKX42kdYNcVNETOdYqye16U5mLnHA4cRmvL4hl WZ/LHqaD2EuN09Q== X-Mailer: b4 0.14.3 Message-ID: <20260611-upgrade-poll-v5-1-497a11c06828@google.com> Subject: [PATCH v5 1/2] rust: poll: use kfree_rcu() for PollCondVar From: Alice Ryhl To: Greg Kroah-Hartman , Carlos Llamas , Christian Brauner , Boqun Feng Cc: "Paul E. McKenney" , Alexander Viro , Jan Kara , Miguel Ojeda , Boqun Feng , "=?utf-8?q?Bj=C3=B6rn_Roy_Baron?=" , Benno Lossin , Andreas Hindborg , Trevor Gross , Danilo Krummrich , Gary Guo , linux-fsdevel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-kernel@vger.kernel.org, Alice Ryhl Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Rust Binder currently uses PollCondVar, but it calls synchronize_rcu() in the destructor, which we would like to avoid. Add a variation of PollCondVar that kfree_rcu() instead. One could avoid the `rcu` field and allocate the rcu_head on drop using a fallback to synchronize_rcu() on ENOMEM. However, I'd prefer to avoid the potential for synchronize_rcu(), and Binder will only use this for a small fraction of processes, so even if it changes which kmalloc bucket it falls into, the extra memory is not a problem. Signed-off-by: Alice Ryhl --- rust/kernel/sync/poll.rs | 73 ++++++++++++++++++++++++++++++++++++++++++++= +++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index 0ec985d560c8..684dfa242b1a 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -5,12 +5,18 @@ //! Utilities for working with `struct poll_table`. =20 use crate::{ + alloc::AllocError, bindings, fs::File, prelude::*, sync::{CondVar, LockClassKey}, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, // }; -use core::{marker::PhantomData, ops::Deref}; =20 /// Creates a [`PollCondVar`] initialiser with the given name and a newly-= created lock class. #[macro_export] @@ -66,6 +72,7 @@ pub fn register_wait(&self, file: &File, cv: &PollCondVar= ) { /// /// [`CondVar`]: crate::sync::CondVar #[pin_data(PinnedDrop)] +#[repr(transparent)] pub struct PollCondVar { #[pin] inner: CondVar, @@ -104,3 +111,67 @@ fn drop(self: Pin<&mut Self>) { unsafe { bindings::synchronize_rcu() }; } } + +/// A [`KBox`] that uses `kfree_rcu`. +/// +/// [`KBox`]: PollCondVar +pub struct PollCondVarBox { + inner: ManuallyDrop>>, +} + +#[pin_data] +#[repr(C)] +struct PollCondVarBoxInner { + #[pin] + inner: PollCondVar, + rcu: Opaque, +} + +// SAFETY: PollCondVar is Send +unsafe impl Send for PollCondVarBoxInner {} +// SAFETY: PollCondVar is Sync +unsafe impl Sync for PollCondVarBoxInner {} + +impl PollCondVarBox { + /// Constructs a new boxed [`PollCondVar`]. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Re= sult { + let b =3D KBox::pin_init( + pin_init!(PollCondVarBoxInner { + inner <- PollCondVar::new(name, key), + rcu: Opaque::uninit(), + }), + GFP_KERNEL, + ) + .map_err(|_| AllocError)?; + + Ok(PollCondVarBox { + inner: ManuallyDrop::new(b), + }) + } +} + +impl Deref for PollCondVarBox { + type Target =3D PollCondVar; + fn deref(&self) -> &PollCondVar { + &self.inner.inner + } +} + +impl Drop for PollCondVarBox { + #[inline] + fn drop(&mut self) { + // SAFETY: ManuallyDrop::take ok because not already taken. + let boxed =3D unsafe { ManuallyDrop::take(&mut self.inner) }; + + // SAFETY: The code below frees the box without calling the actual= destructor of the type, + // but it's okay because it re-implements the destructor using `kf= ree_rcu()` in place of + // `synchronize_rcu()`. + let ptr =3D KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxe= d) }); + + // SAFETY: The pointer points at a valid `wait_queue_head`. + unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queu= e_head.get()) }; + + // SAFETY: This was allocated using `KBox::pin_init`, so it can be= freed with `kvfree`. + unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::()) }; + } +} --=20 2.54.0.1099.g489fc7bff1-goog From nobody Fri Jun 12 07:28:03 2026 Received: from mail-wm1-f73.google.com (mail-wm1-f73.google.com [209.85.128.73]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id F19823BB131 for ; Thu, 11 Jun 2026 09:29:20 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.128.73 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1781170164; cv=none; b=cTfEJNyDxSAYtGSErEVIYAn7LSCnS7wiAMgqfKrIRHHX2ObmerByH8Q1/zG3Fe2U/KHm8EpHKN38pQWY8EGoplw7fzQnIzVxoF/PFzMDjfivImHxbn+80ratX+tKHyFTorGOA/YusA3pi5wMojppG0T01LFxOX/EVOTxJIWaVro= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1781170164; c=relaxed/simple; bh=AoyijZYrXWJlNT7tfAdWNZxkT4dPqvu3zDJVwaV/nYY=; h=Date:In-Reply-To:Mime-Version:References:Message-ID:Subject:From: To:Cc:Content-Type; b=t7krPOq/GqqtXiUfCqQ8ToBcXGgMIvj9DeE7z/g4gNuv9a5Oc/Hcwbi3ihr6pHnx6g1o+znJXwLECdr5dRi8FrZ2BjZPjrMBR/NrbRxGCy93VOEni3Pl4NF2y7BIvDS2Ou6in2udIhejJOazK0kOhW2XE/GhZjXh2juhiBCPkzE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b=hkAXUqFK; arc=none smtp.client-ip=209.85.128.73 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=reject dis=none) header.from=google.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=flex--aliceryhl.bounces.google.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b="hkAXUqFK" Received: by mail-wm1-f73.google.com with SMTP id 5b1f17b1804b1-490b61243easo56224805e9.2 for ; Thu, 11 Jun 2026 02:29:20 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20251104; t=1781170159; x=1781774959; darn=vger.kernel.org; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:from:to:cc:subject:date:message-id:reply-to; bh=mphscAFjnFy4ybtIeT18ZIxWm8T5THYSBs/hbomTrAY=; b=hkAXUqFKFdLXM8uuB16p592g8tFI56MuuYGmxurzSPyCbOUaCHlcCissmqOgZVdBvZ jX3ulVx+OKLvxe0NXL1/eAB3++JU7/Q398+n9hXU4abAEnu8nfP/UQjJQjnOA3d+GgZ4 X69QXpT5NKflm1JmvNpSRkkyFqYJ0a3DPhUyxJKe8bNFVtdaLmoJiXiRNK2APWUMO37z v+sxDVV9wOVPniXIlUgkngN9Zuj/VoHKGyyHXz7tzhIeM2i3bmX3S5ANDb2RYyT2DoQA Yv0qFPUCyeH2GUgyfnSCki9+rIAW3RSW4fS6Nel2w1MyTJh6lcCb5zEkf+9H91VL+INo rouA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20251104; t=1781170159; x=1781774959; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=mphscAFjnFy4ybtIeT18ZIxWm8T5THYSBs/hbomTrAY=; b=PW+1efP69rwzJci5SmnDn0e+Ic9t+JiXSC7qgjhUS+9vwrGhHtHGWeaaO0cDTmLYp6 OhiByzvz5y+WE0XmZsL4bsw48v5fy6x0qr9KjGQmJAiW8zwgSrEi3F4adljl6pm4YWsY qWhHyxerMN2e7EDqhRGwGHgCxfkDSENofv0ycmdeREI+TgOpje7c8j0orH+xCwDNK8HQ rwfT8M57KPy5/SvsF9ab2HXbDXFYa0mihnNhJw9yMwCc4jMPJpzffNnDc+0f6DJsmWWg 4RBpaf4u+3NZRcVpP7ptLcergGKfNqizrMsq/NEGiZRCcfiBkJOBfVmab9SKGV58Viwe DEAA== X-Forwarded-Encrypted: i=1; AFNElJ+uFgQGMD4apWpJlYSFP/hn0q5bK+VewbMzlvRFQjxylWBtZEL3/JLIO/6QlUxkBtju7GwUB4xn5R4aWFU=@vger.kernel.org X-Gm-Message-State: AOJu0YwAOZaq/HfvhLmeU3i0PUL1vIhykUYCPGm2dmDvJ6cdGHuafzsT Ui1R9kIPME0KhAoQK1PFtPU2QfbGvKXseVBhZdR4JhXUk9vD9Fv/fHJJfc9O1sAAJDpVHiGGLXX IC7qqw61tNwsDgi4sOA== X-Received: from wmpi26.prod.google.com ([2002:a05:600c:4b1a:b0:48f:dc9f:6f1a]) (user=aliceryhl job=prod-delivery.src-stubby-dispatcher) by 2002:a05:600c:c093:b0:490:b28d:a6f9 with SMTP id 5b1f17b1804b1-490e55d4a5bmr23696565e9.8.1781170159264; Thu, 11 Jun 2026 02:29:19 -0700 (PDT) Date: Thu, 11 Jun 2026 09:29:01 +0000 In-Reply-To: <20260611-upgrade-poll-v5-0-497a11c06828@google.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 References: <20260611-upgrade-poll-v5-0-497a11c06828@google.com> X-Developer-Key: i=aliceryhl@google.com; a=openpgp; fpr=49F6C1FAA74960F43A5B86A1EE7A392FDE96209F X-Developer-Signature: v=1; a=openpgp-sha256; l=17915; i=aliceryhl@google.com; h=from:subject:message-id; bh=AoyijZYrXWJlNT7tfAdWNZxkT4dPqvu3zDJVwaV/nYY=; b=owEBbQKS/ZANAwAKAQRYvu5YxjlGAcsmYgBqKn/pQFEA2XIVJlm2+LBHe6toRW9d/1TTbJhRf HR+vxTf+36JAjMEAAEKAB0WIQSDkqKUTWQHCvFIvbIEWL7uWMY5RgUCaip/6QAKCRAEWL7uWMY5 Ru1vEACjuCsEaei8rYHxznABYXouuDAHvivyvIqvG50WiJ2LsV/pFraXFR+oVassya97zFnxnns 6a0TIvDBGYjC0OTSNL74fbRsOcVNVJqLe6Rt9CwHuJynTbdSiTHoEj+mS6jeJE4J1nlmeF6ZlF+ uQXtwPiXvxFPud/qOEKFT7GjyH3O4tQwAf3PtIBHaXIE8Xsmm2VOvQxqvdI0GjOOuk4cSJ5DByR qIPoN/3G1Jo+L+ql4NEnOKrjXH8bv1mydU9WANf+Ly5szSI2jsNKt6wjQmNzlE1DZZ3Jb2yOQfq IhCCboBORz8OA4LlIldS+SwJ5SPCvuIk1P0ofCrST4gkgX3lAhTJG0+MZK6VPQ2CVp10C1V+q4o lCqU5HWIyVCMOlaU8pyNhUoPAmMSHvOeu6kDvnkbv66nq1LLoc0djpdZimH+/XFatvSZTGdty/Z n3WocwYSou9PB0jqFDJWd3+6wnOZZu5Fy4bOdIEWBC0ruKmCsswSbYwkUgl4VF+1qb/SHG7pRzI TdKmimb09Dooi773jwUmrRT0Qde8Y6OVuRG+y8oAchHa0O2OospnQdbLKGfpS+NPmRu5qPmwqAT RuaY8+N37AP1kkDGu2pxZ1t0hBYoXCY8t4ngJUPEEcwmFAH0PW76QMGwSXb++7G2K4dqj/XyLfi T2O+9eAnowDOo3Q== X-Mailer: b4 0.14.3 Message-ID: <20260611-upgrade-poll-v5-2-497a11c06828@google.com> Subject: [PATCH v5 2/2] rust_binder: move (e)poll wait queue to Process From: Alice Ryhl To: Greg Kroah-Hartman , Carlos Llamas , Christian Brauner , Boqun Feng Cc: "Paul E. McKenney" , Alexander Viro , Jan Kara , Miguel Ojeda , Boqun Feng , "=?utf-8?q?Bj=C3=B6rn_Roy_Baron?=" , Benno Lossin , Andreas Hindborg , Trevor Gross , Danilo Krummrich , Gary Guo , linux-fsdevel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-kernel@vger.kernel.org, Alice Ryhl Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Most processes do not use Rust Binder with epoll, so avoid paying the synchronize_rcu() cost in drop for those that don't need it. For those that do, we also manage to replace synchronize_rcu() with kfree_rcu(), though we introduce an extra allocation. In case the last ref to an Arc is dropped outside of deferred_release(), this also ensures that synchronize_rcu() is not called in destructor of Arc in other places. Theoretically that could lead to jank by making other syscalls slow, which would be problematic. Signed-off-by: Alice Ryhl --- drivers/android/binder/node.rs | 4 +- drivers/android/binder/process.rs | 67 ++++++++++++++++++++++-------- drivers/android/binder/thread.rs | 78 +++++++++++++++++--------------= ---- drivers/android/binder/transaction.rs | 6 ++- 4 files changed, 95 insertions(+), 60 deletions(-) diff --git a/drivers/android/binder/node.rs b/drivers/android/binder/node.rs index fb27674a8c94..8041677908fd 100644 --- a/drivers/android/binder/node.rs +++ b/drivers/android/binder/node.rs @@ -536,7 +536,7 @@ pub(crate) fn submit_oneway( inner.oneway_todo.push_back(transaction); } else { inner.has_oneway_transaction =3D true; - guard.push_work(transaction)?; + guard.push_work(&self.owner, transaction)?; } Ok(()) } @@ -568,7 +568,7 @@ pub(crate) fn pending_oneway_finished(&self) { let transaction =3D inner.oneway_todo.pop_front(); inner.has_oneway_transaction =3D transaction.is_some(); if let Some(transaction) =3D transaction { - match guard.push_work(transaction) { + match guard.push_work(&self.owner, transaction) { Ok(()) =3D> {} Err((_err, work)) =3D> { // Process is dead. diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/pro= cess.rs index 82c34a93660e..fd09e177f7e8 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -30,7 +30,8 @@ sync::{ aref::ARef, lock::{spinlock::SpinLockBackend, Guard}, - Arc, ArcBorrow, CondVar, CondVarTimeoutResult, SpinLock, UniqueArc, + poll::PollCondVarBox, + Arc, ArcBorrow, CondVar, CondVarTimeoutResult, SetOnce, SpinLock, = UniqueArc, }, task::Task, uaccess::{UserSlice, UserSliceReader}, @@ -172,21 +173,26 @@ fn new() -> Self { /// taken while holding the inner process lock. pub(crate) fn push_work( &mut self, + proc: &Process, work: DLArc, ) -> Result<(), (BinderError, DLArc)> { + let sync =3D work.should_sync_wakeup(); + // Try to find a ready thread to which to push the work. if let Some(thread) =3D self.ready_threads.pop_front() { // Push to thread while holding state lock. This prevents the = thread from giving up // (for example, because of a signal) when we're about to deli= ver work. - match thread.push_work(work) { + match thread.push_work_inner(work, sync) { PushWorkRes::Ok =3D> Ok(()), + PushWorkRes::OkNotifyPoll =3D> { + proc.notify_poll(sync); + Ok(()) + } PushWorkRes::FailedDead(work) =3D> Err((BinderError::new_d= ead(), work)), } } else if self.is_dead { Err((BinderError::new_dead(), work)) } else { - let sync =3D work.should_sync_wakeup(); - // Didn't find a thread waiting for proc work; this can happen // in two scenarios: // 1. All threads are busy handling transactions @@ -194,17 +200,12 @@ pub(crate) fn push_work( // the kernel driver soon and pick up this work. // 2. Threads are using the (e)poll interface, in which case // they may be blocked on the waitqueue without having been - // added to waiting_threads. For this case, we just iterate - // over all threads not handling transaction work, and - // wake them all up. We wake all because we don't know whet= her - // a thread that called into (e)poll is handling non-binder - // work currently. + // added to waiting_threads. For this case, we wake it up + // directly. self.work.push_back(work); =20 // Wake up polling threads, if any. - for thread in self.threads.values() { - thread.notify_if_poll_ready(sync); - } + proc.notify_poll(sync); =20 Ok(()) } @@ -227,11 +228,11 @@ pub(crate) fn update_node_refcount( =20 // If we decided that we need to push work, push either to the pro= cess or to a thread if // one is specified. - if let Some(node) =3D push { + if let Some(pnode) =3D push { if let Some(thread) =3D othread { - thread.push_work_deferred(node); + thread.push_work_deferred(pnode); } else { - let _ =3D self.push_work(node); + let _ =3D self.push_work(&node.owner, pnode); // Nothing to do: `push_work` may fail if the process is d= ead, but that's ok as in // that case, it doesn't care about the notification. } @@ -457,6 +458,12 @@ pub(crate) struct Process { #[pin] node_refs: SpinLock, =20 + // Synchronizes `register_wait` calls to the `PollCondVarBox`. + // + // The `PollCondVarBox` is not stored here because synchronization is + // done for `register_wait` only. Wakeups do not take this lock. + poll: SetOnce, + // Work node for deferred work item. #[pin] defer_work: Work, @@ -516,6 +523,7 @@ fn new(ctx: Arc, cred: ARef) -> Re= sult> { defer_work <- kernel::new_work!("Process::defer_work"), links <- ListLinks::new(), stats: BinderStats::new(), + poll: SetOnce::new(), }), GFP_KERNEL, )?; @@ -715,7 +723,7 @@ fn get_current_thread(self: ArcBorrow<'_, Self>) -> Res= ult> { =20 pub(crate) fn push_work(&self, work: DLArc) -> Bind= erResult { // If push_work fails, drop the work item outside the lock. - let res =3D self.inner.lock().push_work(work); + let res =3D self.inner.lock().push_work(self, work); match res { Ok(()) =3D> Ok(()), Err((err, work)) =3D> { @@ -1004,7 +1012,7 @@ pub(crate) fn inc_ref_done(&self, reader: &mut UserSl= iceReader, strong: bool) -> if let Ok(Some(node)) =3D inner.get_existing_node(ptr, cookie) { if let Some(node) =3D node.inc_ref_done_locked(strong, &mut in= ner) { // This only fails if the process is dead. - let _ =3D inner.push_work(node); + let _ =3D inner.push_work(self, node); } } Ok(()) @@ -1521,6 +1529,15 @@ pub(crate) fn ioctl_freeze(&self, info: &BinderFreez= eInfo) -> Result { } } } + + pub(crate) fn notify_poll(&self, sync: bool) { + if let Some(poll) =3D self.poll.as_ref() { + if sync { + poll.notify_sync(); + } + poll.notify_all(); + } + } } =20 fn get_frozen_status(data: UserSlice) -> Result { @@ -1712,7 +1729,21 @@ pub(crate) fn poll( table: PollTable<'_>, ) -> Result { let thread =3D this.get_current_thread()?; - let (from_proc, mut mask) =3D thread.poll(file, table); + { + let poll =3D loop { + if let Some(poll) =3D this.poll.as_ref() { + break poll; + } + + let poll =3D PollCondVarBox::new(c"Process::poll", kernel:= :static_lock_class!())?; + // Reuse our existing lock to synchronize callers initiali= zing. + let _guard =3D this.node_refs.lock(); + this.poll.populate(poll); + }; + + table.register_wait(file, poll); + } + let (from_proc, mut mask) =3D thread.poll()?; if mask =3D=3D 0 && from_proc && !this.inner.lock().work.is_empty(= ) { mask |=3D bindings::POLLIN; } diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thre= ad.rs index 97d5f31e8fe3..57016971c065 100644 --- a/drivers/android/binder/thread.rs +++ b/drivers/android/binder/thread.rs @@ -9,15 +9,14 @@ =20 use kernel::{ bindings, - fs::{File, LocalFile}, + fs::LocalFile, list::{AtomicTracker, List, ListArc, ListLinks, TryNewListArc}, prelude::*, security, seq_file::SeqFile, seq_print, sync::atomic::{ordering::Relaxed, Atomic}, - sync::poll::{PollCondVar, PollTable}, - sync::{aref::ARef, Arc, SpinLock}, + sync::{aref::ARef, Arc, CondVar, SpinLock}, task::Task, uaccess::{UserPtr, UserSlice, UserSliceReader}, uapi, @@ -225,8 +224,10 @@ fn claim_next(&mut self, size: usize) -> Result= { } } =20 +#[must_use] pub(crate) enum PushWorkRes { Ok, + OkNotifyPoll, FailedDead(DLArc), } =20 @@ -234,6 +235,7 @@ impl PushWorkRes { fn is_ok(&self) -> bool { match self { PushWorkRes::Ok =3D> true, + PushWorkRes::OkNotifyPoll =3D> true, PushWorkRes::FailedDead(_) =3D> false, } } @@ -310,27 +312,32 @@ fn pop_work(&mut self) -> Option> { =20 fn push_work(&mut self, work: DLArc) -> PushWorkRes= { if self.is_dead { - PushWorkRes::FailedDead(work) + return PushWorkRes::FailedDead(work); + } + self.work_list.push_back(work); + self.process_work_list =3D true; + if self.looper_flags & LOOPER_POLL !=3D 0 { + PushWorkRes::OkNotifyPoll } else { - self.work_list.push_back(work); - self.process_work_list =3D true; PushWorkRes::Ok } } =20 - fn push_reply_work(&mut self, code: u32) { + fn push_reply_work(&mut self, code: u32) -> PushWorkRes { if let Ok(work) =3D ListArc::try_from_arc(self.reply_work.clone())= { work.set_error_code(code); - self.push_work(work); + self.push_work(work) } else { pr_warn!("Thread reply work is already in use."); + PushWorkRes::Ok } } =20 fn push_return_work(&mut self, reply: u32) { if let Ok(work) =3D ListArc::try_from_arc(self.return_work.clone()= ) { work.set_error_code(reply); - self.push_work(work); + // Not notifying: Reply to current thread. + let _ =3D self.push_work(work); } else { pr_warn!("Thread return work is already in use."); } @@ -422,7 +429,7 @@ pub(crate) struct Thread { #[pin] inner: SpinLock, #[pin] - work_condvar: PollCondVar, + work_condvar: CondVar, /// Used to insert this thread into the process' `ready_threads` list. /// /// INVARIANT: May never be used for any other list than the `self.pro= cess.ready_threads`. @@ -453,7 +460,7 @@ pub(crate) fn new(id: i32, process: Arc) -> Re= sult> { process, task: ARef::from(&**kernel::current!()), inner <- kernel::new_spinlock!(inner, "Thread::inner"), - work_condvar <- kernel::new_poll_condvar!("Thread::work_co= ndvar"), + work_condvar <- kernel::new_condvar!("Thread::work_condvar= "), links <- ListLinks::new(), links_track <- AtomicTracker::new(), }), @@ -617,7 +624,14 @@ fn get_work(self: &Arc, wait: bool) -> Result) -> Push= WorkRes { let sync =3D work.should_sync_wakeup(); + self.push_work_inner(work, sync) + } =20 + pub(crate) fn push_work_inner( + &self, + work: DLArc, + sync: bool, + ) -> PushWorkRes { let res =3D self.inner.lock().push_work(work); =20 if res.is_ok() { @@ -636,7 +650,8 @@ pub(crate) fn push_work(&self, work: DLArc) -> PushWorkRes { pub(crate) fn push_work_if_looper(&self, work: DLArc) -> BinderResult { let mut inner =3D self.inner.lock(); if inner.is_looper() && !inner.is_dead { - inner.push_work(work); + // Not notifying: Reply to current thread. + let _ =3D inner.push_work(work); Ok(()) } else { drop(inner); @@ -1142,7 +1157,7 @@ fn deliver_single_reply( transaction.set_outstanding(&mut self.process.inner.lock()); } =20 - { + let ret =3D { let mut inner =3D self.inner.lock(); if !inner.pop_transaction_replied(transaction) { return false; @@ -1153,15 +1168,16 @@ fn deliver_single_reply( } =20 match reply { - Ok(work) =3D> { - inner.push_work(work); - } + Ok(work) =3D> inner.push_work(work), Err(code) =3D> inner.push_reply_work(code), } - } + }; =20 // Notify the thread now that we've released the inner lock. self.work_condvar.notify_sync(); + if matches!(ret, PushWorkRes::OkNotifyPoll) { + self.process.notify_poll(true); + } false } =20 @@ -1319,7 +1335,8 @@ fn reply_inner(self: &Arc, info: &mut Transacti= onInfo) -> BinderResult { let process =3D orig.from.process.clone(); let allow_fds =3D orig.flags & TF_ACCEPT_FDS !=3D 0; let reply =3D Transaction::new_reply(self, process, info, allo= w_fds)?; - self.inner.lock().push_work(completion); + // Not notifying: Reply to current thread. + let _ =3D self.inner.lock().push_work(completion); orig.from.deliver_reply(Ok(reply), &orig); Ok(()) })() @@ -1351,7 +1368,8 @@ fn oneway_transaction_inner(self: &Arc, info: &= mut TransactionInfo) -> Bin }; let list_completion =3D DTRWrap::arc_try_new(DeliverCode::new(code= ))?; let completion =3D list_completion.clone_arc(); - self.inner.lock().push_work(list_completion); + // Not notifying: Reply to current thread. + let _ =3D self.inner.lock().push_work(list_completion); match transaction.submit(info) { Ok(()) =3D> Ok(()), Err(err) =3D> { @@ -1551,10 +1569,9 @@ pub(crate) fn write_read(self: &Arc, data: Use= rSlice, wait: bool) -> Resul ret } =20 - pub(crate) fn poll(&self, file: &File, table: PollTable<'_>) -> (bool,= u32) { - table.register_wait(file, &self.work_condvar); + pub(crate) fn poll(&self) -> Result<(bool, u32)> { let mut inner =3D self.inner.lock(); - (inner.should_use_process_work_queue(), inner.poll()) + Ok((inner.should_use_process_work_queue(), inner.poll())) } =20 /// Make the call to `get_work` or `get_work_local` return immediately= , if any. @@ -1571,26 +1588,9 @@ pub(crate) fn exit_looper(&self) { } } =20 - pub(crate) fn notify_if_poll_ready(&self, sync: bool) { - // Determine if we need to notify. This requires the lock. - let inner =3D self.inner.lock(); - let notify =3D inner.looper_flags & LOOPER_POLL !=3D 0 && inner.sh= ould_use_process_work_queue(); - drop(inner); - - // Now that the lock is no longer held, notify the waiters if we h= ave to. - if notify { - if sync { - self.work_condvar.notify_sync(); - } else { - self.work_condvar.notify_one(); - } - } - } - pub(crate) fn release(self: &Arc) { self.inner.lock().is_dead =3D true; =20 - //self.work_condvar.clear(); self.unwind_transaction_stack(); =20 // Cancel all pending work items. diff --git a/drivers/android/binder/transaction.rs b/drivers/android/binder= /transaction.rs index 1d9b66920a21..474f9d231f85 100644 --- a/drivers/android/binder/transaction.rs +++ b/drivers/android/binder/transaction.rs @@ -331,11 +331,15 @@ pub(crate) fn submit(self: DLArc, info: &mut Tr= ansactionInfo) -> BinderRes crate::trace::trace_transaction(false, &self, Some(&thread.tas= k)); match thread.push_work(self) { PushWorkRes::Ok =3D> Ok(()), + PushWorkRes::OkNotifyPoll =3D> { + process.notify_poll(true); + Ok(()) + } PushWorkRes::FailedDead(me) =3D> Err((BinderError::new_dea= d(), me)), } } else { crate::trace::trace_transaction(false, &self, None); - process_inner.push_work(self) + process_inner.push_work(&process, self) }; drop(process_inner); =20 --=20 2.54.0.1099.g489fc7bff1-goog