Issue #345: a poorly-performing but usable subflow (high latency, loss,
bufferbloat) can soak up connection resources and cause head-of-line
blocking of the aggregate stream. Give the default packet scheduler a way
to send less than such a subflow's full congestion window.
Once a subflow has been picked for transmission, flag it for penalisation
when:
- its smoothed delivery rate (avg_pacing_rate) is below half that of the
fastest path, keying on rate, not RTT, so a slow-but-high-throughput
path is left alone;
- the fastest path is cwnd-limited (saturated), so shifting load off the
slow path is worthwhile;
- the subflow is in TCP_CA_Open, so its cwnd is not already being reduced
by loss recovery;
- it has not been penalised in the last RTT.
The reduction halves tcp_snd_cwnd (floor 2) and ssthresh if cwnd
is past it. It is applied in the push path under the subflow socket lock,
which protects snd_cwnd (the scheduler runs under the msk lock). The
congestion control grows the window back, ACK-clocked; that regrowth is
the built-in probe, so no explicit MPTCP-side probing is needed.
Co-developed-by: Matthieu Baerts (NGI0) <matttbe@kernel.org>
Signed-off-by: Shardul Bankar <shardul.b@mpiricsoftware.com>
---
net/mptcp/protocol.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++++---
net/mptcp/protocol.h | 2 ++
2 files changed, 67 insertions(+), 3 deletions(-)
diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
index 954e20bb27de..d31bcb9ad894 100644
--- a/net/mptcp/protocol.c
+++ b/net/mptcp/protocol.c
@@ -1556,6 +1556,44 @@ bool mptcp_subflow_active(struct mptcp_subflow_context *subflow)
#define SSK_MODE_BACKUP 1
#define SSK_MODE_MAX 2
+/* Penalise a subflow whose delivery (pacing) rate is below the fraction
+ * 1 / MPTCP_PENALISE_RATE_RATIO of the fastest path's rate. Keying on rate,
+ * not RTT, throttles only a path whose throughput contribution is small
+ * relative to the head-of-line cost it imposes, and leaves a merely
+ * higher-latency but high-throughput path alone.
+ */
+#define MPTCP_PENALISE_RATE_RATIO 2
+
+/* Rate-limit the penalty to at most once per subflow RTT, so the congestion
+ * control can grow the window back between reductions.
+ */
+static bool mptcp_penalise_throttle_ok(struct mptcp_subflow_context *subflow)
+{
+ struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
+ u32 rtt = usecs_to_jiffies(tcp_sk(ssk)->srtt_us >> 3);
+
+ return tcp_jiffies32 - subflow->last_penalise >= max_t(u32, rtt, 1);
+}
+
+/* Halve the congestion window (and ssthresh, if cwnd is past it) of a subflow
+ * the scheduler flagged. Runs in the push path under the subflow socket lock,
+ * which protects snd_cwnd. The congestion control grows the window back,
+ * ACK-clocked, and that regrowth is the built-in probe, so no explicit probing
+ * is needed.
+ */
+static void mptcp_penalise_cwnd(struct sock *ssk)
+{
+ struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
+ struct tcp_sock *tp = tcp_sk(ssk);
+ u32 cwnd = tcp_snd_cwnd(tp);
+
+ subflow->penalise = false;
+ subflow->last_penalise = tcp_jiffies32;
+ tcp_snd_cwnd_set(tp, max_t(u32, cwnd >> 1, 2));
+ if (cwnd >= tp->snd_ssthresh)
+ tp->snd_ssthresh = max_t(u32, tp->snd_ssthresh >> 1, 2);
+}
+
/* implement the mptcp packet scheduler;
* returns the subflow that will transmit the next DSS
* additionally updates the rtx timeout
@@ -1565,9 +1603,9 @@ struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk)
struct subflow_send_info send_info[SSK_MODE_MAX];
struct mptcp_subflow_context *subflow;
struct sock *sk = (struct sock *)msk;
- u32 pace, burst, wmem;
+ u32 pace, burst, wmem, max_pace = 0;
int i, nr_active = 0;
- struct sock *ssk;
+ struct sock *ssk, *fastest = NULL;
u64 linger_time;
long tout = 0;
@@ -1596,6 +1634,14 @@ struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk)
continue;
}
+ /* track the fastest path by delivery rate; the penalty below
+ * throttles paths that are slow relative to it.
+ */
+ if (pace > max_pace) {
+ max_pace = pace;
+ fastest = ssk;
+ }
+
linger_time = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, pace);
if (linger_time < send_info[backup].linger_time) {
send_info[backup].ssk = ssk;
@@ -1623,12 +1669,25 @@ struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk)
if (!ssk || !sk_stream_memory_free(ssk))
return NULL;
+ /* Flag the chosen subflow for cwnd halving (applied in the push path)
+ * when its delivery rate is a small fraction of the fastest path's and
+ * that fast path is saturated (cwnd-limited), so moving load off the
+ * slow path is worthwhile. Only penalise a path in TCP_CA_Open, one
+ * whose cwnd is not already being shrunk by loss recovery, and at most
+ * once per RTT.
+ */
+ subflow = mptcp_subflow_ctx(ssk);
+ subflow->penalise = fastest && ssk != fastest &&
+ (u64)subflow->avg_pacing_rate * MPTCP_PENALISE_RATE_RATIO < max_pace &&
+ inet_csk(ssk)->icsk_ca_state == TCP_CA_Open &&
+ tcp_is_cwnd_limited(fastest) &&
+ mptcp_penalise_throttle_ok(subflow);
+
burst = min(MPTCP_SEND_BURST_SIZE, mptcp_wnd_end(msk) - msk->snd_nxt);
wmem = READ_ONCE(ssk->sk_wmem_queued);
if (!burst)
return ssk;
- subflow = mptcp_subflow_ctx(ssk);
subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem +
READ_ONCE(ssk->sk_pacing_rate) * burst,
burst + wmem);
@@ -1685,6 +1744,9 @@ static int __subflow_push_pending(struct sock *sk, struct sock *ssk,
struct mptcp_data_frag *dfrag;
int len, copied = 0, err = 0;
+ if (mptcp_subflow_ctx(ssk)->penalise)
+ mptcp_penalise_cwnd(ssk);
+
while ((dfrag = mptcp_send_head(sk))) {
info->sent = dfrag->already_sent;
info->limit = dfrag->data_len;
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index da40c6f3705f..2bf801292563 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -587,6 +587,7 @@ struct mptcp_subflow_context {
__unused : 8;
bool data_avail;
bool scheduled;
+ bool penalise; /* scheduler flagged this subflow for cwnd halving */
bool pm_listener; /* a listener managed by the kernel PM? */
bool fully_established; /* path validated */
u32 lent_mem_frag;
@@ -606,6 +607,7 @@ struct mptcp_subflow_context {
u8 stale_count;
u32 subflow_id;
+ u32 last_penalise; /* tcp_jiffies32 of the last cwnd penalty */
long delegated_status;
unsigned long fail_tout;
--
2.34.1Hi Shardul,
On 26/07/2026 07:55, Shardul Bankar wrote:
> Issue #345: a poorly-performing but usable subflow (high latency, loss,
> bufferbloat) can soak up connection resources and cause head-of-line
> blocking of the aggregate stream. Give the default packet scheduler a way
> to send less than such a subflow's full congestion window.
>
> Once a subflow has been picked for transmission, flag it for penalisation
> when:
> - its smoothed delivery rate (avg_pacing_rate) is below half that of the
> fastest path, keying on rate, not RTT, so a slow-but-high-throughput
> path is left alone;
> - the fastest path is cwnd-limited (saturated), so shifting load off the
> slow path is worthwhile;
> - the subflow is in TCP_CA_Open, so its cwnd is not already being reduced
> by loss recovery;
> - it has not been penalised in the last RTT.
>
> The reduction halves tcp_snd_cwnd (floor 2) and ssthresh if cwnd
> is past it. It is applied in the push path under the subflow socket lock,
> which protects snd_cwnd (the scheduler runs under the msk lock). The
> congestion control grows the window back, ACK-clocked; that regrowth is
> the built-in probe, so no explicit MPTCP-side probing is needed.
>
> Co-developed-by: Matthieu Baerts (NGI0) <matttbe@kernel.org>
> Signed-off-by: Shardul Bankar <shardul.b@mpiricsoftware.com>
> ---
> net/mptcp/protocol.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++++---
> net/mptcp/protocol.h | 2 ++
> 2 files changed, 67 insertions(+), 3 deletions(-)
>
> diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
> index 954e20bb27de..d31bcb9ad894 100644
> --- a/net/mptcp/protocol.c
> +++ b/net/mptcp/protocol.c
> @@ -1556,6 +1556,44 @@ bool mptcp_subflow_active(struct mptcp_subflow_context *subflow)
> #define SSK_MODE_BACKUP 1
> #define SSK_MODE_MAX 2
>
> +/* Penalise a subflow whose delivery (pacing) rate is below the fraction
> + * 1 / MPTCP_PENALISE_RATE_RATIO of the fastest path's rate. Keying on rate,
> + * not RTT, throttles only a path whose throughput contribution is small
> + * relative to the head-of-line cost it imposes, and leaves a merely
> + * higher-latency but high-throughput path alone.
Could you reduce the size of the comments, please? I understood that LLM
tends to leave long comments, but too long is not always good:
explanations can go in the commit message, or be understood by reading
the conditions (or that's a sign the code should be improved). Ideally,
comments should not be needed when reading the code, or limited to one,
with some exceptions for complex cases.
Here for example, I think you could limit the comment to one line:
/* Penalise subflows with pacing rate < this fraction of the fastest path */
> + */
> +#define MPTCP_PENALISE_RATE_RATIO 2
> +
> +/* Rate-limit the penalty to at most once per subflow RTT, so the congestion
> + * control can grow the window back between reductions.
> + */
> +static bool mptcp_penalise_throttle_ok(struct mptcp_subflow_context *subflow)
> +{
> + struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
> + u32 rtt = usecs_to_jiffies(tcp_sk(ssk)->srtt_us >> 3);
> +
> + return tcp_jiffies32 - subflow->last_penalise >= max_t(u32, rtt, 1);
> +}
> +
> +/* Halve the congestion window (and ssthresh, if cwnd is past it) of a subflow
> + * the scheduler flagged. Runs in the push path under the subflow socket lock,
> + * which protects snd_cwnd. The congestion control grows the window back,
> + * ACK-clocked, and that regrowth is the built-in probe, so no explicit probing
> + * is needed.
> + */
Same here: I think what is important to mention is that it is under
subflow socket lock, and you can move the comment about the CC growing
the window back below: no explicit probing required then.
> +static void mptcp_penalise_cwnd(struct sock *ssk)
> +{
> + struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
> + struct tcp_sock *tp = tcp_sk(ssk);
> + u32 cwnd = tcp_snd_cwnd(tp);
> +
> + subflow->penalise = false;
You probably need to check inet_csk(ssk)->icsk_ca_state == TCP_CA_Open
again here, under the subflow socket lock, just in case it has been
modified in between.
> + subflow->last_penalise = tcp_jiffies32;
> + tcp_snd_cwnd_set(tp, max_t(u32, cwnd >> 1, 2));
> + if (cwnd >= tp->snd_ssthresh)
> + tp->snd_ssthresh = max_t(u32, tp->snd_ssthresh >> 1, 2);
> +}
> +
> /* implement the mptcp packet scheduler;
> * returns the subflow that will transmit the next DSS
> * additionally updates the rtx timeout
> @@ -1565,9 +1603,9 @@ struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk)
> struct subflow_send_info send_info[SSK_MODE_MAX];
> struct mptcp_subflow_context *subflow;
> struct sock *sk = (struct sock *)msk;
> - u32 pace, burst, wmem;
> + u32 pace, burst, wmem, max_pace = 0;
Sashiko is (rightly I think) mentioning that the size of these variables
are wrong. I guess a separate fix is needed to change "pace" to
"unsigned long" is required. "max_pace" will get this type too then.
> int i, nr_active = 0;
> - struct sock *ssk;
> + struct sock *ssk, *fastest = NULL;
> u64 linger_time;
> long tout = 0;
>
> @@ -1596,6 +1634,14 @@ struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk)
> continue;
> }
>
> + /* track the fastest path by delivery rate; the penalty below
> + * throttles paths that are slow relative to it.
> + */
> + if (pace > max_pace) {
> + max_pace = pace;
> + fastest = ssk;
> + }
> +
> linger_time = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, pace);
> if (linger_time < send_info[backup].linger_time) {
> send_info[backup].ssk = ssk;
> @@ -1623,12 +1669,25 @@ struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk)
> if (!ssk || !sk_stream_memory_free(ssk))
> return NULL;
>
> + /* Flag the chosen subflow for cwnd halving (applied in the push path)
> + * when its delivery rate is a small fraction of the fastest path's and
> + * that fast path is saturated (cwnd-limited), so moving load off the
> + * slow path is worthwhile. Only penalise a path in TCP_CA_Open, one
> + * whose cwnd is not already being shrunk by loss recovery, and at most
> + * once per RTT.
> + */
Same here. Maybe a comment is not needed, or limited?
> + subflow = mptcp_subflow_ctx(ssk);
> + subflow->penalise = fastest && ssk != fastest &&
> + (u64)subflow->avg_pacing_rate * MPTCP_PENALISE_RATE_RATIO < max_pace &&
(or divide max_pace once before?)
> + inet_csk(ssk)->icsk_ca_state == TCP_CA_Open &&
> + tcp_is_cwnd_limited(fastest) &&
> + mptcp_penalise_throttle_ok(subflow);
> +
> burst = min(MPTCP_SEND_BURST_SIZE, mptcp_wnd_end(msk) - msk->snd_nxt);
> wmem = READ_ONCE(ssk->sk_wmem_queued);
> if (!burst)
> return ssk;
>
> - subflow = mptcp_subflow_ctx(ssk);
> subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem +
> READ_ONCE(ssk->sk_pacing_rate) * burst,
(here as well, it looks like there is an existing bug, and a cast to u64
is probably needed for 32-bit system, as pointed by Sashiko)
> burst + wmem);
> @@ -1685,6 +1744,9 @@ static int __subflow_push_pending(struct sock *sk, struct sock *ssk,
> struct mptcp_data_frag *dfrag;
> int len, copied = 0, err = 0;
>
> + if (mptcp_subflow_ctx(ssk)->penalise)
> + mptcp_penalise_cwnd(ssk);
> +
> while ((dfrag = mptcp_send_head(sk))) {
> info->sent = dfrag->already_sent;
> info->limit = dfrag->data_len;
> diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
> index da40c6f3705f..2bf801292563 100644
> --- a/net/mptcp/protocol.h
> +++ b/net/mptcp/protocol.h
> @@ -587,6 +587,7 @@ struct mptcp_subflow_context {
> __unused : 8;
> bool data_avail;
> bool scheduled;
> + bool penalise; /* scheduler flagged this subflow for cwnd halving */
(If you don't need to read this locklessly, then you can probably use
one unused bit.)
> bool pm_listener; /* a listener managed by the kernel PM? */
> bool fully_established; /* path validated */
> u32 lent_mem_frag;
> @@ -606,6 +607,7 @@ struct mptcp_subflow_context {
> u8 stale_count;
>
> u32 subflow_id;
> + u32 last_penalise; /* tcp_jiffies32 of the last cwnd penalty */
>
> long delegated_status;
> unsigned long fail_tout;
>
Cheers,
Matt
--
Sponsored by the NGI0 Core fund.
Hi Matt,
On Wed, 2026-07-29 at 13:49 +0200, Matthieu Baerts wrote:
> Hi Shardul,
>
> On 26/07/2026 07:55, Shardul Bankar wrote:
> >
>
> Could you reduce the size of the comments, please? I understood that
> LLM
> tends to leave long comments, but too long is not always good:
> explanations can go in the commit message, or be understood by
> reading
> the conditions (or that's a sign the code should be improved).
> Ideally,
> comments should not be needed when reading the code, or limited to
> one,
> with some exceptions for complex cases.
>
> Here for example, I think you could limit the comment to one line:
>
> /* Penalise subflows with pacing rate < this fraction of the fastest
> path */
>
Yes, shortened them to one line each and moved the rationale into the
commit messages.
> > +
> > +/* Halve the congestion window (and ssthresh, if cwnd is past it)
> > of a subflow
> > + * the scheduler flagged. Runs in the push path under the subflow
> > socket lock,
> > + * which protects snd_cwnd. The congestion control grows the
> > window back,
> > + * ACK-clocked, and that regrowth is the built-in probe, so no
> > explicit probing
> > + * is needed.
> > + */
>
> Same here: I think what is important to mention is that it is under
> subflow socket lock, and you can move the comment about the CC
> growing
> the window back below: no explicit probing required then.
>
Ack'ed.
> > +static void mptcp_penalise_cwnd(struct sock *ssk)
> > +{
> > + struct mptcp_subflow_context *subflow =
> > mptcp_subflow_ctx(ssk);
> > + struct tcp_sock *tp = tcp_sk(ssk);
> > + u32 cwnd = tcp_snd_cwnd(tp);
> > +
> > + subflow->penalise = false;
>
> You probably need to check inet_csk(ssk)->icsk_ca_state ==
> TCP_CA_Open
> again here, under the subflow socket lock, just in case it has been
> modified in between.
Added: mptcp_penalise_cwnd() now re-reads it under the subflow lock and
bails if it is no longer Open.
> > + subflow->last_penalise = tcp_jiffies32;
> > + tcp_snd_cwnd_set(tp, max_t(u32, cwnd >> 1, 2));
> > + if (cwnd >= tp->snd_ssthresh)
> > + tp->snd_ssthresh = max_t(u32, tp->snd_ssthresh >>
> > 1, 2);
> > +}
> > +
> > /* implement the mptcp packet scheduler;
> > * returns the subflow that will transmit the next DSS
> > * additionally updates the rtx timeout
> > @@ -1565,9 +1603,9 @@ struct sock *mptcp_subflow_get_send(struct
> > mptcp_sock *msk)
> > struct subflow_send_info send_info[SSK_MODE_MAX];
> > struct mptcp_subflow_context *subflow;
> > struct sock *sk = (struct sock *)msk;
> > - u32 pace, burst, wmem;
> > + u32 pace, burst, wmem, max_pace = 0;
>
> Sashiko is (rightly I think) mentioning that the size of these
> variables
> are wrong. I guess a separate fix is needed to change "pace" to
> "unsigned long" is required. "max_pace" will get this type too then.
>
I am splitting that into its own patch (1/4, Fixes: 3ce0852c86b9).
> >
> > + /* Flag the chosen subflow for cwnd halving (applied in the
> > push path)
> > + * when its delivery rate is a small fraction of the
> > fastest path's and
> > + * that fast path is saturated (cwnd-limited), so moving
> > load off the
> > + * slow path is worthwhile. Only penalise a path in
> > TCP_CA_Open, one
> > + * whose cwnd is not already being shrunk by loss recovery,
> > and at most
> > + * once per RTT.
> > + */
>
> Same here. Maybe a comment is not needed, or limited?
>
Ack'ed.
> > + subflow = mptcp_subflow_ctx(ssk);
> > + subflow->penalise = fastest && ssk != fastest &&
> > + (u64)subflow->avg_pacing_rate *
> > MPTCP_PENALISE_RATE_RATIO < max_pace &&
>
> (or divide max_pace once before?)
>
Changed to avg_pacing_rate < max_pace / MPTCP_PENALISE_RATE_RATIO.
> > + inet_csk(ssk)->icsk_ca_state ==
> > TCP_CA_Open &&
> > + tcp_is_cwnd_limited(fastest) &&
> > + mptcp_penalise_throttle_ok(subflow);
> > +
> > burst = min(MPTCP_SEND_BURST_SIZE, mptcp_wnd_end(msk) -
> > msk->snd_nxt);
> > wmem = READ_ONCE(ssk->sk_wmem_queued);
> > if (!burst)
> > return ssk;
> >
> > - subflow = mptcp_subflow_ctx(ssk);
> > subflow->avg_pacing_rate = div_u64((u64)subflow-
> > >avg_pacing_rate * wmem +
> > READ_ONCE(ssk-
> > >sk_pacing_rate) * burst,
>
> (here as well, it looks like there is an existing bug, and a cast to
> u64
> is probably needed for 32-bit system, as pointed by Sashiko)
>
Added in the same fix as the issue reported above.
> > burst + wmem);
> > @@ -1685,6 +1744,9 @@ static int __subflow_push_pending(struct sock
> > *sk, struct sock *ssk,
> > struct mptcp_data_frag *dfrag;
> > int len, copied = 0, err = 0;
> >
> > + if (mptcp_subflow_ctx(ssk)->penalise)
> > + mptcp_penalise_cwnd(ssk);
> > +
> > while ((dfrag = mptcp_send_head(sk))) {
> > info->sent = dfrag->already_sent;
> > info->limit = dfrag->data_len;
> > diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
> > index da40c6f3705f..2bf801292563 100644
> > --- a/net/mptcp/protocol.h
> > +++ b/net/mptcp/protocol.h
> > @@ -587,6 +587,7 @@ struct mptcp_subflow_context {
> > __unused : 8;
> > bool data_avail;
> > bool scheduled;
> > + bool penalise; /* scheduler flagged this
> > subflow for cwnd halving */
>
> (If you don't need to read this locklessly, then you can probably use
> one unused bit.)
>
It is written under two locks (set in the scheduler under the msk lock,
cleared in the push path under the subflow lock), so a shared bitfield
word would race on the read-modify-write, and I kept it a bool for that
reason. If you see a single-lock way to do it, I am glad to move it to
a bit.
Thanks,
Shardul
Hi Shardul,
Thank you for your reply!
On 07/08/2026 17:18, Shardul Bankar wrote:
> On Wed, 2026-07-29 at 13:49 +0200, Matthieu Baerts wrote:
>> On 26/07/2026 07:55, Shardul Bankar wrote:
(...)
>>> diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
>>> index da40c6f3705f..2bf801292563 100644
>>> --- a/net/mptcp/protocol.h
>>> +++ b/net/mptcp/protocol.h
>>> @@ -587,6 +587,7 @@ struct mptcp_subflow_context {
>>> __unused : 8;
>>> bool data_avail;
>>> bool scheduled;
>>> + bool penalise; /* scheduler flagged this
>>> subflow for cwnd halving */
>>
>> (If you don't need to read this locklessly, then you can probably use
>> one unused bit.)
>>
>
> It is written under two locks (set in the scheduler under the msk lock,
> cleared in the push path under the subflow lock), so a shared bitfield
> word would race on the read-modify-write, and I kept it a bool for that
> reason. If you see a single-lock way to do it, I am glad to move it to
> a bit.
I didn't check all entries from the bitfield a few lines above, but are
they not also written under the subflow lock? (but maybe not under the
msk lock)
But if you saw a risk, fine to keep it like that.
Cheers,
Matt
--
Sponsored by the NGI0 Core fund.
© 2016 - 2026 Red Hat, Inc.