[PULL 15/50] hw/char: sifive_uart: Print uart characters async

Alistair Francis posted 50 patches 1 year, 3 months ago
There is a newer version of this series
[PULL 15/50] hw/char: sifive_uart: Print uart characters async
Posted by Alistair Francis 1 year, 3 months ago
The current approach of using qemu_chr_fe_write() and ignoring the
return values results in dropped characters [1].

Let's update the SiFive UART to use a async sifive_uart_xmit() function
to transmit the characters and apply back pressure to the guest with
the SIFIVE_UART_TXFIFO_FULL status.

This should avoid dropped characters and more realisticly model the
hardware.

1: https://gitlab.com/qemu-project/qemu/-/issues/2114

Signed-off-by: Alistair Francis <alistair.francis@wdc.com>
Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
Tested-by: Thomas Huth <thuth@redhat.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Message-ID: <20240910045419.1252277-3-alistair.francis@wdc.com>
Signed-off-by: Alistair Francis <alistair.francis@wdc.com>
---
 include/hw/char/sifive_uart.h | 16 +++++-
 hw/char/sifive_uart.c         | 97 ++++++++++++++++++++++++++++++++---
 2 files changed, 105 insertions(+), 8 deletions(-)

diff --git a/include/hw/char/sifive_uart.h b/include/hw/char/sifive_uart.h
index 7f6c79f8bd..0846cf6218 100644
--- a/include/hw/char/sifive_uart.h
+++ b/include/hw/char/sifive_uart.h
@@ -24,6 +24,7 @@
 #include "hw/qdev-properties.h"
 #include "hw/sysbus.h"
 #include "qom/object.h"
+#include "qemu/fifo8.h"
 
 enum {
     SIFIVE_UART_TXFIFO        = 0,
@@ -48,9 +49,13 @@ enum {
     SIFIVE_UART_IP_RXWM       = 2  /* Receive watermark interrupt pending */
 };
 
+#define SIFIVE_UART_TXFIFO_FULL    0x80000000
+
 #define SIFIVE_UART_GET_TXCNT(txctrl)   ((txctrl >> 16) & 0x7)
 #define SIFIVE_UART_GET_RXCNT(rxctrl)   ((rxctrl >> 16) & 0x7)
+
 #define SIFIVE_UART_RX_FIFO_SIZE 8
+#define SIFIVE_UART_TX_FIFO_SIZE 8
 
 #define TYPE_SIFIVE_UART "riscv.sifive.uart"
 OBJECT_DECLARE_SIMPLE_TYPE(SiFiveUARTState, SIFIVE_UART)
@@ -63,13 +68,20 @@ struct SiFiveUARTState {
     qemu_irq irq;
     MemoryRegion mmio;
     CharBackend chr;
-    uint8_t rx_fifo[SIFIVE_UART_RX_FIFO_SIZE];
-    uint8_t rx_fifo_len;
+
+    uint32_t txfifo;
     uint32_t ie;
     uint32_t ip;
     uint32_t txctrl;
     uint32_t rxctrl;
     uint32_t div;
+
+    uint8_t rx_fifo[SIFIVE_UART_RX_FIFO_SIZE];
+    uint8_t rx_fifo_len;
+
+    Fifo8 tx_fifo;
+
+    QEMUTimer *fifo_trigger_handle;
 };
 
 SiFiveUARTState *sifive_uart_create(MemoryRegion *address_space, hwaddr base,
diff --git a/hw/char/sifive_uart.c b/hw/char/sifive_uart.c
index 7fc6787f69..aeb45d3601 100644
--- a/hw/char/sifive_uart.c
+++ b/hw/char/sifive_uart.c
@@ -26,6 +26,8 @@
 #include "hw/char/sifive_uart.h"
 #include "hw/qdev-properties-system.h"
 
+#define TX_INTERRUPT_TRIGGER_DELAY_NS 100
+
 /*
  * Not yet implemented:
  *
@@ -64,6 +66,72 @@ static void sifive_uart_update_irq(SiFiveUARTState *s)
     }
 }
 
+static gboolean sifive_uart_xmit(void *do_not_use, GIOCondition cond,
+                                 void *opaque)
+{
+    SiFiveUARTState *s = opaque;
+    int ret;
+    const uint8_t *characters;
+    uint32_t numptr = 0;
+
+    /* instant drain the fifo when there's no back-end */
+    if (!qemu_chr_fe_backend_connected(&s->chr)) {
+        fifo8_reset(&s->tx_fifo);
+        return G_SOURCE_REMOVE;
+    }
+
+    if (fifo8_is_empty(&s->tx_fifo)) {
+        return G_SOURCE_REMOVE;
+    }
+
+    /* Don't pop the FIFO in case the write fails */
+    characters = fifo8_peek_bufptr(&s->tx_fifo,
+                                   fifo8_num_used(&s->tx_fifo), &numptr);
+    ret = qemu_chr_fe_write(&s->chr, characters, numptr);
+
+    if (ret >= 0) {
+        /* We wrote the data, actually pop the fifo */
+        fifo8_pop_bufptr(&s->tx_fifo, ret, NULL);
+    }
+
+    if (!fifo8_is_empty(&s->tx_fifo)) {
+        guint r = qemu_chr_fe_add_watch(&s->chr, G_IO_OUT | G_IO_HUP,
+                                        sifive_uart_xmit, s);
+        if (!r) {
+            fifo8_reset(&s->tx_fifo);
+            return G_SOURCE_REMOVE;
+        }
+    }
+
+    /* Clear the TX Full bit */
+    if (!fifo8_is_full(&s->tx_fifo)) {
+        s->txfifo &= ~SIFIVE_UART_TXFIFO_FULL;
+    }
+
+    sifive_uart_update_irq(s);
+    return G_SOURCE_REMOVE;
+}
+
+static void sifive_uart_write_tx_fifo(SiFiveUARTState *s, const uint8_t *buf,
+                                      int size)
+{
+    uint64_t current_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+
+    if (size > fifo8_num_free(&s->tx_fifo)) {
+        size = fifo8_num_free(&s->tx_fifo);
+        qemu_log_mask(LOG_GUEST_ERROR, "sifive_uart: TX FIFO overflow");
+    }
+
+    fifo8_push_all(&s->tx_fifo, buf, size);
+
+    if (fifo8_is_full(&s->tx_fifo)) {
+        s->txfifo |= SIFIVE_UART_TXFIFO_FULL;
+    }
+
+    timer_mod(s->fifo_trigger_handle, current_time +
+                  TX_INTERRUPT_TRIGGER_DELAY_NS);
+}
+
 static uint64_t
 sifive_uart_read(void *opaque, hwaddr addr, unsigned int size)
 {
@@ -82,7 +150,7 @@ sifive_uart_read(void *opaque, hwaddr addr, unsigned int size)
         return 0x80000000;
 
     case SIFIVE_UART_TXFIFO:
-        return 0; /* Should check tx fifo */
+        return s->txfifo;
     case SIFIVE_UART_IE:
         return s->ie;
     case SIFIVE_UART_IP:
@@ -106,12 +174,10 @@ sifive_uart_write(void *opaque, hwaddr addr,
 {
     SiFiveUARTState *s = opaque;
     uint32_t value = val64;
-    unsigned char ch = value;
 
     switch (addr) {
     case SIFIVE_UART_TXFIFO:
-        qemu_chr_fe_write(&s->chr, &ch, 1);
-        sifive_uart_update_irq(s);
+        sifive_uart_write_tx_fifo(s, (uint8_t *) &value, 1);
         return;
     case SIFIVE_UART_IE:
         s->ie = val64;
@@ -131,6 +197,13 @@ sifive_uart_write(void *opaque, hwaddr addr,
                   __func__, (int)addr, (int)value);
 }
 
+static void fifo_trigger_update(void *opaque)
+{
+    SiFiveUARTState *s = opaque;
+
+    sifive_uart_xmit(NULL, G_IO_OUT, s);
+}
+
 static const MemoryRegionOps sifive_uart_ops = {
     .read = sifive_uart_read,
     .write = sifive_uart_write,
@@ -197,6 +270,9 @@ static void sifive_uart_realize(DeviceState *dev, Error **errp)
 {
     SiFiveUARTState *s = SIFIVE_UART(dev);
 
+    s->fifo_trigger_handle = timer_new_ns(QEMU_CLOCK_VIRTUAL,
+                                          fifo_trigger_update, s);
+
     qemu_chr_fe_set_handlers(&s->chr, sifive_uart_can_rx, sifive_uart_rx,
                              sifive_uart_event, sifive_uart_be_change, s,
                              NULL, true);
@@ -206,12 +282,18 @@ static void sifive_uart_realize(DeviceState *dev, Error **errp)
 static void sifive_uart_reset_enter(Object *obj, ResetType type)
 {
     SiFiveUARTState *s = SIFIVE_UART(obj);
+
+    s->txfifo = 0;
     s->ie = 0;
     s->ip = 0;
     s->txctrl = 0;
     s->rxctrl = 0;
     s->div = 0;
+
     s->rx_fifo_len = 0;
+
+    memset(s->rx_fifo, 0, SIFIVE_UART_RX_FIFO_SIZE);
+    fifo8_create(&s->tx_fifo, SIFIVE_UART_TX_FIFO_SIZE);
 }
 
 static void sifive_uart_reset_hold(Object *obj, ResetType type)
@@ -222,8 +304,8 @@ static void sifive_uart_reset_hold(Object *obj, ResetType type)
 
 static const VMStateDescription vmstate_sifive_uart = {
     .name = TYPE_SIFIVE_UART,
-    .version_id = 1,
-    .minimum_version_id = 1,
+    .version_id = 2,
+    .minimum_version_id = 2,
     .fields = (const VMStateField[]) {
         VMSTATE_UINT8_ARRAY(rx_fifo, SiFiveUARTState,
                             SIFIVE_UART_RX_FIFO_SIZE),
@@ -233,6 +315,9 @@ static const VMStateDescription vmstate_sifive_uart = {
         VMSTATE_UINT32(txctrl, SiFiveUARTState),
         VMSTATE_UINT32(rxctrl, SiFiveUARTState),
         VMSTATE_UINT32(div, SiFiveUARTState),
+        VMSTATE_UINT32(txfifo, SiFiveUARTState),
+        VMSTATE_FIFO8(tx_fifo, SiFiveUARTState),
+        VMSTATE_TIMER_PTR(fifo_trigger_handle, SiFiveUARTState),
         VMSTATE_END_OF_LIST()
     },
 };
-- 
2.47.0


Re: [PULL 15/50] hw/char: sifive_uart: Print uart characters async
Posted by Clément Chigot 12 months ago
Hi Alistair,

I've an issue following this patch. When the system is reset (e.g
using HTIF syscalls), the fifo might not be empty and thus some
characters are lost.
I discovered it on a Windows host. But by extending
"TX_INTERRUPT_TRIGGER_DELAY_NS" to a huge value, I'm able to reproduce
on Linux as well.

I've tried to flush within an unrealized function but it didn't work.
Any suggestions ?

>  static void sifive_uart_reset_enter(Object *obj, ResetType type)
>  {
> ...
> +    fifo8_create(&s->tx_fifo, SIFIVE_UART_TX_FIFO_SIZE);

I'm also wondering if that part could not lead to memory leak.
`fifo8_destroy` is never called and AFAIK, there are ways to reset a
device dynamically (e.g snapshot, though not sure if it's supported
here).

Thanks, Clément
Re: [PULL 15/50] hw/char: sifive_uart: Print uart characters async
Posted by Clément Chigot 11 months, 3 weeks ago
On Fri, Feb 14, 2025 at 1:52 PM Clément Chigot <chigot@adacore.com> wrote:
>
> Hi Alistair,
>
> I've an issue following this patch. When the system is reset (e.g
> using HTIF syscalls), the fifo might not be empty and thus some
> characters are lost.
> I discovered it on a Windows host. But by extending
> "TX_INTERRUPT_TRIGGER_DELAY_NS" to a huge value, I'm able to reproduce
> on Linux as well.

The root cause of my issue was unrelated to these early shutdowns. On
Windows, the character device behind `-serial mon:stdio`
(char-win-stdio) doesn't provide an `add_watch` method. Therefore,
`qemu_chr_fe_add_watch` calls always result in an error, flushing the
fifo. I saw in @Philippe Mathieu-Daudé patch about pl011 that
`G_SOURCE_CONTINUE` is returned instead of calling it and it does
work. @Alistair Francis  do you remember if there was a reason for
calling `add_watch` ?

> I've tried to flush within an unrealized function but it didn't work.
> Any suggestions ?

FTR, I still have found a solution here using
qemu_register_shutdown_notifier. Though I'm wondering if this is
useful: the cases where a shutdown occurs between two "fifo_update"
seems really narrow, but they could happen.
 @Philippe Mathieu-Daudé AFAICT, the new pl011 and other char devices
implementing write fifo have the same issue. Thus, pinging you here to
get your advice.

Thanks,
Clément

> >  static void sifive_uart_reset_enter(Object *obj, ResetType type)
> >  {
> > ...
> > +    fifo8_create(&s->tx_fifo, SIFIVE_UART_TX_FIFO_SIZE);
>
> I'm also wondering if that part could not lead to memory leak.
> `fifo8_destroy` is never called and AFAIK, there are ways to reset a
> device dynamically (e.g snapshot, though not sure if it's supported
> here).
>
> Thanks, Clément
Re: [PULL 15/50] hw/char: sifive_uart: Print uart characters async
Posted by Alistair Francis 11 months, 3 weeks ago
On Sat, Feb 22, 2025 at 1:27 AM Clément Chigot <chigot@adacore.com> wrote:
>
> On Fri, Feb 14, 2025 at 1:52 PM Clément Chigot <chigot@adacore.com> wrote:
> >
> > Hi Alistair,
> >
> > I've an issue following this patch. When the system is reset (e.g
> > using HTIF syscalls), the fifo might not be empty and thus some
> > characters are lost.
> > I discovered it on a Windows host. But by extending
> > "TX_INTERRUPT_TRIGGER_DELAY_NS" to a huge value, I'm able to reproduce
> > on Linux as well.
>
> The root cause of my issue was unrelated to these early shutdowns. On
> Windows, the character device behind `-serial mon:stdio`
> (char-win-stdio) doesn't provide an `add_watch` method. Therefore,
> `qemu_chr_fe_add_watch` calls always result in an error, flushing the
> fifo. I saw in @Philippe Mathieu-Daudé patch about pl011 that
> `G_SOURCE_CONTINUE` is returned instead of calling it and it does
> work. @Alistair Francis  do you remember if there was a reason for
> calling `add_watch` ?

qemu_chr_fe_add_watch() is used in a range of UART devices, I am not
really sure of a different way to write the data out.

I don't see `G_SOURCE_CONTINUE` in pl011.c either

>
> > I've tried to flush within an unrealized function but it didn't work.
> > Any suggestions ?
>
> FTR, I still have found a solution here using
> qemu_register_shutdown_notifier. Though I'm wondering if this is
> useful: the cases where a shutdown occurs between two "fifo_update"
> seems really narrow, but they could happen.

What does the actual hardware do? I don't think it has a shutdown
notifier. I think this is up to the guest to flush the UART.

>  @Philippe Mathieu-Daudé AFAICT, the new pl011 and other char devices
> implementing write fifo have the same issue. Thus, pinging you here to
> get your advice.
>
> Thanks,
> Clément
>
> > >  static void sifive_uart_reset_enter(Object *obj, ResetType type)
> > >  {
> > > ...
> > > +    fifo8_create(&s->tx_fifo, SIFIVE_UART_TX_FIFO_SIZE);
> >
> > I'm also wondering if that part could not lead to memory leak.
> > `fifo8_destroy` is never called and AFAIK, there are ways to reset a
> > device dynamically (e.g snapshot, though not sure if it's supported
> > here).

Good catch, I can send a patch to fix this

Alistair

> >
> > Thanks, Clément
Re: [PULL 15/50] hw/char: sifive_uart: Print uart characters async
Posted by Clément Chigot 11 months, 2 weeks ago
On Mon, Feb 24, 2025 at 5:38 AM Alistair Francis <alistair23@gmail.com> wrote:
>
> On Sat, Feb 22, 2025 at 1:27 AM Clément Chigot <chigot@adacore.com> wrote:
> >
> > On Fri, Feb 14, 2025 at 1:52 PM Clément Chigot <chigot@adacore.com> wrote:
> > >
> > > Hi Alistair,
> > >
> > > I've an issue following this patch. When the system is reset (e.g
> > > using HTIF syscalls), the fifo might not be empty and thus some
> > > characters are lost.
> > > I discovered it on a Windows host. But by extending
> > > "TX_INTERRUPT_TRIGGER_DELAY_NS" to a huge value, I'm able to reproduce
> > > on Linux as well.
> >
> > The root cause of my issue was unrelated to these early shutdowns. On
> > Windows, the character device behind `-serial mon:stdio`
> > (char-win-stdio) doesn't provide an `add_watch` method. Therefore,
> > `qemu_chr_fe_add_watch` calls always result in an error, flushing the
> > fifo. I saw in @Philippe Mathieu-Daudé patch about pl011 that
> > `G_SOURCE_CONTINUE` is returned instead of calling it and it does
> > work. @Alistair Francis  do you remember if there was a reason for
> > calling `add_watch` ?
>
> qemu_chr_fe_add_watch() is used in a range of UART devices, I am not
> really sure of a different way to write the data out.
>
> I don't see `G_SOURCE_CONTINUE` in pl011.c either

The patch has not yet been merged to master AFAICT. But here is the
code replacing `qemu_chr_fe_add_watch` in `*_xmit`:

+    if (!fifo8_is_empty(&s->xmit_fifo)) {
+        /* Reschedule another transmission if we couldn't transmit all. */
+        return G_SOURCE_CONTINUE;
+    }

However, it seems that Peter has detected some issues with those
patches (see [2]). So, I'll wait for the patch to land on master
before anything.

[1] https://mail.gnu.org/archive/html/qemu-devel/2025-02/msg01637.html
[2] https://mail.gnu.org/archive/html/qemu-devel/2025-02/msg04209.html

> > > I've tried to flush within an unrealized function but it didn't work.
> > > Any suggestions ?
> >
> > FTR, I still have found a solution here using
> > qemu_register_shutdown_notifier. Though I'm wondering if this is
> > useful: the cases where a shutdown occurs between two "fifo_update"
> > seems really narrow, but they could happen.
>
> What does the actual hardware do? I don't think it has a shutdown
> notifier. I think this is up to the guest to flush the UART.

I don't have access to real hardware. But the reference manual doesn't
mention any shutdown notifier. So I think you're right.

> >  @Philippe Mathieu-Daudé AFAICT, the new pl011 and other char devices
> > implementing write fifo have the same issue. Thus, pinging you here to
> > get your advice.
> >
> > Thanks,
> > Clément
> >
> > > >  static void sifive_uart_reset_enter(Object *obj, ResetType type)
> > > >  {
> > > > ...
> > > > +    fifo8_create(&s->tx_fifo, SIFIVE_UART_TX_FIFO_SIZE);
> > >
> > > I'm also wondering if that part could not lead to memory leak.
> > > `fifo8_destroy` is never called and AFAIK, there are ways to reset a
> > > device dynamically (e.g snapshot, though not sure if it's supported
> > > here).
>
> Good catch, I can send a patch to fix this
>
> Alistair
>
> > >
> > > Thanks, Clément
Re: [PULL 15/50] hw/char: sifive_uart: Print uart characters async
Posted by Thomas Huth 1 year, 3 months ago
On 31/10/2024 04.52, Alistair Francis wrote:
> The current approach of using qemu_chr_fe_write() and ignoring the
> return values results in dropped characters [1].
> 
> Let's update the SiFive UART to use a async sifive_uart_xmit() function
> to transmit the characters and apply back pressure to the guest with
> the SIFIVE_UART_TXFIFO_FULL status.
> 
> This should avoid dropped characters and more realisticly model the
> hardware.
> 
> 1: https://gitlab.com/qemu-project/qemu/-/issues/2114
> 
> Signed-off-by: Alistair Francis <alistair.francis@wdc.com>
> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
> Tested-by: Thomas Huth <thuth@redhat.com>
> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
> Message-ID: <20240910045419.1252277-3-alistair.francis@wdc.com>
> Signed-off-by: Alistair Francis <alistair.francis@wdc.com>
> ---
...
> @@ -106,12 +174,10 @@ sifive_uart_write(void *opaque, hwaddr addr,
>   {
>       SiFiveUARTState *s = opaque;
>       uint32_t value = val64;
> -    unsigned char ch = value;
>   
>       switch (addr) {
>       case SIFIVE_UART_TXFIFO:
> -        qemu_chr_fe_write(&s->chr, &ch, 1);
> -        sifive_uart_update_irq(s);
> +        sifive_uart_write_tx_fifo(s, (uint8_t *) &value, 1);

This broke the sifive_u machine on big endian hosts. "value" is a 64-bit 
variable, so you cannot assume that you get the 8-bit character with such a 
pointer magic here. I think we have to re-introduce the detour via the 8-bit 
"ch" variable...

  Thomas


Re: [PULL 15/50] hw/char: sifive_uart: Print uart characters async
Posted by Philippe Mathieu-Daudé 1 year, 3 months ago
On 4/11/24 11:38, Thomas Huth wrote:
> On 31/10/2024 04.52, Alistair Francis wrote:
>> The current approach of using qemu_chr_fe_write() and ignoring the
>> return values results in dropped characters [1].
>>
>> Let's update the SiFive UART to use a async sifive_uart_xmit() function
>> to transmit the characters and apply back pressure to the guest with
>> the SIFIVE_UART_TXFIFO_FULL status.
>>
>> This should avoid dropped characters and more realisticly model the
>> hardware.
>>
>> 1: https://gitlab.com/qemu-project/qemu/-/issues/2114
>>
>> Signed-off-by: Alistair Francis <alistair.francis@wdc.com>
>> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
>> Tested-by: Thomas Huth <thuth@redhat.com>
>> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
>> Message-ID: <20240910045419.1252277-3-alistair.francis@wdc.com>
>> Signed-off-by: Alistair Francis <alistair.francis@wdc.com>
>> ---
> ...
>> @@ -106,12 +174,10 @@ sifive_uart_write(void *opaque, hwaddr addr,
>>   {
>>       SiFiveUARTState *s = opaque;
>>       uint32_t value = val64;
>> -    unsigned char ch = value;
>>       switch (addr) {
>>       case SIFIVE_UART_TXFIFO:
>> -        qemu_chr_fe_write(&s->chr, &ch, 1);
>> -        sifive_uart_update_irq(s);
>> +        sifive_uart_write_tx_fifo(s, (uint8_t *) &value, 1);
> 
> This broke the sifive_u machine on big endian hosts. "value" is a 64-bit 
> variable, so you cannot assume that you get the 8-bit character with 
> such a pointer magic here. I think we have to re-introduce the detour 
> via the 8-bit "ch" variable...

Sigh I got corrected for the same mistake but forgot it here:
https://lore.kernel.org/qemu-devel/49918fad-56fc-150e-bb9c-bd00dc67df05@linaro.org/