The 'pci-vga' device allow setting a 'big-endian-framebuffer'
property since commit 3c2784fc864 ("vga: Expose framebuffer
byteorder as a QOM property"). Similarly, the 'virtio-vga'
device since commit 8be61ce2ce3 ("virtio-vga: implement
big-endian-framebuffer property").

Both call vga_common_reset() in their reset handler, respectively
pci_secondary_vga_reset() and virtio_vga_base_reset_hold(), which
reset 'big_endian_fb', overwritting the property. This is not
correct: the hardware is expected to keep its configured
endianness during resets.

Move 'big_endian_fb' assignment from vga_common_reset() to
vga_common_init() which is called once when the common VGA state
is initialized.

'default_endian_fb' value is always target_words_bigendian(),
remove it as it isn't very useful.

Signed-off-by: Philippe Mathieu-Daudé <philmd@linaro.org>
---
 hw/display/vga_int.h | 1 -
 hw/display/vga.c     | 5 ++---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/hw/display/vga_int.h b/hw/display/vga_int.h
index f77c1c11457..5dcdecfd422 100644
--- a/hw/display/vga_int.h
+++ b/hw/display/vga_int.h
@@ -135,7 +135,6 @@ typedef struct VGACommonState {
     bool full_update_text;
     bool full_update_gfx;
     bool big_endian_fb;
-    bool default_endian_fb;
     bool global_vmstate;
     /* hardware mouse cursor support */
     uint32_t invalidated_y_table[VGA_MAX_HEIGHT / 32];
diff --git a/hw/display/vga.c b/hw/display/vga.c
index 892fedc8dce..6dbbbf49073 100644
--- a/hw/display/vga.c
+++ b/hw/display/vga.c
@@ -1873,7 +1873,6 @@ void vga_common_reset(VGACommonState *s)
     s->cursor_start = 0;
     s->cursor_end = 0;
     s->cursor_offset = 0;
-    s->big_endian_fb = s->default_endian_fb;
     memset(s->invalidated_y_table, '\0', sizeof(s->invalidated_y_table));
     memset(s->last_palette, '\0', sizeof(s->last_palette));
     memset(s->last_ch_attr, '\0', sizeof(s->last_ch_attr));
@@ -2117,7 +2116,7 @@ static bool vga_endian_state_needed(void *opaque)
      * default one, thus ensuring backward compatibility for
      * migration of the common case
      */
-    return s->default_endian_fb != s->big_endian_fb;
+    return s->big_endian_fb != target_words_bigendian();
 }
 
 static const VMStateDescription vmstate_vga_endian = {
@@ -2265,7 +2264,7 @@ bool vga_common_init(VGACommonState *s, Object *obj, Error **errp)
      * into a device attribute set by the machine/platform to remove
      * all target endian dependencies from this file.
      */
-    s->default_endian_fb = target_words_bigendian();
+    s->big_endian_fb = target_words_bigendian();
 
     vga_dirty_log_start(s);
 
-- 
2.45.2

On 28/11/24 18:52, Philippe Mathieu-Daudé wrote:
> The 'pci-vga' device allow setting a 'big-endian-framebuffer'
> property since commit 3c2784fc864 ("vga: Expose framebuffer
> byteorder as a QOM property"). Similarly, the 'virtio-vga'
> device since commit 8be61ce2ce3 ("virtio-vga: implement
> big-endian-framebuffer property").
> 
> Both call vga_common_reset() in their reset handler, respectively
> pci_secondary_vga_reset() and virtio_vga_base_reset_hold(), which
> reset 'big_endian_fb', overwritting the property. This is not
> correct: the hardware is expected to keep its configured
> endianness during resets.
> 
> Move 'big_endian_fb' assignment from vga_common_reset() to
> vga_common_init() which is called once when the common VGA state
> is initialized.

If 2 patches are preferred, this part is:

-- >8 --
diff --git a/hw/display/vga.c b/hw/display/vga.c
index 892fedc8dce..b074b58c90d 100644
--- a/hw/display/vga.c
+++ b/hw/display/vga.c
@@ -1875,3 +1875,2 @@ void vga_common_reset(VGACommonState *s)
      s->cursor_offset = 0;
-    s->big_endian_fb = s->default_endian_fb;
      memset(s->invalidated_y_table, '\0', sizeof(s->invalidated_y_table));
@@ -2268,2 +2267,3 @@ bool vga_common_init(VGACommonState *s, Object 
*obj, Error **errp)
      s->default_endian_fb = target_words_bigendian();
+    s->big_endian_fb = s->default_endian_fb;
---

> 
> 'default_endian_fb' value is always target_words_bigendian(),
> remove it as it isn't very useful.

And this one (cleanup):

-- >8 --
diff --git a/hw/display/vga_int.h b/hw/display/vga_int.h
index f77c1c11457..5dcdecfd422 100644
--- a/hw/display/vga_int.h
+++ b/hw/display/vga_int.h
@@ -137,3 +137,2 @@ typedef struct VGACommonState {
      bool big_endian_fb;
-    bool default_endian_fb;
      bool global_vmstate;
diff --git a/hw/display/vga.c b/hw/display/vga.c
index b074b58c90d..65698923daf 100644
--- a/hw/display/vga.c
+++ b/hw/display/vga.c
@@ -2118,3 +2118,3 @@ static bool vga_endian_state_needed(void *opaque)
       */
-    return s->default_endian_fb != s->big_endian_fb;
+    return target_words_bigendian() != s->big_endian_fb;
  }
@@ -2266,4 +2266,3 @@ bool vga_common_init(VGACommonState *s, Object 
*obj, Error **errp)
       */
-    s->default_endian_fb = target_words_bigendian();
-    s->big_endian_fb = s->default_endian_fb;
+    s->big_endian_fb = target_words_bigendian();
---

> 
> Signed-off-by: Philippe Mathieu-Daudé <philmd@linaro.org>
> ---
>   hw/display/vga_int.h | 1 -
>   hw/display/vga.c     | 5 ++---
>   2 files changed, 2 insertions(+), 4 deletions(-)
> 
> diff --git a/hw/display/vga_int.h b/hw/display/vga_int.h
> index f77c1c11457..5dcdecfd422 100644
> --- a/hw/display/vga_int.h
> +++ b/hw/display/vga_int.h
> @@ -135,7 +135,6 @@ typedef struct VGACommonState {
>       bool full_update_text;
>       bool full_update_gfx;
>       bool big_endian_fb;
> -    bool default_endian_fb;
>       bool global_vmstate;
>       /* hardware mouse cursor support */
>       uint32_t invalidated_y_table[VGA_MAX_HEIGHT / 32];
> diff --git a/hw/display/vga.c b/hw/display/vga.c
> index 892fedc8dce..6dbbbf49073 100644
> --- a/hw/display/vga.c
> +++ b/hw/display/vga.c
> @@ -1873,7 +1873,6 @@ void vga_common_reset(VGACommonState *s)
>       s->cursor_start = 0;
>       s->cursor_end = 0;
>       s->cursor_offset = 0;
> -    s->big_endian_fb = s->default_endian_fb;
>       memset(s->invalidated_y_table, '\0', sizeof(s->invalidated_y_table));
>       memset(s->last_palette, '\0', sizeof(s->last_palette));
>       memset(s->last_ch_attr, '\0', sizeof(s->last_ch_attr));
> @@ -2117,7 +2116,7 @@ static bool vga_endian_state_needed(void *opaque)
>        * default one, thus ensuring backward compatibility for
>        * migration of the common case
>        */
> -    return s->default_endian_fb != s->big_endian_fb;
> +    return s->big_endian_fb != target_words_bigendian();
>   }
>   
>   static const VMStateDescription vmstate_vga_endian = {
> @@ -2265,7 +2264,7 @@ bool vga_common_init(VGACommonState *s, Object *obj, Error **errp)
>        * into a device attribute set by the machine/platform to remove
>        * all target endian dependencies from this file.
>        */
> -    s->default_endian_fb = target_words_bigendian();
> +    s->big_endian_fb = target_words_bigendian();
>   
>       vga_dirty_log_start(s);
>