The XHCI console uses DMA, so it's sensitive for relocating its
structures, even if their virtual addresses remain the same. Add a new
console initialization hooks called before+after Xen relocation.
Relocation happens conditionally, but call the hooks unconditionally, as
that simplifies logic in the driver (and if needed, the driver can
easily detect if relocation happened anyway). Thanks to that, a driver
may use it to finalize init steps that need physical address but can be
delayed - this way, it doesn't need un-doing on relocation.
The most important part is the post-relocation hook, but add also
pre-relocation one to simplify clean handling of in-flight data.

Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
---
I considered more limited scope - calling them just around move_xen()
(or even from within that function), but that complicates
iommu_add_extra_reserved_device_memory() call - see the next patch.
As for the post-relocation hook, I have considered a parameter with info
whether the relocation actually happened, but driver can figure it out
on its own anyway.
---
 xen/arch/x86/setup.c       |  8 ++++++++
 xen/drivers/char/console.c | 10 ++++++++++
 xen/drivers/char/serial.c  | 18 ++++++++++++++++++
 xen/include/xen/console.h  |  2 ++
 xen/include/xen/serial.h   |  6 ++++++
 5 files changed, 44 insertions(+)

diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 25189541244d..3ef819a252e4 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -1481,6 +1481,8 @@ void asmlinkage __init noreturn __start_xen(void)
         highmem_start &= ~((1UL << L3_PAGETABLE_SHIFT) - 1);
 #endif
 
+    console_init_pre_relocate();
+
     /*
      * Iterate backwards over all superpage-aligned RAM regions.
      *
@@ -1606,6 +1608,12 @@ void asmlinkage __init noreturn __start_xen(void)
     if ( !xen_phys_start )
         panic("Not enough memory to relocate Xen\n");
 
+    /*
+     * Notify console drivers about relocation, before reusing old Xen's
+     * memory.
+     */
+    console_init_post_relocate();
+
     /* FIXME: Putting a hole in .bss would shatter the large page mapping. */
     if ( using_2M_mapping() )
         efi_boot_mem_unused(NULL, NULL);
diff --git a/xen/drivers/char/console.c b/xen/drivers/char/console.c
index c15987f5bbe2..12898b684b5e 100644
--- a/xen/drivers/char/console.c
+++ b/xen/drivers/char/console.c
@@ -1120,6 +1120,16 @@ void __init console_init_ring(void)
     printk("Allocated console ring of %u KiB.\n", opt_conring_size >> 10);
 }
 
+void __init console_init_pre_relocate(void)
+{
+    serial_init_pre_relocate();
+}
+
+void __init console_init_post_relocate(void)
+{
+    serial_init_post_relocate();
+}
+
 void __init console_init_irq(void)
 {
     serial_init_irq();
diff --git a/xen/drivers/char/serial.c b/xen/drivers/char/serial.c
index 591a00900869..95f7410afa9c 100644
--- a/xen/drivers/char/serial.c
+++ b/xen/drivers/char/serial.c
@@ -447,6 +447,24 @@ void __init serial_init_preirq(void)
             com[i].driver->init_preirq(&com[i]);
 }
 
+void __init serial_init_pre_relocate(void)
+{
+    unsigned int i;
+
+    for ( i = 0; i < ARRAY_SIZE(com); i++ )
+        if ( com[i].driver && com[i].driver->init_pre_relocate )
+            com[i].driver->init_pre_relocate(&com[i]);
+}
+
+void __init serial_init_post_relocate(void)
+{
+    unsigned int i;
+
+    for ( i = 0; i < ARRAY_SIZE(com); i++ )
+        if ( com[i].driver && com[i].driver->init_post_relocate )
+            com[i].driver->init_post_relocate(&com[i]);
+}
+
 void __init serial_init_irq(void)
 {
     unsigned int i;
diff --git a/xen/include/xen/console.h b/xen/include/xen/console.h
index 83cbc9fbdafc..d563777ad9e2 100644
--- a/xen/include/xen/console.h
+++ b/xen/include/xen/console.h
@@ -18,6 +18,8 @@ void console_init_preirq(void);
 void console_init_ring(void);
 void console_init_irq(void);
 void console_init_postirq(void);
+void console_init_pre_relocate(void);
+void console_init_post_relocate(void);
 void console_endboot(void);
 int console_has(const char *device);
 
diff --git a/xen/include/xen/serial.h b/xen/include/xen/serial.h
index 63a82b032dde..1ee3df2624fb 100644
--- a/xen/include/xen/serial.h
+++ b/xen/include/xen/serial.h
@@ -64,6 +64,9 @@ struct uart_driver {
     void (*init_preirq)(struct serial_port *port);
     void (*init_irq)(struct serial_port *port);
     void (*init_postirq)(struct serial_port *port);
+    /* Hooks around optional Xen relocation. */
+    void (*init_pre_relocate)(struct serial_port *port);
+    void (*init_post_relocate)(struct serial_port *port);
     /* Hook to clean up after Xen bootstrap (before domain 0 runs). */
     void (*endboot)(struct serial_port *port);
     /* Driver suspend/resume. */
@@ -103,6 +106,9 @@ struct uart_driver {
 void serial_init_preirq(void);
 void serial_init_irq(void);
 void serial_init_postirq(void);
+/* Notify drivers about Xen relocation (relevant for those using DMA). */
+void serial_init_pre_relocate(void);
+void serial_init_post_relocate(void);
 
 /* Clean-up hook before domain 0 runs. */
 void serial_endboot(void);
-- 
git-series 0.9.1

On 25/05/2025 3:15 pm, Marek Marczykowski-Górecki wrote:
> diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
> index 25189541244d..3ef819a252e4 100644
> --- a/xen/arch/x86/setup.c
> +++ b/xen/arch/x86/setup.c
> @@ -1481,6 +1481,8 @@ void asmlinkage __init noreturn __start_xen(void)
>          highmem_start &= ~((1UL << L3_PAGETABLE_SHIFT) - 1);
>  #endif
>  
> +    console_init_pre_relocate();
> +
>      /*
>       * Iterate backwards over all superpage-aligned RAM regions.
>       *
> @@ -1606,6 +1608,12 @@ void asmlinkage __init noreturn __start_xen(void)
>      if ( !xen_phys_start )
>          panic("Not enough memory to relocate Xen\n");
>  
> +    /*
> +     * Notify console drivers about relocation, before reusing old Xen's
> +     * memory.
> +     */
> +    console_init_post_relocate();
> +

With reference to the next patch, there are printk()'s in this region
which want to work (in case something goes very wrong), so I don't think
setting dbc->suspended is the best approach.

In dbc_uart_init_pre_relocate(), you just wait for all transfers to
complete.  Can't this be merged with post_relocate(), at which point the
intermediate printk()'s will work too?  It will also remove a hook.

Meanwhile, we have things like:

    /* Cache {,compat_}gdt_l1e now that physically relocation is done. */
    this_cpu(gdt_l1e) =
        l1e_from_pfn(virt_to_mfn(boot_gdt), __PAGE_HYPERVISOR_RW);
    if ( IS_ENABLED(CONFIG_PV32) )
        this_cpu(compat_gdt_l1e) =
            l1e_from_pfn(virt_to_mfn(boot_compat_gdt), __PAGE_HYPERVISOR_RW);


in traps_init() which really doesn't belong here, but does belong in
some kind of general "relocation done" mechanism.

I wonder if we want a new type of initcall for this, allowing individual
areas of code to opt in with less boilerpate?

~Andrew

On Mon, May 26, 2025 at 04:08:17PM +0100, Andrew Cooper wrote:
> On 25/05/2025 3:15 pm, Marek Marczykowski-Górecki wrote:
> > diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
> > index 25189541244d..3ef819a252e4 100644
> > --- a/xen/arch/x86/setup.c
> > +++ b/xen/arch/x86/setup.c
> > @@ -1481,6 +1481,8 @@ void asmlinkage __init noreturn __start_xen(void)
> >          highmem_start &= ~((1UL << L3_PAGETABLE_SHIFT) - 1);
> >  #endif
> >  
> > +    console_init_pre_relocate();
> > +
> >      /*
> >       * Iterate backwards over all superpage-aligned RAM regions.
> >       *
> > @@ -1606,6 +1608,12 @@ void asmlinkage __init noreturn __start_xen(void)
> >      if ( !xen_phys_start )
> >          panic("Not enough memory to relocate Xen\n");
> >  
> > +    /*
> > +     * Notify console drivers about relocation, before reusing old Xen's
> > +     * memory.
> > +     */
> > +    console_init_post_relocate();
> > +
> 
> With reference to the next patch, there are printk()'s in this region
> which want to work (in case something goes very wrong), so I don't think
> setting dbc->suspended is the best approach.

I guess the post_relocate hook might be moved a bit earlier, but still,
once relocation happens, the xhci console is not functional until it
gets relocated too (for example - it would post new TRBs into a ring
that isn't actually monitored by the controller).

> In dbc_uart_init_pre_relocate(), you just wait for all transfers to
> complete.  Can't this be merged with post_relocate(), at which point the
> intermediate printk()'s will work too?

Not really, because the structure that driver watches is not the one
that the controller DMA into anymore... The driver would need to watch
the old physical pages (specifically, the event ring buffer there).

>   It will also remove a hook.
> 
> Meanwhile, we have things like:
> 
>     /* Cache {,compat_}gdt_l1e now that physically relocation is done. */
>     this_cpu(gdt_l1e) =
>         l1e_from_pfn(virt_to_mfn(boot_gdt), __PAGE_HYPERVISOR_RW);
>     if ( IS_ENABLED(CONFIG_PV32) )
>         this_cpu(compat_gdt_l1e) =
>             l1e_from_pfn(virt_to_mfn(boot_compat_gdt), __PAGE_HYPERVISOR_RW);
> 
> 
> in traps_init() which really doesn't belong here, but does belong in
> some kind of general "relocation done" mechanism.
> 
> I wonder if we want a new type of initcall for this, allowing individual
> areas of code to opt in with less boilerpate?

Maybe? But for the console specifically, we also need pre-relocation
hook (more context also in the next patch description).

-- 
Best Regards,
Marek Marczykowski-Górecki
Invisible Things Lab

On 26.05.2025 17:39, Marek Marczykowski-Górecki wrote:
> On Mon, May 26, 2025 at 04:08:17PM +0100, Andrew Cooper wrote:
>> On 25/05/2025 3:15 pm, Marek Marczykowski-Górecki wrote:
>>> diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
>>> index 25189541244d..3ef819a252e4 100644
>>> --- a/xen/arch/x86/setup.c
>>> +++ b/xen/arch/x86/setup.c
>>> @@ -1481,6 +1481,8 @@ void asmlinkage __init noreturn __start_xen(void)
>>>          highmem_start &= ~((1UL << L3_PAGETABLE_SHIFT) - 1);
>>>  #endif
>>>  
>>> +    console_init_pre_relocate();
>>> +
>>>      /*
>>>       * Iterate backwards over all superpage-aligned RAM regions.
>>>       *
>>> @@ -1606,6 +1608,12 @@ void asmlinkage __init noreturn __start_xen(void)
>>>      if ( !xen_phys_start )
>>>          panic("Not enough memory to relocate Xen\n");
>>>  
>>> +    /*
>>> +     * Notify console drivers about relocation, before reusing old Xen's
>>> +     * memory.
>>> +     */
>>> +    console_init_post_relocate();
>>> +
>>
>> With reference to the next patch, there are printk()'s in this region
>> which want to work (in case something goes very wrong), so I don't think
>> setting dbc->suspended is the best approach.
> 
> I guess the post_relocate hook might be moved a bit earlier, but still,
> once relocation happens, the xhci console is not functional until it
> gets relocated too (for example - it would post new TRBs into a ring
> that isn't actually monitored by the controller).

Why is it that this ring is dependent upon Xen's position? If the ring was
dynamically allocated, it wouldn't change position when Xen is moved.

Jan

On Thu, Jun 05, 2025 at 04:42:53PM +0200, Jan Beulich wrote:
> On 26.05.2025 17:39, Marek Marczykowski-Górecki wrote:
> > On Mon, May 26, 2025 at 04:08:17PM +0100, Andrew Cooper wrote:
> >> On 25/05/2025 3:15 pm, Marek Marczykowski-Górecki wrote:
> >>> diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
> >>> index 25189541244d..3ef819a252e4 100644
> >>> --- a/xen/arch/x86/setup.c
> >>> +++ b/xen/arch/x86/setup.c
> >>> @@ -1481,6 +1481,8 @@ void asmlinkage __init noreturn __start_xen(void)
> >>>          highmem_start &= ~((1UL << L3_PAGETABLE_SHIFT) - 1);
> >>>  #endif
> >>>  
> >>> +    console_init_pre_relocate();
> >>> +
> >>>      /*
> >>>       * Iterate backwards over all superpage-aligned RAM regions.
> >>>       *
> >>> @@ -1606,6 +1608,12 @@ void asmlinkage __init noreturn __start_xen(void)
> >>>      if ( !xen_phys_start )
> >>>          panic("Not enough memory to relocate Xen\n");
> >>>  
> >>> +    /*
> >>> +     * Notify console drivers about relocation, before reusing old Xen's
> >>> +     * memory.
> >>> +     */
> >>> +    console_init_post_relocate();
> >>> +
> >>
> >> With reference to the next patch, there are printk()'s in this region
> >> which want to work (in case something goes very wrong), so I don't think
> >> setting dbc->suspended is the best approach.
> > 
> > I guess the post_relocate hook might be moved a bit earlier, but still,
> > once relocation happens, the xhci console is not functional until it
> > gets relocated too (for example - it would post new TRBs into a ring
> > that isn't actually monitored by the controller).
> 
> Why is it that this ring is dependent upon Xen's position? If the ring was
> dynamically allocated, it wouldn't change position when Xen is moved.

The console is setup quite early, I don't think I can allocate memory at
this stage, no?

-- 
Best Regards,
Marek Marczykowski-Górecki
Invisible Things Lab

On 05.06.2025 16:51, Marek Marczykowski-Górecki wrote:
> On Thu, Jun 05, 2025 at 04:42:53PM +0200, Jan Beulich wrote:
>> On 26.05.2025 17:39, Marek Marczykowski-Górecki wrote:
>>> On Mon, May 26, 2025 at 04:08:17PM +0100, Andrew Cooper wrote:
>>>> On 25/05/2025 3:15 pm, Marek Marczykowski-Górecki wrote:
>>>>> diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
>>>>> index 25189541244d..3ef819a252e4 100644
>>>>> --- a/xen/arch/x86/setup.c
>>>>> +++ b/xen/arch/x86/setup.c
>>>>> @@ -1481,6 +1481,8 @@ void asmlinkage __init noreturn __start_xen(void)
>>>>>          highmem_start &= ~((1UL << L3_PAGETABLE_SHIFT) - 1);
>>>>>  #endif
>>>>>  
>>>>> +    console_init_pre_relocate();
>>>>> +
>>>>>      /*
>>>>>       * Iterate backwards over all superpage-aligned RAM regions.
>>>>>       *
>>>>> @@ -1606,6 +1608,12 @@ void asmlinkage __init noreturn __start_xen(void)
>>>>>      if ( !xen_phys_start )
>>>>>          panic("Not enough memory to relocate Xen\n");
>>>>>  
>>>>> +    /*
>>>>> +     * Notify console drivers about relocation, before reusing old Xen's
>>>>> +     * memory.
>>>>> +     */
>>>>> +    console_init_post_relocate();
>>>>> +
>>>>
>>>> With reference to the next patch, there are printk()'s in this region
>>>> which want to work (in case something goes very wrong), so I don't think
>>>> setting dbc->suspended is the best approach.
>>>
>>> I guess the post_relocate hook might be moved a bit earlier, but still,
>>> once relocation happens, the xhci console is not functional until it
>>> gets relocated too (for example - it would post new TRBs into a ring
>>> that isn't actually monitored by the controller).
>>
>> Why is it that this ring is dependent upon Xen's position? If the ring was
>> dynamically allocated, it wouldn't change position when Xen is moved.
> 
> The console is setup quite early, I don't think I can allocate memory at
> this stage, no?

But you allocate before Xen is moved, I suppose.

Jan

On Thu, Jun 05, 2025 at 06:05:02PM +0200, Jan Beulich wrote:
> On 05.06.2025 16:51, Marek Marczykowski-Górecki wrote:
> > On Thu, Jun 05, 2025 at 04:42:53PM +0200, Jan Beulich wrote:
> >> Why is it that this ring is dependent upon Xen's position? If the ring was
> >> dynamically allocated, it wouldn't change position when Xen is moved.
> > 
> > The console is setup quite early, I don't think I can allocate memory at
> > this stage, no?
> 
> But you allocate before Xen is moved, I suppose.

Well, I have those buffers in BSS exactly to avoid the need to allocate
them (or rather: have bootloader allocate them for me).

-- 
Best Regards,
Marek Marczykowski-Górecki
Invisible Things Lab

On 05.06.2025 18:08, Marek Marczykowski-Górecki wrote:
> On Thu, Jun 05, 2025 at 06:05:02PM +0200, Jan Beulich wrote:
>> On 05.06.2025 16:51, Marek Marczykowski-Górecki wrote:
>>> On Thu, Jun 05, 2025 at 04:42:53PM +0200, Jan Beulich wrote:
>>>> Why is it that this ring is dependent upon Xen's position? If the ring was
>>>> dynamically allocated, it wouldn't change position when Xen is moved.
>>>
>>> The console is setup quite early, I don't think I can allocate memory at
>>> this stage, no?
>>
>> But you allocate before Xen is moved, I suppose.
> 
> Well, I have those buffers in BSS exactly to avoid the need to allocate
> them (or rather: have bootloader allocate them for me).

Yet them remaining in .bss is now getting in the way. Imo moving them to
.init.* and adding proper allocation is going to be easier than the hook-
ary you are proposing.

Jan

On Fri, Jun 06, 2025 at 08:26:33AM +0200, Jan Beulich wrote:
> On 05.06.2025 18:08, Marek Marczykowski-Górecki wrote:
> > On Thu, Jun 05, 2025 at 06:05:02PM +0200, Jan Beulich wrote:
> >> On 05.06.2025 16:51, Marek Marczykowski-Górecki wrote:
> >>> On Thu, Jun 05, 2025 at 04:42:53PM +0200, Jan Beulich wrote:
> >>>> Why is it that this ring is dependent upon Xen's position? If the ring was
> >>>> dynamically allocated, it wouldn't change position when Xen is moved.
> >>>
> >>> The console is setup quite early, I don't think I can allocate memory at
> >>> this stage, no?
> >>
> >> But you allocate before Xen is moved, I suppose.
> > 
> > Well, I have those buffers in BSS exactly to avoid the need to allocate
> > them (or rather: have bootloader allocate them for me).
> 
> Yet them remaining in .bss is now getting in the way. Imo moving them to
> .init.* and adding proper allocation is going to be easier than the hook-
> ary you are proposing.

So, when would you propose to allocate them? Wouldn't that be yet
another hook?

-- 
Best Regards,
Marek Marczykowski-Górecki
Invisible Things Lab

On 06.06.2025 17:54, Marek Marczykowski-Górecki wrote:
> On Fri, Jun 06, 2025 at 08:26:33AM +0200, Jan Beulich wrote:
>> On 05.06.2025 18:08, Marek Marczykowski-Górecki wrote:
>>> On Thu, Jun 05, 2025 at 06:05:02PM +0200, Jan Beulich wrote:
>>>> On 05.06.2025 16:51, Marek Marczykowski-Górecki wrote:
>>>>> On Thu, Jun 05, 2025 at 04:42:53PM +0200, Jan Beulich wrote:
>>>>>> Why is it that this ring is dependent upon Xen's position? If the ring was
>>>>>> dynamically allocated, it wouldn't change position when Xen is moved.
>>>>>
>>>>> The console is setup quite early, I don't think I can allocate memory at
>>>>> this stage, no?
>>>>
>>>> But you allocate before Xen is moved, I suppose.
>>>
>>> Well, I have those buffers in BSS exactly to avoid the need to allocate
>>> them (or rather: have bootloader allocate them for me).
>>
>> Yet them remaining in .bss is now getting in the way. Imo moving them to
>> .init.* and adding proper allocation is going to be easier than the hook-
>> ary you are proposing.
> 
> So, when would you propose to allocate them? Wouldn't that be yet
> another hook?

Exactly one, yes. Given Andrew's earlier reply my understanding was that to
get things correct in your proposed model would end up requiring more than
one. However, the point in time where move_xen() is called is still too
early to allocate memory a (somewhat) normal way - even the boot allocator
gets seeded only later. So I guess console_init_preirq() may need to inform
its caller how much memory is going to be needed later on (and what address
constraints there are, if any). Then a suitable chunk would need setting
aside kind of like it's done for kexec. That address would then need to be
passed into the new hook.

Jan

On 10/06/2025 8:52 am, Jan Beulich wrote:
> On 06.06.2025 17:54, Marek Marczykowski-Górecki wrote:
>> On Fri, Jun 06, 2025 at 08:26:33AM +0200, Jan Beulich wrote:
>>> On 05.06.2025 18:08, Marek Marczykowski-Górecki wrote:
>>>> On Thu, Jun 05, 2025 at 06:05:02PM +0200, Jan Beulich wrote:
>>>>> On 05.06.2025 16:51, Marek Marczykowski-Górecki wrote:
>>>>>> On Thu, Jun 05, 2025 at 04:42:53PM +0200, Jan Beulich wrote:
>>>>>>> Why is it that this ring is dependent upon Xen's position? If the ring was
>>>>>>> dynamically allocated, it wouldn't change position when Xen is moved.
>>>>>> The console is setup quite early, I don't think I can allocate memory at
>>>>>> this stage, no?
>>>>> But you allocate before Xen is moved, I suppose.
>>>> Well, I have those buffers in BSS exactly to avoid the need to allocate
>>>> them (or rather: have bootloader allocate them for me).
>>> Yet them remaining in .bss is now getting in the way. Imo moving them to
>>> .init.* and adding proper allocation is going to be easier than the hook-
>>> ary you are proposing.
>> So, when would you propose to allocate them? Wouldn't that be yet
>> another hook?
> Exactly one, yes. Given Andrew's earlier reply my understanding was that to
> get things correct in your proposed model would end up requiring more than
> one. However, the point in time where move_xen() is called is still too
> early to allocate memory a (somewhat) normal way - even the boot allocator
> gets seeded only later. So I guess console_init_preirq() may need to inform
> its caller how much memory is going to be needed later on (and what address
> constraints there are, if any). Then a suitable chunk would need setting
> aside kind of like it's done for kexec. That address would then need to be
> passed into the new hook.

How about initialising the console using _va(_pa(xxx)) of the Xen
datastructures?

Xen is mapped into the directmap (pagetable handling depends on this),
so these pointers will work right from the very outset, and will
(intentionally) refer to the old position.

After relocation (specifically, before we free the old Xen image), we
can drain in-flight requests and update the pointers.

~Andrew

On Tue, Jun 10, 2025 at 01:54:04PM +0100, Andrew Cooper wrote:
> On 10/06/2025 8:52 am, Jan Beulich wrote:
> > On 06.06.2025 17:54, Marek Marczykowski-Górecki wrote:
> >> On Fri, Jun 06, 2025 at 08:26:33AM +0200, Jan Beulich wrote:
> >>> On 05.06.2025 18:08, Marek Marczykowski-Górecki wrote:
> >>>> On Thu, Jun 05, 2025 at 06:05:02PM +0200, Jan Beulich wrote:
> >>>>> On 05.06.2025 16:51, Marek Marczykowski-Górecki wrote:
> >>>>>> On Thu, Jun 05, 2025 at 04:42:53PM +0200, Jan Beulich wrote:
> >>>>>>> Why is it that this ring is dependent upon Xen's position? If the ring was
> >>>>>>> dynamically allocated, it wouldn't change position when Xen is moved.
> >>>>>> The console is setup quite early, I don't think I can allocate memory at
> >>>>>> this stage, no?
> >>>>> But you allocate before Xen is moved, I suppose.
> >>>> Well, I have those buffers in BSS exactly to avoid the need to allocate
> >>>> them (or rather: have bootloader allocate them for me).
> >>> Yet them remaining in .bss is now getting in the way. Imo moving them to
> >>> .init.* and adding proper allocation is going to be easier than the hook-
> >>> ary you are proposing.
> >> So, when would you propose to allocate them? Wouldn't that be yet
> >> another hook?
> > Exactly one, yes. Given Andrew's earlier reply my understanding was that to
> > get things correct in your proposed model would end up requiring more than
> > one. However, the point in time where move_xen() is called is still too
> > early to allocate memory a (somewhat) normal way - even the boot allocator
> > gets seeded only later. So I guess console_init_preirq() may need to inform
> > its caller how much memory is going to be needed later on (and what address
> > constraints there are, if any). Then a suitable chunk would need setting
> > aside kind of like it's done for kexec. That address would then need to be
> > passed into the new hook.
> 
> How about initialising the console using _va(_pa(xxx)) of the Xen
> datastructures?
> 
> Xen is mapped into the directmap (pagetable handling depends on this),
> so these pointers will work right from the very outset, and will
> (intentionally) refer to the old position.
> 
> After relocation (specifically, before we free the old Xen image), we
> can drain in-flight requests and update the pointers.

Ah, you mean to use directmap to access the ring pages. Yes, that should
work, and should be enough to not need the pre-relocate hook. The post
relocate would still be needed though (none of existing console init
hooks fits this purpose). I'll try that approach.

-- 
Best Regards,
Marek Marczykowski-Górecki
Invisible Things Lab