macOS's Cocoa event handling must be done on the initial (main) thread

of the process. Furthermore, if library or application code uses

libdispatch, the main dispatch queue must be handling events on the main

thread as well.

So far, this has affected Qemu in both the Cocoa and SDL UIs, although

in different ways: the Cocoa UI replaces the default qemu_main function

with one that spins Qemu's internal main event loop off onto a

background thread. SDL (which uses Cocoa internally) on the other hand

uses a polling approach within Qemu's main event loop. Events are

polled during the SDL UI's dpy_refresh callback, which happens to run

on the main thread by default.

As UIs are mutually exclusive, this works OK as long as nothing else

needs platform-native event handling. In the next patch, a new device is

introduced based on the ParavirtualizedGraphics.framework in macOS.

This uses libdispatch internally, and only works when events are being

handled on the main runloop. With the current system, it works when

using either the Cocoa or the SDL UI. However, it does not when running

headless. Moreover, any attempt to install a similar scheme to the

Cocoa UI's main thread replacement fails when combined with the SDL

UI.

This change tidies up main thread management to be more flexible.

* The qemu_main global function pointer is a custom function for the

main thread, and it may now be NULL. When it is, the main thread

runs the main Qemu loop. This represents the traditional setup.

* When non-null, spawning the main Qemu event loop on a separate

thread is now done centrally rather than inside the Cocoa UI code.

* For most platforms, qemu_main is indeed NULL by default, but on

Darwin, it defaults to a function that runs the CFRunLoop.

* The Cocoa UI sets qemu_main to a function which runs the

NSApplication event handling runloop, as is usual for a Cocoa app.

* The SDL UI overrides the qemu_main function to NULL, thus

specifying that Qemu's main loop must run on the main

thread.

* The GTK UI also overrides the qemu_main function to NULL.

* For other UIs, or in the absence of UIs, the platform's default

behaviour is followed.

This means that on macOS, the platform's runloop events are always

handled, regardless of chosen UI. The new PV graphics device will

thus work in all configurations. There is no functional change on other

operating systems.

Signed-off-by: Phil Dennis-Jordan <phil@philjordan.eu>

Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>

---

v5:

* Simplified the way of setting/clearing the main loop by going back

to setting qemu_main directly, but narrowing the scope of what it

needs to do, and it can now be NULL.

v6:

* Folded function qemu_run_default_main_on_new_thread's code into

main()

* Removed whitespace changes left over on lines near code removed

between v4 and v5

v9:

* Set qemu_main to NULL for GTK UI as well.

include/qemu-main.h | 3 +--

include/qemu/typedefs.h | 1 +

system/main.c | 50 ++++++++++++++++++++++++++++++++++----

ui/cocoa.m | 54 ++++++++++-------------------------------

ui/gtk.c | 3 +++

ui/sdl2.c | 4 +++

6 files changed, 67 insertions(+), 48 deletions(-)

diff --git a/include/qemu-main.h b/include/qemu-main.h

index XXXXXXX..XXXXXXX 100644

--- a/include/qemu-main.h

+++ b/include/qemu-main.h

@@ -XXX,XX +XXX,XX @@

#ifndef QEMU_MAIN_H

#define QEMU_MAIN_H

-int qemu_default_main(void);

-extern int (*qemu_main)(void);

+extern qemu_main_fn qemu_main;

#endif /* QEMU_MAIN_H */

diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h

index XXXXXXX..XXXXXXX 100644

--- a/include/qemu/typedefs.h

+++ b/include/qemu/typedefs.h

@@ -XXX,XX +XXX,XX @@ typedef struct IRQState *qemu_irq;

* Function types

*/

typedef void (*qemu_irq_handler)(void *opaque, int n, int level);

+typedef int (*qemu_main_fn)(void);

#endif /* QEMU_TYPEDEFS_H */

diff --git a/system/main.c b/system/main.c

index XXXXXXX..XXXXXXX 100644

--- a/system/main.c

+++ b/system/main.c

@@ -XXX,XX +XXX,XX @@

#include "qemu/osdep.h"

#include "qemu-main.h"

+#include "qemu/main-loop.h"

#include "sysemu/sysemu.h"

-#ifdef CONFIG_SDL

-#include <SDL.h>

+#ifdef CONFIG_DARWIN

+#include <CoreFoundation/CoreFoundation.h>

#endif

-int qemu_default_main(void)

+static int qemu_default_main(void)

{

int status;

@@ -XXX,XX +XXX,XX @@ int qemu_default_main(void)

return status;

}

-int (*qemu_main)(void) = qemu_default_main;

+/*

+ * Various macOS system libraries, including the Cocoa UI and anything using

+ * libdispatch, such as ParavirtualizedGraphics.framework, requires that the

+ * main runloop, on the main (initial) thread be running or at least regularly

+ * polled for events. A special mode is therefore supported, where the QEMU

+ * main loop runs on a separate thread and the main thread handles the

+ * CF/Cocoa runloop.

+ */

+

+static void *call_qemu_default_main(void *opaque)

+{

+ int status;

+

+ bql_lock();

+ status = qemu_default_main();

+ bql_unlock();

+

+ exit(status);

+}

+

+#ifdef CONFIG_DARWIN

+static int os_darwin_cfrunloop_main(void)

+{

+ CFRunLoopRun();

+ abort();

+}

+

+qemu_main_fn qemu_main = os_darwin_cfrunloop_main;

+#else

+qemu_main_fn qemu_main;

+#endif

int main(int argc, char **argv)

{

+ QemuThread main_loop_thread;

+

qemu_init(argc, argv);

- return qemu_main();

+ if (qemu_main) {

+ qemu_thread_create(&main_loop_thread, "qemu_main",

+ call_qemu_default_main, NULL, QEMU_THREAD_DETACHED);

+ bql_unlock();

+ return qemu_main();

+ } else {

+ qemu_default_main();

+ }

}

diff --git a/ui/cocoa.m b/ui/cocoa.m

index XXXXXXX..XXXXXXX 100644

--- a/ui/cocoa.m

+++ b/ui/cocoa.m

@@ -XXX,XX +XXX,XX @@

int height;

} QEMUScreen;

+@class QemuCocoaPasteboardTypeOwner;

+

static void cocoa_update(DisplayChangeListener *dcl,

int x, int y, int w, int h);

@@ -XXX,XX +XXX,XX @@ static void cocoa_switch(DisplayChangeListener *dcl,

static NSInteger cbchangecount = -1;

static QemuClipboardInfo *cbinfo;

static QemuEvent cbevent;

+static QemuCocoaPasteboardTypeOwner *cbowner;

// Utility functions to run specified code block with the BQL held

typedef void (^CodeBlock)(void);

@@ -XXX,XX +XXX,XX @@ - (void) dealloc

{

COCOA_DEBUG("QemuCocoaAppController: dealloc\n");

- if (cocoaView)

- [cocoaView release];

+ [cocoaView release];

+ [cbowner release];

+ cbowner = nil;

+

[super dealloc];

}

@@ -XXX,XX +XXX,XX @@ - (void)pasteboard:(NSPasteboard *)sender provideDataForType:(NSPasteboardType)t

@end

-static QemuCocoaPasteboardTypeOwner *cbowner;

-

static void cocoa_clipboard_notify(Notifier *notifier, void *data);

static void cocoa_clipboard_request(QemuClipboardInfo *info,

QemuClipboardType type);

@@ -XXX,XX +XXX,XX @@ static void cocoa_clipboard_request(QemuClipboardInfo *info,

}

}

-/*

- * The startup process for the OSX/Cocoa UI is complicated, because

- * OSX insists that the UI runs on the initial main thread, and so we

- * need to start a second thread which runs the qemu_default_main():

- * in main():

- * in cocoa_display_init():

- * assign cocoa_main to qemu_main

- * create application, menus, etc

- * in cocoa_main():

- * create qemu-main thread

- * enter OSX run loop

- */

-

-static void *call_qemu_main(void *opaque)

-{

- int status;

-

- COCOA_DEBUG("Second thread: calling qemu_default_main()\n");

- bql_lock();

- status = qemu_default_main();

- bql_unlock();

- COCOA_DEBUG("Second thread: qemu_default_main() returned, exiting\n");

- [cbowner release];

- exit(status);

-}

-

static int cocoa_main(void)

{

- QemuThread thread;

-

- COCOA_DEBUG("Entered %s()\n", __func__);

-

- bql_unlock();

- qemu_thread_create(&thread, "qemu_main", call_qemu_main,

- NULL, QEMU_THREAD_DETACHED);

-

- // Start the main event loop

COCOA_DEBUG("Main thread: entering OSX run loop\n");

[NSApp run];

COCOA_DEBUG("Main thread: left OSX run loop, which should never happen\n");

@@ -XXX,XX +XXX,XX @@ static void cocoa_display_init(DisplayState *ds, DisplayOptions *opts)

COCOA_DEBUG("qemu_cocoa: cocoa_display_init\n");

- qemu_main = cocoa_main;

-

// Pull this console process up to being a fully-fledged graphical

// app with a menubar and Dock icon

ProcessSerialNumber psn = { 0, kCurrentProcess };

@@ -XXX,XX +XXX,XX @@ static void cocoa_display_init(DisplayState *ds, DisplayOptions *opts)

qemu_clipboard_peer_register(&cbpeer);

[pool release];

+

+ /*

+ * The Cocoa UI will run the NSApplication runloop on the main thread

+ * rather than the default Core Foundation one.

+ */

+ qemu_main = cocoa_main;

}

static QemuDisplay qemu_display_cocoa = {

diff --git a/ui/gtk.c b/ui/gtk.c

index XXXXXXX..XXXXXXX 100644

--- a/ui/gtk.c

+++ b/ui/gtk.c

@@ -XXX,XX +XXX,XX @@

#include "qemu/cutils.h"

#include "qemu/error-report.h"

#include "qemu/main-loop.h"

+#include "qemu-main.h"

#include "ui/console.h"

#include "ui/gtk.h"

@@ -XXX,XX +XXX,XX @@ static void gtk_display_init(DisplayState *ds, DisplayOptions *opts)

#ifdef CONFIG_GTK_CLIPBOARD

gd_clipboard_init(s);

#endif /* CONFIG_GTK_CLIPBOARD */

+

+ qemu_main = NULL;

}

static void early_gtk_display_init(DisplayOptions *opts)

diff --git a/ui/sdl2.c b/ui/sdl2.c

index XXXXXXX..XXXXXXX 100644

--- a/ui/sdl2.c

+++ b/ui/sdl2.c

@@ -XXX,XX +XXX,XX @@

#include "sysemu/sysemu.h"

#include "ui/win32-kbd-hook.h"

#include "qemu/log.h"

+#include "qemu-main.h"

static int sdl2_num_outputs;

static struct sdl2_console *sdl2_console;

@@ -XXX,XX +XXX,XX @@ static void sdl2_display_init(DisplayState *ds, DisplayOptions *o)

}

atexit(sdl_cleanup);

+

+ /* SDL's event polling (in dpy_refresh) must happen on the main thread. */

+ qemu_main = NULL;

}

static QemuDisplay qemu_display_sdl2 = {

--

2.39.3 (Apple Git-145)

MacOS provides a framework (library) that allows any vmm to implement a

paravirtualized 3d graphics passthrough to the host metal stack called

ParavirtualizedGraphics.Framework (PVG). The library abstracts away

almost every aspect of the paravirtualized device model and only provides

and receives callbacks on MMIO access as well as to share memory address

space between the VM and PVG.

This patch implements a QEMU device that drives PVG for the VMApple

variant of it.

Signed-off-by: Alexander Graf <graf@amazon.com>

Co-authored-by: Alexander Graf <graf@amazon.com>

Subsequent changes:

* Cherry-pick/rebase conflict fixes, API use updates.

* Moved from hw/vmapple/ (useful outside that machine type)

* Overhaul of threading model, many thread safety improvements.

* Asynchronous rendering.

* Memory and object lifetime fixes.

* Refactoring to split generic and (vmapple) MMIO variant specific

code.

Implementation wise, most of the complexity lies in the differing threading

models of ParavirtualizedGraphics.framework, which uses libdispatch and

internal locks, versus QEMU, which heavily uses the BQL, especially during

memory-mapped device I/O. Great care has therefore been taken to prevent

deadlocks by never calling into PVG methods while holding the BQL, and

similarly never acquiring the BQL in a callback from PVG. Different strategies

have been used (libdispatch, blocking and non-blocking BHs, RCU, etc.)

depending on the specific requirements at each framework entry and exit point.

Signed-off-by: Phil Dennis-Jordan <phil@philjordan.eu>

---

v2:

* Cherry-pick/rebase conflict fixes

* BQL function renaming

* Moved from hw/vmapple/ (useful outside that machine type)

* Code review comments: Switched to DEFINE_TYPES macro & little endian

MMIO.

* Removed some dead/superfluous code

* Mad set_mode thread & memory safe

* Added migration blocker due to lack of (de-)serialisation.

* Fixes to ObjC refcounting and autorelease pool usage.

* Fixed ObjC new/init misuse

* Switched to ObjC category extension for private property.

* Simplified task memory mapping and made it thread safe.

* Refactoring to split generic and vmapple MMIO variant specific

code.

* Switched to asynchronous MMIO writes on x86-64

* Rendering and graphics update are now done asynchronously

* Fixed cursor handling

* Coding convention fixes

* Removed software cursor compositing

v3:

* Rebased on latest upstream, fixed breakages including switching to Resettable methods.

* Squashed patches dealing with dGPUs, MMIO area size, and GPU picking.

* Allow re-entrant MMIO; this simplifies the code and solves the divergence

between x86-64 and arm64 variants.

v4:

* Renamed '-vmapple' device variant to '-mmio'

* MMIO device type now requires aarch64 host and guest

* Complete overhaul of the glue code for making Qemu's and

ParavirtualizedGraphics.framework's threading and synchronisation models

work together. Calls into PVG are from dispatch queues while the

BQL-holding initiating thread processes AIO context events; callbacks from

PVG are scheduled as BHs on the BQL/main AIO context, awaiting completion

where necessary.

* Guest frame rendering state is covered by the BQL, with only the PVG calls

outside the lock, and serialised on the named render_queue.

* Simplified logic for dropping frames in-flight during mode changes, fixed

bug in pending frames logic.

* Addressed smaller code review notes such as: function naming, object type

declarations, type names/declarations/casts, code formatting, #include

order, over-cautious ObjC retain/release, what goes in init vs realize,

etc.

v5:

* Smaller non-functional fixes in response to review comments, such as using

NULL for the AIO_WAIT_WHILE context argument, type name formatting,

deleting leftover debug code, logging improvements, state struct field

order and documentation improvements, etc.

* Instead of a single condition variable for all synchronous BH job types,

there is now one for each callback block. This reduces the number

of threads being awoken unnecessarily to near zero.

* MMIO device variant: Unified the BH job for raising interrupts.

* Use DMA APIs for PVG framework's guest memory read requests.

* Thread safety improvements: ensure mutable AppleGFXState fields are not

accessed outside the appropriate lock. Added dedicated mutex for the task

list.

* Retain references to MemoryRegions for which there exist mappings in each

PGTask, and for IOSurface mappings.

v6:

* Switched PGTask_s's' mapped_regions from GPtrArray to GArray

* Allow DisplaySurface to manage its own vram now that texture -> vram copy

occurs under BQL.

* Memory mapping operations now use RCU_READ_LOCK_GUARD() where possible

instead of a heavy-weight BH job to acquire the BQL.

* Changed PVG cursor and mode setting callbacks to kick off BHs instead of

libdispatch tasks which then locked the BQL explicitly.

* The single remaining callback which must wait for a BH to complete now

creates an ephemeral QemuSemaphore to await completion.

* Re-removed tracking of mapped surface manager memory regions. Just look up

and ref/unref the memory regions in the map/unmap callbacks.

* Re-ordered functions in apple-gfx.m to group them by area of functionality.

* Improved comments and tweaked some names.

v7:

* Use g_ptr_array_find() helper function

* Error handling coding style tweak

v8:

* Renamed apple_gfx_host_address_for_gpa_range() to

apple_gfx_host_ptr_for_gpa_range(), and made it return a void* instead of

uintptr_t. Fixed up callers and related code.

* Some adjustments to types used.

* Variable naming tweaks for better clarity.

* Fixed leak in unlikely realize error case.

* Fixed typo in unmap call.

* Don't bother with dummy argument for g_ptr_array_find(), NULL works too.

v9:

* Pass device pointer to graphic_console_init().

* Slightly re-ordered initialisation code.

* Simplified error handling during realize().

* Simplified code without functional changes, adjusted code & comment

formatting.

hw/display/Kconfig | 9 +

hw/display/apple-gfx-mmio.m | 281 +++++++++++++

hw/display/apple-gfx.h | 65 +++

hw/display/apple-gfx.m | 764 ++++++++++++++++++++++++++++++++++++

hw/display/meson.build | 4 +

hw/display/trace-events | 28 ++

meson.build | 4 +

7 files changed, 1155 insertions(+)

create mode 100644 hw/display/apple-gfx-mmio.m

create mode 100644 hw/display/apple-gfx.h

create mode 100644 hw/display/apple-gfx.m

diff --git a/hw/display/Kconfig b/hw/display/Kconfig

index XXXXXXX..XXXXXXX 100644

--- a/hw/display/Kconfig

+++ b/hw/display/Kconfig

@@ -XXX,XX +XXX,XX @@ config XLNX_DISPLAYPORT

config DM163

bool

+

+config MAC_PVG

+ bool

+ default y

+

+config MAC_PVG_MMIO

+ bool

+ depends on MAC_PVG && AARCH64

+

diff --git a/hw/display/apple-gfx-mmio.m b/hw/display/apple-gfx-mmio.m

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/hw/display/apple-gfx-mmio.m

@@ -XXX,XX +XXX,XX @@

+/*

+ * QEMU Apple ParavirtualizedGraphics.framework device, MMIO (arm64) variant

+ *

+ * Copyright © 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.

+ *

+ * This work is licensed under the terms of the GNU GPL, version 2 or later.

+ * See the COPYING file in the top-level directory.

+ *

+ * SPDX-License-Identifier: GPL-2.0-or-later

+ *

+ * ParavirtualizedGraphics.framework is a set of libraries that macOS provides

+ * which implements 3d graphics passthrough to the host as well as a

+ * proprietary guest communication channel to drive it. This device model

+ * implements support to drive that library from within QEMU as an MMIO-based

+ * system device for macOS on arm64 VMs.

+ */

+

+#include "qemu/osdep.h"

+#import <ParavirtualizedGraphics/ParavirtualizedGraphics.h>

+#include "apple-gfx.h"

+#include "monitor/monitor.h"

+#include "hw/sysbus.h"

+#include "hw/irq.h"

+#include "trace.h"

+#include "qemu/log.h"

+

+OBJECT_DECLARE_SIMPLE_TYPE(AppleGFXMMIOState, APPLE_GFX_MMIO)

+

+/*

+ * ParavirtualizedGraphics.Framework only ships header files for the PCI

+ * variant which does not include IOSFC descriptors and host devices. We add

+ * their definitions here so that we can also work with the ARM version.

+ */

+typedef bool(^IOSFCRaiseInterrupt)(uint32_t vector);

+typedef bool(^IOSFCUnmapMemory)(

+ void *, void *, void *, void *, void *, void *);

+typedef bool(^IOSFCMapMemory)(

+ uint64_t phys, uint64_t len, bool ro, void **va, void *, void *);

+

+@interface PGDeviceDescriptor (IOSurfaceMapper)

+@property (readwrite, nonatomic) bool usingIOSurfaceMapper;

+@end

+

+@interface PGIOSurfaceHostDeviceDescriptor : NSObject

+-(PGIOSurfaceHostDeviceDescriptor *)init;

+@property (readwrite, nonatomic, copy, nullable) IOSFCMapMemory mapMemory;

+@property (readwrite, nonatomic, copy, nullable) IOSFCUnmapMemory unmapMemory;

+@property (readwrite, nonatomic, copy, nullable) IOSFCRaiseInterrupt raiseInterrupt;

+@end

+

+@interface PGIOSurfaceHostDevice : NSObject

+-(instancetype)initWithDescriptor:(PGIOSurfaceHostDeviceDescriptor *)desc;

+-(uint32_t)mmioReadAtOffset:(size_t)offset;

+-(void)mmioWriteAtOffset:(size_t)offset value:(uint32_t)value;

+@end

+

+struct AppleGFXMapSurfaceMemoryJob;

+struct AppleGFXMMIOState {

+ SysBusDevice parent_obj;

+

+ AppleGFXState common;

+

+ qemu_irq irq_gfx;

+ qemu_irq irq_iosfc;

+ MemoryRegion iomem_iosfc;

+ PGIOSurfaceHostDevice *pgiosfc;

+};

+

+typedef struct AppleGFXMMIOJob {

+ AppleGFXMMIOState *state;

+ uint64_t offset;

+ uint64_t value;

+ bool completed;

+} AppleGFXMMIOJob;

+

+static void iosfc_do_read(void *opaque)

+{

+ AppleGFXMMIOJob *job = opaque;

+ job->value = [job->state->pgiosfc mmioReadAtOffset:job->offset];

+ qatomic_set(&job->completed, true);

+ aio_wait_kick();

+}

+

+static uint64_t iosfc_read(void *opaque, hwaddr offset, unsigned size)

+{

+ AppleGFXMMIOJob job = {

+ .state = opaque,

+ .offset = offset,

+ .completed = false,

+ };

+ dispatch_queue_t queue =

+ dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);

+

+ dispatch_async_f(queue, &job, iosfc_do_read);

+ AIO_WAIT_WHILE(NULL, !qatomic_read(&job.completed));

+

+ trace_apple_gfx_mmio_iosfc_read(offset, job.value);

+ return job.value;

+}

+

+static void iosfc_do_write(void *opaque)

+{

+ AppleGFXMMIOJob *job = opaque;

+ [job->state->pgiosfc mmioWriteAtOffset:job->offset value:job->value];

+ qatomic_set(&job->completed, true);

+ aio_wait_kick();

+}

+

+static void iosfc_write(void *opaque, hwaddr offset, uint64_t val,

+ unsigned size)

+{

+ AppleGFXMMIOJob job = {

+ .state = opaque,

+ .offset = offset,

+ .value = val,

+ .completed = false,

+ };

+ dispatch_queue_t queue =

+ dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);

+

+ dispatch_async_f(queue, &job, iosfc_do_write);

+ AIO_WAIT_WHILE(NULL, !qatomic_read(&job.completed));

+

+ trace_apple_gfx_mmio_iosfc_write(offset, val);

+}

+

+static const MemoryRegionOps apple_iosfc_ops = {

+ .read = iosfc_read,

+ .write = iosfc_write,

+ .endianness = DEVICE_LITTLE_ENDIAN,

+ .valid = {

+ .min_access_size = 4,

+ .max_access_size = 8,

+ },

+ .impl = {

+ .min_access_size = 4,

+ .max_access_size = 8,

+ },

+};

+

+static void raise_irq_bh(void *opaque)

+{

+ qemu_irq *irq = opaque;

+

+ qemu_irq_pulse(*irq);

+}

+

+static void *apple_gfx_mmio_map_surface_memory(uint64_t guest_physical_address,

+ uint64_t length, bool read_only)

+{

+ void *mem;

+ MemoryRegion *region = NULL;

+

+ RCU_READ_LOCK_GUARD();

+ mem = apple_gfx_host_ptr_for_gpa_range(guest_physical_address,

+ length, read_only, &region);

+ if (mem) {

+ memory_region_ref(region);

+ }

+ return mem;

+}

+

+static bool apple_gfx_mmio_unmap_surface_memory(void *ptr)

+{

+ MemoryRegion *region;

+ ram_addr_t offset = 0;

+

+ RCU_READ_LOCK_GUARD();

+ region = memory_region_from_host(ptr, &offset);

+ if (!region) {

+ qemu_log_mask(LOG_GUEST_ERROR, "%s: memory at %p to be unmapped not "

+ "found.\n",

+ __func__, ptr);

+ return false;

+ }

+

+ trace_apple_gfx_iosfc_unmap_memory_region(ptr, region);

+ memory_region_unref(region);

+ return true;

+}

+

+static PGIOSurfaceHostDevice *apple_gfx_prepare_iosurface_host_device(

+ AppleGFXMMIOState *s)

+{

+ PGIOSurfaceHostDeviceDescriptor *iosfc_desc =

+ [PGIOSurfaceHostDeviceDescriptor new];

+ PGIOSurfaceHostDevice *iosfc_host_dev = nil;

+

+ iosfc_desc.mapMemory =

+ ^bool(uint64_t phys, uint64_t len, bool ro, void **va, void *e, void *f) {

+ *va = apple_gfx_mmio_map_surface_memory(phys, len, ro);

+

+ trace_apple_gfx_iosfc_map_memory(phys, len, ro, va, e, f, *va);

+

+ return *va != NULL;

+ };

+

+ iosfc_desc.unmapMemory =

+ ^bool(void *va, void *b, void *c, void *d, void *e, void *f) {

+ return apple_gfx_mmio_unmap_surface_memory(va);

+ };

+

+ iosfc_desc.raiseInterrupt = ^bool(uint32_t vector) {

+ trace_apple_gfx_iosfc_raise_irq(vector);

+ aio_bh_schedule_oneshot(qemu_get_aio_context(),

+ raise_irq_bh, &s->irq_iosfc);

+ return true;

+ };

+

+ iosfc_host_dev =

+ [[PGIOSurfaceHostDevice alloc] initWithDescriptor:iosfc_desc];

+ [iosfc_desc release];

+ return iosfc_host_dev;

+}

+

+static void apple_gfx_mmio_realize(DeviceState *dev, Error **errp)

+{

+ @autoreleasepool {

+ AppleGFXMMIOState *s = APPLE_GFX_MMIO(dev);

+ PGDeviceDescriptor *desc = [PGDeviceDescriptor new];

+

+ desc.raiseInterrupt = ^(uint32_t vector) {

+ trace_apple_gfx_raise_irq(vector);

+ aio_bh_schedule_oneshot(qemu_get_aio_context(),

+ raise_irq_bh, &s->irq_gfx);

+ };

+

+ desc.usingIOSurfaceMapper = true;

+ s->pgiosfc = apple_gfx_prepare_iosurface_host_device(s);

+

+ if (!apple_gfx_common_realize(&s->common, dev, desc, errp)) {

+ [s->pgiosfc release];

+ s->pgiosfc = nil;

+ }

+

+ [desc release];

+ desc = nil;

+ }

+}

+

+static void apple_gfx_mmio_init(Object *obj)

+{

+ AppleGFXMMIOState *s = APPLE_GFX_MMIO(obj);

+

+ apple_gfx_common_init(obj, &s->common, TYPE_APPLE_GFX_MMIO);

+

+ sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->common.iomem_gfx);

+ memory_region_init_io(&s->iomem_iosfc, obj, &apple_iosfc_ops, s,

+ TYPE_APPLE_GFX_MMIO, 0x10000);

+ sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem_iosfc);

+ sysbus_init_irq(SYS_BUS_DEVICE(s), &s->irq_gfx);

+ sysbus_init_irq(SYS_BUS_DEVICE(s), &s->irq_iosfc);

+}

+

+static void apple_gfx_mmio_reset(Object *obj, ResetType type)

+{

+ AppleGFXMMIOState *s = APPLE_GFX_MMIO(obj);

+ [s->common.pgdev reset];

+}

+

+

+static void apple_gfx_mmio_class_init(ObjectClass *klass, void *data)

+{

+ DeviceClass *dc = DEVICE_CLASS(klass);

+ ResettableClass *rc = RESETTABLE_CLASS(klass);

+

+ rc->phases.hold = apple_gfx_mmio_reset;

+ dc->hotpluggable = false;

+ dc->realize = apple_gfx_mmio_realize;

+}

+

+static TypeInfo apple_gfx_mmio_types[] = {

+ {

+ .name = TYPE_APPLE_GFX_MMIO,

+ .parent = TYPE_SYS_BUS_DEVICE,

+ .instance_size = sizeof(AppleGFXMMIOState),

+ .class_init = apple_gfx_mmio_class_init,

+ .instance_init = apple_gfx_mmio_init,

+ }

+};

+DEFINE_TYPES(apple_gfx_mmio_types)

diff --git a/hw/display/apple-gfx.h b/hw/display/apple-gfx.h

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/hw/display/apple-gfx.h

@@ -XXX,XX +XXX,XX @@

+/*

+ * Data structures and functions shared between variants of the macOS

+ * ParavirtualizedGraphics.framework based apple-gfx display adapter.

+ *

+ * SPDX-License-Identifier: GPL-2.0-or-later

+ */

+

+#ifndef QEMU_APPLE_GFX_H

+#define QEMU_APPLE_GFX_H

+

+#define TYPE_APPLE_GFX_MMIO "apple-gfx-mmio"

+#define TYPE_APPLE_GFX_PCI "apple-gfx-pci"

+

+#include "qemu/osdep.h"

+#include <dispatch/dispatch.h>

+#import <ParavirtualizedGraphics/ParavirtualizedGraphics.h>

+#include "qemu/typedefs.h"

+#include "exec/memory.h"

+#include "ui/surface.h"

+

+@class PGDeviceDescriptor;

+@protocol PGDevice;

+@protocol PGDisplay;

+@protocol MTLDevice;

+@protocol MTLTexture;

+@protocol MTLCommandQueue;

+

+typedef QTAILQ_HEAD(, PGTask_s) PGTaskList;

+

+typedef struct AppleGFXState {

+ /* Initialised on init/realize() */

+ MemoryRegion iomem_gfx;

+ id<PGDevice> pgdev;

+ id<PGDisplay> pgdisp;

+ QemuConsole *con;

+ id<MTLDevice> mtl;

+ id<MTLCommandQueue> mtl_queue;

+ dispatch_queue_t render_queue;

+

+ /* List `tasks` is protected by task_mutex */

+ QemuMutex task_mutex;

+ PGTaskList tasks;

+

+ /* Mutable state (BQL protected) */

+ QEMUCursor *cursor;

+ DisplaySurface *surface;

+ id<MTLTexture> texture;

+ int8_t pending_frames; /* # guest frames in the rendering pipeline */

+ bool gfx_update_requested; /* QEMU display system wants a new frame */

+ bool new_frame_ready; /* Guest has rendered a frame, ready to be used */

+ bool using_managed_texture_storage;

+

+ /* Mutable state (atomic) */

+ bool cursor_show;

+} AppleGFXState;

+

+void apple_gfx_common_init(Object *obj, AppleGFXState *s, const char* obj_name);

+bool apple_gfx_common_realize(AppleGFXState *s, DeviceState *dev,

+ PGDeviceDescriptor *desc, Error **errp);

+void *apple_gfx_host_ptr_for_gpa_range(uint64_t guest_physical,

+ uint64_t length, bool read_only,

+ MemoryRegion **mapping_in_region);

+

+#endif

+

diff --git a/hw/display/apple-gfx.m b/hw/display/apple-gfx.m

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/hw/display/apple-gfx.m

@@ -XXX,XX +XXX,XX @@

+/*

+ * QEMU Apple ParavirtualizedGraphics.framework device

+ *

+ * Copyright © 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.

+ *

+ * This work is licensed under the terms of the GNU GPL, version 2 or later.

+ * See the COPYING file in the top-level directory.

+ *

+ * SPDX-License-Identifier: GPL-2.0-or-later

+ *

+ * ParavirtualizedGraphics.framework is a set of libraries that macOS provides

+ * which implements 3d graphics passthrough to the host as well as a

+ * proprietary guest communication channel to drive it. This device model

+ * implements support to drive that library from within QEMU.

+ */

+

+#include "qemu/osdep.h"

+#import <ParavirtualizedGraphics/ParavirtualizedGraphics.h>

+#include <mach/mach_vm.h>

+#include "apple-gfx.h"

+#include "trace.h"

+#include "qemu-main.h"

+#include "exec/address-spaces.h"

+#include "migration/blocker.h"

+#include "monitor/monitor.h"

+#include "qemu/main-loop.h"

+#include "qemu/cutils.h"

+#include "qemu/log.h"

+#include "qapi/visitor.h"

+#include "qapi/error.h"

+#include "sysemu/dma.h"

+#include "ui/console.h"

+

+static const PGDisplayCoord_t apple_gfx_modes[] = {

+ { .x = 1440, .y = 1080 },

+ { .x = 1280, .y = 1024 },

+};

+

+/* ------ PGTask and task operations: new/destroy/map/unmap ------ */

+

+/*

+ * This implements the type declared in <ParavirtualizedGraphics/PGDevice.h>

+ * which is opaque from the framework's point of view. It is used in callbacks

+ * in the form of its typedef PGTask_t, which also already exists in the

+ * framework headers.

+ *

+ * A "task" in PVG terminology represents a host-virtual contiguous address

+ * range which is reserved in a large chunk on task creation. The mapMemory

+ * callback then requests ranges of guest system memory (identified by their

+ * GPA) to be mapped into subranges of this reserved address space.

+ * This type of operation isn't well-supported by QEMU's memory subsystem,

+ * but it is fortunately trivial to achieve with Darwin's mach_vm_remap() call,

+ * which allows us to refer to the same backing memory via multiple virtual

+ * address ranges. The Mach VM APIs are therefore used throughout for managing

+ * task memory.

+ */

+struct PGTask_s {

+ QTAILQ_ENTRY(PGTask_s) node;

+ AppleGFXState *s;

+ mach_vm_address_t address;

+ uint64_t len;

+ /*

+ * All unique MemoryRegions for which a mapping has been created in in this

+ * task, and on which we have thus called memory_region_ref(). There are

+ * usually very few regions of system RAM in total, so we expect this array

+ * to be very short. Therefore, no need for sorting or fancy search

+ * algorithms, linear search will do.

+ * Protected by AppleGFXState's task_mutex.

+ */

+ GPtrArray *mapped_regions;

+};

+

+static Error *apple_gfx_mig_blocker;

+

+static PGTask_t *apple_gfx_new_task(AppleGFXState *s, uint64_t len)

+{

+ mach_vm_address_t task_mem;

+ PGTask_t *task;

+ kern_return_t r;

+

+ r = mach_vm_allocate(mach_task_self(), &task_mem, len, VM_FLAGS_ANYWHERE);

+ if (r != KERN_SUCCESS) {

+ return NULL;

+ }

+

+ task = g_new0(PGTask_t, 1);

+ task->s = s;

+ task->address = task_mem;

+ task->len = len;

+ task->mapped_regions = g_ptr_array_sized_new(2 /* Usually enough */);

+

+ QEMU_LOCK_GUARD(&s->task_mutex);

+ QTAILQ_INSERT_TAIL(&s->tasks, task, node);

+

+ return task;

+}

+

+static void apple_gfx_destroy_task(AppleGFXState *s, PGTask_t *task)

+{

+ GPtrArray *regions = task->mapped_regions;

+ MemoryRegion *region;

+ size_t i;

+

+ for (i = 0; i < regions->len; ++i) {

+ region = g_ptr_array_index(regions, i);

+ memory_region_unref(region);

+ }

+ g_ptr_array_unref(regions);

+

+ mach_vm_deallocate(mach_task_self(), task->address, task->len);

+

+ QEMU_LOCK_GUARD(&s->task_mutex);

+ QTAILQ_REMOVE(&s->tasks, task, node);

+ g_free(task);

+}

+

+void *apple_gfx_host_ptr_for_gpa_range(uint64_t guest_physical,

+ uint64_t length, bool read_only,

+ MemoryRegion **mapping_in_region)

+{

+ MemoryRegion *ram_region;

+ char *host_ptr;

+ hwaddr ram_region_offset = 0;

+ hwaddr ram_region_length = length;

+

+ ram_region = address_space_translate(&address_space_memory,

+ guest_physical,

+ &ram_region_offset,

+ &ram_region_length, !read_only,

+ MEMTXATTRS_UNSPECIFIED);

+

+ if (!ram_region || ram_region_length < length ||

+ !memory_access_is_direct(ram_region, !read_only)) {

+ return NULL;

+ }

+

+ host_ptr = memory_region_get_ram_ptr(ram_region);

+ if (!host_ptr) {

+ return NULL;

+ }

+ host_ptr += ram_region_offset;

+ *mapping_in_region = ram_region;

+ return host_ptr;

+}

+

+static bool apple_gfx_task_map_memory(AppleGFXState *s, PGTask_t *task,

+ uint64_t virtual_offset,

+ PGPhysicalMemoryRange_t *ranges,

+ uint32_t range_count, bool read_only)

+{

+ kern_return_t r;

+ void *source_ptr;

+ mach_vm_address_t target;

+ vm_prot_t cur_protection, max_protection;

+ bool success = true;

+ MemoryRegion *region;

+

+ RCU_READ_LOCK_GUARD();

+ QEMU_LOCK_GUARD(&s->task_mutex);

+

+ trace_apple_gfx_map_memory(task, range_count, virtual_offset, read_only);

+ for (int i = 0; i < range_count; i++) {

+ PGPhysicalMemoryRange_t *range = &ranges[i];

+

+ target = task->address + virtual_offset;

+ virtual_offset += range->physicalLength;

+

+ trace_apple_gfx_map_memory_range(i, range->physicalAddress,

+ range->physicalLength);

+

+ region = NULL;

+ source_ptr = apple_gfx_host_ptr_for_gpa_range(range->physicalAddress,

+ range->physicalLength,

+ read_only, &region);

+ if (!source_ptr) {

+ success = false;

+ continue;

+ }

+

+ if (!g_ptr_array_find(task->mapped_regions, region, NULL)) {

+ g_ptr_array_add(task->mapped_regions, region);

+ memory_region_ref(region);

+ }

+

+ cur_protection = 0;

+ max_protection = 0;

+ /* Map guest RAM at range->physicalAddress into PG task memory range */

+ r = mach_vm_remap(mach_task_self(),

+ &target, range->physicalLength, vm_page_size - 1,

+ VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE,

+ mach_task_self(), (mach_vm_address_t)source_ptr,

+ false /* shared mapping, no copy */,

+ &cur_protection, &max_protection,

+ VM_INHERIT_COPY);

+ trace_apple_gfx_remap(r, source_ptr, target);

+ g_assert(r == KERN_SUCCESS);

+ }

+

+ return success;

+}

+

+static void apple_gfx_task_unmap_memory(AppleGFXState *s, PGTask_t *task,

+ uint64_t virtual_offset, uint64_t length)

+{

+ kern_return_t r;

+ mach_vm_address_t range_address;

+

+ trace_apple_gfx_unmap_memory(task, virtual_offset, length);

+

+ /*

+ * Replace task memory range with fresh 0 pages, undoing the mapping

+ * from guest RAM.

+ */

+ range_address = task->address + virtual_offset;

+ r = mach_vm_allocate(mach_task_self(), &range_address, length,

+ VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE);

+ g_assert(r == KERN_SUCCESS);

+}

+

+/* ------ Rendering and frame management ------ */

+

+static void apple_gfx_render_frame_completed(AppleGFXState *s,

+ uint32_t width, uint32_t height);

+

+static void apple_gfx_render_new_frame_bql_unlock(AppleGFXState *s)

+{

+ BOOL r;

+ bool managed_texture = s->using_managed_texture_storage;

+ uint32_t width = surface_width(s->surface);

+ uint32_t height = surface_height(s->surface);

+ MTLRegion region = MTLRegionMake2D(0, 0, width, height);

+ id<MTLCommandBuffer> command_buffer = [s->mtl_queue commandBuffer];

+ id<MTLTexture> texture = s->texture;

+

+ assert(bql_locked());

+ [texture retain];

+

+ bql_unlock();

+

+ /*

+ * This is not safe to call from the BQL due to PVG-internal locks causing

+ * deadlocks.

+ */

+ r = [s->pgdisp encodeCurrentFrameToCommandBuffer:command_buffer

+ texture:texture

+ region:region];

+ if (!r) {

+ [texture release];

+ bql_lock();

+ --s->pending_frames;

+ bql_unlock();

+ qemu_log_mask(LOG_GUEST_ERROR,

+ "%s: encodeCurrentFrameToCommandBuffer:texture:region: "

+ "failed\n", __func__);

+ return;

+ }

+

+ if (managed_texture) {

+ /* "Managed" textures exist in both VRAM and RAM and must be synced. */

+ id<MTLBlitCommandEncoder> blit = [command_buffer blitCommandEncoder];

+ [blit synchronizeResource:texture];

+ [blit endEncoding];

+ }

+ [texture release];

+ [command_buffer addCompletedHandler:

+ ^(id<MTLCommandBuffer> cb)

+ {