macOS's Cocoa event handling must be done on the initial (main) thread

of the process. Furthermore, if library or application code uses

libdispatch, the main dispatch queue must be handling events on the main

thread as well.

So far, this has affected Qemu in both the Cocoa and SDL UIs, although

in different ways: the Cocoa UI replaces the default qemu_main function

with one that spins Qemu's internal main event loop off onto a

background thread. SDL (which uses Cocoa internally) on the other hand

uses a polling approach within Qemu's main event loop. Events are

polled during the SDL UI's dpy_refresh callback, which happens to run

on the main thread by default.

As UIs are mutually exclusive, this works OK as long as nothing else

needs platform-native event handling. In the next patch, a new device is

introduced based on the ParavirtualizedGraphics.framework in macOS.

This uses libdispatch internally, and only works when events are being

handled on the main runloop. With the current system, it works when

using either the Cocoa or the SDL UI. However, it does not when running

headless. Moreover, any attempt to install a similar scheme to the

Cocoa UI's main thread replacement fails when combined with the SDL

UI.

This change tidies up main thread management to be more flexible.

* The qemu_main global function pointer is a custom function for the

main thread, and it may now be NULL. When it is, the main thread

runs the main Qemu loop. This represents the traditional setup.

* When non-null, spawning the main Qemu event loop on a separate

thread is now done centrally rather than inside the Cocoa UI code.

* For most platforms, qemu_main is indeed NULL by default, but on

Darwin, it defaults to a function that runs the CFRunLoop.

* The Cocoa UI sets qemu_main to a function which runs the

NSApplication event handling runloop, as is usual for a Cocoa app.

* The SDL UI overrides the qemu_main function to NULL, thus

specifying that Qemu's main loop must run on the main

thread.

* For other UIs, or in the absence of UIs, the platform's default

behaviour is followed.

This means that on macOS, the platform's runloop events are always

handled, regardless of chosen UI. The new PV graphics device will

thus work in all configurations. There is no functional change on other

operating systems.

Signed-off-by: Phil Dennis-Jordan <phil@philjordan.eu>

---

v5:

* Simplified the way of setting/clearing the main loop by going back

to setting qemu_main directly, but narrowing the scope of what it

needs to do, and it can now be NULL.

include/qemu-main.h | 3 +--

include/qemu/typedefs.h | 1 +

system/main.c | 56 +++++++++++++++++++++++++++++++++++----

ui/cocoa.m | 58 +++++++++++------------------------------

ui/sdl2.c | 4 +++

5 files changed, 72 insertions(+), 50 deletions(-)

diff --git a/include/qemu-main.h b/include/qemu-main.h

index XXXXXXX..XXXXXXX 100644

--- a/include/qemu-main.h

+++ b/include/qemu-main.h

@@ -XXX,XX +XXX,XX @@

#ifndef QEMU_MAIN_H

#define QEMU_MAIN_H

-int qemu_default_main(void);

-extern int (*qemu_main)(void);

+extern qemu_main_fn qemu_main;

#endif /* QEMU_MAIN_H */

diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h

index XXXXXXX..XXXXXXX 100644

--- a/include/qemu/typedefs.h

+++ b/include/qemu/typedefs.h

@@ -XXX,XX +XXX,XX @@ typedef struct IRQState *qemu_irq;

* Function types

*/

typedef void (*qemu_irq_handler)(void *opaque, int n, int level);

+typedef int (*qemu_main_fn)(void);

#endif /* QEMU_TYPEDEFS_H */

diff --git a/system/main.c b/system/main.c

index XXXXXXX..XXXXXXX 100644

--- a/system/main.c

+++ b/system/main.c

@@ -XXX,XX +XXX,XX @@

#include "qemu/osdep.h"

#include "qemu-main.h"

+#include "qemu/main-loop.h"

#include "sysemu/sysemu.h"

-#ifdef CONFIG_SDL

-#include <SDL.h>

+#ifdef CONFIG_DARWIN

+#include <CoreFoundation/CoreFoundation.h>

#endif

-int qemu_default_main(void)

+static int qemu_default_main(void)

{

int status;

@@ -XXX,XX +XXX,XX @@ int qemu_default_main(void)

return status;

}

-int (*qemu_main)(void) = qemu_default_main;

+/*

+ * Various macOS system libraries, including the Cocoa UI and anything using

+ * libdispatch, such as ParavirtualizedGraphics.framework, requires that the

+ * main runloop, on the main (initial) thread be running or at least regularly

+ * polled for events. A special mode is therefore supported, where the QEMU

+ * main loop runs on a separate thread and the main thread handles the

+ * CF/Cocoa runloop.

+ */

+

+static void *call_qemu_default_main(void *opaque)

+{

+ int status;

+

+ bql_lock();

+ status = qemu_default_main();

+ bql_unlock();

+

+ exit(status);

+}

+

+static void qemu_run_default_main_on_new_thread(void)

+{

+ QemuThread thread;

+

+ qemu_thread_create(&thread, "qemu_main", call_qemu_default_main,

+ NULL, QEMU_THREAD_DETACHED);

+}

+

+

+#ifdef CONFIG_DARWIN

+static int os_darwin_cfrunloop_main(void)

+{

+ CFRunLoopRun();

+ abort();

+}

+

+qemu_main_fn qemu_main = os_darwin_cfrunloop_main;

+#else

+qemu_main_fn qemu_main;

+#endif

int main(int argc, char **argv)

{

qemu_init(argc, argv);

- return qemu_main();

+ if (qemu_main) {

+ qemu_run_default_main_on_new_thread();

+ bql_unlock();

+ return qemu_main();

+ } else {

+ qemu_default_main();

+ }

}

diff --git a/ui/cocoa.m b/ui/cocoa.m

index XXXXXXX..XXXXXXX 100644

--- a/ui/cocoa.m

+++ b/ui/cocoa.m

@@ -XXX,XX +XXX,XX @@

int height;

} QEMUScreen;

+@class QemuCocoaPasteboardTypeOwner;

+

static void cocoa_update(DisplayChangeListener *dcl,

int x, int y, int w, int h);

@@ -XXX,XX +XXX,XX @@ static void cocoa_switch(DisplayChangeListener *dcl,

static NSInteger cbchangecount = -1;

static QemuClipboardInfo *cbinfo;

static QemuEvent cbevent;

+static QemuCocoaPasteboardTypeOwner *cbowner;

// Utility functions to run specified code block with the BQL held

typedef void (^CodeBlock)(void);

@@ -XXX,XX +XXX,XX @@ - (void) dealloc

{

COCOA_DEBUG("QemuCocoaAppController: dealloc\n");

- if (cocoaView)

- [cocoaView release];

+ [cocoaView release];

+ [cbowner release];

+ cbowner = nil;

+

[super dealloc];

}

@@ -XXX,XX +XXX,XX @@ - (void)pasteboard:(NSPasteboard *)sender provideDataForType:(NSPasteboardType)t

@end

-static QemuCocoaPasteboardTypeOwner *cbowner;

-

static void cocoa_clipboard_notify(Notifier *notifier, void *data);

static void cocoa_clipboard_request(QemuClipboardInfo *info,

QemuClipboardType type);

@@ -XXX,XX +XXX,XX @@ static void cocoa_clipboard_request(QemuClipboardInfo *info,

}

}

-/*

- * The startup process for the OSX/Cocoa UI is complicated, because

- * OSX insists that the UI runs on the initial main thread, and so we

- * need to start a second thread which runs the qemu_default_main():

- * in main():

- * in cocoa_display_init():

- * assign cocoa_main to qemu_main

- * create application, menus, etc

- * in cocoa_main():

- * create qemu-main thread

- * enter OSX run loop

- */

-

-static void *call_qemu_main(void *opaque)

-{

- int status;

-

- COCOA_DEBUG("Second thread: calling qemu_default_main()\n");

- bql_lock();

- status = qemu_default_main();

- bql_unlock();

- COCOA_DEBUG("Second thread: qemu_default_main() returned, exiting\n");

- [cbowner release];

- exit(status);

-}

-

static int cocoa_main(void)

{

- QemuThread thread;

-

- COCOA_DEBUG("Entered %s()\n", __func__);

-

- bql_unlock();

- qemu_thread_create(&thread, "qemu_main", call_qemu_main,

- NULL, QEMU_THREAD_DETACHED);

-

- // Start the main event loop

COCOA_DEBUG("Main thread: entering OSX run loop\n");

[NSApp run];

COCOA_DEBUG("Main thread: left OSX run loop, which should never happen\n");

@@ -XXX,XX +XXX,XX @@ static void cocoa_display_init(DisplayState *ds, DisplayOptions *opts)

COCOA_DEBUG("qemu_cocoa: cocoa_display_init\n");

- qemu_main = cocoa_main;

-

// Pull this console process up to being a fully-fledged graphical

// app with a menubar and Dock icon

ProcessSerialNumber psn = { 0, kCurrentProcess };

@@ -XXX,XX +XXX,XX @@ static void cocoa_display_init(DisplayState *ds, DisplayOptions *opts)

qemu_clipboard_peer_register(&cbpeer);

[pool release];

+

+ /*

+ * The Cocoa UI will run the NSApplication runloop on the main thread

+ * rather than the default Core Foundation one.

+ */

+ qemu_main = cocoa_main;

}

static QemuDisplay qemu_display_cocoa = {

- .type = DISPLAY_TYPE_COCOA,

- .init = cocoa_display_init,

+ .type = DISPLAY_TYPE_COCOA,

+ .init = cocoa_display_init,

};

static void register_cocoa(void)

diff --git a/ui/sdl2.c b/ui/sdl2.c

index XXXXXXX..XXXXXXX 100644

--- a/ui/sdl2.c

+++ b/ui/sdl2.c

@@ -XXX,XX +XXX,XX @@

#include "sysemu/sysemu.h"

#include "ui/win32-kbd-hook.h"

#include "qemu/log.h"

+#include "qemu-main.h"

static int sdl2_num_outputs;

static struct sdl2_console *sdl2_console;

@@ -XXX,XX +XXX,XX @@ static void sdl2_display_init(DisplayState *ds, DisplayOptions *o)

}

atexit(sdl_cleanup);

+

+ /* SDL's event polling (in dpy_refresh) must happen on the main thread. */

+ qemu_main = NULL;

}

static QemuDisplay qemu_display_sdl2 = {

--

2.39.3 (Apple Git-145)

MacOS provides a framework (library) that allows any vmm to implement a

paravirtualized 3d graphics passthrough to the host metal stack called

ParavirtualizedGraphics.Framework (PVG). The library abstracts away

almost every aspect of the paravirtualized device model and only provides

and receives callbacks on MMIO access as well as to share memory address

space between the VM and PVG.

This patch implements a QEMU device that drives PVG for the VMApple

variant of it.

Signed-off-by: Alexander Graf <graf@amazon.com>

Co-authored-by: Alexander Graf <graf@amazon.com>

Subsequent changes:

* Cherry-pick/rebase conflict fixes, API use updates.

* Moved from hw/vmapple/ (useful outside that machine type)

* Overhaul of threading model, many thread safety improvements.

* Asynchronous rendering.

* Memory and object lifetime fixes.

* Refactoring to split generic and (vmapple) MMIO variant specific

code.

Signed-off-by: Phil Dennis-Jordan <phil@philjordan.eu>

---

v2:

* Cherry-pick/rebase conflict fixes

* BQL function renaming

* Moved from hw/vmapple/ (useful outside that machine type)

* Code review comments: Switched to DEFINE_TYPES macro & little endian

MMIO.

* Removed some dead/superfluous code

* Mad set_mode thread & memory safe

* Added migration blocker due to lack of (de-)serialisation.

* Fixes to ObjC refcounting and autorelease pool usage.

* Fixed ObjC new/init misuse

* Switched to ObjC category extension for private property.

* Simplified task memory mapping and made it thread safe.

* Refactoring to split generic and vmapple MMIO variant specific

code.

* Switched to asynchronous MMIO writes on x86-64

* Rendering and graphics update are now done asynchronously

* Fixed cursor handling

* Coding convention fixes

* Removed software cursor compositing

v3:

* Rebased on latest upstream, fixed breakages including switching to Resettable methods.

* Squashed patches dealing with dGPUs, MMIO area size, and GPU picking.

* Allow re-entrant MMIO; this simplifies the code and solves the divergence

between x86-64 and arm64 variants.

v4:

* Renamed '-vmapple' device variant to '-mmio'

* MMIO device type now requires aarch64 host and guest

* Complete overhaul of the glue code for making Qemu's and

ParavirtualizedGraphics.framework's threading and synchronisation models

work together. Calls into PVG are from dispatch queues while the

BQL-holding initiating thread processes AIO context events; callbacks from

PVG are scheduled as BHs on the BQL/main AIO context, awaiting completion

where necessary.

* Guest frame rendering state is covered by the BQL, with only the PVG calls

outside the lock, and serialised on the named render_queue.

* Simplified logic for dropping frames in-flight during mode changes, fixed

bug in pending frames logic.

* Addressed smaller code review notes such as: function naming, object type

declarations, type names/declarations/casts, code formatting, #include

order, over-cautious ObjC retain/release, what goes in init vs realize,

etc.

v5:

* Smaller non-functional fixes in response to review comments, such as using

NULL for the AIO_WAIT_WHILE context argument, type name formatting,

deleting leftover debug code, logging improvements, state struct field

order and documentation improvements, etc.

* Instead of a single condition variable for all synchronous BH job types,

there is now one for each callback block. This reduces the number

of threads being awoken unnecessarily to near zero.

* MMIO device variant: Unified the BH job for raising interrupts.

* Use DMA APIs for PVG framework's guest memory read requests.

* Thread safety improvements: ensure mutable AppleGFXState fields are not

accessed outside the appropriate lock. Added dedicated mutex for the task

list.

* Retain references to MemoryRegions for which there exist mappings in each

PGTask, and for IOSurface mappings.

hw/display/Kconfig | 9 +

hw/display/apple-gfx-mmio.m | 387 ++++++++++++++++++

hw/display/apple-gfx.h | 79 ++++

hw/display/apple-gfx.m | 773 ++++++++++++++++++++++++++++++++++++

hw/display/meson.build | 4 +

hw/display/trace-events | 28 ++

meson.build | 4 +

7 files changed, 1284 insertions(+)

create mode 100644 hw/display/apple-gfx-mmio.m

create mode 100644 hw/display/apple-gfx.h

create mode 100644 hw/display/apple-gfx.m

diff --git a/hw/display/Kconfig b/hw/display/Kconfig

index XXXXXXX..XXXXXXX 100644

--- a/hw/display/Kconfig

+++ b/hw/display/Kconfig

@@ -XXX,XX +XXX,XX @@ config XLNX_DISPLAYPORT

config DM163

bool

+

+config MAC_PVG

+ bool

+ default y

+

+config MAC_PVG_MMIO

+ bool

+ depends on MAC_PVG && AARCH64

+

diff --git a/hw/display/apple-gfx-mmio.m b/hw/display/apple-gfx-mmio.m

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/hw/display/apple-gfx-mmio.m

@@ -XXX,XX +XXX,XX @@

+/*

+ * QEMU Apple ParavirtualizedGraphics.framework device, MMIO (arm64) variant

+ *

+ * Copyright © 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.

+ *

+ * This work is licensed under the terms of the GNU GPL, version 2 or later.

+ * See the COPYING file in the top-level directory.

+ *

+ * SPDX-License-Identifier: GPL-2.0-or-later

+ *

+ * ParavirtualizedGraphics.framework is a set of libraries that macOS provides

+ * which implements 3d graphics passthrough to the host as well as a

+ * proprietary guest communication channel to drive it. This device model

+ * implements support to drive that library from within QEMU as an MMIO-based

+ * system device for macOS on arm64 VMs.

+ */

+

+#include "qemu/osdep.h"

+#import <ParavirtualizedGraphics/ParavirtualizedGraphics.h>

+#include "apple-gfx.h"

+#include "monitor/monitor.h"

+#include "hw/sysbus.h"

+#include "hw/irq.h"

+#include "trace.h"

+#include "qemu/log.h"

+

+OBJECT_DECLARE_SIMPLE_TYPE(AppleGFXMMIOState, APPLE_GFX_MMIO)

+

+/*

+ * ParavirtualizedGraphics.Framework only ships header files for the PCI

+ * variant which does not include IOSFC descriptors and host devices. We add

+ * their definitions here so that we can also work with the ARM version.

+ */

+typedef bool(^IOSFCRaiseInterrupt)(uint32_t vector);

+typedef bool(^IOSFCUnmapMemory)(

+ void *, void *, void *, void *, void *, void *);

+typedef bool(^IOSFCMapMemory)(

+ uint64_t phys, uint64_t len, bool ro, void **va, void *, void *);

+

+@interface PGDeviceDescriptor (IOSurfaceMapper)

+@property (readwrite, nonatomic) bool usingIOSurfaceMapper;

+@end

+

+@interface PGIOSurfaceHostDeviceDescriptor : NSObject

+-(PGIOSurfaceHostDeviceDescriptor *)init;

+@property (readwrite, nonatomic, copy, nullable) IOSFCMapMemory mapMemory;

+@property (readwrite, nonatomic, copy, nullable) IOSFCUnmapMemory unmapMemory;

+@property (readwrite, nonatomic, copy, nullable) IOSFCRaiseInterrupt raiseInterrupt;

+@end

+

+@interface PGIOSurfaceHostDevice : NSObject

+-(instancetype)initWithDescriptor:(PGIOSurfaceHostDeviceDescriptor *)desc;

+-(uint32_t)mmioReadAtOffset:(size_t)offset;

+-(void)mmioWriteAtOffset:(size_t)offset value:(uint32_t)value;

+@end

+

+struct AppleGFXMapSurfaceMemoryJob;

+struct AppleGFXMMIOState {

+ SysBusDevice parent_obj;

+

+ AppleGFXState common;

+

+ QemuCond iosfc_map_job_cond;

+ QemuCond iosfc_unmap_job_cond;

+ qemu_irq irq_gfx;

+ qemu_irq irq_iosfc;

+ MemoryRegion iomem_iosfc;

+ PGIOSurfaceHostDevice *pgiosfc;

+

+ GArray *iosfc_mapped_regions; /* array of AppleGFXMMIOMappedRegion */

+};

+

+typedef struct AppleGFXMMIOJob {

+ AppleGFXMMIOState *state;

+ uint64_t offset;

+ uint64_t value;

+ bool completed;

+} AppleGFXMMIOJob;

+

+static void iosfc_do_read(void *opaque)

+{

+ AppleGFXMMIOJob *job = opaque;

+ job->value = [job->state->pgiosfc mmioReadAtOffset:job->offset];

+ qatomic_set(&job->completed, true);

+ aio_wait_kick();

+}

+

+static uint64_t iosfc_read(void *opaque, hwaddr offset, unsigned size)

+{

+ AppleGFXMMIOJob job = {

+ .state = opaque,

+ .offset = offset,

+ .completed = false,

+ };

+ dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);

+

+ dispatch_async_f(queue, &job, iosfc_do_read);

+ AIO_WAIT_WHILE(NULL, !qatomic_read(&job.completed));

+

+ trace_apple_gfx_mmio_iosfc_read(offset, job.value);

+ return job.value;

+}

+

+static void iosfc_do_write(void *opaque)

+{

+ AppleGFXMMIOJob *job = opaque;

+ [job->state->pgiosfc mmioWriteAtOffset:job->offset value:job->value];

+ qatomic_set(&job->completed, true);

+ aio_wait_kick();

+}

+

+static void iosfc_write(void *opaque, hwaddr offset, uint64_t val,

+ unsigned size)

+{

+ AppleGFXMMIOJob job = {

+ .state = opaque,

+ .offset = offset,

+ .value = val,

+ .completed = false,

+ };

+ dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);

+

+ dispatch_async_f(queue, &job, iosfc_do_write);

+ AIO_WAIT_WHILE(NULL, !qatomic_read(&job.completed));

+

+ trace_apple_gfx_mmio_iosfc_write(offset, val);

+}

+

+static const MemoryRegionOps apple_iosfc_ops = {

+ .read = iosfc_read,

+ .write = iosfc_write,

+ .endianness = DEVICE_LITTLE_ENDIAN,

+ .valid = {

+ .min_access_size = 4,

+ .max_access_size = 8,

+ },

+ .impl = {

+ .min_access_size = 4,

+ .max_access_size = 8,

+ },

+};

+

+static void raise_irq(void *opaque)

+{

+ qemu_irq *irq = opaque;

+

+ qemu_irq_pulse(*irq);

+}

+

+typedef struct AppleGFXMapSurfaceMemoryJob {

+ uint64_t guest_physical_address;

+ uint64_t guest_physical_length;

+ void *result_mem;

+ AppleGFXMMIOState *state;

+ bool read_only;

+ bool success;

+ bool done;

+} AppleGFXMapSurfaceMemoryJob;

+

+typedef struct AppleGFXMMIOMappedRegion {

+ MemoryRegion *region;

+ uint64_t map_count;

+ uintptr_t host_virtual_start;

+ uintptr_t host_virtual_end;

+} AppleGFXMMIOMappedRegion;

+

+static void apple_gfx_mmio_map_surface_memory(void *opaque)

+{

+ AppleGFXMapSurfaceMemoryJob *job = opaque;

+ AppleGFXMMIOState *s = job->state;

+ mach_vm_address_t mem;

+ MemoryRegion *region = NULL;

+ GArray *regions = s->iosfc_mapped_regions;

+ AppleGFXMMIOMappedRegion *mapped_region;

+ size_t i;

+

+ mem = apple_gfx_host_address_for_gpa_range(job->guest_physical_address,

+ job->guest_physical_length,

+ job->read_only, &region);

+

+ if (mem != 0) {

+ for (i = 0; i < regions->len; ++i) {

+ mapped_region = &g_array_index(regions, AppleGFXMMIOMappedRegion, i);

+ if (region == mapped_region->region) {

+ ++mapped_region->map_count;

+ break;

+ }

+ }

+

+ if (i >= regions->len) {

+ /* No existing mapping to this region found, keep a reference and save

+ */

+ uintptr_t start = (uintptr_t)memory_region_get_ram_ptr(region);

+ AppleGFXMMIOMappedRegion new_region = {

+ region, 1,

+ start,

+ start + memory_region_size(region)

+ };

+ memory_region_ref(region);

+ g_array_append_val(regions, new_region);

+ trace_apple_gfx_iosfc_map_memory_new_region(

+ i, region, start, new_region.host_virtual_end);

+ }

+ }

+

+ qemu_mutex_lock(&s->common.job_mutex);

+ job->result_mem = (void *)mem;

+ job->success = mem != 0;

+ job->done = true;

+ qemu_cond_broadcast(&s->iosfc_map_job_cond);

+ qemu_mutex_unlock(&s->common.job_mutex);

+}

+

+typedef struct AppleGFXUnmapSurfaceMemoryJob {

+ void *virtual_address;

+ AppleGFXMMIOState *state;

+ bool done;

+} AppleGFXUnmapSurfaceMemoryJob;

+

+static AppleGFXMMIOMappedRegion *find_mapped_region_containing(GArray *regions,

+ uintptr_t va,

+ size_t *index)

+{

+ size_t i;

+ AppleGFXMMIOMappedRegion *mapped_region;

+

+ for (i = 0; i < regions->len; ++i) {

+ mapped_region = &g_array_index(regions, AppleGFXMMIOMappedRegion, i);

+ if (va >= mapped_region->host_virtual_start &&

+ va < mapped_region->host_virtual_end) {

+ *index = i;

+ return mapped_region;

+ }

+ }

+ return NULL;

+}

+

+static void apple_gfx_mmio_unmap_surface_memory(void *opaque)

+{

+ AppleGFXUnmapSurfaceMemoryJob *job = opaque;

+ AppleGFXMMIOState *s = job->state;

+ uintptr_t mem = (uintptr_t)job->virtual_address;

+ GArray *regions = s->iosfc_mapped_regions;

+ size_t region_index;

+ AppleGFXMMIOMappedRegion *mapped_region =

+ find_mapped_region_containing(regions, mem, &region_index);

+

+ if (mapped_region) {

+ trace_apple_gfx_iosfc_unmap_memory_region(mem, region_index, mapped_region->map_count, mapped_region->region);

+ if (--mapped_region->map_count == 0) {

+ memory_region_unref(mapped_region->region);

+ g_array_remove_index_fast(regions, region_index);

+ }

+ } else {

+ qemu_log_mask(LOG_GUEST_ERROR, "%s: memory at %p to be unmapped not "

+ "found in any of %u mapped regions.\n",

+ __func__,

+ job->virtual_address, regions->len);

+ }

+

+ qemu_mutex_lock(&s->common.job_mutex);

+ job->done = true;

+ qemu_cond_broadcast(&s->iosfc_unmap_job_cond);

+ qemu_mutex_unlock(&s->common.job_mutex);

+}

+

+static PGIOSurfaceHostDevice *apple_gfx_prepare_iosurface_host_device(

+ AppleGFXMMIOState *s)

+{

+ PGIOSurfaceHostDeviceDescriptor *iosfc_desc =

+ [PGIOSurfaceHostDeviceDescriptor new];

+ PGIOSurfaceHostDevice *iosfc_host_dev = nil;

+

+ iosfc_desc.mapMemory =

+ ^bool(uint64_t phys, uint64_t len, bool ro, void **va, void *e, void *f) {

+ AppleGFXMapSurfaceMemoryJob job = {

+ .guest_physical_address = phys, .guest_physical_length = len,

+ .read_only = ro, .state = s,

+ };

+

+ aio_bh_schedule_oneshot(qemu_get_aio_context(),

+ apple_gfx_mmio_map_surface_memory, &job);

+ apple_gfx_await_bh_job(&s->common, &s->iosfc_map_job_cond, &job.done);

+

+ *va = job.result_mem;

+

+ trace_apple_gfx_iosfc_map_memory(phys, len, ro, va, e, f, *va,

+ job.success);

+

+ return job.success;

+ };

+

+ iosfc_desc.unmapMemory =

+ ^bool(void *va, void *b, void *c, void *d, void *e, void *f) {

+ AppleGFXUnmapSurfaceMemoryJob job = { va, s };

+ trace_apple_gfx_iosfc_unmap_memory(va, b, c, d, e, f);

+

+ aio_bh_schedule_oneshot(qemu_get_aio_context(),

+ apple_gfx_mmio_unmap_surface_memory, &job);

+ apple_gfx_await_bh_job(&s->common, &s->iosfc_unmap_job_cond, &job.done);

+

+ return true;

+ };

+

+ iosfc_desc.raiseInterrupt = ^bool(uint32_t vector) {

+ trace_apple_gfx_iosfc_raise_irq(vector);

+ aio_bh_schedule_oneshot(qemu_get_aio_context(),

+ raise_irq, &s->irq_iosfc);

+ return true;

+ };

+

+ iosfc_host_dev =

+ [[PGIOSurfaceHostDevice alloc] initWithDescriptor:iosfc_desc];

+ [iosfc_desc release];

+ return iosfc_host_dev;

+}

+

+static void apple_gfx_mmio_realize(DeviceState *dev, Error **errp)

+{

+ @autoreleasepool {

+ AppleGFXMMIOState *s = APPLE_GFX_MMIO(dev);

+ PGDeviceDescriptor *desc = [PGDeviceDescriptor new];

+

+ desc.raiseInterrupt = ^(uint32_t vector) {

+ trace_apple_gfx_raise_irq(vector);

+ aio_bh_schedule_oneshot(qemu_get_aio_context(),

+ raise_irq, &s->irq_gfx);

+ };

+

+ desc.usingIOSurfaceMapper = true;

+ s->pgiosfc = apple_gfx_prepare_iosurface_host_device(s);

+

+ s->iosfc_mapped_regions =

+ g_array_sized_new(false /* no termination */, true /* clear */,

+ sizeof(AppleGFXMMIOMappedRegion),

+ 2 /* Usually no more RAM regions*/);

+

+ apple_gfx_common_realize(&s->common, desc, errp);

+ qemu_cond_init(&s->iosfc_map_job_cond);

+ qemu_cond_init(&s->iosfc_unmap_job_cond);

+

+ [desc release];

+ desc = nil;

+ }

+}

+

+static void apple_gfx_mmio_init(Object *obj)

+{

+ AppleGFXMMIOState *s = APPLE_GFX_MMIO(obj);

+

+ apple_gfx_common_init(obj, &s->common, TYPE_APPLE_GFX_MMIO);

+

+ sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->common.iomem_gfx);

+ memory_region_init_io(&s->iomem_iosfc, obj, &apple_iosfc_ops, s,

+ TYPE_APPLE_GFX_MMIO, 0x10000);

+ sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem_iosfc);

+ sysbus_init_irq(SYS_BUS_DEVICE(s), &s->irq_gfx);

+ sysbus_init_irq(SYS_BUS_DEVICE(s), &s->irq_iosfc);

+}

+

+static void apple_gfx_mmio_reset(Object *obj, ResetType type)

+{

+ AppleGFXMMIOState *s = APPLE_GFX_MMIO(obj);

+ [s->common.pgdev reset];

+}

+

+

+static void apple_gfx_mmio_class_init(ObjectClass *klass, void *data)

+{

+ DeviceClass *dc = DEVICE_CLASS(klass);

+ ResettableClass *rc = RESETTABLE_CLASS(klass);

+

+ rc->phases.hold = apple_gfx_mmio_reset;

+ dc->hotpluggable = false;

+ dc->realize = apple_gfx_mmio_realize;

+}

+

+static TypeInfo apple_gfx_mmio_types[] = {

+ {

+ .name = TYPE_APPLE_GFX_MMIO,

+ .parent = TYPE_SYS_BUS_DEVICE,

+ .instance_size = sizeof(AppleGFXMMIOState),

+ .class_init = apple_gfx_mmio_class_init,

+ .instance_init = apple_gfx_mmio_init,

+ }

+};

+DEFINE_TYPES(apple_gfx_mmio_types)

diff --git a/hw/display/apple-gfx.h b/hw/display/apple-gfx.h

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/hw/display/apple-gfx.h

@@ -XXX,XX +XXX,XX @@

+/*

+ * Data structures and functions shared between variants of the macOS

+ * ParavirtualizedGraphics.framework based apple-gfx display adapter.

+ *

+ * SPDX-License-Identifier: GPL-2.0-or-later

+ */

+

+#ifndef QEMU_APPLE_GFX_H

+#define QEMU_APPLE_GFX_H

+

+#define TYPE_APPLE_GFX_MMIO "apple-gfx-mmio"

+#define TYPE_APPLE_GFX_PCI "apple-gfx-pci"

+

+#include "qemu/osdep.h"

+#include <dispatch/dispatch.h>

+#import <ParavirtualizedGraphics/ParavirtualizedGraphics.h>

+#include "qemu/typedefs.h"

+#include "exec/memory.h"

+#include "ui/surface.h"

+

+@class PGDeviceDescriptor;

+@protocol PGDevice;

+@protocol PGDisplay;

+@protocol MTLDevice;

+@protocol MTLTexture;

+@protocol MTLCommandQueue;

+

+typedef QTAILQ_HEAD(, PGTask_s) PGTaskList;

+

+typedef struct AppleGFXState {

+ /* Initialised on init/realize() */

+ MemoryRegion iomem_gfx;

+ id<PGDevice> pgdev;

+ id<PGDisplay> pgdisp;

+ QemuConsole *con;

+ id<MTLDevice> mtl;

+ id<MTLCommandQueue> mtl_queue;

+ dispatch_queue_t render_queue;

+ /*

+ * QemuMutex & QemuConds for awaiting completion of PVG memory-mapping and

+ * reading requests after submitting them to run in the AIO context.

+ * QemuCond (rather than QemuEvent) are used so multiple concurrent jobs

+ * can be handled safely.

+ * The state associated with each job is tracked in a AppleGFX*Job struct

+ * for each kind of job; instances are allocated on the caller's stack.

+ * This struct also contains the completion flag which is used in

+ * conjunction with the condition variable.

+ */

+ QemuMutex job_mutex;

+ QemuCond task_map_job_cond;

+ QemuCond mem_read_job_cond;

+

+ /* tasks is protected by task_mutex */

+ QemuMutex task_mutex;

+ PGTaskList tasks;

+

+ /* Mutable state (BQL) */

+ QEMUCursor *cursor;

+ bool cursor_show;

+ bool gfx_update_requested;

+ bool new_frame_ready;

+ bool using_managed_texture_storage;

+ int32_t pending_frames;

+ void *vram;

+ DisplaySurface *surface;

+ id<MTLTexture> texture;

+} AppleGFXState;

+

+void apple_gfx_common_init(Object *obj, AppleGFXState *s, const char* obj_name);

+void apple_gfx_common_realize(AppleGFXState *s, PGDeviceDescriptor *desc,

+ Error **errp);

+uintptr_t apple_gfx_host_address_for_gpa_range(uint64_t guest_physical,

+ uint64_t length, bool read_only,

+ MemoryRegion **mapping_in_region);

+void apple_gfx_await_bh_job(AppleGFXState *s, QemuCond *job_cond,

+ bool *job_done_flag);

+

+#endif

+

diff --git a/hw/display/apple-gfx.m b/hw/display/apple-gfx.m

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/hw/display/apple-gfx.m

@@ -XXX,XX +XXX,XX @@

+/*

+ * QEMU Apple ParavirtualizedGraphics.framework device

+ *

+ * Copyright © 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.

+ *

+ * This work is licensed under the terms of the GNU GPL, version 2 or later.

+ * See the COPYING file in the top-level directory.

+ *

+ * SPDX-License-Identifier: GPL-2.0-or-later

+ *

+ * ParavirtualizedGraphics.framework is a set of libraries that macOS provides

+ * which implements 3d graphics passthrough to the host as well as a

+ * proprietary guest communication channel to drive it. This device model

+ * implements support to drive that library from within QEMU.

+ */

+

+#include "qemu/osdep.h"

+#import <ParavirtualizedGraphics/ParavirtualizedGraphics.h>

+#include <mach/mach_vm.h>

+#include "apple-gfx.h"

+#include "trace.h"

+#include "qemu-main.h"

+#include "exec/address-spaces.h"

+#include "migration/blocker.h"

+#include "monitor/monitor.h"

+#include "qemu/main-loop.h"

+#include "qemu/cutils.h"

+#include "qemu/log.h"

+#include "qapi/visitor.h"

+#include "qapi/error.h"

+#include "sysemu/dma.h"

+#include "ui/console.h"

+

+static const PGDisplayCoord_t apple_gfx_modes[] = {

+ { .x = 1440, .y = 1080 },

+ { .x = 1280, .y = 1024 },

+};

+

+/* This implements a type defined in <ParavirtualizedGraphics/PGDevice.h>

+ * which is opaque from the framework's point of view. Typedef PGTask_t already

+ * exists in the framework headers. */

+struct PGTask_s {

+ QTAILQ_ENTRY(PGTask_s) node;

+ AppleGFXState *s;

+ mach_vm_address_t address;

+ uint64_t len;

+ /*

+ * All unique MemoryRegions for which a mapping has been created in in this

+ * task, and on which we have thus called memory_region_ref(). There are

+ * usually very few regions of system RAM in total, so we expect this array

+ * to be very short. Therefore, no need for sorting or fancy search

+ * algorithms, linear search will do. */

+ GArray *mapped_regions;

+};

+

+static Error *apple_gfx_mig_blocker;

+

+static void apple_gfx_render_frame_completed(AppleGFXState *s,

+ uint32_t width, uint32_t height);

+

+static dispatch_queue_t get_background_queue(void)

+{

+ return dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);

+}

+

+static PGTask_t *apple_gfx_new_task(AppleGFXState *s, uint64_t len)

+{

+ mach_vm_address_t task_mem;

+ PGTask_t *task;

+ kern_return_t r;

+

+ r = mach_vm_allocate(mach_task_self(), &task_mem, len, VM_FLAGS_ANYWHERE);

+ if (r != KERN_SUCCESS) {

+ return NULL;

+ }

+

+ task = g_new0(PGTask_t, 1);

+ task->s = s;

+ task->address = task_mem;

+ task->len = len;

+ task->mapped_regions = g_array_sized_new(false /* no termination */,

+ true /* clear */,

+ sizeof(MemoryRegion *),

+ 2 /* Usually no more RAM regions*/);

+

+ QEMU_LOCK_GUARD(&s->task_mutex);

+ QTAILQ_INSERT_TAIL(&s->tasks, task, node);

+

+ return task;

+}

+

+typedef struct AppleGFXIOJob {

+ AppleGFXState *state;

+ uint64_t offset;

+ uint64_t value;

+ bool completed;

+} AppleGFXIOJob;

+

+static void apple_gfx_do_read(void *opaque)

+{

+ AppleGFXIOJob *job = opaque;

+ job->value = [job->state->pgdev mmioReadAtOffset:job->offset];

+ qatomic_set(&job->completed, true);

+ aio_wait_kick();

+}

+

+static uint64_t apple_gfx_read(void *opaque, hwaddr offset, unsigned size)

+{

+ AppleGFXIOJob job = {

+ .state = opaque,

+ .offset = offset,

+ .completed = false,

+ };

+ dispatch_queue_t queue = get_background_queue();

+

+ dispatch_async_f(queue, &job, apple_gfx_do_read);

+ AIO_WAIT_WHILE(NULL, !qatomic_read(&job.completed));

+

+ trace_apple_gfx_read(offset, job.value);

+ return job.value;

+}

+

+static void apple_gfx_do_write(void *opaque)

+{

+ AppleGFXIOJob *job = opaque;

+ [job->state->pgdev mmioWriteAtOffset:job->offset value:job->value];

+ qatomic_set(&job->completed, true);

+ aio_wait_kick();

+}

+

+static void apple_gfx_write(void *opaque, hwaddr offset, uint64_t val,

+ unsigned size)

+{

+ /* The methods mmioReadAtOffset: and especially mmioWriteAtOffset: can

+ * trigger and block on operations on other dispatch queues, which in turn

+ * may call back out on one or more of the callback blocks. For this reason,

+ * and as we are holding the BQL, we invoke the I/O methods on a pool

+ * thread and handle AIO tasks while we wait. Any work in the callbacks

+ * requiring the BQL will in turn schedule BHs which this thread will

+ * process while waiting. */

+ AppleGFXIOJob job = {

+ .state = opaque,

+ .offset = offset,

+ .value = val,

+ .completed = false,

+ };

+ dispatch_queue_t queue = get_background_queue();

+

+ dispatch_async_f(queue, &job, apple_gfx_do_write);

+ AIO_WAIT_WHILE(NULL, !qatomic_read(&job.completed));

+

+ trace_apple_gfx_write(offset, val);

+}

+

+static const MemoryRegionOps apple_gfx_ops = {

+ .read = apple_gfx_read,

+ .write = apple_gfx_write,

+ .endianness = DEVICE_LITTLE_ENDIAN,

+ .valid = {

+ .min_access_size = 4,