The res and ires buffers in struct virtio_gpio_line and struct
vgpio_irq_line respectively are used for DMA_FROM_DEVICE via
virtqueue_add_sgs(). However, within these structs, even though these
elements are tagged as ____cacheline_aligned, adjacent struct elements
can share DMA cachelines on platforms where ARCH_DMA_MINALIGN >
L1_CACHE_BYTES (e.g., arm64 with 128-byte DMA alignment but 64-byte
cache lines).
The existing ____cacheline_aligned annotation aligns to L1_CACHE_BYTES
which is not always sufficient for DMA alignment. For example, with
L1_CACHE_BYTES = 32 and ARCH_DMA_MINALIGN = 128
- irq_lines[0].ires at offset 128
- irq_lines[1].type at offset 192
both in same 128-byte DMA cacheline [128-256)
When the device writes to irq_lines[0].ires and the CPU concurrently
modifies one of irq_lines[1].type/disabled/masked/queued flags,
corruption can occur on non-cache-coherent platforms.
Fix by using __dma_from_device_group_begin()/end() annotations on the
DMA buffers. Drop ____cacheline_aligned - it's not required to isolate
request and response, and keeping them would increase the memory cost.
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
drivers/gpio/gpio-virtio.c | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/drivers/gpio/gpio-virtio.c b/drivers/gpio/gpio-virtio.c
index 17e040991e46..b70294626770 100644
--- a/drivers/gpio/gpio-virtio.c
+++ b/drivers/gpio/gpio-virtio.c
@@ -10,6 +10,7 @@
*/
#include <linux/completion.h>
+#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/gpio/driver.h>
#include <linux/io.h>
@@ -24,8 +25,11 @@
struct virtio_gpio_line {
struct mutex lock; /* Protects line operation */
struct completion completion;
- struct virtio_gpio_request req ____cacheline_aligned;
- struct virtio_gpio_response res ____cacheline_aligned;
+
+ __dma_from_device_group_begin();
+ struct virtio_gpio_request req;
+ struct virtio_gpio_response res;
+ __dma_from_device_group_end();
unsigned int rxlen;
};
@@ -37,8 +41,10 @@ struct vgpio_irq_line {
bool update_pending;
bool queue_pending;
- struct virtio_gpio_irq_request ireq ____cacheline_aligned;
- struct virtio_gpio_irq_response ires ____cacheline_aligned;
+ __dma_from_device_group_begin();
+ struct virtio_gpio_irq_request ireq;
+ struct virtio_gpio_irq_response ires;
+ __dma_from_device_group_end();
};
struct virtio_gpio {
--
MST