Ported pcie-brcmstb bounce buffer implementation to ARM64. (#3144)

Ported pcie-brcmstb bounce buffer implementation to ARM64.
This enables full 4G RAM usage on Raspberry Pi in 64-bit mode.

Signed-off-by: Yaroslav Rosomakho <[email protected]>

Author: yaroslavros

arch/arm64/include/asm/dma-mapping.h

@@ -24,6 +24,27 @@
 #include <xen/xen.h>
 #include <asm/xen/hypervisor.h>
 
+extern void *arm64_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
+			   gfp_t gfp, unsigned long attrs);
+extern void arm64_dma_free(struct device *dev, size_t size, void *cpu_addr,
+			 dma_addr_t handle, unsigned long attrs);
+extern int arm64_dma_mmap(struct device *dev, struct vm_area_struct *vma,
+			void *cpu_addr, dma_addr_t dma_addr, size_t size,
+			unsigned long attrs);
+extern int arm64_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
+		void *cpu_addr, dma_addr_t dma_addr, size_t size,
+		unsigned long attrs);
+extern int arm64_dma_map_sg(struct device *dev, struct scatterlist *sgl, int nelems,
+		enum dma_data_direction dir, unsigned long attrs);
+extern void arm64_dma_unmap_sg(struct device *dev, struct scatterlist *sgl, int,
+		enum dma_data_direction dir, unsigned long attrs);
+extern void arm64_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl, int nelems,
+		enum dma_data_direction dir);
+extern void arm64_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl, int nelems,
+		enum dma_data_direction dir);
+
+
+
 extern const struct dma_map_ops dummy_dma_ops;
 
 static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)

arch/arm64/mm/dma-mapping.c

@@ -138,6 +138,12 @@ static void *__dma_alloc(struct device *dev, size_t size,
 	return NULL;
 }
 
+void *arm64_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
+                           gfp_t gfp, unsigned long attrs)
+{
+        return __dma_alloc(dev, size, handle, gfp, attrs);
+}
+
 static void __dma_free(struct device *dev, size_t size,
 		       void *vaddr, dma_addr_t dma_handle,
 		       unsigned long attrs)
@@ -154,6 +160,12 @@ static void __dma_free(struct device *dev, size_t size,
 	swiotlb_free(dev, size, swiotlb_addr, dma_handle, attrs);
 }
 
+void arm64_dma_free(struct device *dev, size_t size, void *cpu_addr,
+                         dma_addr_t handle, unsigned long attrs)
+{
+        __dma_free(dev, size, cpu_addr, handle, attrs);
+}
+
 static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
 				     unsigned long offset, size_t size,
 				     enum dma_data_direction dir,
@@ -197,6 +209,12 @@ static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
 	return ret;
 }
 
+int arm64_dma_map_sg(struct device *dev, struct scatterlist *sgl, int nelems,
+                enum dma_data_direction dir, unsigned long attrs)
+{
+	return __swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
+}
+
 static void __swiotlb_unmap_sg_attrs(struct device *dev,
 				     struct scatterlist *sgl, int nelems,
 				     enum dma_data_direction dir,
@@ -213,6 +231,12 @@ static void __swiotlb_unmap_sg_attrs(struct device *dev,
 	swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
 }
 
+void arm64_dma_unmap_sg(struct device *dev, struct scatterlist *sgl, int nelems,
+                enum dma_data_direction dir, unsigned long attrs)
+{
+	__swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
+}
+
 static void __swiotlb_sync_single_for_cpu(struct device *dev,
 					  dma_addr_t dev_addr, size_t size,
 					  enum dma_data_direction dir)
@@ -245,6 +269,12 @@ static void __swiotlb_sync_sg_for_cpu(struct device *dev,
 	swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
 }
 
+void arm64_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl, int nelems,
+                enum dma_data_direction dir)
+{
+	__swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
+}
+
 static void __swiotlb_sync_sg_for_device(struct device *dev,
 					 struct scatterlist *sgl, int nelems,
 					 enum dma_data_direction dir)
@@ -259,6 +289,12 @@ static void __swiotlb_sync_sg_for_device(struct device *dev,
 				       sg->length, dir);
 }
 
+void arm64_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl, int nelems,
+                enum dma_data_direction dir)
+{
+	__swiotlb_sync_sg_for_device(dev, sgl, nelems, dir);
+}
+
 static int __swiotlb_mmap_pfn(struct vm_area_struct *vma,
 			      unsigned long pfn, size_t size)
 {
@@ -294,6 +330,13 @@ static int __swiotlb_mmap(struct device *dev,
 	return __swiotlb_mmap_pfn(vma, pfn, size);
 }
 
+int arm64_dma_mmap(struct device *dev, struct vm_area_struct *vma,
+                        void *cpu_addr, dma_addr_t dma_addr, size_t size,
+                        unsigned long attrs)
+{
+	return __swiotlb_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
+}
+
 static int __swiotlb_get_sgtable_page(struct sg_table *sgt,
 				      struct page *page, size_t size)
 {
@@ -314,6 +357,13 @@ static int __swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt,
 	return __swiotlb_get_sgtable_page(sgt, page, size);
 }
 
+int arm64_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
+                void *cpu_addr, dma_addr_t dma_addr, size_t size,
+                unsigned long attrs)
+{
+	return __swiotlb_get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs);
+}
+
 static int __swiotlb_dma_supported(struct device *hwdev, u64 mask)
 {
 	if (swiotlb)

drivers/pci/controller/Makefile

@@ -32,6 +32,9 @@ obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o
 ifdef CONFIG_ARM
 obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb-bounce.o
 endif
+ifdef CONFIG_ARM64
+obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb-bounce64.o
+endif
 
 obj-$(CONFIG_VMD) += vmd.o
 # pcie-hisi.o quirks are needed even without CONFIG_PCIE_DW

drivers/pci/controller/pcie-brcmstb-bounce.h

@@ -6,7 +6,7 @@
 #ifndef _PCIE_BRCMSTB_BOUNCE_H
 #define _PCIE_BRCMSTB_BOUNCE_H
 
-#ifdef CONFIG_ARM
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
 
 int brcm_pcie_bounce_init(struct device *dev, unsigned long buffer_size,
 			  dma_addr_t threshold);