pci-dma.c revision e088a4ad7fa53c3dc3c29f930025f41ccf01953e
162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu/* 262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu * Dynamic DMA mapping support. 362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu */ 462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/types.h> 662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/mm.h> 762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/string.h> 862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/pci.h> 962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/module.h> 1062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/dmar.h> 1162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <asm/iommu.h> 1262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <asm/machvec.h> 1362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/dma-mapping.h> 1462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 1562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <asm/system.h> 1662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 1762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#ifdef CONFIG_DMAR 1862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 1962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/kernel.h> 2062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 2162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <asm/page.h> 2262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 2362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yudma_addr_t bad_dma_address __read_mostly; 2462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua YuEXPORT_SYMBOL(bad_dma_address); 2562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 2662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yustatic int iommu_sac_force __read_mostly; 2762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 2862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuint no_iommu __read_mostly; 2962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#ifdef CONFIG_IOMMU_DEBUG 3062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuint force_iommu __read_mostly = 1; 3162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#else 3262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuint force_iommu __read_mostly; 3362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#endif 3462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 3562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu/* Dummy device used for NULL arguments (normally ISA). Better would 3662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu be probably a smaller DMA mask, but this is bug-to-bug compatible 3762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu to i386. */ 3862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yustruct device fallback_dev = { 3948ef2bb46ce8c2725a171d3afdb0d3563ef2c278Kay Sievers .init_name = "fallback device", 40284901a90a9e0b812ca3f5f852cbbfb60d10249dYang Hongyang .coherent_dma_mask = DMA_BIT_MASK(32), 4162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu .dma_mask = &fallback_dev.coherent_dma_mask, 4262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu}; 4362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 44160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonoriextern struct dma_map_ops intel_dma_ops; 4562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 4662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yustatic int __init pci_iommu_init(void) 4762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu{ 4862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu if (iommu_detected) 4962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu intel_iommu_init(); 5062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 5162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return 0; 5262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu} 5362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 5462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu/* Must execute after PCI subsystem */ 5562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yufs_initcall(pci_iommu_init); 5662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 5762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuvoid pci_iommu_shutdown(void) 5862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu{ 5962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return; 6062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu} 6162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 6262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuvoid __init 6362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuiommu_dma_init(void) 6462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu{ 6562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return; 6662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu} 6762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 6862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuint iommu_dma_supported(struct device *dev, u64 mask) 6962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu{ 70160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori struct dma_map_ops *ops = platform_dma_get_ops(dev); 7162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 72160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori if (ops->dma_supported) 73160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori return ops->dma_supported(dev, mask); 7462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 7562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu /* Copied from i386. Doesn't make much sense, because it will 7662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu only work for pci_alloc_coherent. 7762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu The caller just has to use GFP_DMA in this case. */ 782f4f27d42a301ed147e50c2edbcd27bb8990bc8eYang Hongyang if (mask < DMA_BIT_MASK(24)) 7962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return 0; 8062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 8162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu /* Tell the device to use SAC when IOMMU force is on. This 8262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu allows the driver to use cheaper accesses in some cases. 8362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 8462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu Problem with this is that if we overflow the IOMMU area and 8562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return DAC as fallback address the device may not handle it 8662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu correctly. 8762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 8862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu As a special case some controllers have a 39bit address 8962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu mode that is as efficient as 32bit (aic79xx). Don't force 9062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu SAC for these. Assume all masks <= 40 bits are of this 9162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu type. Normally this doesn't make any difference, but gives 9262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu more gentle handling of IOMMU overflow. */ 9350cf156af7dc68a44409bef636585ef88ebbab34Yang Hongyang if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) { 94e088a4ad7fa53c3dc3c29f930025f41ccf01953eMatthew Wilcox dev_info(dev, "Force SAC with mask %llx\n", mask); 9562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return 0; 9662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu } 9762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 9862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return 1; 9962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu} 10062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua YuEXPORT_SYMBOL(iommu_dma_supported); 10162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 102160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonorivoid __init pci_iommu_alloc(void) 103160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori{ 104160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops = &intel_dma_ops; 105160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori 106160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops->sync_single_for_cpu = machvec_dma_sync_single; 107160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops->sync_sg_for_cpu = machvec_dma_sync_sg; 108160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops->sync_single_for_device = machvec_dma_sync_single; 109160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops->sync_sg_for_device = machvec_dma_sync_sg; 110160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops->dma_supported = iommu_dma_supported; 111160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori 112160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori /* 113160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori * The order of these functions is important for 114160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori * fall-back/fail-over reasons 115160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori */ 116160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori detect_intel_iommu(); 117160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori 118160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori#ifdef CONFIG_SWIOTLB 119160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori pci_swiotlb_init(); 120160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori#endif 121160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori} 122160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori 12362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#endif 124