162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu/* 262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu * Dynamic DMA mapping support. 362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu */ 462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/types.h> 662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/mm.h> 762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/string.h> 862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/pci.h> 962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/module.h> 1062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/dmar.h> 1162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <asm/iommu.h> 1262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <asm/machvec.h> 1362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/dma-mapping.h> 1462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 1562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 16d3f138106b4b40640dc667f0222fd9f137387b32Suresh Siddha#ifdef CONFIG_INTEL_IOMMU 1762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 1862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <linux/kernel.h> 1962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 2062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#include <asm/page.h> 2162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 2262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yudma_addr_t bad_dma_address __read_mostly; 2362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua YuEXPORT_SYMBOL(bad_dma_address); 2462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 2562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yustatic int iommu_sac_force __read_mostly; 2662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 2762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuint no_iommu __read_mostly; 2862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#ifdef CONFIG_IOMMU_DEBUG 2962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuint force_iommu __read_mostly = 1; 3062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#else 3162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuint force_iommu __read_mostly; 3262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#endif 3362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 34aed5d5f4c5ea5da01a774e42cff08c4b4fa59072Fenghua Yuint iommu_pass_through; 35aed5d5f4c5ea5da01a774e42cff08c4b4fa59072Fenghua Yu 36160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonoriextern struct dma_map_ops intel_dma_ops; 3762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 3862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yustatic int __init pci_iommu_init(void) 3962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu{ 4062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu if (iommu_detected) 4162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu intel_iommu_init(); 4262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 4362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return 0; 4462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu} 4562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 4662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu/* Must execute after PCI subsystem */ 4762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yufs_initcall(pci_iommu_init); 4862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 4962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuvoid pci_iommu_shutdown(void) 5062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu{ 5162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return; 5262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu} 5362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 5462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuvoid __init 5562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuiommu_dma_init(void) 5662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu{ 5762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return; 5862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu} 5962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 6062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yuint iommu_dma_supported(struct device *dev, u64 mask) 6162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu{ 6262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu /* Copied from i386. Doesn't make much sense, because it will 6362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu only work for pci_alloc_coherent. 6462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu The caller just has to use GFP_DMA in this case. */ 652f4f27d42a301ed147e50c2edbcd27bb8990bc8eYang Hongyang if (mask < DMA_BIT_MASK(24)) 6662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return 0; 6762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 6862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu /* Tell the device to use SAC when IOMMU force is on. This 6962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu allows the driver to use cheaper accesses in some cases. 7062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 7162fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu Problem with this is that if we overflow the IOMMU area and 7262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return DAC as fallback address the device may not handle it 7362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu correctly. 7462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 7562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu As a special case some controllers have a 39bit address 7662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu mode that is as efficient as 32bit (aic79xx). Don't force 7762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu SAC for these. Assume all masks <= 40 bits are of this 7862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu type. Normally this doesn't make any difference, but gives 7962fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu more gentle handling of IOMMU overflow. */ 8050cf156af7dc68a44409bef636585ef88ebbab34Yang Hongyang if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) { 81e088a4ad7fa53c3dc3c29f930025f41ccf01953eMatthew Wilcox dev_info(dev, "Force SAC with mask %llx\n", mask); 8262fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return 0; 8362fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu } 8462fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 8562fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu return 1; 8662fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu} 8762fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua YuEXPORT_SYMBOL(iommu_dma_supported); 8862fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu 89160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonorivoid __init pci_iommu_alloc(void) 90160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori{ 91160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops = &intel_dma_ops; 92160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori 93160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops->sync_single_for_cpu = machvec_dma_sync_single; 94160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops->sync_sg_for_cpu = machvec_dma_sync_sg; 95160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops->sync_single_for_device = machvec_dma_sync_single; 96160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops->sync_sg_for_device = machvec_dma_sync_sg; 97160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori dma_ops->dma_supported = iommu_dma_supported; 98160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori 99160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori /* 100160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori * The order of these functions is important for 101160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori * fall-back/fail-over reasons 102160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori */ 103160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori detect_intel_iommu(); 104160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori 105160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori#ifdef CONFIG_SWIOTLB 106160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori pci_swiotlb_init(); 107160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori#endif 108160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori} 109160c1d8e40866edfeae7d68816b7005d70acf391FUJITA Tomonori 11062fdd7678a26efadd6ac5c2869543caff77d2df0Fenghua Yu#endif 111