diff options
28 files changed, 869 insertions, 647 deletions
diff --git a/arch/arm/mach-omap2/iommu2.c b/arch/arm/mach-omap2/iommu2.c index f286012783c..eefc37912ef 100644 --- a/arch/arm/mach-omap2/iommu2.c +++ b/arch/arm/mach-omap2/iommu2.c @@ -66,7 +66,7 @@ ((pgsz) == MMU_CAM_PGSZ_4K) ? 0xfffff000 : 0) -static void __iommu_set_twl(struct iommu *obj, bool on) +static void __iommu_set_twl(struct omap_iommu *obj, bool on) { u32 l = iommu_read_reg(obj, MMU_CNTL); @@ -85,7 +85,7 @@ static void __iommu_set_twl(struct iommu *obj, bool on) } -static int omap2_iommu_enable(struct iommu *obj) +static int omap2_iommu_enable(struct omap_iommu *obj) { u32 l, pa; unsigned long timeout; @@ -127,7 +127,7 @@ static int omap2_iommu_enable(struct iommu *obj) return 0; } -static void omap2_iommu_disable(struct iommu *obj) +static void omap2_iommu_disable(struct omap_iommu *obj) { u32 l = iommu_read_reg(obj, MMU_CNTL); @@ -138,12 +138,12 @@ static void omap2_iommu_disable(struct iommu *obj) dev_dbg(obj->dev, "%s is shutting down\n", obj->name); } -static void omap2_iommu_set_twl(struct iommu *obj, bool on) +static void omap2_iommu_set_twl(struct omap_iommu *obj, bool on) { __iommu_set_twl(obj, false); } -static u32 omap2_iommu_fault_isr(struct iommu *obj, u32 *ra) +static u32 omap2_iommu_fault_isr(struct omap_iommu *obj, u32 *ra) { u32 stat, da; u32 errs = 0; @@ -173,13 +173,13 @@ static u32 omap2_iommu_fault_isr(struct iommu *obj, u32 *ra) return errs; } -static void omap2_tlb_read_cr(struct iommu *obj, struct cr_regs *cr) +static void omap2_tlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr) { cr->cam = iommu_read_reg(obj, MMU_READ_CAM); cr->ram = iommu_read_reg(obj, MMU_READ_RAM); } -static void omap2_tlb_load_cr(struct iommu *obj, struct cr_regs *cr) +static void omap2_tlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr) { iommu_write_reg(obj, cr->cam | MMU_CAM_V, MMU_CAM); iommu_write_reg(obj, cr->ram, MMU_RAM); @@ -193,7 +193,8 @@ static u32 omap2_cr_to_virt(struct cr_regs *cr) return cr->cam & mask; } -static struct cr_regs *omap2_alloc_cr(struct iommu *obj, struct iotlb_entry *e) +static struct cr_regs *omap2_alloc_cr(struct omap_iommu *obj, + struct iotlb_entry *e) { struct cr_regs *cr; @@ -230,7 +231,8 @@ static u32 omap2_get_pte_attr(struct iotlb_entry *e) return attr; } -static ssize_t omap2_dump_cr(struct iommu *obj, struct cr_regs *cr, char *buf) +static ssize_t +omap2_dump_cr(struct omap_iommu *obj, struct cr_regs *cr, char *buf) { char *p = buf; @@ -254,7 +256,8 @@ static ssize_t omap2_dump_cr(struct iommu *obj, struct cr_regs *cr, char *buf) goto out; \ } while (0) -static ssize_t omap2_iommu_dump_ctx(struct iommu *obj, char *buf, ssize_t len) +static ssize_t +omap2_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t len) { char *p = buf; @@ -280,7 +283,7 @@ out: return p - buf; } -static void omap2_iommu_save_ctx(struct iommu *obj) +static void omap2_iommu_save_ctx(struct omap_iommu *obj) { int i; u32 *p = obj->ctx; @@ -293,7 +296,7 @@ static void omap2_iommu_save_ctx(struct iommu *obj) BUG_ON(p[0] != IOMMU_ARCH_VERSION); } -static void omap2_iommu_restore_ctx(struct iommu *obj) +static void omap2_iommu_restore_ctx(struct omap_iommu *obj) { int i; u32 *p = obj->ctx; @@ -343,13 +346,13 @@ static const struct iommu_functions omap2_iommu_ops = { static int __init omap2_iommu_init(void) { - return install_iommu_arch(&omap2_iommu_ops); + return omap_install_iommu_arch(&omap2_iommu_ops); } module_init(omap2_iommu_init); static void __exit omap2_iommu_exit(void) { - uninstall_iommu_arch(&omap2_iommu_ops); + omap_uninstall_iommu_arch(&omap2_iommu_ops); } module_exit(omap2_iommu_exit); diff --git a/arch/arm/plat-omap/Kconfig b/arch/arm/plat-omap/Kconfig index 6f4edd3408c..aa59f4247dc 100644 --- a/arch/arm/plat-omap/Kconfig +++ b/arch/arm/plat-omap/Kconfig @@ -134,18 +134,6 @@ config OMAP_MBOX_KFIFO_SIZE This can also be changed at runtime (via the mbox_kfifo_size module parameter). -config OMAP_IOMMU - tristate - -config OMAP_IOMMU_DEBUG - tristate "Export OMAP IOMMU internals in DebugFS" - depends on OMAP_IOMMU && DEBUG_FS - help - Select this to see extensive information about - the internal state of OMAP IOMMU in debugfs. - - Say N unless you know you need this. - config OMAP_IOMMU_IVA2 bool diff --git a/arch/arm/plat-omap/Makefile b/arch/arm/plat-omap/Makefile index f0233e6abcd..985262242f2 100644 --- a/arch/arm/plat-omap/Makefile +++ b/arch/arm/plat-omap/Makefile @@ -18,8 +18,6 @@ obj-$(CONFIG_ARCH_OMAP3) += omap_device.o obj-$(CONFIG_ARCH_OMAP4) += omap_device.o obj-$(CONFIG_OMAP_MCBSP) += mcbsp.o -obj-$(CONFIG_OMAP_IOMMU) += iommu.o iovmm.o -obj-$(CONFIG_OMAP_IOMMU_DEBUG) += iommu-debug.o obj-$(CONFIG_CPU_FREQ) += cpu-omap.o obj-$(CONFIG_OMAP_DM_TIMER) += dmtimer.o diff --git a/arch/arm/plat-omap/include/plat/iommu.h b/arch/arm/plat-omap/include/plat/iommu.h index 174f1b9c8c0..a1d79ee1925 100644 --- a/arch/arm/plat-omap/include/plat/iommu.h +++ b/arch/arm/plat-omap/include/plat/iommu.h @@ -25,16 +25,17 @@ struct iotlb_entry { }; }; -struct iommu { +struct omap_iommu { const char *name; struct module *owner; struct clk *clk; void __iomem *regbase; struct device *dev; void *isr_priv; + struct iommu_domain *domain; unsigned int refcount; - struct mutex iommu_lock; /* global for this whole object */ + spinlock_t iommu_lock; /* global for this whole object */ /* * We don't change iopgd for a situation like pgd for a task, @@ -48,8 +49,6 @@ struct iommu { struct list_head mmap; struct mutex mmap_lock; /* protect mmap */ - int (*isr)(struct iommu *obj, u32 da, u32 iommu_errs, void *priv); - void *ctx; /* iommu context: registres saved area */ u32 da_start; u32 da_end; @@ -81,25 +80,27 @@ struct iotlb_lock { struct iommu_functions { unsigned long version; - int (*enable)(struct iommu *obj); - void (*disable)(struct iommu *obj); - void (*set_twl)(struct iommu *obj, bool on); - u32 (*fault_isr)(struct iommu *obj, u32 *ra); + int (*enable)(struct omap_iommu *obj); + void (*disable)(struct omap_iommu *obj); + void (*set_twl)(struct omap_iommu *obj, bool on); + u32 (*fault_isr)(struct omap_iommu *obj, u32 *ra); - void (*tlb_read_cr)(struct iommu *obj, struct cr_regs *cr); - void (*tlb_load_cr)(struct iommu *obj, struct cr_regs *cr); + void (*tlb_read_cr)(struct omap_iommu *obj, struct cr_regs *cr); + void (*tlb_load_cr)(struct omap_iommu *obj, struct cr_regs *cr); - struct cr_regs *(*alloc_cr)(struct iommu *obj, struct iotlb_entry *e); + struct cr_regs *(*alloc_cr)(struct omap_iommu *obj, + struct iotlb_entry *e); int (*cr_valid)(struct cr_regs *cr); u32 (*cr_to_virt)(struct cr_regs *cr); void (*cr_to_e)(struct cr_regs *cr, struct iotlb_entry *e); - ssize_t (*dump_cr)(struct iommu *obj, struct cr_regs *cr, char *buf); + ssize_t (*dump_cr)(struct omap_iommu *obj, struct cr_regs *cr, + char *buf); u32 (*get_pte_attr)(struct iotlb_entry *e); - void (*save_ctx)(struct iommu *obj); - void (*restore_ctx)(struct iommu *obj); - ssize_t (*dump_ctx)(struct iommu *obj, char *buf, ssize_t len); + void (*save_ctx)(struct omap_iommu *obj); + void (*restore_ctx)(struct omap_iommu *obj); + ssize_t (*dump_ctx)(struct omap_iommu *obj, char *buf, ssize_t len); }; struct iommu_platform_data { @@ -150,40 +151,31 @@ struct iommu_platform_data { /* * global functions */ -extern u32 iommu_arch_version(void); - -extern void iotlb_cr_to_e(struct cr_regs *cr, struct iotlb_entry *e); -extern u32 iotlb_cr_to_virt(struct cr_regs *cr); - -extern int load_iotlb_entry(struct iommu *obj, struct iotlb_entry *e); -extern void iommu_set_twl(struct iommu *obj, bool on); -extern void flush_iotlb_page(struct iommu *obj, u32 da); -extern void flush_iotlb_range(struct iommu *obj, u32 start, u32 end); -extern void flush_iotlb_all(struct iommu *obj); - -extern int iopgtable_store_entry(struct iommu *obj, struct iotlb_entry *e); -extern void iopgtable_lookup_entry(struct iommu *obj, u32 da, u32 **ppgd, - u32 **ppte); -extern size_t iopgtable_clear_entry(struct iommu *obj, u32 iova); - -extern int iommu_set_da_range(struct iommu *obj, u32 start, u32 end); -extern struct iommu *iommu_get(const char *name); -extern void iommu_put(struct iommu *obj); -extern int iommu_set_isr(const char *name, - int (*isr)(struct iommu *obj, u32 da, u32 iommu_errs, +extern u32 omap_iommu_arch_version(void); + +extern void omap_iotlb_cr_to_e(struct cr_regs *cr, struct iotlb_entry *e); + +extern int +omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e); + +extern int omap_iommu_set_isr(const char *name, + int (*isr)(struct omap_iommu *obj, u32 da, u32 iommu_errs, void *priv), void *isr_priv); -extern void iommu_save_ctx(struct iommu *obj); -extern void iommu_restore_ctx(struct iommu *obj); +extern void omap_iommu_save_ctx(struct omap_iommu *obj); +extern void omap_iommu_restore_ctx(struct omap_iommu *obj); -extern int install_iommu_arch(const struct iommu_functions *ops); -extern void uninstall_iommu_arch(const struct iommu_functions *ops); +extern int omap_install_iommu_arch(const struct iommu_functions *ops); +extern void omap_uninstall_iommu_arch(const struct iommu_functions *ops); -extern int foreach_iommu_device(void *data, +extern int omap_foreach_iommu_device(void *data, int (*fn)(struct device *, void *)); -extern ssize_t iommu_dump_ctx(struct iommu *obj, char *buf, ssize_t len); -extern size_t dump_tlb_entries(struct iommu *obj, char *buf, ssize_t len); +extern ssize_t +omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t len); +extern size_t +omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t len); +struct device *omap_find_iommu_device(const char *name); #endif /* __MACH_IOMMU_H */ diff --git a/arch/arm/plat-omap/include/plat/iommu2.h b/arch/arm/plat-omap/include/plat/iommu2.h index 10ad05f410e..d4116b595e4 100644 --- a/arch/arm/plat-omap/include/plat/iommu2.h +++ b/arch/arm/plat-omap/include/plat/iommu2.h @@ -83,12 +83,12 @@ /* * register accessors */ -static inline u32 iommu_read_reg(struct iommu *obj, size_t offs) +static inline u32 iommu_read_reg(struct omap_iommu *obj, size_t offs) { return __raw_readl(obj->regbase + offs); } -static inline void iommu_write_reg(struct iommu *obj, u32 val, size_t offs) +static inline void iommu_write_reg(struct omap_iommu *obj, u32 val, size_t offs) { __raw_writel(val, obj->regbase + offs); } diff --git a/arch/arm/plat-omap/iopgtable.h b/arch/arm/plat-omap/include/plat/iopgtable.h index c3e93bb0911..66a813977d5 100644 --- a/arch/arm/plat-omap/iopgtable.h +++ b/arch/arm/plat-omap/include/plat/iopgtable.h @@ -56,6 +56,19 @@ #define IOPAGE_MASK IOPTE_MASK +/** + * omap_iommu_translate() - va to pa translation + * @d: omap iommu descriptor + * @va: virtual address + * @mask: omap iommu descriptor mask + * + * va to pa translation + */ +static inline phys_addr_t omap_iommu_translate(u32 d, u32 va, u32 mask) +{ + return (d & mask) | (va & (~mask)); +} + /* * some descriptor attributes. */ @@ -64,10 +77,15 @@ #define IOPGD_SUPER (1 << 18 | 2 << 0) #define iopgd_is_table(x) (((x) & 3) == IOPGD_TABLE) +#define iopgd_is_section(x) (((x) & (1 << 18 | 3)) == IOPGD_SECTION) +#define iopgd_is_super(x) (((x) & (1 << 18 | 3)) == IOPGD_SUPER) #define IOPTE_SMALL (2 << 0) #define IOPTE_LARGE (1 << 0) +#define iopte_is_small(x) (((x) & 2) == IOPTE_SMALL) +#define iopte_is_large(x) (((x) & 3) == IOPTE_LARGE) + /* to find an entry in a page-table-directory */ #define iopgd_index(da) (((da) >> IOPGD_SHIFT) & (PTRS_PER_IOPGD - 1)) #define iopgd_offset(obj, da) ((obj)->iopgd + iopgd_index(da)) @@ -97,6 +115,6 @@ static inline u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, } #define to_iommu(dev) \ - (struct iommu *)platform_get_drvdata(to_platform_device(dev)) + (struct omap_iommu *)platform_get_drvdata(to_platform_device(dev)) #endif /* __PLAT_OMAP_IOMMU_H */ diff --git a/arch/arm/plat-omap/include/plat/iovmm.h b/arch/arm/plat-omap/include/plat/iovmm.h index e992b9655fb..6af1a91c0f3 100644 --- a/arch/arm/plat-omap/include/plat/iovmm.h +++ b/arch/arm/plat-omap/include/plat/iovmm.h @@ -13,8 +13,10 @@ #ifndef __IOMMU_MMAP_H #define __IOMMU_MMAP_H +#include <linux/iommu.h> + struct iovm_struct { - struct iommu *iommu; /* iommu object which this belongs to */ + struct omap_iommu *iommu; /* iommu object which this belongs to */ u32 da_start; /* area definition */ u32 da_end; u32 flags; /* IOVMF_: see below */ @@ -70,20 +72,18 @@ struct iovm_struct { #define IOVMF_DA_FIXED (1 << (4 + IOVMF_SW_SHIFT)) -extern struct iovm_struct *find_iovm_area(struct iommu *obj, u32 da); -extern u32 iommu_vmap(struct iommu *obj, u32 da, +extern struct iovm_struct *omap_find_iovm_area(struct omap_iommu *obj, u32 da); +extern u32 +omap_iommu_vmap(struct iommu_domain *domain, struct omap_iommu *obj, u32 da, const struct sg_table *sgt, u32 flags); -extern struct sg_table *iommu_vunmap(struct iommu *obj, u32 da); -extern u32 iommu_vmalloc(struct iommu *obj, u32 da, size_t bytes, - u32 flags); -extern void iommu_vfree(struct iommu *obj, const u32 da); -extern u32 iommu_kmap(struct iommu *obj, u32 da, u32 pa, size_t bytes, - u32 flags); -extern void iommu_kunmap(struct iommu *obj, u32 da); -extern u32 iommu_kmalloc(struct iommu *obj, u32 da, size_t bytes, - u32 flags); -extern void iommu_kfree(struct iommu *obj, u32 da); - -extern void *da_to_va(struct iommu *obj, u32 da); +extern struct sg_table *omap_iommu_vunmap(struct iommu_domain *domain, + struct omap_iommu *obj, u32 da); +extern u32 +omap_iommu_vmalloc(struct iommu_domain *domain, struct omap_iommu *obj, + u32 da, size_t bytes, u32 flags); +extern void +omap_iommu_vfree(struct iommu_domain *domain, struct omap_iommu *obj, + const u32 da); +extern void *omap_da_to_va(struct omap_iommu *obj, u32 da); #endif /* __IOMMU_MMAP_H */ diff --git a/arch/ia64/kvm/kvm-ia64.c b/arch/ia64/kvm/kvm-ia64.c index 8213efe1998..43f4c92816e 100644 --- a/arch/ia64/kvm/kvm-ia64.c +++ b/arch/ia64/kvm/kvm-ia64.c @@ -33,6 +33,7 @@ #include <linux/uaccess.h> #include <linux/iommu.h> #include <linux/intel-iommu.h> +#include <linux/pci.h> #include <asm/pgtable.h> #include <asm/gcc_intrin.h> @@ -204,7 +205,7 @@ int kvm_dev_ioctl_check_extension(long ext) r = KVM_COALESCED_MMIO_PAGE_OFFSET; break; case KVM_CAP_IOMMU: - r = iommu_found(); + r = iommu_present(&pci_bus_type); break; default: r = 0; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index cf269096ead..c38efd7b792 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -44,6 +44,7 @@ #include <linux/perf_event.h> #include <linux/uaccess.h> #include <linux/hash.h> +#include <linux/pci.h> #include <trace/events/kvm.h> #define CREATE_TRACE_POINTS @@ -2123,7 +2124,7 @@ int kvm_dev_ioctl_check_extension(long ext) r = 0; break; case KVM_CAP_IOMMU: - r = iommu_found(); + r = iommu_present(&pci_bus_type); break; case KVM_CAP_MCE: r = KVM_MAX_MCE_BANKS; diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 7d7eaa15e77..5414253b185 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -112,4 +112,23 @@ config IRQ_REMAP To use x2apic mode in the CPU's which support x2APIC enhancements or to support platforms with CPU's having > 8 bit APIC ID, say Y. +# OMAP IOMMU support +config OMAP_IOMMU + bool "OMAP IOMMU Support" + depends on ARCH_OMAP + select IOMMU_API + +config OMAP_IOVMM + tristate "OMAP IO Virtual Memory Manager Support" + depends on OMAP_IOMMU + +config OMAP_IOMMU_DEBUG + tristate "Export OMAP IOMMU/IOVMM internals in DebugFS" + depends on OMAP_IOVMM && DEBUG_FS + help + Select this to see extensive information about + the internal state of OMAP IOMMU/IOVMM in debugfs. + + Say N unless you know you need this. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 6394994a2b9..2f4448794bc 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -4,3 +4,6 @@ obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o obj-$(CONFIG_IRQ_REMAP) += intr_remapping.o +obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o +obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o +obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 0e4227f457a..4ee277a8521 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1283,7 +1283,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom, if (!pte || !IOMMU_PTE_PRESENT(*pte)) continue; - dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1); + dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1); } update_domain(&dma_dom->domain); @@ -2495,7 +2495,7 @@ static unsigned device_dma_ops_init(void) void __init amd_iommu_init_api(void) { - register_iommu(&amd_iommu_ops); + bus_set_iommu(&pci_bus_type, &amd_iommu_ops); } int __init amd_iommu_init_dma_ops(void) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index be1953c239b..bb161d2fa03 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3642,7 +3642,7 @@ int __init intel_iommu_init(void) init_iommu_pm_ops(); - register_iommu(&intel_iommu_ops); + bus_set_iommu(&pci_bus_type, &intel_iommu_ops); bus_register_notifier(&pci_bus_type, &device_nb); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 6e6b6a11b3c..2fb2963df55 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -16,6 +16,8 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include <linux/device.h> +#include <linux/kernel.h> #include <linux/bug.h> #include <linux/types.h> #include <linux/module.h> @@ -23,32 +25,78 @@ #include <linux/errno.h> #include <linux/iommu.h> -static struct iommu_ops *iommu_ops; +static void iommu_bus_init(struct bus_type *bus, struct iommu_ops *ops) +{ +} -void register_iommu(struct iommu_ops *ops) +/** + * bus_set_iommu - set iommu-callbacks for the bus + * @bus: bus. + * @ops: the callbacks provided by the iommu-driver + * + * This function is called by an iommu driver to set the iommu methods + * used for a particular bus. Drivers for devices on that bus can use + * the iommu-api after these ops are registered. + * This special function is needed because IOMMUs are usually devices on + * the bus itself, so the iommu drivers are not initialized when the bus + * is set up. With this function the iommu-driver can set the iommu-ops + * afterwards. + */ +int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops) { - if (iommu_ops) - BUG(); + if (bus->iommu_ops != NULL) + return -EBUSY; + + bus->iommu_ops = ops; + + /* Do IOMMU specific setup for this bus-type */ + iommu_bus_init(bus, ops); - iommu_ops = ops; + return 0; } +EXPORT_SYMBOL_GPL(bus_set_iommu); -bool iommu_found(void) +bool iommu_present(struct bus_type *bus) { - return iommu_ops != NULL; + return bus->iommu_ops != NULL; } -EXPORT_SYMBOL_GPL(iommu_found); +EXPORT_SYMBOL_GPL(iommu_present); -struct iommu_domain *iommu_domain_alloc(void) +/** + * iommu_set_fault_handler() - set a fault handler for an iommu domain + * @domain: iommu domain + * @handler: fault handler + * + * This function should be used by IOMMU users which want to be notified + * whenever an IOMMU fault happens. + * + * The fault handler itself should return 0 on success, and an appropriate + * error code otherwise. + */ +void iommu_set_fault_handler(struct iommu_domain *domain, + iommu_fault_handler_t handler) +{ + BUG_ON(!domain); + + domain->handler = handler; +} +EXPORT_SYMBOL_GPL(iommu_set_fault_handler); + +struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) { struct iommu_domain *domain; int ret; + if (bus == NULL || bus->iommu_ops == NULL) + return NULL; + domain = kmalloc(sizeof(*domain), GFP_KERNEL); if (!domain) return NULL; - ret = iommu_ops->domain_init(domain); + domain->ops = bus->iommu_ops; + + ret = domain->ops->domain_init(domain); if (ret) goto out_free; @@ -63,62 +111,78 @@ EXPORT_SYMBOL_GPL(iommu_domain_alloc); void iommu_domain_free(struct iommu_domain *domain) { - iommu_ops->domain_destroy(domain); + if (likely(domain->ops->domain_destroy != NULL)) + domain->ops->domain_destroy(domain); + kfree(domain); } EXPORT_SYMBOL_GPL(iommu_domain_free); int iommu_attach_device(struct iommu_domain *domain, struct device *dev) { - return iommu_ops->attach_dev(domain, dev); + if (unlikely(domain->ops->attach_dev == NULL)) + return -ENODEV; + + return domain->ops->attach_dev(domain, dev); } EXPORT_SYMBOL_GPL(iommu_attach_device); void iommu_detach_device(struct iommu_domain *domain, struct device *dev) { - iommu_ops->detach_dev(domain, dev); + if (unlikely(domain->ops->detach_dev == NULL)) + return; + + domain->ops->detach_dev(domain, dev); } EXPORT_SYMBOL_GPL(iommu_detach_device); phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, unsigned long iova) { - return iommu_ops->iova_to_phys(domain, iova); + if (unlikely(domain->ops->iova_to_phys == NULL)) + return 0; + + return domain->ops->iova_to_phys(domain, iova); } EXPORT_SYMBOL_GPL(iommu_iova_to_phys); int iommu_domain_has_cap(struct iommu_domain *domain, unsigned long cap) { - return iommu_ops->domain_has_cap(domain, cap); + if (unlikely(domain->ops->domain_has_cap == NULL)) + return 0; + + return domain->ops->domain_has_cap(domain, cap); } EXPORT_SYMBOL_GPL(iommu_domain_has_cap); int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, int gfp_order, int prot) { - unsigned long invalid_mask; size_t size; - size = 0x1000UL << gfp_order; - invalid_mask = size - 1; + if (unlikely(domain->ops->map == NULL)) + return -ENODEV; - BUG_ON((iova | paddr) & invalid_mask); + size = PAGE_SIZE << gfp_order; - return iommu_ops->map(domain, iova, paddr, gfp_order, prot); + BUG_ON(!IS_ALIGNED(iova | paddr, size)); + + return domain->ops->map(domain, iova, paddr, gfp_order, prot); } EXPORT_SYMBOL_GPL(iommu_map); int iommu_unmap(struct iommu_domain *domain, unsigned long iova, int gfp_order) { - unsigned long invalid_mask; size_t size; - size = 0x1000UL << gfp_order; - invalid_mask = size - 1; + if (unlikely(domain->ops->unmap == NULL)) + return -ENODEV; + + size = PAGE_SIZE << gfp_order; - BUG_ON(iova & invalid_mask); + BUG_ON(!IS_ALIGNED(iova, size)); - return iommu_ops->unmap(domain, iova, gfp_order); + return domain->ops->unmap(domain, iova, gfp_order); } EXPORT_SYMBOL_GPL(iommu_unmap); diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 1a584e077c6..5865dd2e28f 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -543,6 +543,13 @@ static int msm_iommu_unmap(struct iommu_domain *domain, unsigned long va, } ret = __flush_iotlb(domain); + + /* + * the IOMMU API requires us to return the order of the unmapped + * page (on success). + */ + if (!ret) + ret = order; fail: spin_unlock_irqrestore(&msm_iommu_lock, flags); return ret; @@ -721,7 +728,7 @@ static void __init setup_iommu_tex_classes(void) static int __init msm_iommu_init(void) { setup_iommu_tex_classes(); - register_iommu(&msm_iommu_ops); + bus_set_iommu(&platform_bus_type, &msm_iommu_ops); return 0; } diff --git a/arch/arm/plat-omap/iommu-debug.c b/drivers/iommu/omap-iommu-debug.c index f07cf2f08e0..9c192e79f80 100644 --- a/arch/arm/plat-omap/iommu-debug.c +++ b/drivers/iommu/omap-iommu-debug.c @@ -21,7 +21,7 @@ #include <plat/iommu.h> #include <plat/iovmm.h> -#include "iopgtable.h" +#include <plat/iopgtable.h> #define MAXCOLUMN 100 /* for short messages */ @@ -32,7 +32,7 @@ static struct dentry *iommu_debug_root; static ssize_t debug_read_ver(struct file *file, char __user *userbuf, size_t count, loff_t *ppos) { - u32 ver = iommu_arch_version(); + u32 ver = omap_iommu_arch_version(); char buf[MAXCOLUMN], *p = buf; p += sprintf(p, "H/W version: %d.%d\n", (ver >> 4) & 0xf , ver & 0xf); @@ -43,7 +43,7 @@ static ssize_t debug_read_ver(struct file *file, char __user *userbuf, static ssize_t debug_read_regs(struct file *file, char __user *userbuf, size_t count, loff_t *ppos) { - struct iommu *obj = file->private_data; + struct omap_iommu *obj = file->private_data; char *p, *buf; ssize_t bytes; @@ -54,7 +54,7 @@ static ssize_t debug_read_regs(struct file *file, char __user *userbuf, mutex_lock(&iommu_debug_lock); - bytes = iommu_dump_ctx(obj, p, count); + bytes = omap_iommu_dump_ctx(obj, p, count); bytes = simple_read_from_buffer(userbuf, count, ppos, buf, bytes); mutex_unlock(&iommu_debug_lock); @@ -66,7 +66,7 @@ static ssize_t debug_read_regs(struct file *file, char __user *userbuf, static ssize_t debug_read_tlb(struct file *file, char __user *userbuf, size_t count, loff_t *ppos) { - struct iommu *obj = file->private_data; + struct omap_iommu *obj = file->private_data; char *p, *buf; ssize_t bytes, rest; @@ -80,7 +80,7 @@ static ssize_t debug_read_tlb(struct file *file, char __user *userbuf, p += sprintf(p, "%8s %8s\n", "cam:", "ram:"); p += sprintf(p, "-----------------------------------------\n"); rest = count - (p - buf); - p += dump_tlb_entries(obj, p, rest); + p += omap_dump_tlb_entries(obj, p, rest); bytes = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf); @@ -96,7 +96,7 @@ static ssize_t debug_write_pagetable(struct file *file, struct iotlb_entry e; struct cr_regs cr; int err; - struct iommu *obj = file->private_data; + struct omap_iommu *obj = file->private_data; char buf[MAXCOLUMN], *p = buf; count = min(count, sizeof(buf)); @@ -113,8 +113,8 @@ static ssize_t debug_write_pagetable(struct file *file, return -EINVAL; } - iotlb_cr_to_e(&cr, &e); - err = iopgtable_store_entry(obj, &e); + omap_iotlb_cr_to_e(&cr, &e); + err = omap_iopgtable_store_entry(obj, &e); if (err) dev_err(obj->dev, "%s: fail to store cr\n", __func__); @@ -136,7 +136,7 @@ static ssize_t debug_write_pagetable(struct file *file, __err; \ }) -static ssize_t dump_ioptable(struct iommu *obj, char *buf, ssize_t len) +static ssize_t dump_ioptable(struct omap_iommu *obj, char *buf, ssize_t len) { int i; u32 *iopgd; @@ -183,7 +183,7 @@ out: static ssize_t debug_read_pagetable(struct file *file, char __user *userbuf, size_t count, loff_t *ppos) { - struct iommu *obj = file->private_data; + struct omap_iommu *obj = file->private_data; char *p, *buf; size_t bytes; @@ -211,7 +211,7 @@ static ssize_t debug_read_pagetable(struct file *file, char __user *userbuf, static ssize_t debug_read_mmap(struct file *file, char __user *userbuf, size_t count, loff_t *ppos) { - struct iommu *obj = file->private_data; + struct omap_iommu *obj = file->private_data; char *p, *buf; struct iovm_struct *tmp; int uninitialized_var(i); @@ -253,7 +253,7 @@ static ssize_t debug_read_mmap(struct file *file, char __user *userbuf, static ssize_t debug_read_mem(struct file *file, char __user *userbuf, size_t count, loff_t *ppos) { - struct iommu *obj = file->private_data; + struct omap_iommu *obj = file->private_data; char *p, *buf; struct iovm_struct *area; ssize_t bytes; @@ -267,7 +267,7 @@ static ssize_t debug_read_mem(struct file *file, char __user *userbuf, mutex_lock(&iommu_debug_lock); - area = find_iovm_area(obj, (u32)ppos); + area = omap_find_iovm_area(obj, (u32)ppos); if (IS_ERR(area)) { bytes = -EINVAL; goto err_out; @@ -286,7 +286,7 @@ err_out: static ssize_t debug_write_mem(struct file *file, const char __user *userbuf, size_t count, loff_t *ppos) { - struct iommu *obj = file->private_data; + struct omap_iommu *obj = file->private_data; struct iovm_struct *area; char *p, *buf; @@ -304,7 +304,7 @@ static ssize_t debug_write_mem(struct file *file, const char __user *userbuf, goto err_out; } - area = find_iovm_area(obj, (u32)ppos); + area = omap_find_iovm_area(obj, (u32)ppos); if (IS_ERR(area)) { count = -EINVAL; goto err_out; @@ -360,7 +360,7 @@ DEBUG_FOPS(mem); static int iommu_debug_register(struct device *dev, void *data) { struct platform_device *pdev = to_platform_device(dev); - struct iommu *obj = platform_get_drvdata(pdev); + struct omap_iommu *obj = platform_get_drvdata(pdev); struct dentry *d, *parent; if (!obj || !obj->dev) @@ -396,7 +396,7 @@ static int __init iommu_debug_init(void) return -ENOMEM; iommu_debug_root = d; - err = foreach_iommu_device(d, iommu_debug_register); + err = omap_foreach_iommu_device(d, iommu_debug_register); if (err) goto err_out; return 0; diff --git a/arch/arm/plat-omap/iommu.c b/drivers/iommu/omap-iommu.c index 34fc31ee908..8f32b2bf758 100644 --- a/arch/arm/plat-omap/iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -18,18 +18,34 @@ #include <linux/ioport.h> #include <linux/clk.h> #include <linux/platform_device.h> +#include <linux/iommu.h> +#include <linux/mutex.h> +#include <linux/spinlock.h> #include <asm/cacheflush.h> #include <plat/iommu.h> -#include "iopgtable.h" +#include <plat/iopgtable.h> #define for_each_iotlb_cr(obj, n, __i, cr) \ for (__i = 0; \ (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \ __i++) +/** + * struct omap_iommu_domain - omap iommu domain + * @pgtable: the page table + * @iommu_dev: an omap iommu device attached to this domain. only a single + * iommu device can be attached for now. + * @lock: domain lock, should be taken when attaching/detaching + */ +struct omap_iommu_domain { + u32 *pgtable; + struct omap_iommu *iommu_dev; + spinlock_t lock; +}; + /* accommodate the difference between omap1 and omap2/3 */ static const struct iommu_functions *arch_iommu; @@ -37,13 +53,13 @@ static struct platform_driver omap_iommu_driver; static struct kmem_cache *iopte_cachep; /** - * install_iommu_arch - Install archtecure specific iommu functions + * omap_install_iommu_arch - Install archtecure specific iommu functions * @ops: a pointer to architecture specific iommu functions * * There are several kind of iommu algorithm(tlb, pagetable) among * omap series. This interface installs such an iommu algorighm. **/ -int install_iommu_arch(const struct iommu_functions *ops) +int omap_install_iommu_arch(const struct iommu_functions *ops) { if (arch_iommu) return -EBUSY; @@ -51,53 +67,53 @@ int install_iommu_arch(const struct iommu_functions *ops) arch_iommu = ops; return 0; } -EXPORT_SYMBOL_GPL(install_iommu_arch); +EXPORT_SYMBOL_GPL(omap_install_iommu_arch); /** - * uninstall_iommu_arch - Uninstall archtecure specific iommu functions + * omap_uninstall_iommu_arch - Uninstall archtecure specific iommu functions * @ops: a pointer to architecture specific iommu functions * * This interface uninstalls the iommu algorighm installed previously. **/ -void uninstall_iommu_arch(const struct iommu_functions *ops) +void omap_uninstall_iommu_arch(const struct iommu_functions *ops) { if (arch_iommu != ops) pr_err("%s: not your arch\n", __func__); arch_iommu = NULL; } -EXPORT_SYMBOL_GPL(uninstall_iommu_arch); +EXPORT_SYMBOL_GPL(omap_uninstall_iommu_arch); /** - * iommu_save_ctx - Save registers for pm off-mode support + * omap_iommu_save_ctx - Save registers for pm off-mode support * @obj: target iommu **/ -void iommu_save_ctx(struct iommu *obj) +void omap_iommu_save_ctx(struct omap_iommu *obj) { arch_iommu->save_ctx(obj); } -EXPORT_SYMBOL_GPL(iommu_save_ctx); +EXPORT_SYMBOL_GPL(omap_iommu_save_ctx); /** - * iommu_restore_ctx - Restore registers for pm off-mode support + * omap_iommu_restore_ctx - Restore registers for pm off-mode support * @obj: target iommu **/ -void iommu_restore_ctx(struct iommu *obj) +void omap_iommu_restore_ctx(struct omap_iommu *obj) { arch_iommu->restore_ctx(obj); } -EXPORT_SYMBOL_GPL(iommu_restore_ctx); +EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx); /** - * iommu_arch_version - Return running iommu arch version + * omap_iommu_arch_version - Return running iommu arch version **/ -u32 iommu_arch_version(void) +u32 omap_iommu_arch_version(void) { return arch_iommu->version; } -EXPORT_SYMBOL_GPL(iommu_arch_version); +EXPORT_SYMBOL_GPL(omap_iommu_arch_version); -static int iommu_enable(struct iommu *obj) +static int iommu_enable(struct omap_iommu *obj) { int err; @@ -115,7 +131,7 @@ static int iommu_enable(struct iommu *obj) return err; } -static void iommu_disable(struct iommu *obj) +static void iommu_disable(struct omap_iommu *obj) { if (!obj) return; @@ -130,13 +146,13 @@ static void iommu_disable(struct iommu *obj) /* * TLB operations */ -void iotlb_cr_to_e(struct cr_regs *cr, struct iotlb_entry *e) +void omap_iotlb_cr_to_e(struct cr_regs *cr, struct iotlb_entry *e) { BUG_ON(!cr || !e); arch_iommu->cr_to_e(cr, e); } -EXPORT_SYMBOL_GPL(iotlb_cr_to_e); +EXPORT_SYMBOL_GPL(omap_iotlb_cr_to_e); static inline int iotlb_cr_valid(struct cr_regs *cr) { @@ -146,7 +162,7 @@ static inline int iotlb_cr_valid(struct cr_regs *cr) return arch_iommu->cr_valid(cr); } -static inline struct cr_regs *iotlb_alloc_cr(struct iommu *obj, +static inline struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj, struct iotlb_entry *e) { if (!e) @@ -155,23 +171,22 @@ static inline struct cr_regs *iotlb_alloc_cr(struct iommu *obj, return arch_iommu->alloc_cr(obj, e); } -u32 iotlb_cr_to_virt(struct cr_regs *cr) +static u32 iotlb_cr_to_virt(struct cr_regs *cr) { return arch_iommu->cr_to_virt(cr); } -EXPORT_SYMBOL_GPL(iotlb_cr_to_virt); static u32 get_iopte_attr(struct iotlb_entry *e) { return arch_iommu->get_pte_attr(e); } -static u32 iommu_report_fault(struct iommu *obj, u32 *da) +static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da) { return arch_iommu->fault_isr(obj, da); } -static void iotlb_lock_get(struct iommu *obj, struct iotlb_lock *l) +static void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l) { u32 val; @@ -182,7 +197,7 @@ static void iotlb_lock_get(struct iommu *obj, struct iotlb_lock *l) } -static void iotlb_lock_set(struct iommu *obj, struct iotlb_lock *l) +static void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l) { u32 val; @@ -192,12 +207,12 @@ static void iotlb_lock_set(struct iommu *obj, struct iotlb_lock *l) iommu_write_reg(obj, val, MMU_LOCK); } -static void iotlb_read_cr(struct iommu *obj, struct cr_regs *cr) +static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr) { arch_iommu->tlb_read_cr(obj, cr); } -static void iotlb_load_cr(struct iommu *obj, struct cr_regs *cr) +static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr) { arch_iommu->tlb_load_cr(obj, cr); @@ -211,7 +226,7 @@ static void iotlb_load_cr(struct iommu *obj, struct cr_regs *cr) * @cr: contents of cam and ram register * @buf: output buffer **/ -static inline ssize_t iotlb_dump_cr(struct iommu *obj, struct cr_regs *cr, +static inline ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr, char *buf) { BUG_ON(!cr || !buf); @@ -220,7 +235,7 @@ static inline ssize_t iotlb_dump_cr(struct iommu *obj, struct cr_regs *cr, } /* only used in iotlb iteration for-loop */ -static struct cr_regs __iotlb_read_cr(struct iommu *obj, int n) +static struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n) { struct cr_regs cr; struct iotlb_lock l; @@ -238,7 +253,8 @@ static struct cr_regs __iotlb_read_cr(struct iommu *obj, int n) * @obj: target iommu * @e: an iommu tlb entry info **/ -int load_iotlb_entry(struct iommu *obj, struct iotlb_entry *e) +#ifdef PREFETCH_IOTLB +static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) { int err = 0; struct iotlb_lock l; @@ -294,7 +310,20 @@ out: clk_disable(obj->clk); return err; } -EXPORT_SYMBOL_GPL(load_iotlb_entry); + +#else /* !PREFETCH_IOTLB */ + +static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) +{ + return 0; +} + +#endif /* !PREFETCH_IOTLB */ + +static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) +{ + return load_iotlb_entry(obj, e); +} /** * flush_iotlb_page - Clear an iommu tlb entry @@ -303,7 +332,7 @@ EXPORT_SYMBOL_GPL(load_iotlb_entry); * * Clear an iommu tlb entry which includes 'da' address. **/ -void flush_iotlb_page(struct iommu *obj, u32 da) +static void flush_iotlb_page(struct omap_iommu *obj, u32 da) { int i; struct cr_regs cr; @@ -332,33 +361,12 @@ void flush_iotlb_page(struct iommu *obj, u32 da) if (i == obj->nr_tlb_entries) dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da); } -EXPORT_SYMBOL_GPL(flush_iotlb_page); - -/** - * flush_iotlb_range - Clear an iommu tlb entries - * @obj: target iommu - * @start: iommu device virtual address(start) - * @end: iommu device virtual address(end) - * - * Clear an iommu tlb entry which includes 'da' address. - **/ -void flush_iotlb_range(struct iommu *obj, u32 start, u32 end) -{ - u32 da = start; - - while (da < end) { - flush_iotlb_page(obj, da); - /* FIXME: Optimize for multiple page size */ - da += IOPTE_SIZE; - } -} -EXPORT_SYMBOL_GPL(flush_iotlb_range); /** * flush_iotlb_all - Clear all iommu tlb entries * @obj: target iommu **/ -void flush_iotlb_all(struct iommu *obj) +static void flush_iotlb_all(struct omap_iommu *obj) { struct iotlb_lock l; @@ -372,28 +380,10 @@ void flush_iotlb_all(struct iommu *obj) clk_disable(obj->clk); } -EXPORT_SYMBOL_GPL(flush_iotlb_all); - -/** - * iommu_set_twl - enable/disable table walking logic - * @obj: target iommu - * @on: enable/disable - * - * Function used to enable/disable TWL. If one wants to work - * exclusively with locked TLB entries and receive notifications - * for TLB miss then call this function to disable TWL. - */ -void iommu_set_twl(struct iommu *obj, bool on) -{ - clk_enable(obj->clk); - arch_iommu->set_twl(obj, on); - clk_disable(obj->clk); -} -EXPORT_SYMBOL_GPL(iommu_set_twl); -#if defined(CONFIG_OMAP_IOMMU_DEBUG_MODULE) +#if defined(CONFIG_OMAP_IOMMU_DEBUG) || defined(CONFIG_OMAP_IOMMU_DEBUG_MODULE) -ssize_t iommu_dump_ctx(struct iommu *obj, char *buf, ssize_t bytes) +ssize_t omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t bytes) { if (!obj || !buf) return -EINVAL; @@ -406,9 +396,10 @@ ssize_t iommu_dump_ctx(struct iommu *obj, char *buf, ssize_t bytes) return bytes; } -EXPORT_SYMBOL_GPL(iommu_dump_ctx); +EXPORT_SYMBOL_GPL(omap_iommu_dump_ctx); -static int __dump_tlb_entries(struct iommu *obj, struct cr_regs *crs, int num) +static int +__dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num) { int i; struct iotlb_lock saved; @@ -431,11 +422,11 @@ static int __dump_tlb_entries(struct iommu *obj, struct cr_regs *crs, int num) } /** - * dump_tlb_entries - dump cr arrays to given buffer + * omap_dump_tlb_entries - dump cr arrays to given buffer * @obj: target iommu * @buf: output buffer **/ -size_t dump_tlb_entries(struct iommu *obj, char *buf, ssize_t bytes) +size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t bytes) { int i, num; struct cr_regs *cr; @@ -455,14 +446,14 @@ size_t dump_tlb_entries(struct iommu *obj, char *buf, ssize_t bytes) return p - buf; } -EXPORT_SYMBOL_GPL(dump_tlb_entries); +EXPORT_SYMBOL_GPL(omap_dump_tlb_entries); -int foreach_iommu_device(void *data, int (*fn)(struct device *, void *)) +int omap_foreach_iommu_device(void *data, int (*fn)(struct device *, void *)) { return driver_for_each_device(&omap_iommu_driver.driver, NULL, data, fn); } -EXPORT_SYMBOL_GPL(foreach_iommu_device); +EXPORT_SYMBOL_GPL(omap_foreach_iommu_device); #endif /* CONFIG_OMAP_IOMMU_DEBUG_MODULE */ @@ -495,7 +486,7 @@ static void iopte_free(u32 *iopte) kmem_cache_free(iopte_cachep, iopte); } -static u32 *iopte_alloc(struct iommu *obj, u32 *iopgd, u32 da) +static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da) { u32 *iopte; @@ -533,7 +524,7 @@ pte_ready: return iopte; } -static int iopgd_alloc_section(struct iommu *obj, u32 da, u32 pa, u32 prot) +static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) { u32 *iopgd = iopgd_offset(obj, da); @@ -548,7 +539,7 @@ static int iopgd_alloc_section(struct iommu *obj, u32 da, u32 pa, u32 prot) return 0; } -static int iopgd_alloc_super(struct iommu *obj, u32 da, u32 pa, u32 prot) +static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) { u32 *iopgd = iopgd_offset(obj, da); int i; @@ -565,7 +556,7 @@ static int iopgd_alloc_super(struct iommu *obj, u32 da, u32 pa, u32 prot) return 0; } -static int iopte_alloc_page(struct iommu *obj, u32 da, u32 pa, u32 prot) +static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) { u32 *iopgd = iopgd_offset(obj, da); u32 *iopte = iopte_alloc(obj, iopgd, da); @@ -582,7 +573,7 @@ static int iopte_alloc_page(struct iommu *obj, u32 da, u32 pa, u32 prot) return 0; } -static int iopte_alloc_large(struct iommu *obj, u32 da, u32 pa, u32 prot) +static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) { u32 *iopgd = iopgd_offset(obj, da); u32 *iopte = iopte_alloc(obj, iopgd, da); @@ -603,9 +594,10 @@ static int iopte_alloc_large(struct iommu *obj, u32 da, u32 pa, u32 prot) return 0; } -static int iopgtable_store_entry_core(struct iommu *obj, struct iotlb_entry *e) +static int +iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e) { - int (*fn)(struct iommu *, u32, u32, u32); + int (*fn)(struct omap_iommu *, u32, u32, u32); u32 prot; int err; @@ -641,23 +633,21 @@ static int iopgtable_store_entry_core(struct iommu *obj, struct iotlb_entry *e) } /** - * iopgtable_store_entry - Make an iommu pte entry + * omap_iopgtable_store_entry - Make an iommu pte entry * @obj: target iommu * @e: an iommu tlb entry info **/ -int iopgtable_store_entry(struct iommu *obj, struct iotlb_entry *e) +int omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e) { int err; flush_iotlb_page(obj, e->da); err = iopgtable_store_entry_core(obj, e); -#ifdef PREFETCH_IOTLB if (!err) - load_iotlb_entry(obj, e); -#endif + prefetch_iotlb_entry(obj, e); return err; } -EXPORT_SYMBOL_GPL(iopgtable_store_entry); +EXPORT_SYMBOL_GPL(omap_iopgtable_store_entry); /** * iopgtable_lookup_entry - Lookup an iommu pte entry @@ -666,7 +656,8 @@ EXPORT_SYMBOL_GPL(iopgtable_store_entry); * @ppgd: iommu pgd entry pointer to be returned * @ppte: iommu pte entry pointer to be returned **/ -void iopgtable_lookup_entry(struct iommu *obj, u32 da, u32 **ppgd, u32 **ppte) +static void +iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte) { u32 *iopgd, *iopte = NULL; @@ -680,9 +671,8 @@ out: *ppgd = iopgd; *ppte = iopte; } -EXPORT_SYMBOL_GPL(iopgtable_lookup_entry); -static size_t iopgtable_clear_entry_core(struct iommu *obj, u32 da) +static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da) { size_t bytes; u32 *iopgd = iopgd_offset(obj, da); @@ -735,7 +725,7 @@ out: * @obj: target iommu * @da: iommu device virtual address **/ -size_t iopgtable_clear_entry(struct iommu *obj, u32 da) +static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da) { size_t bytes; @@ -748,9 +738,8 @@ size_t iopgtable_clear_entry(struct iommu *obj, u32 da) return bytes; } -EXPORT_SYMBOL_GPL(iopgtable_clear_entry); -static void iopgtable_clear_entry_all(struct iommu *obj) +static void iopgtable_clear_entry_all(struct omap_iommu *obj) { int i; @@ -785,7 +774,8 @@ static irqreturn_t iommu_fault_handler(int irq, void *data) { u32 da, errs; u32 *iopgd, *iopte; - struct iommu *obj = data; + struct omap_iommu *obj = data; + struct iommu_domain *domain = obj->domain; if (!obj->refcount) return IRQ_NONE; @@ -797,7 +787,7 @@ static irqreturn_t iommu_fault_handler(int irq, void *data) return IRQ_HANDLED; /* Fault callback or TLB/PTE Dynamic loading */ - if (obj->isr && !obj->isr(obj, da, errs, obj->isr_priv)) + if (!report_iommu_fault(domain, obj->dev, da, 0)) return IRQ_HANDLED; iommu_disable(obj); @@ -821,7 +811,7 @@ static irqreturn_t iommu_fault_handler(int irq, void *data) static int device_match_by_alias(struct device *dev, void *data) { - struct iommu *obj = to_iommu(dev); + struct omap_iommu *obj = to_iommu(dev); const char *name = data; pr_debug("%s: %s %s\n", __func__, obj->name, name); @@ -830,57 +820,55 @@ static int device_match_by_alias(struct device *dev, void *data) } /** - * iommu_set_da_range - Set a valid device address range - * @obj: target iommu - * @start Start of valid range - * @end End of valid range - **/ -int iommu_set_da_range(struct iommu *obj, u32 start, u32 end) + * omap_find_iommu_device() - find an omap iommu device by name + * @name: name of the iommu device + * + * The generic iommu API requires the caller to provide the device + * he wishes to attach to a certain iommu domain. + * + * Drivers generally should not bother with this as it should just + * be taken care of by the DMA-API using dev_archdata. + * + * This function is provided as an interim solution until the latter + * materializes, and omap3isp is fully migrated to the DMA-API. + */ +struct device *omap_find_iommu_device(const char *name) { - - if (!obj) - return -EFAULT; - - if (end < start || !PAGE_ALIGN(start | end)) - return -EINVAL; - - obj->da_start = start; - obj->da_end = end; - - return 0; + return driver_find_device(&omap_iommu_driver.driver, NULL, + (void *)name, + device_match_by_alias); } -EXPORT_SYMBOL_GPL(iommu_set_da_range); +EXPORT_SYMBOL_GPL(omap_find_iommu_device); /** - * iommu_get - Get iommu handler - * @name: target iommu name + * omap_iommu_attach() - attach iommu device to an iommu domain + * @dev: target omap iommu device + * @iopgd: page table **/ -struct iommu *iommu_get(const char *name) +static struct omap_iommu *omap_iommu_attach(struct device *dev, u32 *iopgd) { int err = -ENOMEM; - struct device *dev; - struct iommu *obj; - - dev = driver_find_device(&omap_iommu_driver.driver, NULL, (void *)name, - device_match_by_alias); - if (!dev) - return ERR_PTR(-ENODEV); - - obj = to_iommu(dev); + struct omap_iommu *obj = to_iommu(dev); - mutex_lock(&obj->iommu_lock); + spin_lock(&obj->iommu_lock); - if (obj->refcount++ == 0) { - err = iommu_enable(obj); - if (err) - goto err_enable; - flush_iotlb_all(obj); + /* an iommu device can only be attached once */ + if (++obj->refcount > 1) { + dev_err(dev, "%s: already attached!\n", obj->name); + err = -EBUSY; + goto err_enable; } + obj->iopgd = iopgd; + err = iommu_enable(obj); + if (err) + goto err_enable; + flush_iotlb_all(obj); + if (!try_module_get(obj->owner)) goto err_module; - mutex_unlock(&obj->iommu_lock); + spin_unlock(&obj->iommu_lock); dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); return obj; @@ -890,59 +878,32 @@ err_module: iommu_disable(obj); err_enable: obj->refcount--; - mutex_unlock(&obj->iommu_lock); + spin_unlock(&obj->iommu_lock); return ERR_PTR(err); } -EXPORT_SYMBOL_GPL(iommu_get); /** - * iommu_put - Put back iommu handler + * omap_iommu_detach - release iommu device * @obj: target iommu **/ -void iommu_put(struct iommu *obj) +static void omap_iommu_detach(struct omap_iommu *obj) { if (!obj || IS_ERR(obj)) return; - mutex_lock(&obj->iommu_lock); + spin_lock(&obj->iommu_lock); if (--obj->refcount == 0) iommu_disable(obj); module_put(obj->owner); - mutex_unlock(&obj->iommu_lock); - - dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); -} -EXPORT_SYMBOL_GPL(iommu_put); - -int iommu_set_isr(const char *name, - int (*isr)(struct iommu *obj, u32 da, u32 iommu_errs, - void *priv), - void *isr_priv) -{ - struct device *dev; - struct iommu *obj; + obj->iopgd = NULL; - dev = driver_find_device(&omap_iommu_driver.driver, NULL, (void *)name, - device_match_by_alias); - if (!dev) - return -ENODEV; + spin_unlock(&obj->iommu_lock); - obj = to_iommu(dev); - mutex_lock(&obj->iommu_lock); - if (obj->refcount != 0) { - mutex_unlock(&obj->iommu_lock); - return -EBUSY; - } - obj->isr = isr; - obj->isr_priv = isr_priv; - mutex_unlock(&obj->iommu_lock); - - return 0; + dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); } -EXPORT_SYMBOL_GPL(iommu_set_isr); /* * OMAP Device MMU(IOMMU) detection @@ -950,9 +911,8 @@ EXPORT_SYMBOL_GPL(iommu_set_isr); static int __devinit omap_iommu_probe(struct platform_device *pdev) { int err = -ENODEV; - void *p; int irq; - struct iommu *obj; + struct omap_iommu *obj; struct resource *res; struct iommu_platform_data *pdata = pdev->dev.platform_data; @@ -974,7 +934,7 @@ static int __devinit omap_iommu_probe(struct platform_device *pdev) obj->da_start = pdata->da_start; obj->da_end = pdata->da_end; - mutex_init(&obj->iommu_lock); + spin_lock_init(&obj->iommu_lock); mutex_init(&obj->mmap_lock); spin_lock_init(&obj->page_table_lock); INIT_LIST_HEAD(&obj->mmap); @@ -1009,22 +969,9 @@ static int __devinit omap_iommu_probe(struct platform_device *pdev) goto err_irq; platform_set_drvdata(pdev, obj); - p = (void *)__get_free_pages(GFP_KERNEL, get_order(IOPGD_TABLE_SIZE)); - if (!p) { - err = -ENOMEM; - goto err_pgd; - } - memset(p, 0, IOPGD_TABLE_SIZE); - clean_dcache_area(p, IOPGD_TABLE_SIZE); - obj->iopgd = p; - - BUG_ON(!IS_ALIGNED((unsigned long)obj->iopgd, IOPGD_TABLE_SIZE)); - dev_info(&pdev->dev, "%s registered\n", obj->name); return 0; -err_pgd: - free_irq(irq, obj); err_irq: iounmap(obj->regbase); err_ioremap: @@ -1040,12 +987,11 @@ static int __devexit omap_iommu_remove(struct platform_device *pdev) { int irq; struct resource *res; - struct iommu *obj = platform_get_drvdata(pdev); + struct omap_iommu *obj = platform_get_drvdata(pdev); platform_set_drvdata(pdev, NULL); iopgtable_clear_entry_all(obj); - free_pages((unsigned long)obj->iopgd, get_order(IOPGD_TABLE_SIZE)); irq = platform_get_irq(pdev, 0); free_irq(irq, obj); @@ -1072,6 +1018,201 @@ static void iopte_cachep_ctor(void *iopte) clean_dcache_area(iopte, IOPTE_TABLE_SIZE); } +static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, + phys_addr_t pa, int order, int prot) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + struct omap_iommu *oiommu = omap_domain->iommu_dev; + struct device *dev = oiommu->dev; + size_t bytes = PAGE_SIZE << order; + struct iotlb_entry e; + int omap_pgsz; + u32 ret, flags; + + /* we only support mapping a single iommu page for now */ + omap_pgsz = bytes_to_iopgsz(bytes); + if (omap_pgsz < 0) { + dev_err(dev, "invalid size to map: %d\n", bytes); + return -EINVAL; + } + + dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes); + + flags = omap_pgsz | prot; + + iotlb_init_entry(&e, da, pa, flags); + + ret = omap_iopgtable_store_entry(oiommu, &e); + if (ret) + dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", ret); + + return ret; +} + +static int omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, + int order) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + struct omap_iommu *oiommu = omap_domain->iommu_dev; + struct device *dev = oiommu->dev; + size_t unmap_size; + + dev_dbg(dev, "unmapping da 0x%lx order %d\n", da, order); + + unmap_size = iopgtable_clear_entry(oiommu, da); + + return unmap_size ? get_order(unmap_size) : -EINVAL; +} + +static int +omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + struct omap_iommu *oiommu; + int ret = 0; + + spin_lock(&omap_domain->lock); + + /* only a single device is supported per domain for now */ + if (omap_domain->iommu_dev) { + dev_err(dev, "iommu domain is already attached\n"); + ret = -EBUSY; + goto out; + } + + /* get a handle to and enable the omap iommu */ + oiommu = omap_iommu_attach(dev, omap_domain->pgtable); + if (IS_ERR(oiommu)) { + ret = PTR_ERR(oiommu); + dev_err(dev, "can't get omap iommu: %d\n", ret); + goto out; + } + + omap_domain->iommu_dev = oiommu; + oiommu->domain = domain; + +out: + spin_unlock(&omap_domain->lock); + return ret; +} + +static void omap_iommu_detach_dev(struct iommu_domain *domain, + struct device *dev) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + struct omap_iommu *oiommu = to_iommu(dev); + + spin_lock(&omap_domain->lock); + + /* only a single device is supported per domain for now */ + if (omap_domain->iommu_dev != oiommu) { + dev_err(dev, "invalid iommu device\n"); + goto out; + } + + iopgtable_clear_entry_all(oiommu); + + omap_iommu_detach(oiommu); + + omap_domain->iommu_dev = NULL; + +out: + spin_unlock(&omap_domain->lock); +} + +static int omap_iommu_domain_init(struct iommu_domain *domain) +{ + struct omap_iommu_domain *omap_domain; + + omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL); + if (!omap_domain) { + pr_err("kzalloc failed\n"); + goto out; + } + + omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL); + if (!omap_domain->pgtable) { + pr_err("kzalloc failed\n"); + goto fail_nomem; + } + + /* + * should never fail, but please keep this around to ensure + * we keep the hardware happy + */ + BUG_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE)); + + clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE); + spin_lock_init(&omap_domain->lock); + + domain->priv = omap_domain; + + return 0; + +fail_nomem: + kfree(omap_domain); +out: + return -ENOMEM; +} + +/* assume device was already detached */ +static void omap_iommu_domain_destroy(struct iommu_domain *domain) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + + domain->priv = NULL; + + kfree(omap_domain->pgtable); + kfree(omap_domain); +} + +static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, + unsigned long da) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + struct omap_iommu *oiommu = omap_domain->iommu_dev; + struct device *dev = oiommu->dev; + u32 *pgd, *pte; + phys_addr_t ret = 0; + + iopgtable_lookup_entry(oiommu, da, &pgd, &pte); + + if (pte) { + if (iopte_is_small(*pte)) + ret = omap_iommu_translate(*pte, da, IOPTE_MASK); + else if (iopte_is_large(*pte)) + ret = omap_iommu_translate(*pte, da, IOLARGE_MASK); + else + dev_err(dev, "bogus pte 0x%x", *pte); + } else { + if (iopgd_is_section(*pgd)) + ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK); + else if (iopgd_is_super(*pgd)) + ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK); + else + dev_err(dev, "bogus pgd 0x%x", *pgd); + } + + return ret; +} + +static int omap_iommu_domain_has_cap(struct iommu_domain *domain, + unsigned long cap) +{ + return 0; +} + +static struct iommu_ops omap_iommu_ops = { + .domain_init = omap_iommu_domain_init, + .domain_destroy = omap_iommu_domain_destroy, + .attach_dev = omap_iommu_attach_dev, + .detach_dev = omap_iommu_detach_dev, + .map = omap_iommu_map, + .unmap = omap_iommu_unmap, + .iova_to_phys = omap_iommu_iova_to_phys, + .domain_has_cap = omap_iommu_domain_has_cap, +}; + static int __init omap_iommu_init(void) { struct kmem_cache *p; @@ -1084,6 +1225,8 @@ static int __init omap_iommu_init(void) return -ENOMEM; iopte_cachep = p; + bus_set_iommu(&platform_bus_type, &omap_iommu_ops); + return platform_driver_register(&omap_iommu_driver); } module_init(omap_iommu_init); diff --git a/arch/arm/plat-omap/iovmm.c b/drivers/iommu/omap-iovmm.c index 79e7fedb860..e8fdb8830f6 100644 --- a/arch/arm/plat-omap/iovmm.c +++ b/drivers/iommu/omap-iovmm.c @@ -15,6 +15,7 @@ #include <linux/vmalloc.h> #include <linux/device.h> #include <linux/scatterlist.h> +#include <linux/iommu.h> #include <asm/cacheflush.h> #include <asm/mach/map.h> @@ -22,44 +23,19 @@ #include <plat/iommu.h> #include <plat/iovmm.h> -#include "iopgtable.h" - -/* - * A device driver needs to create address mappings between: - * - * - iommu/device address - * - physical address - * - mpu virtual address - * - * There are 4 possible patterns for them: - * - * |iova/ mapping iommu_ page - * | da pa va (d)-(p)-(v) function type - * --------------------------------------------------------------------------- - * 1 | c c c 1 - 1 - 1 _kmap() / _kunmap() s - * 2 | c c,a c 1 - 1 - 1 _kmalloc()/ _kfree() s - * 3 | c d c 1 - n - 1 _vmap() / _vunmap() s - * 4 | c d,a c 1 - n - 1 _vmalloc()/ _vfree() n* - * - * - * 'iova': device iommu virtual address - * 'da': alias of 'iova' - * 'pa': physical address - * 'va': mpu virtual address - * - * 'c': contiguous memory area - * 'd': discontiguous memory area - * 'a': anonymous memory allocation - * '()': optional feature - * - * 'n': a normal page(4KB) size is used. - * 's': multiple iommu superpage(16MB, 1MB, 64KB, 4KB) size is used. - * - * '*': not yet, but feasible. - */ +#include <plat/iopgtable.h> static struct kmem_cache *iovm_area_cachep; +/* return the offset of the first scatterlist entry in a sg table */ +static unsigned int sgtable_offset(const struct sg_table *sgt) +{ + if (!sgt || !sgt->nents) + return 0; + + return sgt->sgl->offset; +} + /* return total bytes of sg buffers */ static size_t sgtable_len(const struct sg_table *sgt) { @@ -72,11 +48,17 @@ static size_t sgtable_len(const struct sg_table *sgt) for_each_sg(sgt->sgl, sg, sgt->nents, i) { size_t bytes; - bytes = sg->length; + bytes = sg->length + sg->offset; if (!iopgsz_ok(bytes)) { - pr_err("%s: sg[%d] not iommu pagesize(%x)\n", - __func__, i, bytes); + pr_err("%s: sg[%d] not iommu pagesize(%u %u)\n", + __func__, i, bytes, sg->offset); + return 0; + } + + if (i && sg->offset) { + pr_err("%s: sg[%d] offset not allowed in internal " + "entries\n", __func__, i); return 0; } @@ -197,8 +179,8 @@ static void *vmap_sg(const struct sg_table *sgt) u32 pa; int err; - pa = sg_phys(sg); - bytes = sg->length; + pa = sg_phys(sg) - sg->offset; + bytes = sg->length + sg->offset; BUG_ON(bytes != PAGE_SIZE); @@ -224,7 +206,8 @@ static inline void vunmap_sg(const void *va) vunmap(va); } -static struct iovm_struct *__find_iovm_area(struct iommu *obj, const u32 da) +static struct iovm_struct *__find_iovm_area(struct omap_iommu *obj, + const u32 da) { struct iovm_struct *tmp; @@ -246,12 +229,12 @@ static struct iovm_struct *__find_iovm_area(struct iommu *obj, const u32 da) } /** - * find_iovm_area - find iovma which includes @da + * omap_find_iovm_area - find iovma which includes @da * @da: iommu device virtual address * * Find the existing iovma starting at @da */ -struct iovm_struct *find_iovm_area(struct iommu *obj, u32 da) +struct iovm_struct *omap_find_iovm_area(struct omap_iommu *obj, u32 da) { struct iovm_struct *area; @@ -261,13 +244,13 @@ struct iovm_struct *find_iovm_area(struct iommu *obj, u32 da) return area; } -EXPORT_SYMBOL_GPL(find_iovm_area); +EXPORT_SYMBOL_GPL(omap_find_iovm_area); /* * This finds the hole(area) which fits the requested address and len * in iovmas mmap, and returns the new allocated iovma. */ -static struct iovm_struct *alloc_iovm_area(struct iommu *obj, u32 da, +static struct iovm_struct *alloc_iovm_area(struct omap_iommu *obj, u32 da, size_t bytes, u32 flags) { struct iovm_struct *new, *tmp; @@ -342,7 +325,7 @@ found: return new; } -static void free_iovm_area(struct iommu *obj, struct iovm_struct *area) +static void free_iovm_area(struct omap_iommu *obj, struct iovm_struct *area) { size_t bytes; @@ -358,14 +341,14 @@ static void free_iovm_area(struct iommu *obj, struct iovm_struct *area) } /** - * da_to_va - convert (d) to (v) + * omap_da_to_va - convert (d) to (v) * @obj: objective iommu * @da: iommu device virtual address * @va: mpu virtual address * * Returns mpu virtual addr which corresponds to a given device virtual addr */ -void *da_to_va(struct iommu *obj, u32 da) +void *omap_da_to_va(struct omap_iommu *obj, u32 da) { void *va = NULL; struct iovm_struct *area; @@ -383,7 +366,7 @@ out: return va; } -EXPORT_SYMBOL_GPL(da_to_va); +EXPORT_SYMBOL_GPL(omap_da_to_va); static void sgtable_fill_vmalloc(struct sg_table *sgt, void *_va) { @@ -397,7 +380,7 @@ static void sgtable_fill_vmalloc(struct sg_table *sgt, void *_va) const size_t bytes = PAGE_SIZE; /* - * iommu 'superpage' isn't supported with 'iommu_vmalloc()' + * iommu 'superpage' isn't supported with 'omap_iommu_vmalloc()' */ pg = vmalloc_to_page(va); BUG_ON(!pg); @@ -418,74 +401,39 @@ static inline void sgtable_drain_vmalloc(struct sg_table *sgt) BUG_ON(!sgt); } -static void sgtable_fill_kmalloc(struct sg_table *sgt, u32 pa, u32 da, - size_t len) -{ - unsigned int i; - struct scatterlist *sg; - - for_each_sg(sgt->sgl, sg, sgt->nents, i) { - unsigned bytes; - - bytes = max_alignment(da | pa); - bytes = min_t(unsigned, bytes, iopgsz_max(len)); - - BUG_ON(!iopgsz_ok(bytes)); - - sg_set_buf(sg, phys_to_virt(pa), bytes); - /* - * 'pa' is cotinuous(linear). - */ - pa += bytes; - da += bytes; - len -= bytes; - } - BUG_ON(len); -} - -static inline void sgtable_drain_kmalloc(struct sg_table *sgt) -{ - /* - * Actually this is not necessary at all, just exists for - * consistency of the code readability - */ - BUG_ON(!sgt); -} - /* create 'da' <-> 'pa' mapping from 'sgt' */ -static int map_iovm_area(struct iommu *obj, struct iovm_struct *new, - const struct sg_table *sgt, u32 flags) +static int map_iovm_area(struct iommu_domain *domain, struct iovm_struct *new, + const struct sg_table *sgt, u32 flags) { int err; unsigned int i, j; struct scatterlist *sg; u32 da = new->da_start; + int order; - if (!obj || !sgt) + if (!domain || !sgt) return -EINVAL; BUG_ON(!sgtable_ok(sgt)); for_each_sg(sgt->sgl, sg, sgt->nents, i) { u32 pa; - int pgsz; size_t bytes; - struct iotlb_entry e; - pa = sg_phys(sg); - bytes = sg->length; + pa = sg_phys(sg) - sg->offset; + bytes = sg->length + sg->offset; flags &= ~IOVMF_PGSZ_MASK; - pgsz = bytes_to_iopgsz(bytes); - if (pgsz < 0) + + if (bytes_to_iopgsz(bytes) < 0) goto err_out; - flags |= pgsz; + + order = get_order(bytes); pr_debug("%s: [%d] %08x %08x(%x)\n", __func__, i, da, pa, bytes); - iotlb_init_entry(&e, da, pa, flags); - err = iopgtable_store_entry(obj, &e); + err = iommu_map(domain, da, pa, order, flags); if (err) goto err_out; @@ -499,9 +447,11 @@ err_out: for_each_sg(sgt->sgl, sg, i, j) { size_t bytes; - bytes = iopgtable_clear_entry(obj, da); + bytes = sg->length + sg->offset; + order = get_order(bytes); - BUG_ON(!iopgsz_ok(bytes)); + /* ignore failures.. we're already handling one */ + iommu_unmap(domain, da, order); da += bytes; } @@ -509,22 +459,31 @@ err_out: } /* release 'da' <-> 'pa' mapping */ -static void unmap_iovm_area(struct iommu *obj, struct iovm_struct *area) +static void unmap_iovm_area(struct iommu_domain *domain, struct omap_iommu *obj, + struct iovm_struct *area) { u32 start; size_t total = area->da_end - area->da_start; + const struct sg_table *sgt = area->sgt; + struct scatterlist *sg; + int i, err; + BUG_ON(!sgtable_ok(sgt)); BUG_ON((!total) || !IS_ALIGNED(total, PAGE_SIZE)); start = area->da_start; - while (total > 0) { + for_each_sg(sgt->sgl, sg, sgt->nents, i) { size_t bytes; + int order; + + bytes = sg->length + sg->offset; + order = get_order(bytes); + + err = iommu_unmap(domain, start, order); + if (err < 0) + break; - bytes = iopgtable_clear_entry(obj, start); - if (bytes == 0) - bytes = PAGE_SIZE; - else - dev_dbg(obj->dev, "%s: unmap %08x(%x) %08x\n", + dev_dbg(obj->dev, "%s: unmap %08x(%x) %08x\n", __func__, start, bytes, area->flags); BUG_ON(!IS_ALIGNED(bytes, PAGE_SIZE)); @@ -536,7 +495,8 @@ static void unmap_iovm_area(struct iommu *obj, struct iovm_struct *area) } /* template function for all unmapping */ -static struct sg_table *unmap_vm_area(struct iommu *obj, const u32 da, +static struct sg_table *unmap_vm_area(struct iommu_domain *domain, + struct omap_iommu *obj, const u32 da, void (*fn)(const void *), u32 flags) { struct sg_table *sgt = NULL; @@ -562,7 +522,7 @@ static struct sg_table *unmap_vm_area(struct iommu *obj, const u32 da, } sgt = (struct sg_table *)area->sgt; - unmap_iovm_area(obj, area); + unmap_iovm_area(domain, obj, area); fn(area->va); @@ -577,8 +537,9 @@ out: return sgt; } -static u32 map_iommu_region(struct iommu *obj, u32 da, - const struct sg_table *sgt, void *va, size_t bytes, u32 flags) +static u32 map_iommu_region(struct iommu_domain *domain, struct omap_iommu *obj, + u32 da, const struct sg_table *sgt, void *va, + size_t bytes, u32 flags) { int err = -ENOMEM; struct iovm_struct *new; @@ -593,7 +554,7 @@ static u32 map_iommu_region(struct iommu *obj, u32 da, new->va = va; new->sgt = sgt; - if (map_iovm_area(obj, new, sgt, new->flags)) + if (map_iovm_area(domain, new, sgt, new->flags)) goto err_map; mutex_unlock(&obj->mmap_lock); @@ -610,14 +571,16 @@ err_alloc_iovma: return err; } -static inline u32 __iommu_vmap(struct iommu *obj, u32 da, - const struct sg_table *sgt, void *va, size_t bytes, u32 flags) +static inline u32 +__iommu_vmap(struct iommu_domain *domain, struct omap_iommu *obj, + u32 da, const struct sg_table *sgt, + void *va, size_t bytes, u32 flags) { - return map_iommu_region(obj, da, sgt, va, bytes, flags); + return map_iommu_region(domain, obj, da, sgt, va, bytes, flags); } /** - * iommu_vmap - (d)-(p)-(v) address mapper + * omap_iommu_vmap - (d)-(p)-(v) address mapper * @obj: objective iommu * @sgt: address of scatter gather table * @flags: iovma and page property @@ -625,8 +588,8 @@ static inline u32 __iommu_vmap(struct iommu *obj, u32 da, * Creates 1-n-1 mapping with given @sgt and returns @da. * All @sgt element must be io page size aligned. */ -u32 iommu_vmap(struct iommu *obj, u32 da, const struct sg_table *sgt, - u32 flags) +u32 omap_iommu_vmap(struct iommu_domain *domain, struct omap_iommu *obj, u32 da, + const struct sg_table *sgt, u32 flags) { size_t bytes; void *va = NULL; @@ -648,38 +611,41 @@ u32 iommu_vmap(struct iommu *obj, u32 da, const struct sg_table *sgt, flags |= IOVMF_DISCONT; flags |= IOVMF_MMIO; - da = __iommu_vmap(obj, da, sgt, va, bytes, flags); + da = __iommu_vmap(domain, obj, da, sgt, va, bytes, flags); if (IS_ERR_VALUE(da)) vunmap_sg(va); - return da; + return da + sgtable_offset(sgt); } -EXPORT_SYMBOL_GPL(iommu_vmap); +EXPORT_SYMBOL_GPL(omap_iommu_vmap); /** - * iommu_vunmap - release virtual mapping obtained by 'iommu_vmap()' + * omap_iommu_vunmap - release virtual mapping obtained by 'omap_iommu_vmap()' * @obj: objective iommu * @da: iommu device virtual address * * Free the iommu virtually contiguous memory area starting at - * @da, which was returned by 'iommu_vmap()'. + * @da, which was returned by 'omap_iommu_vmap()'. */ -struct sg_table *iommu_vunmap(struct iommu *obj, u32 da) +struct sg_table * +omap_iommu_vunmap(struct iommu_domain *domain, struct omap_iommu *obj, u32 da) { struct sg_table *sgt; /* - * 'sgt' is allocated before 'iommu_vmalloc()' is called. + * 'sgt' is allocated before 'omap_iommu_vmalloc()' is called. * Just returns 'sgt' to the caller to free */ - sgt = unmap_vm_area(obj, da, vunmap_sg, IOVMF_DISCONT | IOVMF_MMIO); + da &= PAGE_MASK; + sgt = unmap_vm_area(domain, obj, da, vunmap_sg, + IOVMF_DISCONT | IOVMF_MMIO); if (!sgt) dev_dbg(obj->dev, "%s: No sgt\n", __func__); return sgt; } -EXPORT_SYMBOL_GPL(iommu_vunmap); +EXPORT_SYMBOL_GPL(omap_iommu_vunmap); /** - * iommu_vmalloc - (d)-(p)-(v) address allocator and mapper + * omap_iommu_vmalloc - (d)-(p)-(v) address allocator and mapper * @obj: objective iommu * @da: contiguous iommu virtual memory * @bytes: allocation size @@ -688,7 +654,9 @@ EXPORT_SYMBOL_GPL(iommu_vunmap); * Allocate @bytes linearly and creates 1-n-1 mapping and returns * @da again, which might be adjusted if 'IOVMF_DA_FIXED' is not set. */ -u32 iommu_vmalloc(struct iommu *obj, u32 da, size_t bytes, u32 flags) +u32 +omap_iommu_vmalloc(struct iommu_domain *domain, struct omap_iommu *obj, u32 da, + size_t bytes, u32 flags) { void *va; struct sg_table *sgt; @@ -712,7 +680,7 @@ u32 iommu_vmalloc(struct iommu *obj, u32 da, size_t bytes, u32 flags) } sgtable_fill_vmalloc(sgt, va); - da = __iommu_vmap(obj, da, sgt, va, bytes, flags); + da = __iommu_vmap(domain, obj, da, sgt, va, bytes, flags); if (IS_ERR_VALUE(da)) goto err_iommu_vmap; @@ -725,158 +693,28 @@ err_sgt_alloc: vfree(va); return da; } -EXPORT_SYMBOL_GPL(iommu_vmalloc); +EXPORT_SYMBOL_GPL(omap_iommu_vmalloc); /** - * iommu_vfree - release memory allocated by 'iommu_vmalloc()' + * omap_iommu_vfree - release memory allocated by 'omap_iommu_vmalloc()' * @obj: objective iommu * @da: iommu device virtual address * * Frees the iommu virtually continuous memory area starting at - * @da, as obtained from 'iommu_vmalloc()'. + * @da, as obtained from 'omap_iommu_vmalloc()'. */ -void iommu_vfree(struct iommu *obj, const u32 da) +void omap_iommu_vfree(struct iommu_domain *domain, struct omap_iommu *obj, + const u32 da) { struct sg_table *sgt; - sgt = unmap_vm_area(obj, da, vfree, IOVMF_DISCONT | IOVMF_ALLOC); + sgt = unmap_vm_area(domain, obj, da, vfree, + IOVMF_DISCONT | IOVMF_ALLOC); if (!sgt) dev_dbg(obj->dev, "%s: No sgt\n", __func__); sgtable_free(sgt); } -EXPORT_SYMBOL_GPL(iommu_vfree); - -static u32 __iommu_kmap(struct iommu *obj, u32 da, u32 pa, void *va, - size_t bytes, u32 flags) -{ - struct sg_table *sgt; - - sgt = sgtable_alloc(bytes, flags, da, pa); - if (IS_ERR(sgt)) - return PTR_ERR(sgt); - - sgtable_fill_kmalloc(sgt, pa, da, bytes); - - da = map_iommu_region(obj, da, sgt, va, bytes, flags); - if (IS_ERR_VALUE(da)) { - sgtable_drain_kmalloc(sgt); - sgtable_free(sgt); - } - - return da; -} - -/** - * iommu_kmap - (d)-(p)-(v) address mapper - * @obj: objective iommu - * @da: contiguous iommu virtual memory - * @pa: contiguous physical memory - * @flags: iovma and page property - * - * Creates 1-1-1 mapping and returns @da again, which can be - * adjusted if 'IOVMF_DA_FIXED' is not set. - */ -u32 iommu_kmap(struct iommu *obj, u32 da, u32 pa, size_t bytes, - u32 flags) -{ - void *va; - - if (!obj || !obj->dev || !bytes) - return -EINVAL; - - bytes = PAGE_ALIGN(bytes); - - va = ioremap(pa, bytes); - if (!va) - return -ENOMEM; - - flags |= IOVMF_LINEAR; - flags |= IOVMF_MMIO; - - da = __iommu_kmap(obj, da, pa, va, bytes, flags); - if (IS_ERR_VALUE(da)) - iounmap(va); - - return da; -} -EXPORT_SYMBOL_GPL(iommu_kmap); - -/** - * iommu_kunmap - release virtual mapping obtained by 'iommu_kmap()' - * @obj: objective iommu - * @da: iommu device virtual address - * - * Frees the iommu virtually contiguous memory area starting at - * @da, which was passed to and was returned by'iommu_kmap()'. - */ -void iommu_kunmap(struct iommu *obj, u32 da) -{ - struct sg_table *sgt; - typedef void (*func_t)(const void *); - - sgt = unmap_vm_area(obj, da, (func_t)iounmap, - IOVMF_LINEAR | IOVMF_MMIO); - if (!sgt) - dev_dbg(obj->dev, "%s: No sgt\n", __func__); - sgtable_free(sgt); -} -EXPORT_SYMBOL_GPL(iommu_kunmap); - -/** - * iommu_kmalloc - (d)-(p)-(v) address allocator and mapper - * @obj: objective iommu - * @da: contiguous iommu virtual memory - * @bytes: bytes for allocation - * @flags: iovma and page property - * - * Allocate @bytes linearly and creates 1-1-1 mapping and returns - * @da again, which might be adjusted if 'IOVMF_DA_FIXED' is not set. - */ -u32 iommu_kmalloc(struct iommu *obj, u32 da, size_t bytes, u32 flags) -{ - void *va; - u32 pa; - - if (!obj || !obj->dev || !bytes) - return -EINVAL; - - bytes = PAGE_ALIGN(bytes); - - va = kmalloc(bytes, GFP_KERNEL | GFP_DMA); - if (!va) - return -ENOMEM; - pa = virt_to_phys(va); - - flags |= IOVMF_LINEAR; - flags |= IOVMF_ALLOC; - - da = __iommu_kmap(obj, da, pa, va, bytes, flags); - if (IS_ERR_VALUE(da)) - kfree(va); - - return da; -} -EXPORT_SYMBOL_GPL(iommu_kmalloc); - -/** - * iommu_kfree - release virtual mapping obtained by 'iommu_kmalloc()' - * @obj: objective iommu - * @da: iommu device virtual address - * - * Frees the iommu virtually contiguous memory area starting at - * @da, which was passed to and was returned by'iommu_kmalloc()'. - */ -void iommu_kfree(struct iommu *obj, u32 da) -{ - struct sg_table *sgt; - - sgt = unmap_vm_area(obj, da, kfree, IOVMF_LINEAR | IOVMF_ALLOC); - if (!sgt) - dev_dbg(obj->dev, "%s: No sgt\n", __func__); - sgtable_free(sgt); -} -EXPORT_SYMBOL_GPL(iommu_kfree); - +EXPORT_SYMBOL_GPL(omap_iommu_vfree); static int __init iovmm_init(void) { diff --git a/drivers/media/video/Kconfig b/drivers/media/video/Kconfig index f574dc012ca..620106937ec 100644 --- a/drivers/media/video/Kconfig +++ b/drivers/media/video/Kconfig @@ -763,8 +763,7 @@ source "drivers/media/video/m5mols/Kconfig" config VIDEO_OMAP3 tristate "OMAP 3 Camera support (EXPERIMENTAL)" - select OMAP_IOMMU - depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API && ARCH_OMAP3 && EXPERIMENTAL + depends on OMAP_IOVMM && VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API && ARCH_OMAP3 && EXPERIMENTAL ---help--- Driver for an OMAP 3 camera controller. diff --git a/drivers/media/video/omap3isp/isp.c b/drivers/media/video/omap3isp/isp.c index 5cea2bbd701..a7ed9859688 100644 --- a/drivers/media/video/omap3isp/isp.c +++ b/drivers/media/video/omap3isp/isp.c @@ -80,6 +80,13 @@ #include "isph3a.h" #include "isphist.h" +/* + * this is provided as an interim solution until omap3isp doesn't need + * any omap-specific iommu API + */ +#define to_iommu(dev) \ + (struct omap_iommu *)platform_get_drvdata(to_platform_device(dev)) + static unsigned int autoidle; module_param(autoidle, int, 0444); MODULE_PARM_DESC(autoidle, "Enable OMAP3ISP AUTOIDLE support"); @@ -1108,7 +1115,7 @@ static void isp_save_ctx(struct isp_device *isp) { isp_save_context(isp, isp_reg_list); if (isp->iommu) - iommu_save_ctx(isp->iommu); + omap_iommu_save_ctx(isp->iommu); } /* @@ -1122,7 +1129,7 @@ static void isp_restore_ctx(struct isp_device *isp) { isp_restore_context(isp, isp_reg_list); if (isp->iommu) - iommu_restore_ctx(isp->iommu); + omap_iommu_restore_ctx(isp->iommu); omap3isp_ccdc_restore_context(isp); omap3isp_preview_restore_context(isp); } @@ -1975,7 +1982,8 @@ static int isp_remove(struct platform_device *pdev) isp_cleanup_modules(isp); omap3isp_get(isp); - iommu_put(isp->iommu); + iommu_detach_device(isp->domain, isp->iommu_dev); + iommu_domain_free(isp->domain); omap3isp_put(isp); free_irq(isp->irq_num, isp); @@ -2123,25 +2131,41 @@ static int isp_probe(struct platform_device *pdev) } /* IOMMU */ - isp->iommu = iommu_get("isp"); - if (IS_ERR_OR_NULL(isp->iommu)) { - isp->iommu = NULL; + isp->iommu_dev = omap_find_iommu_device("isp"); + if (!isp->iommu_dev) { + dev_err(isp->dev, "omap_find_iommu_device failed\n"); ret = -ENODEV; goto error_isp; } + /* to be removed once iommu migration is complete */ + isp->iommu = to_iommu(isp->iommu_dev); + + isp->domain = iommu_domain_alloc(pdev->dev.bus); + if (!isp->domain) { + dev_err(isp->dev, "can't alloc iommu domain\n"); + ret = -ENOMEM; + goto error_isp; + } + + ret = iommu_attach_device(isp->domain, isp->iommu_dev); + if (ret) { + dev_err(&pdev->dev, "can't attach iommu device: %d\n", ret); + goto free_domain; + } + /* Interrupt */ isp->irq_num = platform_get_irq(pdev, 0); if (isp->irq_num <= 0) { dev_err(isp->dev, "No IRQ resource\n"); ret = -ENODEV; - goto error_isp; + goto detach_dev; } if (request_irq(isp->irq_num, isp_isr, IRQF_SHARED, "OMAP3 ISP", isp)) { dev_err(isp->dev, "Unable to request IRQ\n"); ret = -EINVAL; - goto error_isp; + goto detach_dev; } /* Entities */ @@ -2162,8 +2186,11 @@ error_modules: isp_cleanup_modules(isp); error_irq: free_irq(isp->irq_num, isp); +detach_dev: + iommu_detach_device(isp->domain, isp->iommu_dev); +free_domain: + iommu_domain_free(isp->domain); error_isp: - iommu_put(isp->iommu); omap3isp_put(isp); error: isp_put_clocks(isp); diff --git a/drivers/media/video/omap3isp/isp.h b/drivers/media/video/omap3isp/isp.h index 529e582ef94..81fdd85deb6 100644 --- a/drivers/media/video/omap3isp/isp.h +++ b/drivers/media/video/omap3isp/isp.h @@ -32,6 +32,7 @@ #include <linux/io.h> #include <linux/platform_device.h> #include <linux/wait.h> +#include <linux/iommu.h> #include <plat/iommu.h> #include <plat/iovmm.h> @@ -294,7 +295,9 @@ struct isp_device { unsigned int sbl_resources; unsigned int subclk_resources; - struct iommu *iommu; + struct omap_iommu *iommu; + struct iommu_domain *domain; + struct device *iommu_dev; struct isp_platform_callback platform_cb; }; diff --git a/drivers/media/video/omap3isp/ispccdc.c b/drivers/media/video/omap3isp/ispccdc.c index 80796eb0c53..892671922f8 100644 --- a/drivers/media/video/omap3isp/ispccdc.c +++ b/drivers/media/video/omap3isp/ispccdc.c @@ -366,7 +366,7 @@ static void ccdc_lsc_free_request(struct isp_ccdc_device *ccdc, dma_unmap_sg(isp->dev, req->iovm->sgt->sgl, req->iovm->sgt->nents, DMA_TO_DEVICE); if (req->table) - iommu_vfree(isp->iommu, req->table); + omap_iommu_vfree(isp->domain, isp->iommu, req->table); kfree(req); } @@ -438,15 +438,15 @@ static int ccdc_lsc_config(struct isp_ccdc_device *ccdc, req->enable = 1; - req->table = iommu_vmalloc(isp->iommu, 0, req->config.size, - IOMMU_FLAG); + req->table = omap_iommu_vmalloc(isp->domain, isp->iommu, 0, + req->config.size, IOMMU_FLAG); if (IS_ERR_VALUE(req->table)) { req->table = 0; ret = -ENOMEM; goto done; } - req->iovm = find_iovm_area(isp->iommu, req->table); + req->iovm = omap_find_iovm_area(isp->iommu, req->table); if (req->iovm == NULL) { ret = -ENOMEM; goto done; @@ -462,7 +462,7 @@ static int ccdc_lsc_config(struct isp_ccdc_device *ccdc, dma_sync_sg_for_cpu(isp->dev, req->iovm->sgt->sgl, req->iovm->sgt->nents, DMA_TO_DEVICE); - table = da_to_va(isp->iommu, req->table); + table = omap_da_to_va(isp->iommu, req->table); if (copy_from_user(table, config->lsc, req->config.size)) { ret = -EFAULT; goto done; @@ -731,18 +731,19 @@ static int ccdc_config(struct isp_ccdc_device *ccdc, /* * table_new must be 64-bytes aligned, but it's - * already done by iommu_vmalloc(). + * already done by omap_iommu_vmalloc(). */ size = ccdc->fpc.fpnum * 4; - table_new = iommu_vmalloc(isp->iommu, 0, size, - IOMMU_FLAG); + table_new = omap_iommu_vmalloc(isp->domain, isp->iommu, + 0, size, IOMMU_FLAG); if (IS_ERR_VALUE(table_new)) return -ENOMEM; - if (copy_from_user(da_to_va(isp->iommu, table_new), + if (copy_from_user(omap_da_to_va(isp->iommu, table_new), (__force void __user *) ccdc->fpc.fpcaddr, size)) { - iommu_vfree(isp->iommu, table_new); + omap_iommu_vfree(isp->domain, isp->iommu, + table_new); return -EFAULT; } @@ -752,7 +753,7 @@ static int ccdc_config(struct isp_ccdc_device *ccdc, ccdc_configure_fpc(ccdc); if (table_old != 0) - iommu_vfree(isp->iommu, table_old); + omap_iommu_vfree(isp->domain, isp->iommu, table_old); } return ccdc_lsc_config(ccdc, ccdc_struct); @@ -2287,5 +2288,5 @@ void omap3isp_ccdc_cleanup(struct isp_device *isp) ccdc_lsc_free_queue(ccdc, &ccdc->lsc.free_queue); if (ccdc->fpc.fpcaddr != 0) - iommu_vfree(isp->iommu, ccdc->fpc.fpcaddr); + omap_iommu_vfree(isp->domain, isp->iommu, ccdc->fpc.fpcaddr); } diff --git a/drivers/media/video/omap3isp/ispstat.c b/drivers/media/video/omap3isp/ispstat.c index 808065948ac..73290555226 100644 --- a/drivers/media/video/omap3isp/ispstat.c +++ b/drivers/media/video/omap3isp/ispstat.c @@ -366,7 +366,8 @@ static void isp_stat_bufs_free(struct ispstat *stat) dma_unmap_sg(isp->dev, buf->iovm->sgt->sgl, buf->iovm->sgt->nents, DMA_FROM_DEVICE); - iommu_vfree(isp->iommu, buf->iommu_addr); + omap_iommu_vfree(isp->domain, isp->iommu, + buf->iommu_addr); } else { if (!buf->virt_addr) continue; @@ -399,8 +400,8 @@ static int isp_stat_bufs_alloc_iommu(struct ispstat *stat, unsigned int size) struct iovm_struct *iovm; WARN_ON(buf->dma_addr); - buf->iommu_addr = iommu_vmalloc(isp->iommu, 0, size, - IOMMU_FLAG); + buf->iommu_addr = omap_iommu_vmalloc(isp->domain, isp->iommu, 0, + size, IOMMU_FLAG); if (IS_ERR((void *)buf->iommu_addr)) { dev_err(stat->isp->dev, "%s: Can't acquire memory for " @@ -409,7 +410,7 @@ static int isp_stat_bufs_alloc_iommu(struct ispstat *stat, unsigned int size) return -ENOMEM; } - iovm = find_iovm_area(isp->iommu, buf->iommu_addr); + iovm = omap_find_iovm_area(isp->iommu, buf->iommu_addr); if (!iovm || !dma_map_sg(isp->dev, iovm->sgt->sgl, iovm->sgt->nents, DMA_FROM_DEVICE)) { @@ -418,7 +419,7 @@ static int isp_stat_bufs_alloc_iommu(struct ispstat *stat, unsigned int size) } buf->iovm = iovm; - buf->virt_addr = da_to_va(stat->isp->iommu, + buf->virt_addr = omap_da_to_va(stat->isp->iommu, (u32)buf->iommu_addr); buf->empty = 1; dev_dbg(stat->isp->dev, "%s: buffer[%d] allocated." diff --git a/drivers/media/video/omap3isp/ispvideo.c b/drivers/media/video/omap3isp/ispvideo.c index fd965adfd59..912ac071b10 100644 --- a/drivers/media/video/omap3isp/ispvideo.c +++ b/drivers/media/video/omap3isp/ispvideo.c @@ -446,7 +446,7 @@ ispmmu_vmap(struct isp_device *isp, const struct scatterlist *sglist, int sglen) sgt->nents = sglen; sgt->orig_nents = sglen; - da = iommu_vmap(isp->iommu, 0, sgt, IOMMU_FLAG); + da = omap_iommu_vmap(isp->domain, isp->iommu, 0, sgt, IOMMU_FLAG); if (IS_ERR_VALUE(da)) kfree(sgt); @@ -462,7 +462,7 @@ static void ispmmu_vunmap(struct isp_device *isp, dma_addr_t da) { struct sg_table *sgt; - sgt = iommu_vunmap(isp->iommu, (u32)da); + sgt = omap_iommu_vunmap(isp->domain, isp->iommu, (u32)da); kfree(sgt); } diff --git a/include/linux/device.h b/include/linux/device.h index bdcf361ca93..85e78fc7d7f 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -33,6 +33,7 @@ struct class; struct subsys_private; struct bus_type; struct device_node; +struct iommu_ops; struct bus_attribute { struct attribute attr; @@ -67,6 +68,9 @@ extern void bus_remove_file(struct bus_type *, struct bus_attribute *); * @resume: Called to bring a device on this bus out of sleep mode. * @pm: Power management operations of this bus, callback the specific * device driver's pm-ops. + * @iommu_ops IOMMU specific operations for this bus, used to attach IOMMU + * driver implementations to a bus and allow the driver to do + * bus-specific setup * @p: The private data of the driver core, only the driver core can * touch this. * @@ -96,6 +100,8 @@ struct bus_type { const struct dev_pm_ops *pm; + struct iommu_ops *iommu_ops; + struct subsys_private *p; }; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 9940319d6f9..432acc4c054 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -25,15 +25,29 @@ #define IOMMU_WRITE (2) #define IOMMU_CACHE (4) /* DMA cache coherency */ +struct iommu_ops; +struct bus_type; struct device; +struct iommu_domain; + +/* iommu fault flags */ +#define IOMMU_FAULT_READ 0x0 +#define IOMMU_FAULT_WRITE 0x1 + +typedef int (*iommu_fault_handler_t)(struct iommu_domain *, + struct device *, unsigned long, int); struct iommu_domain { + struct iommu_ops *ops; void *priv; + iommu_fault_handler_t handler; }; #define IOMMU_CAP_CACHE_COHERENCY 0x1 #define IOMMU_CAP_INTR_REMAP 0x2 /* isolates device intrs */ +#ifdef CONFIG_IOMMU_API + struct iommu_ops { int (*domain_init)(struct iommu_domain *domain); void (*domain_destroy)(struct iommu_domain *domain); @@ -49,11 +63,9 @@ struct iommu_ops { unsigned long cap); }; -#ifdef CONFIG_IOMMU_API - -extern void register_iommu(struct iommu_ops *ops); -extern bool iommu_found(void); -extern struct iommu_domain *iommu_domain_alloc(void); +extern int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops); +extern bool iommu_present(struct bus_type *bus); +extern struct iommu_domain *iommu_domain_alloc(struct bus_type *bus); extern void iommu_domain_free(struct iommu_domain *domain); extern int iommu_attach_device(struct iommu_domain *domain, struct device *dev); @@ -67,19 +79,58 @@ extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, unsigned long iova); extern int iommu_domain_has_cap(struct iommu_domain *domain, unsigned long cap); +extern void iommu_set_fault_handler(struct iommu_domain *domain, + iommu_fault_handler_t handler); + +/** + * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework + * @domain: the iommu domain where the fault has happened + * @dev: the device where the fault has happened + * @iova: the faulting address + * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...) + * + * This function should be called by the low-level IOMMU implementations + * whenever IOMMU faults happen, to allow high-level users, that are + * interested in such events, to know about them. + * + * This event may be useful for several possible use cases: + * - mere logging of the event + * - dynamic TLB/PTE loading + * - if restarting of the faulting device is required + * + * Returns 0 on success and an appropriate error code otherwise (if dynamic + * PTE/TLB loading will one day be supported, implementations will be able + * to tell whether it succeeded or not according to this return value). + * + * Specifically, -ENOSYS is returned if a fault handler isn't installed + * (though fault handlers can also return -ENOSYS, in case they want to + * elicit the default behavior of the IOMMU drivers). + */ +static inline int report_iommu_fault(struct iommu_domain *domain, + struct device *dev, unsigned long iova, int flags) +{ + int ret = -ENOSYS; -#else /* CONFIG_IOMMU_API */ + /* + * if upper layers showed interest and installed a fault handler, + * invoke it. + */ + if (domain->handler) + ret = domain->handler(domain, dev, iova, flags); -static inline void register_iommu(struct iommu_ops *ops) -{ + return ret; } -static inline bool iommu_found(void) +#else /* CONFIG_IOMMU_API */ + +struct iommu_ops {}; + +static inline bool iommu_present(struct bus_type *bus) { return false; } -static inline struct iommu_domain *iommu_domain_alloc(void) +static inline struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) { return NULL; } @@ -123,6 +174,11 @@ static inline int domain_has_cap(struct iommu_domain *domain, return 0; } +static inline void iommu_set_fault_handler(struct iommu_domain *domain, + iommu_fault_handler_t handler) +{ +} + #endif /* CONFIG_IOMMU_API */ #endif /* __LINUX_IOMMU_H */ diff --git a/lib/dma-debug.c b/lib/dma-debug.c index db07bfd9298..79700fa2dfc 100644 --- a/lib/dma-debug.c +++ b/lib/dma-debug.c @@ -62,6 +62,8 @@ struct dma_debug_entry { #endif }; +typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *); + struct hash_bucket { struct list_head list; spinlock_t lock; @@ -240,18 +242,37 @@ static void put_hash_bucket(struct hash_bucket *bucket, spin_unlock_irqrestore(&bucket->lock, __flags); } +static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b) +{ + return ((a->dev_addr == a->dev_addr) && + (a->dev == b->dev)) ? true : false; +} + +static bool containing_match(struct dma_debug_entry *a, + struct dma_debug_entry *b) +{ + if (a->dev != b->dev) + return false; + + if ((b->dev_addr <= a->dev_addr) && + ((b->dev_addr + b->size) >= (a->dev_addr + a->size))) + return true; + + return false; +} + /* * Search a given entry in the hash bucket list */ -static struct dma_debug_entry *hash_bucket_find(struct hash_bucket *bucket, - struct dma_debug_entry *ref) +static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket, + struct dma_debug_entry *ref, + match_fn match) { struct dma_debug_entry *entry, *ret = NULL; int matches = 0, match_lvl, last_lvl = 0; list_for_each_entry(entry, &bucket->list, list) { - if ((entry->dev_addr != ref->dev_addr) || - (entry->dev != ref->dev)) + if (!match(ref, entry)) continue; /* @@ -293,6 +314,39 @@ static struct dma_debug_entry *hash_bucket_find(struct hash_bucket *bucket, return ret; } +static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket, + struct dma_debug_entry *ref) +{ + return __hash_bucket_find(bucket, ref, exact_match); +} + +static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket, + struct dma_debug_entry *ref, + unsigned long *flags) +{ + + unsigned int max_range = dma_get_max_seg_size(ref->dev); + struct dma_debug_entry *entry, index = *ref; + unsigned int range = 0; + + while (range <= max_range) { + entry = __hash_bucket_find(*bucket, &index, containing_match); + + if (entry) + return entry; + + /* + * Nothing found, go back a hash bucket + */ + put_hash_bucket(*bucket, flags); + range += (1 << HASH_FN_SHIFT); + index.dev_addr -= (1 << HASH_FN_SHIFT); + *bucket = get_hash_bucket(&index, flags); + } + + return NULL; +} + /* * Add an entry to a hash bucket */ @@ -802,7 +856,7 @@ static void check_unmap(struct dma_debug_entry *ref) } bucket = get_hash_bucket(ref, &flags); - entry = hash_bucket_find(bucket, ref); + entry = bucket_find_exact(bucket, ref); if (!entry) { err_printk(ref->dev, NULL, "DMA-API: device driver tries " @@ -902,7 +956,7 @@ static void check_sync(struct device *dev, bucket = get_hash_bucket(ref, &flags); - entry = hash_bucket_find(bucket, ref); + entry = bucket_find_contain(&bucket, ref, &flags); if (!entry) { err_printk(dev, NULL, "DMA-API: device driver tries " @@ -1060,7 +1114,7 @@ static int get_nr_mapped_entries(struct device *dev, int mapped_ents; bucket = get_hash_bucket(ref, &flags); - entry = hash_bucket_find(bucket, ref); + entry = bucket_find_exact(bucket, ref); mapped_ents = 0; if (entry) diff --git a/virt/kvm/iommu.c b/virt/kvm/iommu.c index 967aba133a6..d5f3b8d1e09 100644 --- a/virt/kvm/iommu.c +++ b/virt/kvm/iommu.c @@ -232,12 +232,12 @@ int kvm_iommu_map_guest(struct kvm *kvm) { int r; - if (!iommu_found()) { + if (!iommu_present(&pci_bus_type)) { printk(KERN_ERR "%s: iommu not found\n", __func__); return -ENODEV; } - kvm->arch.iommu_domain = iommu_domain_alloc(); + kvm->arch.iommu_domain = iommu_domain_alloc(&pci_bus_type); if (!kvm->arch.iommu_domain) return -ENOMEM; |