SmartAudio/lichee/linux-4.9/drivers/iommu/sunxi-iommu.c

/*******************************************************************************
 * Copyright (C) 2016-2018, Allwinner Technology CO., LTD.
 * Author: zhuxianbin <zhuxianbin@allwinnertech.com>
 *
 * This file is provided under a dual BSD/GPL license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the
 * GNU General Public License for more details.
 ******************************************************************************/
#include <linux/of_iommu.h>
#include <linux/module.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/interrupt.h>
#include <linux/of_irq.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/iommu.h>
#include <linux/dma-mapping.h>
#include <linux/clk.h>
#include <linux/dma-iommu.h>
#include <linux/sizes.h>
#include <linux/device.h>
#include <asm/cacheflush.h>
#include <linux/pm_runtime.h>

#include "sunxi-iommu.h"

#define _max(x, y) (((u64)(x) > (u64)(y)) ? (x) : (y))

#ifdef CONFIG_ARCH_SUN50IW6
static const char *master[10] = {"DE", "VE_R", "DI", "VE", "CSI", "VP9"};
#endif
#ifdef CONFIG_ARCH_SUN50IW3
static const char *master[10] = {"DE0", "DE1", "VE_R", "VE", "CSI", "VP9"};
#endif
#ifdef CONFIG_ARCH_SUN8IW15
static const char *master[10] = {"DE", "E_EDMA", "E_FE", "VE", "CSI",
						"G2D", "E_BE", "DEBUG_MODE"};
#endif


static struct kmem_cache *iopte_cache;
static struct sunxi_iommu *global_iommu_dev;
static struct sunxi_iommu_domain *global_sunxi_iommu_domain;
struct iommu_domain *global_iommu_domain;

static struct iommu_group *global_group;

static bool tlb_init_flag;

static inline u32 *iopde_offset(u32 *iopd, unsigned int iova)
{
	return iopd + IOPDE_INDEX(iova);
}

static inline u32 *iopte_offset(u32 *ent, unsigned int iova)
{
	return (u32 *)__va(IOPTE_BASE(*ent)) + IOPTE_INDEX(iova);
}

static inline u32 sunxi_iommu_read(struct sunxi_iommu *iommu, u32 offset)
{
	return readl(iommu->base + offset);
}

static inline void sunxi_iommu_write(struct sunxi_iommu *iommu,
						u32 offset, u32 value)
{
	writel(value, iommu->base + offset);
}


void sunxi_enable_device_iommu(unsigned int master_id, bool flag)
{
	struct sunxi_iommu *iommu = global_iommu_dev;
	unsigned long mflag;

	spin_lock_irqsave(&iommu->iommu_lock, mflag);
	if (flag)
		iommu->bypass &= ~(master_id_bitmap[master_id]);
	else
		iommu->bypass |= master_id_bitmap[master_id];
	sunxi_iommu_write(iommu, IOMMU_BYPASS_REG, iommu->bypass);
	spin_unlock_irqrestore(&iommu->iommu_lock, mflag);
}
EXPORT_SYMBOL(sunxi_enable_device_iommu);

static int sunxi_tlb_flush(struct sunxi_iommu *iommu)
{
	int ret;

	sunxi_iommu_write(iommu, IOMMU_TLB_FLUSH_ENABLE_REG, 0x0003003f);
	ret = sunxi_wait_when(
		(sunxi_iommu_read(iommu, IOMMU_TLB_FLUSH_ENABLE_REG)), 2);
	if (ret)
		dev_err(iommu->dev, "Enable flush all request timed out\n");
	return ret;
}

static int
sunxi_tlb_init(struct sunxi_iommu_owner *owner,
				struct iommu_domain *input_domain)
{
	int ret = 0;
	phys_addr_t dte_addr;
	unsigned long mflag;
	struct sunxi_iommu *iommu = owner->data;
	struct sunxi_iommu_domain *sunxi_domain =
		container_of(input_domain, struct sunxi_iommu_domain, domain);

	/* iommu init */
	spin_lock_irqsave(&iommu->iommu_lock, mflag);
	dte_addr = __pa(sunxi_domain->pgtable);
	sunxi_iommu_write(iommu, IOMMU_TTB_REG, dte_addr);
	sunxi_iommu_write(iommu, IOMMU_TLB_PREFETCH_REG, 0x5f);
	sunxi_iommu_write(iommu, IOMMU_INT_ENABLE_REG, 0xffffffff);
	sunxi_iommu_write(iommu, IOMMU_BYPASS_REG, iommu->bypass);
	ret = sunxi_tlb_flush(iommu);
	if (ret) {
		dev_err(iommu->dev, "Enable flush all request timed out\n");
		goto out;
	}
	sunxi_iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0x1);
	sunxi_iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE);
	tlb_init_flag = true;
out:
	spin_unlock_irqrestore(&iommu->iommu_lock, mflag);
	return ret;

}

static int sunxi_tlb_invalid(dma_addr_t iova, u32 mask)
{
	struct sunxi_iommu *iommu = global_iommu_dev;
	int ret = 0;
	unsigned long mflag;

	spin_lock_irqsave(&iommu->iommu_lock, mflag);
	sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova);
	sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, mask);
	sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG, 0x1);
	ret = sunxi_wait_when(
		(sunxi_iommu_read(iommu, IOMMU_TLB_IVLD_ENABLE_REG)&0x1), 2);
	if (ret) {
		dev_err(iommu->dev, "TLB Invalid timed out\n");
		goto out;
	}

out:
	spin_unlock_irqrestore(&iommu->iommu_lock, mflag);
	return ret;
}

static int sunxi_ptw_cache_invalid(dma_addr_t iova)
{
	struct sunxi_iommu *iommu = global_iommu_dev;
	int ret = 0;
	unsigned long mflag;

	spin_lock_irqsave(&iommu->iommu_lock, mflag);
	sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova);
	sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG, 0x1);
	ret = sunxi_wait_when(
		(sunxi_iommu_read(iommu, IOMMU_PC_IVLD_ENABLE_REG)&0x1), 2);
	if (ret) {
		dev_err(iommu->dev, "PTW cache invalid timed out\n");
		goto out;
	}

out:
	spin_unlock_irqrestore(&iommu->iommu_lock, mflag);
	return ret;
}


static int sunxi_alloc_iopte(u32 *sent, int prot)
{
	u32 *pent;
	u32 flags = 0;

	flags |= (prot & IOMMU_READ) ? DENT_READABLE : 0;
	flags |= (prot & IOMMU_WRITE) ? DENT_WRITABLE : 0;

	pent = kmem_cache_zalloc(iopte_cache, GFP_ATOMIC);
	WARN_ON((unsigned long)pent & (PT_SIZE - 1));
	if (!pent) {
		pr_err("%s, %d, malloc failed!\n", __func__, __LINE__);
		return 0;
	}
	*sent = __pa(pent) | DENT_VALID;
	__dma_flush_area(sent, sizeof(*sent));

	return 1;
}

static void sunxi_free_iopte(u32 *pent)
{
	kmem_cache_free(iopte_cache, pent);
}

#if defined(CONFIG_ARCH_SUN50IW6) || defined(CONFIG_ARCH_SUN50IW3) || defined(CONFIG_ARCH_SUN8IW15)
void sunxi_zap_tlb(unsigned long iova, size_t size)
{
	sunxi_tlb_invalid(iova, (u32)IOMMU_PT_MASK);
	sunxi_tlb_invalid(iova + SPAGE_SIZE, (u32)IOMMU_PT_MASK);
	sunxi_tlb_invalid(iova + size, (u32)IOMMU_PT_MASK);
	sunxi_tlb_invalid(iova + size + SPAGE_SIZE, (u32)IOMMU_PT_MASK);
	sunxi_ptw_cache_invalid(iova);
	sunxi_ptw_cache_invalid(iova + SPD_SIZE);
	sunxi_ptw_cache_invalid(iova + size);
	sunxi_ptw_cache_invalid(iova + size + SPD_SIZE);
}
#else
static inline void sunxi_zap_tlb(unsigned long iova, size_t size)
{
}
#endif

static inline u32 sunxi_mk_pte(u32 page, int prot)
{
	u32 flags = 0;

	flags |= (prot & IOMMU_READ) ? SUNXI_PTE_PAGE_READABLE : 0;
	flags |= (prot & IOMMU_WRITE) ? SUNXI_PTE_PAGE_WRITABLE : 0;
	page &= IOMMU_PT_MASK;
	return page | flags | SUNXI_PTE_PAGE_VALID;
}


static int sunxi_iommu_map(struct iommu_domain *domain, unsigned long iova,
	   phys_addr_t paddr, size_t size, int prot)
{
	struct sunxi_iommu_domain *sunxi_domain =
		container_of(domain, struct sunxi_iommu_domain, domain);
	size_t iova_start, iova_end, paddr_start, s_iova_start;
	u32 *dent, *pent;
	int i, j;
	int flush_count = 0;

	WARN_ON(sunxi_domain->pgtable == NULL);
	iova_start = iova & IOMMU_PT_MASK;
	paddr_start = paddr & IOMMU_PT_MASK;
	iova_end = SPAGE_ALIGN(iova+size) - SPAGE_SIZE;
	s_iova_start = iova_start;

	mutex_lock(&sunxi_domain->dt_lock);
	if (IOPDE_INDEX(iova_start) == IOPDE_INDEX(iova_end)) {
		dent = iopde_offset(sunxi_domain->pgtable, iova_start);
		if (!IS_VALID(*dent))
			sunxi_alloc_iopte(dent, prot);
		for (flush_count = 0; iova_start <= iova_end;
			iova_start += SPAGE_SIZE, paddr_start += SPAGE_SIZE) {
			pent = iopte_offset(dent, iova_start);
			WARN_ON(*pent);
			*pent = sunxi_mk_pte(paddr_start, prot);
			++flush_count;
		}
		__dma_flush_area(iopte_offset(dent, s_iova_start),
			flush_count << 2);
		goto out;
	} else {
		dent = iopde_offset(sunxi_domain->pgtable, iova_start);
		if (!IS_VALID(*dent))
			sunxi_alloc_iopte(dent, prot);

		for (flush_count = 0; iova_start < SPDE_ALIGN(s_iova_start + 1);
			iova_start += SPAGE_SIZE, paddr_start += SPAGE_SIZE) {
			pent = iopte_offset(dent, iova_start);
			WARN_ON(*pent);
			*pent = sunxi_mk_pte(paddr_start, prot);
			++flush_count;
		}
		__dma_flush_area(iopte_offset(dent, s_iova_start),
			flush_count << 2);
	}
	if (IOPDE_INDEX(iova_start) < IOPDE_INDEX(iova_end)) {
		for (i = IOPDE_INDEX(iova_start); i < IOPDE_INDEX(iova_end);
			i++, iova_start += SPD_SIZE, paddr_start += SPD_SIZE) {
			dent = iopde_offset(sunxi_domain->pgtable, iova_start);
			if (!IS_VALID(*dent))
				sunxi_alloc_iopte(dent, prot);
			pent = iopte_offset(dent, iova_start);
			for (j = 0; j < NUM_ENTRIES_PTE; j++)
				*(pent + j) =
			sunxi_mk_pte(paddr_start + (j * SPAGE_SIZE), prot);
			__dma_flush_area(pent, PT_SIZE);
		}
	}

	s_iova_start = iova_start;
	dent = iopde_offset(sunxi_domain->pgtable, iova_start);
	if (!IS_VALID(*dent))
		sunxi_alloc_iopte(dent, prot);
	for (flush_count = 0; iova_start <= iova_end;
			iova_start += SPAGE_SIZE, paddr_start += SPAGE_SIZE) {
		pent = iopte_offset(dent, iova_start);
		WARN_ON(*pent);
		*pent = sunxi_mk_pte(paddr_start, prot);
		++flush_count;
	}
	__dma_flush_area(iopte_offset(dent, s_iova_start), flush_count << 2);
out:
	sunxi_zap_tlb(iova, size);
	mutex_unlock(&sunxi_domain->dt_lock);
	return 0;
}

static size_t sunxi_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
		 size_t size)
{
	struct sunxi_iommu_domain *sunxi_domain =
			container_of(domain, struct sunxi_iommu_domain, domain);
	size_t iova_start, iova_end, s_iova_start;
	u32 *dent, *pent;
	int i;
	int flush_count = 0;

	WARN_ON(sunxi_domain->pgtable == NULL);
	iova_start = iova & IOMMU_PT_MASK;
	iova_end = SPAGE_ALIGN(iova+size) - SPAGE_SIZE;
	s_iova_start = iova_start;

	mutex_lock(&sunxi_domain->dt_lock);
	if (IOPDE_INDEX(iova_start) == IOPDE_INDEX(iova_end)) {
		dent = iopde_offset(sunxi_domain->pgtable, iova_start);
		if (IS_VALID(*dent)) {
			for (flush_count = 0; iova_start <= iova_end;
						iova_start += SPAGE_SIZE) {
				pent = iopte_offset(dent, iova_start);
				*pent = 0;
				sunxi_tlb_invalid(iova_start,
							(u32)IOMMU_PT_MASK);
				++flush_count;
			}
			__dma_flush_area(iopte_offset(dent, s_iova_start),
					flush_count << 2);
			/*invalid ptwcache*/
			sunxi_ptw_cache_invalid(s_iova_start);
		}
		goto out;
	} else {
		dent = iopde_offset(sunxi_domain->pgtable, iova_start);
		if (IS_VALID(*dent)) {
			for (flush_count = 0;
				iova_start < SPDE_ALIGN(s_iova_start + 1);
						iova_start += SPAGE_SIZE) {
				pent = iopte_offset(dent, iova_start);
				*pent = 0;
				sunxi_tlb_invalid(iova_start,
							(u32)IOMMU_PT_MASK);
				++flush_count;
			}
			__dma_flush_area(iopte_offset(dent, s_iova_start),
					flush_count << 2);
			/*invalid ptwcache*/
			sunxi_ptw_cache_invalid(s_iova_start);
		}
	}
	if (IOPDE_INDEX(iova_start) < IOPDE_INDEX(iova_end)) {
		for (i = IOPDE_INDEX(iova_start); i < IOPDE_INDEX(iova_end);
			i++, iova_start += SPD_SIZE) {
			dent = iopde_offset(sunxi_domain->pgtable, iova_start);
			if (IS_VALID(*dent)) {
				pent = iopte_offset(dent, iova_start);
				memset(pent, 0, PT_SIZE);
				__dma_flush_area(pent, PT_SIZE);
				sunxi_tlb_invalid(iova_start,
							(u32)IOMMU_PD_MASK);
				*dent = 0;
				__dma_flush_area(dent, sizeof(*dent));
				sunxi_ptw_cache_invalid(iova_start);
				sunxi_free_iopte(pent);
			}
		}
	}
	s_iova_start = iova_start;
	dent = iopde_offset(sunxi_domain->pgtable, iova_start);
	if (IS_VALID(*dent)) {
		for (flush_count = 0; iova_start <= iova_end;
						iova_start += SPAGE_SIZE) {
			pent = iopte_offset(dent, iova_start);
			*pent = 0;
			sunxi_tlb_invalid(iova_start, (u32)IOMMU_PT_MASK);
			++flush_count;
		}
		__dma_flush_area(iopte_offset(dent, s_iova_start),
					flush_count << 2);
		sunxi_ptw_cache_invalid(s_iova_start);
	}
out:
	sunxi_zap_tlb(iova, size);
	mutex_unlock(&sunxi_domain->dt_lock);
	return size;
}

static size_t
sunxi_convert_sg_to_table(struct iommu_domain *domain,
				struct scatterlist *sg, size_t size)
{
	struct scatterlist *cur = sg;
	struct sunxi_iommu_domain *sunxi_domain =
			container_of(domain, struct sunxi_iommu_domain, domain);
	u32 *p = sunxi_domain->sg_buffer;
	int i;
	size_t total_length = 0;
#ifndef CONFIG_ARCH_SUN8IW15P1
	int j = 0;
#endif

#ifdef CONFIG_ARCH_SUN8IW15P1
	WARN_ON(size > MAX_SG_SIZE);
	while (total_length < size) {
#else
	while (j < size) {
#endif
		u32 phys = page_to_phys(sg_page(cur));
		unsigned int length = PAGE_ALIGN(cur->offset + cur->length);

		for (i = 0; i < length / SPAGE_SIZE; i++, p++)
			*p = phys + (i * SPAGE_SIZE);
		cur = sg_next(cur);
		total_length += length;
#ifndef CONFIG_ARCH_SUN8IW15P1
		WARN_ON(total_length > MAX_SG_SIZE);
		++j;
#endif
	}
	return total_length;
}

static size_t
sunxi_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
		 struct scatterlist *sg, unsigned int nents_size, int prot)
{
	struct sunxi_iommu_domain *sunxi_domain =
			container_of(domain, struct sunxi_iommu_domain, domain);
	size_t iova_start, iova_end, s_iova_start;
	u32 *dent, *pent, *paddr_start;
	unsigned int i, j;
	int flush_count = 0;
	size_t total_size = 0;

	WARN_ON(sunxi_domain->pgtable == NULL);
	iova_start = iova & IOMMU_PT_MASK;
	paddr_start = sunxi_domain->sg_buffer;
	s_iova_start = iova_start;
#ifdef CONFIG_ARCH_SUN8IW15P1
	iova_end = SPAGE_ALIGN(iova + nents_size) - SPAGE_SIZE;
#endif
	mutex_lock(&sunxi_domain->dt_lock);
	total_size = sunxi_convert_sg_to_table(domain, sg, nents_size);
#ifndef CONFIG_ARCH_SUN8IW15P1
	iova_end = SPAGE_ALIGN(iova+total_size) - SPAGE_SIZE;
#endif

	if (IOPDE_INDEX(iova_start) == IOPDE_INDEX(iova_end)) {
		dent = iopde_offset(sunxi_domain->pgtable, iova_start);
		if (!IS_VALID(*dent))
			sunxi_alloc_iopte(dent, prot);

		for (flush_count = 0; iova_start <= iova_end;
				iova_start += SPAGE_SIZE, paddr_start++) {
			pent = iopte_offset(dent, iova_start);
			WARN_ON(*pent);
			*pent = sunxi_mk_pte(*paddr_start, prot);
			++flush_count;
		}
		__dma_flush_area(iopte_offset(dent, s_iova_start),
			flush_count << 2);
		goto out;
	} else {
		dent = iopde_offset(sunxi_domain->pgtable, iova_start);
		if (!IS_VALID(*dent))
			sunxi_alloc_iopte(dent, prot);

		for (flush_count = 0; iova_start < SPDE_ALIGN(s_iova_start + 1);
				iova_start += SPAGE_SIZE, paddr_start++) {
			pent = iopte_offset(dent, iova_start);
			WARN_ON(*pent);
			*pent = sunxi_mk_pte(*paddr_start, prot);
			++flush_count;
		}
		__dma_flush_area(iopte_offset(dent, s_iova_start),
			flush_count << 2);
	}
	if (IOPDE_INDEX(iova_start) < IOPDE_INDEX(iova_end)) {
		for (i = IOPDE_INDEX(iova_start); i < IOPDE_INDEX(iova_end);
			i++, iova_start += SPD_SIZE) {
			dent = iopde_offset(sunxi_domain->pgtable, iova_start);
			if (!IS_VALID(*dent))
				sunxi_alloc_iopte(dent, prot);
			pent = iopte_offset(dent, iova_start);
			for (j = 0; j < NUM_ENTRIES_PTE; j++)
				*(pent + j) =
					sunxi_mk_pte(*paddr_start++, prot);
			__dma_flush_area(pent, PT_SIZE);
		}
	}

	s_iova_start = iova_start;
	dent = iopde_offset(sunxi_domain->pgtable, iova_start);
	if (!IS_VALID(*dent))
		sunxi_alloc_iopte(dent, prot);
	for (flush_count = 0; iova_start <= iova_end;
			iova_start += SPAGE_SIZE, paddr_start++) {
		pent = iopte_offset(dent, iova_start);
		WARN_ON(*pent);
		*pent = sunxi_mk_pte(*paddr_start, prot);
		++flush_count;
	}
	__dma_flush_area(iopte_offset(dent, s_iova_start), flush_count << 2);
out:
	sunxi_zap_tlb(iova, nents_size);
	mutex_unlock(&sunxi_domain->dt_lock);
	return total_size;
}

static phys_addr_t
sunxi_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
{
	struct sunxi_iommu_domain *sunxi_domain =
		container_of(domain, struct sunxi_iommu_domain, domain);
	u32 *dent, *pent;
	phys_addr_t ret = 0;

	WARN_ON(sunxi_domain->pgtable == NULL);
	mutex_lock(&sunxi_domain->dt_lock);
	dent = iopde_offset(sunxi_domain->pgtable, iova);
	if (IS_VALID(*dent)) {
		pent = iopte_offset(dent, iova);
		ret = IOPTE_TO_PFN(pent) + IOVA_PAGE_OFT(iova);
	}
	mutex_unlock(&sunxi_domain->dt_lock);
	return ret;
}

static struct iommu_domain *sunxi_iommu_domain_alloc(unsigned type)
{
	struct sunxi_iommu_domain *sunxi_domain;

	if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED)
		return NULL;

	/* we just use one domain */
	if (global_sunxi_iommu_domain)
		return &global_sunxi_iommu_domain->domain;

	sunxi_domain = kzalloc(sizeof(*sunxi_domain), GFP_KERNEL);

	if (!sunxi_domain)
		return NULL;

	sunxi_domain->pgtable = (unsigned int *)__get_free_pages(
				GFP_KERNEL, get_order(PD_SIZE));
	if (!sunxi_domain->pgtable) {
		pr_err("sunxi domain get pgtable failed\n");
		goto err_page;
	}

	sunxi_domain->sg_buffer = (unsigned int *)__get_free_pages(
				GFP_KERNEL, get_order(MAX_SG_TABLE_SIZE));
	if (!sunxi_domain->sg_buffer) {
		pr_err("sunxi domain get sg_buffer failed\n");
		goto err_sg_buffer;
	}

	if (type == IOMMU_DOMAIN_DMA &&
				iommu_get_dma_cookie(&sunxi_domain->domain)) {
		pr_err("sunxi domain get dma cookie failed\n");
		goto err_dma_cookie;
	}

	memset(sunxi_domain->pgtable, 0, PD_SIZE);
	sunxi_domain->domain.geometry.aperture_start = 0;
	sunxi_domain->domain.geometry.aperture_end	 = (1ULL << 32)-1;
	sunxi_domain->domain.geometry.force_aperture = true;
	mutex_init(&sunxi_domain->dt_lock);
	global_sunxi_iommu_domain = sunxi_domain;
	global_iommu_domain = &sunxi_domain->domain;
	return &sunxi_domain->domain;

err_dma_cookie:
err_sg_buffer:
	free_pages((unsigned long)sunxi_domain->pgtable, get_order(PD_SIZE));
	sunxi_domain->pgtable = NULL;
err_page:
	kfree(sunxi_domain);
	return NULL;
}

static void sunxi_iommu_domain_free(struct iommu_domain *domain)
{
	struct sunxi_iommu_domain *sunxi_domain =
		container_of(domain, struct sunxi_iommu_domain, domain);
	int i = 0;
	size_t iova;
	u32 *dent, *pent;

	mutex_lock(&sunxi_domain->dt_lock);
	for (i = 0; i < NUM_ENTRIES_PDE; ++i) {
		dent = sunxi_domain->pgtable + i;
		iova = i << IOMMU_PD_SHIFT;
		if (IS_VALID(*dent)) {
			pent = iopte_offset(dent, iova);
			memset(pent, 0, PT_SIZE);
			__dma_flush_area(pent, PT_SIZE);
			*dent = 0;
			__dma_flush_area(dent, sizeof(*dent));
			sunxi_free_iopte(pent);
		}
	}
	sunxi_tlb_flush(global_iommu_dev);
	mutex_unlock(&sunxi_domain->dt_lock);
	free_pages((unsigned long)sunxi_domain->pgtable, get_order(PD_SIZE));
	sunxi_domain->pgtable = NULL;
	free_pages((unsigned long)sunxi_domain->sg_buffer,
						get_order(MAX_SG_TABLE_SIZE));
	sunxi_domain->sg_buffer = NULL;
	iommu_put_dma_cookie(domain);
	kfree(sunxi_domain);
}

static int
sunxi_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
{
	int ret = 0;

#if defined(CONFIG_IOMMU_DEBUG) && !defined(CONFIG_ARM_DMA_USE_IOMMU)
	do_iommu_attach(dev, NULL, 0, SZ_1G * 4UL);
#endif
	return ret;
}

static void
sunxi_iommu_detach_dev(struct iommu_domain *domain, struct device *dev)
{
		return;
}

static int sunxi_iommu_add_device(struct device *dev)
{
	struct iommu_group *group;
	struct sunxi_iommu_owner *owner = dev->archdata.iommu;
	int ret = 0;

	if (!dev->archdata.iommu) /* Not a iommu client device */
		return -ENODEV;

	group = iommu_group_get_for_dev(dev);

	if (IS_ERR(group)) {
		ret = PTR_ERR(group);
		pr_err("sunxi iommu get group failed, err_no:%d\n", ret);
	}
	if (!tlb_init_flag) {
		ret = sunxi_tlb_init(owner, &global_sunxi_iommu_domain->domain);
		if (ret)
			pr_err("sunxi iommu init tlb failed\n");
	}
	sunxi_enable_device_iommu(owner->tlbid, owner->flag);

	iommu_group_put(group);

	return ret;
}

static void sunxi_iommu_remove_device(struct device *dev)
{
	struct sunxi_iommu_owner *owner = dev->archdata.iommu;

	if (!owner)
		return;

	sunxi_enable_device_iommu(owner->tlbid, false);
	dev->iommu_group = NULL;
	devm_kfree(dev, dev->dma_parms);
	dev->dma_parms = NULL;
	kfree(owner);
	owner = NULL;
	dev->archdata.iommu = NULL;
	iommu_group_remove_device(dev);
}

/* set dma params for master devices */
int sunxi_iommu_set_dma_parms(struct notifier_block *nb,
				unsigned long action, void *data)
{
	struct device *dev = data;

	if (action != IOMMU_GROUP_NOTIFY_BIND_DRIVER)
		return 0;

	dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL);
	if (!dev->dma_parms)
		return -ENOMEM;
	dma_set_max_seg_size(dev, DMA_BIT_MASK(32));

	return 0;
}

struct iommu_group *sunxi_iommu_device_group(struct device *dev)
{
	struct iommu_group *group;
	struct notifier_block *nb;

	if (!global_group) {
		nb = kzalloc(sizeof(nb), GFP_KERNEL);
		if (!nb)
			return ERR_PTR(-ENOMEM);

		global_group = iommu_group_alloc();
		if (IS_ERR(global_group)) {
			pr_err("sunxi iommu alloc group failed\n");
			goto err_group_alloc;
		}

		nb->notifier_call = sunxi_iommu_set_dma_parms;
		if (iommu_group_register_notifier(global_group, nb)) {
			pr_err("sunxi iommu group register notifier failed!\n");
			goto err_notifier;
		}

	}
	group = global_group;
	return group;

err_notifier:
err_group_alloc:
	kfree(nb);
	return ERR_PTR(-EBUSY);
}

static int
sunxi_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
{
	struct sunxi_iommu_owner *owner = dev->archdata.iommu;
	struct platform_device *sysmmu = of_find_device_by_node(args->np);
	struct sunxi_iommu *data;

	if (!sysmmu)
		return -ENODEV;

	data = platform_get_drvdata(sysmmu);
	if (data == NULL)
		return -ENODEV;

	if (!owner) {
		owner = kzalloc(sizeof(*owner), GFP_KERNEL);
		if (!owner)
			return -ENOMEM;
		owner->tlbid = args->args[0];
		owner->flag = args->args[1];
		owner->data = data;
		owner->dev = dev;
		dev->archdata.iommu = owner;
	}

	return 0;
}

#ifdef CONFIG_ARCH_SUN8IW15
void sunxi_set_debug_mode(void)
{
	struct sunxi_iommu *iommu = global_iommu_dev;

	sunxi_iommu_write(iommu,
			IOMMU_VA_CONFIG_REG, 0x80000000);
}

void sunxi_set_prefetch_mode(void)
{
	struct sunxi_iommu *iommu = global_iommu_dev;

	sunxi_iommu_write(iommu,
			IOMMU_VA_CONFIG_REG, 0x00000000);
}

#else
void sunxi_set_debug_mode(void){ return; }
void sunxi_set_prefetch_mode(void){ return; }
#endif

int sunxi_iova_test_write(dma_addr_t iova, u32 val)
{
	struct sunxi_iommu *iommu = global_iommu_dev;
	int retval;

	sunxi_iommu_write(iommu, IOMMU_VA_REG, iova);
	sunxi_iommu_write(iommu, IOMMU_VA_DATA_REG, val);
	sunxi_iommu_write(iommu,
			IOMMU_VA_CONFIG_REG, 0x80000100);
	sunxi_iommu_write(iommu,
			IOMMU_VA_CONFIG_REG, 0x80000101);
	retval = sunxi_wait_when((sunxi_iommu_read(iommu,
				IOMMU_VA_CONFIG_REG) & 0x1), 1);
	if (retval)
		dev_err(iommu->dev,
			"write VA address request timed out\n");
	return retval;
}

unsigned long sunxi_iova_test_read(dma_addr_t iova)
{
	struct sunxi_iommu *iommu = global_iommu_dev;
	unsigned long retval;

	sunxi_iommu_write(iommu, IOMMU_VA_REG, iova);
	sunxi_iommu_write(iommu,
			IOMMU_VA_CONFIG_REG, 0x80000000);
	sunxi_iommu_write(iommu,
			IOMMU_VA_CONFIG_REG, 0x80000001);
	retval = sunxi_wait_when((sunxi_iommu_read(iommu,
				IOMMU_VA_CONFIG_REG) & 0x1), 1);
	if (retval) {
		dev_err(iommu->dev,
			"read VA address request timed out\n");
		retval = 0;
		goto out;
	}
	retval = sunxi_iommu_read(iommu,
			IOMMU_VA_DATA_REG);
out:
	return retval;
}


static struct iommu_ops sunxi_iommu_ops = {
	.pgsize_bitmap = SZ_4K,
	.map  = sunxi_iommu_map,
	.unmap = sunxi_iommu_unmap,
	.map_sg = sunxi_iommu_map_sg,
	.domain_alloc = sunxi_iommu_domain_alloc,
	.domain_free = sunxi_iommu_domain_free,
	.attach_dev = sunxi_iommu_attach_dev,
	.detach_dev = sunxi_iommu_detach_dev,
	.add_device = sunxi_iommu_add_device,
	.remove_device = sunxi_iommu_remove_device,
	.device_group	= sunxi_iommu_device_group,
	.of_xlate = sunxi_iommu_of_xlate,
	.iova_to_phys = sunxi_iommu_iova_to_phys,
};

static int sunxi_iova_invalid_helper(unsigned long iova)
{
	struct sunxi_iommu_domain *sunxi_domain = global_sunxi_iommu_domain;
	u32 *pte_addr, *dte_addr;

	dte_addr = iopde_offset(sunxi_domain->pgtable, iova);
	if ((*dte_addr & 0x3) != 0x1) {
		pr_err("0x%lx is not mapped!\n", iova);
		return 1;
	}
	pte_addr = iopte_offset(dte_addr, iova);
	if ((*pte_addr & 0x2) == 0) {
		pr_err("0x%lx is not mapped!\n", iova);
		return 1;
	}
	pr_err("0x%lx is mapped!\n", iova);
	return 0;
}

static irqreturn_t sunxi_iommu_irq(int irq, void *dev_id)
{

	u32 inter_status_reg = 0;
	u32 addr_reg = 0;
	u32	int_masterid_bitmap = 0;
	u32	data_reg = 0;
	u32	l1_pgint_reg = 0;
	u32	l2_pgint_reg = 0;
	u32	master_id = 0;
	unsigned long mflag;
	struct sunxi_iommu *iommu = dev_id;

	spin_lock_irqsave(&iommu->iommu_lock, mflag);
	inter_status_reg = sunxi_iommu_read(iommu, IOMMU_INT_STA_REG) & 0x3ffff;
	l1_pgint_reg = sunxi_iommu_read(iommu, IOMMU_L1PG_INT_REG);
	l2_pgint_reg = sunxi_iommu_read(iommu, IOMMU_L2PG_INT_REG);

	if (inter_status_reg & MICRO_TLB0_INVALID_INTER_MASK) {
		pr_err("%s Invalid Authority\n", master[0]);
		addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG0);
		data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG0);
	} else if (inter_status_reg & MICRO_TLB1_INVALID_INTER_MASK) {
		pr_err("%s Invalid Authority\n", master[1]);
		addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG1);
		data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG1);
	} else if (inter_status_reg & MICRO_TLB2_INVALID_INTER_MASK) {
		pr_err("%s Invalid Authority\n", master[2]);
		addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG2);
		data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG2);
	} else if (inter_status_reg & MICRO_TLB3_INVALID_INTER_MASK) {
		pr_err("%s Invalid Authority\n", master[3]);
		addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG3);
		data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG3);
	} else if (inter_status_reg & MICRO_TLB4_INVALID_INTER_MASK) {
		pr_err("%s Invalid Authority\n", master[4]);
		addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG4);
		data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG4);
	} else if (inter_status_reg & MICRO_TLB5_INVALID_INTER_MASK) {
		pr_err("%s Invalid Authority\n", master[5]);
		addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG5);
		data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG5);
	}
#ifdef CONFIG_ARCH_SUN8IW15
	else if (inter_status_reg & MICRO_TLB6_INVALID_INTER_MASK) {
		pr_err("%s Invalid Authority\n", master[6]);
		addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG6);
		data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG6);
	} else if (inter_status_reg & L1_PAGETABLE_INVALID_INTER_MASK) {
		pr_err("L1 PageTable Invalid\n");
		addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG7);
		data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG7);
	} else if (inter_status_reg & L2_PAGETABLE_INVALID_INTER_MASK) {
		pr_err("L2 PageTable Invalid\n");
		addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG8);
		data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG8);
	}
#else
	else if (inter_status_reg & L1_PAGETABLE_INVALID_INTER_MASK) {
		pr_err("L1 PageTable Invalid\n");
		addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG6);
		data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG6);
	} else if (inter_status_reg & L2_PAGETABLE_INVALID_INTER_MASK) {
		pr_err("L2 PageTable Invalid\n");
		addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG7);
		data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG7);
	}
#endif
	else
		pr_err("sunxi iommu int error!!!\n");

	if (sunxi_iova_invalid_helper(addr_reg)) {
		int_masterid_bitmap =
			inter_status_reg | l1_pgint_reg | l2_pgint_reg;
		if (int_masterid_bitmap & (1 << 31))
			master_id = 7; /*DEBUG_MODE*/
		else {
			int_masterid_bitmap &= 0xffff;
			master_id = __ffs(int_masterid_bitmap);
		}

		pr_err("%s invalid address: 0x%x, data:0x%x, id:0x%x\n",
			master[master_id], addr_reg, data_reg,
				int_masterid_bitmap);
	}
	sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, addr_reg);
	sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG,
		(u32)IOMMU_PT_MASK);
	sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG, 0x1);
	while (sunxi_iommu_read(iommu, IOMMU_TLB_IVLD_ENABLE_REG) & 0x1)
		;
	sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG,
		addr_reg + 0x2000);
	sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG,
		(u32)IOMMU_PT_MASK);
	sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG, 0x1);
	while (sunxi_iommu_read(iommu, IOMMU_TLB_IVLD_ENABLE_REG) & 0x1)
		;
	sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, addr_reg);
	sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG, 0x1);
	while (sunxi_iommu_read(iommu, IOMMU_PC_IVLD_ENABLE_REG) & 0x1)
		;
	sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG,
		addr_reg + 0x200000);
	sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG, 0x1);
	while (sunxi_iommu_read(iommu, IOMMU_PC_IVLD_ENABLE_REG) & 0x1)
		;

	sunxi_iommu_write(iommu, IOMMU_INT_CLR_REG, inter_status_reg);
	inter_status_reg |= (l1_pgint_reg | l2_pgint_reg);
	inter_status_reg &= 0xffff;
	sunxi_iommu_write(iommu, IOMMU_RESET_REG, ~inter_status_reg);
	sunxi_iommu_write(iommu, IOMMU_RESET_REG, 0xffffffff);
	spin_unlock_irqrestore(&iommu->iommu_lock, mflag);

	return IRQ_HANDLED;
}

static ssize_t sunxi_iommu_enable_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct sunxi_iommu *iommu = global_iommu_dev;
	u32 data;

	spin_lock(&iommu->iommu_lock);
	data = sunxi_iommu_read(iommu, IOMMU_PMU_ENABLE_REG);
	spin_unlock(&iommu->iommu_lock);
	return snprintf(buf, PAGE_SIZE,
		"enable = %d\n", data & 0x1 ? 1 : 0);
}

static ssize_t sunxi_iommu_enable_store(struct device *dev,
					    struct device_attribute *attr,
					    const char *buf, size_t count)
{
	struct sunxi_iommu *iommu = global_iommu_dev;
	unsigned long val;
	u32 data;
	int retval;

	if (kstrtoul(buf, 0, &val))
		return -EINVAL;

	if (val) {
		spin_lock(&iommu->iommu_lock);
		data = sunxi_iommu_read(iommu, IOMMU_PMU_ENABLE_REG);
		sunxi_iommu_write(iommu, IOMMU_PMU_ENABLE_REG, data | 0x1);
		data = sunxi_iommu_read(iommu, IOMMU_PMU_CLR_REG);
		sunxi_iommu_write(iommu, IOMMU_PMU_CLR_REG, data | 0x1);
		retval = sunxi_wait_when((sunxi_iommu_read(iommu,
				IOMMU_PMU_CLR_REG) & 0x1), 1);
		if (retval)
			dev_err(iommu->dev, "Clear PMU Count timed out\n");
		spin_unlock(&iommu->iommu_lock);
	} else {
		spin_lock(&iommu->iommu_lock);
		data = sunxi_iommu_read(iommu, IOMMU_PMU_CLR_REG);
		sunxi_iommu_write(iommu, IOMMU_PMU_CLR_REG, data | 0x1);
		retval = sunxi_wait_when((sunxi_iommu_read(iommu,
				IOMMU_PMU_CLR_REG) & 0x1), 1);
		if (retval)
			dev_err(iommu->dev, "Clear PMU Count timed out\n");
		data = sunxi_iommu_read(iommu, IOMMU_PMU_ENABLE_REG);
		sunxi_iommu_write(iommu, IOMMU_PMU_ENABLE_REG, data & ~0x1);
		spin_unlock(&iommu->iommu_lock);
	}

	return count;
}

static ssize_t sunxi_iommu_profilling_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct sunxi_iommu *iommu = global_iommu_dev;
	u64 micro_tlb0_access_count;
	u64 micro_tlb0_hit_count;
	u64 micro_tlb1_access_count;
	u64 micro_tlb1_hit_count;
	u64 micro_tlb2_access_count;
	u64 micro_tlb2_hit_count;
	u64 micro_tlb3_access_count;
	u64 micro_tlb3_hit_count;
	u64 micro_tlb4_access_count;
	u64 micro_tlb4_hit_count;
	u64 micro_tlb5_access_count;
	u64 micro_tlb5_hit_count;
#ifdef CONFIG_ARCH_SUN8IW15P1
	u64 micro_tlb6_access_count;
	u64 micro_tlb6_hit_count;
#endif
	u64 macrotlb_access_count;
	u64 macrotlb_hit_count;
	u64 ptwcache_access_count;
	u64 ptwcache_hit_count;
	u64 micro_tlb0_latency;
	u64 micro_tlb1_latency;
	u64 micro_tlb2_latency;
	u64 micro_tlb3_latency;
	u64 micro_tlb4_latency;
	u64 micro_tlb5_latency;
#ifdef CONFIG_ARCH_SUN8IW15P1
	u64 micro_tlb6_latency;
	u32 micro_tlb0_max_latency;
	u32 micro_tlb1_max_latency;
	u32 micro_tlb2_max_latency;
	u32 micro_tlb3_max_latency;
	u32 micro_tlb4_max_latency;
	u32 micro_tlb5_max_latency;
	u32 micro_tlb6_max_latency;
#endif

	spin_lock(&iommu->iommu_lock);

#ifndef CONFIG_ARCH_SUN8IW15P1
	macrotlb_access_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG6) &
		0x7ff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG6);
	macrotlb_hit_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG6) &
		0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG6);
#else
	macrotlb_access_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG7) &
		0x7ff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG7);
	macrotlb_hit_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG7) &
		0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG7);
#endif

	micro_tlb0_access_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG0) &
		0x7ff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG0);
	micro_tlb0_hit_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG0) &
		0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG0);

	micro_tlb1_access_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG1) &
		0x7ff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG1);
	micro_tlb1_hit_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG1) &
		0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG1);

	micro_tlb2_access_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG2) &
		0x7ff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG2);
	micro_tlb2_hit_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG2) &
		0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG2);

	micro_tlb3_access_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG3) &
		0x7ff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG3);
	micro_tlb3_hit_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG3) &
		0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG3);

	micro_tlb4_access_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG4) &
		0x7ff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG4);
	micro_tlb4_hit_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG4) &
		0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG4);

	micro_tlb5_access_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG5) &
		0x7ff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG5);
	micro_tlb5_hit_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG5) &
		0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG5);

#ifdef CONFIG_ARCH_SUN8IW15P1
	micro_tlb6_access_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG6) &
		0x7ff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG6);
	micro_tlb6_hit_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG6) &
		0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG6);

	ptwcache_access_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG8) &
		0x7ff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG8);
	ptwcache_hit_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG8) &
		0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG8);
#else
	ptwcache_access_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG7) &
		0x7ff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG7);
	ptwcache_hit_count =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG7) &
		0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG7);
#endif

	micro_tlb0_latency =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG0) &
		0x3ffff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG0);
	micro_tlb1_latency =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG1) &
		0x3ffff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG1);
	micro_tlb2_latency =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG2) &
		0x3ffff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG2);
	micro_tlb3_latency =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG3) &
		0x3ffff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG3);
	micro_tlb4_latency =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG4) &
		0x3ffff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG4);
	micro_tlb5_latency =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG5) &
		0x3ffff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG5);

#ifdef CONFIG_ARCH_SUN8IW15P1
	micro_tlb6_latency =
		((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG6) &
		0x3ffff) << 32) |
		sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG6);

	micro_tlb0_max_latency = sunxi_iommu_read(iommu, IOMMU_PMU_ML_REG0);
	micro_tlb1_max_latency = sunxi_iommu_read(iommu, IOMMU_PMU_ML_REG1);
	micro_tlb2_max_latency = sunxi_iommu_read(iommu, IOMMU_PMU_ML_REG2);
	micro_tlb3_max_latency = sunxi_iommu_read(iommu, IOMMU_PMU_ML_REG3);
	micro_tlb4_max_latency = sunxi_iommu_read(iommu, IOMMU_PMU_ML_REG4);
	micro_tlb5_max_latency = sunxi_iommu_read(iommu, IOMMU_PMU_ML_REG5);
	micro_tlb6_max_latency = sunxi_iommu_read(iommu, IOMMU_PMU_ML_REG6);

#endif
	spin_unlock(&iommu->iommu_lock);

	return snprintf(buf, PAGE_SIZE,
		"%s_access_count = 0x%llx\n"
		"%s_hit_count = 0x%llx\n"
		"%s_access_count = 0x%llx\n"
		"%s_hit_count = 0x%llx\n"
		"%s_access_count = 0x%llx\n"
		"%s_hit_count = 0x%llx\n"
		"%s_access_count = 0x%llx\n"
		"%s_hit_count = 0x%llx\n"
		"%s_access_count = 0x%llx\n"
		"%s_hit_count = 0x%llx\n"
		"%s_access_count = 0x%llx\n"
		"%s_hit_count = 0x%llx\n"
#ifdef CONFIG_ARCH_SUN8IW15P1
		"%s_access_count = 0x%llx\n"
		"%s_hit_count = 0x%llx\n"
#endif
		"macrotlb_access_count = 0x%llx\n"
		"macrotlb_hit_count = 0x%llx\n"
		"ptwcache_access_count = 0x%llx\n"
		"ptwcache_hit_count = 0x%llx\n"
		"%s_total_latency = 0x%llx\n"
		"%s_total_latency = 0x%llx\n"
		"%s_total_latency = 0x%llx\n"
		"%s_total_latency = 0x%llx\n"
		"%s_total_latency = 0x%llx\n"
		"%s_total_latency = 0x%llx\n"
#ifdef CONFIG_ARCH_SUN8IW15P1
		"%s_total_latency = 0x%llx\n"
		"%s_max_latency = 0x%x\n"
		"%s_max_latency = 0x%x\n"
		"%s_max_latency = 0x%x\n"
		"%s_max_latency = 0x%x\n"
		"%s_max_latency = 0x%x\n"
		"%s_max_latency = 0x%x\n"
		"%s_max_latency = 0x%x\n"
#endif
		,
		master[0], micro_tlb0_access_count,
		master[0], micro_tlb0_hit_count,
		master[1], micro_tlb1_access_count,
		master[1], micro_tlb1_hit_count,
		master[2], micro_tlb2_access_count,
		master[2], micro_tlb2_hit_count,
		master[3], micro_tlb3_access_count,
		master[3], micro_tlb3_hit_count,
		master[4], micro_tlb4_access_count,
		master[4], micro_tlb4_hit_count,
		master[5], micro_tlb5_access_count,
		master[5], micro_tlb5_hit_count,
#ifdef CONFIG_ARCH_SUN8IW15P1
		master[6], micro_tlb6_access_count,
		master[6], micro_tlb6_hit_count,
#endif
		macrotlb_access_count,
		macrotlb_hit_count,
		ptwcache_access_count,
		ptwcache_hit_count,
		master[0], micro_tlb0_latency,
		master[1], micro_tlb1_latency,
		master[2], micro_tlb2_latency,
		master[3], micro_tlb3_latency,
		master[4], micro_tlb4_latency,
		master[5], micro_tlb5_latency
#ifdef CONFIG_ARCH_SUN8IW15P1
		,
		master[6], micro_tlb6_latency,
		master[0], micro_tlb0_max_latency,
		master[1], micro_tlb1_max_latency,
		master[2], micro_tlb2_max_latency,
		master[3], micro_tlb3_max_latency,
		master[4], micro_tlb4_max_latency,
		master[5], micro_tlb5_max_latency,
		master[6], micro_tlb6_max_latency
#endif
			);
}

static struct device_attribute sunxi_iommu_enable_attr =
	__ATTR(enable, 0644, sunxi_iommu_enable_show,
	sunxi_iommu_enable_store);
static struct device_attribute sunxi_iommu_profilling_attr =
	__ATTR(profilling, 0444, sunxi_iommu_profilling_show, NULL);

static void sunxi_iommu_sysfs_create(struct platform_device *_pdev)
{
	device_create_file(&_pdev->dev, &sunxi_iommu_enable_attr);
	device_create_file(&_pdev->dev, &sunxi_iommu_profilling_attr);
}

static void sunxi_iommu_sysfs_remove(struct platform_device *_pdev)
{
	device_remove_file(&_pdev->dev, &sunxi_iommu_enable_attr);
	device_remove_file(&_pdev->dev, &sunxi_iommu_profilling_attr);
}

static int sunxi_iommu_probe(struct platform_device *pdev)
{
	int ret, irq;
	struct device *dev = &pdev->dev;
	struct sunxi_iommu *iommu_dev;
	struct resource *res;

	iommu_dev = kzalloc(sizeof(*iommu_dev), GFP_KERNEL);
	if (!iommu_dev)
		return	-ENOMEM;

	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	if (!res) {
		dev_dbg(dev, "Unable to find resource region\n");
		ret = -ENOENT;
		goto err_res;
	}

	iommu_dev->base = devm_ioremap_resource(&pdev->dev, res);
	if (!iommu_dev->base) {
		dev_dbg(dev, "Unable to map IOMEM @ PA:%#x\n",
				(unsigned int)res->start);
		ret = -ENOENT;
		goto err_res;
	}

	iommu_dev->bypass = DEFAULT_BYPASS_VALUE;

	irq = platform_get_irq(pdev, 0);
	if (irq <= 0) {
		dev_dbg(dev, "Unable to find IRQ resource\n");
		goto err_irq;
	}

	pr_info("sunxi iommu: irq = %d\n", irq);

	ret = devm_request_irq(dev, irq, sunxi_iommu_irq, 0,
			dev_name(dev), (void *)iommu_dev);
	if (ret < 0) {
		dev_dbg(dev, "Unabled to register interrupt handler\n");
		goto err_irq;
	}

	iommu_dev->irq = irq;

	iommu_dev->clk = of_clk_get_by_name(dev->of_node, "iommu");
	if (IS_ERR(iommu_dev->clk)) {
		iommu_dev->clk = NULL;
		dev_dbg(dev, "Unable to find clock\n");
		goto err_clk;
	}
	clk_prepare_enable(iommu_dev->clk);

	platform_set_drvdata(pdev, iommu_dev);
	iommu_dev->dev = dev;
	spin_lock_init(&iommu_dev->iommu_lock);
	global_iommu_dev = iommu_dev;

	if (dev->parent)
		pm_runtime_enable(dev);

	sunxi_iommu_sysfs_create(pdev);

	return 0;

err_clk:
	free_irq(irq, iommu_dev);
err_irq:
	devm_iounmap(dev, iommu_dev->base);
err_res:
	kfree(iommu_dev);
	dev_err(dev, "Failed to initialize\n");

	return ret;

}

static int sunxi_iommu_remove(struct platform_device *pdev)
{
	struct sunxi_iommu *mmu_dev = platform_get_drvdata(pdev);

	sunxi_iommu_sysfs_remove(pdev);
	free_irq(mmu_dev->irq, mmu_dev);
	devm_iounmap(mmu_dev->dev, mmu_dev->base);
	kfree(mmu_dev);
	global_iommu_dev = NULL;
	return 0;
}

static const struct of_device_id sunxi_iommu_dt[] = {
	{ .compatible = "allwinner,sunxi-iommu", },
	{ /* sentinel */ },
};

MODULE_DEVICE_TABLE(of, sunxi_iommu_dt);

static struct platform_driver sunxi_iommu_driver = {
	.probe		= sunxi_iommu_probe,
	.remove		= sunxi_iommu_remove,
	.driver		= {
		.owner		= THIS_MODULE,
		.name		= "sunxi-iommu",
		.of_match_table = sunxi_iommu_dt,
	}
};

static int __init sunxi_iommu_of_setup(struct device_node *np)
{
	int ret = 0;
	struct platform_device *pdev;

	iopte_cache = kmem_cache_create("sunxi-iopte-cache", PT_SIZE,
				PT_SIZE, SLAB_HWCACHE_ALIGN, NULL);
	if (!iopte_cache) {
		pr_err("%s: Failed to create sunx-iopte-cacher.\n", __func__);
		return -ENOMEM;
	}
	ret = platform_driver_register(&sunxi_iommu_driver);
	if (ret) {
		pr_err("%s: Failed to register platform driver.\n", __func__);
		goto err_reg_driver;
	}
	pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
	if (IS_ERR(pdev)) {
		ret = PTR_ERR(pdev);
		platform_driver_unregister(&sunxi_iommu_driver);
		kmem_cache_destroy(iopte_cache);
		return ret;
	}
	ret = bus_set_iommu(&platform_bus_type, &sunxi_iommu_ops);
	if (ret) {
		pr_err("%s: Failed to set bus iommu.\n", __func__);
		goto err_set_iommu;
	}
	of_iommu_set_ops(np, &sunxi_iommu_ops);
	return 0;

err_set_iommu:
	platform_driver_unregister(&sunxi_iommu_driver);
err_reg_driver:
	kmem_cache_destroy(iopte_cache);
	return ret;

}

static void __exit sunxi_iommu_exit(void)
{
	kmem_cache_destroy(iopte_cache);
	platform_driver_unregister(&sunxi_iommu_driver);
}


IOMMU_OF_DECLARE(sunxi_iommu_of, "allwinner,sunxi-iommu",
		 sunxi_iommu_of_setup);
module_exit(sunxi_iommu_exit);


MODULE_DESCRIPTION("IOMMU Driver for Allwinner");
MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>");
MODULE_LICENSE("GPL v2");