/* * drivers/dma/sunxi-dma.c * * Copyright (C) 2015-2020 Allwinnertech Co., Ltd * * Author: Sugar * Author: Wim Hwang * * Sunxi DMA controller driver * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "dmaengine.h" #include "virt-dma.h" #if defined(CONFIG_ARCH_SUN8IW1) \ || defined(CONFIG_ARCH_SUN8IW11) \ || defined(CONFIG_ARCH_SUN50IW6)\ || defined(CONFIG_ARCH_SUN8IW12) #define NR_MAX_CHAN 16 /* total of channels */ #elif defined(CONFIG_ARCH_SUN8IW7) \ || defined(CONFIG_ARCH_SUN50IW2) \ || defined(CONFIG_ARCH_SUN50IW3) \ || defined(CONFIG_ARCH_SUN8IW17) #define NR_MAX_CHAN 12 /* total of channels */ #elif defined(CONFIG_ARCH_SUN50IW8) #define NR_MAX_CHAN 10 /* total of channels */ #else #define NR_MAX_CHAN 8 /* total of channels */ #endif #define HIGH_CHAN 8 #define DMA_IRQ_EN(x) (0x000 + ((x) << 2)) /* Interrupt enable register */ #define DMA_IRQ_STAT(x) (0x010 + ((x) << 2)) /* Inetrrupt status register */ #if defined(CONFIG_ARCH_SUN9I) \ || defined(CONFIG_ARCH_SUN8IW7) \ || defined(CONFIG_ARCH_SUN50I)\ || defined(CONFIG_ARCH_SUN8IW12) \ || defined(CONFIG_ARCH_SUN8IW15) \ || defined(CONFIG_ARCH_SUN8IW17) #define DMA_SECU 0x20 /* DMA security register */ #endif #if defined(CONFIG_ARCH_SUN8IW1) #undef DMA_GATE /* no gating register */ #elif defined(CONFIG_ARCH_SUN8IW3) \ || defined(CONFIG_ARCH_SUN8IW5) \ || defined(CONFIG_ARCH_SUN8IW6) \ || defined(CONFIG_ARCH_SUN8IW8) \ || defined(CONFIG_ARCH_SUN8IW9) #define DMA_GATE 0x20 /* DMA gating rgister */ #else #define DMA_GATE 0x28 /* DMA gating rgister */ #endif #define DMA_MCLK_GATE 0x04 #define DMA_COMMON_GATE 0x02 #define DMA_CHAN_GATE 0x01 #define DMA_STAT 0x30 /* DMA Status Register RO */ #define DMA_ENABLE(x) (0x100 + ((x) << 6)) /* Channels enable register */ #define DMA_PAUSE(x) (0x104 + ((x) << 6)) /* DMA Channels pause register */ #define DMA_LLI_ADDR(x) (0x108 + ((x) << 6)) /* Descriptor address register */ #define DMA_CFG(x) (0x10C + ((x) << 6)) /* Configuration register RO */ #define DMA_CUR_SRC(x) (0x110 + ((x) << 6)) /* Current source address RO */ #define DMA_CUR_DST(x) (0x114 + ((x) << 6)) /* Current destination address RO */ #define DMA_CNT(x) (0x118 + ((x) << 6)) /* Byte counter left register RO */ #define DMA_PARA(x) (0x11C + ((x) << 6)) /* Parameter register RO */ #if defined(CONFIG_ARCH_SUN9I) #define LINK_END 0x1FFFF800 /* lastest link must be 0x1ffff800 */ #else #define LINK_END 0xFFFFF800 /* lastest link must be 0xfffff800 */ #endif /* DMA opertions mode */ #if defined(CONFIG_ARCH_SUN8IW1) \ || defined(CONFIG_ARCH_SUN8IW3) \ || defined(CONFIG_ARCH_SUN8IW5) \ || defined(CONFIG_ARCH_SUN8IW6) \ || defined(CONFIG_ARCH_SUN8IW8) \ || defined(CONFIG_ARCH_SUN8IW9) #define DMA_OP_MODE(x) #define SRC_HS_MASK #define DST_HS_MASK #define SET_OP_MODE(d, x, val) do { } while (0) #else #define DMA_OP_MODE(x) (0x128 + ((x) << 6)) /* DMA mode options register */ #define SRC_HS_MASK (0x1 << 2) /* bit 2: Source handshark mode */ #define DST_HS_MASK (0x1 << 3) /* bit 3: Destination handshark mode */ #define SET_OP_MODE(d, x, val) ({ \ writel(val, d->base + DMA_OP_MODE(x)); \ }) #endif #define SHIFT_IRQ_MASK(val, ch) ({ \ (ch) >= HIGH_CHAN \ ? (val) << ((ch - HIGH_CHAN) << 2) \ : (val) << ((ch) << 2); \ }) #define IRQ_HALF 0x01 /* Half package transfer interrupt pending */ #define IRQ_PKG 0x02 /* One package complete interrupt pending */ #define IRQ_QUEUE 0x04 /* All list complete transfer interrupt pending */ /* The detail information of DMA configuration */ #define SRC_WIDTH(x) ((x) << 9) #if defined(CONFIG_ARCH_SUN8IW1) \ || defined(CONFIG_ARCH_SUN8IW3) \ || defined(CONFIG_ARCH_SUN8IW5) \ || defined(CONFIG_ARCH_SUN8IW6) \ || defined(CONFIG_ARCH_SUN8IW8) \ || defined(CONFIG_ARCH_SUN8IW9) #define SRC_BURST(x) ((x) << 7) #else #define SRC_BURST(x) ((x) << 6) #endif #if defined(CONFIG_ARCH_SUN50IW3) \ || defined(CONFIG_ARCH_SUN50IW6)\ || defined(CONFIG_ARCH_SUN8IW12)\ || defined(CONFIG_ARCH_SUN8IW15)\ || defined(CONFIG_ARCH_SUN50IW8)\ || defined(CONFIG_ARCH_SUN8IW17)\ || defined(CONFIG_ARCH_SUN50IW5T) #define SRC_IO_MODE (0x01 << 8) #define SRC_LINEAR_MODE (0x00 << 8) #else #define SRC_IO_MODE (0x01 << 5) #define SRC_LINEAR_MODE (0x00 << 5) #endif #define SRC_DRQ(x) ((x) << 0) #define DST_WIDTH(x) ((x) << 25) #if defined(CONFIG_ARCH_SUN8IW1) \ || defined(CONFIG_ARCH_SUN8IW3) \ || defined(CONFIG_ARCH_SUN8IW5) \ || defined(CONFIG_ARCH_SUN8IW6) \ || defined(CONFIG_ARCH_SUN8IW8) \ || defined(CONFIG_ARCH_SUN8IW9) #define DST_BURST(x) ((x) << 23) #else #define DST_BURST(x) ((x) << 22) #endif #if defined(CONFIG_ARCH_SUN50IW3) \ || defined(CONFIG_ARCH_SUN50IW6)\ || defined(CONFIG_ARCH_SUN8IW12) \ || defined(CONFIG_ARCH_SUN8IW15)\ || defined(CONFIG_ARCH_SUN50IW8)\ || defined(CONFIG_ARCH_SUN8IW17)\ || defined(CONFIG_ARCH_SUN50IW5T) #define DST_IO_MODE (0x01 << 24) #define DST_LINEAR_MODE (0x00 << 24) #else #define DST_IO_MODE (0x01 << 21) #define DST_LINEAR_MODE (0x00 << 21) #endif #define DST_DRQ(x) ((x) << 16) #define CHAN_START 1 #define CHAN_STOP 0 #define CHAN_PAUSE 1 #define CHAN_RESUME 0 #define NORMAL_WAIT (8 << 0) /* * struct sunxi_dma_lli - linked list ltem, the DMA block descriptor * @cfg: DMA configuration * @src: Source address * @dst: Destination address * @len: Length of buffers * @para: Parameter register * @p_lln: Next lli physical address * @v_lln: Next lli virtual address (only for cpu) * @this_phy: Physical address of this lli */ struct sunxi_dma_lli { u32 cfg; u32 src; u32 dst; u32 len; u32 para; u32 p_lln; struct sunxi_dma_lli *v_lln; #ifdef DEBUG u32 this_phy; #define set_this_phy(li, addr) \ ((li)->this_phy = (addr)) #else #define set_this_phy(li, addr) #endif } __packed; struct sunxi_dmadev { struct dma_device dma_dev; void __iomem *base; phys_addr_t pbase; struct clk *ahb_clk; /* AHB clock gate for DMA */ spinlock_t lock; struct tasklet_struct task; struct list_head pending; /* the pending channels list */ struct dma_pool *lli_pool; /* Pool of lli */ }; struct sunxi_desc { struct virt_dma_desc vd; u32 lli_phys; /* physical start for llis */ struct sunxi_dma_lli *lli_virt; /* virtual start for lli */ }; struct sunxi_chan { struct virt_dma_chan vc; struct list_head node; /* queue it to pending list */ struct dma_slave_config cfg; bool cyclic; struct sunxi_desc *desc; u32 irq_type; }; static u64 sunxi_dma_mask = DMA_BIT_MASK(32); static inline struct sunxi_dmadev *to_sunxi_dmadev(struct dma_device *d) { return container_of(d, struct sunxi_dmadev, dma_dev); } static inline struct sunxi_chan *to_sunxi_chan(struct dma_chan *chan) { return container_of(chan, struct sunxi_chan, vc.chan); } static inline struct sunxi_desc *to_sunxi_desc(struct dma_async_tx_descriptor *tx) { return container_of(tx, struct sunxi_desc, vd.tx); } static struct device *chan2dev(struct dma_chan *chan) { return &chan->dev->device; } static struct device *chan2parent(struct dma_chan *chan) { return chan->dev->device.parent; } /* * Fix sconfig's burst size according to sunxi_dmac. We need to convert them as: * 1 -> 0, 4 -> 1, 8 -> 2, 16->3 * * NOTE: burst size 2 is not supported by controller. * * This can be done by finding least significant bit set: n & (n - 1) */ static inline void convert_burst(u32 *maxburst) { if (*maxburst > 1) *maxburst = fls(*maxburst) - 2; else *maxburst = 0; } /* * Fix sconfig's bus width according to at_dmac. * 1 byte -> 0, 2 bytes -> 1, 4 bytes -> 2. */ static inline u8 convert_buswidth(enum dma_slave_buswidth addr_width) { switch (addr_width) { case DMA_SLAVE_BUSWIDTH_2_BYTES: return 1; case DMA_SLAVE_BUSWIDTH_4_BYTES: return 2; default: /* For 1 byte width or fallback */ return 0; } } static size_t sunxi_get_desc_size(struct sunxi_desc *txd) { struct sunxi_dma_lli *lli; size_t size = 0; for (lli = txd->lli_virt; lli != NULL; lli = lli->v_lln) size += lli->len; return size; } /* * sunxi_get_chan_size - get the bytes left of one channel. * @ch: the channel */ static size_t sunxi_get_chan_size(struct sunxi_chan *ch) { struct sunxi_dma_lli *lli; struct sunxi_desc *txd; struct sunxi_dmadev *sdev; size_t size = 0; u32 pos; bool count = false; txd = ch->desc; if (!txd) return 0; sdev = to_sunxi_dmadev(ch->vc.chan.device); pos = readl(sdev->base + DMA_LLI_ADDR(ch->vc.chan.chan_id)); size = readl(sdev->base + DMA_CNT(ch->vc.chan.chan_id)); /* It is the last package, and just read count register */ if (pos == LINK_END) return size; for (lli = txd->lli_virt; lli != NULL; lli = lli->v_lln) { /* Ok, found next lli that is ready be transported */ if (lli->p_lln == pos) { count = true; continue; } if (count) size += lli->len; } return size; } /* * sunxi_free_desc - free the struct sunxi_desc. * @vd: the virt-desc for this chan */ static void sunxi_free_desc(struct virt_dma_desc *vd) { struct sunxi_desc *txd = to_sunxi_desc(&vd->tx); struct sunxi_dmadev *sdev = to_sunxi_dmadev(vd->tx.chan->device); struct sunxi_dma_lli *li_adr, *next_virt; u32 phy, next_phy; if (unlikely(!txd)) return; phy = txd->lli_phys; li_adr = txd->lli_virt; while (li_adr) { next_virt = li_adr->v_lln; next_phy = li_adr->p_lln; dma_pool_free(sdev->lli_pool, li_adr, phy); li_adr = next_virt; phy = next_phy; } txd->vd.tx.callback = NULL; txd->vd.tx.callback_param = NULL; kfree(txd); txd = NULL; } static inline void sunxi_dump_com_regs(struct sunxi_chan *ch) { struct sunxi_dmadev *sdev; sdev = to_sunxi_dmadev(ch->vc.chan.device); pr_debug("Common register:\n" "\tmask0(%04x): 0x%08x\n" "\tmask1(%04x): 0x%08x\n" "\tpend0(%04x): 0x%08x\n" "\tpend1(%04x): 0x%08x\n" #ifdef DMA_SECU "\tsecur(%04x): 0x%08x\n" #endif #ifdef DMA_GATE "\t_gate(%04x): 0x%08x\n" #endif "\tstats(%04x): 0x%08x\n", DMA_IRQ_EN(0), readl(sdev->base + DMA_IRQ_EN(0)), DMA_IRQ_EN(1), readl(sdev->base + DMA_IRQ_EN(1)), DMA_IRQ_STAT(0), readl(sdev->base + DMA_IRQ_STAT(0)), DMA_IRQ_STAT(1), readl(sdev->base + DMA_IRQ_STAT(1)), #ifdef DMA_SECU DMA_SECU, readl(sdev->base + DMA_SECU), #endif #ifdef DMA_GATE DMA_GATE, readl(sdev->base + DMA_GATE), #endif DMA_STAT, readl(sdev->base + DMA_STAT)); } static inline void sunxi_dump_chan_regs(struct sunxi_chan *ch) { struct sunxi_dmadev *sdev = to_sunxi_dmadev(ch->vc.chan.device); u32 chan_num = ch->vc.chan.chan_id; pr_debug("Chan %d reg:\n" "\t___en(%04x): \t0x%08x\n" "\tpause(%04x): \t0x%08x\n" "\tstart(%04x): \t0x%08x\n" "\t__cfg(%04x): \t0x%08x\n" "\t__src(%04x): \t0x%08x\n" "\t__dst(%04x): \t0x%08x\n" "\tcount(%04x): \t0x%08x\n" "\t_para(%04x): \t0x%08x\n\n", chan_num, DMA_ENABLE(chan_num), readl(sdev->base + DMA_ENABLE(chan_num)), DMA_PAUSE(chan_num), readl(sdev->base + DMA_PAUSE(chan_num)), DMA_LLI_ADDR(chan_num), readl(sdev->base + DMA_LLI_ADDR(chan_num)), DMA_CFG(chan_num), readl(sdev->base + DMA_CFG(chan_num)), DMA_CUR_SRC(chan_num), readl(sdev->base + DMA_CUR_SRC(chan_num)), DMA_CUR_DST(chan_num), readl(sdev->base + DMA_CUR_DST(chan_num)), DMA_CNT(chan_num), readl(sdev->base + DMA_CNT(chan_num)), DMA_PARA(chan_num), readl(sdev->base + DMA_PARA(chan_num))); } /* * sunxi_dma_resume - resume channel, which is pause sate. * @chan: the channel to resume */ static int sunxi_dma_resume(struct dma_chan *chan) { struct sunxi_chan *schan = to_sunxi_chan(chan); struct sunxi_dmadev *sdev = to_sunxi_dmadev(schan->vc.chan.device); u32 chan_num = schan->vc.chan.chan_id; writel(CHAN_RESUME, sdev->base + DMA_PAUSE(chan_num)); return 0; } static int sunxi_dma_pause(struct dma_chan *chan) { struct sunxi_chan *schan = to_sunxi_chan(chan); struct sunxi_dmadev *sdev = to_sunxi_dmadev(schan->vc.chan.device); u32 chan_num = schan->vc.chan.chan_id; writel(CHAN_PAUSE, sdev->base + DMA_PAUSE(chan_num)); return 0; } /* * sunxi_terminate_all - stop all descriptors that waiting transfer on chan. * @ch: the channel to stop */ static int sunxi_terminate_all(struct dma_chan *chan) { struct sunxi_chan *schan = to_sunxi_chan(chan); struct sunxi_dmadev *sdev = to_sunxi_dmadev(schan->vc.chan.device); struct virt_dma_desc *vd = NULL; struct virt_dma_chan *vc = NULL; u32 chan_num = schan->vc.chan.chan_id; unsigned long flags; LIST_HEAD(head); spin_lock_irqsave(&schan->vc.lock, flags); spin_lock(&sdev->lock); list_del_init(&schan->node); spin_unlock(&sdev->lock); /* We should entry PAUSE state first to avoid missing data * count which transferring on bus. */ writel(CHAN_PAUSE, sdev->base + DMA_PAUSE(chan_num)); writel(CHAN_STOP, sdev->base + DMA_ENABLE(chan_num)); writel(CHAN_RESUME, sdev->base + DMA_PAUSE(chan_num)); /* At cyclic mode, desc is not be managed by virt-dma, * we need to add it to desc_completed */ if (schan->cyclic) { schan->cyclic = false; if (schan->desc) { vd = &(schan->desc->vd); vc = &(schan->vc); list_add_tail(&vd->node, &vc->desc_completed); } } schan->desc = NULL; vchan_get_all_descriptors(&schan->vc, &head); spin_unlock_irqrestore(&schan->vc.lock, flags); vchan_dma_desc_free_list(&schan->vc, &head); return 0; } /* * sunxi_start_desc - begin to transport the descriptor * @ch: the channel of descriptor */ static void sunxi_start_desc(struct sunxi_chan *ch) { struct virt_dma_desc *vd = vchan_next_desc(&ch->vc); struct sunxi_dmadev *sdev = to_sunxi_dmadev(ch->vc.chan.device); struct sunxi_desc *txd = NULL; u32 chan_num = ch->vc.chan.chan_id; u32 irq_val; u32 high; if (!vd) { while (readl(sdev->base + DMA_STAT) & (1 << chan_num)) cpu_relax(); writel(CHAN_STOP, sdev->base + DMA_ENABLE(chan_num)); return; } /* Delete this desc from the desc_issued list */ list_del(&vd->node); txd = to_sunxi_desc(&vd->tx); ch->desc = txd; if (ch->cyclic) ch->irq_type = IRQ_PKG; else ch->irq_type = IRQ_QUEUE; high = (chan_num >= HIGH_CHAN) ? 1 : 0; irq_val = readl(sdev->base + DMA_IRQ_EN(high)); irq_val |= SHIFT_IRQ_MASK(ch->irq_type, chan_num); writel(irq_val, sdev->base + DMA_IRQ_EN(high)); /* Set the DMA opertions mode */ SET_OP_MODE(sdev, chan_num, SRC_HS_MASK | DST_HS_MASK); /* write the first lli address to register, and start to transfer */ writel(txd->lli_phys, sdev->base + DMA_LLI_ADDR(chan_num)); writel(CHAN_START, sdev->base + DMA_ENABLE(chan_num)); sunxi_dump_com_regs(ch); sunxi_dump_chan_regs(ch); } /* * sunxi_alloc_lli - Allocate a sunxi_lli * @sdev: the sunxi_dmadev * @phy_addr: return the physical address */ void *sunxi_alloc_lli(struct sunxi_dmadev *sdev, u32 *phy_addr) { struct sunxi_dma_lli *l_item; dma_addr_t phy; WARN_TAINT(!sdev->lli_pool, TAINT_WARN, "The dma pool is empty!!\n"); if (unlikely(!sdev->lli_pool)) return NULL; l_item = dma_pool_alloc(sdev->lli_pool, GFP_ATOMIC, &phy); *phy_addr = (u32)phy; set_this_phy(l_item, *phy_addr); return l_item; } /* * sunxi_dump_lli - dump the information for one lli * @shcan: the channel * @lli: a lli to dump */ static inline void sunxi_dump_lli(struct sunxi_chan *schan, struct sunxi_dma_lli *lli) { #ifdef DEBUG dev_dbg(chan2dev(&schan->vc.chan), "\n\tdesc: p - 0x%08x v - 0x%08x\n" "\t\tc - 0x%08x s - 0x%08x d - 0x%08x\n" "\t\tl - 0x%08x p - 0x%08x n - 0x%08x\n", lli->this_phy, (u32)lli, lli->cfg, lli->src, lli->dst, lli->len, lli->para, lli->p_lln); #endif } static void *sunxi_lli_list(struct sunxi_dma_lli *prev, struct sunxi_dma_lli *next, u32 next_phy, struct sunxi_desc *txd) { if ((!prev && !txd) || !next) return NULL; if (!prev) { txd->lli_phys = next_phy; txd->lli_virt = next; } else { prev->p_lln = next_phy; prev->v_lln = next; } next->p_lln = LINK_END; next->v_lln = NULL; return next; } static inline void sunxi_cfg_lli(struct sunxi_dma_lli *lli, dma_addr_t src, dma_addr_t dst, u32 len, struct dma_slave_config *config) { u32 src_width, dst_width; if (!config) return; /* Get the data width */ src_width = convert_buswidth(config->src_addr_width); dst_width = convert_buswidth(config->dst_addr_width); lli->cfg = SRC_BURST(config->src_maxburst) | SRC_WIDTH(src_width) | DST_BURST(config->dst_maxburst) | DST_WIDTH(dst_width); lli->src = (u32)src; lli->dst = (u32)dst; lli->len = len; lli->para = NORMAL_WAIT; } /* * sunxi_dma_tasklet - ensure that the desc's lli be putted into hardware. * @data: sunxi_dmadev */ static void sunxi_dma_tasklet(unsigned long data) { struct sunxi_dmadev *sdev = (struct sunxi_dmadev *)data; LIST_HEAD(head); spin_lock_irq(&sdev->lock); list_splice_tail_init(&sdev->pending, &head); spin_unlock_irq(&sdev->lock); while (!list_empty(&head)) { struct sunxi_chan *c = list_first_entry(&head, struct sunxi_chan, node); spin_lock_irq(&c->vc.lock); list_del_init(&c->node); sunxi_start_desc(c); spin_unlock_irq(&c->vc.lock); } } /* * sunxi_dma_interrupt - interrupt handle. * @irq: irq number * @dev_id: sunxi_dmadev */ static irqreturn_t sunxi_dma_interrupt(int irq, void *dev_id) { struct sunxi_dmadev *sdev = (struct sunxi_dmadev *)dev_id; struct sunxi_chan *ch = NULL; struct sunxi_desc *desc; unsigned long flags; u32 status_lo = 0, status_hi = 0; /* Get the status of irq */ status_lo = readl(sdev->base + DMA_IRQ_STAT(0)); #if NR_MAX_CHAN > HIGH_CHAN status_hi = readl(sdev->base + DMA_IRQ_STAT(1)); #endif dev_dbg(sdev->dma_dev.dev, "[sunxi_dma]: DMA irq status_lo: 0x%08x, status_hi: 0x%08x\n", status_lo, status_hi); /* Clear the bit of irq status */ writel(status_lo, sdev->base + DMA_IRQ_STAT(0)); #if NR_MAX_CHAN > HIGH_CHAN writel(status_hi, sdev->base + DMA_IRQ_STAT(1)); #endif list_for_each_entry(ch, &sdev->dma_dev.channels, vc.chan.device_node) { u32 chan_num = ch->vc.chan.chan_id; u32 status; status = (chan_num >= HIGH_CHAN) ? (status_hi >> ((chan_num - HIGH_CHAN) << 2)) : (status_lo >> (chan_num << 2)); spin_lock_irqsave(&ch->vc.lock, flags); if (!(ch->irq_type & status)) goto unlock; if (!ch->desc) goto unlock; desc = ch->desc; if (ch->cyclic) { struct virt_dma_desc *vd; dma_async_tx_callback cb = NULL; void *cb_data = NULL; vd = &desc->vd; if (vd) { cb = vd->tx.callback; cb_data = vd->tx.callback_param; } spin_unlock_irqrestore(&ch->vc.lock, flags); if (cb) cb(cb_data); spin_lock_irqsave(&ch->vc.lock, flags); } else { ch->desc = NULL; vchan_cookie_complete(&desc->vd); sunxi_start_desc(ch); } unlock: spin_unlock_irqrestore(&ch->vc.lock, flags); } return IRQ_HANDLED; } static struct dma_async_tx_descriptor *sunxi_prep_dma_memcpy( struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) { struct sunxi_chan *schan = to_sunxi_chan(chan); struct sunxi_desc *txd; struct sunxi_dma_lli *l_item; struct sunxi_dmadev *sdev = to_sunxi_dmadev(chan->device); struct dma_slave_config *sconfig = &schan->cfg; u32 phy; dev_dbg(chan2dev(chan), "chan: %d, dest: 0x%08lx, src: 0x%08lx, len: 0x%08zx, flags: 0x%08lx\n", schan->vc.chan.chan_id, (unsigned long)dest, (unsigned long)src, len, flags); if (unlikely(!len)) { dev_dbg(chan2dev(chan), "memcpy length is zero!\n"); return NULL; } txd = kzalloc(sizeof(*txd), GFP_NOWAIT); if (!txd) return NULL; vchan_tx_prep(&schan->vc, &txd->vd, flags); l_item = sunxi_alloc_lli(sdev, &phy); if (!l_item) { sunxi_free_desc(&txd->vd); dev_err(sdev->dma_dev.dev, "Failed to alloc lli memory!\n"); return NULL; } sunxi_cfg_lli(l_item, src, dest, len, sconfig); l_item->cfg |= SRC_DRQ(DRQSRC_SDRAM) | DST_DRQ(DRQDST_SDRAM) | DST_LINEAR_MODE | SRC_LINEAR_MODE; sunxi_lli_list(NULL, l_item, phy, txd); sunxi_dump_lli(schan, l_item); return &txd->vd.tx; } static struct dma_async_tx_descriptor *sunxi_prep_dma_sg( struct dma_chan *chan, struct scatterlist *dst_sg, unsigned int dst_nents, struct scatterlist *src_sg, unsigned int src_nents, unsigned long flags) { struct sunxi_chan *schan = to_sunxi_chan(chan); struct sunxi_dmadev *sdev = to_sunxi_dmadev(chan->device); struct dma_slave_config *sconfig = &schan->cfg; struct sunxi_desc *txd; struct sunxi_dma_lli *l_item, *prev = NULL; u32 phy; if (dst_nents != src_nents) return NULL; if (!dst_nents || !src_nents) return NULL; if (dst_sg == NULL || src_sg == NULL) return NULL; txd = kzalloc(sizeof(*txd), GFP_NOWAIT); if (!txd) return NULL; vchan_tx_prep(&schan->vc, &txd->vd, flags); while ((src_sg != NULL) && (dst_sg != NULL)) { l_item = sunxi_alloc_lli(sdev, &phy); if (!l_item) { sunxi_free_desc(&txd->vd); return NULL; } sunxi_cfg_lli(l_item, sg_dma_address(src_sg), sg_dma_address(dst_sg), sg_dma_len(dst_sg), sconfig); l_item->cfg |= SRC_LINEAR_MODE | DST_LINEAR_MODE | GET_DST_DRQ(sconfig->slave_id) | GET_SRC_DRQ(sconfig->slave_id); prev = sunxi_lli_list(prev, l_item, phy, txd); src_sg = sg_next(src_sg); dst_sg = sg_next(dst_sg); } #ifdef DEBUG pr_debug("[sunxi_dma]: First: 0x%08x\n", txd->lli_phys); for (prev = txd->lli_virt; prev != NULL; prev = prev->v_lln) sunxi_dump_lli(schan, prev); #endif return &txd->vd.tx; } static struct dma_async_tx_descriptor *sunxi_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction dir, unsigned long flags, void *context) { struct sunxi_chan *schan = to_sunxi_chan(chan); struct sunxi_desc *txd; struct sunxi_dma_lli *l_item, *prev = NULL; struct sunxi_dmadev *sdev = to_sunxi_dmadev(chan->device); struct dma_slave_config *sconfig = &schan->cfg; struct scatterlist *sg = NULL; u32 phy; unsigned int i = 0; if (unlikely(!sg_len)) { dev_dbg(chan2dev(chan), "sg length is zero!!\n"); return NULL; } txd = kzalloc(sizeof(*txd), GFP_NOWAIT); if (!txd) return NULL; vchan_tx_prep(&schan->vc, &txd->vd, flags); for_each_sg(sgl, sg, sg_len, i) { l_item = sunxi_alloc_lli(sdev, &phy); if (!l_item) { sunxi_free_desc(&txd->vd); return NULL; } if (dir == DMA_MEM_TO_DEV) { sunxi_cfg_lli(l_item, sg_dma_address(sg), sconfig->dst_addr, sg_dma_len(sg), sconfig); l_item->cfg |= DST_IO_MODE | SRC_LINEAR_MODE | SRC_DRQ(DRQSRC_SDRAM) | GET_DST_DRQ(sconfig->slave_id); } else if (dir == DMA_DEV_TO_MEM) { sunxi_cfg_lli(l_item, sconfig->src_addr, sg_dma_address(sg), sg_dma_len(sg), sconfig); l_item->cfg |= DST_LINEAR_MODE | SRC_IO_MODE | DST_DRQ(DRQDST_SDRAM) | GET_SRC_DRQ(sconfig->slave_id); } prev = sunxi_lli_list(prev, l_item, phy, txd); } #ifdef DEBUG pr_debug("[sunxi_dma]: First: 0x%08x\n", txd->lli_phys); for (prev = txd->lli_virt; prev != NULL; prev = prev->v_lln) sunxi_dump_lli(schan, prev); #endif return &txd->vd.tx; } /** * sunxi_prep_dma_cyclic - prepare the cyclic DMA transfer * @chan: the DMA channel to prepare * @buf_addr: physical DMA address where the buffer starts * @buf_len: total number of bytes for the entire buffer * @period_len: number of bytes for each period * @dir: transfer direction, to or from device * * Must be called before trying to start the transfer. Returns a valid struct * sunxi_cyclic_desc if successful or an ERR_PTR(-errno) if not successful. */ struct dma_async_tx_descriptor *sunxi_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction dir, unsigned long flags) { struct sunxi_desc *txd; struct sunxi_chan *schan = to_sunxi_chan(chan); struct sunxi_dmadev *sdev = to_sunxi_dmadev(chan->device); struct sunxi_dma_lli *l_item, *prev = NULL; struct dma_slave_config *sconfig = &schan->cfg; u32 phy; unsigned int periods = buf_len / period_len; unsigned int i; /* * Not allow duplicate prep dma on cyclic channel. */ if (schan->desc && schan->cyclic) return NULL; txd = kzalloc(sizeof(*txd), GFP_NOWAIT); if (!txd) return NULL; vchan_tx_prep(&schan->vc, &txd->vd, flags); for (i = 0; i < periods; i++) { l_item = sunxi_alloc_lli(sdev, &phy); if (!l_item) { sunxi_free_desc(&txd->vd); return NULL; } if (dir == DMA_MEM_TO_DEV) { sunxi_cfg_lli(l_item, (buf_addr + period_len * i), sconfig->dst_addr, period_len, sconfig); l_item->cfg |= GET_DST_DRQ(sconfig->slave_id) | SRC_LINEAR_MODE | DST_IO_MODE | SRC_DRQ(DRQSRC_SDRAM); } else if (dir == DMA_DEV_TO_MEM) { sunxi_cfg_lli(l_item, sconfig->src_addr, (buf_addr + period_len * i), period_len, sconfig); l_item->cfg |= GET_SRC_DRQ(sconfig->slave_id) | DST_LINEAR_MODE | SRC_IO_MODE | DST_DRQ(DRQDST_SDRAM); } else if (dir == DMA_DEV_TO_DEV) { sunxi_cfg_lli(l_item, sconfig->src_addr, sconfig->dst_addr, period_len, sconfig); l_item->cfg |= GET_SRC_DRQ(sconfig->slave_id) | DST_IO_MODE | SRC_IO_MODE | GET_DST_DRQ(sconfig->slave_id); } prev = sunxi_lli_list(prev, l_item, phy, txd); } /* Make a cyclic list */ prev->p_lln = txd->lli_phys; schan->cyclic = true; #ifdef DEBUG pr_debug("[sunxi_dma]: First: 0x%08x\n", txd->lli_phys); for (prev = txd->lli_virt; prev != NULL; prev = prev->v_lln) sunxi_dump_lli(schan, prev); #endif return &txd->vd.tx; } static int sunxi_set_runtime_config(struct dma_chan *chan, struct dma_slave_config *config) { struct sunxi_chan *schan = to_sunxi_chan(chan); memcpy(&schan->cfg, config, sizeof(struct dma_slave_config)); convert_burst(&schan->cfg.src_maxburst); convert_burst(&schan->cfg.dst_maxburst); return 0; } static enum dma_status sunxi_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { struct sunxi_chan *schan = to_sunxi_chan(chan); struct virt_dma_desc *vd; enum dma_status ret; unsigned long flags; size_t bytes = 0; ret = dma_cookie_status(chan, cookie, txstate); if (ret == DMA_COMPLETE || !txstate) return ret; spin_lock_irqsave(&schan->vc.lock, flags); vd = vchan_find_desc(&schan->vc, cookie); if (vd) bytes = sunxi_get_desc_size(to_sunxi_desc(&vd->tx)); else if (schan->desc && schan->desc->vd.tx.cookie == cookie) bytes = sunxi_get_chan_size(to_sunxi_chan(chan)); /* * This cookie not complete yet * Get number of bytes left in the active transactions and queue */ dma_set_residue(txstate, bytes); spin_unlock_irqrestore(&schan->vc.lock, flags); return ret; } /* * sunxi_issue_pending - try to finish work * @chan: target DMA channel * * It will call vchan_issue_pending(), which can move the desc_submitted * list to desc_issued list. And we will move the chan to pending list of * sunxi_dmadev. */ static void sunxi_issue_pending(struct dma_chan *chan) { struct sunxi_chan *schan = to_sunxi_chan(chan); struct sunxi_dmadev *sdev = to_sunxi_dmadev(chan->device); unsigned long flags; spin_lock_irqsave(&schan->vc.lock, flags); if (vchan_issue_pending(&schan->vc) && !schan->desc) { if (schan->cyclic) { sunxi_start_desc(schan); goto out; } spin_lock(&sdev->lock); if (list_empty(&schan->node)) list_add_tail(&schan->node, &sdev->pending); spin_unlock(&sdev->lock); tasklet_schedule(&sdev->task); } out: spin_unlock_irqrestore(&schan->vc.lock, flags); } static int sunxi_alloc_chan_resources(struct dma_chan *chan) { struct sunxi_chan *schan = to_sunxi_chan(chan); dev_dbg(chan2parent(chan), "%s: Now alloc chan resources!\n", __func__); schan->cyclic = false; return 0; } /* * sunxi_free_chan_resources - free the resources of channel * @chan: the channel to free */ static void sunxi_free_chan_resources(struct dma_chan *chan) { struct sunxi_chan *schan = to_sunxi_chan(chan); vchan_free_chan_resources(&schan->vc); dev_dbg(chan2parent(chan), "%s: Now free chan resources!!\n", __func__); } /* * sunxi_chan_free - free the channel on dmadevice * @sdev: the dmadevice of sunxi */ static inline void sunxi_chan_free(struct sunxi_dmadev *sdev) { struct sunxi_chan *ch; tasklet_kill(&sdev->task); while (!list_empty(&sdev->dma_dev.channels)) { ch = list_first_entry(&sdev->dma_dev.channels, struct sunxi_chan, vc.chan.device_node); list_del(&ch->vc.chan.device_node); tasklet_kill(&ch->vc.task); kfree(ch); } } static void sunxi_dma_hw_init(struct sunxi_dmadev *dev) { struct sunxi_dmadev *sunxi_dev = dev; clk_prepare_enable(sunxi_dev->ahb_clk); #if defined(CONFIG_SUNXI_SMC) sunxi_smc_writel(0xff, sunxi_dev->pbase + DMA_SECU); #endif #if defined(CONFIG_ARCH_SUN8IW3) || \ defined(CONFIG_ARCH_SUN8IW5) || \ defined(CONFIG_ARCH_SUN8IW6) || \ defined(CONFIG_ARCH_SUN8IW8) writel(DMA_MCLK_GATE|DMA_COMMON_GATE|DMA_CHAN_GATE, sunxi_dev->base + DMA_GATE); #endif } static int sunxi_probe(struct platform_device *pdev) { struct sunxi_dmadev *sunxi_dev; struct sunxi_chan *schan; struct resource *res; int irq; int ret, i; pdev->dev.dma_mask = &sunxi_dma_mask; pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32); sunxi_dev = kzalloc(sizeof(struct sunxi_dmadev), GFP_KERNEL); if (!sunxi_dev) return -ENOMEM; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) { ret = -EINVAL; goto io_err; } sunxi_dev->pbase = res->start; sunxi_dev->base = ioremap(res->start, resource_size(res)); if (!sunxi_dev->base) { dev_err(&pdev->dev, "Remap I/O memory failed!\n"); ret = -ENOMEM; goto io_err; } irq = platform_get_irq(pdev, 0); if (irq < 0) { ret = irq; goto irq_err; } ret = request_irq(irq, sunxi_dma_interrupt, IRQF_SHARED, dev_name(&pdev->dev), sunxi_dev); if (ret) { dev_err(&pdev->dev, "NO IRQ found!!!\n"); goto irq_err; } sunxi_dev->ahb_clk = clk_get(&pdev->dev, "dma"); if (!sunxi_dev->ahb_clk) { dev_err(&pdev->dev, "NO clock to dma!!!\n"); ret = -EINVAL; goto clk_err; } sunxi_dev->lli_pool = dma_pool_create(dev_name(&pdev->dev), &pdev->dev, sizeof(struct sunxi_dma_lli), 4/* word alignment */, 0); if (!sunxi_dev->lli_pool) { ret = -ENOMEM; goto pool_err; } platform_set_drvdata(pdev, sunxi_dev); INIT_LIST_HEAD(&sunxi_dev->pending); spin_lock_init(&sunxi_dev->lock); /* Initialize dmaengine */ dma_cap_set(DMA_MEMCPY, sunxi_dev->dma_dev.cap_mask); dma_cap_set(DMA_SLAVE, sunxi_dev->dma_dev.cap_mask); dma_cap_set(DMA_CYCLIC, sunxi_dev->dma_dev.cap_mask); dma_cap_set(DMA_SG, sunxi_dev->dma_dev.cap_mask); INIT_LIST_HEAD(&sunxi_dev->dma_dev.channels); sunxi_dev->dma_dev.device_alloc_chan_resources = sunxi_alloc_chan_resources; sunxi_dev->dma_dev.device_free_chan_resources = sunxi_free_chan_resources; sunxi_dev->dma_dev.device_tx_status = sunxi_tx_status; sunxi_dev->dma_dev.device_issue_pending = sunxi_issue_pending; sunxi_dev->dma_dev.device_prep_dma_sg = sunxi_prep_dma_sg; sunxi_dev->dma_dev.device_prep_slave_sg = sunxi_prep_slave_sg; sunxi_dev->dma_dev.device_prep_dma_cyclic = sunxi_prep_dma_cyclic; sunxi_dev->dma_dev.device_prep_dma_memcpy = sunxi_prep_dma_memcpy; sunxi_dev->dma_dev.device_config = sunxi_set_runtime_config; sunxi_dev->dma_dev.device_pause = sunxi_dma_pause; sunxi_dev->dma_dev.device_resume = sunxi_dma_resume; sunxi_dev->dma_dev.device_terminate_all = sunxi_terminate_all; sunxi_dev->dma_dev.dev = &pdev->dev; tasklet_init(&sunxi_dev->task, sunxi_dma_tasklet, (unsigned long)sunxi_dev); for (i = 0; i < NR_MAX_CHAN; i++) { schan = kzalloc(sizeof(*schan), GFP_KERNEL); if (!schan) { dev_err(&pdev->dev, "no memory for channel\n"); ret = -ENOMEM; goto chan_err; } INIT_LIST_HEAD(&schan->node); sunxi_dev->dma_dev.chancnt++; schan->vc.desc_free = sunxi_free_desc; vchan_init(&schan->vc, &sunxi_dev->dma_dev); } /* Register the sunxi-dma to dmaengine */ ret = dma_async_device_register(&sunxi_dev->dma_dev); if (ret) { dev_warn(&pdev->dev, "Failed to register DMA engine device\n"); goto chan_err; } /* All is ok, and open the clock */ sunxi_dma_hw_init(sunxi_dev); return 0; chan_err: sunxi_chan_free(sunxi_dev); platform_set_drvdata(pdev, NULL); dma_pool_destroy(sunxi_dev->lli_pool); pool_err: clk_put(sunxi_dev->ahb_clk); clk_err: free_irq(irq, sunxi_dev); irq_err: iounmap(sunxi_dev->base); io_err: kfree(sunxi_dev); return ret; } static int sunxi_remove(struct platform_device *pdev) { struct sunxi_dmadev *sunxi_dev = platform_get_drvdata(pdev); dma_async_device_unregister(&sunxi_dev->dma_dev); sunxi_chan_free(sunxi_dev); free_irq(platform_get_irq(pdev, 0), sunxi_dev); dma_pool_destroy(sunxi_dev->lli_pool); clk_disable_unprepare(sunxi_dev->ahb_clk); clk_put(sunxi_dev->ahb_clk); iounmap(sunxi_dev->base); kfree(sunxi_dev); return 0; } static void sunxi_shutdown(struct platform_device *pdev) { struct sunxi_dmadev *sdev = platform_get_drvdata(pdev); clk_disable_unprepare(sdev->ahb_clk); } static int sunxi_suspend_noirq(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct sunxi_dmadev *sunxi_dev = platform_get_drvdata(pdev); clk_disable_unprepare(sunxi_dev->ahb_clk); return 0; } static int sunxi_resume_noirq(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct sunxi_dmadev *sunxi_dev = platform_get_drvdata(pdev); sunxi_dma_hw_init(sunxi_dev); return 0; } static const struct dev_pm_ops sunxi_dev_pm_ops = { .suspend_noirq = sunxi_suspend_noirq, .resume_noirq = sunxi_resume_noirq, .freeze_noirq = sunxi_suspend_noirq, .thaw_noirq = sunxi_resume_noirq, .restore_noirq = sunxi_resume_noirq, .poweroff_noirq = sunxi_suspend_noirq, }; #ifndef CONFIG_OF static struct resource sunxi_dma_reousce[] = { [0] = { .start = DMA_PHYS_BASE, .end = DMA_PHYS_BASE + DMA_PARA(15), .flags = IORESOURCE_MEM, }, [1] = { .start = DMA_IRQ_ID, .end = DMA_IRQ_ID, .flags = IORESOURCE_IRQ, }, }; static struct platform_device sunxi_dma_device = { .name = "sunxi_dmac", .id = -1, .resource = sunxi_dma_reousce, .num_resources = ARRAY_SIZE(sunxi_dma_reousce), .dev = { .dma_mask = &sunxi_dma_mask, .coherent_dma_mask = DMA_BIT_MASK(32), }, }; #else static const struct of_device_id sunxi_dma_match[] = { { .compatible = "allwinner,sun50i-dma", }, { .compatible = "allwinner,sun8i-dma", }, {}, }; #endif static struct platform_driver sunxi_dma_driver = { .probe = sunxi_probe, .remove = sunxi_remove, .shutdown = sunxi_shutdown, .driver = { .name = "sunxi_dmac", .pm = &sunxi_dev_pm_ops, .of_match_table = sunxi_dma_match, }, }; bool sunxi_dma_filter_fn(struct dma_chan *chan, void *param) { bool ret = false; if (chan->device->dev->driver == &sunxi_dma_driver.driver) { const char *p = param; ret = !strcmp("sunxi_dmac", p); pr_debug("[sunxi_rdma]: sunxi_dma_filter_fn: %s\n", p); } return ret; } EXPORT_SYMBOL_GPL(sunxi_dma_filter_fn); static int __init sunxi_dma_init(void) { int ret; #ifndef CONFIG_OF platform_device_register(&sunxi_dma_device); #endif ret = platform_driver_register(&sunxi_dma_driver); return ret; } subsys_initcall(sunxi_dma_init); static void __exit sunxi_dma_exit(void) { platform_driver_unregister(&sunxi_dma_driver); #ifndef CONFIG_OF platform_device_unregister(&sunxi_dma_device); #endif } module_exit(sunxi_dma_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Sunxi DMA Controller driver"); MODULE_AUTHOR("Shuge"); MODULE_AUTHOR("Wim Hwang"); MODULE_ALIAS("platform:sunxi_dmac");