19b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#include <linux/device.h> 29b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#include <linux/dma-mapping.h> 39b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#include <linux/dmaengine.h> 49b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#include <linux/sizes.h> 59b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#include <linux/platform_device.h> 69b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#include <linux/of.h> 79b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 89b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#include "musb_core.h" 99b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 109b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define RNDIS_REG(x) (0x80 + ((x - 1) * 4)) 119b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 129b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define EP_MODE_AUTOREG_NONE 0 139b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define EP_MODE_AUTOREG_ALL_NEOP 1 149b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define EP_MODE_AUTOREG_ALWAYS 3 159b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 169b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define EP_MODE_DMA_TRANSPARENT 0 179b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define EP_MODE_DMA_RNDIS 1 189b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define EP_MODE_DMA_GEN_RNDIS 3 199b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 209b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define USB_CTRL_TX_MODE 0x70 219b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define USB_CTRL_RX_MODE 0x74 229b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define USB_CTRL_AUTOREQ 0xd0 239b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define USB_TDOWN 0xd8 249b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 259b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstruct cppi41_dma_channel { 269b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct dma_channel channel; 279b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_controller *controller; 289b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct musb_hw_ep *hw_ep; 299b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct dma_chan *dc; 309b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_cookie_t cookie; 319b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u8 port_num; 329b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u8 is_tx; 339b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u8 is_allocated; 349b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u8 usb_toggle; 359b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 369b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_addr_t buf_addr; 379b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 total_len; 389b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 prog_len; 399b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 transferred; 409b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 packet_sz; 41a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior struct list_head tx_check; 429267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu int tx_zlp; 439b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior}; 449b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 459b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior#define MUSB_DMA_NUM_CHANNELS 15 469b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 479b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstruct cppi41_dma_controller { 489b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct dma_controller controller; 499b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_channel rx_channel[MUSB_DMA_NUM_CHANNELS]; 509b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_channel tx_channel[MUSB_DMA_NUM_CHANNELS]; 519b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct musb *musb; 52a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior struct hrtimer early_tx; 53a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior struct list_head early_tx_list; 549b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 rx_mode; 559b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 tx_mode; 569b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 auto_req; 579b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior}; 589b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 599b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic void save_rx_toggle(struct cppi41_dma_channel *cppi41_channel) 609b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 619b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u16 csr; 629b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u8 toggle; 639b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 649b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (cppi41_channel->is_tx) 659b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return; 669b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (!is_host_active(cppi41_channel->controller->musb)) 679b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return; 689b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 699b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR); 709b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0; 719b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 729b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->usb_toggle = toggle; 739b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 749b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 759b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel) 769b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 77f50e67853b363b96336718597823ed7a7e8652deDaniel Mack struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep; 78f50e67853b363b96336718597823ed7a7e8652deDaniel Mack struct musb *musb = hw_ep->musb; 799b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u16 csr; 809b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u8 toggle; 819b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 829b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (cppi41_channel->is_tx) 839b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return; 84f50e67853b363b96336718597823ed7a7e8652deDaniel Mack if (!is_host_active(musb)) 859b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return; 869b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 87f50e67853b363b96336718597823ed7a7e8652deDaniel Mack musb_ep_select(musb->mregs, hw_ep->epnum); 88f50e67853b363b96336718597823ed7a7e8652deDaniel Mack csr = musb_readw(hw_ep->regs, MUSB_RXCSR); 899b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0; 909b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 919b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior /* 929b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior * AM335x Advisory 1.0.13: Due to internal synchronisation error the 939b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior * data toggle may reset from DATA1 to DATA0 during receiving data from 949b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior * more than one endpoint. 959b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior */ 969b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (!toggle && toggle == cppi41_channel->usb_toggle) { 979b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr |= MUSB_RXCSR_H_DATATOGGLE | MUSB_RXCSR_H_WR_DATATOGGLE; 989b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writew(cppi41_channel->hw_ep->regs, MUSB_RXCSR, csr); 999b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dev_dbg(cppi41_channel->controller->musb->controller, 1009b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior "Restoring DATA1 toggle.\n"); 1019b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 1029b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 1039b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->usb_toggle = toggle; 1049b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 1059b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 106a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewiorstatic bool musb_is_tx_fifo_empty(struct musb_hw_ep *hw_ep) 107a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior{ 108a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior u8 epnum = hw_ep->epnum; 109a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior struct musb *musb = hw_ep->musb; 110a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior void __iomem *epio = musb->endpoints[epnum].regs; 111a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior u16 csr; 112a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior 113f50e67853b363b96336718597823ed7a7e8652deDaniel Mack musb_ep_select(musb->mregs, hw_ep->epnum); 114a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior csr = musb_readw(epio, MUSB_TXCSR); 115a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior if (csr & MUSB_TXCSR_TXPKTRDY) 116a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior return false; 117a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior return true; 118a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior} 119a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior 120d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewiorstatic void cppi41_dma_callback(void *private_data); 121d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior 122a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewiorstatic void cppi41_trans_done(struct cppi41_dma_channel *cppi41_channel) 1239b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 1249b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep; 1259b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct musb *musb = hw_ep->musb; 1269267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu void __iomem *epio = hw_ep->regs; 1279267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu u16 csr; 1289b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 129aecbc31d767cb549e93a44e50218e20d1bc66b59George Cherian if (!cppi41_channel->prog_len || 130aecbc31d767cb549e93a44e50218e20d1bc66b59George Cherian (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)) { 1319b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 1329b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior /* done, complete */ 1339b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->channel.actual_len = 1349b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->transferred; 1359b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE; 136ff3fcac949187d98684aaf3f1c35c7cae7712649Daniel Mack cppi41_channel->channel.rx_packet_done = true; 1379267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu 1389267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu /* 1399267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu * transmit ZLP using PIO mode for transfers which size is 1409267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu * multiple of EP packet size. 1419267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu */ 1429267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu if (cppi41_channel->tx_zlp && (cppi41_channel->transferred % 1439267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu cppi41_channel->packet_sz) == 0) { 1449267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu musb_ep_select(musb->mregs, hw_ep->epnum); 1459267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu csr = MUSB_TXCSR_MODE | MUSB_TXCSR_TXPKTRDY; 1469267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu musb_writew(epio, MUSB_TXCSR, csr); 1479267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu } 1489b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_dma_completion(musb, hw_ep->epnum, cppi41_channel->is_tx); 1499b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } else { 1509b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior /* next iteration, reload */ 1519b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct dma_chan *dc = cppi41_channel->dc; 1529b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct dma_async_tx_descriptor *dma_desc; 1539b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior enum dma_transfer_direction direction; 1549b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 remain_bytes; 1559b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 1569b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->buf_addr += cppi41_channel->packet_sz; 1579b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 1589b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior remain_bytes = cppi41_channel->total_len; 1599b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior remain_bytes -= cppi41_channel->transferred; 1609b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior remain_bytes = min(remain_bytes, cppi41_channel->packet_sz); 1619b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->prog_len = remain_bytes; 1629b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 1639b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV 1649b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior : DMA_DEV_TO_MEM; 1659b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_desc = dmaengine_prep_slave_single(dc, 1669b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->buf_addr, 1679b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior remain_bytes, 1689b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior direction, 1699b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 170d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior if (WARN_ON(!dma_desc)) 1719b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return; 1729b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 1739b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_desc->callback = cppi41_dma_callback; 174a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior dma_desc->callback_param = &cppi41_channel->channel; 1759b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->cookie = dma_desc->tx_submit(dma_desc); 1769b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_async_issue_pending(dc); 1779b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 1789b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (!cppi41_channel->is_tx) { 179f50e67853b363b96336718597823ed7a7e8652deDaniel Mack musb_ep_select(musb->mregs, hw_ep->epnum); 1809b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr = musb_readw(epio, MUSB_RXCSR); 1819b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr |= MUSB_RXCSR_H_REQPKT; 1829b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writew(epio, MUSB_RXCSR, csr); 1839b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 1849b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 185d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior} 186d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior 187a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewiorstatic enum hrtimer_restart cppi41_recheck_tx_req(struct hrtimer *timer) 188a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior{ 189a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior struct cppi41_dma_controller *controller; 190a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior struct cppi41_dma_channel *cppi41_channel, *n; 191a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior struct musb *musb; 192a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior unsigned long flags; 193a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior enum hrtimer_restart ret = HRTIMER_NORESTART; 194a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior 195a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior controller = container_of(timer, struct cppi41_dma_controller, 196a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior early_tx); 197a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior musb = controller->musb; 198a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior 199a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior spin_lock_irqsave(&musb->lock, flags); 200a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior list_for_each_entry_safe(cppi41_channel, n, &controller->early_tx_list, 201a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior tx_check) { 202a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior bool empty; 203a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep; 204a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior 205a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior empty = musb_is_tx_fifo_empty(hw_ep); 206a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior if (empty) { 207a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior list_del_init(&cppi41_channel->tx_check); 208a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior cppi41_trans_done(cppi41_channel); 209a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior } 210a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior } 211a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior 212d2e6d62c9cbbc2da4211f672dbeea08960e29a80Thomas Gleixner if (!list_empty(&controller->early_tx_list) && 213d2e6d62c9cbbc2da4211f672dbeea08960e29a80Thomas Gleixner !hrtimer_is_queued(&controller->early_tx)) { 214a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior ret = HRTIMER_RESTART; 215a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior hrtimer_forward_now(&controller->early_tx, 216a5e4aa4d770ae96da52c8fa035751d2046e2434fDaniel Mack ktime_set(0, 20 * NSEC_PER_USEC)); 217a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior } 218a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior 219a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior spin_unlock_irqrestore(&musb->lock, flags); 220a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior return ret; 221a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior} 222a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior 223d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewiorstatic void cppi41_dma_callback(void *private_data) 224d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior{ 225d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior struct dma_channel *channel = private_data; 226d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior struct cppi41_dma_channel *cppi41_channel = channel->private_data; 227d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep; 228d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior struct musb *musb = hw_ep->musb; 229d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior unsigned long flags; 230d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior struct dma_tx_state txstate; 231d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior u32 transferred; 232a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior bool empty; 233d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior 234d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior spin_lock_irqsave(&musb->lock, flags); 235d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior 236d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie, 237d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior &txstate); 238d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior transferred = cppi41_channel->prog_len - txstate.residue; 239d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior cppi41_channel->transferred += transferred; 240d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior 241d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior dev_dbg(musb->controller, "DMA transfer done on hw_ep=%d bytes=%d/%d\n", 242d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior hw_ep->epnum, cppi41_channel->transferred, 243d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior cppi41_channel->total_len); 244d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior 245d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior update_rx_toggle(cppi41_channel); 246d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior 247d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior if (cppi41_channel->transferred == cppi41_channel->total_len || 248d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior transferred < cppi41_channel->packet_sz) 249d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior cppi41_channel->prog_len = 0; 250d373a8534d5e1e7a350e40d3c11961a7cd8d530bSebastian Andrzej Siewior 251a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior empty = musb_is_tx_fifo_empty(hw_ep); 252a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior if (empty) { 253a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior cppi41_trans_done(cppi41_channel); 254a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior } else { 255a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior struct cppi41_dma_controller *controller; 256a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior /* 257a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior * On AM335x it has been observed that the TX interrupt fires 258a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior * too early that means the TXFIFO is not yet empty but the DMA 259a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior * engine says that it is done with the transfer. We don't 260a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior * receive a FIFO empty interrupt so the only thing we can do is 261a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior * to poll for the bit. On HS it usually takes 2us, on FS around 262a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior * 110us - 150us depending on the transfer size. 263a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior * We spin on HS (no longer than than 25us and setup a timer on 264a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior * FS to check for the bit and complete the transfer. 265a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior */ 266a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior controller = cppi41_channel->controller; 267a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior 268a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior if (musb->g.speed == USB_SPEED_HIGH) { 269a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior unsigned wait = 25; 270a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior 271a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior do { 272a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior empty = musb_is_tx_fifo_empty(hw_ep); 273a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior if (empty) 274a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior break; 275a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior wait--; 276a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior if (!wait) 277a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior break; 278a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior udelay(1); 279a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior } while (1); 280a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior 281a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior empty = musb_is_tx_fifo_empty(hw_ep); 282a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior if (empty) { 283a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior cppi41_trans_done(cppi41_channel); 284a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior goto out; 285a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior } 286a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior } 287a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior list_add_tail(&cppi41_channel->tx_check, 288a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior &controller->early_tx_list); 289c58d80f523ffc15ef4d062fc7aeb03793fe39701Thomas Gleixner if (!hrtimer_is_queued(&controller->early_tx)) { 29050aea6fca771d6daf3ec24f771da866f7fd836e4Daniel Mack unsigned long usecs = cppi41_channel->total_len / 10; 29150aea6fca771d6daf3ec24f771da866f7fd836e4Daniel Mack 292a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior hrtimer_start_range_ns(&controller->early_tx, 29350aea6fca771d6daf3ec24f771da866f7fd836e4Daniel Mack ktime_set(0, usecs * NSEC_PER_USEC), 294a5e4aa4d770ae96da52c8fa035751d2046e2434fDaniel Mack 20 * NSEC_PER_USEC, 295a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior HRTIMER_MODE_REL); 296a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior } 297a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior } 298a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewiorout: 2999b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior spin_unlock_irqrestore(&musb->lock, flags); 3009b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 3019b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3029b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic u32 update_ep_mode(unsigned ep, unsigned mode, u32 old) 3039b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 3049b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior unsigned shift; 3059b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3069b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior shift = (ep - 1) * 2; 3079b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior old &= ~(3 << shift); 3089b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior old |= mode << shift; 3099b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return old; 3109b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 3119b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3129b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic void cppi41_set_dma_mode(struct cppi41_dma_channel *cppi41_channel, 3139b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior unsigned mode) 3149b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 3159b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_controller *controller = cppi41_channel->controller; 3169b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 port; 3179b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 new_mode; 3189b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 old_mode; 3199b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3209b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (cppi41_channel->is_tx) 3219b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior old_mode = controller->tx_mode; 3229b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior else 3239b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior old_mode = controller->rx_mode; 3249b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior port = cppi41_channel->port_num; 3259b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior new_mode = update_ep_mode(port, mode, old_mode); 3269b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3279b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (new_mode == old_mode) 3289b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return; 3299b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (cppi41_channel->is_tx) { 3309b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior controller->tx_mode = new_mode; 3319b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writel(controller->musb->ctrl_base, USB_CTRL_TX_MODE, 3329b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior new_mode); 3339b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } else { 3349b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior controller->rx_mode = new_mode; 3359b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writel(controller->musb->ctrl_base, USB_CTRL_RX_MODE, 3369b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior new_mode); 3379b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 3389b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 3399b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3409b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic void cppi41_set_autoreq_mode(struct cppi41_dma_channel *cppi41_channel, 3419b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior unsigned mode) 3429b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 3439b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_controller *controller = cppi41_channel->controller; 3449b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 port; 3459b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 new_mode; 3469b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u32 old_mode; 3479b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3489b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior old_mode = controller->auto_req; 3499b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior port = cppi41_channel->port_num; 3509b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior new_mode = update_ep_mode(port, mode, old_mode); 3519b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3529b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (new_mode == old_mode) 3539b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return; 3549b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior controller->auto_req = new_mode; 3559b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writel(controller->musb->ctrl_base, USB_CTRL_AUTOREQ, new_mode); 3569b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 3579b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3589b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic bool cppi41_configure_channel(struct dma_channel *channel, 3599b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u16 packet_sz, u8 mode, 3609b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_addr_t dma_addr, u32 len) 3619b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 3629b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_channel *cppi41_channel = channel->private_data; 3639b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct dma_chan *dc = cppi41_channel->dc; 3649b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct dma_async_tx_descriptor *dma_desc; 3659b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior enum dma_transfer_direction direction; 3669b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct musb *musb = cppi41_channel->controller->musb; 3679b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior unsigned use_gen_rndis = 0; 3689b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3699b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dev_dbg(musb->controller, 3709b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior "configure ep%d/%x packet_sz=%d, mode=%d, dma_addr=0x%llx, len=%d is_tx=%d\n", 3719b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->port_num, RNDIS_REG(cppi41_channel->port_num), 3729b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior packet_sz, mode, (unsigned long long) dma_addr, 3739b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior len, cppi41_channel->is_tx); 3749b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3759b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->buf_addr = dma_addr; 3769b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->total_len = len; 3779b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->transferred = 0; 3789b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->packet_sz = packet_sz; 3799267edaf8cd7b0ef2cd7785c677fe792c077b6abBin Liu cppi41_channel->tx_zlp = (cppi41_channel->is_tx && mode) ? 1 : 0; 3809b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3819b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior /* 3829b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior * Due to AM335x' Advisory 1.0.13 we are not allowed to transfer more 3839b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior * than max packet size at a time. 3849b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior */ 3859b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (cppi41_channel->is_tx) 3869b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior use_gen_rndis = 1; 3879b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3889b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (use_gen_rndis) { 3899b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior /* RNDIS mode */ 3909b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (len > packet_sz) { 3919b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writel(musb->ctrl_base, 3929b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior RNDIS_REG(cppi41_channel->port_num), len); 3939b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior /* gen rndis */ 3949b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_set_dma_mode(cppi41_channel, 3959b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior EP_MODE_DMA_GEN_RNDIS); 3969b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 3979b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior /* auto req */ 3989b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_set_autoreq_mode(cppi41_channel, 3999b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior EP_MODE_AUTOREG_ALL_NEOP); 4009b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } else { 4019b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writel(musb->ctrl_base, 4029b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior RNDIS_REG(cppi41_channel->port_num), 0); 4039b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_set_dma_mode(cppi41_channel, 4049b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior EP_MODE_DMA_TRANSPARENT); 4059b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_set_autoreq_mode(cppi41_channel, 4069b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior EP_MODE_AUTOREG_NONE); 4079b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 4089b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } else { 4099b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior /* fallback mode */ 4109b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_set_dma_mode(cppi41_channel, EP_MODE_DMA_TRANSPARENT); 4119b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREG_NONE); 4129b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior len = min_t(u32, packet_sz, len); 4139b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 4149b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->prog_len = len; 4159b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM; 4169b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_desc = dmaengine_prep_slave_single(dc, dma_addr, len, direction, 4179b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 4189b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (!dma_desc) 4199b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return false; 4209b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4219b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_desc->callback = cppi41_dma_callback; 4229b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_desc->callback_param = channel; 4239b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->cookie = dma_desc->tx_submit(dma_desc); 424ff3fcac949187d98684aaf3f1c35c7cae7712649Daniel Mack cppi41_channel->channel.rx_packet_done = false; 4259b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4269b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior save_rx_toggle(cppi41_channel); 4279b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_async_issue_pending(dc); 4289b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return true; 4299b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 4309b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4319b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic struct dma_channel *cppi41_dma_channel_allocate(struct dma_controller *c, 4329b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct musb_hw_ep *hw_ep, u8 is_tx) 4339b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 4349b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_controller *controller = container_of(c, 4359b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_controller, controller); 4369b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_channel *cppi41_channel = NULL; 4379b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u8 ch_num = hw_ep->epnum - 1; 4389b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4399b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (ch_num >= MUSB_DMA_NUM_CHANNELS) 4409b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return NULL; 4419b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4429b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (is_tx) 4439b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel = &controller->tx_channel[ch_num]; 4449b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior else 4459b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel = &controller->rx_channel[ch_num]; 4469b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4479b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (!cppi41_channel->dc) 4489b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return NULL; 4499b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4509b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (cppi41_channel->is_allocated) 4519b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return NULL; 4529b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4539b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->hw_ep = hw_ep; 4549b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->is_allocated = 1; 4559b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4569b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return &cppi41_channel->channel; 4579b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 4589b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4599b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic void cppi41_dma_channel_release(struct dma_channel *channel) 4609b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 4619b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_channel *cppi41_channel = channel->private_data; 4629b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4639b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (cppi41_channel->is_allocated) { 4649b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->is_allocated = 0; 4659b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior channel->status = MUSB_DMA_STATUS_FREE; 4669b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior channel->actual_len = 0; 4679b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 4689b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 4699b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4709b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic int cppi41_dma_channel_program(struct dma_channel *channel, 4719b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u16 packet_sz, u8 mode, 4729b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_addr_t dma_addr, u32 len) 4739b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 4749b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior int ret; 475f82503f549c70c15e283511270e6713a912fef37George Cherian struct cppi41_dma_channel *cppi41_channel = channel->private_data; 476f82503f549c70c15e283511270e6713a912fef37George Cherian int hb_mult = 0; 4779b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4789b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior BUG_ON(channel->status == MUSB_DMA_STATUS_UNKNOWN || 4799b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior channel->status == MUSB_DMA_STATUS_BUSY); 4809b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 481f82503f549c70c15e283511270e6713a912fef37George Cherian if (is_host_active(cppi41_channel->controller->musb)) { 482f82503f549c70c15e283511270e6713a912fef37George Cherian if (cppi41_channel->is_tx) 483f82503f549c70c15e283511270e6713a912fef37George Cherian hb_mult = cppi41_channel->hw_ep->out_qh->hb_mult; 484f82503f549c70c15e283511270e6713a912fef37George Cherian else 485f82503f549c70c15e283511270e6713a912fef37George Cherian hb_mult = cppi41_channel->hw_ep->in_qh->hb_mult; 486f82503f549c70c15e283511270e6713a912fef37George Cherian } 487f82503f549c70c15e283511270e6713a912fef37George Cherian 4889b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior channel->status = MUSB_DMA_STATUS_BUSY; 4899b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior channel->actual_len = 0; 490f82503f549c70c15e283511270e6713a912fef37George Cherian 491f82503f549c70c15e283511270e6713a912fef37George Cherian if (hb_mult) 492f82503f549c70c15e283511270e6713a912fef37George Cherian packet_sz = hb_mult * (packet_sz & 0x7FF); 493f82503f549c70c15e283511270e6713a912fef37George Cherian 4949b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior ret = cppi41_configure_channel(channel, packet_sz, mode, dma_addr, len); 4959b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (!ret) 4969b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior channel->status = MUSB_DMA_STATUS_FREE; 4979b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 4989b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return ret; 4999b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 5009b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5019b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic int cppi41_is_compatible(struct dma_channel *channel, u16 maxpacket, 5029b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior void *buf, u32 length) 5039b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 5049b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_channel *cppi41_channel = channel->private_data; 5059b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_controller *controller = cppi41_channel->controller; 5069b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct musb *musb = controller->musb; 5079b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5089b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (is_host_active(musb)) { 5099b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior WARN_ON(1); 5109b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return 1; 5119b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 512a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior if (cppi41_channel->hw_ep->ep_in.type != USB_ENDPOINT_XFER_BULK) 513a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior return 0; 51413266fea59f6f55e98d61e66707d784b9e947c84Sebastian Andrzej Siewior if (cppi41_channel->is_tx) 51513266fea59f6f55e98d61e66707d784b9e947c84Sebastian Andrzej Siewior return 1; 51613266fea59f6f55e98d61e66707d784b9e947c84Sebastian Andrzej Siewior /* AM335x Advisory 1.0.13. No workaround for device RX mode */ 5179b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return 0; 5189b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 5199b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5209b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic int cppi41_dma_channel_abort(struct dma_channel *channel) 5219b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 5229b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_channel *cppi41_channel = channel->private_data; 5239b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_controller *controller = cppi41_channel->controller; 5249b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct musb *musb = controller->musb; 5259b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior void __iomem *epio = cppi41_channel->hw_ep->regs; 5269b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior int tdbit; 5279b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior int ret; 5289b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior unsigned is_tx; 5299b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior u16 csr; 5309b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5319b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior is_tx = cppi41_channel->is_tx; 5329b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dev_dbg(musb->controller, "abort channel=%d, is_tx=%d\n", 5339b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->port_num, is_tx); 5349b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5359b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE) 5369b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return 0; 5379b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 538a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior list_del_init(&cppi41_channel->tx_check); 5399b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (is_tx) { 5409b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr = musb_readw(epio, MUSB_TXCSR); 5419b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr &= ~MUSB_TXCSR_DMAENAB; 5429b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writew(epio, MUSB_TXCSR, csr); 5439b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } else { 5449b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr = musb_readw(epio, MUSB_RXCSR); 5459b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr &= ~(MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_DMAENAB); 5469b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writew(epio, MUSB_RXCSR, csr); 5479b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5489b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr = musb_readw(epio, MUSB_RXCSR); 5499b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (csr & MUSB_RXCSR_RXPKTRDY) { 5509b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr |= MUSB_RXCSR_FLUSHFIFO; 5519b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writew(epio, MUSB_RXCSR, csr); 5529b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writew(epio, MUSB_RXCSR, csr); 5539b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 5549b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 5559b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5569b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior tdbit = 1 << cppi41_channel->port_num; 5579b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (is_tx) 5589b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior tdbit <<= 16; 5599b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5609b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior do { 5619b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writel(musb->ctrl_base, USB_TDOWN, tdbit); 5629b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior ret = dmaengine_terminate_all(cppi41_channel->dc); 5639b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } while (ret == -EAGAIN); 5649b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5659b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writel(musb->ctrl_base, USB_TDOWN, tdbit); 5669b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5679b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (is_tx) { 5689b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr = musb_readw(epio, MUSB_TXCSR); 5699b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (csr & MUSB_TXCSR_TXPKTRDY) { 5709b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior csr |= MUSB_TXCSR_FLUSHFIFO; 5719b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_writew(epio, MUSB_TXCSR, csr); 5729b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 5739b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 5749b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5759b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE; 5769b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return 0; 5779b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 5789b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5799b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic void cppi41_release_all_dma_chans(struct cppi41_dma_controller *ctrl) 5809b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 5819b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct dma_chan *dc; 5829b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior int i; 5839b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5849b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior for (i = 0; i < MUSB_DMA_NUM_CHANNELS; i++) { 5859b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dc = ctrl->tx_channel[i].dc; 5869b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (dc) 5879b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_release_channel(dc); 5889b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dc = ctrl->rx_channel[i].dc; 5899b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (dc) 5909b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dma_release_channel(dc); 5919b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 5929b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 5939b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5949b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic void cppi41_dma_controller_stop(struct cppi41_dma_controller *controller) 5959b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 5969b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_release_all_dma_chans(controller); 5979b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 5989b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 5999b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstatic int cppi41_dma_controller_start(struct cppi41_dma_controller *controller) 6009b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 6019b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct musb *musb = controller->musb; 6029b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct device *dev = musb->controller; 6039b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct device_node *np = dev->of_node; 6049b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_channel *cppi41_channel; 6059b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior int count; 6069b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior int i; 6079b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior int ret; 6089b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6099b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior count = of_property_count_strings(np, "dma-names"); 6109b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (count < 0) 6119b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return count; 6129b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6139b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior for (i = 0; i < count; i++) { 6149b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct dma_chan *dc; 6159b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct dma_channel *musb_dma; 6169b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior const char *str; 6179b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior unsigned is_tx; 6189b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior unsigned int port; 6199b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6209b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior ret = of_property_read_string_index(np, "dma-names", i, &str); 6219b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (ret) 6229b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior goto err; 6239b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (!strncmp(str, "tx", 2)) 6249b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior is_tx = 1; 6259b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior else if (!strncmp(str, "rx", 2)) 6269b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior is_tx = 0; 6279b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior else { 6289b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dev_err(dev, "Wrong dmatype %s\n", str); 6299b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior goto err; 6309b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 6319b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior ret = kstrtouint(str + 2, 0, &port); 6329b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (ret) 6339b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior goto err; 6349b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6354805414705df77d946b41df4f8b315e2b089eec4Sebastian Andrzej Siewior ret = -EINVAL; 6369b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (port > MUSB_DMA_NUM_CHANNELS || !port) 6379b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior goto err; 6389b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (is_tx) 6399b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel = &controller->tx_channel[port - 1]; 6409b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior else 6419b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel = &controller->rx_channel[port - 1]; 6429b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6439b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->controller = controller; 6449b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->port_num = port; 6459b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->is_tx = is_tx; 646a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior INIT_LIST_HEAD(&cppi41_channel->tx_check); 6479b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6489b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_dma = &cppi41_channel->channel; 6499b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_dma->private_data = cppi41_channel; 6509b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_dma->status = MUSB_DMA_STATUS_FREE; 6519b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior musb_dma->max_len = SZ_4M; 6529b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6539b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dc = dma_request_slave_channel(dev, str); 6549b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (!dc) { 6555ae477b05fb5edd059f709ff0a7a2f40fb2827d1Rahul Bedarkar dev_err(dev, "Failed to request %s.\n", str); 6564805414705df77d946b41df4f8b315e2b089eec4Sebastian Andrzej Siewior ret = -EPROBE_DEFER; 6579b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior goto err; 6589b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 6599b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_channel->dc = dc; 6609b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 6619b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return 0; 6629b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorerr: 6639b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_release_all_dma_chans(controller); 6644805414705df77d946b41df4f8b315e2b089eec4Sebastian Andrzej Siewior return ret; 6659b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 6669b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6679b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorvoid dma_controller_destroy(struct dma_controller *c) 6689b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 6699b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_controller *controller = container_of(c, 6709b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_controller, controller); 6719b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 672a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior hrtimer_cancel(&controller->early_tx); 6739b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior cppi41_dma_controller_stop(controller); 6749b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior kfree(controller); 6759b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 6769b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6779b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorstruct dma_controller *dma_controller_create(struct musb *musb, 6789b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior void __iomem *base) 6799b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior{ 6809b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior struct cppi41_dma_controller *controller; 6814805414705df77d946b41df4f8b315e2b089eec4Sebastian Andrzej Siewior int ret = 0; 6829b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6839b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (!musb->controller->of_node) { 6849b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior dev_err(musb->controller, "Need DT for the DMA engine.\n"); 6859b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return NULL; 6869b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior } 6879b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6889b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior controller = kzalloc(sizeof(*controller), GFP_KERNEL); 6899b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (!controller) 6909b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior goto kzalloc_fail; 6919b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 692a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior hrtimer_init(&controller->early_tx, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 693a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior controller->early_tx.function = cppi41_recheck_tx_req; 694a655f481d83d6d37bec0a2ddfdd24c30ff8f541fSebastian Andrzej Siewior INIT_LIST_HEAD(&controller->early_tx_list); 6959b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior controller->musb = musb; 6969b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 6979b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior controller->controller.channel_alloc = cppi41_dma_channel_allocate; 6989b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior controller->controller.channel_release = cppi41_dma_channel_release; 6999b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior controller->controller.channel_program = cppi41_dma_channel_program; 7009b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior controller->controller.channel_abort = cppi41_dma_channel_abort; 7019b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior controller->controller.is_compatible = cppi41_is_compatible; 7029b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 7039b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior ret = cppi41_dma_controller_start(controller); 7049b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior if (ret) 7059b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior goto plat_get_fail; 7069b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return &controller->controller; 7079b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior 7089b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorplat_get_fail: 7099b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior kfree(controller); 7109b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewiorkzalloc_fail: 7114805414705df77d946b41df4f8b315e2b089eec4Sebastian Andrzej Siewior if (ret == -EPROBE_DEFER) 7124805414705df77d946b41df4f8b315e2b089eec4Sebastian Andrzej Siewior return ERR_PTR(ret); 7139b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior return NULL; 7149b3452d1fa3c017d3664ff9e6a601daa6e0576ebSebastian Andrzej Siewior} 715