15cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams/* 25cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved. 35cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * 45cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * This program is free software; you can redistribute it and/or modify it 55cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * under the terms of the GNU General Public License as published by the Free 65cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * Software Foundation; either version 2 of the License, or (at your option) 75cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * any later version. 85cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * 95cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * This program is distributed in the hope that it will be useful, but WITHOUT 105cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 115cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 125cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * more details. 135cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * 145cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * You should have received a copy of the GNU General Public License along with 155cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * this program; if not, write to the Free Software Foundation, Inc., 59 165cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 175cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * 185cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * The full GNU General Public License is included in this distribution in the 195cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * file called COPYING. 205cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams */ 215cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams#ifndef IOATDMA_V2_H 225cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams#define IOATDMA_V2_H 235cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 245cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams#include <linux/dmaengine.h> 25abb12dfd50c7580d7dcbd581cf6265ba4d01ea7eDan Williams#include <linux/circ_buf.h> 265cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams#include "dma.h" 275cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams#include "hw.h" 285cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 295cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 305cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williamsextern int ioat_pending_level; 31bf40a6869c9198bdf56fe173961feb89e9f0d961Dan Williamsextern int ioat_ring_alloc_order; 325cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 335cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams/* 345cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * workaround for IOAT ver.3.0 null descriptor issue 355cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * (channel returns error when size is 0) 365cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams */ 375cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams#define NULL_DESC_BUFFER_SIZE 1 385cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 395cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams#define IOAT_MAX_ORDER 16 405cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams#define ioat_get_alloc_order() \ 415cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams (min(ioat_ring_alloc_order, IOAT_MAX_ORDER)) 42a309218acee8606f7e235da20cc826eb06d9b0f6Dan Williams#define ioat_get_max_alloc_order() \ 43a309218acee8606f7e235da20cc826eb06d9b0f6Dan Williams (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER)) 445cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 455cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes 465cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * @base: common ioat channel parameters 475cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * @xfercap_log; log2 of channel max transfer length (for fast division) 485cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * @head: allocated index 495cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * @issued: hardware notification point 505cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * @tail: cleanup index 515cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * @dmacount: identical to 'head' except for occasionally resetting to zero 525cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * @alloc_order: log2 of the number of allocated descriptors 53074cc47679f8b0931d7d5384e95822d82768f149Dan Williams * @produce: number of descriptors to produce at submit time 545cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams * @ring: software ring buffer implementation of hardware ring 55074cc47679f8b0931d7d5384e95822d82768f149Dan Williams * @prep_lock: serializes descriptor preparation (producers) 565cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams */ 575cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williamsstruct ioat2_dma_chan { 585cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams struct ioat_chan_common base; 595cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams size_t xfercap_log; 605cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams u16 head; 615cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams u16 issued; 625cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams u16 tail; 635cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams u16 dmacount; 645cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams u16 alloc_order; 65074cc47679f8b0931d7d5384e95822d82768f149Dan Williams u16 produce; 665cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams struct ioat_ring_ent **ring; 67074cc47679f8b0931d7d5384e95822d82768f149Dan Williams spinlock_t prep_lock; 685cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams}; 695cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 705cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williamsstatic inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c) 715cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams{ 725cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams struct ioat_chan_common *chan = to_chan_common(c); 735cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 745cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams return container_of(chan, struct ioat2_dma_chan, base); 755cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams} 765cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 7721b764e075e74f8af90da9f623aa3e2167484687Dave Jiangstatic inline u32 ioat2_ring_size(struct ioat2_dma_chan *ioat) 785cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams{ 79abb12dfd50c7580d7dcbd581cf6265ba4d01ea7eDan Williams return 1 << ioat->alloc_order; 805cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams} 815cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 825cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams/* count of descriptors in flight with the engine */ 835cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williamsstatic inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat) 845cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams{ 85abb12dfd50c7580d7dcbd581cf6265ba4d01ea7eDan Williams return CIRC_CNT(ioat->head, ioat->tail, ioat2_ring_size(ioat)); 865cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams} 875cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 885cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams/* count of descriptors pending submission to hardware */ 895cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williamsstatic inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat) 905cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams{ 91abb12dfd50c7580d7dcbd581cf6265ba4d01ea7eDan Williams return CIRC_CNT(ioat->head, ioat->issued, ioat2_ring_size(ioat)); 925cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams} 935cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 9421b764e075e74f8af90da9f623aa3e2167484687Dave Jiangstatic inline u32 ioat2_ring_space(struct ioat2_dma_chan *ioat) 955cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams{ 96abb12dfd50c7580d7dcbd581cf6265ba4d01ea7eDan Williams return ioat2_ring_size(ioat) - ioat2_ring_active(ioat); 975cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams} 985cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 995cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williamsstatic inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len) 1005cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams{ 1015cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams u16 num_descs = len >> ioat->xfercap_log; 1025cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 1035cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams num_descs += !!(len & ((1 << ioat->xfercap_log) - 1)); 1045cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams return num_descs; 1055cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams} 1065cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 1072aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams/** 1082aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * struct ioat_ring_ent - wrapper around hardware descriptor 1092aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * @hw: hardware DMA descriptor (for memcpy) 1102aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * @fill: hardware fill descriptor 1112aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * @xor: hardware xor descriptor 1122aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * @xor_ex: hardware xor extension descriptor 1132aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * @pq: hardware pq descriptor 1142aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * @pq_ex: hardware pq extension descriptor 1152aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * @pqu: hardware pq update descriptor 1162aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * @raw: hardware raw (un-typed) descriptor 1172aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * @txd: the generic software descriptor for all engines 1182aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * @len: total transaction length for unmap 119b094ad3be564e7cc59cca4ff0256550d3a55dd3bDan Williams * @result: asynchronous result of validate operations 1202aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams * @id: identifier for debug 1212aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams */ 1222aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams 1235cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williamsstruct ioat_ring_ent { 1242aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams union { 1252aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams struct ioat_dma_descriptor *hw; 1262aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams struct ioat_fill_descriptor *fill; 1272aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams struct ioat_xor_descriptor *xor; 1282aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams struct ioat_xor_ext_descriptor *xor_ex; 1292aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams struct ioat_pq_descriptor *pq; 1302aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams struct ioat_pq_ext_descriptor *pq_ex; 1312aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams struct ioat_pq_update_descriptor *pqu; 1322aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams struct ioat_raw_descriptor *raw; 1332aec048cdc4a5a81163a42a61df903f76a27e737Dan Williams }; 1345cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams size_t len; 135162b96e63e518aa6ff029ce23de12d7f027483bfDan Williams struct dma_async_tx_descriptor txd; 136b094ad3be564e7cc59cca4ff0256550d3a55dd3bDan Williams enum sum_check_flags *result; 1376df9183a153291a2585a8dfe67597fc18c201147Dan Williams #ifdef DEBUG 1386df9183a153291a2585a8dfe67597fc18c201147Dan Williams int id; 1396df9183a153291a2585a8dfe67597fc18c201147Dan Williams #endif 1405cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams}; 1415cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 1425cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williamsstatic inline struct ioat_ring_ent * 1435cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williamsioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx) 1445cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams{ 145abb12dfd50c7580d7dcbd581cf6265ba4d01ea7eDan Williams return ioat->ring[idx & (ioat2_ring_size(ioat) - 1)]; 1465cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams} 1475cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams 14809c8a5b85e5f1e74a19bdd7c85547429d51df1cdDan Williamsstatic inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr) 14909c8a5b85e5f1e74a19bdd7c85547429d51df1cdDan Williams{ 15009c8a5b85e5f1e74a19bdd7c85547429d51df1cdDan Williams struct ioat_chan_common *chan = &ioat->base; 15109c8a5b85e5f1e74a19bdd7c85547429d51df1cdDan Williams 15209c8a5b85e5f1e74a19bdd7c85547429d51df1cdDan Williams writel(addr & 0x00000000FFFFFFFF, 15309c8a5b85e5f1e74a19bdd7c85547429d51df1cdDan Williams chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW); 15409c8a5b85e5f1e74a19bdd7c85547429d51df1cdDan Williams writel(addr >> 32, 15509c8a5b85e5f1e74a19bdd7c85547429d51df1cdDan Williams chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH); 15609c8a5b85e5f1e74a19bdd7c85547429d51df1cdDan Williams} 15709c8a5b85e5f1e74a19bdd7c85547429d51df1cdDan Williams 158345d852391cf3fdc73f23a9ca522c6e7b5eb5a52Dan Williamsint __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca); 159345d852391cf3fdc73f23a9ca522c6e7b5eb5a52Dan Williamsint __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca); 160345d852391cf3fdc73f23a9ca522c6e7b5eb5a52Dan Williamsstruct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); 161345d852391cf3fdc73f23a9ca522c6e7b5eb5a52Dan Williamsstruct dca_provider * __devinit ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); 162074cc47679f8b0931d7d5384e95822d82768f149Dan Williamsint ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs); 163bf40a6869c9198bdf56fe173961feb89e9f0d961Dan Williamsint ioat2_enumerate_channels(struct ioatdma_device *device); 164bf40a6869c9198bdf56fe173961feb89e9f0d961Dan Williamsstruct dma_async_tx_descriptor * 165bf40a6869c9198bdf56fe173961feb89e9f0d961Dan Williamsioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, 166bf40a6869c9198bdf56fe173961feb89e9f0d961Dan Williams dma_addr_t dma_src, size_t len, unsigned long flags); 167bf40a6869c9198bdf56fe173961feb89e9f0d961Dan Williamsvoid ioat2_issue_pending(struct dma_chan *chan); 168bf40a6869c9198bdf56fe173961feb89e9f0d961Dan Williamsint ioat2_alloc_chan_resources(struct dma_chan *c); 169bf40a6869c9198bdf56fe173961feb89e9f0d961Dan Williamsvoid ioat2_free_chan_resources(struct dma_chan *c); 170bf40a6869c9198bdf56fe173961feb89e9f0d961Dan Williamsvoid __ioat2_restart_chan(struct ioat2_dma_chan *ioat); 171bf40a6869c9198bdf56fe173961feb89e9f0d961Dan Williamsbool reshape_ring(struct ioat2_dma_chan *ioat, int order); 172b094ad3be564e7cc59cca4ff0256550d3a55dd3bDan Williamsvoid __ioat2_issue_pending(struct ioat2_dma_chan *ioat); 173aa4d72ae946a4fa40486b871717778734184fa29Dan Williamsvoid ioat2_cleanup_event(unsigned long data); 174e3232714d465c42ac631929b990f5e35e2d8a955Dan Williamsvoid ioat2_timer_event(unsigned long data); 175a6d52d70677e99bdb89b6921c265d0a58c22e597Dan Williamsint ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo); 176a6d52d70677e99bdb89b6921c265d0a58c22e597Dan Williamsint ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo); 1775669e31c5a4874f1634bc0ffba268a6e2fa0cdd2Dan Williamsextern struct kobj_type ioat2_ktype; 178162b96e63e518aa6ff029ce23de12d7f027483bfDan Williamsextern struct kmem_cache *ioat2_cache; 1795cbafa65b92ee4f5b8ba915cddf94b91f186b989Dan Williams#endif /* IOATDMA_V2_H */ 180