1/* 2 * Copyright 2012 Tilera Corporation. All Rights Reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation, version 2. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 11 * NON INFRINGEMENT. See the GNU General Public License for 12 * more details. 13 */ 14 15#ifndef _GXIO_DMA_QUEUE_H_ 16#define _GXIO_DMA_QUEUE_H_ 17 18/* 19 * DMA queue management APIs shared between TRIO and mPIPE. 20 */ 21 22#include <gxio/common.h> 23 24/* The credit counter lives in the high 32 bits. */ 25#define DMA_QUEUE_CREDIT_SHIFT 32 26 27/* 28 * State object that tracks a DMA queue's head and tail indices, as 29 * well as the number of commands posted and completed. The 30 * structure is accessed via a thread-safe, lock-free algorithm. 31 */ 32typedef struct { 33 /* 34 * Address of a MPIPE_EDMA_POST_REGION_VAL_t, 35 * TRIO_PUSH_DMA_REGION_VAL_t, or TRIO_PULL_DMA_REGION_VAL_t 36 * register. These register have identical encodings and provide 37 * information about how many commands have been processed. 38 */ 39 void *post_region_addr; 40 41 /* 42 * A lazily-updated count of how many edescs the hardware has 43 * completed. 44 */ 45 uint64_t hw_complete_count __attribute__ ((aligned(64))); 46 47 /* 48 * High 32 bits are a count of available egress command credits, 49 * low 24 bits are the next egress "slot". 50 */ 51 int64_t credits_and_next_index; 52 53} __gxio_dma_queue_t; 54 55/* Initialize a dma queue. */ 56extern void __gxio_dma_queue_init(__gxio_dma_queue_t *dma_queue, 57 void *post_region_addr, 58 unsigned int num_entries); 59 60/* 61 * Update the "credits_and_next_index" and "hw_complete_count" fields 62 * based on pending hardware completions. Note that some other thread 63 * may have already done this and, importantly, may still be in the 64 * process of updating "credits_and_next_index". 65 */ 66extern void __gxio_dma_queue_update_credits(__gxio_dma_queue_t *dma_queue); 67 68/* Wait for credits to become available. */ 69extern int64_t __gxio_dma_queue_wait_for_credits(__gxio_dma_queue_t *dma_queue, 70 int64_t modifier); 71 72/* Reserve slots in the queue, optionally waiting for slots to become 73 * available, and optionally returning a "completion_slot" suitable for 74 * direct comparison to "hw_complete_count". 75 */ 76static inline int64_t __gxio_dma_queue_reserve(__gxio_dma_queue_t *dma_queue, 77 unsigned int num, bool wait, 78 bool completion) 79{ 80 uint64_t slot; 81 82 /* 83 * Try to reserve 'num' egress command slots. We do this by 84 * constructing a constant that subtracts N credits and adds N to 85 * the index, and using fetchaddgez to only apply it if the credits 86 * count doesn't go negative. 87 */ 88 int64_t modifier = (((int64_t)(-num)) << DMA_QUEUE_CREDIT_SHIFT) | num; 89 int64_t old = 90 __insn_fetchaddgez(&dma_queue->credits_and_next_index, 91 modifier); 92 93 if (unlikely(old + modifier < 0)) { 94 /* 95 * We're out of credits. Try once to get more by checking for 96 * completed egress commands. If that fails, wait or fail. 97 */ 98 __gxio_dma_queue_update_credits(dma_queue); 99 old = __insn_fetchaddgez(&dma_queue->credits_and_next_index, 100 modifier); 101 if (old + modifier < 0) { 102 if (wait) 103 old = __gxio_dma_queue_wait_for_credits 104 (dma_queue, modifier); 105 else 106 return GXIO_ERR_DMA_CREDITS; 107 } 108 } 109 110 /* The bottom 24 bits of old encode the "slot". */ 111 slot = (old & 0xffffff); 112 113 if (completion) { 114 /* 115 * A "completion_slot" is a "slot" which can be compared to 116 * "hw_complete_count" at any time in the future. To convert 117 * "slot" into a "completion_slot", we access "hw_complete_count" 118 * once (knowing that we have reserved a slot, and thus, it will 119 * be "basically" accurate), and combine its high 40 bits with 120 * the 24 bit "slot", and handle "wrapping" by adding "1 << 24" 121 * if the result is LESS than "hw_complete_count". 122 */ 123 uint64_t complete; 124 complete = ACCESS_ONCE(dma_queue->hw_complete_count); 125 slot |= (complete & 0xffffffffff000000); 126 if (slot < complete) 127 slot += 0x1000000; 128 } 129 130 /* 131 * If any of our slots mod 256 were equivalent to 0, go ahead and 132 * collect some egress credits, and update "hw_complete_count", and 133 * make sure the index doesn't overflow into the credits. 134 */ 135 if (unlikely(((old + num) & 0xff) < num)) { 136 __gxio_dma_queue_update_credits(dma_queue); 137 138 /* Make sure the index doesn't overflow into the credits. */ 139#ifdef __BIG_ENDIAN__ 140 *(((uint8_t *)&dma_queue->credits_and_next_index) + 4) = 0; 141#else 142 *(((uint8_t *)&dma_queue->credits_and_next_index) + 3) = 0; 143#endif 144 } 145 146 return slot; 147} 148 149/* Non-inlinable "__gxio_dma_queue_reserve(..., true)". */ 150extern int64_t __gxio_dma_queue_reserve_aux(__gxio_dma_queue_t *dma_queue, 151 unsigned int num, int wait); 152 153/* Check whether a particular "completion slot" has completed. 154 * 155 * Note that this function requires a "completion slot", and thus 156 * cannot be used with the result of any "reserve_fast" function. 157 */ 158extern int __gxio_dma_queue_is_complete(__gxio_dma_queue_t *dma_queue, 159 int64_t completion_slot, int update); 160 161#endif /* !_GXIO_DMA_QUEUE_H_ */ 162