1cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise/* 2cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * 4cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * This software is available to you under a choice of one of two 5cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * licenses. You may choose to be licensed under the terms of the GNU 6cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * General Public License (GPL) Version 2, available from the file 7cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * COPYING in the main directory of this source tree, or the 8cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * OpenIB.org BSD license below: 9cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * 10cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * Redistribution and use in source and binary forms, with or 11cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * without modification, are permitted provided that the following 12cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * conditions are met: 13cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * 14cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * - Redistributions of source code must retain the above 15cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * copyright notice, this list of conditions and the following 16cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * disclaimer. 17cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * 18cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * - Redistributions in binary form must reproduce the above 19cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * copyright notice, this list of conditions and the following 20cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * disclaimer in the documentation and/or other materials 21cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * provided with the distribution. 22cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * 23cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * SOFTWARE. 31cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise */ 32cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 33cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise#include <rdma/ib_umem.h> 3460063497a95e716c9a689af3be2687d261f115b4Arun Sharma#include <linux/atomic.h> 35cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 36cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise#include "iw_cxgb4.h" 37cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 38cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise#define T4_ULPTX_MIN_IO 32 39cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise#define C4IW_MAX_INLINE_SIZE 96 40cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 41cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, 42cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise void *data) 43cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 44cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct sk_buff *skb; 45cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct ulp_mem_io *req; 46cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct ulptx_idata *sc; 47cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u8 wr_len, *to_dp, *from_dp; 48cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int copy_len, num_wqe, i, ret = 0; 49cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_wr_wait wr_wait; 50cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 51cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise addr &= 0x7FFFFFF; 52cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s addr 0x%x len %u\n", __func__, addr, len); 53cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise num_wqe = DIV_ROUND_UP(len, C4IW_MAX_INLINE_SIZE); 54cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4iw_init_wr_wait(&wr_wait); 55cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise for (i = 0; i < num_wqe; i++) { 56cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 57cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise copy_len = len > C4IW_MAX_INLINE_SIZE ? C4IW_MAX_INLINE_SIZE : 58cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise len; 59cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise wr_len = roundup(sizeof *req + sizeof *sc + 60cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise roundup(copy_len, T4_ULPTX_MIN_IO), 16); 61cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 62d3c814e8b2a094dc3bcbe6a0d93ec4824b26e86aDavid Rientjes skb = alloc_skb(wr_len, GFP_KERNEL); 63cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (!skb) 64cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -ENOMEM; 65cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); 66cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 67cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise req = (struct ulp_mem_io *)__skb_put(skb, wr_len); 68cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise memset(req, 0, wr_len); 69cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise INIT_ULPTX_WR(req, wr_len, 0, 0); 70cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 71cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (i == (num_wqe-1)) { 72cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise req->wr.wr_hi = cpu_to_be32(FW_WR_OP(FW_ULPTX_WR) | 73cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise FW_WR_COMPL(1)); 74c8e081a1bf11c5cbac5f2f9f53c040be61d7b29eRoland Dreier req->wr.wr_lo = (__force __be64)(unsigned long) &wr_wait; 75cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } else 76cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise req->wr.wr_hi = cpu_to_be32(FW_WR_OP(FW_ULPTX_WR)); 77cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise req->wr.wr_mid = cpu_to_be32( 78cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise FW_WR_LEN16(DIV_ROUND_UP(wr_len, 16))); 79cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 80cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise req->cmd = cpu_to_be32(ULPTX_CMD(ULP_TX_MEM_WRITE) | (1<<23)); 81cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise req->dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN( 82cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise DIV_ROUND_UP(copy_len, T4_ULPTX_MIN_IO))); 83cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise req->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(req->wr), 84cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 16)); 85cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise req->lock_addr = cpu_to_be32(ULP_MEMIO_ADDR(addr + i * 3)); 86cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 87cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise sc = (struct ulptx_idata *)(req + 1); 88cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise sc->cmd_more = cpu_to_be32(ULPTX_CMD(ULP_TX_SC_IMM)); 89cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise sc->len = cpu_to_be32(roundup(copy_len, T4_ULPTX_MIN_IO)); 90cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 91cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise to_dp = (u8 *)(sc + 1); 92cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise from_dp = (u8 *)data + i * C4IW_MAX_INLINE_SIZE; 93cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (data) 94cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise memcpy(to_dp, from_dp, copy_len); 95cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise else 96cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise memset(to_dp, 0, copy_len); 97cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (copy_len % T4_ULPTX_MIN_IO) 98cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise memset(to_dp + copy_len, 0, T4_ULPTX_MIN_IO - 99cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise (copy_len % T4_ULPTX_MIN_IO)); 100cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = c4iw_ofld_send(rdev, skb); 101cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 102cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ret; 103cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise len -= C4IW_MAX_INLINE_SIZE; 104cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 105cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 106aadc4df3087a33ca7fc37f91a024e7b7efdafa75Steve Wise ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); 107cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ret; 108cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 109cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 110cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise/* 111cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * Build and write a TPT entry. 112cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size, 113cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * pbl_size and pbl_addr 114cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise * OUT: stag index 115cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise */ 116cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, 117cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 *stag, u8 stag_state, u32 pdid, 118cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise enum fw_ri_stag_type type, enum fw_ri_mem_perms perm, 119cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int bind_enabled, u32 zbva, u64 to, 120cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u64 len, u8 page_size, u32 pbl_size, u32 pbl_addr) 121cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 122cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int err; 123cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct fw_ri_tpte tpt; 124cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 stag_idx; 125cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise static atomic_t key; 126cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 127cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (c4iw_fatal_error(rdev)) 128cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -EIO; 129cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 130cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise stag_state = stag_state > 0; 131cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise stag_idx = (*stag) >> 8; 132cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 133cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) { 134cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise stag_idx = c4iw_get_resource(&rdev->resource.tpt_fifo, 135cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise &rdev->resource.tpt_fifo_lock); 136cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (!stag_idx) 137cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -ENOMEM; 138cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise *stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff); 139cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 140cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n", 141cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise __func__, stag_state, type, pdid, stag_idx); 142cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 143cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise /* write TPT entry */ 144cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (reset_tpt_entry) 145cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise memset(&tpt, 0, sizeof(tpt)); 146cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise else { 147cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise tpt.valid_to_pdid = cpu_to_be32(F_FW_RI_TPTE_VALID | 148cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise V_FW_RI_TPTE_STAGKEY((*stag & M_FW_RI_TPTE_STAGKEY)) | 149cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise V_FW_RI_TPTE_STAGSTATE(stag_state) | 150cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise V_FW_RI_TPTE_STAGTYPE(type) | V_FW_RI_TPTE_PDID(pdid)); 151cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise tpt.locread_to_qpid = cpu_to_be32(V_FW_RI_TPTE_PERM(perm) | 152cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise (bind_enabled ? F_FW_RI_TPTE_MWBINDEN : 0) | 153cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise V_FW_RI_TPTE_ADDRTYPE((zbva ? FW_RI_ZERO_BASED_TO : 154cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise FW_RI_VA_BASED_TO))| 155cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise V_FW_RI_TPTE_PS(page_size)); 156cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise tpt.nosnoop_pbladdr = !pbl_size ? 0 : cpu_to_be32( 157cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise V_FW_RI_TPTE_PBLADDR(PBL_OFF(rdev, pbl_addr)>>3)); 158cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise tpt.len_lo = cpu_to_be32((u32)(len & 0xffffffffUL)); 159cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise tpt.va_hi = cpu_to_be32((u32)(to >> 32)); 160cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise tpt.va_lo_fbo = cpu_to_be32((u32)(to & 0xffffffffUL)); 161cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise tpt.dca_mwbcnt_pstag = cpu_to_be32(0); 162cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise tpt.len_hi = cpu_to_be32((u32)(len >> 32)); 163cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 164cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise err = write_adapter_mem(rdev, stag_idx + 165cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise (rdev->lldi.vr->stag.start >> 5), 166cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise sizeof(tpt), &tpt); 167cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 168cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (reset_tpt_entry) 169cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4iw_put_resource(&rdev->resource.tpt_fifo, stag_idx, 170cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise &rdev->resource.tpt_fifo_lock); 171cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return err; 172cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 173cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 174cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int write_pbl(struct c4iw_rdev *rdev, __be64 *pbl, 175cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 pbl_addr, u32 pbl_size) 176cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 177cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int err; 178cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 179cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n", 180cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise __func__, pbl_addr, rdev->lldi.vr->pbl.start, 181cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise pbl_size); 182cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 183cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise err = write_adapter_mem(rdev, pbl_addr >> 5, pbl_size << 3, pbl); 184cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return err; 185cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 186cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 187cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int dereg_mem(struct c4iw_rdev *rdev, u32 stag, u32 pbl_size, 188cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 pbl_addr) 189cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 190cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 191cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise pbl_size, pbl_addr); 192cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 193cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 194cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int allocate_window(struct c4iw_rdev *rdev, u32 * stag, u32 pdid) 195cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 196cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise *stag = T4_STAG_UNSET; 197cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_MW, 0, 0, 0, 198cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 0UL, 0, 0, 0, 0); 199cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 200cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 201cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int deallocate_window(struct c4iw_rdev *rdev, u32 stag) 202cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 203cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0, 204cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 0); 205cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 206cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 207cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int allocate_stag(struct c4iw_rdev *rdev, u32 *stag, u32 pdid, 208cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 pbl_size, u32 pbl_addr) 209cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 210cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise *stag = T4_STAG_UNSET; 211cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_NSMR, 0, 0, 0, 212cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 0UL, 0, 0, pbl_size, pbl_addr); 213cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 214cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 215cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int finish_mem_reg(struct c4iw_mr *mhp, u32 stag) 216cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 217cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 mmid; 218cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 219cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.state = 1; 220cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.stag = stag; 221cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mmid = stag >> 8; 222cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->ibmr.rkey = mhp->ibmr.lkey = stag; 223cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s mmid 0x%x mhp %p\n", __func__, mmid, mhp); 224cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid); 225cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 226cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 227cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php, 228cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_mr *mhp, int shift) 229cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 230cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 stag = T4_STAG_UNSET; 231cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int ret; 232cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 233cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid, 234cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise FW_RI_STAG_NSMR, mhp->attr.perms, 235cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.mw_bind_enable, mhp->attr.zbva, 236cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.va_fbo, mhp->attr.len, shift - 12, 237cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size, mhp->attr.pbl_addr); 238cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 239cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ret; 240cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 241cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = finish_mem_reg(mhp, stag); 242cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 243cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 244cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_addr); 245cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ret; 246cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 247cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 248cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int reregister_mem(struct c4iw_dev *rhp, struct c4iw_pd *php, 249cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_mr *mhp, int shift, int npages) 250cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 251cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 stag; 252cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int ret; 253cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 254cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (npages > mhp->attr.pbl_size) 255cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -ENOMEM; 256cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 257cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise stag = mhp->attr.stag; 258cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid, 259cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise FW_RI_STAG_NSMR, mhp->attr.perms, 260cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.mw_bind_enable, mhp->attr.zbva, 261cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.va_fbo, mhp->attr.len, shift - 12, 262cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size, mhp->attr.pbl_addr); 263cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 264cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ret; 265cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 266cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = finish_mem_reg(mhp, stag); 267cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 268cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 269cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_addr); 270cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 271cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ret; 272cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 273cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 274cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int alloc_pbl(struct c4iw_mr *mhp, int npages) 275cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 276cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_addr = c4iw_pblpool_alloc(&mhp->rhp->rdev, 277cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise npages << 3); 278cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 279cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (!mhp->attr.pbl_addr) 280cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -ENOMEM; 281cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 282cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size = npages; 283cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 284cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return 0; 285cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 286cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 287cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestatic int build_phys_page_list(struct ib_phys_buf *buffer_list, 288cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int num_phys_buf, u64 *iova_start, 289cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u64 *total_size, int *npages, 290cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int *shift, __be64 **page_list) 291cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 292cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u64 mask; 293cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int i, j, n; 294cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 295cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mask = 0; 296cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise *total_size = 0; 297cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise for (i = 0; i < num_phys_buf; ++i) { 298cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (i != 0 && buffer_list[i].addr & ~PAGE_MASK) 299cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -EINVAL; 300cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (i != 0 && i != num_phys_buf - 1 && 301cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise (buffer_list[i].size & ~PAGE_MASK)) 302cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -EINVAL; 303cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise *total_size += buffer_list[i].size; 304cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (i > 0) 305cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mask |= buffer_list[i].addr; 306cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise else 307cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mask |= buffer_list[i].addr & PAGE_MASK; 308cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (i != num_phys_buf - 1) 309cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mask |= buffer_list[i].addr + buffer_list[i].size; 310cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise else 311cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mask |= (buffer_list[i].addr + buffer_list[i].size + 312cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PAGE_SIZE - 1) & PAGE_MASK; 313cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 314cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 315cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (*total_size > 0xFFFFFFFFULL) 316cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -ENOMEM; 317cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 318cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise /* Find largest page shift we can use to cover buffers */ 319cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise for (*shift = PAGE_SHIFT; *shift < 27; ++(*shift)) 320cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if ((1ULL << *shift) & mask) 321cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise break; 322cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 323cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise buffer_list[0].size += buffer_list[0].addr & ((1ULL << *shift) - 1); 324cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise buffer_list[0].addr &= ~0ull << *shift; 325cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 326cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise *npages = 0; 327cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise for (i = 0; i < num_phys_buf; ++i) 328cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise *npages += (buffer_list[i].size + 329cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise (1ULL << *shift) - 1) >> *shift; 330cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 331cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (!*npages) 332cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -EINVAL; 333cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 334cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise *page_list = kmalloc(sizeof(u64) * *npages, GFP_KERNEL); 335cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (!*page_list) 336cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -ENOMEM; 337cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 338cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise n = 0; 339cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise for (i = 0; i < num_phys_buf; ++i) 340cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise for (j = 0; 341cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise j < (buffer_list[i].size + (1ULL << *shift) - 1) >> *shift; 342cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ++j) 343cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise (*page_list)[n++] = cpu_to_be64(buffer_list[i].addr + 344cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ((u64) j << *shift)); 345cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 346cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s va 0x%llx mask 0x%llx shift %d len %lld pbl_size %d\n", 347cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise __func__, (unsigned long long)*iova_start, 348cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise (unsigned long long)mask, *shift, (unsigned long long)*total_size, 349cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise *npages); 350cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 351cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return 0; 352cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 353cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 354cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 355cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseint c4iw_reregister_phys_mem(struct ib_mr *mr, int mr_rereg_mask, 356cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct ib_pd *pd, struct ib_phys_buf *buffer_list, 357cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int num_phys_buf, int acc, u64 *iova_start) 358cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 359cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 360cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_mr mh, *mhp; 361cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_pd *php; 362cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_dev *rhp; 363cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise __be64 *page_list = NULL; 364cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int shift = 0; 365cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u64 total_size; 366cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int npages; 367cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int ret; 368cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 369cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s ib_mr %p ib_pd %p\n", __func__, mr, pd); 370cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 371cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise /* There can be no memory windows */ 372cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (atomic_read(&mr->usecnt)) 373cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -EINVAL; 374cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 375cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp = to_c4iw_mr(mr); 376cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise rhp = mhp->rhp; 377cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise php = to_c4iw_pd(mr->pd); 378cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 379cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise /* make sure we are on the same adapter */ 380cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (rhp != php->rhp) 381cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -EINVAL; 382cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 383cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise memcpy(&mh, mhp, sizeof *mhp); 384cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 385cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (mr_rereg_mask & IB_MR_REREG_PD) 386cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise php = to_c4iw_pd(pd); 387cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (mr_rereg_mask & IB_MR_REREG_ACCESS) { 388cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mh.attr.perms = c4iw_ib_to_tpt_access(acc); 389cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mh.attr.mw_bind_enable = (acc & IB_ACCESS_MW_BIND) == 390cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise IB_ACCESS_MW_BIND; 391cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 392cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (mr_rereg_mask & IB_MR_REREG_TRANS) { 393cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = build_phys_page_list(buffer_list, num_phys_buf, 394cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise iova_start, 395cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise &total_size, &npages, 396cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise &shift, &page_list); 397cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 398cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ret; 399cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 400cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 401cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = reregister_mem(rhp, php, &mh, shift, npages); 402cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(page_list); 403cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 404cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ret; 405cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (mr_rereg_mask & IB_MR_REREG_PD) 406cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pdid = php->pdid; 407cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (mr_rereg_mask & IB_MR_REREG_ACCESS) 408cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.perms = c4iw_ib_to_tpt_access(acc); 409cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (mr_rereg_mask & IB_MR_REREG_TRANS) { 410cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.zbva = 0; 411cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.va_fbo = *iova_start; 412cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.page_size = shift - 12; 413cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.len = (u32) total_size; 414cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size = npages; 415cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 416cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 417cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return 0; 418cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 419cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 420cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestruct ib_mr *c4iw_register_phys_mem(struct ib_pd *pd, 421cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct ib_phys_buf *buffer_list, 422cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int num_phys_buf, int acc, u64 *iova_start) 423cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 424cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise __be64 *page_list; 425cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int shift; 426cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u64 total_size; 427cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int npages; 428cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_dev *rhp; 429cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_pd *php; 430cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_mr *mhp; 431cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int ret; 432cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 433cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s ib_pd %p\n", __func__, pd); 434cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise php = to_c4iw_pd(pd); 435cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise rhp = php->rhp; 436cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 437cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 438cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (!mhp) 439cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(-ENOMEM); 440cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 441cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->rhp = rhp; 442cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 443cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise /* First check that we have enough alignment */ 444cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) { 445cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = -EINVAL; 446cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err; 447cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 448cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 449cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (num_phys_buf > 1 && 450cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) { 451cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = -EINVAL; 452cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err; 453cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 454cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 455cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start, 456cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise &total_size, &npages, &shift, 457cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise &page_list); 458cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 459cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err; 460cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 461cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = alloc_pbl(mhp, npages); 462cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) { 463cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(page_list); 464cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err_pbl; 465cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 466cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 467cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = write_pbl(&mhp->rhp->rdev, page_list, mhp->attr.pbl_addr, 468cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise npages); 469cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(page_list); 470cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 471cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err_pbl; 472cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 473cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pdid = php->pdid; 474cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.zbva = 0; 475cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 476cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.perms = c4iw_ib_to_tpt_access(acc); 477cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.va_fbo = *iova_start; 478cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.page_size = shift - 12; 479cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 480cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.len = (u32) total_size; 481cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size = npages; 482cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = register_mem(rhp, php, mhp, shift); 483cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 484cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err_pbl; 485cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 486cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return &mhp->ibmr; 487cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 488cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseerr_pbl: 489cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 490cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size << 3); 491cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 492cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseerr: 493cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(mhp); 494cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(ret); 495cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 496cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 497cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 498cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestruct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc) 499cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 500cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_dev *rhp; 501cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_pd *php; 502cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_mr *mhp; 503cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int ret; 504cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 stag = T4_STAG_UNSET; 505cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 506cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s ib_pd %p\n", __func__, pd); 507cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise php = to_c4iw_pd(pd); 508cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise rhp = php->rhp; 509cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 510cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 511cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (!mhp) 512cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(-ENOMEM); 513cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 514cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->rhp = rhp; 515cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pdid = php->pdid; 516cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.perms = c4iw_ib_to_tpt_access(acc); 517cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.mw_bind_enable = (acc&IB_ACCESS_MW_BIND) == IB_ACCESS_MW_BIND; 518cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.zbva = 0; 519cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.va_fbo = 0; 520cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.page_size = 0; 521cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.len = ~0UL; 522cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size = 0; 523cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 524cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, php->pdid, 525cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise FW_RI_STAG_NSMR, mhp->attr.perms, 526cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.mw_bind_enable, 0, 0, ~0UL, 0, 0, 0); 527cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 528cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err1; 529cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 530cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = finish_mem_reg(mhp, stag); 531cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 532cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err2; 533cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return &mhp->ibmr; 534cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseerr2: 535cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 536cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_addr); 537cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseerr1: 538cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(mhp); 539cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(ret); 540cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 541cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 542cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestruct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 543cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u64 virt, int acc, struct ib_udata *udata) 544cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 545cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise __be64 *pages; 546cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int shift, n, len; 547cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int i, j, k; 548cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int err = 0; 549cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct ib_umem_chunk *chunk; 550cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_dev *rhp; 551cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_pd *php; 552cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_mr *mhp; 553cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 554cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s ib_pd %p\n", __func__, pd); 555cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 556cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (length == ~0ULL) 557cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(-EINVAL); 558cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 559cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if ((length + start) < start) 560cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(-EINVAL); 561cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 562cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise php = to_c4iw_pd(pd); 563cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise rhp = php->rhp; 564cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 565cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (!mhp) 566cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(-ENOMEM); 567cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 568cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->rhp = rhp; 569cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 570cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0); 571cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (IS_ERR(mhp->umem)) { 572cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise err = PTR_ERR(mhp->umem); 573cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(mhp); 574cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(err); 575cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 576cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 577cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise shift = ffs(mhp->umem->page_size) - 1; 578cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 579cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise n = 0; 580cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise list_for_each_entry(chunk, &mhp->umem->chunk_list, list) 581cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise n += chunk->nents; 582cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 583cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise err = alloc_pbl(mhp, n); 584cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (err) 585cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err; 586cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 587cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise pages = (__be64 *) __get_free_page(GFP_KERNEL); 588cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (!pages) { 589cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise err = -ENOMEM; 590cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err_pbl; 591cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 592cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 593cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise i = n = 0; 594cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 595cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise list_for_each_entry(chunk, &mhp->umem->chunk_list, list) 596cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise for (j = 0; j < chunk->nmap; ++j) { 597cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise len = sg_dma_len(&chunk->page_list[j]) >> shift; 598cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise for (k = 0; k < len; ++k) { 599cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise pages[i++] = cpu_to_be64(sg_dma_address( 600cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise &chunk->page_list[j]) + 601cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->umem->page_size * k); 602cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (i == PAGE_SIZE / sizeof *pages) { 603cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise err = write_pbl(&mhp->rhp->rdev, 604cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise pages, 605cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_addr + (n << 3), i); 606cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (err) 607cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto pbl_done; 608cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise n += i; 609cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise i = 0; 610cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 611cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 612cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 613cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 614cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (i) 615cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise err = write_pbl(&mhp->rhp->rdev, pages, 616cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_addr + (n << 3), i); 617cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 618cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisepbl_done: 619cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise free_page((unsigned long) pages); 620cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (err) 621cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err_pbl; 622cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 623cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pdid = php->pdid; 624cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.zbva = 0; 625cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.perms = c4iw_ib_to_tpt_access(acc); 626cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.va_fbo = virt; 627cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.page_size = shift - 12; 628301c2c3f039a1f9478f6cbef60f2ccd4da9bd4a1Steve Wise mhp->attr.len = length; 629cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 630cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise err = register_mem(rhp, php, mhp, shift); 631cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (err) 632cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err_pbl; 633cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 634cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return &mhp->ibmr; 635cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 636cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseerr_pbl: 637cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 638cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size << 3); 639cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 640cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseerr: 641cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ib_umem_release(mhp->umem); 642cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(mhp); 643cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(err); 644cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 645cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 646cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestruct ib_mw *c4iw_alloc_mw(struct ib_pd *pd) 647cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 648cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_dev *rhp; 649cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_pd *php; 650cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_mw *mhp; 651cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 mmid; 652cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 stag = 0; 653cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int ret; 654cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 655cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise php = to_c4iw_pd(pd); 656cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise rhp = php->rhp; 657cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 658cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (!mhp) 659cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(-ENOMEM); 660cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = allocate_window(&rhp->rdev, &stag, php->pdid); 661cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) { 662cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(mhp); 663cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(ret); 664cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 665cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->rhp = rhp; 666cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pdid = php->pdid; 667cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.type = FW_RI_STAG_MW; 668cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.stag = stag; 669cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mmid = (stag) >> 8; 670cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->ibmw.rkey = stag; 671cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { 672cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise deallocate_window(&rhp->rdev, mhp->attr.stag); 673cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(mhp); 674cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(-ENOMEM); 675cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise } 676cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag); 677cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return &(mhp->ibmw); 678cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 679cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 680cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseint c4iw_dealloc_mw(struct ib_mw *mw) 681cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 682cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_dev *rhp; 683cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_mw *mhp; 684cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 mmid; 685cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 686cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp = to_c4iw_mw(mw); 687cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise rhp = mhp->rhp; 688cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mmid = (mw->rkey) >> 8; 689cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise deallocate_window(&rhp->rdev, mhp->attr.stag); 690cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise remove_handle(rhp, &rhp->mmidr, mmid); 691cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(mhp); 692cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp); 693cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return 0; 694cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 695cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 696cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestruct ib_mr *c4iw_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth) 697cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 698cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_dev *rhp; 699cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_pd *php; 700cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_mr *mhp; 701cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 mmid; 702cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 stag = 0; 703cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int ret = 0; 704cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 705cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise php = to_c4iw_pd(pd); 706cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise rhp = php->rhp; 707cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 708841dba9a5a0dbced4b4bc240f444d47f30ba6724Steve Wise if (!mhp) { 709841dba9a5a0dbced4b4bc240f444d47f30ba6724Steve Wise ret = -ENOMEM; 710cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err; 711841dba9a5a0dbced4b4bc240f444d47f30ba6724Steve Wise } 712cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 713cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->rhp = rhp; 714cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = alloc_pbl(mhp, pbl_depth); 715cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 716cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err1; 717cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size = pbl_depth; 718cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ret = allocate_stag(&rhp->rdev, &stag, php->pdid, 719cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size, mhp->attr.pbl_addr); 720cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (ret) 721cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err2; 722cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pdid = php->pdid; 723cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.type = FW_RI_STAG_NSMR; 724cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.stag = stag; 725cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.state = 1; 726cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mmid = (stag) >> 8; 727cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->ibmr.rkey = mhp->ibmr.lkey = stag; 728841dba9a5a0dbced4b4bc240f444d47f30ba6724Steve Wise if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { 729841dba9a5a0dbced4b4bc240f444d47f30ba6724Steve Wise ret = -ENOMEM; 730cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise goto err3; 731841dba9a5a0dbced4b4bc240f444d47f30ba6724Steve Wise } 732cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 733cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag); 734cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return &(mhp->ibmr); 735cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseerr3: 736cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size, 737cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_addr); 738cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseerr2: 739cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 740cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size << 3); 741cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseerr1: 742cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(mhp); 743cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseerr: 744cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(ret); 745cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 746cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 747cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisestruct ib_fast_reg_page_list *c4iw_alloc_fastreg_pbl(struct ib_device *device, 748cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int page_list_len) 749cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 750cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_fr_page_list *c4pl; 751cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_dev *dev = to_c4iw_dev(device); 752cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise dma_addr_t dma_addr; 753cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise int size = sizeof *c4pl + page_list_len * sizeof(u64); 754cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 755cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4pl = dma_alloc_coherent(&dev->rdev.lldi.pdev->dev, size, 756cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise &dma_addr, GFP_KERNEL); 757cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (!c4pl) 758cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return ERR_PTR(-ENOMEM); 759cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 760f38926aa1dc5fbf7dfc5f97a53377b2e796dedc3FUJITA Tomonori dma_unmap_addr_set(c4pl, mapping, dma_addr); 761cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4pl->dma_addr = dma_addr; 762cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4pl->dev = dev; 763cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4pl->size = size; 764cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4pl->ibpl.page_list = (u64 *)(c4pl + 1); 765cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4pl->ibpl.max_page_list_len = page_list_len; 766cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 767cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return &c4pl->ibpl; 768cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 769cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 770cfdda9d764362ab77b11a410bb928400e6520d57Steve Wisevoid c4iw_free_fastreg_pbl(struct ib_fast_reg_page_list *ibpl) 771cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 772cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_fr_page_list *c4pl = to_c4iw_fr_page_list(ibpl); 773cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 774cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise dma_free_coherent(&c4pl->dev->rdev.lldi.pdev->dev, c4pl->size, 775f38926aa1dc5fbf7dfc5f97a53377b2e796dedc3FUJITA Tomonori c4pl, dma_unmap_addr(c4pl, mapping)); 776cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 777cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 778cfdda9d764362ab77b11a410bb928400e6520d57Steve Wiseint c4iw_dereg_mr(struct ib_mr *ib_mr) 779cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise{ 780cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_dev *rhp; 781cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise struct c4iw_mr *mhp; 782cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise u32 mmid; 783cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 784cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s ib_mr %p\n", __func__, ib_mr); 785cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise /* There can be no memory windows */ 786cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (atomic_read(&ib_mr->usecnt)) 787cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return -EINVAL; 788cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise 789cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp = to_c4iw_mr(ib_mr); 790cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise rhp = mhp->rhp; 791cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mmid = mhp->attr.stag >> 8; 792cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 793cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_addr); 794cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (mhp->attr.pbl_size) 795cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 796cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise mhp->attr.pbl_size << 3); 797cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise remove_handle(rhp, &rhp->mmidr, mmid); 798cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (mhp->kva) 799cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree((void *) (unsigned long) mhp->kva); 800cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise if (mhp->umem) 801cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise ib_umem_release(mhp->umem); 802cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise PDBG("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp); 803cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise kfree(mhp); 804cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise return 0; 805cfdda9d764362ab77b11a410bb928400e6520d57Steve Wise} 806