1b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise/* 2b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Copyright (c) 2006 Chelsio, Inc. All rights reserved. 3b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 4b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * This software is available to you under a choice of one of two 5b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * licenses. You may choose to be licensed under the terms of the GNU 6b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * General Public License (GPL) Version 2, available from the file 7b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * COPYING in the main directory of this source tree, or the 8b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * OpenIB.org BSD license below: 9b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 10b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Redistribution and use in source and binary forms, with or 11b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * without modification, are permitted provided that the following 12b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * conditions are met: 13b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 14b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * - Redistributions of source code must retain the above 15b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * copyright notice, this list of conditions and the following 16b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * disclaimer. 17b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 18b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * - Redistributions in binary form must reproduce the above 19b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * copyright notice, this list of conditions and the following 20b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * disclaimer in the documentation and/or other materials 21b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * provided with the distribution. 22b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 23b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * SOFTWARE. 31b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 325a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 33b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include <asm/byteorder.h> 34b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 35b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include <rdma/iw_cm.h> 36b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include <rdma/ib_verbs.h> 37b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 38b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "cxio_hal.h" 39273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier#include "cxio_resource.h" 40b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "iwch.h" 41b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "iwch_provider.h" 42b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 4313a239330abdda5afcb1161fce68ec214af2cc90Steve Wisestatic int iwch_finish_mem_reg(struct iwch_mr *mhp, u32 stag) 44b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 45b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 mmid; 46b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 47273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.state = 1; 48273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.stag = stag; 49273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mmid = stag >> 8; 50273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->ibmr.rkey = mhp->ibmr.lkey = stag; 51273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier PDBG("%s mmid 0x%x mhp %p\n", __func__, mmid, mhp); 5213a239330abdda5afcb1161fce68ec214af2cc90Steve Wise return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid); 53273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier} 54273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 55273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreierint iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php, 56273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier struct iwch_mr *mhp, int shift) 57273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier{ 58273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier u32 stag; 5913a239330abdda5afcb1161fce68ec214af2cc90Steve Wise int ret; 60b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 61b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (cxio_register_phys_mem(&rhp->rdev, 62b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise &stag, mhp->attr.pdid, 63b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.perms, 64b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.zbva, 65b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.va_fbo, 66b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.len, 67273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier shift - 12, 68273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_size, mhp->attr.pbl_addr)) 69b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 70273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 7113a239330abdda5afcb1161fce68ec214af2cc90Steve Wise ret = iwch_finish_mem_reg(mhp, stag); 7213a239330abdda5afcb1161fce68ec214af2cc90Steve Wise if (ret) 7313a239330abdda5afcb1161fce68ec214af2cc90Steve Wise cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 7413a239330abdda5afcb1161fce68ec214af2cc90Steve Wise mhp->attr.pbl_addr); 7513a239330abdda5afcb1161fce68ec214af2cc90Steve Wise return ret; 76b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 77b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 78b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_reregister_mem(struct iwch_dev *rhp, struct iwch_pd *php, 79b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_mr *mhp, 80b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int shift, 81b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int npages) 82b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 83b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 stag; 8413a239330abdda5afcb1161fce68ec214af2cc90Steve Wise int ret; 85b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 86b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* We could support this... */ 87b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (npages > mhp->attr.pbl_size) 88b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 89b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 90b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise stag = mhp->attr.stag; 91b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (cxio_reregister_phys_mem(&rhp->rdev, 92b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise &stag, mhp->attr.pdid, 93b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.perms, 94b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.zbva, 95b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.va_fbo, 96b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.len, 97273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier shift - 12, 98273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_size, mhp->attr.pbl_addr)) 99b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 100273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 10113a239330abdda5afcb1161fce68ec214af2cc90Steve Wise ret = iwch_finish_mem_reg(mhp, stag); 10213a239330abdda5afcb1161fce68ec214af2cc90Steve Wise if (ret) 10313a239330abdda5afcb1161fce68ec214af2cc90Steve Wise cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 10413a239330abdda5afcb1161fce68ec214af2cc90Steve Wise mhp->attr.pbl_addr); 105273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 10613a239330abdda5afcb1161fce68ec214af2cc90Steve Wise return ret; 107273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier} 108273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 109273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreierint iwch_alloc_pbl(struct iwch_mr *mhp, int npages) 110273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier{ 111273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_addr = cxio_hal_pblpool_alloc(&mhp->rhp->rdev, 112273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier npages << 3); 113273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 114273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier if (!mhp->attr.pbl_addr) 115273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier return -ENOMEM; 116273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 117273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_size = npages; 118273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 119b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 120b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 121b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 122273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreiervoid iwch_free_pbl(struct iwch_mr *mhp) 123273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier{ 124273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier cxio_hal_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 125273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_size << 3); 126273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier} 127273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 128273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreierint iwch_write_pbl(struct iwch_mr *mhp, __be64 *pages, int npages, int offset) 129273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier{ 130273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier return cxio_write_pbl(&mhp->rhp->rdev, pages, 131273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_addr + (offset << 3), npages); 132273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier} 133273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 134b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint build_phys_page_list(struct ib_phys_buf *buffer_list, 135b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int num_phys_buf, 136b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u64 *iova_start, 137b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u64 *total_size, 138b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int *npages, 139b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int *shift, 140b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise __be64 **page_list) 141b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 142b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u64 mask; 143b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int i, j, n; 144b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 145b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mask = 0; 146b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *total_size = 0; 147b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < num_phys_buf; ++i) { 148b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (i != 0 && buffer_list[i].addr & ~PAGE_MASK) 149b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 150b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (i != 0 && i != num_phys_buf - 1 && 151b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (buffer_list[i].size & ~PAGE_MASK)) 152b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 153b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *total_size += buffer_list[i].size; 154b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (i > 0) 155b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mask |= buffer_list[i].addr; 156d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise else 157d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise mask |= buffer_list[i].addr & PAGE_MASK; 158d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise if (i != num_phys_buf - 1) 159d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise mask |= buffer_list[i].addr + buffer_list[i].size; 160d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise else 161d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise mask |= (buffer_list[i].addr + buffer_list[i].size + 162d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise PAGE_SIZE - 1) & PAGE_MASK; 163b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 164b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 165b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (*total_size > 0xFFFFFFFFULL) 166b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 167b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 168b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* Find largest page shift we can use to cover buffers */ 169b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (*shift = PAGE_SHIFT; *shift < 27; ++(*shift)) 17082d416fffb5e8e39e899be7075dbeeac5fb8f0c2Bryan Rosenburg if ((1ULL << *shift) & mask) 17182d416fffb5e8e39e899be7075dbeeac5fb8f0c2Bryan Rosenburg break; 172b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 173b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise buffer_list[0].size += buffer_list[0].addr & ((1ULL << *shift) - 1); 174b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise buffer_list[0].addr &= ~0ull << *shift; 175b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 176b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *npages = 0; 177b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < num_phys_buf; ++i) 178b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *npages += (buffer_list[i].size + 179b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (1ULL << *shift) - 1) >> *shift; 180b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 181b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!*npages) 182b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 183b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 184b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *page_list = kmalloc(sizeof(u64) * *npages, GFP_KERNEL); 185b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!*page_list) 186b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 187b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 188b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise n = 0; 189b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < num_phys_buf; ++i) 190b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (j = 0; 191b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise j < (buffer_list[i].size + (1ULL << *shift) - 1) >> *shift; 192b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++j) 193b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (*page_list)[n++] = cpu_to_be64(buffer_list[i].addr + 194b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ((u64) j << *shift)); 195b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 196b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s va 0x%llx mask 0x%llx shift %d len %lld pbl_size %d\n", 1973371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison __func__, (unsigned long long) *iova_start, 198b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (unsigned long long) mask, *shift, (unsigned long long) *total_size, 199b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *npages); 200b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 201b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 202b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 203b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 204