iwch_mem.c revision 13a239330abdda5afcb1161fce68ec214af2cc90
1b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise/* 2b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Copyright (c) 2006 Chelsio, Inc. All rights reserved. 3b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 4b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * This software is available to you under a choice of one of two 5b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * licenses. You may choose to be licensed under the terms of the GNU 6b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * General Public License (GPL) Version 2, available from the file 7b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * COPYING in the main directory of this source tree, or the 8b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * OpenIB.org BSD license below: 9b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 10b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Redistribution and use in source and binary forms, with or 11b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * without modification, are permitted provided that the following 12b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * conditions are met: 13b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 14b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * - Redistributions of source code must retain the above 15b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * copyright notice, this list of conditions and the following 16b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * disclaimer. 17b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 18b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * - Redistributions in binary form must reproduce the above 19b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * copyright notice, this list of conditions and the following 20b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * disclaimer in the documentation and/or other materials 21b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * provided with the distribution. 22b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 23b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * SOFTWARE. 31b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 32b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include <asm/byteorder.h> 33b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 34b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include <rdma/iw_cm.h> 35b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include <rdma/ib_verbs.h> 36b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 37b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "cxio_hal.h" 38273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier#include "cxio_resource.h" 39b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "iwch.h" 40b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "iwch_provider.h" 41b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 4213a239330abdda5afcb1161fce68ec214af2cc90Steve Wisestatic int iwch_finish_mem_reg(struct iwch_mr *mhp, u32 stag) 43b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 44b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 mmid; 45b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 46273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.state = 1; 47273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.stag = stag; 48273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mmid = stag >> 8; 49273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->ibmr.rkey = mhp->ibmr.lkey = stag; 50273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier PDBG("%s mmid 0x%x mhp %p\n", __func__, mmid, mhp); 5113a239330abdda5afcb1161fce68ec214af2cc90Steve Wise return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid); 52273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier} 53273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 54273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreierint iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php, 55273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier struct iwch_mr *mhp, int shift) 56273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier{ 57273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier u32 stag; 5813a239330abdda5afcb1161fce68ec214af2cc90Steve Wise int ret; 59b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 60b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (cxio_register_phys_mem(&rhp->rdev, 61b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise &stag, mhp->attr.pdid, 62b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.perms, 63b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.zbva, 64b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.va_fbo, 65b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.len, 66273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier shift - 12, 67273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_size, mhp->attr.pbl_addr)) 68b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 69273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 7013a239330abdda5afcb1161fce68ec214af2cc90Steve Wise ret = iwch_finish_mem_reg(mhp, stag); 7113a239330abdda5afcb1161fce68ec214af2cc90Steve Wise if (ret) 7213a239330abdda5afcb1161fce68ec214af2cc90Steve Wise cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 7313a239330abdda5afcb1161fce68ec214af2cc90Steve Wise mhp->attr.pbl_addr); 7413a239330abdda5afcb1161fce68ec214af2cc90Steve Wise return ret; 75b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 76b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 77b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_reregister_mem(struct iwch_dev *rhp, struct iwch_pd *php, 78b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_mr *mhp, 79b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int shift, 80b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int npages) 81b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 82b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 stag; 8313a239330abdda5afcb1161fce68ec214af2cc90Steve Wise int ret; 84b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 85b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* We could support this... */ 86b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (npages > mhp->attr.pbl_size) 87b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 88b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 89b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise stag = mhp->attr.stag; 90b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (cxio_reregister_phys_mem(&rhp->rdev, 91b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise &stag, mhp->attr.pdid, 92b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.perms, 93b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.zbva, 94b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.va_fbo, 95b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.len, 96273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier shift - 12, 97273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_size, mhp->attr.pbl_addr)) 98b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 99273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 10013a239330abdda5afcb1161fce68ec214af2cc90Steve Wise ret = iwch_finish_mem_reg(mhp, stag); 10113a239330abdda5afcb1161fce68ec214af2cc90Steve Wise if (ret) 10213a239330abdda5afcb1161fce68ec214af2cc90Steve Wise cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 10313a239330abdda5afcb1161fce68ec214af2cc90Steve Wise mhp->attr.pbl_addr); 104273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 10513a239330abdda5afcb1161fce68ec214af2cc90Steve Wise return ret; 106273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier} 107273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 108273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreierint iwch_alloc_pbl(struct iwch_mr *mhp, int npages) 109273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier{ 110273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_addr = cxio_hal_pblpool_alloc(&mhp->rhp->rdev, 111273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier npages << 3); 112273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 113273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier if (!mhp->attr.pbl_addr) 114273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier return -ENOMEM; 115273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 116273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_size = npages; 117273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 118b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 119b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 120b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 121273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreiervoid iwch_free_pbl(struct iwch_mr *mhp) 122273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier{ 123273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier cxio_hal_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 124273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_size << 3); 125273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier} 126273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 127273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreierint iwch_write_pbl(struct iwch_mr *mhp, __be64 *pages, int npages, int offset) 128273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier{ 129273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier return cxio_write_pbl(&mhp->rhp->rdev, pages, 130273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier mhp->attr.pbl_addr + (offset << 3), npages); 131273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier} 132273748cc908a901d082b4da5a16b2541c9d78a02Roland Dreier 133b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint build_phys_page_list(struct ib_phys_buf *buffer_list, 134b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int num_phys_buf, 135b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u64 *iova_start, 136b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u64 *total_size, 137b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int *npages, 138b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int *shift, 139b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise __be64 **page_list) 140b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 141b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u64 mask; 142b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int i, j, n; 143b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 144b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mask = 0; 145b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *total_size = 0; 146b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < num_phys_buf; ++i) { 147b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (i != 0 && buffer_list[i].addr & ~PAGE_MASK) 148b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 149b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (i != 0 && i != num_phys_buf - 1 && 150b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (buffer_list[i].size & ~PAGE_MASK)) 151b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 152b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *total_size += buffer_list[i].size; 153b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (i > 0) 154b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mask |= buffer_list[i].addr; 155d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise else 156d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise mask |= buffer_list[i].addr & PAGE_MASK; 157d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise if (i != num_phys_buf - 1) 158d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise mask |= buffer_list[i].addr + buffer_list[i].size; 159d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise else 160d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise mask |= (buffer_list[i].addr + buffer_list[i].size + 161d08ca26ceec4dfbcfdbada4ad728db742ccaecd1Steve Wise PAGE_SIZE - 1) & PAGE_MASK; 162b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 163b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 164b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (*total_size > 0xFFFFFFFFULL) 165b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 166b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 167b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* Find largest page shift we can use to cover buffers */ 168b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (*shift = PAGE_SHIFT; *shift < 27; ++(*shift)) 16982d416fffb5e8e39e899be7075dbeeac5fb8f0c2Bryan Rosenburg if ((1ULL << *shift) & mask) 17082d416fffb5e8e39e899be7075dbeeac5fb8f0c2Bryan Rosenburg break; 171b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 172b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise buffer_list[0].size += buffer_list[0].addr & ((1ULL << *shift) - 1); 173b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise buffer_list[0].addr &= ~0ull << *shift; 174b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 175b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *npages = 0; 176b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < num_phys_buf; ++i) 177b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *npages += (buffer_list[i].size + 178b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (1ULL << *shift) - 1) >> *shift; 179b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 180b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!*npages) 181b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 182b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 183b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *page_list = kmalloc(sizeof(u64) * *npages, GFP_KERNEL); 184b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!*page_list) 185b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 186b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 187b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise n = 0; 188b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < num_phys_buf; ++i) 189b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (j = 0; 190b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise j < (buffer_list[i].size + (1ULL << *shift) - 1) >> *shift; 191b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++j) 192b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (*page_list)[n++] = cpu_to_be64(buffer_list[i].addr + 193b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ((u64) j << *shift)); 194b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 195b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s va 0x%llx mask 0x%llx shift %d len %lld pbl_size %d\n", 1963371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison __func__, (unsigned long long) *iova_start, 197b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (unsigned long long) mask, *shift, (unsigned long long) *total_size, 198b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *npages); 199b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 200b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 201b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 202b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 203