11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. 356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier * Copyright (c) 2005 Cisco Systems. All rights reserved. 42a1d9b7f09aaaacf235656cb32a40ba2c79590b3Roland Dreier * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This software is available to you under a choice of one of two 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * licenses. You may choose to be licensed under the terms of the GNU 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * General Public License (GPL) Version 2, available from the file 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * COPYING in the main directory of this source tree, or the 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * OpenIB.org BSD license below: 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Redistribution and use in source and binary forms, with or 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * without modification, are permitted provided that the following 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * conditions are met: 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - Redistributions of source code must retain the above 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * copyright notice, this list of conditions and the following 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * disclaimer. 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - Redistributions in binary form must reproduce the above 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * copyright notice, this list of conditions and the following 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * disclaimer in the documentation and/or other materials 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * provided with the distribution. 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * SOFTWARE. 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 35dbcf31ba68c92b23ac3f874c4d516bacd5d7518aRoland Dreier#include <linux/mm.h> 36391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin#include <linux/scatterlist.h> 37e8edc6e03a5c8562dc70a6d969f732bdb355a7e7Alexey Dobriyan#include <linux/sched.h> 385a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 39391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 40391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin#include <asm/page.h> 41dbcf31ba68c92b23ac3f874c4d516bacd5d7518aRoland Dreier 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "mthca_memfree.h" 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "mthca_dev.h" 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "mthca_cmd.h" 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We allocate in as big chunks as we can, up to a maximum of 256 KB 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * per chunk. 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsenum { 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds MTHCA_ICM_ALLOC_SIZE = 1 << 18, 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds MTHCA_TABLE_CHUNK_SIZE = 1 << 18 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreierstruct mthca_user_db_table { 56fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier struct mutex mutex; 5756483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier struct { 5856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier u64 uvirt; 5956483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier struct scatterlist mem; 6056483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier int refcount; 6156483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier } page[0]; 6256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier}; 6356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 64391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkinstatic void mthca_free_icm_pages(struct mthca_dev *dev, struct mthca_icm_chunk *chunk) 65391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin{ 66391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin int i; 67391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 68391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin if (chunk->nsg > 0) 69391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages, 70391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin PCI_DMA_BIDIRECTIONAL); 71391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 72391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin for (i = 0; i < chunk->npages; ++i) 7345711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe __free_pages(sg_page(&chunk->mem[i]), 74391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin get_order(chunk->mem[i].length)); 75391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin} 76391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 77391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkinstatic void mthca_free_icm_coherent(struct mthca_dev *dev, struct mthca_icm_chunk *chunk) 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 81391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin for (i = 0; i < chunk->npages; ++i) { 82391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length, 8345711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe lowmem_page_address(sg_page(&chunk->mem[i])), 84391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin sg_dma_address(&chunk->mem[i])); 85391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin } 86391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin} 87391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 88391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkinvoid mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm, int coherent) 89391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin{ 90391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin struct mthca_icm_chunk *chunk, *tmp; 91391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!icm) 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) { 96391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin if (coherent) 97391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin mthca_free_icm_coherent(dev, chunk); 98391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin else 99391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin mthca_free_icm_pages(dev, chunk); 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(chunk); 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(icm); 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 107391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkinstatic int mthca_alloc_icm_pages(struct scatterlist *mem, int order, gfp_t gfp_mask) 108391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin{ 10945711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe struct page *page; 11045711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe 11187afd448b186c885d67a08b7417cd46253b6a9d6Eli Cohen /* 11287afd448b186c885d67a08b7417cd46253b6a9d6Eli Cohen * Use __GFP_ZERO because buggy firmware assumes ICM pages are 11387afd448b186c885d67a08b7417cd46253b6a9d6Eli Cohen * cleared, and subtle failures are seen if they aren't. 11487afd448b186c885d67a08b7417cd46253b6a9d6Eli Cohen */ 11587afd448b186c885d67a08b7417cd46253b6a9d6Eli Cohen page = alloc_pages(gfp_mask | __GFP_ZERO, order); 11645711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe if (!page) 117391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin return -ENOMEM; 118391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 119642f149031d70415d9318b919d50b71e4724adbdJens Axboe sg_set_page(mem, page, PAGE_SIZE << order, 0); 120391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin return 0; 121391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin} 122391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 123391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkinstatic int mthca_alloc_icm_coherent(struct device *dev, struct scatterlist *mem, 124391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin int order, gfp_t gfp_mask) 125391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin{ 126391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order, &sg_dma_address(mem), 127391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin gfp_mask); 128391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin if (!buf) 129391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin return -ENOMEM; 130391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 131391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin sg_set_buf(mem, buf, PAGE_SIZE << order); 132391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin BUG_ON(mem->offset); 133391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin sg_dma_len(mem) = PAGE_SIZE << order; 134391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin return 0; 135391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin} 136391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, 138391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin gfp_t gfp_mask, int coherent) 1391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct mthca_icm *icm; 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct mthca_icm_chunk *chunk = NULL; 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int cur_order; 143391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin int ret; 144391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 145391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin /* We use sg_set_buf for coherent allocs, which assumes low memory */ 146391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM)); 1471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!icm) 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return icm; 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds icm->refcount = 0; 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds INIT_LIST_HEAD(&icm->chunk_list); 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cur_order = get_order(MTHCA_ICM_ALLOC_SIZE); 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (npages > 0) { 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!chunk) { 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds chunk = kmalloc(sizeof *chunk, 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!chunk) 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fail; 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 16445711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe sg_init_table(chunk->mem, MTHCA_ICM_CHUNK_LEN); 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds chunk->npages = 0; 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds chunk->nsg = 0; 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds list_add_tail(&chunk->list, &icm->chunk_list); 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (1 << cur_order > npages) 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds --cur_order; 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 173391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin if (coherent) 174391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin ret = mthca_alloc_icm_coherent(&dev->pdev->dev, 175391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin &chunk->mem[chunk->npages], 176391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin cur_order, gfp_mask); 177391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin else 178391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin ret = mthca_alloc_icm_pages(&chunk->mem[chunk->npages], 179391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin cur_order, gfp_mask); 180391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin 181391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin if (!ret) { 182391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin ++chunk->npages; 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 18411282b32a4ed7bb5fc390c83a1d6b9bc8017882cRoland Dreier if (coherent) 18511282b32a4ed7bb5fc390c83a1d6b9bc8017882cRoland Dreier ++chunk->nsg; 18611282b32a4ed7bb5fc390c83a1d6b9bc8017882cRoland Dreier else if (chunk->npages == MTHCA_ICM_CHUNK_LEN) { 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds chunk->npages, 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds PCI_DMA_BIDIRECTIONAL); 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (chunk->nsg <= 0) 1921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fail; 193391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin } 1941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 195391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin if (chunk->npages == MTHCA_ICM_CHUNK_LEN) 1961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds chunk = NULL; 1971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds npages -= 1 << cur_order; 1991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } else { 2001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds --cur_order; 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (cur_order < 0) 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fail; 2031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 206391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin if (!coherent && chunk) { 2071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, 2081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds chunk->npages, 2091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds PCI_DMA_BIDIRECTIONAL); 2101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (chunk->nsg <= 0) 2121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fail; 2131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return icm; 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsfail: 218391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin mthca_free_icm(dev, icm, coherent); 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj) 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE; 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int ret = 0; 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 227fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_lock(&table->mutex); 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (table->icm[i]) { 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ++table->icm[i]->refcount; 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT, 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | 236391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin __GFP_NOWARN, table->coherent); 2371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!table->icm[i]) { 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = -ENOMEM; 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 242cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues if (mthca_MAP_ICM(dev, table->icm[i], 243cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues table->virt + i * MTHCA_TABLE_CHUNK_SIZE)) { 244391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin mthca_free_icm(dev, table->icm[i], table->coherent); 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds table->icm[i] = NULL; 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = -ENOMEM; 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 2481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ++table->icm[i]->refcount; 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout: 253fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_unlock(&table->mutex); 2541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj) 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 259a03a5a67b243e9a24805ee18272ad25e5b2ca92cRoland Dreier int i; 2601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 261a03a5a67b243e9a24805ee18272ad25e5b2ca92cRoland Dreier if (!mthca_is_memfree(dev)) 262a03a5a67b243e9a24805ee18272ad25e5b2ca92cRoland Dreier return; 263a03a5a67b243e9a24805ee18272ad25e5b2ca92cRoland Dreier 264a03a5a67b243e9a24805ee18272ad25e5b2ca92cRoland Dreier i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE; 265a03a5a67b243e9a24805ee18272ad25e5b2ca92cRoland Dreier 266fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_lock(&table->mutex); 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (--table->icm[i]->refcount == 0) { 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE, 270cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE); 271391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin mthca_free_icm(dev, table->icm[i], table->coherent); 2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds table->icm[i] = NULL; 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 275fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_unlock(&table->mutex); 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 278391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkinvoid *mthca_table_find(struct mthca_icm_table *table, int obj, dma_addr_t *dma_handle) 2790fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin{ 280391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin int idx, offset, dma_offset, i; 2810fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin struct mthca_icm_chunk *chunk; 2820fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin struct mthca_icm *icm; 2830fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin struct page *page = NULL; 2840fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin 2850fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin if (!table->lowmem) 2860fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin return NULL; 2870fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin 288fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_lock(&table->mutex); 2890fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin 2900fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin idx = (obj & (table->num_obj - 1)) * table->obj_size; 2910fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin icm = table->icm[idx / MTHCA_TABLE_CHUNK_SIZE]; 292391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin dma_offset = offset = idx % MTHCA_TABLE_CHUNK_SIZE; 2930fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin 2940fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin if (!icm) 2950fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin goto out; 2960fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin 2970fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin list_for_each_entry(chunk, &icm->chunk_list, list) { 2980fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin for (i = 0; i < chunk->npages; ++i) { 299391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin if (dma_handle && dma_offset >= 0) { 300391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin if (sg_dma_len(&chunk->mem[i]) > dma_offset) 301391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin *dma_handle = sg_dma_address(&chunk->mem[i]) + 302391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin dma_offset; 303391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin dma_offset -= sg_dma_len(&chunk->mem[i]); 304391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin } 305391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin /* DMA mapping can merge pages but not split them, 306391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin * so if we found the page, dma_handle has already 307391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin * been assigned to. */ 30846707e96b7254663139225ab6c9ab9922cd8c435Michael S. Tsirkin if (chunk->mem[i].length > offset) { 30945711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe page = sg_page(&chunk->mem[i]); 3106c7d2a75b512c64c910b69adf32dbaddb461910bMichael S. Tsirkin goto out; 3110fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin } 3120fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin offset -= chunk->mem[i].length; 3130fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin } 3140fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin } 3150fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin 3160fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkinout: 317fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_unlock(&table->mutex); 3180fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin return page ? lowmem_page_address(page) + offset : NULL; 3190fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin} 3200fabd9fb7bdc935f121e6950a2c4eff971dd4c75Michael S. Tsirkin 32186562a139182bb19c984347f9625b61f3e6f7815Roland Dreierint mthca_table_get_range(struct mthca_dev *dev, struct mthca_icm_table *table, 32286562a139182bb19c984347f9625b61f3e6f7815Roland Dreier int start, int end) 32386562a139182bb19c984347f9625b61f3e6f7815Roland Dreier{ 32486562a139182bb19c984347f9625b61f3e6f7815Roland Dreier int inc = MTHCA_TABLE_CHUNK_SIZE / table->obj_size; 32586562a139182bb19c984347f9625b61f3e6f7815Roland Dreier int i, err; 32686562a139182bb19c984347f9625b61f3e6f7815Roland Dreier 32786562a139182bb19c984347f9625b61f3e6f7815Roland Dreier for (i = start; i <= end; i += inc) { 32886562a139182bb19c984347f9625b61f3e6f7815Roland Dreier err = mthca_table_get(dev, table, i); 32986562a139182bb19c984347f9625b61f3e6f7815Roland Dreier if (err) 33086562a139182bb19c984347f9625b61f3e6f7815Roland Dreier goto fail; 33186562a139182bb19c984347f9625b61f3e6f7815Roland Dreier } 33286562a139182bb19c984347f9625b61f3e6f7815Roland Dreier 33386562a139182bb19c984347f9625b61f3e6f7815Roland Dreier return 0; 33486562a139182bb19c984347f9625b61f3e6f7815Roland Dreier 33586562a139182bb19c984347f9625b61f3e6f7815Roland Dreierfail: 33686562a139182bb19c984347f9625b61f3e6f7815Roland Dreier while (i > start) { 33786562a139182bb19c984347f9625b61f3e6f7815Roland Dreier i -= inc; 33886562a139182bb19c984347f9625b61f3e6f7815Roland Dreier mthca_table_put(dev, table, i); 33986562a139182bb19c984347f9625b61f3e6f7815Roland Dreier } 34086562a139182bb19c984347f9625b61f3e6f7815Roland Dreier 34186562a139182bb19c984347f9625b61f3e6f7815Roland Dreier return err; 34286562a139182bb19c984347f9625b61f3e6f7815Roland Dreier} 34386562a139182bb19c984347f9625b61f3e6f7815Roland Dreier 34486562a139182bb19c984347f9625b61f3e6f7815Roland Dreiervoid mthca_table_put_range(struct mthca_dev *dev, struct mthca_icm_table *table, 34586562a139182bb19c984347f9625b61f3e6f7815Roland Dreier int start, int end) 34686562a139182bb19c984347f9625b61f3e6f7815Roland Dreier{ 34786562a139182bb19c984347f9625b61f3e6f7815Roland Dreier int i; 34886562a139182bb19c984347f9625b61f3e6f7815Roland Dreier 349a03a5a67b243e9a24805ee18272ad25e5b2ca92cRoland Dreier if (!mthca_is_memfree(dev)) 350a03a5a67b243e9a24805ee18272ad25e5b2ca92cRoland Dreier return; 351a03a5a67b243e9a24805ee18272ad25e5b2ca92cRoland Dreier 35286562a139182bb19c984347f9625b61f3e6f7815Roland Dreier for (i = start; i <= end; i += MTHCA_TABLE_CHUNK_SIZE / table->obj_size) 35386562a139182bb19c984347f9625b61f3e6f7815Roland Dreier mthca_table_put(dev, table, i); 35486562a139182bb19c984347f9625b61f3e6f7815Roland Dreier} 35586562a139182bb19c984347f9625b61f3e6f7815Roland Dreier 3561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev, 3571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds u64 virt, int obj_size, 3581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int nobj, int reserved, 359391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin int use_lowmem, int use_coherent) 3601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct mthca_icm_table *table; 362c263ff65d5936113cfcbb8139d34122361e2306eRoland Dreier int obj_per_chunk; 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int num_icm; 364d20a40192868082eff6fec729b311cb8463b4a21Roland Dreier unsigned chunk_size; 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 3661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 367c263ff65d5936113cfcbb8139d34122361e2306eRoland Dreier obj_per_chunk = MTHCA_TABLE_CHUNK_SIZE / obj_size; 368c263ff65d5936113cfcbb8139d34122361e2306eRoland Dreier num_icm = DIV_ROUND_UP(nobj, obj_per_chunk); 3691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds table = kmalloc(sizeof *table + num_icm * sizeof *table->icm, GFP_KERNEL); 3711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!table) 3721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 3731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds table->virt = virt; 3751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds table->num_icm = num_icm; 3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds table->num_obj = nobj; 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds table->obj_size = obj_size; 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds table->lowmem = use_lowmem; 379391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin table->coherent = use_coherent; 380fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_init(&table->mutex); 3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < num_icm; ++i) 3831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds table->icm[i] = NULL; 3841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i * MTHCA_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) { 386d20a40192868082eff6fec729b311cb8463b4a21Roland Dreier chunk_size = MTHCA_TABLE_CHUNK_SIZE; 387d20a40192868082eff6fec729b311cb8463b4a21Roland Dreier if ((i + 1) * MTHCA_TABLE_CHUNK_SIZE > nobj * obj_size) 388d20a40192868082eff6fec729b311cb8463b4a21Roland Dreier chunk_size = nobj * obj_size - i * MTHCA_TABLE_CHUNK_SIZE; 389d20a40192868082eff6fec729b311cb8463b4a21Roland Dreier 390d20a40192868082eff6fec729b311cb8463b4a21Roland Dreier table->icm[i] = mthca_alloc_icm(dev, chunk_size >> PAGE_SHIFT, 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) | 392391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin __GFP_NOWARN, use_coherent); 3931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!table->icm[i]) 3941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto err; 395cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues if (mthca_MAP_ICM(dev, table->icm[i], 396cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues virt + i * MTHCA_TABLE_CHUNK_SIZE)) { 397391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin mthca_free_icm(dev, table->icm[i], table->coherent); 3981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds table->icm[i] = NULL; 3991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto err; 4001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 4031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Add a reference to this ICM chunk so that it never 4041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * gets freed (since it contains reserved firmware objects). 4051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 4061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ++table->icm[i]->refcount; 4071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return table; 4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldserr: 4121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < num_icm; ++i) 4131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (table->icm[i]) { 4141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mthca_UNMAP_ICM(dev, virt + i * MTHCA_TABLE_CHUNK_SIZE, 415cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE); 416391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin mthca_free_icm(dev, table->icm[i], table->coherent); 4171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(table); 4201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 4221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table) 4251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 4271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < table->num_icm; ++i) 4291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (table->icm[i]) { 430cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues mthca_UNMAP_ICM(dev, 431cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues table->virt + i * MTHCA_TABLE_CHUNK_SIZE, 432cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE); 433391e4dea7189eef32b0c2d121e7e047110c1b83cMichael S. Tsirkin mthca_free_icm(dev, table->icm[i], table->coherent); 4341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(table); 4371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 43956483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreierstatic u64 mthca_uarc_virt(struct mthca_dev *dev, struct mthca_uar *uar, int page) 4401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return dev->uar_table.uarc_base + 44256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier uar->index * dev->uar_table.uarc_size + 4438d3ef29d6be1e750512e0a9dbea6225290b81d0aIshai Rabinovitz page * MTHCA_ICM_PAGE_SIZE; 4441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 44656483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreierint mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, 44756483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier struct mthca_user_db_table *db_tab, int index, u64 uaddr) 44856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier{ 44945711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe struct page *pages[1]; 45056483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier int ret = 0; 45156483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier int i; 45256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 45356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if (!mthca_is_memfree(dev)) 45456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier return 0; 45556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 45656483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if (index < 0 || index > dev->uar_table.uarc_size / 8) 45756483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier return -EINVAL; 45856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 459fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_lock(&db_tab->mutex); 46056483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 46156483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier i = index / MTHCA_DB_REC_PER_PAGE; 46256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 46356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if ((db_tab->page[i].refcount >= MTHCA_DB_REC_PER_PAGE) || 46456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier (db_tab->page[i].uvirt && db_tab->page[i].uvirt != uaddr) || 46556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier (uaddr & 4095)) { 46656483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier ret = -EINVAL; 46756483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier goto out; 46856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier } 46956483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 47056483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if (db_tab->page[i].refcount) { 47156483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier ++db_tab->page[i].refcount; 47256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier goto out; 47356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier } 47456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 47556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier ret = get_user_pages(current, current->mm, uaddr & PAGE_MASK, 1, 1, 0, 47645711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe pages, NULL); 47756483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if (ret < 0) 47856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier goto out; 47956483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 480642f149031d70415d9318b919d50b71e4724adbdJens Axboe sg_set_page(&db_tab->page[i].mem, pages[0], MTHCA_ICM_PAGE_SIZE, 481642f149031d70415d9318b919d50b71e4724adbdJens Axboe uaddr & ~PAGE_MASK); 48256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 48356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); 48456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if (ret < 0) { 48545711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe put_page(pages[0]); 48656483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier goto out; 48756483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier } 48856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 48956483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem), 490cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues mthca_uarc_virt(dev, uar, i)); 49156483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if (ret) { 49256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); 49345711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe put_page(sg_page(&db_tab->page[i].mem)); 49456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier goto out; 49556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier } 49656483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 49756483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier db_tab->page[i].uvirt = uaddr; 49856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier db_tab->page[i].refcount = 1; 49956483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 50056483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreierout: 501fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_unlock(&db_tab->mutex); 50256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier return ret; 50356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier} 50456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 50556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreiervoid mthca_unmap_user_db(struct mthca_dev *dev, struct mthca_uar *uar, 50656483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier struct mthca_user_db_table *db_tab, int index) 50756483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier{ 50856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if (!mthca_is_memfree(dev)) 50956483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier return; 51056483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 51156483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier /* 51256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier * To make our bookkeeping simpler, we don't unmap DB 51356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier * pages until we clean up the whole db table. 51456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier */ 51556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 516fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_lock(&db_tab->mutex); 51756483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 51856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier --db_tab->page[index / MTHCA_DB_REC_PER_PAGE].refcount; 51956483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 520fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_unlock(&db_tab->mutex); 52156483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier} 52256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 52356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreierstruct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev) 52456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier{ 52556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier struct mthca_user_db_table *db_tab; 52656483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier int npages; 52756483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier int i; 52856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 52956483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if (!mthca_is_memfree(dev)) 53056483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier return NULL; 53156483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 5328d3ef29d6be1e750512e0a9dbea6225290b81d0aIshai Rabinovitz npages = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; 53356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier db_tab = kmalloc(sizeof *db_tab + npages * sizeof *db_tab->page, GFP_KERNEL); 53456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if (!db_tab) 53556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier return ERR_PTR(-ENOMEM); 53656483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 537fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_init(&db_tab->mutex); 53856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier for (i = 0; i < npages; ++i) { 53956483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier db_tab->page[i].refcount = 0; 54056483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier db_tab->page[i].uvirt = 0; 541fe174357eb2deb184c93269846c92adf5743115bRoland Dreier sg_init_table(&db_tab->page[i].mem, 1); 54256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier } 54356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 54456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier return db_tab; 54556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier} 54656483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 54756483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreiervoid mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar, 54856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier struct mthca_user_db_table *db_tab) 54956483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier{ 55056483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier int i; 55156483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 55256483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if (!mthca_is_memfree(dev)) 55356483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier return; 55456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 5558d3ef29d6be1e750512e0a9dbea6225290b81d0aIshai Rabinovitz for (i = 0; i < dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; ++i) { 55656483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier if (db_tab->page[i].uvirt) { 557cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1); 55856483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); 55945711f1af6eff1a6d010703b4862e0d2b9afd056Jens Axboe put_page(sg_page(&db_tab->page[i].mem)); 56056483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier } 56156483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier } 56252d0df153c987e4ad57d15f5df91848f65858e5dJack Morgenstein 56352d0df153c987e4ad57d15f5df91848f65858e5dJack Morgenstein kfree(db_tab); 56456483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier} 56556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier 566c6f5cb7be0ab478e0618e3c2c6ada27f56d1e7fbRoland Dreierint mthca_alloc_db(struct mthca_dev *dev, enum mthca_db_type type, 567c6f5cb7be0ab478e0618e3c2c6ada27f56d1e7fbRoland Dreier u32 qn, __be32 **db) 5681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 5691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int group; 5701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int start, end, dir; 5711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i, j; 5721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct mthca_db_page *page; 5731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int ret = 0; 5741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 575fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_lock(&dev->db_tab->mutex); 5761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds switch (type) { 5781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case MTHCA_DB_TYPE_CQ_ARM: 5791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case MTHCA_DB_TYPE_SQ: 5801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds group = 0; 5811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds start = 0; 5821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds end = dev->db_tab->max_group1; 5831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dir = 1; 5841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 5851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case MTHCA_DB_TYPE_CQ_SET_CI: 5871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case MTHCA_DB_TYPE_RQ: 5881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case MTHCA_DB_TYPE_SRQ: 5891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds group = 1; 5901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds start = dev->db_tab->npages - 1; 5911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds end = dev->db_tab->min_group2; 5921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dir = -1; 5931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 5941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds default: 5962714eb5a4fb2b461b452fb8a2f65c8caacdd12d5Roland Dreier ret = -EINVAL; 5972714eb5a4fb2b461b452fb8a2f65c8caacdd12d5Roland Dreier goto out; 5981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 5991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = start; i != end; i += dir) 6011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (dev->db_tab->page[i].db_rec && 6021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds !bitmap_full(dev->db_tab->page[i].used, 6031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds MTHCA_DB_REC_PER_PAGE)) { 6041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds page = dev->db_tab->page + i; 6051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto found; 6061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 6071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 608018771f435388f22f388eb8658c652086fb3633eRoland Dreier for (i = start; i != end; i += dir) 609018771f435388f22f388eb8658c652086fb3633eRoland Dreier if (!dev->db_tab->page[i].db_rec) { 610018771f435388f22f388eb8658c652086fb3633eRoland Dreier page = dev->db_tab->page + i; 611018771f435388f22f388eb8658c652086fb3633eRoland Dreier goto alloc; 612018771f435388f22f388eb8658c652086fb3633eRoland Dreier } 613018771f435388f22f388eb8658c652086fb3633eRoland Dreier 6141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (dev->db_tab->max_group1 >= dev->db_tab->min_group2 - 1) { 6151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = -ENOMEM; 6161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 6171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 6181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 619018771f435388f22f388eb8658c652086fb3633eRoland Dreier if (group == 0) 620018771f435388f22f388eb8658c652086fb3633eRoland Dreier ++dev->db_tab->max_group1; 621018771f435388f22f388eb8658c652086fb3633eRoland Dreier else 622018771f435388f22f388eb8658c652086fb3633eRoland Dreier --dev->db_tab->min_group2; 623018771f435388f22f388eb8658c652086fb3633eRoland Dreier 6241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds page = dev->db_tab->page + end; 625018771f435388f22f388eb8658c652086fb3633eRoland Dreier 626018771f435388f22f388eb8658c652086fb3633eRoland Dreieralloc: 6278d3ef29d6be1e750512e0a9dbea6225290b81d0aIshai Rabinovitz page->db_rec = dma_alloc_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, 6281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds &page->mapping, GFP_KERNEL); 6291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!page->db_rec) { 6301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = -ENOMEM; 6311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 6321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 6338d3ef29d6be1e750512e0a9dbea6225290b81d0aIshai Rabinovitz memset(page->db_rec, 0, MTHCA_ICM_PAGE_SIZE); 6341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 63556483ec1b70221f8c9838ccc9a89b43d9de66993Roland Dreier ret = mthca_MAP_ICM_page(dev, page->mapping, 636cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues mthca_uarc_virt(dev, &dev->driver_uar, i)); 6371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ret) { 6388d3ef29d6be1e750512e0a9dbea6225290b81d0aIshai Rabinovitz dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, 6391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds page->db_rec, page->mapping); 6401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 6411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 6421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bitmap_zero(page->used, MTHCA_DB_REC_PER_PAGE); 6441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsfound: 6461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds j = find_first_zero_bit(page->used, MTHCA_DB_REC_PER_PAGE); 6471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_bit(j, page->used); 6481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (group == 1) 6501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds j = MTHCA_DB_REC_PER_PAGE - 1 - j; 6511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = i * MTHCA_DB_REC_PER_PAGE + j; 6531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds page->db_rec[j] = cpu_to_be64((qn << 8) | (type << 5)); 6551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 65697f52eb438be7caebe026421545619d8a0c1398aSean Hefty *db = (__be32 *) &page->db_rec[j]; 6571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout: 659fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_unlock(&dev->db_tab->mutex); 6601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 6621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 6631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid mthca_free_db(struct mthca_dev *dev, int type, int db_index) 6651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 6661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i, j; 6671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct mthca_db_page *page; 6681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds i = db_index / MTHCA_DB_REC_PER_PAGE; 6701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds j = db_index % MTHCA_DB_REC_PER_PAGE; 6711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds page = dev->db_tab->page + i; 6731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 674fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_lock(&dev->db_tab->mutex); 6751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds page->db_rec[j] = 0; 6771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (i >= dev->db_tab->min_group2) 6781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds j = MTHCA_DB_REC_PER_PAGE - 1 - j; 6791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds clear_bit(j, page->used); 6801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (bitmap_empty(page->used, MTHCA_DB_REC_PER_PAGE) && 6821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds i >= dev->db_tab->max_group1 - 1) { 683cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1); 6841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6858d3ef29d6be1e750512e0a9dbea6225290b81d0aIshai Rabinovitz dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, 6861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds page->db_rec, page->mapping); 6871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds page->db_rec = NULL; 6881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (i == dev->db_tab->max_group1) { 6901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds --dev->db_tab->max_group1; 6911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* XXX may be able to unmap more pages now */ 6921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 6931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (i == dev->db_tab->min_group2) 6941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ++dev->db_tab->min_group2; 6951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 6961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 697fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_unlock(&dev->db_tab->mutex); 6981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 6991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint mthca_init_db_tab(struct mthca_dev *dev) 7011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 7021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 7031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 704d10ddbf6d7f6699c386d1f41bf542189de32b6beRoland Dreier if (!mthca_is_memfree(dev)) 7051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 7061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dev->db_tab = kmalloc(sizeof *dev->db_tab, GFP_KERNEL); 7081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!dev->db_tab) 7091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 7101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 711fd9cfdd11be3b37b5c919b64b43990f14a1587bdRoland Dreier mutex_init(&dev->db_tab->mutex); 7121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7138d3ef29d6be1e750512e0a9dbea6225290b81d0aIshai Rabinovitz dev->db_tab->npages = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; 7141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dev->db_tab->max_group1 = 0; 7151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dev->db_tab->min_group2 = dev->db_tab->npages - 1; 7161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dev->db_tab->page = kmalloc(dev->db_tab->npages * 7181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds sizeof *dev->db_tab->page, 7191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GFP_KERNEL); 7201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!dev->db_tab->page) { 7211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(dev->db_tab); 7221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 7231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < dev->db_tab->npages; ++i) 7261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dev->db_tab->page[i].db_rec = NULL; 7271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 7291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 7301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid mthca_cleanup_db_tab(struct mthca_dev *dev) 7321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 7331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 7341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 735d10ddbf6d7f6699c386d1f41bf542189de32b6beRoland Dreier if (!mthca_is_memfree(dev)) 7361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 7371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 7391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Because we don't always free our UARC pages when they 7401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * become empty to make mthca_free_db() simpler we need to 7411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * make a sweep through the doorbell pages and free any 7421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * leftover pages now. 7431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 7441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < dev->db_tab->npages; ++i) { 7451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!dev->db_tab->page[i].db_rec) 7461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds continue; 7471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!bitmap_empty(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE)) 7491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mthca_warn(dev, "Kernel UARC page %d not empty\n", i); 7501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 751cdb73db0b6bf7a1bcf5c788f0c8f803facb6e517Goldwyn Rodrigues mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1); 7521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7538d3ef29d6be1e750512e0a9dbea6225290b81d0aIshai Rabinovitz dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, 7541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dev->db_tab->page[i].db_rec, 7551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dev->db_tab->page[i].mapping); 7561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(dev->db_tab->page); 7591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(dev->db_tab); 7601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 761