1fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/* 2fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * IBM eServer eHCA Infiniband device driver for Linux on POWER 3fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 4fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * MR/MW functions 5fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 6fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Authors: Dietmar Decker <ddecker@de.ibm.com> 7fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Christoph Raisch <raisch@de.ibm.com> 85bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen * Hoang-Nam Nguyen <hnguyen@de.ibm.com> 9fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 10fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Copyright (c) 2005 IBM Corporation 11fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 12fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * All rights reserved. 13fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 14fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * This source code is distributed under a dual license of GPL v2.0 and OpenIB 15fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * BSD. 16fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 17fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * OpenIB BSD License 18fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 19fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Redistribution and use in source and binary forms, with or without 20fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * modification, are permitted provided that the following conditions are met: 21fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 22fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Redistributions of source code must retain the above copyright notice, this 23fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * list of conditions and the following disclaimer. 24fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 25fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Redistributions in binary form must reproduce the above copyright notice, 26fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * this list of conditions and the following disclaimer in the documentation 27fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * and/or other materials 28fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * provided with the distribution. 29fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 30fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 31fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 32fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 33fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 34fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 37fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 38fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * POSSIBILITY OF SUCH DAMAGE. 41fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 42fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 435a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 4405d989f948cda7398c9d5089d78a122502e644d2Hoang-Nam Nguyen#include <rdma/ib_umem.h> 4505d989f948cda7398c9d5089d78a122502e644d2Hoang-Nam Nguyen 46fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick#include "ehca_iverbs.h" 47fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick#include "ehca_mrmw.h" 48fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick#include "hcp_if.h" 49fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick#include "hipz_hw.h" 50fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 512492398e616451788bc7c7905cadb8734b082bc7Hoang-Nam Nguyen#define NUM_CHUNKS(length, chunk_size) \ 522492398e616451788bc7c7905cadb8734b082bc7Hoang-Nam Nguyen (((length) + (chunk_size - 1)) / (chunk_size)) 533a31c41901b6bd3937ec36e0e4a930849e270df6Joachim Fenkes 544e4e74cae73325c9f8513fae3a5bd9f79458f4a7Hoang-Nam Nguyen/* max number of rpages (per hcall register_rpages) */ 554e4e74cae73325c9f8513fae3a5bd9f79458f4a7Hoang-Nam Nguyen#define MAX_RPAGES 512 564e4e74cae73325c9f8513fae3a5bd9f79458f4a7Hoang-Nam Nguyen 570cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering/* DMEM toleration management */ 580cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_SECTSHIFT SECTION_SIZE_BITS 590cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_SECTSIZE (1UL << EHCA_SECTSHIFT) 600cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_HUGEPAGESHIFT 34 610cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_HUGEPAGE_SIZE (1UL << EHCA_HUGEPAGESHIFT) 620cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_HUGEPAGE_PFN_MASK ((EHCA_HUGEPAGE_SIZE - 1) >> PAGE_SHIFT) 630cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_INVAL_ADDR 0xFFFFFFFFFFFFFFFFULL 640cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_DIR_INDEX_SHIFT 13 /* 8k Entries in 64k block */ 650cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_TOP_INDEX_SHIFT (EHCA_DIR_INDEX_SHIFT * 2) 660cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_MAP_ENTRIES (1 << EHCA_DIR_INDEX_SHIFT) 670cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_TOP_MAP_SIZE (0x10000) /* currently fixed map size */ 680cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_DIR_MAP_SIZE (0x10000) 690cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_ENT_MAP_SIZE (0x10000) 700cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define EHCA_INDEX_MASK (EHCA_MAP_ENTRIES - 1) 710cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 720cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic unsigned long ehca_mr_len; 730cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 740cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering/* 750cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering * Memory map data structures 760cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering */ 770cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstruct ehca_dir_bmap { 780cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u64 ent[EHCA_MAP_ENTRIES]; 790cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering}; 800cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstruct ehca_top_bmap { 810cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_dir_bmap *dir[EHCA_MAP_ENTRIES]; 820cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering}; 830cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstruct ehca_bmap { 840cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_top_bmap *top[EHCA_MAP_ENTRIES]; 850cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering}; 860cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 870cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic struct ehca_bmap *ehca_bmap; 880cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 89fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickstatic struct kmem_cache *mr_cache; 90fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickstatic struct kmem_cache *mw_cache; 91fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 925bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyenenum ehca_mr_pgsize { 935bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen EHCA_MR_PGSIZE4K = 0x1000L, 945bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen EHCA_MR_PGSIZE64K = 0x10000L, 955bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen EHCA_MR_PGSIZE1M = 0x100000L, 965bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen EHCA_MR_PGSIZE16M = 0x1000000L 975bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen}; 985bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen 993a31c41901b6bd3937ec36e0e4a930849e270df6Joachim Fenkes#define EHCA_MR_PGSHIFT4K 12 1003a31c41901b6bd3937ec36e0e4a930849e270df6Joachim Fenkes#define EHCA_MR_PGSHIFT64K 16 1013a31c41901b6bd3937ec36e0e4a930849e270df6Joachim Fenkes#define EHCA_MR_PGSHIFT1M 20 1023a31c41901b6bd3937ec36e0e4a930849e270df6Joachim Fenkes#define EHCA_MR_PGSHIFT16M 24 1033a31c41901b6bd3937ec36e0e4a930849e270df6Joachim Fenkes 1040cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic u64 ehca_map_vaddr(void *caddr); 1050cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 1065bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyenstatic u32 ehca_encode_hwpage_size(u32 pgsize) 1075bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen{ 1088c08d50d4fc52a9367c356ebbeb194c30fbc7ac8Joachim Fenkes int log = ilog2(pgsize); 1098c08d50d4fc52a9367c356ebbeb194c30fbc7ac8Joachim Fenkes WARN_ON(log < 12 || log > 24 || log & 3); 1108c08d50d4fc52a9367c356ebbeb194c30fbc7ac8Joachim Fenkes return (log - 12) / 4; 1115bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen} 1125bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen 1135bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyenstatic u64 ehca_get_max_hwpage_size(struct ehca_shca *shca) 1145bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen{ 115bd50f8924c684f84416fb58c11eb24619b041f25Kyle McMartin return rounddown_pow_of_two(shca->hca_cap_mr_pgsize); 1165bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen} 1175bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen 118fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickstatic struct ehca_mr *ehca_mr_new(void) 119fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 120fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *me; 121fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 122c376222960ae91d5ffb9197ee36771aaed1d9f90Robert P. J. Day me = kmem_cache_zalloc(mr_cache, GFP_KERNEL); 1232b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen if (me) 124fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick spin_lock_init(&me->mrlock); 1252b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen else 126fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_gen_err("alloc failed"); 127fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 128fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return me; 129fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 130fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 131fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickstatic void ehca_mr_delete(struct ehca_mr *me) 132fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 133fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick kmem_cache_free(mr_cache, me); 134fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 135fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 136fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickstatic struct ehca_mw *ehca_mw_new(void) 137fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 138fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mw *me; 139fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 140c376222960ae91d5ffb9197ee36771aaed1d9f90Robert P. J. Day me = kmem_cache_zalloc(mw_cache, GFP_KERNEL); 1412b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen if (me) 142fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick spin_lock_init(&me->mwlock); 1432b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen else 144fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_gen_err("alloc failed"); 145fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 146fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return me; 147fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 148fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 149fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickstatic void ehca_mw_delete(struct ehca_mw *me) 150fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 151fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick kmem_cache_free(mw_cache, me); 152fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 153fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 154fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 155fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 156fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickstruct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags) 157fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 158fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_mr *ib_mr; 159fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret; 160fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_maxmr; 161fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd); 162fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 163fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(pd->device, struct ehca_shca, ib_device); 164fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 165fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (shca->maxmr) { 166fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_maxmr = ehca_mr_new(); 167fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!e_maxmr) { 168fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "out of memory"); 169fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(-ENOMEM); 170fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto get_dma_mr_exit0; 171fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 172fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1730cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ret = ehca_reg_maxmr(shca, e_maxmr, 174625fbd3a36d836efaaee4b6d9c2fcd25e3654624Sonny Rao (void *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START)), 175fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr_access_flags, e_pd, 176fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick &e_maxmr->ib.ib_mr.lkey, 177fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick &e_maxmr->ib.ib_mr.rkey); 178fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 1793df78f81e070c0e3330ae1bd40385e2f0d6fea2cHoang-Nam Nguyen ehca_mr_delete(e_maxmr); 180fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(ret); 181fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto get_dma_mr_exit0; 182fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 183fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = &e_maxmr->ib.ib_mr; 184fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } else { 185fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "no internal max-MR exist!"); 186fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(-EINVAL); 187fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto get_dma_mr_exit0; 188fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 189fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 190fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickget_dma_mr_exit0: 191fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (IS_ERR(ib_mr)) 192e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(&shca->ib_device, "h_ret=%li pd=%p mr_access_flags=%x", 193fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick PTR_ERR(ib_mr), pd, mr_access_flags); 194fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ib_mr; 195fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_get_dma_mr() */ 196fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 197fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 198fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 199fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickstruct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd, 200fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_phys_buf *phys_buf_array, 201fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int num_phys_buf, 202fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int mr_access_flags, 203fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *iova_start) 204fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 205fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_mr *ib_mr; 206fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret; 207fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_mr; 208fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 209fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(pd->device, struct ehca_shca, ib_device); 210fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd); 211fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 212fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 size; 213fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 214fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((num_phys_buf <= 0) || !phys_buf_array) { 215fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(pd->device, "bad input values: num_phys_buf=%x " 216fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "phys_buf_array=%p", num_phys_buf, phys_buf_array); 217fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(-EINVAL); 218fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto reg_phys_mr_exit0; 219fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 220fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) && 221fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) || 222fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) && 223fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) { 224fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* 225fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Remote Write Access requires Local Write Access 226fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Remote Atomic Access requires Local Write Access 227fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 228fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(pd->device, "bad input values: mr_access_flags=%x", 229fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr_access_flags); 230fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(-EINVAL); 231fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto reg_phys_mr_exit0; 232fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 233fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 234fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* check physical buffer list and calculate size */ 235fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf, 236fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick iova_start, &size); 237fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 238fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(ret); 239fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto reg_phys_mr_exit0; 240fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 241fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((size == 0) || 242fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick (((u64)iova_start + size) < (u64)iova_start)) { 2433750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(pd->device, "bad input values: size=%llx iova_start=%p", 244fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick size, iova_start); 245fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(-EINVAL); 246fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto reg_phys_mr_exit0; 247fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 248fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 249fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr = ehca_mr_new(); 250fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!e_mr) { 251fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(pd->device, "out of memory"); 252fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(-ENOMEM); 253fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto reg_phys_mr_exit0; 254fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 255fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 256fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* register MR on HCA */ 257fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ehca_mr_is_maxmr(size, iova_start)) { 258fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->flags |= EHCA_MR_FLAG_MAXMR; 259fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags, 260fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_pd, &e_mr->ib.ib_mr.lkey, 261fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick &e_mr->ib.ib_mr.rkey); 262fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 263fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(ret); 264fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto reg_phys_mr_exit1; 265fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 266fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } else { 267df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen struct ehca_mr_pginfo pginfo; 268df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen u32 num_kpages; 269df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen u32 num_hwpages; 2705bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen u64 hw_pgsize; 271df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen 272df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size, 273df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen PAGE_SIZE); 2745bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen /* for kernel space we try most possible pgsize */ 2755bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen hw_pgsize = ehca_get_max_hwpage_size(shca); 2765bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size, 2775bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen hw_pgsize); 278df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen memset(&pginfo, 0, sizeof(pginfo)); 279df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.type = EHCA_MR_PGI_PHYS; 280df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.num_kpages = num_kpages; 2815bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo.hwpage_size = hw_pgsize; 282df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.num_hwpages = num_hwpages; 283df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.u.phy.num_phys_buf = num_phys_buf; 284df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.u.phy.phys_buf_array = phys_buf_array; 2855bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo.next_hwpage = 2869511724da9c090da5a6dcf6c557b1cedc4f5cfd7Joachim Fenkes ((u64)iova_start & ~PAGE_MASK) / hw_pgsize; 287fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 288fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags, 289fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_pd, &pginfo, &e_mr->ib.ib_mr.lkey, 2900cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering &e_mr->ib.ib_mr.rkey, EHCA_REG_MR); 291fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 292fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(ret); 293fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto reg_phys_mr_exit1; 294fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 295fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 296fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 297fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful registration of all pages */ 298fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return &e_mr->ib.ib_mr; 299fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 300fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickreg_phys_mr_exit1: 301fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mr_delete(e_mr); 302fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickreg_phys_mr_exit0: 303fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (IS_ERR(ib_mr)) 304e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(pd->device, "h_ret=%li pd=%p phys_buf_array=%p " 305fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "num_phys_buf=%x mr_access_flags=%x iova_start=%p", 306fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick PTR_ERR(ib_mr), pd, phys_buf_array, 307fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick num_phys_buf, mr_access_flags, iova_start); 308fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ib_mr; 309fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_reg_phys_mr() */ 310fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 311fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 312fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 3132b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyenstruct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 3142b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen u64 virt, int mr_access_flags, 3152b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen struct ib_udata *udata) 316fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 317fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_mr *ib_mr; 318fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_mr; 319fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 320fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(pd->device, struct ehca_shca, ib_device); 321fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd); 322df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen struct ehca_mr_pginfo pginfo; 323abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes int ret, page_shift; 324df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen u32 num_kpages; 325df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen u32 num_hwpages; 3265bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen u64 hwpage_size; 327fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 328fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!pd) { 329fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_gen_err("bad pd=%p", pd); 330fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ERR_PTR(-EFAULT); 331fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 332f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier 333fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) && 334fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) || 335fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) && 336fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) { 337fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* 338fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Remote Write Access requires Local Write Access 339fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Remote Atomic Access requires Local Write Access 340fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 341fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(pd->device, "bad input values: mr_access_flags=%x", 342fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr_access_flags); 343fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(-EINVAL); 344fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto reg_user_mr_exit0; 345fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 346fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 347f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier if (length == 0 || virt + length < virt) { 3483750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(pd->device, "bad input values: length=%llx " 3493750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "virt_base=%llx", length, virt); 350fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(-EINVAL); 351fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto reg_user_mr_exit0; 352fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 353fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 354fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr = ehca_mr_new(); 355fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!e_mr) { 356fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(pd->device, "out of memory"); 357fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(-ENOMEM); 358fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto reg_user_mr_exit0; 359fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 360fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 361f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier e_mr->umem = ib_umem_get(pd->uobject->context, start, length, 362cb9fbc5c37b69ac584e61d449cfd590f5ae1f90dArthur Kepner mr_access_flags, 0); 363f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier if (IS_ERR(e_mr->umem)) { 3642b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen ib_mr = (void *)e_mr->umem; 365f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier goto reg_user_mr_exit1; 366f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier } 367f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier 368f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier if (e_mr->umem->page_size != PAGE_SIZE) { 369f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier ehca_err(pd->device, "page size not supported, " 370f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier "e_mr->umem->page_size=%x", e_mr->umem->page_size); 371f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier ib_mr = ERR_PTR(-EINVAL); 372f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier goto reg_user_mr_exit2; 373f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier } 374f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier 375fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* determine number of MR pages */ 376df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE); 3775bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen /* select proper hw_pgsize */ 378abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes page_shift = PAGE_SHIFT; 379abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes if (e_mr->umem->hugetlb) { 380abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes /* determine page_shift, clamp between 4K and 16M */ 381abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes page_shift = (fls64(length - 1) + 3) & ~3; 382abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes page_shift = min(max(page_shift, EHCA_MR_PGSHIFT4K), 383abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes EHCA_MR_PGSHIFT16M); 384abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes } 385abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes hwpage_size = 1UL << page_shift; 386abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes 387abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes /* now that we have the desired page size, shift until it's 388abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes * supported, too. 4K is always supported, so this terminates. 389abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes */ 390abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes while (!(hwpage_size & shca->hca_cap_mr_pgsize)) 391abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes hwpage_size >>= 4; 392fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 3935bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyenreg_user_mr_fallback: 3945bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen num_hwpages = NUM_CHUNKS((virt % hwpage_size) + length, hwpage_size); 395fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* register MR on HCA */ 396df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen memset(&pginfo, 0, sizeof(pginfo)); 397df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.type = EHCA_MR_PGI_USER; 3985bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo.hwpage_size = hwpage_size; 399df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.num_kpages = num_kpages; 400df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.num_hwpages = num_hwpages; 401df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.u.usr.region = e_mr->umem; 4025bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo.next_hwpage = e_mr->umem->offset / hwpage_size; 403eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pginfo.u.usr.next_sg = pginfo.u.usr.region->sg_head.sgl; 4042b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags, 4052b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen e_pd, &pginfo, &e_mr->ib.ib_mr.lkey, 4060cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering &e_mr->ib.ib_mr.rkey, EHCA_REG_MR); 4075bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen if (ret == -EINVAL && pginfo.hwpage_size > PAGE_SIZE) { 4085bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ehca_warn(pd->device, "failed to register mr " 4093750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "with hwpage_size=%llx", hwpage_size); 4105bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ehca_info(pd->device, "try to register mr with " 4115bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen "kpage_size=%lx", PAGE_SIZE); 4125bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen /* 4135bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen * this means kpages are not contiguous for a hw page 4145bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen * try kernel page size as fallback solution 4155bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen */ 4165bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen hwpage_size = PAGE_SIZE; 4175bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen goto reg_user_mr_fallback; 4185bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } 419fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 420fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mr = ERR_PTR(ret); 421f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier goto reg_user_mr_exit2; 422fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 423fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 424fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful registration of all pages */ 425fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return &e_mr->ib.ib_mr; 426fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 427f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreierreg_user_mr_exit2: 428f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier ib_umem_release(e_mr->umem); 429fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickreg_user_mr_exit1: 430fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mr_delete(e_mr); 431fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickreg_user_mr_exit0: 432fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (IS_ERR(ib_mr)) 433e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(pd->device, "rc=%li pd=%p mr_access_flags=%x udata=%p", 434f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier PTR_ERR(ib_mr), pd, mr_access_flags, udata); 435fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ib_mr; 436fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_reg_user_mr() */ 437fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 438fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 439fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 440fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_rereg_phys_mr(struct ib_mr *mr, 441fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int mr_rereg_mask, 442fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_pd *pd, 443fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_phys_buf *phys_buf_array, 444fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int num_phys_buf, 445fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int mr_access_flags, 446fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *iova_start) 447fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 448fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret; 449fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 450fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 451fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(mr->device, struct ehca_shca, ib_device); 452fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr); 453fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 new_size; 454fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *new_start; 455fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 new_acl; 456fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *new_pd; 457fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 tmp_lkey, tmp_rkey; 458fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick unsigned long sl_flags; 459df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen u32 num_kpages = 0; 460df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen u32 num_hwpages = 0; 461df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen struct ehca_mr_pginfo pginfo; 462fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 463fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) { 464fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* TODO not supported, because PHYP rereg hCall needs pages */ 465fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not " 466fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "supported yet, mr_rereg_mask=%x", mr_rereg_mask); 467fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 468fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto rereg_phys_mr_exit0; 469fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 470fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 471fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (mr_rereg_mask & IB_MR_REREG_PD) { 472fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!pd) { 473fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(mr->device, "rereg with bad pd, pd=%p " 474fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "mr_rereg_mask=%x", pd, mr_rereg_mask); 475fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 476fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto rereg_phys_mr_exit0; 477fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 478fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 479fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 480fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((mr_rereg_mask & 481fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) || 482fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick (mr_rereg_mask == 0)) { 483fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 484fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto rereg_phys_mr_exit0; 485fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 486fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 487fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* check other parameters */ 488fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (e_mr == shca->maxmr) { 489fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* should be impossible, however reject to be sure */ 490fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p " 491fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "shca->maxmr=%p mr->lkey=%x", 492fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr, shca->maxmr, mr->lkey); 493fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 494fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto rereg_phys_mr_exit0; 495fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 496fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */ 497fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (e_mr->flags & EHCA_MR_FLAG_FMR) { 498fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(mr->device, "not supported for FMR, mr=%p " 499fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "flags=%x", mr, e_mr->flags); 500fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 501fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto rereg_phys_mr_exit0; 502fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 503fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!phys_buf_array || num_phys_buf <= 0) { 5042b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen ehca_err(mr->device, "bad input values mr_rereg_mask=%x" 505fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick " phys_buf_array=%p num_phys_buf=%x", 506fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr_rereg_mask, phys_buf_array, num_phys_buf); 507fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 508fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto rereg_phys_mr_exit0; 509fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 510fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 511fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((mr_rereg_mask & IB_MR_REREG_ACCESS) && /* change ACL */ 512fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) && 513fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) || 514fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) && 515fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) { 516fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* 517fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Remote Write Access requires Local Write Access 518fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Remote Atomic Access requires Local Write Access 519fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 520fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(mr->device, "bad input values: mr_rereg_mask=%x " 521fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "mr_access_flags=%x", mr_rereg_mask, mr_access_flags); 522fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 523fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto rereg_phys_mr_exit0; 524fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 525fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 526fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* set requested values dependent on rereg request */ 527fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick spin_lock_irqsave(&e_mr->mrlock, sl_flags); 5282b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen new_start = e_mr->start; 5292b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen new_size = e_mr->size; 5302b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen new_acl = e_mr->acl; 5312b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen new_pd = container_of(mr->pd, struct ehca_pd, ib_pd); 532fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 533fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (mr_rereg_mask & IB_MR_REREG_TRANS) { 5345bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen u64 hw_pgsize = ehca_get_max_hwpage_size(shca); 5355bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen 536fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick new_start = iova_start; /* change address */ 537fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* check physical buffer list and calculate size */ 538fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, 539fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick num_phys_buf, iova_start, 540fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick &new_size); 541fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 542fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto rereg_phys_mr_exit1; 543fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((new_size == 0) || 544fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick (((u64)iova_start + new_size) < (u64)iova_start)) { 5453750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(mr->device, "bad input values: new_size=%llx " 546fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "iova_start=%p", new_size, iova_start); 547fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 548fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto rereg_phys_mr_exit1; 549fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 550df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) + 551df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen new_size, PAGE_SIZE); 5525bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen num_hwpages = NUM_CHUNKS(((u64)new_start % hw_pgsize) + 5535bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen new_size, hw_pgsize); 554df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen memset(&pginfo, 0, sizeof(pginfo)); 555df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.type = EHCA_MR_PGI_PHYS; 556df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.num_kpages = num_kpages; 5575bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo.hwpage_size = hw_pgsize; 558df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.num_hwpages = num_hwpages; 559df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.u.phy.num_phys_buf = num_phys_buf; 560df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.u.phy.phys_buf_array = phys_buf_array; 5615bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo.next_hwpage = 5629511724da9c090da5a6dcf6c557b1cedc4f5cfd7Joachim Fenkes ((u64)iova_start & ~PAGE_MASK) / hw_pgsize; 563fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 564fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (mr_rereg_mask & IB_MR_REREG_ACCESS) 565fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick new_acl = mr_access_flags; 566fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (mr_rereg_mask & IB_MR_REREG_PD) 567fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick new_pd = container_of(pd, struct ehca_pd, ib_pd); 568fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 569fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl, 570fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick new_pd, &pginfo, &tmp_lkey, &tmp_rkey); 571fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 572fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto rereg_phys_mr_exit1; 573fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 574fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful reregistration */ 575fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (mr_rereg_mask & IB_MR_REREG_PD) 576fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr->pd = pd; 577fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr->lkey = tmp_lkey; 578fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr->rkey = tmp_rkey; 579fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 580fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickrereg_phys_mr_exit1: 581fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick spin_unlock_irqrestore(&e_mr->mrlock, sl_flags); 582fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickrereg_phys_mr_exit0: 583fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 584e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(mr->device, "ret=%i mr=%p mr_rereg_mask=%x pd=%p " 585fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x " 586fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "iova_start=%p", 587fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret, mr, mr_rereg_mask, pd, phys_buf_array, 588fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick num_phys_buf, mr_access_flags, iova_start); 589fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 590fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_rereg_phys_mr() */ 591fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 592fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 593fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 594fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr) 595fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 596fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret = 0; 597fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 598fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 599fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(mr->device, struct ehca_shca, ib_device); 600fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr); 601fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick unsigned long sl_flags; 6022b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen struct ehca_mr_hipzout_parms hipzout; 603fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 604fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((e_mr->flags & EHCA_MR_FLAG_FMR)) { 605fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p " 606fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "e_mr->flags=%x", mr, e_mr, e_mr->flags); 607fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 608fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto query_mr_exit0; 609fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 610fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 611fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick memset(mr_attr, 0, sizeof(struct ib_mr_attr)); 612fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick spin_lock_irqsave(&e_mr->mrlock, sl_flags); 613fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 614fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout); 615fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 6163750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lli mr=%p " 6173750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "hca_hndl=%llx mr_hndl=%llx lkey=%x", 618fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret, mr, shca->ipz_hca_handle.handle, 619fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ipz_mr_handle.handle, mr->lkey); 620a1a6ff11006c3a056cda9e8b13e7388fba3e69a1Hoang-Nam Nguyen ret = ehca2ib_return_code(h_ret); 621fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto query_mr_exit1; 622fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 623df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen mr_attr->pd = mr->pd; 624fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr_attr->device_virt_addr = hipzout.vaddr; 625df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen mr_attr->size = hipzout.len; 626df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen mr_attr->lkey = hipzout.lkey; 627df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen mr_attr->rkey = hipzout.rkey; 628fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags); 629fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 630fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickquery_mr_exit1: 631fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick spin_unlock_irqrestore(&e_mr->mrlock, sl_flags); 632fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickquery_mr_exit0: 633fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 634e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(mr->device, "ret=%i mr=%p mr_attr=%p", 635fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret, mr, mr_attr); 636fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 637fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_query_mr() */ 638fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 639fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 640fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 641fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_dereg_mr(struct ib_mr *mr) 642fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 643fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret = 0; 644fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 645fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 646fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(mr->device, struct ehca_shca, ib_device); 647fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr); 648fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 649fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((e_mr->flags & EHCA_MR_FLAG_FMR)) { 650fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p " 651fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "e_mr->flags=%x", mr, e_mr, e_mr->flags); 652fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 653fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto dereg_mr_exit0; 654fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } else if (e_mr == shca->maxmr) { 655fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* should be impossible, however reject to be sure */ 656fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(mr->device, "dereg internal max-MR impossible, mr=%p " 657fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "shca->maxmr=%p mr->lkey=%x", 658fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr, shca->maxmr, mr->lkey); 659fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 660fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto dereg_mr_exit0; 661fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 662fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 663fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* TODO: BUSY: MR still has bound window(s) */ 664fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr); 665fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 6663750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(mr->device, "hipz_free_mr failed, h_ret=%lli shca=%p " 6673750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "e_mr=%p hca_hndl=%llx mr_hndl=%llx mr->lkey=%x", 668fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret, shca, e_mr, shca->ipz_hca_handle.handle, 669fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ipz_mr_handle.handle, mr->lkey); 670a1a6ff11006c3a056cda9e8b13e7388fba3e69a1Hoang-Nam Nguyen ret = ehca2ib_return_code(h_ret); 671fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto dereg_mr_exit0; 672fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 673fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 674f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier if (e_mr->umem) 675f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier ib_umem_release(e_mr->umem); 676f7c6a7b5d59980b076abbf2ceeb8735591290285Roland Dreier 677fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful deregistration */ 678fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mr_delete(e_mr); 679fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 680fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickdereg_mr_exit0: 681fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 682e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(mr->device, "ret=%i mr=%p", ret, mr); 683fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 684fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_dereg_mr() */ 685fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 686fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 687fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 6887083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaelistruct ib_mw *ehca_alloc_mw(struct ib_pd *pd, enum ib_mw_type type) 689fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 690fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_mw *ib_mw; 691fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 692fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mw *e_mw; 693fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd); 694fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 695fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(pd->device, struct ehca_shca, ib_device); 6962b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen struct ehca_mw_hipzout_parms hipzout; 697fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 6987083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaeli if (type != IB_MW_TYPE_1) 6997083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaeli return ERR_PTR(-EINVAL); 7007083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaeli 701fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mw = ehca_mw_new(); 702fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!e_mw) { 703fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_mw = ERR_PTR(-ENOMEM); 704fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto alloc_mw_exit0; 705fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 706fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 707fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw, 708fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_pd->fw_pd, &hipzout); 709fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 7103750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(pd->device, "hipz_mw_allocate failed, h_ret=%lli " 7113750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "shca=%p hca_hndl=%llx mw=%p", 712fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret, shca, shca->ipz_hca_handle.handle, e_mw); 713a1a6ff11006c3a056cda9e8b13e7388fba3e69a1Hoang-Nam Nguyen ib_mw = ERR_PTR(ehca2ib_return_code(h_ret)); 714fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto alloc_mw_exit1; 715fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 716fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful MW allocation */ 717fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mw->ipz_mw_handle = hipzout.handle; 718fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mw->ib_mw.rkey = hipzout.rkey; 719fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return &e_mw->ib_mw; 720fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 721fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickalloc_mw_exit1: 722fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mw_delete(e_mw); 723fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickalloc_mw_exit0: 724fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (IS_ERR(ib_mw)) 725e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(pd->device, "h_ret=%li pd=%p", PTR_ERR(ib_mw), pd); 726fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ib_mw; 727fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_alloc_mw() */ 728fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 729fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 730fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 731fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_bind_mw(struct ib_qp *qp, 732fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_mw *mw, 733fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_mw_bind *mw_bind) 734fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 735fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* TODO: not supported up to now */ 736fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_gen_err("bind MW currently not supported by HCAD"); 737fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 738fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -EPERM; 739fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_bind_mw() */ 740fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 741fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 742fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 743fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_dealloc_mw(struct ib_mw *mw) 744fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 745fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 746fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 747fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(mw->device, struct ehca_shca, ib_device); 748fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mw *e_mw = container_of(mw, struct ehca_mw, ib_mw); 749fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 750fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw); 751fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 7523750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(mw->device, "hipz_free_mw failed, h_ret=%lli shca=%p " 7533750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "mw=%p rkey=%x hca_hndl=%llx mw_hndl=%llx", 754fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle, 755fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mw->ipz_mw_handle.handle); 756a1a6ff11006c3a056cda9e8b13e7388fba3e69a1Hoang-Nam Nguyen return ehca2ib_return_code(h_ret); 757fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 758fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful deallocation */ 759fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mw_delete(e_mw); 760fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 761fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_dealloc_mw() */ 762fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 763fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 764fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 765fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickstruct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd, 766fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int mr_access_flags, 767fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_fmr_attr *fmr_attr) 768fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 769fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_fmr *ib_fmr; 770fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 771fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(pd->device, struct ehca_shca, ib_device); 772fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd); 773fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_fmr; 774fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret; 775fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 tmp_lkey, tmp_rkey; 776df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen struct ehca_mr_pginfo pginfo; 7775bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen u64 hw_pgsize; 778fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 779fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* check other parameters */ 780fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) && 781fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) || 782fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) && 783fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) { 784fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* 785fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Remote Write Access requires Local Write Access 786fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Remote Atomic Access requires Local Write Access 787fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 788fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(pd->device, "bad input values: mr_access_flags=%x", 789fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr_access_flags); 790fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_fmr = ERR_PTR(-EINVAL); 791fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto alloc_fmr_exit0; 792fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 793fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (mr_access_flags & IB_ACCESS_MW_BIND) { 794fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(pd->device, "bad input values: mr_access_flags=%x", 795fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr_access_flags); 796fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_fmr = ERR_PTR(-EINVAL); 797fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto alloc_fmr_exit0; 798fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 799fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((fmr_attr->max_pages == 0) || (fmr_attr->max_maps == 0)) { 800fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(pd->device, "bad input values: fmr_attr->max_pages=%x " 801fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "fmr_attr->max_maps=%x fmr_attr->page_shift=%x", 802fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick fmr_attr->max_pages, fmr_attr->max_maps, 803fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick fmr_attr->page_shift); 804fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_fmr = ERR_PTR(-EINVAL); 805fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto alloc_fmr_exit0; 806fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 807abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes 808abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes hw_pgsize = 1 << fmr_attr->page_shift; 809abc39d3672d8af4bf6c943faf85fa8877caccf7eJoachim Fenkes if (!(hw_pgsize & shca->hca_cap_mr_pgsize)) { 810fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x", 811fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick fmr_attr->page_shift); 812fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_fmr = ERR_PTR(-EINVAL); 813fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto alloc_fmr_exit0; 814fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 815fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 816fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr = ehca_mr_new(); 817fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!e_fmr) { 818fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_fmr = ERR_PTR(-ENOMEM); 819fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto alloc_fmr_exit0; 820fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 821fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->flags |= EHCA_MR_FLAG_FMR; 822fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 823fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* register MR on HCA */ 824df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen memset(&pginfo, 0, sizeof(pginfo)); 8258c08d50d4fc52a9367c356ebbeb194c30fbc7ac8Joachim Fenkes pginfo.hwpage_size = hw_pgsize; 8265bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen /* 8275bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen * pginfo.num_hwpages==0, ie register_rpages() will not be called 8285bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen * but deferred to map_phys_fmr() 8295bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen */ 830fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_reg_mr(shca, e_fmr, NULL, 831fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick fmr_attr->max_pages * (1 << fmr_attr->page_shift), 832fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr_access_flags, e_pd, &pginfo, 8330cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering &tmp_lkey, &tmp_rkey, EHCA_REG_MR); 834fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 835fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_fmr = ERR_PTR(ret); 836fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto alloc_fmr_exit1; 837fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 838fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 839fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful */ 8405bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen e_fmr->hwpage_size = hw_pgsize; 841fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->fmr_page_size = 1 << fmr_attr->page_shift; 842fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->fmr_max_pages = fmr_attr->max_pages; 843fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->fmr_max_maps = fmr_attr->max_maps; 844fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->fmr_map_cnt = 0; 845fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return &e_fmr->ib.ib_fmr; 846fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 847fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickalloc_fmr_exit1: 848fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mr_delete(e_fmr); 849fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickalloc_fmr_exit0: 850fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ib_fmr; 851fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_alloc_fmr() */ 852fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 853fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 854fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 855fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_map_phys_fmr(struct ib_fmr *fmr, 856fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *page_list, 857fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int list_len, 858fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 iova) 859fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 860fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret; 861fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 862fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(fmr->device, struct ehca_shca, ib_device); 863fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr); 864fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd = container_of(fmr->pd, struct ehca_pd, ib_pd); 865df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen struct ehca_mr_pginfo pginfo; 866fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 tmp_lkey, tmp_rkey; 867fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 868fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) { 869fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x", 870fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr, e_fmr->flags); 871fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 872fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto map_phys_fmr_exit0; 873fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 874fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_fmr_check_page_list(e_fmr, page_list, list_len); 875fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 876fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto map_phys_fmr_exit0; 877fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (iova % e_fmr->fmr_page_size) { 878fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* only whole-numbered pages */ 8793750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(fmr->device, "bad iova, iova=%llx fmr_page_size=%x", 880fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick iova, e_fmr->fmr_page_size); 881fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 882fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto map_phys_fmr_exit0; 883fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 884fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (e_fmr->fmr_map_cnt >= e_fmr->fmr_max_maps) { 885fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* HCAD does not limit the maps, however trace this anyway */ 886fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_info(fmr->device, "map limit exceeded, fmr=%p " 887fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x", 888fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick fmr, e_fmr->fmr_map_cnt, e_fmr->fmr_max_maps); 889fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 890fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 891df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen memset(&pginfo, 0, sizeof(pginfo)); 892df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.type = EHCA_MR_PGI_FMR; 893df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.num_kpages = list_len; 8945bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo.hwpage_size = e_fmr->hwpage_size; 8955bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo.num_hwpages = 8965bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen list_len * e_fmr->fmr_page_size / pginfo.hwpage_size; 897df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.u.fmr.page_list = page_list; 8985bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo.next_hwpage = 8995bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen (iova & (e_fmr->fmr_page_size-1)) / pginfo.hwpage_size; 900187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size; 901fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 9022b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova, 903fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick list_len * e_fmr->fmr_page_size, 904fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey); 905fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 906fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto map_phys_fmr_exit0; 907fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 908fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful reregistration */ 909fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->fmr_map_cnt++; 910fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->ib.ib_fmr.lkey = tmp_lkey; 911fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->ib.ib_fmr.rkey = tmp_rkey; 912fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 913fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 914fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickmap_phys_fmr_exit0: 915fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 916e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(fmr->device, "ret=%i fmr=%p page_list=%p list_len=%x " 9173750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "iova=%llx", ret, fmr, page_list, list_len, iova); 918fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 919fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_map_phys_fmr() */ 920fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 921fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 922fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 923fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_unmap_fmr(struct list_head *fmr_list) 924fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 925fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret = 0; 926fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_fmr *ib_fmr; 927fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = NULL; 928fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *prev_shca; 929fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_fmr; 930fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 num_fmr = 0; 931fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 unmap_fmr_cnt = 0; 932fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 933fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* check all FMR belong to same SHCA, and check internal flag */ 934fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick list_for_each_entry(ib_fmr, fmr_list, list) { 935fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick prev_shca = shca; 936fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick shca = container_of(ib_fmr->device, struct ehca_shca, 937fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_device); 938fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr); 939fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((shca != prev_shca) && prev_shca) { 940fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p " 941fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "prev_shca=%p e_fmr=%p", 942fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick shca, prev_shca, e_fmr); 943fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 944fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto unmap_fmr_exit0; 945fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 946fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) { 947fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p " 948fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "e_fmr->flags=%x", e_fmr, e_fmr->flags); 949fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 950fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto unmap_fmr_exit0; 951fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 952fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick num_fmr++; 953fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 954fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 955fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* loop over all FMRs to unmap */ 956fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick list_for_each_entry(ib_fmr, fmr_list, list) { 957fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick unmap_fmr_cnt++; 958fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr); 959fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick shca = container_of(ib_fmr->device, struct ehca_shca, 960fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_device); 961fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_unmap_one_fmr(shca, e_fmr); 962fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 963fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* unmap failed, stop unmapping of rest of FMRs */ 964fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "unmap of one FMR failed, " 965fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "stop rest, e_fmr=%p num_fmr=%x " 966fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "unmap_fmr_cnt=%x lkey=%x", e_fmr, num_fmr, 967fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick unmap_fmr_cnt, e_fmr->ib.ib_fmr.lkey); 968fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto unmap_fmr_exit0; 969fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 970fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 971fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 972fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickunmap_fmr_exit0: 973fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 974e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_gen_err("ret=%i fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x", 975fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret, fmr_list, num_fmr, unmap_fmr_cnt); 976fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 977fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_unmap_fmr() */ 978fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 979fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 980fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 981fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_dealloc_fmr(struct ib_fmr *fmr) 982fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 983fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret; 984fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 985fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 986fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(fmr->device, struct ehca_shca, ib_device); 987fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr); 988fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 989fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) { 990fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x", 991fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr, e_fmr->flags); 992fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 993fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto free_fmr_exit0; 994fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 995fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 996fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr); 997fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 9983750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(fmr->device, "hipz_free_mr failed, h_ret=%lli e_fmr=%p " 9993750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "hca_hndl=%llx fmr_hndl=%llx fmr->lkey=%x", 1000fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret, e_fmr, shca->ipz_hca_handle.handle, 1001fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->ipz_mr_handle.handle, fmr->lkey); 1002a1a6ff11006c3a056cda9e8b13e7388fba3e69a1Hoang-Nam Nguyen ret = ehca2ib_return_code(h_ret); 1003fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto free_fmr_exit0; 1004fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1005fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful deregistration */ 1006fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mr_delete(e_fmr); 1007fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 1008fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1009fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickfree_fmr_exit0: 1010fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 1011e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(&shca->ib_device, "ret=%i fmr=%p", ret, fmr); 1012fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 1013fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_dealloc_fmr() */ 1014fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1015fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 1016fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 10170cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca, 10180cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_mr *e_mr, 10190cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_mr_pginfo *pginfo); 10200cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 1021fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_reg_mr(struct ehca_shca *shca, 1022fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_mr, 1023fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *iova_start, 1024fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 size, 1025fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int acl, 1026fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd, 1027fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr_pginfo *pginfo, 1028fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 *lkey, /*OUT*/ 10290cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u32 *rkey, /*OUT*/ 10300cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering enum ehca_reg_type reg_type) 1031fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1032fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret; 1033fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 1034fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 hipz_acl; 10352b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen struct ehca_mr_hipzout_parms hipzout; 1036fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1037fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mrmw_map_acl(acl, &hipz_acl); 10385bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl); 1039fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ehca_use_hp_mr == 1) 10402b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen hipz_acl |= 0x00000001; 1041fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1042fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr, 1043fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick (u64)iova_start, size, hipz_acl, 1044fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_pd->fw_pd, &hipzout); 1045fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 10463750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lli " 10473750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "hca_hndl=%llx", h_ret, shca->ipz_hca_handle.handle); 1048a1a6ff11006c3a056cda9e8b13e7388fba3e69a1Hoang-Nam Nguyen ret = ehca2ib_return_code(h_ret); 1049fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_reg_mr_exit0; 1050fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1051fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1052fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ipz_mr_handle = hipzout.handle; 1053fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 10540cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (reg_type == EHCA_REG_BUSMAP_MR) 10550cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ret = ehca_reg_bmap_mr_rpages(shca, e_mr, pginfo); 10560cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering else if (reg_type == EHCA_REG_MR) 10570cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ret = ehca_reg_mr_rpages(shca, e_mr, pginfo); 10580cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering else 10590cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ret = -EINVAL; 10600cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 1061fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 1062fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_reg_mr_exit1; 1063fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1064fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful registration */ 1065df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_mr->num_kpages = pginfo->num_kpages; 1066df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_mr->num_hwpages = pginfo->num_hwpages; 10675bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen e_mr->hwpage_size = pginfo->hwpage_size; 1068df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_mr->start = iova_start; 1069df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_mr->size = size; 1070df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_mr->acl = acl; 1071fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *lkey = hipzout.lkey; 1072fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *rkey = hipzout.rkey; 1073fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 1074fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1075fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_reg_mr_exit1: 1076fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr); 1077fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 10783750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(&shca->ib_device, "h_ret=%lli shca=%p e_mr=%p " 10793750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "iova_start=%p size=%llx acl=%x e_pd=%p lkey=%x " 10803750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "pginfo=%p num_kpages=%llx num_hwpages=%llx ret=%i", 1081fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret, shca, e_mr, iova_start, size, acl, e_pd, 1082df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen hipzout.lkey, pginfo, pginfo->num_kpages, 1083df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo->num_hwpages, ret); 1084fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, " 1085fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "not recoverable"); 1086fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1087fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_reg_mr_exit0: 1088fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 1089e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p " 10903750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "iova_start=%p size=%llx acl=%x e_pd=%p pginfo=%p " 10913750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "num_kpages=%llx num_hwpages=%llx", 1092fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo, 1093df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo->num_kpages, pginfo->num_hwpages); 1094fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 1095fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_reg_mr() */ 1096fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1097fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 1098fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1099fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_reg_mr_rpages(struct ehca_shca *shca, 1100fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_mr, 1101fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr_pginfo *pginfo) 1102fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1103fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret = 0; 1104fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 1105fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 rnum; 1106fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 rpage; 1107fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 i; 1108fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *kpage; 1109fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 11105bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen if (!pginfo->num_hwpages) /* in case of fmr */ 11115bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen return 0; 11125bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen 1113f2d9136133de257abbd97fec6f624d3a73d1e1fdHoang-Nam Nguyen kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL); 1114fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!kpage) { 1115fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "kpage alloc failed"); 1116fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -ENOMEM; 1117fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_reg_mr_rpages_exit0; 1118fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1119fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 11205bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen /* max MAX_RPAGES ehca mr pages per register call */ 1121df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) { 1122fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1123df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) { 1124df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen rnum = pginfo->num_hwpages % MAX_RPAGES; /* last shot */ 1125fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (rnum == 0) 11264e4e74cae73325c9f8513fae3a5bd9f79458f4a7Hoang-Nam Nguyen rnum = MAX_RPAGES; /* last shot is full */ 1127fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } else 11284e4e74cae73325c9f8513fae3a5bd9f79458f4a7Hoang-Nam Nguyen rnum = MAX_RPAGES; 1129fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1130187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ret = ehca_set_pagebuf(pginfo, rnum, kpage); 1131187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen if (ret) { 1132187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ehca_err(&shca->ib_device, "ehca_set_pagebuf " 1133e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes "bad rc, ret=%i rnum=%x kpage=%p", 11345bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ret, rnum, kpage); 1135187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen goto ehca_reg_mr_rpages_exit1; 1136187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen } 1137187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen 1138187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen if (rnum > 1) { 1139b4b8d1e48e05313b163a36946be1a82e5bc5f9adMichael Ellerman rpage = __pa(kpage); 1140fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!rpage) { 1141fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "kpage=%p i=%x", 1142fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick kpage, i); 1143fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EFAULT; 1144fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_reg_mr_rpages_exit1; 1145fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1146187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen } else 1147187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen rpage = *kpage; 1148fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 11495bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen h_ret = hipz_h_register_rpage_mr( 11505bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen shca->ipz_hca_handle, e_mr, 11515bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ehca_encode_hwpage_size(pginfo->hwpage_size), 11525bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen 0, rpage, rnum); 1153fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1154df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) { 1155fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* 1156fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * check for 'registration complete'==H_SUCCESS 1157fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * and for 'page registered'==H_PAGE_REGISTERED 1158fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 1159fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 1160fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "last " 11613750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "hipz_reg_rpage_mr failed, h_ret=%lli " 11623750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "e_mr=%p i=%x hca_hndl=%llx mr_hndl=%llx" 1163fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick " lkey=%x", h_ret, e_mr, i, 1164fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick shca->ipz_hca_handle.handle, 1165fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ipz_mr_handle.handle, 1166fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ib.ib_mr.lkey); 1167a1a6ff11006c3a056cda9e8b13e7388fba3e69a1Hoang-Nam Nguyen ret = ehca2ib_return_code(h_ret); 1168fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick break; 1169fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } else 1170fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = 0; 1171fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } else if (h_ret != H_PAGE_REGISTERED) { 1172fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, " 11733750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "h_ret=%lli e_mr=%p i=%x lkey=%x hca_hndl=%llx " 11743750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "mr_hndl=%llx", h_ret, e_mr, i, 1175fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ib.ib_mr.lkey, 1176fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick shca->ipz_hca_handle.handle, 1177fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ipz_mr_handle.handle); 1178a1a6ff11006c3a056cda9e8b13e7388fba3e69a1Hoang-Nam Nguyen ret = ehca2ib_return_code(h_ret); 1179fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick break; 1180fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } else 1181fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = 0; 1182fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } /* end for(i) */ 1183fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1184fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1185fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_reg_mr_rpages_exit1: 11867e28db5d8ff63b1cabc221c5cb84a5f45752f1c2Hoang-Nam Nguyen ehca_free_fw_ctrlblock(kpage); 1187fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_reg_mr_rpages_exit0: 1188fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 1189e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p pginfo=%p " 11903750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "num_kpages=%llx num_hwpages=%llx", ret, shca, e_mr, 1191df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo, pginfo->num_kpages, pginfo->num_hwpages); 1192fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 1193fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_reg_mr_rpages() */ 1194fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1195fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 1196fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1197fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickinline int ehca_rereg_mr_rereg1(struct ehca_shca *shca, 1198fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_mr, 1199fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *iova_start, 1200fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 size, 1201fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 acl, 1202fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd, 1203fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr_pginfo *pginfo, 1204fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 *lkey, /*OUT*/ 1205fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 *rkey) /*OUT*/ 1206fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1207fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret; 1208fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 1209fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 hipz_acl; 1210fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *kpage; 1211fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 rpage; 1212fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr_pginfo pginfo_save; 12132b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen struct ehca_mr_hipzout_parms hipzout; 1214fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1215fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mrmw_map_acl(acl, &hipz_acl); 12165bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl); 1217fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1218f2d9136133de257abbd97fec6f624d3a73d1e1fdHoang-Nam Nguyen kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL); 1219fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!kpage) { 1220fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "kpage alloc failed"); 1221fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -ENOMEM; 1222fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_rereg_mr_rereg1_exit0; 1223fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1224fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1225fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick pginfo_save = *pginfo; 1226187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ret = ehca_set_pagebuf(pginfo, pginfo->num_hwpages, kpage); 1227fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 1228fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p " 12293750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "pginfo=%p type=%x num_kpages=%llx num_hwpages=%llx " 1230df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen "kpage=%p", e_mr, pginfo, pginfo->type, 1231df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo->num_kpages, pginfo->num_hwpages, kpage); 1232fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_rereg_mr_rereg1_exit1; 1233fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1234b4b8d1e48e05313b163a36946be1a82e5bc5f9adMichael Ellerman rpage = __pa(kpage); 1235fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!rpage) { 1236fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "kpage=%p", kpage); 1237fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EFAULT; 1238fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_rereg_mr_rereg1_exit1; 1239fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1240fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr, 1241fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick (u64)iova_start, size, hipz_acl, 1242fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_pd->fw_pd, rpage, &hipzout); 1243fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 1244fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* 1245fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * reregistration unsuccessful, try it again with the 3 hCalls, 1246fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * e.g. this is required in case H_MR_CONDITION 1247fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * (MW bound or MR is shared) 1248fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 1249fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed " 12503750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "(Rereg1), h_ret=%lli e_mr=%p", h_ret, e_mr); 1251fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *pginfo = pginfo_save; 1252fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EAGAIN; 12532b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen } else if ((u64 *)hipzout.vaddr != iova_start) { 1254fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "PHYP changed iova_start in " 12553750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "rereg_pmr, iova_start=%p iova_start_out=%llx e_mr=%p " 12563750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "mr_handle=%llx lkey=%x lkey_out=%x", iova_start, 1257fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick hipzout.vaddr, e_mr, e_mr->ipz_mr_handle.handle, 1258fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ib.ib_mr.lkey, hipzout.lkey); 1259fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EFAULT; 1260fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } else { 1261fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* 1262fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * successful reregistration 1263fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * note: start and start_out are identical for eServer HCAs 1264fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 1265df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_mr->num_kpages = pginfo->num_kpages; 1266df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_mr->num_hwpages = pginfo->num_hwpages; 12675bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen e_mr->hwpage_size = pginfo->hwpage_size; 1268df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_mr->start = iova_start; 1269df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_mr->size = size; 1270df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_mr->acl = acl; 1271fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *lkey = hipzout.lkey; 1272fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *rkey = hipzout.rkey; 1273fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1274fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1275fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_rereg_mr_rereg1_exit1: 12767e28db5d8ff63b1cabc221c5cb84a5f45752f1c2Hoang-Nam Nguyen ehca_free_fw_ctrlblock(kpage); 1277fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_rereg_mr_rereg1_exit0: 1278fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ( ret && (ret != -EAGAIN) ) 1279e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(&shca->ib_device, "ret=%i lkey=%x rkey=%x " 12803750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "pginfo=%p num_kpages=%llx num_hwpages=%llx", 1281df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen ret, *lkey, *rkey, pginfo, pginfo->num_kpages, 1282df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo->num_hwpages); 1283fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 1284fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_rereg_mr_rereg1() */ 1285fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1286fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 1287fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1288fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_rereg_mr(struct ehca_shca *shca, 1289fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_mr, 1290fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *iova_start, 1291fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 size, 1292fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int acl, 1293fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd, 1294fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr_pginfo *pginfo, 1295fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 *lkey, 1296fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 *rkey) 1297fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1298fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret = 0; 1299fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 1300fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int rereg_1_hcall = 1; /* 1: use hipz_h_reregister_pmr directly */ 1301fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int rereg_3_hcall = 0; /* 1: use 3 hipz calls for reregistration */ 1302fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1303fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* first determine reregistration hCall(s) */ 1304df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen if ((pginfo->num_hwpages > MAX_RPAGES) || 1305df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen (e_mr->num_hwpages > MAX_RPAGES) || 1306df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen (pginfo->num_hwpages > e_mr->num_hwpages)) { 1307df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen ehca_dbg(&shca->ib_device, "Rereg3 case, " 13083750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "pginfo->num_hwpages=%llx e_mr->num_hwpages=%x", 1309df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo->num_hwpages, e_mr->num_hwpages); 1310fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick rereg_1_hcall = 0; 1311fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick rereg_3_hcall = 1; 1312fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1313fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1314fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (e_mr->flags & EHCA_MR_FLAG_MAXMR) { /* check for max-MR */ 1315fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick rereg_1_hcall = 0; 1316fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick rereg_3_hcall = 1; 1317fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->flags &= ~EHCA_MR_FLAG_MAXMR; 1318fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p", 1319fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr); 1320fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1321fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1322fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (rereg_1_hcall) { 1323fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size, 1324fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick acl, e_pd, pginfo, lkey, rkey); 1325fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 1326fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret == -EAGAIN) 1327fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick rereg_3_hcall = 1; 1328fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick else 1329fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_rereg_mr_exit0; 1330fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1331fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1332fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1333fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (rereg_3_hcall) { 1334fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr save_mr; 1335fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1336fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* first deregister old MR */ 1337fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr); 1338fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 1339fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "hipz_free_mr failed, " 13403750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "h_ret=%lli e_mr=%p hca_hndl=%llx mr_hndl=%llx " 1341fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "mr->lkey=%x", 1342fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret, e_mr, shca->ipz_hca_handle.handle, 1343fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ipz_mr_handle.handle, 1344fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ib.ib_mr.lkey); 1345a1a6ff11006c3a056cda9e8b13e7388fba3e69a1Hoang-Nam Nguyen ret = ehca2ib_return_code(h_ret); 1346fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_rereg_mr_exit0; 1347fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1348fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* clean ehca_mr_t, without changing struct ib_mr and lock */ 1349fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick save_mr = *e_mr; 1350fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mr_deletenew(e_mr); 1351fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1352fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* set some MR values */ 1353fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->flags = save_mr.flags; 13545bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen e_mr->hwpage_size = save_mr.hwpage_size; 1355fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->fmr_page_size = save_mr.fmr_page_size; 1356fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->fmr_max_pages = save_mr.fmr_max_pages; 1357fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->fmr_max_maps = save_mr.fmr_max_maps; 1358fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->fmr_map_cnt = save_mr.fmr_map_cnt; 1359fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1360fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl, 13610cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering e_pd, pginfo, lkey, rkey, EHCA_REG_MR); 1362fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 1363fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 offset = (u64)(&e_mr->flags) - (u64)e_mr; 1364fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick memcpy(&e_mr->flags, &(save_mr.flags), 1365fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick sizeof(struct ehca_mr) - offset); 1366fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_rereg_mr_exit0; 1367fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1368fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1369fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1370fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_rereg_mr_exit0: 1371fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 1372e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p " 13733750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "iova_start=%p size=%llx acl=%x e_pd=%p pginfo=%p " 13743750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "num_kpages=%llx lkey=%x rkey=%x rereg_1_hcall=%x " 1375fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size, 1376df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen acl, e_pd, pginfo, pginfo->num_kpages, *lkey, *rkey, 1377fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick rereg_1_hcall, rereg_3_hcall); 1378fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 1379fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_rereg_mr() */ 1380fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1381fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 1382fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1383fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_unmap_one_fmr(struct ehca_shca *shca, 1384fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_fmr) 1385fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1386fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret = 0; 1387fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 1388fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd = 1389fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd); 1390fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr save_fmr; 1391fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 tmp_lkey, tmp_rkey; 1392df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen struct ehca_mr_pginfo pginfo; 13932b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen struct ehca_mr_hipzout_parms hipzout; 1394187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen struct ehca_mr save_mr; 1395fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1396187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen if (e_fmr->fmr_max_pages <= MAX_RPAGES) { 1397fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* 1398fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * note: after using rereg hcall with len=0, 1399fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * rereg hcall must be used again for registering pages 1400fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 1401fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0, 1402fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 0, 0, e_pd->fw_pd, 0, &hipzout); 1403187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen if (h_ret == H_SUCCESS) { 1404fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful reregistration */ 1405fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->start = NULL; 1406fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->size = 0; 1407fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick tmp_lkey = hipzout.lkey; 1408fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick tmp_rkey = hipzout.rkey; 1409187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen return 0; 1410fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1411187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen /* 1412187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen * should not happen, because length checked above, 1413187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen * FMRs are not shared and no MW bound to FMRs 1414187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen */ 1415187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ehca_err(&shca->ib_device, "hipz_reregister_pmr failed " 14163750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "(Rereg1), h_ret=%lli e_fmr=%p hca_hndl=%llx " 14173750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "mr_hndl=%llx lkey=%x lkey_out=%x", 1418187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen h_ret, e_fmr, shca->ipz_hca_handle.handle, 1419187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen e_fmr->ipz_mr_handle.handle, 1420187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen e_fmr->ib.ib_fmr.lkey, hipzout.lkey); 1421187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen /* try free and rereg */ 1422fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1423fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1424187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen /* first free old FMR */ 1425187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr); 1426187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen if (h_ret != H_SUCCESS) { 1427187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ehca_err(&shca->ib_device, "hipz_free_mr failed, " 14283750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "h_ret=%lli e_fmr=%p hca_hndl=%llx mr_hndl=%llx " 1429187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen "lkey=%x", 1430187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen h_ret, e_fmr, shca->ipz_hca_handle.handle, 1431187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen e_fmr->ipz_mr_handle.handle, 1432187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen e_fmr->ib.ib_fmr.lkey); 1433187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ret = ehca2ib_return_code(h_ret); 1434187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen goto ehca_unmap_one_fmr_exit0; 1435187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen } 1436187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen /* clean ehca_mr_t, without changing lock */ 1437187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen save_fmr = *e_fmr; 1438187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ehca_mr_deletenew(e_fmr); 1439fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1440187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen /* set some MR values */ 1441187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen e_fmr->flags = save_fmr.flags; 14425bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen e_fmr->hwpage_size = save_fmr.hwpage_size; 1443187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen e_fmr->fmr_page_size = save_fmr.fmr_page_size; 1444187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen e_fmr->fmr_max_pages = save_fmr.fmr_max_pages; 1445187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen e_fmr->fmr_max_maps = save_fmr.fmr_max_maps; 1446187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt; 1447187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen e_fmr->acl = save_fmr.acl; 1448fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1449187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen memset(&pginfo, 0, sizeof(pginfo)); 1450187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen pginfo.type = EHCA_MR_PGI_FMR; 1451187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ret = ehca_reg_mr(shca, e_fmr, NULL, 1452187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen (e_fmr->fmr_max_pages * e_fmr->fmr_page_size), 1453187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen e_fmr->acl, e_pd, &pginfo, &tmp_lkey, 14540cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering &tmp_rkey, EHCA_REG_MR); 1455187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen if (ret) { 1456187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr; 1457187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen memcpy(&e_fmr->flags, &(save_mr.flags), 1458187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen sizeof(struct ehca_mr) - offset); 1459fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1460fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1461fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_unmap_one_fmr_exit0: 1462fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 1463e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(&shca->ib_device, "ret=%i tmp_lkey=%x tmp_rkey=%x " 1464187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen "fmr_max_pages=%x", 1465187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages); 1466fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 1467fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_unmap_one_fmr() */ 1468fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1469fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 1470fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1471fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_reg_smr(struct ehca_shca *shca, 1472fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_origmr, 1473fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_newmr, 1474fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *iova_start, 1475fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int acl, 1476fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd, 1477fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 *lkey, /*OUT*/ 1478fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 *rkey) /*OUT*/ 1479fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1480fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret = 0; 1481fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 1482fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 hipz_acl; 14832b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen struct ehca_mr_hipzout_parms hipzout; 1484fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1485fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mrmw_map_acl(acl, &hipz_acl); 14865bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl); 1487fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1488fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr, 1489fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick (u64)iova_start, hipz_acl, e_pd->fw_pd, 1490fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick &hipzout); 1491fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 14923750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli " 1493fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x " 14943750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "e_pd=%p hca_hndl=%llx mr_hndl=%llx lkey=%x", 1495fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd, 1496fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick shca->ipz_hca_handle.handle, 1497fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_origmr->ipz_mr_handle.handle, 1498fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_origmr->ib.ib_mr.lkey); 1499a1a6ff11006c3a056cda9e8b13e7388fba3e69a1Hoang-Nam Nguyen ret = ehca2ib_return_code(h_ret); 1500fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_reg_smr_exit0; 1501fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1502fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful registration */ 1503df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_newmr->num_kpages = e_origmr->num_kpages; 1504df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_newmr->num_hwpages = e_origmr->num_hwpages; 15055bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen e_newmr->hwpage_size = e_origmr->hwpage_size; 1506df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_newmr->start = iova_start; 1507df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_newmr->size = e_origmr->size; 1508df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_newmr->acl = acl; 1509fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_newmr->ipz_mr_handle = hipzout.handle; 1510fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *lkey = hipzout.lkey; 1511fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *rkey = hipzout.rkey; 1512fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 1513fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1514fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_reg_smr_exit0: 1515fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 1516e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(&shca->ib_device, "ret=%i shca=%p e_origmr=%p " 1517fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "e_newmr=%p iova_start=%p acl=%x e_pd=%p", 1518fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd); 1519fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 1520fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_reg_smr() */ 1521fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1522fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 15230cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic inline void *ehca_calc_sectbase(int top, int dir, int idx) 15240cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 15250cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering unsigned long ret = idx; 15260cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ret |= dir << EHCA_DIR_INDEX_SHIFT; 15270cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ret |= top << EHCA_TOP_INDEX_SHIFT; 1528b4b8d1e48e05313b163a36946be1a82e5bc5f9adMichael Ellerman return __va(ret << SECTION_SIZE_BITS); 15290cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 15300cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 15310cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering#define ehca_bmap_valid(entry) \ 15320cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ((u64)entry != (u64)EHCA_INVAL_ADDR) 15330cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 15340cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic u64 ehca_reg_mr_section(int top, int dir, int idx, u64 *kpage, 15350cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_shca *shca, struct ehca_mr *mr, 15360cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_mr_pginfo *pginfo) 15370cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 15380cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u64 h_ret = 0; 15390cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering unsigned long page = 0; 1540b4b8d1e48e05313b163a36946be1a82e5bc5f9adMichael Ellerman u64 rpage = __pa(kpage); 15410cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int page_count; 15420cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 15430cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering void *sectbase = ehca_calc_sectbase(top, dir, idx); 15440cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if ((unsigned long)sectbase & (pginfo->hwpage_size - 1)) { 15450cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_err(&shca->ib_device, "reg_mr_section will probably fail:" 15460cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering "hwpage_size does not fit to " 15470cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering "section start address"); 15480cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 15490cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering page_count = EHCA_SECTSIZE / pginfo->hwpage_size; 15500cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 15510cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering while (page < page_count) { 15520cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u64 rnum; 15530cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering for (rnum = 0; (rnum < MAX_RPAGES) && (page < page_count); 15540cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering rnum++) { 15550cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering void *pg = sectbase + ((page++) * pginfo->hwpage_size); 1556b4b8d1e48e05313b163a36946be1a82e5bc5f9adMichael Ellerman kpage[rnum] = __pa(pg); 15570cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 15580cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 15590cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, mr, 15600cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_encode_hwpage_size(pginfo->hwpage_size), 15610cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 0, rpage, rnum); 15620cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 15630cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if ((h_ret != H_SUCCESS) && (h_ret != H_PAGE_REGISTERED)) { 15640cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_err(&shca->ib_device, "register_rpage_mr failed"); 15650cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return h_ret; 15660cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 15670cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 15680cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return h_ret; 15690cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 15700cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 15710cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic u64 ehca_reg_mr_sections(int top, int dir, u64 *kpage, 15720cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_shca *shca, struct ehca_mr *mr, 15730cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_mr_pginfo *pginfo) 15740cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 15750cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u64 hret = H_SUCCESS; 15760cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int idx; 15770cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 15780cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering for (idx = 0; idx < EHCA_MAP_ENTRIES; idx++) { 15790cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]->ent[idx])) 15800cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering continue; 15810cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 15820cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering hret = ehca_reg_mr_section(top, dir, idx, kpage, shca, mr, 15830cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering pginfo); 15840cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED)) 15850cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return hret; 15860cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 15870cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return hret; 15880cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 15890cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 15900cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic u64 ehca_reg_mr_dir_sections(int top, u64 *kpage, struct ehca_shca *shca, 15910cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_mr *mr, 15920cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_mr_pginfo *pginfo) 15930cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 15940cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u64 hret = H_SUCCESS; 15950cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int dir; 15960cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 15970cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering for (dir = 0; dir < EHCA_MAP_ENTRIES; dir++) { 15980cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir])) 15990cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering continue; 16000cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 16010cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering hret = ehca_reg_mr_sections(top, dir, kpage, shca, mr, pginfo); 16020cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED)) 16030cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return hret; 16040cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 16050cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return hret; 16060cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 1607fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1608fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/* register internal max-MR to internal SHCA */ 1609fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_reg_internal_maxmr( 1610fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca, 1611fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd, 1612fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr **e_maxmr) /*OUT*/ 1613fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1614fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret; 1615fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_mr; 1616fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *iova_start; 1617fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 size_maxmr; 1618df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen struct ehca_mr_pginfo pginfo; 1619fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_phys_buf ib_pbuf; 1620df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen u32 num_kpages; 1621df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen u32 num_hwpages; 16225bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen u64 hw_pgsize; 1623fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 16240cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap) { 16250cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ret = -EFAULT; 16260cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering goto ehca_reg_internal_maxmr_exit0; 16270cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 16280cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 1629fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr = ehca_mr_new(); 1630fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!e_mr) { 1631fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "out of memory"); 1632fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -ENOMEM; 1633fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_reg_internal_maxmr_exit0; 1634fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1635fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->flags |= EHCA_MR_FLAG_MAXMR; 1636fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1637fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* register internal max-MR on HCA */ 16380cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering size_maxmr = ehca_mr_len; 1639625fbd3a36d836efaaee4b6d9c2fcd25e3654624Sonny Rao iova_start = (u64 *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START)); 1640fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_pbuf.addr = 0; 1641fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_pbuf.size = size_maxmr; 1642df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr, 1643df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen PAGE_SIZE); 16445bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen hw_pgsize = ehca_get_max_hwpage_size(shca); 16455bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size_maxmr, 16465bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen hw_pgsize); 1647df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen 1648df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen memset(&pginfo, 0, sizeof(pginfo)); 1649df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.type = EHCA_MR_PGI_PHYS; 1650df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.num_kpages = num_kpages; 1651df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.num_hwpages = num_hwpages; 16525bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo.hwpage_size = hw_pgsize; 1653df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.u.phy.num_phys_buf = 1; 1654df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo.u.phy.phys_buf_array = &ib_pbuf; 1655fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1656fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd, 1657fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick &pginfo, &e_mr->ib.ib_mr.lkey, 16580cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering &e_mr->ib.ib_mr.rkey, EHCA_REG_BUSMAP_MR); 1659fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 1660fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "reg of internal max MR failed, " 16613750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "e_mr=%p iova_start=%p size_maxmr=%llx num_kpages=%x " 1662df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen "num_hwpages=%x", e_mr, iova_start, size_maxmr, 1663df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen num_kpages, num_hwpages); 1664fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_reg_internal_maxmr_exit1; 1665fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1666fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1667fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful registration of all pages */ 1668fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ib.ib_mr.device = e_pd->ib_pd.device; 1669fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ib.ib_mr.pd = &e_pd->ib_pd; 1670fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_mr->ib.ib_mr.uobject = NULL; 1671fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick atomic_inc(&(e_pd->ib_pd.usecnt)); 1672fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick atomic_set(&(e_mr->ib.ib_mr.usecnt), 0); 1673fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *e_maxmr = e_mr; 1674fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 1675fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1676fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_reg_internal_maxmr_exit1: 1677fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mr_delete(e_mr); 1678fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_reg_internal_maxmr_exit0: 1679fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 1680e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(&shca->ib_device, "ret=%i shca=%p e_pd=%p e_maxmr=%p", 1681fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret, shca, e_pd, e_maxmr); 1682fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 1683fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_reg_internal_maxmr() */ 1684fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1685fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 1686fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1687fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_reg_maxmr(struct ehca_shca *shca, 1688fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_newmr, 1689fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *iova_start, 1690fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int acl, 1691fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_pd *e_pd, 1692fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 *lkey, 1693fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 *rkey) 1694fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1695fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 1696fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_origmr = shca->maxmr; 1697fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 hipz_acl; 16982b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyen struct ehca_mr_hipzout_parms hipzout; 1699fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1700fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_mrmw_map_acl(acl, &hipz_acl); 17015bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl); 1702fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1703fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr, 1704fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick (u64)iova_start, hipz_acl, e_pd->fw_pd, 1705fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick &hipzout); 1706fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 17073750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli " 17083750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "e_origmr=%p hca_hndl=%llx mr_hndl=%llx lkey=%x", 1709fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret, e_origmr, shca->ipz_hca_handle.handle, 1710fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_origmr->ipz_mr_handle.handle, 1711fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_origmr->ib.ib_mr.lkey); 1712a1a6ff11006c3a056cda9e8b13e7388fba3e69a1Hoang-Nam Nguyen return ehca2ib_return_code(h_ret); 1713fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1714fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* successful registration */ 1715df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_newmr->num_kpages = e_origmr->num_kpages; 1716df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_newmr->num_hwpages = e_origmr->num_hwpages; 17175bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen e_newmr->hwpage_size = e_origmr->hwpage_size; 1718df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_newmr->start = iova_start; 1719df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_newmr->size = e_origmr->size; 1720df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen e_newmr->acl = acl; 1721fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_newmr->ipz_mr_handle = hipzout.handle; 1722fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *lkey = hipzout.lkey; 1723fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *rkey = hipzout.rkey; 1724fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 1725fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_reg_maxmr() */ 1726fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1727fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 1728fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1729fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_dereg_internal_maxmr(struct ehca_shca *shca) 1730fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1731fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret; 1732fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_mr *e_maxmr; 1733fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_pd *ib_pd; 1734fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1735fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!shca->maxmr) { 1736fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "bad call, shca=%p", shca); 1737fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EINVAL; 1738fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_dereg_internal_maxmr_exit0; 1739fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1740fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1741fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_maxmr = shca->maxmr; 1742fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_pd = e_maxmr->ib.ib_mr.pd; 1743fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */ 1744fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1745fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = ehca_dereg_mr(&e_maxmr->ib.ib_mr); 1746fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) { 1747fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(&shca->ib_device, "dereg internal max-MR failed, " 1748e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes "ret=%i e_maxmr=%p shca=%p lkey=%x", 1749fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey); 1750fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick shca->maxmr = e_maxmr; 1751fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto ehca_dereg_internal_maxmr_exit0; 1752fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1753fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1754fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick atomic_dec(&ib_pd->usecnt); 1755fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1756fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickehca_dereg_internal_maxmr_exit0: 1757fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 1758e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(&shca->ib_device, "ret=%i shca=%p shca->maxmr=%p", 1759fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret, shca, shca->maxmr); 1760fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 1761fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_dereg_internal_maxmr() */ 1762fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1763fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 1764fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1765fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/* 1766fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * check physical buffer array of MR verbs for validness and 1767fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * calculates MR size 1768fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 1769fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array, 1770fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int num_phys_buf, 1771fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *iova_start, 1772fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *size) 1773fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1774fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_phys_buf *pbuf = phys_buf_array; 1775fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 size_count = 0; 1776fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 i; 1777fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1778fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (num_phys_buf == 0) { 1779fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_gen_err("bad phys buf array len, num_phys_buf=0"); 1780fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -EINVAL; 1781fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1782fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* check first buffer */ 1783fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) { 1784fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_gen_err("iova_start/addr mismatch, iova_start=%p " 17853750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "pbuf->addr=%llx pbuf->size=%llx", 1786fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick iova_start, pbuf->addr, pbuf->size); 1787fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -EINVAL; 1788fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1789fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (((pbuf->addr + pbuf->size) % PAGE_SIZE) && 1790fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick (num_phys_buf > 1)) { 17913750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%llx " 17923750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "pbuf->size=%llx", pbuf->addr, pbuf->size); 1793fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -EINVAL; 1794fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1795fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1796fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick for (i = 0; i < num_phys_buf; i++) { 1797fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((i > 0) && (pbuf->addr % PAGE_SIZE)) { 17983750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_gen_err("bad address, i=%x pbuf->addr=%llx " 17993750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "pbuf->size=%llx", 1800fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick i, pbuf->addr, pbuf->size); 1801fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -EINVAL; 1802fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1803fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (((i > 0) && /* not 1st */ 1804fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick (i < (num_phys_buf - 1)) && /* not last */ 1805fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) { 18063750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_gen_err("bad size, i=%x pbuf->size=%llx", 1807fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick i, pbuf->size); 1808fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -EINVAL; 1809fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1810fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick size_count += pbuf->size; 1811fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick pbuf++; 1812fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1813fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1814fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *size = size_count; 1815fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 1816fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_mr_chk_buf_and_calc_size() */ 1817fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1818fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 1819fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1820fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/* check page list of map FMR verb for validness */ 1821fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_fmr_check_page_list(struct ehca_mr *e_fmr, 1822fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *page_list, 1823fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int list_len) 1824fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1825fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 i; 1826fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *page; 1827fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1828fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((list_len == 0) || (list_len > e_fmr->fmr_max_pages)) { 1829fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_gen_err("bad list_len, list_len=%x " 1830fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "e_fmr->fmr_max_pages=%x fmr=%p", 1831fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick list_len, e_fmr->fmr_max_pages, e_fmr); 1832fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -EINVAL; 1833fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1834fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1835fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* each page must be aligned */ 1836fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick page = page_list; 1837fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick for (i = 0; i < list_len; i++) { 1838fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (*page % e_fmr->fmr_page_size) { 18393750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_gen_err("bad page, i=%x *page=%llx page=%p fmr=%p " 1840fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "fmr_page_size=%x", i, *page, page, e_fmr, 1841fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick e_fmr->fmr_page_size); 1842fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -EINVAL; 1843fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1844fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick page++; 1845fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1846fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1847fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 1848fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_fmr_check_page_list() */ 1849fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1850fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 1851fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1852187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen/* PAGE_SIZE >= pginfo->hwpage_size */ 1853187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyenstatic int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo, 1854187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen u32 number, 1855187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen u64 *kpage) 1856fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 1857fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret = 0; 1858187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen u64 pgaddr; 1859fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 j = 0; 18605bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size; 1861eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas struct scatterlist **sg = &pginfo->u.usr.next_sg; 1862eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas 1863eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas while (*sg != NULL) { 1864eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pgaddr = page_to_pfn(sg_page(*sg)) 1865eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas << PAGE_SHIFT; 1866eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas *kpage = pgaddr + (pginfo->next_hwpage * 1867eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pginfo->hwpage_size); 1868eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas if (!(*kpage)) { 1869eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas ehca_gen_err("pgaddr=%llx " 1870eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas "sg_dma_address=%llx " 1871eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas "entry=%llx next_hwpage=%llx", 1872eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pgaddr, (u64)sg_dma_address(*sg), 1873eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pginfo->u.usr.next_nmap, 1874eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pginfo->next_hwpage); 1875eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas return -EFAULT; 1876fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1877eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas (pginfo->hwpage_cnt)++; 1878eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas (pginfo->next_hwpage)++; 1879eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas kpage++; 1880eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas if (pginfo->next_hwpage % hwpages_per_kpage == 0) { 1881eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas (pginfo->kpage_cnt)++; 1882eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas (pginfo->u.usr.next_nmap)++; 1883eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pginfo->next_hwpage = 0; 1884eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas *sg = sg_next(*sg); 1885eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas } 1886eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas j++; 1887eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas if (j >= number) 1888187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen break; 1889fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 1890eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas 1891fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 1892187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen} 1893fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 18945bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen/* 18955bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen * check given pages for contiguous layout 18965bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen * last page addr is returned in prev_pgaddr for further check 18975bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen */ 1898eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadasstatic int ehca_check_kpages_per_ate(struct scatterlist **sg, 1899eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas int num_pages, 19005bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen u64 *prev_pgaddr) 19015bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen{ 1902eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas for (; *sg && num_pages > 0; *sg = sg_next(*sg), num_pages--) { 1903eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas u64 pgaddr = page_to_pfn(sg_page(*sg)) << PAGE_SHIFT; 19044da27d6d5b92c8fe4b3a3e5bcf42606d9e4a6fc8Joachim Fenkes if (ehca_debug_level >= 3) 19053750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_gen_dbg("chunk_page=%llx value=%016llx", pgaddr, 1906b4b8d1e48e05313b163a36946be1a82e5bc5f9adMichael Ellerman *(u64 *)__va(pgaddr)); 19075bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen if (pgaddr - PAGE_SIZE != *prev_pgaddr) { 19083750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_gen_err("uncontiguous page found pgaddr=%llx " 1909eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas "prev_pgaddr=%llx entries_left_in_hwpage=%x", 1910eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pgaddr, *prev_pgaddr, num_pages); 19115bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen return -EINVAL; 19125bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } 19135bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen *prev_pgaddr = pgaddr; 19145bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } 19155bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen return 0; 19165bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen} 19175bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen 19185bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen/* PAGE_SIZE < pginfo->hwpage_size */ 19195bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyenstatic int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo, 19205bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen u32 number, 19215bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen u64 *kpage) 19225bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen{ 19235bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen int ret = 0; 19245bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen u64 pgaddr, prev_pgaddr; 19255bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen u32 j = 0; 19265bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE; 19275bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen int nr_kpages = kpages_per_hwpage; 1928eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas struct scatterlist **sg = &pginfo->u.usr.next_sg; 19295bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen 1930eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas while (*sg != NULL) { 1931eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas 1932eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas if (nr_kpages == kpages_per_hwpage) { 1933eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pgaddr = (page_to_pfn(sg_page(*sg)) 1934eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas << PAGE_SHIFT); 1935eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas *kpage = pgaddr; 1936eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas if (!(*kpage)) { 1937eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas ehca_gen_err("pgaddr=%llx entry=%llx", 1938eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pgaddr, pginfo->u.usr.next_nmap); 1939eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas ret = -EFAULT; 1940eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas return ret; 1941eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas } 1942eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas /* 1943eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas * The first page in a hwpage must be aligned; 1944eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas * the first MR page is exempt from this rule. 1945eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas */ 1946eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas if (pgaddr & (pginfo->hwpage_size - 1)) { 1947eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas if (pginfo->hwpage_cnt) { 1948eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas ehca_gen_err( 1949eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas "invalid alignment " 1950eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas "pgaddr=%llx entry=%llx " 1951eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas "mr_pgsize=%llx", 1952eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pgaddr, pginfo->u.usr.next_nmap, 1953eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pginfo->hwpage_size); 19545bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ret = -EFAULT; 19555bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen return ret; 19565bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } 1957eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas /* first MR page */ 1958eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pginfo->kpage_cnt = 1959eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas (pgaddr & 1960eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas (pginfo->hwpage_size - 1)) >> 1961eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas PAGE_SHIFT; 1962eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas nr_kpages -= pginfo->kpage_cnt; 1963eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas *kpage = pgaddr & 1964eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas ~(pginfo->hwpage_size - 1); 19655bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } 1966eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas if (ehca_debug_level >= 3) { 1967eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas u64 val = *(u64 *)__va(pgaddr); 1968eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas ehca_gen_dbg("kpage=%llx page=%llx " 1969eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas "value=%016llx", 1970eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas *kpage, pgaddr, val); 19715bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } 1972eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas prev_pgaddr = pgaddr; 1973eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas *sg = sg_next(*sg); 1974eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pginfo->kpage_cnt++; 1975eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pginfo->u.usr.next_nmap++; 1976eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas nr_kpages--; 1977eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas if (!nr_kpages) 1978eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas goto next_kpage; 1979eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas continue; 1980eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas } 1981eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas 1982eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas ret = ehca_check_kpages_per_ate(sg, nr_kpages, 1983eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas &prev_pgaddr); 1984eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas if (ret) 1985eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas return ret; 1986eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pginfo->kpage_cnt += nr_kpages; 1987eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas pginfo->u.usr.next_nmap += nr_kpages; 19885bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen 19895bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyennext_kpage: 1990eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas nr_kpages = kpages_per_hwpage; 1991eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas (pginfo->hwpage_cnt)++; 1992eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas kpage++; 1993eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas j++; 1994eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas if (j >= number) 19955bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen break; 19965bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } 1997eeb8461e36c99fdf2d058751be924a2aab215005Yishai Hadas 19985bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen return ret; 19995bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen} 20005bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen 2001e8e91f6b4dc1179a70b0d21241b769c0ebfaa129Roland Dreierstatic int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo, 2002e8e91f6b4dc1179a70b0d21241b769c0ebfaa129Roland Dreier u32 number, u64 *kpage) 2003fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 2004fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret = 0; 2005187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen struct ib_phys_buf *pbuf; 2006187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen u64 num_hw, offs_hw; 2007187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen u32 i = 0; 2008187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen 2009187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen /* loop over desired phys_buf_array entries */ 2010187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen while (i < number) { 2011187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf; 20125bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen num_hw = NUM_CHUNKS((pbuf->addr % pginfo->hwpage_size) + 20135bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pbuf->size, pginfo->hwpage_size); 20145bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen offs_hw = (pbuf->addr & ~(pginfo->hwpage_size - 1)) / 20155bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo->hwpage_size; 2016187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen while (pginfo->next_hwpage < offs_hw + num_hw) { 2017187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen /* sanity check */ 2018187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen if ((pginfo->kpage_cnt >= pginfo->num_kpages) || 2019187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen (pginfo->hwpage_cnt >= pginfo->num_hwpages)) { 2020187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ehca_gen_err("kpage_cnt >= num_kpages, " 20213750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "kpage_cnt=%llx num_kpages=%llx " 20223750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "hwpage_cnt=%llx " 20233750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "num_hwpages=%llx i=%x", 2024187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen pginfo->kpage_cnt, 2025187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen pginfo->num_kpages, 2026187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen pginfo->hwpage_cnt, 2027187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen pginfo->num_hwpages, i); 2028187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen return -EFAULT; 2029187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen } 2030aa97c32c1e6d6564cc385c36bfd28a2663a9bad6Michael Ellerman *kpage = (pbuf->addr & ~(pginfo->hwpage_size - 1)) + 2031aa97c32c1e6d6564cc385c36bfd28a2663a9bad6Michael Ellerman (pginfo->next_hwpage * pginfo->hwpage_size); 2032187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen if ( !(*kpage) && pbuf->addr ) { 20333750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_gen_err("pbuf->addr=%llx pbuf->size=%llx " 20343750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "next_hwpage=%llx", pbuf->addr, 20355bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pbuf->size, pginfo->next_hwpage); 2036187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen return -EFAULT; 2037fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 2038df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen (pginfo->hwpage_cnt)++; 2039df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen (pginfo->next_hwpage)++; 20405bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen if (PAGE_SIZE >= pginfo->hwpage_size) { 20415bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen if (pginfo->next_hwpage % 20425bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen (PAGE_SIZE / pginfo->hwpage_size) == 0) 20435bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen (pginfo->kpage_cnt)++; 20445bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } else 20455bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo->kpage_cnt += pginfo->hwpage_size / 20465bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen PAGE_SIZE; 2047187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen kpage++; 2048187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen i++; 2049187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen if (i >= number) break; 2050fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 2051187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen if (pginfo->next_hwpage >= offs_hw + num_hw) { 2052187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen (pginfo->u.phy.next_buf)++; 2053187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen pginfo->next_hwpage = 0; 2054187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen } 2055187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen } 2056187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen return ret; 2057187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen} 2058187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen 2059e8e91f6b4dc1179a70b0d21241b769c0ebfaa129Roland Dreierstatic int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo, 2060e8e91f6b4dc1179a70b0d21241b769c0ebfaa129Roland Dreier u32 number, u64 *kpage) 2061187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen{ 2062187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen int ret = 0; 2063187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen u64 *fmrlist; 2064187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen u32 i; 2065187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen 2066187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen /* loop over desired page_list entries */ 2067187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem; 2068187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen for (i = 0; i < number; i++) { 2069aa97c32c1e6d6564cc385c36bfd28a2663a9bad6Michael Ellerman *kpage = (*fmrlist & ~(pginfo->hwpage_size - 1)) + 2070aa97c32c1e6d6564cc385c36bfd28a2663a9bad6Michael Ellerman pginfo->next_hwpage * pginfo->hwpage_size; 2071187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen if ( !(*kpage) ) { 20723750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_gen_err("*fmrlist=%llx fmrlist=%p " 20733750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "next_listelem=%llx next_hwpage=%llx", 2074187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen *fmrlist, fmrlist, 2075187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen pginfo->u.fmr.next_listelem, 2076df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen pginfo->next_hwpage); 2077187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen return -EFAULT; 2078fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 2079df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen (pginfo->hwpage_cnt)++; 20805bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen if (pginfo->u.fmr.fmr_pgsize >= pginfo->hwpage_size) { 20815bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen if (pginfo->next_hwpage % 20825bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen (pginfo->u.fmr.fmr_pgsize / 20835bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo->hwpage_size) == 0) { 20845bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen (pginfo->kpage_cnt)++; 20855bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen (pginfo->u.fmr.next_listelem)++; 20865bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen fmrlist++; 20875bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo->next_hwpage = 0; 20885bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } else 20895bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen (pginfo->next_hwpage)++; 20905bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } else { 20915bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen unsigned int cnt_per_hwpage = pginfo->hwpage_size / 20925bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo->u.fmr.fmr_pgsize; 20935bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen unsigned int j; 20945bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen u64 prev = *kpage; 20955bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen /* check if adrs are contiguous */ 20965bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen for (j = 1; j < cnt_per_hwpage; j++) { 2097aa97c32c1e6d6564cc385c36bfd28a2663a9bad6Michael Ellerman u64 p = fmrlist[j] & ~(pginfo->hwpage_size - 1); 20985bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen if (prev + pginfo->u.fmr.fmr_pgsize != p) { 20995bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ehca_gen_err("uncontiguous fmr pages " 21003750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "found prev=%llx p=%llx " 21015bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen "idx=%x", prev, p, i + j); 21025bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen return -EINVAL; 21035bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } 21045bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen prev = p; 21055bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen } 21065bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo->kpage_cnt += cnt_per_hwpage; 21075bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen pginfo->u.fmr.next_listelem += cnt_per_hwpage; 21085bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen fmrlist += cnt_per_hwpage; 2109fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 21105bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen kpage++; 2111187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen } 2112187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen return ret; 2113187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen} 2114187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen 2115187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen/* setup page buffer from page info */ 2116187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyenint ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo, 2117187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen u32 number, 2118187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen u64 *kpage) 2119187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen{ 2120187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen int ret; 2121187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen 2122187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen switch (pginfo->type) { 2123187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen case EHCA_MR_PGI_PHYS: 2124187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ret = ehca_set_pagebuf_phys(pginfo, number, kpage); 2125187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen break; 2126187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen case EHCA_MR_PGI_USER: 21275bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ret = PAGE_SIZE >= pginfo->hwpage_size ? 21285bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ehca_set_pagebuf_user1(pginfo, number, kpage) : 21295bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen ehca_set_pagebuf_user2(pginfo, number, kpage); 2130187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen break; 2131187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen case EHCA_MR_PGI_FMR: 2132187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen ret = ehca_set_pagebuf_fmr(pginfo, number, kpage); 2133187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen break; 2134187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen default: 2135fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_gen_err("bad pginfo->type=%x", pginfo->type); 2136fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = -EFAULT; 2137187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen break; 2138fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 2139fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 2140187c72e31f92791ec70395b80aa9883f2edad97fHoang-Nam Nguyen} /* end ehca_set_pagebuf() */ 2141fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2142fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 2143fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2144fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/* 2145fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * check MR if it is a max-MR, i.e. uses whole memory 2146fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * in case it's a max-MR 1 is returned, else 0 2147fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 2148fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_mr_is_maxmr(u64 size, 2149fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 *iova_start) 2150fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 2151fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* a MR is treated as max-MR only if it fits following: */ 21520cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if ((size == ehca_mr_len) && 2153625fbd3a36d836efaaee4b6d9c2fcd25e3654624Sonny Rao (iova_start == (void *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START)))) { 2154fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_gen_dbg("this is a max-MR"); 2155fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 1; 2156fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } else 2157fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 2158fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_mr_is_maxmr() */ 2159fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2160fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 2161fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2162fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/* map access control for MR/MW. This routine is used for MR and MW. */ 2163fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickvoid ehca_mrmw_map_acl(int ib_acl, 2164fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 *hipz_acl) 2165fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 2166fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *hipz_acl = 0; 2167fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ib_acl & IB_ACCESS_REMOTE_READ) 2168fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *hipz_acl |= HIPZ_ACCESSCTRL_R_READ; 2169fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ib_acl & IB_ACCESS_REMOTE_WRITE) 2170fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *hipz_acl |= HIPZ_ACCESSCTRL_R_WRITE; 2171fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ib_acl & IB_ACCESS_REMOTE_ATOMIC) 2172fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *hipz_acl |= HIPZ_ACCESSCTRL_R_ATOMIC; 2173fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ib_acl & IB_ACCESS_LOCAL_WRITE) 2174fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *hipz_acl |= HIPZ_ACCESSCTRL_L_WRITE; 2175fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ib_acl & IB_ACCESS_MW_BIND) 2176fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *hipz_acl |= HIPZ_ACCESSCTRL_MW_BIND; 2177fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_mrmw_map_acl() */ 2178fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2179fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 2180fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2181fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/* sets page size in hipz access control for MR/MW. */ 21825bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyenvoid ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl) /*INOUT*/ 2183fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 21845bb7d9290cd23a55906e4fe7a7fedecf29468c81Hoang-Nam Nguyen *hipz_acl |= (ehca_encode_hwpage_size(pgsize) << 24); 2185fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_mrmw_set_pgsize_hipz_acl() */ 2186fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2187fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 2188fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2189fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/* 2190fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * reverse map access control for MR/MW. 2191fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * This routine is used for MR and MW. 2192fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 2193fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickvoid ehca_mrmw_reverse_map_acl(const u32 *hipz_acl, 2194fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int *ib_acl) /*OUT*/ 2195fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 2196fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *ib_acl = 0; 2197fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (*hipz_acl & HIPZ_ACCESSCTRL_R_READ) 2198fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *ib_acl |= IB_ACCESS_REMOTE_READ; 2199fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (*hipz_acl & HIPZ_ACCESSCTRL_R_WRITE) 2200fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *ib_acl |= IB_ACCESS_REMOTE_WRITE; 2201fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (*hipz_acl & HIPZ_ACCESSCTRL_R_ATOMIC) 2202fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *ib_acl |= IB_ACCESS_REMOTE_ATOMIC; 2203fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (*hipz_acl & HIPZ_ACCESSCTRL_L_WRITE) 2204fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *ib_acl |= IB_ACCESS_LOCAL_WRITE; 2205fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (*hipz_acl & HIPZ_ACCESSCTRL_MW_BIND) 2206fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick *ib_acl |= IB_ACCESS_MW_BIND; 2207fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_mrmw_reverse_map_acl() */ 2208fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2209fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2210fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/*----------------------------------------------------------------------*/ 2211fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2212fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/* 2213fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * MR destructor and constructor 2214fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * used in Reregister MR verb, sets all fields in ehca_mr_t to 0, 2215fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * except struct ib_mr and spinlock 2216fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 2217fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickvoid ehca_mr_deletenew(struct ehca_mr *mr) 2218fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 2219df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen mr->flags = 0; 2220df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen mr->num_kpages = 0; 2221df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen mr->num_hwpages = 0; 2222df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen mr->acl = 0; 2223df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen mr->start = NULL; 2224fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr->fmr_page_size = 0; 2225fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr->fmr_max_pages = 0; 2226df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen mr->fmr_max_maps = 0; 2227df17bfd4a030f7d986de14210f4b21876a7a2989Hoang-Nam Nguyen mr->fmr_map_cnt = 0; 2228fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick memset(&mr->ipz_mr_handle, 0, sizeof(mr->ipz_mr_handle)); 2229fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick memset(&mr->galpas, 0, sizeof(mr->galpas)); 2230fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} /* end ehca_mr_deletenew() */ 2231fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2232fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_init_mrmw_cache(void) 2233fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 2234fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr_cache = kmem_cache_create("ehca_cache_mr", 2235fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick sizeof(struct ehca_mr), 0, 2236fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick SLAB_HWCACHE_ALIGN, 223720c2df83d25c6a95affe6157a4c9cac4cf5ffaacPaul Mundt NULL); 2238fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!mr_cache) 2239fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -ENOMEM; 2240fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mw_cache = kmem_cache_create("ehca_cache_mw", 2241fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick sizeof(struct ehca_mw), 0, 2242fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick SLAB_HWCACHE_ALIGN, 224320c2df83d25c6a95affe6157a4c9cac4cf5ffaacPaul Mundt NULL); 2244fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!mw_cache) { 2245fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick kmem_cache_destroy(mr_cache); 2246fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick mr_cache = NULL; 2247fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -ENOMEM; 2248fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 2249fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 2250fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 2251fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 2252fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickvoid ehca_cleanup_mrmw_cache(void) 2253fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 2254fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (mr_cache) 2255fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick kmem_cache_destroy(mr_cache); 2256fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (mw_cache) 2257fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick kmem_cache_destroy(mw_cache); 2258fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 22590cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 22600cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic inline int ehca_init_top_bmap(struct ehca_top_bmap *ehca_top_bmap, 22610cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int dir) 22620cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 22630cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap_valid(ehca_top_bmap->dir[dir])) { 22640cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_top_bmap->dir[dir] = 22650cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering kmalloc(sizeof(struct ehca_dir_bmap), GFP_KERNEL); 22660cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_top_bmap->dir[dir]) 22670cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return -ENOMEM; 22680cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering /* Set map block to 0xFF according to EHCA_INVAL_ADDR */ 22690cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering memset(ehca_top_bmap->dir[dir], 0xFF, EHCA_ENT_MAP_SIZE); 22700cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 22710cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return 0; 22720cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 22730cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 22740cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic inline int ehca_init_bmap(struct ehca_bmap *ehca_bmap, int top, int dir) 22750cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 22760cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap_valid(ehca_bmap->top[top])) { 22770cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_bmap->top[top] = 22780cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering kmalloc(sizeof(struct ehca_top_bmap), GFP_KERNEL); 22790cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap->top[top]) 22800cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return -ENOMEM; 22810cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering /* Set map block to 0xFF according to EHCA_INVAL_ADDR */ 22820cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering memset(ehca_bmap->top[top], 0xFF, EHCA_DIR_MAP_SIZE); 22830cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 22840cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return ehca_init_top_bmap(ehca_bmap->top[top], dir); 22850cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 22860cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 22870cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic inline int ehca_calc_index(unsigned long i, unsigned long s) 22880cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 22890cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return (i >> s) & EHCA_INDEX_MASK; 22900cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 22910cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 22920cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringvoid ehca_destroy_busmap(void) 22930cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 22940cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int top, dir; 22950cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 22960cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap) 22970cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return; 22980cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 22990cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering for (top = 0; top < EHCA_MAP_ENTRIES; top++) { 23000cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap_valid(ehca_bmap->top[top])) 23010cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering continue; 23020cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering for (dir = 0; dir < EHCA_MAP_ENTRIES; dir++) { 23030cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir])) 23040cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering continue; 23050cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23060cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering kfree(ehca_bmap->top[top]->dir[dir]); 23070cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 23080cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23090cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering kfree(ehca_bmap->top[top]); 23100cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 23110cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23120cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering kfree(ehca_bmap); 23130cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_bmap = NULL; 23140cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 23150cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23160cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic int ehca_update_busmap(unsigned long pfn, unsigned long nr_pages) 23170cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 23180cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering unsigned long i, start_section, end_section; 23190cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int top, dir, idx; 23200cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23210cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!nr_pages) 23220cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return 0; 23230cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23240cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap) { 23250cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_bmap = kmalloc(sizeof(struct ehca_bmap), GFP_KERNEL); 23260cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap) 23270cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return -ENOMEM; 23280cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering /* Set map block to 0xFF according to EHCA_INVAL_ADDR */ 23290cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering memset(ehca_bmap, 0xFF, EHCA_TOP_MAP_SIZE); 23300cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 23310cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 2332aa97c32c1e6d6564cc385c36bfd28a2663a9bad6Michael Ellerman start_section = (pfn * PAGE_SIZE) / EHCA_SECTSIZE; 2333aa97c32c1e6d6564cc385c36bfd28a2663a9bad6Michael Ellerman end_section = ((pfn + nr_pages) * PAGE_SIZE) / EHCA_SECTSIZE; 23340cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering for (i = start_section; i < end_section; i++) { 23350cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int ret; 23360cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering top = ehca_calc_index(i, EHCA_TOP_INDEX_SHIFT); 23370cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering dir = ehca_calc_index(i, EHCA_DIR_INDEX_SHIFT); 23380cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering idx = i & EHCA_INDEX_MASK; 23390cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23400cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ret = ehca_init_bmap(ehca_bmap, top, dir); 23410cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (ret) { 23420cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_destroy_busmap(); 23430cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return ret; 23440cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 23450cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_bmap->top[top]->dir[dir]->ent[idx] = ehca_mr_len; 23460cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_mr_len += EHCA_SECTSIZE; 23470cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 23480cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return 0; 23490cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 23500cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23510cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic int ehca_is_hugepage(unsigned long pfn) 23520cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 23530cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int page_order; 23540cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23550cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (pfn & EHCA_HUGEPAGE_PFN_MASK) 23560cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return 0; 23570cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23580cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering page_order = compound_order(pfn_to_page(pfn)); 23590cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (page_order + PAGE_SHIFT != EHCA_HUGEPAGESHIFT) 23600cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return 0; 23610cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23620cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return 1; 23630cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 23640cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23650cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic int ehca_create_busmap_callback(unsigned long initial_pfn, 23660cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering unsigned long total_nr_pages, void *arg) 23670cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 23680cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int ret; 23690cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering unsigned long pfn, start_pfn, end_pfn, nr_pages; 23700cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23710cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if ((total_nr_pages * PAGE_SIZE) < EHCA_HUGEPAGE_SIZE) 23720cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return ehca_update_busmap(initial_pfn, total_nr_pages); 23730cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23740cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering /* Given chunk is >= 16GB -> check for hugepages */ 23750cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering start_pfn = initial_pfn; 23760cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering end_pfn = initial_pfn + total_nr_pages; 23770cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering pfn = start_pfn; 23780cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23790cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering while (pfn < end_pfn) { 23800cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (ehca_is_hugepage(pfn)) { 23810cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering /* Add mem found in front of the hugepage */ 23820cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering nr_pages = pfn - start_pfn; 23830cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ret = ehca_update_busmap(start_pfn, nr_pages); 23840cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (ret) 23850cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return ret; 23860cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering /* Skip the hugepage */ 23870cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering pfn += (EHCA_HUGEPAGE_SIZE / PAGE_SIZE); 23880cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering start_pfn = pfn; 23890cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } else 23900cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering pfn += (EHCA_SECTSIZE / PAGE_SIZE); 23910cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 23920cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23930cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering /* Add mem found behind the hugepage(s) */ 23940cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering nr_pages = pfn - start_pfn; 23950cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return ehca_update_busmap(start_pfn, nr_pages); 23960cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 23970cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 23980cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringint ehca_create_busmap(void) 23990cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 24000cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int ret; 24010cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24020cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_mr_len = 0; 2403908eedc6168bd92e89f90d89fa389065a36358faKAMEZAWA Hiroyuki ret = walk_system_ram_range(0, 1ULL << MAX_PHYSMEM_BITS, NULL, 24040cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_create_busmap_callback); 24050cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return ret; 24060cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 24070cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24080cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca, 24090cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_mr *e_mr, 24100cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct ehca_mr_pginfo *pginfo) 24110cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 24120cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int top; 24130cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u64 hret, *kpage; 24140cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24150cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL); 24160cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!kpage) { 24170cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_err(&shca->ib_device, "kpage alloc failed"); 24180cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return -ENOMEM; 24190cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 24200cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering for (top = 0; top < EHCA_MAP_ENTRIES; top++) { 24210cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap_valid(ehca_bmap->top[top])) 24220cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering continue; 24230cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering hret = ehca_reg_mr_dir_sections(top, kpage, shca, e_mr, pginfo); 24240cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if ((hret != H_PAGE_REGISTERED) && (hret != H_SUCCESS)) 24250cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering break; 24260cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 24270cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24280cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_free_fw_ctrlblock(kpage); 24290cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24300cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (hret == H_SUCCESS) 24310cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return 0; /* Everything is fine */ 24320cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering else { 24330cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering ehca_err(&shca->ib_device, "ehca_reg_bmap_mr_rpages failed, " 24340cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering "h_ret=%lli e_mr=%p top=%x lkey=%x " 24350cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering "hca_hndl=%llx mr_hndl=%llx", hret, e_mr, top, 24360cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering e_mr->ib.ib_mr.lkey, 24370cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering shca->ipz_hca_handle.handle, 24380cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering e_mr->ipz_mr_handle.handle); 24390cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return ehca2ib_return_code(hret); 24400cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 24410cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 24420cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24430cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic u64 ehca_map_vaddr(void *caddr) 24440cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 24450cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int top, dir, idx; 24460cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering unsigned long abs_addr, offset; 24470cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u64 entry; 24480cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24490cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap) 24500cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return EHCA_INVAL_ADDR; 24510cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 2452b4b8d1e48e05313b163a36946be1a82e5bc5f9adMichael Ellerman abs_addr = __pa(caddr); 24530cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering top = ehca_calc_index(abs_addr, EHCA_TOP_INDEX_SHIFT + EHCA_SECTSHIFT); 24540cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap_valid(ehca_bmap->top[top])) 24550cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return EHCA_INVAL_ADDR; 24560cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24570cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering dir = ehca_calc_index(abs_addr, EHCA_DIR_INDEX_SHIFT + EHCA_SECTSHIFT); 24580cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir])) 24590cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return EHCA_INVAL_ADDR; 24600cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24610cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering idx = ehca_calc_index(abs_addr, EHCA_SECTSHIFT); 24620cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24630cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering entry = ehca_bmap->top[top]->dir[dir]->ent[idx]; 24640cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (ehca_bmap_valid(entry)) { 24650cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering offset = (unsigned long)caddr & (EHCA_SECTSIZE - 1); 24660cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return entry | offset; 24670cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } else 24680cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return EHCA_INVAL_ADDR; 24690cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 24700cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24710cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic int ehca_dma_mapping_error(struct ib_device *dev, u64 dma_addr) 24720cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 24730cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return dma_addr == EHCA_INVAL_ADDR; 24740cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 24750cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24760cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic u64 ehca_dma_map_single(struct ib_device *dev, void *cpu_addr, 24770cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering size_t size, enum dma_data_direction direction) 24780cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 24790cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (cpu_addr) 24800cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return ehca_map_vaddr(cpu_addr); 24810cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering else 24820cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return EHCA_INVAL_ADDR; 24830cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 24840cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24850cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic void ehca_dma_unmap_single(struct ib_device *dev, u64 addr, size_t size, 24860cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering enum dma_data_direction direction) 24870cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 24880cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering /* This is only a stub; nothing to be done here */ 24890cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 24900cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24910cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic u64 ehca_dma_map_page(struct ib_device *dev, struct page *page, 24920cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering unsigned long offset, size_t size, 24930cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering enum dma_data_direction direction) 24940cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 24950cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u64 addr; 24960cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 24970cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (offset + size > PAGE_SIZE) 24980cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return EHCA_INVAL_ADDR; 24990cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25000cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering addr = ehca_map_vaddr(page_address(page)); 25010cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (!ehca_dma_mapping_error(dev, addr)) 25020cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering addr += offset; 25030cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25040cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return addr; 25050cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 25060cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25070cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic void ehca_dma_unmap_page(struct ib_device *dev, u64 addr, size_t size, 25080cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering enum dma_data_direction direction) 25090cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 25100cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering /* This is only a stub; nothing to be done here */ 25110cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 25120cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25130cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic int ehca_dma_map_sg(struct ib_device *dev, struct scatterlist *sgl, 25140cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int nents, enum dma_data_direction direction) 25150cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 25160cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct scatterlist *sg; 25170cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int i; 25180cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25190cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering for_each_sg(sgl, sg, nents, i) { 25200cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u64 addr; 25210cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering addr = ehca_map_vaddr(sg_virt(sg)); 25220cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (ehca_dma_mapping_error(dev, addr)) 25230cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return 0; 25240cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25250cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering sg->dma_address = addr; 25260cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering sg->dma_length = sg->length; 25270cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 25280cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return nents; 25290cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 25300cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25310cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic void ehca_dma_unmap_sg(struct ib_device *dev, struct scatterlist *sg, 25320cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering int nents, enum dma_data_direction direction) 25330cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 25340cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering /* This is only a stub; nothing to be done here */ 25350cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 25360cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25370cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic void ehca_dma_sync_single_for_cpu(struct ib_device *dev, u64 addr, 25380cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering size_t size, 25390cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering enum dma_data_direction dir) 25400cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 25410cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering dma_sync_single_for_cpu(dev->dma_device, addr, size, dir); 25420cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 25430cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25440cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic void ehca_dma_sync_single_for_device(struct ib_device *dev, u64 addr, 25450cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering size_t size, 25460cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering enum dma_data_direction dir) 25470cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 25480cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering dma_sync_single_for_device(dev->dma_device, addr, size, dir); 25490cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 25500cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25510cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic void *ehca_dma_alloc_coherent(struct ib_device *dev, size_t size, 25520cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u64 *dma_handle, gfp_t flag) 25530cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 25540cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering struct page *p; 25550cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering void *addr = NULL; 25560cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering u64 dma_addr; 25570cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25580cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering p = alloc_pages(flag, get_order(size)); 25590cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (p) { 25600cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering addr = page_address(p); 25610cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering dma_addr = ehca_map_vaddr(addr); 25620cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (ehca_dma_mapping_error(dev, dma_addr)) { 25630cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering free_pages((unsigned long)addr, get_order(size)); 25640cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return NULL; 25650cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 25660cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (dma_handle) 25670cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering *dma_handle = dma_addr; 25680cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return addr; 25690cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering } 25700cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering return NULL; 25710cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 25720cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25730cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstatic void ehca_dma_free_coherent(struct ib_device *dev, size_t size, 25740cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering void *cpu_addr, u64 dma_handle) 25750cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering{ 25760cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering if (cpu_addr && size) 25770cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering free_pages((unsigned long)cpu_addr, get_order(size)); 25780cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering} 25790cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25800cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering 25810cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Heringstruct ib_dma_mapping_ops ehca_dma_mapping_ops = { 25820cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering .mapping_error = ehca_dma_mapping_error, 25830cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering .map_single = ehca_dma_map_single, 25840cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering .unmap_single = ehca_dma_unmap_single, 25850cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering .map_page = ehca_dma_map_page, 25860cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering .unmap_page = ehca_dma_unmap_page, 25870cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering .map_sg = ehca_dma_map_sg, 25880cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering .unmap_sg = ehca_dma_unmap_sg, 25890cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering .sync_single_for_cpu = ehca_dma_sync_single_for_cpu, 25900cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering .sync_single_for_device = ehca_dma_sync_single_for_device, 25910cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering .alloc_coherent = ehca_dma_alloc_coherent, 25920cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering .free_coherent = ehca_dma_free_coherent, 25930cf89dcdbc53f2b43e4ce7419b6ff47f4309c2ebHannes Hering}; 2594