1e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz/* 2e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. 3e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * 4e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * This software is available to you under a choice of one of two 5e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * licenses. You may choose to be licensed under the terms of the GNU 6e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * General Public License (GPL) Version 2, available from the file 7e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * COPYING in the main directory of this source tree, or the 8e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * OpenIB.org BSD license below: 9e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * 10e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * Redistribution and use in source and binary forms, with or 11e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * without modification, are permitted provided that the following 12e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * conditions are met: 13e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * 14e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * - Redistributions of source code must retain the above 15e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * copyright notice, this list of conditions and the following 16e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * disclaimer. 17e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * 18e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * - Redistributions in binary form must reproduce the above 19e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * copyright notice, this list of conditions and the following 20e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * disclaimer in the documentation and/or other materials 21e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * provided with the distribution. 22e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * 23e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * SOFTWARE. 31e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz */ 32e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz#include <linux/kernel.h> 33e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz#include <linux/slab.h> 34e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz#include <linux/mm.h> 35e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz#include <linux/scatterlist.h> 36e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz#include <linux/kfifo.h> 37e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz#include <scsi/scsi_cmnd.h> 38e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz#include <scsi/scsi_host.h> 39e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 40e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz#include "iscsi_iser.h" 41e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 42e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz/* Register user buffer memory and initialize passive rdma 43e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * dto descriptor. Total data size is stored in 442261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie * iser_task->data[ISER_DIR_IN].data_len 45e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz */ 462261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christiestatic int iser_prepare_read_cmd(struct iscsi_task *task, 47e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz unsigned int edtl) 48e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 49e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz{ 502261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iscsi_iser_task *iser_task = task->dd_data; 51e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz struct iser_regd_buf *regd_buf; 52e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz int err; 532261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iser_hdr *hdr = &iser_task->desc.iser_header; 542261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iser_data_buf *buf_in = &iser_task->data[ISER_DIR_IN]; 55e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 562261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie err = iser_dma_map_task_data(iser_task, 57e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz buf_in, 58e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz ISER_DIR_IN, 59e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz DMA_FROM_DEVICE); 60e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (err) 61e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return err; 62e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 632261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie if (edtl > iser_task->data[ISER_DIR_IN].data_len) { 64e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_err("Total data length: %ld, less than EDTL: " 65e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz "%d, in READ cmd BHS itt: %d, conn: 0x%p\n", 662261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_task->data[ISER_DIR_IN].data_len, edtl, 672261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie task->itt, iser_task->iser_conn); 68e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return -EINVAL; 69e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 70e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 712261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie err = iser_reg_rdma_mem(iser_task,ISER_DIR_IN); 72e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (err) { 73e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_err("Failed to set up Data-IN RDMA\n"); 74e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return err; 75e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 762261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie regd_buf = &iser_task->rdma_regd[ISER_DIR_IN]; 77e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 78e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz hdr->flags |= ISER_RSV; 79e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz hdr->read_stag = cpu_to_be32(regd_buf->reg.rkey); 80e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz hdr->read_va = cpu_to_be64(regd_buf->reg.va); 81e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 82e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n", 832261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie task->itt, regd_buf->reg.rkey, 84e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz (unsigned long long)regd_buf->reg.va); 85e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 86e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return 0; 87e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz} 88e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 89e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz/* Register user buffer memory and initialize passive rdma 90e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * dto descriptor. Total data size is stored in 912261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie * task->data[ISER_DIR_OUT].data_len 92e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz */ 93e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitzstatic int 942261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christieiser_prepare_write_cmd(struct iscsi_task *task, 95e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz unsigned int imm_sz, 96e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz unsigned int unsol_sz, 97e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz unsigned int edtl) 98e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz{ 992261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iscsi_iser_task *iser_task = task->dd_data; 100e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz struct iser_regd_buf *regd_buf; 101e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz int err; 1022261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iser_hdr *hdr = &iser_task->desc.iser_header; 1032261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT]; 104f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz struct ib_sge *tx_dsg = &iser_task->desc.tx_sg[1]; 105e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 1062261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie err = iser_dma_map_task_data(iser_task, 107e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz buf_out, 108e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz ISER_DIR_OUT, 109e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz DMA_TO_DEVICE); 110e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (err) 111e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return err; 112e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 1132261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie if (edtl > iser_task->data[ISER_DIR_OUT].data_len) { 114e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_err("Total data length: %ld, less than EDTL: %d, " 115e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz "in WRITE cmd BHS itt: %d, conn: 0x%p\n", 1162261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_task->data[ISER_DIR_OUT].data_len, 1172261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie edtl, task->itt, task->conn); 118e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return -EINVAL; 119e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 120e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 1212261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie err = iser_reg_rdma_mem(iser_task,ISER_DIR_OUT); 122e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (err != 0) { 123e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_err("Failed to register write cmd RDMA mem\n"); 124e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return err; 125e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 126e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 1272261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT]; 128e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 129e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (unsol_sz < edtl) { 130e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz hdr->flags |= ISER_WSV; 131e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz hdr->write_stag = cpu_to_be32(regd_buf->reg.rkey); 132e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz hdr->write_va = cpu_to_be64(regd_buf->reg.va + unsol_sz); 133e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 134e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X " 135e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz "VA:%#llX + unsol:%d\n", 1362261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie task->itt, regd_buf->reg.rkey, 137e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz (unsigned long long)regd_buf->reg.va, unsol_sz); 138e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 139e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 140e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (imm_sz > 0) { 141e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n", 1422261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie task->itt, imm_sz); 143f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_dsg->addr = regd_buf->reg.va; 144f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_dsg->length = imm_sz; 145f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_dsg->lkey = regd_buf->reg.lkey; 146f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz iser_task->desc.num_sge = 2; 147e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 148e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 149e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return 0; 150e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz} 151e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 152e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz/* creates a new tx descriptor and adds header regd buffer */ 153f19624aa92003969ba822cd3c552800965aa530bOr Gerlitzstatic void iser_create_send_desc(struct iser_conn *ib_conn, 154f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz struct iser_tx_desc *tx_desc) 155e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz{ 156f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz struct iser_device *device = ib_conn->device; 157e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 158f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz ib_dma_sync_single_for_cpu(device->ib_device, 159f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); 160e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 161e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 162e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz tx_desc->iser_header.flags = ISER_VER; 163e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 164f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_desc->num_sge = 1; 165f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz 166f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz if (tx_desc->tx_sg[0].lkey != device->mr->lkey) { 167f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_desc->tx_sg[0].lkey = device->mr->lkey; 168f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz iser_dbg("sdesc %p lkey mismatch, fixing\n", tx_desc); 169f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz } 170e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz} 171e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 172f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz 17389e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitzint iser_alloc_rx_descriptors(struct iser_conn *ib_conn) 174bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz{ 175bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz int i, j; 176bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz u64 dma_addr; 177bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz struct iser_rx_desc *rx_desc; 178bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz struct ib_sge *rx_sg; 179bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz struct iser_device *device = ib_conn->device; 180bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 181bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz ib_conn->rx_descs = kmalloc(ISER_QP_MAX_RECV_DTOS * 182bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz sizeof(struct iser_rx_desc), GFP_KERNEL); 183bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz if (!ib_conn->rx_descs) 184bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz goto rx_desc_alloc_fail; 185bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 186bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_desc = ib_conn->rx_descs; 187bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 188bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz for (i = 0; i < ISER_QP_MAX_RECV_DTOS; i++, rx_desc++) { 189bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc, 190bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 191bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz if (ib_dma_mapping_error(device->ib_device, dma_addr)) 192bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz goto rx_desc_dma_map_failed; 193bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 194bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_desc->dma_addr = dma_addr; 195bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 196bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_sg = &rx_desc->rx_sg; 197bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_sg->addr = rx_desc->dma_addr; 198bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_sg->length = ISER_RX_PAYLOAD_SIZE; 199bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_sg->lkey = device->mr->lkey; 200bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz } 201bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 202bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz ib_conn->rx_desc_head = 0; 203bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz return 0; 204bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 205bcc60c381d857ced653e912cbe6121294773e147Or Gerlitzrx_desc_dma_map_failed: 206bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_desc = ib_conn->rx_descs; 207bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz for (j = 0; j < i; j++, rx_desc++) 208bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 209bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 210bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz kfree(ib_conn->rx_descs); 211bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz ib_conn->rx_descs = NULL; 212bcc60c381d857ced653e912cbe6121294773e147Or Gerlitzrx_desc_alloc_fail: 213bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz iser_err("failed allocating rx descriptors / data buffers\n"); 214bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz return -ENOMEM; 215bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz} 216bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 217bcc60c381d857ced653e912cbe6121294773e147Or Gerlitzvoid iser_free_rx_descriptors(struct iser_conn *ib_conn) 218bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz{ 219bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz int i; 220bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz struct iser_rx_desc *rx_desc; 221bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz struct iser_device *device = ib_conn->device; 222bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 223bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz if (!ib_conn->rx_descs) 224bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz return; 225bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 226bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_desc = ib_conn->rx_descs; 227bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz for (i = 0; i < ISER_QP_MAX_RECV_DTOS; i++, rx_desc++) 228bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 229bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 230bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz kfree(ib_conn->rx_descs); 231bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz} 232bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 23389e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitzstatic int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req) 234e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz{ 235e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz struct iscsi_iser_conn *iser_conn = conn->dd_data; 236e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 23789e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz iser_dbg("req op %x flags %x\n", req->opcode, req->flags); 23889e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz /* check if this is the last login - going to full feature phase */ 23989e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz if ((req->flags & ISCSI_FULL_FEATURE_PHASE) != ISCSI_FULL_FEATURE_PHASE) 24089e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz return 0; 241e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 24289e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz /* 24389e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz * Check that there is one posted recv buffer (for the last login 24489e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz * response) and no posted send buffers left - they must have been 24589e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz * consumed during previous login phases. 24689e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz */ 24789e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz WARN_ON(iser_conn->ib_conn->post_recv_buf_count != 1); 24889e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz WARN_ON(atomic_read(&iser_conn->ib_conn->post_send_buf_count) != 0); 249bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 25089e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz iser_dbg("Initially post: %d\n", ISER_MIN_POSTED_RX); 251e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz /* Initial post receive buffers */ 252bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz if (iser_post_recvm(iser_conn->ib_conn, ISER_MIN_POSTED_RX)) 253bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz return -ENOMEM; 254bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 255e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return 0; 256e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz} 257e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 258e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz/** 259e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * iser_send_command - send command PDU 260e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz */ 2612747fdb25726caa1a89229f43d99ca50af72576aMike Christieint iser_send_command(struct iscsi_conn *conn, 2622261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iscsi_task *task) 263e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz{ 264e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz struct iscsi_iser_conn *iser_conn = conn->dd_data; 2652261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iscsi_iser_task *iser_task = task->dd_data; 266e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz unsigned long edtl; 267bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz int err; 268e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz struct iser_data_buf *data_buf; 269123521830c0ea35055b900d2ff0b73bb129e08cbNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr; 2702261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct scsi_cmnd *sc = task->sc; 271f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz struct iser_tx_desc *tx_desc = &iser_task->desc; 272e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 273e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz edtl = ntohl(hdr->data_length); 274e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 275e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz /* build the tx desc regd header and add it to the tx desc dto */ 276f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_desc->type = ISCSI_TX_SCSI_COMMAND; 277f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz iser_create_send_desc(iser_conn->ib_conn, tx_desc); 278e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 279e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (hdr->flags & ISCSI_FLAG_CMD_READ) 2802261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie data_buf = &iser_task->data[ISER_DIR_IN]; 281e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz else 2822261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie data_buf = &iser_task->data[ISER_DIR_OUT]; 283e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 284da9c0c770e775e655e3f77c96d91ee557b117adbFUJITA Tomonori if (scsi_sg_count(sc)) { /* using a scatter list */ 285da9c0c770e775e655e3f77c96d91ee557b117adbFUJITA Tomonori data_buf->buf = scsi_sglist(sc); 286da9c0c770e775e655e3f77c96d91ee557b117adbFUJITA Tomonori data_buf->size = scsi_sg_count(sc); 287e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 288e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 289da9c0c770e775e655e3f77c96d91ee557b117adbFUJITA Tomonori data_buf->data_len = scsi_bufflen(sc); 290e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 291e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (hdr->flags & ISCSI_FLAG_CMD_READ) { 2922261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie err = iser_prepare_read_cmd(task, edtl); 293e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (err) 294e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz goto send_command_error; 295e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 296e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (hdr->flags & ISCSI_FLAG_CMD_WRITE) { 2972261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie err = iser_prepare_write_cmd(task, 2982261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie task->imm_count, 2992261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie task->imm_count + 3000f9c7449ce050759d10424048b96d1bd0d59dcc1Mike Christie task->unsol_r2t.data_length, 301e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz edtl); 302e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (err) 303e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz goto send_command_error; 304e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 305e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 3062261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_task->status = ISER_TASK_STATUS_STARTED; 307e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 308f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz err = iser_post_send(iser_conn->ib_conn, tx_desc); 309e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (!err) 310e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return 0; 311e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 312e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitzsend_command_error: 3132261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_err("conn %p failed task->itt %d err %d\n",conn, task->itt, err); 314e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return err; 315e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz} 316e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 317e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz/** 318e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * iser_send_data_out - send data out PDU 319e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz */ 3202747fdb25726caa1a89229f43d99ca50af72576aMike Christieint iser_send_data_out(struct iscsi_conn *conn, 3212261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iscsi_task *task, 322e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz struct iscsi_data *hdr) 323e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz{ 324e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz struct iscsi_iser_conn *iser_conn = conn->dd_data; 3252261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iscsi_iser_task *iser_task = task->dd_data; 326f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz struct iser_tx_desc *tx_desc = NULL; 327f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz struct iser_regd_buf *regd_buf; 328e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz unsigned long buf_offset; 329e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz unsigned long data_seg_len; 3300a22ab92f51478796d5f3997f4f5922409c98b10Erez Zilber uint32_t itt; 331e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz int err = 0; 332f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz struct ib_sge *tx_dsg; 333f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz 3340a22ab92f51478796d5f3997f4f5922409c98b10Erez Zilber itt = (__force uint32_t)hdr->itt; 335e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz data_seg_len = ntoh24(hdr->dlength); 336e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz buf_offset = ntohl(hdr->offset); 337e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 338e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_dbg("%s itt %d dseg_len %d offset %d\n", 339e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz __func__,(int)itt,(int)data_seg_len,(int)buf_offset); 340e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 341528f4e8c8341706a354ff96daf615e678e9b296fOr Gerlitz tx_desc = kmem_cache_zalloc(ig.desc_cache, GFP_ATOMIC); 342e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (tx_desc == NULL) { 343e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_err("Failed to alloc desc for post dataout\n"); 344e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return -ENOMEM; 345e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 346e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 347e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz tx_desc->type = ISCSI_TX_DATAOUT; 348f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_desc->iser_header.flags = ISER_VER; 349e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr)); 350e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 351f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz /* build the tx desc */ 352f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz iser_initialize_task_headers(task, tx_desc); 353e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 354f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT]; 355f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_dsg = &tx_desc->tx_sg[1]; 356f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_dsg->addr = regd_buf->reg.va + buf_offset; 357f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_dsg->length = data_seg_len; 358f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_dsg->lkey = regd_buf->reg.lkey; 359f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_desc->num_sge = 2; 360e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 3612261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) { 362e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_err("Offset:%ld & DSL:%ld in Data-Out " 363e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz "inconsistent with total len:%ld, itt:%d\n", 364e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz buf_offset, data_seg_len, 3652261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_task->data[ISER_DIR_OUT].data_len, itt); 366e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz err = -EINVAL; 367e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz goto send_data_out_error; 368e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 369e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n", 370e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz itt, buf_offset, data_seg_len); 371e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 372e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 373f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz err = iser_post_send(iser_conn->ib_conn, tx_desc); 374e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (!err) 375e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return 0; 376e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 377e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitzsend_data_out_error: 378e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz kmem_cache_free(ig.desc_cache, tx_desc); 379e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_err("conn %p failed err %d\n",conn, err); 380e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return err; 381e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz} 382e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 383e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitzint iser_send_control(struct iscsi_conn *conn, 3842261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iscsi_task *task) 385e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz{ 386e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz struct iscsi_iser_conn *iser_conn = conn->dd_data; 3872261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iscsi_iser_task *iser_task = task->dd_data; 388f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz struct iser_tx_desc *mdesc = &iser_task->desc; 389e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz unsigned long data_seg_len; 390f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz int err = 0; 391e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz struct iser_device *device; 3922c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz struct iser_conn *ib_conn = iser_conn->ib_conn; 393e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 394e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz /* build the tx desc regd header and add it to the tx desc dto */ 395e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz mdesc->type = ISCSI_TX_CONTROL; 396f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz iser_create_send_desc(iser_conn->ib_conn, mdesc); 397e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 398e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz device = iser_conn->ib_conn->device; 399e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 4002261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie data_seg_len = ntoh24(task->hdr->dlength); 401e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 402e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (data_seg_len > 0) { 403f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz struct ib_sge *tx_dsg = &mdesc->tx_sg[1]; 404f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz if (task != conn->login_task) { 405f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz iser_err("data present on non login task!!!\n"); 406f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz goto send_control_error; 407f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz } 4082c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz 4092c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz ib_dma_sync_single_for_cpu(device->ib_device, 4102c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz ib_conn->login_req_dma, task->data_count, 4112c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz DMA_TO_DEVICE); 4122c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz 4132c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz memcpy(iser_conn->ib_conn->login_req_buf, task->data, 414f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz task->data_count); 4152c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz 4162c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz ib_dma_sync_single_for_device(device->ib_device, 4172c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz ib_conn->login_req_dma, task->data_count, 4182c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz DMA_TO_DEVICE); 4192c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz 4202c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz tx_dsg->addr = iser_conn->ib_conn->login_req_dma; 421200ae1a08bec8f3fedfcfe94c892d9a024db4e46Or Gerlitz tx_dsg->length = task->data_count; 422f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz tx_dsg->lkey = device->mr->lkey; 423f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz mdesc->num_sge = 2; 424e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 425e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 426bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz if (task == conn->login_task) { 427bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz err = iser_post_recvl(iser_conn->ib_conn); 428bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz if (err) 429bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz goto send_control_error; 43089e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz err = iser_post_rx_bufs(conn, task->hdr); 43189e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz if (err) 43289e984e2c2cd14f77ccb26c47726ac7f13b70ae8Or Gerlitz goto send_control_error; 433e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 434e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 435f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz err = iser_post_send(iser_conn->ib_conn, mdesc); 436e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (!err) 437e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return 0; 438e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 439e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitzsend_control_error: 440e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz iser_err("conn %p failed err %d\n",conn, err); 441e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz return err; 442e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz} 443e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 444e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz/** 445e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * iser_rcv_dto_completion - recv DTO completion 446e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz */ 447bcc60c381d857ced653e912cbe6121294773e147Or Gerlitzvoid iser_rcv_completion(struct iser_rx_desc *rx_desc, 448bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz unsigned long rx_xfer_len, 449bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz struct iser_conn *ib_conn) 450e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz{ 451bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz struct iscsi_iser_conn *conn = ib_conn->iser_conn; 452e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz struct iscsi_hdr *hdr; 453bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz u64 rx_dma; 454bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz int rx_buflen, outstanding, count, err; 455bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 456bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz /* differentiate between login to all other PDUs */ 4572c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz if ((char *)rx_desc == ib_conn->login_resp_buf) { 4582c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz rx_dma = ib_conn->login_resp_dma; 459bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_buflen = ISER_RX_LOGIN_SIZE; 460bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz } else { 461bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_dma = rx_desc->dma_addr; 462bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_buflen = ISER_RX_PAYLOAD_SIZE; 463bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz } 464e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 465bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma, 466bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_buflen, DMA_FROM_DEVICE); 467e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 468bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz hdr = &rx_desc->iscsi_header; 469e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 470bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, 471bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz hdr->itt, (int)(rx_xfer_len - ISER_HEADERS_LEN)); 472e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 473bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz iscsi_iser_recv(conn->iscsi_conn, hdr, 474bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_desc->data, rx_xfer_len - ISER_HEADERS_LEN); 475e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 476bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma, 477bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz rx_buflen, DMA_FROM_DEVICE); 478e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 479e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz /* decrementing conn->post_recv_buf_count only --after-- freeing the * 480e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * task eliminates the need to worry on tasks which are completed in * 481e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * parallel to the execution of iser_conn_term. So the code that waits * 482e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * for the posted rx bufs refcount to become zero handles everything */ 483704315f082d473b34047817f0a6a01924f38501eOr Gerlitz conn->ib_conn->post_recv_buf_count--; 484bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 4852c4ce609347f2a45792c8d9ebb5af11217766cb6Or Gerlitz if (rx_dma == ib_conn->login_resp_dma) 486bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz return; 487bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz 488704315f082d473b34047817f0a6a01924f38501eOr Gerlitz outstanding = ib_conn->post_recv_buf_count; 489bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz if (outstanding + ISER_MIN_POSTED_RX <= ISER_QP_MAX_RECV_DTOS) { 490bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz count = min(ISER_QP_MAX_RECV_DTOS - outstanding, 491bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz ISER_MIN_POSTED_RX); 492bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz err = iser_post_recvm(ib_conn, count); 493bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz if (err) 494bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz iser_err("posting %d rx bufs err %d\n", count, err); 495bcc60c381d857ced653e912cbe6121294773e147Or Gerlitz } 496e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz} 497e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 498f19624aa92003969ba822cd3c552800965aa530bOr Gerlitzvoid iser_snd_completion(struct iser_tx_desc *tx_desc, 499f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz struct iser_conn *ib_conn) 500e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz{ 5012261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie struct iscsi_task *task; 502f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz struct iser_device *device = ib_conn->device; 503e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 504f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz if (tx_desc->type == ISCSI_TX_DATAOUT) { 505f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr, 506f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz ISER_HEADERS_LEN, DMA_TO_DEVICE); 507e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz kmem_cache_free(ig.desc_cache, tx_desc); 508f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz } 509e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 51087e8df7a273c7c1acb864c90b5253609c44375a6Erez Zilber atomic_dec(&ib_conn->post_send_buf_count); 511e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 512e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz if (tx_desc->type == ISCSI_TX_CONTROL) { 513e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz /* this arithmetic is legal by libiscsi dd_data allocation */ 5142261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie task = (void *) ((long)(void *)tx_desc - 5152261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie sizeof(struct iscsi_task)); 5162261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie if (task->hdr->itt == RESERVED_ITT) 5172261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iscsi_put_task(task); 518e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 519e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz} 520e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 5212261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christievoid iser_task_rdma_init(struct iscsi_iser_task *iser_task) 522e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 523e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz{ 5242261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_task->status = ISER_TASK_STATUS_INIT; 525e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 5262261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_task->dir[ISER_DIR_IN] = 0; 5272261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_task->dir[ISER_DIR_OUT] = 0; 528e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 5292261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_task->data[ISER_DIR_IN].data_len = 0; 5302261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_task->data[ISER_DIR_OUT].data_len = 0; 531e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 5322261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie memset(&iser_task->rdma_regd[ISER_DIR_IN], 0, 533e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz sizeof(struct iser_regd_buf)); 5342261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie memset(&iser_task->rdma_regd[ISER_DIR_OUT], 0, 535e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz sizeof(struct iser_regd_buf)); 536e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz} 537e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 5382261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christievoid iser_task_rdma_finalize(struct iscsi_iser_task *iser_task) 539e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz{ 54074a2078061409e027ccb51a28cf6174c31ab8f99Erez Zilber int is_rdma_aligned = 1; 5413104a2175dc04b7a597acea90f19b033abcfc7d8Erez Zilber struct iser_regd_buf *regd; 542e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 543e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz /* if we were reading, copy back to unaligned sglist, 544e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz * anyway dma_unmap and free the copy 545e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz */ 5462261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie if (iser_task->data_copy[ISER_DIR_IN].copy_buf != NULL) { 54774a2078061409e027ccb51a28cf6174c31ab8f99Erez Zilber is_rdma_aligned = 0; 5482261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_IN); 54974a2078061409e027ccb51a28cf6174c31ab8f99Erez Zilber } 5502261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie if (iser_task->data_copy[ISER_DIR_OUT].copy_buf != NULL) { 55174a2078061409e027ccb51a28cf6174c31ab8f99Erez Zilber is_rdma_aligned = 0; 5522261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_OUT); 55374a2078061409e027ccb51a28cf6174c31ab8f99Erez Zilber } 554e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 5552261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie if (iser_task->dir[ISER_DIR_IN]) { 5562261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie regd = &iser_task->rdma_regd[ISER_DIR_IN]; 557f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz if (regd->reg.is_fmr) 558f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz iser_unreg_mem(®d->reg); 559e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 560e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 5612261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie if (iser_task->dir[ISER_DIR_OUT]) { 5622261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie regd = &iser_task->rdma_regd[ISER_DIR_OUT]; 563f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz if (regd->reg.is_fmr) 564f19624aa92003969ba822cd3c552800965aa530bOr Gerlitz iser_unreg_mem(®d->reg); 565e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz } 566e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz 56774a2078061409e027ccb51a28cf6174c31ab8f99Erez Zilber /* if the data was unaligned, it was already unmapped and then copied */ 56874a2078061409e027ccb51a28cf6174c31ab8f99Erez Zilber if (is_rdma_aligned) 5692261ec3d686e35c1a6088ab7f00a1d02b528b994Mike Christie iser_dma_unmap_task_data(iser_task); 570e85b24b5e7de9f507c6253183d089370f37618c5Or Gerlitz} 571