be_main.c revision 457ff3b7dc3796d8778286217ad304ff122e948f
1/** 2 * Copyright (C) 2005 - 2010 ServerEngines 3 * All rights reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License version 2 7 * as published by the Free Software Foundation. The full GNU General 8 * Public License is included in this distribution in the file called COPYING. 9 * 10 * Written by: Jayamohan Kallickal (jayamohank@serverengines.com) 11 * 12 * Contact Information: 13 * linux-drivers@serverengines.com 14 * 15 * ServerEngines 16 * 209 N. Fair Oaks Ave 17 * Sunnyvale, CA 94085 18 * 19 */ 20#include <linux/reboot.h> 21#include <linux/delay.h> 22#include <linux/slab.h> 23#include <linux/interrupt.h> 24#include <linux/blkdev.h> 25#include <linux/pci.h> 26#include <linux/string.h> 27#include <linux/kernel.h> 28#include <linux/semaphore.h> 29 30#include <scsi/libiscsi.h> 31#include <scsi/scsi_transport_iscsi.h> 32#include <scsi/scsi_transport.h> 33#include <scsi/scsi_cmnd.h> 34#include <scsi/scsi_device.h> 35#include <scsi/scsi_host.h> 36#include <scsi/scsi.h> 37#include "be_main.h" 38#include "be_iscsi.h" 39#include "be_mgmt.h" 40 41static unsigned int be_iopoll_budget = 10; 42static unsigned int be_max_phys_size = 64; 43static unsigned int enable_msix = 1; 44 45MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table); 46MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR); 47MODULE_AUTHOR("ServerEngines Corporation"); 48MODULE_LICENSE("GPL"); 49module_param(be_iopoll_budget, int, 0); 50module_param(enable_msix, int, 0); 51module_param(be_max_phys_size, uint, S_IRUGO); 52MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically" 53 "contiguous memory that can be allocated." 54 "Range is 16 - 128"); 55 56static int beiscsi_slave_configure(struct scsi_device *sdev) 57{ 58 blk_queue_max_segment_size(sdev->request_queue, 65536); 59 return 0; 60} 61 62static int beiscsi_eh_abort(struct scsi_cmnd *sc) 63{ 64 struct iscsi_cls_session *cls_session; 65 struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr; 66 struct beiscsi_io_task *aborted_io_task; 67 struct iscsi_conn *conn; 68 struct beiscsi_conn *beiscsi_conn; 69 struct beiscsi_hba *phba; 70 struct iscsi_session *session; 71 struct invalidate_command_table *inv_tbl; 72 unsigned int cid, tag, num_invalidate; 73 74 cls_session = starget_to_session(scsi_target(sc->device)); 75 session = cls_session->dd_data; 76 77 spin_lock_bh(&session->lock); 78 if (!aborted_task || !aborted_task->sc) { 79 /* we raced */ 80 spin_unlock_bh(&session->lock); 81 return SUCCESS; 82 } 83 84 aborted_io_task = aborted_task->dd_data; 85 if (!aborted_io_task->scsi_cmnd) { 86 /* raced or invalid command */ 87 spin_unlock_bh(&session->lock); 88 return SUCCESS; 89 } 90 spin_unlock_bh(&session->lock); 91 conn = aborted_task->conn; 92 beiscsi_conn = conn->dd_data; 93 phba = beiscsi_conn->phba; 94 95 /* invalidate iocb */ 96 cid = beiscsi_conn->beiscsi_conn_cid; 97 inv_tbl = phba->inv_tbl; 98 memset(inv_tbl, 0x0, sizeof(*inv_tbl)); 99 inv_tbl->cid = cid; 100 inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index; 101 num_invalidate = 1; 102 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, cid); 103 if (!tag) { 104 shost_printk(KERN_WARNING, phba->shost, 105 "mgmt_invalidate_icds could not be" 106 " submitted\n"); 107 return FAILED; 108 } else { 109 wait_event_interruptible(phba->ctrl.mcc_wait[tag], 110 phba->ctrl.mcc_numtag[tag]); 111 free_mcc_tag(&phba->ctrl, tag); 112 } 113 114 return iscsi_eh_abort(sc); 115} 116 117static int beiscsi_eh_device_reset(struct scsi_cmnd *sc) 118{ 119 struct iscsi_task *abrt_task; 120 struct beiscsi_io_task *abrt_io_task; 121 struct iscsi_conn *conn; 122 struct beiscsi_conn *beiscsi_conn; 123 struct beiscsi_hba *phba; 124 struct iscsi_session *session; 125 struct iscsi_cls_session *cls_session; 126 struct invalidate_command_table *inv_tbl; 127 unsigned int cid, tag, i, num_invalidate; 128 int rc = FAILED; 129 130 /* invalidate iocbs */ 131 cls_session = starget_to_session(scsi_target(sc->device)); 132 session = cls_session->dd_data; 133 spin_lock_bh(&session->lock); 134 if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) 135 goto unlock; 136 137 conn = session->leadconn; 138 beiscsi_conn = conn->dd_data; 139 phba = beiscsi_conn->phba; 140 cid = beiscsi_conn->beiscsi_conn_cid; 141 inv_tbl = phba->inv_tbl; 142 memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN); 143 num_invalidate = 0; 144 for (i = 0; i < conn->session->cmds_max; i++) { 145 abrt_task = conn->session->cmds[i]; 146 abrt_io_task = abrt_task->dd_data; 147 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE) 148 continue; 149 150 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun) 151 continue; 152 153 inv_tbl->cid = cid; 154 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index; 155 num_invalidate++; 156 inv_tbl++; 157 } 158 spin_unlock_bh(&session->lock); 159 inv_tbl = phba->inv_tbl; 160 161 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, cid); 162 if (!tag) { 163 shost_printk(KERN_WARNING, phba->shost, 164 "mgmt_invalidate_icds could not be" 165 " submitted\n"); 166 return FAILED; 167 } else { 168 wait_event_interruptible(phba->ctrl.mcc_wait[tag], 169 phba->ctrl.mcc_numtag[tag]); 170 free_mcc_tag(&phba->ctrl, tag); 171 } 172 173 return iscsi_eh_device_reset(sc); 174unlock: 175 spin_unlock_bh(&session->lock); 176 return rc; 177} 178 179/*------------------- PCI Driver operations and data ----------------- */ 180static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = { 181 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, 182 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, 183 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, 184 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, 185 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) }, 186 { 0 } 187}; 188MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table); 189 190static struct scsi_host_template beiscsi_sht = { 191 .module = THIS_MODULE, 192 .name = "ServerEngines 10Gbe open-iscsi Initiator Driver", 193 .proc_name = DRV_NAME, 194 .queuecommand = iscsi_queuecommand, 195 .change_queue_depth = iscsi_change_queue_depth, 196 .slave_configure = beiscsi_slave_configure, 197 .target_alloc = iscsi_target_alloc, 198 .eh_abort_handler = beiscsi_eh_abort, 199 .eh_device_reset_handler = beiscsi_eh_device_reset, 200 .eh_target_reset_handler = iscsi_eh_session_reset, 201 .sg_tablesize = BEISCSI_SGLIST_ELEMENTS, 202 .can_queue = BE2_IO_DEPTH, 203 .this_id = -1, 204 .max_sectors = BEISCSI_MAX_SECTORS, 205 .cmd_per_lun = BEISCSI_CMD_PER_LUN, 206 .use_clustering = ENABLE_CLUSTERING, 207}; 208 209static struct scsi_transport_template *beiscsi_scsi_transport; 210 211static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev) 212{ 213 struct beiscsi_hba *phba; 214 struct Scsi_Host *shost; 215 216 shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0); 217 if (!shost) { 218 dev_err(&pcidev->dev, "beiscsi_hba_alloc -" 219 "iscsi_host_alloc failed\n"); 220 return NULL; 221 } 222 shost->dma_boundary = pcidev->dma_mask; 223 shost->max_id = BE2_MAX_SESSIONS; 224 shost->max_channel = 0; 225 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN; 226 shost->max_lun = BEISCSI_NUM_MAX_LUN; 227 shost->transportt = beiscsi_scsi_transport; 228 phba = iscsi_host_priv(shost); 229 memset(phba, 0, sizeof(*phba)); 230 phba->shost = shost; 231 phba->pcidev = pci_dev_get(pcidev); 232 pci_set_drvdata(pcidev, phba); 233 234 if (iscsi_host_add(shost, &phba->pcidev->dev)) 235 goto free_devices; 236 return phba; 237 238free_devices: 239 pci_dev_put(phba->pcidev); 240 iscsi_host_free(phba->shost); 241 return NULL; 242} 243 244static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba) 245{ 246 if (phba->csr_va) { 247 iounmap(phba->csr_va); 248 phba->csr_va = NULL; 249 } 250 if (phba->db_va) { 251 iounmap(phba->db_va); 252 phba->db_va = NULL; 253 } 254 if (phba->pci_va) { 255 iounmap(phba->pci_va); 256 phba->pci_va = NULL; 257 } 258} 259 260static int beiscsi_map_pci_bars(struct beiscsi_hba *phba, 261 struct pci_dev *pcidev) 262{ 263 u8 __iomem *addr; 264 int pcicfg_reg; 265 266 addr = ioremap_nocache(pci_resource_start(pcidev, 2), 267 pci_resource_len(pcidev, 2)); 268 if (addr == NULL) 269 return -ENOMEM; 270 phba->ctrl.csr = addr; 271 phba->csr_va = addr; 272 phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2); 273 274 addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024); 275 if (addr == NULL) 276 goto pci_map_err; 277 phba->ctrl.db = addr; 278 phba->db_va = addr; 279 phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4); 280 281 if (phba->generation == BE_GEN2) 282 pcicfg_reg = 1; 283 else 284 pcicfg_reg = 0; 285 286 addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg), 287 pci_resource_len(pcidev, pcicfg_reg)); 288 289 if (addr == NULL) 290 goto pci_map_err; 291 phba->ctrl.pcicfg = addr; 292 phba->pci_va = addr; 293 phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg); 294 return 0; 295 296pci_map_err: 297 beiscsi_unmap_pci_function(phba); 298 return -ENOMEM; 299} 300 301static int beiscsi_enable_pci(struct pci_dev *pcidev) 302{ 303 int ret; 304 305 ret = pci_enable_device(pcidev); 306 if (ret) { 307 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device " 308 "failed. Returning -ENODEV\n"); 309 return ret; 310 } 311 312 pci_set_master(pcidev); 313 if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) { 314 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32)); 315 if (ret) { 316 dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n"); 317 pci_disable_device(pcidev); 318 return ret; 319 } 320 } 321 return 0; 322} 323 324static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev) 325{ 326 struct be_ctrl_info *ctrl = &phba->ctrl; 327 struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced; 328 struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem; 329 int status = 0; 330 331 ctrl->pdev = pdev; 332 status = beiscsi_map_pci_bars(phba, pdev); 333 if (status) 334 return status; 335 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 336 mbox_mem_alloc->va = pci_alloc_consistent(pdev, 337 mbox_mem_alloc->size, 338 &mbox_mem_alloc->dma); 339 if (!mbox_mem_alloc->va) { 340 beiscsi_unmap_pci_function(phba); 341 status = -ENOMEM; 342 return status; 343 } 344 345 mbox_mem_align->size = sizeof(struct be_mcc_mailbox); 346 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); 347 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 348 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox)); 349 spin_lock_init(&ctrl->mbox_lock); 350 spin_lock_init(&phba->ctrl.mcc_lock); 351 spin_lock_init(&phba->ctrl.mcc_cq_lock); 352 353 return status; 354} 355 356static void beiscsi_get_params(struct beiscsi_hba *phba) 357{ 358 phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count 359 - (phba->fw_config.iscsi_cid_count 360 + BE2_TMFS 361 + BE2_NOPOUT_REQ)); 362 phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count; 363 phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2; 364 phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;; 365 phba->params.num_sge_per_io = BE2_SGE; 366 phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ; 367 phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ; 368 phba->params.eq_timer = 64; 369 phba->params.num_eq_entries = 370 (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2 371 + BE2_TMFS) / 512) + 1) * 512; 372 phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024) 373 ? 1024 : phba->params.num_eq_entries; 374 SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n", 375 phba->params.num_eq_entries); 376 phba->params.num_cq_entries = 377 (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2 378 + BE2_TMFS) / 512) + 1) * 512; 379 phba->params.wrbs_per_cxn = 256; 380} 381 382static void hwi_ring_eq_db(struct beiscsi_hba *phba, 383 unsigned int id, unsigned int clr_interrupt, 384 unsigned int num_processed, 385 unsigned char rearm, unsigned char event) 386{ 387 u32 val = 0; 388 val |= id & DB_EQ_RING_ID_MASK; 389 if (rearm) 390 val |= 1 << DB_EQ_REARM_SHIFT; 391 if (clr_interrupt) 392 val |= 1 << DB_EQ_CLR_SHIFT; 393 if (event) 394 val |= 1 << DB_EQ_EVNT_SHIFT; 395 val |= num_processed << DB_EQ_NUM_POPPED_SHIFT; 396 iowrite32(val, phba->db_va + DB_EQ_OFFSET); 397} 398 399/** 400 * be_isr_mcc - The isr routine of the driver. 401 * @irq: Not used 402 * @dev_id: Pointer to host adapter structure 403 */ 404static irqreturn_t be_isr_mcc(int irq, void *dev_id) 405{ 406 struct beiscsi_hba *phba; 407 struct be_eq_entry *eqe = NULL; 408 struct be_queue_info *eq; 409 struct be_queue_info *mcc; 410 unsigned int num_eq_processed; 411 struct be_eq_obj *pbe_eq; 412 unsigned long flags; 413 414 pbe_eq = dev_id; 415 eq = &pbe_eq->q; 416 phba = pbe_eq->phba; 417 mcc = &phba->ctrl.mcc_obj.cq; 418 eqe = queue_tail_node(eq); 419 if (!eqe) 420 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n"); 421 422 num_eq_processed = 0; 423 424 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 425 & EQE_VALID_MASK) { 426 if (((eqe->dw[offsetof(struct amap_eq_entry, 427 resource_id) / 32] & 428 EQE_RESID_MASK) >> 16) == mcc->id) { 429 spin_lock_irqsave(&phba->isr_lock, flags); 430 phba->todo_mcc_cq = 1; 431 spin_unlock_irqrestore(&phba->isr_lock, flags); 432 } 433 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 434 queue_tail_inc(eq); 435 eqe = queue_tail_node(eq); 436 num_eq_processed++; 437 } 438 if (phba->todo_mcc_cq) 439 queue_work(phba->wq, &phba->work_cqs); 440 if (num_eq_processed) 441 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1); 442 443 return IRQ_HANDLED; 444} 445 446/** 447 * be_isr_msix - The isr routine of the driver. 448 * @irq: Not used 449 * @dev_id: Pointer to host adapter structure 450 */ 451static irqreturn_t be_isr_msix(int irq, void *dev_id) 452{ 453 struct beiscsi_hba *phba; 454 struct be_eq_entry *eqe = NULL; 455 struct be_queue_info *eq; 456 struct be_queue_info *cq; 457 unsigned int num_eq_processed; 458 struct be_eq_obj *pbe_eq; 459 unsigned long flags; 460 461 pbe_eq = dev_id; 462 eq = &pbe_eq->q; 463 cq = pbe_eq->cq; 464 eqe = queue_tail_node(eq); 465 if (!eqe) 466 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n"); 467 468 phba = pbe_eq->phba; 469 num_eq_processed = 0; 470 if (blk_iopoll_enabled) { 471 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 472 & EQE_VALID_MASK) { 473 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) 474 blk_iopoll_sched(&pbe_eq->iopoll); 475 476 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 477 queue_tail_inc(eq); 478 eqe = queue_tail_node(eq); 479 num_eq_processed++; 480 } 481 if (num_eq_processed) 482 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1); 483 484 return IRQ_HANDLED; 485 } else { 486 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 487 & EQE_VALID_MASK) { 488 spin_lock_irqsave(&phba->isr_lock, flags); 489 phba->todo_cq = 1; 490 spin_unlock_irqrestore(&phba->isr_lock, flags); 491 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 492 queue_tail_inc(eq); 493 eqe = queue_tail_node(eq); 494 num_eq_processed++; 495 } 496 if (phba->todo_cq) 497 queue_work(phba->wq, &phba->work_cqs); 498 499 if (num_eq_processed) 500 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1); 501 502 return IRQ_HANDLED; 503 } 504} 505 506/** 507 * be_isr - The isr routine of the driver. 508 * @irq: Not used 509 * @dev_id: Pointer to host adapter structure 510 */ 511static irqreturn_t be_isr(int irq, void *dev_id) 512{ 513 struct beiscsi_hba *phba; 514 struct hwi_controller *phwi_ctrlr; 515 struct hwi_context_memory *phwi_context; 516 struct be_eq_entry *eqe = NULL; 517 struct be_queue_info *eq; 518 struct be_queue_info *cq; 519 struct be_queue_info *mcc; 520 unsigned long flags, index; 521 unsigned int num_mcceq_processed, num_ioeq_processed; 522 struct be_ctrl_info *ctrl; 523 struct be_eq_obj *pbe_eq; 524 int isr; 525 526 phba = dev_id; 527 ctrl = &phba->ctrl;; 528 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET + 529 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE)); 530 if (!isr) 531 return IRQ_NONE; 532 533 phwi_ctrlr = phba->phwi_ctrlr; 534 phwi_context = phwi_ctrlr->phwi_ctxt; 535 pbe_eq = &phwi_context->be_eq[0]; 536 537 eq = &phwi_context->be_eq[0].q; 538 mcc = &phba->ctrl.mcc_obj.cq; 539 index = 0; 540 eqe = queue_tail_node(eq); 541 if (!eqe) 542 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n"); 543 544 num_ioeq_processed = 0; 545 num_mcceq_processed = 0; 546 if (blk_iopoll_enabled) { 547 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 548 & EQE_VALID_MASK) { 549 if (((eqe->dw[offsetof(struct amap_eq_entry, 550 resource_id) / 32] & 551 EQE_RESID_MASK) >> 16) == mcc->id) { 552 spin_lock_irqsave(&phba->isr_lock, flags); 553 phba->todo_mcc_cq = 1; 554 spin_unlock_irqrestore(&phba->isr_lock, flags); 555 num_mcceq_processed++; 556 } else { 557 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) 558 blk_iopoll_sched(&pbe_eq->iopoll); 559 num_ioeq_processed++; 560 } 561 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 562 queue_tail_inc(eq); 563 eqe = queue_tail_node(eq); 564 } 565 if (num_ioeq_processed || num_mcceq_processed) { 566 if (phba->todo_mcc_cq) 567 queue_work(phba->wq, &phba->work_cqs); 568 569 if ((num_mcceq_processed) && (!num_ioeq_processed)) 570 hwi_ring_eq_db(phba, eq->id, 0, 571 (num_ioeq_processed + 572 num_mcceq_processed) , 1, 1); 573 else 574 hwi_ring_eq_db(phba, eq->id, 0, 575 (num_ioeq_processed + 576 num_mcceq_processed), 0, 1); 577 578 return IRQ_HANDLED; 579 } else 580 return IRQ_NONE; 581 } else { 582 cq = &phwi_context->be_cq[0]; 583 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 584 & EQE_VALID_MASK) { 585 586 if (((eqe->dw[offsetof(struct amap_eq_entry, 587 resource_id) / 32] & 588 EQE_RESID_MASK) >> 16) != cq->id) { 589 spin_lock_irqsave(&phba->isr_lock, flags); 590 phba->todo_mcc_cq = 1; 591 spin_unlock_irqrestore(&phba->isr_lock, flags); 592 } else { 593 spin_lock_irqsave(&phba->isr_lock, flags); 594 phba->todo_cq = 1; 595 spin_unlock_irqrestore(&phba->isr_lock, flags); 596 } 597 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 598 queue_tail_inc(eq); 599 eqe = queue_tail_node(eq); 600 num_ioeq_processed++; 601 } 602 if (phba->todo_cq || phba->todo_mcc_cq) 603 queue_work(phba->wq, &phba->work_cqs); 604 605 if (num_ioeq_processed) { 606 hwi_ring_eq_db(phba, eq->id, 0, 607 num_ioeq_processed, 1, 1); 608 return IRQ_HANDLED; 609 } else 610 return IRQ_NONE; 611 } 612} 613 614static int beiscsi_init_irqs(struct beiscsi_hba *phba) 615{ 616 struct pci_dev *pcidev = phba->pcidev; 617 struct hwi_controller *phwi_ctrlr; 618 struct hwi_context_memory *phwi_context; 619 int ret, msix_vec, i = 0; 620 char desc[32]; 621 622 phwi_ctrlr = phba->phwi_ctrlr; 623 phwi_context = phwi_ctrlr->phwi_ctxt; 624 625 if (phba->msix_enabled) { 626 for (i = 0; i < phba->num_cpus; i++) { 627 sprintf(desc, "beiscsi_msix_%04x", i); 628 msix_vec = phba->msix_entries[i].vector; 629 ret = request_irq(msix_vec, be_isr_msix, 0, desc, 630 &phwi_context->be_eq[i]); 631 } 632 msix_vec = phba->msix_entries[i].vector; 633 ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc", 634 &phwi_context->be_eq[i]); 635 } else { 636 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED, 637 "beiscsi", phba); 638 if (ret) { 639 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-" 640 "Failed to register irq\\n"); 641 return ret; 642 } 643 } 644 return 0; 645} 646 647static void hwi_ring_cq_db(struct beiscsi_hba *phba, 648 unsigned int id, unsigned int num_processed, 649 unsigned char rearm, unsigned char event) 650{ 651 u32 val = 0; 652 val |= id & DB_CQ_RING_ID_MASK; 653 if (rearm) 654 val |= 1 << DB_CQ_REARM_SHIFT; 655 val |= num_processed << DB_CQ_NUM_POPPED_SHIFT; 656 iowrite32(val, phba->db_va + DB_CQ_OFFSET); 657} 658 659static unsigned int 660beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn, 661 struct beiscsi_hba *phba, 662 unsigned short cid, 663 struct pdu_base *ppdu, 664 unsigned long pdu_len, 665 void *pbuffer, unsigned long buf_len) 666{ 667 struct iscsi_conn *conn = beiscsi_conn->conn; 668 struct iscsi_session *session = conn->session; 669 struct iscsi_task *task; 670 struct beiscsi_io_task *io_task; 671 struct iscsi_hdr *login_hdr; 672 673 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] & 674 PDUBASE_OPCODE_MASK) { 675 case ISCSI_OP_NOOP_IN: 676 pbuffer = NULL; 677 buf_len = 0; 678 break; 679 case ISCSI_OP_ASYNC_EVENT: 680 break; 681 case ISCSI_OP_REJECT: 682 WARN_ON(!pbuffer); 683 WARN_ON(!(buf_len == 48)); 684 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n"); 685 break; 686 case ISCSI_OP_LOGIN_RSP: 687 case ISCSI_OP_TEXT_RSP: 688 task = conn->login_task; 689 io_task = task->dd_data; 690 login_hdr = (struct iscsi_hdr *)ppdu; 691 login_hdr->itt = io_task->libiscsi_itt; 692 break; 693 default: 694 shost_printk(KERN_WARNING, phba->shost, 695 "Unrecognized opcode 0x%x in async msg\n", 696 (ppdu-> 697 dw[offsetof(struct amap_pdu_base, opcode) / 32] 698 & PDUBASE_OPCODE_MASK)); 699 return 1; 700 } 701 702 spin_lock_bh(&session->lock); 703 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len); 704 spin_unlock_bh(&session->lock); 705 return 0; 706} 707 708static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba) 709{ 710 struct sgl_handle *psgl_handle; 711 712 if (phba->io_sgl_hndl_avbl) { 713 SE_DEBUG(DBG_LVL_8, 714 "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n", 715 phba->io_sgl_alloc_index); 716 psgl_handle = phba->io_sgl_hndl_base[phba-> 717 io_sgl_alloc_index]; 718 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL; 719 phba->io_sgl_hndl_avbl--; 720 if (phba->io_sgl_alloc_index == (phba->params. 721 ios_per_ctrl - 1)) 722 phba->io_sgl_alloc_index = 0; 723 else 724 phba->io_sgl_alloc_index++; 725 } else 726 psgl_handle = NULL; 727 return psgl_handle; 728} 729 730static void 731free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) 732{ 733 SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n", 734 phba->io_sgl_free_index); 735 if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) { 736 /* 737 * this can happen if clean_task is called on a task that 738 * failed in xmit_task or alloc_pdu. 739 */ 740 SE_DEBUG(DBG_LVL_8, 741 "Double Free in IO SGL io_sgl_free_index=%d," 742 "value there=%p\n", phba->io_sgl_free_index, 743 phba->io_sgl_hndl_base[phba->io_sgl_free_index]); 744 return; 745 } 746 phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle; 747 phba->io_sgl_hndl_avbl++; 748 if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1)) 749 phba->io_sgl_free_index = 0; 750 else 751 phba->io_sgl_free_index++; 752} 753 754/** 755 * alloc_wrb_handle - To allocate a wrb handle 756 * @phba: The hba pointer 757 * @cid: The cid to use for allocation 758 * 759 * This happens under session_lock until submission to chip 760 */ 761struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid) 762{ 763 struct hwi_wrb_context *pwrb_context; 764 struct hwi_controller *phwi_ctrlr; 765 struct wrb_handle *pwrb_handle, *pwrb_handle_tmp; 766 767 phwi_ctrlr = phba->phwi_ctrlr; 768 pwrb_context = &phwi_ctrlr->wrb_context[cid]; 769 if (pwrb_context->wrb_handles_available >= 2) { 770 pwrb_handle = pwrb_context->pwrb_handle_base[ 771 pwrb_context->alloc_index]; 772 pwrb_context->wrb_handles_available--; 773 if (pwrb_context->alloc_index == 774 (phba->params.wrbs_per_cxn - 1)) 775 pwrb_context->alloc_index = 0; 776 else 777 pwrb_context->alloc_index++; 778 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[ 779 pwrb_context->alloc_index]; 780 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index; 781 } else 782 pwrb_handle = NULL; 783 return pwrb_handle; 784} 785 786/** 787 * free_wrb_handle - To free the wrb handle back to pool 788 * @phba: The hba pointer 789 * @pwrb_context: The context to free from 790 * @pwrb_handle: The wrb_handle to free 791 * 792 * This happens under session_lock until submission to chip 793 */ 794static void 795free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context, 796 struct wrb_handle *pwrb_handle) 797{ 798 pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle; 799 pwrb_context->wrb_handles_available++; 800 if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1)) 801 pwrb_context->free_index = 0; 802 else 803 pwrb_context->free_index++; 804 805 SE_DEBUG(DBG_LVL_8, 806 "FREE WRB: pwrb_handle=%p free_index=0x%x" 807 "wrb_handles_available=%d\n", 808 pwrb_handle, pwrb_context->free_index, 809 pwrb_context->wrb_handles_available); 810} 811 812static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba) 813{ 814 struct sgl_handle *psgl_handle; 815 816 if (phba->eh_sgl_hndl_avbl) { 817 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index]; 818 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL; 819 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n", 820 phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index); 821 phba->eh_sgl_hndl_avbl--; 822 if (phba->eh_sgl_alloc_index == 823 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 824 1)) 825 phba->eh_sgl_alloc_index = 0; 826 else 827 phba->eh_sgl_alloc_index++; 828 } else 829 psgl_handle = NULL; 830 return psgl_handle; 831} 832 833void 834free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) 835{ 836 837 SE_DEBUG(DBG_LVL_8, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d\n", 838 phba->eh_sgl_free_index); 839 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) { 840 /* 841 * this can happen if clean_task is called on a task that 842 * failed in xmit_task or alloc_pdu. 843 */ 844 SE_DEBUG(DBG_LVL_8, 845 "Double Free in eh SGL ,eh_sgl_free_index=%d\n", 846 phba->eh_sgl_free_index); 847 return; 848 } 849 phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle; 850 phba->eh_sgl_hndl_avbl++; 851 if (phba->eh_sgl_free_index == 852 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1)) 853 phba->eh_sgl_free_index = 0; 854 else 855 phba->eh_sgl_free_index++; 856} 857 858static void 859be_complete_io(struct beiscsi_conn *beiscsi_conn, 860 struct iscsi_task *task, struct sol_cqe *psol) 861{ 862 struct beiscsi_io_task *io_task = task->dd_data; 863 struct be_status_bhs *sts_bhs = 864 (struct be_status_bhs *)io_task->cmd_bhs; 865 struct iscsi_conn *conn = beiscsi_conn->conn; 866 unsigned int sense_len; 867 unsigned char *sense; 868 u32 resid = 0, exp_cmdsn, max_cmdsn; 869 u8 rsp, status, flags; 870 871 exp_cmdsn = (psol-> 872 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] 873 & SOL_EXP_CMD_SN_MASK); 874 max_cmdsn = ((psol-> 875 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] 876 & SOL_EXP_CMD_SN_MASK) + 877 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 878 / 32] & SOL_CMD_WND_MASK) >> 24) - 1); 879 rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32] 880 & SOL_RESP_MASK) >> 16); 881 status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32] 882 & SOL_STS_MASK) >> 8); 883 flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] 884 & SOL_FLAGS_MASK) >> 24) | 0x80; 885 886 task->sc->result = (DID_OK << 16) | status; 887 if (rsp != ISCSI_STATUS_CMD_COMPLETED) { 888 task->sc->result = DID_ERROR << 16; 889 goto unmap; 890 } 891 892 /* bidi not initially supported */ 893 if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) { 894 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 895 32] & SOL_RES_CNT_MASK); 896 897 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW)) 898 task->sc->result = DID_ERROR << 16; 899 900 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) { 901 scsi_set_resid(task->sc, resid); 902 if (!status && (scsi_bufflen(task->sc) - resid < 903 task->sc->underflow)) 904 task->sc->result = DID_ERROR << 16; 905 } 906 } 907 908 if (status == SAM_STAT_CHECK_CONDITION) { 909 unsigned short *slen = (unsigned short *)sts_bhs->sense_info; 910 sense = sts_bhs->sense_info + sizeof(unsigned short); 911 sense_len = cpu_to_be16(*slen); 912 memcpy(task->sc->sense_buffer, sense, 913 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE)); 914 } 915 916 if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) { 917 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32] 918 & SOL_RES_CNT_MASK) 919 conn->rxdata_octets += (psol-> 920 dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32] 921 & SOL_RES_CNT_MASK); 922 } 923unmap: 924 scsi_dma_unmap(io_task->scsi_cmnd); 925 iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn); 926} 927 928static void 929be_complete_logout(struct beiscsi_conn *beiscsi_conn, 930 struct iscsi_task *task, struct sol_cqe *psol) 931{ 932 struct iscsi_logout_rsp *hdr; 933 struct beiscsi_io_task *io_task = task->dd_data; 934 struct iscsi_conn *conn = beiscsi_conn->conn; 935 936 hdr = (struct iscsi_logout_rsp *)task->hdr; 937 hdr->opcode = ISCSI_OP_LOGOUT_RSP; 938 hdr->t2wait = 5; 939 hdr->t2retain = 0; 940 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] 941 & SOL_FLAGS_MASK) >> 24) | 0x80; 942 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 943 32] & SOL_RESP_MASK); 944 hdr->exp_cmdsn = cpu_to_be32(psol-> 945 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] 946 & SOL_EXP_CMD_SN_MASK); 947 hdr->max_cmdsn = be32_to_cpu((psol-> 948 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] 949 & SOL_EXP_CMD_SN_MASK) + 950 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 951 / 32] & SOL_CMD_WND_MASK) >> 24) - 1); 952 hdr->dlength[0] = 0; 953 hdr->dlength[1] = 0; 954 hdr->dlength[2] = 0; 955 hdr->hlength = 0; 956 hdr->itt = io_task->libiscsi_itt; 957 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 958} 959 960static void 961be_complete_tmf(struct beiscsi_conn *beiscsi_conn, 962 struct iscsi_task *task, struct sol_cqe *psol) 963{ 964 struct iscsi_tm_rsp *hdr; 965 struct iscsi_conn *conn = beiscsi_conn->conn; 966 struct beiscsi_io_task *io_task = task->dd_data; 967 968 hdr = (struct iscsi_tm_rsp *)task->hdr; 969 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP; 970 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] 971 & SOL_FLAGS_MASK) >> 24) | 0x80; 972 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 973 32] & SOL_RESP_MASK); 974 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe, 975 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK); 976 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe, 977 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) + 978 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 979 / 32] & SOL_CMD_WND_MASK) >> 24) - 1); 980 hdr->itt = io_task->libiscsi_itt; 981 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 982} 983 984static void 985hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn, 986 struct beiscsi_hba *phba, struct sol_cqe *psol) 987{ 988 struct hwi_wrb_context *pwrb_context; 989 struct wrb_handle *pwrb_handle = NULL; 990 struct hwi_controller *phwi_ctrlr; 991 struct iscsi_task *task; 992 struct beiscsi_io_task *io_task; 993 struct iscsi_conn *conn = beiscsi_conn->conn; 994 struct iscsi_session *session = conn->session; 995 996 phwi_ctrlr = phba->phwi_ctrlr; 997 pwrb_context = &phwi_ctrlr->wrb_context[((psol-> 998 dw[offsetof(struct amap_sol_cqe, cid) / 32] & 999 SOL_CID_MASK) >> 6) - 1000 phba->fw_config.iscsi_cid_start]; 1001 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol-> 1002 dw[offsetof(struct amap_sol_cqe, wrb_index) / 1003 32] & SOL_WRB_INDEX_MASK) >> 16)]; 1004 task = pwrb_handle->pio_handle; 1005 1006 io_task = task->dd_data; 1007 spin_lock(&phba->mgmt_sgl_lock); 1008 free_mgmt_sgl_handle(phba, io_task->psgl_handle); 1009 spin_unlock(&phba->mgmt_sgl_lock); 1010 spin_lock_bh(&session->lock); 1011 free_wrb_handle(phba, pwrb_context, pwrb_handle); 1012 spin_unlock_bh(&session->lock); 1013} 1014 1015static void 1016be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn, 1017 struct iscsi_task *task, struct sol_cqe *psol) 1018{ 1019 struct iscsi_nopin *hdr; 1020 struct iscsi_conn *conn = beiscsi_conn->conn; 1021 struct beiscsi_io_task *io_task = task->dd_data; 1022 1023 hdr = (struct iscsi_nopin *)task->hdr; 1024 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] 1025 & SOL_FLAGS_MASK) >> 24) | 0x80; 1026 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe, 1027 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK); 1028 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe, 1029 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) + 1030 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 1031 / 32] & SOL_CMD_WND_MASK) >> 24) - 1); 1032 hdr->opcode = ISCSI_OP_NOOP_IN; 1033 hdr->itt = io_task->libiscsi_itt; 1034 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 1035} 1036 1037static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn, 1038 struct beiscsi_hba *phba, struct sol_cqe *psol) 1039{ 1040 struct hwi_wrb_context *pwrb_context; 1041 struct wrb_handle *pwrb_handle; 1042 struct iscsi_wrb *pwrb = NULL; 1043 struct hwi_controller *phwi_ctrlr; 1044 struct iscsi_task *task; 1045 unsigned int type; 1046 struct iscsi_conn *conn = beiscsi_conn->conn; 1047 struct iscsi_session *session = conn->session; 1048 1049 phwi_ctrlr = phba->phwi_ctrlr; 1050 pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof 1051 (struct amap_sol_cqe, cid) / 32] 1052 & SOL_CID_MASK) >> 6) - 1053 phba->fw_config.iscsi_cid_start]; 1054 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol-> 1055 dw[offsetof(struct amap_sol_cqe, wrb_index) / 1056 32] & SOL_WRB_INDEX_MASK) >> 16)]; 1057 task = pwrb_handle->pio_handle; 1058 pwrb = pwrb_handle->pwrb; 1059 type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] & 1060 WRB_TYPE_MASK) >> 28; 1061 1062 spin_lock_bh(&session->lock); 1063 switch (type) { 1064 case HWH_TYPE_IO: 1065 case HWH_TYPE_IO_RD: 1066 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == 1067 ISCSI_OP_NOOP_OUT) 1068 be_complete_nopin_resp(beiscsi_conn, task, psol); 1069 else 1070 be_complete_io(beiscsi_conn, task, psol); 1071 break; 1072 1073 case HWH_TYPE_LOGOUT: 1074 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT) 1075 be_complete_logout(beiscsi_conn, task, psol); 1076 else 1077 be_complete_tmf(beiscsi_conn, task, psol); 1078 1079 break; 1080 1081 case HWH_TYPE_LOGIN: 1082 SE_DEBUG(DBG_LVL_1, 1083 "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd" 1084 "- Solicited path\n"); 1085 break; 1086 1087 case HWH_TYPE_NOP: 1088 be_complete_nopin_resp(beiscsi_conn, task, psol); 1089 break; 1090 1091 default: 1092 shost_printk(KERN_WARNING, phba->shost, 1093 "In hwi_complete_cmd, unknown type = %d" 1094 "wrb_index 0x%x CID 0x%x\n", type, 1095 ((psol->dw[offsetof(struct amap_iscsi_wrb, 1096 type) / 32] & SOL_WRB_INDEX_MASK) >> 16), 1097 ((psol->dw[offsetof(struct amap_sol_cqe, 1098 cid) / 32] & SOL_CID_MASK) >> 6)); 1099 break; 1100 } 1101 1102 spin_unlock_bh(&session->lock); 1103} 1104 1105static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context 1106 *pasync_ctx, unsigned int is_header, 1107 unsigned int host_write_ptr) 1108{ 1109 if (is_header) 1110 return &pasync_ctx->async_entry[host_write_ptr]. 1111 header_busy_list; 1112 else 1113 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list; 1114} 1115 1116static struct async_pdu_handle * 1117hwi_get_async_handle(struct beiscsi_hba *phba, 1118 struct beiscsi_conn *beiscsi_conn, 1119 struct hwi_async_pdu_context *pasync_ctx, 1120 struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index) 1121{ 1122 struct be_bus_address phys_addr; 1123 struct list_head *pbusy_list; 1124 struct async_pdu_handle *pasync_handle = NULL; 1125 int buffer_len = 0; 1126 unsigned char buffer_index = -1; 1127 unsigned char is_header = 0; 1128 1129 phys_addr.u.a32.address_lo = 1130 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] - 1131 ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32] 1132 & PDUCQE_DPL_MASK) >> 16); 1133 phys_addr.u.a32.address_hi = 1134 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32]; 1135 1136 phys_addr.u.a64.address = 1137 *((unsigned long long *)(&phys_addr.u.a64.address)); 1138 1139 switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32] 1140 & PDUCQE_CODE_MASK) { 1141 case UNSOL_HDR_NOTIFY: 1142 is_header = 1; 1143 1144 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1, 1145 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, 1146 index) / 32] & PDUCQE_INDEX_MASK)); 1147 1148 buffer_len = (unsigned int)(phys_addr.u.a64.address - 1149 pasync_ctx->async_header.pa_base.u.a64.address); 1150 1151 buffer_index = buffer_len / 1152 pasync_ctx->async_header.buffer_size; 1153 1154 break; 1155 case UNSOL_DATA_NOTIFY: 1156 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe-> 1157 dw[offsetof(struct amap_i_t_dpdu_cqe, 1158 index) / 32] & PDUCQE_INDEX_MASK)); 1159 buffer_len = (unsigned long)(phys_addr.u.a64.address - 1160 pasync_ctx->async_data.pa_base.u. 1161 a64.address); 1162 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size; 1163 break; 1164 default: 1165 pbusy_list = NULL; 1166 shost_printk(KERN_WARNING, phba->shost, 1167 "Unexpected code=%d\n", 1168 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, 1169 code) / 32] & PDUCQE_CODE_MASK); 1170 return NULL; 1171 } 1172 1173 WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries)); 1174 WARN_ON(list_empty(pbusy_list)); 1175 list_for_each_entry(pasync_handle, pbusy_list, link) { 1176 WARN_ON(pasync_handle->consumed); 1177 if (pasync_handle->index == buffer_index) 1178 break; 1179 } 1180 1181 WARN_ON(!pasync_handle); 1182 1183 pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid - 1184 phba->fw_config.iscsi_cid_start; 1185 pasync_handle->is_header = is_header; 1186 pasync_handle->buffer_len = ((pdpdu_cqe-> 1187 dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32] 1188 & PDUCQE_DPL_MASK) >> 16); 1189 1190 *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, 1191 index) / 32] & PDUCQE_INDEX_MASK); 1192 return pasync_handle; 1193} 1194 1195static unsigned int 1196hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx, 1197 unsigned int is_header, unsigned int cq_index) 1198{ 1199 struct list_head *pbusy_list; 1200 struct async_pdu_handle *pasync_handle; 1201 unsigned int num_entries, writables = 0; 1202 unsigned int *pep_read_ptr, *pwritables; 1203 1204 1205 if (is_header) { 1206 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr; 1207 pwritables = &pasync_ctx->async_header.writables; 1208 num_entries = pasync_ctx->async_header.num_entries; 1209 } else { 1210 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr; 1211 pwritables = &pasync_ctx->async_data.writables; 1212 num_entries = pasync_ctx->async_data.num_entries; 1213 } 1214 1215 while ((*pep_read_ptr) != cq_index) { 1216 (*pep_read_ptr)++; 1217 *pep_read_ptr = (*pep_read_ptr) % num_entries; 1218 1219 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header, 1220 *pep_read_ptr); 1221 if (writables == 0) 1222 WARN_ON(list_empty(pbusy_list)); 1223 1224 if (!list_empty(pbusy_list)) { 1225 pasync_handle = list_entry(pbusy_list->next, 1226 struct async_pdu_handle, 1227 link); 1228 WARN_ON(!pasync_handle); 1229 pasync_handle->consumed = 1; 1230 } 1231 1232 writables++; 1233 } 1234 1235 if (!writables) { 1236 SE_DEBUG(DBG_LVL_1, 1237 "Duplicate notification received - index 0x%x!!\n", 1238 cq_index); 1239 WARN_ON(1); 1240 } 1241 1242 *pwritables = *pwritables + writables; 1243 return 0; 1244} 1245 1246static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba, 1247 unsigned int cri) 1248{ 1249 struct hwi_controller *phwi_ctrlr; 1250 struct hwi_async_pdu_context *pasync_ctx; 1251 struct async_pdu_handle *pasync_handle, *tmp_handle; 1252 struct list_head *plist; 1253 unsigned int i = 0; 1254 1255 phwi_ctrlr = phba->phwi_ctrlr; 1256 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); 1257 1258 plist = &pasync_ctx->async_entry[cri].wait_queue.list; 1259 1260 list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) { 1261 list_del(&pasync_handle->link); 1262 1263 if (i == 0) { 1264 list_add_tail(&pasync_handle->link, 1265 &pasync_ctx->async_header.free_list); 1266 pasync_ctx->async_header.free_entries++; 1267 i++; 1268 } else { 1269 list_add_tail(&pasync_handle->link, 1270 &pasync_ctx->async_data.free_list); 1271 pasync_ctx->async_data.free_entries++; 1272 i++; 1273 } 1274 } 1275 1276 INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list); 1277 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0; 1278 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0; 1279 return 0; 1280} 1281 1282static struct phys_addr * 1283hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx, 1284 unsigned int is_header, unsigned int host_write_ptr) 1285{ 1286 struct phys_addr *pasync_sge = NULL; 1287 1288 if (is_header) 1289 pasync_sge = pasync_ctx->async_header.ring_base; 1290 else 1291 pasync_sge = pasync_ctx->async_data.ring_base; 1292 1293 return pasync_sge + host_write_ptr; 1294} 1295 1296static void hwi_post_async_buffers(struct beiscsi_hba *phba, 1297 unsigned int is_header) 1298{ 1299 struct hwi_controller *phwi_ctrlr; 1300 struct hwi_async_pdu_context *pasync_ctx; 1301 struct async_pdu_handle *pasync_handle; 1302 struct list_head *pfree_link, *pbusy_list; 1303 struct phys_addr *pasync_sge; 1304 unsigned int ring_id, num_entries; 1305 unsigned int host_write_num; 1306 unsigned int writables; 1307 unsigned int i = 0; 1308 u32 doorbell = 0; 1309 1310 phwi_ctrlr = phba->phwi_ctrlr; 1311 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); 1312 1313 if (is_header) { 1314 num_entries = pasync_ctx->async_header.num_entries; 1315 writables = min(pasync_ctx->async_header.writables, 1316 pasync_ctx->async_header.free_entries); 1317 pfree_link = pasync_ctx->async_header.free_list.next; 1318 host_write_num = pasync_ctx->async_header.host_write_ptr; 1319 ring_id = phwi_ctrlr->default_pdu_hdr.id; 1320 } else { 1321 num_entries = pasync_ctx->async_data.num_entries; 1322 writables = min(pasync_ctx->async_data.writables, 1323 pasync_ctx->async_data.free_entries); 1324 pfree_link = pasync_ctx->async_data.free_list.next; 1325 host_write_num = pasync_ctx->async_data.host_write_ptr; 1326 ring_id = phwi_ctrlr->default_pdu_data.id; 1327 } 1328 1329 writables = (writables / 8) * 8; 1330 if (writables) { 1331 for (i = 0; i < writables; i++) { 1332 pbusy_list = 1333 hwi_get_async_busy_list(pasync_ctx, is_header, 1334 host_write_num); 1335 pasync_handle = 1336 list_entry(pfree_link, struct async_pdu_handle, 1337 link); 1338 WARN_ON(!pasync_handle); 1339 pasync_handle->consumed = 0; 1340 1341 pfree_link = pfree_link->next; 1342 1343 pasync_sge = hwi_get_ring_address(pasync_ctx, 1344 is_header, host_write_num); 1345 1346 pasync_sge->hi = pasync_handle->pa.u.a32.address_lo; 1347 pasync_sge->lo = pasync_handle->pa.u.a32.address_hi; 1348 1349 list_move(&pasync_handle->link, pbusy_list); 1350 1351 host_write_num++; 1352 host_write_num = host_write_num % num_entries; 1353 } 1354 1355 if (is_header) { 1356 pasync_ctx->async_header.host_write_ptr = 1357 host_write_num; 1358 pasync_ctx->async_header.free_entries -= writables; 1359 pasync_ctx->async_header.writables -= writables; 1360 pasync_ctx->async_header.busy_entries += writables; 1361 } else { 1362 pasync_ctx->async_data.host_write_ptr = host_write_num; 1363 pasync_ctx->async_data.free_entries -= writables; 1364 pasync_ctx->async_data.writables -= writables; 1365 pasync_ctx->async_data.busy_entries += writables; 1366 } 1367 1368 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK; 1369 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT; 1370 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT; 1371 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK) 1372 << DB_DEF_PDU_CQPROC_SHIFT; 1373 1374 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET); 1375 } 1376} 1377 1378static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba, 1379 struct beiscsi_conn *beiscsi_conn, 1380 struct i_t_dpdu_cqe *pdpdu_cqe) 1381{ 1382 struct hwi_controller *phwi_ctrlr; 1383 struct hwi_async_pdu_context *pasync_ctx; 1384 struct async_pdu_handle *pasync_handle = NULL; 1385 unsigned int cq_index = -1; 1386 1387 phwi_ctrlr = phba->phwi_ctrlr; 1388 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); 1389 1390 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx, 1391 pdpdu_cqe, &cq_index); 1392 BUG_ON(pasync_handle->is_header != 0); 1393 if (pasync_handle->consumed == 0) 1394 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header, 1395 cq_index); 1396 1397 hwi_free_async_msg(phba, pasync_handle->cri); 1398 hwi_post_async_buffers(phba, pasync_handle->is_header); 1399} 1400 1401static unsigned int 1402hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn, 1403 struct beiscsi_hba *phba, 1404 struct hwi_async_pdu_context *pasync_ctx, unsigned short cri) 1405{ 1406 struct list_head *plist; 1407 struct async_pdu_handle *pasync_handle; 1408 void *phdr = NULL; 1409 unsigned int hdr_len = 0, buf_len = 0; 1410 unsigned int status, index = 0, offset = 0; 1411 void *pfirst_buffer = NULL; 1412 unsigned int num_buf = 0; 1413 1414 plist = &pasync_ctx->async_entry[cri].wait_queue.list; 1415 1416 list_for_each_entry(pasync_handle, plist, link) { 1417 if (index == 0) { 1418 phdr = pasync_handle->pbuffer; 1419 hdr_len = pasync_handle->buffer_len; 1420 } else { 1421 buf_len = pasync_handle->buffer_len; 1422 if (!num_buf) { 1423 pfirst_buffer = pasync_handle->pbuffer; 1424 num_buf++; 1425 } 1426 memcpy(pfirst_buffer + offset, 1427 pasync_handle->pbuffer, buf_len); 1428 offset = buf_len; 1429 } 1430 index++; 1431 } 1432 1433 status = beiscsi_process_async_pdu(beiscsi_conn, phba, 1434 (beiscsi_conn->beiscsi_conn_cid - 1435 phba->fw_config.iscsi_cid_start), 1436 phdr, hdr_len, pfirst_buffer, 1437 buf_len); 1438 1439 if (status == 0) 1440 hwi_free_async_msg(phba, cri); 1441 return 0; 1442} 1443 1444static unsigned int 1445hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn, 1446 struct beiscsi_hba *phba, 1447 struct async_pdu_handle *pasync_handle) 1448{ 1449 struct hwi_async_pdu_context *pasync_ctx; 1450 struct hwi_controller *phwi_ctrlr; 1451 unsigned int bytes_needed = 0, status = 0; 1452 unsigned short cri = pasync_handle->cri; 1453 struct pdu_base *ppdu; 1454 1455 phwi_ctrlr = phba->phwi_ctrlr; 1456 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); 1457 1458 list_del(&pasync_handle->link); 1459 if (pasync_handle->is_header) { 1460 pasync_ctx->async_header.busy_entries--; 1461 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) { 1462 hwi_free_async_msg(phba, cri); 1463 BUG(); 1464 } 1465 1466 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0; 1467 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1; 1468 pasync_ctx->async_entry[cri].wait_queue.hdr_len = 1469 (unsigned short)pasync_handle->buffer_len; 1470 list_add_tail(&pasync_handle->link, 1471 &pasync_ctx->async_entry[cri].wait_queue.list); 1472 1473 ppdu = pasync_handle->pbuffer; 1474 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base, 1475 data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) & 1476 0xFFFF0000) | ((be16_to_cpu((ppdu-> 1477 dw[offsetof(struct amap_pdu_base, data_len_lo) / 32] 1478 & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF)); 1479 1480 if (status == 0) { 1481 pasync_ctx->async_entry[cri].wait_queue.bytes_needed = 1482 bytes_needed; 1483 1484 if (bytes_needed == 0) 1485 status = hwi_fwd_async_msg(beiscsi_conn, phba, 1486 pasync_ctx, cri); 1487 } 1488 } else { 1489 pasync_ctx->async_data.busy_entries--; 1490 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) { 1491 list_add_tail(&pasync_handle->link, 1492 &pasync_ctx->async_entry[cri].wait_queue. 1493 list); 1494 pasync_ctx->async_entry[cri].wait_queue. 1495 bytes_received += 1496 (unsigned short)pasync_handle->buffer_len; 1497 1498 if (pasync_ctx->async_entry[cri].wait_queue. 1499 bytes_received >= 1500 pasync_ctx->async_entry[cri].wait_queue. 1501 bytes_needed) 1502 status = hwi_fwd_async_msg(beiscsi_conn, phba, 1503 pasync_ctx, cri); 1504 } 1505 } 1506 return status; 1507} 1508 1509static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn, 1510 struct beiscsi_hba *phba, 1511 struct i_t_dpdu_cqe *pdpdu_cqe) 1512{ 1513 struct hwi_controller *phwi_ctrlr; 1514 struct hwi_async_pdu_context *pasync_ctx; 1515 struct async_pdu_handle *pasync_handle = NULL; 1516 unsigned int cq_index = -1; 1517 1518 phwi_ctrlr = phba->phwi_ctrlr; 1519 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); 1520 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx, 1521 pdpdu_cqe, &cq_index); 1522 1523 if (pasync_handle->consumed == 0) 1524 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header, 1525 cq_index); 1526 hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle); 1527 hwi_post_async_buffers(phba, pasync_handle->is_header); 1528} 1529 1530static void beiscsi_process_mcc_isr(struct beiscsi_hba *phba) 1531{ 1532 struct be_queue_info *mcc_cq; 1533 struct be_mcc_compl *mcc_compl; 1534 unsigned int num_processed = 0; 1535 1536 mcc_cq = &phba->ctrl.mcc_obj.cq; 1537 mcc_compl = queue_tail_node(mcc_cq); 1538 mcc_compl->flags = le32_to_cpu(mcc_compl->flags); 1539 while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) { 1540 1541 if (num_processed >= 32) { 1542 hwi_ring_cq_db(phba, mcc_cq->id, 1543 num_processed, 0, 0); 1544 num_processed = 0; 1545 } 1546 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) { 1547 /* Interpret flags as an async trailer */ 1548 if (is_link_state_evt(mcc_compl->flags)) 1549 /* Interpret compl as a async link evt */ 1550 beiscsi_async_link_state_process(phba, 1551 (struct be_async_event_link_state *) mcc_compl); 1552 else 1553 SE_DEBUG(DBG_LVL_1, 1554 " Unsupported Async Event, flags" 1555 " = 0x%08x\n", mcc_compl->flags); 1556 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) { 1557 be_mcc_compl_process_isr(&phba->ctrl, mcc_compl); 1558 atomic_dec(&phba->ctrl.mcc_obj.q.used); 1559 } 1560 1561 mcc_compl->flags = 0; 1562 queue_tail_inc(mcc_cq); 1563 mcc_compl = queue_tail_node(mcc_cq); 1564 mcc_compl->flags = le32_to_cpu(mcc_compl->flags); 1565 num_processed++; 1566 } 1567 1568 if (num_processed > 0) 1569 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0); 1570 1571} 1572 1573static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq) 1574{ 1575 struct be_queue_info *cq; 1576 struct sol_cqe *sol; 1577 struct dmsg_cqe *dmsg; 1578 unsigned int num_processed = 0; 1579 unsigned int tot_nump = 0; 1580 struct beiscsi_conn *beiscsi_conn; 1581 struct beiscsi_endpoint *beiscsi_ep; 1582 struct iscsi_endpoint *ep; 1583 struct beiscsi_hba *phba; 1584 1585 cq = pbe_eq->cq; 1586 sol = queue_tail_node(cq); 1587 phba = pbe_eq->phba; 1588 1589 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] & 1590 CQE_VALID_MASK) { 1591 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe)); 1592 1593 ep = phba->ep_array[(u32) ((sol-> 1594 dw[offsetof(struct amap_sol_cqe, cid) / 32] & 1595 SOL_CID_MASK) >> 6) - 1596 phba->fw_config.iscsi_cid_start]; 1597 1598 beiscsi_ep = ep->dd_data; 1599 beiscsi_conn = beiscsi_ep->conn; 1600 1601 if (num_processed >= 32) { 1602 hwi_ring_cq_db(phba, cq->id, 1603 num_processed, 0, 0); 1604 tot_nump += num_processed; 1605 num_processed = 0; 1606 } 1607 1608 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) / 1609 32] & CQE_CODE_MASK) { 1610 case SOL_CMD_COMPLETE: 1611 hwi_complete_cmd(beiscsi_conn, phba, sol); 1612 break; 1613 case DRIVERMSG_NOTIFY: 1614 SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY\n"); 1615 dmsg = (struct dmsg_cqe *)sol; 1616 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol); 1617 break; 1618 case UNSOL_HDR_NOTIFY: 1619 SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n"); 1620 hwi_process_default_pdu_ring(beiscsi_conn, phba, 1621 (struct i_t_dpdu_cqe *)sol); 1622 break; 1623 case UNSOL_DATA_NOTIFY: 1624 SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n"); 1625 hwi_process_default_pdu_ring(beiscsi_conn, phba, 1626 (struct i_t_dpdu_cqe *)sol); 1627 break; 1628 case CXN_INVALIDATE_INDEX_NOTIFY: 1629 case CMD_INVALIDATED_NOTIFY: 1630 case CXN_INVALIDATE_NOTIFY: 1631 SE_DEBUG(DBG_LVL_1, 1632 "Ignoring CQ Error notification for cmd/cxn" 1633 "invalidate\n"); 1634 break; 1635 case SOL_CMD_KILLED_DATA_DIGEST_ERR: 1636 case CMD_KILLED_INVALID_STATSN_RCVD: 1637 case CMD_KILLED_INVALID_R2T_RCVD: 1638 case CMD_CXN_KILLED_LUN_INVALID: 1639 case CMD_CXN_KILLED_ICD_INVALID: 1640 case CMD_CXN_KILLED_ITT_INVALID: 1641 case CMD_CXN_KILLED_SEQ_OUTOFORDER: 1642 case CMD_CXN_KILLED_INVALID_DATASN_RCVD: 1643 SE_DEBUG(DBG_LVL_1, 1644 "CQ Error notification for cmd.. " 1645 "code %d cid 0x%x\n", 1646 sol->dw[offsetof(struct amap_sol_cqe, code) / 1647 32] & CQE_CODE_MASK, 1648 (sol->dw[offsetof(struct amap_sol_cqe, cid) / 1649 32] & SOL_CID_MASK)); 1650 break; 1651 case UNSOL_DATA_DIGEST_ERROR_NOTIFY: 1652 SE_DEBUG(DBG_LVL_1, 1653 "Digest error on def pdu ring, dropping..\n"); 1654 hwi_flush_default_pdu_buffer(phba, beiscsi_conn, 1655 (struct i_t_dpdu_cqe *) sol); 1656 break; 1657 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL: 1658 case CXN_KILLED_BURST_LEN_MISMATCH: 1659 case CXN_KILLED_AHS_RCVD: 1660 case CXN_KILLED_HDR_DIGEST_ERR: 1661 case CXN_KILLED_UNKNOWN_HDR: 1662 case CXN_KILLED_STALE_ITT_TTT_RCVD: 1663 case CXN_KILLED_INVALID_ITT_TTT_RCVD: 1664 case CXN_KILLED_TIMED_OUT: 1665 case CXN_KILLED_FIN_RCVD: 1666 case CXN_KILLED_BAD_UNSOL_PDU_RCVD: 1667 case CXN_KILLED_BAD_WRB_INDEX_ERROR: 1668 case CXN_KILLED_OVER_RUN_RESIDUAL: 1669 case CXN_KILLED_UNDER_RUN_RESIDUAL: 1670 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN: 1671 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID " 1672 "0x%x...\n", 1673 sol->dw[offsetof(struct amap_sol_cqe, code) / 1674 32] & CQE_CODE_MASK, 1675 (sol->dw[offsetof(struct amap_sol_cqe, cid) / 1676 32] & CQE_CID_MASK)); 1677 iscsi_conn_failure(beiscsi_conn->conn, 1678 ISCSI_ERR_CONN_FAILED); 1679 break; 1680 case CXN_KILLED_RST_SENT: 1681 case CXN_KILLED_RST_RCVD: 1682 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset" 1683 "received/sent on CID 0x%x...\n", 1684 sol->dw[offsetof(struct amap_sol_cqe, code) / 1685 32] & CQE_CODE_MASK, 1686 (sol->dw[offsetof(struct amap_sol_cqe, cid) / 1687 32] & CQE_CID_MASK)); 1688 iscsi_conn_failure(beiscsi_conn->conn, 1689 ISCSI_ERR_CONN_FAILED); 1690 break; 1691 default: 1692 SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d " 1693 "received on CID 0x%x...\n", 1694 sol->dw[offsetof(struct amap_sol_cqe, code) / 1695 32] & CQE_CODE_MASK, 1696 (sol->dw[offsetof(struct amap_sol_cqe, cid) / 1697 32] & CQE_CID_MASK)); 1698 break; 1699 } 1700 1701 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0); 1702 queue_tail_inc(cq); 1703 sol = queue_tail_node(cq); 1704 num_processed++; 1705 } 1706 1707 if (num_processed > 0) { 1708 tot_nump += num_processed; 1709 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0); 1710 } 1711 return tot_nump; 1712} 1713 1714void beiscsi_process_all_cqs(struct work_struct *work) 1715{ 1716 unsigned long flags; 1717 struct hwi_controller *phwi_ctrlr; 1718 struct hwi_context_memory *phwi_context; 1719 struct be_eq_obj *pbe_eq; 1720 struct beiscsi_hba *phba = 1721 container_of(work, struct beiscsi_hba, work_cqs); 1722 1723 phwi_ctrlr = phba->phwi_ctrlr; 1724 phwi_context = phwi_ctrlr->phwi_ctxt; 1725 if (phba->msix_enabled) 1726 pbe_eq = &phwi_context->be_eq[phba->num_cpus]; 1727 else 1728 pbe_eq = &phwi_context->be_eq[0]; 1729 1730 if (phba->todo_mcc_cq) { 1731 spin_lock_irqsave(&phba->isr_lock, flags); 1732 phba->todo_mcc_cq = 0; 1733 spin_unlock_irqrestore(&phba->isr_lock, flags); 1734 beiscsi_process_mcc_isr(phba); 1735 } 1736 1737 if (phba->todo_cq) { 1738 spin_lock_irqsave(&phba->isr_lock, flags); 1739 phba->todo_cq = 0; 1740 spin_unlock_irqrestore(&phba->isr_lock, flags); 1741 beiscsi_process_cq(pbe_eq); 1742 } 1743} 1744 1745static int be_iopoll(struct blk_iopoll *iop, int budget) 1746{ 1747 static unsigned int ret; 1748 struct beiscsi_hba *phba; 1749 struct be_eq_obj *pbe_eq; 1750 1751 pbe_eq = container_of(iop, struct be_eq_obj, iopoll); 1752 ret = beiscsi_process_cq(pbe_eq); 1753 if (ret < budget) { 1754 phba = pbe_eq->phba; 1755 blk_iopoll_complete(iop); 1756 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id); 1757 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1); 1758 } 1759 return ret; 1760} 1761 1762static void 1763hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg, 1764 unsigned int num_sg, struct beiscsi_io_task *io_task) 1765{ 1766 struct iscsi_sge *psgl; 1767 unsigned short sg_len, index; 1768 unsigned int sge_len = 0; 1769 unsigned long long addr; 1770 struct scatterlist *l_sg; 1771 unsigned int offset; 1772 1773 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb, 1774 io_task->bhs_pa.u.a32.address_lo); 1775 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb, 1776 io_task->bhs_pa.u.a32.address_hi); 1777 1778 l_sg = sg; 1779 for (index = 0; (index < num_sg) && (index < 2); index++, 1780 sg = sg_next(sg)) { 1781 if (index == 0) { 1782 sg_len = sg_dma_len(sg); 1783 addr = (u64) sg_dma_address(sg); 1784 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb, 1785 ((u32)(addr & 0xFFFFFFFF))); 1786 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb, 1787 ((u32)(addr >> 32))); 1788 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb, 1789 sg_len); 1790 sge_len = sg_len; 1791 } else { 1792 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset, 1793 pwrb, sge_len); 1794 sg_len = sg_dma_len(sg); 1795 addr = (u64) sg_dma_address(sg); 1796 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb, 1797 ((u32)(addr & 0xFFFFFFFF))); 1798 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb, 1799 ((u32)(addr >> 32))); 1800 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb, 1801 sg_len); 1802 } 1803 } 1804 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; 1805 memset(psgl, 0, sizeof(*psgl) * BE2_SGE); 1806 1807 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2); 1808 1809 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 1810 io_task->bhs_pa.u.a32.address_hi); 1811 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 1812 io_task->bhs_pa.u.a32.address_lo); 1813 1814 if (num_sg == 1) { 1815 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1816 1); 1817 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 1818 0); 1819 } else if (num_sg == 2) { 1820 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1821 0); 1822 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 1823 1); 1824 } else { 1825 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1826 0); 1827 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 1828 0); 1829 } 1830 sg = l_sg; 1831 psgl++; 1832 psgl++; 1833 offset = 0; 1834 for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) { 1835 sg_len = sg_dma_len(sg); 1836 addr = (u64) sg_dma_address(sg); 1837 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 1838 (addr & 0xFFFFFFFF)); 1839 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 1840 (addr >> 32)); 1841 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len); 1842 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset); 1843 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); 1844 offset += sg_len; 1845 } 1846 psgl--; 1847 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); 1848} 1849 1850static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task) 1851{ 1852 struct iscsi_sge *psgl; 1853 unsigned long long addr; 1854 struct beiscsi_io_task *io_task = task->dd_data; 1855 struct beiscsi_conn *beiscsi_conn = io_task->conn; 1856 struct beiscsi_hba *phba = beiscsi_conn->phba; 1857 1858 io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2; 1859 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb, 1860 io_task->bhs_pa.u.a32.address_lo); 1861 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb, 1862 io_task->bhs_pa.u.a32.address_hi); 1863 1864 if (task->data) { 1865 if (task->data_count) { 1866 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1); 1867 addr = (u64) pci_map_single(phba->pcidev, 1868 task->data, 1869 task->data_count, 1); 1870 } else { 1871 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); 1872 addr = 0; 1873 } 1874 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb, 1875 ((u32)(addr & 0xFFFFFFFF))); 1876 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb, 1877 ((u32)(addr >> 32))); 1878 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb, 1879 task->data_count); 1880 1881 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1); 1882 } else { 1883 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); 1884 addr = 0; 1885 } 1886 1887 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; 1888 1889 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len); 1890 1891 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 1892 io_task->bhs_pa.u.a32.address_hi); 1893 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 1894 io_task->bhs_pa.u.a32.address_lo); 1895 if (task->data) { 1896 psgl++; 1897 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0); 1898 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0); 1899 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0); 1900 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0); 1901 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0); 1902 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); 1903 1904 psgl++; 1905 if (task->data) { 1906 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 1907 ((u32)(addr & 0xFFFFFFFF))); 1908 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 1909 ((u32)(addr >> 32))); 1910 } 1911 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106); 1912 } 1913 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); 1914} 1915 1916static void beiscsi_find_mem_req(struct beiscsi_hba *phba) 1917{ 1918 unsigned int num_cq_pages, num_async_pdu_buf_pages; 1919 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn; 1920 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages; 1921 1922 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \ 1923 sizeof(struct sol_cqe)); 1924 num_async_pdu_buf_pages = 1925 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ 1926 phba->params.defpdu_hdr_sz); 1927 num_async_pdu_buf_sgl_pages = 1928 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ 1929 sizeof(struct phys_addr)); 1930 num_async_pdu_data_pages = 1931 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ 1932 phba->params.defpdu_data_sz); 1933 num_async_pdu_data_sgl_pages = 1934 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ 1935 sizeof(struct phys_addr)); 1936 1937 phba->params.hwi_ws_sz = sizeof(struct hwi_controller); 1938 1939 phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 * 1940 BE_ISCSI_PDU_HEADER_SIZE; 1941 phba->mem_req[HWI_MEM_ADDN_CONTEXT] = 1942 sizeof(struct hwi_context_memory); 1943 1944 1945 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb) 1946 * (phba->params.wrbs_per_cxn) 1947 * phba->params.cxns_per_ctrl; 1948 wrb_sz_per_cxn = sizeof(struct wrb_handle) * 1949 (phba->params.wrbs_per_cxn); 1950 phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) * 1951 phba->params.cxns_per_ctrl); 1952 1953 phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) * 1954 phba->params.icds_per_ctrl; 1955 phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) * 1956 phba->params.num_sge_per_io * phba->params.icds_per_ctrl; 1957 1958 phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] = 1959 num_async_pdu_buf_pages * PAGE_SIZE; 1960 phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] = 1961 num_async_pdu_data_pages * PAGE_SIZE; 1962 phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] = 1963 num_async_pdu_buf_sgl_pages * PAGE_SIZE; 1964 phba->mem_req[HWI_MEM_ASYNC_DATA_RING] = 1965 num_async_pdu_data_sgl_pages * PAGE_SIZE; 1966 phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] = 1967 phba->params.asyncpdus_per_ctrl * 1968 sizeof(struct async_pdu_handle); 1969 phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] = 1970 phba->params.asyncpdus_per_ctrl * 1971 sizeof(struct async_pdu_handle); 1972 phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] = 1973 sizeof(struct hwi_async_pdu_context) + 1974 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry)); 1975} 1976 1977static int beiscsi_alloc_mem(struct beiscsi_hba *phba) 1978{ 1979 struct be_mem_descriptor *mem_descr; 1980 dma_addr_t bus_add; 1981 struct mem_array *mem_arr, *mem_arr_orig; 1982 unsigned int i, j, alloc_size, curr_alloc_size; 1983 1984 phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL); 1985 if (!phba->phwi_ctrlr) 1986 return -ENOMEM; 1987 1988 phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr), 1989 GFP_KERNEL); 1990 if (!phba->init_mem) { 1991 kfree(phba->phwi_ctrlr); 1992 return -ENOMEM; 1993 } 1994 1995 mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT, 1996 GFP_KERNEL); 1997 if (!mem_arr_orig) { 1998 kfree(phba->init_mem); 1999 kfree(phba->phwi_ctrlr); 2000 return -ENOMEM; 2001 } 2002 2003 mem_descr = phba->init_mem; 2004 for (i = 0; i < SE_MEM_MAX; i++) { 2005 j = 0; 2006 mem_arr = mem_arr_orig; 2007 alloc_size = phba->mem_req[i]; 2008 memset(mem_arr, 0, sizeof(struct mem_array) * 2009 BEISCSI_MAX_FRAGS_INIT); 2010 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size); 2011 do { 2012 mem_arr->virtual_address = pci_alloc_consistent( 2013 phba->pcidev, 2014 curr_alloc_size, 2015 &bus_add); 2016 if (!mem_arr->virtual_address) { 2017 if (curr_alloc_size <= BE_MIN_MEM_SIZE) 2018 goto free_mem; 2019 if (curr_alloc_size - 2020 rounddown_pow_of_two(curr_alloc_size)) 2021 curr_alloc_size = rounddown_pow_of_two 2022 (curr_alloc_size); 2023 else 2024 curr_alloc_size = curr_alloc_size / 2; 2025 } else { 2026 mem_arr->bus_address.u. 2027 a64.address = (__u64) bus_add; 2028 mem_arr->size = curr_alloc_size; 2029 alloc_size -= curr_alloc_size; 2030 curr_alloc_size = min(be_max_phys_size * 2031 1024, alloc_size); 2032 j++; 2033 mem_arr++; 2034 } 2035 } while (alloc_size); 2036 mem_descr->num_elements = j; 2037 mem_descr->size_in_bytes = phba->mem_req[i]; 2038 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j, 2039 GFP_KERNEL); 2040 if (!mem_descr->mem_array) 2041 goto free_mem; 2042 2043 memcpy(mem_descr->mem_array, mem_arr_orig, 2044 sizeof(struct mem_array) * j); 2045 mem_descr++; 2046 } 2047 kfree(mem_arr_orig); 2048 return 0; 2049free_mem: 2050 mem_descr->num_elements = j; 2051 while ((i) || (j)) { 2052 for (j = mem_descr->num_elements; j > 0; j--) { 2053 pci_free_consistent(phba->pcidev, 2054 mem_descr->mem_array[j - 1].size, 2055 mem_descr->mem_array[j - 1]. 2056 virtual_address, 2057 (unsigned long)mem_descr-> 2058 mem_array[j - 1]. 2059 bus_address.u.a64.address); 2060 } 2061 if (i) { 2062 i--; 2063 kfree(mem_descr->mem_array); 2064 mem_descr--; 2065 } 2066 } 2067 kfree(mem_arr_orig); 2068 kfree(phba->init_mem); 2069 kfree(phba->phwi_ctrlr); 2070 return -ENOMEM; 2071} 2072 2073static int beiscsi_get_memory(struct beiscsi_hba *phba) 2074{ 2075 beiscsi_find_mem_req(phba); 2076 return beiscsi_alloc_mem(phba); 2077} 2078 2079static void iscsi_init_global_templates(struct beiscsi_hba *phba) 2080{ 2081 struct pdu_data_out *pdata_out; 2082 struct pdu_nop_out *pnop_out; 2083 struct be_mem_descriptor *mem_descr; 2084 2085 mem_descr = phba->init_mem; 2086 mem_descr += ISCSI_MEM_GLOBAL_HEADER; 2087 pdata_out = 2088 (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address; 2089 memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE); 2090 2091 AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out, 2092 IIOC_SCSI_DATA); 2093 2094 pnop_out = 2095 (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0]. 2096 virtual_address + BE_ISCSI_PDU_HEADER_SIZE); 2097 2098 memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE); 2099 AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF); 2100 AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1); 2101 AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0); 2102} 2103 2104static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba) 2105{ 2106 struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb; 2107 struct wrb_handle *pwrb_handle; 2108 struct hwi_controller *phwi_ctrlr; 2109 struct hwi_wrb_context *pwrb_context; 2110 struct iscsi_wrb *pwrb; 2111 unsigned int num_cxn_wrbh; 2112 unsigned int num_cxn_wrb, j, idx, index; 2113 2114 mem_descr_wrbh = phba->init_mem; 2115 mem_descr_wrbh += HWI_MEM_WRBH; 2116 2117 mem_descr_wrb = phba->init_mem; 2118 mem_descr_wrb += HWI_MEM_WRB; 2119 2120 idx = 0; 2121 pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address; 2122 num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) / 2123 ((sizeof(struct wrb_handle)) * 2124 phba->params.wrbs_per_cxn)); 2125 phwi_ctrlr = phba->phwi_ctrlr; 2126 2127 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) { 2128 pwrb_context = &phwi_ctrlr->wrb_context[index]; 2129 pwrb_context->pwrb_handle_base = 2130 kzalloc(sizeof(struct wrb_handle *) * 2131 phba->params.wrbs_per_cxn, GFP_KERNEL); 2132 pwrb_context->pwrb_handle_basestd = 2133 kzalloc(sizeof(struct wrb_handle *) * 2134 phba->params.wrbs_per_cxn, GFP_KERNEL); 2135 if (num_cxn_wrbh) { 2136 pwrb_context->alloc_index = 0; 2137 pwrb_context->wrb_handles_available = 0; 2138 for (j = 0; j < phba->params.wrbs_per_cxn; j++) { 2139 pwrb_context->pwrb_handle_base[j] = pwrb_handle; 2140 pwrb_context->pwrb_handle_basestd[j] = 2141 pwrb_handle; 2142 pwrb_context->wrb_handles_available++; 2143 pwrb_handle->wrb_index = j; 2144 pwrb_handle++; 2145 } 2146 pwrb_context->free_index = 0; 2147 num_cxn_wrbh--; 2148 } else { 2149 idx++; 2150 pwrb_handle = 2151 mem_descr_wrbh->mem_array[idx].virtual_address; 2152 num_cxn_wrbh = 2153 ((mem_descr_wrbh->mem_array[idx].size) / 2154 ((sizeof(struct wrb_handle)) * 2155 phba->params.wrbs_per_cxn)); 2156 pwrb_context->alloc_index = 0; 2157 for (j = 0; j < phba->params.wrbs_per_cxn; j++) { 2158 pwrb_context->pwrb_handle_base[j] = pwrb_handle; 2159 pwrb_context->pwrb_handle_basestd[j] = 2160 pwrb_handle; 2161 pwrb_context->wrb_handles_available++; 2162 pwrb_handle->wrb_index = j; 2163 pwrb_handle++; 2164 } 2165 pwrb_context->free_index = 0; 2166 num_cxn_wrbh--; 2167 } 2168 } 2169 idx = 0; 2170 pwrb = mem_descr_wrb->mem_array[idx].virtual_address; 2171 num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) / 2172 ((sizeof(struct iscsi_wrb) * 2173 phba->params.wrbs_per_cxn)); 2174 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) { 2175 pwrb_context = &phwi_ctrlr->wrb_context[index]; 2176 if (num_cxn_wrb) { 2177 for (j = 0; j < phba->params.wrbs_per_cxn; j++) { 2178 pwrb_handle = pwrb_context->pwrb_handle_base[j]; 2179 pwrb_handle->pwrb = pwrb; 2180 pwrb++; 2181 } 2182 num_cxn_wrb--; 2183 } else { 2184 idx++; 2185 pwrb = mem_descr_wrb->mem_array[idx].virtual_address; 2186 num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) / 2187 ((sizeof(struct iscsi_wrb) * 2188 phba->params.wrbs_per_cxn)); 2189 for (j = 0; j < phba->params.wrbs_per_cxn; j++) { 2190 pwrb_handle = pwrb_context->pwrb_handle_base[j]; 2191 pwrb_handle->pwrb = pwrb; 2192 pwrb++; 2193 } 2194 num_cxn_wrb--; 2195 } 2196 } 2197} 2198 2199static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba) 2200{ 2201 struct hwi_controller *phwi_ctrlr; 2202 struct hba_parameters *p = &phba->params; 2203 struct hwi_async_pdu_context *pasync_ctx; 2204 struct async_pdu_handle *pasync_header_h, *pasync_data_h; 2205 unsigned int index; 2206 struct be_mem_descriptor *mem_descr; 2207 2208 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2209 mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT; 2210 2211 phwi_ctrlr = phba->phwi_ctrlr; 2212 phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *) 2213 mem_descr->mem_array[0].virtual_address; 2214 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx; 2215 memset(pasync_ctx, 0, sizeof(*pasync_ctx)); 2216 2217 pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl; 2218 pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz; 2219 pasync_ctx->async_data.buffer_size = p->defpdu_data_sz; 2220 pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl; 2221 2222 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2223 mem_descr += HWI_MEM_ASYNC_HEADER_BUF; 2224 if (mem_descr->mem_array[0].virtual_address) { 2225 SE_DEBUG(DBG_LVL_8, 2226 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF" 2227 "va=%p\n", mem_descr->mem_array[0].virtual_address); 2228 } else 2229 shost_printk(KERN_WARNING, phba->shost, 2230 "No Virtual address\n"); 2231 2232 pasync_ctx->async_header.va_base = 2233 mem_descr->mem_array[0].virtual_address; 2234 2235 pasync_ctx->async_header.pa_base.u.a64.address = 2236 mem_descr->mem_array[0].bus_address.u.a64.address; 2237 2238 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2239 mem_descr += HWI_MEM_ASYNC_HEADER_RING; 2240 if (mem_descr->mem_array[0].virtual_address) { 2241 SE_DEBUG(DBG_LVL_8, 2242 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING" 2243 "va=%p\n", mem_descr->mem_array[0].virtual_address); 2244 } else 2245 shost_printk(KERN_WARNING, phba->shost, 2246 "No Virtual address\n"); 2247 pasync_ctx->async_header.ring_base = 2248 mem_descr->mem_array[0].virtual_address; 2249 2250 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2251 mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE; 2252 if (mem_descr->mem_array[0].virtual_address) { 2253 SE_DEBUG(DBG_LVL_8, 2254 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE" 2255 "va=%p\n", mem_descr->mem_array[0].virtual_address); 2256 } else 2257 shost_printk(KERN_WARNING, phba->shost, 2258 "No Virtual address\n"); 2259 2260 pasync_ctx->async_header.handle_base = 2261 mem_descr->mem_array[0].virtual_address; 2262 pasync_ctx->async_header.writables = 0; 2263 INIT_LIST_HEAD(&pasync_ctx->async_header.free_list); 2264 2265 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2266 mem_descr += HWI_MEM_ASYNC_DATA_BUF; 2267 if (mem_descr->mem_array[0].virtual_address) { 2268 SE_DEBUG(DBG_LVL_8, 2269 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF" 2270 "va=%p\n", mem_descr->mem_array[0].virtual_address); 2271 } else 2272 shost_printk(KERN_WARNING, phba->shost, 2273 "No Virtual address\n"); 2274 pasync_ctx->async_data.va_base = 2275 mem_descr->mem_array[0].virtual_address; 2276 pasync_ctx->async_data.pa_base.u.a64.address = 2277 mem_descr->mem_array[0].bus_address.u.a64.address; 2278 2279 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2280 mem_descr += HWI_MEM_ASYNC_DATA_RING; 2281 if (mem_descr->mem_array[0].virtual_address) { 2282 SE_DEBUG(DBG_LVL_8, 2283 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING" 2284 "va=%p\n", mem_descr->mem_array[0].virtual_address); 2285 } else 2286 shost_printk(KERN_WARNING, phba->shost, 2287 "No Virtual address\n"); 2288 2289 pasync_ctx->async_data.ring_base = 2290 mem_descr->mem_array[0].virtual_address; 2291 2292 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2293 mem_descr += HWI_MEM_ASYNC_DATA_HANDLE; 2294 if (!mem_descr->mem_array[0].virtual_address) 2295 shost_printk(KERN_WARNING, phba->shost, 2296 "No Virtual address\n"); 2297 2298 pasync_ctx->async_data.handle_base = 2299 mem_descr->mem_array[0].virtual_address; 2300 pasync_ctx->async_data.writables = 0; 2301 INIT_LIST_HEAD(&pasync_ctx->async_data.free_list); 2302 2303 pasync_header_h = 2304 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base; 2305 pasync_data_h = 2306 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base; 2307 2308 for (index = 0; index < p->asyncpdus_per_ctrl; index++) { 2309 pasync_header_h->cri = -1; 2310 pasync_header_h->index = (char)index; 2311 INIT_LIST_HEAD(&pasync_header_h->link); 2312 pasync_header_h->pbuffer = 2313 (void *)((unsigned long) 2314 (pasync_ctx->async_header.va_base) + 2315 (p->defpdu_hdr_sz * index)); 2316 2317 pasync_header_h->pa.u.a64.address = 2318 pasync_ctx->async_header.pa_base.u.a64.address + 2319 (p->defpdu_hdr_sz * index); 2320 2321 list_add_tail(&pasync_header_h->link, 2322 &pasync_ctx->async_header.free_list); 2323 pasync_header_h++; 2324 pasync_ctx->async_header.free_entries++; 2325 pasync_ctx->async_header.writables++; 2326 2327 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list); 2328 INIT_LIST_HEAD(&pasync_ctx->async_entry[index]. 2329 header_busy_list); 2330 pasync_data_h->cri = -1; 2331 pasync_data_h->index = (char)index; 2332 INIT_LIST_HEAD(&pasync_data_h->link); 2333 pasync_data_h->pbuffer = 2334 (void *)((unsigned long) 2335 (pasync_ctx->async_data.va_base) + 2336 (p->defpdu_data_sz * index)); 2337 2338 pasync_data_h->pa.u.a64.address = 2339 pasync_ctx->async_data.pa_base.u.a64.address + 2340 (p->defpdu_data_sz * index); 2341 2342 list_add_tail(&pasync_data_h->link, 2343 &pasync_ctx->async_data.free_list); 2344 pasync_data_h++; 2345 pasync_ctx->async_data.free_entries++; 2346 pasync_ctx->async_data.writables++; 2347 2348 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list); 2349 } 2350 2351 pasync_ctx->async_header.host_write_ptr = 0; 2352 pasync_ctx->async_header.ep_read_ptr = -1; 2353 pasync_ctx->async_data.host_write_ptr = 0; 2354 pasync_ctx->async_data.ep_read_ptr = -1; 2355} 2356 2357static int 2358be_sgl_create_contiguous(void *virtual_address, 2359 u64 physical_address, u32 length, 2360 struct be_dma_mem *sgl) 2361{ 2362 WARN_ON(!virtual_address); 2363 WARN_ON(!physical_address); 2364 WARN_ON(!length > 0); 2365 WARN_ON(!sgl); 2366 2367 sgl->va = virtual_address; 2368 sgl->dma = (unsigned long)physical_address; 2369 sgl->size = length; 2370 2371 return 0; 2372} 2373 2374static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl) 2375{ 2376 memset(sgl, 0, sizeof(*sgl)); 2377} 2378 2379static void 2380hwi_build_be_sgl_arr(struct beiscsi_hba *phba, 2381 struct mem_array *pmem, struct be_dma_mem *sgl) 2382{ 2383 if (sgl->va) 2384 be_sgl_destroy_contiguous(sgl); 2385 2386 be_sgl_create_contiguous(pmem->virtual_address, 2387 pmem->bus_address.u.a64.address, 2388 pmem->size, sgl); 2389} 2390 2391static void 2392hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba, 2393 struct mem_array *pmem, struct be_dma_mem *sgl) 2394{ 2395 if (sgl->va) 2396 be_sgl_destroy_contiguous(sgl); 2397 2398 be_sgl_create_contiguous((unsigned char *)pmem->virtual_address, 2399 pmem->bus_address.u.a64.address, 2400 pmem->size, sgl); 2401} 2402 2403static int be_fill_queue(struct be_queue_info *q, 2404 u16 len, u16 entry_size, void *vaddress) 2405{ 2406 struct be_dma_mem *mem = &q->dma_mem; 2407 2408 memset(q, 0, sizeof(*q)); 2409 q->len = len; 2410 q->entry_size = entry_size; 2411 mem->size = len * entry_size; 2412 mem->va = vaddress; 2413 if (!mem->va) 2414 return -ENOMEM; 2415 memset(mem->va, 0, mem->size); 2416 return 0; 2417} 2418 2419static int beiscsi_create_eqs(struct beiscsi_hba *phba, 2420 struct hwi_context_memory *phwi_context) 2421{ 2422 unsigned int i, num_eq_pages; 2423 int ret, eq_for_mcc; 2424 struct be_queue_info *eq; 2425 struct be_dma_mem *mem; 2426 void *eq_vaddress; 2427 dma_addr_t paddr; 2428 2429 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \ 2430 sizeof(struct be_eq_entry)); 2431 2432 if (phba->msix_enabled) 2433 eq_for_mcc = 1; 2434 else 2435 eq_for_mcc = 0; 2436 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) { 2437 eq = &phwi_context->be_eq[i].q; 2438 mem = &eq->dma_mem; 2439 phwi_context->be_eq[i].phba = phba; 2440 eq_vaddress = pci_alloc_consistent(phba->pcidev, 2441 num_eq_pages * PAGE_SIZE, 2442 &paddr); 2443 if (!eq_vaddress) 2444 goto create_eq_error; 2445 2446 mem->va = eq_vaddress; 2447 ret = be_fill_queue(eq, phba->params.num_eq_entries, 2448 sizeof(struct be_eq_entry), eq_vaddress); 2449 if (ret) { 2450 shost_printk(KERN_ERR, phba->shost, 2451 "be_fill_queue Failed for EQ\n"); 2452 goto create_eq_error; 2453 } 2454 2455 mem->dma = paddr; 2456 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq, 2457 phwi_context->cur_eqd); 2458 if (ret) { 2459 shost_printk(KERN_ERR, phba->shost, 2460 "beiscsi_cmd_eq_create" 2461 "Failedfor EQ\n"); 2462 goto create_eq_error; 2463 } 2464 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id); 2465 } 2466 return 0; 2467create_eq_error: 2468 for (i = 0; i < (phba->num_cpus + 1); i++) { 2469 eq = &phwi_context->be_eq[i].q; 2470 mem = &eq->dma_mem; 2471 if (mem->va) 2472 pci_free_consistent(phba->pcidev, num_eq_pages 2473 * PAGE_SIZE, 2474 mem->va, mem->dma); 2475 } 2476 return ret; 2477} 2478 2479static int beiscsi_create_cqs(struct beiscsi_hba *phba, 2480 struct hwi_context_memory *phwi_context) 2481{ 2482 unsigned int i, num_cq_pages; 2483 int ret; 2484 struct be_queue_info *cq, *eq; 2485 struct be_dma_mem *mem; 2486 struct be_eq_obj *pbe_eq; 2487 void *cq_vaddress; 2488 dma_addr_t paddr; 2489 2490 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \ 2491 sizeof(struct sol_cqe)); 2492 2493 for (i = 0; i < phba->num_cpus; i++) { 2494 cq = &phwi_context->be_cq[i]; 2495 eq = &phwi_context->be_eq[i].q; 2496 pbe_eq = &phwi_context->be_eq[i]; 2497 pbe_eq->cq = cq; 2498 pbe_eq->phba = phba; 2499 mem = &cq->dma_mem; 2500 cq_vaddress = pci_alloc_consistent(phba->pcidev, 2501 num_cq_pages * PAGE_SIZE, 2502 &paddr); 2503 if (!cq_vaddress) 2504 goto create_cq_error; 2505 ret = be_fill_queue(cq, phba->params.num_cq_entries, 2506 sizeof(struct sol_cqe), cq_vaddress); 2507 if (ret) { 2508 shost_printk(KERN_ERR, phba->shost, 2509 "be_fill_queue Failed for ISCSI CQ\n"); 2510 goto create_cq_error; 2511 } 2512 2513 mem->dma = paddr; 2514 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false, 2515 false, 0); 2516 if (ret) { 2517 shost_printk(KERN_ERR, phba->shost, 2518 "beiscsi_cmd_eq_create" 2519 "Failed for ISCSI CQ\n"); 2520 goto create_cq_error; 2521 } 2522 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n", 2523 cq->id, eq->id); 2524 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n"); 2525 } 2526 return 0; 2527 2528create_cq_error: 2529 for (i = 0; i < phba->num_cpus; i++) { 2530 cq = &phwi_context->be_cq[i]; 2531 mem = &cq->dma_mem; 2532 if (mem->va) 2533 pci_free_consistent(phba->pcidev, num_cq_pages 2534 * PAGE_SIZE, 2535 mem->va, mem->dma); 2536 } 2537 return ret; 2538 2539} 2540 2541static int 2542beiscsi_create_def_hdr(struct beiscsi_hba *phba, 2543 struct hwi_context_memory *phwi_context, 2544 struct hwi_controller *phwi_ctrlr, 2545 unsigned int def_pdu_ring_sz) 2546{ 2547 unsigned int idx; 2548 int ret; 2549 struct be_queue_info *dq, *cq; 2550 struct be_dma_mem *mem; 2551 struct be_mem_descriptor *mem_descr; 2552 void *dq_vaddress; 2553 2554 idx = 0; 2555 dq = &phwi_context->be_def_hdrq; 2556 cq = &phwi_context->be_cq[0]; 2557 mem = &dq->dma_mem; 2558 mem_descr = phba->init_mem; 2559 mem_descr += HWI_MEM_ASYNC_HEADER_RING; 2560 dq_vaddress = mem_descr->mem_array[idx].virtual_address; 2561 ret = be_fill_queue(dq, mem_descr->mem_array[0].size / 2562 sizeof(struct phys_addr), 2563 sizeof(struct phys_addr), dq_vaddress); 2564 if (ret) { 2565 shost_printk(KERN_ERR, phba->shost, 2566 "be_fill_queue Failed for DEF PDU HDR\n"); 2567 return ret; 2568 } 2569 mem->dma = (unsigned long)mem_descr->mem_array[idx]. 2570 bus_address.u.a64.address; 2571 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq, 2572 def_pdu_ring_sz, 2573 phba->params.defpdu_hdr_sz); 2574 if (ret) { 2575 shost_printk(KERN_ERR, phba->shost, 2576 "be_cmd_create_default_pdu_queue Failed DEFHDR\n"); 2577 return ret; 2578 } 2579 phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id; 2580 SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n", 2581 phwi_context->be_def_hdrq.id); 2582 hwi_post_async_buffers(phba, 1); 2583 return 0; 2584} 2585 2586static int 2587beiscsi_create_def_data(struct beiscsi_hba *phba, 2588 struct hwi_context_memory *phwi_context, 2589 struct hwi_controller *phwi_ctrlr, 2590 unsigned int def_pdu_ring_sz) 2591{ 2592 unsigned int idx; 2593 int ret; 2594 struct be_queue_info *dataq, *cq; 2595 struct be_dma_mem *mem; 2596 struct be_mem_descriptor *mem_descr; 2597 void *dq_vaddress; 2598 2599 idx = 0; 2600 dataq = &phwi_context->be_def_dataq; 2601 cq = &phwi_context->be_cq[0]; 2602 mem = &dataq->dma_mem; 2603 mem_descr = phba->init_mem; 2604 mem_descr += HWI_MEM_ASYNC_DATA_RING; 2605 dq_vaddress = mem_descr->mem_array[idx].virtual_address; 2606 ret = be_fill_queue(dataq, mem_descr->mem_array[0].size / 2607 sizeof(struct phys_addr), 2608 sizeof(struct phys_addr), dq_vaddress); 2609 if (ret) { 2610 shost_printk(KERN_ERR, phba->shost, 2611 "be_fill_queue Failed for DEF PDU DATA\n"); 2612 return ret; 2613 } 2614 mem->dma = (unsigned long)mem_descr->mem_array[idx]. 2615 bus_address.u.a64.address; 2616 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq, 2617 def_pdu_ring_sz, 2618 phba->params.defpdu_data_sz); 2619 if (ret) { 2620 shost_printk(KERN_ERR, phba->shost, 2621 "be_cmd_create_default_pdu_queue Failed" 2622 " for DEF PDU DATA\n"); 2623 return ret; 2624 } 2625 phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id; 2626 SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n", 2627 phwi_context->be_def_dataq.id); 2628 hwi_post_async_buffers(phba, 0); 2629 SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED\n"); 2630 return 0; 2631} 2632 2633static int 2634beiscsi_post_pages(struct beiscsi_hba *phba) 2635{ 2636 struct be_mem_descriptor *mem_descr; 2637 struct mem_array *pm_arr; 2638 unsigned int page_offset, i; 2639 struct be_dma_mem sgl; 2640 int status; 2641 2642 mem_descr = phba->init_mem; 2643 mem_descr += HWI_MEM_SGE; 2644 pm_arr = mem_descr->mem_array; 2645 2646 page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io * 2647 phba->fw_config.iscsi_icd_start) / PAGE_SIZE; 2648 for (i = 0; i < mem_descr->num_elements; i++) { 2649 hwi_build_be_sgl_arr(phba, pm_arr, &sgl); 2650 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl, 2651 page_offset, 2652 (pm_arr->size / PAGE_SIZE)); 2653 page_offset += pm_arr->size / PAGE_SIZE; 2654 if (status != 0) { 2655 shost_printk(KERN_ERR, phba->shost, 2656 "post sgl failed.\n"); 2657 return status; 2658 } 2659 pm_arr++; 2660 } 2661 SE_DEBUG(DBG_LVL_8, "POSTED PAGES\n"); 2662 return 0; 2663} 2664 2665static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q) 2666{ 2667 struct be_dma_mem *mem = &q->dma_mem; 2668 if (mem->va) 2669 pci_free_consistent(phba->pcidev, mem->size, 2670 mem->va, mem->dma); 2671} 2672 2673static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q, 2674 u16 len, u16 entry_size) 2675{ 2676 struct be_dma_mem *mem = &q->dma_mem; 2677 2678 memset(q, 0, sizeof(*q)); 2679 q->len = len; 2680 q->entry_size = entry_size; 2681 mem->size = len * entry_size; 2682 mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma); 2683 if (!mem->va) 2684 return -1; 2685 memset(mem->va, 0, mem->size); 2686 return 0; 2687} 2688 2689static int 2690beiscsi_create_wrb_rings(struct beiscsi_hba *phba, 2691 struct hwi_context_memory *phwi_context, 2692 struct hwi_controller *phwi_ctrlr) 2693{ 2694 unsigned int wrb_mem_index, offset, size, num_wrb_rings; 2695 u64 pa_addr_lo; 2696 unsigned int idx, num, i; 2697 struct mem_array *pwrb_arr; 2698 void *wrb_vaddr; 2699 struct be_dma_mem sgl; 2700 struct be_mem_descriptor *mem_descr; 2701 int status; 2702 2703 idx = 0; 2704 mem_descr = phba->init_mem; 2705 mem_descr += HWI_MEM_WRB; 2706 pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl, 2707 GFP_KERNEL); 2708 if (!pwrb_arr) { 2709 shost_printk(KERN_ERR, phba->shost, 2710 "Memory alloc failed in create wrb ring.\n"); 2711 return -ENOMEM; 2712 } 2713 wrb_vaddr = mem_descr->mem_array[idx].virtual_address; 2714 pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address; 2715 num_wrb_rings = mem_descr->mem_array[idx].size / 2716 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb)); 2717 2718 for (num = 0; num < phba->params.cxns_per_ctrl; num++) { 2719 if (num_wrb_rings) { 2720 pwrb_arr[num].virtual_address = wrb_vaddr; 2721 pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo; 2722 pwrb_arr[num].size = phba->params.wrbs_per_cxn * 2723 sizeof(struct iscsi_wrb); 2724 wrb_vaddr += pwrb_arr[num].size; 2725 pa_addr_lo += pwrb_arr[num].size; 2726 num_wrb_rings--; 2727 } else { 2728 idx++; 2729 wrb_vaddr = mem_descr->mem_array[idx].virtual_address; 2730 pa_addr_lo = mem_descr->mem_array[idx].\ 2731 bus_address.u.a64.address; 2732 num_wrb_rings = mem_descr->mem_array[idx].size / 2733 (phba->params.wrbs_per_cxn * 2734 sizeof(struct iscsi_wrb)); 2735 pwrb_arr[num].virtual_address = wrb_vaddr; 2736 pwrb_arr[num].bus_address.u.a64.address\ 2737 = pa_addr_lo; 2738 pwrb_arr[num].size = phba->params.wrbs_per_cxn * 2739 sizeof(struct iscsi_wrb); 2740 wrb_vaddr += pwrb_arr[num].size; 2741 pa_addr_lo += pwrb_arr[num].size; 2742 num_wrb_rings--; 2743 } 2744 } 2745 for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 2746 wrb_mem_index = 0; 2747 offset = 0; 2748 size = 0; 2749 2750 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl); 2751 status = be_cmd_wrbq_create(&phba->ctrl, &sgl, 2752 &phwi_context->be_wrbq[i]); 2753 if (status != 0) { 2754 shost_printk(KERN_ERR, phba->shost, 2755 "wrbq create failed."); 2756 kfree(pwrb_arr); 2757 return status; 2758 } 2759 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i]. 2760 id; 2761 } 2762 kfree(pwrb_arr); 2763 return 0; 2764} 2765 2766static void free_wrb_handles(struct beiscsi_hba *phba) 2767{ 2768 unsigned int index; 2769 struct hwi_controller *phwi_ctrlr; 2770 struct hwi_wrb_context *pwrb_context; 2771 2772 phwi_ctrlr = phba->phwi_ctrlr; 2773 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) { 2774 pwrb_context = &phwi_ctrlr->wrb_context[index]; 2775 kfree(pwrb_context->pwrb_handle_base); 2776 kfree(pwrb_context->pwrb_handle_basestd); 2777 } 2778} 2779 2780static void be_mcc_queues_destroy(struct beiscsi_hba *phba) 2781{ 2782 struct be_queue_info *q; 2783 struct be_ctrl_info *ctrl = &phba->ctrl; 2784 2785 q = &phba->ctrl.mcc_obj.q; 2786 if (q->created) 2787 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ); 2788 be_queue_free(phba, q); 2789 2790 q = &phba->ctrl.mcc_obj.cq; 2791 if (q->created) 2792 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ); 2793 be_queue_free(phba, q); 2794} 2795 2796static void hwi_cleanup(struct beiscsi_hba *phba) 2797{ 2798 struct be_queue_info *q; 2799 struct be_ctrl_info *ctrl = &phba->ctrl; 2800 struct hwi_controller *phwi_ctrlr; 2801 struct hwi_context_memory *phwi_context; 2802 int i, eq_num; 2803 2804 phwi_ctrlr = phba->phwi_ctrlr; 2805 phwi_context = phwi_ctrlr->phwi_ctxt; 2806 for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 2807 q = &phwi_context->be_wrbq[i]; 2808 if (q->created) 2809 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ); 2810 } 2811 free_wrb_handles(phba); 2812 2813 q = &phwi_context->be_def_hdrq; 2814 if (q->created) 2815 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ); 2816 2817 q = &phwi_context->be_def_dataq; 2818 if (q->created) 2819 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ); 2820 2821 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL); 2822 2823 for (i = 0; i < (phba->num_cpus); i++) { 2824 q = &phwi_context->be_cq[i]; 2825 if (q->created) 2826 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ); 2827 } 2828 if (phba->msix_enabled) 2829 eq_num = 1; 2830 else 2831 eq_num = 0; 2832 for (i = 0; i < (phba->num_cpus + eq_num); i++) { 2833 q = &phwi_context->be_eq[i].q; 2834 if (q->created) 2835 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ); 2836 } 2837 be_mcc_queues_destroy(phba); 2838} 2839 2840static int be_mcc_queues_create(struct beiscsi_hba *phba, 2841 struct hwi_context_memory *phwi_context) 2842{ 2843 struct be_queue_info *q, *cq; 2844 struct be_ctrl_info *ctrl = &phba->ctrl; 2845 2846 /* Alloc MCC compl queue */ 2847 cq = &phba->ctrl.mcc_obj.cq; 2848 if (be_queue_alloc(phba, cq, MCC_CQ_LEN, 2849 sizeof(struct be_mcc_compl))) 2850 goto err; 2851 /* Ask BE to create MCC compl queue; */ 2852 if (phba->msix_enabled) { 2853 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq 2854 [phba->num_cpus].q, false, true, 0)) 2855 goto mcc_cq_free; 2856 } else { 2857 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q, 2858 false, true, 0)) 2859 goto mcc_cq_free; 2860 } 2861 2862 /* Alloc MCC queue */ 2863 q = &phba->ctrl.mcc_obj.q; 2864 if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) 2865 goto mcc_cq_destroy; 2866 2867 /* Ask BE to create MCC queue */ 2868 if (beiscsi_cmd_mccq_create(phba, q, cq)) 2869 goto mcc_q_free; 2870 2871 return 0; 2872 2873mcc_q_free: 2874 be_queue_free(phba, q); 2875mcc_cq_destroy: 2876 beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ); 2877mcc_cq_free: 2878 be_queue_free(phba, cq); 2879err: 2880 return -1; 2881} 2882 2883static int find_num_cpus(void) 2884{ 2885 int num_cpus = 0; 2886 2887 num_cpus = num_online_cpus(); 2888 if (num_cpus >= MAX_CPUS) 2889 num_cpus = MAX_CPUS - 1; 2890 2891 SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", num_cpus); 2892 return num_cpus; 2893} 2894 2895static int hwi_init_port(struct beiscsi_hba *phba) 2896{ 2897 struct hwi_controller *phwi_ctrlr; 2898 struct hwi_context_memory *phwi_context; 2899 unsigned int def_pdu_ring_sz; 2900 struct be_ctrl_info *ctrl = &phba->ctrl; 2901 int status; 2902 2903 def_pdu_ring_sz = 2904 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr); 2905 phwi_ctrlr = phba->phwi_ctrlr; 2906 phwi_context = phwi_ctrlr->phwi_ctxt; 2907 phwi_context->max_eqd = 0; 2908 phwi_context->min_eqd = 0; 2909 phwi_context->cur_eqd = 64; 2910 be_cmd_fw_initialize(&phba->ctrl); 2911 2912 status = beiscsi_create_eqs(phba, phwi_context); 2913 if (status != 0) { 2914 shost_printk(KERN_ERR, phba->shost, "EQ not created\n"); 2915 goto error; 2916 } 2917 2918 status = be_mcc_queues_create(phba, phwi_context); 2919 if (status != 0) 2920 goto error; 2921 2922 status = mgmt_check_supported_fw(ctrl, phba); 2923 if (status != 0) { 2924 shost_printk(KERN_ERR, phba->shost, 2925 "Unsupported fw version\n"); 2926 goto error; 2927 } 2928 2929 status = beiscsi_create_cqs(phba, phwi_context); 2930 if (status != 0) { 2931 shost_printk(KERN_ERR, phba->shost, "CQ not created\n"); 2932 goto error; 2933 } 2934 2935 status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr, 2936 def_pdu_ring_sz); 2937 if (status != 0) { 2938 shost_printk(KERN_ERR, phba->shost, 2939 "Default Header not created\n"); 2940 goto error; 2941 } 2942 2943 status = beiscsi_create_def_data(phba, phwi_context, 2944 phwi_ctrlr, def_pdu_ring_sz); 2945 if (status != 0) { 2946 shost_printk(KERN_ERR, phba->shost, 2947 "Default Data not created\n"); 2948 goto error; 2949 } 2950 2951 status = beiscsi_post_pages(phba); 2952 if (status != 0) { 2953 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n"); 2954 goto error; 2955 } 2956 2957 status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr); 2958 if (status != 0) { 2959 shost_printk(KERN_ERR, phba->shost, 2960 "WRB Rings not created\n"); 2961 goto error; 2962 } 2963 2964 SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n"); 2965 return 0; 2966 2967error: 2968 shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed"); 2969 hwi_cleanup(phba); 2970 return -ENOMEM; 2971} 2972 2973static int hwi_init_controller(struct beiscsi_hba *phba) 2974{ 2975 struct hwi_controller *phwi_ctrlr; 2976 2977 phwi_ctrlr = phba->phwi_ctrlr; 2978 if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) { 2979 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba-> 2980 init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address; 2981 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p\n", 2982 phwi_ctrlr->phwi_ctxt); 2983 } else { 2984 shost_printk(KERN_ERR, phba->shost, 2985 "HWI_MEM_ADDN_CONTEXT is more than one element." 2986 "Failing to load\n"); 2987 return -ENOMEM; 2988 } 2989 2990 iscsi_init_global_templates(phba); 2991 beiscsi_init_wrb_handle(phba); 2992 hwi_init_async_pdu_ctx(phba); 2993 if (hwi_init_port(phba) != 0) { 2994 shost_printk(KERN_ERR, phba->shost, 2995 "hwi_init_controller failed\n"); 2996 return -ENOMEM; 2997 } 2998 return 0; 2999} 3000 3001static void beiscsi_free_mem(struct beiscsi_hba *phba) 3002{ 3003 struct be_mem_descriptor *mem_descr; 3004 int i, j; 3005 3006 mem_descr = phba->init_mem; 3007 i = 0; 3008 j = 0; 3009 for (i = 0; i < SE_MEM_MAX; i++) { 3010 for (j = mem_descr->num_elements; j > 0; j--) { 3011 pci_free_consistent(phba->pcidev, 3012 mem_descr->mem_array[j - 1].size, 3013 mem_descr->mem_array[j - 1].virtual_address, 3014 (unsigned long)mem_descr->mem_array[j - 1]. 3015 bus_address.u.a64.address); 3016 } 3017 kfree(mem_descr->mem_array); 3018 mem_descr++; 3019 } 3020 kfree(phba->init_mem); 3021 kfree(phba->phwi_ctrlr); 3022} 3023 3024static int beiscsi_init_controller(struct beiscsi_hba *phba) 3025{ 3026 int ret = -ENOMEM; 3027 3028 ret = beiscsi_get_memory(phba); 3029 if (ret < 0) { 3030 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -" 3031 "Failed in beiscsi_alloc_memory\n"); 3032 return ret; 3033 } 3034 3035 ret = hwi_init_controller(phba); 3036 if (ret) 3037 goto free_init; 3038 SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller"); 3039 return 0; 3040 3041free_init: 3042 beiscsi_free_mem(phba); 3043 return -ENOMEM; 3044} 3045 3046static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba) 3047{ 3048 struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg; 3049 struct sgl_handle *psgl_handle; 3050 struct iscsi_sge *pfrag; 3051 unsigned int arr_index, i, idx; 3052 3053 phba->io_sgl_hndl_avbl = 0; 3054 phba->eh_sgl_hndl_avbl = 0; 3055 3056 mem_descr_sglh = phba->init_mem; 3057 mem_descr_sglh += HWI_MEM_SGLH; 3058 if (1 == mem_descr_sglh->num_elements) { 3059 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) * 3060 phba->params.ios_per_ctrl, 3061 GFP_KERNEL); 3062 if (!phba->io_sgl_hndl_base) { 3063 shost_printk(KERN_ERR, phba->shost, 3064 "Mem Alloc Failed. Failing to load\n"); 3065 return -ENOMEM; 3066 } 3067 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) * 3068 (phba->params.icds_per_ctrl - 3069 phba->params.ios_per_ctrl), 3070 GFP_KERNEL); 3071 if (!phba->eh_sgl_hndl_base) { 3072 kfree(phba->io_sgl_hndl_base); 3073 shost_printk(KERN_ERR, phba->shost, 3074 "Mem Alloc Failed. Failing to load\n"); 3075 return -ENOMEM; 3076 } 3077 } else { 3078 shost_printk(KERN_ERR, phba->shost, 3079 "HWI_MEM_SGLH is more than one element." 3080 "Failing to load\n"); 3081 return -ENOMEM; 3082 } 3083 3084 arr_index = 0; 3085 idx = 0; 3086 while (idx < mem_descr_sglh->num_elements) { 3087 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address; 3088 3089 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size / 3090 sizeof(struct sgl_handle)); i++) { 3091 if (arr_index < phba->params.ios_per_ctrl) { 3092 phba->io_sgl_hndl_base[arr_index] = psgl_handle; 3093 phba->io_sgl_hndl_avbl++; 3094 arr_index++; 3095 } else { 3096 phba->eh_sgl_hndl_base[arr_index - 3097 phba->params.ios_per_ctrl] = 3098 psgl_handle; 3099 arr_index++; 3100 phba->eh_sgl_hndl_avbl++; 3101 } 3102 psgl_handle++; 3103 } 3104 idx++; 3105 } 3106 SE_DEBUG(DBG_LVL_8, 3107 "phba->io_sgl_hndl_avbl=%d" 3108 "phba->eh_sgl_hndl_avbl=%d\n", 3109 phba->io_sgl_hndl_avbl, 3110 phba->eh_sgl_hndl_avbl); 3111 mem_descr_sg = phba->init_mem; 3112 mem_descr_sg += HWI_MEM_SGE; 3113 SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d\n", 3114 mem_descr_sg->num_elements); 3115 arr_index = 0; 3116 idx = 0; 3117 while (idx < mem_descr_sg->num_elements) { 3118 pfrag = mem_descr_sg->mem_array[idx].virtual_address; 3119 3120 for (i = 0; 3121 i < (mem_descr_sg->mem_array[idx].size) / 3122 (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io); 3123 i++) { 3124 if (arr_index < phba->params.ios_per_ctrl) 3125 psgl_handle = phba->io_sgl_hndl_base[arr_index]; 3126 else 3127 psgl_handle = phba->eh_sgl_hndl_base[arr_index - 3128 phba->params.ios_per_ctrl]; 3129 psgl_handle->pfrag = pfrag; 3130 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0); 3131 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0); 3132 pfrag += phba->params.num_sge_per_io; 3133 psgl_handle->sgl_index = 3134 phba->fw_config.iscsi_icd_start + arr_index++; 3135 } 3136 idx++; 3137 } 3138 phba->io_sgl_free_index = 0; 3139 phba->io_sgl_alloc_index = 0; 3140 phba->eh_sgl_free_index = 0; 3141 phba->eh_sgl_alloc_index = 0; 3142 return 0; 3143} 3144 3145static int hba_setup_cid_tbls(struct beiscsi_hba *phba) 3146{ 3147 int i, new_cid; 3148 3149 phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl, 3150 GFP_KERNEL); 3151 if (!phba->cid_array) { 3152 shost_printk(KERN_ERR, phba->shost, 3153 "Failed to allocate memory in " 3154 "hba_setup_cid_tbls\n"); 3155 return -ENOMEM; 3156 } 3157 phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) * 3158 phba->params.cxns_per_ctrl * 2, GFP_KERNEL); 3159 if (!phba->ep_array) { 3160 shost_printk(KERN_ERR, phba->shost, 3161 "Failed to allocate memory in " 3162 "hba_setup_cid_tbls\n"); 3163 kfree(phba->cid_array); 3164 return -ENOMEM; 3165 } 3166 new_cid = phba->fw_config.iscsi_cid_start; 3167 for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 3168 phba->cid_array[i] = new_cid; 3169 new_cid += 2; 3170 } 3171 phba->avlbl_cids = phba->params.cxns_per_ctrl; 3172 return 0; 3173} 3174 3175static unsigned char hwi_enable_intr(struct beiscsi_hba *phba) 3176{ 3177 struct be_ctrl_info *ctrl = &phba->ctrl; 3178 struct hwi_controller *phwi_ctrlr; 3179 struct hwi_context_memory *phwi_context; 3180 struct be_queue_info *eq; 3181 u8 __iomem *addr; 3182 u32 reg, i; 3183 u32 enabled; 3184 3185 phwi_ctrlr = phba->phwi_ctrlr; 3186 phwi_context = phwi_ctrlr->phwi_ctxt; 3187 3188 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg + 3189 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET); 3190 reg = ioread32(addr); 3191 SE_DEBUG(DBG_LVL_8, "reg =x%08x\n", reg); 3192 3193 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 3194 if (!enabled) { 3195 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 3196 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p\n", reg, addr); 3197 iowrite32(reg, addr); 3198 if (!phba->msix_enabled) { 3199 eq = &phwi_context->be_eq[0].q; 3200 SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id); 3201 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); 3202 } else { 3203 for (i = 0; i <= phba->num_cpus; i++) { 3204 eq = &phwi_context->be_eq[i].q; 3205 SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id); 3206 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); 3207 } 3208 } 3209 } 3210 return true; 3211} 3212 3213static void hwi_disable_intr(struct beiscsi_hba *phba) 3214{ 3215 struct be_ctrl_info *ctrl = &phba->ctrl; 3216 3217 u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET; 3218 u32 reg = ioread32(addr); 3219 3220 u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 3221 if (enabled) { 3222 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 3223 iowrite32(reg, addr); 3224 } else 3225 shost_printk(KERN_WARNING, phba->shost, 3226 "In hwi_disable_intr, Already Disabled\n"); 3227} 3228 3229static int beiscsi_init_port(struct beiscsi_hba *phba) 3230{ 3231 int ret; 3232 3233 ret = beiscsi_init_controller(phba); 3234 if (ret < 0) { 3235 shost_printk(KERN_ERR, phba->shost, 3236 "beiscsi_dev_probe - Failed in" 3237 "beiscsi_init_controller\n"); 3238 return ret; 3239 } 3240 ret = beiscsi_init_sgl_handle(phba); 3241 if (ret < 0) { 3242 shost_printk(KERN_ERR, phba->shost, 3243 "beiscsi_dev_probe - Failed in" 3244 "beiscsi_init_sgl_handle\n"); 3245 goto do_cleanup_ctrlr; 3246 } 3247 3248 if (hba_setup_cid_tbls(phba)) { 3249 shost_printk(KERN_ERR, phba->shost, 3250 "Failed in hba_setup_cid_tbls\n"); 3251 kfree(phba->io_sgl_hndl_base); 3252 kfree(phba->eh_sgl_hndl_base); 3253 goto do_cleanup_ctrlr; 3254 } 3255 3256 return ret; 3257 3258do_cleanup_ctrlr: 3259 hwi_cleanup(phba); 3260 return ret; 3261} 3262 3263static void hwi_purge_eq(struct beiscsi_hba *phba) 3264{ 3265 struct hwi_controller *phwi_ctrlr; 3266 struct hwi_context_memory *phwi_context; 3267 struct be_queue_info *eq; 3268 struct be_eq_entry *eqe = NULL; 3269 int i, eq_msix; 3270 unsigned int num_processed; 3271 3272 phwi_ctrlr = phba->phwi_ctrlr; 3273 phwi_context = phwi_ctrlr->phwi_ctxt; 3274 if (phba->msix_enabled) 3275 eq_msix = 1; 3276 else 3277 eq_msix = 0; 3278 3279 for (i = 0; i < (phba->num_cpus + eq_msix); i++) { 3280 eq = &phwi_context->be_eq[i].q; 3281 eqe = queue_tail_node(eq); 3282 num_processed = 0; 3283 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 3284 & EQE_VALID_MASK) { 3285 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 3286 queue_tail_inc(eq); 3287 eqe = queue_tail_node(eq); 3288 num_processed++; 3289 } 3290 3291 if (num_processed) 3292 hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1); 3293 } 3294} 3295 3296static void beiscsi_clean_port(struct beiscsi_hba *phba) 3297{ 3298 unsigned char mgmt_status; 3299 3300 mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0); 3301 if (mgmt_status) 3302 shost_printk(KERN_WARNING, phba->shost, 3303 "mgmt_epfw_cleanup FAILED\n"); 3304 3305 hwi_purge_eq(phba); 3306 hwi_cleanup(phba); 3307 kfree(phba->io_sgl_hndl_base); 3308 kfree(phba->eh_sgl_hndl_base); 3309 kfree(phba->cid_array); 3310 kfree(phba->ep_array); 3311} 3312 3313void 3314beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn, 3315 struct beiscsi_offload_params *params) 3316{ 3317 struct wrb_handle *pwrb_handle; 3318 struct iscsi_target_context_update_wrb *pwrb = NULL; 3319 struct be_mem_descriptor *mem_descr; 3320 struct beiscsi_hba *phba = beiscsi_conn->phba; 3321 u32 doorbell = 0; 3322 3323 /* 3324 * We can always use 0 here because it is reserved by libiscsi for 3325 * login/startup related tasks. 3326 */ 3327 pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid - 3328 phba->fw_config.iscsi_cid_start)); 3329 pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb; 3330 memset(pwrb, 0, sizeof(*pwrb)); 3331 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 3332 max_burst_length, pwrb, params->dw[offsetof 3333 (struct amap_beiscsi_offload_params, 3334 max_burst_length) / 32]); 3335 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 3336 max_send_data_segment_length, pwrb, 3337 params->dw[offsetof(struct amap_beiscsi_offload_params, 3338 max_send_data_segment_length) / 32]); 3339 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 3340 first_burst_length, 3341 pwrb, 3342 params->dw[offsetof(struct amap_beiscsi_offload_params, 3343 first_burst_length) / 32]); 3344 3345 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb, 3346 (params->dw[offsetof(struct amap_beiscsi_offload_params, 3347 erl) / 32] & OFFLD_PARAMS_ERL)); 3348 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb, 3349 (params->dw[offsetof(struct amap_beiscsi_offload_params, 3350 dde) / 32] & OFFLD_PARAMS_DDE) >> 2); 3351 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb, 3352 (params->dw[offsetof(struct amap_beiscsi_offload_params, 3353 hde) / 32] & OFFLD_PARAMS_HDE) >> 3); 3354 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb, 3355 (params->dw[offsetof(struct amap_beiscsi_offload_params, 3356 ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4); 3357 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb, 3358 (params->dw[offsetof(struct amap_beiscsi_offload_params, 3359 imd) / 32] & OFFLD_PARAMS_IMD) >> 5); 3360 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn, 3361 pwrb, 3362 (params->dw[offsetof(struct amap_beiscsi_offload_params, 3363 exp_statsn) / 32] + 1)); 3364 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb, 3365 0x7); 3366 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx, 3367 pwrb, pwrb_handle->wrb_index); 3368 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb, 3369 pwrb, pwrb_handle->nxt_wrb_index); 3370 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 3371 session_state, pwrb, 0); 3372 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack, 3373 pwrb, 1); 3374 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq, 3375 pwrb, 0); 3376 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb, 3377 0); 3378 3379 mem_descr = phba->init_mem; 3380 mem_descr += ISCSI_MEM_GLOBAL_HEADER; 3381 3382 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 3383 pad_buffer_addr_hi, pwrb, 3384 mem_descr->mem_array[0].bus_address.u.a32.address_hi); 3385 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 3386 pad_buffer_addr_lo, pwrb, 3387 mem_descr->mem_array[0].bus_address.u.a32.address_lo); 3388 3389 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb)); 3390 3391 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 3392 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK) 3393 << DB_DEF_PDU_WRB_INDEX_SHIFT; 3394 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 3395 3396 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET); 3397} 3398 3399static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt, 3400 int *index, int *age) 3401{ 3402 *index = (int)itt; 3403 if (age) 3404 *age = conn->session->age; 3405} 3406 3407/** 3408 * beiscsi_alloc_pdu - allocates pdu and related resources 3409 * @task: libiscsi task 3410 * @opcode: opcode of pdu for task 3411 * 3412 * This is called with the session lock held. It will allocate 3413 * the wrb and sgl if needed for the command. And it will prep 3414 * the pdu's itt. beiscsi_parse_pdu will later translate 3415 * the pdu itt to the libiscsi task itt. 3416 */ 3417static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode) 3418{ 3419 struct beiscsi_io_task *io_task = task->dd_data; 3420 struct iscsi_conn *conn = task->conn; 3421 struct beiscsi_conn *beiscsi_conn = conn->dd_data; 3422 struct beiscsi_hba *phba = beiscsi_conn->phba; 3423 struct hwi_wrb_context *pwrb_context; 3424 struct hwi_controller *phwi_ctrlr; 3425 itt_t itt; 3426 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess; 3427 dma_addr_t paddr; 3428 3429 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool, 3430 GFP_KERNEL, &paddr); 3431 if (!io_task->cmd_bhs) 3432 return -ENOMEM; 3433 io_task->bhs_pa.u.a64.address = paddr; 3434 io_task->libiscsi_itt = (itt_t)task->itt; 3435 io_task->pwrb_handle = alloc_wrb_handle(phba, 3436 beiscsi_conn->beiscsi_conn_cid - 3437 phba->fw_config.iscsi_cid_start 3438 ); 3439 io_task->conn = beiscsi_conn; 3440 3441 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr; 3442 task->hdr_max = sizeof(struct be_cmd_bhs); 3443 3444 if (task->sc) { 3445 spin_lock(&phba->io_sgl_lock); 3446 io_task->psgl_handle = alloc_io_sgl_handle(phba); 3447 spin_unlock(&phba->io_sgl_lock); 3448 if (!io_task->psgl_handle) 3449 goto free_hndls; 3450 } else { 3451 io_task->scsi_cmnd = NULL; 3452 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) { 3453 if (!beiscsi_conn->login_in_progress) { 3454 spin_lock(&phba->mgmt_sgl_lock); 3455 io_task->psgl_handle = (struct sgl_handle *) 3456 alloc_mgmt_sgl_handle(phba); 3457 spin_unlock(&phba->mgmt_sgl_lock); 3458 if (!io_task->psgl_handle) 3459 goto free_hndls; 3460 3461 beiscsi_conn->login_in_progress = 1; 3462 beiscsi_conn->plogin_sgl_handle = 3463 io_task->psgl_handle; 3464 } else { 3465 io_task->psgl_handle = 3466 beiscsi_conn->plogin_sgl_handle; 3467 } 3468 } else { 3469 spin_lock(&phba->mgmt_sgl_lock); 3470 io_task->psgl_handle = alloc_mgmt_sgl_handle(phba); 3471 spin_unlock(&phba->mgmt_sgl_lock); 3472 if (!io_task->psgl_handle) 3473 goto free_hndls; 3474 } 3475 } 3476 itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle-> 3477 wrb_index << 16) | (unsigned int) 3478 (io_task->psgl_handle->sgl_index)); 3479 io_task->pwrb_handle->pio_handle = task; 3480 3481 io_task->cmd_bhs->iscsi_hdr.itt = itt; 3482 return 0; 3483 3484free_hndls: 3485 phwi_ctrlr = phba->phwi_ctrlr; 3486 pwrb_context = &phwi_ctrlr->wrb_context[ 3487 beiscsi_conn->beiscsi_conn_cid - 3488 phba->fw_config.iscsi_cid_start]; 3489 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle); 3490 io_task->pwrb_handle = NULL; 3491 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs, 3492 io_task->bhs_pa.u.a64.address); 3493 SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed\n"); 3494 return -ENOMEM; 3495} 3496 3497static void beiscsi_cleanup_task(struct iscsi_task *task) 3498{ 3499 struct beiscsi_io_task *io_task = task->dd_data; 3500 struct iscsi_conn *conn = task->conn; 3501 struct beiscsi_conn *beiscsi_conn = conn->dd_data; 3502 struct beiscsi_hba *phba = beiscsi_conn->phba; 3503 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess; 3504 struct hwi_wrb_context *pwrb_context; 3505 struct hwi_controller *phwi_ctrlr; 3506 3507 phwi_ctrlr = phba->phwi_ctrlr; 3508 pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid 3509 - phba->fw_config.iscsi_cid_start]; 3510 if (io_task->pwrb_handle) { 3511 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle); 3512 io_task->pwrb_handle = NULL; 3513 } 3514 3515 if (io_task->cmd_bhs) { 3516 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs, 3517 io_task->bhs_pa.u.a64.address); 3518 } 3519 3520 if (task->sc) { 3521 if (io_task->psgl_handle) { 3522 spin_lock(&phba->io_sgl_lock); 3523 free_io_sgl_handle(phba, io_task->psgl_handle); 3524 spin_unlock(&phba->io_sgl_lock); 3525 io_task->psgl_handle = NULL; 3526 } 3527 } else { 3528 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) 3529 return; 3530 if (io_task->psgl_handle) { 3531 spin_lock(&phba->mgmt_sgl_lock); 3532 free_mgmt_sgl_handle(phba, io_task->psgl_handle); 3533 spin_unlock(&phba->mgmt_sgl_lock); 3534 io_task->psgl_handle = NULL; 3535 } 3536 } 3537} 3538 3539static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg, 3540 unsigned int num_sg, unsigned int xferlen, 3541 unsigned int writedir) 3542{ 3543 3544 struct beiscsi_io_task *io_task = task->dd_data; 3545 struct iscsi_conn *conn = task->conn; 3546 struct beiscsi_conn *beiscsi_conn = conn->dd_data; 3547 struct beiscsi_hba *phba = beiscsi_conn->phba; 3548 struct iscsi_wrb *pwrb = NULL; 3549 unsigned int doorbell = 0; 3550 3551 pwrb = io_task->pwrb_handle->pwrb; 3552 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0; 3553 io_task->bhs_len = sizeof(struct be_cmd_bhs); 3554 3555 if (writedir) { 3556 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48); 3557 AMAP_SET_BITS(struct amap_pdu_data_out, itt, 3558 &io_task->cmd_bhs->iscsi_data_pdu, 3559 (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt); 3560 AMAP_SET_BITS(struct amap_pdu_data_out, opcode, 3561 &io_task->cmd_bhs->iscsi_data_pdu, 3562 ISCSI_OPCODE_SCSI_DATA_OUT); 3563 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit, 3564 &io_task->cmd_bhs->iscsi_data_pdu, 1); 3565 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 3566 INI_WR_CMD); 3567 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1); 3568 } else { 3569 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 3570 INI_RD_CMD); 3571 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); 3572 } 3573 memcpy(&io_task->cmd_bhs->iscsi_data_pdu. 3574 dw[offsetof(struct amap_pdu_data_out, lun) / 32], 3575 io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun)); 3576 3577 AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb, 3578 cpu_to_be16((unsigned short)io_task->cmd_bhs->iscsi_hdr. 3579 lun[0])); 3580 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen); 3581 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb, 3582 io_task->pwrb_handle->wrb_index); 3583 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 3584 be32_to_cpu(task->cmdsn)); 3585 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb, 3586 io_task->psgl_handle->sgl_index); 3587 3588 hwi_write_sgl(pwrb, sg, num_sg, io_task); 3589 3590 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb, 3591 io_task->pwrb_handle->nxt_wrb_index); 3592 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb)); 3593 3594 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 3595 doorbell |= (io_task->pwrb_handle->wrb_index & 3596 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT; 3597 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 3598 3599 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET); 3600 return 0; 3601} 3602 3603static int beiscsi_mtask(struct iscsi_task *task) 3604{ 3605 struct beiscsi_io_task *io_task = task->dd_data; 3606 struct iscsi_conn *conn = task->conn; 3607 struct beiscsi_conn *beiscsi_conn = conn->dd_data; 3608 struct beiscsi_hba *phba = beiscsi_conn->phba; 3609 struct iscsi_wrb *pwrb = NULL; 3610 unsigned int doorbell = 0; 3611 unsigned int cid; 3612 3613 cid = beiscsi_conn->beiscsi_conn_cid; 3614 pwrb = io_task->pwrb_handle->pwrb; 3615 memset(pwrb, 0, sizeof(*pwrb)); 3616 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 3617 be32_to_cpu(task->cmdsn)); 3618 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb, 3619 io_task->pwrb_handle->wrb_index); 3620 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb, 3621 io_task->psgl_handle->sgl_index); 3622 3623 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) { 3624 case ISCSI_OP_LOGIN: 3625 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 3626 TGT_DM_CMD); 3627 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 3628 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1); 3629 hwi_write_buffer(pwrb, task); 3630 break; 3631 case ISCSI_OP_NOOP_OUT: 3632 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 3633 INI_RD_CMD); 3634 if (task->hdr->ttt == ISCSI_RESERVED_TAG) 3635 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 3636 else 3637 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1); 3638 hwi_write_buffer(pwrb, task); 3639 break; 3640 case ISCSI_OP_TEXT: 3641 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 3642 TGT_DM_CMD); 3643 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 3644 hwi_write_buffer(pwrb, task); 3645 break; 3646 case ISCSI_OP_SCSI_TMFUNC: 3647 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 3648 INI_TMF_CMD); 3649 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 3650 hwi_write_buffer(pwrb, task); 3651 break; 3652 case ISCSI_OP_LOGOUT: 3653 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 3654 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 3655 HWH_TYPE_LOGOUT); 3656 hwi_write_buffer(pwrb, task); 3657 break; 3658 3659 default: 3660 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported\n", 3661 task->hdr->opcode & ISCSI_OPCODE_MASK); 3662 return -EINVAL; 3663 } 3664 3665 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, 3666 task->data_count); 3667 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb, 3668 io_task->pwrb_handle->nxt_wrb_index); 3669 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb)); 3670 3671 doorbell |= cid & DB_WRB_POST_CID_MASK; 3672 doorbell |= (io_task->pwrb_handle->wrb_index & 3673 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT; 3674 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 3675 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET); 3676 return 0; 3677} 3678 3679static int beiscsi_task_xmit(struct iscsi_task *task) 3680{ 3681 struct beiscsi_io_task *io_task = task->dd_data; 3682 struct scsi_cmnd *sc = task->sc; 3683 struct scatterlist *sg; 3684 int num_sg; 3685 unsigned int writedir = 0, xferlen = 0; 3686 3687 if (!sc) 3688 return beiscsi_mtask(task); 3689 3690 io_task->scsi_cmnd = sc; 3691 num_sg = scsi_dma_map(sc); 3692 if (num_sg < 0) { 3693 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n") 3694 return num_sg; 3695 } 3696 SE_DEBUG(DBG_LVL_4, "xferlen=0x%08x scmd=%p num_sg=%d sernum=%lu\n", 3697 (scsi_bufflen(sc)), sc, num_sg, sc->serial_number); 3698 xferlen = scsi_bufflen(sc); 3699 sg = scsi_sglist(sc); 3700 if (sc->sc_data_direction == DMA_TO_DEVICE) { 3701 writedir = 1; 3702 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x\n", 3703 task->imm_count); 3704 } else 3705 writedir = 0; 3706 return beiscsi_iotask(task, sg, num_sg, xferlen, writedir); 3707} 3708 3709static void beiscsi_remove(struct pci_dev *pcidev) 3710{ 3711 struct beiscsi_hba *phba = NULL; 3712 struct hwi_controller *phwi_ctrlr; 3713 struct hwi_context_memory *phwi_context; 3714 struct be_eq_obj *pbe_eq; 3715 unsigned int i, msix_vec; 3716 3717 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev); 3718 if (!phba) { 3719 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n"); 3720 return; 3721 } 3722 3723 phwi_ctrlr = phba->phwi_ctrlr; 3724 phwi_context = phwi_ctrlr->phwi_ctxt; 3725 hwi_disable_intr(phba); 3726 if (phba->msix_enabled) { 3727 for (i = 0; i <= phba->num_cpus; i++) { 3728 msix_vec = phba->msix_entries[i].vector; 3729 free_irq(msix_vec, &phwi_context->be_eq[i]); 3730 } 3731 } else 3732 if (phba->pcidev->irq) 3733 free_irq(phba->pcidev->irq, phba); 3734 pci_disable_msix(phba->pcidev); 3735 destroy_workqueue(phba->wq); 3736 if (blk_iopoll_enabled) 3737 for (i = 0; i < phba->num_cpus; i++) { 3738 pbe_eq = &phwi_context->be_eq[i]; 3739 blk_iopoll_disable(&pbe_eq->iopoll); 3740 } 3741 3742 beiscsi_clean_port(phba); 3743 beiscsi_free_mem(phba); 3744 beiscsi_unmap_pci_function(phba); 3745 pci_free_consistent(phba->pcidev, 3746 phba->ctrl.mbox_mem_alloced.size, 3747 phba->ctrl.mbox_mem_alloced.va, 3748 phba->ctrl.mbox_mem_alloced.dma); 3749 iscsi_host_remove(phba->shost); 3750 pci_dev_put(phba->pcidev); 3751 iscsi_host_free(phba->shost); 3752} 3753 3754static void beiscsi_msix_enable(struct beiscsi_hba *phba) 3755{ 3756 int i, status; 3757 3758 for (i = 0; i <= phba->num_cpus; i++) 3759 phba->msix_entries[i].entry = i; 3760 3761 status = pci_enable_msix(phba->pcidev, phba->msix_entries, 3762 (phba->num_cpus + 1)); 3763 if (!status) 3764 phba->msix_enabled = true; 3765 3766 return; 3767} 3768 3769static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev, 3770 const struct pci_device_id *id) 3771{ 3772 struct beiscsi_hba *phba = NULL; 3773 struct hwi_controller *phwi_ctrlr; 3774 struct hwi_context_memory *phwi_context; 3775 struct be_eq_obj *pbe_eq; 3776 int ret, msix_vec, num_cpus, i; 3777 3778 ret = beiscsi_enable_pci(pcidev); 3779 if (ret < 0) { 3780 dev_err(&pcidev->dev, "beiscsi_dev_probe-" 3781 " Failed to enable pci device\n"); 3782 return ret; 3783 } 3784 3785 phba = beiscsi_hba_alloc(pcidev); 3786 if (!phba) { 3787 dev_err(&pcidev->dev, "beiscsi_dev_probe-" 3788 " Failed in beiscsi_hba_alloc\n"); 3789 goto disable_pci; 3790 } 3791 3792 switch (pcidev->device) { 3793 case BE_DEVICE_ID1: 3794 case OC_DEVICE_ID1: 3795 case OC_DEVICE_ID2: 3796 phba->generation = BE_GEN2; 3797 break; 3798 case BE_DEVICE_ID2: 3799 case OC_DEVICE_ID3: 3800 phba->generation = BE_GEN3; 3801 break; 3802 default: 3803 phba->generation = 0; 3804 } 3805 3806 if (enable_msix) 3807 num_cpus = find_num_cpus(); 3808 else 3809 num_cpus = 1; 3810 phba->num_cpus = num_cpus; 3811 SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", phba->num_cpus); 3812 3813 if (enable_msix) 3814 beiscsi_msix_enable(phba); 3815 ret = be_ctrl_init(phba, pcidev); 3816 if (ret) { 3817 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" 3818 "Failed in be_ctrl_init\n"); 3819 goto hba_free; 3820 } 3821 3822 spin_lock_init(&phba->io_sgl_lock); 3823 spin_lock_init(&phba->mgmt_sgl_lock); 3824 spin_lock_init(&phba->isr_lock); 3825 ret = mgmt_get_fw_config(&phba->ctrl, phba); 3826 if (ret != 0) { 3827 shost_printk(KERN_ERR, phba->shost, 3828 "Error getting fw config\n"); 3829 goto free_port; 3830 } 3831 phba->shost->max_id = phba->fw_config.iscsi_cid_count; 3832 beiscsi_get_params(phba); 3833 phba->shost->can_queue = phba->params.ios_per_ctrl; 3834 ret = beiscsi_init_port(phba); 3835 if (ret < 0) { 3836 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" 3837 "Failed in beiscsi_init_port\n"); 3838 goto free_port; 3839 } 3840 3841 for (i = 0; i < MAX_MCC_CMD ; i++) { 3842 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]); 3843 phba->ctrl.mcc_tag[i] = i + 1; 3844 phba->ctrl.mcc_numtag[i + 1] = 0; 3845 phba->ctrl.mcc_tag_available++; 3846 } 3847 3848 phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0; 3849 3850 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u", 3851 phba->shost->host_no); 3852 phba->wq = create_workqueue(phba->wq_name); 3853 if (!phba->wq) { 3854 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" 3855 "Failed to allocate work queue\n"); 3856 goto free_twq; 3857 } 3858 3859 INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs); 3860 3861 phwi_ctrlr = phba->phwi_ctrlr; 3862 phwi_context = phwi_ctrlr->phwi_ctxt; 3863 if (blk_iopoll_enabled) { 3864 for (i = 0; i < phba->num_cpus; i++) { 3865 pbe_eq = &phwi_context->be_eq[i]; 3866 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget, 3867 be_iopoll); 3868 blk_iopoll_enable(&pbe_eq->iopoll); 3869 } 3870 } 3871 ret = beiscsi_init_irqs(phba); 3872 if (ret < 0) { 3873 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" 3874 "Failed to beiscsi_init_irqs\n"); 3875 goto free_blkenbld; 3876 } 3877 ret = hwi_enable_intr(phba); 3878 if (ret < 0) { 3879 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" 3880 "Failed to hwi_enable_intr\n"); 3881 goto free_ctrlr; 3882 } 3883 SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED\n\n\n"); 3884 return 0; 3885 3886free_ctrlr: 3887 if (phba->msix_enabled) { 3888 for (i = 0; i <= phba->num_cpus; i++) { 3889 msix_vec = phba->msix_entries[i].vector; 3890 free_irq(msix_vec, &phwi_context->be_eq[i]); 3891 } 3892 } else 3893 if (phba->pcidev->irq) 3894 free_irq(phba->pcidev->irq, phba); 3895 pci_disable_msix(phba->pcidev); 3896free_blkenbld: 3897 destroy_workqueue(phba->wq); 3898 if (blk_iopoll_enabled) 3899 for (i = 0; i < phba->num_cpus; i++) { 3900 pbe_eq = &phwi_context->be_eq[i]; 3901 blk_iopoll_disable(&pbe_eq->iopoll); 3902 } 3903free_twq: 3904 beiscsi_clean_port(phba); 3905 beiscsi_free_mem(phba); 3906free_port: 3907 pci_free_consistent(phba->pcidev, 3908 phba->ctrl.mbox_mem_alloced.size, 3909 phba->ctrl.mbox_mem_alloced.va, 3910 phba->ctrl.mbox_mem_alloced.dma); 3911 beiscsi_unmap_pci_function(phba); 3912hba_free: 3913 iscsi_host_remove(phba->shost); 3914 pci_dev_put(phba->pcidev); 3915 iscsi_host_free(phba->shost); 3916disable_pci: 3917 pci_disable_device(pcidev); 3918 return ret; 3919} 3920 3921struct iscsi_transport beiscsi_iscsi_transport = { 3922 .owner = THIS_MODULE, 3923 .name = DRV_NAME, 3924 .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO | 3925 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD, 3926 .param_mask = ISCSI_MAX_RECV_DLENGTH | 3927 ISCSI_MAX_XMIT_DLENGTH | 3928 ISCSI_HDRDGST_EN | 3929 ISCSI_DATADGST_EN | 3930 ISCSI_INITIAL_R2T_EN | 3931 ISCSI_MAX_R2T | 3932 ISCSI_IMM_DATA_EN | 3933 ISCSI_FIRST_BURST | 3934 ISCSI_MAX_BURST | 3935 ISCSI_PDU_INORDER_EN | 3936 ISCSI_DATASEQ_INORDER_EN | 3937 ISCSI_ERL | 3938 ISCSI_CONN_PORT | 3939 ISCSI_CONN_ADDRESS | 3940 ISCSI_EXP_STATSN | 3941 ISCSI_PERSISTENT_PORT | 3942 ISCSI_PERSISTENT_ADDRESS | 3943 ISCSI_TARGET_NAME | ISCSI_TPGT | 3944 ISCSI_USERNAME | ISCSI_PASSWORD | 3945 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN | 3946 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO | 3947 ISCSI_LU_RESET_TMO | 3948 ISCSI_PING_TMO | ISCSI_RECV_TMO | 3949 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME, 3950 .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS | 3951 ISCSI_HOST_INITIATOR_NAME, 3952 .create_session = beiscsi_session_create, 3953 .destroy_session = beiscsi_session_destroy, 3954 .create_conn = beiscsi_conn_create, 3955 .bind_conn = beiscsi_conn_bind, 3956 .destroy_conn = iscsi_conn_teardown, 3957 .set_param = beiscsi_set_param, 3958 .get_conn_param = beiscsi_conn_get_param, 3959 .get_session_param = iscsi_session_get_param, 3960 .get_host_param = beiscsi_get_host_param, 3961 .start_conn = beiscsi_conn_start, 3962 .stop_conn = iscsi_conn_stop, 3963 .send_pdu = iscsi_conn_send_pdu, 3964 .xmit_task = beiscsi_task_xmit, 3965 .cleanup_task = beiscsi_cleanup_task, 3966 .alloc_pdu = beiscsi_alloc_pdu, 3967 .parse_pdu_itt = beiscsi_parse_pdu, 3968 .get_stats = beiscsi_conn_get_stats, 3969 .ep_connect = beiscsi_ep_connect, 3970 .ep_poll = beiscsi_ep_poll, 3971 .ep_disconnect = beiscsi_ep_disconnect, 3972 .session_recovery_timedout = iscsi_session_recovery_timedout, 3973}; 3974 3975static struct pci_driver beiscsi_pci_driver = { 3976 .name = DRV_NAME, 3977 .probe = beiscsi_dev_probe, 3978 .remove = beiscsi_remove, 3979 .id_table = beiscsi_pci_id_table 3980}; 3981 3982 3983static int __init beiscsi_module_init(void) 3984{ 3985 int ret; 3986 3987 beiscsi_scsi_transport = 3988 iscsi_register_transport(&beiscsi_iscsi_transport); 3989 if (!beiscsi_scsi_transport) { 3990 SE_DEBUG(DBG_LVL_1, 3991 "beiscsi_module_init - Unable to register beiscsi" 3992 "transport.\n"); 3993 return -ENOMEM; 3994 } 3995 SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p\n", 3996 &beiscsi_iscsi_transport); 3997 3998 ret = pci_register_driver(&beiscsi_pci_driver); 3999 if (ret) { 4000 SE_DEBUG(DBG_LVL_1, 4001 "beiscsi_module_init - Unable to register" 4002 "beiscsi pci driver.\n"); 4003 goto unregister_iscsi_transport; 4004 } 4005 return 0; 4006 4007unregister_iscsi_transport: 4008 iscsi_unregister_transport(&beiscsi_iscsi_transport); 4009 return ret; 4010} 4011 4012static void __exit beiscsi_module_exit(void) 4013{ 4014 pci_unregister_driver(&beiscsi_pci_driver); 4015 iscsi_unregister_transport(&beiscsi_iscsi_transport); 4016} 4017 4018module_init(beiscsi_module_init); 4019module_exit(beiscsi_module_exit); 4020