Lines Matching refs:ic
66 static void rds_iw_send_unmap_rdma(struct rds_iw_connection *ic,
70 ib_dma_unmap_sg(ic->i_cm_id->device,
77 static void rds_iw_send_unmap_rm(struct rds_iw_connection *ic,
83 rdsdebug("ic %p send %p rm %p\n", ic, send, rm);
85 ib_dma_unmap_sg(ic->i_cm_id->device,
90 rds_iw_send_unmap_rdma(ic, &rm->rdma);
128 void rds_iw_send_init_ring(struct rds_iw_connection *ic)
133 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {
148 sge = rds_iw_data_sge(ic, send->s_sge);
151 sge = rds_iw_header_sge(ic, send->s_sge);
152 sge->addr = ic->i_send_hdrs_dma + (i * sizeof(struct rds_header));
156 send->s_mr = ib_alloc_fast_reg_mr(ic->i_pd, fastreg_message_size);
163 ic->i_cm_id->device, fastreg_message_size);
171 void rds_iw_send_clear_ring(struct rds_iw_connection *ic)
176 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {
184 rds_iw_send_unmap_rm(ic, send, IB_WC_WR_FLUSH_ERR);
186 rds_iw_send_unmap_rdma(ic, send->s_op);
199 struct rds_iw_connection *ic = conn->c_transport_data;
225 ic->i_fastreg_posted = 0;
230 ic->i_fastreg_posted = 1;
235 if (time_after(jiffies, ic->i_ack_queued + HZ/2))
237 rds_iw_ack_send_complete(ic);
241 oldest = rds_iw_ring_oldest(&ic->i_send_ring);
243 completed = rds_iw_ring_completed(&ic->i_send_ring, wc.wr_id, oldest);
246 send = &ic->i_sends[oldest];
252 rds_iw_send_unmap_rm(ic, send, wc.status);
285 oldest = (oldest + 1) % ic->i_send_ring.w_nr;
288 rds_iw_ring_free(&ic->i_send_ring, completed);
348 int rds_iw_send_grab_credits(struct rds_iw_connection *ic,
355 if (!ic->i_flowctl)
360 oldval = newval = atomic_read(&ic->i_credits);
372 struct rds_connection *conn = ic->i_cm_id->context;
394 if (atomic_cmpxchg(&ic->i_credits, oldval, newval) != oldval)
403 struct rds_iw_connection *ic = conn->c_transport_data;
410 IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits)),
413 atomic_add(IB_SET_SEND_CREDITS(credits), &ic->i_credits);
424 struct rds_iw_connection *ic = conn->c_transport_data;
429 atomic_add(IB_SET_POST_CREDITS(posted), &ic->i_credits);
443 if (IB_GET_POST_CREDITS(atomic_read(&ic->i_credits)) >= 16)
444 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
448 rds_iw_xmit_populate_wr(struct rds_iw_connection *ic,
455 WARN_ON(pos != send - ic->i_sends);
465 sge = rds_iw_data_sge(ic, send->s_sge);
468 sge->lkey = rds_iw_local_dma_lkey(ic);
470 sge = rds_iw_header_sge(ic, send->s_sge);
478 sge->addr = ic->i_send_hdrs_dma + (pos * sizeof(struct rds_header));
480 sge->lkey = rds_iw_local_dma_lkey(ic);
499 struct rds_iw_connection *ic = conn->c_transport_data;
500 struct ib_device *dev = ic->i_cm_id->device;
521 if (rds_rdma_cookie_key(rm->m_rdma_cookie) && !ic->i_fastreg_posted) {
532 work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, i, &pos);
541 if (ic->i_flowctl) {
542 credit_alloc = rds_iw_send_grab_credits(ic, work_alloc, &posted, 0, RDS_MAX_ADV_CREDIT);
545 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - credit_alloc);
558 if (!ic->i_rm) {
570 rdsdebug("ic %p mapping rm %p: %d\n", ic, rm, rm->data.op_count);
573 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);
581 ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs;
582 ic->i_unsignaled_bytes = rds_iw_sysctl_max_unsig_bytes;
584 ic->i_rm = rm;
611 rm->m_inc.i_hdr.h_ack = cpu_to_be64(rds_iw_piggyb_ack(ic));
617 rds_iw_send_grab_credits(ic, 0, &posted, 1, RDS_MAX_ADV_CREDIT - adv_credits);
622 send = &ic->i_sends[pos];
649 rds_iw_xmit_populate_wr(ic, send, pos, 0, 0, send_flags);
657 send = &ic->i_sends[pos];
660 rds_iw_xmit_populate_wr(ic, send, pos,
669 if (ic->i_unsignaled_wrs-- == 0) {
670 ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs;
674 ic->i_unsignaled_bytes -= len;
675 if (ic->i_unsignaled_bytes <= 0) {
676 ic->i_unsignaled_bytes = rds_iw_sysctl_max_unsig_bytes;
699 memcpy(&ic->i_send_hdrs[pos], &rm->m_inc.i_hdr, sizeof(struct rds_header));
702 struct rds_header *hdr = &ic->i_send_hdrs[pos];
710 struct rds_header *hdr = &ic->i_send_hdrs[pos];
723 pos = (pos + 1) % ic->i_send_ring.w_nr;
733 prev->s_rm = ic->i_rm;
735 ic->i_rm = NULL;
739 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - i);
742 if (ic->i_flowctl && i < credit_alloc)
747 ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr);
748 rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,
754 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);
756 ic->i_rm = prev->s_rm;
768 static void rds_iw_build_send_fastreg(struct rds_iw_device *rds_iwdev, struct rds_iw_connection *ic, struct rds_iw_send_work *send, int nent, int len, u64 sg_addr)
790 struct rds_iw_connection *ic = conn->c_transport_data;
807 rds_iwdev = ib_get_client_data(ic->i_cm_id->device, &rds_iw_client);
811 op->op_count = ib_dma_map_sg(ic->i_cm_id->device,
814 rdsdebug("ic %p mapping op %p: %d\n", ic, op, op->op_count);
826 work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, 1, &fr_pos);
828 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);
841 work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, i, &pos);
843 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);
849 send = &ic->i_sends[pos];
851 first = prev = &ic->i_sends[fr_pos];
868 if (ic->i_unsignaled_wrs-- == 0) {
869 ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs;
898 len = ib_sg_dma_len(ic->i_cm_id->device, scat);
901 send->s_page_list->page_list[j] = ib_sg_dma_address(ic->i_cm_id->device, scat);
903 send->s_sge[j].addr = ib_sg_dma_address(ic->i_cm_id->device, scat);
905 send->s_sge[j].lkey = rds_iw_local_dma_lkey(ic);
909 rdsdebug("ic %p sent %d remote_addr %llu\n", ic, sent, remote_addr);
919 send->s_sge[0].lkey = ic->i_sends[fr_pos].s_mr->lkey;
926 if (++send == &ic->i_sends[ic->i_send_ring.w_nr])
927 send = ic->i_sends;
935 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - i);
946 rds_iw_build_send_fastreg(rds_iwdev, ic, &ic->i_sends[fr_pos],
952 ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr);
953 rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,
959 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);
969 struct rds_iw_connection *ic = conn->c_transport_data;
973 rds_iw_attempt_ack(ic);