1/* 2 * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License as 6 * published by the Free Software Foundation; either version 2 of the 7 * License, or any later version. 8 * 9 * This program is distributed in the hope that it will be useful, but 10 * WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 * General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 17 */ 18 19FILE_LICENCE ( GPL2_OR_LATER ); 20 21#include <stdint.h> 22#include <string.h> 23#include <byteswap.h> 24#include <errno.h> 25#include <gpxe/list.h> 26#include <gpxe/infiniband.h> 27#include <gpxe/ib_mi.h> 28#include <gpxe/ib_mcast.h> 29 30/** @file 31 * 32 * Infiniband multicast groups 33 * 34 */ 35 36/** 37 * Generate multicast membership MAD 38 * 39 * @v ibdev Infiniband device 40 * @v gid Multicast GID 41 * @v join Join (rather than leave) group 42 * @v mad MAD to fill in 43 */ 44static void ib_mcast_mad ( struct ib_device *ibdev, struct ib_gid *gid, 45 int join, union ib_mad *mad ) { 46 struct ib_mad_sa *sa = &mad->sa; 47 48 /* Construct multicast membership record request */ 49 memset ( sa, 0, sizeof ( *sa ) ); 50 sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; 51 sa->mad_hdr.class_version = IB_SA_CLASS_VERSION; 52 sa->mad_hdr.method = 53 ( join ? IB_MGMT_METHOD_SET : IB_MGMT_METHOD_DELETE ); 54 sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC ); 55 sa->sa_hdr.comp_mask[1] = 56 htonl ( IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID | 57 IB_SA_MCMEMBER_REC_JOIN_STATE ); 58 sa->sa_data.mc_member_record.scope__join_state = 1; 59 memcpy ( &sa->sa_data.mc_member_record.mgid, gid, 60 sizeof ( sa->sa_data.mc_member_record.mgid ) ); 61 memcpy ( &sa->sa_data.mc_member_record.port_gid, &ibdev->gid, 62 sizeof ( sa->sa_data.mc_member_record.port_gid ) ); 63} 64 65/** 66 * Handle multicast membership record join response 67 * 68 * @v ibdev Infiniband device 69 * @v mi Management interface 70 * @v madx Management transaction 71 * @v rc Status code 72 * @v mad Received MAD (or NULL on error) 73 * @v av Source address vector (or NULL on error) 74 */ 75static void ib_mcast_complete ( struct ib_device *ibdev, 76 struct ib_mad_interface *mi __unused, 77 struct ib_mad_transaction *madx, 78 int rc, union ib_mad *mad, 79 struct ib_address_vector *av __unused ) { 80 struct ib_mc_membership *membership = ib_madx_get_ownerdata ( madx ); 81 struct ib_queue_pair *qp = membership->qp; 82 struct ib_gid *gid = &membership->gid; 83 struct ib_mc_member_record *mc_member_record = 84 &mad->sa.sa_data.mc_member_record; 85 int joined; 86 unsigned long qkey; 87 88 /* Report failures */ 89 if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) )) 90 rc = -ENOTCONN; 91 if ( rc != 0 ) { 92 DBGC ( ibdev, "IBDEV %p QPN %lx join failed: %s\n", 93 ibdev, qp->qpn, strerror ( rc ) ); 94 goto out; 95 } 96 97 /* Extract values from MAD */ 98 joined = ( mad->hdr.method == IB_MGMT_METHOD_GET_RESP ); 99 qkey = ntohl ( mc_member_record->qkey ); 100 DBGC ( ibdev, "IBDEV %p QPN %lx %s %08x:%08x:%08x:%08x qkey %lx\n", 101 ibdev, qp->qpn, ( joined ? "joined" : "left" ), 102 ntohl ( gid->u.dwords[0] ), ntohl ( gid->u.dwords[1] ), 103 ntohl ( gid->u.dwords[2] ), ntohl ( gid->u.dwords[3] ), 104 qkey ); 105 106 /* Set queue key */ 107 qp->qkey = qkey; 108 if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) { 109 DBGC ( ibdev, "IBDEV %p QPN %lx could not modify qkey: %s\n", 110 ibdev, qp->qpn, strerror ( rc ) ); 111 goto out; 112 } 113 114 out: 115 /* Destroy the completed transaction */ 116 ib_destroy_madx ( ibdev, mi, madx ); 117 membership->madx = NULL; 118 119 /* Hand off to upper completion handler */ 120 membership->complete ( ibdev, qp, membership, rc, mad ); 121} 122 123/** Multicast membership management transaction completion operations */ 124static struct ib_mad_transaction_operations ib_mcast_op = { 125 .complete = ib_mcast_complete, 126}; 127 128/** 129 * Join multicast group 130 * 131 * @v ibdev Infiniband device 132 * @v qp Queue pair 133 * @v membership Multicast group membership 134 * @v gid Multicast GID to join 135 * @v joined Join completion handler 136 * @ret rc Return status code 137 */ 138int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp, 139 struct ib_mc_membership *membership, struct ib_gid *gid, 140 void ( * complete ) ( struct ib_device *ibdev, 141 struct ib_queue_pair *qp, 142 struct ib_mc_membership *membership, 143 int rc, union ib_mad *mad ) ) { 144 union ib_mad mad; 145 int rc; 146 147 DBGC ( ibdev, "IBDEV %p QPN %lx joining %08x:%08x:%08x:%08x\n", 148 ibdev, qp->qpn, ntohl ( gid->u.dwords[0] ), 149 ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ), 150 ntohl ( gid->u.dwords[3] ) ); 151 152 /* Initialise structure */ 153 membership->qp = qp; 154 memcpy ( &membership->gid, gid, sizeof ( membership->gid ) ); 155 membership->complete = complete; 156 157 /* Attach queue pair to multicast GID */ 158 if ( ( rc = ib_mcast_attach ( ibdev, qp, gid ) ) != 0 ) { 159 DBGC ( ibdev, "IBDEV %p QPN %lx could not attach: %s\n", 160 ibdev, qp->qpn, strerror ( rc ) ); 161 goto err_mcast_attach; 162 } 163 164 /* Initiate multicast membership join */ 165 ib_mcast_mad ( ibdev, gid, 1, &mad ); 166 membership->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL, 167 &ib_mcast_op ); 168 if ( ! membership->madx ) { 169 DBGC ( ibdev, "IBDEV %p QPN %lx could not create join " 170 "transaction\n", ibdev, qp->qpn ); 171 rc = -ENOMEM; 172 goto err_create_madx; 173 } 174 ib_madx_set_ownerdata ( membership->madx, membership ); 175 176 return 0; 177 178 ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx ); 179 err_create_madx: 180 ib_mcast_detach ( ibdev, qp, gid ); 181 err_mcast_attach: 182 return rc; 183} 184 185/** 186 * Leave multicast group 187 * 188 * @v ibdev Infiniband device 189 * @v qp Queue pair 190 * @v membership Multicast group membership 191 */ 192void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp, 193 struct ib_mc_membership *membership ) { 194 struct ib_gid *gid = &membership->gid; 195 union ib_mad mad; 196 int rc; 197 198 DBGC ( ibdev, "IBDEV %p QPN %lx leaving %08x:%08x:%08x:%08x\n", 199 ibdev, qp->qpn, ntohl ( gid->u.dwords[0] ), 200 ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ), 201 ntohl ( gid->u.dwords[3] ) ); 202 203 /* Detach from multicast GID */ 204 ib_mcast_detach ( ibdev, qp, &membership->gid ); 205 206 /* Cancel multicast membership join, if applicable */ 207 if ( membership->madx ) { 208 ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx ); 209 membership->madx = NULL; 210 } 211 212 /* Send a single group leave MAD */ 213 ib_mcast_mad ( ibdev, &membership->gid, 0, &mad ); 214 if ( ( rc = ib_mi_send ( ibdev, ibdev->gsi, &mad, NULL ) ) != 0 ) { 215 DBGC ( ibdev, "IBDEV %p QPN %lx could not send leave request: " 216 "%s\n", ibdev, qp->qpn, strerror ( rc ) ); 217 } 218} 219