1/* 2 * Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License as 6 * published by the Free Software Foundation; either version 2 of the 7 * License, or any later version. 8 * 9 * This program is distributed in the hope that it will be useful, but 10 * WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 * General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 17 */ 18 19FILE_LICENCE ( GPL2_OR_LATER ); 20 21#include <stdint.h> 22#include <stdlib.h> 23#include <string.h> 24#include <errno.h> 25#include <byteswap.h> 26#include <gpxe/iobuf.h> 27#include <gpxe/infiniband.h> 28#include <gpxe/ib_packet.h> 29 30/** 31 * @file 32 * 33 * Infiniband Packet Formats 34 * 35 */ 36 37/** 38 * Add IB headers 39 * 40 * @v ibdev Infiniband device 41 * @v iobuf I/O buffer to contain headers 42 * @v qp Queue pair 43 * @v payload_len Payload length 44 * @v av Address vector 45 */ 46int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf, 47 struct ib_queue_pair *qp, size_t payload_len, 48 const struct ib_address_vector *av ) { 49 struct ib_local_route_header *lrh; 50 struct ib_global_route_header *grh; 51 struct ib_base_transport_header *bth; 52 struct ib_datagram_extended_transport_header *deth; 53 size_t orig_iob_len = iob_len ( iobuf ); 54 size_t pad_len; 55 size_t lrh_len; 56 size_t grh_len; 57 unsigned int vl; 58 unsigned int lnh; 59 60 DBGC2 ( ibdev, "IBDEV %p TX %04x:%08lx => %04x:%08lx (key %08lx)\n", 61 ibdev, ibdev->lid, qp->ext_qpn, av->lid, av->qpn, av->qkey ); 62 63 /* Calculate packet length */ 64 pad_len = ( (-payload_len) & 0x3 ); 65 payload_len += pad_len; 66 payload_len += 4; /* ICRC */ 67 68 /* Reserve space for headers */ 69 orig_iob_len = iob_len ( iobuf ); 70 deth = iob_push ( iobuf, sizeof ( *deth ) ); 71 bth = iob_push ( iobuf, sizeof ( *bth ) ); 72 grh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len ); 73 grh = ( av->gid_present ? 74 iob_push ( iobuf, sizeof ( *grh ) ) : NULL ); 75 lrh = iob_push ( iobuf, sizeof ( *lrh ) ); 76 lrh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len ); 77 78 /* Construct LRH */ 79 vl = ( ( qp->ext_qpn == IB_QPN_SMI ) ? IB_VL_SMP : IB_VL_DEFAULT ); 80 lrh->vl__lver = ( vl << 4 ); 81 lnh = ( grh ? IB_LNH_GRH : IB_LNH_BTH ); 82 lrh->sl__lnh = ( ( av->sl << 4 ) | lnh ); 83 lrh->dlid = htons ( av->lid ); 84 lrh->length = htons ( lrh_len >> 2 ); 85 lrh->slid = htons ( ibdev->lid ); 86 87 /* Construct GRH, if required */ 88 if ( grh ) { 89 grh->ipver__tclass__flowlabel = 90 htonl ( IB_GRH_IPVER_IPv6 << 28 ); 91 grh->paylen = htons ( grh_len ); 92 grh->nxthdr = IB_GRH_NXTHDR_IBA; 93 grh->hoplmt = 0; 94 memcpy ( &grh->sgid, &ibdev->gid, sizeof ( grh->sgid ) ); 95 memcpy ( &grh->dgid, &av->gid, sizeof ( grh->dgid ) ); 96 } 97 98 /* Construct BTH */ 99 bth->opcode = BTH_OPCODE_UD_SEND; 100 bth->se__m__padcnt__tver = ( pad_len << 4 ); 101 bth->pkey = htons ( ibdev->pkey ); 102 bth->dest_qp = htonl ( av->qpn ); 103 bth->ack__psn = htonl ( ( qp->send.psn++ ) & 0xffffffUL ); 104 105 /* Construct DETH */ 106 deth->qkey = htonl ( av->qkey ); 107 deth->src_qp = htonl ( qp->ext_qpn ); 108 109 DBGCP_HDA ( ibdev, 0, iobuf->data, 110 ( iob_len ( iobuf ) - orig_iob_len ) ); 111 112 return 0; 113} 114 115/** 116 * Remove IB headers 117 * 118 * @v ibdev Infiniband device 119 * @v iobuf I/O buffer containing headers 120 * @v qp Queue pair to fill in, or NULL 121 * @v payload_len Payload length to fill in, or NULL 122 * @v av Address vector to fill in 123 */ 124int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf, 125 struct ib_queue_pair **qp, size_t *payload_len, 126 struct ib_address_vector *av ) { 127 struct ib_local_route_header *lrh; 128 struct ib_global_route_header *grh; 129 struct ib_base_transport_header *bth; 130 struct ib_datagram_extended_transport_header *deth; 131 size_t orig_iob_len = iob_len ( iobuf ); 132 unsigned int lnh; 133 size_t pad_len; 134 unsigned long qpn; 135 unsigned int lid; 136 137 /* Clear return values */ 138 if ( qp ) 139 *qp = NULL; 140 if ( payload_len ) 141 *payload_len = 0; 142 memset ( av, 0, sizeof ( *av ) ); 143 144 /* Extract LRH */ 145 if ( iob_len ( iobuf ) < sizeof ( *lrh ) ) { 146 DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for LRH\n", 147 ibdev, iob_len ( iobuf ) ); 148 return -EINVAL; 149 } 150 lrh = iobuf->data; 151 iob_pull ( iobuf, sizeof ( *lrh ) ); 152 av->lid = ntohs ( lrh->slid ); 153 av->sl = ( lrh->sl__lnh >> 4 ); 154 lnh = ( lrh->sl__lnh & 0x3 ); 155 lid = ntohs ( lrh->dlid ); 156 157 /* Reject unsupported packets */ 158 if ( ! ( ( lnh == IB_LNH_BTH ) || ( lnh == IB_LNH_GRH ) ) ) { 159 DBGC ( ibdev, "IBDEV %p RX unsupported LNH %x\n", 160 ibdev, lnh ); 161 return -ENOTSUP; 162 } 163 164 /* Extract GRH, if present */ 165 if ( lnh == IB_LNH_GRH ) { 166 if ( iob_len ( iobuf ) < sizeof ( *grh ) ) { 167 DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) " 168 "for GRH\n", ibdev, iob_len ( iobuf ) ); 169 return -EINVAL; 170 } 171 grh = iobuf->data; 172 iob_pull ( iobuf, sizeof ( *grh ) ); 173 av->gid_present = 1; 174 memcpy ( &av->gid, &grh->sgid, sizeof ( av->gid ) ); 175 } else { 176 grh = NULL; 177 } 178 179 /* Extract BTH */ 180 if ( iob_len ( iobuf ) < sizeof ( *bth ) ) { 181 DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for BTH\n", 182 ibdev, iob_len ( iobuf ) ); 183 return -EINVAL; 184 } 185 bth = iobuf->data; 186 iob_pull ( iobuf, sizeof ( *bth ) ); 187 if ( bth->opcode != BTH_OPCODE_UD_SEND ) { 188 DBGC ( ibdev, "IBDEV %p unsupported BTH opcode %x\n", 189 ibdev, bth->opcode ); 190 return -ENOTSUP; 191 } 192 qpn = ntohl ( bth->dest_qp ); 193 194 /* Extract DETH */ 195 if ( iob_len ( iobuf ) < sizeof ( *deth ) ) { 196 DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for DETH\n", 197 ibdev, iob_len ( iobuf ) ); 198 return -EINVAL; 199 } 200 deth = iobuf->data; 201 iob_pull ( iobuf, sizeof ( *deth ) ); 202 av->qpn = ntohl ( deth->src_qp ); 203 av->qkey = ntohl ( deth->qkey ); 204 205 /* Calculate payload length, if applicable */ 206 if ( payload_len ) { 207 pad_len = ( ( bth->se__m__padcnt__tver >> 4 ) & 0x3 ); 208 *payload_len = ( ( ntohs ( lrh->length ) << 2 ) 209 - ( orig_iob_len - iob_len ( iobuf ) ) 210 - pad_len - 4 /* ICRC */ ); 211 } 212 213 /* Determine destination QP, if applicable */ 214 if ( qp ) { 215 if ( IB_LID_MULTICAST ( lid ) && grh ) { 216 if ( ! ( *qp = ib_find_qp_mgid ( ibdev, &grh->dgid ))){ 217 DBGC ( ibdev, "IBDEV %p RX for unknown MGID " 218 "%08x:%08x:%08x:%08x\n", ibdev, 219 ntohl ( grh->dgid.u.dwords[0] ), 220 ntohl ( grh->dgid.u.dwords[1] ), 221 ntohl ( grh->dgid.u.dwords[2] ), 222 ntohl ( grh->dgid.u.dwords[3] ) ); 223 return -ENODEV; 224 } 225 } else { 226 if ( ! ( *qp = ib_find_qp_qpn ( ibdev, qpn ) ) ) { 227 DBGC ( ibdev, "IBDEV %p RX for nonexistent " 228 "QPN %lx\n", ibdev, qpn ); 229 return -ENODEV; 230 } 231 } 232 assert ( *qp ); 233 } 234 235 DBGC2 ( ibdev, "IBDEV %p RX %04x:%08lx <= %04x:%08lx (key %08x)\n", 236 ibdev, lid, ( IB_LID_MULTICAST( lid ) ? 237 ( qp ? (*qp)->ext_qpn : -1UL ) : qpn ), 238 av->lid, av->qpn, ntohl ( deth->qkey ) ); 239 DBGCP_HDA ( ibdev, 0, 240 ( iobuf->data - ( orig_iob_len - iob_len ( iobuf ) ) ), 241 ( orig_iob_len - iob_len ( iobuf ) ) ); 242 243 return 0; 244} 245