176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/* 276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * Copyright (C) 2009 Michael Brown <mbrown@fensystems.co.uk>. 376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * This program is free software; you can redistribute it and/or 576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * modify it under the terms of the GNU General Public License as 676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * published by the Free Software Foundation; either version 2 of the 776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * License, or any later version. 876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * This program is distributed in the hope that it will be useful, but 1076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * WITHOUT ANY WARRANTY; without even the implied warranty of 1176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 1276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * General Public License for more details. 1376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 1476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * You should have received a copy of the GNU General Public License 1576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * along with this program; if not, write to the Free Software 1676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 1776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 1876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 1976d05dc695b06c4e987bb8078f78032441e1430cGreg HartmanFILE_LICENCE ( GPL2_OR_LATER ); 2076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 2176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#include <stdint.h> 2276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#include <stdlib.h> 2376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#include <string.h> 2476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#include <errno.h> 2576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#include <stdio.h> 2676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#include <unistd.h> 2776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#include <byteswap.h> 2876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#include <gpxe/infiniband.h> 2976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#include <gpxe/iobuf.h> 3076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#include <gpxe/ib_mi.h> 3176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 3276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** 3376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @file 3476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 3576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * Infiniband management interfaces 3676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 3776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 3876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 3976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** Management interface number of send WQEs 4076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 4176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * This is a policy decision. 4276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 4376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#define IB_MI_NUM_SEND_WQES 4 4476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 4576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** Management interface number of receive WQEs 4676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 4776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * This is a policy decision. 4876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 4976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#define IB_MI_NUM_RECV_WQES 2 5076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 5176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** Management interface number of completion queue entries 5276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 5376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * This is a policy decision 5476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 5576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#define IB_MI_NUM_CQES 8 5676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 5776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** TID magic signature */ 5876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman#define IB_MI_TID_MAGIC ( ( 'g' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' ) 5976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 6076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** TID to use for next MAD */ 6176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartmanstatic unsigned int next_tid; 6276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 6376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** 6476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * Handle received MAD 6576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 6676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v ibdev Infiniband device 6776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v mi Management interface 6876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v mad Received MAD 6976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v av Source address vector 7076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @ret rc Return status code 7176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 7276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartmanstatic int ib_mi_handle ( struct ib_device *ibdev, 7376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_interface *mi, 7476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman union ib_mad *mad, 7576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_address_vector *av ) { 7676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_hdr *hdr = &mad->hdr; 7776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_transaction *madx; 7876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_agent *agent; 7976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 8076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Look for a matching transaction by TID */ 8176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman list_for_each_entry ( madx, &mi->madx, list ) { 8276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( memcmp ( &hdr->tid, &madx->mad.hdr.tid, 8376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman sizeof ( hdr->tid ) ) != 0 ) 8476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman continue; 8576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Found a matching transaction */ 8676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman madx->op->complete ( ibdev, mi, madx, 0, mad, av ); 8776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return 0; 8876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 8976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 9076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* If there is no matching transaction, look for a listening agent */ 9176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman for_each_table_entry ( agent, IB_MAD_AGENTS ) { 9276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( ( ( agent->mgmt_class & IB_MGMT_CLASS_MASK ) != 9376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ( hdr->mgmt_class & IB_MGMT_CLASS_MASK ) ) || 9476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ( agent->class_version != hdr->class_version ) || 9576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ( agent->attr_id != hdr->attr_id ) ) 9676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman continue; 9776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Found a matching agent */ 9876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman agent->handle ( ibdev, mi, mad, av ); 9976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return 0; 10076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 10176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 10276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Otherwise, ignore it */ 10376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p RX TID %08x%08x ignored\n", 10476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) ); 10576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return -ENOTSUP; 10676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman} 10776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 10876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** 10976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * Complete receive via management interface 11076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 11176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 11276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v ibdev Infiniband device 11376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v qp Queue pair 11476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v av Address vector 11576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v iobuf I/O buffer 11676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v rc Completion status code 11776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 11876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartmanstatic void ib_mi_complete_recv ( struct ib_device *ibdev, 11976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_queue_pair *qp, 12076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_address_vector *av, 12176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct io_buffer *iobuf, int rc ) { 12276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_interface *mi = ib_qp_get_ownerdata ( qp ); 12376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman union ib_mad *mad; 12476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_hdr *hdr; 12576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 12676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Ignore errors */ 12776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( rc != 0 ) { 12876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p RX error: %s\n", mi, strerror ( rc ) ); 12976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman goto out; 13076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 13176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 13276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Sanity checks */ 13376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( iob_len ( iobuf ) != sizeof ( *mad ) ) { 13476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p RX bad size (%zd bytes)\n", 13576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi, iob_len ( iobuf ) ); 13676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC_HDA ( mi, 0, iobuf->data, iob_len ( iobuf ) ); 13776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman goto out; 13876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 13976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mad = iobuf->data; 14076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman hdr = &mad->hdr; 14176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( hdr->base_version != IB_MGMT_BASE_VERSION ) { 14276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p RX unsupported base version %x\n", 14376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi, hdr->base_version ); 14476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC_HDA ( mi, 0, mad, sizeof ( *mad ) ); 14576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman goto out; 14676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 14776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p RX TID %08x%08x (%02x,%02x,%02x,%04x) status " 14876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman "%04x\n", mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ), 14976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman hdr->mgmt_class, hdr->class_version, hdr->method, 15076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ntohs ( hdr->attr_id ), ntohs ( hdr->status ) ); 15176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) ); 15276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 15376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Handle MAD */ 15476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( ( rc = ib_mi_handle ( ibdev, mi, mad, av ) ) != 0 ) 15576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman goto out; 15676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 15776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman out: 15876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman free_iob ( iobuf ); 15976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman} 16076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 16176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** Management interface completion operations */ 16276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartmanstatic struct ib_completion_queue_operations ib_mi_completion_ops = { 16376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman .complete_recv = ib_mi_complete_recv, 16476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman}; 16576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 16676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** 16776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * Transmit MAD 16876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 16976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v ibdev Infiniband device 17076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v mi Management interface 17176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v mad MAD 17276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v av Destination address vector 17376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @ret rc Return status code 17476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 17576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartmanint ib_mi_send ( struct ib_device *ibdev, struct ib_mad_interface *mi, 17676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman union ib_mad *mad, struct ib_address_vector *av ) { 17776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_hdr *hdr = &mad->hdr; 17876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct io_buffer *iobuf; 17976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman int rc; 18076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 18176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Set common fields */ 18276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman hdr->base_version = IB_MGMT_BASE_VERSION; 18376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( ( hdr->tid[0] == 0 ) && ( hdr->tid[1] == 0 ) ) { 18476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman hdr->tid[0] = htonl ( IB_MI_TID_MAGIC ); 18576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman hdr->tid[1] = htonl ( ++next_tid ); 18676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 18776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p TX TID %08x%08x (%02x,%02x,%02x,%04x) status " 18876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman "%04x\n", mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ), 18976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman hdr->mgmt_class, hdr->class_version, hdr->method, 19076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ntohs ( hdr->attr_id ), ntohs ( hdr->status ) ); 19176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) ); 19276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 19376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Construct directed route portion of response, if necessary */ 19476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ) { 19576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_smp *smp = &mad->smp; 19676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman unsigned int hop_pointer; 19776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman unsigned int hop_count; 19876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 19976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman smp->mad_hdr.status |= htons ( IB_SMP_STATUS_D_INBOUND ); 20076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman hop_pointer = smp->mad_hdr.class_specific.smp.hop_pointer; 20176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman hop_count = smp->mad_hdr.class_specific.smp.hop_count; 20276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman assert ( hop_count == hop_pointer ); 20376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( hop_pointer < ( sizeof ( smp->return_path.hops ) / 20476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman sizeof ( smp->return_path.hops[0] ) ) ) { 20576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman smp->return_path.hops[hop_pointer] = ibdev->port; 20676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } else { 20776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p TX TID %08x%08x invalid hop pointer " 20876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman "%d\n", mi, ntohl ( hdr->tid[0] ), 20976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ntohl ( hdr->tid[1] ), hop_pointer ); 21076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return -EINVAL; 21176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 21276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 21376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 21476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Construct I/O buffer */ 21576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman iobuf = alloc_iob ( sizeof ( *mad ) ); 21676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( ! iobuf ) { 21776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p could not allocate buffer for TID " 21876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman "%08x%08x\n", 21976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) ); 22076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return -ENOMEM; 22176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 22276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman memcpy ( iob_put ( iobuf, sizeof ( *mad ) ), mad, sizeof ( *mad ) ); 22376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 22476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Send I/O buffer */ 22576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( ( rc = ib_post_send ( ibdev, mi->qp, av, iobuf ) ) != 0 ) { 22676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p TX TID %08x%08x failed: %s\n", 22776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ), 22876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman strerror ( rc ) ); 22976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman free_iob ( iobuf ); 23076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return rc; 23176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 23276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 23376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return 0; 23476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman} 23576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 23676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** 23776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * Handle management transaction timer expiry 23876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 23976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v timer Retry timer 24076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v expired Failure indicator 24176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 24276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartmanstatic void ib_mi_timer_expired ( struct retry_timer *timer, int expired ) { 24376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_transaction *madx = 24476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman container_of ( timer, struct ib_mad_transaction, timer ); 24576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_interface *mi = madx->mi; 24676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_device *ibdev = mi->ibdev; 24776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_hdr *hdr = &madx->mad.hdr; 24876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 24976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Abandon transaction if we have tried too many times */ 25076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( expired ) { 25176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p abandoning TID %08x%08x\n", 25276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) ); 25376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman madx->op->complete ( ibdev, mi, madx, -ETIMEDOUT, NULL, NULL ); 25476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return; 25576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 25676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 25776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Restart retransmission timer */ 25876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman start_timer ( timer ); 25976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 26076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Resend MAD */ 26176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ib_mi_send ( ibdev, mi, &madx->mad, &madx->av ); 26276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman} 26376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 26476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** 26576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * Create management transaction 26676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 26776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v ibdev Infiniband device 26876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v mi Management interface 26976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v mad MAD to send 27076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v av Destination address, or NULL to use SM's GSI 27176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v op Management transaction operations 27276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @ret madx Management transaction, or NULL 27376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 27476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartmanstruct ib_mad_transaction * 27576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartmanib_create_madx ( struct ib_device *ibdev, struct ib_mad_interface *mi, 27676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman union ib_mad *mad, struct ib_address_vector *av, 27776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_transaction_operations *op ) { 27876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_transaction *madx; 27976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 28076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Allocate and initialise structure */ 28176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman madx = zalloc ( sizeof ( *madx ) ); 28276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( ! madx ) 28376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return NULL; 28476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman madx->mi = mi; 28576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman madx->timer.expired = ib_mi_timer_expired; 28676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman madx->op = op; 28776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 28876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Determine address vector */ 28976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( av ) { 29076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman memcpy ( &madx->av, av, sizeof ( madx->av ) ); 29176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } else { 29276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman madx->av.lid = ibdev->sm_lid; 29376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman madx->av.sl = ibdev->sm_sl; 29476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman madx->av.qpn = IB_QPN_GSI; 29576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman madx->av.qkey = IB_QKEY_GSI; 29676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 29776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 29876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Copy MAD */ 29976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman memcpy ( &madx->mad, mad, sizeof ( madx->mad ) ); 30076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 30176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Add to list and start timer to send initial MAD */ 30276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman list_add ( &madx->list, &mi->madx ); 30376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman start_timer_nodelay ( &madx->timer ); 30476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 30576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return madx; 30676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman} 30776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 30876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** 30976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * Destroy management transaction 31076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 31176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v ibdev Infiniband device 31276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v mi Management interface 31376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v madx Management transaction 31476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 31576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartmanvoid ib_destroy_madx ( struct ib_device *ibdev __unused, 31676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_interface *mi __unused, 31776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_transaction *madx ) { 31876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 31976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Stop timer and remove from list */ 32076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman stop_timer ( &madx->timer ); 32176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman list_del ( &madx->list ); 32276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 32376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Free transaction */ 32476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman free ( madx ); 32576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman} 32676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 32776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** 32876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * Create management interface 32976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 33076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v ibdev Infiniband device 33176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v type Queue pair type 33276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @ret mi Management agent, or NULL 33376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 33476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartmanstruct ib_mad_interface * ib_create_mi ( struct ib_device *ibdev, 33576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman enum ib_queue_pair_type type ) { 33676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_interface *mi; 33776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman int rc; 33876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 33976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Allocate and initialise fields */ 34076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi = zalloc ( sizeof ( *mi ) ); 34176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( ! mi ) 34276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman goto err_alloc; 34376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi->ibdev = ibdev; 34476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman INIT_LIST_HEAD ( &mi->madx ); 34576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 34676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Create completion queue */ 34776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi->cq = ib_create_cq ( ibdev, IB_MI_NUM_CQES, &ib_mi_completion_ops ); 34876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( ! mi->cq ) { 34976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p could not allocate completion queue\n", mi ); 35076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman goto err_create_cq; 35176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 35276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 35376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Create queue pair */ 35476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi->qp = ib_create_qp ( ibdev, type, IB_MI_NUM_SEND_WQES, mi->cq, 35576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman IB_MI_NUM_RECV_WQES, mi->cq ); 35676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( ! mi->qp ) { 35776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p could not allocate queue pair\n", mi ); 35876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman goto err_create_qp; 35976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 36076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ib_qp_set_ownerdata ( mi->qp, mi ); 36176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p (%s) running on QPN %#lx\n", 36276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi, ( ( type == IB_QPT_SMI ) ? "SMI" : "GSI" ), mi->qp->qpn ); 36376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 36476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Set queue key */ 36576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi->qp->qkey = ( ( type == IB_QPT_SMI ) ? IB_QKEY_SMI : IB_QKEY_GSI ); 36676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman if ( ( rc = ib_modify_qp ( ibdev, mi->qp ) ) != 0 ) { 36776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p could not set queue key: %s\n", 36876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi, strerror ( rc ) ); 36976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman goto err_modify_qp; 37076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 37176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 37276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Fill receive ring */ 37376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ib_refill_recv ( ibdev, mi->qp ); 37476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return mi; 37576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 37676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman err_modify_qp: 37776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ib_destroy_qp ( ibdev, mi->qp ); 37876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman err_create_qp: 37976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ib_destroy_cq ( ibdev, mi->cq ); 38076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman err_create_cq: 38176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman free ( mi ); 38276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman err_alloc: 38376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman return NULL; 38476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman} 38576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 38676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman/** 38776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * Destroy management interface 38876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * 38976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman * @v mi Management interface 39076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman */ 39176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartmanvoid ib_destroy_mi ( struct ib_device *ibdev, struct ib_mad_interface *mi ) { 39276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_transaction *madx; 39376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman struct ib_mad_transaction *tmp; 39476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 39576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman /* Flush any outstanding requests */ 39676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman list_for_each_entry_safe ( madx, tmp, &mi->madx, list ) { 39776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman DBGC ( mi, "MI %p destroyed while TID %08x%08x in progress\n", 39876d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman mi, ntohl ( madx->mad.hdr.tid[0] ), 39976d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ntohl ( madx->mad.hdr.tid[1] ) ); 40076d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman madx->op->complete ( ibdev, mi, madx, -ECANCELED, NULL, NULL ); 40176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman } 40276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman 40376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ib_destroy_qp ( ibdev, mi->qp ); 40476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman ib_destroy_cq ( ibdev, mi->cq ); 40576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman free ( mi ); 40676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman} 407