1fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick/* 2fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * IBM eServer eHCA Infiniband device driver for Linux on POWER 3fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 4fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Completion queue handling 5fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 6fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Authors: Waleri Fomin <fomin@de.ibm.com> 7fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Khadija Souissi <souissi@de.ibm.com> 8fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Reinhard Ernst <rernst@de.ibm.com> 9fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Heiko J Schick <schickhj@de.ibm.com> 10fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Hoang-Nam Nguyen <hnguyen@de.ibm.com> 11fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 12fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 13fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Copyright (c) 2005 IBM Corporation 14fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 15fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * All rights reserved. 16fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 17fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * This source code is distributed under a dual license of GPL v2.0 and OpenIB 18fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * BSD. 19fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 20fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * OpenIB BSD License 21fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 22fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Redistribution and use in source and binary forms, with or without 23fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * modification, are permitted provided that the following conditions are met: 24fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 25fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Redistributions of source code must retain the above copyright notice, this 26fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * list of conditions and the following disclaimer. 27fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 28fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * Redistributions in binary form must reproduce the above copyright notice, 29fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * this list of conditions and the following disclaimer in the documentation 30fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * and/or other materials 31fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * provided with the distribution. 32fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * 33fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 34fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 37fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 38fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 39fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 40fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 41fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 42fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 43fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * POSSIBILITY OF SUCH DAMAGE. 44fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 45fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 465a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 475a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo 48fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick#include "ehca_iverbs.h" 49fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick#include "ehca_classes.h" 50fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick#include "ehca_irq.h" 51fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick#include "hcp_if.h" 52fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 53fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickstatic struct kmem_cache *cq_cache; 54fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 55fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp) 56fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 57fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick unsigned int qp_num = qp->real_qp_num; 58fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick unsigned int key = qp_num & (QP_HASHTAB_LEN-1); 599844b71baa60270110eabaa9589d3260443d1a71Joachim Fenkes unsigned long flags; 60fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 619844b71baa60270110eabaa9589d3260443d1a71Joachim Fenkes spin_lock_irqsave(&cq->spinlock, flags); 62fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]); 639844b71baa60270110eabaa9589d3260443d1a71Joachim Fenkes spin_unlock_irqrestore(&cq->spinlock, flags); 64fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 65fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x", 66fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq->cq_number, qp_num); 67fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 68fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 69fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 70fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 71fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num) 72fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 73fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int ret = -EINVAL; 74fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1); 75fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct hlist_node *iter; 76fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_qp *qp; 779844b71baa60270110eabaa9589d3260443d1a71Joachim Fenkes unsigned long flags; 78fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 799844b71baa60270110eabaa9589d3260443d1a71Joachim Fenkes spin_lock_irqsave(&cq->spinlock, flags); 80fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick hlist_for_each(iter, &cq->qp_hashtab[key]) { 81fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick qp = hlist_entry(iter, struct ehca_qp, list_entries); 82fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (qp->real_qp_num == real_qp_num) { 83fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick hlist_del(iter); 84fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_dbg(cq->ib_cq.device, 85fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "removed qp from cq .cq_num=%x real_qp_num=%x", 86fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq->cq_number, real_qp_num); 87fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = 0; 88fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick break; 89fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 90fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 919844b71baa60270110eabaa9589d3260443d1a71Joachim Fenkes spin_unlock_irqrestore(&cq->spinlock, flags); 92fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ret) 93fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(cq->ib_cq.device, 94fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "qp not found cq_num=%x real_qp_num=%x", 95fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq->cq_number, real_qp_num); 96fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 97fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 98fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 99fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1002b94397adc68c2f0f851539884cc426e03444a26Hoang-Nam Nguyenstruct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num) 101fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 102fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_qp *ret = NULL; 103fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1); 104fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct hlist_node *iter; 105fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_qp *qp; 106fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick hlist_for_each(iter, &cq->qp_hashtab[key]) { 107fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick qp = hlist_entry(iter, struct ehca_qp, list_entries); 108fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (qp->real_qp_num == real_qp_num) { 109fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ret = qp; 110fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick break; 111fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 112fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 113fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ret; 114fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 115fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 116f4fd0b224d60044d2da5ca02f8f2b5150c1d8731Michael S. Tsirkinstruct ib_cq *ehca_create_cq(struct ib_device *device, int cqe, int comp_vector, 117fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_ucontext *context, 118fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_udata *udata) 119fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 120fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick static const u32 additional_cqe = 20; 121fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_cq *cq; 122fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_cq *my_cq; 123fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = 124fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick container_of(device, struct ehca_shca, ib_device); 125fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ipz_adapter_handle adapter_handle; 126fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_alloc_cq_parms param; /* h_call's out parameters */ 127fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct h_galpa gal; 128fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick void *vpage; 129fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u32 counter; 130fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 rpage, cqx_fec, h_ret; 131cbbbce1de2763cdf923b0e85c1a13877a6b61025Tejun Heo int ipz_rc, i; 132fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick unsigned long flags; 133fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 134fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (cqe >= 0xFFFFFFFF - 64 - additional_cqe) 135fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ERR_PTR(-EINVAL); 136fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 13719f4282149147b4a3e8c670373dc73ddd5d5faccStefan Roscher if (!atomic_add_unless(&shca->num_cqs, 1, shca->max_num_cqs)) { 138d227fa7288adebe5ba37fa8e4a589c977d4e4a34Stefan Roscher ehca_err(device, "Unable to create CQ, max number of %i " 13919f4282149147b4a3e8c670373dc73ddd5d5faccStefan Roscher "CQs reached.", shca->max_num_cqs); 140d227fa7288adebe5ba37fa8e4a589c977d4e4a34Stefan Roscher ehca_err(device, "To increase the maximum number of CQs " 141d227fa7288adebe5ba37fa8e4a589c977d4e4a34Stefan Roscher "use the number_of_cqs module parameter.\n"); 142d227fa7288adebe5ba37fa8e4a589c977d4e4a34Stefan Roscher return ERR_PTR(-ENOSPC); 143d227fa7288adebe5ba37fa8e4a589c977d4e4a34Stefan Roscher } 144d227fa7288adebe5ba37fa8e4a589c977d4e4a34Stefan Roscher 145c376222960ae91d5ffb9197ee36771aaed1d9f90Robert P. J. Day my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL); 146fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!my_cq) { 147fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(device, "Out of memory for ehca_cq struct device=%p", 148fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick device); 149d227fa7288adebe5ba37fa8e4a589c977d4e4a34Stefan Roscher atomic_dec(&shca->num_cqs); 150fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ERR_PTR(-ENOMEM); 151fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 152fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 153fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick memset(¶m, 0, sizeof(struct ehca_alloc_cq_parms)); 154fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 155fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick spin_lock_init(&my_cq->spinlock); 156fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick spin_lock_init(&my_cq->cb_lock); 157fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick spin_lock_init(&my_cq->task_lock); 15828db6beb420c756c61dd44d9f2786a0677159e74Joachim Fenkes atomic_set(&my_cq->nr_events, 0); 15931726798bd8fbef6244b28cf962f4a4c45793deaHoang-Nam Nguyen init_waitqueue_head(&my_cq->wait_completion); 160fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 161fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq = &my_cq->ib_cq; 162fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 163fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick adapter_handle = shca->ipz_hca_handle; 164fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick param.eq_handle = shca->eq.ipz_eq_handle; 165fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 166cbbbce1de2763cdf923b0e85c1a13877a6b61025Tejun Heo idr_preload(GFP_KERNEL); 167cbbbce1de2763cdf923b0e85c1a13877a6b61025Tejun Heo write_lock_irqsave(&ehca_cq_idr_lock, flags); 168cbbbce1de2763cdf923b0e85c1a13877a6b61025Tejun Heo my_cq->token = idr_alloc(&ehca_cq_idr, my_cq, 0, 0x2000000, GFP_NOWAIT); 169cbbbce1de2763cdf923b0e85c1a13877a6b61025Tejun Heo write_unlock_irqrestore(&ehca_cq_idr_lock, flags); 170cbbbce1de2763cdf923b0e85c1a13877a6b61025Tejun Heo idr_preload_end(); 171fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 172cbbbce1de2763cdf923b0e85c1a13877a6b61025Tejun Heo if (my_cq->token < 0) { 173fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq = ERR_PTR(-ENOMEM); 174fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(device, "Can't allocate new idr entry. device=%p", 175fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick device); 176fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto create_cq_exit1; 177fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 178fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 179fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* 180fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * CQs maximum depth is 4GB-64, but we need additional 20 as buffer 181fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick * for receiving errors CQEs. 182fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick */ 183fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick param.nr_cqe = cqe + additional_cqe; 184fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, ¶m); 185fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 186fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 187fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(device, "hipz_h_alloc_resource_cq() failed " 1883750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "h_ret=%lli device=%p", h_ret, device); 189fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq = ERR_PTR(ehca2ib_return_code(h_ret)); 190fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto create_cq_exit2; 191fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 192fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 193e2f81daf23efde23d8cac1fc253d41838f0347cfStefan Roscher <stefan.roscher at ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages, 194e2f81daf23efde23d8cac1fc253d41838f0347cfStefan Roscher <stefan.roscher at EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0); 195fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!ipz_rc) { 196e37221928bf685d63ba5319746eafe463d61e330Joachim Fenkes ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%i device=%p", 197fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ipz_rc, device); 198fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq = ERR_PTR(-EINVAL); 199fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto create_cq_exit3; 200fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 201fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 202fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick for (counter = 0; counter < param.act_pages; counter++) { 203fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue); 204fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!vpage) { 205fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(device, "ipz_qpageit_get_inc() " 206fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "returns NULL device=%p", device); 207fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq = ERR_PTR(-EAGAIN); 208fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto create_cq_exit4; 209fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 210b4b8d1e48e05313b163a36946be1a82e5bc5f9adMichael Ellerman rpage = __pa(vpage); 211fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 212fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_register_rpage_cq(adapter_handle, 213fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick my_cq->ipz_cq_handle, 214fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick &my_cq->pf, 215fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 0, 216fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 0, 217fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick rpage, 218fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 1, 219fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick my_cq->galpas. 220fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick kernel); 221fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 222fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret < H_SUCCESS) { 223fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(device, "hipz_h_register_rpage_cq() failed " 2243750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "ehca_cq=%p cq_num=%x h_ret=%lli counter=%i " 225fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "act_pages=%i", my_cq, my_cq->cq_number, 226fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret, counter, param.act_pages); 227fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq = ERR_PTR(-EINVAL); 228fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto create_cq_exit4; 229fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 230fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 231fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (counter == (param.act_pages - 1)) { 232fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue); 233fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if ((h_ret != H_SUCCESS) || vpage) { 234fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(device, "Registration of pages not " 235fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "complete ehca_cq=%p cq_num=%x " 2363750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "h_ret=%lli", my_cq, my_cq->cq_number, 237fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret); 238fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq = ERR_PTR(-EAGAIN); 239fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto create_cq_exit4; 240fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 241fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } else { 242fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_PAGE_REGISTERED) { 243fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(device, "Registration of page failed " 2443750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "ehca_cq=%p cq_num=%x h_ret=%lli " 245fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "counter=%i act_pages=%i", 246fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick my_cq, my_cq->cq_number, 247fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret, counter, param.act_pages); 248fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq = ERR_PTR(-ENOMEM); 249fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick goto create_cq_exit4; 250fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 251fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 252fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 253fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 254fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ipz_qeit_reset(&my_cq->ipz_queue); 255fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 256fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick gal = my_cq->galpas.kernel; 257fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec)); 2583750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%llx", 259fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick my_cq, my_cq->cq_number, cqx_fec); 260fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 261fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick my_cq->ib_cq.cqe = my_cq->nr_of_entries = 262fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick param.act_nr_of_entries - additional_cqe; 263fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff; 264fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 265fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick for (i = 0; i < QP_HASHTAB_LEN; i++) 266fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]); 267fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 268b9012e0a4255c93e1d81f1ccee591de6414b5955Alexander Schmidt INIT_LIST_HEAD(&my_cq->sqp_err_list); 269b9012e0a4255c93e1d81f1ccee591de6414b5955Alexander Schmidt INIT_LIST_HEAD(&my_cq->rqp_err_list); 270b9012e0a4255c93e1d81f1ccee591de6414b5955Alexander Schmidt 271fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (context) { 272fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ipz_queue *ipz_queue = &my_cq->ipz_queue; 273fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_create_cq_resp resp; 274fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick memset(&resp, 0, sizeof(resp)); 275fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick resp.cq_number = my_cq->cq_number; 276fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick resp.token = my_cq->token; 277fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick resp.ipz_queue.qe_size = ipz_queue->qe_size; 278fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg; 279fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick resp.ipz_queue.queue_length = ipz_queue->queue_length; 280fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick resp.ipz_queue.pagesize = ipz_queue->pagesize; 281fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick resp.ipz_queue.toggle_state = ipz_queue->toggle_state; 282e390d3b52f791fcea26312ba4982cda82052727bHoang-Nam Nguyen resp.fw_handle_ofs = (u32) 283e390d3b52f791fcea26312ba4982cda82052727bHoang-Nam Nguyen (my_cq->galpas.user.fw_handle & (PAGE_SIZE - 1)); 284fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (ib_copy_to_udata(udata, &resp, sizeof(resp))) { 285fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(device, "Copy to udata failed."); 2865bdb0f02add5994b0bc17494f4726925ca5d6ba1Yann Droneaud cq = ERR_PTR(-EFAULT); 2874c34bdf58c0a3b305ebd9b5e74011ca1fd6d964dHoang-Nam Nguyen goto create_cq_exit4; 288fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 289fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 290fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 291fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return cq; 292fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 293fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickcreate_cq_exit4: 294e2f81daf23efde23d8cac1fc253d41838f0347cfStefan Roscher <stefan.roscher at ipz_queue_dtor(NULL, &my_cq->ipz_queue); 295fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 296fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickcreate_cq_exit3: 297fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1); 298fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) 299fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p " 3003750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell "cq_num=%x h_ret=%lli", my_cq, my_cq->cq_number, h_ret); 301fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 302fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickcreate_cq_exit2: 30326ed687fdd541c2542b79dcd75fb2c82eb36f189Joachim Fenkes write_lock_irqsave(&ehca_cq_idr_lock, flags); 304fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick idr_remove(&ehca_cq_idr, my_cq->token); 30526ed687fdd541c2542b79dcd75fb2c82eb36f189Joachim Fenkes write_unlock_irqrestore(&ehca_cq_idr_lock, flags); 306fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 307fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickcreate_cq_exit1: 308fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick kmem_cache_free(cq_cache, my_cq); 309fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 310d227fa7288adebe5ba37fa8e4a589c977d4e4a34Stefan Roscher atomic_dec(&shca->num_cqs); 311fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return cq; 312fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 313fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 314fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_destroy_cq(struct ib_cq *cq) 315fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 316fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick u64 h_ret; 317fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq); 318fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick int cq_num = my_cq->cq_number; 319fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ib_device *device = cq->device; 320fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ehca_shca *shca = container_of(device, struct ehca_shca, 321fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ib_device); 322fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle; 323fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick unsigned long flags; 324fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 3254c34bdf58c0a3b305ebd9b5e74011ca1fd6d964dHoang-Nam Nguyen if (cq->uobject) { 3264c34bdf58c0a3b305ebd9b5e74011ca1fd6d964dHoang-Nam Nguyen if (my_cq->mm_count_galpa || my_cq->mm_count_queue) { 3274c34bdf58c0a3b305ebd9b5e74011ca1fd6d964dHoang-Nam Nguyen ehca_err(device, "Resources still referenced in " 3284c34bdf58c0a3b305ebd9b5e74011ca1fd6d964dHoang-Nam Nguyen "user space cq_num=%x", my_cq->cq_number); 3294c34bdf58c0a3b305ebd9b5e74011ca1fd6d964dHoang-Nam Nguyen return -EINVAL; 3304c34bdf58c0a3b305ebd9b5e74011ca1fd6d964dHoang-Nam Nguyen } 3314c34bdf58c0a3b305ebd9b5e74011ca1fd6d964dHoang-Nam Nguyen } 3324c34bdf58c0a3b305ebd9b5e74011ca1fd6d964dHoang-Nam Nguyen 33328db6beb420c756c61dd44d9f2786a0677159e74Joachim Fenkes /* 33428db6beb420c756c61dd44d9f2786a0677159e74Joachim Fenkes * remove the CQ from the idr first to make sure 33528db6beb420c756c61dd44d9f2786a0677159e74Joachim Fenkes * no more interrupt tasklets will touch this CQ 33628db6beb420c756c61dd44d9f2786a0677159e74Joachim Fenkes */ 33726ed687fdd541c2542b79dcd75fb2c82eb36f189Joachim Fenkes write_lock_irqsave(&ehca_cq_idr_lock, flags); 338fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick idr_remove(&ehca_cq_idr, my_cq->token); 33926ed687fdd541c2542b79dcd75fb2c82eb36f189Joachim Fenkes write_unlock_irqrestore(&ehca_cq_idr_lock, flags); 340fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 34128db6beb420c756c61dd44d9f2786a0677159e74Joachim Fenkes /* now wait until all pending events have completed */ 34228db6beb420c756c61dd44d9f2786a0677159e74Joachim Fenkes wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events)); 34328db6beb420c756c61dd44d9f2786a0677159e74Joachim Fenkes 34428db6beb420c756c61dd44d9f2786a0677159e74Joachim Fenkes /* nobody's using our CQ any longer -- we can destroy it */ 345fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0); 346fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret == H_R_STATE) { 347fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* cq in err: read err data and destroy it forcibly */ 3483750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_dbg(device, "ehca_cq=%p cq_num=%x resource=%llx in err " 349fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "state. Try to delete it forcibly.", 350fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick my_cq, cq_num, my_cq->ipz_cq_handle.handle); 351fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle); 352fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1); 353fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret == H_SUCCESS) 354fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_dbg(device, "cq_num=%x deleted successfully.", 355fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq_num); 356fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 357fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (h_ret != H_SUCCESS) { 3583750f60557b68776eb749859ad68af70d1a01ad0Stephen Rothwell ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lli " 359fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num); 360fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return ehca2ib_return_code(h_ret); 361fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick } 362e2f81daf23efde23d8cac1fc253d41838f0347cfStefan Roscher <stefan.roscher at ipz_queue_dtor(NULL, &my_cq->ipz_queue); 363fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick kmem_cache_free(cq_cache, my_cq); 364fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 365d227fa7288adebe5ba37fa8e4a589c977d4e4a34Stefan Roscher atomic_dec(&shca->num_cqs); 366fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 367fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 368fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 369fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata) 370fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 371fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick /* TODO: proper resize needs to be done */ 372fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick ehca_err(cq->device, "not implemented yet"); 373fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 374fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -EFAULT; 375fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 376fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 377fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickint ehca_init_cq_cache(void) 378fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 379fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick cq_cache = kmem_cache_create("ehca_cache_cq", 380fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick sizeof(struct ehca_cq), 0, 381fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick SLAB_HWCACHE_ALIGN, 38220c2df83d25c6a95affe6157a4c9cac4cf5ffaacPaul Mundt NULL); 383fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (!cq_cache) 384fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return -ENOMEM; 385fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick return 0; 386fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 387fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick 388fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schickvoid ehca_cleanup_cq_cache(void) 389fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick{ 390fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick if (cq_cache) 391fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick kmem_cache_destroy(cq_cache); 392fab97220c9e409a98b1956ba677ddd2dd43b0b95Heiko J Schick} 393