Merge git://git.linux-nfs.org/pub/linux/nfs-2.6
[linux-2.6] / drivers / infiniband / hw / ehca / ehca_reqs.c
1 /*
2  *  IBM eServer eHCA Infiniband device driver for Linux on POWER
3  *
4  *  post_send/recv, poll_cq, req_notify
5  *
6  *  Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
7  *           Waleri Fomin <fomin@de.ibm.com>
8  *           Joachim Fenkes <fenkes@de.ibm.com>
9  *           Reinhard Ernst <rernst@de.ibm.com>
10  *
11  *  Copyright (c) 2005 IBM Corporation
12  *
13  *  All rights reserved.
14  *
15  *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
16  *  BSD.
17  *
18  * OpenIB BSD License
19  *
20  * Redistribution and use in source and binary forms, with or without
21  * modification, are permitted provided that the following conditions are met:
22  *
23  * Redistributions of source code must retain the above copyright notice, this
24  * list of conditions and the following disclaimer.
25  *
26  * Redistributions in binary form must reproduce the above copyright notice,
27  * this list of conditions and the following disclaimer in the documentation
28  * and/or other materials
29  * provided with the distribution.
30  *
31  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
32  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
35  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
36  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
37  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
38  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
39  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
41  * POSSIBILITY OF SUCH DAMAGE.
42  */
43
44
45 #include <asm-powerpc/system.h>
46 #include "ehca_classes.h"
47 #include "ehca_tools.h"
48 #include "ehca_qes.h"
49 #include "ehca_iverbs.h"
50 #include "hcp_if.h"
51 #include "hipz_fns.h"
52
53 static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
54                                   struct ehca_wqe *wqe_p,
55                                   struct ib_recv_wr *recv_wr)
56 {
57         u8 cnt_ds;
58         if (unlikely((recv_wr->num_sge < 0) ||
59                      (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
60                 ehca_gen_err("Invalid number of WQE SGE. "
61                          "num_sqe=%x max_nr_of_sg=%x",
62                          recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
63                 return -EINVAL; /* invalid SG list length */
64         }
65
66         /* clear wqe header until sglist */
67         memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
68
69         wqe_p->work_request_id = recv_wr->wr_id;
70         wqe_p->nr_of_data_seg = recv_wr->num_sge;
71
72         for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
73                 wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
74                         recv_wr->sg_list[cnt_ds].addr;
75                 wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
76                         recv_wr->sg_list[cnt_ds].lkey;
77                 wqe_p->u.all_rcv.sg_list[cnt_ds].length =
78                         recv_wr->sg_list[cnt_ds].length;
79         }
80
81         if (ehca_debug_level) {
82                 ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p", ipz_rqueue);
83                 ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
84         }
85
86         return 0;
87 }
88
89 #if defined(DEBUG_GSI_SEND_WR)
90
91 /* need ib_mad struct */
92 #include <rdma/ib_mad.h>
93
94 static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
95 {
96         int idx;
97         int j;
98         while (send_wr) {
99                 struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
100                 struct ib_sge *sge = send_wr->sg_list;
101                 ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
102                              "send_flags=%x opcode=%x",idx, send_wr->wr_id,
103                              send_wr->num_sge, send_wr->send_flags,
104                              send_wr->opcode);
105                 if (mad_hdr) {
106                         ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
107                                      "mgmt_class=%x class_version=%x method=%x "
108                                      "status=%x class_specific=%x tid=%lx "
109                                      "attr_id=%x resv=%x attr_mod=%x",
110                                      idx, mad_hdr->base_version,
111                                      mad_hdr->mgmt_class,
112                                      mad_hdr->class_version, mad_hdr->method,
113                                      mad_hdr->status, mad_hdr->class_specific,
114                                      mad_hdr->tid, mad_hdr->attr_id,
115                                      mad_hdr->resv,
116                                      mad_hdr->attr_mod);
117                 }
118                 for (j = 0; j < send_wr->num_sge; j++) {
119                         u8 *data = (u8 *) abs_to_virt(sge->addr);
120                         ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
121                                      "lkey=%x",
122                                      idx, j, data, sge->length, sge->lkey);
123                         /* assume length is n*16 */
124                         ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
125                                  idx, j);
126                         sge++;
127                 } /* eof for j */
128                 idx++;
129                 send_wr = send_wr->next;
130         } /* eof while send_wr */
131 }
132
133 #endif /* DEBUG_GSI_SEND_WR */
134
135 static inline int ehca_write_swqe(struct ehca_qp *qp,
136                                   struct ehca_wqe *wqe_p,
137                                   const struct ib_send_wr *send_wr)
138 {
139         u32 idx;
140         u64 dma_length;
141         struct ehca_av *my_av;
142         u32 remote_qkey = send_wr->wr.ud.remote_qkey;
143
144         if (unlikely((send_wr->num_sge < 0) ||
145                      (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
146                 ehca_gen_err("Invalid number of WQE SGE. "
147                          "num_sqe=%x max_nr_of_sg=%x",
148                          send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
149                 return -EINVAL; /* invalid SG list length */
150         }
151
152         /* clear wqe header until sglist */
153         memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
154
155         wqe_p->work_request_id = send_wr->wr_id;
156
157         switch (send_wr->opcode) {
158         case IB_WR_SEND:
159         case IB_WR_SEND_WITH_IMM:
160                 wqe_p->optype = WQE_OPTYPE_SEND;
161                 break;
162         case IB_WR_RDMA_WRITE:
163         case IB_WR_RDMA_WRITE_WITH_IMM:
164                 wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
165                 break;
166         case IB_WR_RDMA_READ:
167                 wqe_p->optype = WQE_OPTYPE_RDMAREAD;
168                 break;
169         default:
170                 ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
171                 return -EINVAL; /* invalid opcode */
172         }
173
174         wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
175
176         wqe_p->wr_flag = 0;
177
178         if (send_wr->send_flags & IB_SEND_SIGNALED)
179                 wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
180
181         if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
182             send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
183                 /* this might not work as long as HW does not support it */
184                 wqe_p->immediate_data = be32_to_cpu(send_wr->imm_data);
185                 wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
186         }
187
188         wqe_p->nr_of_data_seg = send_wr->num_sge;
189
190         switch (qp->qp_type) {
191         case IB_QPT_SMI:
192         case IB_QPT_GSI:
193                 /* no break is intential here */
194         case IB_QPT_UD:
195                 /* IB 1.2 spec C10-15 compliance */
196                 if (send_wr->wr.ud.remote_qkey & 0x80000000)
197                         remote_qkey = qp->qkey;
198
199                 wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
200                 wqe_p->local_ee_context_qkey = remote_qkey;
201                 if (!send_wr->wr.ud.ah) {
202                         ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
203                         return -EINVAL;
204                 }
205                 my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
206                 wqe_p->u.ud_av.ud_av = my_av->av;
207
208                 /*
209                  * omitted check of IB_SEND_INLINE
210                  * since HW does not support it
211                  */
212                 for (idx = 0; idx < send_wr->num_sge; idx++) {
213                         wqe_p->u.ud_av.sg_list[idx].vaddr =
214                                 send_wr->sg_list[idx].addr;
215                         wqe_p->u.ud_av.sg_list[idx].lkey =
216                                 send_wr->sg_list[idx].lkey;
217                         wqe_p->u.ud_av.sg_list[idx].length =
218                                 send_wr->sg_list[idx].length;
219                 } /* eof for idx */
220                 if (qp->qp_type == IB_QPT_SMI ||
221                     qp->qp_type == IB_QPT_GSI)
222                         wqe_p->u.ud_av.ud_av.pmtu = 1;
223                 if (qp->qp_type == IB_QPT_GSI) {
224                         wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
225 #ifdef DEBUG_GSI_SEND_WR
226                         trace_send_wr_ud(send_wr);
227 #endif /* DEBUG_GSI_SEND_WR */
228                 }
229                 break;
230
231         case IB_QPT_UC:
232                 if (send_wr->send_flags & IB_SEND_FENCE)
233                         wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
234                 /* no break is intentional here */
235         case IB_QPT_RC:
236                 /* TODO: atomic not implemented */
237                 wqe_p->u.nud.remote_virtual_adress =
238                         send_wr->wr.rdma.remote_addr;
239                 wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
240
241                 /*
242                  * omitted checking of IB_SEND_INLINE
243                  * since HW does not support it
244                  */
245                 dma_length = 0;
246                 for (idx = 0; idx < send_wr->num_sge; idx++) {
247                         wqe_p->u.nud.sg_list[idx].vaddr =
248                                 send_wr->sg_list[idx].addr;
249                         wqe_p->u.nud.sg_list[idx].lkey =
250                                 send_wr->sg_list[idx].lkey;
251                         wqe_p->u.nud.sg_list[idx].length =
252                                 send_wr->sg_list[idx].length;
253                         dma_length += send_wr->sg_list[idx].length;
254                 } /* eof idx */
255                 wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
256
257                 break;
258
259         default:
260                 ehca_gen_err("Invalid qptype=%x", qp->qp_type);
261                 return -EINVAL;
262         }
263
264         if (ehca_debug_level) {
265                 ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
266                 ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
267         }
268         return 0;
269 }
270
271 /* map_ib_wc_status converts raw cqe_status to ib_wc_status */
272 static inline void map_ib_wc_status(u32 cqe_status,
273                                     enum ib_wc_status *wc_status)
274 {
275         if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) {
276                 switch (cqe_status & 0x3F) {
277                 case 0x01:
278                 case 0x21:
279                         *wc_status = IB_WC_LOC_LEN_ERR;
280                         break;
281                 case 0x02:
282                 case 0x22:
283                         *wc_status = IB_WC_LOC_QP_OP_ERR;
284                         break;
285                 case 0x03:
286                 case 0x23:
287                         *wc_status = IB_WC_LOC_EEC_OP_ERR;
288                         break;
289                 case 0x04:
290                 case 0x24:
291                         *wc_status = IB_WC_LOC_PROT_ERR;
292                         break;
293                 case 0x05:
294                 case 0x25:
295                         *wc_status = IB_WC_WR_FLUSH_ERR;
296                         break;
297                 case 0x06:
298                         *wc_status = IB_WC_MW_BIND_ERR;
299                         break;
300                 case 0x07: /* remote error - look into bits 20:24 */
301                         switch ((cqe_status
302                                  & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) {
303                         case 0x0:
304                                 /*
305                                  * PSN Sequence Error!
306                                  * couldn't find a matching status!
307                                  */
308                                 *wc_status = IB_WC_GENERAL_ERR;
309                                 break;
310                         case 0x1:
311                                 *wc_status = IB_WC_REM_INV_REQ_ERR;
312                                 break;
313                         case 0x2:
314                                 *wc_status = IB_WC_REM_ACCESS_ERR;
315                                 break;
316                         case 0x3:
317                                 *wc_status = IB_WC_REM_OP_ERR;
318                                 break;
319                         case 0x4:
320                                 *wc_status = IB_WC_REM_INV_RD_REQ_ERR;
321                                 break;
322                         }
323                         break;
324                 case 0x08:
325                         *wc_status = IB_WC_RETRY_EXC_ERR;
326                         break;
327                 case 0x09:
328                         *wc_status = IB_WC_RNR_RETRY_EXC_ERR;
329                         break;
330                 case 0x0A:
331                 case 0x2D:
332                         *wc_status = IB_WC_REM_ABORT_ERR;
333                         break;
334                 case 0x0B:
335                 case 0x2E:
336                         *wc_status = IB_WC_INV_EECN_ERR;
337                         break;
338                 case 0x0C:
339                 case 0x2F:
340                         *wc_status = IB_WC_INV_EEC_STATE_ERR;
341                         break;
342                 case 0x0D:
343                         *wc_status = IB_WC_BAD_RESP_ERR;
344                         break;
345                 case 0x10:
346                         /* WQE purged */
347                         *wc_status = IB_WC_WR_FLUSH_ERR;
348                         break;
349                 default:
350                         *wc_status = IB_WC_FATAL_ERR;
351
352                 }
353         } else
354                 *wc_status = IB_WC_SUCCESS;
355 }
356
357 int ehca_post_send(struct ib_qp *qp,
358                    struct ib_send_wr *send_wr,
359                    struct ib_send_wr **bad_send_wr)
360 {
361         struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
362         struct ib_send_wr *cur_send_wr;
363         struct ehca_wqe *wqe_p;
364         int wqe_cnt = 0;
365         int ret = 0;
366         unsigned long flags;
367
368         /* LOCK the QUEUE */
369         spin_lock_irqsave(&my_qp->spinlock_s, flags);
370
371         /* loop processes list of send reqs */
372         for (cur_send_wr = send_wr; cur_send_wr != NULL;
373              cur_send_wr = cur_send_wr->next) {
374                 u64 start_offset = my_qp->ipz_squeue.current_q_offset;
375                 /* get pointer next to free WQE */
376                 wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
377                 if (unlikely(!wqe_p)) {
378                         /* too many posted work requests: queue overflow */
379                         if (bad_send_wr)
380                                 *bad_send_wr = cur_send_wr;
381                         if (wqe_cnt == 0) {
382                                 ret = -ENOMEM;
383                                 ehca_err(qp->device, "Too many posted WQEs "
384                                          "qp_num=%x", qp->qp_num);
385                         }
386                         goto post_send_exit0;
387                 }
388                 /* write a SEND WQE into the QUEUE */
389                 ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr);
390                 /*
391                  * if something failed,
392                  * reset the free entry pointer to the start value
393                  */
394                 if (unlikely(ret)) {
395                         my_qp->ipz_squeue.current_q_offset = start_offset;
396                         *bad_send_wr = cur_send_wr;
397                         if (wqe_cnt == 0) {
398                                 ret = -EINVAL;
399                                 ehca_err(qp->device, "Could not write WQE "
400                                          "qp_num=%x", qp->qp_num);
401                         }
402                         goto post_send_exit0;
403                 }
404                 wqe_cnt++;
405                 ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d",
406                          my_qp, qp->qp_num, wqe_cnt);
407         } /* eof for cur_send_wr */
408
409 post_send_exit0:
410         iosync(); /* serialize GAL register access */
411         hipz_update_sqa(my_qp, wqe_cnt);
412         spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
413         return ret;
414 }
415
416 static int internal_post_recv(struct ehca_qp *my_qp,
417                               struct ib_device *dev,
418                               struct ib_recv_wr *recv_wr,
419                               struct ib_recv_wr **bad_recv_wr)
420 {
421         struct ib_recv_wr *cur_recv_wr;
422         struct ehca_wqe *wqe_p;
423         int wqe_cnt = 0;
424         int ret = 0;
425         unsigned long flags;
426
427         if (unlikely(!HAS_RQ(my_qp))) {
428                 ehca_err(dev, "QP has no RQ  ehca_qp=%p qp_num=%x ext_type=%d",
429                          my_qp, my_qp->real_qp_num, my_qp->ext_type);
430                 return -ENODEV;
431         }
432
433         /* LOCK the QUEUE */
434         spin_lock_irqsave(&my_qp->spinlock_r, flags);
435
436         /* loop processes list of send reqs */
437         for (cur_recv_wr = recv_wr; cur_recv_wr != NULL;
438              cur_recv_wr = cur_recv_wr->next) {
439                 u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
440                 /* get pointer next to free WQE */
441                 wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
442                 if (unlikely(!wqe_p)) {
443                         /* too many posted work requests: queue overflow */
444                         if (bad_recv_wr)
445                                 *bad_recv_wr = cur_recv_wr;
446                         if (wqe_cnt == 0) {
447                                 ret = -ENOMEM;
448                                 ehca_err(dev, "Too many posted WQEs "
449                                          "qp_num=%x", my_qp->real_qp_num);
450                         }
451                         goto post_recv_exit0;
452                 }
453                 /* write a RECV WQE into the QUEUE */
454                 ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, cur_recv_wr);
455                 /*
456                  * if something failed,
457                  * reset the free entry pointer to the start value
458                  */
459                 if (unlikely(ret)) {
460                         my_qp->ipz_rqueue.current_q_offset = start_offset;
461                         *bad_recv_wr = cur_recv_wr;
462                         if (wqe_cnt == 0) {
463                                 ret = -EINVAL;
464                                 ehca_err(dev, "Could not write WQE "
465                                          "qp_num=%x", my_qp->real_qp_num);
466                         }
467                         goto post_recv_exit0;
468                 }
469                 wqe_cnt++;
470                 ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d",
471                          my_qp, my_qp->real_qp_num, wqe_cnt);
472         } /* eof for cur_recv_wr */
473
474 post_recv_exit0:
475         iosync(); /* serialize GAL register access */
476         hipz_update_rqa(my_qp, wqe_cnt);
477         spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
478         return ret;
479 }
480
481 int ehca_post_recv(struct ib_qp *qp,
482                    struct ib_recv_wr *recv_wr,
483                    struct ib_recv_wr **bad_recv_wr)
484 {
485         return internal_post_recv(container_of(qp, struct ehca_qp, ib_qp),
486                                   qp->device, recv_wr, bad_recv_wr);
487 }
488
489 int ehca_post_srq_recv(struct ib_srq *srq,
490                        struct ib_recv_wr *recv_wr,
491                        struct ib_recv_wr **bad_recv_wr)
492 {
493         return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq),
494                                   srq->device, recv_wr, bad_recv_wr);
495 }
496
497 /*
498  * ib_wc_opcode table converts ehca wc opcode to ib
499  * Since we use zero to indicate invalid opcode, the actual ib opcode must
500  * be decremented!!!
501  */
502 static const u8 ib_wc_opcode[255] = {
503         [0x01] = IB_WC_RECV+1,
504         [0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
505         [0x04] = IB_WC_BIND_MW+1,
506         [0x08] = IB_WC_FETCH_ADD+1,
507         [0x10] = IB_WC_COMP_SWAP+1,
508         [0x20] = IB_WC_RDMA_WRITE+1,
509         [0x40] = IB_WC_RDMA_READ+1,
510         [0x80] = IB_WC_SEND+1
511 };
512
513 /* internal function to poll one entry of cq */
514 static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
515 {
516         int ret = 0;
517         struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
518         struct ehca_cqe *cqe;
519         struct ehca_qp *my_qp;
520         int cqe_count = 0;
521
522 poll_cq_one_read_cqe:
523         cqe = (struct ehca_cqe *)
524                 ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
525         if (!cqe) {
526                 ret = -EAGAIN;
527                 ehca_dbg(cq->device, "Completion queue is empty ehca_cq=%p "
528                          "cq_num=%x ret=%x", my_cq, my_cq->cq_number, ret);
529                 goto  poll_cq_one_exit0;
530         }
531
532         /* prevents loads being reordered across this point */
533         rmb();
534
535         cqe_count++;
536         if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
537                 struct ehca_qp *qp=ehca_cq_get_qp(my_cq, cqe->local_qp_number);
538                 int purgeflag;
539                 unsigned long flags;
540                 if (!qp) {
541                         ehca_err(cq->device, "cq_num=%x qp_num=%x "
542                                  "could not find qp -> ignore cqe",
543                                  my_cq->cq_number, cqe->local_qp_number);
544                         ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
545                                  my_cq->cq_number, cqe->local_qp_number);
546                         /* ignore this purged cqe */
547                         goto poll_cq_one_read_cqe;
548                 }
549                 spin_lock_irqsave(&qp->spinlock_s, flags);
550                 purgeflag = qp->sqerr_purgeflag;
551                 spin_unlock_irqrestore(&qp->spinlock_s, flags);
552
553                 if (purgeflag) {
554                         ehca_dbg(cq->device, "Got CQE with purged bit qp_num=%x "
555                                  "src_qp=%x",
556                                  cqe->local_qp_number, cqe->remote_qp_number);
557                         if (ehca_debug_level)
558                                 ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
559                                          cqe->local_qp_number,
560                                          cqe->remote_qp_number);
561                         /*
562                          * ignore this to avoid double cqes of bad wqe
563                          * that caused sqe and turn off purge flag
564                          */
565                         qp->sqerr_purgeflag = 0;
566                         goto poll_cq_one_read_cqe;
567                 }
568         }
569
570         /* tracing cqe */
571         if (unlikely(ehca_debug_level)) {
572                 ehca_dbg(cq->device,
573                          "Received COMPLETION ehca_cq=%p cq_num=%x -----",
574                          my_cq, my_cq->cq_number);
575                 ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
576                          my_cq, my_cq->cq_number);
577                 ehca_dbg(cq->device,
578                          "ehca_cq=%p cq_num=%x -------------------------",
579                          my_cq, my_cq->cq_number);
580         }
581
582         /* we got a completion! */
583         wc->wr_id = cqe->work_request_id;
584
585         /* eval ib_wc_opcode */
586         wc->opcode = ib_wc_opcode[cqe->optype]-1;
587         if (unlikely(wc->opcode == -1)) {
588                 ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
589                          "ehca_cq=%p cq_num=%x",
590                          cqe->optype, cqe->status, my_cq, my_cq->cq_number);
591                 /* dump cqe for other infos */
592                 ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
593                          my_cq, my_cq->cq_number);
594                 /* update also queue adder to throw away this entry!!! */
595                 goto poll_cq_one_exit0;
596         }
597         /* eval ib_wc_status */
598         if (unlikely(cqe->status & WC_STATUS_ERROR_BIT)) {
599                 /* complete with errors */
600                 map_ib_wc_status(cqe->status, &wc->status);
601                 wc->vendor_err = wc->status;
602         } else
603                 wc->status = IB_WC_SUCCESS;
604
605         read_lock(&ehca_qp_idr_lock);
606         my_qp = idr_find(&ehca_qp_idr, cqe->qp_token);
607         wc->qp = &my_qp->ib_qp;
608         read_unlock(&ehca_qp_idr_lock);
609
610         wc->byte_len = cqe->nr_bytes_transferred;
611         wc->pkey_index = cqe->pkey_index;
612         wc->slid = cqe->rlid;
613         wc->dlid_path_bits = cqe->dlid;
614         wc->src_qp = cqe->remote_qp_number;
615         wc->wc_flags = cqe->w_completion_flags;
616         wc->imm_data = cpu_to_be32(cqe->immediate_data);
617         wc->sl = cqe->service_level;
618
619         if (unlikely(wc->status != IB_WC_SUCCESS))
620                 ehca_dbg(cq->device,
621                          "ehca_cq=%p cq_num=%x WARNING unsuccessful cqe "
622                          "OPType=%x status=%x qp_num=%x src_qp=%x wr_id=%lx "
623                          "cqe=%p", my_cq, my_cq->cq_number, cqe->optype,
624                          cqe->status, cqe->local_qp_number,
625                          cqe->remote_qp_number, cqe->work_request_id, cqe);
626
627 poll_cq_one_exit0:
628         if (cqe_count > 0)
629                 hipz_update_feca(my_cq, cqe_count);
630
631         return ret;
632 }
633
634 int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
635 {
636         struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
637         int nr;
638         struct ib_wc *current_wc = wc;
639         int ret = 0;
640         unsigned long flags;
641
642         if (num_entries < 1) {
643                 ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
644                          "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
645                 ret = -EINVAL;
646                 goto poll_cq_exit0;
647         }
648
649         spin_lock_irqsave(&my_cq->spinlock, flags);
650         for (nr = 0; nr < num_entries; nr++) {
651                 ret = ehca_poll_cq_one(cq, current_wc);
652                 if (ret)
653                         break;
654                 current_wc++;
655         } /* eof for nr */
656         spin_unlock_irqrestore(&my_cq->spinlock, flags);
657         if (ret == -EAGAIN  || !ret)
658                 ret = nr;
659
660 poll_cq_exit0:
661         return ret;
662 }
663
664 int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags)
665 {
666         struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
667         int ret = 0;
668
669         switch (notify_flags & IB_CQ_SOLICITED_MASK) {
670         case IB_CQ_SOLICITED:
671                 hipz_set_cqx_n0(my_cq, 1);
672                 break;
673         case IB_CQ_NEXT_COMP:
674                 hipz_set_cqx_n1(my_cq, 1);
675                 break;
676         default:
677                 return -EINVAL;
678         }
679
680         if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
681                 unsigned long spl_flags;
682                 spin_lock_irqsave(&my_cq->spinlock, spl_flags);
683                 ret = ipz_qeit_is_valid(&my_cq->ipz_queue);
684                 spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
685         }
686
687         return ret;
688 }