Merge branches 'topic/asoc', 'topic/misc-fixes' and 'topic/hda' into for-linus
[linux-2.6] / drivers / infiniband / hw / ehca / ehca_reqs.c
1 /*
2  *  IBM eServer eHCA Infiniband device driver for Linux on POWER
3  *
4  *  post_send/recv, poll_cq, req_notify
5  *
6  *  Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
7  *           Waleri Fomin <fomin@de.ibm.com>
8  *           Joachim Fenkes <fenkes@de.ibm.com>
9  *           Reinhard Ernst <rernst@de.ibm.com>
10  *
11  *  Copyright (c) 2005 IBM Corporation
12  *
13  *  All rights reserved.
14  *
15  *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
16  *  BSD.
17  *
18  * OpenIB BSD License
19  *
20  * Redistribution and use in source and binary forms, with or without
21  * modification, are permitted provided that the following conditions are met:
22  *
23  * Redistributions of source code must retain the above copyright notice, this
24  * list of conditions and the following disclaimer.
25  *
26  * Redistributions in binary form must reproduce the above copyright notice,
27  * this list of conditions and the following disclaimer in the documentation
28  * and/or other materials
29  * provided with the distribution.
30  *
31  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
32  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
35  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
36  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
37  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
38  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
39  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
41  * POSSIBILITY OF SUCH DAMAGE.
42  */
43
44
45 #include <asm/system.h>
46 #include "ehca_classes.h"
47 #include "ehca_tools.h"
48 #include "ehca_qes.h"
49 #include "ehca_iverbs.h"
50 #include "hcp_if.h"
51 #include "hipz_fns.h"
52
53 /* in RC traffic, insert an empty RDMA READ every this many packets */
54 #define ACK_CIRC_THRESHOLD 2000000
55
56 static u64 replace_wr_id(u64 wr_id, u16 idx)
57 {
58         u64 ret;
59
60         ret = wr_id & ~QMAP_IDX_MASK;
61         ret |= idx & QMAP_IDX_MASK;
62
63         return ret;
64 }
65
66 static u16 get_app_wr_id(u64 wr_id)
67 {
68         return wr_id & QMAP_IDX_MASK;
69 }
70
71 static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
72                                   struct ehca_wqe *wqe_p,
73                                   struct ib_recv_wr *recv_wr,
74                                   u32 rq_map_idx)
75 {
76         u8 cnt_ds;
77         if (unlikely((recv_wr->num_sge < 0) ||
78                      (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
79                 ehca_gen_err("Invalid number of WQE SGE. "
80                          "num_sqe=%x max_nr_of_sg=%x",
81                          recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
82                 return -EINVAL; /* invalid SG list length */
83         }
84
85         /* clear wqe header until sglist */
86         memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
87
88         wqe_p->work_request_id = replace_wr_id(recv_wr->wr_id, rq_map_idx);
89         wqe_p->nr_of_data_seg = recv_wr->num_sge;
90
91         for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
92                 wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
93                         recv_wr->sg_list[cnt_ds].addr;
94                 wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
95                         recv_wr->sg_list[cnt_ds].lkey;
96                 wqe_p->u.all_rcv.sg_list[cnt_ds].length =
97                         recv_wr->sg_list[cnt_ds].length;
98         }
99
100         if (ehca_debug_level >= 3) {
101                 ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p",
102                              ipz_rqueue);
103                 ehca_dmp(wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
104         }
105
106         return 0;
107 }
108
109 #if defined(DEBUG_GSI_SEND_WR)
110
111 /* need ib_mad struct */
112 #include <rdma/ib_mad.h>
113
114 static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
115 {
116         int idx;
117         int j;
118         while (send_wr) {
119                 struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
120                 struct ib_sge *sge = send_wr->sg_list;
121                 ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
122                              "send_flags=%x opcode=%x", idx, send_wr->wr_id,
123                              send_wr->num_sge, send_wr->send_flags,
124                              send_wr->opcode);
125                 if (mad_hdr) {
126                         ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
127                                      "mgmt_class=%x class_version=%x method=%x "
128                                      "status=%x class_specific=%x tid=%lx "
129                                      "attr_id=%x resv=%x attr_mod=%x",
130                                      idx, mad_hdr->base_version,
131                                      mad_hdr->mgmt_class,
132                                      mad_hdr->class_version, mad_hdr->method,
133                                      mad_hdr->status, mad_hdr->class_specific,
134                                      mad_hdr->tid, mad_hdr->attr_id,
135                                      mad_hdr->resv,
136                                      mad_hdr->attr_mod);
137                 }
138                 for (j = 0; j < send_wr->num_sge; j++) {
139                         u8 *data = (u8 *)abs_to_virt(sge->addr);
140                         ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
141                                      "lkey=%x",
142                                      idx, j, data, sge->length, sge->lkey);
143                         /* assume length is n*16 */
144                         ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
145                                  idx, j);
146                         sge++;
147                 } /* eof for j */
148                 idx++;
149                 send_wr = send_wr->next;
150         } /* eof while send_wr */
151 }
152
153 #endif /* DEBUG_GSI_SEND_WR */
154
155 static inline int ehca_write_swqe(struct ehca_qp *qp,
156                                   struct ehca_wqe *wqe_p,
157                                   const struct ib_send_wr *send_wr,
158                                   u32 sq_map_idx,
159                                   int hidden)
160 {
161         u32 idx;
162         u64 dma_length;
163         struct ehca_av *my_av;
164         u32 remote_qkey = send_wr->wr.ud.remote_qkey;
165         struct ehca_qmap_entry *qmap_entry = &qp->sq_map.map[sq_map_idx];
166
167         if (unlikely((send_wr->num_sge < 0) ||
168                      (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
169                 ehca_gen_err("Invalid number of WQE SGE. "
170                          "num_sqe=%x max_nr_of_sg=%x",
171                          send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
172                 return -EINVAL; /* invalid SG list length */
173         }
174
175         /* clear wqe header until sglist */
176         memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
177
178         wqe_p->work_request_id = replace_wr_id(send_wr->wr_id, sq_map_idx);
179
180         qmap_entry->app_wr_id = get_app_wr_id(send_wr->wr_id);
181         qmap_entry->reported = 0;
182
183         switch (send_wr->opcode) {
184         case IB_WR_SEND:
185         case IB_WR_SEND_WITH_IMM:
186                 wqe_p->optype = WQE_OPTYPE_SEND;
187                 break;
188         case IB_WR_RDMA_WRITE:
189         case IB_WR_RDMA_WRITE_WITH_IMM:
190                 wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
191                 break;
192         case IB_WR_RDMA_READ:
193                 wqe_p->optype = WQE_OPTYPE_RDMAREAD;
194                 break;
195         default:
196                 ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
197                 return -EINVAL; /* invalid opcode */
198         }
199
200         wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
201
202         wqe_p->wr_flag = 0;
203
204         if ((send_wr->send_flags & IB_SEND_SIGNALED ||
205             qp->init_attr.sq_sig_type == IB_SIGNAL_ALL_WR)
206             && !hidden)
207                 wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
208
209         if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
210             send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
211                 /* this might not work as long as HW does not support it */
212                 wqe_p->immediate_data = be32_to_cpu(send_wr->ex.imm_data);
213                 wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
214         }
215
216         wqe_p->nr_of_data_seg = send_wr->num_sge;
217
218         switch (qp->qp_type) {
219         case IB_QPT_SMI:
220         case IB_QPT_GSI:
221                 /* no break is intential here */
222         case IB_QPT_UD:
223                 /* IB 1.2 spec C10-15 compliance */
224                 if (send_wr->wr.ud.remote_qkey & 0x80000000)
225                         remote_qkey = qp->qkey;
226
227                 wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
228                 wqe_p->local_ee_context_qkey = remote_qkey;
229                 if (unlikely(!send_wr->wr.ud.ah)) {
230                         ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
231                         return -EINVAL;
232                 }
233                 if (unlikely(send_wr->wr.ud.remote_qpn == 0)) {
234                         ehca_gen_err("dest QP# is 0. qp=%x", qp->real_qp_num);
235                         return -EINVAL;
236                 }
237                 my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
238                 wqe_p->u.ud_av.ud_av = my_av->av;
239
240                 /*
241                  * omitted check of IB_SEND_INLINE
242                  * since HW does not support it
243                  */
244                 for (idx = 0; idx < send_wr->num_sge; idx++) {
245                         wqe_p->u.ud_av.sg_list[idx].vaddr =
246                                 send_wr->sg_list[idx].addr;
247                         wqe_p->u.ud_av.sg_list[idx].lkey =
248                                 send_wr->sg_list[idx].lkey;
249                         wqe_p->u.ud_av.sg_list[idx].length =
250                                 send_wr->sg_list[idx].length;
251                 } /* eof for idx */
252                 if (qp->qp_type == IB_QPT_SMI ||
253                     qp->qp_type == IB_QPT_GSI)
254                         wqe_p->u.ud_av.ud_av.pmtu = 1;
255                 if (qp->qp_type == IB_QPT_GSI) {
256                         wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
257 #ifdef DEBUG_GSI_SEND_WR
258                         trace_send_wr_ud(send_wr);
259 #endif /* DEBUG_GSI_SEND_WR */
260                 }
261                 break;
262
263         case IB_QPT_UC:
264                 if (send_wr->send_flags & IB_SEND_FENCE)
265                         wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
266                 /* no break is intentional here */
267         case IB_QPT_RC:
268                 /* TODO: atomic not implemented */
269                 wqe_p->u.nud.remote_virtual_adress =
270                         send_wr->wr.rdma.remote_addr;
271                 wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
272
273                 /*
274                  * omitted checking of IB_SEND_INLINE
275                  * since HW does not support it
276                  */
277                 dma_length = 0;
278                 for (idx = 0; idx < send_wr->num_sge; idx++) {
279                         wqe_p->u.nud.sg_list[idx].vaddr =
280                                 send_wr->sg_list[idx].addr;
281                         wqe_p->u.nud.sg_list[idx].lkey =
282                                 send_wr->sg_list[idx].lkey;
283                         wqe_p->u.nud.sg_list[idx].length =
284                                 send_wr->sg_list[idx].length;
285                         dma_length += send_wr->sg_list[idx].length;
286                 } /* eof idx */
287                 wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
288
289                 /* unsolicited ack circumvention */
290                 if (send_wr->opcode == IB_WR_RDMA_READ) {
291                         /* on RDMA read, switch on and reset counters */
292                         qp->message_count = qp->packet_count = 0;
293                         qp->unsol_ack_circ = 1;
294                 } else
295                         /* else estimate #packets */
296                         qp->packet_count += (dma_length >> qp->mtu_shift) + 1;
297
298                 break;
299
300         default:
301                 ehca_gen_err("Invalid qptype=%x", qp->qp_type);
302                 return -EINVAL;
303         }
304
305         if (ehca_debug_level >= 3) {
306                 ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
307                 ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
308         }
309         return 0;
310 }
311
312 /* map_ib_wc_status converts raw cqe_status to ib_wc_status */
313 static inline void map_ib_wc_status(u32 cqe_status,
314                                     enum ib_wc_status *wc_status)
315 {
316         if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) {
317                 switch (cqe_status & 0x3F) {
318                 case 0x01:
319                 case 0x21:
320                         *wc_status = IB_WC_LOC_LEN_ERR;
321                         break;
322                 case 0x02:
323                 case 0x22:
324                         *wc_status = IB_WC_LOC_QP_OP_ERR;
325                         break;
326                 case 0x03:
327                 case 0x23:
328                         *wc_status = IB_WC_LOC_EEC_OP_ERR;
329                         break;
330                 case 0x04:
331                 case 0x24:
332                         *wc_status = IB_WC_LOC_PROT_ERR;
333                         break;
334                 case 0x05:
335                 case 0x25:
336                         *wc_status = IB_WC_WR_FLUSH_ERR;
337                         break;
338                 case 0x06:
339                         *wc_status = IB_WC_MW_BIND_ERR;
340                         break;
341                 case 0x07: /* remote error - look into bits 20:24 */
342                         switch ((cqe_status
343                                  & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) {
344                         case 0x0:
345                                 /*
346                                  * PSN Sequence Error!
347                                  * couldn't find a matching status!
348                                  */
349                                 *wc_status = IB_WC_GENERAL_ERR;
350                                 break;
351                         case 0x1:
352                                 *wc_status = IB_WC_REM_INV_REQ_ERR;
353                                 break;
354                         case 0x2:
355                                 *wc_status = IB_WC_REM_ACCESS_ERR;
356                                 break;
357                         case 0x3:
358                                 *wc_status = IB_WC_REM_OP_ERR;
359                                 break;
360                         case 0x4:
361                                 *wc_status = IB_WC_REM_INV_RD_REQ_ERR;
362                                 break;
363                         }
364                         break;
365                 case 0x08:
366                         *wc_status = IB_WC_RETRY_EXC_ERR;
367                         break;
368                 case 0x09:
369                         *wc_status = IB_WC_RNR_RETRY_EXC_ERR;
370                         break;
371                 case 0x0A:
372                 case 0x2D:
373                         *wc_status = IB_WC_REM_ABORT_ERR;
374                         break;
375                 case 0x0B:
376                 case 0x2E:
377                         *wc_status = IB_WC_INV_EECN_ERR;
378                         break;
379                 case 0x0C:
380                 case 0x2F:
381                         *wc_status = IB_WC_INV_EEC_STATE_ERR;
382                         break;
383                 case 0x0D:
384                         *wc_status = IB_WC_BAD_RESP_ERR;
385                         break;
386                 case 0x10:
387                         /* WQE purged */
388                         *wc_status = IB_WC_WR_FLUSH_ERR;
389                         break;
390                 default:
391                         *wc_status = IB_WC_FATAL_ERR;
392
393                 }
394         } else
395                 *wc_status = IB_WC_SUCCESS;
396 }
397
398 static inline int post_one_send(struct ehca_qp *my_qp,
399                          struct ib_send_wr *cur_send_wr,
400                          struct ib_send_wr **bad_send_wr,
401                          int hidden)
402 {
403         struct ehca_wqe *wqe_p;
404         int ret;
405         u32 sq_map_idx;
406         u64 start_offset = my_qp->ipz_squeue.current_q_offset;
407
408         /* get pointer next to free WQE */
409         wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
410         if (unlikely(!wqe_p)) {
411                 /* too many posted work requests: queue overflow */
412                 if (bad_send_wr)
413                         *bad_send_wr = cur_send_wr;
414                 ehca_err(my_qp->ib_qp.device, "Too many posted WQEs "
415                          "qp_num=%x", my_qp->ib_qp.qp_num);
416                 return -ENOMEM;
417         }
418
419         /*
420          * Get the index of the WQE in the send queue. The same index is used
421          * for writing into the sq_map.
422          */
423         sq_map_idx = start_offset / my_qp->ipz_squeue.qe_size;
424
425         /* write a SEND WQE into the QUEUE */
426         ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr, sq_map_idx, hidden);
427         /*
428          * if something failed,
429          * reset the free entry pointer to the start value
430          */
431         if (unlikely(ret)) {
432                 my_qp->ipz_squeue.current_q_offset = start_offset;
433                 if (bad_send_wr)
434                         *bad_send_wr = cur_send_wr;
435                 ehca_err(my_qp->ib_qp.device, "Could not write WQE "
436                          "qp_num=%x", my_qp->ib_qp.qp_num);
437                 return -EINVAL;
438         }
439
440         return 0;
441 }
442
443 int ehca_post_send(struct ib_qp *qp,
444                    struct ib_send_wr *send_wr,
445                    struct ib_send_wr **bad_send_wr)
446 {
447         struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
448         struct ib_send_wr *cur_send_wr;
449         int wqe_cnt = 0;
450         int ret = 0;
451         unsigned long flags;
452
453         /* Reject WR if QP is in RESET, INIT or RTR state */
454         if (unlikely(my_qp->state < IB_QPS_RTS)) {
455                 ehca_err(qp->device, "Invalid QP state  qp_state=%d qpn=%x",
456                          my_qp->state, qp->qp_num);
457                 return -EINVAL;
458         }
459
460         /* LOCK the QUEUE */
461         spin_lock_irqsave(&my_qp->spinlock_s, flags);
462
463         /* Send an empty extra RDMA read if:
464          *  1) there has been an RDMA read on this connection before
465          *  2) no RDMA read occurred for ACK_CIRC_THRESHOLD link packets
466          *  3) we can be sure that any previous extra RDMA read has been
467          *     processed so we don't overflow the SQ
468          */
469         if (unlikely(my_qp->unsol_ack_circ &&
470                      my_qp->packet_count > ACK_CIRC_THRESHOLD &&
471                      my_qp->message_count > my_qp->init_attr.cap.max_send_wr)) {
472                 /* insert an empty RDMA READ to fix up the remote QP state */
473                 struct ib_send_wr circ_wr;
474                 memset(&circ_wr, 0, sizeof(circ_wr));
475                 circ_wr.opcode = IB_WR_RDMA_READ;
476                 post_one_send(my_qp, &circ_wr, NULL, 1); /* ignore retcode */
477                 wqe_cnt++;
478                 ehca_dbg(qp->device, "posted circ wr  qp_num=%x", qp->qp_num);
479                 my_qp->message_count = my_qp->packet_count = 0;
480         }
481
482         /* loop processes list of send reqs */
483         for (cur_send_wr = send_wr; cur_send_wr != NULL;
484              cur_send_wr = cur_send_wr->next) {
485                 ret = post_one_send(my_qp, cur_send_wr, bad_send_wr, 0);
486                 if (unlikely(ret)) {
487                         /* if one or more WQEs were successful, don't fail */
488                         if (wqe_cnt)
489                                 ret = 0;
490                         goto post_send_exit0;
491                 }
492                 wqe_cnt++;
493         } /* eof for cur_send_wr */
494
495 post_send_exit0:
496         iosync(); /* serialize GAL register access */
497         hipz_update_sqa(my_qp, wqe_cnt);
498         if (unlikely(ret || ehca_debug_level >= 2))
499                 ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
500                          my_qp, qp->qp_num, wqe_cnt, ret);
501         my_qp->message_count += wqe_cnt;
502         spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
503         return ret;
504 }
505
506 static int internal_post_recv(struct ehca_qp *my_qp,
507                               struct ib_device *dev,
508                               struct ib_recv_wr *recv_wr,
509                               struct ib_recv_wr **bad_recv_wr)
510 {
511         struct ib_recv_wr *cur_recv_wr;
512         struct ehca_wqe *wqe_p;
513         int wqe_cnt = 0;
514         int ret = 0;
515         u32 rq_map_idx;
516         unsigned long flags;
517         struct ehca_qmap_entry *qmap_entry;
518
519         if (unlikely(!HAS_RQ(my_qp))) {
520                 ehca_err(dev, "QP has no RQ  ehca_qp=%p qp_num=%x ext_type=%d",
521                          my_qp, my_qp->real_qp_num, my_qp->ext_type);
522                 return -ENODEV;
523         }
524
525         /* LOCK the QUEUE */
526         spin_lock_irqsave(&my_qp->spinlock_r, flags);
527
528         /* loop processes list of send reqs */
529         for (cur_recv_wr = recv_wr; cur_recv_wr != NULL;
530              cur_recv_wr = cur_recv_wr->next) {
531                 u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
532                 /* get pointer next to free WQE */
533                 wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
534                 if (unlikely(!wqe_p)) {
535                         /* too many posted work requests: queue overflow */
536                         if (bad_recv_wr)
537                                 *bad_recv_wr = cur_recv_wr;
538                         if (wqe_cnt == 0) {
539                                 ret = -ENOMEM;
540                                 ehca_err(dev, "Too many posted WQEs "
541                                          "qp_num=%x", my_qp->real_qp_num);
542                         }
543                         goto post_recv_exit0;
544                 }
545                 /*
546                  * Get the index of the WQE in the recv queue. The same index
547                  * is used for writing into the rq_map.
548                  */
549                 rq_map_idx = start_offset / my_qp->ipz_rqueue.qe_size;
550
551                 /* write a RECV WQE into the QUEUE */
552                 ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, cur_recv_wr,
553                                 rq_map_idx);
554                 /*
555                  * if something failed,
556                  * reset the free entry pointer to the start value
557                  */
558                 if (unlikely(ret)) {
559                         my_qp->ipz_rqueue.current_q_offset = start_offset;
560                         *bad_recv_wr = cur_recv_wr;
561                         if (wqe_cnt == 0) {
562                                 ret = -EINVAL;
563                                 ehca_err(dev, "Could not write WQE "
564                                          "qp_num=%x", my_qp->real_qp_num);
565                         }
566                         goto post_recv_exit0;
567                 }
568
569                 qmap_entry = &my_qp->rq_map.map[rq_map_idx];
570                 qmap_entry->app_wr_id = get_app_wr_id(cur_recv_wr->wr_id);
571                 qmap_entry->reported = 0;
572
573                 wqe_cnt++;
574         } /* eof for cur_recv_wr */
575
576 post_recv_exit0:
577         iosync(); /* serialize GAL register access */
578         hipz_update_rqa(my_qp, wqe_cnt);
579         if (unlikely(ret || ehca_debug_level >= 2))
580             ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
581                      my_qp, my_qp->real_qp_num, wqe_cnt, ret);
582         spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
583         return ret;
584 }
585
586 int ehca_post_recv(struct ib_qp *qp,
587                    struct ib_recv_wr *recv_wr,
588                    struct ib_recv_wr **bad_recv_wr)
589 {
590         struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
591
592         /* Reject WR if QP is in RESET state */
593         if (unlikely(my_qp->state == IB_QPS_RESET)) {
594                 ehca_err(qp->device, "Invalid QP state  qp_state=%d qpn=%x",
595                          my_qp->state, qp->qp_num);
596                 return -EINVAL;
597         }
598
599         return internal_post_recv(my_qp, qp->device, recv_wr, bad_recv_wr);
600 }
601
602 int ehca_post_srq_recv(struct ib_srq *srq,
603                        struct ib_recv_wr *recv_wr,
604                        struct ib_recv_wr **bad_recv_wr)
605 {
606         return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq),
607                                   srq->device, recv_wr, bad_recv_wr);
608 }
609
610 /*
611  * ib_wc_opcode table converts ehca wc opcode to ib
612  * Since we use zero to indicate invalid opcode, the actual ib opcode must
613  * be decremented!!!
614  */
615 static const u8 ib_wc_opcode[255] = {
616         [0x01] = IB_WC_RECV+1,
617         [0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
618         [0x04] = IB_WC_BIND_MW+1,
619         [0x08] = IB_WC_FETCH_ADD+1,
620         [0x10] = IB_WC_COMP_SWAP+1,
621         [0x20] = IB_WC_RDMA_WRITE+1,
622         [0x40] = IB_WC_RDMA_READ+1,
623         [0x80] = IB_WC_SEND+1
624 };
625
626 /* internal function to poll one entry of cq */
627 static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
628 {
629         int ret = 0, qmap_tail_idx;
630         struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
631         struct ehca_cqe *cqe;
632         struct ehca_qp *my_qp;
633         struct ehca_qmap_entry *qmap_entry;
634         struct ehca_queue_map *qmap;
635         int cqe_count = 0, is_error;
636
637 repoll:
638         cqe = (struct ehca_cqe *)
639                 ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
640         if (!cqe) {
641                 ret = -EAGAIN;
642                 if (ehca_debug_level >= 3)
643                         ehca_dbg(cq->device, "Completion queue is empty  "
644                                  "my_cq=%p cq_num=%x", my_cq, my_cq->cq_number);
645                 goto poll_cq_one_exit0;
646         }
647
648         /* prevents loads being reordered across this point */
649         rmb();
650
651         cqe_count++;
652         if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
653                 struct ehca_qp *qp;
654                 int purgeflag;
655                 unsigned long flags;
656
657                 qp = ehca_cq_get_qp(my_cq, cqe->local_qp_number);
658                 if (!qp) {
659                         ehca_err(cq->device, "cq_num=%x qp_num=%x "
660                                  "could not find qp -> ignore cqe",
661                                  my_cq->cq_number, cqe->local_qp_number);
662                         ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
663                                  my_cq->cq_number, cqe->local_qp_number);
664                         /* ignore this purged cqe */
665                         goto repoll;
666                 }
667                 spin_lock_irqsave(&qp->spinlock_s, flags);
668                 purgeflag = qp->sqerr_purgeflag;
669                 spin_unlock_irqrestore(&qp->spinlock_s, flags);
670
671                 if (purgeflag) {
672                         ehca_dbg(cq->device,
673                                  "Got CQE with purged bit qp_num=%x src_qp=%x",
674                                  cqe->local_qp_number, cqe->remote_qp_number);
675                         if (ehca_debug_level >= 2)
676                                 ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
677                                          cqe->local_qp_number,
678                                          cqe->remote_qp_number);
679                         /*
680                          * ignore this to avoid double cqes of bad wqe
681                          * that caused sqe and turn off purge flag
682                          */
683                         qp->sqerr_purgeflag = 0;
684                         goto repoll;
685                 }
686         }
687
688         is_error = cqe->status & WC_STATUS_ERROR_BIT;
689
690         /* trace error CQEs if debug_level >= 1, trace all CQEs if >= 3 */
691         if (unlikely(ehca_debug_level >= 3 || (ehca_debug_level && is_error))) {
692                 ehca_dbg(cq->device,
693                          "Received %sCOMPLETION ehca_cq=%p cq_num=%x -----",
694                          is_error ? "ERROR " : "", my_cq, my_cq->cq_number);
695                 ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
696                          my_cq, my_cq->cq_number);
697                 ehca_dbg(cq->device,
698                          "ehca_cq=%p cq_num=%x -------------------------",
699                          my_cq, my_cq->cq_number);
700         }
701
702         read_lock(&ehca_qp_idr_lock);
703         my_qp = idr_find(&ehca_qp_idr, cqe->qp_token);
704         read_unlock(&ehca_qp_idr_lock);
705         if (!my_qp)
706                 goto repoll;
707         wc->qp = &my_qp->ib_qp;
708
709         if (is_error) {
710                 /*
711                  * set left_to_poll to 0 because in error state, we will not
712                  * get any additional CQEs
713                  */
714                 ehca_add_to_err_list(my_qp, 1);
715                 my_qp->sq_map.left_to_poll = 0;
716
717                 if (HAS_RQ(my_qp))
718                         ehca_add_to_err_list(my_qp, 0);
719                 my_qp->rq_map.left_to_poll = 0;
720         }
721
722         qmap_tail_idx = get_app_wr_id(cqe->work_request_id);
723         if (!(cqe->w_completion_flags & WC_SEND_RECEIVE_BIT))
724                 /* We got a send completion. */
725                 qmap = &my_qp->sq_map;
726         else
727                 /* We got a receive completion. */
728                 qmap = &my_qp->rq_map;
729
730         qmap_entry = &qmap->map[qmap_tail_idx];
731         if (qmap_entry->reported) {
732                 ehca_warn(cq->device, "Double cqe on qp_num=%#x",
733                                 my_qp->real_qp_num);
734                 /* found a double cqe, discard it and read next one */
735                 goto repoll;
736         }
737
738         wc->wr_id = replace_wr_id(cqe->work_request_id, qmap_entry->app_wr_id);
739         qmap_entry->reported = 1;
740
741         /* this is a proper completion, we need to advance the tail pointer */
742         if (++qmap->tail == qmap->entries)
743                 qmap->tail = 0;
744
745         /* if left_to_poll is decremented to 0, add the QP to the error list */
746         if (qmap->left_to_poll > 0) {
747                 qmap->left_to_poll--;
748                 if ((my_qp->sq_map.left_to_poll == 0) &&
749                                 (my_qp->rq_map.left_to_poll == 0)) {
750                         ehca_add_to_err_list(my_qp, 1);
751                         if (HAS_RQ(my_qp))
752                                 ehca_add_to_err_list(my_qp, 0);
753                 }
754         }
755
756         /* eval ib_wc_opcode */
757         wc->opcode = ib_wc_opcode[cqe->optype]-1;
758         if (unlikely(wc->opcode == -1)) {
759                 ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
760                          "ehca_cq=%p cq_num=%x",
761                          cqe->optype, cqe->status, my_cq, my_cq->cq_number);
762                 /* dump cqe for other infos */
763                 ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
764                          my_cq, my_cq->cq_number);
765                 /* update also queue adder to throw away this entry!!! */
766                 goto repoll;
767         }
768
769         /* eval ib_wc_status */
770         if (unlikely(is_error)) {
771                 /* complete with errors */
772                 map_ib_wc_status(cqe->status, &wc->status);
773                 wc->vendor_err = wc->status;
774         } else
775                 wc->status = IB_WC_SUCCESS;
776
777         wc->byte_len = cqe->nr_bytes_transferred;
778         wc->pkey_index = cqe->pkey_index;
779         wc->slid = cqe->rlid;
780         wc->dlid_path_bits = cqe->dlid;
781         wc->src_qp = cqe->remote_qp_number;
782         wc->wc_flags = cqe->w_completion_flags;
783         wc->ex.imm_data = cpu_to_be32(cqe->immediate_data);
784         wc->sl = cqe->service_level;
785
786 poll_cq_one_exit0:
787         if (cqe_count > 0)
788                 hipz_update_feca(my_cq, cqe_count);
789
790         return ret;
791 }
792
793 static int generate_flush_cqes(struct ehca_qp *my_qp, struct ib_cq *cq,
794                                struct ib_wc *wc, int num_entries,
795                                struct ipz_queue *ipz_queue, int on_sq)
796 {
797         int nr = 0;
798         struct ehca_wqe *wqe;
799         u64 offset;
800         struct ehca_queue_map *qmap;
801         struct ehca_qmap_entry *qmap_entry;
802
803         if (on_sq)
804                 qmap = &my_qp->sq_map;
805         else
806                 qmap = &my_qp->rq_map;
807
808         qmap_entry = &qmap->map[qmap->tail];
809
810         while ((nr < num_entries) && (qmap_entry->reported == 0)) {
811                 /* generate flush CQE */
812                 memset(wc, 0, sizeof(*wc));
813
814                 offset = qmap->tail * ipz_queue->qe_size;
815                 wqe = (struct ehca_wqe *)ipz_qeit_calc(ipz_queue, offset);
816                 if (!wqe) {
817                         ehca_err(cq->device, "Invalid wqe offset=%#lx on "
818                                  "qp_num=%#x", offset, my_qp->real_qp_num);
819                         return nr;
820                 }
821
822                 wc->wr_id = replace_wr_id(wqe->work_request_id,
823                                           qmap_entry->app_wr_id);
824
825                 if (on_sq) {
826                         switch (wqe->optype) {
827                         case WQE_OPTYPE_SEND:
828                                 wc->opcode = IB_WC_SEND;
829                                 break;
830                         case WQE_OPTYPE_RDMAWRITE:
831                                 wc->opcode = IB_WC_RDMA_WRITE;
832                                 break;
833                         case WQE_OPTYPE_RDMAREAD:
834                                 wc->opcode = IB_WC_RDMA_READ;
835                                 break;
836                         default:
837                                 ehca_err(cq->device, "Invalid optype=%x",
838                                                 wqe->optype);
839                                 return nr;
840                         }
841                 } else
842                         wc->opcode = IB_WC_RECV;
843
844                 if (wqe->wr_flag & WQE_WRFLAG_IMM_DATA_PRESENT) {
845                         wc->ex.imm_data = wqe->immediate_data;
846                         wc->wc_flags |= IB_WC_WITH_IMM;
847                 }
848
849                 wc->status = IB_WC_WR_FLUSH_ERR;
850
851                 wc->qp = &my_qp->ib_qp;
852
853                 /* mark as reported and advance tail pointer */
854                 qmap_entry->reported = 1;
855                 if (++qmap->tail == qmap->entries)
856                         qmap->tail = 0;
857                 qmap_entry = &qmap->map[qmap->tail];
858
859                 wc++; nr++;
860         }
861
862         return nr;
863
864 }
865
866 int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
867 {
868         struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
869         int nr;
870         struct ehca_qp *err_qp;
871         struct ib_wc *current_wc = wc;
872         int ret = 0;
873         unsigned long flags;
874         int entries_left = num_entries;
875
876         if (num_entries < 1) {
877                 ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
878                          "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
879                 ret = -EINVAL;
880                 goto poll_cq_exit0;
881         }
882
883         spin_lock_irqsave(&my_cq->spinlock, flags);
884
885         /* generate flush cqes for send queues */
886         list_for_each_entry(err_qp, &my_cq->sqp_err_list, sq_err_node) {
887                 nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
888                                 &err_qp->ipz_squeue, 1);
889                 entries_left -= nr;
890                 current_wc += nr;
891
892                 if (entries_left == 0)
893                         break;
894         }
895
896         /* generate flush cqes for receive queues */
897         list_for_each_entry(err_qp, &my_cq->rqp_err_list, rq_err_node) {
898                 nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
899                                 &err_qp->ipz_rqueue, 0);
900                 entries_left -= nr;
901                 current_wc += nr;
902
903                 if (entries_left == 0)
904                         break;
905         }
906
907         for (nr = 0; nr < entries_left; nr++) {
908                 ret = ehca_poll_cq_one(cq, current_wc);
909                 if (ret)
910                         break;
911                 current_wc++;
912         } /* eof for nr */
913         entries_left -= nr;
914
915         spin_unlock_irqrestore(&my_cq->spinlock, flags);
916         if (ret == -EAGAIN  || !ret)
917                 ret = num_entries - entries_left;
918
919 poll_cq_exit0:
920         return ret;
921 }
922
923 int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags)
924 {
925         struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
926         int ret = 0;
927
928         switch (notify_flags & IB_CQ_SOLICITED_MASK) {
929         case IB_CQ_SOLICITED:
930                 hipz_set_cqx_n0(my_cq, 1);
931                 break;
932         case IB_CQ_NEXT_COMP:
933                 hipz_set_cqx_n1(my_cq, 1);
934                 break;
935         default:
936                 return -EINVAL;
937         }
938
939         if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
940                 unsigned long spl_flags;
941                 spin_lock_irqsave(&my_cq->spinlock, spl_flags);
942                 ret = ipz_qeit_is_valid(&my_cq->ipz_queue);
943                 spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
944         }
945
946         return ret;
947 }