[ARM] Merge AT91 and devel branches
[linux-2.6] / drivers / infiniband / hw / amso1100 / c2_qp.c
1 /*
2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Cisco Systems. All rights reserved.
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
6  * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
7  *
8  * This software is available to you under a choice of one of two
9  * licenses.  You may choose to be licensed under the terms of the GNU
10  * General Public License (GPL) Version 2, available from the file
11  * COPYING in the main directory of this source tree, or the
12  * OpenIB.org BSD license below:
13  *
14  *     Redistribution and use in source and binary forms, with or
15  *     without modification, are permitted provided that the following
16  *     conditions are met:
17  *
18  *      - Redistributions of source code must retain the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer.
21  *
22  *      - Redistributions in binary form must reproduce the above
23  *        copyright notice, this list of conditions and the following
24  *        disclaimer in the documentation and/or other materials
25  *        provided with the distribution.
26  *
27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34  * SOFTWARE.
35  *
36  */
37
38 #include <linux/delay.h>
39
40 #include "c2.h"
41 #include "c2_vq.h"
42 #include "c2_status.h"
43
44 #define C2_MAX_ORD_PER_QP 128
45 #define C2_MAX_IRD_PER_QP 128
46
47 #define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
48 #define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
49 #define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
50
51 #define NO_SUPPORT -1
52 static const u8 c2_opcode[] = {
53         [IB_WR_SEND] = C2_WR_TYPE_SEND,
54         [IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
55         [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
56         [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
57         [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
58         [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
59         [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
60 };
61
62 static int to_c2_state(enum ib_qp_state ib_state)
63 {
64         switch (ib_state) {
65         case IB_QPS_RESET:
66                 return C2_QP_STATE_IDLE;
67         case IB_QPS_RTS:
68                 return C2_QP_STATE_RTS;
69         case IB_QPS_SQD:
70                 return C2_QP_STATE_CLOSING;
71         case IB_QPS_SQE:
72                 return C2_QP_STATE_CLOSING;
73         case IB_QPS_ERR:
74                 return C2_QP_STATE_ERROR;
75         default:
76                 return -1;
77         }
78 }
79
80 static int to_ib_state(enum c2_qp_state c2_state)
81 {
82         switch (c2_state) {
83         case C2_QP_STATE_IDLE:
84                 return IB_QPS_RESET;
85         case C2_QP_STATE_CONNECTING:
86                 return IB_QPS_RTR;
87         case C2_QP_STATE_RTS:
88                 return IB_QPS_RTS;
89         case C2_QP_STATE_CLOSING:
90                 return IB_QPS_SQD;
91         case C2_QP_STATE_ERROR:
92                 return IB_QPS_ERR;
93         case C2_QP_STATE_TERMINATE:
94                 return IB_QPS_SQE;
95         default:
96                 return -1;
97         }
98 }
99
100 static const char *to_ib_state_str(int ib_state)
101 {
102         static const char *state_str[] = {
103                 "IB_QPS_RESET",
104                 "IB_QPS_INIT",
105                 "IB_QPS_RTR",
106                 "IB_QPS_RTS",
107                 "IB_QPS_SQD",
108                 "IB_QPS_SQE",
109                 "IB_QPS_ERR"
110         };
111         if (ib_state < IB_QPS_RESET ||
112             ib_state > IB_QPS_ERR)
113                 return "<invalid IB QP state>";
114
115         ib_state -= IB_QPS_RESET;
116         return state_str[ib_state];
117 }
118
119 void c2_set_qp_state(struct c2_qp *qp, int c2_state)
120 {
121         int new_state = to_ib_state(c2_state);
122
123         pr_debug("%s: qp[%p] state modify %s --> %s\n",
124                __FUNCTION__,
125                 qp,
126                 to_ib_state_str(qp->state),
127                 to_ib_state_str(new_state));
128         qp->state = new_state;
129 }
130
131 #define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
132
133 int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
134                  struct ib_qp_attr *attr, int attr_mask)
135 {
136         struct c2wr_qp_modify_req wr;
137         struct c2wr_qp_modify_rep *reply;
138         struct c2_vq_req *vq_req;
139         unsigned long flags;
140         u8 next_state;
141         int err;
142
143         pr_debug("%s:%d qp=%p, %s --> %s\n",
144                 __FUNCTION__, __LINE__,
145                 qp,
146                 to_ib_state_str(qp->state),
147                 to_ib_state_str(attr->qp_state));
148
149         vq_req = vq_req_alloc(c2dev);
150         if (!vq_req)
151                 return -ENOMEM;
152
153         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
154         wr.hdr.context = (unsigned long) vq_req;
155         wr.rnic_handle = c2dev->adapter_handle;
156         wr.qp_handle = qp->adapter_handle;
157         wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
158         wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
159         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
160         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
161
162         if (attr_mask & IB_QP_STATE) {
163                 /* Ensure the state is valid */
164                 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR)
165                         return -EINVAL;
166
167                 wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
168
169                 if (attr->qp_state == IB_QPS_ERR) {
170                         spin_lock_irqsave(&qp->lock, flags);
171                         if (qp->cm_id && qp->state == IB_QPS_RTS) {
172                                 pr_debug("Generating CLOSE event for QP-->ERR, "
173                                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
174                                 /* Generate an CLOSE event */
175                                 vq_req->cm_id = qp->cm_id;
176                                 vq_req->event = IW_CM_EVENT_CLOSE;
177                         }
178                         spin_unlock_irqrestore(&qp->lock, flags);
179                 }
180                 next_state =  attr->qp_state;
181
182         } else if (attr_mask & IB_QP_CUR_STATE) {
183
184                 if (attr->cur_qp_state != IB_QPS_RTR &&
185                     attr->cur_qp_state != IB_QPS_RTS &&
186                     attr->cur_qp_state != IB_QPS_SQD &&
187                     attr->cur_qp_state != IB_QPS_SQE)
188                         return -EINVAL;
189                 else
190                         wr.next_qp_state =
191                             cpu_to_be32(to_c2_state(attr->cur_qp_state));
192
193                 next_state = attr->cur_qp_state;
194
195         } else {
196                 err = 0;
197                 goto bail0;
198         }
199
200         /* reference the request struct */
201         vq_req_get(c2dev, vq_req);
202
203         err = vq_send_wr(c2dev, (union c2wr *) & wr);
204         if (err) {
205                 vq_req_put(c2dev, vq_req);
206                 goto bail0;
207         }
208
209         err = vq_wait_for_reply(c2dev, vq_req);
210         if (err)
211                 goto bail0;
212
213         reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
214         if (!reply) {
215                 err = -ENOMEM;
216                 goto bail0;
217         }
218
219         err = c2_errno(reply);
220         if (!err)
221                 qp->state = next_state;
222 #ifdef DEBUG
223         else
224                 pr_debug("%s: c2_errno=%d\n", __FUNCTION__, err);
225 #endif
226         /*
227          * If we're going to error and generating the event here, then
228          * we need to remove the reference because there will be no
229          * close event generated by the adapter
230         */
231         spin_lock_irqsave(&qp->lock, flags);
232         if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
233                 qp->cm_id->rem_ref(qp->cm_id);
234                 qp->cm_id = NULL;
235         }
236         spin_unlock_irqrestore(&qp->lock, flags);
237
238         vq_repbuf_free(c2dev, reply);
239       bail0:
240         vq_req_free(c2dev, vq_req);
241
242         pr_debug("%s:%d qp=%p, cur_state=%s\n",
243                 __FUNCTION__, __LINE__,
244                 qp,
245                 to_ib_state_str(qp->state));
246         return err;
247 }
248
249 int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
250                           int ord, int ird)
251 {
252         struct c2wr_qp_modify_req wr;
253         struct c2wr_qp_modify_rep *reply;
254         struct c2_vq_req *vq_req;
255         int err;
256
257         vq_req = vq_req_alloc(c2dev);
258         if (!vq_req)
259                 return -ENOMEM;
260
261         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
262         wr.hdr.context = (unsigned long) vq_req;
263         wr.rnic_handle = c2dev->adapter_handle;
264         wr.qp_handle = qp->adapter_handle;
265         wr.ord = cpu_to_be32(ord);
266         wr.ird = cpu_to_be32(ird);
267         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
268         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
269         wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
270
271         /* reference the request struct */
272         vq_req_get(c2dev, vq_req);
273
274         err = vq_send_wr(c2dev, (union c2wr *) & wr);
275         if (err) {
276                 vq_req_put(c2dev, vq_req);
277                 goto bail0;
278         }
279
280         err = vq_wait_for_reply(c2dev, vq_req);
281         if (err)
282                 goto bail0;
283
284         reply = (struct c2wr_qp_modify_rep *) (unsigned long)
285                 vq_req->reply_msg;
286         if (!reply) {
287                 err = -ENOMEM;
288                 goto bail0;
289         }
290
291         err = c2_errno(reply);
292         vq_repbuf_free(c2dev, reply);
293       bail0:
294         vq_req_free(c2dev, vq_req);
295         return err;
296 }
297
298 static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
299 {
300         struct c2_vq_req *vq_req;
301         struct c2wr_qp_destroy_req wr;
302         struct c2wr_qp_destroy_rep *reply;
303         unsigned long flags;
304         int err;
305
306         /*
307          * Allocate a verb request message
308          */
309         vq_req = vq_req_alloc(c2dev);
310         if (!vq_req) {
311                 return -ENOMEM;
312         }
313
314         /*
315          * Initialize the WR
316          */
317         c2_wr_set_id(&wr, CCWR_QP_DESTROY);
318         wr.hdr.context = (unsigned long) vq_req;
319         wr.rnic_handle = c2dev->adapter_handle;
320         wr.qp_handle = qp->adapter_handle;
321
322         /*
323          * reference the request struct.  dereferenced in the int handler.
324          */
325         vq_req_get(c2dev, vq_req);
326
327         spin_lock_irqsave(&qp->lock, flags);
328         if (qp->cm_id && qp->state == IB_QPS_RTS) {
329                 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
330                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
331                 /* Generate an CLOSE event */
332                 vq_req->qp = qp;
333                 vq_req->cm_id = qp->cm_id;
334                 vq_req->event = IW_CM_EVENT_CLOSE;
335         }
336         spin_unlock_irqrestore(&qp->lock, flags);
337
338         /*
339          * Send WR to adapter
340          */
341         err = vq_send_wr(c2dev, (union c2wr *) & wr);
342         if (err) {
343                 vq_req_put(c2dev, vq_req);
344                 goto bail0;
345         }
346
347         /*
348          * Wait for reply from adapter
349          */
350         err = vq_wait_for_reply(c2dev, vq_req);
351         if (err) {
352                 goto bail0;
353         }
354
355         /*
356          * Process reply
357          */
358         reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
359         if (!reply) {
360                 err = -ENOMEM;
361                 goto bail0;
362         }
363
364         spin_lock_irqsave(&qp->lock, flags);
365         if (qp->cm_id) {
366                 qp->cm_id->rem_ref(qp->cm_id);
367                 qp->cm_id = NULL;
368         }
369         spin_unlock_irqrestore(&qp->lock, flags);
370
371         vq_repbuf_free(c2dev, reply);
372       bail0:
373         vq_req_free(c2dev, vq_req);
374         return err;
375 }
376
377 static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
378 {
379         int ret;
380
381         do {
382                 spin_lock_irq(&c2dev->qp_table.lock);
383                 ret = idr_get_new_above(&c2dev->qp_table.idr, qp,
384                                         c2dev->qp_table.last++, &qp->qpn);
385                 spin_unlock_irq(&c2dev->qp_table.lock);
386         } while ((ret == -EAGAIN) &&
387                  idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL));
388         return ret;
389 }
390
391 static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
392 {
393         spin_lock_irq(&c2dev->qp_table.lock);
394         idr_remove(&c2dev->qp_table.idr, qpn);
395         spin_unlock_irq(&c2dev->qp_table.lock);
396 }
397
398 struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
399 {
400         unsigned long flags;
401         struct c2_qp *qp;
402
403         spin_lock_irqsave(&c2dev->qp_table.lock, flags);
404         qp = idr_find(&c2dev->qp_table.idr, qpn);
405         spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
406         return qp;
407 }
408
409 int c2_alloc_qp(struct c2_dev *c2dev,
410                 struct c2_pd *pd,
411                 struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
412 {
413         struct c2wr_qp_create_req wr;
414         struct c2wr_qp_create_rep *reply;
415         struct c2_vq_req *vq_req;
416         struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
417         struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
418         unsigned long peer_pa;
419         u32 q_size, msg_size, mmap_size;
420         void __iomem *mmap;
421         int err;
422
423         err = c2_alloc_qpn(c2dev, qp);
424         if (err)
425                 return err;
426         qp->ibqp.qp_num = qp->qpn;
427         qp->ibqp.qp_type = IB_QPT_RC;
428
429         /* Allocate the SQ and RQ shared pointers */
430         qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
431                                          &qp->sq_mq.shared_dma, GFP_KERNEL);
432         if (!qp->sq_mq.shared) {
433                 err = -ENOMEM;
434                 goto bail0;
435         }
436
437         qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
438                                          &qp->rq_mq.shared_dma, GFP_KERNEL);
439         if (!qp->rq_mq.shared) {
440                 err = -ENOMEM;
441                 goto bail1;
442         }
443
444         /* Allocate the verbs request */
445         vq_req = vq_req_alloc(c2dev);
446         if (vq_req == NULL) {
447                 err = -ENOMEM;
448                 goto bail2;
449         }
450
451         /* Initialize the work request */
452         memset(&wr, 0, sizeof(wr));
453         c2_wr_set_id(&wr, CCWR_QP_CREATE);
454         wr.hdr.context = (unsigned long) vq_req;
455         wr.rnic_handle = c2dev->adapter_handle;
456         wr.sq_cq_handle = send_cq->adapter_handle;
457         wr.rq_cq_handle = recv_cq->adapter_handle;
458         wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
459         wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
460         wr.srq_handle = 0;
461         wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
462                                QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
463         wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
464         wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
465         wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
466         wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
467         wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
468         wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
469         wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
470         wr.pd_id = pd->pd_id;
471         wr.user_context = (unsigned long) qp;
472
473         vq_req_get(c2dev, vq_req);
474
475         /* Send the WR to the adapter */
476         err = vq_send_wr(c2dev, (union c2wr *) & wr);
477         if (err) {
478                 vq_req_put(c2dev, vq_req);
479                 goto bail3;
480         }
481
482         /* Wait for the verb reply  */
483         err = vq_wait_for_reply(c2dev, vq_req);
484         if (err) {
485                 goto bail3;
486         }
487
488         /* Process the reply */
489         reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
490         if (!reply) {
491                 err = -ENOMEM;
492                 goto bail3;
493         }
494
495         if ((err = c2_wr_get_result(reply)) != 0) {
496                 goto bail4;
497         }
498
499         /* Fill in the kernel QP struct */
500         atomic_set(&qp->refcount, 1);
501         qp->adapter_handle = reply->qp_handle;
502         qp->state = IB_QPS_RESET;
503         qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
504         qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
505         qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
506
507         /* Initialize the SQ MQ */
508         q_size = be32_to_cpu(reply->sq_depth);
509         msg_size = be32_to_cpu(reply->sq_msg_size);
510         peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
511         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
512         mmap = ioremap_nocache(peer_pa, mmap_size);
513         if (!mmap) {
514                 err = -ENOMEM;
515                 goto bail5;
516         }
517
518         c2_mq_req_init(&qp->sq_mq,
519                        be32_to_cpu(reply->sq_mq_index),
520                        q_size,
521                        msg_size,
522                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
523                        mmap,                            /* peer */
524                        C2_MQ_ADAPTER_TARGET);
525
526         /* Initialize the RQ mq */
527         q_size = be32_to_cpu(reply->rq_depth);
528         msg_size = be32_to_cpu(reply->rq_msg_size);
529         peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
530         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
531         mmap = ioremap_nocache(peer_pa, mmap_size);
532         if (!mmap) {
533                 err = -ENOMEM;
534                 goto bail6;
535         }
536
537         c2_mq_req_init(&qp->rq_mq,
538                        be32_to_cpu(reply->rq_mq_index),
539                        q_size,
540                        msg_size,
541                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
542                        mmap,                            /* peer */
543                        C2_MQ_ADAPTER_TARGET);
544
545         vq_repbuf_free(c2dev, reply);
546         vq_req_free(c2dev, vq_req);
547
548         return 0;
549
550       bail6:
551         iounmap(qp->sq_mq.peer);
552       bail5:
553         destroy_qp(c2dev, qp);
554       bail4:
555         vq_repbuf_free(c2dev, reply);
556       bail3:
557         vq_req_free(c2dev, vq_req);
558       bail2:
559         c2_free_mqsp(qp->rq_mq.shared);
560       bail1:
561         c2_free_mqsp(qp->sq_mq.shared);
562       bail0:
563         c2_free_qpn(c2dev, qp->qpn);
564         return err;
565 }
566
567 static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
568 {
569         if (send_cq == recv_cq)
570                 spin_lock_irq(&send_cq->lock);
571         else if (send_cq > recv_cq) {
572                 spin_lock_irq(&send_cq->lock);
573                 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
574         } else {
575                 spin_lock_irq(&recv_cq->lock);
576                 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
577         }
578 }
579
580 static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
581 {
582         if (send_cq == recv_cq)
583                 spin_unlock_irq(&send_cq->lock);
584         else if (send_cq > recv_cq) {
585                 spin_unlock(&recv_cq->lock);
586                 spin_unlock_irq(&send_cq->lock);
587         } else {
588                 spin_unlock(&send_cq->lock);
589                 spin_unlock_irq(&recv_cq->lock);
590         }
591 }
592
593 void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
594 {
595         struct c2_cq *send_cq;
596         struct c2_cq *recv_cq;
597
598         send_cq = to_c2cq(qp->ibqp.send_cq);
599         recv_cq = to_c2cq(qp->ibqp.recv_cq);
600
601         /*
602          * Lock CQs here, so that CQ polling code can do QP lookup
603          * without taking a lock.
604          */
605         c2_lock_cqs(send_cq, recv_cq);
606         c2_free_qpn(c2dev, qp->qpn);
607         c2_unlock_cqs(send_cq, recv_cq);
608
609         /*
610          * Destory qp in the rnic...
611          */
612         destroy_qp(c2dev, qp);
613
614         /*
615          * Mark any unreaped CQEs as null and void.
616          */
617         c2_cq_clean(c2dev, qp, send_cq->cqn);
618         if (send_cq != recv_cq)
619                 c2_cq_clean(c2dev, qp, recv_cq->cqn);
620         /*
621          * Unmap the MQs and return the shared pointers
622          * to the message pool.
623          */
624         iounmap(qp->sq_mq.peer);
625         iounmap(qp->rq_mq.peer);
626         c2_free_mqsp(qp->sq_mq.shared);
627         c2_free_mqsp(qp->rq_mq.shared);
628
629         atomic_dec(&qp->refcount);
630         wait_event(qp->wait, !atomic_read(&qp->refcount));
631 }
632
633 /*
634  * Function: move_sgl
635  *
636  * Description:
637  * Move an SGL from the user's work request struct into a CCIL Work Request
638  * message, swapping to WR byte order and ensure the total length doesn't
639  * overflow.
640  *
641  * IN:
642  * dst          - ptr to CCIL Work Request message SGL memory.
643  * src          - ptr to the consumers SGL memory.
644  *
645  * OUT: none
646  *
647  * Return:
648  * CCIL status codes.
649  */
650 static int
651 move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
652          u8 * actual_count)
653 {
654         u32 tot = 0;            /* running total */
655         u8 acount = 0;          /* running total non-0 len sge's */
656
657         while (count > 0) {
658                 /*
659                  * If the addition of this SGE causes the
660                  * total SGL length to exceed 2^32-1, then
661                  * fail-n-bail.
662                  *
663                  * If the current total plus the next element length
664                  * wraps, then it will go negative and be less than the
665                  * current total...
666                  */
667                 if ((tot + src->length) < tot) {
668                         return -EINVAL;
669                 }
670                 /*
671                  * Bug: 1456 (as well as 1498 & 1643)
672                  * Skip over any sge's supplied with len=0
673                  */
674                 if (src->length) {
675                         tot += src->length;
676                         dst->stag = cpu_to_be32(src->lkey);
677                         dst->to = cpu_to_be64(src->addr);
678                         dst->length = cpu_to_be32(src->length);
679                         dst++;
680                         acount++;
681                 }
682                 src++;
683                 count--;
684         }
685
686         if (acount == 0) {
687                 /*
688                  * Bug: 1476 (as well as 1498, 1456 and 1643)
689                  * Setup the SGL in the WR to make it easier for the RNIC.
690                  * This way, the FW doesn't have to deal with special cases.
691                  * Setting length=0 should be sufficient.
692                  */
693                 dst->stag = 0;
694                 dst->to = 0;
695                 dst->length = 0;
696         }
697
698         *p_len = tot;
699         *actual_count = acount;
700         return 0;
701 }
702
703 /*
704  * Function: c2_activity (private function)
705  *
706  * Description:
707  * Post an mq index to the host->adapter activity fifo.
708  *
709  * IN:
710  * c2dev        - ptr to c2dev structure
711  * mq_index     - mq index to post
712  * shared       - value most recently written to shared
713  *
714  * OUT:
715  *
716  * Return:
717  * none
718  */
719 static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
720 {
721         /*
722          * First read the register to see if the FIFO is full, and if so,
723          * spin until it's not.  This isn't perfect -- there is no
724          * synchronization among the clients of the register, but in
725          * practice it prevents multiple CPU from hammering the bus
726          * with PCI RETRY. Note that when this does happen, the card
727          * cannot get on the bus and the card and system hang in a
728          * deadlock -- thus the need for this code. [TOT]
729          */
730         while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000)
731                 udelay(10);
732
733         __raw_writel(C2_HINT_MAKE(mq_index, shared),
734                      c2dev->regs + PCI_BAR0_ADAPTER_HINT);
735 }
736
737 /*
738  * Function: qp_wr_post
739  *
740  * Description:
741  * This in-line function allocates a MQ msg, then moves the host-copy of
742  * the completed WR into msg.  Then it posts the message.
743  *
744  * IN:
745  * q            - ptr to user MQ.
746  * wr           - ptr to host-copy of the WR.
747  * qp           - ptr to user qp
748  * size         - Number of bytes to post.  Assumed to be divisible by 4.
749  *
750  * OUT: none
751  *
752  * Return:
753  * CCIL status codes.
754  */
755 static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
756 {
757         union c2wr *msg;
758
759         msg = c2_mq_alloc(q);
760         if (msg == NULL) {
761                 return -EINVAL;
762         }
763 #ifdef CCMSGMAGIC
764         ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
765 #endif
766
767         /*
768          * Since all header fields in the WR are the same as the
769          * CQE, set the following so the adapter need not.
770          */
771         c2_wr_set_result(wr, CCERR_PENDING);
772
773         /*
774          * Copy the wr down to the adapter
775          */
776         memcpy((void *) msg, (void *) wr, size);
777
778         c2_mq_produce(q);
779         return 0;
780 }
781
782
783 int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
784                  struct ib_send_wr **bad_wr)
785 {
786         struct c2_dev *c2dev = to_c2dev(ibqp->device);
787         struct c2_qp *qp = to_c2qp(ibqp);
788         union c2wr wr;
789         unsigned long lock_flags;
790         int err = 0;
791
792         u32 flags;
793         u32 tot_len;
794         u8 actual_sge_count;
795         u32 msg_size;
796
797         if (qp->state > IB_QPS_RTS)
798                 return -EINVAL;
799
800         while (ib_wr) {
801
802                 flags = 0;
803                 wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
804                 if (ib_wr->send_flags & IB_SEND_SIGNALED) {
805                         flags |= SQ_SIGNALED;
806                 }
807
808                 switch (ib_wr->opcode) {
809                 case IB_WR_SEND:
810                         if (ib_wr->send_flags & IB_SEND_SOLICITED) {
811                                 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
812                                 msg_size = sizeof(struct c2wr_send_req);
813                         } else {
814                                 c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
815                                 msg_size = sizeof(struct c2wr_send_req);
816                         }
817
818                         wr.sqwr.send.remote_stag = 0;
819                         msg_size += sizeof(struct c2_data_addr) * ib_wr->num_sge;
820                         if (ib_wr->num_sge > qp->send_sgl_depth) {
821                                 err = -EINVAL;
822                                 break;
823                         }
824                         if (ib_wr->send_flags & IB_SEND_FENCE) {
825                                 flags |= SQ_READ_FENCE;
826                         }
827                         err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
828                                        ib_wr->sg_list,
829                                        ib_wr->num_sge,
830                                        &tot_len, &actual_sge_count);
831                         wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
832                         c2_wr_set_sge_count(&wr, actual_sge_count);
833                         break;
834                 case IB_WR_RDMA_WRITE:
835                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
836                         msg_size = sizeof(struct c2wr_rdma_write_req) +
837                             (sizeof(struct c2_data_addr) * ib_wr->num_sge);
838                         if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
839                                 err = -EINVAL;
840                                 break;
841                         }
842                         if (ib_wr->send_flags & IB_SEND_FENCE) {
843                                 flags |= SQ_READ_FENCE;
844                         }
845                         wr.sqwr.rdma_write.remote_stag =
846                             cpu_to_be32(ib_wr->wr.rdma.rkey);
847                         wr.sqwr.rdma_write.remote_to =
848                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
849                         err = move_sgl((struct c2_data_addr *)
850                                        & (wr.sqwr.rdma_write.data),
851                                        ib_wr->sg_list,
852                                        ib_wr->num_sge,
853                                        &tot_len, &actual_sge_count);
854                         wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
855                         c2_wr_set_sge_count(&wr, actual_sge_count);
856                         break;
857                 case IB_WR_RDMA_READ:
858                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
859                         msg_size = sizeof(struct c2wr_rdma_read_req);
860
861                         /* IWarp only suppots 1 sge for RDMA reads */
862                         if (ib_wr->num_sge > 1) {
863                                 err = -EINVAL;
864                                 break;
865                         }
866
867                         /*
868                          * Move the local and remote stag/to/len into the WR.
869                          */
870                         wr.sqwr.rdma_read.local_stag =
871                             cpu_to_be32(ib_wr->sg_list->lkey);
872                         wr.sqwr.rdma_read.local_to =
873                             cpu_to_be64(ib_wr->sg_list->addr);
874                         wr.sqwr.rdma_read.remote_stag =
875                             cpu_to_be32(ib_wr->wr.rdma.rkey);
876                         wr.sqwr.rdma_read.remote_to =
877                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
878                         wr.sqwr.rdma_read.length =
879                             cpu_to_be32(ib_wr->sg_list->length);
880                         break;
881                 default:
882                         /* error */
883                         msg_size = 0;
884                         err = -EINVAL;
885                         break;
886                 }
887
888                 /*
889                  * If we had an error on the last wr build, then
890                  * break out.  Possible errors include bogus WR
891                  * type, and a bogus SGL length...
892                  */
893                 if (err) {
894                         break;
895                 }
896
897                 /*
898                  * Store flags
899                  */
900                 c2_wr_set_flags(&wr, flags);
901
902                 /*
903                  * Post the puppy!
904                  */
905                 spin_lock_irqsave(&qp->lock, lock_flags);
906                 err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
907                 if (err) {
908                         spin_unlock_irqrestore(&qp->lock, lock_flags);
909                         break;
910                 }
911
912                 /*
913                  * Enqueue mq index to activity FIFO.
914                  */
915                 c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
916                 spin_unlock_irqrestore(&qp->lock, lock_flags);
917
918                 ib_wr = ib_wr->next;
919         }
920
921         if (err)
922                 *bad_wr = ib_wr;
923         return err;
924 }
925
926 int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
927                     struct ib_recv_wr **bad_wr)
928 {
929         struct c2_dev *c2dev = to_c2dev(ibqp->device);
930         struct c2_qp *qp = to_c2qp(ibqp);
931         union c2wr wr;
932         unsigned long lock_flags;
933         int err = 0;
934
935         if (qp->state > IB_QPS_RTS)
936                 return -EINVAL;
937
938         /*
939          * Try and post each work request
940          */
941         while (ib_wr) {
942                 u32 tot_len;
943                 u8 actual_sge_count;
944
945                 if (ib_wr->num_sge > qp->recv_sgl_depth) {
946                         err = -EINVAL;
947                         break;
948                 }
949
950                 /*
951                  * Create local host-copy of the WR
952                  */
953                 wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
954                 c2_wr_set_id(&wr, CCWR_RECV);
955                 c2_wr_set_flags(&wr, 0);
956
957                 /* sge_count is limited to eight bits. */
958                 BUG_ON(ib_wr->num_sge >= 256);
959                 err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
960                                ib_wr->sg_list,
961                                ib_wr->num_sge, &tot_len, &actual_sge_count);
962                 c2_wr_set_sge_count(&wr, actual_sge_count);
963
964                 /*
965                  * If we had an error on the last wr build, then
966                  * break out.  Possible errors include bogus WR
967                  * type, and a bogus SGL length...
968                  */
969                 if (err) {
970                         break;
971                 }
972
973                 spin_lock_irqsave(&qp->lock, lock_flags);
974                 err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
975                 if (err) {
976                         spin_unlock_irqrestore(&qp->lock, lock_flags);
977                         break;
978                 }
979
980                 /*
981                  * Enqueue mq index to activity FIFO
982                  */
983                 c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
984                 spin_unlock_irqrestore(&qp->lock, lock_flags);
985
986                 ib_wr = ib_wr->next;
987         }
988
989         if (err)
990                 *bad_wr = ib_wr;
991         return err;
992 }
993
994 void __devinit c2_init_qp_table(struct c2_dev *c2dev)
995 {
996         spin_lock_init(&c2dev->qp_table.lock);
997         idr_init(&c2dev->qp_table.idr);
998 }
999
1000 void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev)
1001 {
1002         idr_destroy(&c2dev->qp_table.idr);
1003 }