Merge branch 'for-linus' of master.kernel.org:/pub/scm/linux/kernel/git/dtor/input
[linux-2.6] / drivers / infiniband / core / cma.c
1 /*
2  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
3  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
4  * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
5  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
6  *
7  * This Software is licensed under one of the following licenses:
8  *
9  * 1) under the terms of the "Common Public License 1.0" a copy of which is
10  *    available from the Open Source Initiative, see
11  *    http://www.opensource.org/licenses/cpl.php.
12  *
13  * 2) under the terms of the "The BSD License" a copy of which is
14  *    available from the Open Source Initiative, see
15  *    http://www.opensource.org/licenses/bsd-license.php.
16  *
17  * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
18  *    copy of which is available from the Open Source Initiative, see
19  *    http://www.opensource.org/licenses/gpl-license.php.
20  *
21  * Licensee has the right to choose one of the above licenses.
22  *
23  * Redistributions of source code must retain the above copyright
24  * notice and one of the license notices.
25  *
26  * Redistributions in binary form must reproduce both the above copyright
27  * notice, one of the license notices in the documentation
28  * and/or other materials provided with the distribution.
29  *
30  */
31
32 #include <linux/completion.h>
33 #include <linux/in.h>
34 #include <linux/in6.h>
35 #include <linux/mutex.h>
36 #include <linux/random.h>
37 #include <linux/idr.h>
38 #include <linux/inetdevice.h>
39
40 #include <net/tcp.h>
41
42 #include <rdma/rdma_cm.h>
43 #include <rdma/rdma_cm_ib.h>
44 #include <rdma/ib_cache.h>
45 #include <rdma/ib_cm.h>
46 #include <rdma/ib_sa.h>
47 #include <rdma/iw_cm.h>
48
49 MODULE_AUTHOR("Sean Hefty");
50 MODULE_DESCRIPTION("Generic RDMA CM Agent");
51 MODULE_LICENSE("Dual BSD/GPL");
52
53 #define CMA_CM_RESPONSE_TIMEOUT 20
54 #define CMA_MAX_CM_RETRIES 15
55
56 static void cma_add_one(struct ib_device *device);
57 static void cma_remove_one(struct ib_device *device);
58
59 static struct ib_client cma_client = {
60         .name   = "cma",
61         .add    = cma_add_one,
62         .remove = cma_remove_one
63 };
64
65 static struct ib_sa_client sa_client;
66 static struct rdma_addr_client addr_client;
67 static LIST_HEAD(dev_list);
68 static LIST_HEAD(listen_any_list);
69 static DEFINE_MUTEX(lock);
70 static struct workqueue_struct *cma_wq;
71 static DEFINE_IDR(sdp_ps);
72 static DEFINE_IDR(tcp_ps);
73 static DEFINE_IDR(udp_ps);
74 static DEFINE_IDR(ipoib_ps);
75 static int next_port;
76
77 struct cma_device {
78         struct list_head        list;
79         struct ib_device        *device;
80         __be64                  node_guid;
81         struct completion       comp;
82         atomic_t                refcount;
83         struct list_head        id_list;
84 };
85
86 enum cma_state {
87         CMA_IDLE,
88         CMA_ADDR_QUERY,
89         CMA_ADDR_RESOLVED,
90         CMA_ROUTE_QUERY,
91         CMA_ROUTE_RESOLVED,
92         CMA_CONNECT,
93         CMA_DISCONNECT,
94         CMA_ADDR_BOUND,
95         CMA_LISTEN,
96         CMA_DEVICE_REMOVAL,
97         CMA_DESTROYING
98 };
99
100 struct rdma_bind_list {
101         struct idr              *ps;
102         struct hlist_head       owners;
103         unsigned short          port;
104 };
105
106 /*
107  * Device removal can occur at anytime, so we need extra handling to
108  * serialize notifying the user of device removal with other callbacks.
109  * We do this by disabling removal notification while a callback is in process,
110  * and reporting it after the callback completes.
111  */
112 struct rdma_id_private {
113         struct rdma_cm_id       id;
114
115         struct rdma_bind_list   *bind_list;
116         struct hlist_node       node;
117         struct list_head        list;
118         struct list_head        listen_list;
119         struct cma_device       *cma_dev;
120         struct list_head        mc_list;
121
122         enum cma_state          state;
123         spinlock_t              lock;
124         struct completion       comp;
125         atomic_t                refcount;
126         wait_queue_head_t       wait_remove;
127         atomic_t                dev_remove;
128
129         int                     backlog;
130         int                     timeout_ms;
131         struct ib_sa_query      *query;
132         int                     query_id;
133         union {
134                 struct ib_cm_id *ib;
135                 struct iw_cm_id *iw;
136         } cm_id;
137
138         u32                     seq_num;
139         u32                     qkey;
140         u32                     qp_num;
141         u8                      srq;
142 };
143
144 struct cma_multicast {
145         struct rdma_id_private *id_priv;
146         union {
147                 struct ib_sa_multicast *ib;
148         } multicast;
149         struct list_head        list;
150         void                    *context;
151         struct sockaddr         addr;
152         u8                      pad[sizeof(struct sockaddr_in6) -
153                                     sizeof(struct sockaddr)];
154 };
155
156 struct cma_work {
157         struct work_struct      work;
158         struct rdma_id_private  *id;
159         enum cma_state          old_state;
160         enum cma_state          new_state;
161         struct rdma_cm_event    event;
162 };
163
164 union cma_ip_addr {
165         struct in6_addr ip6;
166         struct {
167                 __u32 pad[3];
168                 __u32 addr;
169         } ip4;
170 };
171
172 struct cma_hdr {
173         u8 cma_version;
174         u8 ip_version;  /* IP version: 7:4 */
175         __u16 port;
176         union cma_ip_addr src_addr;
177         union cma_ip_addr dst_addr;
178 };
179
180 struct sdp_hh {
181         u8 bsdh[16];
182         u8 sdp_version; /* Major version: 7:4 */
183         u8 ip_version;  /* IP version: 7:4 */
184         u8 sdp_specific1[10];
185         __u16 port;
186         __u16 sdp_specific2;
187         union cma_ip_addr src_addr;
188         union cma_ip_addr dst_addr;
189 };
190
191 struct sdp_hah {
192         u8 bsdh[16];
193         u8 sdp_version;
194 };
195
196 #define CMA_VERSION 0x00
197 #define SDP_MAJ_VERSION 0x2
198
199 static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp)
200 {
201         unsigned long flags;
202         int ret;
203
204         spin_lock_irqsave(&id_priv->lock, flags);
205         ret = (id_priv->state == comp);
206         spin_unlock_irqrestore(&id_priv->lock, flags);
207         return ret;
208 }
209
210 static int cma_comp_exch(struct rdma_id_private *id_priv,
211                          enum cma_state comp, enum cma_state exch)
212 {
213         unsigned long flags;
214         int ret;
215
216         spin_lock_irqsave(&id_priv->lock, flags);
217         if ((ret = (id_priv->state == comp)))
218                 id_priv->state = exch;
219         spin_unlock_irqrestore(&id_priv->lock, flags);
220         return ret;
221 }
222
223 static enum cma_state cma_exch(struct rdma_id_private *id_priv,
224                                enum cma_state exch)
225 {
226         unsigned long flags;
227         enum cma_state old;
228
229         spin_lock_irqsave(&id_priv->lock, flags);
230         old = id_priv->state;
231         id_priv->state = exch;
232         spin_unlock_irqrestore(&id_priv->lock, flags);
233         return old;
234 }
235
236 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
237 {
238         return hdr->ip_version >> 4;
239 }
240
241 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
242 {
243         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
244 }
245
246 static inline u8 sdp_get_majv(u8 sdp_version)
247 {
248         return sdp_version >> 4;
249 }
250
251 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
252 {
253         return hh->ip_version >> 4;
254 }
255
256 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
257 {
258         hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
259 }
260
261 static inline int cma_is_ud_ps(enum rdma_port_space ps)
262 {
263         return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB);
264 }
265
266 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
267                               struct cma_device *cma_dev)
268 {
269         atomic_inc(&cma_dev->refcount);
270         id_priv->cma_dev = cma_dev;
271         id_priv->id.device = cma_dev->device;
272         list_add_tail(&id_priv->list, &cma_dev->id_list);
273 }
274
275 static inline void cma_deref_dev(struct cma_device *cma_dev)
276 {
277         if (atomic_dec_and_test(&cma_dev->refcount))
278                 complete(&cma_dev->comp);
279 }
280
281 static void cma_detach_from_dev(struct rdma_id_private *id_priv)
282 {
283         list_del(&id_priv->list);
284         cma_deref_dev(id_priv->cma_dev);
285         id_priv->cma_dev = NULL;
286 }
287
288 static int cma_set_qkey(struct ib_device *device, u8 port_num,
289                         enum rdma_port_space ps,
290                         struct rdma_dev_addr *dev_addr, u32 *qkey)
291 {
292         struct ib_sa_mcmember_rec rec;
293         int ret = 0;
294
295         switch (ps) {
296         case RDMA_PS_UDP:
297                 *qkey = RDMA_UDP_QKEY;
298                 break;
299         case RDMA_PS_IPOIB:
300                 ib_addr_get_mgid(dev_addr, &rec.mgid);
301                 ret = ib_sa_get_mcmember_rec(device, port_num, &rec.mgid, &rec);
302                 *qkey = be32_to_cpu(rec.qkey);
303                 break;
304         default:
305                 break;
306         }
307         return ret;
308 }
309
310 static int cma_acquire_dev(struct rdma_id_private *id_priv)
311 {
312         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
313         struct cma_device *cma_dev;
314         union ib_gid gid;
315         int ret = -ENODEV;
316
317         switch (rdma_node_get_transport(dev_addr->dev_type)) {
318         case RDMA_TRANSPORT_IB:
319                 ib_addr_get_sgid(dev_addr, &gid);
320                 break;
321         case RDMA_TRANSPORT_IWARP:
322                 iw_addr_get_sgid(dev_addr, &gid);
323                 break;
324         default:
325                 return -ENODEV;
326         }
327
328         list_for_each_entry(cma_dev, &dev_list, list) {
329                 ret = ib_find_cached_gid(cma_dev->device, &gid,
330                                          &id_priv->id.port_num, NULL);
331                 if (!ret) {
332                         ret = cma_set_qkey(cma_dev->device,
333                                            id_priv->id.port_num,
334                                            id_priv->id.ps, dev_addr,
335                                            &id_priv->qkey);
336                         if (!ret)
337                                 cma_attach_to_dev(id_priv, cma_dev);
338                         break;
339                 }
340         }
341         return ret;
342 }
343
344 static void cma_deref_id(struct rdma_id_private *id_priv)
345 {
346         if (atomic_dec_and_test(&id_priv->refcount))
347                 complete(&id_priv->comp);
348 }
349
350 static void cma_release_remove(struct rdma_id_private *id_priv)
351 {
352         if (atomic_dec_and_test(&id_priv->dev_remove))
353                 wake_up(&id_priv->wait_remove);
354 }
355
356 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
357                                   void *context, enum rdma_port_space ps)
358 {
359         struct rdma_id_private *id_priv;
360
361         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
362         if (!id_priv)
363                 return ERR_PTR(-ENOMEM);
364
365         id_priv->state = CMA_IDLE;
366         id_priv->id.context = context;
367         id_priv->id.event_handler = event_handler;
368         id_priv->id.ps = ps;
369         spin_lock_init(&id_priv->lock);
370         init_completion(&id_priv->comp);
371         atomic_set(&id_priv->refcount, 1);
372         init_waitqueue_head(&id_priv->wait_remove);
373         atomic_set(&id_priv->dev_remove, 0);
374         INIT_LIST_HEAD(&id_priv->listen_list);
375         INIT_LIST_HEAD(&id_priv->mc_list);
376         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
377
378         return &id_priv->id;
379 }
380 EXPORT_SYMBOL(rdma_create_id);
381
382 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
383 {
384         struct ib_qp_attr qp_attr;
385         int qp_attr_mask, ret;
386
387         qp_attr.qp_state = IB_QPS_INIT;
388         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
389         if (ret)
390                 return ret;
391
392         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
393         if (ret)
394                 return ret;
395
396         qp_attr.qp_state = IB_QPS_RTR;
397         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
398         if (ret)
399                 return ret;
400
401         qp_attr.qp_state = IB_QPS_RTS;
402         qp_attr.sq_psn = 0;
403         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
404
405         return ret;
406 }
407
408 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
409 {
410         struct ib_qp_attr qp_attr;
411         int qp_attr_mask, ret;
412
413         qp_attr.qp_state = IB_QPS_INIT;
414         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
415         if (ret)
416                 return ret;
417
418         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
419 }
420
421 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
422                    struct ib_qp_init_attr *qp_init_attr)
423 {
424         struct rdma_id_private *id_priv;
425         struct ib_qp *qp;
426         int ret;
427
428         id_priv = container_of(id, struct rdma_id_private, id);
429         if (id->device != pd->device)
430                 return -EINVAL;
431
432         qp = ib_create_qp(pd, qp_init_attr);
433         if (IS_ERR(qp))
434                 return PTR_ERR(qp);
435
436         if (cma_is_ud_ps(id_priv->id.ps))
437                 ret = cma_init_ud_qp(id_priv, qp);
438         else
439                 ret = cma_init_conn_qp(id_priv, qp);
440         if (ret)
441                 goto err;
442
443         id->qp = qp;
444         id_priv->qp_num = qp->qp_num;
445         id_priv->srq = (qp->srq != NULL);
446         return 0;
447 err:
448         ib_destroy_qp(qp);
449         return ret;
450 }
451 EXPORT_SYMBOL(rdma_create_qp);
452
453 void rdma_destroy_qp(struct rdma_cm_id *id)
454 {
455         ib_destroy_qp(id->qp);
456 }
457 EXPORT_SYMBOL(rdma_destroy_qp);
458
459 static int cma_modify_qp_rtr(struct rdma_cm_id *id)
460 {
461         struct ib_qp_attr qp_attr;
462         int qp_attr_mask, ret;
463
464         if (!id->qp)
465                 return 0;
466
467         /* Need to update QP attributes from default values. */
468         qp_attr.qp_state = IB_QPS_INIT;
469         ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
470         if (ret)
471                 return ret;
472
473         ret = ib_modify_qp(id->qp, &qp_attr, qp_attr_mask);
474         if (ret)
475                 return ret;
476
477         qp_attr.qp_state = IB_QPS_RTR;
478         ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
479         if (ret)
480                 return ret;
481
482         return ib_modify_qp(id->qp, &qp_attr, qp_attr_mask);
483 }
484
485 static int cma_modify_qp_rts(struct rdma_cm_id *id)
486 {
487         struct ib_qp_attr qp_attr;
488         int qp_attr_mask, ret;
489
490         if (!id->qp)
491                 return 0;
492
493         qp_attr.qp_state = IB_QPS_RTS;
494         ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
495         if (ret)
496                 return ret;
497
498         return ib_modify_qp(id->qp, &qp_attr, qp_attr_mask);
499 }
500
501 static int cma_modify_qp_err(struct rdma_cm_id *id)
502 {
503         struct ib_qp_attr qp_attr;
504
505         if (!id->qp)
506                 return 0;
507
508         qp_attr.qp_state = IB_QPS_ERR;
509         return ib_modify_qp(id->qp, &qp_attr, IB_QP_STATE);
510 }
511
512 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
513                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
514 {
515         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
516         int ret;
517
518         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
519                                   ib_addr_get_pkey(dev_addr),
520                                   &qp_attr->pkey_index);
521         if (ret)
522                 return ret;
523
524         qp_attr->port_num = id_priv->id.port_num;
525         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
526
527         if (cma_is_ud_ps(id_priv->id.ps)) {
528                 qp_attr->qkey = id_priv->qkey;
529                 *qp_attr_mask |= IB_QP_QKEY;
530         } else {
531                 qp_attr->qp_access_flags = 0;
532                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
533         }
534         return 0;
535 }
536
537 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
538                        int *qp_attr_mask)
539 {
540         struct rdma_id_private *id_priv;
541         int ret = 0;
542
543         id_priv = container_of(id, struct rdma_id_private, id);
544         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
545         case RDMA_TRANSPORT_IB:
546                 if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps))
547                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
548                 else
549                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
550                                                  qp_attr_mask);
551                 if (qp_attr->qp_state == IB_QPS_RTR)
552                         qp_attr->rq_psn = id_priv->seq_num;
553                 break;
554         case RDMA_TRANSPORT_IWARP:
555                 if (!id_priv->cm_id.iw) {
556                         qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE;
557                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
558                 } else
559                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
560                                                  qp_attr_mask);
561                 break;
562         default:
563                 ret = -ENOSYS;
564                 break;
565         }
566
567         return ret;
568 }
569 EXPORT_SYMBOL(rdma_init_qp_attr);
570
571 static inline int cma_zero_addr(struct sockaddr *addr)
572 {
573         struct in6_addr *ip6;
574
575         if (addr->sa_family == AF_INET)
576                 return ZERONET(((struct sockaddr_in *) addr)->sin_addr.s_addr);
577         else {
578                 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
579                 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
580                         ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
581         }
582 }
583
584 static inline int cma_loopback_addr(struct sockaddr *addr)
585 {
586         return LOOPBACK(((struct sockaddr_in *) addr)->sin_addr.s_addr);
587 }
588
589 static inline int cma_any_addr(struct sockaddr *addr)
590 {
591         return cma_zero_addr(addr) || cma_loopback_addr(addr);
592 }
593
594 static inline __be16 cma_port(struct sockaddr *addr)
595 {
596         if (addr->sa_family == AF_INET)
597                 return ((struct sockaddr_in *) addr)->sin_port;
598         else
599                 return ((struct sockaddr_in6 *) addr)->sin6_port;
600 }
601
602 static inline int cma_any_port(struct sockaddr *addr)
603 {
604         return !cma_port(addr);
605 }
606
607 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
608                             u8 *ip_ver, __u16 *port,
609                             union cma_ip_addr **src, union cma_ip_addr **dst)
610 {
611         switch (ps) {
612         case RDMA_PS_SDP:
613                 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
614                     SDP_MAJ_VERSION)
615                         return -EINVAL;
616
617                 *ip_ver = sdp_get_ip_ver(hdr);
618                 *port   = ((struct sdp_hh *) hdr)->port;
619                 *src    = &((struct sdp_hh *) hdr)->src_addr;
620                 *dst    = &((struct sdp_hh *) hdr)->dst_addr;
621                 break;
622         default:
623                 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
624                         return -EINVAL;
625
626                 *ip_ver = cma_get_ip_ver(hdr);
627                 *port   = ((struct cma_hdr *) hdr)->port;
628                 *src    = &((struct cma_hdr *) hdr)->src_addr;
629                 *dst    = &((struct cma_hdr *) hdr)->dst_addr;
630                 break;
631         }
632
633         if (*ip_ver != 4 && *ip_ver != 6)
634                 return -EINVAL;
635         return 0;
636 }
637
638 static void cma_save_net_info(struct rdma_addr *addr,
639                               struct rdma_addr *listen_addr,
640                               u8 ip_ver, __u16 port,
641                               union cma_ip_addr *src, union cma_ip_addr *dst)
642 {
643         struct sockaddr_in *listen4, *ip4;
644         struct sockaddr_in6 *listen6, *ip6;
645
646         switch (ip_ver) {
647         case 4:
648                 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
649                 ip4 = (struct sockaddr_in *) &addr->src_addr;
650                 ip4->sin_family = listen4->sin_family;
651                 ip4->sin_addr.s_addr = dst->ip4.addr;
652                 ip4->sin_port = listen4->sin_port;
653
654                 ip4 = (struct sockaddr_in *) &addr->dst_addr;
655                 ip4->sin_family = listen4->sin_family;
656                 ip4->sin_addr.s_addr = src->ip4.addr;
657                 ip4->sin_port = port;
658                 break;
659         case 6:
660                 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
661                 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
662                 ip6->sin6_family = listen6->sin6_family;
663                 ip6->sin6_addr = dst->ip6;
664                 ip6->sin6_port = listen6->sin6_port;
665
666                 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
667                 ip6->sin6_family = listen6->sin6_family;
668                 ip6->sin6_addr = src->ip6;
669                 ip6->sin6_port = port;
670                 break;
671         default:
672                 break;
673         }
674 }
675
676 static inline int cma_user_data_offset(enum rdma_port_space ps)
677 {
678         switch (ps) {
679         case RDMA_PS_SDP:
680                 return 0;
681         default:
682                 return sizeof(struct cma_hdr);
683         }
684 }
685
686 static void cma_cancel_route(struct rdma_id_private *id_priv)
687 {
688         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
689         case RDMA_TRANSPORT_IB:
690                 if (id_priv->query)
691                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
692                 break;
693         default:
694                 break;
695         }
696 }
697
698 static inline int cma_internal_listen(struct rdma_id_private *id_priv)
699 {
700         return (id_priv->state == CMA_LISTEN) && id_priv->cma_dev &&
701                cma_any_addr(&id_priv->id.route.addr.src_addr);
702 }
703
704 static void cma_destroy_listen(struct rdma_id_private *id_priv)
705 {
706         cma_exch(id_priv, CMA_DESTROYING);
707
708         if (id_priv->cma_dev) {
709                 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
710                 case RDMA_TRANSPORT_IB:
711                         if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
712                                 ib_destroy_cm_id(id_priv->cm_id.ib);
713                         break;
714                 case RDMA_TRANSPORT_IWARP:
715                         if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
716                                 iw_destroy_cm_id(id_priv->cm_id.iw);
717                         break;
718                 default:
719                         break;
720                 }
721                 cma_detach_from_dev(id_priv);
722         }
723         list_del(&id_priv->listen_list);
724
725         cma_deref_id(id_priv);
726         wait_for_completion(&id_priv->comp);
727
728         kfree(id_priv);
729 }
730
731 static void cma_cancel_listens(struct rdma_id_private *id_priv)
732 {
733         struct rdma_id_private *dev_id_priv;
734
735         mutex_lock(&lock);
736         list_del(&id_priv->list);
737
738         while (!list_empty(&id_priv->listen_list)) {
739                 dev_id_priv = list_entry(id_priv->listen_list.next,
740                                          struct rdma_id_private, listen_list);
741                 cma_destroy_listen(dev_id_priv);
742         }
743         mutex_unlock(&lock);
744 }
745
746 static void cma_cancel_operation(struct rdma_id_private *id_priv,
747                                  enum cma_state state)
748 {
749         switch (state) {
750         case CMA_ADDR_QUERY:
751                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
752                 break;
753         case CMA_ROUTE_QUERY:
754                 cma_cancel_route(id_priv);
755                 break;
756         case CMA_LISTEN:
757                 if (cma_any_addr(&id_priv->id.route.addr.src_addr) &&
758                     !id_priv->cma_dev)
759                         cma_cancel_listens(id_priv);
760                 break;
761         default:
762                 break;
763         }
764 }
765
766 static void cma_release_port(struct rdma_id_private *id_priv)
767 {
768         struct rdma_bind_list *bind_list = id_priv->bind_list;
769
770         if (!bind_list)
771                 return;
772
773         mutex_lock(&lock);
774         hlist_del(&id_priv->node);
775         if (hlist_empty(&bind_list->owners)) {
776                 idr_remove(bind_list->ps, bind_list->port);
777                 kfree(bind_list);
778         }
779         mutex_unlock(&lock);
780 }
781
782 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
783 {
784         struct cma_multicast *mc;
785
786         while (!list_empty(&id_priv->mc_list)) {
787                 mc = container_of(id_priv->mc_list.next,
788                                   struct cma_multicast, list);
789                 list_del(&mc->list);
790                 ib_sa_free_multicast(mc->multicast.ib);
791                 kfree(mc);
792         }
793 }
794
795 void rdma_destroy_id(struct rdma_cm_id *id)
796 {
797         struct rdma_id_private *id_priv;
798         enum cma_state state;
799
800         id_priv = container_of(id, struct rdma_id_private, id);
801         state = cma_exch(id_priv, CMA_DESTROYING);
802         cma_cancel_operation(id_priv, state);
803
804         mutex_lock(&lock);
805         if (id_priv->cma_dev) {
806                 mutex_unlock(&lock);
807                 switch (rdma_node_get_transport(id->device->node_type)) {
808                 case RDMA_TRANSPORT_IB:
809                         if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
810                                 ib_destroy_cm_id(id_priv->cm_id.ib);
811                         break;
812                 case RDMA_TRANSPORT_IWARP:
813                         if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
814                                 iw_destroy_cm_id(id_priv->cm_id.iw);
815                         break;
816                 default:
817                         break;
818                 }
819                 cma_leave_mc_groups(id_priv);
820                 mutex_lock(&lock);
821                 cma_detach_from_dev(id_priv);
822         }
823         mutex_unlock(&lock);
824
825         cma_release_port(id_priv);
826         cma_deref_id(id_priv);
827         wait_for_completion(&id_priv->comp);
828
829         kfree(id_priv->id.route.path_rec);
830         kfree(id_priv);
831 }
832 EXPORT_SYMBOL(rdma_destroy_id);
833
834 static int cma_rep_recv(struct rdma_id_private *id_priv)
835 {
836         int ret;
837
838         ret = cma_modify_qp_rtr(&id_priv->id);
839         if (ret)
840                 goto reject;
841
842         ret = cma_modify_qp_rts(&id_priv->id);
843         if (ret)
844                 goto reject;
845
846         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
847         if (ret)
848                 goto reject;
849
850         return 0;
851 reject:
852         cma_modify_qp_err(&id_priv->id);
853         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
854                        NULL, 0, NULL, 0);
855         return ret;
856 }
857
858 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
859 {
860         if (id_priv->id.ps == RDMA_PS_SDP &&
861             sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
862             SDP_MAJ_VERSION)
863                 return -EINVAL;
864
865         return 0;
866 }
867
868 static void cma_set_rep_event_data(struct rdma_cm_event *event,
869                                    struct ib_cm_rep_event_param *rep_data,
870                                    void *private_data)
871 {
872         event->param.conn.private_data = private_data;
873         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
874         event->param.conn.responder_resources = rep_data->responder_resources;
875         event->param.conn.initiator_depth = rep_data->initiator_depth;
876         event->param.conn.flow_control = rep_data->flow_control;
877         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
878         event->param.conn.srq = rep_data->srq;
879         event->param.conn.qp_num = rep_data->remote_qpn;
880 }
881
882 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
883 {
884         struct rdma_id_private *id_priv = cm_id->context;
885         struct rdma_cm_event event;
886         int ret = 0;
887
888         atomic_inc(&id_priv->dev_remove);
889         if (!cma_comp(id_priv, CMA_CONNECT))
890                 goto out;
891
892         memset(&event, 0, sizeof event);
893         switch (ib_event->event) {
894         case IB_CM_REQ_ERROR:
895         case IB_CM_REP_ERROR:
896                 event.event = RDMA_CM_EVENT_UNREACHABLE;
897                 event.status = -ETIMEDOUT;
898                 break;
899         case IB_CM_REP_RECEIVED:
900                 event.status = cma_verify_rep(id_priv, ib_event->private_data);
901                 if (event.status)
902                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
903                 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
904                         event.status = cma_rep_recv(id_priv);
905                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
906                                                      RDMA_CM_EVENT_ESTABLISHED;
907                 } else
908                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
909                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
910                                        ib_event->private_data);
911                 break;
912         case IB_CM_RTU_RECEIVED:
913         case IB_CM_USER_ESTABLISHED:
914                 event.event = RDMA_CM_EVENT_ESTABLISHED;
915                 break;
916         case IB_CM_DREQ_ERROR:
917                 event.status = -ETIMEDOUT; /* fall through */
918         case IB_CM_DREQ_RECEIVED:
919         case IB_CM_DREP_RECEIVED:
920                 if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT))
921                         goto out;
922                 event.event = RDMA_CM_EVENT_DISCONNECTED;
923                 break;
924         case IB_CM_TIMEWAIT_EXIT:
925         case IB_CM_MRA_RECEIVED:
926                 /* ignore event */
927                 goto out;
928         case IB_CM_REJ_RECEIVED:
929                 cma_modify_qp_err(&id_priv->id);
930                 event.status = ib_event->param.rej_rcvd.reason;
931                 event.event = RDMA_CM_EVENT_REJECTED;
932                 event.param.conn.private_data = ib_event->private_data;
933                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
934                 break;
935         default:
936                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d",
937                        ib_event->event);
938                 goto out;
939         }
940
941         ret = id_priv->id.event_handler(&id_priv->id, &event);
942         if (ret) {
943                 /* Destroy the CM ID by returning a non-zero value. */
944                 id_priv->cm_id.ib = NULL;
945                 cma_exch(id_priv, CMA_DESTROYING);
946                 cma_release_remove(id_priv);
947                 rdma_destroy_id(&id_priv->id);
948                 return ret;
949         }
950 out:
951         cma_release_remove(id_priv);
952         return ret;
953 }
954
955 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
956                                                struct ib_cm_event *ib_event)
957 {
958         struct rdma_id_private *id_priv;
959         struct rdma_cm_id *id;
960         struct rdma_route *rt;
961         union cma_ip_addr *src, *dst;
962         __u16 port;
963         u8 ip_ver;
964
965         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
966                              &ip_ver, &port, &src, &dst))
967                 goto err;
968
969         id = rdma_create_id(listen_id->event_handler, listen_id->context,
970                             listen_id->ps);
971         if (IS_ERR(id))
972                 goto err;
973
974         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
975                           ip_ver, port, src, dst);
976
977         rt = &id->route;
978         rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
979         rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
980                                GFP_KERNEL);
981         if (!rt->path_rec)
982                 goto destroy_id;
983
984         rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
985         if (rt->num_paths == 2)
986                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
987
988         ib_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
989         ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
990         ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
991         rt->addr.dev_addr.dev_type = RDMA_NODE_IB_CA;
992
993         id_priv = container_of(id, struct rdma_id_private, id);
994         id_priv->state = CMA_CONNECT;
995         return id_priv;
996
997 destroy_id:
998         rdma_destroy_id(id);
999 err:
1000         return NULL;
1001 }
1002
1003 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1004                                               struct ib_cm_event *ib_event)
1005 {
1006         struct rdma_id_private *id_priv;
1007         struct rdma_cm_id *id;
1008         union cma_ip_addr *src, *dst;
1009         __u16 port;
1010         u8 ip_ver;
1011         int ret;
1012
1013         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1014                             listen_id->ps);
1015         if (IS_ERR(id))
1016                 return NULL;
1017
1018
1019         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1020                              &ip_ver, &port, &src, &dst))
1021                 goto err;
1022
1023         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1024                           ip_ver, port, src, dst);
1025
1026         ret = rdma_translate_ip(&id->route.addr.src_addr,
1027                                 &id->route.addr.dev_addr);
1028         if (ret)
1029                 goto err;
1030
1031         id_priv = container_of(id, struct rdma_id_private, id);
1032         id_priv->state = CMA_CONNECT;
1033         return id_priv;
1034 err:
1035         rdma_destroy_id(id);
1036         return NULL;
1037 }
1038
1039 static void cma_set_req_event_data(struct rdma_cm_event *event,
1040                                    struct ib_cm_req_event_param *req_data,
1041                                    void *private_data, int offset)
1042 {
1043         event->param.conn.private_data = private_data + offset;
1044         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1045         event->param.conn.responder_resources = req_data->responder_resources;
1046         event->param.conn.initiator_depth = req_data->initiator_depth;
1047         event->param.conn.flow_control = req_data->flow_control;
1048         event->param.conn.retry_count = req_data->retry_count;
1049         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1050         event->param.conn.srq = req_data->srq;
1051         event->param.conn.qp_num = req_data->remote_qpn;
1052 }
1053
1054 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1055 {
1056         struct rdma_id_private *listen_id, *conn_id;
1057         struct rdma_cm_event event;
1058         int offset, ret;
1059
1060         listen_id = cm_id->context;
1061         atomic_inc(&listen_id->dev_remove);
1062         if (!cma_comp(listen_id, CMA_LISTEN)) {
1063                 ret = -ECONNABORTED;
1064                 goto out;
1065         }
1066
1067         memset(&event, 0, sizeof event);
1068         offset = cma_user_data_offset(listen_id->id.ps);
1069         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1070         if (cma_is_ud_ps(listen_id->id.ps)) {
1071                 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1072                 event.param.ud.private_data = ib_event->private_data + offset;
1073                 event.param.ud.private_data_len =
1074                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1075         } else {
1076                 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1077                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1078                                        ib_event->private_data, offset);
1079         }
1080         if (!conn_id) {
1081                 ret = -ENOMEM;
1082                 goto out;
1083         }
1084
1085         atomic_inc(&conn_id->dev_remove);
1086         mutex_lock(&lock);
1087         ret = cma_acquire_dev(conn_id);
1088         mutex_unlock(&lock);
1089         if (ret)
1090                 goto release_conn_id;
1091
1092         conn_id->cm_id.ib = cm_id;
1093         cm_id->context = conn_id;
1094         cm_id->cm_handler = cma_ib_handler;
1095
1096         ret = conn_id->id.event_handler(&conn_id->id, &event);
1097         if (!ret)
1098                 goto out;
1099
1100         /* Destroy the CM ID by returning a non-zero value. */
1101         conn_id->cm_id.ib = NULL;
1102
1103 release_conn_id:
1104         cma_exch(conn_id, CMA_DESTROYING);
1105         cma_release_remove(conn_id);
1106         rdma_destroy_id(&conn_id->id);
1107
1108 out:
1109         cma_release_remove(listen_id);
1110         return ret;
1111 }
1112
1113 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1114 {
1115         return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1116 }
1117
1118 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1119                                  struct ib_cm_compare_data *compare)
1120 {
1121         struct cma_hdr *cma_data, *cma_mask;
1122         struct sdp_hh *sdp_data, *sdp_mask;
1123         __u32 ip4_addr;
1124         struct in6_addr ip6_addr;
1125
1126         memset(compare, 0, sizeof *compare);
1127         cma_data = (void *) compare->data;
1128         cma_mask = (void *) compare->mask;
1129         sdp_data = (void *) compare->data;
1130         sdp_mask = (void *) compare->mask;
1131
1132         switch (addr->sa_family) {
1133         case AF_INET:
1134                 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1135                 if (ps == RDMA_PS_SDP) {
1136                         sdp_set_ip_ver(sdp_data, 4);
1137                         sdp_set_ip_ver(sdp_mask, 0xF);
1138                         sdp_data->dst_addr.ip4.addr = ip4_addr;
1139                         sdp_mask->dst_addr.ip4.addr = ~0;
1140                 } else {
1141                         cma_set_ip_ver(cma_data, 4);
1142                         cma_set_ip_ver(cma_mask, 0xF);
1143                         cma_data->dst_addr.ip4.addr = ip4_addr;
1144                         cma_mask->dst_addr.ip4.addr = ~0;
1145                 }
1146                 break;
1147         case AF_INET6:
1148                 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1149                 if (ps == RDMA_PS_SDP) {
1150                         sdp_set_ip_ver(sdp_data, 6);
1151                         sdp_set_ip_ver(sdp_mask, 0xF);
1152                         sdp_data->dst_addr.ip6 = ip6_addr;
1153                         memset(&sdp_mask->dst_addr.ip6, 0xFF,
1154                                sizeof sdp_mask->dst_addr.ip6);
1155                 } else {
1156                         cma_set_ip_ver(cma_data, 6);
1157                         cma_set_ip_ver(cma_mask, 0xF);
1158                         cma_data->dst_addr.ip6 = ip6_addr;
1159                         memset(&cma_mask->dst_addr.ip6, 0xFF,
1160                                sizeof cma_mask->dst_addr.ip6);
1161                 }
1162                 break;
1163         default:
1164                 break;
1165         }
1166 }
1167
1168 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1169 {
1170         struct rdma_id_private *id_priv = iw_id->context;
1171         struct rdma_cm_event event;
1172         struct sockaddr_in *sin;
1173         int ret = 0;
1174
1175         memset(&event, 0, sizeof event);
1176         atomic_inc(&id_priv->dev_remove);
1177
1178         switch (iw_event->event) {
1179         case IW_CM_EVENT_CLOSE:
1180                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1181                 break;
1182         case IW_CM_EVENT_CONNECT_REPLY:
1183                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1184                 *sin = iw_event->local_addr;
1185                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1186                 *sin = iw_event->remote_addr;
1187                 switch (iw_event->status) {
1188                 case 0:
1189                         event.event = RDMA_CM_EVENT_ESTABLISHED;
1190                         break;
1191                 case -ECONNRESET:
1192                 case -ECONNREFUSED:
1193                         event.event = RDMA_CM_EVENT_REJECTED;
1194                         break;
1195                 case -ETIMEDOUT:
1196                         event.event = RDMA_CM_EVENT_UNREACHABLE;
1197                         break;
1198                 default:
1199                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1200                         break;
1201                 }
1202                 break;
1203         case IW_CM_EVENT_ESTABLISHED:
1204                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1205                 break;
1206         default:
1207                 BUG_ON(1);
1208         }
1209
1210         event.status = iw_event->status;
1211         event.param.conn.private_data = iw_event->private_data;
1212         event.param.conn.private_data_len = iw_event->private_data_len;
1213         ret = id_priv->id.event_handler(&id_priv->id, &event);
1214         if (ret) {
1215                 /* Destroy the CM ID by returning a non-zero value. */
1216                 id_priv->cm_id.iw = NULL;
1217                 cma_exch(id_priv, CMA_DESTROYING);
1218                 cma_release_remove(id_priv);
1219                 rdma_destroy_id(&id_priv->id);
1220                 return ret;
1221         }
1222
1223         cma_release_remove(id_priv);
1224         return ret;
1225 }
1226
1227 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1228                                struct iw_cm_event *iw_event)
1229 {
1230         struct rdma_cm_id *new_cm_id;
1231         struct rdma_id_private *listen_id, *conn_id;
1232         struct sockaddr_in *sin;
1233         struct net_device *dev = NULL;
1234         struct rdma_cm_event event;
1235         int ret;
1236
1237         listen_id = cm_id->context;
1238         atomic_inc(&listen_id->dev_remove);
1239         if (!cma_comp(listen_id, CMA_LISTEN)) {
1240                 ret = -ECONNABORTED;
1241                 goto out;
1242         }
1243
1244         /* Create a new RDMA id for the new IW CM ID */
1245         new_cm_id = rdma_create_id(listen_id->id.event_handler,
1246                                    listen_id->id.context,
1247                                    RDMA_PS_TCP);
1248         if (!new_cm_id) {
1249                 ret = -ENOMEM;
1250                 goto out;
1251         }
1252         conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1253         atomic_inc(&conn_id->dev_remove);
1254         conn_id->state = CMA_CONNECT;
1255
1256         dev = ip_dev_find(iw_event->local_addr.sin_addr.s_addr);
1257         if (!dev) {
1258                 ret = -EADDRNOTAVAIL;
1259                 cma_release_remove(conn_id);
1260                 rdma_destroy_id(new_cm_id);
1261                 goto out;
1262         }
1263         ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1264         if (ret) {
1265                 cma_release_remove(conn_id);
1266                 rdma_destroy_id(new_cm_id);
1267                 goto out;
1268         }
1269
1270         mutex_lock(&lock);
1271         ret = cma_acquire_dev(conn_id);
1272         mutex_unlock(&lock);
1273         if (ret) {
1274                 cma_release_remove(conn_id);
1275                 rdma_destroy_id(new_cm_id);
1276                 goto out;
1277         }
1278
1279         conn_id->cm_id.iw = cm_id;
1280         cm_id->context = conn_id;
1281         cm_id->cm_handler = cma_iw_handler;
1282
1283         sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1284         *sin = iw_event->local_addr;
1285         sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1286         *sin = iw_event->remote_addr;
1287
1288         memset(&event, 0, sizeof event);
1289         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1290         event.param.conn.private_data = iw_event->private_data;
1291         event.param.conn.private_data_len = iw_event->private_data_len;
1292         ret = conn_id->id.event_handler(&conn_id->id, &event);
1293         if (ret) {
1294                 /* User wants to destroy the CM ID */
1295                 conn_id->cm_id.iw = NULL;
1296                 cma_exch(conn_id, CMA_DESTROYING);
1297                 cma_release_remove(conn_id);
1298                 rdma_destroy_id(&conn_id->id);
1299         }
1300
1301 out:
1302         if (dev)
1303                 dev_put(dev);
1304         cma_release_remove(listen_id);
1305         return ret;
1306 }
1307
1308 static int cma_ib_listen(struct rdma_id_private *id_priv)
1309 {
1310         struct ib_cm_compare_data compare_data;
1311         struct sockaddr *addr;
1312         __be64 svc_id;
1313         int ret;
1314
1315         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler,
1316                                             id_priv);
1317         if (IS_ERR(id_priv->cm_id.ib))
1318                 return PTR_ERR(id_priv->cm_id.ib);
1319
1320         addr = &id_priv->id.route.addr.src_addr;
1321         svc_id = cma_get_service_id(id_priv->id.ps, addr);
1322         if (cma_any_addr(addr))
1323                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1324         else {
1325                 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1326                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1327         }
1328
1329         if (ret) {
1330                 ib_destroy_cm_id(id_priv->cm_id.ib);
1331                 id_priv->cm_id.ib = NULL;
1332         }
1333
1334         return ret;
1335 }
1336
1337 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1338 {
1339         int ret;
1340         struct sockaddr_in *sin;
1341
1342         id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device,
1343                                             iw_conn_req_handler,
1344                                             id_priv);
1345         if (IS_ERR(id_priv->cm_id.iw))
1346                 return PTR_ERR(id_priv->cm_id.iw);
1347
1348         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1349         id_priv->cm_id.iw->local_addr = *sin;
1350
1351         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1352
1353         if (ret) {
1354                 iw_destroy_cm_id(id_priv->cm_id.iw);
1355                 id_priv->cm_id.iw = NULL;
1356         }
1357
1358         return ret;
1359 }
1360
1361 static int cma_listen_handler(struct rdma_cm_id *id,
1362                               struct rdma_cm_event *event)
1363 {
1364         struct rdma_id_private *id_priv = id->context;
1365
1366         id->context = id_priv->id.context;
1367         id->event_handler = id_priv->id.event_handler;
1368         return id_priv->id.event_handler(id, event);
1369 }
1370
1371 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1372                               struct cma_device *cma_dev)
1373 {
1374         struct rdma_id_private *dev_id_priv;
1375         struct rdma_cm_id *id;
1376         int ret;
1377
1378         id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps);
1379         if (IS_ERR(id))
1380                 return;
1381
1382         dev_id_priv = container_of(id, struct rdma_id_private, id);
1383
1384         dev_id_priv->state = CMA_ADDR_BOUND;
1385         memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
1386                ip_addr_size(&id_priv->id.route.addr.src_addr));
1387
1388         cma_attach_to_dev(dev_id_priv, cma_dev);
1389         list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1390
1391         ret = rdma_listen(id, id_priv->backlog);
1392         if (ret)
1393                 goto err;
1394
1395         return;
1396 err:
1397         cma_destroy_listen(dev_id_priv);
1398 }
1399
1400 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1401 {
1402         struct cma_device *cma_dev;
1403
1404         mutex_lock(&lock);
1405         list_add_tail(&id_priv->list, &listen_any_list);
1406         list_for_each_entry(cma_dev, &dev_list, list)
1407                 cma_listen_on_dev(id_priv, cma_dev);
1408         mutex_unlock(&lock);
1409 }
1410
1411 static int cma_bind_any(struct rdma_cm_id *id, sa_family_t af)
1412 {
1413         struct sockaddr_in addr_in;
1414
1415         memset(&addr_in, 0, sizeof addr_in);
1416         addr_in.sin_family = af;
1417         return rdma_bind_addr(id, (struct sockaddr *) &addr_in);
1418 }
1419
1420 int rdma_listen(struct rdma_cm_id *id, int backlog)
1421 {
1422         struct rdma_id_private *id_priv;
1423         int ret;
1424
1425         id_priv = container_of(id, struct rdma_id_private, id);
1426         if (id_priv->state == CMA_IDLE) {
1427                 ret = cma_bind_any(id, AF_INET);
1428                 if (ret)
1429                         return ret;
1430         }
1431
1432         if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN))
1433                 return -EINVAL;
1434
1435         id_priv->backlog = backlog;
1436         if (id->device) {
1437                 switch (rdma_node_get_transport(id->device->node_type)) {
1438                 case RDMA_TRANSPORT_IB:
1439                         ret = cma_ib_listen(id_priv);
1440                         if (ret)
1441                                 goto err;
1442                         break;
1443                 case RDMA_TRANSPORT_IWARP:
1444                         ret = cma_iw_listen(id_priv, backlog);
1445                         if (ret)
1446                                 goto err;
1447                         break;
1448                 default:
1449                         ret = -ENOSYS;
1450                         goto err;
1451                 }
1452         } else
1453                 cma_listen_on_all(id_priv);
1454
1455         return 0;
1456 err:
1457         id_priv->backlog = 0;
1458         cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND);
1459         return ret;
1460 }
1461 EXPORT_SYMBOL(rdma_listen);
1462
1463 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1464                               void *context)
1465 {
1466         struct cma_work *work = context;
1467         struct rdma_route *route;
1468
1469         route = &work->id->id.route;
1470
1471         if (!status) {
1472                 route->num_paths = 1;
1473                 *route->path_rec = *path_rec;
1474         } else {
1475                 work->old_state = CMA_ROUTE_QUERY;
1476                 work->new_state = CMA_ADDR_RESOLVED;
1477                 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1478                 work->event.status = status;
1479         }
1480
1481         queue_work(cma_wq, &work->work);
1482 }
1483
1484 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1485                               struct cma_work *work)
1486 {
1487         struct rdma_dev_addr *addr = &id_priv->id.route.addr.dev_addr;
1488         struct ib_sa_path_rec path_rec;
1489
1490         memset(&path_rec, 0, sizeof path_rec);
1491         ib_addr_get_sgid(addr, &path_rec.sgid);
1492         ib_addr_get_dgid(addr, &path_rec.dgid);
1493         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(addr));
1494         path_rec.numb_path = 1;
1495
1496         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1497                                 id_priv->id.port_num, &path_rec,
1498                                 IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1499                                 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH,
1500                                 timeout_ms, GFP_KERNEL,
1501                                 cma_query_handler, work, &id_priv->query);
1502
1503         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1504 }
1505
1506 static void cma_work_handler(struct work_struct *_work)
1507 {
1508         struct cma_work *work = container_of(_work, struct cma_work, work);
1509         struct rdma_id_private *id_priv = work->id;
1510         int destroy = 0;
1511
1512         atomic_inc(&id_priv->dev_remove);
1513         if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1514                 goto out;
1515
1516         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1517                 cma_exch(id_priv, CMA_DESTROYING);
1518                 destroy = 1;
1519         }
1520 out:
1521         cma_release_remove(id_priv);
1522         cma_deref_id(id_priv);
1523         if (destroy)
1524                 rdma_destroy_id(&id_priv->id);
1525         kfree(work);
1526 }
1527
1528 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1529 {
1530         struct rdma_route *route = &id_priv->id.route;
1531         struct cma_work *work;
1532         int ret;
1533
1534         work = kzalloc(sizeof *work, GFP_KERNEL);
1535         if (!work)
1536                 return -ENOMEM;
1537
1538         work->id = id_priv;
1539         INIT_WORK(&work->work, cma_work_handler);
1540         work->old_state = CMA_ROUTE_QUERY;
1541         work->new_state = CMA_ROUTE_RESOLVED;
1542         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1543
1544         route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1545         if (!route->path_rec) {
1546                 ret = -ENOMEM;
1547                 goto err1;
1548         }
1549
1550         ret = cma_query_ib_route(id_priv, timeout_ms, work);
1551         if (ret)
1552                 goto err2;
1553
1554         return 0;
1555 err2:
1556         kfree(route->path_rec);
1557         route->path_rec = NULL;
1558 err1:
1559         kfree(work);
1560         return ret;
1561 }
1562
1563 int rdma_set_ib_paths(struct rdma_cm_id *id,
1564                       struct ib_sa_path_rec *path_rec, int num_paths)
1565 {
1566         struct rdma_id_private *id_priv;
1567         int ret;
1568
1569         id_priv = container_of(id, struct rdma_id_private, id);
1570         if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED))
1571                 return -EINVAL;
1572
1573         id->route.path_rec = kmalloc(sizeof *path_rec * num_paths, GFP_KERNEL);
1574         if (!id->route.path_rec) {
1575                 ret = -ENOMEM;
1576                 goto err;
1577         }
1578
1579         memcpy(id->route.path_rec, path_rec, sizeof *path_rec * num_paths);
1580         return 0;
1581 err:
1582         cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED);
1583         return ret;
1584 }
1585 EXPORT_SYMBOL(rdma_set_ib_paths);
1586
1587 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1588 {
1589         struct cma_work *work;
1590
1591         work = kzalloc(sizeof *work, GFP_KERNEL);
1592         if (!work)
1593                 return -ENOMEM;
1594
1595         work->id = id_priv;
1596         INIT_WORK(&work->work, cma_work_handler);
1597         work->old_state = CMA_ROUTE_QUERY;
1598         work->new_state = CMA_ROUTE_RESOLVED;
1599         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1600         queue_work(cma_wq, &work->work);
1601         return 0;
1602 }
1603
1604 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1605 {
1606         struct rdma_id_private *id_priv;
1607         int ret;
1608
1609         id_priv = container_of(id, struct rdma_id_private, id);
1610         if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY))
1611                 return -EINVAL;
1612
1613         atomic_inc(&id_priv->refcount);
1614         switch (rdma_node_get_transport(id->device->node_type)) {
1615         case RDMA_TRANSPORT_IB:
1616                 ret = cma_resolve_ib_route(id_priv, timeout_ms);
1617                 break;
1618         case RDMA_TRANSPORT_IWARP:
1619                 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1620                 break;
1621         default:
1622                 ret = -ENOSYS;
1623                 break;
1624         }
1625         if (ret)
1626                 goto err;
1627
1628         return 0;
1629 err:
1630         cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED);
1631         cma_deref_id(id_priv);
1632         return ret;
1633 }
1634 EXPORT_SYMBOL(rdma_resolve_route);
1635
1636 static int cma_bind_loopback(struct rdma_id_private *id_priv)
1637 {
1638         struct cma_device *cma_dev;
1639         struct ib_port_attr port_attr;
1640         union ib_gid gid;
1641         u16 pkey;
1642         int ret;
1643         u8 p;
1644
1645         mutex_lock(&lock);
1646         if (list_empty(&dev_list)) {
1647                 ret = -ENODEV;
1648                 goto out;
1649         }
1650         list_for_each_entry(cma_dev, &dev_list, list)
1651                 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
1652                         if (!ib_query_port(cma_dev->device, p, &port_attr) &&
1653                             port_attr.state == IB_PORT_ACTIVE)
1654                                 goto port_found;
1655
1656         p = 1;
1657         cma_dev = list_entry(dev_list.next, struct cma_device, list);
1658
1659 port_found:
1660         ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
1661         if (ret)
1662                 goto out;
1663
1664         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
1665         if (ret)
1666                 goto out;
1667
1668         ib_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1669         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
1670         id_priv->id.port_num = p;
1671         cma_attach_to_dev(id_priv, cma_dev);
1672 out:
1673         mutex_unlock(&lock);
1674         return ret;
1675 }
1676
1677 static void addr_handler(int status, struct sockaddr *src_addr,
1678                          struct rdma_dev_addr *dev_addr, void *context)
1679 {
1680         struct rdma_id_private *id_priv = context;
1681         struct rdma_cm_event event;
1682
1683         memset(&event, 0, sizeof event);
1684         atomic_inc(&id_priv->dev_remove);
1685
1686         /*
1687          * Grab mutex to block rdma_destroy_id() from removing the device while
1688          * we're trying to acquire it.
1689          */
1690         mutex_lock(&lock);
1691         if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) {
1692                 mutex_unlock(&lock);
1693                 goto out;
1694         }
1695
1696         if (!status && !id_priv->cma_dev)
1697                 status = cma_acquire_dev(id_priv);
1698         mutex_unlock(&lock);
1699
1700         if (status) {
1701                 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND))
1702                         goto out;
1703                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
1704                 event.status = status;
1705         } else {
1706                 memcpy(&id_priv->id.route.addr.src_addr, src_addr,
1707                        ip_addr_size(src_addr));
1708                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1709         }
1710
1711         if (id_priv->id.event_handler(&id_priv->id, &event)) {
1712                 cma_exch(id_priv, CMA_DESTROYING);
1713                 cma_release_remove(id_priv);
1714                 cma_deref_id(id_priv);
1715                 rdma_destroy_id(&id_priv->id);
1716                 return;
1717         }
1718 out:
1719         cma_release_remove(id_priv);
1720         cma_deref_id(id_priv);
1721 }
1722
1723 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
1724 {
1725         struct cma_work *work;
1726         struct sockaddr_in *src_in, *dst_in;
1727         union ib_gid gid;
1728         int ret;
1729
1730         work = kzalloc(sizeof *work, GFP_KERNEL);
1731         if (!work)
1732                 return -ENOMEM;
1733
1734         if (!id_priv->cma_dev) {
1735                 ret = cma_bind_loopback(id_priv);
1736                 if (ret)
1737                         goto err;
1738         }
1739
1740         ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1741         ib_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
1742
1743         if (cma_zero_addr(&id_priv->id.route.addr.src_addr)) {
1744                 src_in = (struct sockaddr_in *)&id_priv->id.route.addr.src_addr;
1745                 dst_in = (struct sockaddr_in *)&id_priv->id.route.addr.dst_addr;
1746                 src_in->sin_family = dst_in->sin_family;
1747                 src_in->sin_addr.s_addr = dst_in->sin_addr.s_addr;
1748         }
1749
1750         work->id = id_priv;
1751         INIT_WORK(&work->work, cma_work_handler);
1752         work->old_state = CMA_ADDR_QUERY;
1753         work->new_state = CMA_ADDR_RESOLVED;
1754         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1755         queue_work(cma_wq, &work->work);
1756         return 0;
1757 err:
1758         kfree(work);
1759         return ret;
1760 }
1761
1762 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1763                          struct sockaddr *dst_addr)
1764 {
1765         if (src_addr && src_addr->sa_family)
1766                 return rdma_bind_addr(id, src_addr);
1767         else
1768                 return cma_bind_any(id, dst_addr->sa_family);
1769 }
1770
1771 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1772                       struct sockaddr *dst_addr, int timeout_ms)
1773 {
1774         struct rdma_id_private *id_priv;
1775         int ret;
1776
1777         id_priv = container_of(id, struct rdma_id_private, id);
1778         if (id_priv->state == CMA_IDLE) {
1779                 ret = cma_bind_addr(id, src_addr, dst_addr);
1780                 if (ret)
1781                         return ret;
1782         }
1783
1784         if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY))
1785                 return -EINVAL;
1786
1787         atomic_inc(&id_priv->refcount);
1788         memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
1789         if (cma_any_addr(dst_addr))
1790                 ret = cma_resolve_loopback(id_priv);
1791         else
1792                 ret = rdma_resolve_ip(&addr_client, &id->route.addr.src_addr,
1793                                       dst_addr, &id->route.addr.dev_addr,
1794                                       timeout_ms, addr_handler, id_priv);
1795         if (ret)
1796                 goto err;
1797
1798         return 0;
1799 err:
1800         cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND);
1801         cma_deref_id(id_priv);
1802         return ret;
1803 }
1804 EXPORT_SYMBOL(rdma_resolve_addr);
1805
1806 static void cma_bind_port(struct rdma_bind_list *bind_list,
1807                           struct rdma_id_private *id_priv)
1808 {
1809         struct sockaddr_in *sin;
1810
1811         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1812         sin->sin_port = htons(bind_list->port);
1813         id_priv->bind_list = bind_list;
1814         hlist_add_head(&id_priv->node, &bind_list->owners);
1815 }
1816
1817 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
1818                           unsigned short snum)
1819 {
1820         struct rdma_bind_list *bind_list;
1821         int port, ret;
1822
1823         bind_list = kmalloc(sizeof *bind_list, GFP_KERNEL);
1824         if (!bind_list)
1825                 return -ENOMEM;
1826
1827         do {
1828                 ret = idr_get_new_above(ps, bind_list, snum, &port);
1829         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1830
1831         if (ret)
1832                 goto err1;
1833
1834         if (port != snum) {
1835                 ret = -EADDRNOTAVAIL;
1836                 goto err2;
1837         }
1838
1839         bind_list->ps = ps;
1840         bind_list->port = (unsigned short) port;
1841         cma_bind_port(bind_list, id_priv);
1842         return 0;
1843 err2:
1844         idr_remove(ps, port);
1845 err1:
1846         kfree(bind_list);
1847         return ret;
1848 }
1849
1850 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
1851 {
1852         struct rdma_bind_list *bind_list;
1853         int port, ret;
1854
1855         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
1856         if (!bind_list)
1857                 return -ENOMEM;
1858
1859 retry:
1860         do {
1861                 ret = idr_get_new_above(ps, bind_list, next_port, &port);
1862         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1863
1864         if (ret)
1865                 goto err1;
1866
1867         if (port > sysctl_local_port_range[1]) {
1868                 if (next_port != sysctl_local_port_range[0]) {
1869                         idr_remove(ps, port);
1870                         next_port = sysctl_local_port_range[0];
1871                         goto retry;
1872                 }
1873                 ret = -EADDRNOTAVAIL;
1874                 goto err2;
1875         }
1876
1877         if (port == sysctl_local_port_range[1])
1878                 next_port = sysctl_local_port_range[0];
1879         else
1880                 next_port = port + 1;
1881
1882         bind_list->ps = ps;
1883         bind_list->port = (unsigned short) port;
1884         cma_bind_port(bind_list, id_priv);
1885         return 0;
1886 err2:
1887         idr_remove(ps, port);
1888 err1:
1889         kfree(bind_list);
1890         return ret;
1891 }
1892
1893 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
1894 {
1895         struct rdma_id_private *cur_id;
1896         struct sockaddr_in *sin, *cur_sin;
1897         struct rdma_bind_list *bind_list;
1898         struct hlist_node *node;
1899         unsigned short snum;
1900
1901         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1902         snum = ntohs(sin->sin_port);
1903         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
1904                 return -EACCES;
1905
1906         bind_list = idr_find(ps, snum);
1907         if (!bind_list)
1908                 return cma_alloc_port(ps, id_priv, snum);
1909
1910         /*
1911          * We don't support binding to any address if anyone is bound to
1912          * a specific address on the same port.
1913          */
1914         if (cma_any_addr(&id_priv->id.route.addr.src_addr))
1915                 return -EADDRNOTAVAIL;
1916
1917         hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
1918                 if (cma_any_addr(&cur_id->id.route.addr.src_addr))
1919                         return -EADDRNOTAVAIL;
1920
1921                 cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr;
1922                 if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr)
1923                         return -EADDRINUSE;
1924         }
1925
1926         cma_bind_port(bind_list, id_priv);
1927         return 0;
1928 }
1929
1930 static int cma_get_port(struct rdma_id_private *id_priv)
1931 {
1932         struct idr *ps;
1933         int ret;
1934
1935         switch (id_priv->id.ps) {
1936         case RDMA_PS_SDP:
1937                 ps = &sdp_ps;
1938                 break;
1939         case RDMA_PS_TCP:
1940                 ps = &tcp_ps;
1941                 break;
1942         case RDMA_PS_UDP:
1943                 ps = &udp_ps;
1944                 break;
1945         case RDMA_PS_IPOIB:
1946                 ps = &ipoib_ps;
1947                 break;
1948         default:
1949                 return -EPROTONOSUPPORT;
1950         }
1951
1952         mutex_lock(&lock);
1953         if (cma_any_port(&id_priv->id.route.addr.src_addr))
1954                 ret = cma_alloc_any_port(ps, id_priv);
1955         else
1956                 ret = cma_use_port(ps, id_priv);
1957         mutex_unlock(&lock);
1958
1959         return ret;
1960 }
1961
1962 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
1963 {
1964         struct rdma_id_private *id_priv;
1965         int ret;
1966
1967         if (addr->sa_family != AF_INET)
1968                 return -EAFNOSUPPORT;
1969
1970         id_priv = container_of(id, struct rdma_id_private, id);
1971         if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND))
1972                 return -EINVAL;
1973
1974         if (!cma_any_addr(addr)) {
1975                 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
1976                 if (ret)
1977                         goto err1;
1978
1979                 mutex_lock(&lock);
1980                 ret = cma_acquire_dev(id_priv);
1981                 mutex_unlock(&lock);
1982                 if (ret)
1983                         goto err1;
1984         }
1985
1986         memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
1987         ret = cma_get_port(id_priv);
1988         if (ret)
1989                 goto err2;
1990
1991         return 0;
1992 err2:
1993         if (!cma_any_addr(addr)) {
1994                 mutex_lock(&lock);
1995                 cma_detach_from_dev(id_priv);
1996                 mutex_unlock(&lock);
1997         }
1998 err1:
1999         cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE);
2000         return ret;
2001 }
2002 EXPORT_SYMBOL(rdma_bind_addr);
2003
2004 static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2005                           struct rdma_route *route)
2006 {
2007         struct sockaddr_in *src4, *dst4;
2008         struct cma_hdr *cma_hdr;
2009         struct sdp_hh *sdp_hdr;
2010
2011         src4 = (struct sockaddr_in *) &route->addr.src_addr;
2012         dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2013
2014         switch (ps) {
2015         case RDMA_PS_SDP:
2016                 sdp_hdr = hdr;
2017                 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2018                         return -EINVAL;
2019                 sdp_set_ip_ver(sdp_hdr, 4);
2020                 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2021                 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2022                 sdp_hdr->port = src4->sin_port;
2023                 break;
2024         default:
2025                 cma_hdr = hdr;
2026                 cma_hdr->cma_version = CMA_VERSION;
2027                 cma_set_ip_ver(cma_hdr, 4);
2028                 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2029                 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2030                 cma_hdr->port = src4->sin_port;
2031                 break;
2032         }
2033         return 0;
2034 }
2035
2036 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2037                                 struct ib_cm_event *ib_event)
2038 {
2039         struct rdma_id_private *id_priv = cm_id->context;
2040         struct rdma_cm_event event;
2041         struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2042         int ret = 0;
2043
2044         memset(&event, 0, sizeof event);
2045         atomic_inc(&id_priv->dev_remove);
2046         if (!cma_comp(id_priv, CMA_CONNECT))
2047                 goto out;
2048
2049         switch (ib_event->event) {
2050         case IB_CM_SIDR_REQ_ERROR:
2051                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2052                 event.status = -ETIMEDOUT;
2053                 break;
2054         case IB_CM_SIDR_REP_RECEIVED:
2055                 event.param.ud.private_data = ib_event->private_data;
2056                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2057                 if (rep->status != IB_SIDR_SUCCESS) {
2058                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2059                         event.status = ib_event->param.sidr_rep_rcvd.status;
2060                         break;
2061                 }
2062                 if (id_priv->qkey != rep->qkey) {
2063                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2064                         event.status = -EINVAL;
2065                         break;
2066                 }
2067                 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2068                                      id_priv->id.route.path_rec,
2069                                      &event.param.ud.ah_attr);
2070                 event.param.ud.qp_num = rep->qpn;
2071                 event.param.ud.qkey = rep->qkey;
2072                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2073                 event.status = 0;
2074                 break;
2075         default:
2076                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d",
2077                        ib_event->event);
2078                 goto out;
2079         }
2080
2081         ret = id_priv->id.event_handler(&id_priv->id, &event);
2082         if (ret) {
2083                 /* Destroy the CM ID by returning a non-zero value. */
2084                 id_priv->cm_id.ib = NULL;
2085                 cma_exch(id_priv, CMA_DESTROYING);
2086                 cma_release_remove(id_priv);
2087                 rdma_destroy_id(&id_priv->id);
2088                 return ret;
2089         }
2090 out:
2091         cma_release_remove(id_priv);
2092         return ret;
2093 }
2094
2095 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2096                               struct rdma_conn_param *conn_param)
2097 {
2098         struct ib_cm_sidr_req_param req;
2099         struct rdma_route *route;
2100         int ret;
2101
2102         req.private_data_len = sizeof(struct cma_hdr) +
2103                                conn_param->private_data_len;
2104         req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2105         if (!req.private_data)
2106                 return -ENOMEM;
2107
2108         if (conn_param->private_data && conn_param->private_data_len)
2109                 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2110                        conn_param->private_data, conn_param->private_data_len);
2111
2112         route = &id_priv->id.route;
2113         ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2114         if (ret)
2115                 goto out;
2116
2117         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device,
2118                                             cma_sidr_rep_handler, id_priv);
2119         if (IS_ERR(id_priv->cm_id.ib)) {
2120                 ret = PTR_ERR(id_priv->cm_id.ib);
2121                 goto out;
2122         }
2123
2124         req.path = route->path_rec;
2125         req.service_id = cma_get_service_id(id_priv->id.ps,
2126                                             &route->addr.dst_addr);
2127         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2128         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2129
2130         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2131         if (ret) {
2132                 ib_destroy_cm_id(id_priv->cm_id.ib);
2133                 id_priv->cm_id.ib = NULL;
2134         }
2135 out:
2136         kfree(req.private_data);
2137         return ret;
2138 }
2139
2140 static int cma_connect_ib(struct rdma_id_private *id_priv,
2141                           struct rdma_conn_param *conn_param)
2142 {
2143         struct ib_cm_req_param req;
2144         struct rdma_route *route;
2145         void *private_data;
2146         int offset, ret;
2147
2148         memset(&req, 0, sizeof req);
2149         offset = cma_user_data_offset(id_priv->id.ps);
2150         req.private_data_len = offset + conn_param->private_data_len;
2151         private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2152         if (!private_data)
2153                 return -ENOMEM;
2154
2155         if (conn_param->private_data && conn_param->private_data_len)
2156                 memcpy(private_data + offset, conn_param->private_data,
2157                        conn_param->private_data_len);
2158
2159         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler,
2160                                             id_priv);
2161         if (IS_ERR(id_priv->cm_id.ib)) {
2162                 ret = PTR_ERR(id_priv->cm_id.ib);
2163                 goto out;
2164         }
2165
2166         route = &id_priv->id.route;
2167         ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2168         if (ret)
2169                 goto out;
2170         req.private_data = private_data;
2171
2172         req.primary_path = &route->path_rec[0];
2173         if (route->num_paths == 2)
2174                 req.alternate_path = &route->path_rec[1];
2175
2176         req.service_id = cma_get_service_id(id_priv->id.ps,
2177                                             &route->addr.dst_addr);
2178         req.qp_num = id_priv->qp_num;
2179         req.qp_type = IB_QPT_RC;
2180         req.starting_psn = id_priv->seq_num;
2181         req.responder_resources = conn_param->responder_resources;
2182         req.initiator_depth = conn_param->initiator_depth;
2183         req.flow_control = conn_param->flow_control;
2184         req.retry_count = conn_param->retry_count;
2185         req.rnr_retry_count = conn_param->rnr_retry_count;
2186         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2187         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2188         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2189         req.srq = id_priv->srq ? 1 : 0;
2190
2191         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2192 out:
2193         if (ret && !IS_ERR(id_priv->cm_id.ib)) {
2194                 ib_destroy_cm_id(id_priv->cm_id.ib);
2195                 id_priv->cm_id.ib = NULL;
2196         }
2197
2198         kfree(private_data);
2199         return ret;
2200 }
2201
2202 static int cma_connect_iw(struct rdma_id_private *id_priv,
2203                           struct rdma_conn_param *conn_param)
2204 {
2205         struct iw_cm_id *cm_id;
2206         struct sockaddr_in* sin;
2207         int ret;
2208         struct iw_cm_conn_param iw_param;
2209
2210         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2211         if (IS_ERR(cm_id)) {
2212                 ret = PTR_ERR(cm_id);
2213                 goto out;
2214         }
2215
2216         id_priv->cm_id.iw = cm_id;
2217
2218         sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2219         cm_id->local_addr = *sin;
2220
2221         sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2222         cm_id->remote_addr = *sin;
2223
2224         ret = cma_modify_qp_rtr(&id_priv->id);
2225         if (ret)
2226                 goto out;
2227
2228         iw_param.ord = conn_param->initiator_depth;
2229         iw_param.ird = conn_param->responder_resources;
2230         iw_param.private_data = conn_param->private_data;
2231         iw_param.private_data_len = conn_param->private_data_len;
2232         if (id_priv->id.qp)
2233                 iw_param.qpn = id_priv->qp_num;
2234         else
2235                 iw_param.qpn = conn_param->qp_num;
2236         ret = iw_cm_connect(cm_id, &iw_param);
2237 out:
2238         if (ret && !IS_ERR(cm_id)) {
2239                 iw_destroy_cm_id(cm_id);
2240                 id_priv->cm_id.iw = NULL;
2241         }
2242         return ret;
2243 }
2244
2245 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2246 {
2247         struct rdma_id_private *id_priv;
2248         int ret;
2249
2250         id_priv = container_of(id, struct rdma_id_private, id);
2251         if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT))
2252                 return -EINVAL;
2253
2254         if (!id->qp) {
2255                 id_priv->qp_num = conn_param->qp_num;
2256                 id_priv->srq = conn_param->srq;
2257         }
2258
2259         switch (rdma_node_get_transport(id->device->node_type)) {
2260         case RDMA_TRANSPORT_IB:
2261                 if (cma_is_ud_ps(id->ps))
2262                         ret = cma_resolve_ib_udp(id_priv, conn_param);
2263                 else
2264                         ret = cma_connect_ib(id_priv, conn_param);
2265                 break;
2266         case RDMA_TRANSPORT_IWARP:
2267                 ret = cma_connect_iw(id_priv, conn_param);
2268                 break;
2269         default:
2270                 ret = -ENOSYS;
2271                 break;
2272         }
2273         if (ret)
2274                 goto err;
2275
2276         return 0;
2277 err:
2278         cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED);
2279         return ret;
2280 }
2281 EXPORT_SYMBOL(rdma_connect);
2282
2283 static int cma_accept_ib(struct rdma_id_private *id_priv,
2284                          struct rdma_conn_param *conn_param)
2285 {
2286         struct ib_cm_rep_param rep;
2287         struct ib_qp_attr qp_attr;
2288         int qp_attr_mask, ret;
2289
2290         if (id_priv->id.qp) {
2291                 ret = cma_modify_qp_rtr(&id_priv->id);
2292                 if (ret)
2293                         goto out;
2294
2295                 qp_attr.qp_state = IB_QPS_RTS;
2296                 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, &qp_attr,
2297                                          &qp_attr_mask);
2298                 if (ret)
2299                         goto out;
2300
2301                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
2302                 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
2303                 if (ret)
2304                         goto out;
2305         }
2306
2307         memset(&rep, 0, sizeof rep);
2308         rep.qp_num = id_priv->qp_num;
2309         rep.starting_psn = id_priv->seq_num;
2310         rep.private_data = conn_param->private_data;
2311         rep.private_data_len = conn_param->private_data_len;
2312         rep.responder_resources = conn_param->responder_resources;
2313         rep.initiator_depth = conn_param->initiator_depth;
2314         rep.target_ack_delay = CMA_CM_RESPONSE_TIMEOUT;
2315         rep.failover_accepted = 0;
2316         rep.flow_control = conn_param->flow_control;
2317         rep.rnr_retry_count = conn_param->rnr_retry_count;
2318         rep.srq = id_priv->srq ? 1 : 0;
2319
2320         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2321 out:
2322         return ret;
2323 }
2324
2325 static int cma_accept_iw(struct rdma_id_private *id_priv,
2326                   struct rdma_conn_param *conn_param)
2327 {
2328         struct iw_cm_conn_param iw_param;
2329         int ret;
2330
2331         ret = cma_modify_qp_rtr(&id_priv->id);
2332         if (ret)
2333                 return ret;
2334
2335         iw_param.ord = conn_param->initiator_depth;
2336         iw_param.ird = conn_param->responder_resources;
2337         iw_param.private_data = conn_param->private_data;
2338         iw_param.private_data_len = conn_param->private_data_len;
2339         if (id_priv->id.qp) {
2340                 iw_param.qpn = id_priv->qp_num;
2341         } else
2342                 iw_param.qpn = conn_param->qp_num;
2343
2344         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2345 }
2346
2347 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2348                              enum ib_cm_sidr_status status,
2349                              const void *private_data, int private_data_len)
2350 {
2351         struct ib_cm_sidr_rep_param rep;
2352
2353         memset(&rep, 0, sizeof rep);
2354         rep.status = status;
2355         if (status == IB_SIDR_SUCCESS) {
2356                 rep.qp_num = id_priv->qp_num;
2357                 rep.qkey = id_priv->qkey;
2358         }
2359         rep.private_data = private_data;
2360         rep.private_data_len = private_data_len;
2361
2362         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2363 }
2364
2365 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2366 {
2367         struct rdma_id_private *id_priv;
2368         int ret;
2369
2370         id_priv = container_of(id, struct rdma_id_private, id);
2371         if (!cma_comp(id_priv, CMA_CONNECT))
2372                 return -EINVAL;
2373
2374         if (!id->qp && conn_param) {
2375                 id_priv->qp_num = conn_param->qp_num;
2376                 id_priv->srq = conn_param->srq;
2377         }
2378
2379         switch (rdma_node_get_transport(id->device->node_type)) {
2380         case RDMA_TRANSPORT_IB:
2381                 if (cma_is_ud_ps(id->ps))
2382                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2383                                                 conn_param->private_data,
2384                                                 conn_param->private_data_len);
2385                 else if (conn_param)
2386                         ret = cma_accept_ib(id_priv, conn_param);
2387                 else
2388                         ret = cma_rep_recv(id_priv);
2389                 break;
2390         case RDMA_TRANSPORT_IWARP:
2391                 ret = cma_accept_iw(id_priv, conn_param);
2392                 break;
2393         default:
2394                 ret = -ENOSYS;
2395                 break;
2396         }
2397
2398         if (ret)
2399                 goto reject;
2400
2401         return 0;
2402 reject:
2403         cma_modify_qp_err(id);
2404         rdma_reject(id, NULL, 0);
2405         return ret;
2406 }
2407 EXPORT_SYMBOL(rdma_accept);
2408
2409 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2410 {
2411         struct rdma_id_private *id_priv;
2412         int ret;
2413
2414         id_priv = container_of(id, struct rdma_id_private, id);
2415         if (!cma_comp(id_priv, CMA_CONNECT))
2416                 return -EINVAL;
2417
2418         switch (id->device->node_type) {
2419         case RDMA_NODE_IB_CA:
2420                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2421                 break;
2422         default:
2423                 ret = 0;
2424                 break;
2425         }
2426         return ret;
2427 }
2428 EXPORT_SYMBOL(rdma_notify);
2429
2430 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
2431                 u8 private_data_len)
2432 {
2433         struct rdma_id_private *id_priv;
2434         int ret;
2435
2436         id_priv = container_of(id, struct rdma_id_private, id);
2437         if (!cma_comp(id_priv, CMA_CONNECT))
2438                 return -EINVAL;
2439
2440         switch (rdma_node_get_transport(id->device->node_type)) {
2441         case RDMA_TRANSPORT_IB:
2442                 if (cma_is_ud_ps(id->ps))
2443                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
2444                                                 private_data, private_data_len);
2445                 else
2446                         ret = ib_send_cm_rej(id_priv->cm_id.ib,
2447                                              IB_CM_REJ_CONSUMER_DEFINED, NULL,
2448                                              0, private_data, private_data_len);
2449                 break;
2450         case RDMA_TRANSPORT_IWARP:
2451                 ret = iw_cm_reject(id_priv->cm_id.iw,
2452                                    private_data, private_data_len);
2453                 break;
2454         default:
2455                 ret = -ENOSYS;
2456                 break;
2457         }
2458         return ret;
2459 }
2460 EXPORT_SYMBOL(rdma_reject);
2461
2462 int rdma_disconnect(struct rdma_cm_id *id)
2463 {
2464         struct rdma_id_private *id_priv;
2465         int ret;
2466
2467         id_priv = container_of(id, struct rdma_id_private, id);
2468         if (!cma_comp(id_priv, CMA_CONNECT) &&
2469             !cma_comp(id_priv, CMA_DISCONNECT))
2470                 return -EINVAL;
2471
2472         switch (rdma_node_get_transport(id->device->node_type)) {
2473         case RDMA_TRANSPORT_IB:
2474                 ret = cma_modify_qp_err(id);
2475                 if (ret)
2476                         goto out;
2477                 /* Initiate or respond to a disconnect. */
2478                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
2479                         ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
2480                 break;
2481         case RDMA_TRANSPORT_IWARP:
2482                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
2483                 break;
2484         default:
2485                 ret = -EINVAL;
2486                 break;
2487         }
2488 out:
2489         return ret;
2490 }
2491 EXPORT_SYMBOL(rdma_disconnect);
2492
2493 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
2494 {
2495         struct rdma_id_private *id_priv;
2496         struct cma_multicast *mc = multicast->context;
2497         struct rdma_cm_event event;
2498         int ret;
2499
2500         id_priv = mc->id_priv;
2501         atomic_inc(&id_priv->dev_remove);
2502         if (!cma_comp(id_priv, CMA_ADDR_BOUND) &&
2503             !cma_comp(id_priv, CMA_ADDR_RESOLVED))
2504                 goto out;
2505
2506         if (!status && id_priv->id.qp)
2507                 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
2508                                          multicast->rec.mlid);
2509
2510         memset(&event, 0, sizeof event);
2511         event.status = status;
2512         event.param.ud.private_data = mc->context;
2513         if (!status) {
2514                 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
2515                 ib_init_ah_from_mcmember(id_priv->id.device,
2516                                          id_priv->id.port_num, &multicast->rec,
2517                                          &event.param.ud.ah_attr);
2518                 event.param.ud.qp_num = 0xFFFFFF;
2519                 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
2520         } else
2521                 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
2522
2523         ret = id_priv->id.event_handler(&id_priv->id, &event);
2524         if (ret) {
2525                 cma_exch(id_priv, CMA_DESTROYING);
2526                 cma_release_remove(id_priv);
2527                 rdma_destroy_id(&id_priv->id);
2528                 return 0;
2529         }
2530 out:
2531         cma_release_remove(id_priv);
2532         return 0;
2533 }
2534
2535 static void cma_set_mgid(struct rdma_id_private *id_priv,
2536                          struct sockaddr *addr, union ib_gid *mgid)
2537 {
2538         unsigned char mc_map[MAX_ADDR_LEN];
2539         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2540         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
2541         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
2542
2543         if (cma_any_addr(addr)) {
2544                 memset(mgid, 0, sizeof *mgid);
2545         } else if ((addr->sa_family == AF_INET6) &&
2546                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFF10A01B) ==
2547                                                                  0xFF10A01B)) {
2548                 /* IPv6 address is an SA assigned MGID. */
2549                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
2550         } else {
2551                 ip_ib_mc_map(sin->sin_addr.s_addr, mc_map);
2552                 if (id_priv->id.ps == RDMA_PS_UDP)
2553                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
2554                 mc_map[8] = ib_addr_get_pkey(dev_addr) >> 8;
2555                 mc_map[9] = (unsigned char) ib_addr_get_pkey(dev_addr);
2556                 *mgid = *(union ib_gid *) (mc_map + 4);
2557         }
2558 }
2559
2560 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
2561                                  struct cma_multicast *mc)
2562 {
2563         struct ib_sa_mcmember_rec rec;
2564         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2565         ib_sa_comp_mask comp_mask;
2566         int ret;
2567
2568         ib_addr_get_mgid(dev_addr, &rec.mgid);
2569         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
2570                                      &rec.mgid, &rec);
2571         if (ret)
2572                 return ret;
2573
2574         cma_set_mgid(id_priv, &mc->addr, &rec.mgid);
2575         if (id_priv->id.ps == RDMA_PS_UDP)
2576                 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
2577         ib_addr_get_sgid(dev_addr, &rec.port_gid);
2578         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2579         rec.join_state = 1;
2580
2581         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
2582                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
2583                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
2584                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
2585                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
2586
2587         mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
2588                                                 id_priv->id.port_num, &rec,
2589                                                 comp_mask, GFP_KERNEL,
2590                                                 cma_ib_mc_handler, mc);
2591         if (IS_ERR(mc->multicast.ib))
2592                 return PTR_ERR(mc->multicast.ib);
2593
2594         return 0;
2595 }
2596
2597 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
2598                         void *context)
2599 {
2600         struct rdma_id_private *id_priv;
2601         struct cma_multicast *mc;
2602         int ret;
2603
2604         id_priv = container_of(id, struct rdma_id_private, id);
2605         if (!cma_comp(id_priv, CMA_ADDR_BOUND) &&
2606             !cma_comp(id_priv, CMA_ADDR_RESOLVED))
2607                 return -EINVAL;
2608
2609         mc = kmalloc(sizeof *mc, GFP_KERNEL);
2610         if (!mc)
2611                 return -ENOMEM;
2612
2613         memcpy(&mc->addr, addr, ip_addr_size(addr));
2614         mc->context = context;
2615         mc->id_priv = id_priv;
2616
2617         spin_lock(&id_priv->lock);
2618         list_add(&mc->list, &id_priv->mc_list);
2619         spin_unlock(&id_priv->lock);
2620
2621         switch (rdma_node_get_transport(id->device->node_type)) {
2622         case RDMA_TRANSPORT_IB:
2623                 ret = cma_join_ib_multicast(id_priv, mc);
2624                 break;
2625         default:
2626                 ret = -ENOSYS;
2627                 break;
2628         }
2629
2630         if (ret) {
2631                 spin_lock_irq(&id_priv->lock);
2632                 list_del(&mc->list);
2633                 spin_unlock_irq(&id_priv->lock);
2634                 kfree(mc);
2635         }
2636         return ret;
2637 }
2638 EXPORT_SYMBOL(rdma_join_multicast);
2639
2640 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
2641 {
2642         struct rdma_id_private *id_priv;
2643         struct cma_multicast *mc;
2644
2645         id_priv = container_of(id, struct rdma_id_private, id);
2646         spin_lock_irq(&id_priv->lock);
2647         list_for_each_entry(mc, &id_priv->mc_list, list) {
2648                 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
2649                         list_del(&mc->list);
2650                         spin_unlock_irq(&id_priv->lock);
2651
2652                         if (id->qp)
2653                                 ib_detach_mcast(id->qp,
2654                                                 &mc->multicast.ib->rec.mgid,
2655                                                 mc->multicast.ib->rec.mlid);
2656                         ib_sa_free_multicast(mc->multicast.ib);
2657                         kfree(mc);
2658                         return;
2659                 }
2660         }
2661         spin_unlock_irq(&id_priv->lock);
2662 }
2663 EXPORT_SYMBOL(rdma_leave_multicast);
2664
2665 static void cma_add_one(struct ib_device *device)
2666 {
2667         struct cma_device *cma_dev;
2668         struct rdma_id_private *id_priv;
2669
2670         cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
2671         if (!cma_dev)
2672                 return;
2673
2674         cma_dev->device = device;
2675         cma_dev->node_guid = device->node_guid;
2676
2677         init_completion(&cma_dev->comp);
2678         atomic_set(&cma_dev->refcount, 1);
2679         INIT_LIST_HEAD(&cma_dev->id_list);
2680         ib_set_client_data(device, &cma_client, cma_dev);
2681
2682         mutex_lock(&lock);
2683         list_add_tail(&cma_dev->list, &dev_list);
2684         list_for_each_entry(id_priv, &listen_any_list, list)
2685                 cma_listen_on_dev(id_priv, cma_dev);
2686         mutex_unlock(&lock);
2687 }
2688
2689 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
2690 {
2691         struct rdma_cm_event event;
2692         enum cma_state state;
2693
2694         /* Record that we want to remove the device */
2695         state = cma_exch(id_priv, CMA_DEVICE_REMOVAL);
2696         if (state == CMA_DESTROYING)
2697                 return 0;
2698
2699         cma_cancel_operation(id_priv, state);
2700         wait_event(id_priv->wait_remove, !atomic_read(&id_priv->dev_remove));
2701
2702         /* Check for destruction from another callback. */
2703         if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL))
2704                 return 0;
2705
2706         memset(&event, 0, sizeof event);
2707         event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
2708         return id_priv->id.event_handler(&id_priv->id, &event);
2709 }
2710
2711 static void cma_process_remove(struct cma_device *cma_dev)
2712 {
2713         struct rdma_id_private *id_priv;
2714         int ret;
2715
2716         mutex_lock(&lock);
2717         while (!list_empty(&cma_dev->id_list)) {
2718                 id_priv = list_entry(cma_dev->id_list.next,
2719                                      struct rdma_id_private, list);
2720
2721                 if (cma_internal_listen(id_priv)) {
2722                         cma_destroy_listen(id_priv);
2723                         continue;
2724                 }
2725
2726                 list_del_init(&id_priv->list);
2727                 atomic_inc(&id_priv->refcount);
2728                 mutex_unlock(&lock);
2729
2730                 ret = cma_remove_id_dev(id_priv);
2731                 cma_deref_id(id_priv);
2732                 if (ret)
2733                         rdma_destroy_id(&id_priv->id);
2734
2735                 mutex_lock(&lock);
2736         }
2737         mutex_unlock(&lock);
2738
2739         cma_deref_dev(cma_dev);
2740         wait_for_completion(&cma_dev->comp);
2741 }
2742
2743 static void cma_remove_one(struct ib_device *device)
2744 {
2745         struct cma_device *cma_dev;
2746
2747         cma_dev = ib_get_client_data(device, &cma_client);
2748         if (!cma_dev)
2749                 return;
2750
2751         mutex_lock(&lock);
2752         list_del(&cma_dev->list);
2753         mutex_unlock(&lock);
2754
2755         cma_process_remove(cma_dev);
2756         kfree(cma_dev);
2757 }
2758
2759 static int cma_init(void)
2760 {
2761         int ret;
2762
2763         get_random_bytes(&next_port, sizeof next_port);
2764         next_port = (next_port % (sysctl_local_port_range[1] -
2765                                   sysctl_local_port_range[0])) +
2766                     sysctl_local_port_range[0];
2767         cma_wq = create_singlethread_workqueue("rdma_cm");
2768         if (!cma_wq)
2769                 return -ENOMEM;
2770
2771         ib_sa_register_client(&sa_client);
2772         rdma_addr_register_client(&addr_client);
2773
2774         ret = ib_register_client(&cma_client);
2775         if (ret)
2776                 goto err;
2777         return 0;
2778
2779 err:
2780         rdma_addr_unregister_client(&addr_client);
2781         ib_sa_unregister_client(&sa_client);
2782         destroy_workqueue(cma_wq);
2783         return ret;
2784 }
2785
2786 static void cma_cleanup(void)
2787 {
2788         ib_unregister_client(&cma_client);
2789         rdma_addr_unregister_client(&addr_client);
2790         ib_sa_unregister_client(&sa_client);
2791         destroy_workqueue(cma_wq);
2792         idr_destroy(&sdp_ps);
2793         idr_destroy(&tcp_ps);
2794         idr_destroy(&udp_ps);
2795         idr_destroy(&ipoib_ps);
2796 }
2797
2798 module_init(cma_init);
2799 module_exit(cma_cleanup);