git.oblomov.eu Git - linux-2.6/blob - drivers/infiniband/core/cma.c

   1 /*
   2  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   3  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   4  * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
   5  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   6  *
   7  * This Software is licensed under one of the following licenses:
   8  *
   9  * 1) under the terms of the "Common Public License 1.0" a copy of which is
  10  *    available from the Open Source Initiative, see
  11  *    http://www.opensource.org/licenses/cpl.php.
  12  *
  13  * 2) under the terms of the "The BSD License" a copy of which is
  14  *    available from the Open Source Initiative, see
  15  *    http://www.opensource.org/licenses/bsd-license.php.
  16  *
  17  * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
  18  *    copy of which is available from the Open Source Initiative, see
  19  *    http://www.opensource.org/licenses/gpl-license.php.
  20  *
  21  * Licensee has the right to choose one of the above licenses.
  22  *
  23  * Redistributions of source code must retain the above copyright
  24  * notice and one of the license notices.
  25  *
  26  * Redistributions in binary form must reproduce both the above copyright
  27  * notice, one of the license notices in the documentation
  28  * and/or other materials provided with the distribution.
  29  *
  30  */
  31
  32 #include <linux/completion.h>
  33 #include <linux/in.h>
  34 #include <linux/in6.h>
  35 #include <linux/mutex.h>
  36 #include <linux/random.h>
  37 #include <linux/idr.h>
  38 #include <linux/inetdevice.h>
  39
  40 #include <net/tcp.h>
  41
  42 #include <rdma/rdma_cm.h>
  43 #include <rdma/rdma_cm_ib.h>
  44 #include <rdma/ib_cache.h>
  45 #include <rdma/ib_cm.h>
  46 #include <rdma/ib_sa.h>
  47 #include <rdma/iw_cm.h>
  48
  49 MODULE_AUTHOR("Sean Hefty");
  50 MODULE_DESCRIPTION("Generic RDMA CM Agent");
  51 MODULE_LICENSE("Dual BSD/GPL");
  52
  53 #define CMA_CM_RESPONSE_TIMEOUT 20
  54 #define CMA_MAX_CM_RETRIES 15
  55
  56 static void cma_add_one(struct ib_device *device);
  57 static void cma_remove_one(struct ib_device *device);
  58
  59 static struct ib_client cma_client = {
  60         .name   = "cma",
  61         .add    = cma_add_one,
  62         .remove = cma_remove_one
  63 };
  64
  65 static struct ib_sa_client sa_client;
  66 static struct rdma_addr_client addr_client;
  67 static LIST_HEAD(dev_list);
  68 static LIST_HEAD(listen_any_list);
  69 static DEFINE_MUTEX(lock);
  70 static struct workqueue_struct *cma_wq;
  71 static DEFINE_IDR(sdp_ps);
  72 static DEFINE_IDR(tcp_ps);
  73 static DEFINE_IDR(udp_ps);
  74 static DEFINE_IDR(ipoib_ps);
  75 static int next_port;
  76
  77 struct cma_device {
  78         struct list_head        list;
  79         struct ib_device        *device;
  80         __be64                  node_guid;
  81         struct completion       comp;
  82         atomic_t                refcount;
  83         struct list_head        id_list;
  84 };
  85
  86 enum cma_state {
  87         CMA_IDLE,
  88         CMA_ADDR_QUERY,
  89         CMA_ADDR_RESOLVED,
  90         CMA_ROUTE_QUERY,
  91         CMA_ROUTE_RESOLVED,
  92         CMA_CONNECT,
  93         CMA_DISCONNECT,
  94         CMA_ADDR_BOUND,
  95         CMA_LISTEN,
  96         CMA_DEVICE_REMOVAL,
  97         CMA_DESTROYING
  98 };
  99
 100 struct rdma_bind_list {
 101         struct idr              *ps;
 102         struct hlist_head       owners;
 103         unsigned short          port;
 104 };
 105
 106 /*
 107  * Device removal can occur at anytime, so we need extra handling to
 108  * serialize notifying the user of device removal with other callbacks.
 109  * We do this by disabling removal notification while a callback is in process,
 110  * and reporting it after the callback completes.
 111  */
 112 struct rdma_id_private {
 113         struct rdma_cm_id       id;
 114
 115         struct rdma_bind_list   *bind_list;
 116         struct hlist_node       node;
 117         struct list_head        list;
 118         struct list_head        listen_list;
 119         struct cma_device       *cma_dev;
 120         struct list_head        mc_list;
 121
 122         enum cma_state          state;
 123         spinlock_t              lock;
 124         struct completion       comp;
 125         atomic_t                refcount;
 126         wait_queue_head_t       wait_remove;
 127         atomic_t                dev_remove;
 128
 129         int                     backlog;
 130         int                     timeout_ms;
 131         struct ib_sa_query      *query;
 132         int                     query_id;
 133         union {
 134                 struct ib_cm_id *ib;
 135                 struct iw_cm_id *iw;
 136         } cm_id;
 137
 138         u32                     seq_num;
 139         u32                     qkey;
 140         u32                     qp_num;
 141         u8                      srq;
 142 };
 143
 144 struct cma_multicast {
 145         struct rdma_id_private *id_priv;
 146         union {
 147                 struct ib_sa_multicast *ib;
 148         } multicast;
 149         struct list_head        list;
 150         void                    *context;
 151         struct sockaddr         addr;
 152         u8                      pad[sizeof(struct sockaddr_in6) -
 153                                     sizeof(struct sockaddr)];
 154 };
 155
 156 struct cma_work {
 157         struct work_struct      work;
 158         struct rdma_id_private  *id;
 159         enum cma_state          old_state;
 160         enum cma_state          new_state;
 161         struct rdma_cm_event    event;
 162 };
 163
 164 union cma_ip_addr {
 165         struct in6_addr ip6;
 166         struct {
 167                 __u32 pad[3];
 168                 __u32 addr;
 169         } ip4;
 170 };
 171
 172 struct cma_hdr {
 173         u8 cma_version;
 174         u8 ip_version;  /* IP version: 7:4 */
 175         __u16 port;
 176         union cma_ip_addr src_addr;
 177         union cma_ip_addr dst_addr;
 178 };
 179
 180 struct sdp_hh {
 181         u8 bsdh[16];
 182         u8 sdp_version; /* Major version: 7:4 */
 183         u8 ip_version;  /* IP version: 7:4 */
 184         u8 sdp_specific1[10];
 185         __u16 port;
 186         __u16 sdp_specific2;
 187         union cma_ip_addr src_addr;
 188         union cma_ip_addr dst_addr;
 189 };
 190
 191 struct sdp_hah {
 192         u8 bsdh[16];
 193         u8 sdp_version;
 194 };
 195
 196 #define CMA_VERSION 0x00
 197 #define SDP_MAJ_VERSION 0x2
 198
 199 static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp)
 200 {
 201         unsigned long flags;
 202         int ret;
 203
 204         spin_lock_irqsave(&id_priv->lock, flags);
 205         ret = (id_priv->state == comp);
 206         spin_unlock_irqrestore(&id_priv->lock, flags);
 207         return ret;
 208 }
 209
 210 static int cma_comp_exch(struct rdma_id_private *id_priv,
 211                          enum cma_state comp, enum cma_state exch)
 212 {
 213         unsigned long flags;
 214         int ret;
 215
 216         spin_lock_irqsave(&id_priv->lock, flags);
 217         if ((ret = (id_priv->state == comp)))
 218                 id_priv->state = exch;
 219         spin_unlock_irqrestore(&id_priv->lock, flags);
 220         return ret;
 221 }
 222
 223 static enum cma_state cma_exch(struct rdma_id_private *id_priv,
 224                                enum cma_state exch)
 225 {
 226         unsigned long flags;
 227         enum cma_state old;
 228
 229         spin_lock_irqsave(&id_priv->lock, flags);
 230         old = id_priv->state;
 231         id_priv->state = exch;
 232         spin_unlock_irqrestore(&id_priv->lock, flags);
 233         return old;
 234 }
 235
 236 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
 237 {
 238         return hdr->ip_version >> 4;
 239 }
 240
 241 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 242 {
 243         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 244 }
 245
 246 static inline u8 sdp_get_majv(u8 sdp_version)
 247 {
 248         return sdp_version >> 4;
 249 }
 250
 251 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
 252 {
 253         return hh->ip_version >> 4;
 254 }
 255
 256 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
 257 {
 258         hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
 259 }
 260
 261 static inline int cma_is_ud_ps(enum rdma_port_space ps)
 262 {
 263         return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB);
 264 }
 265
 266 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 267                               struct cma_device *cma_dev)
 268 {
 269         atomic_inc(&cma_dev->refcount);
 270         id_priv->cma_dev = cma_dev;
 271         id_priv->id.device = cma_dev->device;
 272         list_add_tail(&id_priv->list, &cma_dev->id_list);
 273 }
 274
 275 static inline void cma_deref_dev(struct cma_device *cma_dev)
 276 {
 277         if (atomic_dec_and_test(&cma_dev->refcount))
 278                 complete(&cma_dev->comp);
 279 }
 280
 281 static void cma_detach_from_dev(struct rdma_id_private *id_priv)
 282 {
 283         list_del(&id_priv->list);
 284         cma_deref_dev(id_priv->cma_dev);
 285         id_priv->cma_dev = NULL;
 286 }
 287
 288 static int cma_set_qkey(struct ib_device *device, u8 port_num,
 289                         enum rdma_port_space ps,
 290                         struct rdma_dev_addr *dev_addr, u32 *qkey)
 291 {
 292         struct ib_sa_mcmember_rec rec;
 293         int ret = 0;
 294
 295         switch (ps) {
 296         case RDMA_PS_UDP:
 297                 *qkey = RDMA_UDP_QKEY;
 298                 break;
 299         case RDMA_PS_IPOIB:
 300                 ib_addr_get_mgid(dev_addr, &rec.mgid);
 301                 ret = ib_sa_get_mcmember_rec(device, port_num, &rec.mgid, &rec);
 302                 *qkey = be32_to_cpu(rec.qkey);
 303                 break;
 304         default:
 305                 break;
 306         }
 307         return ret;
 308 }
 309
 310 static int cma_acquire_dev(struct rdma_id_private *id_priv)
 311 {
 312         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 313         struct cma_device *cma_dev;
 314         union ib_gid gid;
 315         int ret = -ENODEV;
 316
 317         switch (rdma_node_get_transport(dev_addr->dev_type)) {
 318         case RDMA_TRANSPORT_IB:
 319                 ib_addr_get_sgid(dev_addr, &gid);
 320                 break;
 321         case RDMA_TRANSPORT_IWARP:
 322                 iw_addr_get_sgid(dev_addr, &gid);
 323                 break;
 324         default:
 325                 return -ENODEV;
 326         }
 327
 328         list_for_each_entry(cma_dev, &dev_list, list) {
 329                 ret = ib_find_cached_gid(cma_dev->device, &gid,
 330                                          &id_priv->id.port_num, NULL);
 331                 if (!ret) {
 332                         ret = cma_set_qkey(cma_dev->device,
 333                                            id_priv->id.port_num,
 334                                            id_priv->id.ps, dev_addr,
 335                                            &id_priv->qkey);
 336                         if (!ret)
 337                                 cma_attach_to_dev(id_priv, cma_dev);
 338                         break;
 339                 }
 340         }
 341         return ret;
 342 }
 343
 344 static void cma_deref_id(struct rdma_id_private *id_priv)
 345 {
 346         if (atomic_dec_and_test(&id_priv->refcount))
 347                 complete(&id_priv->comp);
 348 }
 349
 350 static void cma_release_remove(struct rdma_id_private *id_priv)
 351 {
 352         if (atomic_dec_and_test(&id_priv->dev_remove))
 353                 wake_up(&id_priv->wait_remove);
 354 }
 355
 356 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
 357                                   void *context, enum rdma_port_space ps)
 358 {
 359         struct rdma_id_private *id_priv;
 360
 361         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 362         if (!id_priv)
 363                 return ERR_PTR(-ENOMEM);
 364
 365         id_priv->state = CMA_IDLE;
 366         id_priv->id.context = context;
 367         id_priv->id.event_handler = event_handler;
 368         id_priv->id.ps = ps;
 369         spin_lock_init(&id_priv->lock);
 370         init_completion(&id_priv->comp);
 371         atomic_set(&id_priv->refcount, 1);
 372         init_waitqueue_head(&id_priv->wait_remove);
 373         atomic_set(&id_priv->dev_remove, 0);
 374         INIT_LIST_HEAD(&id_priv->listen_list);
 375         INIT_LIST_HEAD(&id_priv->mc_list);
 376         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
 377
 378         return &id_priv->id;
 379 }
 380 EXPORT_SYMBOL(rdma_create_id);
 381
 382 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 383 {
 384         struct ib_qp_attr qp_attr;
 385         int qp_attr_mask, ret;
 386
 387         qp_attr.qp_state = IB_QPS_INIT;
 388         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 389         if (ret)
 390                 return ret;
 391
 392         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 393         if (ret)
 394                 return ret;
 395
 396         qp_attr.qp_state = IB_QPS_RTR;
 397         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
 398         if (ret)
 399                 return ret;
 400
 401         qp_attr.qp_state = IB_QPS_RTS;
 402         qp_attr.sq_psn = 0;
 403         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
 404
 405         return ret;
 406 }
 407
 408 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 409 {
 410         struct ib_qp_attr qp_attr;
 411         int qp_attr_mask, ret;
 412
 413         qp_attr.qp_state = IB_QPS_INIT;
 414         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 415         if (ret)
 416                 return ret;
 417
 418         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 419 }
 420
 421 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 422                    struct ib_qp_init_attr *qp_init_attr)
 423 {
 424         struct rdma_id_private *id_priv;
 425         struct ib_qp *qp;
 426         int ret;
 427
 428         id_priv = container_of(id, struct rdma_id_private, id);
 429         if (id->device != pd->device)
 430                 return -EINVAL;
 431
 432         qp = ib_create_qp(pd, qp_init_attr);
 433         if (IS_ERR(qp))
 434                 return PTR_ERR(qp);
 435
 436         if (cma_is_ud_ps(id_priv->id.ps))
 437                 ret = cma_init_ud_qp(id_priv, qp);
 438         else
 439                 ret = cma_init_conn_qp(id_priv, qp);
 440         if (ret)
 441                 goto err;
 442
 443         id->qp = qp;
 444         id_priv->qp_num = qp->qp_num;
 445         id_priv->srq = (qp->srq != NULL);
 446         return 0;
 447 err:
 448         ib_destroy_qp(qp);
 449         return ret;
 450 }
 451 EXPORT_SYMBOL(rdma_create_qp);
 452
 453 void rdma_destroy_qp(struct rdma_cm_id *id)
 454 {
 455         ib_destroy_qp(id->qp);
 456 }
 457 EXPORT_SYMBOL(rdma_destroy_qp);
 458
 459 static int cma_modify_qp_rtr(struct rdma_cm_id *id)
 460 {
 461         struct ib_qp_attr qp_attr;
 462         int qp_attr_mask, ret;
 463
 464         if (!id->qp)
 465                 return 0;
 466
 467         /* Need to update QP attributes from default values. */
 468         qp_attr.qp_state = IB_QPS_INIT;
 469         ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
 470         if (ret)
 471                 return ret;
 472
 473         ret = ib_modify_qp(id->qp, &qp_attr, qp_attr_mask);
 474         if (ret)
 475                 return ret;
 476
 477         qp_attr.qp_state = IB_QPS_RTR;
 478         ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
 479         if (ret)
 480                 return ret;
 481
 482         return ib_modify_qp(id->qp, &qp_attr, qp_attr_mask);
 483 }
 484
 485 static int cma_modify_qp_rts(struct rdma_cm_id *id)
 486 {
 487         struct ib_qp_attr qp_attr;
 488         int qp_attr_mask, ret;
 489
 490         if (!id->qp)
 491                 return 0;
 492
 493         qp_attr.qp_state = IB_QPS_RTS;
 494         ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
 495         if (ret)
 496                 return ret;
 497
 498         return ib_modify_qp(id->qp, &qp_attr, qp_attr_mask);
 499 }
 500
 501 static int cma_modify_qp_err(struct rdma_cm_id *id)
 502 {
 503         struct ib_qp_attr qp_attr;
 504
 505         if (!id->qp)
 506                 return 0;
 507
 508         qp_attr.qp_state = IB_QPS_ERR;
 509         return ib_modify_qp(id->qp, &qp_attr, IB_QP_STATE);
 510 }
 511
 512 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
 513                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
 514 {
 515         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 516         int ret;
 517
 518         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
 519                                   ib_addr_get_pkey(dev_addr),
 520                                   &qp_attr->pkey_index);
 521         if (ret)
 522                 return ret;
 523
 524         qp_attr->port_num = id_priv->id.port_num;
 525         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
 526
 527         if (cma_is_ud_ps(id_priv->id.ps)) {
 528                 qp_attr->qkey = id_priv->qkey;
 529                 *qp_attr_mask |= IB_QP_QKEY;
 530         } else {
 531                 qp_attr->qp_access_flags = 0;
 532                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
 533         }
 534         return 0;
 535 }
 536
 537 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
 538                        int *qp_attr_mask)
 539 {
 540         struct rdma_id_private *id_priv;
 541         int ret = 0;
 542
 543         id_priv = container_of(id, struct rdma_id_private, id);
 544         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 545         case RDMA_TRANSPORT_IB:
 546                 if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps))
 547                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
 548                 else
 549                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
 550                                                  qp_attr_mask);
 551                 if (qp_attr->qp_state == IB_QPS_RTR)
 552                         qp_attr->rq_psn = id_priv->seq_num;
 553                 break;
 554         case RDMA_TRANSPORT_IWARP:
 555                 if (!id_priv->cm_id.iw) {
 556                         qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE;
 557                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
 558                 } else
 559                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
 560                                                  qp_attr_mask);
 561                 break;
 562         default:
 563                 ret = -ENOSYS;
 564                 break;
 565         }
 566
 567         return ret;
 568 }
 569 EXPORT_SYMBOL(rdma_init_qp_attr);
 570
 571 static inline int cma_zero_addr(struct sockaddr *addr)
 572 {
 573         struct in6_addr *ip6;
 574
 575         if (addr->sa_family == AF_INET)
 576                 return ZERONET(((struct sockaddr_in *) addr)->sin_addr.s_addr);
 577         else {
 578                 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
 579                 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
 580                         ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
 581         }
 582 }
 583
 584 static inline int cma_loopback_addr(struct sockaddr *addr)
 585 {
 586         return LOOPBACK(((struct sockaddr_in *) addr)->sin_addr.s_addr);
 587 }
 588
 589 static inline int cma_any_addr(struct sockaddr *addr)
 590 {
 591         return cma_zero_addr(addr) || cma_loopback_addr(addr);
 592 }
 593
 594 static inline __be16 cma_port(struct sockaddr *addr)
 595 {
 596         if (addr->sa_family == AF_INET)
 597                 return ((struct sockaddr_in *) addr)->sin_port;
 598         else
 599                 return ((struct sockaddr_in6 *) addr)->sin6_port;
 600 }
 601
 602 static inline int cma_any_port(struct sockaddr *addr)
 603 {
 604         return !cma_port(addr);
 605 }
 606
 607 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
 608                             u8 *ip_ver, __u16 *port,
 609                             union cma_ip_addr **src, union cma_ip_addr **dst)
 610 {
 611         switch (ps) {
 612         case RDMA_PS_SDP:
 613                 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
 614                     SDP_MAJ_VERSION)
 615                         return -EINVAL;
 616
 617                 *ip_ver = sdp_get_ip_ver(hdr);
 618                 *port   = ((struct sdp_hh *) hdr)->port;
 619                 *src    = &((struct sdp_hh *) hdr)->src_addr;
 620                 *dst    = &((struct sdp_hh *) hdr)->dst_addr;
 621                 break;
 622         default:
 623                 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
 624                         return -EINVAL;
 625
 626                 *ip_ver = cma_get_ip_ver(hdr);
 627                 *port   = ((struct cma_hdr *) hdr)->port;
 628                 *src    = &((struct cma_hdr *) hdr)->src_addr;
 629                 *dst    = &((struct cma_hdr *) hdr)->dst_addr;
 630                 break;
 631         }
 632
 633         if (*ip_ver != 4 && *ip_ver != 6)
 634                 return -EINVAL;
 635         return 0;
 636 }
 637
 638 static void cma_save_net_info(struct rdma_addr *addr,
 639                               struct rdma_addr *listen_addr,
 640                               u8 ip_ver, __u16 port,
 641                               union cma_ip_addr *src, union cma_ip_addr *dst)
 642 {
 643         struct sockaddr_in *listen4, *ip4;
 644         struct sockaddr_in6 *listen6, *ip6;
 645
 646         switch (ip_ver) {
 647         case 4:
 648                 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
 649                 ip4 = (struct sockaddr_in *) &addr->src_addr;
 650                 ip4->sin_family = listen4->sin_family;
 651                 ip4->sin_addr.s_addr = dst->ip4.addr;
 652                 ip4->sin_port = listen4->sin_port;
 653
 654                 ip4 = (struct sockaddr_in *) &addr->dst_addr;
 655                 ip4->sin_family = listen4->sin_family;
 656                 ip4->sin_addr.s_addr = src->ip4.addr;
 657                 ip4->sin_port = port;
 658                 break;
 659         case 6:
 660                 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
 661                 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
 662                 ip6->sin6_family = listen6->sin6_family;
 663                 ip6->sin6_addr = dst->ip6;
 664                 ip6->sin6_port = listen6->sin6_port;
 665
 666                 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
 667                 ip6->sin6_family = listen6->sin6_family;
 668                 ip6->sin6_addr = src->ip6;
 669                 ip6->sin6_port = port;
 670                 break;
 671         default:
 672                 break;
 673         }
 674 }
 675
 676 static inline int cma_user_data_offset(enum rdma_port_space ps)
 677 {
 678         switch (ps) {
 679         case RDMA_PS_SDP:
 680                 return 0;
 681         default:
 682                 return sizeof(struct cma_hdr);
 683         }
 684 }
 685
 686 static void cma_cancel_route(struct rdma_id_private *id_priv)
 687 {
 688         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 689         case RDMA_TRANSPORT_IB:
 690                 if (id_priv->query)
 691                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
 692                 break;
 693         default:
 694                 break;
 695         }
 696 }
 697
 698 static inline int cma_internal_listen(struct rdma_id_private *id_priv)
 699 {
 700         return (id_priv->state == CMA_LISTEN) && id_priv->cma_dev &&
 701                cma_any_addr(&id_priv->id.route.addr.src_addr);
 702 }
 703
 704 static void cma_destroy_listen(struct rdma_id_private *id_priv)
 705 {
 706         cma_exch(id_priv, CMA_DESTROYING);
 707
 708         if (id_priv->cma_dev) {
 709                 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 710                 case RDMA_TRANSPORT_IB:
 711                         if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
 712                                 ib_destroy_cm_id(id_priv->cm_id.ib);
 713                         break;
 714                 case RDMA_TRANSPORT_IWARP:
 715                         if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
 716                                 iw_destroy_cm_id(id_priv->cm_id.iw);
 717                         break;
 718                 default:
 719                         break;
 720                 }
 721                 cma_detach_from_dev(id_priv);
 722         }
 723         list_del(&id_priv->listen_list);
 724
 725         cma_deref_id(id_priv);
 726         wait_for_completion(&id_priv->comp);
 727
 728         kfree(id_priv);
 729 }
 730
 731 static void cma_cancel_listens(struct rdma_id_private *id_priv)
 732 {
 733         struct rdma_id_private *dev_id_priv;
 734
 735         mutex_lock(&lock);
 736         list_del(&id_priv->list);
 737
 738         while (!list_empty(&id_priv->listen_list)) {
 739                 dev_id_priv = list_entry(id_priv->listen_list.next,
 740                                          struct rdma_id_private, listen_list);
 741                 cma_destroy_listen(dev_id_priv);
 742         }
 743         mutex_unlock(&lock);
 744 }
 745
 746 static void cma_cancel_operation(struct rdma_id_private *id_priv,
 747                                  enum cma_state state)
 748 {
 749         switch (state) {
 750         case CMA_ADDR_QUERY:
 751                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
 752                 break;
 753         case CMA_ROUTE_QUERY:
 754                 cma_cancel_route(id_priv);
 755                 break;
 756         case CMA_LISTEN:
 757                 if (cma_any_addr(&id_priv->id.route.addr.src_addr) &&
 758                     !id_priv->cma_dev)
 759                         cma_cancel_listens(id_priv);
 760                 break;
 761         default:
 762                 break;
 763         }
 764 }
 765
 766 static void cma_release_port(struct rdma_id_private *id_priv)
 767 {
 768         struct rdma_bind_list *bind_list = id_priv->bind_list;
 769
 770         if (!bind_list)
 771                 return;
 772
 773         mutex_lock(&lock);
 774         hlist_del(&id_priv->node);
 775         if (hlist_empty(&bind_list->owners)) {
 776                 idr_remove(bind_list->ps, bind_list->port);
 777                 kfree(bind_list);
 778         }
 779         mutex_unlock(&lock);
 780 }
 781
 782 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
 783 {
 784         struct cma_multicast *mc;
 785
 786         while (!list_empty(&id_priv->mc_list)) {
 787                 mc = container_of(id_priv->mc_list.next,
 788                                   struct cma_multicast, list);
 789                 list_del(&mc->list);
 790                 ib_sa_free_multicast(mc->multicast.ib);
 791                 kfree(mc);
 792         }
 793 }
 794
 795 void rdma_destroy_id(struct rdma_cm_id *id)
 796 {
 797         struct rdma_id_private *id_priv;
 798         enum cma_state state;
 799
 800         id_priv = container_of(id, struct rdma_id_private, id);
 801         state = cma_exch(id_priv, CMA_DESTROYING);
 802         cma_cancel_operation(id_priv, state);
 803
 804         mutex_lock(&lock);
 805         if (id_priv->cma_dev) {
 806                 mutex_unlock(&lock);
 807                 switch (rdma_node_get_transport(id->device->node_type)) {
 808                 case RDMA_TRANSPORT_IB:
 809                         if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
 810                                 ib_destroy_cm_id(id_priv->cm_id.ib);
 811                         break;
 812                 case RDMA_TRANSPORT_IWARP:
 813                         if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
 814                                 iw_destroy_cm_id(id_priv->cm_id.iw);
 815                         break;
 816                 default:
 817                         break;
 818                 }
 819                 cma_leave_mc_groups(id_priv);
 820                 mutex_lock(&lock);
 821                 cma_detach_from_dev(id_priv);
 822         }
 823         mutex_unlock(&lock);
 824
 825         cma_release_port(id_priv);
 826         cma_deref_id(id_priv);
 827         wait_for_completion(&id_priv->comp);
 828
 829         kfree(id_priv->id.route.path_rec);
 830         kfree(id_priv);
 831 }
 832 EXPORT_SYMBOL(rdma_destroy_id);
 833
 834 static int cma_rep_recv(struct rdma_id_private *id_priv)
 835 {
 836         int ret;
 837
 838         ret = cma_modify_qp_rtr(&id_priv->id);
 839         if (ret)
 840                 goto reject;
 841
 842         ret = cma_modify_qp_rts(&id_priv->id);
 843         if (ret)
 844                 goto reject;
 845
 846         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
 847         if (ret)
 848                 goto reject;
 849
 850         return 0;
 851 reject:
 852         cma_modify_qp_err(&id_priv->id);
 853         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
 854                        NULL, 0, NULL, 0);
 855         return ret;
 856 }
 857
 858 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
 859 {
 860         if (id_priv->id.ps == RDMA_PS_SDP &&
 861             sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
 862             SDP_MAJ_VERSION)
 863                 return -EINVAL;
 864
 865         return 0;
 866 }
 867
 868 static void cma_set_rep_event_data(struct rdma_cm_event *event,
 869                                    struct ib_cm_rep_event_param *rep_data,
 870                                    void *private_data)
 871 {
 872         event->param.conn.private_data = private_data;
 873         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
 874         event->param.conn.responder_resources = rep_data->responder_resources;
 875         event->param.conn.initiator_depth = rep_data->initiator_depth;
 876         event->param.conn.flow_control = rep_data->flow_control;
 877         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
 878         event->param.conn.srq = rep_data->srq;
 879         event->param.conn.qp_num = rep_data->remote_qpn;
 880 }
 881
 882 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
 883 {
 884         struct rdma_id_private *id_priv = cm_id->context;
 885         struct rdma_cm_event event;
 886         int ret = 0;
 887
 888         atomic_inc(&id_priv->dev_remove);
 889         if (!cma_comp(id_priv, CMA_CONNECT))
 890                 goto out;
 891
 892         memset(&event, 0, sizeof event);
 893         switch (ib_event->event) {
 894         case IB_CM_REQ_ERROR:
 895         case IB_CM_REP_ERROR:
 896                 event.event = RDMA_CM_EVENT_UNREACHABLE;
 897                 event.status = -ETIMEDOUT;
 898                 break;
 899         case IB_CM_REP_RECEIVED:
 900                 event.status = cma_verify_rep(id_priv, ib_event->private_data);
 901                 if (event.status)
 902                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
 903                 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
 904                         event.status = cma_rep_recv(id_priv);
 905                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
 906                                                      RDMA_CM_EVENT_ESTABLISHED;
 907                 } else
 908                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
 909                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
 910                                        ib_event->private_data);
 911                 break;
 912         case IB_CM_RTU_RECEIVED:
 913         case IB_CM_USER_ESTABLISHED:
 914                 event.event = RDMA_CM_EVENT_ESTABLISHED;
 915                 break;
 916         case IB_CM_DREQ_ERROR:
 917                 event.status = -ETIMEDOUT; /* fall through */
 918         case IB_CM_DREQ_RECEIVED:
 919         case IB_CM_DREP_RECEIVED:
 920                 if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT))
 921                         goto out;
 922                 event.event = RDMA_CM_EVENT_DISCONNECTED;
 923                 break;
 924         case IB_CM_TIMEWAIT_EXIT:
 925         case IB_CM_MRA_RECEIVED:
 926                 /* ignore event */
 927                 goto out;
 928         case IB_CM_REJ_RECEIVED:
 929                 cma_modify_qp_err(&id_priv->id);
 930                 event.status = ib_event->param.rej_rcvd.reason;
 931                 event.event = RDMA_CM_EVENT_REJECTED;
 932                 event.param.conn.private_data = ib_event->private_data;
 933                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
 934                 break;
 935         default:
 936                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d",
 937                        ib_event->event);
 938                 goto out;
 939         }
 940
 941         ret = id_priv->id.event_handler(&id_priv->id, &event);
 942         if (ret) {
 943                 /* Destroy the CM ID by returning a non-zero value. */
 944                 id_priv->cm_id.ib = NULL;
 945                 cma_exch(id_priv, CMA_DESTROYING);
 946                 cma_release_remove(id_priv);
 947                 rdma_destroy_id(&id_priv->id);
 948                 return ret;
 949         }
 950 out:
 951         cma_release_remove(id_priv);
 952         return ret;
 953 }
 954
 955 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
 956                                                struct ib_cm_event *ib_event)
 957 {
 958         struct rdma_id_private *id_priv;
 959         struct rdma_cm_id *id;
 960         struct rdma_route *rt;
 961         union cma_ip_addr *src, *dst;
 962         __u16 port;
 963         u8 ip_ver;
 964
 965         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
 966                              &ip_ver, &port, &src, &dst))
 967                 goto err;
 968
 969         id = rdma_create_id(listen_id->event_handler, listen_id->context,
 970                             listen_id->ps);
 971         if (IS_ERR(id))
 972                 goto err;
 973
 974         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
 975                           ip_ver, port, src, dst);
 976
 977         rt = &id->route;
 978         rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
 979         rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
 980                                GFP_KERNEL);
 981         if (!rt->path_rec)
 982                 goto destroy_id;
 983
 984         rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
 985         if (rt->num_paths == 2)
 986                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
 987
 988         ib_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
 989         ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
 990         ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
 991         rt->addr.dev_addr.dev_type = RDMA_NODE_IB_CA;
 992
 993         id_priv = container_of(id, struct rdma_id_private, id);
 994         id_priv->state = CMA_CONNECT;
 995         return id_priv;
 996
 997 destroy_id:
 998         rdma_destroy_id(id);
 999 err:
1000         return NULL;
1001 }
1002
1003 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1004                                               struct ib_cm_event *ib_event)
1005 {
1006         struct rdma_id_private *id_priv;
1007         struct rdma_cm_id *id;
1008         union cma_ip_addr *src, *dst;
1009         __u16 port;
1010         u8 ip_ver;
1011         int ret;
1012
1013         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1014                             listen_id->ps);
1015         if (IS_ERR(id))
1016                 return NULL;
1017
1018
1019         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1020                              &ip_ver, &port, &src, &dst))
1021                 goto err;
1022
1023         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1024                           ip_ver, port, src, dst);
1025
1026         ret = rdma_translate_ip(&id->route.addr.src_addr,
1027                                 &id->route.addr.dev_addr);
1028         if (ret)
1029                 goto err;
1030
1031         id_priv = container_of(id, struct rdma_id_private, id);
1032         id_priv->state = CMA_CONNECT;
1033         return id_priv;
1034 err:
1035         rdma_destroy_id(id);
1036         return NULL;
1037 }
1038
1039 static void cma_set_req_event_data(struct rdma_cm_event *event,
1040                                    struct ib_cm_req_event_param *req_data,
1041                                    void *private_data, int offset)
1042 {
1043         event->param.conn.private_data = private_data + offset;
1044         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1045         event->param.conn.responder_resources = req_data->responder_resources;
1046         event->param.conn.initiator_depth = req_data->initiator_depth;
1047         event->param.conn.flow_control = req_data->flow_control;
1048         event->param.conn.retry_count = req_data->retry_count;
1049         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1050         event->param.conn.srq = req_data->srq;
1051         event->param.conn.qp_num = req_data->remote_qpn;
1052 }
1053
1054 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1055 {
1056         struct rdma_id_private *listen_id, *conn_id;
1057         struct rdma_cm_event event;
1058         int offset, ret;
1059
1060         listen_id = cm_id->context;
1061         atomic_inc(&listen_id->dev_remove);
1062         if (!cma_comp(listen_id, CMA_LISTEN)) {
1063                 ret = -ECONNABORTED;
1064                 goto out;
1065         }
1066
1067         memset(&event, 0, sizeof event);
1068         offset = cma_user_data_offset(listen_id->id.ps);
1069         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1070         if (cma_is_ud_ps(listen_id->id.ps)) {
1071                 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1072                 event.param.ud.private_data = ib_event->private_data + offset;
1073                 event.param.ud.private_data_len =
1074                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1075         } else {
1076                 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1077                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1078                                        ib_event->private_data, offset);
1079         }
1080         if (!conn_id) {
1081                 ret = -ENOMEM;
1082                 goto out;
1083         }
1084
1085         atomic_inc(&conn_id->dev_remove);
1086         mutex_lock(&lock);
1087         ret = cma_acquire_dev(conn_id);
1088         mutex_unlock(&lock);
1089         if (ret)
1090                 goto release_conn_id;
1091
1092         conn_id->cm_id.ib = cm_id;
1093         cm_id->context = conn_id;
1094         cm_id->cm_handler = cma_ib_handler;
1095
1096         ret = conn_id->id.event_handler(&conn_id->id, &event);
1097         if (!ret)
1098                 goto out;
1099
1100         /* Destroy the CM ID by returning a non-zero value. */
1101         conn_id->cm_id.ib = NULL;
1102
1103 release_conn_id:
1104         cma_exch(conn_id, CMA_DESTROYING);
1105         cma_release_remove(conn_id);
1106         rdma_destroy_id(&conn_id->id);
1107
1108 out:
1109         cma_release_remove(listen_id);
1110         return ret;
1111 }
1112
1113 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1114 {
1115         return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1116 }
1117
1118 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1119                                  struct ib_cm_compare_data *compare)
1120 {
1121         struct cma_hdr *cma_data, *cma_mask;
1122         struct sdp_hh *sdp_data, *sdp_mask;
1123         __u32 ip4_addr;
1124         struct in6_addr ip6_addr;
1125
1126         memset(compare, 0, sizeof *compare);
1127         cma_data = (void *) compare->data;
1128         cma_mask = (void *) compare->mask;
1129         sdp_data = (void *) compare->data;
1130         sdp_mask = (void *) compare->mask;
1131
1132         switch (addr->sa_family) {
1133         case AF_INET:
1134                 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1135                 if (ps == RDMA_PS_SDP) {
1136                         sdp_set_ip_ver(sdp_data, 4);
1137                         sdp_set_ip_ver(sdp_mask, 0xF);
1138                         sdp_data->dst_addr.ip4.addr = ip4_addr;
1139                         sdp_mask->dst_addr.ip4.addr = ~0;
1140                 } else {
1141                         cma_set_ip_ver(cma_data, 4);
1142                         cma_set_ip_ver(cma_mask, 0xF);
1143                         cma_data->dst_addr.ip4.addr = ip4_addr;
1144                         cma_mask->dst_addr.ip4.addr = ~0;
1145                 }
1146                 break;
1147         case AF_INET6:
1148                 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1149                 if (ps == RDMA_PS_SDP) {
1150                         sdp_set_ip_ver(sdp_data, 6);
1151                         sdp_set_ip_ver(sdp_mask, 0xF);
1152                         sdp_data->dst_addr.ip6 = ip6_addr;
1153                         memset(&sdp_mask->dst_addr.ip6, 0xFF,
1154                                sizeof sdp_mask->dst_addr.ip6);
1155                 } else {
1156                         cma_set_ip_ver(cma_data, 6);
1157                         cma_set_ip_ver(cma_mask, 0xF);
1158                         cma_data->dst_addr.ip6 = ip6_addr;
1159                         memset(&cma_mask->dst_addr.ip6, 0xFF,
1160                                sizeof cma_mask->dst_addr.ip6);
1161                 }
1162                 break;
1163         default:
1164                 break;
1165         }
1166 }
1167
1168 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1169 {
1170         struct rdma_id_private *id_priv = iw_id->context;
1171         struct rdma_cm_event event;
1172         struct sockaddr_in *sin;
1173         int ret = 0;
1174
1175         memset(&event, 0, sizeof event);
1176         atomic_inc(&id_priv->dev_remove);
1177
1178         switch (iw_event->event) {
1179         case IW_CM_EVENT_CLOSE:
1180                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1181                 break;
1182         case IW_CM_EVENT_CONNECT_REPLY:
1183                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1184                 *sin = iw_event->local_addr;
1185                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1186                 *sin = iw_event->remote_addr;
1187                 switch (iw_event->status) {
1188                 case 0:
1189                         event.event = RDMA_CM_EVENT_ESTABLISHED;
1190                         break;
1191                 case -ECONNRESET:
1192                 case -ECONNREFUSED:
1193                         event.event = RDMA_CM_EVENT_REJECTED;
1194                         break;
1195                 case -ETIMEDOUT:
1196                         event.event = RDMA_CM_EVENT_UNREACHABLE;
1197                         break;
1198                 default:
1199                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1200                         break;
1201                 }
1202                 break;
1203         case IW_CM_EVENT_ESTABLISHED:
1204                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1205                 break;
1206         default:
1207                 BUG_ON(1);
1208         }
1209
1210         event.status = iw_event->status;
1211         event.param.conn.private_data = iw_event->private_data;
1212         event.param.conn.private_data_len = iw_event->private_data_len;
1213         ret = id_priv->id.event_handler(&id_priv->id, &event);
1214         if (ret) {
1215                 /* Destroy the CM ID by returning a non-zero value. */
1216                 id_priv->cm_id.iw = NULL;
1217                 cma_exch(id_priv, CMA_DESTROYING);
1218                 cma_release_remove(id_priv);
1219                 rdma_destroy_id(&id_priv->id);
1220                 return ret;
1221         }
1222
1223         cma_release_remove(id_priv);
1224         return ret;
1225 }
1226
1227 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1228                                struct iw_cm_event *iw_event)
1229 {
1230         struct rdma_cm_id *new_cm_id;
1231         struct rdma_id_private *listen_id, *conn_id;
1232         struct sockaddr_in *sin;
1233         struct net_device *dev = NULL;
1234         struct rdma_cm_event event;
1235         int ret;
1236
1237         listen_id = cm_id->context;
1238         atomic_inc(&listen_id->dev_remove);
1239         if (!cma_comp(listen_id, CMA_LISTEN)) {
1240                 ret = -ECONNABORTED;
1241                 goto out;
1242         }
1243
1244         /* Create a new RDMA id for the new IW CM ID */
1245         new_cm_id = rdma_create_id(listen_id->id.event_handler,
1246                                    listen_id->id.context,
1247                                    RDMA_PS_TCP);
1248         if (!new_cm_id) {
1249                 ret = -ENOMEM;
1250                 goto out;
1251         }
1252         conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1253         atomic_inc(&conn_id->dev_remove);
1254         conn_id->state = CMA_CONNECT;
1255
1256         dev = ip_dev_find(iw_event->local_addr.sin_addr.s_addr);
1257         if (!dev) {
1258                 ret = -EADDRNOTAVAIL;
1259                 cma_release_remove(conn_id);
1260                 rdma_destroy_id(new_cm_id);
1261                 goto out;
1262         }
1263         ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1264         if (ret) {
1265                 cma_release_remove(conn_id);
1266                 rdma_destroy_id(new_cm_id);
1267                 goto out;
1268         }
1269
1270         mutex_lock(&lock);
1271         ret = cma_acquire_dev(conn_id);
1272         mutex_unlock(&lock);
1273         if (ret) {
1274                 cma_release_remove(conn_id);
1275                 rdma_destroy_id(new_cm_id);
1276                 goto out;
1277         }
1278
1279         conn_id->cm_id.iw = cm_id;
1280         cm_id->context = conn_id;
1281         cm_id->cm_handler = cma_iw_handler;
1282
1283         sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1284         *sin = iw_event->local_addr;
1285         sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1286         *sin = iw_event->remote_addr;
1287
1288         memset(&event, 0, sizeof event);
1289         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1290         event.param.conn.private_data = iw_event->private_data;
1291         event.param.conn.private_data_len = iw_event->private_data_len;
1292         ret = conn_id->id.event_handler(&conn_id->id, &event);
1293         if (ret) {
1294                 /* User wants to destroy the CM ID */
1295                 conn_id->cm_id.iw = NULL;
1296                 cma_exch(conn_id, CMA_DESTROYING);
1297                 cma_release_remove(conn_id);
1298                 rdma_destroy_id(&conn_id->id);
1299         }
1300
1301 out:
1302         if (dev)
1303                 dev_put(dev);
1304         cma_release_remove(listen_id);
1305         return ret;
1306 }
1307
1308 static int cma_ib_listen(struct rdma_id_private *id_priv)
1309 {
1310         struct ib_cm_compare_data compare_data;
1311         struct sockaddr *addr;
1312         __be64 svc_id;
1313         int ret;
1314
1315         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler,
1316                                             id_priv);
1317         if (IS_ERR(id_priv->cm_id.ib))
1318                 return PTR_ERR(id_priv->cm_id.ib);
1319
1320         addr = &id_priv->id.route.addr.src_addr;
1321         svc_id = cma_get_service_id(id_priv->id.ps, addr);
1322         if (cma_any_addr(addr))
1323                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1324         else {
1325                 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1326                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1327         }
1328
1329         if (ret) {
1330                 ib_destroy_cm_id(id_priv->cm_id.ib);
1331                 id_priv->cm_id.ib = NULL;
1332         }
1333
1334         return ret;
1335 }
1336
1337 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1338 {
1339         int ret;
1340         struct sockaddr_in *sin;
1341
1342         id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device,
1343                                             iw_conn_req_handler,
1344                                             id_priv);
1345         if (IS_ERR(id_priv->cm_id.iw))
1346                 return PTR_ERR(id_priv->cm_id.iw);
1347
1348         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1349         id_priv->cm_id.iw->local_addr = *sin;
1350
1351         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1352
1353         if (ret) {
1354                 iw_destroy_cm_id(id_priv->cm_id.iw);
1355                 id_priv->cm_id.iw = NULL;
1356         }
1357
1358         return ret;
1359 }
1360
1361 static int cma_listen_handler(struct rdma_cm_id *id,
1362                               struct rdma_cm_event *event)
1363 {
1364         struct rdma_id_private *id_priv = id->context;
1365
1366         id->context = id_priv->id.context;
1367         id->event_handler = id_priv->id.event_handler;
1368         return id_priv->id.event_handler(id, event);
1369 }
1370
1371 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1372                               struct cma_device *cma_dev)
1373 {
1374         struct rdma_id_private *dev_id_priv;
1375         struct rdma_cm_id *id;
1376         int ret;
1377
1378         id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps);
1379         if (IS_ERR(id))
1380                 return;
1381
1382         dev_id_priv = container_of(id, struct rdma_id_private, id);
1383
1384         dev_id_priv->state = CMA_ADDR_BOUND;
1385         memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
1386                ip_addr_size(&id_priv->id.route.addr.src_addr));
1387
1388         cma_attach_to_dev(dev_id_priv, cma_dev);
1389         list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1390
1391         ret = rdma_listen(id, id_priv->backlog);
1392         if (ret)
1393                 goto err;
1394
1395         return;
1396 err:
1397         cma_destroy_listen(dev_id_priv);
1398 }
1399
1400 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1401 {
1402         struct cma_device *cma_dev;
1403
1404         mutex_lock(&lock);
1405         list_add_tail(&id_priv->list, &listen_any_list);
1406         list_for_each_entry(cma_dev, &dev_list, list)
1407                 cma_listen_on_dev(id_priv, cma_dev);
1408         mutex_unlock(&lock);
1409 }
1410
1411 static int cma_bind_any(struct rdma_cm_id *id, sa_family_t af)
1412 {
1413         struct sockaddr_in addr_in;
1414
1415         memset(&addr_in, 0, sizeof addr_in);
1416         addr_in.sin_family = af;
1417         return rdma_bind_addr(id, (struct sockaddr *) &addr_in);
1418 }
1419
1420 int rdma_listen(struct rdma_cm_id *id, int backlog)
1421 {
1422         struct rdma_id_private *id_priv;
1423         int ret;
1424
1425         id_priv = container_of(id, struct rdma_id_private, id);
1426         if (id_priv->state == CMA_IDLE) {
1427                 ret = cma_bind_any(id, AF_INET);
1428                 if (ret)
1429                         return ret;
1430         }
1431
1432         if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN))
1433                 return -EINVAL;
1434
1435         id_priv->backlog = backlog;
1436         if (id->device) {
1437                 switch (rdma_node_get_transport(id->device->node_type)) {
1438                 case RDMA_TRANSPORT_IB:
1439                         ret = cma_ib_listen(id_priv);
1440                         if (ret)
1441                                 goto err;
1442                         break;
1443                 case RDMA_TRANSPORT_IWARP:
1444                         ret = cma_iw_listen(id_priv, backlog);
1445                         if (ret)
1446                                 goto err;
1447                         break;
1448                 default:
1449                         ret = -ENOSYS;
1450                         goto err;
1451                 }
1452         } else
1453                 cma_listen_on_all(id_priv);
1454
1455         return 0;
1456 err:
1457         id_priv->backlog = 0;
1458         cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND);
1459         return ret;
1460 }
1461 EXPORT_SYMBOL(rdma_listen);
1462
1463 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1464                               void *context)
1465 {
1466         struct cma_work *work = context;
1467         struct rdma_route *route;
1468
1469         route = &work->id->id.route;
1470
1471         if (!status) {
1472                 route->num_paths = 1;
1473                 *route->path_rec = *path_rec;
1474         } else {
1475                 work->old_state = CMA_ROUTE_QUERY;
1476                 work->new_state = CMA_ADDR_RESOLVED;
1477                 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1478                 work->event.status = status;
1479         }
1480
1481         queue_work(cma_wq, &work->work);
1482 }
1483
1484 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1485                               struct cma_work *work)
1486 {
1487         struct rdma_dev_addr *addr = &id_priv->id.route.addr.dev_addr;
1488         struct ib_sa_path_rec path_rec;
1489
1490         memset(&path_rec, 0, sizeof path_rec);
1491         ib_addr_get_sgid(addr, &path_rec.sgid);
1492         ib_addr_get_dgid(addr, &path_rec.dgid);
1493         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(addr));
1494         path_rec.numb_path = 1;
1495
1496         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1497                                 id_priv->id.port_num, &path_rec,
1498                                 IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1499                                 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH,
1500                                 timeout_ms, GFP_KERNEL,
1501                                 cma_query_handler, work, &id_priv->query);
1502
1503         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1504 }
1505
1506 static void cma_work_handler(struct work_struct *_work)
1507 {
1508         struct cma_work *work = container_of(_work, struct cma_work, work);
1509         struct rdma_id_private *id_priv = work->id;
1510         int destroy = 0;
1511
1512         atomic_inc(&id_priv->dev_remove);
1513         if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1514                 goto out;
1515
1516         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1517                 cma_exch(id_priv, CMA_DESTROYING);
1518                 destroy = 1;
1519         }
1520 out:
1521         cma_release_remove(id_priv);
1522         cma_deref_id(id_priv);
1523         if (destroy)
1524                 rdma_destroy_id(&id_priv->id);
1525         kfree(work);
1526 }
1527
1528 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1529 {
1530         struct rdma_route *route = &id_priv->id.route;
1531         struct cma_work *work;
1532         int ret;
1533
1534         work = kzalloc(sizeof *work, GFP_KERNEL);
1535         if (!work)
1536                 return -ENOMEM;
1537
1538         work->id = id_priv;
1539         INIT_WORK(&work->work, cma_work_handler);
1540         work->old_state = CMA_ROUTE_QUERY;
1541         work->new_state = CMA_ROUTE_RESOLVED;
1542         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1543
1544         route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1545         if (!route->path_rec) {
1546                 ret = -ENOMEM;
1547                 goto err1;
1548         }
1549
1550         ret = cma_query_ib_route(id_priv, timeout_ms, work);
1551         if (ret)
1552                 goto err2;
1553
1554         return 0;
1555 err2:
1556         kfree(route->path_rec);
1557         route->path_rec = NULL;
1558 err1:
1559         kfree(work);
1560         return ret;
1561 }
1562
1563 int rdma_set_ib_paths(struct rdma_cm_id *id,
1564                       struct ib_sa_path_rec *path_rec, int num_paths)
1565 {
1566         struct rdma_id_private *id_priv;
1567         int ret;
1568
1569         id_priv = container_of(id, struct rdma_id_private, id);
1570         if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED))
1571                 return -EINVAL;
1572
1573         id->route.path_rec = kmalloc(sizeof *path_rec * num_paths, GFP_KERNEL);
1574         if (!id->route.path_rec) {
1575                 ret = -ENOMEM;
1576                 goto err;
1577         }
1578
1579         memcpy(id->route.path_rec, path_rec, sizeof *path_rec * num_paths);
1580         return 0;
1581 err:
1582         cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED);
1583         return ret;
1584 }
1585 EXPORT_SYMBOL(rdma_set_ib_paths);
1586
1587 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1588 {
1589         struct cma_work *work;
1590
1591         work = kzalloc(sizeof *work, GFP_KERNEL);
1592         if (!work)
1593                 return -ENOMEM;
1594
1595         work->id = id_priv;
1596         INIT_WORK(&work->work, cma_work_handler);
1597         work->old_state = CMA_ROUTE_QUERY;
1598         work->new_state = CMA_ROUTE_RESOLVED;
1599         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1600         queue_work(cma_wq, &work->work);
1601         return 0;
1602 }
1603
1604 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1605 {
1606         struct rdma_id_private *id_priv;
1607         int ret;
1608
1609         id_priv = container_of(id, struct rdma_id_private, id);
1610         if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY))
1611                 return -EINVAL;
1612
1613         atomic_inc(&id_priv->refcount);
1614         switch (rdma_node_get_transport(id->device->node_type)) {
1615         case RDMA_TRANSPORT_IB:
1616                 ret = cma_resolve_ib_route(id_priv, timeout_ms);
1617                 break;
1618         case RDMA_TRANSPORT_IWARP:
1619                 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1620                 break;
1621         default:
1622                 ret = -ENOSYS;
1623                 break;
1624         }
1625         if (ret)
1626                 goto err;
1627
1628         return 0;
1629 err:
1630         cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED);
1631         cma_deref_id(id_priv);
1632         return ret;
1633 }
1634 EXPORT_SYMBOL(rdma_resolve_route);
1635
1636 static int cma_bind_loopback(struct rdma_id_private *id_priv)
1637 {
1638         struct cma_device *cma_dev;
1639         struct ib_port_attr port_attr;
1640         union ib_gid gid;
1641         u16 pkey;
1642         int ret;
1643         u8 p;
1644
1645         mutex_lock(&lock);
1646         if (list_empty(&dev_list)) {
1647                 ret = -ENODEV;
1648                 goto out;
1649         }
1650         list_for_each_entry(cma_dev, &dev_list, list)
1651                 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
1652                         if (!ib_query_port(cma_dev->device, p, &port_attr) &&
1653                             port_attr.state == IB_PORT_ACTIVE)
1654                                 goto port_found;
1655
1656         p = 1;
1657         cma_dev = list_entry(dev_list.next, struct cma_device, list);
1658
1659 port_found:
1660         ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
1661         if (ret)
1662                 goto out;
1663
1664         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
1665         if (ret)
1666                 goto out;
1667
1668         ib_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1669         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
1670         id_priv->id.port_num = p;
1671         cma_attach_to_dev(id_priv, cma_dev);
1672 out:
1673         mutex_unlock(&lock);
1674         return ret;
1675 }
1676
1677 static void addr_handler(int status, struct sockaddr *src_addr,
1678                          struct rdma_dev_addr *dev_addr, void *context)
1679 {
1680         struct rdma_id_private *id_priv = context;
1681         struct rdma_cm_event event;
1682
1683         memset(&event, 0, sizeof event);
1684         atomic_inc(&id_priv->dev_remove);
1685
1686         /*
1687          * Grab mutex to block rdma_destroy_id() from removing the device while
1688          * we're trying to acquire it.
1689          */
1690         mutex_lock(&lock);
1691         if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) {
1692                 mutex_unlock(&lock);
1693                 goto out;
1694         }
1695
1696         if (!status && !id_priv->cma_dev)
1697                 status = cma_acquire_dev(id_priv);
1698         mutex_unlock(&lock);
1699
1700         if (status) {
1701                 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND))
1702                         goto out;
1703                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
1704                 event.status = status;
1705         } else {
1706                 memcpy(&id_priv->id.route.addr.src_addr, src_addr,
1707                        ip_addr_size(src_addr));
1708                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1709         }
1710
1711         if (id_priv->id.event_handler(&id_priv->id, &event)) {
1712                 cma_exch(id_priv, CMA_DESTROYING);
1713                 cma_release_remove(id_priv);
1714                 cma_deref_id(id_priv);
1715                 rdma_destroy_id(&id_priv->id);
1716                 return;
1717         }
1718 out:
1719         cma_release_remove(id_priv);
1720         cma_deref_id(id_priv);
1721 }
1722
1723 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
1724 {
1725         struct cma_work *work;
1726         struct sockaddr_in *src_in, *dst_in;
1727         union ib_gid gid;
1728         int ret;
1729
1730         work = kzalloc(sizeof *work, GFP_KERNEL);
1731         if (!work)
1732                 return -ENOMEM;
1733
1734         if (!id_priv->cma_dev) {
1735                 ret = cma_bind_loopback(id_priv);
1736                 if (ret)
1737                         goto err;
1738         }
1739
1740         ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1741         ib_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
1742
1743         if (cma_zero_addr(&id_priv->id.route.addr.src_addr)) {
1744                 src_in = (struct sockaddr_in *)&id_priv->id.route.addr.src_addr;
1745                 dst_in = (struct sockaddr_in *)&id_priv->id.route.addr.dst_addr;
1746                 src_in->sin_family = dst_in->sin_family;
1747                 src_in->sin_addr.s_addr = dst_in->sin_addr.s_addr;
1748         }
1749
1750         work->id = id_priv;
1751         INIT_WORK(&work->work, cma_work_handler);
1752         work->old_state = CMA_ADDR_QUERY;
1753         work->new_state = CMA_ADDR_RESOLVED;
1754         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1755         queue_work(cma_wq, &work->work);
1756         return 0;
1757 err:
1758         kfree(work);
1759         return ret;
1760 }
1761
1762 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1763                          struct sockaddr *dst_addr)
1764 {
1765         if (src_addr && src_addr->sa_family)
1766                 return rdma_bind_addr(id, src_addr);
1767         else
1768                 return cma_bind_any(id, dst_addr->sa_family);
1769 }
1770
1771 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1772                       struct sockaddr *dst_addr, int timeout_ms)
1773 {
1774         struct rdma_id_private *id_priv;
1775         int ret;
1776
1777         id_priv = container_of(id, struct rdma_id_private, id);
1778         if (id_priv->state == CMA_IDLE) {
1779                 ret = cma_bind_addr(id, src_addr, dst_addr);
1780                 if (ret)
1781                         return ret;
1782         }
1783
1784         if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY))
1785                 return -EINVAL;
1786
1787         atomic_inc(&id_priv->refcount);
1788         memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
1789         if (cma_any_addr(dst_addr))
1790                 ret = cma_resolve_loopback(id_priv);
1791         else
1792                 ret = rdma_resolve_ip(&addr_client, &id->route.addr.src_addr,
1793                                       dst_addr, &id->route.addr.dev_addr,
1794                                       timeout_ms, addr_handler, id_priv);
1795         if (ret)
1796                 goto err;
1797
1798         return 0;
1799 err:
1800         cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND);
1801         cma_deref_id(id_priv);
1802         return ret;
1803 }
1804 EXPORT_SYMBOL(rdma_resolve_addr);
1805
1806 static void cma_bind_port(struct rdma_bind_list *bind_list,
1807                           struct rdma_id_private *id_priv)
1808 {
1809         struct sockaddr_in *sin;
1810
1811         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1812         sin->sin_port = htons(bind_list->port);
1813         id_priv->bind_list = bind_list;
1814         hlist_add_head(&id_priv->node, &bind_list->owners);
1815 }
1816
1817 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
1818                           unsigned short snum)
1819 {
1820         struct rdma_bind_list *bind_list;
1821         int port, ret;
1822
1823         bind_list = kmalloc(sizeof *bind_list, GFP_KERNEL);
1824         if (!bind_list)
1825                 return -ENOMEM;
1826
1827         do {
1828                 ret = idr_get_new_above(ps, bind_list, snum, &port);
1829         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1830
1831         if (ret)
1832                 goto err1;
1833
1834         if (port != snum) {
1835                 ret = -EADDRNOTAVAIL;
1836                 goto err2;
1837         }
1838
1839         bind_list->ps = ps;
1840         bind_list->port = (unsigned short) port;
1841         cma_bind_port(bind_list, id_priv);
1842         return 0;
1843 err2:
1844         idr_remove(ps, port);
1845 err1:
1846         kfree(bind_list);
1847         return ret;
1848 }
1849
1850 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
1851 {
1852         struct rdma_bind_list *bind_list;
1853         int port, ret;
1854
1855         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
1856         if (!bind_list)
1857                 return -ENOMEM;
1858
1859 retry:
1860         do {
1861                 ret = idr_get_new_above(ps, bind_list, next_port, &port);
1862         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1863
1864         if (ret)
1865                 goto err1;
1866
1867         if (port > sysctl_local_port_range[1]) {
1868                 if (next_port != sysctl_local_port_range[0]) {
1869                         idr_remove(ps, port);
1870                         next_port = sysctl_local_port_range[0];
1871                         goto retry;
1872                 }
1873                 ret = -EADDRNOTAVAIL;
1874                 goto err2;
1875         }
1876
1877         if (port == sysctl_local_port_range[1])
1878                 next_port = sysctl_local_port_range[0];
1879         else
1880                 next_port = port + 1;
1881
1882         bind_list->ps = ps;
1883         bind_list->port = (unsigned short) port;
1884         cma_bind_port(bind_list, id_priv);
1885         return 0;
1886 err2:
1887         idr_remove(ps, port);
1888 err1:
1889         kfree(bind_list);
1890         return ret;
1891 }
1892
1893 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
1894 {
1895         struct rdma_id_private *cur_id;
1896         struct sockaddr_in *sin, *cur_sin;
1897         struct rdma_bind_list *bind_list;
1898         struct hlist_node *node;
1899         unsigned short snum;
1900
1901         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1902         snum = ntohs(sin->sin_port);
1903         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
1904                 return -EACCES;
1905
1906         bind_list = idr_find(ps, snum);
1907         if (!bind_list)
1908                 return cma_alloc_port(ps, id_priv, snum);
1909
1910         /*
1911          * We don't support binding to any address if anyone is bound to
1912          * a specific address on the same port.
1913          */
1914         if (cma_any_addr(&id_priv->id.route.addr.src_addr))
1915                 return -EADDRNOTAVAIL;
1916
1917         hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
1918                 if (cma_any_addr(&cur_id->id.route.addr.src_addr))
1919                         return -EADDRNOTAVAIL;
1920
1921                 cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr;
1922                 if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr)
1923                         return -EADDRINUSE;
1924         }
1925
1926         cma_bind_port(bind_list, id_priv);
1927         return 0;
1928 }
1929
1930 static int cma_get_port(struct rdma_id_private *id_priv)
1931 {
1932         struct idr *ps;
1933         int ret;
1934
1935         switch (id_priv->id.ps) {
1936         case RDMA_PS_SDP:
1937                 ps = &sdp_ps;
1938                 break;
1939         case RDMA_PS_TCP:
1940                 ps = &tcp_ps;
1941                 break;
1942         case RDMA_PS_UDP:
1943                 ps = &udp_ps;
1944                 break;
1945         case RDMA_PS_IPOIB:
1946                 ps = &ipoib_ps;
1947                 break;
1948         default:
1949                 return -EPROTONOSUPPORT;
1950         }
1951
1952         mutex_lock(&lock);
1953         if (cma_any_port(&id_priv->id.route.addr.src_addr))
1954                 ret = cma_alloc_any_port(ps, id_priv);
1955         else
1956                 ret = cma_use_port(ps, id_priv);
1957         mutex_unlock(&lock);
1958
1959         return ret;
1960 }
1961
1962 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
1963 {
1964         struct rdma_id_private *id_priv;
1965         int ret;
1966
1967         if (addr->sa_family != AF_INET)
1968                 return -EAFNOSUPPORT;
1969
1970         id_priv = container_of(id, struct rdma_id_private, id);
1971         if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND))
1972                 return -EINVAL;
1973
1974         if (!cma_any_addr(addr)) {
1975                 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
1976                 if (ret)
1977                         goto err1;
1978
1979                 mutex_lock(&lock);
1980                 ret = cma_acquire_dev(id_priv);
1981                 mutex_unlock(&lock);
1982                 if (ret)
1983                         goto err1;
1984         }
1985
1986         memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
1987         ret = cma_get_port(id_priv);
1988         if (ret)
1989                 goto err2;
1990
1991         return 0;
1992 err2:
1993         if (!cma_any_addr(addr)) {
1994                 mutex_lock(&lock);
1995                 cma_detach_from_dev(id_priv);
1996                 mutex_unlock(&lock);
1997         }
1998 err1:
1999         cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE);
2000         return ret;
2001 }
2002 EXPORT_SYMBOL(rdma_bind_addr);
2003
2004 static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2005                           struct rdma_route *route)
2006 {
2007         struct sockaddr_in *src4, *dst4;
2008         struct cma_hdr *cma_hdr;
2009         struct sdp_hh *sdp_hdr;
2010
2011         src4 = (struct sockaddr_in *) &route->addr.src_addr;
2012         dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2013
2014         switch (ps) {
2015         case RDMA_PS_SDP:
2016                 sdp_hdr = hdr;
2017                 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2018                         return -EINVAL;
2019                 sdp_set_ip_ver(sdp_hdr, 4);
2020                 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2021                 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2022                 sdp_hdr->port = src4->sin_port;
2023                 break;
2024         default:
2025                 cma_hdr = hdr;
2026                 cma_hdr->cma_version = CMA_VERSION;
2027                 cma_set_ip_ver(cma_hdr, 4);
2028                 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2029                 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2030                 cma_hdr->port = src4->sin_port;
2031                 break;
2032         }
2033         return 0;
2034 }
2035
2036 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2037                                 struct ib_cm_event *ib_event)
2038 {
2039         struct rdma_id_private *id_priv = cm_id->context;
2040         struct rdma_cm_event event;
2041         struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2042         int ret = 0;
2043
2044         memset(&event, 0, sizeof event);
2045         atomic_inc(&id_priv->dev_remove);
2046         if (!cma_comp(id_priv, CMA_CONNECT))
2047                 goto out;
2048
2049         switch (ib_event->event) {
2050         case IB_CM_SIDR_REQ_ERROR:
2051                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2052                 event.status = -ETIMEDOUT;
2053                 break;
2054         case IB_CM_SIDR_REP_RECEIVED:
2055                 event.param.ud.private_data = ib_event->private_data;
2056                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2057                 if (rep->status != IB_SIDR_SUCCESS) {
2058                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2059                         event.status = ib_event->param.sidr_rep_rcvd.status;
2060                         break;
2061                 }
2062                 if (id_priv->qkey != rep->qkey) {
2063                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2064                         event.status = -EINVAL;
2065                         break;
2066                 }
2067                 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2068                                      id_priv->id.route.path_rec,
2069                                      &event.param.ud.ah_attr);
2070                 event.param.ud.qp_num = rep->qpn;
2071                 event.param.ud.qkey = rep->qkey;
2072                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2073                 event.status = 0;
2074                 break;
2075         default:
2076                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d",
2077                        ib_event->event);
2078                 goto out;
2079         }
2080
2081         ret = id_priv->id.event_handler(&id_priv->id, &event);
2082         if (ret) {
2083                 /* Destroy the CM ID by returning a non-zero value. */
2084                 id_priv->cm_id.ib = NULL;
2085                 cma_exch(id_priv, CMA_DESTROYING);
2086                 cma_release_remove(id_priv);
2087                 rdma_destroy_id(&id_priv->id);
2088                 return ret;
2089         }
2090 out:
2091         cma_release_remove(id_priv);
2092         return ret;
2093 }
2094
2095 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2096                               struct rdma_conn_param *conn_param)
2097 {
2098         struct ib_cm_sidr_req_param req;
2099         struct rdma_route *route;
2100         int ret;
2101
2102         req.private_data_len = sizeof(struct cma_hdr) +
2103                                conn_param->private_data_len;
2104         req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2105         if (!req.private_data)
2106                 return -ENOMEM;
2107
2108         if (conn_param->private_data && conn_param->private_data_len)
2109                 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2110                        conn_param->private_data, conn_param->private_data_len);
2111
2112         route = &id_priv->id.route;
2113         ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2114         if (ret)
2115                 goto out;
2116
2117         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device,
2118                                             cma_sidr_rep_handler, id_priv);
2119         if (IS_ERR(id_priv->cm_id.ib)) {
2120                 ret = PTR_ERR(id_priv->cm_id.ib);
2121                 goto out;
2122         }
2123
2124         req.path = route->path_rec;
2125         req.service_id = cma_get_service_id(id_priv->id.ps,
2126                                             &route->addr.dst_addr);
2127         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2128         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2129
2130         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2131         if (ret) {
2132                 ib_destroy_cm_id(id_priv->cm_id.ib);
2133                 id_priv->cm_id.ib = NULL;
2134         }
2135 out:
2136         kfree(req.private_data);
2137         return ret;
2138 }
2139
2140 static int cma_connect_ib(struct rdma_id_private *id_priv,
2141                           struct rdma_conn_param *conn_param)
2142 {
2143         struct ib_cm_req_param req;
2144         struct rdma_route *route;
2145         void *private_data;
2146         int offset, ret;
2147
2148         memset(&req, 0, sizeof req);
2149         offset = cma_user_data_offset(id_priv->id.ps);
2150         req.private_data_len = offset + conn_param->private_data_len;
2151         private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2152         if (!private_data)
2153                 return -ENOMEM;
2154
2155         if (conn_param->private_data && conn_param->private_data_len)
2156                 memcpy(private_data + offset, conn_param->private_data,
2157                        conn_param->private_data_len);
2158
2159         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler,
2160                                             id_priv);
2161         if (IS_ERR(id_priv->cm_id.ib)) {
2162                 ret = PTR_ERR(id_priv->cm_id.ib);
2163                 goto out;
2164         }
2165
2166         route = &id_priv->id.route;
2167         ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2168         if (ret)
2169                 goto out;
2170         req.private_data = private_data;
2171
2172         req.primary_path = &route->path_rec[0];
2173         if (route->num_paths == 2)
2174                 req.alternate_path = &route->path_rec[1];
2175
2176         req.service_id = cma_get_service_id(id_priv->id.ps,
2177                                             &route->addr.dst_addr);
2178         req.qp_num = id_priv->qp_num;
2179         req.qp_type = IB_QPT_RC;
2180         req.starting_psn = id_priv->seq_num;
2181         req.responder_resources = conn_param->responder_resources;
2182         req.initiator_depth = conn_param->initiator_depth;
2183         req.flow_control = conn_param->flow_control;
2184         req.retry_count = conn_param->retry_count;
2185         req.rnr_retry_count = conn_param->rnr_retry_count;
2186         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2187         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2188         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2189         req.srq = id_priv->srq ? 1 : 0;
2190
2191         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2192 out:
2193         if (ret && !IS_ERR(id_priv->cm_id.ib)) {
2194                 ib_destroy_cm_id(id_priv->cm_id.ib);
2195                 id_priv->cm_id.ib = NULL;
2196         }
2197
2198         kfree(private_data);
2199         return ret;
2200 }
2201
2202 static int cma_connect_iw(struct rdma_id_private *id_priv,
2203                           struct rdma_conn_param *conn_param)
2204 {
2205         struct iw_cm_id *cm_id;
2206         struct sockaddr_in* sin;
2207         int ret;
2208         struct iw_cm_conn_param iw_param;
2209
2210         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2211         if (IS_ERR(cm_id)) {
2212                 ret = PTR_ERR(cm_id);
2213                 goto out;
2214         }
2215
2216         id_priv->cm_id.iw = cm_id;
2217
2218         sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2219         cm_id->local_addr = *sin;
2220
2221         sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2222         cm_id->remote_addr = *sin;
2223
2224         ret = cma_modify_qp_rtr(&id_priv->id);
2225         if (ret)
2226                 goto out;
2227
2228         iw_param.ord = conn_param->initiator_depth;
2229         iw_param.ird = conn_param->responder_resources;
2230         iw_param.private_data = conn_param->private_data;
2231         iw_param.private_data_len = conn_param->private_data_len;
2232         if (id_priv->id.qp)
2233                 iw_param.qpn = id_priv->qp_num;
2234         else
2235                 iw_param.qpn = conn_param->qp_num;
2236         ret = iw_cm_connect(cm_id, &iw_param);
2237 out:
2238         if (ret && !IS_ERR(cm_id)) {
2239                 iw_destroy_cm_id(cm_id);
2240                 id_priv->cm_id.iw = NULL;
2241         }
2242         return ret;
2243 }
2244
2245 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2246 {
2247         struct rdma_id_private *id_priv;
2248         int ret;
2249
2250         id_priv = container_of(id, struct rdma_id_private, id);
2251         if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT))
2252                 return -EINVAL;
2253
2254         if (!id->qp) {
2255                 id_priv->qp_num = conn_param->qp_num;
2256                 id_priv->srq = conn_param->srq;
2257         }
2258
2259         switch (rdma_node_get_transport(id->device->node_type)) {
2260         case RDMA_TRANSPORT_IB:
2261                 if (cma_is_ud_ps(id->ps))
2262                         ret = cma_resolve_ib_udp(id_priv, conn_param);
2263                 else
2264                         ret = cma_connect_ib(id_priv, conn_param);
2265                 break;
2266         case RDMA_TRANSPORT_IWARP:
2267                 ret = cma_connect_iw(id_priv, conn_param);
2268                 break;
2269         default:
2270                 ret = -ENOSYS;
2271                 break;
2272         }
2273         if (ret)
2274                 goto err;
2275
2276         return 0;
2277 err:
2278         cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED);
2279         return ret;
2280 }
2281 EXPORT_SYMBOL(rdma_connect);
2282
2283 static int cma_accept_ib(struct rdma_id_private *id_priv,
2284                          struct rdma_conn_param *conn_param)
2285 {
2286         struct ib_cm_rep_param rep;
2287         struct ib_qp_attr qp_attr;
2288         int qp_attr_mask, ret;
2289
2290         if (id_priv->id.qp) {
2291                 ret = cma_modify_qp_rtr(&id_priv->id);
2292                 if (ret)
2293                         goto out;
2294
2295                 qp_attr.qp_state = IB_QPS_RTS;
2296                 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, &qp_attr,
2297                                          &qp_attr_mask);
2298                 if (ret)
2299                         goto out;
2300
2301                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
2302                 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
2303                 if (ret)
2304                         goto out;
2305         }
2306
2307         memset(&rep, 0, sizeof rep);
2308         rep.qp_num = id_priv->qp_num;
2309         rep.starting_psn = id_priv->seq_num;
2310         rep.private_data = conn_param->private_data;
2311         rep.private_data_len = conn_param->private_data_len;
2312         rep.responder_resources = conn_param->responder_resources;
2313         rep.initiator_depth = conn_param->initiator_depth;
2314         rep.target_ack_delay = CMA_CM_RESPONSE_TIMEOUT;
2315         rep.failover_accepted = 0;
2316         rep.flow_control = conn_param->flow_control;
2317         rep.rnr_retry_count = conn_param->rnr_retry_count;
2318         rep.srq = id_priv->srq ? 1 : 0;
2319
2320         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2321 out:
2322         return ret;
2323 }
2324
2325 static int cma_accept_iw(struct rdma_id_private *id_priv,
2326                   struct rdma_conn_param *conn_param)
2327 {
2328         struct iw_cm_conn_param iw_param;
2329         int ret;
2330
2331         ret = cma_modify_qp_rtr(&id_priv->id);
2332         if (ret)
2333                 return ret;
2334
2335         iw_param.ord = conn_param->initiator_depth;
2336         iw_param.ird = conn_param->responder_resources;
2337         iw_param.private_data = conn_param->private_data;
2338         iw_param.private_data_len = conn_param->private_data_len;
2339         if (id_priv->id.qp) {
2340                 iw_param.qpn = id_priv->qp_num;
2341         } else
2342                 iw_param.qpn = conn_param->qp_num;
2343
2344         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2345 }
2346
2347 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2348                              enum ib_cm_sidr_status status,
2349                              const void *private_data, int private_data_len)
2350 {
2351         struct ib_cm_sidr_rep_param rep;
2352
2353         memset(&rep, 0, sizeof rep);
2354         rep.status = status;
2355         if (status == IB_SIDR_SUCCESS) {
2356                 rep.qp_num = id_priv->qp_num;
2357                 rep.qkey = id_priv->qkey;
2358         }
2359         rep.private_data = private_data;
2360         rep.private_data_len = private_data_len;
2361
2362         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2363 }
2364
2365 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2366 {
2367         struct rdma_id_private *id_priv;
2368         int ret;
2369
2370         id_priv = container_of(id, struct rdma_id_private, id);
2371         if (!cma_comp(id_priv, CMA_CONNECT))
2372                 return -EINVAL;
2373
2374         if (!id->qp && conn_param) {
2375                 id_priv->qp_num = conn_param->qp_num;
2376                 id_priv->srq = conn_param->srq;
2377         }
2378
2379         switch (rdma_node_get_transport(id->device->node_type)) {
2380         case RDMA_TRANSPORT_IB:
2381                 if (cma_is_ud_ps(id->ps))
2382                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2383                                                 conn_param->private_data,
2384                                                 conn_param->private_data_len);
2385                 else if (conn_param)
2386                         ret = cma_accept_ib(id_priv, conn_param);
2387                 else
2388                         ret = cma_rep_recv(id_priv);
2389                 break;
2390         case RDMA_TRANSPORT_IWARP:
2391                 ret = cma_accept_iw(id_priv, conn_param);
2392                 break;
2393         default:
2394                 ret = -ENOSYS;
2395                 break;
2396         }
2397
2398         if (ret)
2399                 goto reject;
2400
2401         return 0;
2402 reject:
2403         cma_modify_qp_err(id);
2404         rdma_reject(id, NULL, 0);
2405         return ret;
2406 }
2407 EXPORT_SYMBOL(rdma_accept);
2408
2409 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2410 {
2411         struct rdma_id_private *id_priv;
2412         int ret;
2413
2414         id_priv = container_of(id, struct rdma_id_private, id);
2415         if (!cma_comp(id_priv, CMA_CONNECT))
2416                 return -EINVAL;
2417
2418         switch (id->device->node_type) {
2419         case RDMA_NODE_IB_CA:
2420                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2421                 break;
2422         default:
2423                 ret = 0;
2424                 break;
2425         }
2426         return ret;
2427 }
2428 EXPORT_SYMBOL(rdma_notify);
2429
2430 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
2431                 u8 private_data_len)
2432 {
2433         struct rdma_id_private *id_priv;
2434         int ret;
2435
2436         id_priv = container_of(id, struct rdma_id_private, id);
2437         if (!cma_comp(id_priv, CMA_CONNECT))
2438                 return -EINVAL;
2439
2440         switch (rdma_node_get_transport(id->device->node_type)) {
2441         case RDMA_TRANSPORT_IB:
2442                 if (cma_is_ud_ps(id->ps))
2443                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
2444                                                 private_data, private_data_len);
2445                 else
2446                         ret = ib_send_cm_rej(id_priv->cm_id.ib,
2447                                              IB_CM_REJ_CONSUMER_DEFINED, NULL,
2448                                              0, private_data, private_data_len);
2449                 break;
2450         case RDMA_TRANSPORT_IWARP:
2451                 ret = iw_cm_reject(id_priv->cm_id.iw,
2452                                    private_data, private_data_len);
2453                 break;
2454         default:
2455                 ret = -ENOSYS;
2456                 break;
2457         }
2458         return ret;
2459 }
2460 EXPORT_SYMBOL(rdma_reject);
2461
2462 int rdma_disconnect(struct rdma_cm_id *id)
2463 {
2464         struct rdma_id_private *id_priv;
2465         int ret;
2466
2467         id_priv = container_of(id, struct rdma_id_private, id);
2468         if (!cma_comp(id_priv, CMA_CONNECT) &&
2469             !cma_comp(id_priv, CMA_DISCONNECT))
2470                 return -EINVAL;
2471
2472         switch (rdma_node_get_transport(id->device->node_type)) {
2473         case RDMA_TRANSPORT_IB:
2474                 ret = cma_modify_qp_err(id);
2475                 if (ret)
2476                         goto out;
2477                 /* Initiate or respond to a disconnect. */
2478                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
2479                         ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
2480                 break;
2481         case RDMA_TRANSPORT_IWARP:
2482                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
2483                 break;
2484         default:
2485                 ret = -EINVAL;
2486                 break;
2487         }
2488 out:
2489         return ret;
2490 }
2491 EXPORT_SYMBOL(rdma_disconnect);
2492
2493 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
2494 {
2495         struct rdma_id_private *id_priv;
2496         struct cma_multicast *mc = multicast->context;
2497         struct rdma_cm_event event;
2498         int ret;
2499
2500         id_priv = mc->id_priv;
2501         atomic_inc(&id_priv->dev_remove);
2502         if (!cma_comp(id_priv, CMA_ADDR_BOUND) &&
2503             !cma_comp(id_priv, CMA_ADDR_RESOLVED))
2504                 goto out;
2505
2506         if (!status && id_priv->id.qp)
2507                 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
2508                                          multicast->rec.mlid);
2509
2510         memset(&event, 0, sizeof event);
2511         event.status = status;
2512         event.param.ud.private_data = mc->context;
2513         if (!status) {
2514                 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
2515                 ib_init_ah_from_mcmember(id_priv->id.device,
2516                                          id_priv->id.port_num, &multicast->rec,
2517                                          &event.param.ud.ah_attr);
2518                 event.param.ud.qp_num = 0xFFFFFF;
2519                 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
2520         } else
2521                 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
2522
2523         ret = id_priv->id.event_handler(&id_priv->id, &event);
2524         if (ret) {
2525                 cma_exch(id_priv, CMA_DESTROYING);
2526                 cma_release_remove(id_priv);
2527                 rdma_destroy_id(&id_priv->id);
2528                 return 0;
2529         }
2530 out:
2531         cma_release_remove(id_priv);
2532         return 0;
2533 }
2534
2535 static void cma_set_mgid(struct rdma_id_private *id_priv,
2536                          struct sockaddr *addr, union ib_gid *mgid)
2537 {
2538         unsigned char mc_map[MAX_ADDR_LEN];
2539         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2540         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
2541         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
2542
2543         if (cma_any_addr(addr)) {
2544                 memset(mgid, 0, sizeof *mgid);
2545         } else if ((addr->sa_family == AF_INET6) &&
2546                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFF10A01B) ==
2547                                                                  0xFF10A01B)) {
2548                 /* IPv6 address is an SA assigned MGID. */
2549                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
2550         } else {
2551                 ip_ib_mc_map(sin->sin_addr.s_addr, mc_map);
2552                 if (id_priv->id.ps == RDMA_PS_UDP)
2553                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
2554                 mc_map[8] = ib_addr_get_pkey(dev_addr) >> 8;
2555                 mc_map[9] = (unsigned char) ib_addr_get_pkey(dev_addr);
2556                 *mgid = *(union ib_gid *) (mc_map + 4);
2557         }
2558 }
2559
2560 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
2561                                  struct cma_multicast *mc)
2562 {
2563         struct ib_sa_mcmember_rec rec;
2564         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2565         ib_sa_comp_mask comp_mask;
2566         int ret;
2567
2568         ib_addr_get_mgid(dev_addr, &rec.mgid);
2569         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
2570                                      &rec.mgid, &rec);
2571         if (ret)
2572                 return ret;
2573
2574         cma_set_mgid(id_priv, &mc->addr, &rec.mgid);
2575         if (id_priv->id.ps == RDMA_PS_UDP)
2576                 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
2577         ib_addr_get_sgid(dev_addr, &rec.port_gid);
2578         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2579         rec.join_state = 1;
2580
2581         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
2582                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
2583                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
2584                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
2585                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
2586
2587         mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
2588                                                 id_priv->id.port_num, &rec,
2589                                                 comp_mask, GFP_KERNEL,
2590                                                 cma_ib_mc_handler, mc);
2591         if (IS_ERR(mc->multicast.ib))
2592                 return PTR_ERR(mc->multicast.ib);
2593
2594         return 0;
2595 }
2596
2597 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
2598                         void *context)
2599 {
2600         struct rdma_id_private *id_priv;
2601         struct cma_multicast *mc;
2602         int ret;
2603
2604         id_priv = container_of(id, struct rdma_id_private, id);
2605         if (!cma_comp(id_priv, CMA_ADDR_BOUND) &&
2606             !cma_comp(id_priv, CMA_ADDR_RESOLVED))
2607                 return -EINVAL;
2608
2609         mc = kmalloc(sizeof *mc, GFP_KERNEL);
2610         if (!mc)
2611                 return -ENOMEM;
2612
2613         memcpy(&mc->addr, addr, ip_addr_size(addr));
2614         mc->context = context;
2615         mc->id_priv = id_priv;
2616
2617         spin_lock(&id_priv->lock);
2618         list_add(&mc->list, &id_priv->mc_list);
2619         spin_unlock(&id_priv->lock);
2620
2621         switch (rdma_node_get_transport(id->device->node_type)) {
2622         case RDMA_TRANSPORT_IB:
2623                 ret = cma_join_ib_multicast(id_priv, mc);
2624                 break;
2625         default:
2626                 ret = -ENOSYS;
2627                 break;
2628         }
2629
2630         if (ret) {
2631                 spin_lock_irq(&id_priv->lock);
2632                 list_del(&mc->list);
2633                 spin_unlock_irq(&id_priv->lock);
2634                 kfree(mc);
2635         }
2636         return ret;
2637 }
2638 EXPORT_SYMBOL(rdma_join_multicast);
2639
2640 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
2641 {
2642         struct rdma_id_private *id_priv;
2643         struct cma_multicast *mc;
2644
2645         id_priv = container_of(id, struct rdma_id_private, id);
2646         spin_lock_irq(&id_priv->lock);
2647         list_for_each_entry(mc, &id_priv->mc_list, list) {
2648                 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
2649                         list_del(&mc->list);
2650                         spin_unlock_irq(&id_priv->lock);
2651
2652                         if (id->qp)
2653                                 ib_detach_mcast(id->qp,
2654                                                 &mc->multicast.ib->rec.mgid,
2655                                                 mc->multicast.ib->rec.mlid);
2656                         ib_sa_free_multicast(mc->multicast.ib);
2657                         kfree(mc);
2658                         return;
2659                 }
2660         }
2661         spin_unlock_irq(&id_priv->lock);
2662 }
2663 EXPORT_SYMBOL(rdma_leave_multicast);
2664
2665 static void cma_add_one(struct ib_device *device)
2666 {
2667         struct cma_device *cma_dev;
2668         struct rdma_id_private *id_priv;
2669
2670         cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
2671         if (!cma_dev)
2672                 return;
2673
2674         cma_dev->device = device;
2675         cma_dev->node_guid = device->node_guid;
2676
2677         init_completion(&cma_dev->comp);
2678         atomic_set(&cma_dev->refcount, 1);
2679         INIT_LIST_HEAD(&cma_dev->id_list);
2680         ib_set_client_data(device, &cma_client, cma_dev);
2681
2682         mutex_lock(&lock);
2683         list_add_tail(&cma_dev->list, &dev_list);
2684         list_for_each_entry(id_priv, &listen_any_list, list)
2685                 cma_listen_on_dev(id_priv, cma_dev);
2686         mutex_unlock(&lock);
2687 }
2688
2689 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
2690 {
2691         struct rdma_cm_event event;
2692         enum cma_state state;
2693
2694         /* Record that we want to remove the device */
2695         state = cma_exch(id_priv, CMA_DEVICE_REMOVAL);
2696         if (state == CMA_DESTROYING)
2697                 return 0;
2698
2699         cma_cancel_operation(id_priv, state);
2700         wait_event(id_priv->wait_remove, !atomic_read(&id_priv->dev_remove));
2701
2702         /* Check for destruction from another callback. */
2703         if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL))
2704                 return 0;
2705
2706         memset(&event, 0, sizeof event);
2707         event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
2708         return id_priv->id.event_handler(&id_priv->id, &event);
2709 }
2710
2711 static void cma_process_remove(struct cma_device *cma_dev)
2712 {
2713         struct rdma_id_private *id_priv;
2714         int ret;
2715
2716         mutex_lock(&lock);
2717         while (!list_empty(&cma_dev->id_list)) {
2718                 id_priv = list_entry(cma_dev->id_list.next,
2719                                      struct rdma_id_private, list);
2720
2721                 if (cma_internal_listen(id_priv)) {
2722                         cma_destroy_listen(id_priv);
2723                         continue;
2724                 }
2725
2726                 list_del_init(&id_priv->list);
2727                 atomic_inc(&id_priv->refcount);
2728                 mutex_unlock(&lock);
2729
2730                 ret = cma_remove_id_dev(id_priv);
2731                 cma_deref_id(id_priv);
2732                 if (ret)
2733                         rdma_destroy_id(&id_priv->id);
2734
2735                 mutex_lock(&lock);
2736         }
2737         mutex_unlock(&lock);
2738
2739         cma_deref_dev(cma_dev);
2740         wait_for_completion(&cma_dev->comp);
2741 }
2742
2743 static void cma_remove_one(struct ib_device *device)
2744 {
2745         struct cma_device *cma_dev;
2746
2747         cma_dev = ib_get_client_data(device, &cma_client);
2748         if (!cma_dev)
2749                 return;
2750
2751         mutex_lock(&lock);
2752         list_del(&cma_dev->list);
2753         mutex_unlock(&lock);
2754
2755         cma_process_remove(cma_dev);
2756         kfree(cma_dev);
2757 }
2758
2759 static int cma_init(void)
2760 {
2761         int ret;
2762
2763         get_random_bytes(&next_port, sizeof next_port);
2764         next_port = (next_port % (sysctl_local_port_range[1] -
2765                                   sysctl_local_port_range[0])) +
2766                     sysctl_local_port_range[0];
2767         cma_wq = create_singlethread_workqueue("rdma_cm");
2768         if (!cma_wq)
2769                 return -ENOMEM;
2770
2771         ib_sa_register_client(&sa_client);
2772         rdma_addr_register_client(&addr_client);
2773
2774         ret = ib_register_client(&cma_client);
2775         if (ret)
2776                 goto err;
2777         return 0;
2778
2779 err:
2780         rdma_addr_unregister_client(&addr_client);
2781         ib_sa_unregister_client(&sa_client);
2782         destroy_workqueue(cma_wq);
2783         return ret;
2784 }
2785
2786 static void cma_cleanup(void)
2787 {
2788         ib_unregister_client(&cma_client);
2789         rdma_addr_unregister_client(&addr_client);
2790         ib_sa_unregister_client(&sa_client);
2791         destroy_workqueue(cma_wq);
2792         idr_destroy(&sdp_ps);
2793         idr_destroy(&tcp_ps);
2794         idr_destroy(&udp_ps);
2795         idr_destroy(&ipoib_ps);
2796 }
2797
2798 module_init(cma_init);
2799 module_exit(cma_cleanup);