git.oblomov.eu Git - linux-2.6/blob - net/sunrpc/xprtrdma/verbs.c

   1 /*
   2  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the BSD-type
   8  * license below:
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  *      Redistributions of source code must retain the above copyright
  15  *      notice, this list of conditions and the following disclaimer.
  16  *
  17  *      Redistributions in binary form must reproduce the above
  18  *      copyright notice, this list of conditions and the following
  19  *      disclaimer in the documentation and/or other materials provided
  20  *      with the distribution.
  21  *
  22  *      Neither the name of the Network Appliance, Inc. nor the names of
  23  *      its contributors may be used to endorse or promote products
  24  *      derived from this software without specific prior written
  25  *      permission.
  26  *
  27  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  28  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  29  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  30  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  31  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  32  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  33  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  34  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  35  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  36  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  37  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  38  */
  39
  40 /*
  41  * verbs.c
  42  *
  43  * Encapsulates the major functions managing:
  44  *  o adapters
  45  *  o endpoints
  46  *  o connections
  47  *  o buffer memory
  48  */
  49
  50 #include <linux/pci.h>  /* for Tavor hack below */
  51
  52 #include "xprt_rdma.h"
  53
  54 /*
  55  * Globals/Macros
  56  */
  57
  58 #ifdef RPC_DEBUG
  59 # define RPCDBG_FACILITY        RPCDBG_TRANS
  60 #endif
  61
  62 /*
  63  * internal functions
  64  */
  65
  66 /*
  67  * handle replies in tasklet context, using a single, global list
  68  * rdma tasklet function -- just turn around and call the func
  69  * for all replies on the list
  70  */
  71
  72 static DEFINE_SPINLOCK(rpcrdma_tk_lock_g);
  73 static LIST_HEAD(rpcrdma_tasklets_g);
  74
  75 static void
  76 rpcrdma_run_tasklet(unsigned long data)
  77 {
  78         struct rpcrdma_rep *rep;
  79         void (*func)(struct rpcrdma_rep *);
  80         unsigned long flags;
  81
  82         data = data;
  83         spin_lock_irqsave(&rpcrdma_tk_lock_g, flags);
  84         while (!list_empty(&rpcrdma_tasklets_g)) {
  85                 rep = list_entry(rpcrdma_tasklets_g.next,
  86                                  struct rpcrdma_rep, rr_list);
  87                 list_del(&rep->rr_list);
  88                 func = rep->rr_func;
  89                 rep->rr_func = NULL;
  90                 spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags);
  91
  92                 if (func)
  93                         func(rep);
  94                 else
  95                         rpcrdma_recv_buffer_put(rep);
  96
  97                 spin_lock_irqsave(&rpcrdma_tk_lock_g, flags);
  98         }
  99         spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags);
 100 }
 101
 102 static DECLARE_TASKLET(rpcrdma_tasklet_g, rpcrdma_run_tasklet, 0UL);
 103
 104 static inline void
 105 rpcrdma_schedule_tasklet(struct rpcrdma_rep *rep)
 106 {
 107         unsigned long flags;
 108
 109         spin_lock_irqsave(&rpcrdma_tk_lock_g, flags);
 110         list_add_tail(&rep->rr_list, &rpcrdma_tasklets_g);
 111         spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags);
 112         tasklet_schedule(&rpcrdma_tasklet_g);
 113 }
 114
 115 static void
 116 rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context)
 117 {
 118         struct rpcrdma_ep *ep = context;
 119
 120         dprintk("RPC:       %s: QP error %X on device %s ep %p\n",
 121                 __func__, event->event, event->device->name, context);
 122         if (ep->rep_connected == 1) {
 123                 ep->rep_connected = -EIO;
 124                 ep->rep_func(ep);
 125                 wake_up_all(&ep->rep_connect_wait);
 126         }
 127 }
 128
 129 static void
 130 rpcrdma_cq_async_error_upcall(struct ib_event *event, void *context)
 131 {
 132         struct rpcrdma_ep *ep = context;
 133
 134         dprintk("RPC:       %s: CQ error %X on device %s ep %p\n",
 135                 __func__, event->event, event->device->name, context);
 136         if (ep->rep_connected == 1) {
 137                 ep->rep_connected = -EIO;
 138                 ep->rep_func(ep);
 139                 wake_up_all(&ep->rep_connect_wait);
 140         }
 141 }
 142
 143 static inline
 144 void rpcrdma_event_process(struct ib_wc *wc)
 145 {
 146         struct rpcrdma_rep *rep =
 147                         (struct rpcrdma_rep *)(unsigned long) wc->wr_id;
 148
 149         dprintk("RPC:       %s: event rep %p status %X opcode %X length %u\n",
 150                 __func__, rep, wc->status, wc->opcode, wc->byte_len);
 151
 152         if (!rep) /* send or bind completion that we don't care about */
 153                 return;
 154
 155         if (IB_WC_SUCCESS != wc->status) {
 156                 dprintk("RPC:       %s: %s WC status %X, connection lost\n",
 157                         __func__, (wc->opcode & IB_WC_RECV) ? "recv" : "send",
 158                          wc->status);
 159                 rep->rr_len = ~0U;
 160                 rpcrdma_schedule_tasklet(rep);
 161                 return;
 162         }
 163
 164         switch (wc->opcode) {
 165         case IB_WC_RECV:
 166                 rep->rr_len = wc->byte_len;
 167                 ib_dma_sync_single_for_cpu(
 168                         rdmab_to_ia(rep->rr_buffer)->ri_id->device,
 169                         rep->rr_iov.addr, rep->rr_len, DMA_FROM_DEVICE);
 170                 /* Keep (only) the most recent credits, after check validity */
 171                 if (rep->rr_len >= 16) {
 172                         struct rpcrdma_msg *p =
 173                                         (struct rpcrdma_msg *) rep->rr_base;
 174                         unsigned int credits = ntohl(p->rm_credit);
 175                         if (credits == 0) {
 176                                 dprintk("RPC:       %s: server"
 177                                         " dropped credits to 0!\n", __func__);
 178                                 /* don't deadlock */
 179                                 credits = 1;
 180                         } else if (credits > rep->rr_buffer->rb_max_requests) {
 181                                 dprintk("RPC:       %s: server"
 182                                         " over-crediting: %d (%d)\n",
 183                                         __func__, credits,
 184                                         rep->rr_buffer->rb_max_requests);
 185                                 credits = rep->rr_buffer->rb_max_requests;
 186                         }
 187                         atomic_set(&rep->rr_buffer->rb_credits, credits);
 188                 }
 189                 /* fall through */
 190         case IB_WC_BIND_MW:
 191                 rpcrdma_schedule_tasklet(rep);
 192                 break;
 193         default:
 194                 dprintk("RPC:       %s: unexpected WC event %X\n",
 195                         __func__, wc->opcode);
 196                 break;
 197         }
 198 }
 199
 200 static inline int
 201 rpcrdma_cq_poll(struct ib_cq *cq)
 202 {
 203         struct ib_wc wc;
 204         int rc;
 205
 206         for (;;) {
 207                 rc = ib_poll_cq(cq, 1, &wc);
 208                 if (rc < 0) {
 209                         dprintk("RPC:       %s: ib_poll_cq failed %i\n",
 210                                 __func__, rc);
 211                         return rc;
 212                 }
 213                 if (rc == 0)
 214                         break;
 215
 216                 rpcrdma_event_process(&wc);
 217         }
 218
 219         return 0;
 220 }
 221
 222 /*
 223  * rpcrdma_cq_event_upcall
 224  *
 225  * This upcall handles recv, send, bind and unbind events.
 226  * It is reentrant but processes single events in order to maintain
 227  * ordering of receives to keep server credits.
 228  *
 229  * It is the responsibility of the scheduled tasklet to return
 230  * recv buffers to the pool. NOTE: this affects synchronization of
 231  * connection shutdown. That is, the structures required for
 232  * the completion of the reply handler must remain intact until
 233  * all memory has been reclaimed.
 234  *
 235  * Note that send events are suppressed and do not result in an upcall.
 236  */
 237 static void
 238 rpcrdma_cq_event_upcall(struct ib_cq *cq, void *context)
 239 {
 240         int rc;
 241
 242         rc = rpcrdma_cq_poll(cq);
 243         if (rc)
 244                 return;
 245
 246         rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
 247         if (rc) {
 248                 dprintk("RPC:       %s: ib_req_notify_cq failed %i\n",
 249                         __func__, rc);
 250                 return;
 251         }
 252
 253         rpcrdma_cq_poll(cq);
 254 }
 255
 256 #ifdef RPC_DEBUG
 257 static const char * const conn[] = {
 258         "address resolved",
 259         "address error",
 260         "route resolved",
 261         "route error",
 262         "connect request",
 263         "connect response",
 264         "connect error",
 265         "unreachable",
 266         "rejected",
 267         "established",
 268         "disconnected",
 269         "device removal"
 270 };
 271 #endif
 272
 273 static int
 274 rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event)
 275 {
 276         struct rpcrdma_xprt *xprt = id->context;
 277         struct rpcrdma_ia *ia = &xprt->rx_ia;
 278         struct rpcrdma_ep *ep = &xprt->rx_ep;
 279         struct sockaddr_in *addr = (struct sockaddr_in *) &ep->rep_remote_addr;
 280         struct ib_qp_attr attr;
 281         struct ib_qp_init_attr iattr;
 282         int connstate = 0;
 283
 284         switch (event->event) {
 285         case RDMA_CM_EVENT_ADDR_RESOLVED:
 286         case RDMA_CM_EVENT_ROUTE_RESOLVED:
 287                 complete(&ia->ri_done);
 288                 break;
 289         case RDMA_CM_EVENT_ADDR_ERROR:
 290                 ia->ri_async_rc = -EHOSTUNREACH;
 291                 dprintk("RPC:       %s: CM address resolution error, ep 0x%p\n",
 292                         __func__, ep);
 293                 complete(&ia->ri_done);
 294                 break;
 295         case RDMA_CM_EVENT_ROUTE_ERROR:
 296                 ia->ri_async_rc = -ENETUNREACH;
 297                 dprintk("RPC:       %s: CM route resolution error, ep 0x%p\n",
 298                         __func__, ep);
 299                 complete(&ia->ri_done);
 300                 break;
 301         case RDMA_CM_EVENT_ESTABLISHED:
 302                 connstate = 1;
 303                 ib_query_qp(ia->ri_id->qp, &attr,
 304                         IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC,
 305                         &iattr);
 306                 dprintk("RPC:       %s: %d responder resources"
 307                         " (%d initiator)\n",
 308                         __func__, attr.max_dest_rd_atomic, attr.max_rd_atomic);
 309                 goto connected;
 310         case RDMA_CM_EVENT_CONNECT_ERROR:
 311                 connstate = -ENOTCONN;
 312                 goto connected;
 313         case RDMA_CM_EVENT_UNREACHABLE:
 314                 connstate = -ENETDOWN;
 315                 goto connected;
 316         case RDMA_CM_EVENT_REJECTED:
 317                 connstate = -ECONNREFUSED;
 318                 goto connected;
 319         case RDMA_CM_EVENT_DISCONNECTED:
 320                 connstate = -ECONNABORTED;
 321                 goto connected;
 322         case RDMA_CM_EVENT_DEVICE_REMOVAL:
 323                 connstate = -ENODEV;
 324 connected:
 325                 dprintk("RPC:       %s: %s: %u.%u.%u.%u:%u"
 326                         " (ep 0x%p event 0x%x)\n",
 327                         __func__,
 328                         (event->event <= 11) ? conn[event->event] :
 329                                                 "unknown connection error",
 330                         NIPQUAD(addr->sin_addr.s_addr),
 331                         ntohs(addr->sin_port),
 332                         ep, event->event);
 333                 atomic_set(&rpcx_to_rdmax(ep->rep_xprt)->rx_buf.rb_credits, 1);
 334                 dprintk("RPC:       %s: %sconnected\n",
 335                                         __func__, connstate > 0 ? "" : "dis");
 336                 ep->rep_connected = connstate;
 337                 ep->rep_func(ep);
 338                 wake_up_all(&ep->rep_connect_wait);
 339                 break;
 340         default:
 341                 ia->ri_async_rc = -EINVAL;
 342                 dprintk("RPC:       %s: unexpected CM event %X\n",
 343                         __func__, event->event);
 344                 complete(&ia->ri_done);
 345                 break;
 346         }
 347
 348         return 0;
 349 }
 350
 351 static struct rdma_cm_id *
 352 rpcrdma_create_id(struct rpcrdma_xprt *xprt,
 353                         struct rpcrdma_ia *ia, struct sockaddr *addr)
 354 {
 355         struct rdma_cm_id *id;
 356         int rc;
 357
 358         id = rdma_create_id(rpcrdma_conn_upcall, xprt, RDMA_PS_TCP);
 359         if (IS_ERR(id)) {
 360                 rc = PTR_ERR(id);
 361                 dprintk("RPC:       %s: rdma_create_id() failed %i\n",
 362                         __func__, rc);
 363                 return id;
 364         }
 365
 366         ia->ri_async_rc = 0;
 367         rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT);
 368         if (rc) {
 369                 dprintk("RPC:       %s: rdma_resolve_addr() failed %i\n",
 370                         __func__, rc);
 371                 goto out;
 372         }
 373         wait_for_completion(&ia->ri_done);
 374         rc = ia->ri_async_rc;
 375         if (rc)
 376                 goto out;
 377
 378         ia->ri_async_rc = 0;
 379         rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT);
 380         if (rc) {
 381                 dprintk("RPC:       %s: rdma_resolve_route() failed %i\n",
 382                         __func__, rc);
 383                 goto out;
 384         }
 385         wait_for_completion(&ia->ri_done);
 386         rc = ia->ri_async_rc;
 387         if (rc)
 388                 goto out;
 389
 390         return id;
 391
 392 out:
 393         rdma_destroy_id(id);
 394         return ERR_PTR(rc);
 395 }
 396
 397 /*
 398  * Drain any cq, prior to teardown.
 399  */
 400 static void
 401 rpcrdma_clean_cq(struct ib_cq *cq)
 402 {
 403         struct ib_wc wc;
 404         int count = 0;
 405
 406         while (1 == ib_poll_cq(cq, 1, &wc))
 407                 ++count;
 408
 409         if (count)
 410                 dprintk("RPC:       %s: flushed %d events (last 0x%x)\n",
 411                         __func__, count, wc.opcode);
 412 }
 413
 414 /*
 415  * Exported functions.
 416  */
 417
 418 /*
 419  * Open and initialize an Interface Adapter.
 420  *  o initializes fields of struct rpcrdma_ia, including
 421  *    interface and provider attributes and protection zone.
 422  */
 423 int
 424 rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
 425 {
 426         int rc, mem_priv;
 427         struct ib_device_attr devattr;
 428         struct rpcrdma_ia *ia = &xprt->rx_ia;
 429
 430         init_completion(&ia->ri_done);
 431
 432         ia->ri_id = rpcrdma_create_id(xprt, ia, addr);
 433         if (IS_ERR(ia->ri_id)) {
 434                 rc = PTR_ERR(ia->ri_id);
 435                 goto out1;
 436         }
 437
 438         ia->ri_pd = ib_alloc_pd(ia->ri_id->device);
 439         if (IS_ERR(ia->ri_pd)) {
 440                 rc = PTR_ERR(ia->ri_pd);
 441                 dprintk("RPC:       %s: ib_alloc_pd() failed %i\n",
 442                         __func__, rc);
 443                 goto out2;
 444         }
 445
 446         /*
 447          * Query the device to determine if the requested memory
 448          * registration strategy is supported. If it isn't, set the
 449          * strategy to a globally supported model.
 450          */
 451         rc = ib_query_device(ia->ri_id->device, &devattr);
 452         if (rc) {
 453                 dprintk("RPC:       %s: ib_query_device failed %d\n",
 454                         __func__, rc);
 455                 goto out2;
 456         }
 457
 458         if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) {
 459                 ia->ri_have_dma_lkey = 1;
 460                 ia->ri_dma_lkey = ia->ri_id->device->local_dma_lkey;
 461         }
 462
 463         switch (memreg) {
 464         case RPCRDMA_MEMWINDOWS:
 465         case RPCRDMA_MEMWINDOWS_ASYNC:
 466                 if (!(devattr.device_cap_flags & IB_DEVICE_MEM_WINDOW)) {
 467                         dprintk("RPC:       %s: MEMWINDOWS registration "
 468                                 "specified but not supported by adapter, "
 469                                 "using slower RPCRDMA_REGISTER\n",
 470                                 __func__);
 471                         memreg = RPCRDMA_REGISTER;
 472                 }
 473                 break;
 474         case RPCRDMA_MTHCAFMR:
 475                 if (!ia->ri_id->device->alloc_fmr) {
 476 #if RPCRDMA_PERSISTENT_REGISTRATION
 477                         dprintk("RPC:       %s: MTHCAFMR registration "
 478                                 "specified but not supported by adapter, "
 479                                 "using riskier RPCRDMA_ALLPHYSICAL\n",
 480                                 __func__);
 481                         memreg = RPCRDMA_ALLPHYSICAL;
 482 #else
 483                         dprintk("RPC:       %s: MTHCAFMR registration "
 484                                 "specified but not supported by adapter, "
 485                                 "using slower RPCRDMA_REGISTER\n",
 486                                 __func__);
 487                         memreg = RPCRDMA_REGISTER;
 488 #endif
 489                 }
 490                 break;
 491         case RPCRDMA_FRMR:
 492                 /* Requires both frmr reg and local dma lkey */
 493                 if ((devattr.device_cap_flags &
 494                      (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) !=
 495                     (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) {
 496 #if RPCRDMA_PERSISTENT_REGISTRATION
 497                         dprintk("RPC:       %s: FRMR registration "
 498                                 "specified but not supported by adapter, "
 499                                 "using riskier RPCRDMA_ALLPHYSICAL\n",
 500                                 __func__);
 501                         memreg = RPCRDMA_ALLPHYSICAL;
 502 #else
 503                         dprintk("RPC:       %s: FRMR registration "
 504                                 "specified but not supported by adapter, "
 505                                 "using slower RPCRDMA_REGISTER\n",
 506                                 __func__);
 507                         memreg = RPCRDMA_REGISTER;
 508 #endif
 509                 }
 510                 break;
 511         }
 512
 513         /*
 514          * Optionally obtain an underlying physical identity mapping in
 515          * order to do a memory window-based bind. This base registration
 516          * is protected from remote access - that is enabled only by binding
 517          * for the specific bytes targeted during each RPC operation, and
 518          * revoked after the corresponding completion similar to a storage
 519          * adapter.
 520          */
 521         switch (memreg) {
 522         case RPCRDMA_BOUNCEBUFFERS:
 523         case RPCRDMA_REGISTER:
 524         case RPCRDMA_FRMR:
 525                 break;
 526 #if RPCRDMA_PERSISTENT_REGISTRATION
 527         case RPCRDMA_ALLPHYSICAL:
 528                 mem_priv = IB_ACCESS_LOCAL_WRITE |
 529                                 IB_ACCESS_REMOTE_WRITE |
 530                                 IB_ACCESS_REMOTE_READ;
 531                 goto register_setup;
 532 #endif
 533         case RPCRDMA_MEMWINDOWS_ASYNC:
 534         case RPCRDMA_MEMWINDOWS:
 535                 mem_priv = IB_ACCESS_LOCAL_WRITE |
 536                                 IB_ACCESS_MW_BIND;
 537                 goto register_setup;
 538         case RPCRDMA_MTHCAFMR:
 539                 if (ia->ri_have_dma_lkey)
 540                         break;
 541                 mem_priv = IB_ACCESS_LOCAL_WRITE;
 542         register_setup:
 543                 ia->ri_bind_mem = ib_get_dma_mr(ia->ri_pd, mem_priv);
 544                 if (IS_ERR(ia->ri_bind_mem)) {
 545                         printk(KERN_ALERT "%s: ib_get_dma_mr for "
 546                                 "phys register failed with %lX\n\t"
 547                                 "Will continue with degraded performance\n",
 548                                 __func__, PTR_ERR(ia->ri_bind_mem));
 549                         memreg = RPCRDMA_REGISTER;
 550                         ia->ri_bind_mem = NULL;
 551                 }
 552                 break;
 553         default:
 554                 printk(KERN_ERR "%s: invalid memory registration mode %d\n",
 555                                 __func__, memreg);
 556                 rc = -EINVAL;
 557                 goto out2;
 558         }
 559         dprintk("RPC:       %s: memory registration strategy is %d\n",
 560                 __func__, memreg);
 561
 562         /* Else will do memory reg/dereg for each chunk */
 563         ia->ri_memreg_strategy = memreg;
 564
 565         return 0;
 566 out2:
 567         rdma_destroy_id(ia->ri_id);
 568 out1:
 569         return rc;
 570 }
 571
 572 /*
 573  * Clean up/close an IA.
 574  *   o if event handles and PD have been initialized, free them.
 575  *   o close the IA
 576  */
 577 void
 578 rpcrdma_ia_close(struct rpcrdma_ia *ia)
 579 {
 580         int rc;
 581
 582         dprintk("RPC:       %s: entering\n", __func__);
 583         if (ia->ri_bind_mem != NULL) {
 584                 rc = ib_dereg_mr(ia->ri_bind_mem);
 585                 dprintk("RPC:       %s: ib_dereg_mr returned %i\n",
 586                         __func__, rc);
 587         }
 588         if (ia->ri_id != NULL && !IS_ERR(ia->ri_id) && ia->ri_id->qp)
 589                 rdma_destroy_qp(ia->ri_id);
 590         if (ia->ri_pd != NULL && !IS_ERR(ia->ri_pd)) {
 591                 rc = ib_dealloc_pd(ia->ri_pd);
 592                 dprintk("RPC:       %s: ib_dealloc_pd returned %i\n",
 593                         __func__, rc);
 594         }
 595         if (ia->ri_id != NULL && !IS_ERR(ia->ri_id))
 596                 rdma_destroy_id(ia->ri_id);
 597 }
 598
 599 /*
 600  * Create unconnected endpoint.
 601  */
 602 int
 603 rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
 604                                 struct rpcrdma_create_data_internal *cdata)
 605 {
 606         struct ib_device_attr devattr;
 607         int rc, err;
 608
 609         rc = ib_query_device(ia->ri_id->device, &devattr);
 610         if (rc) {
 611                 dprintk("RPC:       %s: ib_query_device failed %d\n",
 612                         __func__, rc);
 613                 return rc;
 614         }
 615
 616         /* check provider's send/recv wr limits */
 617         if (cdata->max_requests > devattr.max_qp_wr)
 618                 cdata->max_requests = devattr.max_qp_wr;
 619
 620         ep->rep_attr.event_handler = rpcrdma_qp_async_error_upcall;
 621         ep->rep_attr.qp_context = ep;
 622         /* send_cq and recv_cq initialized below */
 623         ep->rep_attr.srq = NULL;
 624         ep->rep_attr.cap.max_send_wr = cdata->max_requests;
 625         switch (ia->ri_memreg_strategy) {
 626         case RPCRDMA_FRMR:
 627                 /* Add room for frmr register and invalidate WRs */
 628                 ep->rep_attr.cap.max_send_wr *= 3;
 629                 if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr)
 630                         return -EINVAL;
 631                 break;
 632         case RPCRDMA_MEMWINDOWS_ASYNC:
 633         case RPCRDMA_MEMWINDOWS:
 634                 /* Add room for mw_binds+unbinds - overkill! */
 635                 ep->rep_attr.cap.max_send_wr++;
 636                 ep->rep_attr.cap.max_send_wr *= (2 * RPCRDMA_MAX_SEGS);
 637                 if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr)
 638                         return -EINVAL;
 639                 break;
 640         default:
 641                 break;
 642         }
 643         ep->rep_attr.cap.max_recv_wr = cdata->max_requests;
 644         ep->rep_attr.cap.max_send_sge = (cdata->padding ? 4 : 2);
 645         ep->rep_attr.cap.max_recv_sge = 1;
 646         ep->rep_attr.cap.max_inline_data = 0;
 647         ep->rep_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 648         ep->rep_attr.qp_type = IB_QPT_RC;
 649         ep->rep_attr.port_num = ~0;
 650
 651         dprintk("RPC:       %s: requested max: dtos: send %d recv %d; "
 652                 "iovs: send %d recv %d\n",
 653                 __func__,
 654                 ep->rep_attr.cap.max_send_wr,
 655                 ep->rep_attr.cap.max_recv_wr,
 656                 ep->rep_attr.cap.max_send_sge,
 657                 ep->rep_attr.cap.max_recv_sge);
 658
 659         /* set trigger for requesting send completion */
 660         ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 /*  - 1*/;
 661         switch (ia->ri_memreg_strategy) {
 662         case RPCRDMA_MEMWINDOWS_ASYNC:
 663         case RPCRDMA_MEMWINDOWS:
 664                 ep->rep_cqinit -= RPCRDMA_MAX_SEGS;
 665                 break;
 666         default:
 667                 break;
 668         }
 669         if (ep->rep_cqinit <= 2)
 670                 ep->rep_cqinit = 0;
 671         INIT_CQCOUNT(ep);
 672         ep->rep_ia = ia;
 673         init_waitqueue_head(&ep->rep_connect_wait);
 674
 675         /*
 676          * Create a single cq for receive dto and mw_bind (only ever
 677          * care about unbind, really). Send completions are suppressed.
 678          * Use single threaded tasklet upcalls to maintain ordering.
 679          */
 680         ep->rep_cq = ib_create_cq(ia->ri_id->device, rpcrdma_cq_event_upcall,
 681                                   rpcrdma_cq_async_error_upcall, NULL,
 682                                   ep->rep_attr.cap.max_recv_wr +
 683                                   ep->rep_attr.cap.max_send_wr + 1, 0);
 684         if (IS_ERR(ep->rep_cq)) {
 685                 rc = PTR_ERR(ep->rep_cq);
 686                 dprintk("RPC:       %s: ib_create_cq failed: %i\n",
 687                         __func__, rc);
 688                 goto out1;
 689         }
 690
 691         rc = ib_req_notify_cq(ep->rep_cq, IB_CQ_NEXT_COMP);
 692         if (rc) {
 693                 dprintk("RPC:       %s: ib_req_notify_cq failed: %i\n",
 694                         __func__, rc);
 695                 goto out2;
 696         }
 697
 698         ep->rep_attr.send_cq = ep->rep_cq;
 699         ep->rep_attr.recv_cq = ep->rep_cq;
 700
 701         /* Initialize cma parameters */
 702
 703         /* RPC/RDMA does not use private data */
 704         ep->rep_remote_cma.private_data = NULL;
 705         ep->rep_remote_cma.private_data_len = 0;
 706
 707         /* Client offers RDMA Read but does not initiate */
 708         switch (ia->ri_memreg_strategy) {
 709         case RPCRDMA_BOUNCEBUFFERS:
 710                 ep->rep_remote_cma.responder_resources = 0;
 711                 break;
 712         case RPCRDMA_MTHCAFMR:
 713         case RPCRDMA_REGISTER:
 714         case RPCRDMA_FRMR:
 715                 ep->rep_remote_cma.responder_resources = cdata->max_requests *
 716                                 (RPCRDMA_MAX_DATA_SEGS / 8);
 717                 break;
 718         case RPCRDMA_MEMWINDOWS:
 719         case RPCRDMA_MEMWINDOWS_ASYNC:
 720 #if RPCRDMA_PERSISTENT_REGISTRATION
 721         case RPCRDMA_ALLPHYSICAL:
 722 #endif
 723                 ep->rep_remote_cma.responder_resources = cdata->max_requests *
 724                                 (RPCRDMA_MAX_DATA_SEGS / 2);
 725                 break;
 726         default:
 727                 break;
 728         }
 729         if (ep->rep_remote_cma.responder_resources > devattr.max_qp_rd_atom)
 730                 ep->rep_remote_cma.responder_resources = devattr.max_qp_rd_atom;
 731         ep->rep_remote_cma.initiator_depth = 0;
 732
 733         ep->rep_remote_cma.retry_count = 7;
 734         ep->rep_remote_cma.flow_control = 0;
 735         ep->rep_remote_cma.rnr_retry_count = 0;
 736
 737         return 0;
 738
 739 out2:
 740         err = ib_destroy_cq(ep->rep_cq);
 741         if (err)
 742                 dprintk("RPC:       %s: ib_destroy_cq returned %i\n",
 743                         __func__, err);
 744 out1:
 745         return rc;
 746 }
 747
 748 /*
 749  * rpcrdma_ep_destroy
 750  *
 751  * Disconnect and destroy endpoint. After this, the only
 752  * valid operations on the ep are to free it (if dynamically
 753  * allocated) or re-create it.
 754  *
 755  * The caller's error handling must be sure to not leak the endpoint
 756  * if this function fails.
 757  */
 758 int
 759 rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
 760 {
 761         int rc;
 762
 763         dprintk("RPC:       %s: entering, connected is %d\n",
 764                 __func__, ep->rep_connected);
 765
 766         if (ia->ri_id->qp) {
 767                 rc = rpcrdma_ep_disconnect(ep, ia);
 768                 if (rc)
 769                         dprintk("RPC:       %s: rpcrdma_ep_disconnect"
 770                                 " returned %i\n", __func__, rc);
 771         }
 772
 773         ep->rep_func = NULL;
 774
 775         /* padding - could be done in rpcrdma_buffer_destroy... */
 776         if (ep->rep_pad_mr) {
 777                 rpcrdma_deregister_internal(ia, ep->rep_pad_mr, &ep->rep_pad);
 778                 ep->rep_pad_mr = NULL;
 779         }
 780
 781         if (ia->ri_id->qp) {
 782                 rdma_destroy_qp(ia->ri_id);
 783                 ia->ri_id->qp = NULL;
 784         }
 785
 786         rpcrdma_clean_cq(ep->rep_cq);
 787         rc = ib_destroy_cq(ep->rep_cq);
 788         if (rc)
 789                 dprintk("RPC:       %s: ib_destroy_cq returned %i\n",
 790                         __func__, rc);
 791
 792         return rc;
 793 }
 794
 795 /*
 796  * Connect unconnected endpoint.
 797  */
 798 int
 799 rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
 800 {
 801         struct rdma_cm_id *id;
 802         int rc = 0;
 803         int retry_count = 0;
 804         int reconnect = (ep->rep_connected != 0);
 805
 806         if (reconnect) {
 807                 struct rpcrdma_xprt *xprt;
 808 retry:
 809                 rc = rpcrdma_ep_disconnect(ep, ia);
 810                 if (rc && rc != -ENOTCONN)
 811                         dprintk("RPC:       %s: rpcrdma_ep_disconnect"
 812                                 " status %i\n", __func__, rc);
 813                 rpcrdma_clean_cq(ep->rep_cq);
 814
 815                 xprt = container_of(ia, struct rpcrdma_xprt, rx_ia);
 816                 id = rpcrdma_create_id(xprt, ia,
 817                                 (struct sockaddr *)&xprt->rx_data.addr);
 818                 if (IS_ERR(id)) {
 819                         rc = PTR_ERR(id);
 820                         goto out;
 821                 }
 822                 /* TEMP TEMP TEMP - fail if new device:
 823                  * Deregister/remarshal *all* requests!
 824                  * Close and recreate adapter, pd, etc!
 825                  * Re-determine all attributes still sane!
 826                  * More stuff I haven't thought of!
 827                  * Rrrgh!
 828                  */
 829                 if (ia->ri_id->device != id->device) {
 830                         printk("RPC:       %s: can't reconnect on "
 831                                 "different device!\n", __func__);
 832                         rdma_destroy_id(id);
 833                         rc = -ENETDOWN;
 834                         goto out;
 835                 }
 836                 /* END TEMP */
 837                 rdma_destroy_id(ia->ri_id);
 838                 ia->ri_id = id;
 839         }
 840
 841         rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr);
 842         if (rc) {
 843                 dprintk("RPC:       %s: rdma_create_qp failed %i\n",
 844                         __func__, rc);
 845                 goto out;
 846         }
 847
 848 /* XXX Tavor device performs badly with 2K MTU! */
 849 if (strnicmp(ia->ri_id->device->dma_device->bus->name, "pci", 3) == 0) {
 850         struct pci_dev *pcid = to_pci_dev(ia->ri_id->device->dma_device);
 851         if (pcid->device == PCI_DEVICE_ID_MELLANOX_TAVOR &&
 852             (pcid->vendor == PCI_VENDOR_ID_MELLANOX ||
 853              pcid->vendor == PCI_VENDOR_ID_TOPSPIN)) {
 854                 struct ib_qp_attr attr = {
 855                         .path_mtu = IB_MTU_1024
 856                 };
 857                 rc = ib_modify_qp(ia->ri_id->qp, &attr, IB_QP_PATH_MTU);
 858         }
 859 }
 860
 861         /* Theoretically a client initiator_depth > 0 is not needed,
 862          * but many peers fail to complete the connection unless they
 863          * == responder_resources! */
 864         if (ep->rep_remote_cma.initiator_depth !=
 865                                 ep->rep_remote_cma.responder_resources)
 866                 ep->rep_remote_cma.initiator_depth =
 867                         ep->rep_remote_cma.responder_resources;
 868
 869         ep->rep_connected = 0;
 870
 871         rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma);
 872         if (rc) {
 873                 dprintk("RPC:       %s: rdma_connect() failed with %i\n",
 874                                 __func__, rc);
 875                 goto out;
 876         }
 877
 878         if (reconnect)
 879                 return 0;
 880
 881         wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 0);
 882
 883         /*
 884          * Check state. A non-peer reject indicates no listener
 885          * (ECONNREFUSED), which may be a transient state. All
 886          * others indicate a transport condition which has already
 887          * undergone a best-effort.
 888          */
 889         if (ep->rep_connected == -ECONNREFUSED
 890             && ++retry_count <= RDMA_CONNECT_RETRY_MAX) {
 891                 dprintk("RPC:       %s: non-peer_reject, retry\n", __func__);
 892                 goto retry;
 893         }
 894         if (ep->rep_connected <= 0) {
 895                 /* Sometimes, the only way to reliably connect to remote
 896                  * CMs is to use same nonzero values for ORD and IRD. */
 897                 ep->rep_remote_cma.initiator_depth =
 898                                         ep->rep_remote_cma.responder_resources;
 899                 if (ep->rep_remote_cma.initiator_depth == 0)
 900                         ++ep->rep_remote_cma.initiator_depth;
 901                 if (ep->rep_remote_cma.responder_resources == 0)
 902                         ++ep->rep_remote_cma.responder_resources;
 903                 if (retry_count++ == 0)
 904                         goto retry;
 905                 rc = ep->rep_connected;
 906         } else {
 907                 dprintk("RPC:       %s: connected\n", __func__);
 908         }
 909
 910 out:
 911         if (rc)
 912                 ep->rep_connected = rc;
 913         return rc;
 914 }
 915
 916 /*
 917  * rpcrdma_ep_disconnect
 918  *
 919  * This is separate from destroy to facilitate the ability
 920  * to reconnect without recreating the endpoint.
 921  *
 922  * This call is not reentrant, and must not be made in parallel
 923  * on the same endpoint.
 924  */
 925 int
 926 rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
 927 {
 928         int rc;
 929
 930         rpcrdma_clean_cq(ep->rep_cq);
 931         rc = rdma_disconnect(ia->ri_id);
 932         if (!rc) {
 933                 /* returns without wait if not connected */
 934                 wait_event_interruptible(ep->rep_connect_wait,
 935                                                         ep->rep_connected != 1);
 936                 dprintk("RPC:       %s: after wait, %sconnected\n", __func__,
 937                         (ep->rep_connected == 1) ? "still " : "dis");
 938         } else {
 939                 dprintk("RPC:       %s: rdma_disconnect %i\n", __func__, rc);
 940                 ep->rep_connected = rc;
 941         }
 942         return rc;
 943 }
 944
 945 /*
 946  * Initialize buffer memory
 947  */
 948 int
 949 rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep,
 950         struct rpcrdma_ia *ia, struct rpcrdma_create_data_internal *cdata)
 951 {
 952         char *p;
 953         size_t len;
 954         int i, rc;
 955         struct rpcrdma_mw *r;
 956
 957         buf->rb_max_requests = cdata->max_requests;
 958         spin_lock_init(&buf->rb_lock);
 959         atomic_set(&buf->rb_credits, 1);
 960
 961         /* Need to allocate:
 962          *   1.  arrays for send and recv pointers
 963          *   2.  arrays of struct rpcrdma_req to fill in pointers
 964          *   3.  array of struct rpcrdma_rep for replies
 965          *   4.  padding, if any
 966          *   5.  mw's, fmr's or frmr's, if any
 967          * Send/recv buffers in req/rep need to be registered
 968          */
 969
 970         len = buf->rb_max_requests *
 971                 (sizeof(struct rpcrdma_req *) + sizeof(struct rpcrdma_rep *));
 972         len += cdata->padding;
 973         switch (ia->ri_memreg_strategy) {
 974         case RPCRDMA_FRMR:
 975                 len += buf->rb_max_requests * RPCRDMA_MAX_SEGS *
 976                                 sizeof(struct rpcrdma_mw);
 977                 break;
 978         case RPCRDMA_MTHCAFMR:
 979                 /* TBD we are perhaps overallocating here */
 980                 len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS *
 981                                 sizeof(struct rpcrdma_mw);
 982                 break;
 983         case RPCRDMA_MEMWINDOWS_ASYNC:
 984         case RPCRDMA_MEMWINDOWS:
 985                 len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS *
 986                                 sizeof(struct rpcrdma_mw);
 987                 break;
 988         default:
 989                 break;
 990         }
 991
 992         /* allocate 1, 4 and 5 in one shot */
 993         p = kzalloc(len, GFP_KERNEL);
 994         if (p == NULL) {
 995                 dprintk("RPC:       %s: req_t/rep_t/pad kzalloc(%zd) failed\n",
 996                         __func__, len);
 997                 rc = -ENOMEM;
 998                 goto out;
 999         }
1000         buf->rb_pool = p;       /* for freeing it later */
1001
1002         buf->rb_send_bufs = (struct rpcrdma_req **) p;
1003         p = (char *) &buf->rb_send_bufs[buf->rb_max_requests];
1004         buf->rb_recv_bufs = (struct rpcrdma_rep **) p;
1005         p = (char *) &buf->rb_recv_bufs[buf->rb_max_requests];
1006
1007         /*
1008          * Register the zeroed pad buffer, if any.
1009          */
1010         if (cdata->padding) {
1011                 rc = rpcrdma_register_internal(ia, p, cdata->padding,
1012                                             &ep->rep_pad_mr, &ep->rep_pad);
1013                 if (rc)
1014                         goto out;
1015         }
1016         p += cdata->padding;
1017
1018         /*
1019          * Allocate the fmr's, or mw's for mw_bind chunk registration.
1020          * We "cycle" the mw's in order to minimize rkey reuse,
1021          * and also reduce unbind-to-bind collision.
1022          */
1023         INIT_LIST_HEAD(&buf->rb_mws);
1024         r = (struct rpcrdma_mw *)p;
1025         switch (ia->ri_memreg_strategy) {
1026         case RPCRDMA_FRMR:
1027                 for (i = buf->rb_max_requests * RPCRDMA_MAX_SEGS; i; i--) {
1028                         r->r.frmr.fr_mr = ib_alloc_fast_reg_mr(ia->ri_pd,
1029                                                          RPCRDMA_MAX_SEGS);
1030                         if (IS_ERR(r->r.frmr.fr_mr)) {
1031                                 rc = PTR_ERR(r->r.frmr.fr_mr);
1032                                 dprintk("RPC:       %s: ib_alloc_fast_reg_mr"
1033                                         " failed %i\n", __func__, rc);
1034                                 goto out;
1035                         }
1036                         r->r.frmr.fr_pgl =
1037                                 ib_alloc_fast_reg_page_list(ia->ri_id->device,
1038                                                             RPCRDMA_MAX_SEGS);
1039                         if (IS_ERR(r->r.frmr.fr_pgl)) {
1040                                 rc = PTR_ERR(r->r.frmr.fr_pgl);
1041                                 dprintk("RPC:       %s: "
1042                                         "ib_alloc_fast_reg_page_list "
1043                                         "failed %i\n", __func__, rc);
1044                                 goto out;
1045                         }
1046                         list_add(&r->mw_list, &buf->rb_mws);
1047                         ++r;
1048                 }
1049                 break;
1050         case RPCRDMA_MTHCAFMR:
1051                 /* TBD we are perhaps overallocating here */
1052                 for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {
1053                         static struct ib_fmr_attr fa =
1054                                 { RPCRDMA_MAX_DATA_SEGS, 1, PAGE_SHIFT };
1055                         r->r.fmr = ib_alloc_fmr(ia->ri_pd,
1056                                 IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ,
1057                                 &fa);
1058                         if (IS_ERR(r->r.fmr)) {
1059                                 rc = PTR_ERR(r->r.fmr);
1060                                 dprintk("RPC:       %s: ib_alloc_fmr"
1061                                         " failed %i\n", __func__, rc);
1062                                 goto out;
1063                         }
1064                         list_add(&r->mw_list, &buf->rb_mws);
1065                         ++r;
1066                 }
1067                 break;
1068         case RPCRDMA_MEMWINDOWS_ASYNC:
1069         case RPCRDMA_MEMWINDOWS:
1070                 /* Allocate one extra request's worth, for full cycling */
1071                 for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {
1072                         r->r.mw = ib_alloc_mw(ia->ri_pd);
1073                         if (IS_ERR(r->r.mw)) {
1074                                 rc = PTR_ERR(r->r.mw);
1075                                 dprintk("RPC:       %s: ib_alloc_mw"
1076                                         " failed %i\n", __func__, rc);
1077                                 goto out;
1078                         }
1079                         list_add(&r->mw_list, &buf->rb_mws);
1080                         ++r;
1081                 }
1082                 break;
1083         default:
1084                 break;
1085         }
1086
1087         /*
1088          * Allocate/init the request/reply buffers. Doing this
1089          * using kmalloc for now -- one for each buf.
1090          */
1091         for (i = 0; i < buf->rb_max_requests; i++) {
1092                 struct rpcrdma_req *req;
1093                 struct rpcrdma_rep *rep;
1094
1095                 len = cdata->inline_wsize + sizeof(struct rpcrdma_req);
1096                 /* RPC layer requests *double* size + 1K RPC_SLACK_SPACE! */
1097                 /* Typical ~2400b, so rounding up saves work later */
1098                 if (len < 4096)
1099                         len = 4096;
1100                 req = kmalloc(len, GFP_KERNEL);
1101                 if (req == NULL) {
1102                         dprintk("RPC:       %s: request buffer %d alloc"
1103                                 " failed\n", __func__, i);
1104                         rc = -ENOMEM;
1105                         goto out;
1106                 }
1107                 memset(req, 0, sizeof(struct rpcrdma_req));
1108                 buf->rb_send_bufs[i] = req;
1109                 buf->rb_send_bufs[i]->rl_buffer = buf;
1110
1111                 rc = rpcrdma_register_internal(ia, req->rl_base,
1112                                 len - offsetof(struct rpcrdma_req, rl_base),
1113                                 &buf->rb_send_bufs[i]->rl_handle,
1114                                 &buf->rb_send_bufs[i]->rl_iov);
1115                 if (rc)
1116                         goto out;
1117
1118                 buf->rb_send_bufs[i]->rl_size = len-sizeof(struct rpcrdma_req);
1119
1120                 len = cdata->inline_rsize + sizeof(struct rpcrdma_rep);
1121                 rep = kmalloc(len, GFP_KERNEL);
1122                 if (rep == NULL) {
1123                         dprintk("RPC:       %s: reply buffer %d alloc failed\n",
1124                                 __func__, i);
1125                         rc = -ENOMEM;
1126                         goto out;
1127                 }
1128                 memset(rep, 0, sizeof(struct rpcrdma_rep));
1129                 buf->rb_recv_bufs[i] = rep;
1130                 buf->rb_recv_bufs[i]->rr_buffer = buf;
1131                 init_waitqueue_head(&rep->rr_unbind);
1132
1133                 rc = rpcrdma_register_internal(ia, rep->rr_base,
1134                                 len - offsetof(struct rpcrdma_rep, rr_base),
1135                                 &buf->rb_recv_bufs[i]->rr_handle,
1136                                 &buf->rb_recv_bufs[i]->rr_iov);
1137                 if (rc)
1138                         goto out;
1139
1140         }
1141         dprintk("RPC:       %s: max_requests %d\n",
1142                 __func__, buf->rb_max_requests);
1143         /* done */
1144         return 0;
1145 out:
1146         rpcrdma_buffer_destroy(buf);
1147         return rc;
1148 }
1149
1150 /*
1151  * Unregister and destroy buffer memory. Need to deal with
1152  * partial initialization, so it's callable from failed create.
1153  * Must be called before destroying endpoint, as registrations
1154  * reference it.
1155  */
1156 void
1157 rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
1158 {
1159         int rc, i;
1160         struct rpcrdma_ia *ia = rdmab_to_ia(buf);
1161         struct rpcrdma_mw *r;
1162
1163         /* clean up in reverse order from create
1164          *   1.  recv mr memory (mr free, then kfree)
1165          *   1a. bind mw memory
1166          *   2.  send mr memory (mr free, then kfree)
1167          *   3.  padding (if any) [moved to rpcrdma_ep_destroy]
1168          *   4.  arrays
1169          */
1170         dprintk("RPC:       %s: entering\n", __func__);
1171
1172         for (i = 0; i < buf->rb_max_requests; i++) {
1173                 if (buf->rb_recv_bufs && buf->rb_recv_bufs[i]) {
1174                         rpcrdma_deregister_internal(ia,
1175                                         buf->rb_recv_bufs[i]->rr_handle,
1176                                         &buf->rb_recv_bufs[i]->rr_iov);
1177                         kfree(buf->rb_recv_bufs[i]);
1178                 }
1179                 if (buf->rb_send_bufs && buf->rb_send_bufs[i]) {
1180                         while (!list_empty(&buf->rb_mws)) {
1181                                 r = list_entry(buf->rb_mws.next,
1182                                         struct rpcrdma_mw, mw_list);
1183                                 list_del(&r->mw_list);
1184                                 switch (ia->ri_memreg_strategy) {
1185                                 case RPCRDMA_FRMR:
1186                                         rc = ib_dereg_mr(r->r.frmr.fr_mr);
1187                                         if (rc)
1188                                                 dprintk("RPC:       %s:"
1189                                                         " ib_dereg_mr"
1190                                                         " failed %i\n",
1191                                                         __func__, rc);
1192                                         ib_free_fast_reg_page_list(r->r.frmr.fr_pgl);
1193                                         break;
1194                                 case RPCRDMA_MTHCAFMR:
1195                                         rc = ib_dealloc_fmr(r->r.fmr);
1196                                         if (rc)
1197                                                 dprintk("RPC:       %s:"
1198                                                         " ib_dealloc_fmr"
1199                                                         " failed %i\n",
1200                                                         __func__, rc);
1201                                         break;
1202                                 case RPCRDMA_MEMWINDOWS_ASYNC:
1203                                 case RPCRDMA_MEMWINDOWS:
1204                                         rc = ib_dealloc_mw(r->r.mw);
1205                                         if (rc)
1206                                                 dprintk("RPC:       %s:"
1207                                                         " ib_dealloc_mw"
1208                                                         " failed %i\n",
1209                                                         __func__, rc);
1210                                         break;
1211                                 default:
1212                                         break;
1213                                 }
1214                         }
1215                         rpcrdma_deregister_internal(ia,
1216                                         buf->rb_send_bufs[i]->rl_handle,
1217                                         &buf->rb_send_bufs[i]->rl_iov);
1218                         kfree(buf->rb_send_bufs[i]);
1219                 }
1220         }
1221
1222         kfree(buf->rb_pool);
1223 }
1224
1225 /*
1226  * Get a set of request/reply buffers.
1227  *
1228  * Reply buffer (if needed) is attached to send buffer upon return.
1229  * Rule:
1230  *    rb_send_index and rb_recv_index MUST always be pointing to the
1231  *    *next* available buffer (non-NULL). They are incremented after
1232  *    removing buffers, and decremented *before* returning them.
1233  */
1234 struct rpcrdma_req *
1235 rpcrdma_buffer_get(struct rpcrdma_buffer *buffers)
1236 {
1237         struct rpcrdma_req *req;
1238         unsigned long flags;
1239         int i;
1240         struct rpcrdma_mw *r;
1241
1242         spin_lock_irqsave(&buffers->rb_lock, flags);
1243         if (buffers->rb_send_index == buffers->rb_max_requests) {
1244                 spin_unlock_irqrestore(&buffers->rb_lock, flags);
1245                 dprintk("RPC:       %s: out of request buffers\n", __func__);
1246                 return ((struct rpcrdma_req *)NULL);
1247         }
1248
1249         req = buffers->rb_send_bufs[buffers->rb_send_index];
1250         if (buffers->rb_send_index < buffers->rb_recv_index) {
1251                 dprintk("RPC:       %s: %d extra receives outstanding (ok)\n",
1252                         __func__,
1253                         buffers->rb_recv_index - buffers->rb_send_index);
1254                 req->rl_reply = NULL;
1255         } else {
1256                 req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index];
1257                 buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL;
1258         }
1259         buffers->rb_send_bufs[buffers->rb_send_index++] = NULL;
1260         if (!list_empty(&buffers->rb_mws)) {
1261                 i = RPCRDMA_MAX_SEGS - 1;
1262                 do {
1263                         r = list_entry(buffers->rb_mws.next,
1264                                         struct rpcrdma_mw, mw_list);
1265                         list_del(&r->mw_list);
1266                         req->rl_segments[i].mr_chunk.rl_mw = r;
1267                 } while (--i >= 0);
1268         }
1269         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1270         return req;
1271 }
1272
1273 /*
1274  * Put request/reply buffers back into pool.
1275  * Pre-decrement counter/array index.
1276  */
1277 void
1278 rpcrdma_buffer_put(struct rpcrdma_req *req)
1279 {
1280         struct rpcrdma_buffer *buffers = req->rl_buffer;
1281         struct rpcrdma_ia *ia = rdmab_to_ia(buffers);
1282         int i;
1283         unsigned long flags;
1284
1285         BUG_ON(req->rl_nchunks != 0);
1286         spin_lock_irqsave(&buffers->rb_lock, flags);
1287         buffers->rb_send_bufs[--buffers->rb_send_index] = req;
1288         req->rl_niovs = 0;
1289         if (req->rl_reply) {
1290                 buffers->rb_recv_bufs[--buffers->rb_recv_index] = req->rl_reply;
1291                 init_waitqueue_head(&req->rl_reply->rr_unbind);
1292                 req->rl_reply->rr_func = NULL;
1293                 req->rl_reply = NULL;
1294         }
1295         switch (ia->ri_memreg_strategy) {
1296         case RPCRDMA_FRMR:
1297         case RPCRDMA_MTHCAFMR:
1298         case RPCRDMA_MEMWINDOWS_ASYNC:
1299         case RPCRDMA_MEMWINDOWS:
1300                 /*
1301                  * Cycle mw's back in reverse order, and "spin" them.
1302                  * This delays and scrambles reuse as much as possible.
1303                  */
1304                 i = 1;
1305                 do {
1306                         struct rpcrdma_mw **mw;
1307                         mw = &req->rl_segments[i].mr_chunk.rl_mw;
1308                         list_add_tail(&(*mw)->mw_list, &buffers->rb_mws);
1309                         *mw = NULL;
1310                 } while (++i < RPCRDMA_MAX_SEGS);
1311                 list_add_tail(&req->rl_segments[0].mr_chunk.rl_mw->mw_list,
1312                                         &buffers->rb_mws);
1313                 req->rl_segments[0].mr_chunk.rl_mw = NULL;
1314                 break;
1315         default:
1316                 break;
1317         }
1318         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1319 }
1320
1321 /*
1322  * Recover reply buffers from pool.
1323  * This happens when recovering from error conditions.
1324  * Post-increment counter/array index.
1325  */
1326 void
1327 rpcrdma_recv_buffer_get(struct rpcrdma_req *req)
1328 {
1329         struct rpcrdma_buffer *buffers = req->rl_buffer;
1330         unsigned long flags;
1331
1332         if (req->rl_iov.length == 0)    /* special case xprt_rdma_allocate() */
1333                 buffers = ((struct rpcrdma_req *) buffers)->rl_buffer;
1334         spin_lock_irqsave(&buffers->rb_lock, flags);
1335         if (buffers->rb_recv_index < buffers->rb_max_requests) {
1336                 req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index];
1337                 buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL;
1338         }
1339         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1340 }
1341
1342 /*
1343  * Put reply buffers back into pool when not attached to
1344  * request. This happens in error conditions, and when
1345  * aborting unbinds. Pre-decrement counter/array index.
1346  */
1347 void
1348 rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep)
1349 {
1350         struct rpcrdma_buffer *buffers = rep->rr_buffer;
1351         unsigned long flags;
1352
1353         rep->rr_func = NULL;
1354         spin_lock_irqsave(&buffers->rb_lock, flags);
1355         buffers->rb_recv_bufs[--buffers->rb_recv_index] = rep;
1356         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1357 }
1358
1359 /*
1360  * Wrappers for internal-use kmalloc memory registration, used by buffer code.
1361  */
1362
1363 int
1364 rpcrdma_register_internal(struct rpcrdma_ia *ia, void *va, int len,
1365                                 struct ib_mr **mrp, struct ib_sge *iov)
1366 {
1367         struct ib_phys_buf ipb;
1368         struct ib_mr *mr;
1369         int rc;
1370
1371         /*
1372          * All memory passed here was kmalloc'ed, therefore phys-contiguous.
1373          */
1374         iov->addr = ib_dma_map_single(ia->ri_id->device,
1375                         va, len, DMA_BIDIRECTIONAL);
1376         iov->length = len;
1377
1378         if (ia->ri_have_dma_lkey) {
1379                 *mrp = NULL;
1380                 iov->lkey = ia->ri_dma_lkey;
1381                 return 0;
1382         } else if (ia->ri_bind_mem != NULL) {
1383                 *mrp = NULL;
1384                 iov->lkey = ia->ri_bind_mem->lkey;
1385                 return 0;
1386         }
1387
1388         ipb.addr = iov->addr;
1389         ipb.size = iov->length;
1390         mr = ib_reg_phys_mr(ia->ri_pd, &ipb, 1,
1391                         IB_ACCESS_LOCAL_WRITE, &iov->addr);
1392
1393         dprintk("RPC:       %s: phys convert: 0x%llx "
1394                         "registered 0x%llx length %d\n",
1395                         __func__, (unsigned long long)ipb.addr,
1396                         (unsigned long long)iov->addr, len);
1397
1398         if (IS_ERR(mr)) {
1399                 *mrp = NULL;
1400                 rc = PTR_ERR(mr);
1401                 dprintk("RPC:       %s: failed with %i\n", __func__, rc);
1402         } else {
1403                 *mrp = mr;
1404                 iov->lkey = mr->lkey;
1405                 rc = 0;
1406         }
1407
1408         return rc;
1409 }
1410
1411 int
1412 rpcrdma_deregister_internal(struct rpcrdma_ia *ia,
1413                                 struct ib_mr *mr, struct ib_sge *iov)
1414 {
1415         int rc;
1416
1417         ib_dma_unmap_single(ia->ri_id->device,
1418                         iov->addr, iov->length, DMA_BIDIRECTIONAL);
1419
1420         if (NULL == mr)
1421                 return 0;
1422
1423         rc = ib_dereg_mr(mr);
1424         if (rc)
1425                 dprintk("RPC:       %s: ib_dereg_mr failed %i\n", __func__, rc);
1426         return rc;
1427 }
1428
1429 /*
1430  * Wrappers for chunk registration, shared by read/write chunk code.
1431  */
1432
1433 static void
1434 rpcrdma_map_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg, int writing)
1435 {
1436         seg->mr_dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1437         seg->mr_dmalen = seg->mr_len;
1438         if (seg->mr_page)
1439                 seg->mr_dma = ib_dma_map_page(ia->ri_id->device,
1440                                 seg->mr_page, offset_in_page(seg->mr_offset),
1441                                 seg->mr_dmalen, seg->mr_dir);
1442         else
1443                 seg->mr_dma = ib_dma_map_single(ia->ri_id->device,
1444                                 seg->mr_offset,
1445                                 seg->mr_dmalen, seg->mr_dir);
1446 }
1447
1448 static void
1449 rpcrdma_unmap_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg)
1450 {
1451         if (seg->mr_page)
1452                 ib_dma_unmap_page(ia->ri_id->device,
1453                                 seg->mr_dma, seg->mr_dmalen, seg->mr_dir);
1454         else
1455                 ib_dma_unmap_single(ia->ri_id->device,
1456                                 seg->mr_dma, seg->mr_dmalen, seg->mr_dir);
1457 }
1458
1459 static int
1460 rpcrdma_register_frmr_external(struct rpcrdma_mr_seg *seg,
1461                         int *nsegs, int writing, struct rpcrdma_ia *ia,
1462                         struct rpcrdma_xprt *r_xprt)
1463 {
1464         struct rpcrdma_mr_seg *seg1 = seg;
1465         struct ib_send_wr frmr_wr, *bad_wr;
1466         u8 key;
1467         int len, pageoff;
1468         int i, rc;
1469
1470         pageoff = offset_in_page(seg1->mr_offset);
1471         seg1->mr_offset -= pageoff;     /* start of page */
1472         seg1->mr_len += pageoff;
1473         len = -pageoff;
1474         if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
1475                 *nsegs = RPCRDMA_MAX_DATA_SEGS;
1476         for (i = 0; i < *nsegs;) {
1477                 rpcrdma_map_one(ia, seg, writing);
1478                 seg1->mr_chunk.rl_mw->r.frmr.fr_pgl->page_list[i] = seg->mr_dma;
1479                 len += seg->mr_len;
1480                 ++seg;
1481                 ++i;
1482                 /* Check for holes */
1483                 if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||
1484                     offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))
1485                         break;
1486         }
1487         dprintk("RPC:       %s: Using frmr %p to map %d segments\n",
1488                 __func__, seg1->mr_chunk.rl_mw, i);
1489
1490         /* Bump the key */
1491         key = (u8)(seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey & 0x000000FF);
1492         ib_update_fast_reg_key(seg1->mr_chunk.rl_mw->r.frmr.fr_mr, ++key);
1493
1494         /* Prepare FRMR WR */
1495         memset(&frmr_wr, 0, sizeof frmr_wr);
1496         frmr_wr.opcode = IB_WR_FAST_REG_MR;
1497         frmr_wr.send_flags = 0;                 /* unsignaled */
1498         frmr_wr.wr.fast_reg.iova_start = (unsigned long)seg1->mr_dma;
1499         frmr_wr.wr.fast_reg.page_list = seg1->mr_chunk.rl_mw->r.frmr.fr_pgl;
1500         frmr_wr.wr.fast_reg.page_list_len = i;
1501         frmr_wr.wr.fast_reg.page_shift = PAGE_SHIFT;
1502         frmr_wr.wr.fast_reg.length = i << PAGE_SHIFT;
1503         frmr_wr.wr.fast_reg.access_flags = (writing ?
1504                                 IB_ACCESS_REMOTE_WRITE : IB_ACCESS_REMOTE_READ);
1505         frmr_wr.wr.fast_reg.rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1506         DECR_CQCOUNT(&r_xprt->rx_ep);
1507
1508         rc = ib_post_send(ia->ri_id->qp, &frmr_wr, &bad_wr);
1509
1510         if (rc) {
1511                 dprintk("RPC:       %s: failed ib_post_send for register,"
1512                         " status %i\n", __func__, rc);
1513                 while (i--)
1514                         rpcrdma_unmap_one(ia, --seg);
1515         } else {
1516                 seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1517                 seg1->mr_base = seg1->mr_dma + pageoff;
1518                 seg1->mr_nsegs = i;
1519                 seg1->mr_len = len;
1520         }
1521         *nsegs = i;
1522         return rc;
1523 }
1524
1525 static int
1526 rpcrdma_deregister_frmr_external(struct rpcrdma_mr_seg *seg,
1527                         struct rpcrdma_ia *ia, struct rpcrdma_xprt *r_xprt)
1528 {
1529         struct rpcrdma_mr_seg *seg1 = seg;
1530         struct ib_send_wr invalidate_wr, *bad_wr;
1531         int rc;
1532
1533         while (seg1->mr_nsegs--)
1534                 rpcrdma_unmap_one(ia, seg++);
1535
1536         memset(&invalidate_wr, 0, sizeof invalidate_wr);
1537         invalidate_wr.opcode = IB_WR_LOCAL_INV;
1538         invalidate_wr.send_flags = 0;                   /* unsignaled */
1539         invalidate_wr.ex.invalidate_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1540         DECR_CQCOUNT(&r_xprt->rx_ep);
1541
1542         rc = ib_post_send(ia->ri_id->qp, &invalidate_wr, &bad_wr);
1543         if (rc)
1544                 dprintk("RPC:       %s: failed ib_post_send for invalidate,"
1545                         " status %i\n", __func__, rc);
1546         return rc;
1547 }
1548
1549 static int
1550 rpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg,
1551                         int *nsegs, int writing, struct rpcrdma_ia *ia)
1552 {
1553         struct rpcrdma_mr_seg *seg1 = seg;
1554         u64 physaddrs[RPCRDMA_MAX_DATA_SEGS];
1555         int len, pageoff, i, rc;
1556
1557         pageoff = offset_in_page(seg1->mr_offset);
1558         seg1->mr_offset -= pageoff;     /* start of page */
1559         seg1->mr_len += pageoff;
1560         len = -pageoff;
1561         if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
1562                 *nsegs = RPCRDMA_MAX_DATA_SEGS;
1563         for (i = 0; i < *nsegs;) {
1564                 rpcrdma_map_one(ia, seg, writing);
1565                 physaddrs[i] = seg->mr_dma;
1566                 len += seg->mr_len;
1567                 ++seg;
1568                 ++i;
1569                 /* Check for holes */
1570                 if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||
1571                     offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))
1572                         break;
1573         }
1574         rc = ib_map_phys_fmr(seg1->mr_chunk.rl_mw->r.fmr,
1575                                 physaddrs, i, seg1->mr_dma);
1576         if (rc) {
1577                 dprintk("RPC:       %s: failed ib_map_phys_fmr "
1578                         "%u@0x%llx+%i (%d)... status %i\n", __func__,
1579                         len, (unsigned long long)seg1->mr_dma,
1580                         pageoff, i, rc);
1581                 while (i--)
1582                         rpcrdma_unmap_one(ia, --seg);
1583         } else {
1584                 seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.fmr->rkey;
1585                 seg1->mr_base = seg1->mr_dma + pageoff;
1586                 seg1->mr_nsegs = i;
1587                 seg1->mr_len = len;
1588         }
1589         *nsegs = i;
1590         return rc;
1591 }
1592
1593 static int
1594 rpcrdma_deregister_fmr_external(struct rpcrdma_mr_seg *seg,
1595                         struct rpcrdma_ia *ia)
1596 {
1597         struct rpcrdma_mr_seg *seg1 = seg;
1598         LIST_HEAD(l);
1599         int rc;
1600
1601         list_add(&seg1->mr_chunk.rl_mw->r.fmr->list, &l);
1602         rc = ib_unmap_fmr(&l);
1603         while (seg1->mr_nsegs--)
1604                 rpcrdma_unmap_one(ia, seg++);
1605         if (rc)
1606                 dprintk("RPC:       %s: failed ib_unmap_fmr,"
1607                         " status %i\n", __func__, rc);
1608         return rc;
1609 }
1610
1611 static int
1612 rpcrdma_register_memwin_external(struct rpcrdma_mr_seg *seg,
1613                         int *nsegs, int writing, struct rpcrdma_ia *ia,
1614                         struct rpcrdma_xprt *r_xprt)
1615 {
1616         int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :
1617                                   IB_ACCESS_REMOTE_READ);
1618         struct ib_mw_bind param;
1619         int rc;
1620
1621         *nsegs = 1;
1622         rpcrdma_map_one(ia, seg, writing);
1623         param.mr = ia->ri_bind_mem;
1624         param.wr_id = 0ULL;     /* no send cookie */
1625         param.addr = seg->mr_dma;
1626         param.length = seg->mr_len;
1627         param.send_flags = 0;
1628         param.mw_access_flags = mem_priv;
1629
1630         DECR_CQCOUNT(&r_xprt->rx_ep);
1631         rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, &param);
1632         if (rc) {
1633                 dprintk("RPC:       %s: failed ib_bind_mw "
1634                         "%u@0x%llx status %i\n",
1635                         __func__, seg->mr_len,
1636                         (unsigned long long)seg->mr_dma, rc);
1637                 rpcrdma_unmap_one(ia, seg);
1638         } else {
1639                 seg->mr_rkey = seg->mr_chunk.rl_mw->r.mw->rkey;
1640                 seg->mr_base = param.addr;
1641                 seg->mr_nsegs = 1;
1642         }
1643         return rc;
1644 }
1645
1646 static int
1647 rpcrdma_deregister_memwin_external(struct rpcrdma_mr_seg *seg,
1648                         struct rpcrdma_ia *ia,
1649                         struct rpcrdma_xprt *r_xprt, void **r)
1650 {
1651         struct ib_mw_bind param;
1652         LIST_HEAD(l);
1653         int rc;
1654
1655         BUG_ON(seg->mr_nsegs != 1);
1656         param.mr = ia->ri_bind_mem;
1657         param.addr = 0ULL;      /* unbind */
1658         param.length = 0;
1659         param.mw_access_flags = 0;
1660         if (*r) {
1661                 param.wr_id = (u64) (unsigned long) *r;
1662                 param.send_flags = IB_SEND_SIGNALED;
1663                 INIT_CQCOUNT(&r_xprt->rx_ep);
1664         } else {
1665                 param.wr_id = 0ULL;
1666                 param.send_flags = 0;
1667                 DECR_CQCOUNT(&r_xprt->rx_ep);
1668         }
1669         rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, &param);
1670         rpcrdma_unmap_one(ia, seg);
1671         if (rc)
1672                 dprintk("RPC:       %s: failed ib_(un)bind_mw,"
1673                         " status %i\n", __func__, rc);
1674         else
1675                 *r = NULL;      /* will upcall on completion */
1676         return rc;
1677 }
1678
1679 static int
1680 rpcrdma_register_default_external(struct rpcrdma_mr_seg *seg,
1681                         int *nsegs, int writing, struct rpcrdma_ia *ia)
1682 {
1683         int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :
1684                                   IB_ACCESS_REMOTE_READ);
1685         struct rpcrdma_mr_seg *seg1 = seg;
1686         struct ib_phys_buf ipb[RPCRDMA_MAX_DATA_SEGS];
1687         int len, i, rc = 0;
1688
1689         if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
1690                 *nsegs = RPCRDMA_MAX_DATA_SEGS;
1691         for (len = 0, i = 0; i < *nsegs;) {
1692                 rpcrdma_map_one(ia, seg, writing);
1693                 ipb[i].addr = seg->mr_dma;
1694                 ipb[i].size = seg->mr_len;
1695                 len += seg->mr_len;
1696                 ++seg;
1697                 ++i;
1698                 /* Check for holes */
1699                 if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||
1700                     offset_in_page((seg-1)->mr_offset+(seg-1)->mr_len))
1701                         break;
1702         }
1703         seg1->mr_base = seg1->mr_dma;
1704         seg1->mr_chunk.rl_mr = ib_reg_phys_mr(ia->ri_pd,
1705                                 ipb, i, mem_priv, &seg1->mr_base);
1706         if (IS_ERR(seg1->mr_chunk.rl_mr)) {
1707                 rc = PTR_ERR(seg1->mr_chunk.rl_mr);
1708                 dprintk("RPC:       %s: failed ib_reg_phys_mr "
1709                         "%u@0x%llx (%d)... status %i\n",
1710                         __func__, len,
1711                         (unsigned long long)seg1->mr_dma, i, rc);
1712                 while (i--)
1713                         rpcrdma_unmap_one(ia, --seg);
1714         } else {
1715                 seg1->mr_rkey = seg1->mr_chunk.rl_mr->rkey;
1716                 seg1->mr_nsegs = i;
1717                 seg1->mr_len = len;
1718         }
1719         *nsegs = i;
1720         return rc;
1721 }
1722
1723 static int
1724 rpcrdma_deregister_default_external(struct rpcrdma_mr_seg *seg,
1725                         struct rpcrdma_ia *ia)
1726 {
1727         struct rpcrdma_mr_seg *seg1 = seg;
1728         int rc;
1729
1730         rc = ib_dereg_mr(seg1->mr_chunk.rl_mr);
1731         seg1->mr_chunk.rl_mr = NULL;
1732         while (seg1->mr_nsegs--)
1733                 rpcrdma_unmap_one(ia, seg++);
1734         if (rc)
1735                 dprintk("RPC:       %s: failed ib_dereg_mr,"
1736                         " status %i\n", __func__, rc);
1737         return rc;
1738 }
1739
1740 int
1741 rpcrdma_register_external(struct rpcrdma_mr_seg *seg,
1742                         int nsegs, int writing, struct rpcrdma_xprt *r_xprt)
1743 {
1744         struct rpcrdma_ia *ia = &r_xprt->rx_ia;
1745         int rc = 0;
1746
1747         switch (ia->ri_memreg_strategy) {
1748
1749 #if RPCRDMA_PERSISTENT_REGISTRATION
1750         case RPCRDMA_ALLPHYSICAL:
1751                 rpcrdma_map_one(ia, seg, writing);
1752                 seg->mr_rkey = ia->ri_bind_mem->rkey;
1753                 seg->mr_base = seg->mr_dma;
1754                 seg->mr_nsegs = 1;
1755                 nsegs = 1;
1756                 break;
1757 #endif
1758
1759         /* Registration using frmr registration */
1760         case RPCRDMA_FRMR:
1761                 rc = rpcrdma_register_frmr_external(seg, &nsegs, writing, ia, r_xprt);
1762                 break;
1763
1764         /* Registration using fmr memory registration */
1765         case RPCRDMA_MTHCAFMR:
1766                 rc = rpcrdma_register_fmr_external(seg, &nsegs, writing, ia);
1767                 break;
1768
1769         /* Registration using memory windows */
1770         case RPCRDMA_MEMWINDOWS_ASYNC:
1771         case RPCRDMA_MEMWINDOWS:
1772                 rc = rpcrdma_register_memwin_external(seg, &nsegs, writing, ia, r_xprt);
1773                 break;
1774
1775         /* Default registration each time */
1776         default:
1777                 rc = rpcrdma_register_default_external(seg, &nsegs, writing, ia);
1778                 break;
1779         }
1780         if (rc)
1781                 return -1;
1782
1783         return nsegs;
1784 }
1785
1786 int
1787 rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg,
1788                 struct rpcrdma_xprt *r_xprt, void *r)
1789 {
1790         struct rpcrdma_ia *ia = &r_xprt->rx_ia;
1791         int nsegs = seg->mr_nsegs, rc;
1792
1793         switch (ia->ri_memreg_strategy) {
1794
1795 #if RPCRDMA_PERSISTENT_REGISTRATION
1796         case RPCRDMA_ALLPHYSICAL:
1797                 BUG_ON(nsegs != 1);
1798                 rpcrdma_unmap_one(ia, seg);
1799                 rc = 0;
1800                 break;
1801 #endif
1802
1803         case RPCRDMA_FRMR:
1804                 rc = rpcrdma_deregister_frmr_external(seg, ia, r_xprt);
1805                 break;
1806
1807         case RPCRDMA_MTHCAFMR:
1808                 rc = rpcrdma_deregister_fmr_external(seg, ia);
1809                 break;
1810
1811         case RPCRDMA_MEMWINDOWS_ASYNC:
1812         case RPCRDMA_MEMWINDOWS:
1813                 rc = rpcrdma_deregister_memwin_external(seg, ia, r_xprt, &r);
1814                 break;
1815
1816         default:
1817                 rc = rpcrdma_deregister_default_external(seg, ia);
1818                 break;
1819         }
1820         if (r) {
1821                 struct rpcrdma_rep *rep = r;
1822                 void (*func)(struct rpcrdma_rep *) = rep->rr_func;
1823                 rep->rr_func = NULL;
1824                 func(rep);      /* dereg done, callback now */
1825         }
1826         return nsegs;
1827 }
1828
1829 /*
1830  * Prepost any receive buffer, then post send.
1831  *
1832  * Receive buffer is donated to hardware, reclaimed upon recv completion.
1833  */
1834 int
1835 rpcrdma_ep_post(struct rpcrdma_ia *ia,
1836                 struct rpcrdma_ep *ep,
1837                 struct rpcrdma_req *req)
1838 {
1839         struct ib_send_wr send_wr, *send_wr_fail;
1840         struct rpcrdma_rep *rep = req->rl_reply;
1841         int rc;
1842
1843         if (rep) {
1844                 rc = rpcrdma_ep_post_recv(ia, ep, rep);
1845                 if (rc)
1846                         goto out;
1847                 req->rl_reply = NULL;
1848         }
1849
1850         send_wr.next = NULL;
1851         send_wr.wr_id = 0ULL;   /* no send cookie */
1852         send_wr.sg_list = req->rl_send_iov;
1853         send_wr.num_sge = req->rl_niovs;
1854         send_wr.opcode = IB_WR_SEND;
1855         if (send_wr.num_sge == 4)       /* no need to sync any pad (constant) */
1856                 ib_dma_sync_single_for_device(ia->ri_id->device,
1857                         req->rl_send_iov[3].addr, req->rl_send_iov[3].length,
1858                         DMA_TO_DEVICE);
1859         ib_dma_sync_single_for_device(ia->ri_id->device,
1860                 req->rl_send_iov[1].addr, req->rl_send_iov[1].length,
1861                 DMA_TO_DEVICE);
1862         ib_dma_sync_single_for_device(ia->ri_id->device,
1863                 req->rl_send_iov[0].addr, req->rl_send_iov[0].length,
1864                 DMA_TO_DEVICE);
1865
1866         if (DECR_CQCOUNT(ep) > 0)
1867                 send_wr.send_flags = 0;
1868         else { /* Provider must take a send completion every now and then */
1869                 INIT_CQCOUNT(ep);
1870                 send_wr.send_flags = IB_SEND_SIGNALED;
1871         }
1872
1873         rc = ib_post_send(ia->ri_id->qp, &send_wr, &send_wr_fail);
1874         if (rc)
1875                 dprintk("RPC:       %s: ib_post_send returned %i\n", __func__,
1876                         rc);
1877 out:
1878         return rc;
1879 }
1880
1881 /*
1882  * (Re)post a receive buffer.
1883  */
1884 int
1885 rpcrdma_ep_post_recv(struct rpcrdma_ia *ia,
1886                      struct rpcrdma_ep *ep,
1887                      struct rpcrdma_rep *rep)
1888 {
1889         struct ib_recv_wr recv_wr, *recv_wr_fail;
1890         int rc;
1891
1892         recv_wr.next = NULL;
1893         recv_wr.wr_id = (u64) (unsigned long) rep;
1894         recv_wr.sg_list = &rep->rr_iov;
1895         recv_wr.num_sge = 1;
1896
1897         ib_dma_sync_single_for_cpu(ia->ri_id->device,
1898                 rep->rr_iov.addr, rep->rr_iov.length, DMA_BIDIRECTIONAL);
1899
1900         DECR_CQCOUNT(ep);
1901         rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail);
1902
1903         if (rc)
1904                 dprintk("RPC:       %s: ib_post_recv returned %i\n", __func__,
1905                         rc);
1906         return rc;
1907 }