git.oblomov.eu Git - linux-2.6/blob - drivers/scsi/cxgb3i/cxgb3i_offload.c

   1 /*
   2  * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
   3  *
   4  * Copyright (C) 2003-2008 Chelsio Communications.  All rights reserved.
   5  *
   6  * This program is distributed in the hope that it will be useful, but WITHOUT
   7  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   8  * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
   9  * release for licensing terms and conditions.
  10  *
  11  * Written by:  Dimitris Michailidis (dm@chelsio.com)
  12  *              Karen Xie (kxie@chelsio.com)
  13  */
  14
  15 #include <linux/if_vlan.h>
  16 #include <linux/version.h>
  17
  18 #include "cxgb3_defs.h"
  19 #include "cxgb3_ctl_defs.h"
  20 #include "firmware_exports.h"
  21 #include "cxgb3i_offload.h"
  22 #include "cxgb3i_pdu.h"
  23 #include "cxgb3i_ddp.h"
  24
  25 #ifdef __DEBUG_C3CN_CONN__
  26 #define c3cn_conn_debug         cxgb3i_log_info
  27 #else
  28 #define c3cn_conn_debug(fmt...)
  29 #endif
  30
  31 #ifdef __DEBUG_C3CN_TX__
  32 #define c3cn_tx_debug         cxgb3i_log_debug
  33 #else
  34 #define c3cn_tx_debug(fmt...)
  35 #endif
  36
  37 #ifdef __DEBUG_C3CN_RX__
  38 #define c3cn_rx_debug         cxgb3i_log_debug
  39 #else
  40 #define c3cn_rx_debug(fmt...)
  41 #endif
  42
  43 /*
  44  * module parameters releated to offloaded iscsi connection
  45  */
  46 static int cxgb3_rcv_win = 256 * 1024;
  47 module_param(cxgb3_rcv_win, int, 0644);
  48 MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)");
  49
  50 static int cxgb3_snd_win = 64 * 1024;
  51 module_param(cxgb3_snd_win, int, 0644);
  52 MODULE_PARM_DESC(cxgb3_snd_win, "TCP send window in bytes (default=64KB)");
  53
  54 static int cxgb3_rx_credit_thres = 10 * 1024;
  55 module_param(cxgb3_rx_credit_thres, int, 0644);
  56 MODULE_PARM_DESC(rx_credit_thres,
  57                  "RX credits return threshold in bytes (default=10KB)");
  58
  59 static unsigned int cxgb3_max_connect = 8 * 1024;
  60 module_param(cxgb3_max_connect, uint, 0644);
  61 MODULE_PARM_DESC(cxgb3_max_connect, "Max. # of connections (default=8092)");
  62
  63 static unsigned int cxgb3_sport_base = 20000;
  64 module_param(cxgb3_sport_base, uint, 0644);
  65 MODULE_PARM_DESC(cxgb3_sport_base, "starting port number (default=20000)");
  66
  67 /*
  68  * cxgb3i tcp connection data(per adapter) list
  69  */
  70 static LIST_HEAD(cdata_list);
  71 static DEFINE_RWLOCK(cdata_rwlock);
  72
  73 static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion);
  74 static void c3cn_release_offload_resources(struct s3_conn *c3cn);
  75
  76 /*
  77  * iscsi source port management
  78  *
  79  * Find a free source port in the port allocation map. We use a very simple
  80  * rotor scheme to look for the next free port.
  81  *
  82  * If a source port has been specified make sure that it doesn't collide with
  83  * our normal source port allocation map.  If it's outside the range of our
  84  * allocation/deallocation scheme just let them use it.
  85  *
  86  * If the source port is outside our allocation range, the caller is
  87  * responsible for keeping track of their port usage.
  88  */
  89 static int c3cn_get_port(struct s3_conn *c3cn, struct cxgb3i_sdev_data *cdata)
  90 {
  91         unsigned int start;
  92         int idx;
  93
  94         if (!cdata)
  95                 goto error_out;
  96
  97         if (c3cn->saddr.sin_port != 0) {
  98                 idx = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
  99                 if (idx < 0 || idx >= cxgb3_max_connect)
 100                         return 0;
 101                 if (!test_and_set_bit(idx, cdata->sport_map))
 102                         return -EADDRINUSE;
 103         }
 104
 105         /* the sport_map_next may not be accurate but that is okay, sport_map
 106            should be */
 107         start = idx = cdata->sport_map_next;
 108         do {
 109                 if (++idx >= cxgb3_max_connect)
 110                         idx = 0;
 111                 if (!(test_and_set_bit(idx, cdata->sport_map))) {
 112                         c3cn->saddr.sin_port = htons(cxgb3_sport_base + idx);
 113                         cdata->sport_map_next = idx;
 114                         c3cn_conn_debug("%s reserve port %u.\n",
 115                                         cdata->cdev->name,
 116                                         cxgb3_sport_base + idx);
 117                         return 0;
 118                 }
 119         } while (idx != start);
 120
 121 error_out:
 122         return -EADDRNOTAVAIL;
 123 }
 124
 125 static void c3cn_put_port(struct s3_conn *c3cn)
 126 {
 127         struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(c3cn->cdev);
 128
 129         if (c3cn->saddr.sin_port) {
 130                 int idx = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
 131
 132                 c3cn->saddr.sin_port = 0;
 133                 if (idx < 0 || idx >= cxgb3_max_connect)
 134                         return;
 135                 clear_bit(idx, cdata->sport_map);
 136                 c3cn_conn_debug("%s, release port %u.\n",
 137                                 cdata->cdev->name, cxgb3_sport_base + idx);
 138         }
 139 }
 140
 141 static inline void c3cn_set_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
 142 {
 143         __set_bit(flag, &c3cn->flags);
 144         c3cn_conn_debug("c3cn 0x%p, set %d, s %u, f 0x%lx.\n",
 145                         c3cn, flag, c3cn->state, c3cn->flags);
 146 }
 147
 148 static inline void c3cn_clear_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
 149 {
 150         __clear_bit(flag, &c3cn->flags);
 151         c3cn_conn_debug("c3cn 0x%p, clear %d, s %u, f 0x%lx.\n",
 152                         c3cn, flag, c3cn->state, c3cn->flags);
 153 }
 154
 155 static inline int c3cn_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
 156 {
 157         if (c3cn == NULL)
 158                 return 0;
 159         return test_bit(flag, &c3cn->flags);
 160 }
 161
 162 static void c3cn_set_state(struct s3_conn *c3cn, int state)
 163 {
 164         c3cn_conn_debug("c3cn 0x%p state -> %u.\n", c3cn, state);
 165         c3cn->state = state;
 166 }
 167
 168 static inline void c3cn_hold(struct s3_conn *c3cn)
 169 {
 170         atomic_inc(&c3cn->refcnt);
 171 }
 172
 173 static inline void c3cn_put(struct s3_conn *c3cn)
 174 {
 175         if (atomic_dec_and_test(&c3cn->refcnt)) {
 176                 c3cn_conn_debug("free c3cn 0x%p, s %u, f 0x%lx.\n",
 177                                 c3cn, c3cn->state, c3cn->flags);
 178                 kfree(c3cn);
 179         }
 180 }
 181
 182 static void c3cn_closed(struct s3_conn *c3cn)
 183 {
 184         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 185                          c3cn, c3cn->state, c3cn->flags);
 186
 187         c3cn_put_port(c3cn);
 188         c3cn_release_offload_resources(c3cn);
 189         c3cn_set_state(c3cn, C3CN_STATE_CLOSED);
 190         cxgb3i_conn_closing(c3cn);
 191 }
 192
 193 /*
 194  * CPL (Chelsio Protocol Language) defines a message passing interface between
 195  * the host driver and T3 asic.
 196  * The section below implments CPLs that related to iscsi tcp connection
 197  * open/close/abort and data send/receive.
 198  */
 199
 200 /*
 201  * CPL connection active open request: host ->
 202  */
 203 static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)
 204 {
 205         int i = 0;
 206
 207         while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu)
 208                 ++i;
 209         return i;
 210 }
 211
 212 static unsigned int select_mss(struct s3_conn *c3cn, unsigned int pmtu)
 213 {
 214         unsigned int idx;
 215         struct dst_entry *dst = c3cn->dst_cache;
 216         struct t3cdev *cdev = c3cn->cdev;
 217         const struct t3c_data *td = T3C_DATA(cdev);
 218         u16 advmss = dst_metric(dst, RTAX_ADVMSS);
 219
 220         if (advmss > pmtu - 40)
 221                 advmss = pmtu - 40;
 222         if (advmss < td->mtus[0] - 40)
 223                 advmss = td->mtus[0] - 40;
 224         idx = find_best_mtu(td, advmss + 40);
 225         return idx;
 226 }
 227
 228 static inline int compute_wscale(int win)
 229 {
 230         int wscale = 0;
 231         while (wscale < 14 && (65535<<wscale) < win)
 232                 wscale++;
 233         return wscale;
 234 }
 235
 236 static inline unsigned int calc_opt0h(struct s3_conn *c3cn)
 237 {
 238         int wscale = compute_wscale(cxgb3_rcv_win);
 239         return  V_KEEP_ALIVE(1) |
 240                 F_TCAM_BYPASS |
 241                 V_WND_SCALE(wscale) |
 242                 V_MSS_IDX(c3cn->mss_idx);
 243 }
 244
 245 static inline unsigned int calc_opt0l(struct s3_conn *c3cn)
 246 {
 247         return  V_ULP_MODE(ULP_MODE_ISCSI) |
 248                 V_RCV_BUFSIZ(cxgb3_rcv_win>>10);
 249 }
 250
 251 static void make_act_open_req(struct s3_conn *c3cn, struct sk_buff *skb,
 252                               unsigned int atid, const struct l2t_entry *e)
 253 {
 254         struct cpl_act_open_req *req;
 255
 256         c3cn_conn_debug("c3cn 0x%p, atid 0x%x.\n", c3cn, atid);
 257
 258         skb->priority = CPL_PRIORITY_SETUP;
 259         req = (struct cpl_act_open_req *)__skb_put(skb, sizeof(*req));
 260         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 261         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
 262         req->local_port = c3cn->saddr.sin_port;
 263         req->peer_port = c3cn->daddr.sin_port;
 264         req->local_ip = c3cn->saddr.sin_addr.s_addr;
 265         req->peer_ip = c3cn->daddr.sin_addr.s_addr;
 266         req->opt0h = htonl(calc_opt0h(c3cn) | V_L2T_IDX(e->idx) |
 267                            V_TX_CHANNEL(e->smt_idx));
 268         req->opt0l = htonl(calc_opt0l(c3cn));
 269         req->params = 0;
 270 }
 271
 272 static void fail_act_open(struct s3_conn *c3cn, int errno)
 273 {
 274         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 275                         c3cn, c3cn->state, c3cn->flags);
 276         c3cn->err = errno;
 277         c3cn_closed(c3cn);
 278 }
 279
 280 static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
 281 {
 282         struct s3_conn *c3cn = (struct s3_conn *)skb->sk;
 283
 284         c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
 285
 286         c3cn_hold(c3cn);
 287         spin_lock_bh(&c3cn->lock);
 288         if (c3cn->state == C3CN_STATE_CONNECTING)
 289                 fail_act_open(c3cn, EHOSTUNREACH);
 290         spin_unlock_bh(&c3cn->lock);
 291         c3cn_put(c3cn);
 292         __kfree_skb(skb);
 293 }
 294
 295 /*
 296  * CPL connection close request: host ->
 297  *
 298  * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
 299  * the write queue (i.e., after any unsent txt data).
 300  */
 301 static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb,
 302                        int flags)
 303 {
 304         CXGB3_SKB_CB(skb)->seq = c3cn->write_seq;
 305         CXGB3_SKB_CB(skb)->flags = flags;
 306         __skb_queue_tail(&c3cn->write_queue, skb);
 307 }
 308
 309 static void send_close_req(struct s3_conn *c3cn)
 310 {
 311         struct sk_buff *skb = c3cn->cpl_close;
 312         struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
 313         unsigned int tid = c3cn->tid;
 314
 315         c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
 316                         c3cn, c3cn->state, c3cn->flags);
 317
 318         c3cn->cpl_close = NULL;
 319
 320         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
 321         req->wr.wr_lo = htonl(V_WR_TID(tid));
 322         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
 323         req->rsvd = htonl(c3cn->write_seq);
 324
 325         skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND);
 326         if (c3cn->state != C3CN_STATE_CONNECTING)
 327                 c3cn_push_tx_frames(c3cn, 1);
 328 }
 329
 330 /*
 331  * CPL connection abort request: host ->
 332  *
 333  * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
 334  * for the same connection and also that we do not try to send a message
 335  * after the connection has closed.
 336  */
 337 static void abort_arp_failure(struct t3cdev *cdev, struct sk_buff *skb)
 338 {
 339         struct cpl_abort_req *req = cplhdr(skb);
 340
 341         c3cn_conn_debug("tdev 0x%p.\n", cdev);
 342
 343         req->cmd = CPL_ABORT_NO_RST;
 344         cxgb3_ofld_send(cdev, skb);
 345 }
 346
 347 static inline void c3cn_purge_write_queue(struct s3_conn *c3cn)
 348 {
 349         struct sk_buff *skb;
 350
 351         while ((skb = __skb_dequeue(&c3cn->write_queue)))
 352                 __kfree_skb(skb);
 353 }
 354
 355 static void send_abort_req(struct s3_conn *c3cn)
 356 {
 357         struct sk_buff *skb = c3cn->cpl_abort_req;
 358         struct cpl_abort_req *req;
 359         unsigned int tid = c3cn->tid;
 360
 361         if (unlikely(c3cn->state == C3CN_STATE_ABORTING) || !skb ||
 362                      !c3cn->cdev)
 363                 return;
 364
 365         c3cn_set_state(c3cn, C3CN_STATE_ABORTING);
 366
 367         c3cn_conn_debug("c3cn 0x%p, flag ABORT_RPL + ABORT_SHUT.\n", c3cn);
 368
 369         c3cn_set_flag(c3cn, C3CN_ABORT_RPL_PENDING);
 370
 371         /* Purge the send queue so we don't send anything after an abort. */
 372         c3cn_purge_write_queue(c3cn);
 373
 374         c3cn->cpl_abort_req = NULL;
 375         req = (struct cpl_abort_req *)skb->head;
 376
 377         skb->priority = CPL_PRIORITY_DATA;
 378         set_arp_failure_handler(skb, abort_arp_failure);
 379
 380         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
 381         req->wr.wr_lo = htonl(V_WR_TID(tid));
 382         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
 383         req->rsvd0 = htonl(c3cn->snd_nxt);
 384         req->rsvd1 = !c3cn_flag(c3cn, C3CN_TX_DATA_SENT);
 385         req->cmd = CPL_ABORT_SEND_RST;
 386
 387         l2t_send(c3cn->cdev, skb, c3cn->l2t);
 388 }
 389
 390 /*
 391  * CPL connection abort reply: host ->
 392  *
 393  * Send an ABORT_RPL message in response of the ABORT_REQ received.
 394  */
 395 static void send_abort_rpl(struct s3_conn *c3cn, int rst_status)
 396 {
 397         struct sk_buff *skb = c3cn->cpl_abort_rpl;
 398         struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
 399
 400         c3cn->cpl_abort_rpl = NULL;
 401
 402         skb->priority = CPL_PRIORITY_DATA;
 403         rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
 404         rpl->wr.wr_lo = htonl(V_WR_TID(c3cn->tid));
 405         OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, c3cn->tid));
 406         rpl->cmd = rst_status;
 407
 408         cxgb3_ofld_send(c3cn->cdev, skb);
 409 }
 410
 411 /*
 412  * CPL connection rx data ack: host ->
 413  * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
 414  * credits sent.
 415  */
 416 static u32 send_rx_credits(struct s3_conn *c3cn, u32 credits, u32 dack)
 417 {
 418         struct sk_buff *skb;
 419         struct cpl_rx_data_ack *req;
 420
 421         skb = alloc_skb(sizeof(*req), GFP_ATOMIC);
 422         if (!skb)
 423                 return 0;
 424
 425         req = (struct cpl_rx_data_ack *)__skb_put(skb, sizeof(*req));
 426         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 427         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, c3cn->tid));
 428         req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
 429         skb->priority = CPL_PRIORITY_ACK;
 430         cxgb3_ofld_send(c3cn->cdev, skb);
 431         return credits;
 432 }
 433
 434 /*
 435  * CPL connection tx data: host ->
 436  *
 437  * Send iscsi PDU via TX_DATA CPL message. Returns the number of
 438  * credits sent.
 439  * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
 440  * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
 441  */
 442
 443 /*
 444  * For ULP connections HW may inserts digest bytes into the pdu. Those digest
 445  * bytes are not sent by the host but are part of the TCP payload and therefore
 446  * consume TCP sequence space.
 447  */
 448 static const unsigned int cxgb3_ulp_extra_len[] = { 0, 4, 4, 8 };
 449 static inline unsigned int ulp_extra_len(const struct sk_buff *skb)
 450 {
 451         return cxgb3_ulp_extra_len[skb_ulp_mode(skb) & 3];
 452 }
 453
 454 static unsigned int wrlen __read_mostly;
 455
 456 /*
 457  * The number of WRs needed for an skb depends on the number of fragments
 458  * in the skb and whether it has any payload in its main body.  This maps the
 459  * length of the gather list represented by an skb into the # of necessary WRs.
 460  *
 461  * The max. length of an skb is controlled by the max pdu size which is ~16K.
 462  * Also, assume the min. fragment length is the sector size (512), then add
 463  * extra fragment counts for iscsi bhs and payload padding.
 464  */
 465 #define SKB_WR_LIST_SIZE        (16384/512 + 3)
 466 static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
 467
 468 static void s3_init_wr_tab(unsigned int wr_len)
 469 {
 470         int i;
 471
 472         if (skb_wrs[1])         /* already initialized */
 473                 return;
 474
 475         for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
 476                 int sgl_len = (3 * i) / 2 + (i & 1);
 477
 478                 sgl_len += 3;
 479                 skb_wrs[i] = (sgl_len <= wr_len
 480                               ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
 481         }
 482
 483         wrlen = wr_len * 8;
 484 }
 485
 486 static inline void reset_wr_list(struct s3_conn *c3cn)
 487 {
 488         c3cn->wr_pending_head = NULL;
 489 }
 490
 491 /*
 492  * Add a WR to a connections's list of pending WRs.  This is a singly-linked
 493  * list of sk_buffs operating as a FIFO.  The head is kept in wr_pending_head
 494  * and the tail in wr_pending_tail.
 495  */
 496 static inline void enqueue_wr(struct s3_conn *c3cn,
 497                               struct sk_buff *skb)
 498 {
 499         skb_wr_data(skb) = NULL;
 500
 501         /*
 502          * We want to take an extra reference since both us and the driver
 503          * need to free the packet before it's really freed. We know there's
 504          * just one user currently so we use atomic_set rather than skb_get
 505          * to avoid the atomic op.
 506          */
 507         atomic_set(&skb->users, 2);
 508
 509         if (!c3cn->wr_pending_head)
 510                 c3cn->wr_pending_head = skb;
 511         else
 512                 skb_wr_data(skb) = skb;
 513         c3cn->wr_pending_tail = skb;
 514 }
 515
 516 static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
 517 {
 518         return c3cn->wr_pending_head;
 519 }
 520
 521 static inline void free_wr_skb(struct sk_buff *skb)
 522 {
 523         kfree_skb(skb);
 524 }
 525
 526 static inline struct sk_buff *dequeue_wr(struct s3_conn *c3cn)
 527 {
 528         struct sk_buff *skb = c3cn->wr_pending_head;
 529
 530         if (likely(skb)) {
 531                 /* Don't bother clearing the tail */
 532                 c3cn->wr_pending_head = skb_wr_data(skb);
 533                 skb_wr_data(skb) = NULL;
 534         }
 535         return skb;
 536 }
 537
 538 static void purge_wr_queue(struct s3_conn *c3cn)
 539 {
 540         struct sk_buff *skb;
 541         while ((skb = dequeue_wr(c3cn)) != NULL)
 542                 free_wr_skb(skb);
 543 }
 544
 545 static inline void make_tx_data_wr(struct s3_conn *c3cn, struct sk_buff *skb,
 546                                    int len)
 547 {
 548         struct tx_data_wr *req;
 549
 550         skb_reset_transport_header(skb);
 551         req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
 552         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
 553         req->wr_lo = htonl(V_WR_TID(c3cn->tid));
 554         req->sndseq = htonl(c3cn->snd_nxt);
 555         /* len includes the length of any HW ULP additions */
 556         req->len = htonl(len);
 557         req->param = htonl(V_TX_PORT(c3cn->l2t->smt_idx));
 558         /* V_TX_ULP_SUBMODE sets both the mode and submode */
 559         req->flags = htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb)) |
 560                            V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
 561
 562         if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
 563                 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
 564                                     V_TX_CPU_IDX(c3cn->qset));
 565                 /* Sendbuffer is in units of 32KB. */
 566                 req->param |= htonl(V_TX_SNDBUF(cxgb3_snd_win >> 15));
 567                 c3cn_set_flag(c3cn, C3CN_TX_DATA_SENT);
 568         }
 569 }
 570
 571 /**
 572  * c3cn_push_tx_frames -- start transmit
 573  * @c3cn: the offloaded connection
 574  * @req_completion: request wr_ack or not
 575  *
 576  * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
 577  * connection's send queue and sends them on to T3.  Must be called with the
 578  * connection's lock held.  Returns the amount of send buffer space that was
 579  * freed as a result of sending queued data to T3.
 580  */
 581 static void arp_failure_discard(struct t3cdev *cdev, struct sk_buff *skb)
 582 {
 583         kfree_skb(skb);
 584 }
 585
 586 static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion)
 587 {
 588         int total_size = 0;
 589         struct sk_buff *skb;
 590         struct t3cdev *cdev;
 591         struct cxgb3i_sdev_data *cdata;
 592
 593         if (unlikely(c3cn->state == C3CN_STATE_CONNECTING ||
 594                      c3cn->state == C3CN_STATE_CLOSE_WAIT_1 ||
 595                      c3cn->state == C3CN_STATE_ABORTING)) {
 596                 c3cn_tx_debug("c3cn 0x%p, in closing state %u.\n",
 597                               c3cn, c3cn->state);
 598                 return 0;
 599         }
 600
 601         cdev = c3cn->cdev;
 602         cdata = CXGB3_SDEV_DATA(cdev);
 603
 604         while (c3cn->wr_avail
 605                && (skb = skb_peek(&c3cn->write_queue)) != NULL) {
 606                 int len = skb->len;     /* length before skb_push */
 607                 int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
 608                 int wrs_needed = skb_wrs[frags];
 609
 610                 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
 611                         wrs_needed = 1;
 612
 613                 WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
 614
 615                 if (c3cn->wr_avail < wrs_needed) {
 616                         c3cn_tx_debug("c3cn 0x%p, skb len %u/%u, frag %u, "
 617                                       "wr %d < %u.\n",
 618                                       c3cn, skb->len, skb->datalen, frags,
 619                                       wrs_needed, c3cn->wr_avail);
 620                         break;
 621                 }
 622
 623                 __skb_unlink(skb, &c3cn->write_queue);
 624                 skb->priority = CPL_PRIORITY_DATA;
 625                 skb->csum = wrs_needed; /* remember this until the WR_ACK */
 626                 c3cn->wr_avail -= wrs_needed;
 627                 c3cn->wr_unacked += wrs_needed;
 628                 enqueue_wr(c3cn, skb);
 629
 630                 if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) {
 631                         len += ulp_extra_len(skb);
 632                         make_tx_data_wr(c3cn, skb, len);
 633                         c3cn->snd_nxt += len;
 634                         if ((req_completion
 635                              && c3cn->wr_unacked == wrs_needed)
 636                             || (CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL)
 637                             || c3cn->wr_unacked >= c3cn->wr_max / 2) {
 638                                 struct work_request_hdr *wr = cplhdr(skb);
 639
 640                                 wr->wr_hi |= htonl(F_WR_COMPL);
 641                                 c3cn->wr_unacked = 0;
 642                         }
 643                         CXGB3_SKB_CB(skb)->flags &= ~C3CB_FLAG_NEED_HDR;
 644                 }
 645
 646                 total_size += skb->truesize;
 647                 set_arp_failure_handler(skb, arp_failure_discard);
 648                 l2t_send(cdev, skb, c3cn->l2t);
 649         }
 650         return total_size;
 651 }
 652
 653 /*
 654  * process_cpl_msg: -> host
 655  * Top-level CPL message processing used by most CPL messages that
 656  * pertain to connections.
 657  */
 658 static inline void process_cpl_msg(void (*fn)(struct s3_conn *,
 659                                               struct sk_buff *),
 660                                    struct s3_conn *c3cn,
 661                                    struct sk_buff *skb)
 662 {
 663         spin_lock_bh(&c3cn->lock);
 664         fn(c3cn, skb);
 665         spin_unlock_bh(&c3cn->lock);
 666 }
 667
 668 /*
 669  * process_cpl_msg_ref: -> host
 670  * Similar to process_cpl_msg() but takes an extra connection reference around
 671  * the call to the handler.  Should be used if the handler may drop a
 672  * connection reference.
 673  */
 674 static inline void process_cpl_msg_ref(void (*fn) (struct s3_conn *,
 675                                                    struct sk_buff *),
 676                                        struct s3_conn *c3cn,
 677                                        struct sk_buff *skb)
 678 {
 679         c3cn_hold(c3cn);
 680         process_cpl_msg(fn, c3cn, skb);
 681         c3cn_put(c3cn);
 682 }
 683
 684 /*
 685  * Process a CPL_ACT_ESTABLISH message: -> host
 686  * Updates connection state from an active establish CPL message.  Runs with
 687  * the connection lock held.
 688  */
 689
 690 static inline void s3_free_atid(struct t3cdev *cdev, unsigned int tid)
 691 {
 692         struct s3_conn *c3cn = cxgb3_free_atid(cdev, tid);
 693         if (c3cn)
 694                 c3cn_put(c3cn);
 695 }
 696
 697 static void c3cn_established(struct s3_conn *c3cn, u32 snd_isn,
 698                              unsigned int opt)
 699 {
 700         c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
 701
 702         c3cn->write_seq = c3cn->snd_nxt = c3cn->snd_una = snd_isn;
 703
 704         /*
 705          * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
 706          * pass through opt0.
 707          */
 708         if (cxgb3_rcv_win > (M_RCV_BUFSIZ << 10))
 709                 c3cn->rcv_wup -= cxgb3_rcv_win - (M_RCV_BUFSIZ << 10);
 710
 711         dst_confirm(c3cn->dst_cache);
 712
 713         smp_mb();
 714
 715         c3cn_set_state(c3cn, C3CN_STATE_ESTABLISHED);
 716 }
 717
 718 static void process_act_establish(struct s3_conn *c3cn, struct sk_buff *skb)
 719 {
 720         struct cpl_act_establish *req = cplhdr(skb);
 721         u32 rcv_isn = ntohl(req->rcv_isn);      /* real RCV_ISN + 1 */
 722
 723         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 724                         c3cn, c3cn->state, c3cn->flags);
 725
 726         if (unlikely(c3cn->state != C3CN_STATE_CONNECTING))
 727                 cxgb3i_log_error("TID %u expected SYN_SENT, got EST., s %u\n",
 728                                  c3cn->tid, c3cn->state);
 729
 730         c3cn->copied_seq = c3cn->rcv_wup = c3cn->rcv_nxt = rcv_isn;
 731         c3cn_established(c3cn, ntohl(req->snd_isn), ntohs(req->tcp_opt));
 732
 733         __kfree_skb(skb);
 734
 735         if (unlikely(c3cn_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED)))
 736                 /* upper layer has requested closing */
 737                 send_abort_req(c3cn);
 738         else if (c3cn_push_tx_frames(c3cn, 1))
 739                 cxgb3i_conn_tx_open(c3cn);
 740 }
 741
 742 static int do_act_establish(struct t3cdev *cdev, struct sk_buff *skb,
 743                             void *ctx)
 744 {
 745         struct cpl_act_establish *req = cplhdr(skb);
 746         unsigned int tid = GET_TID(req);
 747         unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
 748         struct s3_conn *c3cn = ctx;
 749         struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
 750
 751         c3cn_conn_debug("rcv, tid 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
 752                         tid, c3cn, c3cn->state, c3cn->flags);
 753
 754         c3cn->tid = tid;
 755         c3cn_hold(c3cn);
 756         cxgb3_insert_tid(cdata->cdev, cdata->client, c3cn, tid);
 757         s3_free_atid(cdev, atid);
 758
 759         c3cn->qset = G_QNUM(ntohl(skb->csum));
 760
 761         process_cpl_msg(process_act_establish, c3cn, skb);
 762         return 0;
 763 }
 764
 765 /*
 766  * Process a CPL_ACT_OPEN_RPL message: -> host
 767  * Handle active open failures.
 768  */
 769 static int act_open_rpl_status_to_errno(int status)
 770 {
 771         switch (status) {
 772         case CPL_ERR_CONN_RESET:
 773                 return ECONNREFUSED;
 774         case CPL_ERR_ARP_MISS:
 775                 return EHOSTUNREACH;
 776         case CPL_ERR_CONN_TIMEDOUT:
 777                 return ETIMEDOUT;
 778         case CPL_ERR_TCAM_FULL:
 779                 return ENOMEM;
 780         case CPL_ERR_CONN_EXIST:
 781                 cxgb3i_log_error("ACTIVE_OPEN_RPL: 4-tuple in use\n");
 782                 return EADDRINUSE;
 783         default:
 784                 return EIO;
 785         }
 786 }
 787
 788 static void act_open_retry_timer(unsigned long data)
 789 {
 790         struct sk_buff *skb;
 791         struct s3_conn *c3cn = (struct s3_conn *)data;
 792
 793         c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
 794
 795         spin_lock_bh(&c3cn->lock);
 796         skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_ATOMIC);
 797         if (!skb)
 798                 fail_act_open(c3cn, ENOMEM);
 799         else {
 800                 skb->sk = (struct sock *)c3cn;
 801                 set_arp_failure_handler(skb, act_open_req_arp_failure);
 802                 make_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
 803                 l2t_send(c3cn->cdev, skb, c3cn->l2t);
 804         }
 805         spin_unlock_bh(&c3cn->lock);
 806         c3cn_put(c3cn);
 807 }
 808
 809 static void process_act_open_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
 810 {
 811         struct cpl_act_open_rpl *rpl = cplhdr(skb);
 812
 813         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 814                         c3cn, c3cn->state, c3cn->flags);
 815
 816         if (rpl->status == CPL_ERR_CONN_EXIST &&
 817             c3cn->retry_timer.function != act_open_retry_timer) {
 818                 c3cn->retry_timer.function = act_open_retry_timer;
 819                 if (!mod_timer(&c3cn->retry_timer, jiffies + HZ / 2))
 820                         c3cn_hold(c3cn);
 821         } else
 822                 fail_act_open(c3cn, act_open_rpl_status_to_errno(rpl->status));
 823         __kfree_skb(skb);
 824 }
 825
 826 static int do_act_open_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 827 {
 828         struct s3_conn *c3cn = ctx;
 829         struct cpl_act_open_rpl *rpl = cplhdr(skb);
 830
 831         c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
 832                         rpl->status, c3cn, c3cn->state, c3cn->flags);
 833
 834         if (rpl->status != CPL_ERR_TCAM_FULL &&
 835             rpl->status != CPL_ERR_CONN_EXIST &&
 836             rpl->status != CPL_ERR_ARP_MISS)
 837                 cxgb3_queue_tid_release(cdev, GET_TID(rpl));
 838
 839         process_cpl_msg_ref(process_act_open_rpl, c3cn, skb);
 840         return 0;
 841 }
 842
 843 /*
 844  * Process PEER_CLOSE CPL messages: -> host
 845  * Handle peer FIN.
 846  */
 847 static void process_peer_close(struct s3_conn *c3cn, struct sk_buff *skb)
 848 {
 849         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 850                         c3cn, c3cn->state, c3cn->flags);
 851
 852         if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
 853                 goto out;
 854
 855         switch (c3cn->state) {
 856         case C3CN_STATE_ESTABLISHED:
 857                 c3cn_set_state(c3cn, C3CN_STATE_PASSIVE_CLOSE);
 858                 break;
 859         case C3CN_STATE_ACTIVE_CLOSE:
 860                 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_2);
 861                 break;
 862         case C3CN_STATE_CLOSE_WAIT_1:
 863                 c3cn_closed(c3cn);
 864                 break;
 865         case C3CN_STATE_ABORTING:
 866                 break;
 867         default:
 868                 cxgb3i_log_error("%s: peer close, TID %u in bad state %u\n",
 869                                  c3cn->cdev->name, c3cn->tid, c3cn->state);
 870         }
 871
 872         cxgb3i_conn_closing(c3cn);
 873 out:
 874         __kfree_skb(skb);
 875 }
 876
 877 static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 878 {
 879         struct s3_conn *c3cn = ctx;
 880
 881         c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
 882                         c3cn, c3cn->state, c3cn->flags);
 883         process_cpl_msg_ref(process_peer_close, c3cn, skb);
 884         return 0;
 885 }
 886
 887 /*
 888  * Process CLOSE_CONN_RPL CPL message: -> host
 889  * Process a peer ACK to our FIN.
 890  */
 891 static void process_close_con_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
 892 {
 893         struct cpl_close_con_rpl *rpl = cplhdr(skb);
 894
 895         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 896                         c3cn, c3cn->state, c3cn->flags);
 897
 898         c3cn->snd_una = ntohl(rpl->snd_nxt) - 1;        /* exclude FIN */
 899
 900         if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
 901                 goto out;
 902
 903         switch (c3cn->state) {
 904         case C3CN_STATE_ACTIVE_CLOSE:
 905                 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_1);
 906                 break;
 907         case C3CN_STATE_CLOSE_WAIT_1:
 908         case C3CN_STATE_CLOSE_WAIT_2:
 909                 c3cn_closed(c3cn);
 910                 break;
 911         case C3CN_STATE_ABORTING:
 912                 break;
 913         default:
 914                 cxgb3i_log_error("%s: close_rpl, TID %u in bad state %u\n",
 915                                  c3cn->cdev->name, c3cn->tid, c3cn->state);
 916         }
 917
 918 out:
 919         kfree_skb(skb);
 920 }
 921
 922 static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
 923                             void *ctx)
 924 {
 925         struct s3_conn *c3cn = ctx;
 926
 927         c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
 928                          c3cn, c3cn->state, c3cn->flags);
 929
 930         process_cpl_msg_ref(process_close_con_rpl, c3cn, skb);
 931         return 0;
 932 }
 933
 934 /*
 935  * Process ABORT_REQ_RSS CPL message: -> host
 936  * Process abort requests.  If we are waiting for an ABORT_RPL we ignore this
 937  * request except that we need to reply to it.
 938  */
 939
 940 static int abort_status_to_errno(struct s3_conn *c3cn, int abort_reason,
 941                                  int *need_rst)
 942 {
 943         switch (abort_reason) {
 944         case CPL_ERR_BAD_SYN: /* fall through */
 945         case CPL_ERR_CONN_RESET:
 946                 return c3cn->state > C3CN_STATE_ESTABLISHED ?
 947                         EPIPE : ECONNRESET;
 948         case CPL_ERR_XMIT_TIMEDOUT:
 949         case CPL_ERR_PERSIST_TIMEDOUT:
 950         case CPL_ERR_FINWAIT2_TIMEDOUT:
 951         case CPL_ERR_KEEPALIVE_TIMEDOUT:
 952                 return ETIMEDOUT;
 953         default:
 954                 return EIO;
 955         }
 956 }
 957
 958 static void process_abort_req(struct s3_conn *c3cn, struct sk_buff *skb)
 959 {
 960         int rst_status = CPL_ABORT_NO_RST;
 961         const struct cpl_abort_req_rss *req = cplhdr(skb);
 962
 963         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 964                         c3cn, c3cn->state, c3cn->flags);
 965
 966         if (!c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD)) {
 967                 c3cn_set_flag(c3cn, C3CN_ABORT_REQ_RCVD);
 968                 c3cn_set_state(c3cn, C3CN_STATE_ABORTING);
 969                 __kfree_skb(skb);
 970                 return;
 971         }
 972
 973         c3cn_clear_flag(c3cn, C3CN_ABORT_REQ_RCVD);
 974         send_abort_rpl(c3cn, rst_status);
 975
 976         if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
 977                 c3cn->err =
 978                     abort_status_to_errno(c3cn, req->status, &rst_status);
 979                 c3cn_closed(c3cn);
 980         }
 981 }
 982
 983 static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 984 {
 985         const struct cpl_abort_req_rss *req = cplhdr(skb);
 986         struct s3_conn *c3cn = ctx;
 987
 988         c3cn_conn_debug("rcv, c3cn 0x%p, s 0x%x, f 0x%lx.\n",
 989                         c3cn, c3cn->state, c3cn->flags);
 990
 991         if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
 992             req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
 993                 __kfree_skb(skb);
 994                 return 0;
 995         }
 996
 997         process_cpl_msg_ref(process_abort_req, c3cn, skb);
 998         return 0;
 999 }
1000
1001 /*
1002  * Process ABORT_RPL_RSS CPL message: -> host
1003  * Process abort replies.  We only process these messages if we anticipate
1004  * them as the coordination between SW and HW in this area is somewhat lacking
1005  * and sometimes we get ABORT_RPLs after we are done with the connection that
1006  * originated the ABORT_REQ.
1007  */
1008 static void process_abort_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
1009 {
1010         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1011                         c3cn, c3cn->state, c3cn->flags);
1012
1013         if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
1014                 if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_RCVD))
1015                         c3cn_set_flag(c3cn, C3CN_ABORT_RPL_RCVD);
1016                 else {
1017                         c3cn_clear_flag(c3cn, C3CN_ABORT_RPL_RCVD);
1018                         c3cn_clear_flag(c3cn, C3CN_ABORT_RPL_PENDING);
1019                         if (c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD))
1020                                 cxgb3i_log_error("%s tid %u, ABORT_RPL_RSS\n",
1021                                                  c3cn->cdev->name, c3cn->tid);
1022                         c3cn_closed(c3cn);
1023                 }
1024         }
1025         __kfree_skb(skb);
1026 }
1027
1028 static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
1029 {
1030         struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
1031         struct s3_conn *c3cn = ctx;
1032
1033         c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, 0x%lx.\n",
1034                         rpl->status, c3cn, c3cn ? c3cn->state : 0,
1035                         c3cn ? c3cn->flags : 0UL);
1036
1037         /*
1038          * Ignore replies to post-close aborts indicating that the abort was
1039          * requested too late.  These connections are terminated when we get
1040          * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
1041          * arrives the TID is either no longer used or it has been recycled.
1042          */
1043         if (rpl->status == CPL_ERR_ABORT_FAILED)
1044                 goto discard;
1045
1046         /*
1047          * Sometimes we've already closed the connection, e.g., a post-close
1048          * abort races with ABORT_REQ_RSS, the latter frees the connection
1049          * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
1050          * but FW turns the ABORT_REQ into a regular one and so we get
1051          * ABORT_RPL_RSS with status 0 and no connection.
1052          */
1053         if (!c3cn)
1054                 goto discard;
1055
1056         process_cpl_msg_ref(process_abort_rpl, c3cn, skb);
1057         return 0;
1058
1059 discard:
1060         __kfree_skb(skb);
1061         return 0;
1062 }
1063
1064 /*
1065  * Process RX_ISCSI_HDR CPL message: -> host
1066  * Handle received PDUs, the payload could be DDP'ed. If not, the payload
1067  * follow after the bhs.
1068  */
1069 static void process_rx_iscsi_hdr(struct s3_conn *c3cn, struct sk_buff *skb)
1070 {
1071         struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
1072         struct cpl_iscsi_hdr_norss data_cpl;
1073         struct cpl_rx_data_ddp_norss ddp_cpl;
1074         unsigned int hdr_len, data_len, status;
1075         unsigned int len;
1076         int err;
1077
1078         if (unlikely(c3cn->state >= C3CN_STATE_PASSIVE_CLOSE)) {
1079                 if (c3cn->state != C3CN_STATE_ABORTING)
1080                         send_abort_req(c3cn);
1081                 __kfree_skb(skb);
1082                 return;
1083         }
1084
1085         CXGB3_SKB_CB(skb)->seq = ntohl(hdr_cpl->seq);
1086         CXGB3_SKB_CB(skb)->flags = 0;
1087
1088         skb_reset_transport_header(skb);
1089         __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
1090
1091         len = hdr_len = ntohs(hdr_cpl->len);
1092         /* msg coalesce is off or not enough data received */
1093         if (skb->len <= hdr_len) {
1094                 cxgb3i_log_error("%s: TID %u, ISCSI_HDR, skb len %u < %u.\n",
1095                                  c3cn->cdev->name, c3cn->tid,
1096                                  skb->len, hdr_len);
1097                 goto abort_conn;
1098         }
1099
1100         err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
1101                             sizeof(ddp_cpl));
1102         if (err < 0)
1103                 goto abort_conn;
1104
1105         skb_ulp_mode(skb) = ULP2_FLAG_DATA_READY;
1106         skb_ulp_pdulen(skb) = ntohs(ddp_cpl.len);
1107         skb_ulp_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
1108         status = ntohl(ddp_cpl.ddp_status);
1109
1110         c3cn_rx_debug("rx skb 0x%p, len %u, pdulen %u, ddp status 0x%x.\n",
1111                       skb, skb->len, skb_ulp_pdulen(skb), status);
1112
1113         if (status & (1 << RX_DDP_STATUS_HCRC_SHIFT))
1114                 skb_ulp_mode(skb) |= ULP2_FLAG_HCRC_ERROR;
1115         if (status & (1 << RX_DDP_STATUS_DCRC_SHIFT))
1116                 skb_ulp_mode(skb) |= ULP2_FLAG_DCRC_ERROR;
1117         if (status & (1 << RX_DDP_STATUS_PAD_SHIFT))
1118                 skb_ulp_mode(skb) |= ULP2_FLAG_PAD_ERROR;
1119
1120         if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
1121                 err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
1122                 if (err < 0)
1123                         goto abort_conn;
1124                 data_len = ntohs(data_cpl.len);
1125                 len += sizeof(data_cpl) + data_len;
1126         } else if (status & (1 << RX_DDP_STATUS_DDP_SHIFT))
1127                 skb_ulp_mode(skb) |= ULP2_FLAG_DATA_DDPED;
1128
1129         c3cn->rcv_nxt = ntohl(ddp_cpl.seq) + skb_ulp_pdulen(skb);
1130         __pskb_trim(skb, len);
1131         __skb_queue_tail(&c3cn->receive_queue, skb);
1132         cxgb3i_conn_pdu_ready(c3cn);
1133
1134         return;
1135
1136 abort_conn:
1137         send_abort_req(c3cn);
1138         __kfree_skb(skb);
1139 }
1140
1141 static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
1142 {
1143         struct s3_conn *c3cn = ctx;
1144
1145         process_cpl_msg(process_rx_iscsi_hdr, c3cn, skb);
1146         return 0;
1147 }
1148
1149 /*
1150  * Process TX_DATA_ACK CPL messages: -> host
1151  * Process an acknowledgment of WR completion.  Advance snd_una and send the
1152  * next batch of work requests from the write queue.
1153  */
1154 static void process_wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
1155 {
1156         struct cpl_wr_ack *hdr = cplhdr(skb);
1157         unsigned int credits = ntohs(hdr->credits);
1158         u32 snd_una = ntohl(hdr->snd_una);
1159
1160         c3cn->wr_avail += credits;
1161         if (c3cn->wr_unacked > c3cn->wr_max - c3cn->wr_avail)
1162                 c3cn->wr_unacked = c3cn->wr_max - c3cn->wr_avail;
1163
1164         while (credits) {
1165                 struct sk_buff *p = peek_wr(c3cn);
1166
1167                 if (unlikely(!p)) {
1168                         cxgb3i_log_error("%u WR_ACK credits for TID %u with "
1169                                          "nothing pending, state %u\n",
1170                                          credits, c3cn->tid, c3cn->state);
1171                         break;
1172                 }
1173                 if (unlikely(credits < p->csum)) {
1174                         p->csum -= credits;
1175                         break;
1176                 } else {
1177                         dequeue_wr(c3cn);
1178                         credits -= p->csum;
1179                         free_wr_skb(p);
1180                 }
1181         }
1182
1183         if (unlikely(before(snd_una, c3cn->snd_una)))
1184                 goto out_free;
1185
1186         if (c3cn->snd_una != snd_una) {
1187                 c3cn->snd_una = snd_una;
1188                 dst_confirm(c3cn->dst_cache);
1189         }
1190
1191         if (skb_queue_len(&c3cn->write_queue) && c3cn_push_tx_frames(c3cn, 0))
1192                 cxgb3i_conn_tx_open(c3cn);
1193 out_free:
1194         __kfree_skb(skb);
1195 }
1196
1197 static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
1198 {
1199         struct s3_conn *c3cn = ctx;
1200
1201         process_cpl_msg(process_wr_ack, c3cn, skb);
1202         return 0;
1203 }
1204
1205 /*
1206  * for each connection, pre-allocate skbs needed for close/abort requests. So
1207  * that we can service the request right away.
1208  */
1209 static void c3cn_free_cpl_skbs(struct s3_conn *c3cn)
1210 {
1211         if (c3cn->cpl_close)
1212                 kfree_skb(c3cn->cpl_close);
1213         if (c3cn->cpl_abort_req)
1214                 kfree_skb(c3cn->cpl_abort_req);
1215         if (c3cn->cpl_abort_rpl)
1216                 kfree_skb(c3cn->cpl_abort_rpl);
1217 }
1218
1219 static int c3cn_alloc_cpl_skbs(struct s3_conn *c3cn)
1220 {
1221         c3cn->cpl_close = alloc_skb(sizeof(struct cpl_close_con_req),
1222                                    GFP_KERNEL);
1223         if (!c3cn->cpl_close)
1224                 return -ENOMEM;
1225         skb_put(c3cn->cpl_close, sizeof(struct cpl_close_con_req));
1226
1227         c3cn->cpl_abort_req = alloc_skb(sizeof(struct cpl_abort_req),
1228                                         GFP_KERNEL);
1229         if (!c3cn->cpl_abort_req)
1230                 goto free_cpl_skbs;
1231         skb_put(c3cn->cpl_abort_req, sizeof(struct cpl_abort_req));
1232
1233         c3cn->cpl_abort_rpl = alloc_skb(sizeof(struct cpl_abort_rpl),
1234                                         GFP_KERNEL);
1235         if (!c3cn->cpl_abort_rpl)
1236                 goto free_cpl_skbs;
1237         skb_put(c3cn->cpl_abort_rpl, sizeof(struct cpl_abort_rpl));
1238
1239         return 0;
1240
1241 free_cpl_skbs:
1242         c3cn_free_cpl_skbs(c3cn);
1243         return -ENOMEM;
1244 }
1245
1246 /**
1247  * c3cn_release_offload_resources - release offload resource
1248  * @c3cn: the offloaded iscsi tcp connection.
1249  * Release resources held by an offload connection (TID, L2T entry, etc.)
1250  */
1251 static void c3cn_release_offload_resources(struct s3_conn *c3cn)
1252 {
1253         struct t3cdev *cdev = c3cn->cdev;
1254         unsigned int tid = c3cn->tid;
1255
1256         if (!cdev)
1257                 return;
1258
1259         c3cn->qset = 0;
1260
1261         c3cn_free_cpl_skbs(c3cn);
1262
1263         if (c3cn->wr_avail != c3cn->wr_max) {
1264                 purge_wr_queue(c3cn);
1265                 reset_wr_list(c3cn);
1266         }
1267
1268         if (c3cn->l2t) {
1269                 l2t_release(L2DATA(cdev), c3cn->l2t);
1270                 c3cn->l2t = NULL;
1271         }
1272
1273         if (c3cn->state == C3CN_STATE_CONNECTING) /* we have ATID */
1274                 s3_free_atid(cdev, tid);
1275         else {          /* we have TID */
1276                 cxgb3_remove_tid(cdev, (void *)c3cn, tid);
1277                 c3cn_put(c3cn);
1278         }
1279
1280         c3cn->cdev = NULL;
1281 }
1282
1283 /**
1284  * cxgb3i_c3cn_create - allocate and initialize an s3_conn structure
1285  * returns the s3_conn structure allocated.
1286  */
1287 struct s3_conn *cxgb3i_c3cn_create(void)
1288 {
1289         struct s3_conn *c3cn;
1290
1291         c3cn = kzalloc(sizeof(*c3cn), GFP_KERNEL);
1292         if (!c3cn)
1293                 return NULL;
1294
1295         /* pre-allocate close/abort cpl, so we don't need to wait for memory
1296            when close/abort is requested. */
1297         if (c3cn_alloc_cpl_skbs(c3cn) < 0)
1298                 goto free_c3cn;
1299
1300         c3cn_conn_debug("alloc c3cn 0x%p.\n", c3cn);
1301
1302         c3cn->flags = 0;
1303         spin_lock_init(&c3cn->lock);
1304         atomic_set(&c3cn->refcnt, 1);
1305         skb_queue_head_init(&c3cn->receive_queue);
1306         skb_queue_head_init(&c3cn->write_queue);
1307         setup_timer(&c3cn->retry_timer, NULL, (unsigned long)c3cn);
1308         rwlock_init(&c3cn->callback_lock);
1309
1310         return c3cn;
1311
1312 free_c3cn:
1313         kfree(c3cn);
1314         return NULL;
1315 }
1316
1317 static void c3cn_active_close(struct s3_conn *c3cn)
1318 {
1319         int data_lost;
1320         int close_req = 0;
1321
1322         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1323                          c3cn, c3cn->state, c3cn->flags);
1324
1325         dst_confirm(c3cn->dst_cache);
1326
1327         c3cn_hold(c3cn);
1328         spin_lock_bh(&c3cn->lock);
1329
1330         data_lost = skb_queue_len(&c3cn->receive_queue);
1331         __skb_queue_purge(&c3cn->receive_queue);
1332
1333         switch (c3cn->state) {
1334         case C3CN_STATE_CLOSED:
1335         case C3CN_STATE_ACTIVE_CLOSE:
1336         case C3CN_STATE_CLOSE_WAIT_1:
1337         case C3CN_STATE_CLOSE_WAIT_2:
1338         case C3CN_STATE_ABORTING:
1339                 /* nothing need to be done */
1340                 break;
1341         case C3CN_STATE_CONNECTING:
1342                 /* defer until cpl_act_open_rpl or cpl_act_establish */
1343                 c3cn_set_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED);
1344                 break;
1345         case C3CN_STATE_ESTABLISHED:
1346                 close_req = 1;
1347                 c3cn_set_state(c3cn, C3CN_STATE_ACTIVE_CLOSE);
1348                 break;
1349         case C3CN_STATE_PASSIVE_CLOSE:
1350                 close_req = 1;
1351                 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_2);
1352                 break;
1353         }
1354
1355         if (close_req) {
1356                 if (data_lost)
1357                         /* Unread data was tossed, zap the connection. */
1358                         send_abort_req(c3cn);
1359                 else
1360                         send_close_req(c3cn);
1361         }
1362
1363         spin_unlock_bh(&c3cn->lock);
1364         c3cn_put(c3cn);
1365 }
1366
1367 /**
1368  * cxgb3i_c3cn_release - close and release an iscsi tcp connection and any
1369  *      resource held
1370  * @c3cn: the iscsi tcp connection
1371  */
1372 void cxgb3i_c3cn_release(struct s3_conn *c3cn)
1373 {
1374         c3cn_conn_debug("c3cn 0x%p, s %u, f 0x%lx.\n",
1375                         c3cn, c3cn->state, c3cn->flags);
1376         if (likely(c3cn->state != C3CN_STATE_CONNECTING))
1377                 c3cn_active_close(c3cn);
1378         else
1379                 c3cn_set_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED);
1380         c3cn_put(c3cn);
1381 }
1382
1383 static int is_cxgb3_dev(struct net_device *dev)
1384 {
1385         struct cxgb3i_sdev_data *cdata;
1386
1387         write_lock(&cdata_rwlock);
1388         list_for_each_entry(cdata, &cdata_list, list) {
1389                 struct adap_ports *ports = &cdata->ports;
1390                 int i;
1391
1392                 for (i = 0; i < ports->nports; i++)
1393                         if (dev == ports->lldevs[i]) {
1394                                 write_unlock(&cdata_rwlock);
1395                                 return 1;
1396                         }
1397         }
1398         write_unlock(&cdata_rwlock);
1399         return 0;
1400 }
1401
1402 /**
1403  * cxgb3_egress_dev - return the cxgb3 egress device
1404  * @root_dev: the root device anchoring the search
1405  * @c3cn: the connection used to determine egress port in bonding mode
1406  * @context: in bonding mode, indicates a connection set up or failover
1407  *
1408  * Return egress device or NULL if the egress device isn't one of our ports.
1409  */
1410 static struct net_device *cxgb3_egress_dev(struct net_device *root_dev,
1411                                            struct s3_conn *c3cn,
1412                                            int context)
1413 {
1414         while (root_dev) {
1415                 if (root_dev->priv_flags & IFF_802_1Q_VLAN)
1416                         root_dev = vlan_dev_real_dev(root_dev);
1417                 else if (is_cxgb3_dev(root_dev))
1418                         return root_dev;
1419                 else
1420                         return NULL;
1421         }
1422         return NULL;
1423 }
1424
1425 static struct rtable *find_route(__be32 saddr, __be32 daddr,
1426                                  __be16 sport, __be16 dport)
1427 {
1428         struct rtable *rt;
1429         struct flowi fl = {
1430                 .oif = 0,
1431                 .nl_u = {
1432                          .ip4_u = {
1433                                    .daddr = daddr,
1434                                    .saddr = saddr,
1435                                    .tos = 0 } },
1436                 .proto = IPPROTO_TCP,
1437                 .uli_u = {
1438                           .ports = {
1439                                     .sport = sport,
1440                                     .dport = dport } } };
1441
1442         if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0))
1443                 return NULL;
1444         return rt;
1445 }
1446
1447 /*
1448  * Assign offload parameters to some connection fields.
1449  */
1450 static void init_offload_conn(struct s3_conn *c3cn,
1451                               struct t3cdev *cdev,
1452                               struct dst_entry *dst)
1453 {
1454         BUG_ON(c3cn->cdev != cdev);
1455         c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs;
1456         c3cn->wr_unacked = 0;
1457         c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst));
1458
1459         reset_wr_list(c3cn);
1460 }
1461
1462 static int initiate_act_open(struct s3_conn *c3cn, struct net_device *dev)
1463 {
1464         struct cxgb3i_sdev_data *cdata = NDEV2CDATA(dev);
1465         struct t3cdev *cdev = cdata->cdev;
1466         struct dst_entry *dst = c3cn->dst_cache;
1467         struct sk_buff *skb;
1468
1469         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1470                         c3cn, c3cn->state, c3cn->flags);
1471         /*
1472          * Initialize connection data.  Note that the flags and ULP mode are
1473          * initialized higher up ...
1474          */
1475         c3cn->dev = dev;
1476         c3cn->cdev = cdev;
1477         c3cn->tid = cxgb3_alloc_atid(cdev, cdata->client, c3cn);
1478         if (c3cn->tid < 0)
1479                 goto out_err;
1480
1481         c3cn->qset = 0;
1482         c3cn->l2t = t3_l2t_get(cdev, dst->neighbour, dev);
1483         if (!c3cn->l2t)
1484                 goto free_tid;
1485
1486         skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_KERNEL);
1487         if (!skb)
1488                 goto free_l2t;
1489
1490         skb->sk = (struct sock *)c3cn;
1491         set_arp_failure_handler(skb, act_open_req_arp_failure);
1492
1493         c3cn_hold(c3cn);
1494
1495         init_offload_conn(c3cn, cdev, dst);
1496         c3cn->err = 0;
1497
1498         make_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
1499         l2t_send(cdev, skb, c3cn->l2t);
1500         return 0;
1501
1502 free_l2t:
1503         l2t_release(L2DATA(cdev), c3cn->l2t);
1504 free_tid:
1505         s3_free_atid(cdev, c3cn->tid);
1506         c3cn->tid = 0;
1507 out_err:
1508         return -1;
1509 }
1510
1511
1512 /**
1513  * cxgb3i_c3cn_connect - initiates an iscsi tcp connection to a given address
1514  * @c3cn: the iscsi tcp connection
1515  * @usin: destination address
1516  *
1517  * return 0 if active open request is sent, < 0 otherwise.
1518  */
1519 int cxgb3i_c3cn_connect(struct s3_conn *c3cn, struct sockaddr_in *usin)
1520 {
1521         struct rtable *rt;
1522         struct net_device *dev;
1523         struct cxgb3i_sdev_data *cdata;
1524         struct t3cdev *cdev;
1525         __be32 sipv4;
1526         int err;
1527
1528         if (usin->sin_family != AF_INET)
1529                 return -EAFNOSUPPORT;
1530
1531         c3cn->daddr.sin_port = usin->sin_port;
1532         c3cn->daddr.sin_addr.s_addr = usin->sin_addr.s_addr;
1533
1534         rt = find_route(c3cn->saddr.sin_addr.s_addr,
1535                         c3cn->daddr.sin_addr.s_addr,
1536                         c3cn->saddr.sin_port,
1537                         c3cn->daddr.sin_port);
1538         if (rt == NULL) {
1539                 c3cn_conn_debug("NO route to 0x%x, port %u.\n",
1540                                 c3cn->daddr.sin_addr.s_addr,
1541                                 ntohs(c3cn->daddr.sin_port));
1542                 return -ENETUNREACH;
1543         }
1544
1545         if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
1546                 c3cn_conn_debug("multi-cast route to 0x%x, port %u.\n",
1547                                 c3cn->daddr.sin_addr.s_addr,
1548                                 ntohs(c3cn->daddr.sin_port));
1549                 ip_rt_put(rt);
1550                 return -ENETUNREACH;
1551         }
1552
1553         if (!c3cn->saddr.sin_addr.s_addr)
1554                 c3cn->saddr.sin_addr.s_addr = rt->rt_src;
1555
1556         /* now commit destination to connection */
1557         c3cn->dst_cache = &rt->u.dst;
1558
1559         /* try to establish an offloaded connection */
1560         dev = cxgb3_egress_dev(c3cn->dst_cache->dev, c3cn, 0);
1561         if (dev == NULL) {
1562                 c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn);
1563                 return -ENETUNREACH;
1564         }
1565         cdata = NDEV2CDATA(dev);
1566         cdev = cdata->cdev;
1567
1568         /* get a source port if one hasn't been provided */
1569         err = c3cn_get_port(c3cn, cdata);
1570         if (err)
1571                 return err;
1572
1573         c3cn_conn_debug("c3cn 0x%p get port %u.\n",
1574                         c3cn, ntohs(c3cn->saddr.sin_port));
1575
1576         sipv4 = cxgb3i_get_private_ipv4addr(dev);
1577         if (!sipv4) {
1578                 c3cn_conn_debug("c3cn 0x%p, iscsi ip not configured.\n", c3cn);
1579                 sipv4 = c3cn->saddr.sin_addr.s_addr;
1580                 cxgb3i_set_private_ipv4addr(dev, sipv4);
1581         } else
1582                 c3cn->saddr.sin_addr.s_addr = sipv4;
1583
1584         c3cn_conn_debug("c3cn 0x%p, %u.%u.%u.%u,%u-%u.%u.%u.%u,%u SYN_SENT.\n",
1585                         c3cn, NIPQUAD(c3cn->saddr.sin_addr.s_addr),
1586                         ntohs(c3cn->saddr.sin_port),
1587                         NIPQUAD(c3cn->daddr.sin_addr.s_addr),
1588                         ntohs(c3cn->daddr.sin_port));
1589
1590         c3cn_set_state(c3cn, C3CN_STATE_CONNECTING);
1591         if (!initiate_act_open(c3cn, dev))
1592                 return 0;
1593
1594         /*
1595          * If we get here, we don't have an offload connection so simply
1596          * return a failure.
1597          */
1598         err = -ENOTSUPP;
1599
1600         /*
1601          * This trashes the connection and releases the local port,
1602          * if necessary.
1603          */
1604         c3cn_conn_debug("c3cn 0x%p -> CLOSED.\n", c3cn);
1605         c3cn_set_state(c3cn, C3CN_STATE_CLOSED);
1606         ip_rt_put(rt);
1607         c3cn_put_port(c3cn);
1608         c3cn->daddr.sin_port = 0;
1609         return err;
1610 }
1611
1612 /**
1613  * cxgb3i_c3cn_rx_credits - ack received tcp data.
1614  * @c3cn: iscsi tcp connection
1615  * @copied: # of bytes processed
1616  *
1617  * Called after some received data has been read.  It returns RX credits
1618  * to the HW for the amount of data processed.
1619  */
1620 void cxgb3i_c3cn_rx_credits(struct s3_conn *c3cn, int copied)
1621 {
1622         struct t3cdev *cdev;
1623         int must_send;
1624         u32 credits, dack = 0;
1625
1626         if (c3cn->state != C3CN_STATE_ESTABLISHED)
1627                 return;
1628
1629         credits = c3cn->copied_seq - c3cn->rcv_wup;
1630         if (unlikely(!credits))
1631                 return;
1632
1633         cdev = c3cn->cdev;
1634
1635         if (unlikely(cxgb3_rx_credit_thres == 0))
1636                 return;
1637
1638         dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
1639
1640         /*
1641          * For coalescing to work effectively ensure the receive window has
1642          * at least 16KB left.
1643          */
1644         must_send = credits + 16384 >= cxgb3_rcv_win;
1645
1646         if (must_send || credits >= cxgb3_rx_credit_thres)
1647                 c3cn->rcv_wup += send_rx_credits(c3cn, credits, dack);
1648 }
1649
1650 /**
1651  * cxgb3i_c3cn_send_pdus - send the skbs containing iscsi pdus
1652  * @c3cn: iscsi tcp connection
1653  * @skb: skb contains the iscsi pdu
1654  *
1655  * Add a list of skbs to a connection send queue. The skbs must comply with
1656  * the max size limit of the device and have a headroom of at least
1657  * TX_HEADER_LEN bytes.
1658  * Return # of bytes queued.
1659  */
1660 int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb)
1661 {
1662         struct sk_buff *next;
1663         int err, copied = 0;
1664
1665         spin_lock_bh(&c3cn->lock);
1666
1667         if (c3cn->state != C3CN_STATE_ESTABLISHED) {
1668                 c3cn_tx_debug("c3cn 0x%p, not in est. state %u.\n",
1669                               c3cn, c3cn->state);
1670                 err = -EAGAIN;
1671                 goto out_err;
1672         }
1673
1674         err = -EPIPE;
1675         if (c3cn->err) {
1676                 c3cn_tx_debug("c3cn 0x%p, err %d.\n", c3cn, c3cn->err);
1677                 goto out_err;
1678         }
1679
1680         while (skb) {
1681                 int frags = skb_shinfo(skb)->nr_frags +
1682                                 (skb->len != skb->data_len);
1683
1684                 if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) {
1685                         c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn);
1686                         err = -EINVAL;
1687                         goto out_err;
1688                 }
1689
1690                 if (frags >= SKB_WR_LIST_SIZE) {
1691                         cxgb3i_log_error("c3cn 0x%p, tx frags %d, len %u,%u.\n",
1692                                          c3cn, skb_shinfo(skb)->nr_frags,
1693                                          skb->len, skb->data_len);
1694                         err = -EINVAL;
1695                         goto out_err;
1696                 }
1697
1698                 next = skb->next;
1699                 skb->next = NULL;
1700                 skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR);
1701                 copied += skb->len;
1702                 c3cn->write_seq += skb->len + ulp_extra_len(skb);
1703                 skb = next;
1704         }
1705 done:
1706         if (likely(skb_queue_len(&c3cn->write_queue)))
1707                 c3cn_push_tx_frames(c3cn, 1);
1708         spin_unlock_bh(&c3cn->lock);
1709         return copied;
1710
1711 out_err:
1712         if (copied == 0 && err == -EPIPE)
1713                 copied = c3cn->err ? c3cn->err : -EPIPE;
1714         goto done;
1715 }
1716
1717 static void sdev_data_cleanup(struct cxgb3i_sdev_data *cdata)
1718 {
1719         struct adap_ports *ports = &cdata->ports;
1720         int i;
1721
1722         for (i = 0; i < ports->nports; i++)
1723                 NDEV2CDATA(ports->lldevs[i]) = NULL;
1724         cxgb3i_free_big_mem(cdata);
1725 }
1726
1727 void cxgb3i_sdev_cleanup(void)
1728 {
1729         struct cxgb3i_sdev_data *cdata;
1730
1731         write_lock(&cdata_rwlock);
1732         list_for_each_entry(cdata, &cdata_list, list) {
1733                 list_del(&cdata->list);
1734                 sdev_data_cleanup(cdata);
1735         }
1736         write_unlock(&cdata_rwlock);
1737 }
1738
1739 int cxgb3i_sdev_init(cxgb3_cpl_handler_func *cpl_handlers)
1740 {
1741         cpl_handlers[CPL_ACT_ESTABLISH] = do_act_establish;
1742         cpl_handlers[CPL_ACT_OPEN_RPL] = do_act_open_rpl;
1743         cpl_handlers[CPL_PEER_CLOSE] = do_peer_close;
1744         cpl_handlers[CPL_ABORT_REQ_RSS] = do_abort_req;
1745         cpl_handlers[CPL_ABORT_RPL_RSS] = do_abort_rpl;
1746         cpl_handlers[CPL_CLOSE_CON_RPL] = do_close_con_rpl;
1747         cpl_handlers[CPL_TX_DMA_ACK] = do_wr_ack;
1748         cpl_handlers[CPL_ISCSI_HDR] = do_iscsi_hdr;
1749
1750         if (cxgb3_max_connect > CXGB3I_MAX_CONN)
1751                 cxgb3_max_connect = CXGB3I_MAX_CONN;
1752         return 0;
1753 }
1754
1755 /**
1756  * cxgb3i_sdev_add - allocate and initialize resources for each adapter found
1757  * @cdev:       t3cdev adapter
1758  * @client:     cxgb3 driver client
1759  */
1760 void cxgb3i_sdev_add(struct t3cdev *cdev, struct cxgb3_client *client)
1761 {
1762         struct cxgb3i_sdev_data *cdata;
1763         struct ofld_page_info rx_page_info;
1764         unsigned int wr_len;
1765         int mapsize = DIV_ROUND_UP(cxgb3_max_connect,
1766                                    8 * sizeof(unsigned long));
1767         int i;
1768
1769         cdata =  cxgb3i_alloc_big_mem(sizeof(*cdata) + mapsize, GFP_KERNEL);
1770         if (!cdata)
1771                 return;
1772
1773         if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0 ||
1774             cdev->ctl(cdev, GET_PORTS, &cdata->ports) < 0 ||
1775             cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info) < 0)
1776                 goto free_cdata;
1777
1778         s3_init_wr_tab(wr_len);
1779
1780         INIT_LIST_HEAD(&cdata->list);
1781         cdata->cdev = cdev;
1782         cdata->client = client;
1783
1784         for (i = 0; i < cdata->ports.nports; i++)
1785                 NDEV2CDATA(cdata->ports.lldevs[i]) = cdata;
1786
1787         write_lock(&cdata_rwlock);
1788         list_add_tail(&cdata->list, &cdata_list);
1789         write_unlock(&cdata_rwlock);
1790
1791         return;
1792
1793 free_cdata:
1794         cxgb3i_free_big_mem(cdata);
1795 }
1796
1797 /**
1798  * cxgb3i_sdev_remove - free the allocated resources for the adapter
1799  * @cdev:       t3cdev adapter
1800  */
1801 void cxgb3i_sdev_remove(struct t3cdev *cdev)
1802 {
1803         struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
1804
1805         write_lock(&cdata_rwlock);
1806         list_del(&cdata->list);
1807         write_unlock(&cdata_rwlock);
1808
1809         sdev_data_cleanup(cdata);
1810 }