Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ieee1394...
[linux-2.6] / drivers / net / mlx4 / main.c
1 /*
2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5  * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/errno.h>
39 #include <linux/pci.h>
40 #include <linux/dma-mapping.h>
41
42 #include <linux/mlx4/device.h>
43 #include <linux/mlx4/doorbell.h>
44
45 #include "mlx4.h"
46 #include "fw.h"
47 #include "icm.h"
48
49 MODULE_AUTHOR("Roland Dreier");
50 MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver");
51 MODULE_LICENSE("Dual BSD/GPL");
52 MODULE_VERSION(DRV_VERSION);
53
54 #ifdef CONFIG_MLX4_DEBUG
55
56 int mlx4_debug_level = 0;
57 module_param_named(debug_level, mlx4_debug_level, int, 0644);
58 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
59
60 #endif /* CONFIG_MLX4_DEBUG */
61
62 #ifdef CONFIG_PCI_MSI
63
64 static int msi_x;
65 module_param(msi_x, int, 0444);
66 MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");
67
68 #else /* CONFIG_PCI_MSI */
69
70 #define msi_x (0)
71
72 #endif /* CONFIG_PCI_MSI */
73
74 static const char mlx4_version[] __devinitdata =
75         DRV_NAME ": Mellanox ConnectX core driver v"
76         DRV_VERSION " (" DRV_RELDATE ")\n";
77
78 static struct mlx4_profile default_profile = {
79         .num_qp         = 1 << 16,
80         .num_srq        = 1 << 16,
81         .rdmarc_per_qp  = 4,
82         .num_cq         = 1 << 16,
83         .num_mcg        = 1 << 13,
84         .num_mpt        = 1 << 17,
85         .num_mtt        = 1 << 20,
86 };
87
88 static int __devinit mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
89 {
90         int err;
91         int i;
92
93         err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
94         if (err) {
95                 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
96                 return err;
97         }
98
99         if (dev_cap->min_page_sz > PAGE_SIZE) {
100                 mlx4_err(dev, "HCA minimum page size of %d bigger than "
101                          "kernel PAGE_SIZE of %ld, aborting.\n",
102                          dev_cap->min_page_sz, PAGE_SIZE);
103                 return -ENODEV;
104         }
105         if (dev_cap->num_ports > MLX4_MAX_PORTS) {
106                 mlx4_err(dev, "HCA has %d ports, but we only support %d, "
107                          "aborting.\n",
108                          dev_cap->num_ports, MLX4_MAX_PORTS);
109                 return -ENODEV;
110         }
111
112         if (dev_cap->uar_size > pci_resource_len(dev->pdev, 2)) {
113                 mlx4_err(dev, "HCA reported UAR size of 0x%x bigger than "
114                          "PCI resource 2 size of 0x%llx, aborting.\n",
115                          dev_cap->uar_size,
116                          (unsigned long long) pci_resource_len(dev->pdev, 2));
117                 return -ENODEV;
118         }
119
120         dev->caps.num_ports          = dev_cap->num_ports;
121         for (i = 1; i <= dev->caps.num_ports; ++i) {
122                 dev->caps.vl_cap[i]         = dev_cap->max_vl[i];
123                 dev->caps.mtu_cap[i]        = dev_cap->max_mtu[i];
124                 dev->caps.gid_table_len[i]  = dev_cap->max_gids[i];
125                 dev->caps.pkey_table_len[i] = dev_cap->max_pkeys[i];
126                 dev->caps.port_width_cap[i] = dev_cap->max_port_width[i];
127         }
128
129         dev->caps.num_uars           = dev_cap->uar_size / PAGE_SIZE;
130         dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay;
131         dev->caps.bf_reg_size        = dev_cap->bf_reg_size;
132         dev->caps.bf_regs_per_page   = dev_cap->bf_regs_per_page;
133         dev->caps.max_sq_sg          = dev_cap->max_sq_sg;
134         dev->caps.max_rq_sg          = dev_cap->max_rq_sg;
135         dev->caps.max_wqes           = dev_cap->max_qp_sz;
136         dev->caps.max_qp_init_rdma   = dev_cap->max_requester_per_qp;
137         dev->caps.reserved_qps       = dev_cap->reserved_qps;
138         dev->caps.max_srq_wqes       = dev_cap->max_srq_sz;
139         dev->caps.max_srq_sge        = dev_cap->max_rq_sg - 1;
140         dev->caps.reserved_srqs      = dev_cap->reserved_srqs;
141         dev->caps.max_sq_desc_sz     = dev_cap->max_sq_desc_sz;
142         dev->caps.max_rq_desc_sz     = dev_cap->max_rq_desc_sz;
143         dev->caps.num_qp_per_mgm     = MLX4_QP_PER_MGM;
144         /*
145          * Subtract 1 from the limit because we need to allocate a
146          * spare CQE so the HCA HW can tell the difference between an
147          * empty CQ and a full CQ.
148          */
149         dev->caps.max_cqes           = dev_cap->max_cq_sz - 1;
150         dev->caps.reserved_cqs       = dev_cap->reserved_cqs;
151         dev->caps.reserved_eqs       = dev_cap->reserved_eqs;
152         dev->caps.reserved_mtts      = dev_cap->reserved_mtts;
153         dev->caps.reserved_mrws      = dev_cap->reserved_mrws;
154         dev->caps.reserved_uars      = dev_cap->reserved_uars;
155         dev->caps.reserved_pds       = dev_cap->reserved_pds;
156         dev->caps.mtt_entry_sz       = MLX4_MTT_ENTRY_PER_SEG * dev_cap->mtt_entry_sz;
157         dev->caps.page_size_cap      = ~(u32) (dev_cap->min_page_sz - 1);
158         dev->caps.flags              = dev_cap->flags;
159         dev->caps.stat_rate_support  = dev_cap->stat_rate_support;
160
161         return 0;
162 }
163
164 static int __devinit mlx4_load_fw(struct mlx4_dev *dev)
165 {
166         struct mlx4_priv *priv = mlx4_priv(dev);
167         int err;
168
169         priv->fw.fw_icm = mlx4_alloc_icm(dev, priv->fw.fw_pages,
170                                          GFP_HIGHUSER | __GFP_NOWARN);
171         if (!priv->fw.fw_icm) {
172                 mlx4_err(dev, "Couldn't allocate FW area, aborting.\n");
173                 return -ENOMEM;
174         }
175
176         err = mlx4_MAP_FA(dev, priv->fw.fw_icm);
177         if (err) {
178                 mlx4_err(dev, "MAP_FA command failed, aborting.\n");
179                 goto err_free;
180         }
181
182         err = mlx4_RUN_FW(dev);
183         if (err) {
184                 mlx4_err(dev, "RUN_FW command failed, aborting.\n");
185                 goto err_unmap_fa;
186         }
187
188         return 0;
189
190 err_unmap_fa:
191         mlx4_UNMAP_FA(dev);
192
193 err_free:
194         mlx4_free_icm(dev, priv->fw.fw_icm);
195         return err;
196 }
197
198 static int __devinit mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base,
199                                           int cmpt_entry_sz)
200 {
201         struct mlx4_priv *priv = mlx4_priv(dev);
202         int err;
203
204         err = mlx4_init_icm_table(dev, &priv->qp_table.cmpt_table,
205                                   cmpt_base +
206                                   ((u64) (MLX4_CMPT_TYPE_QP *
207                                           cmpt_entry_sz) << MLX4_CMPT_SHIFT),
208                                   cmpt_entry_sz, dev->caps.num_qps,
209                                   dev->caps.reserved_qps, 0);
210         if (err)
211                 goto err;
212
213         err = mlx4_init_icm_table(dev, &priv->srq_table.cmpt_table,
214                                   cmpt_base +
215                                   ((u64) (MLX4_CMPT_TYPE_SRQ *
216                                           cmpt_entry_sz) << MLX4_CMPT_SHIFT),
217                                   cmpt_entry_sz, dev->caps.num_srqs,
218                                   dev->caps.reserved_srqs, 0);
219         if (err)
220                 goto err_qp;
221
222         err = mlx4_init_icm_table(dev, &priv->cq_table.cmpt_table,
223                                   cmpt_base +
224                                   ((u64) (MLX4_CMPT_TYPE_CQ *
225                                           cmpt_entry_sz) << MLX4_CMPT_SHIFT),
226                                   cmpt_entry_sz, dev->caps.num_cqs,
227                                   dev->caps.reserved_cqs, 0);
228         if (err)
229                 goto err_srq;
230
231         err = mlx4_init_icm_table(dev, &priv->eq_table.cmpt_table,
232                                   cmpt_base +
233                                   ((u64) (MLX4_CMPT_TYPE_EQ *
234                                           cmpt_entry_sz) << MLX4_CMPT_SHIFT),
235                                   cmpt_entry_sz,
236                                   roundup_pow_of_two(MLX4_NUM_EQ +
237                                                      dev->caps.reserved_eqs),
238                                   MLX4_NUM_EQ + dev->caps.reserved_eqs, 0);
239         if (err)
240                 goto err_cq;
241
242         return 0;
243
244 err_cq:
245         mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
246
247 err_srq:
248         mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
249
250 err_qp:
251         mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
252
253 err:
254         return err;
255 }
256
257 static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
258                                    struct mlx4_dev_cap *dev_cap,
259                                    struct mlx4_init_hca_param *init_hca,
260                                    u64 icm_size)
261 {
262         struct mlx4_priv *priv = mlx4_priv(dev);
263         u64 aux_pages;
264         int err;
265
266         err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages);
267         if (err) {
268                 mlx4_err(dev, "SET_ICM_SIZE command failed, aborting.\n");
269                 return err;
270         }
271
272         mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory.\n",
273                  (unsigned long long) icm_size >> 10,
274                  (unsigned long long) aux_pages << 2);
275
276         priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages,
277                                           GFP_HIGHUSER | __GFP_NOWARN);
278         if (!priv->fw.aux_icm) {
279                 mlx4_err(dev, "Couldn't allocate aux memory, aborting.\n");
280                 return -ENOMEM;
281         }
282
283         err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm);
284         if (err) {
285                 mlx4_err(dev, "MAP_ICM_AUX command failed, aborting.\n");
286                 goto err_free_aux;
287         }
288
289         err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz);
290         if (err) {
291                 mlx4_err(dev, "Failed to map cMPT context memory, aborting.\n");
292                 goto err_unmap_aux;
293         }
294
295         err = mlx4_map_eq_icm(dev, init_hca->eqc_base);
296         if (err) {
297                 mlx4_err(dev, "Failed to map EQ context memory, aborting.\n");
298                 goto err_unmap_cmpt;
299         }
300
301         err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table,
302                                   init_hca->mtt_base,
303                                   dev->caps.mtt_entry_sz,
304                                   dev->caps.num_mtt_segs,
305                                   dev->caps.reserved_mtts, 1);
306         if (err) {
307                 mlx4_err(dev, "Failed to map MTT context memory, aborting.\n");
308                 goto err_unmap_eq;
309         }
310
311         err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table,
312                                   init_hca->dmpt_base,
313                                   dev_cap->dmpt_entry_sz,
314                                   dev->caps.num_mpts,
315                                   dev->caps.reserved_mrws, 1);
316         if (err) {
317                 mlx4_err(dev, "Failed to map dMPT context memory, aborting.\n");
318                 goto err_unmap_mtt;
319         }
320
321         err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table,
322                                   init_hca->qpc_base,
323                                   dev_cap->qpc_entry_sz,
324                                   dev->caps.num_qps,
325                                   dev->caps.reserved_qps, 0);
326         if (err) {
327                 mlx4_err(dev, "Failed to map QP context memory, aborting.\n");
328                 goto err_unmap_dmpt;
329         }
330
331         err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table,
332                                   init_hca->auxc_base,
333                                   dev_cap->aux_entry_sz,
334                                   dev->caps.num_qps,
335                                   dev->caps.reserved_qps, 0);
336         if (err) {
337                 mlx4_err(dev, "Failed to map AUXC context memory, aborting.\n");
338                 goto err_unmap_qp;
339         }
340
341         err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table,
342                                   init_hca->altc_base,
343                                   dev_cap->altc_entry_sz,
344                                   dev->caps.num_qps,
345                                   dev->caps.reserved_qps, 0);
346         if (err) {
347                 mlx4_err(dev, "Failed to map ALTC context memory, aborting.\n");
348                 goto err_unmap_auxc;
349         }
350
351         err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table,
352                                   init_hca->rdmarc_base,
353                                   dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift,
354                                   dev->caps.num_qps,
355                                   dev->caps.reserved_qps, 0);
356         if (err) {
357                 mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n");
358                 goto err_unmap_altc;
359         }
360
361         err = mlx4_init_icm_table(dev, &priv->cq_table.table,
362                                   init_hca->cqc_base,
363                                   dev_cap->cqc_entry_sz,
364                                   dev->caps.num_cqs,
365                                   dev->caps.reserved_cqs, 0);
366         if (err) {
367                 mlx4_err(dev, "Failed to map CQ context memory, aborting.\n");
368                 goto err_unmap_rdmarc;
369         }
370
371         err = mlx4_init_icm_table(dev, &priv->srq_table.table,
372                                   init_hca->srqc_base,
373                                   dev_cap->srq_entry_sz,
374                                   dev->caps.num_srqs,
375                                   dev->caps.reserved_srqs, 0);
376         if (err) {
377                 mlx4_err(dev, "Failed to map SRQ context memory, aborting.\n");
378                 goto err_unmap_cq;
379         }
380
381         /*
382          * It's not strictly required, but for simplicity just map the
383          * whole multicast group table now.  The table isn't very big
384          * and it's a lot easier than trying to track ref counts.
385          */
386         err = mlx4_init_icm_table(dev, &priv->mcg_table.table,
387                                   init_hca->mc_base, MLX4_MGM_ENTRY_SIZE,
388                                   dev->caps.num_mgms + dev->caps.num_amgms,
389                                   dev->caps.num_mgms + dev->caps.num_amgms,
390                                   0);
391         if (err) {
392                 mlx4_err(dev, "Failed to map MCG context memory, aborting.\n");
393                 goto err_unmap_srq;
394         }
395
396         return 0;
397
398 err_unmap_srq:
399         mlx4_cleanup_icm_table(dev, &priv->srq_table.table);
400
401 err_unmap_cq:
402         mlx4_cleanup_icm_table(dev, &priv->cq_table.table);
403
404 err_unmap_rdmarc:
405         mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);
406
407 err_unmap_altc:
408         mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);
409
410 err_unmap_auxc:
411         mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);
412
413 err_unmap_qp:
414         mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);
415
416 err_unmap_dmpt:
417         mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);
418
419 err_unmap_mtt:
420         mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);
421
422 err_unmap_eq:
423         mlx4_unmap_eq_icm(dev);
424
425 err_unmap_cmpt:
426         mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);
427         mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
428         mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
429         mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
430
431 err_unmap_aux:
432         mlx4_UNMAP_ICM_AUX(dev);
433
434 err_free_aux:
435         mlx4_free_icm(dev, priv->fw.aux_icm);
436
437         return err;
438 }
439
440 static void mlx4_free_icms(struct mlx4_dev *dev)
441 {
442         struct mlx4_priv *priv = mlx4_priv(dev);
443
444         mlx4_cleanup_icm_table(dev, &priv->mcg_table.table);
445         mlx4_cleanup_icm_table(dev, &priv->srq_table.table);
446         mlx4_cleanup_icm_table(dev, &priv->cq_table.table);
447         mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);
448         mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);
449         mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);
450         mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);
451         mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);
452         mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);
453         mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);
454         mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
455         mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
456         mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
457         mlx4_unmap_eq_icm(dev);
458
459         mlx4_UNMAP_ICM_AUX(dev);
460         mlx4_free_icm(dev, priv->fw.aux_icm);
461 }
462
463 static void mlx4_close_hca(struct mlx4_dev *dev)
464 {
465         mlx4_CLOSE_HCA(dev, 0);
466         mlx4_free_icms(dev);
467         mlx4_UNMAP_FA(dev);
468         mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm);
469 }
470
471 static int __devinit mlx4_init_hca(struct mlx4_dev *dev)
472 {
473         struct mlx4_priv          *priv = mlx4_priv(dev);
474         struct mlx4_adapter        adapter;
475         struct mlx4_dev_cap        dev_cap;
476         struct mlx4_profile        profile;
477         struct mlx4_init_hca_param init_hca;
478         u64 icm_size;
479         int err;
480
481         err = mlx4_QUERY_FW(dev);
482         if (err) {
483                 mlx4_err(dev, "QUERY_FW command failed, aborting.\n");
484                 return err;
485         }
486
487         err = mlx4_load_fw(dev);
488         if (err) {
489                 mlx4_err(dev, "Failed to start FW, aborting.\n");
490                 return err;
491         }
492
493         err = mlx4_dev_cap(dev, &dev_cap);
494         if (err) {
495                 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
496                 goto err_stop_fw;
497         }
498
499         profile = default_profile;
500
501         icm_size = mlx4_make_profile(dev, &profile, &dev_cap, &init_hca);
502         if ((long long) icm_size < 0) {
503                 err = icm_size;
504                 goto err_stop_fw;
505         }
506
507         init_hca.log_uar_sz = ilog2(dev->caps.num_uars);
508
509         err = mlx4_init_icm(dev, &dev_cap, &init_hca, icm_size);
510         if (err)
511                 goto err_stop_fw;
512
513         err = mlx4_INIT_HCA(dev, &init_hca);
514         if (err) {
515                 mlx4_err(dev, "INIT_HCA command failed, aborting.\n");
516                 goto err_free_icm;
517         }
518
519         err = mlx4_QUERY_ADAPTER(dev, &adapter);
520         if (err) {
521                 mlx4_err(dev, "QUERY_ADAPTER command failed, aborting.\n");
522                 goto err_close;
523         }
524
525         priv->eq_table.inta_pin = adapter.inta_pin;
526         priv->rev_id            = adapter.revision_id;
527         memcpy(priv->board_id, adapter.board_id, sizeof priv->board_id);
528
529         return 0;
530
531 err_close:
532         mlx4_close_hca(dev);
533
534 err_free_icm:
535         mlx4_free_icms(dev);
536
537 err_stop_fw:
538         mlx4_UNMAP_FA(dev);
539         mlx4_free_icm(dev, priv->fw.fw_icm);
540
541         return err;
542 }
543
544 static int __devinit mlx4_setup_hca(struct mlx4_dev *dev)
545 {
546         struct mlx4_priv *priv = mlx4_priv(dev);
547         int err;
548
549         err = mlx4_init_uar_table(dev);
550         if (err) {
551                 mlx4_err(dev, "Failed to initialize "
552                          "user access region table, aborting.\n");
553                 return err;
554         }
555
556         err = mlx4_uar_alloc(dev, &priv->driver_uar);
557         if (err) {
558                 mlx4_err(dev, "Failed to allocate driver access region, "
559                          "aborting.\n");
560                 goto err_uar_table_free;
561         }
562
563         priv->kar = ioremap(priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
564         if (!priv->kar) {
565                 mlx4_err(dev, "Couldn't map kernel access region, "
566                          "aborting.\n");
567                 err = -ENOMEM;
568                 goto err_uar_free;
569         }
570
571         err = mlx4_init_pd_table(dev);
572         if (err) {
573                 mlx4_err(dev, "Failed to initialize "
574                          "protection domain table, aborting.\n");
575                 goto err_kar_unmap;
576         }
577
578         err = mlx4_init_mr_table(dev);
579         if (err) {
580                 mlx4_err(dev, "Failed to initialize "
581                          "memory region table, aborting.\n");
582                 goto err_pd_table_free;
583         }
584
585         mlx4_map_catas_buf(dev);
586
587         err = mlx4_init_eq_table(dev);
588         if (err) {
589                 mlx4_err(dev, "Failed to initialize "
590                          "event queue table, aborting.\n");
591                 goto err_catas_buf;
592         }
593
594         err = mlx4_cmd_use_events(dev);
595         if (err) {
596                 mlx4_err(dev, "Failed to switch to event-driven "
597                          "firmware commands, aborting.\n");
598                 goto err_eq_table_free;
599         }
600
601         err = mlx4_NOP(dev);
602         if (err) {
603                 mlx4_err(dev, "NOP command failed to generate interrupt "
604                          "(IRQ %d), aborting.\n",
605                          priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
606                 if (dev->flags & MLX4_FLAG_MSI_X)
607                         mlx4_err(dev, "Try again with MSI-X disabled.\n");
608                 else
609                         mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n");
610
611                 goto err_cmd_poll;
612         }
613
614         mlx4_dbg(dev, "NOP command IRQ test passed\n");
615
616         err = mlx4_init_cq_table(dev);
617         if (err) {
618                 mlx4_err(dev, "Failed to initialize "
619                          "completion queue table, aborting.\n");
620                 goto err_cmd_poll;
621         }
622
623         err = mlx4_init_srq_table(dev);
624         if (err) {
625                 mlx4_err(dev, "Failed to initialize "
626                          "shared receive queue table, aborting.\n");
627                 goto err_cq_table_free;
628         }
629
630         err = mlx4_init_qp_table(dev);
631         if (err) {
632                 mlx4_err(dev, "Failed to initialize "
633                          "queue pair table, aborting.\n");
634                 goto err_srq_table_free;
635         }
636
637         err = mlx4_init_mcg_table(dev);
638         if (err) {
639                 mlx4_err(dev, "Failed to initialize "
640                          "multicast group table, aborting.\n");
641                 goto err_qp_table_free;
642         }
643
644         return 0;
645
646 err_qp_table_free:
647         mlx4_cleanup_qp_table(dev);
648
649 err_srq_table_free:
650         mlx4_cleanup_srq_table(dev);
651
652 err_cq_table_free:
653         mlx4_cleanup_cq_table(dev);
654
655 err_cmd_poll:
656         mlx4_cmd_use_polling(dev);
657
658 err_eq_table_free:
659         mlx4_cleanup_eq_table(dev);
660
661 err_catas_buf:
662         mlx4_unmap_catas_buf(dev);
663         mlx4_cleanup_mr_table(dev);
664
665 err_pd_table_free:
666         mlx4_cleanup_pd_table(dev);
667
668 err_kar_unmap:
669         iounmap(priv->kar);
670
671 err_uar_free:
672         mlx4_uar_free(dev, &priv->driver_uar);
673
674 err_uar_table_free:
675         mlx4_cleanup_uar_table(dev);
676         return err;
677 }
678
679 static void __devinit mlx4_enable_msi_x(struct mlx4_dev *dev)
680 {
681         struct mlx4_priv *priv = mlx4_priv(dev);
682         struct msix_entry entries[MLX4_NUM_EQ];
683         int err;
684         int i;
685
686         if (msi_x) {
687                 for (i = 0; i < MLX4_NUM_EQ; ++i)
688                         entries[i].entry = i;
689
690                 err = pci_enable_msix(dev->pdev, entries, ARRAY_SIZE(entries));
691                 if (err) {
692                         if (err > 0)
693                                 mlx4_info(dev, "Only %d MSI-X vectors available, "
694                                           "not using MSI-X\n", err);
695                         goto no_msi;
696                 }
697
698                 for (i = 0; i < MLX4_NUM_EQ; ++i)
699                         priv->eq_table.eq[i].irq = entries[i].vector;
700
701                 dev->flags |= MLX4_FLAG_MSI_X;
702                 return;
703         }
704
705 no_msi:
706         for (i = 0; i < MLX4_NUM_EQ; ++i)
707                 priv->eq_table.eq[i].irq = dev->pdev->irq;
708 }
709
710 static int __devinit mlx4_init_one(struct pci_dev *pdev,
711                                    const struct pci_device_id *id)
712 {
713         static int mlx4_version_printed;
714         struct mlx4_priv *priv;
715         struct mlx4_dev *dev;
716         int err;
717
718         if (!mlx4_version_printed) {
719                 printk(KERN_INFO "%s", mlx4_version);
720                 ++mlx4_version_printed;
721         }
722
723         printk(KERN_INFO PFX "Initializing %s\n",
724                pci_name(pdev));
725
726         err = pci_enable_device(pdev);
727         if (err) {
728                 dev_err(&pdev->dev, "Cannot enable PCI device, "
729                         "aborting.\n");
730                 return err;
731         }
732
733         /*
734          * Check for BARs.  We expect 0: 1MB, 2: 8MB, 4: DDR (may not
735          * be present)
736          */
737         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
738             pci_resource_len(pdev, 0) != 1 << 20) {
739                 dev_err(&pdev->dev, "Missing DCS, aborting.\n");
740                 err = -ENODEV;
741                 goto err_disable_pdev;
742         }
743         if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
744                 dev_err(&pdev->dev, "Missing UAR, aborting.\n");
745                 err = -ENODEV;
746                 goto err_disable_pdev;
747         }
748
749         err = pci_request_region(pdev, 0, DRV_NAME);
750         if (err) {
751                 dev_err(&pdev->dev, "Cannot request control region, aborting.\n");
752                 goto err_disable_pdev;
753         }
754
755         err = pci_request_region(pdev, 2, DRV_NAME);
756         if (err) {
757                 dev_err(&pdev->dev, "Cannot request UAR region, aborting.\n");
758                 goto err_release_bar0;
759         }
760
761         pci_set_master(pdev);
762
763         err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
764         if (err) {
765                 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n");
766                 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
767                 if (err) {
768                         dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n");
769                         goto err_release_bar2;
770                 }
771         }
772         err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
773         if (err) {
774                 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit "
775                          "consistent PCI DMA mask.\n");
776                 err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
777                 if (err) {
778                         dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, "
779                                 "aborting.\n");
780                         goto err_release_bar2;
781                 }
782         }
783
784         priv = kzalloc(sizeof *priv, GFP_KERNEL);
785         if (!priv) {
786                 dev_err(&pdev->dev, "Device struct alloc failed, "
787                         "aborting.\n");
788                 err = -ENOMEM;
789                 goto err_release_bar2;
790         }
791
792         dev       = &priv->dev;
793         dev->pdev = pdev;
794         INIT_LIST_HEAD(&priv->ctx_list);
795         spin_lock_init(&priv->ctx_lock);
796
797         /*
798          * Now reset the HCA before we touch the PCI capabilities or
799          * attempt a firmware command, since a boot ROM may have left
800          * the HCA in an undefined state.
801          */
802         err = mlx4_reset(dev);
803         if (err) {
804                 mlx4_err(dev, "Failed to reset HCA, aborting.\n");
805                 goto err_free_dev;
806         }
807
808         mlx4_enable_msi_x(dev);
809
810         if (mlx4_cmd_init(dev)) {
811                 mlx4_err(dev, "Failed to init command interface, aborting.\n");
812                 goto err_free_dev;
813         }
814
815         err = mlx4_init_hca(dev);
816         if (err)
817                 goto err_cmd;
818
819         err = mlx4_setup_hca(dev);
820         if (err)
821                 goto err_close;
822
823         err = mlx4_register_device(dev);
824         if (err)
825                 goto err_cleanup;
826
827         pci_set_drvdata(pdev, dev);
828
829         return 0;
830
831 err_cleanup:
832         mlx4_cleanup_mcg_table(dev);
833         mlx4_cleanup_qp_table(dev);
834         mlx4_cleanup_srq_table(dev);
835         mlx4_cleanup_cq_table(dev);
836         mlx4_cmd_use_polling(dev);
837         mlx4_cleanup_eq_table(dev);
838
839         mlx4_unmap_catas_buf(dev);
840
841         mlx4_cleanup_mr_table(dev);
842         mlx4_cleanup_pd_table(dev);
843         mlx4_cleanup_uar_table(dev);
844
845 err_close:
846         mlx4_close_hca(dev);
847
848 err_cmd:
849         mlx4_cmd_cleanup(dev);
850
851 err_free_dev:
852         if (dev->flags & MLX4_FLAG_MSI_X)
853                 pci_disable_msix(pdev);
854
855         kfree(priv);
856
857 err_release_bar2:
858         pci_release_region(pdev, 2);
859
860 err_release_bar0:
861         pci_release_region(pdev, 0);
862
863 err_disable_pdev:
864         pci_disable_device(pdev);
865         pci_set_drvdata(pdev, NULL);
866         return err;
867 }
868
869 static void __devexit mlx4_remove_one(struct pci_dev *pdev)
870 {
871         struct mlx4_dev  *dev  = pci_get_drvdata(pdev);
872         struct mlx4_priv *priv = mlx4_priv(dev);
873         int p;
874
875         if (dev) {
876                 mlx4_unregister_device(dev);
877
878                 for (p = 1; p <= dev->caps.num_ports; ++p)
879                         mlx4_CLOSE_PORT(dev, p);
880
881                 mlx4_cleanup_mcg_table(dev);
882                 mlx4_cleanup_qp_table(dev);
883                 mlx4_cleanup_srq_table(dev);
884                 mlx4_cleanup_cq_table(dev);
885                 mlx4_cmd_use_polling(dev);
886                 mlx4_cleanup_eq_table(dev);
887
888                 mlx4_unmap_catas_buf(dev);
889
890                 mlx4_cleanup_mr_table(dev);
891                 mlx4_cleanup_pd_table(dev);
892
893                 iounmap(priv->kar);
894                 mlx4_uar_free(dev, &priv->driver_uar);
895                 mlx4_cleanup_uar_table(dev);
896                 mlx4_close_hca(dev);
897                 mlx4_cmd_cleanup(dev);
898
899                 if (dev->flags & MLX4_FLAG_MSI_X)
900                         pci_disable_msix(pdev);
901
902                 kfree(priv);
903                 pci_release_region(pdev, 2);
904                 pci_release_region(pdev, 0);
905                 pci_disable_device(pdev);
906                 pci_set_drvdata(pdev, NULL);
907         }
908 }
909
910 static struct pci_device_id mlx4_pci_table[] = {
911         { PCI_VDEVICE(MELLANOX, 0x6340) }, /* MT25408 "Hermon" SDR */
912         { PCI_VDEVICE(MELLANOX, 0x634a) }, /* MT25408 "Hermon" DDR */
913         { PCI_VDEVICE(MELLANOX, 0x6354) }, /* MT25408 "Hermon" QDR */
914         { 0, }
915 };
916
917 MODULE_DEVICE_TABLE(pci, mlx4_pci_table);
918
919 static struct pci_driver mlx4_driver = {
920         .name           = DRV_NAME,
921         .id_table       = mlx4_pci_table,
922         .probe          = mlx4_init_one,
923         .remove         = __devexit_p(mlx4_remove_one)
924 };
925
926 static int __init mlx4_init(void)
927 {
928         int ret;
929
930         ret = pci_register_driver(&mlx4_driver);
931         return ret < 0 ? ret : 0;
932 }
933
934 static void __exit mlx4_cleanup(void)
935 {
936         pci_unregister_driver(&mlx4_driver);
937 }
938
939 module_init(mlx4_init);
940 module_exit(mlx4_cleanup);