[IB] mthca: report page size capability
[linux-2.6] / drivers / infiniband / hw / mthca / mthca_main.c
1 /*
2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  * $Id: mthca_main.c 1396 2004-12-28 04:10:27Z roland $
35  */
36
37 #include <linux/config.h>
38 #include <linux/module.h>
39 #include <linux/init.h>
40 #include <linux/errno.h>
41 #include <linux/pci.h>
42 #include <linux/interrupt.h>
43
44 #include "mthca_dev.h"
45 #include "mthca_config_reg.h"
46 #include "mthca_cmd.h"
47 #include "mthca_profile.h"
48 #include "mthca_memfree.h"
49
50 MODULE_AUTHOR("Roland Dreier");
51 MODULE_DESCRIPTION("Mellanox InfiniBand HCA low-level driver");
52 MODULE_LICENSE("Dual BSD/GPL");
53 MODULE_VERSION(DRV_VERSION);
54
55 #ifdef CONFIG_PCI_MSI
56
57 static int msi_x = 0;
58 module_param(msi_x, int, 0444);
59 MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");
60
61 static int msi = 0;
62 module_param(msi, int, 0444);
63 MODULE_PARM_DESC(msi, "attempt to use MSI if nonzero");
64
65 #else /* CONFIG_PCI_MSI */
66
67 #define msi_x (0)
68 #define msi   (0)
69
70 #endif /* CONFIG_PCI_MSI */
71
72 static const char mthca_version[] __devinitdata =
73         DRV_NAME ": Mellanox InfiniBand HCA driver v"
74         DRV_VERSION " (" DRV_RELDATE ")\n";
75
76 static struct mthca_profile default_profile = {
77         .num_qp            = 1 << 16,
78         .rdb_per_qp        = 4,
79         .num_cq            = 1 << 16,
80         .num_mcg           = 1 << 13,
81         .num_mpt           = 1 << 17,
82         .num_mtt           = 1 << 20,
83         .num_udav          = 1 << 15,   /* Tavor only */
84         .fmr_reserved_mtts = 1 << 18,   /* Tavor only */
85         .uarc_size         = 1 << 18,   /* Arbel only */
86 };
87
88 static int __devinit mthca_tune_pci(struct mthca_dev *mdev)
89 {
90         int cap;
91         u16 val;
92
93         /* First try to max out Read Byte Count */
94         cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX);
95         if (cap) {
96                 if (pci_read_config_word(mdev->pdev, cap + PCI_X_CMD, &val)) {
97                         mthca_err(mdev, "Couldn't read PCI-X command register, "
98                                   "aborting.\n");
99                         return -ENODEV;
100                 }
101                 val = (val & ~PCI_X_CMD_MAX_READ) | (3 << 2);
102                 if (pci_write_config_word(mdev->pdev, cap + PCI_X_CMD, val)) {
103                         mthca_err(mdev, "Couldn't write PCI-X command register, "
104                                   "aborting.\n");
105                         return -ENODEV;
106                 }
107         } else if (!(mdev->mthca_flags & MTHCA_FLAG_PCIE))
108                 mthca_info(mdev, "No PCI-X capability, not setting RBC.\n");
109
110         cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_EXP);
111         if (cap) {
112                 if (pci_read_config_word(mdev->pdev, cap + PCI_EXP_DEVCTL, &val)) {
113                         mthca_err(mdev, "Couldn't read PCI Express device control "
114                                   "register, aborting.\n");
115                         return -ENODEV;
116                 }
117                 val = (val & ~PCI_EXP_DEVCTL_READRQ) | (5 << 12);
118                 if (pci_write_config_word(mdev->pdev, cap + PCI_EXP_DEVCTL, val)) {
119                         mthca_err(mdev, "Couldn't write PCI Express device control "
120                                   "register, aborting.\n");
121                         return -ENODEV;
122                 }
123         } else if (mdev->mthca_flags & MTHCA_FLAG_PCIE)
124                 mthca_info(mdev, "No PCI Express capability, "
125                            "not setting Max Read Request Size.\n");
126
127         return 0;
128 }
129
130 static int __devinit mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim *dev_lim)
131 {
132         int err;
133         u8 status;
134
135         err = mthca_QUERY_DEV_LIM(mdev, dev_lim, &status);
136         if (err) {
137                 mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
138                 return err;
139         }
140         if (status) {
141                 mthca_err(mdev, "QUERY_DEV_LIM returned status 0x%02x, "
142                           "aborting.\n", status);
143                 return -EINVAL;
144         }
145         if (dev_lim->min_page_sz > PAGE_SIZE) {
146                 mthca_err(mdev, "HCA minimum page size of %d bigger than "
147                           "kernel PAGE_SIZE of %ld, aborting.\n",
148                           dev_lim->min_page_sz, PAGE_SIZE);
149                 return -ENODEV;
150         }
151         if (dev_lim->num_ports > MTHCA_MAX_PORTS) {
152                 mthca_err(mdev, "HCA has %d ports, but we only support %d, "
153                           "aborting.\n",
154                           dev_lim->num_ports, MTHCA_MAX_PORTS);
155                 return -ENODEV;
156         }
157
158         mdev->limits.num_ports          = dev_lim->num_ports;
159         mdev->limits.vl_cap             = dev_lim->max_vl;
160         mdev->limits.mtu_cap            = dev_lim->max_mtu;
161         mdev->limits.gid_table_len      = dev_lim->max_gids;
162         mdev->limits.pkey_table_len     = dev_lim->max_pkeys;
163         mdev->limits.local_ca_ack_delay = dev_lim->local_ca_ack_delay;
164         mdev->limits.max_sg             = dev_lim->max_sg;
165         mdev->limits.max_wqes           = dev_lim->max_qp_sz;
166         mdev->limits.max_qp_init_rdma   = dev_lim->max_requester_per_qp;
167         mdev->limits.reserved_qps       = dev_lim->reserved_qps;
168         mdev->limits.max_srq_wqes       = dev_lim->max_srq_sz;
169         mdev->limits.reserved_srqs      = dev_lim->reserved_srqs;
170         mdev->limits.reserved_eecs      = dev_lim->reserved_eecs;
171         /*
172          * Subtract 1 from the limit because we need to allocate a
173          * spare CQE so the HCA HW can tell the difference between an
174          * empty CQ and a full CQ.
175          */
176         mdev->limits.max_cqes           = dev_lim->max_cq_sz - 1;
177         mdev->limits.reserved_cqs       = dev_lim->reserved_cqs;
178         mdev->limits.reserved_eqs       = dev_lim->reserved_eqs;
179         mdev->limits.reserved_mtts      = dev_lim->reserved_mtts;
180         mdev->limits.reserved_mrws      = dev_lim->reserved_mrws;
181         mdev->limits.reserved_uars      = dev_lim->reserved_uars;
182         mdev->limits.reserved_pds       = dev_lim->reserved_pds;
183         mdev->limits.port_width_cap     = dev_lim->max_port_width;
184         mdev->limits.page_size_cap      = ~(u32) (dev_lim->min_page_sz - 1);
185         mdev->limits.flags              = dev_lim->flags;
186
187         /* IB_DEVICE_RESIZE_MAX_WR not supported by driver.
188            May be doable since hardware supports it for SRQ.
189
190            IB_DEVICE_N_NOTIFY_CQ is supported by hardware but not by driver.
191
192            IB_DEVICE_SRQ_RESIZE is supported by hardware but SRQ is not
193            supported by driver. */
194         mdev->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
195                 IB_DEVICE_PORT_ACTIVE_EVENT |
196                 IB_DEVICE_SYS_IMAGE_GUID |
197                 IB_DEVICE_RC_RNR_NAK_GEN;
198
199         if (dev_lim->flags & DEV_LIM_FLAG_BAD_PKEY_CNTR)
200                 mdev->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
201
202         if (dev_lim->flags & DEV_LIM_FLAG_BAD_QKEY_CNTR)
203                 mdev->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
204
205         if (dev_lim->flags & DEV_LIM_FLAG_RAW_MULTI)
206                 mdev->device_cap_flags |= IB_DEVICE_RAW_MULTI;
207
208         if (dev_lim->flags & DEV_LIM_FLAG_AUTO_PATH_MIG)
209                 mdev->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
210
211         if (dev_lim->flags & DEV_LIM_FLAG_UD_AV_PORT_ENFORCE)
212                 mdev->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE;
213
214         if (dev_lim->flags & DEV_LIM_FLAG_SRQ)
215                 mdev->mthca_flags |= MTHCA_FLAG_SRQ;
216
217         return 0;
218 }
219
220 static int __devinit mthca_init_tavor(struct mthca_dev *mdev)
221 {
222         u8 status;
223         int err;
224         struct mthca_dev_lim        dev_lim;
225         struct mthca_profile        profile;
226         struct mthca_init_hca_param init_hca;
227
228         err = mthca_SYS_EN(mdev, &status);
229         if (err) {
230                 mthca_err(mdev, "SYS_EN command failed, aborting.\n");
231                 return err;
232         }
233         if (status) {
234                 mthca_err(mdev, "SYS_EN returned status 0x%02x, "
235                           "aborting.\n", status);
236                 return -EINVAL;
237         }
238
239         err = mthca_QUERY_FW(mdev, &status);
240         if (err) {
241                 mthca_err(mdev, "QUERY_FW command failed, aborting.\n");
242                 goto err_disable;
243         }
244         if (status) {
245                 mthca_err(mdev, "QUERY_FW returned status 0x%02x, "
246                           "aborting.\n", status);
247                 err = -EINVAL;
248                 goto err_disable;
249         }
250         err = mthca_QUERY_DDR(mdev, &status);
251         if (err) {
252                 mthca_err(mdev, "QUERY_DDR command failed, aborting.\n");
253                 goto err_disable;
254         }
255         if (status) {
256                 mthca_err(mdev, "QUERY_DDR returned status 0x%02x, "
257                           "aborting.\n", status);
258                 err = -EINVAL;
259                 goto err_disable;
260         }
261
262         err = mthca_dev_lim(mdev, &dev_lim);
263
264         profile = default_profile;
265         profile.num_uar   = dev_lim.uar_size / PAGE_SIZE;
266         profile.uarc_size = 0;
267         if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
268                 profile.num_srq = dev_lim.max_srqs;
269
270         err = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
271         if (err < 0)
272                 goto err_disable;
273
274         err = mthca_INIT_HCA(mdev, &init_hca, &status);
275         if (err) {
276                 mthca_err(mdev, "INIT_HCA command failed, aborting.\n");
277                 goto err_disable;
278         }
279         if (status) {
280                 mthca_err(mdev, "INIT_HCA returned status 0x%02x, "
281                           "aborting.\n", status);
282                 err = -EINVAL;
283                 goto err_disable;
284         }
285
286         return 0;
287
288 err_disable:
289         mthca_SYS_DIS(mdev, &status);
290
291         return err;
292 }
293
294 static int __devinit mthca_load_fw(struct mthca_dev *mdev)
295 {
296         u8 status;
297         int err;
298
299         /* FIXME: use HCA-attached memory for FW if present */
300
301         mdev->fw.arbel.fw_icm =
302                 mthca_alloc_icm(mdev, mdev->fw.arbel.fw_pages,
303                                 GFP_HIGHUSER | __GFP_NOWARN);
304         if (!mdev->fw.arbel.fw_icm) {
305                 mthca_err(mdev, "Couldn't allocate FW area, aborting.\n");
306                 return -ENOMEM;
307         }
308
309         err = mthca_MAP_FA(mdev, mdev->fw.arbel.fw_icm, &status);
310         if (err) {
311                 mthca_err(mdev, "MAP_FA command failed, aborting.\n");
312                 goto err_free;
313         }
314         if (status) {
315                 mthca_err(mdev, "MAP_FA returned status 0x%02x, aborting.\n", status);
316                 err = -EINVAL;
317                 goto err_free;
318         }
319         err = mthca_RUN_FW(mdev, &status);
320         if (err) {
321                 mthca_err(mdev, "RUN_FW command failed, aborting.\n");
322                 goto err_unmap_fa;
323         }
324         if (status) {
325                 mthca_err(mdev, "RUN_FW returned status 0x%02x, aborting.\n", status);
326                 err = -EINVAL;
327                 goto err_unmap_fa;
328         }
329
330         return 0;
331
332 err_unmap_fa:
333         mthca_UNMAP_FA(mdev, &status);
334
335 err_free:
336         mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
337         return err;
338 }
339
340 static int __devinit mthca_init_icm(struct mthca_dev *mdev,
341                                     struct mthca_dev_lim *dev_lim,
342                                     struct mthca_init_hca_param *init_hca,
343                                     u64 icm_size)
344 {
345         u64 aux_pages;
346         u8 status;
347         int err;
348
349         err = mthca_SET_ICM_SIZE(mdev, icm_size, &aux_pages, &status);
350         if (err) {
351                 mthca_err(mdev, "SET_ICM_SIZE command failed, aborting.\n");
352                 return err;
353         }
354         if (status) {
355                 mthca_err(mdev, "SET_ICM_SIZE returned status 0x%02x, "
356                           "aborting.\n", status);
357                 return -EINVAL;
358         }
359
360         mthca_dbg(mdev, "%lld KB of HCA context requires %lld KB aux memory.\n",
361                   (unsigned long long) icm_size >> 10,
362                   (unsigned long long) aux_pages << 2);
363
364         mdev->fw.arbel.aux_icm = mthca_alloc_icm(mdev, aux_pages,
365                                                  GFP_HIGHUSER | __GFP_NOWARN);
366         if (!mdev->fw.arbel.aux_icm) {
367                 mthca_err(mdev, "Couldn't allocate aux memory, aborting.\n");
368                 return -ENOMEM;
369         }
370
371         err = mthca_MAP_ICM_AUX(mdev, mdev->fw.arbel.aux_icm, &status);
372         if (err) {
373                 mthca_err(mdev, "MAP_ICM_AUX command failed, aborting.\n");
374                 goto err_free_aux;
375         }
376         if (status) {
377                 mthca_err(mdev, "MAP_ICM_AUX returned status 0x%02x, aborting.\n", status);
378                 err = -EINVAL;
379                 goto err_free_aux;
380         }
381
382         err = mthca_map_eq_icm(mdev, init_hca->eqc_base);
383         if (err) {
384                 mthca_err(mdev, "Failed to map EQ context memory, aborting.\n");
385                 goto err_unmap_aux;
386         }
387
388         mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base,
389                                                          MTHCA_MTT_SEG_SIZE,
390                                                          mdev->limits.num_mtt_segs,
391                                                          mdev->limits.reserved_mtts, 1);
392         if (!mdev->mr_table.mtt_table) {
393                 mthca_err(mdev, "Failed to map MTT context memory, aborting.\n");
394                 err = -ENOMEM;
395                 goto err_unmap_eq;
396         }
397
398         mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base,
399                                                          dev_lim->mpt_entry_sz,
400                                                          mdev->limits.num_mpts,
401                                                          mdev->limits.reserved_mrws, 1);
402         if (!mdev->mr_table.mpt_table) {
403                 mthca_err(mdev, "Failed to map MPT context memory, aborting.\n");
404                 err = -ENOMEM;
405                 goto err_unmap_mtt;
406         }
407
408         mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base,
409                                                         dev_lim->qpc_entry_sz,
410                                                         mdev->limits.num_qps,
411                                                         mdev->limits.reserved_qps, 0);
412         if (!mdev->qp_table.qp_table) {
413                 mthca_err(mdev, "Failed to map QP context memory, aborting.\n");
414                 err = -ENOMEM;
415                 goto err_unmap_mpt;
416         }
417
418         mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base,
419                                                          dev_lim->eqpc_entry_sz,
420                                                          mdev->limits.num_qps,
421                                                          mdev->limits.reserved_qps, 0);
422         if (!mdev->qp_table.eqp_table) {
423                 mthca_err(mdev, "Failed to map EQP context memory, aborting.\n");
424                 err = -ENOMEM;
425                 goto err_unmap_qp;
426         }
427
428         mdev->qp_table.rdb_table = mthca_alloc_icm_table(mdev, init_hca->rdb_base,
429                                                          MTHCA_RDB_ENTRY_SIZE,
430                                                          mdev->limits.num_qps <<
431                                                          mdev->qp_table.rdb_shift,
432                                                          0, 0);
433         if (!mdev->qp_table.rdb_table) {
434                 mthca_err(mdev, "Failed to map RDB context memory, aborting\n");
435                 err = -ENOMEM;
436                 goto err_unmap_eqp;
437         }
438
439        mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base,
440                                                     dev_lim->cqc_entry_sz,
441                                                     mdev->limits.num_cqs,
442                                                     mdev->limits.reserved_cqs, 0);
443         if (!mdev->cq_table.table) {
444                 mthca_err(mdev, "Failed to map CQ context memory, aborting.\n");
445                 err = -ENOMEM;
446                 goto err_unmap_rdb;
447         }
448
449         if (mdev->mthca_flags & MTHCA_FLAG_SRQ) {
450                 mdev->srq_table.table =
451                         mthca_alloc_icm_table(mdev, init_hca->srqc_base,
452                                               dev_lim->srq_entry_sz,
453                                               mdev->limits.num_srqs,
454                                               mdev->limits.reserved_srqs, 0);
455                 if (!mdev->srq_table.table) {
456                         mthca_err(mdev, "Failed to map SRQ context memory, "
457                                   "aborting.\n");
458                         err = -ENOMEM;
459                         goto err_unmap_cq;
460                 }
461         }
462
463         /*
464          * It's not strictly required, but for simplicity just map the
465          * whole multicast group table now.  The table isn't very big
466          * and it's a lot easier than trying to track ref counts.
467          */
468         mdev->mcg_table.table = mthca_alloc_icm_table(mdev, init_hca->mc_base,
469                                                       MTHCA_MGM_ENTRY_SIZE,
470                                                       mdev->limits.num_mgms +
471                                                       mdev->limits.num_amgms,
472                                                       mdev->limits.num_mgms +
473                                                       mdev->limits.num_amgms,
474                                                       0);
475         if (!mdev->mcg_table.table) {
476                 mthca_err(mdev, "Failed to map MCG context memory, aborting.\n");
477                 err = -ENOMEM;
478                 goto err_unmap_srq;
479         }
480
481         return 0;
482
483 err_unmap_srq:
484         if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
485                 mthca_free_icm_table(mdev, mdev->srq_table.table);
486
487 err_unmap_cq:
488         mthca_free_icm_table(mdev, mdev->cq_table.table);
489
490 err_unmap_rdb:
491         mthca_free_icm_table(mdev, mdev->qp_table.rdb_table);
492
493 err_unmap_eqp:
494         mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
495
496 err_unmap_qp:
497         mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
498
499 err_unmap_mpt:
500         mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
501
502 err_unmap_mtt:
503         mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
504
505 err_unmap_eq:
506         mthca_unmap_eq_icm(mdev);
507
508 err_unmap_aux:
509         mthca_UNMAP_ICM_AUX(mdev, &status);
510
511 err_free_aux:
512         mthca_free_icm(mdev, mdev->fw.arbel.aux_icm);
513
514         return err;
515 }
516
517 static void mthca_free_icms(struct mthca_dev *mdev)
518 {
519         u8 status;
520
521         mthca_free_icm_table(mdev, mdev->mcg_table.table);
522         if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
523                 mthca_free_icm_table(mdev, mdev->srq_table.table);
524         mthca_free_icm_table(mdev, mdev->cq_table.table);
525         mthca_free_icm_table(mdev, mdev->qp_table.rdb_table);
526         mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
527         mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
528         mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
529         mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
530         mthca_unmap_eq_icm(mdev);
531
532         mthca_UNMAP_ICM_AUX(mdev, &status);
533         mthca_free_icm(mdev, mdev->fw.arbel.aux_icm);
534 }
535
536 static int __devinit mthca_init_arbel(struct mthca_dev *mdev)
537 {
538         struct mthca_dev_lim        dev_lim;
539         struct mthca_profile        profile;
540         struct mthca_init_hca_param init_hca;
541         u64 icm_size;
542         u8 status;
543         int err;
544
545         err = mthca_QUERY_FW(mdev, &status);
546         if (err) {
547                 mthca_err(mdev, "QUERY_FW command failed, aborting.\n");
548                 return err;
549         }
550         if (status) {
551                 mthca_err(mdev, "QUERY_FW returned status 0x%02x, "
552                           "aborting.\n", status);
553                 return -EINVAL;
554         }
555
556         err = mthca_ENABLE_LAM(mdev, &status);
557         if (err) {
558                 mthca_err(mdev, "ENABLE_LAM command failed, aborting.\n");
559                 return err;
560         }
561         if (status == MTHCA_CMD_STAT_LAM_NOT_PRE) {
562                 mthca_dbg(mdev, "No HCA-attached memory (running in MemFree mode)\n");
563                 mdev->mthca_flags |= MTHCA_FLAG_NO_LAM;
564         } else if (status) {
565                 mthca_err(mdev, "ENABLE_LAM returned status 0x%02x, "
566                           "aborting.\n", status);
567                 return -EINVAL;
568         }
569
570         err = mthca_load_fw(mdev);
571         if (err) {
572                 mthca_err(mdev, "Failed to start FW, aborting.\n");
573                 goto err_disable;
574         }
575
576         err = mthca_dev_lim(mdev, &dev_lim);
577         if (err) {
578                 mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
579                 goto err_stop_fw;
580         }
581
582         profile = default_profile;
583         profile.num_uar  = dev_lim.uar_size / PAGE_SIZE;
584         profile.num_udav = 0;
585         if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
586                 profile.num_srq = dev_lim.max_srqs;
587
588         icm_size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
589         if ((int) icm_size < 0) {
590                 err = icm_size;
591                 goto err_stop_fw;
592         }
593
594         err = mthca_init_icm(mdev, &dev_lim, &init_hca, icm_size);
595         if (err)
596                 goto err_stop_fw;
597
598         err = mthca_INIT_HCA(mdev, &init_hca, &status);
599         if (err) {
600                 mthca_err(mdev, "INIT_HCA command failed, aborting.\n");
601                 goto err_free_icm;
602         }
603         if (status) {
604                 mthca_err(mdev, "INIT_HCA returned status 0x%02x, "
605                           "aborting.\n", status);
606                 err = -EINVAL;
607                 goto err_free_icm;
608         }
609
610         return 0;
611
612 err_free_icm:
613         mthca_free_icms(mdev);
614
615 err_stop_fw:
616         mthca_UNMAP_FA(mdev, &status);
617         mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
618
619 err_disable:
620         if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
621                 mthca_DISABLE_LAM(mdev, &status);
622
623         return err;
624 }
625
626 static void mthca_close_hca(struct mthca_dev *mdev)
627 {
628         u8 status;
629
630         mthca_CLOSE_HCA(mdev, 0, &status);
631
632         if (mthca_is_memfree(mdev)) {
633                 mthca_free_icms(mdev);
634
635                 mthca_UNMAP_FA(mdev, &status);
636                 mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
637
638                 if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
639                         mthca_DISABLE_LAM(mdev, &status);
640         } else
641                 mthca_SYS_DIS(mdev, &status);
642 }
643
644 static int __devinit mthca_init_hca(struct mthca_dev *mdev)
645 {
646         u8 status;
647         int err;
648         struct mthca_adapter adapter;
649
650         if (mthca_is_memfree(mdev))
651                 err = mthca_init_arbel(mdev);
652         else
653                 err = mthca_init_tavor(mdev);
654
655         if (err)
656                 return err;
657
658         err = mthca_QUERY_ADAPTER(mdev, &adapter, &status);
659         if (err) {
660                 mthca_err(mdev, "QUERY_ADAPTER command failed, aborting.\n");
661                 goto err_close;
662         }
663         if (status) {
664                 mthca_err(mdev, "QUERY_ADAPTER returned status 0x%02x, "
665                           "aborting.\n", status);
666                 err = -EINVAL;
667                 goto err_close;
668         }
669
670         mdev->eq_table.inta_pin = adapter.inta_pin;
671         mdev->rev_id            = adapter.revision_id;
672         memcpy(mdev->board_id, adapter.board_id, sizeof mdev->board_id);
673
674         return 0;
675
676 err_close:
677         mthca_close_hca(mdev);
678         return err;
679 }
680
681 static int __devinit mthca_setup_hca(struct mthca_dev *dev)
682 {
683         int err;
684         u8 status;
685
686         MTHCA_INIT_DOORBELL_LOCK(&dev->doorbell_lock);
687
688         err = mthca_init_uar_table(dev);
689         if (err) {
690                 mthca_err(dev, "Failed to initialize "
691                           "user access region table, aborting.\n");
692                 return err;
693         }
694
695         err = mthca_uar_alloc(dev, &dev->driver_uar);
696         if (err) {
697                 mthca_err(dev, "Failed to allocate driver access region, "
698                           "aborting.\n");
699                 goto err_uar_table_free;
700         }
701
702         dev->kar = ioremap(dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
703         if (!dev->kar) {
704                 mthca_err(dev, "Couldn't map kernel access region, "
705                           "aborting.\n");
706                 err = -ENOMEM;
707                 goto err_uar_free;
708         }
709
710         err = mthca_init_pd_table(dev);
711         if (err) {
712                 mthca_err(dev, "Failed to initialize "
713                           "protection domain table, aborting.\n");
714                 goto err_kar_unmap;
715         }
716
717         err = mthca_init_mr_table(dev);
718         if (err) {
719                 mthca_err(dev, "Failed to initialize "
720                           "memory region table, aborting.\n");
721                 goto err_pd_table_free;
722         }
723
724         err = mthca_pd_alloc(dev, 1, &dev->driver_pd);
725         if (err) {
726                 mthca_err(dev, "Failed to create driver PD, "
727                           "aborting.\n");
728                 goto err_mr_table_free;
729         }
730
731         err = mthca_init_eq_table(dev);
732         if (err) {
733                 mthca_err(dev, "Failed to initialize "
734                           "event queue table, aborting.\n");
735                 goto err_pd_free;
736         }
737
738         err = mthca_cmd_use_events(dev);
739         if (err) {
740                 mthca_err(dev, "Failed to switch to event-driven "
741                           "firmware commands, aborting.\n");
742                 goto err_eq_table_free;
743         }
744
745         err = mthca_NOP(dev, &status);
746         if (err || status) {
747                 mthca_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting.\n",
748                           dev->mthca_flags & MTHCA_FLAG_MSI_X ?
749                           dev->eq_table.eq[MTHCA_EQ_CMD].msi_x_vector :
750                           dev->pdev->irq);
751                 if (dev->mthca_flags & (MTHCA_FLAG_MSI | MTHCA_FLAG_MSI_X))
752                         mthca_err(dev, "Try again with MSI/MSI-X disabled.\n");
753                 else
754                         mthca_err(dev, "BIOS or ACPI interrupt routing problem?\n");
755
756                 goto err_cmd_poll;
757         }
758
759         mthca_dbg(dev, "NOP command IRQ test passed\n");
760
761         err = mthca_init_cq_table(dev);
762         if (err) {
763                 mthca_err(dev, "Failed to initialize "
764                           "completion queue table, aborting.\n");
765                 goto err_cmd_poll;
766         }
767
768         err = mthca_init_srq_table(dev);
769         if (err) {
770                 mthca_err(dev, "Failed to initialize "
771                           "shared receive queue table, aborting.\n");
772                 goto err_cq_table_free;
773         }
774
775         err = mthca_init_qp_table(dev);
776         if (err) {
777                 mthca_err(dev, "Failed to initialize "
778                           "queue pair table, aborting.\n");
779                 goto err_srq_table_free;
780         }
781
782         err = mthca_init_av_table(dev);
783         if (err) {
784                 mthca_err(dev, "Failed to initialize "
785                           "address vector table, aborting.\n");
786                 goto err_qp_table_free;
787         }
788
789         err = mthca_init_mcg_table(dev);
790         if (err) {
791                 mthca_err(dev, "Failed to initialize "
792                           "multicast group table, aborting.\n");
793                 goto err_av_table_free;
794         }
795
796         return 0;
797
798 err_av_table_free:
799         mthca_cleanup_av_table(dev);
800
801 err_qp_table_free:
802         mthca_cleanup_qp_table(dev);
803
804 err_srq_table_free:
805         mthca_cleanup_srq_table(dev);
806
807 err_cq_table_free:
808         mthca_cleanup_cq_table(dev);
809
810 err_cmd_poll:
811         mthca_cmd_use_polling(dev);
812
813 err_eq_table_free:
814         mthca_cleanup_eq_table(dev);
815
816 err_pd_free:
817         mthca_pd_free(dev, &dev->driver_pd);
818
819 err_mr_table_free:
820         mthca_cleanup_mr_table(dev);
821
822 err_pd_table_free:
823         mthca_cleanup_pd_table(dev);
824
825 err_kar_unmap:
826         iounmap(dev->kar);
827
828 err_uar_free:
829         mthca_uar_free(dev, &dev->driver_uar);
830
831 err_uar_table_free:
832         mthca_cleanup_uar_table(dev);
833         return err;
834 }
835
836 static int __devinit mthca_request_regions(struct pci_dev *pdev,
837                                            int ddr_hidden)
838 {
839         int err;
840
841         /*
842          * We can't just use pci_request_regions() because the MSI-X
843          * table is right in the middle of the first BAR.  If we did
844          * pci_request_region and grab all of the first BAR, then
845          * setting up MSI-X would fail, since the PCI core wants to do
846          * request_mem_region on the MSI-X vector table.
847          *
848          * So just request what we need right now, and request any
849          * other regions we need when setting up EQs.
850          */
851         if (!request_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
852                                 MTHCA_HCR_SIZE, DRV_NAME))
853                 return -EBUSY;
854
855         err = pci_request_region(pdev, 2, DRV_NAME);
856         if (err)
857                 goto err_bar2_failed;
858
859         if (!ddr_hidden) {
860                 err = pci_request_region(pdev, 4, DRV_NAME);
861                 if (err)
862                         goto err_bar4_failed;
863         }
864
865         return 0;
866
867 err_bar4_failed:
868         pci_release_region(pdev, 2);
869
870 err_bar2_failed:
871         release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
872                            MTHCA_HCR_SIZE);
873
874         return err;
875 }
876
877 static void mthca_release_regions(struct pci_dev *pdev,
878                                   int ddr_hidden)
879 {
880         if (!ddr_hidden)
881                 pci_release_region(pdev, 4);
882
883         pci_release_region(pdev, 2);
884
885         release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
886                            MTHCA_HCR_SIZE);
887 }
888
889 static int __devinit mthca_enable_msi_x(struct mthca_dev *mdev)
890 {
891         struct msix_entry entries[3];
892         int err;
893
894         entries[0].entry = 0;
895         entries[1].entry = 1;
896         entries[2].entry = 2;
897
898         err = pci_enable_msix(mdev->pdev, entries, ARRAY_SIZE(entries));
899         if (err) {
900                 if (err > 0)
901                         mthca_info(mdev, "Only %d MSI-X vectors available, "
902                                    "not using MSI-X\n", err);
903                 return err;
904         }
905
906         mdev->eq_table.eq[MTHCA_EQ_COMP ].msi_x_vector = entries[0].vector;
907         mdev->eq_table.eq[MTHCA_EQ_ASYNC].msi_x_vector = entries[1].vector;
908         mdev->eq_table.eq[MTHCA_EQ_CMD  ].msi_x_vector = entries[2].vector;
909
910         return 0;
911 }
912
913 /* Types of supported HCA */
914 enum {
915         TAVOR,                  /* MT23108                        */
916         ARBEL_COMPAT,           /* MT25208 in Tavor compat mode   */
917         ARBEL_NATIVE,           /* MT25208 with extended features */
918         SINAI                   /* MT25204 */
919 };
920
921 #define MTHCA_FW_VER(major, minor, subminor) \
922         (((u64) (major) << 32) | ((u64) (minor) << 16) | (u64) (subminor))
923
924 static struct {
925         u64 latest_fw;
926         int is_memfree;
927         int is_pcie;
928 } mthca_hca_table[] = {
929         [TAVOR]        = { .latest_fw = MTHCA_FW_VER(3, 3, 3), .is_memfree = 0, .is_pcie = 0 },
930         [ARBEL_COMPAT] = { .latest_fw = MTHCA_FW_VER(4, 7, 0), .is_memfree = 0, .is_pcie = 1 },
931         [ARBEL_NATIVE] = { .latest_fw = MTHCA_FW_VER(5, 1, 0), .is_memfree = 1, .is_pcie = 1 },
932         [SINAI]        = { .latest_fw = MTHCA_FW_VER(1, 0, 1), .is_memfree = 1, .is_pcie = 1 }
933 };
934
935 static int __devinit mthca_init_one(struct pci_dev *pdev,
936                                     const struct pci_device_id *id)
937 {
938         static int mthca_version_printed = 0;
939         int ddr_hidden = 0;
940         int err;
941         struct mthca_dev *mdev;
942
943         if (!mthca_version_printed) {
944                 printk(KERN_INFO "%s", mthca_version);
945                 ++mthca_version_printed;
946         }
947
948         printk(KERN_INFO PFX "Initializing %s\n",
949                pci_name(pdev));
950
951         if (id->driver_data >= ARRAY_SIZE(mthca_hca_table)) {
952                 printk(KERN_ERR PFX "%s has invalid driver data %lx\n",
953                        pci_name(pdev), id->driver_data);
954                 return -ENODEV;
955         }
956
957         err = pci_enable_device(pdev);
958         if (err) {
959                 dev_err(&pdev->dev, "Cannot enable PCI device, "
960                         "aborting.\n");
961                 return err;
962         }
963
964         /*
965          * Check for BARs.  We expect 0: 1MB, 2: 8MB, 4: DDR (may not
966          * be present)
967          */
968         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
969             pci_resource_len(pdev, 0) != 1 << 20) {
970                 dev_err(&pdev->dev, "Missing DCS, aborting.\n");
971                 err = -ENODEV;
972                 goto err_disable_pdev;
973         }
974         if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM) ||
975             pci_resource_len(pdev, 2) != 1 << 23) {
976                 dev_err(&pdev->dev, "Missing UAR, aborting.\n");
977                 err = -ENODEV;
978                 goto err_disable_pdev;
979         }
980         if (!(pci_resource_flags(pdev, 4) & IORESOURCE_MEM))
981                 ddr_hidden = 1;
982
983         err = mthca_request_regions(pdev, ddr_hidden);
984         if (err) {
985                 dev_err(&pdev->dev, "Cannot obtain PCI resources, "
986                         "aborting.\n");
987                 goto err_disable_pdev;
988         }
989
990         pci_set_master(pdev);
991
992         err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
993         if (err) {
994                 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n");
995                 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
996                 if (err) {
997                         dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n");
998                         goto err_free_res;
999                 }
1000         }
1001         err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
1002         if (err) {
1003                 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit "
1004                          "consistent PCI DMA mask.\n");
1005                 err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
1006                 if (err) {
1007                         dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, "
1008                                 "aborting.\n");
1009                         goto err_free_res;
1010                 }
1011         }
1012
1013         mdev = (struct mthca_dev *) ib_alloc_device(sizeof *mdev);
1014         if (!mdev) {
1015                 dev_err(&pdev->dev, "Device struct alloc failed, "
1016                         "aborting.\n");
1017                 err = -ENOMEM;
1018                 goto err_free_res;
1019         }
1020
1021         mdev->pdev = pdev;
1022
1023         if (ddr_hidden)
1024                 mdev->mthca_flags |= MTHCA_FLAG_DDR_HIDDEN;
1025         if (mthca_hca_table[id->driver_data].is_memfree)
1026                 mdev->mthca_flags |= MTHCA_FLAG_MEMFREE;
1027         if (mthca_hca_table[id->driver_data].is_pcie)
1028                 mdev->mthca_flags |= MTHCA_FLAG_PCIE;
1029
1030         /*
1031          * Now reset the HCA before we touch the PCI capabilities or
1032          * attempt a firmware command, since a boot ROM may have left
1033          * the HCA in an undefined state.
1034          */
1035         err = mthca_reset(mdev);
1036         if (err) {
1037                 mthca_err(mdev, "Failed to reset HCA, aborting.\n");
1038                 goto err_free_dev;
1039         }
1040
1041         if (msi_x && !mthca_enable_msi_x(mdev))
1042                 mdev->mthca_flags |= MTHCA_FLAG_MSI_X;
1043         if (msi && !(mdev->mthca_flags & MTHCA_FLAG_MSI_X) &&
1044             !pci_enable_msi(pdev))
1045                 mdev->mthca_flags |= MTHCA_FLAG_MSI;
1046
1047         if (mthca_cmd_init(mdev)) {
1048                 mthca_err(mdev, "Failed to init command interface, aborting.\n");
1049                 goto err_free_dev;
1050         }
1051
1052         err = mthca_tune_pci(mdev);
1053         if (err)
1054                 goto err_cmd;
1055
1056         err = mthca_init_hca(mdev);
1057         if (err)
1058                 goto err_cmd;
1059
1060         if (mdev->fw_ver < mthca_hca_table[id->driver_data].latest_fw) {
1061                 mthca_warn(mdev, "HCA FW version %d.%d.%d is old (%d.%d.%d is current).\n",
1062                            (int) (mdev->fw_ver >> 32), (int) (mdev->fw_ver >> 16) & 0xffff,
1063                            (int) (mdev->fw_ver & 0xffff),
1064                            (int) (mthca_hca_table[id->driver_data].latest_fw >> 32),
1065                            (int) (mthca_hca_table[id->driver_data].latest_fw >> 16) & 0xffff,
1066                            (int) (mthca_hca_table[id->driver_data].latest_fw & 0xffff));
1067                 mthca_warn(mdev, "If you have problems, try updating your HCA FW.\n");
1068         }
1069
1070         err = mthca_setup_hca(mdev);
1071         if (err)
1072                 goto err_close;
1073
1074         err = mthca_register_device(mdev);
1075         if (err)
1076                 goto err_cleanup;
1077
1078         err = mthca_create_agents(mdev);
1079         if (err)
1080                 goto err_unregister;
1081
1082         pci_set_drvdata(pdev, mdev);
1083
1084         return 0;
1085
1086 err_unregister:
1087         mthca_unregister_device(mdev);
1088
1089 err_cleanup:
1090         mthca_cleanup_mcg_table(mdev);
1091         mthca_cleanup_av_table(mdev);
1092         mthca_cleanup_qp_table(mdev);
1093         mthca_cleanup_srq_table(mdev);
1094         mthca_cleanup_cq_table(mdev);
1095         mthca_cmd_use_polling(mdev);
1096         mthca_cleanup_eq_table(mdev);
1097
1098         mthca_pd_free(mdev, &mdev->driver_pd);
1099
1100         mthca_cleanup_mr_table(mdev);
1101         mthca_cleanup_pd_table(mdev);
1102         mthca_cleanup_uar_table(mdev);
1103
1104 err_close:
1105         mthca_close_hca(mdev);
1106
1107 err_cmd:
1108         mthca_cmd_cleanup(mdev);
1109
1110 err_free_dev:
1111         if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1112                 pci_disable_msix(pdev);
1113         if (mdev->mthca_flags & MTHCA_FLAG_MSI)
1114                 pci_disable_msi(pdev);
1115
1116         ib_dealloc_device(&mdev->ib_dev);
1117
1118 err_free_res:
1119         mthca_release_regions(pdev, ddr_hidden);
1120
1121 err_disable_pdev:
1122         pci_disable_device(pdev);
1123         pci_set_drvdata(pdev, NULL);
1124         return err;
1125 }
1126
1127 static void __devexit mthca_remove_one(struct pci_dev *pdev)
1128 {
1129         struct mthca_dev *mdev = pci_get_drvdata(pdev);
1130         u8 status;
1131         int p;
1132
1133         if (mdev) {
1134                 mthca_free_agents(mdev);
1135                 mthca_unregister_device(mdev);
1136
1137                 for (p = 1; p <= mdev->limits.num_ports; ++p)
1138                         mthca_CLOSE_IB(mdev, p, &status);
1139
1140                 mthca_cleanup_mcg_table(mdev);
1141                 mthca_cleanup_av_table(mdev);
1142                 mthca_cleanup_qp_table(mdev);
1143                 mthca_cleanup_srq_table(mdev);
1144                 mthca_cleanup_cq_table(mdev);
1145                 mthca_cmd_use_polling(mdev);
1146                 mthca_cleanup_eq_table(mdev);
1147
1148                 mthca_pd_free(mdev, &mdev->driver_pd);
1149
1150                 mthca_cleanup_mr_table(mdev);
1151                 mthca_cleanup_pd_table(mdev);
1152
1153                 iounmap(mdev->kar);
1154                 mthca_uar_free(mdev, &mdev->driver_uar);
1155                 mthca_cleanup_uar_table(mdev);
1156                 mthca_close_hca(mdev);
1157                 mthca_cmd_cleanup(mdev);
1158
1159                 if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1160                         pci_disable_msix(pdev);
1161                 if (mdev->mthca_flags & MTHCA_FLAG_MSI)
1162                         pci_disable_msi(pdev);
1163
1164                 ib_dealloc_device(&mdev->ib_dev);
1165                 mthca_release_regions(pdev, mdev->mthca_flags &
1166                                       MTHCA_FLAG_DDR_HIDDEN);
1167                 pci_disable_device(pdev);
1168                 pci_set_drvdata(pdev, NULL);
1169         }
1170 }
1171
1172 static struct pci_device_id mthca_pci_table[] = {
1173         { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR),
1174           .driver_data = TAVOR },
1175         { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_TAVOR),
1176           .driver_data = TAVOR },
1177         { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
1178           .driver_data = ARBEL_COMPAT },
1179         { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
1180           .driver_data = ARBEL_COMPAT },
1181         { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL),
1182           .driver_data = ARBEL_NATIVE },
1183         { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL),
1184           .driver_data = ARBEL_NATIVE },
1185         { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI),
1186           .driver_data = SINAI },
1187         { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI),
1188           .driver_data = SINAI },
1189         { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI_OLD),
1190           .driver_data = SINAI },
1191         { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI_OLD),
1192           .driver_data = SINAI },
1193         { 0, }
1194 };
1195
1196 MODULE_DEVICE_TABLE(pci, mthca_pci_table);
1197
1198 static struct pci_driver mthca_driver = {
1199         .name           = DRV_NAME,
1200         .owner          = THIS_MODULE,
1201         .id_table       = mthca_pci_table,
1202         .probe          = mthca_init_one,
1203         .remove         = __devexit_p(mthca_remove_one)
1204 };
1205
1206 static int __init mthca_init(void)
1207 {
1208         int ret;
1209
1210         ret = pci_register_driver(&mthca_driver);
1211         return ret < 0 ? ret : 0;
1212 }
1213
1214 static void __exit mthca_cleanup(void)
1215 {
1216         pci_unregister_driver(&mthca_driver);
1217 }
1218
1219 module_init(mthca_init);
1220 module_exit(mthca_cleanup);