2 * drivers/pci/pcie/aer/aerdrv_core.c
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
8 * This file implements the core part of PCI-Express AER. When an pci-express
9 * error is delivered, an error message will be collected and printed to
10 * console, then, an error recovery procedure will be executed by following
11 * the pci error recovery rules.
13 * Copyright (C) 2006 Intel Corp.
14 * Tom Long Nguyen (tom.l.nguyen@intel.com)
15 * Zhang Yanmin (yanmin.zhang@intel.com)
19 #include <linux/module.h>
20 #include <linux/pci.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
24 #include <linux/suspend.h>
25 #include <linux/delay.h>
29 module_param(forceload, bool, 0);
31 #define PCI_CFG_SPACE_SIZE (0x100)
32 int pci_find_aer_capability(struct pci_dev *dev)
37 /* Check if it's a pci-express device */
38 pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
42 /* Check if it supports pci-express AER */
43 pos = PCI_CFG_SPACE_SIZE;
45 if (pci_read_config_dword(dev, pos, ®32))
48 /* some broken boards return ~0 */
49 if (reg32 == 0xffffffff)
52 if (PCI_EXT_CAP_ID(reg32) == PCI_EXT_CAP_ID_ERR)
61 int pci_enable_pcie_error_reporting(struct pci_dev *dev)
66 pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
70 pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, ®16);
73 PCI_EXP_DEVCTL_NFERE |
76 pci_write_config_word(dev, pos+PCI_EXP_DEVCTL,
81 int pci_disable_pcie_error_reporting(struct pci_dev *dev)
86 pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
90 pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, ®16);
91 reg16 = reg16 & ~(PCI_EXP_DEVCTL_CERE |
92 PCI_EXP_DEVCTL_NFERE |
95 pci_write_config_word(dev, pos+PCI_EXP_DEVCTL,
100 int pci_cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
105 pos = pci_find_aer_capability(dev);
109 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
110 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
111 if (dev->error_state == pci_channel_io_normal)
112 status &= ~mask; /* Clear corresponding nonfatal bits */
114 status &= mask; /* Clear corresponding fatal bits */
115 pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
120 int pci_cleanup_aer_correct_error_status(struct pci_dev *dev)
125 pos = pci_find_aer_capability(dev);
129 pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status);
130 pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS, status);
135 static int find_device_iter(struct device *device, void *data)
138 u16 id = *(unsigned long *)data;
139 u8 secondary, subordinate, d_bus = id >> 8;
141 if (device->bus == &pci_bus_type) {
142 dev = to_pci_dev(device);
143 if (id == ((dev->bus->number << 8) | dev->devfn)) {
147 *(unsigned long*)data = (unsigned long)device;
152 * If device is P2P, check if it is an upstream?
154 if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) {
155 pci_read_config_byte(dev, PCI_SECONDARY_BUS,
157 pci_read_config_byte(dev, PCI_SUBORDINATE_BUS,
159 if (d_bus >= secondary && d_bus <= subordinate) {
160 *(unsigned long*)data = (unsigned long)device;
170 * find_source_device - search through device hierarchy for source device
171 * @p_dev: pointer to Root Port pci_dev data structure
172 * @id: device ID of agent who sends an error message to this Root Port
174 * Invoked when error is detected at the Root Port.
176 static struct device* find_source_device(struct pci_dev *parent, u16 id)
178 struct pci_dev *dev = parent;
179 struct device *device;
180 unsigned long device_addr;
183 /* Is Root Port an agent that sends error message? */
184 if (id == ((dev->bus->number << 8) | dev->devfn))
189 if ((status = device_for_each_child(&dev->dev,
190 &device_addr, find_device_iter))) {
191 device = (struct device*)device_addr;
192 dev = to_pci_dev(device);
193 if (id == ((dev->bus->number << 8) | dev->devfn))
201 static void report_error_detected(struct pci_dev *dev, void *data)
203 pci_ers_result_t vote;
204 struct pci_error_handlers *err_handler;
205 struct aer_broadcast_data *result_data;
206 result_data = (struct aer_broadcast_data *) data;
208 dev->error_state = result_data->state;
211 !dev->driver->err_handler ||
212 !dev->driver->err_handler->error_detected) {
213 if (result_data->state == pci_channel_io_frozen &&
214 !(dev->hdr_type & PCI_HEADER_TYPE_BRIDGE)) {
216 * In case of fatal recovery, if one of down-
217 * stream device has no driver. We might be
218 * unable to recover because a later insmod
219 * of a driver for this device is unaware of
222 printk(KERN_DEBUG "Device ID[%s] has %s\n",
223 dev->dev.bus_id, (dev->driver) ?
224 "no AER-aware driver" : "no driver");
229 err_handler = dev->driver->err_handler;
230 vote = err_handler->error_detected(dev, result_data->state);
231 result_data->result = merge_result(result_data->result, vote);
235 static void report_mmio_enabled(struct pci_dev *dev, void *data)
237 pci_ers_result_t vote;
238 struct pci_error_handlers *err_handler;
239 struct aer_broadcast_data *result_data;
240 result_data = (struct aer_broadcast_data *) data;
243 !dev->driver->err_handler ||
244 !dev->driver->err_handler->mmio_enabled)
247 err_handler = dev->driver->err_handler;
248 vote = err_handler->mmio_enabled(dev);
249 result_data->result = merge_result(result_data->result, vote);
253 static void report_slot_reset(struct pci_dev *dev, void *data)
255 pci_ers_result_t vote;
256 struct pci_error_handlers *err_handler;
257 struct aer_broadcast_data *result_data;
258 result_data = (struct aer_broadcast_data *) data;
261 !dev->driver->err_handler ||
262 !dev->driver->err_handler->slot_reset)
265 err_handler = dev->driver->err_handler;
266 vote = err_handler->slot_reset(dev);
267 result_data->result = merge_result(result_data->result, vote);
271 static void report_resume(struct pci_dev *dev, void *data)
273 struct pci_error_handlers *err_handler;
275 dev->error_state = pci_channel_io_normal;
278 !dev->driver->err_handler ||
279 !dev->driver->err_handler->slot_reset)
282 err_handler = dev->driver->err_handler;
283 err_handler->resume(dev);
288 * broadcast_error_message - handle message broadcast to downstream drivers
289 * @device: pointer to from where in a hierarchy message is broadcasted down
290 * @api: callback to be broadcasted
291 * @state: error state
293 * Invoked during error recovery process. Once being invoked, the content
294 * of error severity will be broadcasted to all downstream drivers in a
295 * hierarchy in question.
297 static pci_ers_result_t broadcast_error_message(struct pci_dev *dev,
298 enum pci_channel_state state,
300 void (*cb)(struct pci_dev *, void *))
302 struct aer_broadcast_data result_data;
304 printk(KERN_DEBUG "Broadcast %s message\n", error_mesg);
305 result_data.state = state;
306 if (cb == report_error_detected)
307 result_data.result = PCI_ERS_RESULT_CAN_RECOVER;
309 result_data.result = PCI_ERS_RESULT_RECOVERED;
311 if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) {
313 * If the error is reported by a bridge, we think this error
314 * is related to the downstream link of the bridge, so we
315 * do error recovery on all subordinates of the bridge instead
316 * of the bridge and clear the error status of the bridge.
318 if (cb == report_error_detected)
319 dev->error_state = state;
320 pci_walk_bus(dev->subordinate, cb, &result_data);
321 if (cb == report_resume) {
322 pci_cleanup_aer_uncorrect_error_status(dev);
323 dev->error_state = pci_channel_io_normal;
328 * If the error is reported by an end point, we think this
329 * error is related to the upstream link of the end point.
331 pci_walk_bus(dev->bus, cb, &result_data);
334 return result_data.result;
337 struct find_aer_service_data {
338 struct pcie_port_service_driver *aer_driver;
342 static int find_aer_service_iter(struct device *device, void *data)
344 struct device_driver *driver;
345 struct pcie_port_service_driver *service_driver;
346 struct pcie_device *pcie_dev;
347 struct find_aer_service_data *result;
349 result = (struct find_aer_service_data *) data;
351 if (device->bus == &pcie_port_bus_type) {
352 pcie_dev = to_pcie_device(device);
353 if (pcie_dev->id.port_type == PCIE_SW_DOWNSTREAM_PORT)
354 result->is_downstream = 1;
356 driver = device->driver;
358 service_driver = to_service_driver(driver);
359 if (service_driver->id_table->service_type ==
360 PCIE_PORT_SERVICE_AER) {
361 result->aer_driver = service_driver;
370 static void find_aer_service(struct pci_dev *dev,
371 struct find_aer_service_data *data)
374 retval = device_for_each_child(&dev->dev, data, find_aer_service_iter);
377 static pci_ers_result_t reset_link(struct pcie_device *aerdev,
380 struct pci_dev *udev;
381 pci_ers_result_t status;
382 struct find_aer_service_data data;
384 if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE)
387 udev= dev->bus->self;
389 data.is_downstream = 0;
390 data.aer_driver = NULL;
391 find_aer_service(udev, &data);
394 * Use the aer driver of the error agent firstly.
395 * If it hasn't the aer driver, use the root port's
397 if (!data.aer_driver || !data.aer_driver->reset_link) {
398 if (data.is_downstream &&
399 aerdev->device.driver &&
400 to_service_driver(aerdev->device.driver)->reset_link) {
402 to_service_driver(aerdev->device.driver);
404 printk(KERN_DEBUG "No link-reset support to Device ID"
407 return PCI_ERS_RESULT_DISCONNECT;
411 status = data.aer_driver->reset_link(udev);
412 if (status != PCI_ERS_RESULT_RECOVERED) {
413 printk(KERN_DEBUG "Link reset at upstream Device ID"
416 return PCI_ERS_RESULT_DISCONNECT;
423 * do_recovery - handle nonfatal/fatal error recovery process
424 * @aerdev: pointer to a pcie_device data structure of root port
425 * @dev: pointer to a pci_dev data structure of agent detecting an error
426 * @severity: error severity type
428 * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
429 * error detected message to all downstream drivers within a hierarchy in
430 * question and return the returned code.
432 static pci_ers_result_t do_recovery(struct pcie_device *aerdev,
436 pci_ers_result_t status, result = PCI_ERS_RESULT_RECOVERED;
437 enum pci_channel_state state;
439 if (severity == AER_FATAL)
440 state = pci_channel_io_frozen;
442 state = pci_channel_io_normal;
444 status = broadcast_error_message(dev,
447 report_error_detected);
449 if (severity == AER_FATAL) {
450 result = reset_link(aerdev, dev);
451 if (result != PCI_ERS_RESULT_RECOVERED) {
452 /* TODO: Should panic here? */
457 if (status == PCI_ERS_RESULT_CAN_RECOVER)
458 status = broadcast_error_message(dev,
461 report_mmio_enabled);
463 if (status == PCI_ERS_RESULT_NEED_RESET) {
465 * TODO: Should call platform-specific
466 * functions to reset slot before calling
467 * drivers' slot_reset callbacks?
469 status = broadcast_error_message(dev,
475 if (status == PCI_ERS_RESULT_RECOVERED)
476 broadcast_error_message(dev,
485 * handle_error_source - handle logging error into an event log
486 * @aerdev: pointer to pcie_device data structure of the root port
487 * @dev: pointer to pci_dev data structure of error source device
488 * @info: comprehensive error information
490 * Invoked when an error being detected by Root Port.
492 static void handle_error_source(struct pcie_device * aerdev,
494 struct aer_err_info info)
496 pci_ers_result_t status = 0;
499 if (info.severity == AER_CORRECTABLE) {
501 * Correctable error does not need software intevention.
502 * No need to go through error recovery process.
504 pos = pci_find_aer_capability(dev);
506 pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS,
509 status = do_recovery(aerdev, dev, info.severity);
510 if (status == PCI_ERS_RESULT_RECOVERED) {
511 printk(KERN_DEBUG "AER driver successfully recovered\n");
513 /* TODO: Should kernel panic here? */
514 printk(KERN_DEBUG "AER driver didn't recover\n");
520 * aer_enable_rootport - enable Root Port's interrupts when receiving messages
521 * @rpc: pointer to a Root Port data structure
523 * Invoked when PCIE bus loads AER service driver.
525 void aer_enable_rootport(struct aer_rpc *rpc)
527 struct pci_dev *pdev = rpc->rpd->port;
532 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
533 /* Clear PCIE Capability's Device Status */
534 pci_read_config_word(pdev, pos+PCI_EXP_DEVSTA, ®16);
535 pci_write_config_word(pdev, pos+PCI_EXP_DEVSTA, reg16);
537 /* Disable system error generation in response to error messages */
538 pci_read_config_word(pdev, pos + PCI_EXP_RTCTL, ®16);
539 reg16 &= ~(SYSTEM_ERROR_INTR_ON_MESG_MASK);
540 pci_write_config_word(pdev, pos + PCI_EXP_RTCTL, reg16);
542 aer_pos = pci_find_aer_capability(pdev);
543 /* Clear error status */
544 pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, ®32);
545 pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, reg32);
546 pci_read_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, ®32);
547 pci_write_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, reg32);
548 pci_read_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, ®32);
549 pci_write_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, reg32);
551 /* Enable Root Port device reporting error itself */
552 pci_read_config_word(pdev, pos+PCI_EXP_DEVCTL, ®16);
554 PCI_EXP_DEVCTL_CERE |
555 PCI_EXP_DEVCTL_NFERE |
556 PCI_EXP_DEVCTL_FERE |
558 pci_write_config_word(pdev, pos+PCI_EXP_DEVCTL,
561 /* Enable Root Port's interrupt in response to error messages */
562 pci_write_config_dword(pdev,
563 aer_pos + PCI_ERR_ROOT_COMMAND,
564 ROOT_PORT_INTR_ON_MESG_MASK);
568 * disable_root_aer - disable Root Port's interrupts when receiving messages
569 * @rpc: pointer to a Root Port data structure
571 * Invoked when PCIE bus unloads AER service driver.
573 static void disable_root_aer(struct aer_rpc *rpc)
575 struct pci_dev *pdev = rpc->rpd->port;
579 pos = pci_find_aer_capability(pdev);
580 /* Disable Root's interrupt in response to error messages */
581 pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, 0);
583 /* Clear Root's error status reg */
584 pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, ®32);
585 pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, reg32);
589 * get_e_source - retrieve an error source
590 * @rpc: pointer to the root port which holds an error
592 * Invoked by DPC handler to consume an error.
594 static struct aer_err_source* get_e_source(struct aer_rpc *rpc)
596 struct aer_err_source *e_source;
599 /* Lock access to Root error producer/consumer index */
600 spin_lock_irqsave(&rpc->e_lock, flags);
601 if (rpc->prod_idx == rpc->cons_idx) {
602 spin_unlock_irqrestore(&rpc->e_lock, flags);
605 e_source = &rpc->e_sources[rpc->cons_idx];
607 if (rpc->cons_idx == AER_ERROR_SOURCES_MAX)
609 spin_unlock_irqrestore(&rpc->e_lock, flags);
614 static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
618 pos = pci_find_aer_capability(dev);
620 /* The device might not support AER */
624 if (info->severity == AER_CORRECTABLE) {
625 pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS,
627 if (!(info->status & ERR_CORRECTABLE_ERROR_MASK))
628 return AER_UNSUCCESS;
629 } else if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE ||
630 info->severity == AER_NONFATAL) {
632 /* Link is still healthy for IO reads */
633 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS,
635 if (!(info->status & ERR_UNCORRECTABLE_ERROR_MASK))
636 return AER_UNSUCCESS;
638 if (info->status & AER_LOG_TLP_MASKS) {
639 info->flags |= AER_TLP_HEADER_VALID_FLAG;
640 pci_read_config_dword(dev,
641 pos + PCI_ERR_HEADER_LOG, &info->tlp.dw0);
642 pci_read_config_dword(dev,
643 pos + PCI_ERR_HEADER_LOG + 4, &info->tlp.dw1);
644 pci_read_config_dword(dev,
645 pos + PCI_ERR_HEADER_LOG + 8, &info->tlp.dw2);
646 pci_read_config_dword(dev,
647 pos + PCI_ERR_HEADER_LOG + 12, &info->tlp.dw3);
655 * aer_isr_one_error - consume an error detected by root port
656 * @p_device: pointer to error root port service device
657 * @e_src: pointer to an error source
659 static void aer_isr_one_error(struct pcie_device *p_device,
660 struct aer_err_source *e_src)
662 struct device *s_device;
663 struct aer_err_info e_info = {0, 0, 0,};
668 * There is a possibility that both correctable error and
669 * uncorrectable error being logged. Report correctable error first.
671 for (i = 1; i & ROOT_ERR_STATUS_MASKS ; i <<= 2) {
674 if (!(e_src->status & i))
677 /* Init comprehensive error information */
678 if (i & PCI_ERR_ROOT_COR_RCV) {
679 id = ERR_COR_ID(e_src->id);
680 e_info.severity = AER_CORRECTABLE;
682 id = ERR_UNCOR_ID(e_src->id);
683 e_info.severity = ((e_src->status >> 6) & 1);
686 (PCI_ERR_ROOT_MULTI_COR_RCV |
687 PCI_ERR_ROOT_MULTI_UNCOR_RCV))
688 e_info.flags |= AER_MULTI_ERROR_VALID_FLAG;
689 if (!(s_device = find_source_device(p_device->port, id))) {
690 printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
694 if (get_device_error_info(to_pci_dev(s_device), &e_info) ==
696 aer_print_error(to_pci_dev(s_device), &e_info);
697 handle_error_source(p_device,
698 to_pci_dev(s_device),
705 * aer_isr - consume errors detected by root port
706 * @work: definition of this work item
708 * Invoked, as DPC, when root port records new detected error
710 void aer_isr(struct work_struct *work)
712 struct aer_rpc *rpc = container_of(work, struct aer_rpc, dpc_handler);
713 struct pcie_device *p_device = rpc->rpd;
714 struct aer_err_source *e_src;
716 mutex_lock(&rpc->rpc_mutex);
717 e_src = get_e_source(rpc);
719 aer_isr_one_error(p_device, e_src);
720 e_src = get_e_source(rpc);
722 mutex_unlock(&rpc->rpc_mutex);
724 wake_up(&rpc->wait_release);
728 * aer_delete_rootport - disable root port aer and delete service data
729 * @rpc: pointer to a root port device being deleted
731 * Invoked when AER service unloaded on a specific Root Port
733 void aer_delete_rootport(struct aer_rpc *rpc)
735 /* Disable root port AER itself */
736 disable_root_aer(rpc);
742 * aer_init - provide AER initialization
743 * @dev: pointer to AER pcie device
745 * Invoked when AER service driver is loaded.
747 int aer_init(struct pcie_device *dev)
749 if (aer_osc_setup(dev) && !forceload)
755 EXPORT_SYMBOL_GPL(pci_find_aer_capability);
756 EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting);
757 EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting);
758 EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status);
759 EXPORT_SYMBOL_GPL(pci_cleanup_aer_correct_error_status);