cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

pci-driver.c (44203B)


      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
      4 * (C) Copyright 2007 Novell Inc.
      5 */
      6
      7#include <linux/pci.h>
      8#include <linux/module.h>
      9#include <linux/init.h>
     10#include <linux/device.h>
     11#include <linux/mempolicy.h>
     12#include <linux/string.h>
     13#include <linux/slab.h>
     14#include <linux/sched.h>
     15#include <linux/sched/isolation.h>
     16#include <linux/cpu.h>
     17#include <linux/pm_runtime.h>
     18#include <linux/suspend.h>
     19#include <linux/kexec.h>
     20#include <linux/of_device.h>
     21#include <linux/acpi.h>
     22#include <linux/dma-map-ops.h>
     23#include <linux/iommu.h>
     24#include "pci.h"
     25#include "pcie/portdrv.h"
     26
     27struct pci_dynid {
     28	struct list_head node;
     29	struct pci_device_id id;
     30};
     31
     32/**
     33 * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
     34 * @drv: target pci driver
     35 * @vendor: PCI vendor ID
     36 * @device: PCI device ID
     37 * @subvendor: PCI subvendor ID
     38 * @subdevice: PCI subdevice ID
     39 * @class: PCI class
     40 * @class_mask: PCI class mask
     41 * @driver_data: private driver data
     42 *
     43 * Adds a new dynamic pci device ID to this driver and causes the
     44 * driver to probe for all devices again.  @drv must have been
     45 * registered prior to calling this function.
     46 *
     47 * CONTEXT:
     48 * Does GFP_KERNEL allocation.
     49 *
     50 * RETURNS:
     51 * 0 on success, -errno on failure.
     52 */
     53int pci_add_dynid(struct pci_driver *drv,
     54		  unsigned int vendor, unsigned int device,
     55		  unsigned int subvendor, unsigned int subdevice,
     56		  unsigned int class, unsigned int class_mask,
     57		  unsigned long driver_data)
     58{
     59	struct pci_dynid *dynid;
     60
     61	dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
     62	if (!dynid)
     63		return -ENOMEM;
     64
     65	dynid->id.vendor = vendor;
     66	dynid->id.device = device;
     67	dynid->id.subvendor = subvendor;
     68	dynid->id.subdevice = subdevice;
     69	dynid->id.class = class;
     70	dynid->id.class_mask = class_mask;
     71	dynid->id.driver_data = driver_data;
     72
     73	spin_lock(&drv->dynids.lock);
     74	list_add_tail(&dynid->node, &drv->dynids.list);
     75	spin_unlock(&drv->dynids.lock);
     76
     77	return driver_attach(&drv->driver);
     78}
     79EXPORT_SYMBOL_GPL(pci_add_dynid);
     80
     81static void pci_free_dynids(struct pci_driver *drv)
     82{
     83	struct pci_dynid *dynid, *n;
     84
     85	spin_lock(&drv->dynids.lock);
     86	list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
     87		list_del(&dynid->node);
     88		kfree(dynid);
     89	}
     90	spin_unlock(&drv->dynids.lock);
     91}
     92
     93/**
     94 * pci_match_id - See if a PCI device matches a given pci_id table
     95 * @ids: array of PCI device ID structures to search in
     96 * @dev: the PCI device structure to match against.
     97 *
     98 * Used by a driver to check whether a PCI device is in its list of
     99 * supported devices.  Returns the matching pci_device_id structure or
    100 * %NULL if there is no match.
    101 *
    102 * Deprecated; don't use this as it will not catch any dynamic IDs
    103 * that a driver might want to check for.
    104 */
    105const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
    106					 struct pci_dev *dev)
    107{
    108	if (ids) {
    109		while (ids->vendor || ids->subvendor || ids->class_mask) {
    110			if (pci_match_one_device(ids, dev))
    111				return ids;
    112			ids++;
    113		}
    114	}
    115	return NULL;
    116}
    117EXPORT_SYMBOL(pci_match_id);
    118
    119static const struct pci_device_id pci_device_id_any = {
    120	.vendor = PCI_ANY_ID,
    121	.device = PCI_ANY_ID,
    122	.subvendor = PCI_ANY_ID,
    123	.subdevice = PCI_ANY_ID,
    124};
    125
    126/**
    127 * pci_match_device - See if a device matches a driver's list of IDs
    128 * @drv: the PCI driver to match against
    129 * @dev: the PCI device structure to match against
    130 *
    131 * Used by a driver to check whether a PCI device is in its list of
    132 * supported devices or in the dynids list, which may have been augmented
    133 * via the sysfs "new_id" file.  Returns the matching pci_device_id
    134 * structure or %NULL if there is no match.
    135 */
    136static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
    137						    struct pci_dev *dev)
    138{
    139	struct pci_dynid *dynid;
    140	const struct pci_device_id *found_id = NULL, *ids;
    141
    142	/* When driver_override is set, only bind to the matching driver */
    143	if (dev->driver_override && strcmp(dev->driver_override, drv->name))
    144		return NULL;
    145
    146	/* Look at the dynamic ids first, before the static ones */
    147	spin_lock(&drv->dynids.lock);
    148	list_for_each_entry(dynid, &drv->dynids.list, node) {
    149		if (pci_match_one_device(&dynid->id, dev)) {
    150			found_id = &dynid->id;
    151			break;
    152		}
    153	}
    154	spin_unlock(&drv->dynids.lock);
    155
    156	if (found_id)
    157		return found_id;
    158
    159	for (ids = drv->id_table; (found_id = pci_match_id(ids, dev));
    160	     ids = found_id + 1) {
    161		/*
    162		 * The match table is split based on driver_override.
    163		 * In case override_only was set, enforce driver_override
    164		 * matching.
    165		 */
    166		if (found_id->override_only) {
    167			if (dev->driver_override)
    168				return found_id;
    169		} else {
    170			return found_id;
    171		}
    172	}
    173
    174	/* driver_override will always match, send a dummy id */
    175	if (dev->driver_override)
    176		return &pci_device_id_any;
    177	return NULL;
    178}
    179
    180/**
    181 * new_id_store - sysfs frontend to pci_add_dynid()
    182 * @driver: target device driver
    183 * @buf: buffer for scanning device ID data
    184 * @count: input size
    185 *
    186 * Allow PCI IDs to be added to an existing driver via sysfs.
    187 */
    188static ssize_t new_id_store(struct device_driver *driver, const char *buf,
    189			    size_t count)
    190{
    191	struct pci_driver *pdrv = to_pci_driver(driver);
    192	const struct pci_device_id *ids = pdrv->id_table;
    193	u32 vendor, device, subvendor = PCI_ANY_ID,
    194		subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
    195	unsigned long driver_data = 0;
    196	int fields = 0;
    197	int retval = 0;
    198
    199	fields = sscanf(buf, "%x %x %x %x %x %x %lx",
    200			&vendor, &device, &subvendor, &subdevice,
    201			&class, &class_mask, &driver_data);
    202	if (fields < 2)
    203		return -EINVAL;
    204
    205	if (fields != 7) {
    206		struct pci_dev *pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
    207		if (!pdev)
    208			return -ENOMEM;
    209
    210		pdev->vendor = vendor;
    211		pdev->device = device;
    212		pdev->subsystem_vendor = subvendor;
    213		pdev->subsystem_device = subdevice;
    214		pdev->class = class;
    215
    216		if (pci_match_device(pdrv, pdev))
    217			retval = -EEXIST;
    218
    219		kfree(pdev);
    220
    221		if (retval)
    222			return retval;
    223	}
    224
    225	/* Only accept driver_data values that match an existing id_table
    226	   entry */
    227	if (ids) {
    228		retval = -EINVAL;
    229		while (ids->vendor || ids->subvendor || ids->class_mask) {
    230			if (driver_data == ids->driver_data) {
    231				retval = 0;
    232				break;
    233			}
    234			ids++;
    235		}
    236		if (retval)	/* No match */
    237			return retval;
    238	}
    239
    240	retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
    241			       class, class_mask, driver_data);
    242	if (retval)
    243		return retval;
    244	return count;
    245}
    246static DRIVER_ATTR_WO(new_id);
    247
    248/**
    249 * remove_id_store - remove a PCI device ID from this driver
    250 * @driver: target device driver
    251 * @buf: buffer for scanning device ID data
    252 * @count: input size
    253 *
    254 * Removes a dynamic pci device ID to this driver.
    255 */
    256static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
    257			       size_t count)
    258{
    259	struct pci_dynid *dynid, *n;
    260	struct pci_driver *pdrv = to_pci_driver(driver);
    261	u32 vendor, device, subvendor = PCI_ANY_ID,
    262		subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
    263	int fields = 0;
    264	size_t retval = -ENODEV;
    265
    266	fields = sscanf(buf, "%x %x %x %x %x %x",
    267			&vendor, &device, &subvendor, &subdevice,
    268			&class, &class_mask);
    269	if (fields < 2)
    270		return -EINVAL;
    271
    272	spin_lock(&pdrv->dynids.lock);
    273	list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
    274		struct pci_device_id *id = &dynid->id;
    275		if ((id->vendor == vendor) &&
    276		    (id->device == device) &&
    277		    (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
    278		    (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
    279		    !((id->class ^ class) & class_mask)) {
    280			list_del(&dynid->node);
    281			kfree(dynid);
    282			retval = count;
    283			break;
    284		}
    285	}
    286	spin_unlock(&pdrv->dynids.lock);
    287
    288	return retval;
    289}
    290static DRIVER_ATTR_WO(remove_id);
    291
    292static struct attribute *pci_drv_attrs[] = {
    293	&driver_attr_new_id.attr,
    294	&driver_attr_remove_id.attr,
    295	NULL,
    296};
    297ATTRIBUTE_GROUPS(pci_drv);
    298
    299struct drv_dev_and_id {
    300	struct pci_driver *drv;
    301	struct pci_dev *dev;
    302	const struct pci_device_id *id;
    303};
    304
    305static long local_pci_probe(void *_ddi)
    306{
    307	struct drv_dev_and_id *ddi = _ddi;
    308	struct pci_dev *pci_dev = ddi->dev;
    309	struct pci_driver *pci_drv = ddi->drv;
    310	struct device *dev = &pci_dev->dev;
    311	int rc;
    312
    313	/*
    314	 * Unbound PCI devices are always put in D0, regardless of
    315	 * runtime PM status.  During probe, the device is set to
    316	 * active and the usage count is incremented.  If the driver
    317	 * supports runtime PM, it should call pm_runtime_put_noidle(),
    318	 * or any other runtime PM helper function decrementing the usage
    319	 * count, in its probe routine and pm_runtime_get_noresume() in
    320	 * its remove routine.
    321	 */
    322	pm_runtime_get_sync(dev);
    323	pci_dev->driver = pci_drv;
    324	rc = pci_drv->probe(pci_dev, ddi->id);
    325	if (!rc)
    326		return rc;
    327	if (rc < 0) {
    328		pci_dev->driver = NULL;
    329		pm_runtime_put_sync(dev);
    330		return rc;
    331	}
    332	/*
    333	 * Probe function should return < 0 for failure, 0 for success
    334	 * Treat values > 0 as success, but warn.
    335	 */
    336	pci_warn(pci_dev, "Driver probe function unexpectedly returned %d\n",
    337		 rc);
    338	return 0;
    339}
    340
    341static bool pci_physfn_is_probed(struct pci_dev *dev)
    342{
    343#ifdef CONFIG_PCI_IOV
    344	return dev->is_virtfn && dev->physfn->is_probed;
    345#else
    346	return false;
    347#endif
    348}
    349
    350static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
    351			  const struct pci_device_id *id)
    352{
    353	int error, node, cpu;
    354	struct drv_dev_and_id ddi = { drv, dev, id };
    355
    356	/*
    357	 * Execute driver initialization on node where the device is
    358	 * attached.  This way the driver likely allocates its local memory
    359	 * on the right node.
    360	 */
    361	node = dev_to_node(&dev->dev);
    362	dev->is_probed = 1;
    363
    364	cpu_hotplug_disable();
    365
    366	/*
    367	 * Prevent nesting work_on_cpu() for the case where a Virtual Function
    368	 * device is probed from work_on_cpu() of the Physical device.
    369	 */
    370	if (node < 0 || node >= MAX_NUMNODES || !node_online(node) ||
    371	    pci_physfn_is_probed(dev)) {
    372		cpu = nr_cpu_ids;
    373	} else {
    374		cpumask_var_t wq_domain_mask;
    375
    376		if (!zalloc_cpumask_var(&wq_domain_mask, GFP_KERNEL)) {
    377			error = -ENOMEM;
    378			goto out;
    379		}
    380		cpumask_and(wq_domain_mask,
    381			    housekeeping_cpumask(HK_TYPE_WQ),
    382			    housekeeping_cpumask(HK_TYPE_DOMAIN));
    383
    384		cpu = cpumask_any_and(cpumask_of_node(node),
    385				      wq_domain_mask);
    386		free_cpumask_var(wq_domain_mask);
    387	}
    388
    389	if (cpu < nr_cpu_ids)
    390		error = work_on_cpu(cpu, local_pci_probe, &ddi);
    391	else
    392		error = local_pci_probe(&ddi);
    393out:
    394	dev->is_probed = 0;
    395	cpu_hotplug_enable();
    396	return error;
    397}
    398
    399/**
    400 * __pci_device_probe - check if a driver wants to claim a specific PCI device
    401 * @drv: driver to call to check if it wants the PCI device
    402 * @pci_dev: PCI device being probed
    403 *
    404 * returns 0 on success, else error.
    405 * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
    406 */
    407static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
    408{
    409	const struct pci_device_id *id;
    410	int error = 0;
    411
    412	if (drv->probe) {
    413		error = -ENODEV;
    414
    415		id = pci_match_device(drv, pci_dev);
    416		if (id)
    417			error = pci_call_probe(drv, pci_dev, id);
    418	}
    419	return error;
    420}
    421
    422int __weak pcibios_alloc_irq(struct pci_dev *dev)
    423{
    424	return 0;
    425}
    426
    427void __weak pcibios_free_irq(struct pci_dev *dev)
    428{
    429}
    430
    431#ifdef CONFIG_PCI_IOV
    432static inline bool pci_device_can_probe(struct pci_dev *pdev)
    433{
    434	return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe ||
    435		pdev->driver_override);
    436}
    437#else
    438static inline bool pci_device_can_probe(struct pci_dev *pdev)
    439{
    440	return true;
    441}
    442#endif
    443
    444static int pci_device_probe(struct device *dev)
    445{
    446	int error;
    447	struct pci_dev *pci_dev = to_pci_dev(dev);
    448	struct pci_driver *drv = to_pci_driver(dev->driver);
    449
    450	if (!pci_device_can_probe(pci_dev))
    451		return -ENODEV;
    452
    453	pci_assign_irq(pci_dev);
    454
    455	error = pcibios_alloc_irq(pci_dev);
    456	if (error < 0)
    457		return error;
    458
    459	pci_dev_get(pci_dev);
    460	error = __pci_device_probe(drv, pci_dev);
    461	if (error) {
    462		pcibios_free_irq(pci_dev);
    463		pci_dev_put(pci_dev);
    464	}
    465
    466	return error;
    467}
    468
    469static void pci_device_remove(struct device *dev)
    470{
    471	struct pci_dev *pci_dev = to_pci_dev(dev);
    472	struct pci_driver *drv = pci_dev->driver;
    473
    474	if (drv->remove) {
    475		pm_runtime_get_sync(dev);
    476		drv->remove(pci_dev);
    477		pm_runtime_put_noidle(dev);
    478	}
    479	pcibios_free_irq(pci_dev);
    480	pci_dev->driver = NULL;
    481	pci_iov_remove(pci_dev);
    482
    483	/* Undo the runtime PM settings in local_pci_probe() */
    484	pm_runtime_put_sync(dev);
    485
    486	/*
    487	 * If the device is still on, set the power state as "unknown",
    488	 * since it might change by the next time we load the driver.
    489	 */
    490	if (pci_dev->current_state == PCI_D0)
    491		pci_dev->current_state = PCI_UNKNOWN;
    492
    493	/*
    494	 * We would love to complain here if pci_dev->is_enabled is set, that
    495	 * the driver should have called pci_disable_device(), but the
    496	 * unfortunate fact is there are too many odd BIOS and bridge setups
    497	 * that don't like drivers doing that all of the time.
    498	 * Oh well, we can dream of sane hardware when we sleep, no matter how
    499	 * horrible the crap we have to deal with is when we are awake...
    500	 */
    501
    502	pci_dev_put(pci_dev);
    503}
    504
    505static void pci_device_shutdown(struct device *dev)
    506{
    507	struct pci_dev *pci_dev = to_pci_dev(dev);
    508	struct pci_driver *drv = pci_dev->driver;
    509
    510	pm_runtime_resume(dev);
    511
    512	if (drv && drv->shutdown)
    513		drv->shutdown(pci_dev);
    514
    515	/*
    516	 * If this is a kexec reboot, turn off Bus Master bit on the
    517	 * device to tell it to not continue to do DMA. Don't touch
    518	 * devices in D3cold or unknown states.
    519	 * If it is not a kexec reboot, firmware will hit the PCI
    520	 * devices with big hammer and stop their DMA any way.
    521	 */
    522	if (kexec_in_progress && (pci_dev->current_state <= PCI_D3hot))
    523		pci_clear_master(pci_dev);
    524}
    525
    526#ifdef CONFIG_PM_SLEEP
    527
    528/* Auxiliary functions used for system resume */
    529
    530/**
    531 * pci_restore_standard_config - restore standard config registers of PCI device
    532 * @pci_dev: PCI device to handle
    533 */
    534static int pci_restore_standard_config(struct pci_dev *pci_dev)
    535{
    536	pci_update_current_state(pci_dev, PCI_UNKNOWN);
    537
    538	if (pci_dev->current_state != PCI_D0) {
    539		int error = pci_set_power_state(pci_dev, PCI_D0);
    540		if (error)
    541			return error;
    542	}
    543
    544	pci_restore_state(pci_dev);
    545	pci_pme_restore(pci_dev);
    546	return 0;
    547}
    548#endif /* CONFIG_PM_SLEEP */
    549
    550#ifdef CONFIG_PM
    551
    552/* Auxiliary functions used for system resume and run-time resume */
    553
    554static void pci_pm_default_resume(struct pci_dev *pci_dev)
    555{
    556	pci_fixup_device(pci_fixup_resume, pci_dev);
    557	pci_enable_wake(pci_dev, PCI_D0, false);
    558}
    559
    560static void pci_pm_power_up_and_verify_state(struct pci_dev *pci_dev)
    561{
    562	pci_power_up(pci_dev);
    563	pci_update_current_state(pci_dev, PCI_D0);
    564}
    565
    566static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
    567{
    568	pci_pm_power_up_and_verify_state(pci_dev);
    569	pci_restore_state(pci_dev);
    570	pci_pme_restore(pci_dev);
    571}
    572
    573static void pci_pm_bridge_power_up_actions(struct pci_dev *pci_dev)
    574{
    575	pci_bridge_wait_for_secondary_bus(pci_dev);
    576	/*
    577	 * When powering on a bridge from D3cold, the whole hierarchy may be
    578	 * powered on into D0uninitialized state, resume them to give them a
    579	 * chance to suspend again
    580	 */
    581	pci_resume_bus(pci_dev->subordinate);
    582}
    583
    584#endif /* CONFIG_PM */
    585
    586#ifdef CONFIG_PM_SLEEP
    587
    588/*
    589 * Default "suspend" method for devices that have no driver provided suspend,
    590 * or not even a driver at all (second part).
    591 */
    592static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
    593{
    594	/*
    595	 * mark its power state as "unknown", since we don't know if
    596	 * e.g. the BIOS will change its device state when we suspend.
    597	 */
    598	if (pci_dev->current_state == PCI_D0)
    599		pci_dev->current_state = PCI_UNKNOWN;
    600}
    601
    602/*
    603 * Default "resume" method for devices that have no driver provided resume,
    604 * or not even a driver at all (second part).
    605 */
    606static int pci_pm_reenable_device(struct pci_dev *pci_dev)
    607{
    608	int retval;
    609
    610	/* if the device was enabled before suspend, re-enable */
    611	retval = pci_reenable_device(pci_dev);
    612	/*
    613	 * if the device was busmaster before the suspend, make it busmaster
    614	 * again
    615	 */
    616	if (pci_dev->is_busmaster)
    617		pci_set_master(pci_dev);
    618
    619	return retval;
    620}
    621
    622static int pci_legacy_suspend(struct device *dev, pm_message_t state)
    623{
    624	struct pci_dev *pci_dev = to_pci_dev(dev);
    625	struct pci_driver *drv = pci_dev->driver;
    626
    627	if (drv && drv->suspend) {
    628		pci_power_t prev = pci_dev->current_state;
    629		int error;
    630
    631		error = drv->suspend(pci_dev, state);
    632		suspend_report_result(dev, drv->suspend, error);
    633		if (error)
    634			return error;
    635
    636		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
    637		    && pci_dev->current_state != PCI_UNKNOWN) {
    638			pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
    639				      "PCI PM: Device state not saved by %pS\n",
    640				      drv->suspend);
    641		}
    642	}
    643
    644	pci_fixup_device(pci_fixup_suspend, pci_dev);
    645
    646	return 0;
    647}
    648
    649static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
    650{
    651	struct pci_dev *pci_dev = to_pci_dev(dev);
    652
    653	if (!pci_dev->state_saved)
    654		pci_save_state(pci_dev);
    655
    656	pci_pm_set_unknown_state(pci_dev);
    657
    658	pci_fixup_device(pci_fixup_suspend_late, pci_dev);
    659
    660	return 0;
    661}
    662
    663static int pci_legacy_resume(struct device *dev)
    664{
    665	struct pci_dev *pci_dev = to_pci_dev(dev);
    666	struct pci_driver *drv = pci_dev->driver;
    667
    668	pci_fixup_device(pci_fixup_resume, pci_dev);
    669
    670	return drv && drv->resume ?
    671			drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
    672}
    673
    674/* Auxiliary functions used by the new power management framework */
    675
    676static void pci_pm_default_suspend(struct pci_dev *pci_dev)
    677{
    678	/* Disable non-bridge devices without PM support */
    679	if (!pci_has_subordinate(pci_dev))
    680		pci_disable_enabled_device(pci_dev);
    681}
    682
    683static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
    684{
    685	struct pci_driver *drv = pci_dev->driver;
    686	bool ret = drv && (drv->suspend || drv->resume);
    687
    688	/*
    689	 * Legacy PM support is used by default, so warn if the new framework is
    690	 * supported as well.  Drivers are supposed to support either the
    691	 * former, or the latter, but not both at the same time.
    692	 */
    693	pci_WARN(pci_dev, ret && drv->driver.pm, "device %04x:%04x\n",
    694		 pci_dev->vendor, pci_dev->device);
    695
    696	return ret;
    697}
    698
    699/* New power management framework */
    700
    701static int pci_pm_prepare(struct device *dev)
    702{
    703	struct pci_dev *pci_dev = to_pci_dev(dev);
    704	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
    705
    706	if (pm && pm->prepare) {
    707		int error = pm->prepare(dev);
    708		if (error < 0)
    709			return error;
    710
    711		if (!error && dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_PREPARE))
    712			return 0;
    713	}
    714	if (pci_dev_need_resume(pci_dev))
    715		return 0;
    716
    717	/*
    718	 * The PME setting needs to be adjusted here in case the direct-complete
    719	 * optimization is used with respect to this device.
    720	 */
    721	pci_dev_adjust_pme(pci_dev);
    722	return 1;
    723}
    724
    725static void pci_pm_complete(struct device *dev)
    726{
    727	struct pci_dev *pci_dev = to_pci_dev(dev);
    728
    729	pci_dev_complete_resume(pci_dev);
    730	pm_generic_complete(dev);
    731
    732	/* Resume device if platform firmware has put it in reset-power-on */
    733	if (pm_runtime_suspended(dev) && pm_resume_via_firmware()) {
    734		pci_power_t pre_sleep_state = pci_dev->current_state;
    735
    736		pci_refresh_power_state(pci_dev);
    737		/*
    738		 * On platforms with ACPI this check may also trigger for
    739		 * devices sharing power resources if one of those power
    740		 * resources has been activated as a result of a change of the
    741		 * power state of another device sharing it.  However, in that
    742		 * case it is also better to resume the device, in general.
    743		 */
    744		if (pci_dev->current_state < pre_sleep_state)
    745			pm_request_resume(dev);
    746	}
    747}
    748
    749#else /* !CONFIG_PM_SLEEP */
    750
    751#define pci_pm_prepare	NULL
    752#define pci_pm_complete	NULL
    753
    754#endif /* !CONFIG_PM_SLEEP */
    755
    756#ifdef CONFIG_SUSPEND
    757static void pcie_pme_root_status_cleanup(struct pci_dev *pci_dev)
    758{
    759	/*
    760	 * Some BIOSes forget to clear Root PME Status bits after system
    761	 * wakeup, which breaks ACPI-based runtime wakeup on PCI Express.
    762	 * Clear those bits now just in case (shouldn't hurt).
    763	 */
    764	if (pci_is_pcie(pci_dev) &&
    765	    (pci_pcie_type(pci_dev) == PCI_EXP_TYPE_ROOT_PORT ||
    766	     pci_pcie_type(pci_dev) == PCI_EXP_TYPE_RC_EC))
    767		pcie_clear_root_pme_status(pci_dev);
    768}
    769
    770static int pci_pm_suspend(struct device *dev)
    771{
    772	struct pci_dev *pci_dev = to_pci_dev(dev);
    773	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
    774
    775	pci_dev->skip_bus_pm = false;
    776
    777	if (pci_has_legacy_pm_support(pci_dev))
    778		return pci_legacy_suspend(dev, PMSG_SUSPEND);
    779
    780	if (!pm) {
    781		pci_pm_default_suspend(pci_dev);
    782		return 0;
    783	}
    784
    785	/*
    786	 * PCI devices suspended at run time may need to be resumed at this
    787	 * point, because in general it may be necessary to reconfigure them for
    788	 * system suspend.  Namely, if the device is expected to wake up the
    789	 * system from the sleep state, it may have to be reconfigured for this
    790	 * purpose, or if the device is not expected to wake up the system from
    791	 * the sleep state, it should be prevented from signaling wakeup events
    792	 * going forward.
    793	 *
    794	 * Also if the driver of the device does not indicate that its system
    795	 * suspend callbacks can cope with runtime-suspended devices, it is
    796	 * better to resume the device from runtime suspend here.
    797	 */
    798	if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
    799	    pci_dev_need_resume(pci_dev)) {
    800		pm_runtime_resume(dev);
    801		pci_dev->state_saved = false;
    802	} else {
    803		pci_dev_adjust_pme(pci_dev);
    804	}
    805
    806	if (pm->suspend) {
    807		pci_power_t prev = pci_dev->current_state;
    808		int error;
    809
    810		error = pm->suspend(dev);
    811		suspend_report_result(dev, pm->suspend, error);
    812		if (error)
    813			return error;
    814
    815		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
    816		    && pci_dev->current_state != PCI_UNKNOWN) {
    817			pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
    818				      "PCI PM: State of device not saved by %pS\n",
    819				      pm->suspend);
    820		}
    821	}
    822
    823	return 0;
    824}
    825
    826static int pci_pm_suspend_late(struct device *dev)
    827{
    828	if (dev_pm_skip_suspend(dev))
    829		return 0;
    830
    831	pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
    832
    833	return pm_generic_suspend_late(dev);
    834}
    835
    836static int pci_pm_suspend_noirq(struct device *dev)
    837{
    838	struct pci_dev *pci_dev = to_pci_dev(dev);
    839	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
    840
    841	if (dev_pm_skip_suspend(dev))
    842		return 0;
    843
    844	if (pci_has_legacy_pm_support(pci_dev))
    845		return pci_legacy_suspend_late(dev, PMSG_SUSPEND);
    846
    847	if (!pm) {
    848		pci_save_state(pci_dev);
    849		goto Fixup;
    850	}
    851
    852	if (pm->suspend_noirq) {
    853		pci_power_t prev = pci_dev->current_state;
    854		int error;
    855
    856		error = pm->suspend_noirq(dev);
    857		suspend_report_result(dev, pm->suspend_noirq, error);
    858		if (error)
    859			return error;
    860
    861		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
    862		    && pci_dev->current_state != PCI_UNKNOWN) {
    863			pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
    864				      "PCI PM: State of device not saved by %pS\n",
    865				      pm->suspend_noirq);
    866			goto Fixup;
    867		}
    868	}
    869
    870	if (pci_dev->skip_bus_pm) {
    871		/*
    872		 * Either the device is a bridge with a child in D0 below it, or
    873		 * the function is running for the second time in a row without
    874		 * going through full resume, which is possible only during
    875		 * suspend-to-idle in a spurious wakeup case.  The device should
    876		 * be in D0 at this point, but if it is a bridge, it may be
    877		 * necessary to save its state.
    878		 */
    879		if (!pci_dev->state_saved)
    880			pci_save_state(pci_dev);
    881	} else if (!pci_dev->state_saved) {
    882		pci_save_state(pci_dev);
    883		if (pci_power_manageable(pci_dev))
    884			pci_prepare_to_sleep(pci_dev);
    885	}
    886
    887	pci_dbg(pci_dev, "PCI PM: Suspend power state: %s\n",
    888		pci_power_name(pci_dev->current_state));
    889
    890	if (pci_dev->current_state == PCI_D0) {
    891		pci_dev->skip_bus_pm = true;
    892		/*
    893		 * Per PCI PM r1.2, table 6-1, a bridge must be in D0 if any
    894		 * downstream device is in D0, so avoid changing the power state
    895		 * of the parent bridge by setting the skip_bus_pm flag for it.
    896		 */
    897		if (pci_dev->bus->self)
    898			pci_dev->bus->self->skip_bus_pm = true;
    899	}
    900
    901	if (pci_dev->skip_bus_pm && pm_suspend_no_platform()) {
    902		pci_dbg(pci_dev, "PCI PM: Skipped\n");
    903		goto Fixup;
    904	}
    905
    906	pci_pm_set_unknown_state(pci_dev);
    907
    908	/*
    909	 * Some BIOSes from ASUS have a bug: If a USB EHCI host controller's
    910	 * PCI COMMAND register isn't 0, the BIOS assumes that the controller
    911	 * hasn't been quiesced and tries to turn it off.  If the controller
    912	 * is already in D3, this can hang or cause memory corruption.
    913	 *
    914	 * Since the value of the COMMAND register doesn't matter once the
    915	 * device has been suspended, we can safely set it to 0 here.
    916	 */
    917	if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
    918		pci_write_config_word(pci_dev, PCI_COMMAND, 0);
    919
    920Fixup:
    921	pci_fixup_device(pci_fixup_suspend_late, pci_dev);
    922
    923	/*
    924	 * If the target system sleep state is suspend-to-idle, it is sufficient
    925	 * to check whether or not the device's wakeup settings are good for
    926	 * runtime PM.  Otherwise, the pm_resume_via_firmware() check will cause
    927	 * pci_pm_complete() to take care of fixing up the device's state
    928	 * anyway, if need be.
    929	 */
    930	if (device_can_wakeup(dev) && !device_may_wakeup(dev))
    931		dev->power.may_skip_resume = false;
    932
    933	return 0;
    934}
    935
    936static int pci_pm_resume_noirq(struct device *dev)
    937{
    938	struct pci_dev *pci_dev = to_pci_dev(dev);
    939	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
    940	pci_power_t prev_state = pci_dev->current_state;
    941	bool skip_bus_pm = pci_dev->skip_bus_pm;
    942
    943	if (dev_pm_skip_resume(dev))
    944		return 0;
    945
    946	/*
    947	 * In the suspend-to-idle case, devices left in D0 during suspend will
    948	 * stay in D0, so it is not necessary to restore or update their
    949	 * configuration here and attempting to put them into D0 again is
    950	 * pointless, so avoid doing that.
    951	 */
    952	if (!(skip_bus_pm && pm_suspend_no_platform()))
    953		pci_pm_default_resume_early(pci_dev);
    954
    955	pci_fixup_device(pci_fixup_resume_early, pci_dev);
    956	pcie_pme_root_status_cleanup(pci_dev);
    957
    958	if (!skip_bus_pm && prev_state == PCI_D3cold)
    959		pci_pm_bridge_power_up_actions(pci_dev);
    960
    961	if (pci_has_legacy_pm_support(pci_dev))
    962		return 0;
    963
    964	if (pm && pm->resume_noirq)
    965		return pm->resume_noirq(dev);
    966
    967	return 0;
    968}
    969
    970static int pci_pm_resume_early(struct device *dev)
    971{
    972	if (dev_pm_skip_resume(dev))
    973		return 0;
    974
    975	return pm_generic_resume_early(dev);
    976}
    977
    978static int pci_pm_resume(struct device *dev)
    979{
    980	struct pci_dev *pci_dev = to_pci_dev(dev);
    981	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
    982
    983	/*
    984	 * This is necessary for the suspend error path in which resume is
    985	 * called without restoring the standard config registers of the device.
    986	 */
    987	if (pci_dev->state_saved)
    988		pci_restore_standard_config(pci_dev);
    989
    990	if (pci_has_legacy_pm_support(pci_dev))
    991		return pci_legacy_resume(dev);
    992
    993	pci_pm_default_resume(pci_dev);
    994
    995	if (pm) {
    996		if (pm->resume)
    997			return pm->resume(dev);
    998	} else {
    999		pci_pm_reenable_device(pci_dev);
   1000	}
   1001
   1002	return 0;
   1003}
   1004
   1005#else /* !CONFIG_SUSPEND */
   1006
   1007#define pci_pm_suspend		NULL
   1008#define pci_pm_suspend_late	NULL
   1009#define pci_pm_suspend_noirq	NULL
   1010#define pci_pm_resume		NULL
   1011#define pci_pm_resume_early	NULL
   1012#define pci_pm_resume_noirq	NULL
   1013
   1014#endif /* !CONFIG_SUSPEND */
   1015
   1016#ifdef CONFIG_HIBERNATE_CALLBACKS
   1017
   1018static int pci_pm_freeze(struct device *dev)
   1019{
   1020	struct pci_dev *pci_dev = to_pci_dev(dev);
   1021	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
   1022
   1023	if (pci_has_legacy_pm_support(pci_dev))
   1024		return pci_legacy_suspend(dev, PMSG_FREEZE);
   1025
   1026	if (!pm) {
   1027		pci_pm_default_suspend(pci_dev);
   1028		return 0;
   1029	}
   1030
   1031	/*
   1032	 * Resume all runtime-suspended devices before creating a snapshot
   1033	 * image of system memory, because the restore kernel generally cannot
   1034	 * be expected to always handle them consistently and they need to be
   1035	 * put into the runtime-active metastate during system resume anyway,
   1036	 * so it is better to ensure that the state saved in the image will be
   1037	 * always consistent with that.
   1038	 */
   1039	pm_runtime_resume(dev);
   1040	pci_dev->state_saved = false;
   1041
   1042	if (pm->freeze) {
   1043		int error;
   1044
   1045		error = pm->freeze(dev);
   1046		suspend_report_result(dev, pm->freeze, error);
   1047		if (error)
   1048			return error;
   1049	}
   1050
   1051	return 0;
   1052}
   1053
   1054static int pci_pm_freeze_noirq(struct device *dev)
   1055{
   1056	struct pci_dev *pci_dev = to_pci_dev(dev);
   1057	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
   1058
   1059	if (pci_has_legacy_pm_support(pci_dev))
   1060		return pci_legacy_suspend_late(dev, PMSG_FREEZE);
   1061
   1062	if (pm && pm->freeze_noirq) {
   1063		int error;
   1064
   1065		error = pm->freeze_noirq(dev);
   1066		suspend_report_result(dev, pm->freeze_noirq, error);
   1067		if (error)
   1068			return error;
   1069	}
   1070
   1071	if (!pci_dev->state_saved)
   1072		pci_save_state(pci_dev);
   1073
   1074	pci_pm_set_unknown_state(pci_dev);
   1075
   1076	return 0;
   1077}
   1078
   1079static int pci_pm_thaw_noirq(struct device *dev)
   1080{
   1081	struct pci_dev *pci_dev = to_pci_dev(dev);
   1082	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
   1083
   1084	/*
   1085	 * The pm->thaw_noirq() callback assumes the device has been
   1086	 * returned to D0 and its config state has been restored.
   1087	 *
   1088	 * In addition, pci_restore_state() restores MSI-X state in MMIO
   1089	 * space, which requires the device to be in D0, so return it to D0
   1090	 * in case the driver's "freeze" callbacks put it into a low-power
   1091	 * state.
   1092	 */
   1093	pci_pm_power_up_and_verify_state(pci_dev);
   1094	pci_restore_state(pci_dev);
   1095
   1096	if (pci_has_legacy_pm_support(pci_dev))
   1097		return 0;
   1098
   1099	if (pm && pm->thaw_noirq)
   1100		return pm->thaw_noirq(dev);
   1101
   1102	return 0;
   1103}
   1104
   1105static int pci_pm_thaw(struct device *dev)
   1106{
   1107	struct pci_dev *pci_dev = to_pci_dev(dev);
   1108	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
   1109	int error = 0;
   1110
   1111	if (pci_has_legacy_pm_support(pci_dev))
   1112		return pci_legacy_resume(dev);
   1113
   1114	if (pm) {
   1115		if (pm->thaw)
   1116			error = pm->thaw(dev);
   1117	} else {
   1118		pci_pm_reenable_device(pci_dev);
   1119	}
   1120
   1121	pci_dev->state_saved = false;
   1122
   1123	return error;
   1124}
   1125
   1126static int pci_pm_poweroff(struct device *dev)
   1127{
   1128	struct pci_dev *pci_dev = to_pci_dev(dev);
   1129	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
   1130
   1131	if (pci_has_legacy_pm_support(pci_dev))
   1132		return pci_legacy_suspend(dev, PMSG_HIBERNATE);
   1133
   1134	if (!pm) {
   1135		pci_pm_default_suspend(pci_dev);
   1136		return 0;
   1137	}
   1138
   1139	/* The reason to do that is the same as in pci_pm_suspend(). */
   1140	if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
   1141	    pci_dev_need_resume(pci_dev)) {
   1142		pm_runtime_resume(dev);
   1143		pci_dev->state_saved = false;
   1144	} else {
   1145		pci_dev_adjust_pme(pci_dev);
   1146	}
   1147
   1148	if (pm->poweroff) {
   1149		int error;
   1150
   1151		error = pm->poweroff(dev);
   1152		suspend_report_result(dev, pm->poweroff, error);
   1153		if (error)
   1154			return error;
   1155	}
   1156
   1157	return 0;
   1158}
   1159
   1160static int pci_pm_poweroff_late(struct device *dev)
   1161{
   1162	if (dev_pm_skip_suspend(dev))
   1163		return 0;
   1164
   1165	pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
   1166
   1167	return pm_generic_poweroff_late(dev);
   1168}
   1169
   1170static int pci_pm_poweroff_noirq(struct device *dev)
   1171{
   1172	struct pci_dev *pci_dev = to_pci_dev(dev);
   1173	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
   1174
   1175	if (dev_pm_skip_suspend(dev))
   1176		return 0;
   1177
   1178	if (pci_has_legacy_pm_support(pci_dev))
   1179		return pci_legacy_suspend_late(dev, PMSG_HIBERNATE);
   1180
   1181	if (!pm) {
   1182		pci_fixup_device(pci_fixup_suspend_late, pci_dev);
   1183		return 0;
   1184	}
   1185
   1186	if (pm->poweroff_noirq) {
   1187		int error;
   1188
   1189		error = pm->poweroff_noirq(dev);
   1190		suspend_report_result(dev, pm->poweroff_noirq, error);
   1191		if (error)
   1192			return error;
   1193	}
   1194
   1195	if (!pci_dev->state_saved && !pci_has_subordinate(pci_dev))
   1196		pci_prepare_to_sleep(pci_dev);
   1197
   1198	/*
   1199	 * The reason for doing this here is the same as for the analogous code
   1200	 * in pci_pm_suspend_noirq().
   1201	 */
   1202	if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
   1203		pci_write_config_word(pci_dev, PCI_COMMAND, 0);
   1204
   1205	pci_fixup_device(pci_fixup_suspend_late, pci_dev);
   1206
   1207	return 0;
   1208}
   1209
   1210static int pci_pm_restore_noirq(struct device *dev)
   1211{
   1212	struct pci_dev *pci_dev = to_pci_dev(dev);
   1213	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
   1214
   1215	pci_pm_default_resume_early(pci_dev);
   1216	pci_fixup_device(pci_fixup_resume_early, pci_dev);
   1217
   1218	if (pci_has_legacy_pm_support(pci_dev))
   1219		return 0;
   1220
   1221	if (pm && pm->restore_noirq)
   1222		return pm->restore_noirq(dev);
   1223
   1224	return 0;
   1225}
   1226
   1227static int pci_pm_restore(struct device *dev)
   1228{
   1229	struct pci_dev *pci_dev = to_pci_dev(dev);
   1230	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
   1231
   1232	/*
   1233	 * This is necessary for the hibernation error path in which restore is
   1234	 * called without restoring the standard config registers of the device.
   1235	 */
   1236	if (pci_dev->state_saved)
   1237		pci_restore_standard_config(pci_dev);
   1238
   1239	if (pci_has_legacy_pm_support(pci_dev))
   1240		return pci_legacy_resume(dev);
   1241
   1242	pci_pm_default_resume(pci_dev);
   1243
   1244	if (pm) {
   1245		if (pm->restore)
   1246			return pm->restore(dev);
   1247	} else {
   1248		pci_pm_reenable_device(pci_dev);
   1249	}
   1250
   1251	return 0;
   1252}
   1253
   1254#else /* !CONFIG_HIBERNATE_CALLBACKS */
   1255
   1256#define pci_pm_freeze		NULL
   1257#define pci_pm_freeze_noirq	NULL
   1258#define pci_pm_thaw		NULL
   1259#define pci_pm_thaw_noirq	NULL
   1260#define pci_pm_poweroff		NULL
   1261#define pci_pm_poweroff_late	NULL
   1262#define pci_pm_poweroff_noirq	NULL
   1263#define pci_pm_restore		NULL
   1264#define pci_pm_restore_noirq	NULL
   1265
   1266#endif /* !CONFIG_HIBERNATE_CALLBACKS */
   1267
   1268#ifdef CONFIG_PM
   1269
   1270static int pci_pm_runtime_suspend(struct device *dev)
   1271{
   1272	struct pci_dev *pci_dev = to_pci_dev(dev);
   1273	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
   1274	pci_power_t prev = pci_dev->current_state;
   1275	int error;
   1276
   1277	/*
   1278	 * If pci_dev->driver is not set (unbound), we leave the device in D0,
   1279	 * but it may go to D3cold when the bridge above it runtime suspends.
   1280	 * Save its config space in case that happens.
   1281	 */
   1282	if (!pci_dev->driver) {
   1283		pci_save_state(pci_dev);
   1284		return 0;
   1285	}
   1286
   1287	pci_dev->state_saved = false;
   1288	if (pm && pm->runtime_suspend) {
   1289		error = pm->runtime_suspend(dev);
   1290		/*
   1291		 * -EBUSY and -EAGAIN is used to request the runtime PM core
   1292		 * to schedule a new suspend, so log the event only with debug
   1293		 * log level.
   1294		 */
   1295		if (error == -EBUSY || error == -EAGAIN) {
   1296			pci_dbg(pci_dev, "can't suspend now (%ps returned %d)\n",
   1297				pm->runtime_suspend, error);
   1298			return error;
   1299		} else if (error) {
   1300			pci_err(pci_dev, "can't suspend (%ps returned %d)\n",
   1301				pm->runtime_suspend, error);
   1302			return error;
   1303		}
   1304	}
   1305
   1306	pci_fixup_device(pci_fixup_suspend, pci_dev);
   1307
   1308	if (pm && pm->runtime_suspend
   1309	    && !pci_dev->state_saved && pci_dev->current_state != PCI_D0
   1310	    && pci_dev->current_state != PCI_UNKNOWN) {
   1311		pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
   1312			      "PCI PM: State of device not saved by %pS\n",
   1313			      pm->runtime_suspend);
   1314		return 0;
   1315	}
   1316
   1317	if (!pci_dev->state_saved) {
   1318		pci_save_state(pci_dev);
   1319		pci_finish_runtime_suspend(pci_dev);
   1320	}
   1321
   1322	return 0;
   1323}
   1324
   1325static int pci_pm_runtime_resume(struct device *dev)
   1326{
   1327	struct pci_dev *pci_dev = to_pci_dev(dev);
   1328	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
   1329	pci_power_t prev_state = pci_dev->current_state;
   1330	int error = 0;
   1331
   1332	/*
   1333	 * Restoring config space is necessary even if the device is not bound
   1334	 * to a driver because although we left it in D0, it may have gone to
   1335	 * D3cold when the bridge above it runtime suspended.
   1336	 */
   1337	pci_pm_default_resume_early(pci_dev);
   1338
   1339	if (!pci_dev->driver)
   1340		return 0;
   1341
   1342	pci_fixup_device(pci_fixup_resume_early, pci_dev);
   1343	pci_pm_default_resume(pci_dev);
   1344
   1345	if (prev_state == PCI_D3cold)
   1346		pci_pm_bridge_power_up_actions(pci_dev);
   1347
   1348	if (pm && pm->runtime_resume)
   1349		error = pm->runtime_resume(dev);
   1350
   1351	return error;
   1352}
   1353
   1354static int pci_pm_runtime_idle(struct device *dev)
   1355{
   1356	struct pci_dev *pci_dev = to_pci_dev(dev);
   1357	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
   1358
   1359	/*
   1360	 * If pci_dev->driver is not set (unbound), the device should
   1361	 * always remain in D0 regardless of the runtime PM status
   1362	 */
   1363	if (!pci_dev->driver)
   1364		return 0;
   1365
   1366	if (!pm)
   1367		return -ENOSYS;
   1368
   1369	if (pm->runtime_idle)
   1370		return pm->runtime_idle(dev);
   1371
   1372	return 0;
   1373}
   1374
   1375static const struct dev_pm_ops pci_dev_pm_ops = {
   1376	.prepare = pci_pm_prepare,
   1377	.complete = pci_pm_complete,
   1378	.suspend = pci_pm_suspend,
   1379	.suspend_late = pci_pm_suspend_late,
   1380	.resume = pci_pm_resume,
   1381	.resume_early = pci_pm_resume_early,
   1382	.freeze = pci_pm_freeze,
   1383	.thaw = pci_pm_thaw,
   1384	.poweroff = pci_pm_poweroff,
   1385	.poweroff_late = pci_pm_poweroff_late,
   1386	.restore = pci_pm_restore,
   1387	.suspend_noirq = pci_pm_suspend_noirq,
   1388	.resume_noirq = pci_pm_resume_noirq,
   1389	.freeze_noirq = pci_pm_freeze_noirq,
   1390	.thaw_noirq = pci_pm_thaw_noirq,
   1391	.poweroff_noirq = pci_pm_poweroff_noirq,
   1392	.restore_noirq = pci_pm_restore_noirq,
   1393	.runtime_suspend = pci_pm_runtime_suspend,
   1394	.runtime_resume = pci_pm_runtime_resume,
   1395	.runtime_idle = pci_pm_runtime_idle,
   1396};
   1397
   1398#define PCI_PM_OPS_PTR	(&pci_dev_pm_ops)
   1399
   1400#else /* !CONFIG_PM */
   1401
   1402#define pci_pm_runtime_suspend	NULL
   1403#define pci_pm_runtime_resume	NULL
   1404#define pci_pm_runtime_idle	NULL
   1405
   1406#define PCI_PM_OPS_PTR	NULL
   1407
   1408#endif /* !CONFIG_PM */
   1409
   1410/**
   1411 * __pci_register_driver - register a new pci driver
   1412 * @drv: the driver structure to register
   1413 * @owner: owner module of drv
   1414 * @mod_name: module name string
   1415 *
   1416 * Adds the driver structure to the list of registered drivers.
   1417 * Returns a negative value on error, otherwise 0.
   1418 * If no error occurred, the driver remains registered even if
   1419 * no device was claimed during registration.
   1420 */
   1421int __pci_register_driver(struct pci_driver *drv, struct module *owner,
   1422			  const char *mod_name)
   1423{
   1424	/* initialize common driver fields */
   1425	drv->driver.name = drv->name;
   1426	drv->driver.bus = &pci_bus_type;
   1427	drv->driver.owner = owner;
   1428	drv->driver.mod_name = mod_name;
   1429	drv->driver.groups = drv->groups;
   1430	drv->driver.dev_groups = drv->dev_groups;
   1431
   1432	spin_lock_init(&drv->dynids.lock);
   1433	INIT_LIST_HEAD(&drv->dynids.list);
   1434
   1435	/* register with core */
   1436	return driver_register(&drv->driver);
   1437}
   1438EXPORT_SYMBOL(__pci_register_driver);
   1439
   1440/**
   1441 * pci_unregister_driver - unregister a pci driver
   1442 * @drv: the driver structure to unregister
   1443 *
   1444 * Deletes the driver structure from the list of registered PCI drivers,
   1445 * gives it a chance to clean up by calling its remove() function for
   1446 * each device it was responsible for, and marks those devices as
   1447 * driverless.
   1448 */
   1449
   1450void pci_unregister_driver(struct pci_driver *drv)
   1451{
   1452	driver_unregister(&drv->driver);
   1453	pci_free_dynids(drv);
   1454}
   1455EXPORT_SYMBOL(pci_unregister_driver);
   1456
   1457static struct pci_driver pci_compat_driver = {
   1458	.name = "compat"
   1459};
   1460
   1461/**
   1462 * pci_dev_driver - get the pci_driver of a device
   1463 * @dev: the device to query
   1464 *
   1465 * Returns the appropriate pci_driver structure or %NULL if there is no
   1466 * registered driver for the device.
   1467 */
   1468struct pci_driver *pci_dev_driver(const struct pci_dev *dev)
   1469{
   1470	if (dev->driver)
   1471		return dev->driver;
   1472	else {
   1473		int i;
   1474		for (i = 0; i <= PCI_ROM_RESOURCE; i++)
   1475			if (dev->resource[i].flags & IORESOURCE_BUSY)
   1476				return &pci_compat_driver;
   1477	}
   1478	return NULL;
   1479}
   1480EXPORT_SYMBOL(pci_dev_driver);
   1481
   1482/**
   1483 * pci_bus_match - Tell if a PCI device structure has a matching PCI device id structure
   1484 * @dev: the PCI device structure to match against
   1485 * @drv: the device driver to search for matching PCI device id structures
   1486 *
   1487 * Used by a driver to check whether a PCI device present in the
   1488 * system is in its list of supported devices. Returns the matching
   1489 * pci_device_id structure or %NULL if there is no match.
   1490 */
   1491static int pci_bus_match(struct device *dev, struct device_driver *drv)
   1492{
   1493	struct pci_dev *pci_dev = to_pci_dev(dev);
   1494	struct pci_driver *pci_drv;
   1495	const struct pci_device_id *found_id;
   1496
   1497	if (!pci_dev->match_driver)
   1498		return 0;
   1499
   1500	pci_drv = to_pci_driver(drv);
   1501	found_id = pci_match_device(pci_drv, pci_dev);
   1502	if (found_id)
   1503		return 1;
   1504
   1505	return 0;
   1506}
   1507
   1508/**
   1509 * pci_dev_get - increments the reference count of the pci device structure
   1510 * @dev: the device being referenced
   1511 *
   1512 * Each live reference to a device should be refcounted.
   1513 *
   1514 * Drivers for PCI devices should normally record such references in
   1515 * their probe() methods, when they bind to a device, and release
   1516 * them by calling pci_dev_put(), in their disconnect() methods.
   1517 *
   1518 * A pointer to the device with the incremented reference counter is returned.
   1519 */
   1520struct pci_dev *pci_dev_get(struct pci_dev *dev)
   1521{
   1522	if (dev)
   1523		get_device(&dev->dev);
   1524	return dev;
   1525}
   1526EXPORT_SYMBOL(pci_dev_get);
   1527
   1528/**
   1529 * pci_dev_put - release a use of the pci device structure
   1530 * @dev: device that's been disconnected
   1531 *
   1532 * Must be called when a user of a device is finished with it.  When the last
   1533 * user of the device calls this function, the memory of the device is freed.
   1534 */
   1535void pci_dev_put(struct pci_dev *dev)
   1536{
   1537	if (dev)
   1538		put_device(&dev->dev);
   1539}
   1540EXPORT_SYMBOL(pci_dev_put);
   1541
   1542static int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
   1543{
   1544	struct pci_dev *pdev;
   1545
   1546	if (!dev)
   1547		return -ENODEV;
   1548
   1549	pdev = to_pci_dev(dev);
   1550
   1551	if (add_uevent_var(env, "PCI_CLASS=%04X", pdev->class))
   1552		return -ENOMEM;
   1553
   1554	if (add_uevent_var(env, "PCI_ID=%04X:%04X", pdev->vendor, pdev->device))
   1555		return -ENOMEM;
   1556
   1557	if (add_uevent_var(env, "PCI_SUBSYS_ID=%04X:%04X", pdev->subsystem_vendor,
   1558			   pdev->subsystem_device))
   1559		return -ENOMEM;
   1560
   1561	if (add_uevent_var(env, "PCI_SLOT_NAME=%s", pci_name(pdev)))
   1562		return -ENOMEM;
   1563
   1564	if (add_uevent_var(env, "MODALIAS=pci:v%08Xd%08Xsv%08Xsd%08Xbc%02Xsc%02Xi%02X",
   1565			   pdev->vendor, pdev->device,
   1566			   pdev->subsystem_vendor, pdev->subsystem_device,
   1567			   (u8)(pdev->class >> 16), (u8)(pdev->class >> 8),
   1568			   (u8)(pdev->class)))
   1569		return -ENOMEM;
   1570
   1571	return 0;
   1572}
   1573
   1574#if defined(CONFIG_PCIEAER) || defined(CONFIG_EEH)
   1575/**
   1576 * pci_uevent_ers - emit a uevent during recovery path of PCI device
   1577 * @pdev: PCI device undergoing error recovery
   1578 * @err_type: type of error event
   1579 */
   1580void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type)
   1581{
   1582	int idx = 0;
   1583	char *envp[3];
   1584
   1585	switch (err_type) {
   1586	case PCI_ERS_RESULT_NONE:
   1587	case PCI_ERS_RESULT_CAN_RECOVER:
   1588		envp[idx++] = "ERROR_EVENT=BEGIN_RECOVERY";
   1589		envp[idx++] = "DEVICE_ONLINE=0";
   1590		break;
   1591	case PCI_ERS_RESULT_RECOVERED:
   1592		envp[idx++] = "ERROR_EVENT=SUCCESSFUL_RECOVERY";
   1593		envp[idx++] = "DEVICE_ONLINE=1";
   1594		break;
   1595	case PCI_ERS_RESULT_DISCONNECT:
   1596		envp[idx++] = "ERROR_EVENT=FAILED_RECOVERY";
   1597		envp[idx++] = "DEVICE_ONLINE=0";
   1598		break;
   1599	default:
   1600		break;
   1601	}
   1602
   1603	if (idx > 0) {
   1604		envp[idx++] = NULL;
   1605		kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, envp);
   1606	}
   1607}
   1608#endif
   1609
   1610static int pci_bus_num_vf(struct device *dev)
   1611{
   1612	return pci_num_vf(to_pci_dev(dev));
   1613}
   1614
   1615/**
   1616 * pci_dma_configure - Setup DMA configuration
   1617 * @dev: ptr to dev structure
   1618 *
   1619 * Function to update PCI devices's DMA configuration using the same
   1620 * info from the OF node or ACPI node of host bridge's parent (if any).
   1621 */
   1622static int pci_dma_configure(struct device *dev)
   1623{
   1624	struct pci_driver *driver = to_pci_driver(dev->driver);
   1625	struct device *bridge;
   1626	int ret = 0;
   1627
   1628	bridge = pci_get_host_bridge_device(to_pci_dev(dev));
   1629
   1630	if (IS_ENABLED(CONFIG_OF) && bridge->parent &&
   1631	    bridge->parent->of_node) {
   1632		ret = of_dma_configure(dev, bridge->parent->of_node, true);
   1633	} else if (has_acpi_companion(bridge)) {
   1634		struct acpi_device *adev = to_acpi_device_node(bridge->fwnode);
   1635
   1636		ret = acpi_dma_configure(dev, acpi_get_dma_attr(adev));
   1637	}
   1638
   1639	pci_put_host_bridge_device(bridge);
   1640
   1641	if (!ret && !driver->driver_managed_dma) {
   1642		ret = iommu_device_use_default_domain(dev);
   1643		if (ret)
   1644			arch_teardown_dma_ops(dev);
   1645	}
   1646
   1647	return ret;
   1648}
   1649
   1650static void pci_dma_cleanup(struct device *dev)
   1651{
   1652	struct pci_driver *driver = to_pci_driver(dev->driver);
   1653
   1654	if (!driver->driver_managed_dma)
   1655		iommu_device_unuse_default_domain(dev);
   1656}
   1657
   1658struct bus_type pci_bus_type = {
   1659	.name		= "pci",
   1660	.match		= pci_bus_match,
   1661	.uevent		= pci_uevent,
   1662	.probe		= pci_device_probe,
   1663	.remove		= pci_device_remove,
   1664	.shutdown	= pci_device_shutdown,
   1665	.dev_groups	= pci_dev_groups,
   1666	.bus_groups	= pci_bus_groups,
   1667	.drv_groups	= pci_drv_groups,
   1668	.pm		= PCI_PM_OPS_PTR,
   1669	.num_vf		= pci_bus_num_vf,
   1670	.dma_configure	= pci_dma_configure,
   1671	.dma_cleanup	= pci_dma_cleanup,
   1672};
   1673EXPORT_SYMBOL(pci_bus_type);
   1674
   1675#ifdef CONFIG_PCIEPORTBUS
   1676static int pcie_port_bus_match(struct device *dev, struct device_driver *drv)
   1677{
   1678	struct pcie_device *pciedev;
   1679	struct pcie_port_service_driver *driver;
   1680
   1681	if (drv->bus != &pcie_port_bus_type || dev->bus != &pcie_port_bus_type)
   1682		return 0;
   1683
   1684	pciedev = to_pcie_device(dev);
   1685	driver = to_service_driver(drv);
   1686
   1687	if (driver->service != pciedev->service)
   1688		return 0;
   1689
   1690	if (driver->port_type != PCIE_ANY_PORT &&
   1691	    driver->port_type != pci_pcie_type(pciedev->port))
   1692		return 0;
   1693
   1694	return 1;
   1695}
   1696
   1697struct bus_type pcie_port_bus_type = {
   1698	.name		= "pci_express",
   1699	.match		= pcie_port_bus_match,
   1700};
   1701EXPORT_SYMBOL_GPL(pcie_port_bus_type);
   1702#endif
   1703
   1704static int __init pci_driver_init(void)
   1705{
   1706	int ret;
   1707
   1708	ret = bus_register(&pci_bus_type);
   1709	if (ret)
   1710		return ret;
   1711
   1712#ifdef CONFIG_PCIEPORTBUS
   1713	ret = bus_register(&pcie_port_bus_type);
   1714	if (ret)
   1715		return ret;
   1716#endif
   1717	dma_debug_add_bus(&pci_bus_type);
   1718	return 0;
   1719}
   1720postcore_initcall(pci_driver_init);