cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

ipa_main.c (25106B)


      1// SPDX-License-Identifier: GPL-2.0
      2
      3/* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
      4 * Copyright (C) 2018-2021 Linaro Ltd.
      5 */
      6
      7#include <linux/types.h>
      8#include <linux/atomic.h>
      9#include <linux/bitfield.h>
     10#include <linux/device.h>
     11#include <linux/bug.h>
     12#include <linux/io.h>
     13#include <linux/firmware.h>
     14#include <linux/module.h>
     15#include <linux/of.h>
     16#include <linux/of_device.h>
     17#include <linux/of_address.h>
     18#include <linux/pm_runtime.h>
     19#include <linux/qcom_scm.h>
     20#include <linux/soc/qcom/mdt_loader.h>
     21
     22#include "ipa.h"
     23#include "ipa_power.h"
     24#include "ipa_data.h"
     25#include "ipa_endpoint.h"
     26#include "ipa_resource.h"
     27#include "ipa_cmd.h"
     28#include "ipa_reg.h"
     29#include "ipa_mem.h"
     30#include "ipa_table.h"
     31#include "ipa_smp2p.h"
     32#include "ipa_modem.h"
     33#include "ipa_uc.h"
     34#include "ipa_interrupt.h"
     35#include "gsi_trans.h"
     36#include "ipa_sysfs.h"
     37
     38/**
     39 * DOC: The IP Accelerator
     40 *
     41 * This driver supports the Qualcomm IP Accelerator (IPA), which is a
     42 * networking component found in many Qualcomm SoCs.  The IPA is connected
     43 * to the application processor (AP), but is also connected (and partially
     44 * controlled by) other "execution environments" (EEs), such as a modem.
     45 *
     46 * The IPA is the conduit between the AP and the modem that carries network
     47 * traffic.  This driver presents a network interface representing the
     48 * connection of the modem to external (e.g. LTE) networks.
     49 *
     50 * The IPA provides protocol checksum calculation, offloading this work
     51 * from the AP.  The IPA offers additional functionality, including routing,
     52 * filtering, and NAT support, but that more advanced functionality is not
     53 * currently supported.  Despite that, some resources--including routing
     54 * tables and filter tables--are defined in this driver because they must
     55 * be initialized even when the advanced hardware features are not used.
     56 *
     57 * There are two distinct layers that implement the IPA hardware, and this
     58 * is reflected in the organization of the driver.  The generic software
     59 * interface (GSI) is an integral component of the IPA, providing a
     60 * well-defined communication layer between the AP subsystem and the IPA
     61 * core.  The GSI implements a set of "channels" used for communication
     62 * between the AP and the IPA.
     63 *
     64 * The IPA layer uses GSI channels to implement its "endpoints".  And while
     65 * a GSI channel carries data between the AP and the IPA, a pair of IPA
     66 * endpoints is used to carry traffic between two EEs.  Specifically, the main
     67 * modem network interface is implemented by two pairs of endpoints:  a TX
     68 * endpoint on the AP coupled with an RX endpoint on the modem; and another
     69 * RX endpoint on the AP receiving data from a TX endpoint on the modem.
     70 */
     71
     72/* The name of the GSI firmware file relative to /lib/firmware */
     73#define IPA_FW_PATH_DEFAULT	"ipa_fws.mdt"
     74#define IPA_PAS_ID		15
     75
     76/* Shift of 19.2 MHz timestamp to achieve lower resolution timestamps */
     77#define DPL_TIMESTAMP_SHIFT	14	/* ~1.172 kHz, ~853 usec per tick */
     78#define TAG_TIMESTAMP_SHIFT	14
     79#define NAT_TIMESTAMP_SHIFT	24	/* ~1.144 Hz, ~874 msec per tick */
     80
     81/* Divider for 19.2 MHz crystal oscillator clock to get common timer clock */
     82#define IPA_XO_CLOCK_DIVIDER	192	/* 1 is subtracted where used */
     83
     84/**
     85 * ipa_setup() - Set up IPA hardware
     86 * @ipa:	IPA pointer
     87 *
     88 * Perform initialization that requires issuing immediate commands on
     89 * the command TX endpoint.  If the modem is doing GSI firmware load
     90 * and initialization, this function will be called when an SMP2P
     91 * interrupt has been signaled by the modem.  Otherwise it will be
     92 * called from ipa_probe() after GSI firmware has been successfully
     93 * loaded, authenticated, and started by Trust Zone.
     94 */
     95int ipa_setup(struct ipa *ipa)
     96{
     97	struct ipa_endpoint *exception_endpoint;
     98	struct ipa_endpoint *command_endpoint;
     99	struct device *dev = &ipa->pdev->dev;
    100	int ret;
    101
    102	ret = gsi_setup(&ipa->gsi);
    103	if (ret)
    104		return ret;
    105
    106	ret = ipa_power_setup(ipa);
    107	if (ret)
    108		goto err_gsi_teardown;
    109
    110	ipa_endpoint_setup(ipa);
    111
    112	/* We need to use the AP command TX endpoint to perform other
    113	 * initialization, so we enable first.
    114	 */
    115	command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];
    116	ret = ipa_endpoint_enable_one(command_endpoint);
    117	if (ret)
    118		goto err_endpoint_teardown;
    119
    120	ret = ipa_mem_setup(ipa);	/* No matching teardown required */
    121	if (ret)
    122		goto err_command_disable;
    123
    124	ret = ipa_table_setup(ipa);	/* No matching teardown required */
    125	if (ret)
    126		goto err_command_disable;
    127
    128	/* Enable the exception handling endpoint, and tell the hardware
    129	 * to use it by default.
    130	 */
    131	exception_endpoint = ipa->name_map[IPA_ENDPOINT_AP_LAN_RX];
    132	ret = ipa_endpoint_enable_one(exception_endpoint);
    133	if (ret)
    134		goto err_command_disable;
    135
    136	ipa_endpoint_default_route_set(ipa, exception_endpoint->endpoint_id);
    137
    138	/* We're all set.  Now prepare for communication with the modem */
    139	ret = ipa_qmi_setup(ipa);
    140	if (ret)
    141		goto err_default_route_clear;
    142
    143	ipa->setup_complete = true;
    144
    145	dev_info(dev, "IPA driver setup completed successfully\n");
    146
    147	return 0;
    148
    149err_default_route_clear:
    150	ipa_endpoint_default_route_clear(ipa);
    151	ipa_endpoint_disable_one(exception_endpoint);
    152err_command_disable:
    153	ipa_endpoint_disable_one(command_endpoint);
    154err_endpoint_teardown:
    155	ipa_endpoint_teardown(ipa);
    156	ipa_power_teardown(ipa);
    157err_gsi_teardown:
    158	gsi_teardown(&ipa->gsi);
    159
    160	return ret;
    161}
    162
    163/**
    164 * ipa_teardown() - Inverse of ipa_setup()
    165 * @ipa:	IPA pointer
    166 */
    167static void ipa_teardown(struct ipa *ipa)
    168{
    169	struct ipa_endpoint *exception_endpoint;
    170	struct ipa_endpoint *command_endpoint;
    171
    172	/* We're going to tear everything down, as if setup never completed */
    173	ipa->setup_complete = false;
    174
    175	ipa_qmi_teardown(ipa);
    176	ipa_endpoint_default_route_clear(ipa);
    177	exception_endpoint = ipa->name_map[IPA_ENDPOINT_AP_LAN_RX];
    178	ipa_endpoint_disable_one(exception_endpoint);
    179	command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];
    180	ipa_endpoint_disable_one(command_endpoint);
    181	ipa_endpoint_teardown(ipa);
    182	ipa_power_teardown(ipa);
    183	gsi_teardown(&ipa->gsi);
    184}
    185
    186/* Configure bus access behavior for IPA components */
    187static void ipa_hardware_config_comp(struct ipa *ipa)
    188{
    189	u32 val;
    190
    191	/* Nothing to configure prior to IPA v4.0 */
    192	if (ipa->version < IPA_VERSION_4_0)
    193		return;
    194
    195	val = ioread32(ipa->reg_virt + IPA_REG_COMP_CFG_OFFSET);
    196
    197	if (ipa->version == IPA_VERSION_4_0) {
    198		val &= ~IPA_QMB_SELECT_CONS_EN_FMASK;
    199		val &= ~IPA_QMB_SELECT_PROD_EN_FMASK;
    200		val &= ~IPA_QMB_SELECT_GLOBAL_EN_FMASK;
    201	} else if (ipa->version < IPA_VERSION_4_5) {
    202		val |= GSI_MULTI_AXI_MASTERS_DIS_FMASK;
    203	} else {
    204		/* For IPA v4.5 IPA_FULL_FLUSH_WAIT_RSC_CLOSE_EN is 0 */
    205	}
    206
    207	val |= GSI_MULTI_INORDER_RD_DIS_FMASK;
    208	val |= GSI_MULTI_INORDER_WR_DIS_FMASK;
    209
    210	iowrite32(val, ipa->reg_virt + IPA_REG_COMP_CFG_OFFSET);
    211}
    212
    213/* Configure DDR and (possibly) PCIe max read/write QSB values */
    214static void
    215ipa_hardware_config_qsb(struct ipa *ipa, const struct ipa_data *data)
    216{
    217	const struct ipa_qsb_data *data0;
    218	const struct ipa_qsb_data *data1;
    219	u32 val;
    220
    221	/* QMB 0 represents DDR; QMB 1 (if present) represents PCIe */
    222	data0 = &data->qsb_data[IPA_QSB_MASTER_DDR];
    223	if (data->qsb_count > 1)
    224		data1 = &data->qsb_data[IPA_QSB_MASTER_PCIE];
    225
    226	/* Max outstanding write accesses for QSB masters */
    227	val = u32_encode_bits(data0->max_writes, GEN_QMB_0_MAX_WRITES_FMASK);
    228	if (data->qsb_count > 1)
    229		val |= u32_encode_bits(data1->max_writes,
    230				       GEN_QMB_1_MAX_WRITES_FMASK);
    231	iowrite32(val, ipa->reg_virt + IPA_REG_QSB_MAX_WRITES_OFFSET);
    232
    233	/* Max outstanding read accesses for QSB masters */
    234	val = u32_encode_bits(data0->max_reads, GEN_QMB_0_MAX_READS_FMASK);
    235	if (ipa->version >= IPA_VERSION_4_0)
    236		val |= u32_encode_bits(data0->max_reads_beats,
    237				       GEN_QMB_0_MAX_READS_BEATS_FMASK);
    238	if (data->qsb_count > 1) {
    239		val |= u32_encode_bits(data1->max_reads,
    240				       GEN_QMB_1_MAX_READS_FMASK);
    241		if (ipa->version >= IPA_VERSION_4_0)
    242			val |= u32_encode_bits(data1->max_reads_beats,
    243					       GEN_QMB_1_MAX_READS_BEATS_FMASK);
    244	}
    245	iowrite32(val, ipa->reg_virt + IPA_REG_QSB_MAX_READS_OFFSET);
    246}
    247
    248/* The internal inactivity timer clock is used for the aggregation timer */
    249#define TIMER_FREQUENCY	32000		/* 32 KHz inactivity timer clock */
    250
    251/* Compute the value to use in the COUNTER_CFG register AGGR_GRANULARITY
    252 * field to represent the given number of microseconds.  The value is one
    253 * less than the number of timer ticks in the requested period.  0 is not
    254 * a valid granularity value (so for example @usec must be at least 16 for
    255 * a TIMER_FREQUENCY of 32000).
    256 */
    257static __always_inline u32 ipa_aggr_granularity_val(u32 usec)
    258{
    259	return DIV_ROUND_CLOSEST(usec * TIMER_FREQUENCY, USEC_PER_SEC) - 1;
    260}
    261
    262/* IPA uses unified Qtime starting at IPA v4.5, implementing various
    263 * timestamps and timers independent of the IPA core clock rate.  The
    264 * Qtimer is based on a 56-bit timestamp incremented at each tick of
    265 * a 19.2 MHz SoC crystal oscillator (XO clock).
    266 *
    267 * For IPA timestamps (tag, NAT, data path logging) a lower resolution
    268 * timestamp is achieved by shifting the Qtimer timestamp value right
    269 * some number of bits to produce the low-order bits of the coarser
    270 * granularity timestamp.
    271 *
    272 * For timers, a common timer clock is derived from the XO clock using
    273 * a divider (we use 192, to produce a 100kHz timer clock).  From
    274 * this common clock, three "pulse generators" are used to produce
    275 * timer ticks at a configurable frequency.  IPA timers (such as
    276 * those used for aggregation or head-of-line block handling) now
    277 * define their period based on one of these pulse generators.
    278 */
    279static void ipa_qtime_config(struct ipa *ipa)
    280{
    281	u32 val;
    282
    283	/* Timer clock divider must be disabled when we change the rate */
    284	iowrite32(0, ipa->reg_virt + IPA_REG_TIMERS_XO_CLK_DIV_CFG_OFFSET);
    285
    286	/* Set DPL time stamp resolution to use Qtime (instead of 1 msec) */
    287	val = u32_encode_bits(DPL_TIMESTAMP_SHIFT, DPL_TIMESTAMP_LSB_FMASK);
    288	val |= u32_encode_bits(1, DPL_TIMESTAMP_SEL_FMASK);
    289	/* Configure tag and NAT Qtime timestamp resolution as well */
    290	val |= u32_encode_bits(TAG_TIMESTAMP_SHIFT, TAG_TIMESTAMP_LSB_FMASK);
    291	val |= u32_encode_bits(NAT_TIMESTAMP_SHIFT, NAT_TIMESTAMP_LSB_FMASK);
    292	iowrite32(val, ipa->reg_virt + IPA_REG_QTIME_TIMESTAMP_CFG_OFFSET);
    293
    294	/* Set granularity of pulse generators used for other timers */
    295	val = u32_encode_bits(IPA_GRAN_100_US, GRAN_0_FMASK);
    296	val |= u32_encode_bits(IPA_GRAN_1_MS, GRAN_1_FMASK);
    297	val |= u32_encode_bits(IPA_GRAN_1_MS, GRAN_2_FMASK);
    298	iowrite32(val, ipa->reg_virt + IPA_REG_TIMERS_PULSE_GRAN_CFG_OFFSET);
    299
    300	/* Actual divider is 1 more than value supplied here */
    301	val = u32_encode_bits(IPA_XO_CLOCK_DIVIDER - 1, DIV_VALUE_FMASK);
    302	iowrite32(val, ipa->reg_virt + IPA_REG_TIMERS_XO_CLK_DIV_CFG_OFFSET);
    303
    304	/* Divider value is set; re-enable the common timer clock divider */
    305	val |= u32_encode_bits(1, DIV_ENABLE_FMASK);
    306	iowrite32(val, ipa->reg_virt + IPA_REG_TIMERS_XO_CLK_DIV_CFG_OFFSET);
    307}
    308
    309static void ipa_idle_indication_cfg(struct ipa *ipa,
    310				    u32 enter_idle_debounce_thresh,
    311				    bool const_non_idle_enable)
    312{
    313	u32 offset;
    314	u32 val;
    315
    316	val = u32_encode_bits(enter_idle_debounce_thresh,
    317			      ENTER_IDLE_DEBOUNCE_THRESH_FMASK);
    318	if (const_non_idle_enable)
    319		val |= CONST_NON_IDLE_ENABLE_FMASK;
    320
    321	offset = ipa_reg_idle_indication_cfg_offset(ipa->version);
    322	iowrite32(val, ipa->reg_virt + offset);
    323}
    324
    325/**
    326 * ipa_hardware_dcd_config() - Enable dynamic clock division on IPA
    327 * @ipa:	IPA pointer
    328 *
    329 * Configures when the IPA signals it is idle to the global clock
    330 * controller, which can respond by scaling down the clock to save
    331 * power.
    332 */
    333static void ipa_hardware_dcd_config(struct ipa *ipa)
    334{
    335	/* Recommended values for IPA 3.5 and later according to IPA HPG */
    336	ipa_idle_indication_cfg(ipa, 256, false);
    337}
    338
    339static void ipa_hardware_dcd_deconfig(struct ipa *ipa)
    340{
    341	/* Power-on reset values */
    342	ipa_idle_indication_cfg(ipa, 0, true);
    343}
    344
    345/**
    346 * ipa_hardware_config() - Primitive hardware initialization
    347 * @ipa:	IPA pointer
    348 * @data:	IPA configuration data
    349 */
    350static void ipa_hardware_config(struct ipa *ipa, const struct ipa_data *data)
    351{
    352	enum ipa_version version = ipa->version;
    353	u32 granularity;
    354	u32 val;
    355
    356	/* IPA v4.5+ has no backward compatibility register */
    357	if (version < IPA_VERSION_4_5) {
    358		val = data->backward_compat;
    359		iowrite32(val, ipa->reg_virt + IPA_REG_BCR_OFFSET);
    360	}
    361
    362	/* Implement some hardware workarounds */
    363	if (version >= IPA_VERSION_4_0 && version < IPA_VERSION_4_5) {
    364		/* Disable PA mask to allow HOLB drop */
    365		val = ioread32(ipa->reg_virt + IPA_REG_TX_CFG_OFFSET);
    366		val &= ~PA_MASK_EN_FMASK;
    367		iowrite32(val, ipa->reg_virt + IPA_REG_TX_CFG_OFFSET);
    368
    369		/* Enable open global clocks in the CLKON configuration */
    370		val = GLOBAL_FMASK | GLOBAL_2X_CLK_FMASK;
    371	} else if (version == IPA_VERSION_3_1) {
    372		val = MISC_FMASK;	/* Disable MISC clock gating */
    373	} else {
    374		val = 0;		/* No CLKON configuration needed */
    375	}
    376	if (val)
    377		iowrite32(val, ipa->reg_virt + IPA_REG_CLKON_CFG_OFFSET);
    378
    379	ipa_hardware_config_comp(ipa);
    380
    381	/* Configure system bus limits */
    382	ipa_hardware_config_qsb(ipa, data);
    383
    384	if (version < IPA_VERSION_4_5) {
    385		/* Configure aggregation timer granularity */
    386		granularity = ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY);
    387		val = u32_encode_bits(granularity, AGGR_GRANULARITY_FMASK);
    388		iowrite32(val, ipa->reg_virt + IPA_REG_COUNTER_CFG_OFFSET);
    389	} else {
    390		ipa_qtime_config(ipa);
    391	}
    392
    393	/* IPA v4.2 does not support hashed tables, so disable them */
    394	if (version == IPA_VERSION_4_2) {
    395		u32 offset = ipa_reg_filt_rout_hash_en_offset(version);
    396
    397		iowrite32(0, ipa->reg_virt + offset);
    398	}
    399
    400	/* Enable dynamic clock division */
    401	ipa_hardware_dcd_config(ipa);
    402}
    403
    404/**
    405 * ipa_hardware_deconfig() - Inverse of ipa_hardware_config()
    406 * @ipa:	IPA pointer
    407 *
    408 * This restores the power-on reset values (even if they aren't different)
    409 */
    410static void ipa_hardware_deconfig(struct ipa *ipa)
    411{
    412	/* Mostly we just leave things as we set them. */
    413	ipa_hardware_dcd_deconfig(ipa);
    414}
    415
    416/**
    417 * ipa_config() - Configure IPA hardware
    418 * @ipa:	IPA pointer
    419 * @data:	IPA configuration data
    420 *
    421 * Perform initialization requiring IPA power to be enabled.
    422 */
    423static int ipa_config(struct ipa *ipa, const struct ipa_data *data)
    424{
    425	int ret;
    426
    427	ipa_hardware_config(ipa, data);
    428
    429	ret = ipa_mem_config(ipa);
    430	if (ret)
    431		goto err_hardware_deconfig;
    432
    433	ipa->interrupt = ipa_interrupt_config(ipa);
    434	if (IS_ERR(ipa->interrupt)) {
    435		ret = PTR_ERR(ipa->interrupt);
    436		ipa->interrupt = NULL;
    437		goto err_mem_deconfig;
    438	}
    439
    440	ipa_uc_config(ipa);
    441
    442	ret = ipa_endpoint_config(ipa);
    443	if (ret)
    444		goto err_uc_deconfig;
    445
    446	ipa_table_config(ipa);		/* No deconfig required */
    447
    448	/* Assign resource limitation to each group; no deconfig required */
    449	ret = ipa_resource_config(ipa, data->resource_data);
    450	if (ret)
    451		goto err_endpoint_deconfig;
    452
    453	ret = ipa_modem_config(ipa);
    454	if (ret)
    455		goto err_endpoint_deconfig;
    456
    457	return 0;
    458
    459err_endpoint_deconfig:
    460	ipa_endpoint_deconfig(ipa);
    461err_uc_deconfig:
    462	ipa_uc_deconfig(ipa);
    463	ipa_interrupt_deconfig(ipa->interrupt);
    464	ipa->interrupt = NULL;
    465err_mem_deconfig:
    466	ipa_mem_deconfig(ipa);
    467err_hardware_deconfig:
    468	ipa_hardware_deconfig(ipa);
    469
    470	return ret;
    471}
    472
    473/**
    474 * ipa_deconfig() - Inverse of ipa_config()
    475 * @ipa:	IPA pointer
    476 */
    477static void ipa_deconfig(struct ipa *ipa)
    478{
    479	ipa_modem_deconfig(ipa);
    480	ipa_endpoint_deconfig(ipa);
    481	ipa_uc_deconfig(ipa);
    482	ipa_interrupt_deconfig(ipa->interrupt);
    483	ipa->interrupt = NULL;
    484	ipa_mem_deconfig(ipa);
    485	ipa_hardware_deconfig(ipa);
    486}
    487
    488static int ipa_firmware_load(struct device *dev)
    489{
    490	const struct firmware *fw;
    491	struct device_node *node;
    492	struct resource res;
    493	phys_addr_t phys;
    494	const char *path;
    495	ssize_t size;
    496	void *virt;
    497	int ret;
    498
    499	node = of_parse_phandle(dev->of_node, "memory-region", 0);
    500	if (!node) {
    501		dev_err(dev, "DT error getting \"memory-region\" property\n");
    502		return -EINVAL;
    503	}
    504
    505	ret = of_address_to_resource(node, 0, &res);
    506	of_node_put(node);
    507	if (ret) {
    508		dev_err(dev, "error %d getting \"memory-region\" resource\n",
    509			ret);
    510		return ret;
    511	}
    512
    513	/* Use name from DTB if specified; use default for *any* error */
    514	ret = of_property_read_string(dev->of_node, "firmware-name", &path);
    515	if (ret) {
    516		dev_dbg(dev, "error %d getting \"firmware-name\" resource\n",
    517			ret);
    518		path = IPA_FW_PATH_DEFAULT;
    519	}
    520
    521	ret = request_firmware(&fw, path, dev);
    522	if (ret) {
    523		dev_err(dev, "error %d requesting \"%s\"\n", ret, path);
    524		return ret;
    525	}
    526
    527	phys = res.start;
    528	size = (size_t)resource_size(&res);
    529	virt = memremap(phys, size, MEMREMAP_WC);
    530	if (!virt) {
    531		dev_err(dev, "unable to remap firmware memory\n");
    532		ret = -ENOMEM;
    533		goto out_release_firmware;
    534	}
    535
    536	ret = qcom_mdt_load(dev, fw, path, IPA_PAS_ID, virt, phys, size, NULL);
    537	if (ret)
    538		dev_err(dev, "error %d loading \"%s\"\n", ret, path);
    539	else if ((ret = qcom_scm_pas_auth_and_reset(IPA_PAS_ID)))
    540		dev_err(dev, "error %d authenticating \"%s\"\n", ret, path);
    541
    542	memunmap(virt);
    543out_release_firmware:
    544	release_firmware(fw);
    545
    546	return ret;
    547}
    548
    549static const struct of_device_id ipa_match[] = {
    550	{
    551		.compatible	= "qcom,msm8998-ipa",
    552		.data		= &ipa_data_v3_1,
    553	},
    554	{
    555		.compatible	= "qcom,sdm845-ipa",
    556		.data		= &ipa_data_v3_5_1,
    557	},
    558	{
    559		.compatible	= "qcom,sc7180-ipa",
    560		.data		= &ipa_data_v4_2,
    561	},
    562	{
    563		.compatible	= "qcom,sdx55-ipa",
    564		.data		= &ipa_data_v4_5,
    565	},
    566	{
    567		.compatible	= "qcom,sm8350-ipa",
    568		.data		= &ipa_data_v4_9,
    569	},
    570	{
    571		.compatible	= "qcom,sc7280-ipa",
    572		.data		= &ipa_data_v4_11,
    573	},
    574	{ },
    575};
    576MODULE_DEVICE_TABLE(of, ipa_match);
    577
    578/* Check things that can be validated at build time.  This just
    579 * groups these things BUILD_BUG_ON() calls don't clutter the rest
    580 * of the code.
    581 * */
    582static void ipa_validate_build(void)
    583{
    584	/* At one time we assumed a 64-bit build, allowing some do_div()
    585	 * calls to be replaced by simple division or modulo operations.
    586	 * We currently only perform divide and modulo operations on u32,
    587	 * u16, or size_t objects, and of those only size_t has any chance
    588	 * of being a 64-bit value.  (It should be guaranteed 32 bits wide
    589	 * on a 32-bit build, but there is no harm in verifying that.)
    590	 */
    591	BUILD_BUG_ON(!IS_ENABLED(CONFIG_64BIT) && sizeof(size_t) != 4);
    592
    593	/* Code assumes the EE ID for the AP is 0 (zeroed structure field) */
    594	BUILD_BUG_ON(GSI_EE_AP != 0);
    595
    596	/* There's no point if we have no channels or event rings */
    597	BUILD_BUG_ON(!GSI_CHANNEL_COUNT_MAX);
    598	BUILD_BUG_ON(!GSI_EVT_RING_COUNT_MAX);
    599
    600	/* GSI hardware design limits */
    601	BUILD_BUG_ON(GSI_CHANNEL_COUNT_MAX > 32);
    602	BUILD_BUG_ON(GSI_EVT_RING_COUNT_MAX > 31);
    603
    604	/* The number of TREs in a transaction is limited by the channel's
    605	 * TLV FIFO size.  A transaction structure uses 8-bit fields
    606	 * to represents the number of TREs it has allocated and used.
    607	 */
    608	BUILD_BUG_ON(GSI_TLV_MAX > U8_MAX);
    609
    610	/* This is used as a divisor */
    611	BUILD_BUG_ON(!IPA_AGGR_GRANULARITY);
    612
    613	/* Aggregation granularity value can't be 0, and must fit */
    614	BUILD_BUG_ON(!ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY));
    615	BUILD_BUG_ON(ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY) >
    616			field_max(AGGR_GRANULARITY_FMASK));
    617}
    618
    619static bool ipa_version_valid(enum ipa_version version)
    620{
    621	switch (version) {
    622	case IPA_VERSION_3_0:
    623	case IPA_VERSION_3_1:
    624	case IPA_VERSION_3_5:
    625	case IPA_VERSION_3_5_1:
    626	case IPA_VERSION_4_0:
    627	case IPA_VERSION_4_1:
    628	case IPA_VERSION_4_2:
    629	case IPA_VERSION_4_5:
    630	case IPA_VERSION_4_7:
    631	case IPA_VERSION_4_9:
    632	case IPA_VERSION_4_11:
    633		return true;
    634
    635	default:
    636		return false;
    637	}
    638}
    639
    640/**
    641 * ipa_probe() - IPA platform driver probe function
    642 * @pdev:	Platform device pointer
    643 *
    644 * Return:	0 if successful, or a negative error code (possibly
    645 *		EPROBE_DEFER)
    646 *
    647 * This is the main entry point for the IPA driver.  Initialization proceeds
    648 * in several stages:
    649 *   - The "init" stage involves activities that can be initialized without
    650 *     access to the IPA hardware.
    651 *   - The "config" stage requires IPA power to be active so IPA registers
    652 *     can be accessed, but does not require the use of IPA immediate commands.
    653 *   - The "setup" stage uses IPA immediate commands, and so requires the GSI
    654 *     layer to be initialized.
    655 *
    656 * A Boolean Device Tree "modem-init" property determines whether GSI
    657 * initialization will be performed by the AP (Trust Zone) or the modem.
    658 * If the AP does GSI initialization, the setup phase is entered after
    659 * this has completed successfully.  Otherwise the modem initializes
    660 * the GSI layer and signals it has finished by sending an SMP2P interrupt
    661 * to the AP; this triggers the start if IPA setup.
    662 */
    663static int ipa_probe(struct platform_device *pdev)
    664{
    665	struct device *dev = &pdev->dev;
    666	const struct ipa_data *data;
    667	struct ipa_power *power;
    668	bool modem_init;
    669	struct ipa *ipa;
    670	int ret;
    671
    672	ipa_validate_build();
    673
    674	/* Get configuration data early; needed for power initialization */
    675	data = of_device_get_match_data(dev);
    676	if (!data) {
    677		dev_err(dev, "matched hardware not supported\n");
    678		return -ENODEV;
    679	}
    680
    681	if (!ipa_version_valid(data->version)) {
    682		dev_err(dev, "invalid IPA version\n");
    683		return -EINVAL;
    684	}
    685
    686	/* If we need Trust Zone, make sure it's available */
    687	modem_init = of_property_read_bool(dev->of_node, "modem-init");
    688	if (!modem_init)
    689		if (!qcom_scm_is_available())
    690			return -EPROBE_DEFER;
    691
    692	/* The clock and interconnects might not be ready when we're
    693	 * probed, so might return -EPROBE_DEFER.
    694	 */
    695	power = ipa_power_init(dev, data->power_data);
    696	if (IS_ERR(power))
    697		return PTR_ERR(power);
    698
    699	/* No more EPROBE_DEFER.  Allocate and initialize the IPA structure */
    700	ipa = kzalloc(sizeof(*ipa), GFP_KERNEL);
    701	if (!ipa) {
    702		ret = -ENOMEM;
    703		goto err_power_exit;
    704	}
    705
    706	ipa->pdev = pdev;
    707	dev_set_drvdata(dev, ipa);
    708	ipa->power = power;
    709	ipa->version = data->version;
    710	init_completion(&ipa->completion);
    711
    712	ret = ipa_reg_init(ipa);
    713	if (ret)
    714		goto err_kfree_ipa;
    715
    716	ret = ipa_mem_init(ipa, data->mem_data);
    717	if (ret)
    718		goto err_reg_exit;
    719
    720	ret = gsi_init(&ipa->gsi, pdev, ipa->version, data->endpoint_count,
    721		       data->endpoint_data);
    722	if (ret)
    723		goto err_mem_exit;
    724
    725	/* Result is a non-zero mask of endpoints that support filtering */
    726	ipa->filter_map = ipa_endpoint_init(ipa, data->endpoint_count,
    727					    data->endpoint_data);
    728	if (!ipa->filter_map) {
    729		ret = -EINVAL;
    730		goto err_gsi_exit;
    731	}
    732
    733	ret = ipa_table_init(ipa);
    734	if (ret)
    735		goto err_endpoint_exit;
    736
    737	ret = ipa_smp2p_init(ipa, modem_init);
    738	if (ret)
    739		goto err_table_exit;
    740
    741	/* Power needs to be active for config and setup */
    742	ret = pm_runtime_get_sync(dev);
    743	if (WARN_ON(ret < 0))
    744		goto err_power_put;
    745
    746	ret = ipa_config(ipa, data);
    747	if (ret)
    748		goto err_power_put;
    749
    750	dev_info(dev, "IPA driver initialized");
    751
    752	/* If the modem is doing early initialization, it will trigger a
    753	 * call to ipa_setup() when it has finished.  In that case we're
    754	 * done here.
    755	 */
    756	if (modem_init)
    757		goto done;
    758
    759	/* Otherwise we need to load the firmware and have Trust Zone validate
    760	 * and install it.  If that succeeds we can proceed with setup.
    761	 */
    762	ret = ipa_firmware_load(dev);
    763	if (ret)
    764		goto err_deconfig;
    765
    766	ret = ipa_setup(ipa);
    767	if (ret)
    768		goto err_deconfig;
    769done:
    770	pm_runtime_mark_last_busy(dev);
    771	(void)pm_runtime_put_autosuspend(dev);
    772
    773	return 0;
    774
    775err_deconfig:
    776	ipa_deconfig(ipa);
    777err_power_put:
    778	pm_runtime_put_noidle(dev);
    779	ipa_smp2p_exit(ipa);
    780err_table_exit:
    781	ipa_table_exit(ipa);
    782err_endpoint_exit:
    783	ipa_endpoint_exit(ipa);
    784err_gsi_exit:
    785	gsi_exit(&ipa->gsi);
    786err_mem_exit:
    787	ipa_mem_exit(ipa);
    788err_reg_exit:
    789	ipa_reg_exit(ipa);
    790err_kfree_ipa:
    791	kfree(ipa);
    792err_power_exit:
    793	ipa_power_exit(power);
    794
    795	return ret;
    796}
    797
    798static int ipa_remove(struct platform_device *pdev)
    799{
    800	struct ipa *ipa = dev_get_drvdata(&pdev->dev);
    801	struct ipa_power *power = ipa->power;
    802	struct device *dev = &pdev->dev;
    803	int ret;
    804
    805	/* Prevent the modem from triggering a call to ipa_setup().  This
    806	 * also ensures a modem-initiated setup that's underway completes.
    807	 */
    808	ipa_smp2p_irq_disable_setup(ipa);
    809
    810	ret = pm_runtime_get_sync(dev);
    811	if (WARN_ON(ret < 0))
    812		goto out_power_put;
    813
    814	if (ipa->setup_complete) {
    815		ret = ipa_modem_stop(ipa);
    816		/* If starting or stopping is in progress, try once more */
    817		if (ret == -EBUSY) {
    818			usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC);
    819			ret = ipa_modem_stop(ipa);
    820		}
    821		if (ret)
    822			return ret;
    823
    824		ipa_teardown(ipa);
    825	}
    826
    827	ipa_deconfig(ipa);
    828out_power_put:
    829	pm_runtime_put_noidle(dev);
    830	ipa_smp2p_exit(ipa);
    831	ipa_table_exit(ipa);
    832	ipa_endpoint_exit(ipa);
    833	gsi_exit(&ipa->gsi);
    834	ipa_mem_exit(ipa);
    835	ipa_reg_exit(ipa);
    836	kfree(ipa);
    837	ipa_power_exit(power);
    838
    839	return 0;
    840}
    841
    842static void ipa_shutdown(struct platform_device *pdev)
    843{
    844	int ret;
    845
    846	ret = ipa_remove(pdev);
    847	if (ret)
    848		dev_err(&pdev->dev, "shutdown: remove returned %d\n", ret);
    849}
    850
    851static const struct attribute_group *ipa_attribute_groups[] = {
    852	&ipa_attribute_group,
    853	&ipa_feature_attribute_group,
    854	&ipa_modem_attribute_group,
    855	NULL,
    856};
    857
    858static struct platform_driver ipa_driver = {
    859	.probe		= ipa_probe,
    860	.remove		= ipa_remove,
    861	.shutdown	= ipa_shutdown,
    862	.driver	= {
    863		.name		= "ipa",
    864		.pm		= &ipa_pm_ops,
    865		.of_match_table	= ipa_match,
    866		.dev_groups	= ipa_attribute_groups,
    867	},
    868};
    869
    870module_platform_driver(ipa_driver);
    871
    872MODULE_LICENSE("GPL v2");
    873MODULE_DESCRIPTION("Qualcomm IP Accelerator device driver");