cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

irq_affinity.c (7296B)


      1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
      2/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
      3
      4#include "mlx5_core.h"
      5#include "mlx5_irq.h"
      6#include "pci_irq.h"
      7
      8static void cpu_put(struct mlx5_irq_pool *pool, int cpu)
      9{
     10	pool->irqs_per_cpu[cpu]--;
     11}
     12
     13static void cpu_get(struct mlx5_irq_pool *pool, int cpu)
     14{
     15	pool->irqs_per_cpu[cpu]++;
     16}
     17
     18/* Gets the least loaded CPU. e.g.: the CPU with least IRQs bound to it */
     19static int cpu_get_least_loaded(struct mlx5_irq_pool *pool,
     20				const struct cpumask *req_mask)
     21{
     22	int best_cpu = -1;
     23	int cpu;
     24
     25	for_each_cpu_and(cpu, req_mask, cpu_online_mask) {
     26		/* CPU has zero IRQs on it. No need to search any more CPUs. */
     27		if (!pool->irqs_per_cpu[cpu]) {
     28			best_cpu = cpu;
     29			break;
     30		}
     31		if (best_cpu < 0)
     32			best_cpu = cpu;
     33		if (pool->irqs_per_cpu[cpu] < pool->irqs_per_cpu[best_cpu])
     34			best_cpu = cpu;
     35	}
     36	if (best_cpu == -1) {
     37		/* There isn't online CPUs in req_mask */
     38		mlx5_core_err(pool->dev, "NO online CPUs in req_mask (%*pbl)\n",
     39			      cpumask_pr_args(req_mask));
     40		best_cpu = cpumask_first(cpu_online_mask);
     41	}
     42	pool->irqs_per_cpu[best_cpu]++;
     43	return best_cpu;
     44}
     45
     46/* Creating an IRQ from irq_pool */
     47static struct mlx5_irq *
     48irq_pool_request_irq(struct mlx5_irq_pool *pool, const struct cpumask *req_mask)
     49{
     50	cpumask_var_t auto_mask;
     51	struct mlx5_irq *irq;
     52	u32 irq_index;
     53	int err;
     54
     55	if (!zalloc_cpumask_var(&auto_mask, GFP_KERNEL))
     56		return ERR_PTR(-ENOMEM);
     57	err = xa_alloc(&pool->irqs, &irq_index, NULL, pool->xa_num_irqs, GFP_KERNEL);
     58	if (err)
     59		return ERR_PTR(err);
     60	if (pool->irqs_per_cpu) {
     61		if (cpumask_weight(req_mask) > 1)
     62			/* if req_mask contain more then one CPU, set the least loadad CPU
     63			 * of req_mask
     64			 */
     65			cpumask_set_cpu(cpu_get_least_loaded(pool, req_mask), auto_mask);
     66		else
     67			cpu_get(pool, cpumask_first(req_mask));
     68	}
     69	irq = mlx5_irq_alloc(pool, irq_index, cpumask_empty(auto_mask) ? req_mask : auto_mask);
     70	free_cpumask_var(auto_mask);
     71	return irq;
     72}
     73
     74/* Looking for the IRQ with the smallest refcount that fits req_mask.
     75 * If pool is sf_comp_pool, then we are looking for an IRQ with any of the
     76 * requested CPUs in req_mask.
     77 * for example: req_mask = 0xf, irq0_mask = 0x10, irq1_mask = 0x1. irq0_mask
     78 * isn't subset of req_mask, so we will skip it. irq1_mask is subset of req_mask,
     79 * we don't skip it.
     80 * If pool is sf_ctrl_pool, then all IRQs have the same mask, so any IRQ will
     81 * fit. And since mask is subset of itself, we will pass the first if bellow.
     82 */
     83static struct mlx5_irq *
     84irq_pool_find_least_loaded(struct mlx5_irq_pool *pool, const struct cpumask *req_mask)
     85{
     86	int start = pool->xa_num_irqs.min;
     87	int end = pool->xa_num_irqs.max;
     88	struct mlx5_irq *irq = NULL;
     89	struct mlx5_irq *iter;
     90	int irq_refcount = 0;
     91	unsigned long index;
     92
     93	lockdep_assert_held(&pool->lock);
     94	xa_for_each_range(&pool->irqs, index, iter, start, end) {
     95		struct cpumask *iter_mask = mlx5_irq_get_affinity_mask(iter);
     96		int iter_refcount = mlx5_irq_read_locked(iter);
     97
     98		if (!cpumask_subset(iter_mask, req_mask))
     99			/* skip IRQs with a mask which is not subset of req_mask */
    100			continue;
    101		if (iter_refcount < pool->min_threshold)
    102			/* If we found an IRQ with less than min_thres, return it */
    103			return iter;
    104		if (!irq || iter_refcount < irq_refcount) {
    105			/* In case we won't find an IRQ with less than min_thres,
    106			 * keep a pointer to the least used IRQ
    107			 */
    108			irq_refcount = iter_refcount;
    109			irq = iter;
    110		}
    111	}
    112	return irq;
    113}
    114
    115/**
    116 * mlx5_irq_affinity_request - request an IRQ according to the given mask.
    117 * @pool: IRQ pool to request from.
    118 * @req_mask: cpumask requested for this IRQ.
    119 *
    120 * This function returns a pointer to IRQ, or ERR_PTR in case of error.
    121 */
    122struct mlx5_irq *
    123mlx5_irq_affinity_request(struct mlx5_irq_pool *pool, const struct cpumask *req_mask)
    124{
    125	struct mlx5_irq *least_loaded_irq, *new_irq;
    126
    127	mutex_lock(&pool->lock);
    128	least_loaded_irq = irq_pool_find_least_loaded(pool, req_mask);
    129	if (least_loaded_irq &&
    130	    mlx5_irq_read_locked(least_loaded_irq) < pool->min_threshold)
    131		goto out;
    132	/* We didn't find an IRQ with less than min_thres, try to allocate a new IRQ */
    133	new_irq = irq_pool_request_irq(pool, req_mask);
    134	if (IS_ERR(new_irq)) {
    135		if (!least_loaded_irq) {
    136			/* We failed to create an IRQ and we didn't find an IRQ */
    137			mlx5_core_err(pool->dev, "Didn't find a matching IRQ. err = %ld\n",
    138				      PTR_ERR(new_irq));
    139			mutex_unlock(&pool->lock);
    140			return new_irq;
    141		}
    142		/* We failed to create a new IRQ for the requested affinity,
    143		 * sharing existing IRQ.
    144		 */
    145		goto out;
    146	}
    147	least_loaded_irq = new_irq;
    148	goto unlock;
    149out:
    150	mlx5_irq_get_locked(least_loaded_irq);
    151	if (mlx5_irq_read_locked(least_loaded_irq) > pool->max_threshold)
    152		mlx5_core_dbg(pool->dev, "IRQ %u overloaded, pool_name: %s, %u EQs on this irq\n",
    153			      pci_irq_vector(pool->dev->pdev,
    154					     mlx5_irq_get_index(least_loaded_irq)), pool->name,
    155			      mlx5_irq_read_locked(least_loaded_irq) / MLX5_EQ_REFS_PER_IRQ);
    156unlock:
    157	mutex_unlock(&pool->lock);
    158	return least_loaded_irq;
    159}
    160
    161void mlx5_irq_affinity_irqs_release(struct mlx5_core_dev *dev, struct mlx5_irq **irqs,
    162				    int num_irqs)
    163{
    164	struct mlx5_irq_pool *pool = mlx5_irq_pool_get(dev);
    165	int i;
    166
    167	for (i = 0; i < num_irqs; i++) {
    168		int cpu = cpumask_first(mlx5_irq_get_affinity_mask(irqs[i]));
    169
    170		synchronize_irq(pci_irq_vector(pool->dev->pdev,
    171					       mlx5_irq_get_index(irqs[i])));
    172		if (mlx5_irq_put(irqs[i]))
    173			if (pool->irqs_per_cpu)
    174				cpu_put(pool, cpu);
    175	}
    176}
    177
    178/**
    179 * mlx5_irq_affinity_irqs_request_auto - request one or more IRQs for mlx5 device.
    180 * @dev: mlx5 device that is requesting the IRQs.
    181 * @nirqs: number of IRQs to request.
    182 * @irqs: an output array of IRQs pointers.
    183 *
    184 * Each IRQ is bounded to at most 1 CPU.
    185 * This function is requesting IRQs according to the default assignment.
    186 * The default assignment policy is:
    187 * - in each iteration, request the least loaded IRQ which is not bound to any
    188 *   CPU of the previous IRQs requested.
    189 *
    190 * This function returns the number of IRQs requested, (which might be smaller than
    191 * @nirqs), if successful, or a negative error code in case of an error.
    192 */
    193int mlx5_irq_affinity_irqs_request_auto(struct mlx5_core_dev *dev, int nirqs,
    194					struct mlx5_irq **irqs)
    195{
    196	struct mlx5_irq_pool *pool = mlx5_irq_pool_get(dev);
    197	cpumask_var_t req_mask;
    198	struct mlx5_irq *irq;
    199	int i = 0;
    200
    201	if (!zalloc_cpumask_var(&req_mask, GFP_KERNEL))
    202		return -ENOMEM;
    203	cpumask_copy(req_mask, cpu_online_mask);
    204	for (i = 0; i < nirqs; i++) {
    205		if (mlx5_irq_pool_is_sf_pool(pool))
    206			irq = mlx5_irq_affinity_request(pool, req_mask);
    207		else
    208			/* In case SF pool doesn't exists, fallback to the PF IRQs.
    209			 * The PF IRQs are already allocated and binded to CPU
    210			 * at this point. Hence, only an index is needed.
    211			 */
    212			irq = mlx5_irq_request(dev, i, NULL);
    213		if (IS_ERR(irq))
    214			break;
    215		irqs[i] = irq;
    216		cpumask_clear_cpu(cpumask_first(mlx5_irq_get_affinity_mask(irq)), req_mask);
    217		mlx5_core_dbg(pool->dev, "IRQ %u mapped to cpu %*pbl, %u EQs on this irq\n",
    218			      pci_irq_vector(dev->pdev, mlx5_irq_get_index(irq)),
    219			      cpumask_pr_args(mlx5_irq_get_affinity_mask(irq)),
    220			      mlx5_irq_read_locked(irq) / MLX5_EQ_REFS_PER_IRQ);
    221	}
    222	free_cpumask_var(req_mask);
    223	if (!i)
    224		return PTR_ERR(irq);
    225	return i;
    226}