cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

blk-ioprio.c (6621B)


      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * Block rq-qos policy for assigning an I/O priority class to requests.
      4 *
      5 * Using an rq-qos policy for assigning I/O priority class has two advantages
      6 * over using the ioprio_set() system call:
      7 *
      8 * - This policy is cgroup based so it has all the advantages of cgroups.
      9 * - While ioprio_set() does not affect page cache writeback I/O, this rq-qos
     10 *   controller affects page cache writeback I/O for filesystems that support
     11 *   assiociating a cgroup with writeback I/O. See also
     12 *   Documentation/admin-guide/cgroup-v2.rst.
     13 */
     14
     15#include <linux/blk-mq.h>
     16#include <linux/blk_types.h>
     17#include <linux/kernel.h>
     18#include <linux/module.h>
     19#include "blk-cgroup.h"
     20#include "blk-ioprio.h"
     21#include "blk-rq-qos.h"
     22
     23/**
     24 * enum prio_policy - I/O priority class policy.
     25 * @POLICY_NO_CHANGE: (default) do not modify the I/O priority class.
     26 * @POLICY_NONE_TO_RT: modify IOPRIO_CLASS_NONE into IOPRIO_CLASS_RT.
     27 * @POLICY_RESTRICT_TO_BE: modify IOPRIO_CLASS_NONE and IOPRIO_CLASS_RT into
     28 *		IOPRIO_CLASS_BE.
     29 * @POLICY_ALL_TO_IDLE: change the I/O priority class into IOPRIO_CLASS_IDLE.
     30 *
     31 * See also <linux/ioprio.h>.
     32 */
     33enum prio_policy {
     34	POLICY_NO_CHANGE	= 0,
     35	POLICY_NONE_TO_RT	= 1,
     36	POLICY_RESTRICT_TO_BE	= 2,
     37	POLICY_ALL_TO_IDLE	= 3,
     38};
     39
     40static const char *policy_name[] = {
     41	[POLICY_NO_CHANGE]	= "no-change",
     42	[POLICY_NONE_TO_RT]	= "none-to-rt",
     43	[POLICY_RESTRICT_TO_BE]	= "restrict-to-be",
     44	[POLICY_ALL_TO_IDLE]	= "idle",
     45};
     46
     47static struct blkcg_policy ioprio_policy;
     48
     49/**
     50 * struct ioprio_blkg - Per (cgroup, request queue) data.
     51 * @pd: blkg_policy_data structure.
     52 */
     53struct ioprio_blkg {
     54	struct blkg_policy_data pd;
     55};
     56
     57/**
     58 * struct ioprio_blkcg - Per cgroup data.
     59 * @cpd: blkcg_policy_data structure.
     60 * @prio_policy: One of the IOPRIO_CLASS_* values. See also <linux/ioprio.h>.
     61 */
     62struct ioprio_blkcg {
     63	struct blkcg_policy_data cpd;
     64	enum prio_policy	 prio_policy;
     65	bool			 prio_set;
     66};
     67
     68static inline struct ioprio_blkg *pd_to_ioprio(struct blkg_policy_data *pd)
     69{
     70	return pd ? container_of(pd, struct ioprio_blkg, pd) : NULL;
     71}
     72
     73static struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg)
     74{
     75	return container_of(blkcg_to_cpd(blkcg, &ioprio_policy),
     76			    struct ioprio_blkcg, cpd);
     77}
     78
     79static struct ioprio_blkcg *
     80ioprio_blkcg_from_css(struct cgroup_subsys_state *css)
     81{
     82	return blkcg_to_ioprio_blkcg(css_to_blkcg(css));
     83}
     84
     85static struct ioprio_blkcg *ioprio_blkcg_from_bio(struct bio *bio)
     86{
     87	struct blkg_policy_data *pd = blkg_to_pd(bio->bi_blkg, &ioprio_policy);
     88
     89	if (!pd)
     90		return NULL;
     91
     92	return blkcg_to_ioprio_blkcg(pd->blkg->blkcg);
     93}
     94
     95static int ioprio_show_prio_policy(struct seq_file *sf, void *v)
     96{
     97	struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf));
     98
     99	seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]);
    100	return 0;
    101}
    102
    103static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf,
    104				      size_t nbytes, loff_t off)
    105{
    106	struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of));
    107	int ret;
    108
    109	if (off != 0)
    110		return -EIO;
    111	/* kernfs_fop_write_iter() terminates 'buf' with '\0'. */
    112	ret = sysfs_match_string(policy_name, buf);
    113	if (ret < 0)
    114		return ret;
    115	blkcg->prio_policy = ret;
    116	blkcg->prio_set = true;
    117	return nbytes;
    118}
    119
    120static struct blkg_policy_data *
    121ioprio_alloc_pd(gfp_t gfp, struct request_queue *q, struct blkcg *blkcg)
    122{
    123	struct ioprio_blkg *ioprio_blkg;
    124
    125	ioprio_blkg = kzalloc(sizeof(*ioprio_blkg), gfp);
    126	if (!ioprio_blkg)
    127		return NULL;
    128
    129	return &ioprio_blkg->pd;
    130}
    131
    132static void ioprio_free_pd(struct blkg_policy_data *pd)
    133{
    134	struct ioprio_blkg *ioprio_blkg = pd_to_ioprio(pd);
    135
    136	kfree(ioprio_blkg);
    137}
    138
    139static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp)
    140{
    141	struct ioprio_blkcg *blkcg;
    142
    143	blkcg = kzalloc(sizeof(*blkcg), gfp);
    144	if (!blkcg)
    145		return NULL;
    146	blkcg->prio_policy = POLICY_NO_CHANGE;
    147	return &blkcg->cpd;
    148}
    149
    150static void ioprio_free_cpd(struct blkcg_policy_data *cpd)
    151{
    152	struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd);
    153
    154	kfree(blkcg);
    155}
    156
    157#define IOPRIO_ATTRS						\
    158	{							\
    159		.name		= "prio.class",			\
    160		.seq_show	= ioprio_show_prio_policy,	\
    161		.write		= ioprio_set_prio_policy,	\
    162	},							\
    163	{ } /* sentinel */
    164
    165/* cgroup v2 attributes */
    166static struct cftype ioprio_files[] = {
    167	IOPRIO_ATTRS
    168};
    169
    170/* cgroup v1 attributes */
    171static struct cftype ioprio_legacy_files[] = {
    172	IOPRIO_ATTRS
    173};
    174
    175static struct blkcg_policy ioprio_policy = {
    176	.dfl_cftypes	= ioprio_files,
    177	.legacy_cftypes = ioprio_legacy_files,
    178
    179	.cpd_alloc_fn	= ioprio_alloc_cpd,
    180	.cpd_free_fn	= ioprio_free_cpd,
    181
    182	.pd_alloc_fn	= ioprio_alloc_pd,
    183	.pd_free_fn	= ioprio_free_pd,
    184};
    185
    186struct blk_ioprio {
    187	struct rq_qos rqos;
    188};
    189
    190static void blkcg_ioprio_track(struct rq_qos *rqos, struct request *rq,
    191			       struct bio *bio)
    192{
    193	struct ioprio_blkcg *blkcg = ioprio_blkcg_from_bio(bio);
    194	u16 prio;
    195
    196	if (!blkcg->prio_set)
    197		return;
    198
    199	/*
    200	 * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers
    201	 * correspond to a lower priority. Hence, the max_t() below selects
    202	 * the lower priority of bi_ioprio and the cgroup I/O priority class.
    203	 * If the cgroup policy has been set to POLICY_NO_CHANGE == 0, the
    204	 * bio I/O priority is not modified. If the bio I/O priority equals
    205	 * IOPRIO_CLASS_NONE, the cgroup I/O priority is assigned to the bio.
    206	 */
    207	prio = max_t(u16, bio->bi_ioprio,
    208			IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0));
    209	if (prio > bio->bi_ioprio)
    210		bio->bi_ioprio = prio;
    211}
    212
    213static void blkcg_ioprio_exit(struct rq_qos *rqos)
    214{
    215	struct blk_ioprio *blkioprio_blkg =
    216		container_of(rqos, typeof(*blkioprio_blkg), rqos);
    217
    218	blkcg_deactivate_policy(rqos->q, &ioprio_policy);
    219	kfree(blkioprio_blkg);
    220}
    221
    222static struct rq_qos_ops blkcg_ioprio_ops = {
    223	.track	= blkcg_ioprio_track,
    224	.exit	= blkcg_ioprio_exit,
    225};
    226
    227int blk_ioprio_init(struct request_queue *q)
    228{
    229	struct blk_ioprio *blkioprio_blkg;
    230	struct rq_qos *rqos;
    231	int ret;
    232
    233	blkioprio_blkg = kzalloc(sizeof(*blkioprio_blkg), GFP_KERNEL);
    234	if (!blkioprio_blkg)
    235		return -ENOMEM;
    236
    237	ret = blkcg_activate_policy(q, &ioprio_policy);
    238	if (ret) {
    239		kfree(blkioprio_blkg);
    240		return ret;
    241	}
    242
    243	rqos = &blkioprio_blkg->rqos;
    244	rqos->id = RQ_QOS_IOPRIO;
    245	rqos->ops = &blkcg_ioprio_ops;
    246	rqos->q = q;
    247
    248	/*
    249	 * Registering the rq-qos policy after activating the blk-cgroup
    250	 * policy guarantees that ioprio_blkcg_from_bio(bio) != NULL in the
    251	 * rq-qos callbacks.
    252	 */
    253	rq_qos_add(q, rqos);
    254
    255	return 0;
    256}
    257
    258static int __init ioprio_init(void)
    259{
    260	return blkcg_policy_register(&ioprio_policy);
    261}
    262
    263static void __exit ioprio_exit(void)
    264{
    265	blkcg_policy_unregister(&ioprio_policy);
    266}
    267
    268module_init(ioprio_init);
    269module_exit(ioprio_exit);