blk-ioprio.c (6621B)
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Block rq-qos policy for assigning an I/O priority class to requests. 4 * 5 * Using an rq-qos policy for assigning I/O priority class has two advantages 6 * over using the ioprio_set() system call: 7 * 8 * - This policy is cgroup based so it has all the advantages of cgroups. 9 * - While ioprio_set() does not affect page cache writeback I/O, this rq-qos 10 * controller affects page cache writeback I/O for filesystems that support 11 * assiociating a cgroup with writeback I/O. See also 12 * Documentation/admin-guide/cgroup-v2.rst. 13 */ 14 15#include <linux/blk-mq.h> 16#include <linux/blk_types.h> 17#include <linux/kernel.h> 18#include <linux/module.h> 19#include "blk-cgroup.h" 20#include "blk-ioprio.h" 21#include "blk-rq-qos.h" 22 23/** 24 * enum prio_policy - I/O priority class policy. 25 * @POLICY_NO_CHANGE: (default) do not modify the I/O priority class. 26 * @POLICY_NONE_TO_RT: modify IOPRIO_CLASS_NONE into IOPRIO_CLASS_RT. 27 * @POLICY_RESTRICT_TO_BE: modify IOPRIO_CLASS_NONE and IOPRIO_CLASS_RT into 28 * IOPRIO_CLASS_BE. 29 * @POLICY_ALL_TO_IDLE: change the I/O priority class into IOPRIO_CLASS_IDLE. 30 * 31 * See also <linux/ioprio.h>. 32 */ 33enum prio_policy { 34 POLICY_NO_CHANGE = 0, 35 POLICY_NONE_TO_RT = 1, 36 POLICY_RESTRICT_TO_BE = 2, 37 POLICY_ALL_TO_IDLE = 3, 38}; 39 40static const char *policy_name[] = { 41 [POLICY_NO_CHANGE] = "no-change", 42 [POLICY_NONE_TO_RT] = "none-to-rt", 43 [POLICY_RESTRICT_TO_BE] = "restrict-to-be", 44 [POLICY_ALL_TO_IDLE] = "idle", 45}; 46 47static struct blkcg_policy ioprio_policy; 48 49/** 50 * struct ioprio_blkg - Per (cgroup, request queue) data. 51 * @pd: blkg_policy_data structure. 52 */ 53struct ioprio_blkg { 54 struct blkg_policy_data pd; 55}; 56 57/** 58 * struct ioprio_blkcg - Per cgroup data. 59 * @cpd: blkcg_policy_data structure. 60 * @prio_policy: One of the IOPRIO_CLASS_* values. See also <linux/ioprio.h>. 61 */ 62struct ioprio_blkcg { 63 struct blkcg_policy_data cpd; 64 enum prio_policy prio_policy; 65 bool prio_set; 66}; 67 68static inline struct ioprio_blkg *pd_to_ioprio(struct blkg_policy_data *pd) 69{ 70 return pd ? container_of(pd, struct ioprio_blkg, pd) : NULL; 71} 72 73static struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg) 74{ 75 return container_of(blkcg_to_cpd(blkcg, &ioprio_policy), 76 struct ioprio_blkcg, cpd); 77} 78 79static struct ioprio_blkcg * 80ioprio_blkcg_from_css(struct cgroup_subsys_state *css) 81{ 82 return blkcg_to_ioprio_blkcg(css_to_blkcg(css)); 83} 84 85static struct ioprio_blkcg *ioprio_blkcg_from_bio(struct bio *bio) 86{ 87 struct blkg_policy_data *pd = blkg_to_pd(bio->bi_blkg, &ioprio_policy); 88 89 if (!pd) 90 return NULL; 91 92 return blkcg_to_ioprio_blkcg(pd->blkg->blkcg); 93} 94 95static int ioprio_show_prio_policy(struct seq_file *sf, void *v) 96{ 97 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf)); 98 99 seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]); 100 return 0; 101} 102 103static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf, 104 size_t nbytes, loff_t off) 105{ 106 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of)); 107 int ret; 108 109 if (off != 0) 110 return -EIO; 111 /* kernfs_fop_write_iter() terminates 'buf' with '\0'. */ 112 ret = sysfs_match_string(policy_name, buf); 113 if (ret < 0) 114 return ret; 115 blkcg->prio_policy = ret; 116 blkcg->prio_set = true; 117 return nbytes; 118} 119 120static struct blkg_policy_data * 121ioprio_alloc_pd(gfp_t gfp, struct request_queue *q, struct blkcg *blkcg) 122{ 123 struct ioprio_blkg *ioprio_blkg; 124 125 ioprio_blkg = kzalloc(sizeof(*ioprio_blkg), gfp); 126 if (!ioprio_blkg) 127 return NULL; 128 129 return &ioprio_blkg->pd; 130} 131 132static void ioprio_free_pd(struct blkg_policy_data *pd) 133{ 134 struct ioprio_blkg *ioprio_blkg = pd_to_ioprio(pd); 135 136 kfree(ioprio_blkg); 137} 138 139static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp) 140{ 141 struct ioprio_blkcg *blkcg; 142 143 blkcg = kzalloc(sizeof(*blkcg), gfp); 144 if (!blkcg) 145 return NULL; 146 blkcg->prio_policy = POLICY_NO_CHANGE; 147 return &blkcg->cpd; 148} 149 150static void ioprio_free_cpd(struct blkcg_policy_data *cpd) 151{ 152 struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd); 153 154 kfree(blkcg); 155} 156 157#define IOPRIO_ATTRS \ 158 { \ 159 .name = "prio.class", \ 160 .seq_show = ioprio_show_prio_policy, \ 161 .write = ioprio_set_prio_policy, \ 162 }, \ 163 { } /* sentinel */ 164 165/* cgroup v2 attributes */ 166static struct cftype ioprio_files[] = { 167 IOPRIO_ATTRS 168}; 169 170/* cgroup v1 attributes */ 171static struct cftype ioprio_legacy_files[] = { 172 IOPRIO_ATTRS 173}; 174 175static struct blkcg_policy ioprio_policy = { 176 .dfl_cftypes = ioprio_files, 177 .legacy_cftypes = ioprio_legacy_files, 178 179 .cpd_alloc_fn = ioprio_alloc_cpd, 180 .cpd_free_fn = ioprio_free_cpd, 181 182 .pd_alloc_fn = ioprio_alloc_pd, 183 .pd_free_fn = ioprio_free_pd, 184}; 185 186struct blk_ioprio { 187 struct rq_qos rqos; 188}; 189 190static void blkcg_ioprio_track(struct rq_qos *rqos, struct request *rq, 191 struct bio *bio) 192{ 193 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_bio(bio); 194 u16 prio; 195 196 if (!blkcg->prio_set) 197 return; 198 199 /* 200 * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers 201 * correspond to a lower priority. Hence, the max_t() below selects 202 * the lower priority of bi_ioprio and the cgroup I/O priority class. 203 * If the cgroup policy has been set to POLICY_NO_CHANGE == 0, the 204 * bio I/O priority is not modified. If the bio I/O priority equals 205 * IOPRIO_CLASS_NONE, the cgroup I/O priority is assigned to the bio. 206 */ 207 prio = max_t(u16, bio->bi_ioprio, 208 IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0)); 209 if (prio > bio->bi_ioprio) 210 bio->bi_ioprio = prio; 211} 212 213static void blkcg_ioprio_exit(struct rq_qos *rqos) 214{ 215 struct blk_ioprio *blkioprio_blkg = 216 container_of(rqos, typeof(*blkioprio_blkg), rqos); 217 218 blkcg_deactivate_policy(rqos->q, &ioprio_policy); 219 kfree(blkioprio_blkg); 220} 221 222static struct rq_qos_ops blkcg_ioprio_ops = { 223 .track = blkcg_ioprio_track, 224 .exit = blkcg_ioprio_exit, 225}; 226 227int blk_ioprio_init(struct request_queue *q) 228{ 229 struct blk_ioprio *blkioprio_blkg; 230 struct rq_qos *rqos; 231 int ret; 232 233 blkioprio_blkg = kzalloc(sizeof(*blkioprio_blkg), GFP_KERNEL); 234 if (!blkioprio_blkg) 235 return -ENOMEM; 236 237 ret = blkcg_activate_policy(q, &ioprio_policy); 238 if (ret) { 239 kfree(blkioprio_blkg); 240 return ret; 241 } 242 243 rqos = &blkioprio_blkg->rqos; 244 rqos->id = RQ_QOS_IOPRIO; 245 rqos->ops = &blkcg_ioprio_ops; 246 rqos->q = q; 247 248 /* 249 * Registering the rq-qos policy after activating the blk-cgroup 250 * policy guarantees that ioprio_blkcg_from_bio(bio) != NULL in the 251 * rq-qos callbacks. 252 */ 253 rq_qos_add(q, rqos); 254 255 return 0; 256} 257 258static int __init ioprio_init(void) 259{ 260 return blkcg_policy_register(&ioprio_policy); 261} 262 263static void __exit ioprio_exit(void) 264{ 265 blkcg_policy_unregister(&ioprio_policy); 266} 267 268module_init(ioprio_init); 269module_exit(ioprio_exit);