cpuset.h (8131B)
1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _LINUX_CPUSET_H 3#define _LINUX_CPUSET_H 4/* 5 * cpuset interface 6 * 7 * Copyright (C) 2003 BULL SA 8 * Copyright (C) 2004-2006 Silicon Graphics, Inc. 9 * 10 */ 11 12#include <linux/sched.h> 13#include <linux/sched/topology.h> 14#include <linux/sched/task.h> 15#include <linux/cpumask.h> 16#include <linux/nodemask.h> 17#include <linux/mm.h> 18#include <linux/mmu_context.h> 19#include <linux/jump_label.h> 20 21#ifdef CONFIG_CPUSETS 22 23/* 24 * Static branch rewrites can happen in an arbitrary order for a given 25 * key. In code paths where we need to loop with read_mems_allowed_begin() and 26 * read_mems_allowed_retry() to get a consistent view of mems_allowed, we need 27 * to ensure that begin() always gets rewritten before retry() in the 28 * disabled -> enabled transition. If not, then if local irqs are disabled 29 * around the loop, we can deadlock since retry() would always be 30 * comparing the latest value of the mems_allowed seqcount against 0 as 31 * begin() still would see cpusets_enabled() as false. The enabled -> disabled 32 * transition should happen in reverse order for the same reasons (want to stop 33 * looking at real value of mems_allowed.sequence in retry() first). 34 */ 35extern struct static_key_false cpusets_pre_enable_key; 36extern struct static_key_false cpusets_enabled_key; 37extern struct static_key_false cpusets_insane_config_key; 38 39static inline bool cpusets_enabled(void) 40{ 41 return static_branch_unlikely(&cpusets_enabled_key); 42} 43 44static inline void cpuset_inc(void) 45{ 46 static_branch_inc_cpuslocked(&cpusets_pre_enable_key); 47 static_branch_inc_cpuslocked(&cpusets_enabled_key); 48} 49 50static inline void cpuset_dec(void) 51{ 52 static_branch_dec_cpuslocked(&cpusets_enabled_key); 53 static_branch_dec_cpuslocked(&cpusets_pre_enable_key); 54} 55 56/* 57 * This will get enabled whenever a cpuset configuration is considered 58 * unsupportable in general. E.g. movable only node which cannot satisfy 59 * any non movable allocations (see update_nodemask). Page allocator 60 * needs to make additional checks for those configurations and this 61 * check is meant to guard those checks without any overhead for sane 62 * configurations. 63 */ 64static inline bool cpusets_insane_config(void) 65{ 66 return static_branch_unlikely(&cpusets_insane_config_key); 67} 68 69extern int cpuset_init(void); 70extern void cpuset_init_smp(void); 71extern void cpuset_force_rebuild(void); 72extern void cpuset_update_active_cpus(void); 73extern void cpuset_wait_for_hotplug(void); 74extern void cpuset_read_lock(void); 75extern void cpuset_read_unlock(void); 76extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); 77extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); 78extern nodemask_t cpuset_mems_allowed(struct task_struct *p); 79#define cpuset_current_mems_allowed (current->mems_allowed) 80void cpuset_init_current_mems_allowed(void); 81int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask); 82 83extern bool __cpuset_node_allowed(int node, gfp_t gfp_mask); 84 85static inline bool cpuset_node_allowed(int node, gfp_t gfp_mask) 86{ 87 if (cpusets_enabled()) 88 return __cpuset_node_allowed(node, gfp_mask); 89 return true; 90} 91 92static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) 93{ 94 return __cpuset_node_allowed(zone_to_nid(z), gfp_mask); 95} 96 97static inline bool cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) 98{ 99 if (cpusets_enabled()) 100 return __cpuset_zone_allowed(z, gfp_mask); 101 return true; 102} 103 104extern int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, 105 const struct task_struct *tsk2); 106 107#define cpuset_memory_pressure_bump() \ 108 do { \ 109 if (cpuset_memory_pressure_enabled) \ 110 __cpuset_memory_pressure_bump(); \ 111 } while (0) 112extern int cpuset_memory_pressure_enabled; 113extern void __cpuset_memory_pressure_bump(void); 114 115extern void cpuset_task_status_allowed(struct seq_file *m, 116 struct task_struct *task); 117extern int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns, 118 struct pid *pid, struct task_struct *tsk); 119 120extern int cpuset_mem_spread_node(void); 121extern int cpuset_slab_spread_node(void); 122 123static inline int cpuset_do_page_mem_spread(void) 124{ 125 return task_spread_page(current); 126} 127 128static inline int cpuset_do_slab_mem_spread(void) 129{ 130 return task_spread_slab(current); 131} 132 133extern bool current_cpuset_is_being_rebound(void); 134 135extern void rebuild_sched_domains(void); 136 137extern void cpuset_print_current_mems_allowed(void); 138 139/* 140 * read_mems_allowed_begin is required when making decisions involving 141 * mems_allowed such as during page allocation. mems_allowed can be updated in 142 * parallel and depending on the new value an operation can fail potentially 143 * causing process failure. A retry loop with read_mems_allowed_begin and 144 * read_mems_allowed_retry prevents these artificial failures. 145 */ 146static inline unsigned int read_mems_allowed_begin(void) 147{ 148 if (!static_branch_unlikely(&cpusets_pre_enable_key)) 149 return 0; 150 151 return read_seqcount_begin(¤t->mems_allowed_seq); 152} 153 154/* 155 * If this returns true, the operation that took place after 156 * read_mems_allowed_begin may have failed artificially due to a concurrent 157 * update of mems_allowed. It is up to the caller to retry the operation if 158 * appropriate. 159 */ 160static inline bool read_mems_allowed_retry(unsigned int seq) 161{ 162 if (!static_branch_unlikely(&cpusets_enabled_key)) 163 return false; 164 165 return read_seqcount_retry(¤t->mems_allowed_seq, seq); 166} 167 168static inline void set_mems_allowed(nodemask_t nodemask) 169{ 170 unsigned long flags; 171 172 task_lock(current); 173 local_irq_save(flags); 174 write_seqcount_begin(¤t->mems_allowed_seq); 175 current->mems_allowed = nodemask; 176 write_seqcount_end(¤t->mems_allowed_seq); 177 local_irq_restore(flags); 178 task_unlock(current); 179} 180 181#else /* !CONFIG_CPUSETS */ 182 183static inline bool cpusets_enabled(void) { return false; } 184 185static inline bool cpusets_insane_config(void) { return false; } 186 187static inline int cpuset_init(void) { return 0; } 188static inline void cpuset_init_smp(void) {} 189 190static inline void cpuset_force_rebuild(void) { } 191 192static inline void cpuset_update_active_cpus(void) 193{ 194 partition_sched_domains(1, NULL, NULL); 195} 196 197static inline void cpuset_wait_for_hotplug(void) { } 198 199static inline void cpuset_read_lock(void) { } 200static inline void cpuset_read_unlock(void) { } 201 202static inline void cpuset_cpus_allowed(struct task_struct *p, 203 struct cpumask *mask) 204{ 205 cpumask_copy(mask, task_cpu_possible_mask(p)); 206} 207 208static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p) 209{ 210 return false; 211} 212 213static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) 214{ 215 return node_possible_map; 216} 217 218#define cpuset_current_mems_allowed (node_states[N_MEMORY]) 219static inline void cpuset_init_current_mems_allowed(void) {} 220 221static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) 222{ 223 return 1; 224} 225 226static inline bool cpuset_node_allowed(int node, gfp_t gfp_mask) 227{ 228 return true; 229} 230 231static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) 232{ 233 return true; 234} 235 236static inline bool cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) 237{ 238 return true; 239} 240 241static inline int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, 242 const struct task_struct *tsk2) 243{ 244 return 1; 245} 246 247static inline void cpuset_memory_pressure_bump(void) {} 248 249static inline void cpuset_task_status_allowed(struct seq_file *m, 250 struct task_struct *task) 251{ 252} 253 254static inline int cpuset_mem_spread_node(void) 255{ 256 return 0; 257} 258 259static inline int cpuset_slab_spread_node(void) 260{ 261 return 0; 262} 263 264static inline int cpuset_do_page_mem_spread(void) 265{ 266 return 0; 267} 268 269static inline int cpuset_do_slab_mem_spread(void) 270{ 271 return 0; 272} 273 274static inline bool current_cpuset_is_being_rebound(void) 275{ 276 return false; 277} 278 279static inline void rebuild_sched_domains(void) 280{ 281 partition_sched_domains(1, NULL, NULL); 282} 283 284static inline void cpuset_print_current_mems_allowed(void) 285{ 286} 287 288static inline void set_mems_allowed(nodemask_t nodemask) 289{ 290} 291 292static inline unsigned int read_mems_allowed_begin(void) 293{ 294 return 0; 295} 296 297static inline bool read_mems_allowed_retry(unsigned int seq) 298{ 299 return false; 300} 301 302#endif /* !CONFIG_CPUSETS */ 303 304#endif /* _LINUX_CPUSET_H */