ktlb.S (5966B)
1/* SPDX-License-Identifier: GPL-2.0 */ 2/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling. 3 * 4 * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net> 5 * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de) 6 * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) 7 * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 8 */ 9 10#include <linux/pgtable.h> 11#include <asm/head.h> 12#include <asm/asi.h> 13#include <asm/page.h> 14#include <asm/tsb.h> 15 16 .text 17 .align 32 18 19kvmap_itlb: 20 /* g6: TAG TARGET */ 21 mov TLB_TAG_ACCESS, %g4 22 ldxa [%g4] ASI_IMMU, %g4 23 24 /* The kernel executes in context zero, therefore we do not 25 * need to clear the context ID bits out of %g4 here. 26 */ 27 28 /* sun4v_itlb_miss branches here with the missing virtual 29 * address already loaded into %g4 30 */ 31kvmap_itlb_4v: 32 33 /* Catch kernel NULL pointer calls. */ 34 sethi %hi(PAGE_SIZE), %g5 35 cmp %g4, %g5 36 blu,pn %xcc, kvmap_itlb_longpath 37 nop 38 39 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load) 40 41kvmap_itlb_tsb_miss: 42 sethi %hi(LOW_OBP_ADDRESS), %g5 43 cmp %g4, %g5 44 blu,pn %xcc, kvmap_itlb_vmalloc_addr 45 mov 0x1, %g5 46 sllx %g5, 32, %g5 47 cmp %g4, %g5 48 blu,pn %xcc, kvmap_itlb_obp 49 nop 50 51kvmap_itlb_vmalloc_addr: 52 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) 53 54 TSB_LOCK_TAG(%g1, %g2, %g7) 55 TSB_WRITE(%g1, %g5, %g6) 56 57 /* fallthrough to TLB load */ 58 59kvmap_itlb_load: 60 61661: stxa %g5, [%g0] ASI_ITLB_DATA_IN 62 retry 63 .section .sun4v_2insn_patch, "ax" 64 .word 661b 65 nop 66 nop 67 .previous 68 69 /* For sun4v the ASI_ITLB_DATA_IN store and the retry 70 * instruction get nop'd out and we get here to branch 71 * to the sun4v tlb load code. The registers are setup 72 * as follows: 73 * 74 * %g4: vaddr 75 * %g5: PTE 76 * %g6: TAG 77 * 78 * The sun4v TLB load wants the PTE in %g3 so we fix that 79 * up here. 80 */ 81 ba,pt %xcc, sun4v_itlb_load 82 mov %g5, %g3 83 84kvmap_itlb_longpath: 85 86661: rdpr %pstate, %g5 87 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate 88 .section .sun4v_2insn_patch, "ax" 89 .word 661b 90 SET_GL(1) 91 nop 92 .previous 93 94 rdpr %tpc, %g5 95 ba,pt %xcc, sparc64_realfault_common 96 mov FAULT_CODE_ITLB, %g4 97 98kvmap_itlb_obp: 99 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath) 100 101 TSB_LOCK_TAG(%g1, %g2, %g7) 102 103 TSB_WRITE(%g1, %g5, %g6) 104 105 ba,pt %xcc, kvmap_itlb_load 106 nop 107 108kvmap_dtlb_obp: 109 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath) 110 111 TSB_LOCK_TAG(%g1, %g2, %g7) 112 113 TSB_WRITE(%g1, %g5, %g6) 114 115 ba,pt %xcc, kvmap_dtlb_load 116 nop 117 118kvmap_linear_early: 119 sethi %hi(kern_linear_pte_xor), %g7 120 ldx [%g7 + %lo(kern_linear_pte_xor)], %g2 121 ba,pt %xcc, kvmap_dtlb_tsb4m_load 122 xor %g2, %g4, %g5 123 124 .align 32 125kvmap_dtlb_tsb4m_load: 126 TSB_LOCK_TAG(%g1, %g2, %g7) 127 TSB_WRITE(%g1, %g5, %g6) 128 ba,pt %xcc, kvmap_dtlb_load 129 nop 130 131kvmap_dtlb: 132 /* %g6: TAG TARGET */ 133 mov TLB_TAG_ACCESS, %g4 134 ldxa [%g4] ASI_DMMU, %g4 135 136 /* The kernel executes in context zero, therefore we do not 137 * need to clear the context ID bits out of %g4 here. 138 */ 139 140 /* sun4v_dtlb_miss branches here with the missing virtual 141 * address already loaded into %g4 142 */ 143kvmap_dtlb_4v: 144 brgez,pn %g4, kvmap_dtlb_nonlinear 145 nop 146 147#ifdef CONFIG_DEBUG_PAGEALLOC 148 /* Index through the base page size TSB even for linear 149 * mappings when using page allocation debugging. 150 */ 151 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) 152#else 153 /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */ 154 KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) 155#endif 156 /* Linear mapping TSB lookup failed. Fallthrough to kernel 157 * page table based lookup. 158 */ 159 .globl kvmap_linear_patch 160kvmap_linear_patch: 161 ba,a,pt %xcc, kvmap_linear_early 162 163kvmap_dtlb_vmalloc_addr: 164 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) 165 166 TSB_LOCK_TAG(%g1, %g2, %g7) 167 TSB_WRITE(%g1, %g5, %g6) 168 169 /* fallthrough to TLB load */ 170 171kvmap_dtlb_load: 172 173661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB 174 retry 175 .section .sun4v_2insn_patch, "ax" 176 .word 661b 177 nop 178 nop 179 .previous 180 181 /* For sun4v the ASI_DTLB_DATA_IN store and the retry 182 * instruction get nop'd out and we get here to branch 183 * to the sun4v tlb load code. The registers are setup 184 * as follows: 185 * 186 * %g4: vaddr 187 * %g5: PTE 188 * %g6: TAG 189 * 190 * The sun4v TLB load wants the PTE in %g3 so we fix that 191 * up here. 192 */ 193 ba,pt %xcc, sun4v_dtlb_load 194 mov %g5, %g3 195 196#ifdef CONFIG_SPARSEMEM_VMEMMAP 197kvmap_vmemmap: 198 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) 199 ba,a,pt %xcc, kvmap_dtlb_load 200#endif 201 202kvmap_dtlb_nonlinear: 203 /* Catch kernel NULL pointer derefs. */ 204 sethi %hi(PAGE_SIZE), %g5 205 cmp %g4, %g5 206 bleu,pn %xcc, kvmap_dtlb_longpath 207 nop 208 209#ifdef CONFIG_SPARSEMEM_VMEMMAP 210 /* Do not use the TSB for vmemmap. */ 211 sethi %hi(VMEMMAP_BASE), %g5 212 ldx [%g5 + %lo(VMEMMAP_BASE)], %g5 213 cmp %g4,%g5 214 bgeu,pn %xcc, kvmap_vmemmap 215 nop 216#endif 217 218 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) 219 220kvmap_dtlb_tsbmiss: 221 sethi %hi(MODULES_VADDR), %g5 222 cmp %g4, %g5 223 blu,pn %xcc, kvmap_dtlb_longpath 224 sethi %hi(VMALLOC_END), %g5 225 ldx [%g5 + %lo(VMALLOC_END)], %g5 226 cmp %g4, %g5 227 bgeu,pn %xcc, kvmap_dtlb_longpath 228 nop 229 230kvmap_check_obp: 231 sethi %hi(LOW_OBP_ADDRESS), %g5 232 cmp %g4, %g5 233 blu,pn %xcc, kvmap_dtlb_vmalloc_addr 234 mov 0x1, %g5 235 sllx %g5, 32, %g5 236 cmp %g4, %g5 237 blu,pn %xcc, kvmap_dtlb_obp 238 nop 239 ba,pt %xcc, kvmap_dtlb_vmalloc_addr 240 nop 241 242kvmap_dtlb_longpath: 243 244661: rdpr %pstate, %g5 245 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate 246 .section .sun4v_2insn_patch, "ax" 247 .word 661b 248 SET_GL(1) 249 ldxa [%g0] ASI_SCRATCHPAD, %g5 250 .previous 251 252 rdpr %tl, %g3 253 cmp %g3, 1 254 255661: mov TLB_TAG_ACCESS, %g4 256 ldxa [%g4] ASI_DMMU, %g5 257 .section .sun4v_2insn_patch, "ax" 258 .word 661b 259 ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5 260 nop 261 .previous 262 263 /* The kernel executes in context zero, therefore we do not 264 * need to clear the context ID bits out of %g5 here. 265 */ 266 267 be,pt %xcc, sparc64_realfault_common 268 mov FAULT_CODE_DTLB, %g4 269 ba,pt %xcc, winfix_trampoline 270 nop