sun4v_tlb_miss.S (10836B)
1/* SPDX-License-Identifier: GPL-2.0 */ 2/* sun4v_tlb_miss.S: Sun4v TLB miss handlers. 3 * 4 * Copyright (C) 2006 <davem@davemloft.net> 5 */ 6 7 .text 8 .align 32 9 10 /* Load ITLB fault information into VADDR and CTX, using BASE. */ 11#define LOAD_ITLB_INFO(BASE, VADDR, CTX) \ 12 ldx [BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \ 13 ldx [BASE + HV_FAULT_I_CTX_OFFSET], CTX; 14 15 /* Load DTLB fault information into VADDR and CTX, using BASE. */ 16#define LOAD_DTLB_INFO(BASE, VADDR, CTX) \ 17 ldx [BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \ 18 ldx [BASE + HV_FAULT_D_CTX_OFFSET], CTX; 19 20 /* DEST = (VADDR >> 22) 21 * 22 * Branch to ZERO_CTX_LABEL if context is zero. 23 */ 24#define COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \ 25 srlx VADDR, 22, DEST; \ 26 brz,pn CTX, ZERO_CTX_LABEL; \ 27 nop; 28 29 /* Create TSB pointer. This is something like: 30 * 31 * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; 32 * tsb_base = tsb_reg & ~0x7UL; 33 * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask); 34 * tsb_ptr = tsb_base + (tsb_index * 16); 35 */ 36#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \ 37 and TSB_PTR, 0x7, TMP1; \ 38 mov 512, TMP2; \ 39 andn TSB_PTR, 0x7, TSB_PTR; \ 40 sllx TMP2, TMP1, TMP2; \ 41 srlx VADDR, HASH_SHIFT, TMP1; \ 42 sub TMP2, 1, TMP2; \ 43 and TMP1, TMP2, TMP1; \ 44 sllx TMP1, 4, TMP1; \ 45 add TSB_PTR, TMP1, TSB_PTR; 46 47sun4v_itlb_miss: 48 /* Load MMU Miss base into %g2. */ 49 ldxa [%g0] ASI_SCRATCHPAD, %g2 50 51 /* Load UTSB reg into %g1. */ 52 mov SCRATCHPAD_UTSBREG1, %g1 53 ldxa [%g1] ASI_SCRATCHPAD, %g1 54 55 LOAD_ITLB_INFO(%g2, %g4, %g5) 56 COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v) 57 COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) 58 59 /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ 60 ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 61 cmp %g2, %g6 62 bne,a,pn %xcc, tsb_miss_page_table_walk 63 mov FAULT_CODE_ITLB, %g3 64 andcc %g3, _PAGE_EXEC_4V, %g0 65 be,a,pn %xcc, tsb_do_fault 66 mov FAULT_CODE_ITLB, %g3 67 68 /* We have a valid entry, make hypervisor call to load 69 * I-TLB and return from trap. 70 * 71 * %g3: PTE 72 * %g4: vaddr 73 */ 74sun4v_itlb_load: 75 ldxa [%g0] ASI_SCRATCHPAD, %g6 76 mov %o0, %g1 ! save %o0 77 mov %o1, %g2 ! save %o1 78 mov %o2, %g5 ! save %o2 79 mov %o3, %g7 ! save %o3 80 mov %g4, %o0 ! vaddr 81 ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 ! ctx 82 mov %g3, %o2 ! PTE 83 mov HV_MMU_IMMU, %o3 ! flags 84 ta HV_MMU_MAP_ADDR_TRAP 85 brnz,pn %o0, sun4v_itlb_error 86 mov %g2, %o1 ! restore %o1 87 mov %g1, %o0 ! restore %o0 88 mov %g5, %o2 ! restore %o2 89 mov %g7, %o3 ! restore %o3 90 91 retry 92 93sun4v_dtlb_miss: 94 /* Load MMU Miss base into %g2. */ 95 ldxa [%g0] ASI_SCRATCHPAD, %g2 96 97 /* Load UTSB reg into %g1. */ 98 mov SCRATCHPAD_UTSBREG1, %g1 99 ldxa [%g1] ASI_SCRATCHPAD, %g1 100 101 LOAD_DTLB_INFO(%g2, %g4, %g5) 102 COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v) 103 COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) 104 105 /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ 106 ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 107 cmp %g2, %g6 108 bne,a,pn %xcc, tsb_miss_page_table_walk 109 mov FAULT_CODE_DTLB, %g3 110 111 /* We have a valid entry, make hypervisor call to load 112 * D-TLB and return from trap. 113 * 114 * %g3: PTE 115 * %g4: vaddr 116 */ 117sun4v_dtlb_load: 118 ldxa [%g0] ASI_SCRATCHPAD, %g6 119 mov %o0, %g1 ! save %o0 120 mov %o1, %g2 ! save %o1 121 mov %o2, %g5 ! save %o2 122 mov %o3, %g7 ! save %o3 123 mov %g4, %o0 ! vaddr 124 ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 ! ctx 125 mov %g3, %o2 ! PTE 126 mov HV_MMU_DMMU, %o3 ! flags 127 ta HV_MMU_MAP_ADDR_TRAP 128 brnz,pn %o0, sun4v_dtlb_error 129 mov %g2, %o1 ! restore %o1 130 mov %g1, %o0 ! restore %o0 131 mov %g5, %o2 ! restore %o2 132 mov %g7, %o3 ! restore %o3 133 134 retry 135 136sun4v_dtlb_prot: 137 SET_GL(1) 138 139 /* Load MMU Miss base into %g5. */ 140 ldxa [%g0] ASI_SCRATCHPAD, %g5 141 142 ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5 143 rdpr %tl, %g1 144 cmp %g1, 1 145 bgu,pn %xcc, winfix_trampoline 146 mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 147 ba,pt %xcc, sparc64_realfault_common 148 nop 149 150 /* Called from trap table: 151 * %g4: vaddr 152 * %g5: context 153 * %g6: TAG TARGET 154 */ 155sun4v_itsb_miss: 156 mov SCRATCHPAD_UTSBREG1, %g1 157 ldxa [%g1] ASI_SCRATCHPAD, %g1 158 brz,pn %g5, kvmap_itlb_4v 159 mov FAULT_CODE_ITLB, %g3 160 ba,a,pt %xcc, sun4v_tsb_miss_common 161 162 /* Called from trap table: 163 * %g4: vaddr 164 * %g5: context 165 * %g6: TAG TARGET 166 */ 167sun4v_dtsb_miss: 168 mov SCRATCHPAD_UTSBREG1, %g1 169 ldxa [%g1] ASI_SCRATCHPAD, %g1 170 brz,pn %g5, kvmap_dtlb_4v 171 mov FAULT_CODE_DTLB, %g3 172 173 /* fallthrough */ 174 175sun4v_tsb_miss_common: 176 COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7) 177 178 sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 179 180#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 181 mov SCRATCHPAD_UTSBREG2, %g5 182 ldxa [%g5] ASI_SCRATCHPAD, %g5 183 cmp %g5, -1 184 be,pt %xcc, 80f 185 nop 186 COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7) 187 188 /* That clobbered %g2, reload it. */ 189 ldxa [%g0] ASI_SCRATCHPAD, %g2 190 sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 191 19280: stx %g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP] 193#endif 194 195 ba,pt %xcc, tsb_miss_page_table_walk_sun4v_fastpath 196 ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7 197 198sun4v_itlb_error: 199 rdpr %tl, %g1 200 cmp %g1, 1 201 ble,pt %icc, sun4v_bad_ra 202 or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_ITLB, %g1 203 204 sethi %hi(sun4v_err_itlb_vaddr), %g1 205 stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)] 206 sethi %hi(sun4v_err_itlb_ctx), %g1 207 ldxa [%g0] ASI_SCRATCHPAD, %g6 208 ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 209 stx %o1, [%g1 + %lo(sun4v_err_itlb_ctx)] 210 sethi %hi(sun4v_err_itlb_pte), %g1 211 stx %g3, [%g1 + %lo(sun4v_err_itlb_pte)] 212 sethi %hi(sun4v_err_itlb_error), %g1 213 stx %o0, [%g1 + %lo(sun4v_err_itlb_error)] 214 215 sethi %hi(1f), %g7 216 rdpr %tl, %g4 217 ba,pt %xcc, etraptl1 2181: or %g7, %lo(1f), %g7 219 mov %l4, %o1 220 call sun4v_itlb_error_report 221 add %sp, PTREGS_OFF, %o0 222 223 /* NOTREACHED */ 224 225sun4v_dtlb_error: 226 rdpr %tl, %g1 227 cmp %g1, 1 228 ble,pt %icc, sun4v_bad_ra 229 or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_DTLB, %g1 230 231 sethi %hi(sun4v_err_dtlb_vaddr), %g1 232 stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)] 233 sethi %hi(sun4v_err_dtlb_ctx), %g1 234 ldxa [%g0] ASI_SCRATCHPAD, %g6 235 ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 236 stx %o1, [%g1 + %lo(sun4v_err_dtlb_ctx)] 237 sethi %hi(sun4v_err_dtlb_pte), %g1 238 stx %g3, [%g1 + %lo(sun4v_err_dtlb_pte)] 239 sethi %hi(sun4v_err_dtlb_error), %g1 240 stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)] 241 242 sethi %hi(1f), %g7 243 rdpr %tl, %g4 244 ba,pt %xcc, etraptl1 2451: or %g7, %lo(1f), %g7 246 mov %l4, %o1 247 call sun4v_dtlb_error_report 248 add %sp, PTREGS_OFF, %o0 249 250 /* NOTREACHED */ 251 252sun4v_bad_ra: 253 or %g0, %g4, %g5 254 ba,pt %xcc, sparc64_realfault_common 255 or %g1, %g0, %g4 256 257 /* NOTREACHED */ 258 259 /* Instruction Access Exception, tl0. */ 260sun4v_iacc: 261 ldxa [%g0] ASI_SCRATCHPAD, %g2 262 ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 263 ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 264 ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 265 sllx %g3, 16, %g3 266 or %g5, %g3, %g5 267 ba,pt %xcc, etrap 268 rd %pc, %g7 269 mov %l4, %o1 270 mov %l5, %o2 271 call sun4v_insn_access_exception 272 add %sp, PTREGS_OFF, %o0 273 ba,a,pt %xcc, rtrap 274 275 /* Instruction Access Exception, tl1. */ 276sun4v_iacc_tl1: 277 ldxa [%g0] ASI_SCRATCHPAD, %g2 278 ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 279 ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 280 ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 281 sllx %g3, 16, %g3 282 or %g5, %g3, %g5 283 ba,pt %xcc, etraptl1 284 rd %pc, %g7 285 mov %l4, %o1 286 mov %l5, %o2 287 call sun4v_insn_access_exception_tl1 288 add %sp, PTREGS_OFF, %o0 289 ba,a,pt %xcc, rtrap 290 291 /* Data Access Exception, tl0. */ 292sun4v_dacc: 293 ldxa [%g0] ASI_SCRATCHPAD, %g2 294 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 295 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 296 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 297 sllx %g3, 16, %g3 298 or %g5, %g3, %g5 299 ba,pt %xcc, etrap 300 rd %pc, %g7 301 mov %l4, %o1 302 mov %l5, %o2 303 call sun4v_data_access_exception 304 add %sp, PTREGS_OFF, %o0 305 ba,a,pt %xcc, rtrap 306 307 /* Data Access Exception, tl1. */ 308sun4v_dacc_tl1: 309 ldxa [%g0] ASI_SCRATCHPAD, %g2 310 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 311 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 312 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 313 sllx %g3, 16, %g3 314 or %g5, %g3, %g5 315 ba,pt %xcc, etraptl1 316 rd %pc, %g7 317 mov %l4, %o1 318 mov %l5, %o2 319 call sun4v_data_access_exception_tl1 320 add %sp, PTREGS_OFF, %o0 321 ba,a,pt %xcc, rtrap 322 323 /* Memory Address Unaligned. */ 324sun4v_mna: 325 /* Window fixup? */ 326 rdpr %tl, %g2 327 cmp %g2, 1 328 ble,pt %icc, 1f 329 nop 330 331 SET_GL(1) 332 ldxa [%g0] ASI_SCRATCHPAD, %g2 333 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5 334 mov HV_FAULT_TYPE_UNALIGNED, %g3 335 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g4 336 sllx %g3, 16, %g3 337 or %g4, %g3, %g4 338 ba,pt %xcc, winfix_mna 339 rdpr %tpc, %g3 340 /* not reached */ 341 3421: ldxa [%g0] ASI_SCRATCHPAD, %g2 343 mov HV_FAULT_TYPE_UNALIGNED, %g3 344 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 345 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 346 sllx %g3, 16, %g3 347 or %g5, %g3, %g5 348 349 ba,pt %xcc, etrap 350 rd %pc, %g7 351 mov %l4, %o1 352 mov %l5, %o2 353 call sun4v_do_mna 354 add %sp, PTREGS_OFF, %o0 355 ba,a,pt %xcc, rtrap 356 nop 357 358 /* Privileged Action. */ 359sun4v_privact: 360 ba,pt %xcc, etrap 361 rd %pc, %g7 362 call do_privact 363 add %sp, PTREGS_OFF, %o0 364 ba,a,pt %xcc, rtrap 365 366 /* Unaligned ldd float, tl0. */ 367sun4v_lddfmna: 368 ldxa [%g0] ASI_SCRATCHPAD, %g2 369 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 370 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 371 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 372 sllx %g3, 16, %g3 373 or %g5, %g3, %g5 374 ba,pt %xcc, etrap 375 rd %pc, %g7 376 mov %l4, %o1 377 mov %l5, %o2 378 call handle_lddfmna 379 add %sp, PTREGS_OFF, %o0 380 ba,a,pt %xcc, rtrap 381 382 /* Unaligned std float, tl0. */ 383sun4v_stdfmna: 384 ldxa [%g0] ASI_SCRATCHPAD, %g2 385 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 386 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 387 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 388 sllx %g3, 16, %g3 389 or %g5, %g3, %g5 390 ba,pt %xcc, etrap 391 rd %pc, %g7 392 mov %l4, %o1 393 mov %l5, %o2 394 call handle_stdfmna 395 add %sp, PTREGS_OFF, %o0 396 ba,a,pt %xcc, rtrap 397 398#define BRANCH_ALWAYS 0x10680000 399#define NOP 0x01000000 400#define SUN4V_DO_PATCH(OLD, NEW) \ 401 sethi %hi(NEW), %g1; \ 402 or %g1, %lo(NEW), %g1; \ 403 sethi %hi(OLD), %g2; \ 404 or %g2, %lo(OLD), %g2; \ 405 sub %g1, %g2, %g1; \ 406 sethi %hi(BRANCH_ALWAYS), %g3; \ 407 sll %g1, 11, %g1; \ 408 srl %g1, 11 + 2, %g1; \ 409 or %g3, %lo(BRANCH_ALWAYS), %g3; \ 410 or %g3, %g1, %g3; \ 411 stw %g3, [%g2]; \ 412 sethi %hi(NOP), %g3; \ 413 or %g3, %lo(NOP), %g3; \ 414 stw %g3, [%g2 + 0x4]; \ 415 flush %g2; 416 417 .globl sun4v_patch_tlb_handlers 418 .type sun4v_patch_tlb_handlers,#function 419sun4v_patch_tlb_handlers: 420 SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss) 421 SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss) 422 SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss) 423 SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss) 424 SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot) 425 SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot) 426 SUN4V_DO_PATCH(tl0_iax, sun4v_iacc) 427 SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1) 428 SUN4V_DO_PATCH(tl0_dax, sun4v_dacc) 429 SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1) 430 SUN4V_DO_PATCH(tl0_mna, sun4v_mna) 431 SUN4V_DO_PATCH(tl1_mna, sun4v_mna) 432 SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna) 433 SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna) 434 SUN4V_DO_PATCH(tl0_privact, sun4v_privact) 435 retl 436 nop 437 .size sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers