align.S (12272B)
1/* 2 * arch/xtensa/kernel/align.S 3 * 4 * Handle unalignment exceptions in kernel space. 5 * 6 * This file is subject to the terms and conditions of the GNU General 7 * Public License. See the file "COPYING" in the main directory of 8 * this archive for more details. 9 * 10 * Copyright (C) 2001 - 2005 Tensilica, Inc. 11 * Copyright (C) 2014 Cadence Design Systems Inc. 12 * 13 * Rewritten by Chris Zankel <chris@zankel.net> 14 * 15 * Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> 16 * and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca> 17 */ 18 19#include <linux/linkage.h> 20#include <asm/current.h> 21#include <asm/asm-offsets.h> 22#include <asm/asmmacro.h> 23#include <asm/processor.h> 24 25#if XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION 26 27/* First-level exception handler for unaligned exceptions. 28 * 29 * Note: This handler works only for kernel exceptions. Unaligned user 30 * access should get a seg fault. 31 */ 32 33/* Big and little endian 16-bit values are located in 34 * different halves of a register. HWORD_START helps to 35 * abstract the notion of extracting a 16-bit value from a 36 * register. 37 * We also have to define new shifting instructions because 38 * lsb and msb are on 'opposite' ends in a register for 39 * different endian machines. 40 * 41 * Assume a memory region in ascending address: 42 * 0 1 2 3|4 5 6 7 43 * 44 * When loading one word into a register, the content of that register is: 45 * LE 3 2 1 0, 7 6 5 4 46 * BE 0 1 2 3, 4 5 6 7 47 * 48 * Masking the bits of the higher/lower address means: 49 * LE X X 0 0, 0 0 X X 50 * BE 0 0 X X, X X 0 0 51 * 52 * Shifting to higher/lower addresses, means: 53 * LE shift left / shift right 54 * BE shift right / shift left 55 * 56 * Extracting 16 bits from a 32 bit reg. value to higher/lower address means: 57 * LE mask 0 0 X X / shift left 58 * BE shift left / mask 0 0 X X 59 */ 60 61#if XCHAL_HAVE_WINDOWED 62#define UNALIGNED_USER_EXCEPTION 63#endif 64 65#if XCHAL_HAVE_BE 66 67#define HWORD_START 16 68#define INSN_OP0 28 69#define INSN_T 24 70#define INSN_OP1 16 71 72.macro __ssa8r r; ssa8l \r; .endm 73.macro __sh r, s; srl \r, \s; .endm 74.macro __sl r, s; sll \r, \s; .endm 75.macro __exth r, s; extui \r, \s, 0, 16; .endm 76.macro __extl r, s; slli \r, \s, 16; .endm 77 78#else 79 80#define HWORD_START 0 81#define INSN_OP0 0 82#define INSN_T 4 83#define INSN_OP1 12 84 85.macro __ssa8r r; ssa8b \r; .endm 86.macro __sh r, s; sll \r, \s; .endm 87.macro __sl r, s; srl \r, \s; .endm 88.macro __exth r, s; slli \r, \s, 16; .endm 89.macro __extl r, s; extui \r, \s, 0, 16; .endm 90 91#endif 92 93/* 94 * xxxx xxxx = imm8 field 95 * yyyy = imm4 field 96 * ssss = s field 97 * tttt = t field 98 * 99 * 16 0 100 * ------------------- 101 * L32I.N yyyy ssss tttt 1000 102 * S32I.N yyyy ssss tttt 1001 103 * 104 * 23 0 105 * ----------------------------- 106 * res 0000 0010 107 * L16UI xxxx xxxx 0001 ssss tttt 0010 108 * L32I xxxx xxxx 0010 ssss tttt 0010 109 * XXX 0011 ssss tttt 0010 110 * XXX 0100 ssss tttt 0010 111 * S16I xxxx xxxx 0101 ssss tttt 0010 112 * S32I xxxx xxxx 0110 ssss tttt 0010 113 * XXX 0111 ssss tttt 0010 114 * XXX 1000 ssss tttt 0010 115 * L16SI xxxx xxxx 1001 ssss tttt 0010 116 * XXX 1010 0010 117 * **L32AI xxxx xxxx 1011 ssss tttt 0010 unsupported 118 * XXX 1100 0010 119 * XXX 1101 0010 120 * XXX 1110 0010 121 * **S32RI xxxx xxxx 1111 ssss tttt 0010 unsupported 122 * ----------------------------- 123 * ^ ^ ^ 124 * sub-opcode (NIBBLE_R) -+ | | 125 * t field (NIBBLE_T) -----------+ | 126 * major opcode (NIBBLE_OP0) --------------+ 127 */ 128 129#define OP0_L32I_N 0x8 /* load immediate narrow */ 130#define OP0_S32I_N 0x9 /* store immediate narrow */ 131#define OP1_SI_MASK 0x4 /* OP1 bit set for stores */ 132#define OP1_SI_BIT 2 /* OP1 bit number for stores */ 133 134#define OP1_L32I 0x2 135#define OP1_L16UI 0x1 136#define OP1_L16SI 0x9 137#define OP1_L32AI 0xb 138 139#define OP1_S32I 0x6 140#define OP1_S16I 0x5 141#define OP1_S32RI 0xf 142 143/* 144 * Entry condition: 145 * 146 * a0: trashed, original value saved on stack (PT_AREG0) 147 * a1: a1 148 * a2: new stack pointer, original in DEPC 149 * a3: a3 150 * depc: a2, original value saved on stack (PT_DEPC) 151 * excsave_1: dispatch table 152 * 153 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 154 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 155 */ 156 157 .literal_position 158ENTRY(fast_unaligned) 159 160 /* Note: We don't expect the address to be aligned on a word 161 * boundary. After all, the processor generated that exception 162 * and it would be a hardware fault. 163 */ 164 165 /* Save some working register */ 166 167 s32i a4, a2, PT_AREG4 168 s32i a5, a2, PT_AREG5 169 s32i a6, a2, PT_AREG6 170 s32i a7, a2, PT_AREG7 171 s32i a8, a2, PT_AREG8 172 173 rsr a0, depc 174 s32i a0, a2, PT_AREG2 175 s32i a3, a2, PT_AREG3 176 177 rsr a3, excsave1 178 movi a4, fast_unaligned_fixup 179 s32i a4, a3, EXC_TABLE_FIXUP 180 181 /* Keep value of SAR in a0 */ 182 183 rsr a0, sar 184 rsr a8, excvaddr # load unaligned memory address 185 186 /* Now, identify one of the following load/store instructions. 187 * 188 * The only possible danger of a double exception on the 189 * following l32i instructions is kernel code in vmalloc 190 * memory. The processor was just executing at the EPC_1 191 * address, and indeed, already fetched the instruction. That 192 * guarantees a TLB mapping, which hasn't been replaced by 193 * this unaligned exception handler that uses only static TLB 194 * mappings. However, high-level interrupt handlers might 195 * modify TLB entries, so for the generic case, we register a 196 * TABLE_FIXUP handler here, too. 197 */ 198 199 /* a3...a6 saved on stack, a2 = SP */ 200 201 /* Extract the instruction that caused the unaligned access. */ 202 203 rsr a7, epc1 # load exception address 204 movi a3, ~3 205 and a3, a3, a7 # mask lower bits 206 207 l32i a4, a3, 0 # load 2 words 208 l32i a5, a3, 4 209 210 __ssa8 a7 211 __src_b a4, a4, a5 # a4 has the instruction 212 213 /* Analyze the instruction (load or store?). */ 214 215 extui a5, a4, INSN_OP0, 4 # get insn.op0 nibble 216 217#if XCHAL_HAVE_DENSITY 218 _beqi a5, OP0_L32I_N, .Lload # L32I.N, jump 219 addi a6, a5, -OP0_S32I_N 220 _beqz a6, .Lstore # S32I.N, do a store 221#endif 222 /* 'store indicator bit' not set, jump */ 223 _bbci.l a4, OP1_SI_BIT + INSN_OP1, .Lload 224 225 /* Store: Jump to table entry to get the value in the source register.*/ 226 227.Lstore:movi a5, .Lstore_table # table 228 extui a6, a4, INSN_T, 4 # get source register 229 addx8 a5, a6, a5 230 jx a5 # jump into table 231 232 /* Load: Load memory address. */ 233 234.Lload: movi a3, ~3 235 and a3, a3, a8 # align memory address 236 237 __ssa8 a8 238#ifdef UNALIGNED_USER_EXCEPTION 239 addi a3, a3, 8 240 l32e a5, a3, -8 241 l32e a6, a3, -4 242#else 243 l32i a5, a3, 0 244 l32i a6, a3, 4 245#endif 246 __src_b a3, a5, a6 # a3 has the data word 247 248#if XCHAL_HAVE_DENSITY 249 addi a7, a7, 2 # increment PC (assume 16-bit insn) 250 251 extui a5, a4, INSN_OP0, 4 252 _beqi a5, OP0_L32I_N, 1f # l32i.n: jump 253 254 addi a7, a7, 1 255#else 256 addi a7, a7, 3 257#endif 258 259 extui a5, a4, INSN_OP1, 4 260 _beqi a5, OP1_L32I, 1f # l32i: jump 261 262 extui a3, a3, 0, 16 # extract lower 16 bits 263 _beqi a5, OP1_L16UI, 1f 264 addi a5, a5, -OP1_L16SI 265 _bnez a5, .Linvalid_instruction_load 266 267 /* sign extend value */ 268 269 slli a3, a3, 16 270 srai a3, a3, 16 271 272 /* Set target register. */ 273 2741: 275 extui a4, a4, INSN_T, 4 # extract target register 276 movi a5, .Lload_table 277 addx8 a4, a4, a5 278 jx a4 # jump to entry for target register 279 280 .align 8 281.Lload_table: 282 s32i a3, a2, PT_AREG0; _j .Lexit; .align 8 283 mov a1, a3; _j .Lexit; .align 8 # fishy?? 284 s32i a3, a2, PT_AREG2; _j .Lexit; .align 8 285 s32i a3, a2, PT_AREG3; _j .Lexit; .align 8 286 s32i a3, a2, PT_AREG4; _j .Lexit; .align 8 287 s32i a3, a2, PT_AREG5; _j .Lexit; .align 8 288 s32i a3, a2, PT_AREG6; _j .Lexit; .align 8 289 s32i a3, a2, PT_AREG7; _j .Lexit; .align 8 290 s32i a3, a2, PT_AREG8; _j .Lexit; .align 8 291 mov a9, a3 ; _j .Lexit; .align 8 292 mov a10, a3 ; _j .Lexit; .align 8 293 mov a11, a3 ; _j .Lexit; .align 8 294 mov a12, a3 ; _j .Lexit; .align 8 295 mov a13, a3 ; _j .Lexit; .align 8 296 mov a14, a3 ; _j .Lexit; .align 8 297 mov a15, a3 ; _j .Lexit; .align 8 298 299.Lstore_table: 300 l32i a3, a2, PT_AREG0; _j 1f; .align 8 301 mov a3, a1; _j 1f; .align 8 # fishy?? 302 l32i a3, a2, PT_AREG2; _j 1f; .align 8 303 l32i a3, a2, PT_AREG3; _j 1f; .align 8 304 l32i a3, a2, PT_AREG4; _j 1f; .align 8 305 l32i a3, a2, PT_AREG5; _j 1f; .align 8 306 l32i a3, a2, PT_AREG6; _j 1f; .align 8 307 l32i a3, a2, PT_AREG7; _j 1f; .align 8 308 l32i a3, a2, PT_AREG8; _j 1f; .align 8 309 mov a3, a9 ; _j 1f; .align 8 310 mov a3, a10 ; _j 1f; .align 8 311 mov a3, a11 ; _j 1f; .align 8 312 mov a3, a12 ; _j 1f; .align 8 313 mov a3, a13 ; _j 1f; .align 8 314 mov a3, a14 ; _j 1f; .align 8 315 mov a3, a15 ; _j 1f; .align 8 316 317 /* We cannot handle this exception. */ 318 319 .extern _kernel_exception 320.Linvalid_instruction_load: 321.Linvalid_instruction_store: 322 323 movi a4, 0 324 rsr a3, excsave1 325 s32i a4, a3, EXC_TABLE_FIXUP 326 327 /* Restore a4...a8 and SAR, set SP, and jump to default exception. */ 328 329 l32i a8, a2, PT_AREG8 330 l32i a7, a2, PT_AREG7 331 l32i a6, a2, PT_AREG6 332 l32i a5, a2, PT_AREG5 333 l32i a4, a2, PT_AREG4 334 wsr a0, sar 335 mov a1, a2 336 337 rsr a0, ps 338 bbsi.l a0, PS_UM_BIT, 2f # jump if user mode 339 340 movi a0, _kernel_exception 341 jx a0 342 3432: movi a0, _user_exception 344 jx a0 345 3461: # a7: instruction pointer, a4: instruction, a3: value 347 348 movi a6, 0 # mask: ffffffff:00000000 349 350#if XCHAL_HAVE_DENSITY 351 addi a7, a7, 2 # incr. PC,assume 16-bit instruction 352 353 extui a5, a4, INSN_OP0, 4 # extract OP0 354 addi a5, a5, -OP0_S32I_N 355 _beqz a5, 1f # s32i.n: jump 356 357 addi a7, a7, 1 # increment PC, 32-bit instruction 358#else 359 addi a7, a7, 3 # increment PC, 32-bit instruction 360#endif 361 362 extui a5, a4, INSN_OP1, 4 # extract OP1 363 _beqi a5, OP1_S32I, 1f # jump if 32 bit store 364 _bnei a5, OP1_S16I, .Linvalid_instruction_store 365 366 movi a5, -1 367 __extl a3, a3 # get 16-bit value 368 __exth a6, a5 # get 16-bit mask ffffffff:ffff0000 369 370 /* Get memory address */ 371 3721: 373 movi a4, ~3 374 and a4, a4, a8 # align memory address 375 376 /* Insert value into memory */ 377 378 movi a5, -1 # mask: ffffffff:XXXX0000 379#ifdef UNALIGNED_USER_EXCEPTION 380 addi a4, a4, 8 381#endif 382 383 __ssa8r a8 384 __src_b a8, a5, a6 # lo-mask F..F0..0 (BE) 0..0F..F (LE) 385 __src_b a6, a6, a5 # hi-mask 0..0F..F (BE) F..F0..0 (LE) 386#ifdef UNALIGNED_USER_EXCEPTION 387 l32e a5, a4, -8 388#else 389 l32i a5, a4, 0 # load lower address word 390#endif 391 and a5, a5, a8 # mask 392 __sh a8, a3 # shift value 393 or a5, a5, a8 # or with original value 394#ifdef UNALIGNED_USER_EXCEPTION 395 s32e a5, a4, -8 396 l32e a8, a4, -4 397#else 398 s32i a5, a4, 0 # store 399 l32i a8, a4, 4 # same for upper address word 400#endif 401 __sl a5, a3 402 and a6, a8, a6 403 or a6, a6, a5 404#ifdef UNALIGNED_USER_EXCEPTION 405 s32e a6, a4, -4 406#else 407 s32i a6, a4, 4 408#endif 409 410.Lexit: 411#if XCHAL_HAVE_LOOPS 412 rsr a4, lend # check if we reached LEND 413 bne a7, a4, 1f 414 rsr a4, lcount # and LCOUNT != 0 415 beqz a4, 1f 416 addi a4, a4, -1 # decrement LCOUNT and set 417 rsr a7, lbeg # set PC to LBEGIN 418 wsr a4, lcount 419#endif 420 4211: wsr a7, epc1 # skip emulated instruction 422 423 /* Update icount if we're single-stepping in userspace. */ 424 rsr a4, icountlevel 425 beqz a4, 1f 426 bgeui a4, LOCKLEVEL + 1, 1f 427 rsr a4, icount 428 addi a4, a4, 1 429 wsr a4, icount 4301: 431 movi a4, 0 432 rsr a3, excsave1 433 s32i a4, a3, EXC_TABLE_FIXUP 434 435 /* Restore working register */ 436 437 l32i a8, a2, PT_AREG8 438 l32i a7, a2, PT_AREG7 439 l32i a6, a2, PT_AREG6 440 l32i a5, a2, PT_AREG5 441 l32i a4, a2, PT_AREG4 442 l32i a3, a2, PT_AREG3 443 444 /* restore SAR and return */ 445 446 wsr a0, sar 447 l32i a0, a2, PT_AREG0 448 l32i a2, a2, PT_AREG2 449 rfe 450 451ENDPROC(fast_unaligned) 452 453ENTRY(fast_unaligned_fixup) 454 455 l32i a2, a3, EXC_TABLE_DOUBLE_SAVE 456 wsr a3, excsave1 457 458 l32i a8, a2, PT_AREG8 459 l32i a7, a2, PT_AREG7 460 l32i a6, a2, PT_AREG6 461 l32i a5, a2, PT_AREG5 462 l32i a4, a2, PT_AREG4 463 l32i a0, a2, PT_AREG2 464 xsr a0, depc # restore depc and a0 465 wsr a0, sar 466 467 rsr a0, exccause 468 s32i a0, a2, PT_DEPC # mark as a regular exception 469 470 rsr a0, ps 471 bbsi.l a0, PS_UM_BIT, 1f # jump if user mode 472 473 rsr a0, exccause 474 addx4 a0, a0, a3 # find entry in table 475 l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler 476 l32i a3, a2, PT_AREG3 477 jx a0 4781: 479 rsr a0, exccause 480 addx4 a0, a0, a3 # find entry in table 481 l32i a0, a0, EXC_TABLE_FAST_USER # load handler 482 l32i a3, a2, PT_AREG3 483 jx a0 484 485ENDPROC(fast_unaligned_fixup) 486 487#endif /* XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION */