smmu-common.c (17418B)
1/* 2 * Copyright (C) 2014-2016 Broadcom Corporation 3 * Copyright (c) 2017 Red Hat, Inc. 4 * Written by Prem Mallappa, Eric Auger 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * Author: Prem Mallappa <pmallapp@broadcom.com> 16 * 17 */ 18 19#include "qemu/osdep.h" 20#include "trace.h" 21#include "exec/target_page.h" 22#include "hw/core/cpu.h" 23#include "hw/qdev-properties.h" 24#include "qapi/error.h" 25#include "qemu/jhash.h" 26#include "qemu/module.h" 27 28#include "qemu/error-report.h" 29#include "hw/arm/smmu-common.h" 30#include "smmu-internal.h" 31 32/* IOTLB Management */ 33 34static guint smmu_iotlb_key_hash(gconstpointer v) 35{ 36 SMMUIOTLBKey *key = (SMMUIOTLBKey *)v; 37 uint32_t a, b, c; 38 39 /* Jenkins hash */ 40 a = b = c = JHASH_INITVAL + sizeof(*key); 41 a += key->asid + key->level + key->tg; 42 b += extract64(key->iova, 0, 32); 43 c += extract64(key->iova, 32, 32); 44 45 __jhash_mix(a, b, c); 46 __jhash_final(a, b, c); 47 48 return c; 49} 50 51static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) 52{ 53 SMMUIOTLBKey *k1 = (SMMUIOTLBKey *)v1, *k2 = (SMMUIOTLBKey *)v2; 54 55 return (k1->asid == k2->asid) && (k1->iova == k2->iova) && 56 (k1->level == k2->level) && (k1->tg == k2->tg); 57} 58 59SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint64_t iova, 60 uint8_t tg, uint8_t level) 61{ 62 SMMUIOTLBKey key = {.asid = asid, .iova = iova, .tg = tg, .level = level}; 63 64 return key; 65} 66 67SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg, 68 SMMUTransTableInfo *tt, hwaddr iova) 69{ 70 uint8_t tg = (tt->granule_sz - 10) / 2; 71 uint8_t inputsize = 64 - tt->tsz; 72 uint8_t stride = tt->granule_sz - 3; 73 uint8_t level = 4 - (inputsize - 4) / stride; 74 SMMUTLBEntry *entry = NULL; 75 76 while (level <= 3) { 77 uint64_t subpage_size = 1ULL << level_shift(level, tt->granule_sz); 78 uint64_t mask = subpage_size - 1; 79 SMMUIOTLBKey key; 80 81 key = smmu_get_iotlb_key(cfg->asid, iova & ~mask, tg, level); 82 entry = g_hash_table_lookup(bs->iotlb, &key); 83 if (entry) { 84 break; 85 } 86 level++; 87 } 88 89 if (entry) { 90 cfg->iotlb_hits++; 91 trace_smmu_iotlb_lookup_hit(cfg->asid, iova, 92 cfg->iotlb_hits, cfg->iotlb_misses, 93 100 * cfg->iotlb_hits / 94 (cfg->iotlb_hits + cfg->iotlb_misses)); 95 } else { 96 cfg->iotlb_misses++; 97 trace_smmu_iotlb_lookup_miss(cfg->asid, iova, 98 cfg->iotlb_hits, cfg->iotlb_misses, 99 100 * cfg->iotlb_hits / 100 (cfg->iotlb_hits + cfg->iotlb_misses)); 101 } 102 return entry; 103} 104 105void smmu_iotlb_insert(SMMUState *bs, SMMUTransCfg *cfg, SMMUTLBEntry *new) 106{ 107 SMMUIOTLBKey *key = g_new0(SMMUIOTLBKey, 1); 108 uint8_t tg = (new->granule - 10) / 2; 109 110 if (g_hash_table_size(bs->iotlb) >= SMMU_IOTLB_MAX_SIZE) { 111 smmu_iotlb_inv_all(bs); 112 } 113 114 *key = smmu_get_iotlb_key(cfg->asid, new->entry.iova, tg, new->level); 115 trace_smmu_iotlb_insert(cfg->asid, new->entry.iova, tg, new->level); 116 g_hash_table_insert(bs->iotlb, key, new); 117} 118 119inline void smmu_iotlb_inv_all(SMMUState *s) 120{ 121 trace_smmu_iotlb_inv_all(); 122 g_hash_table_remove_all(s->iotlb); 123} 124 125static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value, 126 gpointer user_data) 127{ 128 uint16_t asid = *(uint16_t *)user_data; 129 SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; 130 131 return SMMU_IOTLB_ASID(*iotlb_key) == asid; 132} 133 134static gboolean smmu_hash_remove_by_asid_iova(gpointer key, gpointer value, 135 gpointer user_data) 136{ 137 SMMUTLBEntry *iter = (SMMUTLBEntry *)value; 138 IOMMUTLBEntry *entry = &iter->entry; 139 SMMUIOTLBPageInvInfo *info = (SMMUIOTLBPageInvInfo *)user_data; 140 SMMUIOTLBKey iotlb_key = *(SMMUIOTLBKey *)key; 141 142 if (info->asid >= 0 && info->asid != SMMU_IOTLB_ASID(iotlb_key)) { 143 return false; 144 } 145 return ((info->iova & ~entry->addr_mask) == entry->iova) || 146 ((entry->iova & ~info->mask) == info->iova); 147} 148 149inline void 150smmu_iotlb_inv_iova(SMMUState *s, int asid, dma_addr_t iova, 151 uint8_t tg, uint64_t num_pages, uint8_t ttl) 152{ 153 /* if tg is not set we use 4KB range invalidation */ 154 uint8_t granule = tg ? tg * 2 + 10 : 12; 155 156 if (ttl && (num_pages == 1) && (asid >= 0)) { 157 SMMUIOTLBKey key = smmu_get_iotlb_key(asid, iova, tg, ttl); 158 159 if (g_hash_table_remove(s->iotlb, &key)) { 160 return; 161 } 162 /* 163 * if the entry is not found, let's see if it does not 164 * belong to a larger IOTLB entry 165 */ 166 } 167 168 SMMUIOTLBPageInvInfo info = { 169 .asid = asid, .iova = iova, 170 .mask = (num_pages * 1 << granule) - 1}; 171 172 g_hash_table_foreach_remove(s->iotlb, 173 smmu_hash_remove_by_asid_iova, 174 &info); 175} 176 177inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid) 178{ 179 trace_smmu_iotlb_inv_asid(asid); 180 g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid, &asid); 181} 182 183/* VMSAv8-64 Translation */ 184 185/** 186 * get_pte - Get the content of a page table entry located at 187 * @base_addr[@index] 188 */ 189static int get_pte(dma_addr_t baseaddr, uint32_t index, uint64_t *pte, 190 SMMUPTWEventInfo *info) 191{ 192 int ret; 193 dma_addr_t addr = baseaddr + index * sizeof(*pte); 194 195 /* TODO: guarantee 64-bit single-copy atomicity */ 196 ret = dma_memory_read(&address_space_memory, addr, pte, sizeof(*pte)); 197 198 if (ret != MEMTX_OK) { 199 info->type = SMMU_PTW_ERR_WALK_EABT; 200 info->addr = addr; 201 return -EINVAL; 202 } 203 trace_smmu_get_pte(baseaddr, index, addr, *pte); 204 return 0; 205} 206 207/* VMSAv8-64 Translation Table Format Descriptor Decoding */ 208 209/** 210 * get_page_pte_address - returns the L3 descriptor output address, 211 * ie. the page frame 212 * ARM ARM spec: Figure D4-17 VMSAv8-64 level 3 descriptor format 213 */ 214static inline hwaddr get_page_pte_address(uint64_t pte, int granule_sz) 215{ 216 return PTE_ADDRESS(pte, granule_sz); 217} 218 219/** 220 * get_table_pte_address - return table descriptor output address, 221 * ie. address of next level table 222 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats 223 */ 224static inline hwaddr get_table_pte_address(uint64_t pte, int granule_sz) 225{ 226 return PTE_ADDRESS(pte, granule_sz); 227} 228 229/** 230 * get_block_pte_address - return block descriptor output address and block size 231 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats 232 */ 233static inline hwaddr get_block_pte_address(uint64_t pte, int level, 234 int granule_sz, uint64_t *bsz) 235{ 236 int n = level_shift(level, granule_sz); 237 238 *bsz = 1ULL << n; 239 return PTE_ADDRESS(pte, n); 240} 241 242SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) 243{ 244 bool tbi = extract64(iova, 55, 1) ? TBI1(cfg->tbi) : TBI0(cfg->tbi); 245 uint8_t tbi_byte = tbi * 8; 246 247 if (cfg->tt[0].tsz && 248 !extract64(iova, 64 - cfg->tt[0].tsz, cfg->tt[0].tsz - tbi_byte)) { 249 /* there is a ttbr0 region and we are in it (high bits all zero) */ 250 return &cfg->tt[0]; 251 } else if (cfg->tt[1].tsz && 252 !extract64(iova, 64 - cfg->tt[1].tsz, cfg->tt[1].tsz - tbi_byte)) { 253 /* there is a ttbr1 region and we are in it (high bits all one) */ 254 return &cfg->tt[1]; 255 } else if (!cfg->tt[0].tsz) { 256 /* ttbr0 region is "everything not in the ttbr1 region" */ 257 return &cfg->tt[0]; 258 } else if (!cfg->tt[1].tsz) { 259 /* ttbr1 region is "everything not in the ttbr0 region" */ 260 return &cfg->tt[1]; 261 } 262 /* in the gap between the two regions, this is a Translation fault */ 263 return NULL; 264} 265 266/** 267 * smmu_ptw_64 - VMSAv8-64 Walk of the page tables for a given IOVA 268 * @cfg: translation config 269 * @iova: iova to translate 270 * @perm: access type 271 * @tlbe: SMMUTLBEntry (out) 272 * @info: handle to an error info 273 * 274 * Return 0 on success, < 0 on error. In case of error, @info is filled 275 * and tlbe->perm is set to IOMMU_NONE. 276 * Upon success, @tlbe is filled with translated_addr and entry 277 * permission rights. 278 */ 279static int smmu_ptw_64(SMMUTransCfg *cfg, 280 dma_addr_t iova, IOMMUAccessFlags perm, 281 SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) 282{ 283 dma_addr_t baseaddr, indexmask; 284 int stage = cfg->stage; 285 SMMUTransTableInfo *tt = select_tt(cfg, iova); 286 uint8_t level, granule_sz, inputsize, stride; 287 288 if (!tt || tt->disabled) { 289 info->type = SMMU_PTW_ERR_TRANSLATION; 290 goto error; 291 } 292 293 granule_sz = tt->granule_sz; 294 stride = granule_sz - 3; 295 inputsize = 64 - tt->tsz; 296 level = 4 - (inputsize - 4) / stride; 297 indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1; 298 baseaddr = extract64(tt->ttb, 0, 48); 299 baseaddr &= ~indexmask; 300 301 while (level <= 3) { 302 uint64_t subpage_size = 1ULL << level_shift(level, granule_sz); 303 uint64_t mask = subpage_size - 1; 304 uint32_t offset = iova_level_offset(iova, inputsize, level, granule_sz); 305 uint64_t pte, gpa; 306 dma_addr_t pte_addr = baseaddr + offset * sizeof(pte); 307 uint8_t ap; 308 309 if (get_pte(baseaddr, offset, &pte, info)) { 310 goto error; 311 } 312 trace_smmu_ptw_level(level, iova, subpage_size, 313 baseaddr, offset, pte); 314 315 if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) { 316 trace_smmu_ptw_invalid_pte(stage, level, baseaddr, 317 pte_addr, offset, pte); 318 break; 319 } 320 321 if (is_table_pte(pte, level)) { 322 ap = PTE_APTABLE(pte); 323 324 if (is_permission_fault(ap, perm) && !tt->had) { 325 info->type = SMMU_PTW_ERR_PERMISSION; 326 goto error; 327 } 328 baseaddr = get_table_pte_address(pte, granule_sz); 329 level++; 330 continue; 331 } else if (is_page_pte(pte, level)) { 332 gpa = get_page_pte_address(pte, granule_sz); 333 trace_smmu_ptw_page_pte(stage, level, iova, 334 baseaddr, pte_addr, pte, gpa); 335 } else { 336 uint64_t block_size; 337 338 gpa = get_block_pte_address(pte, level, granule_sz, 339 &block_size); 340 trace_smmu_ptw_block_pte(stage, level, baseaddr, 341 pte_addr, pte, iova, gpa, 342 block_size >> 20); 343 } 344 ap = PTE_AP(pte); 345 if (is_permission_fault(ap, perm)) { 346 info->type = SMMU_PTW_ERR_PERMISSION; 347 goto error; 348 } 349 350 tlbe->entry.translated_addr = gpa; 351 tlbe->entry.iova = iova & ~mask; 352 tlbe->entry.addr_mask = mask; 353 tlbe->entry.perm = PTE_AP_TO_PERM(ap); 354 tlbe->level = level; 355 tlbe->granule = granule_sz; 356 return 0; 357 } 358 info->type = SMMU_PTW_ERR_TRANSLATION; 359 360error: 361 tlbe->entry.perm = IOMMU_NONE; 362 return -EINVAL; 363} 364 365/** 366 * smmu_ptw - Walk the page tables for an IOVA, according to @cfg 367 * 368 * @cfg: translation configuration 369 * @iova: iova to translate 370 * @perm: tentative access type 371 * @tlbe: returned entry 372 * @info: ptw event handle 373 * 374 * return 0 on success 375 */ 376inline int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, 377 SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) 378{ 379 if (!cfg->aa64) { 380 /* 381 * This code path is not entered as we check this while decoding 382 * the configuration data in the derived SMMU model. 383 */ 384 g_assert_not_reached(); 385 } 386 387 return smmu_ptw_64(cfg, iova, perm, tlbe, info); 388} 389 390/** 391 * The bus number is used for lookup when SID based invalidation occurs. 392 * In that case we lazily populate the SMMUPciBus array from the bus hash 393 * table. At the time the SMMUPciBus is created (smmu_find_add_as), the bus 394 * numbers may not be always initialized yet. 395 */ 396SMMUPciBus *smmu_find_smmu_pcibus(SMMUState *s, uint8_t bus_num) 397{ 398 SMMUPciBus *smmu_pci_bus = s->smmu_pcibus_by_bus_num[bus_num]; 399 GHashTableIter iter; 400 401 if (smmu_pci_bus) { 402 return smmu_pci_bus; 403 } 404 405 g_hash_table_iter_init(&iter, s->smmu_pcibus_by_busptr); 406 while (g_hash_table_iter_next(&iter, NULL, (void **)&smmu_pci_bus)) { 407 if (pci_bus_num(smmu_pci_bus->bus) == bus_num) { 408 s->smmu_pcibus_by_bus_num[bus_num] = smmu_pci_bus; 409 return smmu_pci_bus; 410 } 411 } 412 413 return NULL; 414} 415 416static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn) 417{ 418 SMMUState *s = opaque; 419 SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_pcibus_by_busptr, bus); 420 SMMUDevice *sdev; 421 static unsigned int index; 422 423 if (!sbus) { 424 sbus = g_malloc0(sizeof(SMMUPciBus) + 425 sizeof(SMMUDevice *) * SMMU_PCI_DEVFN_MAX); 426 sbus->bus = bus; 427 g_hash_table_insert(s->smmu_pcibus_by_busptr, bus, sbus); 428 } 429 430 sdev = sbus->pbdev[devfn]; 431 if (!sdev) { 432 char *name = g_strdup_printf("%s-%d-%d", s->mrtypename, devfn, index++); 433 434 sdev = sbus->pbdev[devfn] = g_new0(SMMUDevice, 1); 435 436 sdev->smmu = s; 437 sdev->bus = bus; 438 sdev->devfn = devfn; 439 440 memory_region_init_iommu(&sdev->iommu, sizeof(sdev->iommu), 441 s->mrtypename, 442 OBJECT(s), name, 1ULL << SMMU_MAX_VA_BITS); 443 address_space_init(&sdev->as, 444 MEMORY_REGION(&sdev->iommu), name); 445 trace_smmu_add_mr(name); 446 g_free(name); 447 } 448 449 return &sdev->as; 450} 451 452IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid) 453{ 454 uint8_t bus_n, devfn; 455 SMMUPciBus *smmu_bus; 456 SMMUDevice *smmu; 457 458 bus_n = PCI_BUS_NUM(sid); 459 smmu_bus = smmu_find_smmu_pcibus(s, bus_n); 460 if (smmu_bus) { 461 devfn = SMMU_PCI_DEVFN(sid); 462 smmu = smmu_bus->pbdev[devfn]; 463 if (smmu) { 464 return &smmu->iommu; 465 } 466 } 467 return NULL; 468} 469 470/* Unmap the whole notifier's range */ 471static void smmu_unmap_notifier_range(IOMMUNotifier *n) 472{ 473 IOMMUTLBEvent event; 474 475 event.type = IOMMU_NOTIFIER_UNMAP; 476 event.entry.target_as = &address_space_memory; 477 event.entry.iova = n->start; 478 event.entry.perm = IOMMU_NONE; 479 event.entry.addr_mask = n->end - n->start; 480 481 memory_region_notify_iommu_one(n, &event); 482} 483 484/* Unmap all notifiers attached to @mr */ 485inline void smmu_inv_notifiers_mr(IOMMUMemoryRegion *mr) 486{ 487 IOMMUNotifier *n; 488 489 trace_smmu_inv_notifiers_mr(mr->parent_obj.name); 490 IOMMU_NOTIFIER_FOREACH(n, mr) { 491 smmu_unmap_notifier_range(n); 492 } 493} 494 495/* Unmap all notifiers of all mr's */ 496void smmu_inv_notifiers_all(SMMUState *s) 497{ 498 SMMUDevice *sdev; 499 500 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { 501 smmu_inv_notifiers_mr(&sdev->iommu); 502 } 503} 504 505static void smmu_base_realize(DeviceState *dev, Error **errp) 506{ 507 SMMUState *s = ARM_SMMU(dev); 508 SMMUBaseClass *sbc = ARM_SMMU_GET_CLASS(dev); 509 Error *local_err = NULL; 510 511 sbc->parent_realize(dev, &local_err); 512 if (local_err) { 513 error_propagate(errp, local_err); 514 return; 515 } 516 s->configs = g_hash_table_new_full(NULL, NULL, NULL, g_free); 517 s->iotlb = g_hash_table_new_full(smmu_iotlb_key_hash, smmu_iotlb_key_equal, 518 g_free, g_free); 519 s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL); 520 521 if (s->primary_bus) { 522 pci_setup_iommu(s->primary_bus, smmu_find_add_as, s); 523 } else { 524 error_setg(errp, "SMMU is not attached to any PCI bus!"); 525 } 526} 527 528static void smmu_base_reset(DeviceState *dev) 529{ 530 SMMUState *s = ARM_SMMU(dev); 531 532 g_hash_table_remove_all(s->configs); 533 g_hash_table_remove_all(s->iotlb); 534} 535 536static Property smmu_dev_properties[] = { 537 DEFINE_PROP_UINT8("bus_num", SMMUState, bus_num, 0), 538 DEFINE_PROP_LINK("primary-bus", SMMUState, primary_bus, "PCI", PCIBus *), 539 DEFINE_PROP_END_OF_LIST(), 540}; 541 542static void smmu_base_class_init(ObjectClass *klass, void *data) 543{ 544 DeviceClass *dc = DEVICE_CLASS(klass); 545 SMMUBaseClass *sbc = ARM_SMMU_CLASS(klass); 546 547 device_class_set_props(dc, smmu_dev_properties); 548 device_class_set_parent_realize(dc, smmu_base_realize, 549 &sbc->parent_realize); 550 dc->reset = smmu_base_reset; 551} 552 553static const TypeInfo smmu_base_info = { 554 .name = TYPE_ARM_SMMU, 555 .parent = TYPE_SYS_BUS_DEVICE, 556 .instance_size = sizeof(SMMUState), 557 .class_data = NULL, 558 .class_size = sizeof(SMMUBaseClass), 559 .class_init = smmu_base_class_init, 560 .abstract = true, 561}; 562 563static void smmu_base_register_types(void) 564{ 565 type_register_static(&smmu_base_info); 566} 567 568type_init(smmu_base_register_types) 569