virt/kvm/arm/vgic-v3-emul.c

   1 /*
   2  * GICv3 distributor and redistributor emulation
   3  *
   4  * GICv3 emulation is currently only supported on a GICv3 host (because
   5  * we rely on the hardware's CPU interface virtualization support), but
   6  * supports both hardware with or without the optional GICv2 backwards
   7  * compatibility features.
   8  *
   9  * Limitations of the emulation:
  10  * (RAZ/WI: read as zero, write ignore, RAO/WI: read as one, write ignore)
  11  * - We do not support LPIs (yet). TYPER.LPIS is reported as 0 and is RAZ/WI.
  12  * - We do not support the message based interrupts (MBIs) triggered by
  13  *   writes to the GICD_{SET,CLR}SPI_* registers. TYPER.MBIS is reported as 0.
  14  * - We do not support the (optional) backwards compatibility feature.
  15  *   GICD_CTLR.ARE resets to 1 and is RAO/WI. If the _host_ GIC supports
  16  *   the compatiblity feature, you can use a GICv2 in the guest, though.
  17  * - We only support a single security state. GICD_CTLR.DS is 1 and is RAO/WI.
  18  * - Priorities are not emulated (same as the GICv2 emulation). Linux
  19  *   as a guest is fine with this, because it does not use priorities.
  20  * - We only support Group1 interrupts. Again Linux uses only those.
  21  *
  22  * Copyright (C) 2014 ARM Ltd.
  23  * Author: Andre Przywara <andre.przywara@arm.com>
  24  *
  25  * This program is free software; you can redistribute it and/or modify
  26  * it under the terms of the GNU General Public License version 2 as
  27  * published by the Free Software Foundation.
  28  *
  29  * This program is distributed in the hope that it will be useful,
  30  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  31  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  32  * GNU General Public License for more details.
  33  *
  34  * You should have received a copy of the GNU General Public License
  35  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  36  */
  37
  38 #include <linux/cpu.h>
  39 #include <linux/kvm.h>
  40 #include <linux/kvm_host.h>
  41 #include <linux/interrupt.h>
  42
  43 #include <linux/irqchip/arm-gic-v3.h>
  44 #include <kvm/arm_vgic.h>
  45
  46 #include <asm/kvm_emulate.h>
  47 #include <asm/kvm_arm.h>
  48 #include <asm/kvm_mmu.h>
  49
  50 #include "vgic.h"
  51
  52 static bool handle_mmio_rao_wi(struct kvm_vcpu *vcpu,
  53                                struct kvm_exit_mmio *mmio, phys_addr_t offset)
  54 {
  55         u32 reg = 0xffffffff;
  56
  57         vgic_reg_access(mmio, &reg, offset,
  58                         ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
  59
  60         return false;
  61 }
  62
  63 static bool handle_mmio_ctlr(struct kvm_vcpu *vcpu,
  64                              struct kvm_exit_mmio *mmio, phys_addr_t offset)
  65 {
  66         u32 reg = 0;
  67
  68         /*
  69          * Force ARE and DS to 1, the guest cannot change this.
  70          * For the time being we only support Group1 interrupts.
  71          */
  72         if (vcpu->kvm->arch.vgic.enabled)
  73                 reg = GICD_CTLR_ENABLE_SS_G1;
  74         reg |= GICD_CTLR_ARE_NS | GICD_CTLR_DS;
  75
  76         vgic_reg_access(mmio, &reg, offset,
  77                         ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
  78         if (mmio->is_write) {
  79                 if (reg & GICD_CTLR_ENABLE_SS_G0)
  80                         kvm_info("guest tried to enable unsupported Group0 interrupts\n");
  81                 vcpu->kvm->arch.vgic.enabled = !!(reg & GICD_CTLR_ENABLE_SS_G1);
  82                 vgic_update_state(vcpu->kvm);
  83                 return true;
  84         }
  85         return false;
  86 }
  87
  88 /*
  89  * As this implementation does not provide compatibility
  90  * with GICv2 (ARE==1), we report zero CPUs in bits [5..7].
  91  * Also LPIs and MBIs are not supported, so we set the respective bits to 0.
  92  * Also we report at most 2**10=1024 interrupt IDs (to match 1024 SPIs).
  93  */
  94 #define INTERRUPT_ID_BITS 10
  95 static bool handle_mmio_typer(struct kvm_vcpu *vcpu,
  96                               struct kvm_exit_mmio *mmio, phys_addr_t offset)
  97 {
  98         u32 reg;
  99
 100         reg = (min(vcpu->kvm->arch.vgic.nr_irqs, 1024) >> 5) - 1;
 101
 102         reg |= (INTERRUPT_ID_BITS - 1) << 19;
 103
 104         vgic_reg_access(mmio, &reg, offset,
 105                         ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
 106
 107         return false;
 108 }
 109
 110 static bool handle_mmio_iidr(struct kvm_vcpu *vcpu,
 111                              struct kvm_exit_mmio *mmio, phys_addr_t offset)
 112 {
 113         u32 reg;
 114
 115         reg = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
 116         vgic_reg_access(mmio, &reg, offset,
 117                         ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
 118
 119         return false;
 120 }
 121
 122 static bool handle_mmio_set_enable_reg_dist(struct kvm_vcpu *vcpu,
 123                                             struct kvm_exit_mmio *mmio,
 124                                             phys_addr_t offset)
 125 {
 126         if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
 127                 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
 128                                               vcpu->vcpu_id,
 129                                               ACCESS_WRITE_SETBIT);
 130
 131         vgic_reg_access(mmio, NULL, offset,
 132                         ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
 133         return false;
 134 }
 135
 136 static bool handle_mmio_clear_enable_reg_dist(struct kvm_vcpu *vcpu,
 137                                               struct kvm_exit_mmio *mmio,
 138                                               phys_addr_t offset)
 139 {
 140         if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
 141                 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
 142                                               vcpu->vcpu_id,
 143                                               ACCESS_WRITE_CLEARBIT);
 144
 145         vgic_reg_access(mmio, NULL, offset,
 146                         ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
 147         return false;
 148 }
 149
 150 static bool handle_mmio_set_pending_reg_dist(struct kvm_vcpu *vcpu,
 151                                              struct kvm_exit_mmio *mmio,
 152                                              phys_addr_t offset)
 153 {
 154         if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
 155                 return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset,
 156                                                    vcpu->vcpu_id);
 157
 158         vgic_reg_access(mmio, NULL, offset,
 159                         ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
 160         return false;
 161 }
 162
 163 static bool handle_mmio_clear_pending_reg_dist(struct kvm_vcpu *vcpu,
 164                                                struct kvm_exit_mmio *mmio,
 165                                                phys_addr_t offset)
 166 {
 167         if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
 168                 return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset,
 169                                                      vcpu->vcpu_id);
 170
 171         vgic_reg_access(mmio, NULL, offset,
 172                         ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
 173         return false;
 174 }
 175
 176 static bool handle_mmio_priority_reg_dist(struct kvm_vcpu *vcpu,
 177                                           struct kvm_exit_mmio *mmio,
 178                                           phys_addr_t offset)
 179 {
 180         u32 *reg;
 181
 182         if (unlikely(offset < VGIC_NR_PRIVATE_IRQS)) {
 183                 vgic_reg_access(mmio, NULL, offset,
 184                                 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
 185                 return false;
 186         }
 187
 188         reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
 189                                    vcpu->vcpu_id, offset);
 190         vgic_reg_access(mmio, reg, offset,
 191                 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
 192         return false;
 193 }
 194
 195 static bool handle_mmio_cfg_reg_dist(struct kvm_vcpu *vcpu,
 196                                      struct kvm_exit_mmio *mmio,
 197                                      phys_addr_t offset)
 198 {
 199         u32 *reg;
 200
 201         if (unlikely(offset < VGIC_NR_PRIVATE_IRQS / 4)) {
 202                 vgic_reg_access(mmio, NULL, offset,
 203                                 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
 204                 return false;
 205         }
 206
 207         reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
 208                                   vcpu->vcpu_id, offset >> 1);
 209
 210         return vgic_handle_cfg_reg(reg, mmio, offset);
 211 }
 212
 213 /*
 214  * We use a compressed version of the MPIDR (all 32 bits in one 32-bit word)
 215  * when we store the target MPIDR written by the guest.
 216  */
 217 static u32 compress_mpidr(unsigned long mpidr)
 218 {
 219         u32 ret;
 220
 221         ret = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 222         ret |= MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8;
 223         ret |= MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16;
 224         ret |= MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24;
 225
 226         return ret;
 227 }
 228
 229 static unsigned long uncompress_mpidr(u32 value)
 230 {
 231         unsigned long mpidr;
 232
 233         mpidr  = ((value >>  0) & 0xFF) << MPIDR_LEVEL_SHIFT(0);
 234         mpidr |= ((value >>  8) & 0xFF) << MPIDR_LEVEL_SHIFT(1);
 235         mpidr |= ((value >> 16) & 0xFF) << MPIDR_LEVEL_SHIFT(2);
 236         mpidr |= (u64)((value >> 24) & 0xFF) << MPIDR_LEVEL_SHIFT(3);
 237
 238         return mpidr;
 239 }
 240
 241 /*
 242  * Lookup the given MPIDR value to get the vcpu_id (if there is one)
 243  * and store that in the irq_spi_cpu[] array.
 244  * This limits the number of VCPUs to 255 for now, extending the data
 245  * type (or storing kvm_vcpu pointers) should lift the limit.
 246  * Store the original MPIDR value in an extra array to support read-as-written.
 247  * Unallocated MPIDRs are translated to a special value and caught
 248  * before any array accesses.
 249  */
 250 static bool handle_mmio_route_reg(struct kvm_vcpu *vcpu,
 251                                   struct kvm_exit_mmio *mmio,
 252                                   phys_addr_t offset)
 253 {
 254         struct kvm *kvm = vcpu->kvm;
 255         struct vgic_dist *dist = &kvm->arch.vgic;
 256         int spi;
 257         u32 reg;
 258         int vcpu_id;
 259         unsigned long *bmap, mpidr;
 260
 261         /*
 262          * The upper 32 bits of each 64 bit register are zero,
 263          * as we don't support Aff3.
 264          */
 265         if ((offset & 4)) {
 266                 vgic_reg_access(mmio, NULL, offset,
 267                                 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
 268                 return false;
 269         }
 270
 271         /* This region only covers SPIs, so no handling of private IRQs here. */
 272         spi = offset / 8;
 273
 274         /* get the stored MPIDR for this IRQ */
 275         mpidr = uncompress_mpidr(dist->irq_spi_mpidr[spi]);
 276         reg = mpidr;
 277
 278         vgic_reg_access(mmio, &reg, offset,
 279                         ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
 280
 281         if (!mmio->is_write)
 282                 return false;
 283
 284         /*
 285          * Now clear the currently assigned vCPU from the map, making room
 286          * for the new one to be written below
 287          */
 288         vcpu = kvm_mpidr_to_vcpu(kvm, mpidr);
 289         if (likely(vcpu)) {
 290                 vcpu_id = vcpu->vcpu_id;
 291                 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]);
 292                 __clear_bit(spi, bmap);
 293         }
 294
 295         dist->irq_spi_mpidr[spi] = compress_mpidr(reg);
 296         vcpu = kvm_mpidr_to_vcpu(kvm, reg & MPIDR_HWID_BITMASK);
 297
 298         /*
 299          * The spec says that non-existent MPIDR values should not be
 300          * forwarded to any existent (v)CPU, but should be able to become
 301          * pending anyway. We simply keep the irq_spi_target[] array empty, so
 302          * the interrupt will never be injected.
 303          * irq_spi_cpu[irq] gets a magic value in this case.
 304          */
 305         if (likely(vcpu)) {
 306                 vcpu_id = vcpu->vcpu_id;
 307                 dist->irq_spi_cpu[spi] = vcpu_id;
 308                 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]);
 309                 __set_bit(spi, bmap);
 310         } else {
 311                 dist->irq_spi_cpu[spi] = VCPU_NOT_ALLOCATED;
 312         }
 313
 314         vgic_update_state(kvm);
 315
 316         return true;
 317 }
 318
 319 /*
 320  * We should be careful about promising too much when a guest reads
 321  * this register. Don't claim to be like any hardware implementation,
 322  * but just report the GIC as version 3 - which is what a Linux guest
 323  * would check.
 324  */
 325 static bool handle_mmio_idregs(struct kvm_vcpu *vcpu,
 326                                struct kvm_exit_mmio *mmio,
 327                                phys_addr_t offset)
 328 {
 329         u32 reg = 0;
 330
 331         switch (offset + GICD_IDREGS) {
 332         case GICD_PIDR2:
 333                 reg = 0x3b;
 334                 break;
 335         }
 336
 337         vgic_reg_access(mmio, &reg, offset,
 338                         ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
 339
 340         return false;
 341 }
 342
 343 static const struct vgic_io_range vgic_v3_dist_ranges[] = {
 344         {
 345                 .base           = GICD_CTLR,
 346                 .len            = 0x04,
 347                 .bits_per_irq   = 0,
 348                 .handle_mmio    = handle_mmio_ctlr,
 349         },
 350         {
 351                 .base           = GICD_TYPER,
 352                 .len            = 0x04,
 353                 .bits_per_irq   = 0,
 354                 .handle_mmio    = handle_mmio_typer,
 355         },
 356         {
 357                 .base           = GICD_IIDR,
 358                 .len            = 0x04,
 359                 .bits_per_irq   = 0,
 360                 .handle_mmio    = handle_mmio_iidr,
 361         },
 362         {
 363                 /* this register is optional, it is RAZ/WI if not implemented */
 364                 .base           = GICD_STATUSR,
 365                 .len            = 0x04,
 366                 .bits_per_irq   = 0,
 367                 .handle_mmio    = handle_mmio_raz_wi,
 368         },
 369         {
 370                 /* this write only register is WI when TYPER.MBIS=0 */
 371                 .base           = GICD_SETSPI_NSR,
 372                 .len            = 0x04,
 373                 .bits_per_irq   = 0,
 374                 .handle_mmio    = handle_mmio_raz_wi,
 375         },
 376         {
 377                 /* this write only register is WI when TYPER.MBIS=0 */
 378                 .base           = GICD_CLRSPI_NSR,
 379                 .len            = 0x04,
 380                 .bits_per_irq   = 0,
 381                 .handle_mmio    = handle_mmio_raz_wi,
 382         },
 383         {
 384                 /* this is RAZ/WI when DS=1 */
 385                 .base           = GICD_SETSPI_SR,
 386                 .len            = 0x04,
 387                 .bits_per_irq   = 0,
 388                 .handle_mmio    = handle_mmio_raz_wi,
 389         },
 390         {
 391                 /* this is RAZ/WI when DS=1 */
 392                 .base           = GICD_CLRSPI_SR,
 393                 .len            = 0x04,
 394                 .bits_per_irq   = 0,
 395                 .handle_mmio    = handle_mmio_raz_wi,
 396         },
 397         {
 398                 .base           = GICD_IGROUPR,
 399                 .len            = 0x80,
 400                 .bits_per_irq   = 1,
 401                 .handle_mmio    = handle_mmio_rao_wi,
 402         },
 403         {
 404                 .base           = GICD_ISENABLER,
 405                 .len            = 0x80,
 406                 .bits_per_irq   = 1,
 407                 .handle_mmio    = handle_mmio_set_enable_reg_dist,
 408         },
 409         {
 410                 .base           = GICD_ICENABLER,
 411                 .len            = 0x80,
 412                 .bits_per_irq   = 1,
 413                 .handle_mmio    = handle_mmio_clear_enable_reg_dist,
 414         },
 415         {
 416                 .base           = GICD_ISPENDR,
 417                 .len            = 0x80,
 418                 .bits_per_irq   = 1,
 419                 .handle_mmio    = handle_mmio_set_pending_reg_dist,
 420         },
 421         {
 422                 .base           = GICD_ICPENDR,
 423                 .len            = 0x80,
 424                 .bits_per_irq   = 1,
 425                 .handle_mmio    = handle_mmio_clear_pending_reg_dist,
 426         },
 427         {
 428                 .base           = GICD_ISACTIVER,
 429                 .len            = 0x80,
 430                 .bits_per_irq   = 1,
 431                 .handle_mmio    = handle_mmio_raz_wi,
 432         },
 433         {
 434                 .base           = GICD_ICACTIVER,
 435                 .len            = 0x80,
 436                 .bits_per_irq   = 1,
 437                 .handle_mmio    = handle_mmio_raz_wi,
 438         },
 439         {
 440                 .base           = GICD_IPRIORITYR,
 441                 .len            = 0x400,
 442                 .bits_per_irq   = 8,
 443                 .handle_mmio    = handle_mmio_priority_reg_dist,
 444         },
 445         {
 446                 /* TARGETSRn is RES0 when ARE=1 */
 447                 .base           = GICD_ITARGETSR,
 448                 .len            = 0x400,
 449                 .bits_per_irq   = 8,
 450                 .handle_mmio    = handle_mmio_raz_wi,
 451         },
 452         {
 453                 .base           = GICD_ICFGR,
 454                 .len            = 0x100,
 455                 .bits_per_irq   = 2,
 456                 .handle_mmio    = handle_mmio_cfg_reg_dist,
 457         },
 458         {
 459                 /* this is RAZ/WI when DS=1 */
 460                 .base           = GICD_IGRPMODR,
 461                 .len            = 0x80,
 462                 .bits_per_irq   = 1,
 463                 .handle_mmio    = handle_mmio_raz_wi,
 464         },
 465         {
 466                 /* this is RAZ/WI when DS=1 */
 467                 .base           = GICD_NSACR,
 468                 .len            = 0x100,
 469                 .bits_per_irq   = 2,
 470                 .handle_mmio    = handle_mmio_raz_wi,
 471         },
 472         {
 473                 /* this is RAZ/WI when ARE=1 */
 474                 .base           = GICD_SGIR,
 475                 .len            = 0x04,
 476                 .handle_mmio    = handle_mmio_raz_wi,
 477         },
 478         {
 479                 /* this is RAZ/WI when ARE=1 */
 480                 .base           = GICD_CPENDSGIR,
 481                 .len            = 0x10,
 482                 .handle_mmio    = handle_mmio_raz_wi,
 483         },
 484         {
 485                 /* this is RAZ/WI when ARE=1 */
 486                 .base           = GICD_SPENDSGIR,
 487                 .len            = 0x10,
 488                 .handle_mmio    = handle_mmio_raz_wi,
 489         },
 490         {
 491                 .base           = GICD_IROUTER + 0x100,
 492                 .len            = 0x1ee0,
 493                 .bits_per_irq   = 64,
 494                 .handle_mmio    = handle_mmio_route_reg,
 495         },
 496         {
 497                 .base           = GICD_IDREGS,
 498                 .len            = 0x30,
 499                 .bits_per_irq   = 0,
 500                 .handle_mmio    = handle_mmio_idregs,
 501         },
 502         {},
 503 };
 504
 505 static bool handle_mmio_ctlr_redist(struct kvm_vcpu *vcpu,
 506                                     struct kvm_exit_mmio *mmio,
 507                                     phys_addr_t offset)
 508 {
 509         /* since we don't support LPIs, this register is zero for now */
 510         vgic_reg_access(mmio, NULL, offset,
 511                         ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
 512         return false;
 513 }
 514
 515 static bool handle_mmio_typer_redist(struct kvm_vcpu *vcpu,
 516                                      struct kvm_exit_mmio *mmio,
 517                                      phys_addr_t offset)
 518 {
 519         u32 reg;
 520         u64 mpidr;
 521         struct kvm_vcpu *redist_vcpu = mmio->private;
 522         int target_vcpu_id = redist_vcpu->vcpu_id;
 523
 524         /* the upper 32 bits contain the affinity value */
 525         if ((offset & ~3) == 4) {
 526                 mpidr = kvm_vcpu_get_mpidr_aff(redist_vcpu);
 527                 reg = compress_mpidr(mpidr);
 528
 529                 vgic_reg_access(mmio, &reg, offset,
 530                                 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
 531                 return false;
 532         }
 533
 534         reg = redist_vcpu->vcpu_id << 8;
 535         if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1)
 536                 reg |= GICR_TYPER_LAST;
 537         vgic_reg_access(mmio, &reg, offset,
 538                         ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
 539         return false;
 540 }
 541
 542 static bool handle_mmio_set_enable_reg_redist(struct kvm_vcpu *vcpu,
 543                                               struct kvm_exit_mmio *mmio,
 544                                               phys_addr_t offset)
 545 {
 546         struct kvm_vcpu *redist_vcpu = mmio->private;
 547
 548         return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
 549                                       redist_vcpu->vcpu_id,
 550                                       ACCESS_WRITE_SETBIT);
 551 }
 552
 553 static bool handle_mmio_clear_enable_reg_redist(struct kvm_vcpu *vcpu,
 554                                                 struct kvm_exit_mmio *mmio,
 555                                                 phys_addr_t offset)
 556 {
 557         struct kvm_vcpu *redist_vcpu = mmio->private;
 558
 559         return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
 560                                       redist_vcpu->vcpu_id,
 561                                       ACCESS_WRITE_CLEARBIT);
 562 }
 563
 564 static bool handle_mmio_set_pending_reg_redist(struct kvm_vcpu *vcpu,
 565                                                struct kvm_exit_mmio *mmio,
 566                                                phys_addr_t offset)
 567 {
 568         struct kvm_vcpu *redist_vcpu = mmio->private;
 569
 570         return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset,
 571                                            redist_vcpu->vcpu_id);
 572 }
 573
 574 static bool handle_mmio_clear_pending_reg_redist(struct kvm_vcpu *vcpu,
 575                                                  struct kvm_exit_mmio *mmio,
 576                                                  phys_addr_t offset)
 577 {
 578         struct kvm_vcpu *redist_vcpu = mmio->private;
 579
 580         return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset,
 581                                              redist_vcpu->vcpu_id);
 582 }
 583
 584 static bool handle_mmio_priority_reg_redist(struct kvm_vcpu *vcpu,
 585                                             struct kvm_exit_mmio *mmio,
 586                                             phys_addr_t offset)
 587 {
 588         struct kvm_vcpu *redist_vcpu = mmio->private;
 589         u32 *reg;
 590
 591         reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
 592                                    redist_vcpu->vcpu_id, offset);
 593         vgic_reg_access(mmio, reg, offset,
 594                         ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
 595         return false;
 596 }
 597
 598 static bool handle_mmio_cfg_reg_redist(struct kvm_vcpu *vcpu,
 599                                        struct kvm_exit_mmio *mmio,
 600                                        phys_addr_t offset)
 601 {
 602         struct kvm_vcpu *redist_vcpu = mmio->private;
 603
 604         u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
 605                                        redist_vcpu->vcpu_id, offset >> 1);
 606
 607         return vgic_handle_cfg_reg(reg, mmio, offset);
 608 }
 609
 610 #define SGI_base(x) ((x) + SZ_64K)
 611
 612 static const struct vgic_io_range vgic_redist_ranges[] = {
 613         {
 614                 .base           = GICR_CTLR,
 615                 .len            = 0x04,
 616                 .bits_per_irq   = 0,
 617                 .handle_mmio    = handle_mmio_ctlr_redist,
 618         },
 619         {
 620                 .base           = GICR_TYPER,
 621                 .len            = 0x08,
 622                 .bits_per_irq   = 0,
 623                 .handle_mmio    = handle_mmio_typer_redist,
 624         },
 625         {
 626                 .base           = GICR_IIDR,
 627                 .len            = 0x04,
 628                 .bits_per_irq   = 0,
 629                 .handle_mmio    = handle_mmio_iidr,
 630         },
 631         {
 632                 .base           = GICR_WAKER,
 633                 .len            = 0x04,
 634                 .bits_per_irq   = 0,
 635                 .handle_mmio    = handle_mmio_raz_wi,
 636         },
 637         {
 638                 .base           = GICR_IDREGS,
 639                 .len            = 0x30,
 640                 .bits_per_irq   = 0,
 641                 .handle_mmio    = handle_mmio_idregs,
 642         },
 643         {
 644                 .base           = SGI_base(GICR_IGROUPR0),
 645                 .len            = 0x04,
 646                 .bits_per_irq   = 1,
 647                 .handle_mmio    = handle_mmio_rao_wi,
 648         },
 649         {
 650                 .base           = SGI_base(GICR_ISENABLER0),
 651                 .len            = 0x04,
 652                 .bits_per_irq   = 1,
 653                 .handle_mmio    = handle_mmio_set_enable_reg_redist,
 654         },
 655         {
 656                 .base           = SGI_base(GICR_ICENABLER0),
 657                 .len            = 0x04,
 658                 .bits_per_irq   = 1,
 659                 .handle_mmio    = handle_mmio_clear_enable_reg_redist,
 660         },
 661         {
 662                 .base           = SGI_base(GICR_ISPENDR0),
 663                 .len            = 0x04,
 664                 .bits_per_irq   = 1,
 665                 .handle_mmio    = handle_mmio_set_pending_reg_redist,
 666         },
 667         {
 668                 .base           = SGI_base(GICR_ICPENDR0),
 669                 .len            = 0x04,
 670                 .bits_per_irq   = 1,
 671                 .handle_mmio    = handle_mmio_clear_pending_reg_redist,
 672         },
 673         {
 674                 .base           = SGI_base(GICR_ISACTIVER0),
 675                 .len            = 0x04,
 676                 .bits_per_irq   = 1,
 677                 .handle_mmio    = handle_mmio_raz_wi,
 678         },
 679         {
 680                 .base           = SGI_base(GICR_ICACTIVER0),
 681                 .len            = 0x04,
 682                 .bits_per_irq   = 1,
 683                 .handle_mmio    = handle_mmio_raz_wi,
 684         },
 685         {
 686                 .base           = SGI_base(GICR_IPRIORITYR0),
 687                 .len            = 0x20,
 688                 .bits_per_irq   = 8,
 689                 .handle_mmio    = handle_mmio_priority_reg_redist,
 690         },
 691         {
 692                 .base           = SGI_base(GICR_ICFGR0),
 693                 .len            = 0x08,
 694                 .bits_per_irq   = 2,
 695                 .handle_mmio    = handle_mmio_cfg_reg_redist,
 696         },
 697         {
 698                 .base           = SGI_base(GICR_IGRPMODR0),
 699                 .len            = 0x04,
 700                 .bits_per_irq   = 1,
 701                 .handle_mmio    = handle_mmio_raz_wi,
 702         },
 703         {
 704                 .base           = SGI_base(GICR_NSACR),
 705                 .len            = 0x04,
 706                 .handle_mmio    = handle_mmio_raz_wi,
 707         },
 708         {},
 709 };
 710
 711 /*
 712  * This function splits accesses between the distributor and the two
 713  * redistributor parts (private/SPI). As each redistributor is accessible
 714  * from any CPU, we have to determine the affected VCPU by taking the faulting
 715  * address into account. We then pass this VCPU to the handler function via
 716  * the private parameter.
 717  */
 718 #define SGI_BASE_OFFSET SZ_64K
 719 static bool vgic_v3_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
 720                                 struct kvm_exit_mmio *mmio)
 721 {
 722         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 723         unsigned long dbase = dist->vgic_dist_base;
 724         unsigned long rdbase = dist->vgic_redist_base;
 725         int nrcpus = atomic_read(&vcpu->kvm->online_vcpus);
 726         int vcpu_id;
 727
 728         if (is_in_range(mmio->phys_addr, mmio->len, dbase, GIC_V3_DIST_SIZE)) {
 729                 return vgic_handle_mmio_range(vcpu, run, mmio,
 730                                               vgic_v3_dist_ranges, dbase);
 731         }
 732
 733         if (!is_in_range(mmio->phys_addr, mmio->len, rdbase,
 734             GIC_V3_REDIST_SIZE * nrcpus))
 735                 return false;
 736
 737         vcpu_id = (mmio->phys_addr - rdbase) / GIC_V3_REDIST_SIZE;
 738         rdbase += (vcpu_id * GIC_V3_REDIST_SIZE);
 739         mmio->private = kvm_get_vcpu(vcpu->kvm, vcpu_id);
 740
 741         return vgic_handle_mmio_range(vcpu, run, mmio, vgic_redist_ranges,
 742                                       rdbase);
 743 }
 744
 745 static bool vgic_v3_queue_sgi(struct kvm_vcpu *vcpu, int irq)
 746 {
 747         if (vgic_queue_irq(vcpu, 0, irq)) {
 748                 vgic_dist_irq_clear_pending(vcpu, irq);
 749                 vgic_cpu_irq_clear(vcpu, irq);
 750                 return true;
 751         }
 752
 753         return false;
 754 }
 755
 756 static int vgic_v3_map_resources(struct kvm *kvm,
 757                                  const struct vgic_params *params)
 758 {
 759         int ret = 0;
 760         struct vgic_dist *dist = &kvm->arch.vgic;
 761
 762         if (!irqchip_in_kernel(kvm))
 763                 return 0;
 764
 765         mutex_lock(&kvm->lock);
 766
 767         if (vgic_ready(kvm))
 768                 goto out;
 769
 770         if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) ||
 771             IS_VGIC_ADDR_UNDEF(dist->vgic_redist_base)) {
 772                 kvm_err("Need to set vgic distributor addresses first\n");
 773                 ret = -ENXIO;
 774                 goto out;
 775         }
 776
 777         /*
 778          * For a VGICv3 we require the userland to explicitly initialize
 779          * the VGIC before we need to use it.
 780          */
 781         if (!vgic_initialized(kvm)) {
 782                 ret = -EBUSY;
 783                 goto out;
 784         }
 785
 786         kvm->arch.vgic.ready = true;
 787 out:
 788         if (ret)
 789                 kvm_vgic_destroy(kvm);
 790         mutex_unlock(&kvm->lock);
 791         return ret;
 792 }
 793
 794 static int vgic_v3_init_model(struct kvm *kvm)
 795 {
 796         int i;
 797         u32 mpidr;
 798         struct vgic_dist *dist = &kvm->arch.vgic;
 799         int nr_spis = dist->nr_irqs - VGIC_NR_PRIVATE_IRQS;
 800
 801         dist->irq_spi_mpidr = kcalloc(nr_spis, sizeof(dist->irq_spi_mpidr[0]),
 802                                       GFP_KERNEL);
 803
 804         if (!dist->irq_spi_mpidr)
 805                 return -ENOMEM;
 806
 807         /* Initialize the target VCPUs for each IRQ to VCPU 0 */
 808         mpidr = compress_mpidr(kvm_vcpu_get_mpidr_aff(kvm_get_vcpu(kvm, 0)));
 809         for (i = VGIC_NR_PRIVATE_IRQS; i < dist->nr_irqs; i++) {
 810                 dist->irq_spi_cpu[i - VGIC_NR_PRIVATE_IRQS] = 0;
 811                 dist->irq_spi_mpidr[i - VGIC_NR_PRIVATE_IRQS] = mpidr;
 812                 vgic_bitmap_set_irq_val(dist->irq_spi_target, 0, i, 1);
 813         }
 814
 815         return 0;
 816 }
 817
 818 /* GICv3 does not keep track of SGI sources anymore. */
 819 static void vgic_v3_add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
 820 {
 821 }
 822
 823 void vgic_v3_init_emulation(struct kvm *kvm)
 824 {
 825         struct vgic_dist *dist = &kvm->arch.vgic;
 826
 827         dist->vm_ops.handle_mmio = vgic_v3_handle_mmio;
 828         dist->vm_ops.queue_sgi = vgic_v3_queue_sgi;
 829         dist->vm_ops.add_sgi_source = vgic_v3_add_sgi_source;
 830         dist->vm_ops.init_model = vgic_v3_init_model;
 831         dist->vm_ops.map_resources = vgic_v3_map_resources;
 832
 833         kvm->arch.max_vcpus = KVM_MAX_VCPUS;
 834 }
 835
 836 /*
 837  * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI
 838  * generation register ICC_SGI1R_EL1) with a given VCPU.
 839  * If the VCPU's MPIDR matches, return the level0 affinity, otherwise
 840  * return -1.
 841  */
 842 static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu)
 843 {
 844         unsigned long affinity;
 845         int level0;
 846
 847         /*
 848          * Split the current VCPU's MPIDR into affinity level 0 and the
 849          * rest as this is what we have to compare against.
 850          */
 851         affinity = kvm_vcpu_get_mpidr_aff(vcpu);
 852         level0 = MPIDR_AFFINITY_LEVEL(affinity, 0);
 853         affinity &= ~MPIDR_LEVEL_MASK;
 854
 855         /* bail out if the upper three levels don't match */
 856         if (sgi_aff != affinity)
 857                 return -1;
 858
 859         /* Is this VCPU's bit set in the mask ? */
 860         if (!(sgi_cpu_mask & BIT(level0)))
 861                 return -1;
 862
 863         return level0;
 864 }
 865
 866 #define SGI_AFFINITY_LEVEL(reg, level) \
 867         ((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \
 868         >> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level))
 869
 870 /**
 871  * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs
 872  * @vcpu: The VCPU requesting a SGI
 873  * @reg: The value written into the ICC_SGI1R_EL1 register by that VCPU
 874  *
 875  * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register.
 876  * This will trap in sys_regs.c and call this function.
 877  * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the
 878  * target processors as well as a bitmask of 16 Aff0 CPUs.
 879  * If the interrupt routing mode bit is not set, we iterate over all VCPUs to
 880  * check for matching ones. If this bit is set, we signal all, but not the
 881  * calling VCPU.
 882  */
 883 void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg)
 884 {
 885         struct kvm *kvm = vcpu->kvm;
 886         struct kvm_vcpu *c_vcpu;
 887         struct vgic_dist *dist = &kvm->arch.vgic;
 888         u16 target_cpus;
 889         u64 mpidr;
 890         int sgi, c;
 891         int vcpu_id = vcpu->vcpu_id;
 892         bool broadcast;
 893         int updated = 0;
 894
 895         sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT;
 896         broadcast = reg & BIT(ICC_SGI1R_IRQ_ROUTING_MODE_BIT);
 897         target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT;
 898         mpidr = SGI_AFFINITY_LEVEL(reg, 3);
 899         mpidr |= SGI_AFFINITY_LEVEL(reg, 2);
 900         mpidr |= SGI_AFFINITY_LEVEL(reg, 1);
 901
 902         /*
 903          * We take the dist lock here, because we come from the sysregs
 904          * code path and not from the MMIO one (which already takes the lock).
 905          */
 906         spin_lock(&dist->lock);
 907
 908         /*
 909          * We iterate over all VCPUs to find the MPIDRs matching the request.
 910          * If we have handled one CPU, we clear it's bit to detect early
 911          * if we are already finished. This avoids iterating through all
 912          * VCPUs when most of the times we just signal a single VCPU.
 913          */
 914         kvm_for_each_vcpu(c, c_vcpu, kvm) {
 915
 916                 /* Exit early if we have dealt with all requested CPUs */
 917                 if (!broadcast && target_cpus == 0)
 918                         break;
 919
 920                  /* Don't signal the calling VCPU */
 921                 if (broadcast && c == vcpu_id)
 922                         continue;
 923
 924                 if (!broadcast) {
 925                         int level0;
 926
 927                         level0 = match_mpidr(mpidr, target_cpus, c_vcpu);
 928                         if (level0 == -1)
 929                                 continue;
 930
 931                         /* remove this matching VCPU from the mask */
 932                         target_cpus &= ~BIT(level0);
 933                 }
 934
 935                 /* Flag the SGI as pending */
 936                 vgic_dist_irq_set_pending(c_vcpu, sgi);
 937                 updated = 1;
 938                 kvm_debug("SGI%d from CPU%d to CPU%d\n", sgi, vcpu_id, c);
 939         }
 940         if (updated)
 941                 vgic_update_state(vcpu->kvm);
 942         spin_unlock(&dist->lock);
 943         if (updated)
 944                 vgic_kick_vcpus(vcpu->kvm);
 945 }
 946
 947 static int vgic_v3_create(struct kvm_device *dev, u32 type)
 948 {
 949         return kvm_vgic_create(dev->kvm, type);
 950 }
 951
 952 static void vgic_v3_destroy(struct kvm_device *dev)
 953 {
 954         kfree(dev);
 955 }
 956
 957 static int vgic_v3_set_attr(struct kvm_device *dev,
 958                             struct kvm_device_attr *attr)
 959 {
 960         int ret;
 961
 962         ret = vgic_set_common_attr(dev, attr);
 963         if (ret != -ENXIO)
 964                 return ret;
 965
 966         switch (attr->group) {
 967         case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
 968         case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
 969                 return -ENXIO;
 970         }
 971
 972         return -ENXIO;
 973 }
 974
 975 static int vgic_v3_get_attr(struct kvm_device *dev,
 976                             struct kvm_device_attr *attr)
 977 {
 978         int ret;
 979
 980         ret = vgic_get_common_attr(dev, attr);
 981         if (ret != -ENXIO)
 982                 return ret;
 983
 984         switch (attr->group) {
 985         case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
 986         case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
 987                 return -ENXIO;
 988         }
 989
 990         return -ENXIO;
 991 }
 992
 993 static int vgic_v3_has_attr(struct kvm_device *dev,
 994                             struct kvm_device_attr *attr)
 995 {
 996         switch (attr->group) {
 997         case KVM_DEV_ARM_VGIC_GRP_ADDR:
 998                 switch (attr->attr) {
 999                 case KVM_VGIC_V2_ADDR_TYPE_DIST:
1000                 case KVM_VGIC_V2_ADDR_TYPE_CPU:
1001                         return -ENXIO;
1002                 case KVM_VGIC_V3_ADDR_TYPE_DIST:
1003                 case KVM_VGIC_V3_ADDR_TYPE_REDIST:
1004                         return 0;
1005                 }
1006                 break;
1007         case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
1008         case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
1009                 return -ENXIO;
1010         case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
1011                 return 0;
1012         case KVM_DEV_ARM_VGIC_GRP_CTRL:
1013                 switch (attr->attr) {
1014                 case KVM_DEV_ARM_VGIC_CTRL_INIT:
1015                         return 0;
1016                 }
1017         }
1018         return -ENXIO;
1019 }
1020
1021 struct kvm_device_ops kvm_arm_vgic_v3_ops = {
1022         .name = "kvm-arm-vgic-v3",
1023         .create = vgic_v3_create,
1024         .destroy = vgic_v3_destroy,
1025         .set_attr = vgic_v3_set_attr,
1026         .get_attr = vgic_v3_get_attr,
1027         .has_attr = vgic_v3_has_attr,
1028 };