virt/kvm/arm/vgic.c

   1 /*
   2  * Copyright (C) 2012 ARM Ltd.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_host.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/io.h>
  24 #include <linux/of.h>
  25 #include <linux/of_address.h>
  26 #include <linux/of_irq.h>
  27 #include <linux/uaccess.h>
  28
  29 #include <linux/irqchip/arm-gic.h>
  30
  31 #include <asm/kvm_emulate.h>
  32 #include <asm/kvm_arm.h>
  33 #include <asm/kvm_mmu.h>
  34 #include <trace/events/kvm.h>
  35 #include <asm/kvm.h>
  36 #include <kvm/iodev.h>
  37
  38 /*
  39  * How the whole thing works (courtesy of Christoffer Dall):
  40  *
  41  * - At any time, the dist->irq_pending_on_cpu is the oracle that knows if
  42  *   something is pending on the CPU interface.
  43  * - Interrupts that are pending on the distributor are stored on the
  44  *   vgic.irq_pending vgic bitmap (this bitmap is updated by both user land
  45  *   ioctls and guest mmio ops, and other in-kernel peripherals such as the
  46  *   arch. timers).
  47  * - Every time the bitmap changes, the irq_pending_on_cpu oracle is
  48  *   recalculated
  49  * - To calculate the oracle, we need info for each cpu from
  50  *   compute_pending_for_cpu, which considers:
  51  *   - PPI: dist->irq_pending & dist->irq_enable
  52  *   - SPI: dist->irq_pending & dist->irq_enable & dist->irq_spi_target
  53  *   - irq_spi_target is a 'formatted' version of the GICD_ITARGETSRn
  54  *     registers, stored on each vcpu. We only keep one bit of
  55  *     information per interrupt, making sure that only one vcpu can
  56  *     accept the interrupt.
  57  * - If any of the above state changes, we must recalculate the oracle.
  58  * - The same is true when injecting an interrupt, except that we only
  59  *   consider a single interrupt at a time. The irq_spi_cpu array
  60  *   contains the target CPU for each SPI.
  61  *
  62  * The handling of level interrupts adds some extra complexity. We
  63  * need to track when the interrupt has been EOIed, so we can sample
  64  * the 'line' again. This is achieved as such:
  65  *
  66  * - When a level interrupt is moved onto a vcpu, the corresponding
  67  *   bit in irq_queued is set. As long as this bit is set, the line
  68  *   will be ignored for further interrupts. The interrupt is injected
  69  *   into the vcpu with the GICH_LR_EOI bit set (generate a
  70  *   maintenance interrupt on EOI).
  71  * - When the interrupt is EOIed, the maintenance interrupt fires,
  72  *   and clears the corresponding bit in irq_queued. This allows the
  73  *   interrupt line to be sampled again.
  74  * - Note that level-triggered interrupts can also be set to pending from
  75  *   writes to GICD_ISPENDRn and lowering the external input line does not
  76  *   cause the interrupt to become inactive in such a situation.
  77  *   Conversely, writes to GICD_ICPENDRn do not cause the interrupt to become
  78  *   inactive as long as the external input line is held high.
  79  */
  80
  81 #include "vgic.h"
  82
  83 static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu);
  84 static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu);
  85 static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr);
  86 static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr, struct vgic_lr lr_desc);
  87
  88 static const struct vgic_ops *vgic_ops;
  89 static const struct vgic_params *vgic;
  90
  91 static void add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
  92 {
  93         vcpu->kvm->arch.vgic.vm_ops.add_sgi_source(vcpu, irq, source);
  94 }
  95
  96 static bool queue_sgi(struct kvm_vcpu *vcpu, int irq)
  97 {
  98         return vcpu->kvm->arch.vgic.vm_ops.queue_sgi(vcpu, irq);
  99 }
 100
 101 int kvm_vgic_map_resources(struct kvm *kvm)
 102 {
 103         return kvm->arch.vgic.vm_ops.map_resources(kvm, vgic);
 104 }
 105
 106 /*
 107  * struct vgic_bitmap contains a bitmap made of unsigned longs, but
 108  * extracts u32s out of them.
 109  *
 110  * This does not work on 64-bit BE systems, because the bitmap access
 111  * will store two consecutive 32-bit words with the higher-addressed
 112  * register's bits at the lower index and the lower-addressed register's
 113  * bits at the higher index.
 114  *
 115  * Therefore, swizzle the register index when accessing the 32-bit word
 116  * registers to access the right register's value.
 117  */
 118 #if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 64
 119 #define REG_OFFSET_SWIZZLE      1
 120 #else
 121 #define REG_OFFSET_SWIZZLE      0
 122 #endif
 123
 124 static int vgic_init_bitmap(struct vgic_bitmap *b, int nr_cpus, int nr_irqs)
 125 {
 126         int nr_longs;
 127
 128         nr_longs = nr_cpus + BITS_TO_LONGS(nr_irqs - VGIC_NR_PRIVATE_IRQS);
 129
 130         b->private = kzalloc(sizeof(unsigned long) * nr_longs, GFP_KERNEL);
 131         if (!b->private)
 132                 return -ENOMEM;
 133
 134         b->shared = b->private + nr_cpus;
 135
 136         return 0;
 137 }
 138
 139 static void vgic_free_bitmap(struct vgic_bitmap *b)
 140 {
 141         kfree(b->private);
 142         b->private = NULL;
 143         b->shared = NULL;
 144 }
 145
 146 /*
 147  * Call this function to convert a u64 value to an unsigned long * bitmask
 148  * in a way that works on both 32-bit and 64-bit LE and BE platforms.
 149  *
 150  * Warning: Calling this function may modify *val.
 151  */
 152 static unsigned long *u64_to_bitmask(u64 *val)
 153 {
 154 #if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 32
 155         *val = (*val >> 32) | (*val << 32);
 156 #endif
 157         return (unsigned long *)val;
 158 }
 159
 160 u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x, int cpuid, u32 offset)
 161 {
 162         offset >>= 2;
 163         if (!offset)
 164                 return (u32 *)(x->private + cpuid) + REG_OFFSET_SWIZZLE;
 165         else
 166                 return (u32 *)(x->shared) + ((offset - 1) ^ REG_OFFSET_SWIZZLE);
 167 }
 168
 169 static int vgic_bitmap_get_irq_val(struct vgic_bitmap *x,
 170                                    int cpuid, int irq)
 171 {
 172         if (irq < VGIC_NR_PRIVATE_IRQS)
 173                 return test_bit(irq, x->private + cpuid);
 174
 175         return test_bit(irq - VGIC_NR_PRIVATE_IRQS, x->shared);
 176 }
 177
 178 void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
 179                              int irq, int val)
 180 {
 181         unsigned long *reg;
 182
 183         if (irq < VGIC_NR_PRIVATE_IRQS) {
 184                 reg = x->private + cpuid;
 185         } else {
 186                 reg = x->shared;
 187                 irq -= VGIC_NR_PRIVATE_IRQS;
 188         }
 189
 190         if (val)
 191                 set_bit(irq, reg);
 192         else
 193                 clear_bit(irq, reg);
 194 }
 195
 196 static unsigned long *vgic_bitmap_get_cpu_map(struct vgic_bitmap *x, int cpuid)
 197 {
 198         return x->private + cpuid;
 199 }
 200
 201 unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x)
 202 {
 203         return x->shared;
 204 }
 205
 206 static int vgic_init_bytemap(struct vgic_bytemap *x, int nr_cpus, int nr_irqs)
 207 {
 208         int size;
 209
 210         size  = nr_cpus * VGIC_NR_PRIVATE_IRQS;
 211         size += nr_irqs - VGIC_NR_PRIVATE_IRQS;
 212
 213         x->private = kzalloc(size, GFP_KERNEL);
 214         if (!x->private)
 215                 return -ENOMEM;
 216
 217         x->shared = x->private + nr_cpus * VGIC_NR_PRIVATE_IRQS / sizeof(u32);
 218         return 0;
 219 }
 220
 221 static void vgic_free_bytemap(struct vgic_bytemap *b)
 222 {
 223         kfree(b->private);
 224         b->private = NULL;
 225         b->shared = NULL;
 226 }
 227
 228 u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset)
 229 {
 230         u32 *reg;
 231
 232         if (offset < VGIC_NR_PRIVATE_IRQS) {
 233                 reg = x->private;
 234                 offset += cpuid * VGIC_NR_PRIVATE_IRQS;
 235         } else {
 236                 reg = x->shared;
 237                 offset -= VGIC_NR_PRIVATE_IRQS;
 238         }
 239
 240         return reg + (offset / sizeof(u32));
 241 }
 242
 243 #define VGIC_CFG_LEVEL  0
 244 #define VGIC_CFG_EDGE   1
 245
 246 static bool vgic_irq_is_edge(struct kvm_vcpu *vcpu, int irq)
 247 {
 248         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 249         int irq_val;
 250
 251         irq_val = vgic_bitmap_get_irq_val(&dist->irq_cfg, vcpu->vcpu_id, irq);
 252         return irq_val == VGIC_CFG_EDGE;
 253 }
 254
 255 static int vgic_irq_is_enabled(struct kvm_vcpu *vcpu, int irq)
 256 {
 257         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 258
 259         return vgic_bitmap_get_irq_val(&dist->irq_enabled, vcpu->vcpu_id, irq);
 260 }
 261
 262 static int vgic_irq_is_queued(struct kvm_vcpu *vcpu, int irq)
 263 {
 264         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 265
 266         return vgic_bitmap_get_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq);
 267 }
 268
 269 static int vgic_irq_is_active(struct kvm_vcpu *vcpu, int irq)
 270 {
 271         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 272
 273         return vgic_bitmap_get_irq_val(&dist->irq_active, vcpu->vcpu_id, irq);
 274 }
 275
 276 static void vgic_irq_set_queued(struct kvm_vcpu *vcpu, int irq)
 277 {
 278         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 279
 280         vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 1);
 281 }
 282
 283 static void vgic_irq_clear_queued(struct kvm_vcpu *vcpu, int irq)
 284 {
 285         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 286
 287         vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 0);
 288 }
 289
 290 static void vgic_irq_set_active(struct kvm_vcpu *vcpu, int irq)
 291 {
 292         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 293
 294         vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 1);
 295 }
 296
 297 static void vgic_irq_clear_active(struct kvm_vcpu *vcpu, int irq)
 298 {
 299         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 300
 301         vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 0);
 302 }
 303
 304 static int vgic_dist_irq_get_level(struct kvm_vcpu *vcpu, int irq)
 305 {
 306         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 307
 308         return vgic_bitmap_get_irq_val(&dist->irq_level, vcpu->vcpu_id, irq);
 309 }
 310
 311 static void vgic_dist_irq_set_level(struct kvm_vcpu *vcpu, int irq)
 312 {
 313         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 314
 315         vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 1);
 316 }
 317
 318 static void vgic_dist_irq_clear_level(struct kvm_vcpu *vcpu, int irq)
 319 {
 320         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 321
 322         vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 0);
 323 }
 324
 325 static int vgic_dist_irq_soft_pend(struct kvm_vcpu *vcpu, int irq)
 326 {
 327         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 328
 329         return vgic_bitmap_get_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq);
 330 }
 331
 332 static void vgic_dist_irq_clear_soft_pend(struct kvm_vcpu *vcpu, int irq)
 333 {
 334         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 335
 336         vgic_bitmap_set_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq, 0);
 337 }
 338
 339 static int vgic_dist_irq_is_pending(struct kvm_vcpu *vcpu, int irq)
 340 {
 341         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 342
 343         return vgic_bitmap_get_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq);
 344 }
 345
 346 void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq)
 347 {
 348         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 349
 350         vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 1);
 351 }
 352
 353 void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq)
 354 {
 355         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 356
 357         vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 0);
 358 }
 359
 360 static void vgic_cpu_irq_set(struct kvm_vcpu *vcpu, int irq)
 361 {
 362         if (irq < VGIC_NR_PRIVATE_IRQS)
 363                 set_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
 364         else
 365                 set_bit(irq - VGIC_NR_PRIVATE_IRQS,
 366                         vcpu->arch.vgic_cpu.pending_shared);
 367 }
 368
 369 void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq)
 370 {
 371         if (irq < VGIC_NR_PRIVATE_IRQS)
 372                 clear_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
 373         else
 374                 clear_bit(irq - VGIC_NR_PRIVATE_IRQS,
 375                           vcpu->arch.vgic_cpu.pending_shared);
 376 }
 377
 378 static bool vgic_can_sample_irq(struct kvm_vcpu *vcpu, int irq)
 379 {
 380         return vgic_irq_is_edge(vcpu, irq) || !vgic_irq_is_queued(vcpu, irq);
 381 }
 382
 383 /**
 384  * vgic_reg_access - access vgic register
 385  * @mmio:   pointer to the data describing the mmio access
 386  * @reg:    pointer to the virtual backing of vgic distributor data
 387  * @offset: least significant 2 bits used for word offset
 388  * @mode:   ACCESS_ mode (see defines above)
 389  *
 390  * Helper to make vgic register access easier using one of the access
 391  * modes defined for vgic register access
 392  * (read,raz,write-ignored,setbit,clearbit,write)
 393  */
 394 void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
 395                      phys_addr_t offset, int mode)
 396 {
 397         int word_offset = (offset & 3) * 8;
 398         u32 mask = (1UL << (mmio->len * 8)) - 1;
 399         u32 regval;
 400
 401         /*
 402          * Any alignment fault should have been delivered to the guest
 403          * directly (ARM ARM B3.12.7 "Prioritization of aborts").
 404          */
 405
 406         if (reg) {
 407                 regval = *reg;
 408         } else {
 409                 BUG_ON(mode != (ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED));
 410                 regval = 0;
 411         }
 412
 413         if (mmio->is_write) {
 414                 u32 data = mmio_data_read(mmio, mask) << word_offset;
 415                 switch (ACCESS_WRITE_MASK(mode)) {
 416                 case ACCESS_WRITE_IGNORED:
 417                         return;
 418
 419                 case ACCESS_WRITE_SETBIT:
 420                         regval |= data;
 421                         break;
 422
 423                 case ACCESS_WRITE_CLEARBIT:
 424                         regval &= ~data;
 425                         break;
 426
 427                 case ACCESS_WRITE_VALUE:
 428                         regval = (regval & ~(mask << word_offset)) | data;
 429                         break;
 430                 }
 431                 *reg = regval;
 432         } else {
 433                 switch (ACCESS_READ_MASK(mode)) {
 434                 case ACCESS_READ_RAZ:
 435                         regval = 0;
 436                         /* fall through */
 437
 438                 case ACCESS_READ_VALUE:
 439                         mmio_data_write(mmio, mask, regval >> word_offset);
 440                 }
 441         }
 442 }
 443
 444 bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
 445                         phys_addr_t offset)
 446 {
 447         vgic_reg_access(mmio, NULL, offset,
 448                         ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
 449         return false;
 450 }
 451
 452 bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
 453                             phys_addr_t offset, int vcpu_id, int access)
 454 {
 455         u32 *reg;
 456         int mode = ACCESS_READ_VALUE | access;
 457         struct kvm_vcpu *target_vcpu = kvm_get_vcpu(kvm, vcpu_id);
 458
 459         reg = vgic_bitmap_get_reg(&kvm->arch.vgic.irq_enabled, vcpu_id, offset);
 460         vgic_reg_access(mmio, reg, offset, mode);
 461         if (mmio->is_write) {
 462                 if (access & ACCESS_WRITE_CLEARBIT) {
 463                         if (offset < 4) /* Force SGI enabled */
 464                                 *reg |= 0xffff;
 465                         vgic_retire_disabled_irqs(target_vcpu);
 466                 }
 467                 vgic_update_state(kvm);
 468                 return true;
 469         }
 470
 471         return false;
 472 }
 473
 474 bool vgic_handle_set_pending_reg(struct kvm *kvm,
 475                                  struct kvm_exit_mmio *mmio,
 476                                  phys_addr_t offset, int vcpu_id)
 477 {
 478         u32 *reg, orig;
 479         u32 level_mask;
 480         int mode = ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT;
 481         struct vgic_dist *dist = &kvm->arch.vgic;
 482
 483         reg = vgic_bitmap_get_reg(&dist->irq_cfg, vcpu_id, offset);
 484         level_mask = (~(*reg));
 485
 486         /* Mark both level and edge triggered irqs as pending */
 487         reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
 488         orig = *reg;
 489         vgic_reg_access(mmio, reg, offset, mode);
 490
 491         if (mmio->is_write) {
 492                 /* Set the soft-pending flag only for level-triggered irqs */
 493                 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
 494                                           vcpu_id, offset);
 495                 vgic_reg_access(mmio, reg, offset, mode);
 496                 *reg &= level_mask;
 497
 498                 /* Ignore writes to SGIs */
 499                 if (offset < 2) {
 500                         *reg &= ~0xffff;
 501                         *reg |= orig & 0xffff;
 502                 }
 503
 504                 vgic_update_state(kvm);
 505                 return true;
 506         }
 507
 508         return false;
 509 }
 510
 511 bool vgic_handle_clear_pending_reg(struct kvm *kvm,
 512                                    struct kvm_exit_mmio *mmio,
 513                                    phys_addr_t offset, int vcpu_id)
 514 {
 515         u32 *level_active;
 516         u32 *reg, orig;
 517         int mode = ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT;
 518         struct vgic_dist *dist = &kvm->arch.vgic;
 519
 520         reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
 521         orig = *reg;
 522         vgic_reg_access(mmio, reg, offset, mode);
 523         if (mmio->is_write) {
 524                 /* Re-set level triggered level-active interrupts */
 525                 level_active = vgic_bitmap_get_reg(&dist->irq_level,
 526                                           vcpu_id, offset);
 527                 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
 528                 *reg |= *level_active;
 529
 530                 /* Ignore writes to SGIs */
 531                 if (offset < 2) {
 532                         *reg &= ~0xffff;
 533                         *reg |= orig & 0xffff;
 534                 }
 535
 536                 /* Clear soft-pending flags */
 537                 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
 538                                           vcpu_id, offset);
 539                 vgic_reg_access(mmio, reg, offset, mode);
 540
 541                 vgic_update_state(kvm);
 542                 return true;
 543         }
 544         return false;
 545 }
 546
 547 bool vgic_handle_set_active_reg(struct kvm *kvm,
 548                                 struct kvm_exit_mmio *mmio,
 549                                 phys_addr_t offset, int vcpu_id)
 550 {
 551         u32 *reg;
 552         struct vgic_dist *dist = &kvm->arch.vgic;
 553
 554         reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
 555         vgic_reg_access(mmio, reg, offset,
 556                         ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
 557
 558         if (mmio->is_write) {
 559                 vgic_update_state(kvm);
 560                 return true;
 561         }
 562
 563         return false;
 564 }
 565
 566 bool vgic_handle_clear_active_reg(struct kvm *kvm,
 567                                   struct kvm_exit_mmio *mmio,
 568                                   phys_addr_t offset, int vcpu_id)
 569 {
 570         u32 *reg;
 571         struct vgic_dist *dist = &kvm->arch.vgic;
 572
 573         reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
 574         vgic_reg_access(mmio, reg, offset,
 575                         ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
 576
 577         if (mmio->is_write) {
 578                 vgic_update_state(kvm);
 579                 return true;
 580         }
 581
 582         return false;
 583 }
 584
 585 static u32 vgic_cfg_expand(u16 val)
 586 {
 587         u32 res = 0;
 588         int i;
 589
 590         /*
 591          * Turn a 16bit value like abcd...mnop into a 32bit word
 592          * a0b0c0d0...m0n0o0p0, which is what the HW cfg register is.
 593          */
 594         for (i = 0; i < 16; i++)
 595                 res |= ((val >> i) & VGIC_CFG_EDGE) << (2 * i + 1);
 596
 597         return res;
 598 }
 599
 600 static u16 vgic_cfg_compress(u32 val)
 601 {
 602         u16 res = 0;
 603         int i;
 604
 605         /*
 606          * Turn a 32bit word a0b0c0d0...m0n0o0p0 into 16bit value like
 607          * abcd...mnop which is what we really care about.
 608          */
 609         for (i = 0; i < 16; i++)
 610                 res |= ((val >> (i * 2 + 1)) & VGIC_CFG_EDGE) << i;
 611
 612         return res;
 613 }
 614
 615 /*
 616  * The distributor uses 2 bits per IRQ for the CFG register, but the
 617  * LSB is always 0. As such, we only keep the upper bit, and use the
 618  * two above functions to compress/expand the bits
 619  */
 620 bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
 621                          phys_addr_t offset)
 622 {
 623         u32 val;
 624
 625         if (offset & 4)
 626                 val = *reg >> 16;
 627         else
 628                 val = *reg & 0xffff;
 629
 630         val = vgic_cfg_expand(val);
 631         vgic_reg_access(mmio, &val, offset,
 632                         ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
 633         if (mmio->is_write) {
 634                 if (offset < 8) {
 635                         *reg = ~0U; /* Force PPIs/SGIs to 1 */
 636                         return false;
 637                 }
 638
 639                 val = vgic_cfg_compress(val);
 640                 if (offset & 4) {
 641                         *reg &= 0xffff;
 642                         *reg |= val << 16;
 643                 } else {
 644                         *reg &= 0xffff << 16;
 645                         *reg |= val;
 646                 }
 647         }
 648
 649         return false;
 650 }
 651
 652 /**
 653  * vgic_unqueue_irqs - move pending/active IRQs from LRs to the distributor
 654  * @vgic_cpu: Pointer to the vgic_cpu struct holding the LRs
 655  *
 656  * Move any IRQs that have already been assigned to LRs back to the
 657  * emulated distributor state so that the complete emulated state can be read
 658  * from the main emulation structures without investigating the LRs.
 659  */
 660 void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
 661 {
 662         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 663         int i;
 664
 665         for_each_set_bit(i, vgic_cpu->lr_used, vgic_cpu->nr_lr) {
 666                 struct vgic_lr lr = vgic_get_lr(vcpu, i);
 667
 668                 /*
 669                  * There are three options for the state bits:
 670                  *
 671                  * 01: pending
 672                  * 10: active
 673                  * 11: pending and active
 674                  */
 675                 BUG_ON(!(lr.state & LR_STATE_MASK));
 676
 677                 /* Reestablish SGI source for pending and active IRQs */
 678                 if (lr.irq < VGIC_NR_SGIS)
 679                         add_sgi_source(vcpu, lr.irq, lr.source);
 680
 681                 /*
 682                  * If the LR holds an active (10) or a pending and active (11)
 683                  * interrupt then move the active state to the
 684                  * distributor tracking bit.
 685                  */
 686                 if (lr.state & LR_STATE_ACTIVE) {
 687                         vgic_irq_set_active(vcpu, lr.irq);
 688                         lr.state &= ~LR_STATE_ACTIVE;
 689                 }
 690
 691                 /*
 692                  * Reestablish the pending state on the distributor and the
 693                  * CPU interface.  It may have already been pending, but that
 694                  * is fine, then we are only setting a few bits that were
 695                  * already set.
 696                  */
 697                 if (lr.state & LR_STATE_PENDING) {
 698                         vgic_dist_irq_set_pending(vcpu, lr.irq);
 699                         lr.state &= ~LR_STATE_PENDING;
 700                 }
 701
 702                 vgic_set_lr(vcpu, i, lr);
 703
 704                 /*
 705                  * Mark the LR as free for other use.
 706                  */
 707                 BUG_ON(lr.state & LR_STATE_MASK);
 708                 vgic_retire_lr(i, lr.irq, vcpu);
 709                 vgic_irq_clear_queued(vcpu, lr.irq);
 710
 711                 /* Finally update the VGIC state. */
 712                 vgic_update_state(vcpu->kvm);
 713         }
 714 }
 715
 716 const
 717 struct vgic_io_range *vgic_find_range(const struct vgic_io_range *ranges,
 718                                       int len, gpa_t offset)
 719 {
 720         while (ranges->len) {
 721                 if (offset >= ranges->base &&
 722                     (offset + len) <= (ranges->base + ranges->len))
 723                         return ranges;
 724                 ranges++;
 725         }
 726
 727         return NULL;
 728 }
 729
 730 static bool vgic_validate_access(const struct vgic_dist *dist,
 731                                  const struct vgic_io_range *range,
 732                                  unsigned long offset)
 733 {
 734         int irq;
 735
 736         if (!range->bits_per_irq)
 737                 return true;    /* Not an irq-based access */
 738
 739         irq = offset * 8 / range->bits_per_irq;
 740         if (irq >= dist->nr_irqs)
 741                 return false;
 742
 743         return true;
 744 }
 745
 746 /*
 747  * Call the respective handler function for the given range.
 748  * We split up any 64 bit accesses into two consecutive 32 bit
 749  * handler calls and merge the result afterwards.
 750  * We do this in a little endian fashion regardless of the host's
 751  * or guest's endianness, because the GIC is always LE and the rest of
 752  * the code (vgic_reg_access) also puts it in a LE fashion already.
 753  * At this point we have already identified the handle function, so
 754  * range points to that one entry and offset is relative to this.
 755  */
 756 static bool call_range_handler(struct kvm_vcpu *vcpu,
 757                                struct kvm_exit_mmio *mmio,
 758                                unsigned long offset,
 759                                const struct vgic_io_range *range)
 760 {
 761         u32 *data32 = (void *)mmio->data;
 762         struct kvm_exit_mmio mmio32;
 763         bool ret;
 764
 765         if (likely(mmio->len <= 4))
 766                 return range->handle_mmio(vcpu, mmio, offset);
 767
 768         /*
 769          * Any access bigger than 4 bytes (that we currently handle in KVM)
 770          * is actually 8 bytes long, caused by a 64-bit access
 771          */
 772
 773         mmio32.len = 4;
 774         mmio32.is_write = mmio->is_write;
 775         mmio32.private = mmio->private;
 776
 777         mmio32.phys_addr = mmio->phys_addr + 4;
 778         if (mmio->is_write)
 779                 *(u32 *)mmio32.data = data32[1];
 780         ret = range->handle_mmio(vcpu, &mmio32, offset + 4);
 781         if (!mmio->is_write)
 782                 data32[1] = *(u32 *)mmio32.data;
 783
 784         mmio32.phys_addr = mmio->phys_addr;
 785         if (mmio->is_write)
 786                 *(u32 *)mmio32.data = data32[0];
 787         ret |= range->handle_mmio(vcpu, &mmio32, offset);
 788         if (!mmio->is_write)
 789                 data32[0] = *(u32 *)mmio32.data;
 790
 791         return ret;
 792 }
 793
 794 /**
 795  * vgic_handle_mmio_range - handle an in-kernel MMIO access
 796  * @vcpu:       pointer to the vcpu performing the access
 797  * @run:        pointer to the kvm_run structure
 798  * @mmio:       pointer to the data describing the access
 799  * @ranges:     array of MMIO ranges in a given region
 800  * @mmio_base:  base address of that region
 801  *
 802  * returns true if the MMIO access could be performed
 803  */
 804 bool vgic_handle_mmio_range(struct kvm_vcpu *vcpu, struct kvm_run *run,
 805                             struct kvm_exit_mmio *mmio,
 806                             const struct vgic_io_range *ranges,
 807                             unsigned long mmio_base)
 808 {
 809         const struct vgic_io_range *range;
 810         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 811         bool updated_state;
 812         unsigned long offset;
 813
 814         offset = mmio->phys_addr - mmio_base;
 815         range = vgic_find_range(ranges, mmio->len, offset);
 816         if (unlikely(!range || !range->handle_mmio)) {
 817                 pr_warn("Unhandled access %d %08llx %d\n",
 818                         mmio->is_write, mmio->phys_addr, mmio->len);
 819                 return false;
 820         }
 821
 822         spin_lock(&vcpu->kvm->arch.vgic.lock);
 823         offset -= range->base;
 824         if (vgic_validate_access(dist, range, offset)) {
 825                 updated_state = call_range_handler(vcpu, mmio, offset, range);
 826         } else {
 827                 if (!mmio->is_write)
 828                         memset(mmio->data, 0, mmio->len);
 829                 updated_state = false;
 830         }
 831         spin_unlock(&vcpu->kvm->arch.vgic.lock);
 832         kvm_prepare_mmio(run, mmio);
 833         kvm_handle_mmio_return(vcpu, run);
 834
 835         if (updated_state)
 836                 vgic_kick_vcpus(vcpu->kvm);
 837
 838         return true;
 839 }
 840
 841 /**
 842  * vgic_handle_mmio_access - handle an in-kernel MMIO access
 843  * This is called by the read/write KVM IO device wrappers below.
 844  * @vcpu:       pointer to the vcpu performing the access
 845  * @this:       pointer to the KVM IO device in charge
 846  * @addr:       guest physical address of the access
 847  * @len:        size of the access
 848  * @val:        pointer to the data region
 849  * @is_write:   read or write access
 850  *
 851  * returns true if the MMIO access could be performed
 852  */
 853 static int vgic_handle_mmio_access(struct kvm_vcpu *vcpu,
 854                                    struct kvm_io_device *this, gpa_t addr,
 855                                    int len, void *val, bool is_write)
 856 {
 857         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 858         struct vgic_io_device *iodev = container_of(this,
 859                                                     struct vgic_io_device, dev);
 860         struct kvm_run *run = vcpu->run;
 861         const struct vgic_io_range *range;
 862         struct kvm_exit_mmio mmio;
 863         bool updated_state;
 864         gpa_t offset;
 865
 866         offset = addr - iodev->addr;
 867         range = vgic_find_range(iodev->reg_ranges, len, offset);
 868         if (unlikely(!range || !range->handle_mmio)) {
 869                 pr_warn("Unhandled access %d %08llx %d\n", is_write, addr, len);
 870                 return -ENXIO;
 871         }
 872
 873         mmio.phys_addr = addr;
 874         mmio.len = len;
 875         mmio.is_write = is_write;
 876         if (is_write)
 877                 memcpy(mmio.data, val, len);
 878         mmio.private = iodev->redist_vcpu;
 879
 880         spin_lock(&dist->lock);
 881         offset -= range->base;
 882         if (vgic_validate_access(dist, range, offset)) {
 883                 updated_state = call_range_handler(vcpu, &mmio, offset, range);
 884                 if (!is_write)
 885                         memcpy(val, mmio.data, len);
 886         } else {
 887                 if (!is_write)
 888                         memset(val, 0, len);
 889                 updated_state = false;
 890         }
 891         spin_unlock(&dist->lock);
 892         kvm_prepare_mmio(run, &mmio);
 893         kvm_handle_mmio_return(vcpu, run);
 894
 895         if (updated_state)
 896                 vgic_kick_vcpus(vcpu->kvm);
 897
 898         return 0;
 899 }
 900
 901 /**
 902  * vgic_handle_mmio - handle an in-kernel MMIO access for the GIC emulation
 903  * @vcpu:      pointer to the vcpu performing the access
 904  * @run:       pointer to the kvm_run structure
 905  * @mmio:      pointer to the data describing the access
 906  *
 907  * returns true if the MMIO access has been performed in kernel space,
 908  * and false if it needs to be emulated in user space.
 909  * Calls the actual handling routine for the selected VGIC model.
 910  */
 911 bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
 912                       struct kvm_exit_mmio *mmio)
 913 {
 914         if (!irqchip_in_kernel(vcpu->kvm))
 915                 return false;
 916
 917         /*
 918          * This will currently call either vgic_v2_handle_mmio() or
 919          * vgic_v3_handle_mmio(), which in turn will call
 920          * vgic_handle_mmio_range() defined above.
 921          */
 922         return vcpu->kvm->arch.vgic.vm_ops.handle_mmio(vcpu, run, mmio);
 923 }
 924
 925 static int vgic_handle_mmio_read(struct kvm_vcpu *vcpu,
 926                                  struct kvm_io_device *this,
 927                                  gpa_t addr, int len, void *val)
 928 {
 929         return vgic_handle_mmio_access(vcpu, this, addr, len, val, false);
 930 }
 931
 932 static int vgic_handle_mmio_write(struct kvm_vcpu *vcpu,
 933                                   struct kvm_io_device *this,
 934                                   gpa_t addr, int len, const void *val)
 935 {
 936         return vgic_handle_mmio_access(vcpu, this, addr, len, (void *)val,
 937                                        true);
 938 }
 939
 940 struct kvm_io_device_ops vgic_io_ops = {
 941         .read   = vgic_handle_mmio_read,
 942         .write  = vgic_handle_mmio_write,
 943 };
 944
 945 /**
 946  * vgic_register_kvm_io_dev - register VGIC register frame on the KVM I/O bus
 947  * @kvm:            The VM structure pointer
 948  * @base:           The (guest) base address for the register frame
 949  * @len:            Length of the register frame window
 950  * @ranges:         Describing the handler functions for each register
 951  * @redist_vcpu_id: The VCPU ID to pass on to the handlers on call
 952  * @iodev:          Points to memory to be passed on to the handler
 953  *
 954  * @iodev stores the parameters of this function to be usable by the handler
 955  * respectively the dispatcher function (since the KVM I/O bus framework lacks
 956  * an opaque parameter). Initialization is done in this function, but the
 957  * reference should be valid and unique for the whole VGIC lifetime.
 958  * If the register frame is not mapped for a specific VCPU, pass -1 to
 959  * @redist_vcpu_id.
 960  */
 961 int vgic_register_kvm_io_dev(struct kvm *kvm, gpa_t base, int len,
 962                              const struct vgic_io_range *ranges,
 963                              int redist_vcpu_id,
 964                              struct vgic_io_device *iodev)
 965 {
 966         struct kvm_vcpu *vcpu = NULL;
 967         int ret;
 968
 969         if (redist_vcpu_id >= 0)
 970                 vcpu = kvm_get_vcpu(kvm, redist_vcpu_id);
 971
 972         iodev->addr             = base;
 973         iodev->len              = len;
 974         iodev->reg_ranges       = ranges;
 975         iodev->redist_vcpu      = vcpu;
 976
 977         kvm_iodevice_init(&iodev->dev, &vgic_io_ops);
 978
 979         mutex_lock(&kvm->slots_lock);
 980
 981         ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, base, len,
 982                                       &iodev->dev);
 983         mutex_unlock(&kvm->slots_lock);
 984
 985         /* Mark the iodev as invalid if registration fails. */
 986         if (ret)
 987                 iodev->dev.ops = NULL;
 988
 989         return ret;
 990 }
 991
 992 static int vgic_nr_shared_irqs(struct vgic_dist *dist)
 993 {
 994         return dist->nr_irqs - VGIC_NR_PRIVATE_IRQS;
 995 }
 996
 997 static int compute_active_for_cpu(struct kvm_vcpu *vcpu)
 998 {
 999         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1000         unsigned long *active, *enabled, *act_percpu, *act_shared;
1001         unsigned long active_private, active_shared;
1002         int nr_shared = vgic_nr_shared_irqs(dist);
1003         int vcpu_id;
1004
1005         vcpu_id = vcpu->vcpu_id;
1006         act_percpu = vcpu->arch.vgic_cpu.active_percpu;
1007         act_shared = vcpu->arch.vgic_cpu.active_shared;
1008
1009         active = vgic_bitmap_get_cpu_map(&dist->irq_active, vcpu_id);
1010         enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
1011         bitmap_and(act_percpu, active, enabled, VGIC_NR_PRIVATE_IRQS);
1012
1013         active = vgic_bitmap_get_shared_map(&dist->irq_active);
1014         enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
1015         bitmap_and(act_shared, active, enabled, nr_shared);
1016         bitmap_and(act_shared, act_shared,
1017                    vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
1018                    nr_shared);
1019
1020         active_private = find_first_bit(act_percpu, VGIC_NR_PRIVATE_IRQS);
1021         active_shared = find_first_bit(act_shared, nr_shared);
1022
1023         return (active_private < VGIC_NR_PRIVATE_IRQS ||
1024                 active_shared < nr_shared);
1025 }
1026
1027 static int compute_pending_for_cpu(struct kvm_vcpu *vcpu)
1028 {
1029         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1030         unsigned long *pending, *enabled, *pend_percpu, *pend_shared;
1031         unsigned long pending_private, pending_shared;
1032         int nr_shared = vgic_nr_shared_irqs(dist);
1033         int vcpu_id;
1034
1035         vcpu_id = vcpu->vcpu_id;
1036         pend_percpu = vcpu->arch.vgic_cpu.pending_percpu;
1037         pend_shared = vcpu->arch.vgic_cpu.pending_shared;
1038
1039         pending = vgic_bitmap_get_cpu_map(&dist->irq_pending, vcpu_id);
1040         enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
1041         bitmap_and(pend_percpu, pending, enabled, VGIC_NR_PRIVATE_IRQS);
1042
1043         pending = vgic_bitmap_get_shared_map(&dist->irq_pending);
1044         enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
1045         bitmap_and(pend_shared, pending, enabled, nr_shared);
1046         bitmap_and(pend_shared, pend_shared,
1047                    vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
1048                    nr_shared);
1049
1050         pending_private = find_first_bit(pend_percpu, VGIC_NR_PRIVATE_IRQS);
1051         pending_shared = find_first_bit(pend_shared, nr_shared);
1052         return (pending_private < VGIC_NR_PRIVATE_IRQS ||
1053                 pending_shared < vgic_nr_shared_irqs(dist));
1054 }
1055
1056 /*
1057  * Update the interrupt state and determine which CPUs have pending
1058  * or active interrupts. Must be called with distributor lock held.
1059  */
1060 void vgic_update_state(struct kvm *kvm)
1061 {
1062         struct vgic_dist *dist = &kvm->arch.vgic;
1063         struct kvm_vcpu *vcpu;
1064         int c;
1065
1066         if (!dist->enabled) {
1067                 set_bit(0, dist->irq_pending_on_cpu);
1068                 return;
1069         }
1070
1071         kvm_for_each_vcpu(c, vcpu, kvm) {
1072                 if (compute_pending_for_cpu(vcpu))
1073                         set_bit(c, dist->irq_pending_on_cpu);
1074
1075                 if (compute_active_for_cpu(vcpu))
1076                         set_bit(c, dist->irq_active_on_cpu);
1077                 else
1078                         clear_bit(c, dist->irq_active_on_cpu);
1079         }
1080 }
1081
1082 static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr)
1083 {
1084         return vgic_ops->get_lr(vcpu, lr);
1085 }
1086
1087 static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr,
1088                                struct vgic_lr vlr)
1089 {
1090         vgic_ops->set_lr(vcpu, lr, vlr);
1091 }
1092
1093 static void vgic_sync_lr_elrsr(struct kvm_vcpu *vcpu, int lr,
1094                                struct vgic_lr vlr)
1095 {
1096         vgic_ops->sync_lr_elrsr(vcpu, lr, vlr);
1097 }
1098
1099 static inline u64 vgic_get_elrsr(struct kvm_vcpu *vcpu)
1100 {
1101         return vgic_ops->get_elrsr(vcpu);
1102 }
1103
1104 static inline u64 vgic_get_eisr(struct kvm_vcpu *vcpu)
1105 {
1106         return vgic_ops->get_eisr(vcpu);
1107 }
1108
1109 static inline u32 vgic_get_interrupt_status(struct kvm_vcpu *vcpu)
1110 {
1111         return vgic_ops->get_interrupt_status(vcpu);
1112 }
1113
1114 static inline void vgic_enable_underflow(struct kvm_vcpu *vcpu)
1115 {
1116         vgic_ops->enable_underflow(vcpu);
1117 }
1118
1119 static inline void vgic_disable_underflow(struct kvm_vcpu *vcpu)
1120 {
1121         vgic_ops->disable_underflow(vcpu);
1122 }
1123
1124 void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
1125 {
1126         vgic_ops->get_vmcr(vcpu, vmcr);
1127 }
1128
1129 void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
1130 {
1131         vgic_ops->set_vmcr(vcpu, vmcr);
1132 }
1133
1134 static inline void vgic_enable(struct kvm_vcpu *vcpu)
1135 {
1136         vgic_ops->enable(vcpu);
1137 }
1138
1139 static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu)
1140 {
1141         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1142         struct vgic_lr vlr = vgic_get_lr(vcpu, lr_nr);
1143
1144         vlr.state = 0;
1145         vgic_set_lr(vcpu, lr_nr, vlr);
1146         clear_bit(lr_nr, vgic_cpu->lr_used);
1147         vgic_cpu->vgic_irq_lr_map[irq] = LR_EMPTY;
1148 }
1149
1150 /*
1151  * An interrupt may have been disabled after being made pending on the
1152  * CPU interface (the classic case is a timer running while we're
1153  * rebooting the guest - the interrupt would kick as soon as the CPU
1154  * interface gets enabled, with deadly consequences).
1155  *
1156  * The solution is to examine already active LRs, and check the
1157  * interrupt is still enabled. If not, just retire it.
1158  */
1159 static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu)
1160 {
1161         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1162         int lr;
1163
1164         for_each_set_bit(lr, vgic_cpu->lr_used, vgic->nr_lr) {
1165                 struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
1166
1167                 if (!vgic_irq_is_enabled(vcpu, vlr.irq)) {
1168                         vgic_retire_lr(lr, vlr.irq, vcpu);
1169                         if (vgic_irq_is_queued(vcpu, vlr.irq))
1170                                 vgic_irq_clear_queued(vcpu, vlr.irq);
1171                 }
1172         }
1173 }
1174
1175 static void vgic_queue_irq_to_lr(struct kvm_vcpu *vcpu, int irq,
1176                                  int lr_nr, struct vgic_lr vlr)
1177 {
1178         if (vgic_irq_is_active(vcpu, irq)) {
1179                 vlr.state |= LR_STATE_ACTIVE;
1180                 kvm_debug("Set active, clear distributor: 0x%x\n", vlr.state);
1181                 vgic_irq_clear_active(vcpu, irq);
1182                 vgic_update_state(vcpu->kvm);
1183         } else if (vgic_dist_irq_is_pending(vcpu, irq)) {
1184                 vlr.state |= LR_STATE_PENDING;
1185                 kvm_debug("Set pending: 0x%x\n", vlr.state);
1186         }
1187
1188         if (!vgic_irq_is_edge(vcpu, irq))
1189                 vlr.state |= LR_EOI_INT;
1190
1191         vgic_set_lr(vcpu, lr_nr, vlr);
1192 }
1193
1194 /*
1195  * Queue an interrupt to a CPU virtual interface. Return true on success,
1196  * or false if it wasn't possible to queue it.
1197  * sgi_source must be zero for any non-SGI interrupts.
1198  */
1199 bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
1200 {
1201         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1202         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1203         struct vgic_lr vlr;
1204         int lr;
1205
1206         /* Sanitize the input... */
1207         BUG_ON(sgi_source_id & ~7);
1208         BUG_ON(sgi_source_id && irq >= VGIC_NR_SGIS);
1209         BUG_ON(irq >= dist->nr_irqs);
1210
1211         kvm_debug("Queue IRQ%d\n", irq);
1212
1213         lr = vgic_cpu->vgic_irq_lr_map[irq];
1214
1215         /* Do we have an active interrupt for the same CPUID? */
1216         if (lr != LR_EMPTY) {
1217                 vlr = vgic_get_lr(vcpu, lr);
1218                 if (vlr.source == sgi_source_id) {
1219                         kvm_debug("LR%d piggyback for IRQ%d\n", lr, vlr.irq);
1220                         BUG_ON(!test_bit(lr, vgic_cpu->lr_used));
1221                         vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
1222                         return true;
1223                 }
1224         }
1225
1226         /* Try to use another LR for this interrupt */
1227         lr = find_first_zero_bit((unsigned long *)vgic_cpu->lr_used,
1228                                vgic->nr_lr);
1229         if (lr >= vgic->nr_lr)
1230                 return false;
1231
1232         kvm_debug("LR%d allocated for IRQ%d %x\n", lr, irq, sgi_source_id);
1233         vgic_cpu->vgic_irq_lr_map[irq] = lr;
1234         set_bit(lr, vgic_cpu->lr_used);
1235
1236         vlr.irq = irq;
1237         vlr.source = sgi_source_id;
1238         vlr.state = 0;
1239         vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
1240
1241         return true;
1242 }
1243
1244 static bool vgic_queue_hwirq(struct kvm_vcpu *vcpu, int irq)
1245 {
1246         if (!vgic_can_sample_irq(vcpu, irq))
1247                 return true; /* level interrupt, already queued */
1248
1249         if (vgic_queue_irq(vcpu, 0, irq)) {
1250                 if (vgic_irq_is_edge(vcpu, irq)) {
1251                         vgic_dist_irq_clear_pending(vcpu, irq);
1252                         vgic_cpu_irq_clear(vcpu, irq);
1253                 } else {
1254                         vgic_irq_set_queued(vcpu, irq);
1255                 }
1256
1257                 return true;
1258         }
1259
1260         return false;
1261 }
1262
1263 /*
1264  * Fill the list registers with pending interrupts before running the
1265  * guest.
1266  */
1267 static void __kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1268 {
1269         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1270         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1271         unsigned long *pa_percpu, *pa_shared;
1272         int i, vcpu_id;
1273         int overflow = 0;
1274         int nr_shared = vgic_nr_shared_irqs(dist);
1275
1276         vcpu_id = vcpu->vcpu_id;
1277
1278         pa_percpu = vcpu->arch.vgic_cpu.pend_act_percpu;
1279         pa_shared = vcpu->arch.vgic_cpu.pend_act_shared;
1280
1281         bitmap_or(pa_percpu, vgic_cpu->pending_percpu, vgic_cpu->active_percpu,
1282                   VGIC_NR_PRIVATE_IRQS);
1283         bitmap_or(pa_shared, vgic_cpu->pending_shared, vgic_cpu->active_shared,
1284                   nr_shared);
1285         /*
1286          * We may not have any pending interrupt, or the interrupts
1287          * may have been serviced from another vcpu. In all cases,
1288          * move along.
1289          */
1290         if (!kvm_vgic_vcpu_pending_irq(vcpu) && !kvm_vgic_vcpu_active_irq(vcpu))
1291                 goto epilog;
1292
1293         /* SGIs */
1294         for_each_set_bit(i, pa_percpu, VGIC_NR_SGIS) {
1295                 if (!queue_sgi(vcpu, i))
1296                         overflow = 1;
1297         }
1298
1299         /* PPIs */
1300         for_each_set_bit_from(i, pa_percpu, VGIC_NR_PRIVATE_IRQS) {
1301                 if (!vgic_queue_hwirq(vcpu, i))
1302                         overflow = 1;
1303         }
1304
1305         /* SPIs */
1306         for_each_set_bit(i, pa_shared, nr_shared) {
1307                 if (!vgic_queue_hwirq(vcpu, i + VGIC_NR_PRIVATE_IRQS))
1308                         overflow = 1;
1309         }
1310
1311
1312
1313
1314 epilog:
1315         if (overflow) {
1316                 vgic_enable_underflow(vcpu);
1317         } else {
1318                 vgic_disable_underflow(vcpu);
1319                 /*
1320                  * We're about to run this VCPU, and we've consumed
1321                  * everything the distributor had in store for
1322                  * us. Claim we don't have anything pending. We'll
1323                  * adjust that if needed while exiting.
1324                  */
1325                 clear_bit(vcpu_id, dist->irq_pending_on_cpu);
1326         }
1327 }
1328
1329 static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
1330 {
1331         u32 status = vgic_get_interrupt_status(vcpu);
1332         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1333         bool level_pending = false;
1334         struct kvm *kvm = vcpu->kvm;
1335
1336         kvm_debug("STATUS = %08x\n", status);
1337
1338         if (status & INT_STATUS_EOI) {
1339                 /*
1340                  * Some level interrupts have been EOIed. Clear their
1341                  * active bit.
1342                  */
1343                 u64 eisr = vgic_get_eisr(vcpu);
1344                 unsigned long *eisr_ptr = u64_to_bitmask(&eisr);
1345                 int lr;
1346
1347                 for_each_set_bit(lr, eisr_ptr, vgic->nr_lr) {
1348                         struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
1349                         WARN_ON(vgic_irq_is_edge(vcpu, vlr.irq));
1350
1351                         spin_lock(&dist->lock);
1352                         vgic_irq_clear_queued(vcpu, vlr.irq);
1353                         WARN_ON(vlr.state & LR_STATE_MASK);
1354                         vlr.state = 0;
1355                         vgic_set_lr(vcpu, lr, vlr);
1356
1357                         /*
1358                          * If the IRQ was EOIed it was also ACKed and we we
1359                          * therefore assume we can clear the soft pending
1360                          * state (should it had been set) for this interrupt.
1361                          *
1362                          * Note: if the IRQ soft pending state was set after
1363                          * the IRQ was acked, it actually shouldn't be
1364                          * cleared, but we have no way of knowing that unless
1365                          * we start trapping ACKs when the soft-pending state
1366                          * is set.
1367                          */
1368                         vgic_dist_irq_clear_soft_pend(vcpu, vlr.irq);
1369
1370                         /*
1371                          * kvm_notify_acked_irq calls kvm_set_irq()
1372                          * to reset the IRQ level. Need to release the
1373                          * lock for kvm_set_irq to grab it.
1374                          */
1375                         spin_unlock(&dist->lock);
1376
1377                         kvm_notify_acked_irq(kvm, 0,
1378                                              vlr.irq - VGIC_NR_PRIVATE_IRQS);
1379                         spin_lock(&dist->lock);
1380
1381                         /* Any additional pending interrupt? */
1382                         if (vgic_dist_irq_get_level(vcpu, vlr.irq)) {
1383                                 vgic_cpu_irq_set(vcpu, vlr.irq);
1384                                 level_pending = true;
1385                         } else {
1386                                 vgic_dist_irq_clear_pending(vcpu, vlr.irq);
1387                                 vgic_cpu_irq_clear(vcpu, vlr.irq);
1388                         }
1389
1390                         spin_unlock(&dist->lock);
1391
1392                         /*
1393                          * Despite being EOIed, the LR may not have
1394                          * been marked as empty.
1395                          */
1396                         vgic_sync_lr_elrsr(vcpu, lr, vlr);
1397                 }
1398         }
1399
1400         if (status & INT_STATUS_UNDERFLOW)
1401                 vgic_disable_underflow(vcpu);
1402
1403         return level_pending;
1404 }
1405
1406 /* Sync back the VGIC state after a guest run */
1407 static void __kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1408 {
1409         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1410         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1411         u64 elrsr;
1412         unsigned long *elrsr_ptr;
1413         int lr, pending;
1414         bool level_pending;
1415
1416         level_pending = vgic_process_maintenance(vcpu);
1417         elrsr = vgic_get_elrsr(vcpu);
1418         elrsr_ptr = u64_to_bitmask(&elrsr);
1419
1420         /* Clear mappings for empty LRs */
1421         for_each_set_bit(lr, elrsr_ptr, vgic->nr_lr) {
1422                 struct vgic_lr vlr;
1423
1424                 if (!test_and_clear_bit(lr, vgic_cpu->lr_used))
1425                         continue;
1426
1427                 vlr = vgic_get_lr(vcpu, lr);
1428
1429                 BUG_ON(vlr.irq >= dist->nr_irqs);
1430                 vgic_cpu->vgic_irq_lr_map[vlr.irq] = LR_EMPTY;
1431         }
1432
1433         /* Check if we still have something up our sleeve... */
1434         pending = find_first_zero_bit(elrsr_ptr, vgic->nr_lr);
1435         if (level_pending || pending < vgic->nr_lr)
1436                 set_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
1437 }
1438
1439 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1440 {
1441         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1442
1443         if (!irqchip_in_kernel(vcpu->kvm))
1444                 return;
1445
1446         spin_lock(&dist->lock);
1447         __kvm_vgic_flush_hwstate(vcpu);
1448         spin_unlock(&dist->lock);
1449 }
1450
1451 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1452 {
1453         if (!irqchip_in_kernel(vcpu->kvm))
1454                 return;
1455
1456         __kvm_vgic_sync_hwstate(vcpu);
1457 }
1458
1459 int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
1460 {
1461         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1462
1463         if (!irqchip_in_kernel(vcpu->kvm))
1464                 return 0;
1465
1466         return test_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
1467 }
1468
1469 int kvm_vgic_vcpu_active_irq(struct kvm_vcpu *vcpu)
1470 {
1471         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1472
1473         if (!irqchip_in_kernel(vcpu->kvm))
1474                 return 0;
1475
1476         return test_bit(vcpu->vcpu_id, dist->irq_active_on_cpu);
1477 }
1478
1479
1480 void vgic_kick_vcpus(struct kvm *kvm)
1481 {
1482         struct kvm_vcpu *vcpu;
1483         int c;
1484
1485         /*
1486          * We've injected an interrupt, time to find out who deserves
1487          * a good kick...
1488          */
1489         kvm_for_each_vcpu(c, vcpu, kvm) {
1490                 if (kvm_vgic_vcpu_pending_irq(vcpu))
1491                         kvm_vcpu_kick(vcpu);
1492         }
1493 }
1494
1495 static int vgic_validate_injection(struct kvm_vcpu *vcpu, int irq, int level)
1496 {
1497         int edge_triggered = vgic_irq_is_edge(vcpu, irq);
1498
1499         /*
1500          * Only inject an interrupt if:
1501          * - edge triggered and we have a rising edge
1502          * - level triggered and we change level
1503          */
1504         if (edge_triggered) {
1505                 int state = vgic_dist_irq_is_pending(vcpu, irq);
1506                 return level > state;
1507         } else {
1508                 int state = vgic_dist_irq_get_level(vcpu, irq);
1509                 return level != state;
1510         }
1511 }
1512
1513 static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
1514                                   unsigned int irq_num, bool level)
1515 {
1516         struct vgic_dist *dist = &kvm->arch.vgic;
1517         struct kvm_vcpu *vcpu;
1518         int edge_triggered, level_triggered;
1519         int enabled;
1520         bool ret = true, can_inject = true;
1521
1522         spin_lock(&dist->lock);
1523
1524         vcpu = kvm_get_vcpu(kvm, cpuid);
1525         edge_triggered = vgic_irq_is_edge(vcpu, irq_num);
1526         level_triggered = !edge_triggered;
1527
1528         if (!vgic_validate_injection(vcpu, irq_num, level)) {
1529                 ret = false;
1530                 goto out;
1531         }
1532
1533         if (irq_num >= VGIC_NR_PRIVATE_IRQS) {
1534                 cpuid = dist->irq_spi_cpu[irq_num - VGIC_NR_PRIVATE_IRQS];
1535                 if (cpuid == VCPU_NOT_ALLOCATED) {
1536                         /* Pretend we use CPU0, and prevent injection */
1537                         cpuid = 0;
1538                         can_inject = false;
1539                 }
1540                 vcpu = kvm_get_vcpu(kvm, cpuid);
1541         }
1542
1543         kvm_debug("Inject IRQ%d level %d CPU%d\n", irq_num, level, cpuid);
1544
1545         if (level) {
1546                 if (level_triggered)
1547                         vgic_dist_irq_set_level(vcpu, irq_num);
1548                 vgic_dist_irq_set_pending(vcpu, irq_num);
1549         } else {
1550                 if (level_triggered) {
1551                         vgic_dist_irq_clear_level(vcpu, irq_num);
1552                         if (!vgic_dist_irq_soft_pend(vcpu, irq_num))
1553                                 vgic_dist_irq_clear_pending(vcpu, irq_num);
1554                 }
1555
1556                 ret = false;
1557                 goto out;
1558         }
1559
1560         enabled = vgic_irq_is_enabled(vcpu, irq_num);
1561
1562         if (!enabled || !can_inject) {
1563                 ret = false;
1564                 goto out;
1565         }
1566
1567         if (!vgic_can_sample_irq(vcpu, irq_num)) {
1568                 /*
1569                  * Level interrupt in progress, will be picked up
1570                  * when EOId.
1571                  */
1572                 ret = false;
1573                 goto out;
1574         }
1575
1576         if (level) {
1577                 vgic_cpu_irq_set(vcpu, irq_num);
1578                 set_bit(cpuid, dist->irq_pending_on_cpu);
1579         }
1580
1581 out:
1582         spin_unlock(&dist->lock);
1583
1584         return ret ? cpuid : -EINVAL;
1585 }
1586
1587 /**
1588  * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic
1589  * @kvm:     The VM structure pointer
1590  * @cpuid:   The CPU for PPIs
1591  * @irq_num: The IRQ number that is assigned to the device
1592  * @level:   Edge-triggered:  true:  to trigger the interrupt
1593  *                            false: to ignore the call
1594  *           Level-sensitive  true:  activates an interrupt
1595  *                            false: deactivates an interrupt
1596  *
1597  * The GIC is not concerned with devices being active-LOW or active-HIGH for
1598  * level-sensitive interrupts.  You can think of the level parameter as 1
1599  * being HIGH and 0 being LOW and all devices being active-HIGH.
1600  */
1601 int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
1602                         bool level)
1603 {
1604         int ret = 0;
1605         int vcpu_id;
1606
1607         if (unlikely(!vgic_initialized(kvm))) {
1608                 /*
1609                  * We only provide the automatic initialization of the VGIC
1610                  * for the legacy case of a GICv2. Any other type must
1611                  * be explicitly initialized once setup with the respective
1612                  * KVM device call.
1613                  */
1614                 if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2) {
1615                         ret = -EBUSY;
1616                         goto out;
1617                 }
1618                 mutex_lock(&kvm->lock);
1619                 ret = vgic_init(kvm);
1620                 mutex_unlock(&kvm->lock);
1621
1622                 if (ret)
1623                         goto out;
1624         }
1625
1626         vcpu_id = vgic_update_irq_pending(kvm, cpuid, irq_num, level);
1627         if (vcpu_id >= 0) {
1628                 /* kick the specified vcpu */
1629                 kvm_vcpu_kick(kvm_get_vcpu(kvm, vcpu_id));
1630         }
1631
1632 out:
1633         return ret;
1634 }
1635
1636 static irqreturn_t vgic_maintenance_handler(int irq, void *data)
1637 {
1638         /*
1639          * We cannot rely on the vgic maintenance interrupt to be
1640          * delivered synchronously. This means we can only use it to
1641          * exit the VM, and we perform the handling of EOIed
1642          * interrupts on the exit path (see vgic_process_maintenance).
1643          */
1644         return IRQ_HANDLED;
1645 }
1646
1647 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
1648 {
1649         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1650
1651         kfree(vgic_cpu->pending_shared);
1652         kfree(vgic_cpu->active_shared);
1653         kfree(vgic_cpu->pend_act_shared);
1654         kfree(vgic_cpu->vgic_irq_lr_map);
1655         vgic_cpu->pending_shared = NULL;
1656         vgic_cpu->active_shared = NULL;
1657         vgic_cpu->pend_act_shared = NULL;
1658         vgic_cpu->vgic_irq_lr_map = NULL;
1659 }
1660
1661 static int vgic_vcpu_init_maps(struct kvm_vcpu *vcpu, int nr_irqs)
1662 {
1663         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1664
1665         int sz = (nr_irqs - VGIC_NR_PRIVATE_IRQS) / 8;
1666         vgic_cpu->pending_shared = kzalloc(sz, GFP_KERNEL);
1667         vgic_cpu->active_shared = kzalloc(sz, GFP_KERNEL);
1668         vgic_cpu->pend_act_shared = kzalloc(sz, GFP_KERNEL);
1669         vgic_cpu->vgic_irq_lr_map = kmalloc(nr_irqs, GFP_KERNEL);
1670
1671         if (!vgic_cpu->pending_shared
1672                 || !vgic_cpu->active_shared
1673                 || !vgic_cpu->pend_act_shared
1674                 || !vgic_cpu->vgic_irq_lr_map) {
1675                 kvm_vgic_vcpu_destroy(vcpu);
1676                 return -ENOMEM;
1677         }
1678
1679         memset(vgic_cpu->vgic_irq_lr_map, LR_EMPTY, nr_irqs);
1680
1681         /*
1682          * Store the number of LRs per vcpu, so we don't have to go
1683          * all the way to the distributor structure to find out. Only
1684          * assembly code should use this one.
1685          */
1686         vgic_cpu->nr_lr = vgic->nr_lr;
1687
1688         return 0;
1689 }
1690
1691 /**
1692  * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
1693  *
1694  * The host's GIC naturally limits the maximum amount of VCPUs a guest
1695  * can use.
1696  */
1697 int kvm_vgic_get_max_vcpus(void)
1698 {
1699         return vgic->max_gic_vcpus;
1700 }
1701
1702 void kvm_vgic_destroy(struct kvm *kvm)
1703 {
1704         struct vgic_dist *dist = &kvm->arch.vgic;
1705         struct kvm_vcpu *vcpu;
1706         int i;
1707
1708         kvm_for_each_vcpu(i, vcpu, kvm)
1709                 kvm_vgic_vcpu_destroy(vcpu);
1710
1711         vgic_free_bitmap(&dist->irq_enabled);
1712         vgic_free_bitmap(&dist->irq_level);
1713         vgic_free_bitmap(&dist->irq_pending);
1714         vgic_free_bitmap(&dist->irq_soft_pend);
1715         vgic_free_bitmap(&dist->irq_queued);
1716         vgic_free_bitmap(&dist->irq_cfg);
1717         vgic_free_bytemap(&dist->irq_priority);
1718         if (dist->irq_spi_target) {
1719                 for (i = 0; i < dist->nr_cpus; i++)
1720                         vgic_free_bitmap(&dist->irq_spi_target[i]);
1721         }
1722         kfree(dist->irq_sgi_sources);
1723         kfree(dist->irq_spi_cpu);
1724         kfree(dist->irq_spi_mpidr);
1725         kfree(dist->irq_spi_target);
1726         kfree(dist->irq_pending_on_cpu);
1727         kfree(dist->irq_active_on_cpu);
1728         dist->irq_sgi_sources = NULL;
1729         dist->irq_spi_cpu = NULL;
1730         dist->irq_spi_target = NULL;
1731         dist->irq_pending_on_cpu = NULL;
1732         dist->irq_active_on_cpu = NULL;
1733         dist->nr_cpus = 0;
1734 }
1735
1736 /*
1737  * Allocate and initialize the various data structures. Must be called
1738  * with kvm->lock held!
1739  */
1740 int vgic_init(struct kvm *kvm)
1741 {
1742         struct vgic_dist *dist = &kvm->arch.vgic;
1743         struct kvm_vcpu *vcpu;
1744         int nr_cpus, nr_irqs;
1745         int ret, i, vcpu_id;
1746
1747         if (vgic_initialized(kvm))
1748                 return 0;
1749
1750         nr_cpus = dist->nr_cpus = atomic_read(&kvm->online_vcpus);
1751         if (!nr_cpus)           /* No vcpus? Can't be good... */
1752                 return -ENODEV;
1753
1754         /*
1755          * If nobody configured the number of interrupts, use the
1756          * legacy one.
1757          */
1758         if (!dist->nr_irqs)
1759                 dist->nr_irqs = VGIC_NR_IRQS_LEGACY;
1760
1761         nr_irqs = dist->nr_irqs;
1762
1763         ret  = vgic_init_bitmap(&dist->irq_enabled, nr_cpus, nr_irqs);
1764         ret |= vgic_init_bitmap(&dist->irq_level, nr_cpus, nr_irqs);
1765         ret |= vgic_init_bitmap(&dist->irq_pending, nr_cpus, nr_irqs);
1766         ret |= vgic_init_bitmap(&dist->irq_soft_pend, nr_cpus, nr_irqs);
1767         ret |= vgic_init_bitmap(&dist->irq_queued, nr_cpus, nr_irqs);
1768         ret |= vgic_init_bitmap(&dist->irq_active, nr_cpus, nr_irqs);
1769         ret |= vgic_init_bitmap(&dist->irq_cfg, nr_cpus, nr_irqs);
1770         ret |= vgic_init_bytemap(&dist->irq_priority, nr_cpus, nr_irqs);
1771
1772         if (ret)
1773                 goto out;
1774
1775         dist->irq_sgi_sources = kzalloc(nr_cpus * VGIC_NR_SGIS, GFP_KERNEL);
1776         dist->irq_spi_cpu = kzalloc(nr_irqs - VGIC_NR_PRIVATE_IRQS, GFP_KERNEL);
1777         dist->irq_spi_target = kzalloc(sizeof(*dist->irq_spi_target) * nr_cpus,
1778                                        GFP_KERNEL);
1779         dist->irq_pending_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
1780                                            GFP_KERNEL);
1781         dist->irq_active_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
1782                                            GFP_KERNEL);
1783         if (!dist->irq_sgi_sources ||
1784             !dist->irq_spi_cpu ||
1785             !dist->irq_spi_target ||
1786             !dist->irq_pending_on_cpu ||
1787             !dist->irq_active_on_cpu) {
1788                 ret = -ENOMEM;
1789                 goto out;
1790         }
1791
1792         for (i = 0; i < nr_cpus; i++)
1793                 ret |= vgic_init_bitmap(&dist->irq_spi_target[i],
1794                                         nr_cpus, nr_irqs);
1795
1796         if (ret)
1797                 goto out;
1798
1799         ret = kvm->arch.vgic.vm_ops.init_model(kvm);
1800         if (ret)
1801                 goto out;
1802
1803         kvm_for_each_vcpu(vcpu_id, vcpu, kvm) {
1804                 ret = vgic_vcpu_init_maps(vcpu, nr_irqs);
1805                 if (ret) {
1806                         kvm_err("VGIC: Failed to allocate vcpu memory\n");
1807                         break;
1808                 }
1809
1810                 for (i = 0; i < dist->nr_irqs; i++) {
1811                         if (i < VGIC_NR_PPIS)
1812                                 vgic_bitmap_set_irq_val(&dist->irq_enabled,
1813                                                         vcpu->vcpu_id, i, 1);
1814                         if (i < VGIC_NR_PRIVATE_IRQS)
1815                                 vgic_bitmap_set_irq_val(&dist->irq_cfg,
1816                                                         vcpu->vcpu_id, i,
1817                                                         VGIC_CFG_EDGE);
1818                 }
1819
1820                 vgic_enable(vcpu);
1821         }
1822
1823 out:
1824         if (ret)
1825                 kvm_vgic_destroy(kvm);
1826
1827         return ret;
1828 }
1829
1830 static int init_vgic_model(struct kvm *kvm, int type)
1831 {
1832         switch (type) {
1833         case KVM_DEV_TYPE_ARM_VGIC_V2:
1834                 vgic_v2_init_emulation(kvm);
1835                 break;
1836 #ifdef CONFIG_ARM_GIC_V3
1837         case KVM_DEV_TYPE_ARM_VGIC_V3:
1838                 vgic_v3_init_emulation(kvm);
1839                 break;
1840 #endif
1841         default:
1842                 return -ENODEV;
1843         }
1844
1845         if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus)
1846                 return -E2BIG;
1847
1848         return 0;
1849 }
1850
1851 int kvm_vgic_create(struct kvm *kvm, u32 type)
1852 {
1853         int i, vcpu_lock_idx = -1, ret;
1854         struct kvm_vcpu *vcpu;
1855
1856         mutex_lock(&kvm->lock);
1857
1858         if (irqchip_in_kernel(kvm)) {
1859                 ret = -EEXIST;
1860                 goto out;
1861         }
1862
1863         /*
1864          * This function is also called by the KVM_CREATE_IRQCHIP handler,
1865          * which had no chance yet to check the availability of the GICv2
1866          * emulation. So check this here again. KVM_CREATE_DEVICE does
1867          * the proper checks already.
1868          */
1869         if (type == KVM_DEV_TYPE_ARM_VGIC_V2 && !vgic->can_emulate_gicv2)
1870                 return -ENODEV;
1871
1872         /*
1873          * Any time a vcpu is run, vcpu_load is called which tries to grab the
1874          * vcpu->mutex.  By grabbing the vcpu->mutex of all VCPUs we ensure
1875          * that no other VCPUs are run while we create the vgic.
1876          */
1877         ret = -EBUSY;
1878         kvm_for_each_vcpu(i, vcpu, kvm) {
1879                 if (!mutex_trylock(&vcpu->mutex))
1880                         goto out_unlock;
1881                 vcpu_lock_idx = i;
1882         }
1883
1884         kvm_for_each_vcpu(i, vcpu, kvm) {
1885                 if (vcpu->arch.has_run_once)
1886                         goto out_unlock;
1887         }
1888         ret = 0;
1889
1890         ret = init_vgic_model(kvm, type);
1891         if (ret)
1892                 goto out_unlock;
1893
1894         spin_lock_init(&kvm->arch.vgic.lock);
1895         kvm->arch.vgic.in_kernel = true;
1896         kvm->arch.vgic.vgic_model = type;
1897         kvm->arch.vgic.vctrl_base = vgic->vctrl_base;
1898         kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
1899         kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
1900         kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF;
1901
1902 out_unlock:
1903         for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
1904                 vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
1905                 mutex_unlock(&vcpu->mutex);
1906         }
1907
1908 out:
1909         mutex_unlock(&kvm->lock);
1910         return ret;
1911 }
1912
1913 static int vgic_ioaddr_overlap(struct kvm *kvm)
1914 {
1915         phys_addr_t dist = kvm->arch.vgic.vgic_dist_base;
1916         phys_addr_t cpu = kvm->arch.vgic.vgic_cpu_base;
1917
1918         if (IS_VGIC_ADDR_UNDEF(dist) || IS_VGIC_ADDR_UNDEF(cpu))
1919                 return 0;
1920         if ((dist <= cpu && dist + KVM_VGIC_V2_DIST_SIZE > cpu) ||
1921             (cpu <= dist && cpu + KVM_VGIC_V2_CPU_SIZE > dist))
1922                 return -EBUSY;
1923         return 0;
1924 }
1925
1926 static int vgic_ioaddr_assign(struct kvm *kvm, phys_addr_t *ioaddr,
1927                               phys_addr_t addr, phys_addr_t size)
1928 {
1929         int ret;
1930
1931         if (addr & ~KVM_PHYS_MASK)
1932                 return -E2BIG;
1933
1934         if (addr & (SZ_4K - 1))
1935                 return -EINVAL;
1936
1937         if (!IS_VGIC_ADDR_UNDEF(*ioaddr))
1938                 return -EEXIST;
1939         if (addr + size < addr)
1940                 return -EINVAL;
1941
1942         *ioaddr = addr;
1943         ret = vgic_ioaddr_overlap(kvm);
1944         if (ret)
1945                 *ioaddr = VGIC_ADDR_UNDEF;
1946
1947         return ret;
1948 }
1949
1950 /**
1951  * kvm_vgic_addr - set or get vgic VM base addresses
1952  * @kvm:   pointer to the vm struct
1953  * @type:  the VGIC addr type, one of KVM_VGIC_V[23]_ADDR_TYPE_XXX
1954  * @addr:  pointer to address value
1955  * @write: if true set the address in the VM address space, if false read the
1956  *          address
1957  *
1958  * Set or get the vgic base addresses for the distributor and the virtual CPU
1959  * interface in the VM physical address space.  These addresses are properties
1960  * of the emulated core/SoC and therefore user space initially knows this
1961  * information.
1962  */
1963 int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)
1964 {
1965         int r = 0;
1966         struct vgic_dist *vgic = &kvm->arch.vgic;
1967         int type_needed;
1968         phys_addr_t *addr_ptr, block_size;
1969         phys_addr_t alignment;
1970
1971         mutex_lock(&kvm->lock);
1972         switch (type) {
1973         case KVM_VGIC_V2_ADDR_TYPE_DIST:
1974                 type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
1975                 addr_ptr = &vgic->vgic_dist_base;
1976                 block_size = KVM_VGIC_V2_DIST_SIZE;
1977                 alignment = SZ_4K;
1978                 break;
1979         case KVM_VGIC_V2_ADDR_TYPE_CPU:
1980                 type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
1981                 addr_ptr = &vgic->vgic_cpu_base;
1982                 block_size = KVM_VGIC_V2_CPU_SIZE;
1983                 alignment = SZ_4K;
1984                 break;
1985 #ifdef CONFIG_ARM_GIC_V3
1986         case KVM_VGIC_V3_ADDR_TYPE_DIST:
1987                 type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
1988                 addr_ptr = &vgic->vgic_dist_base;
1989                 block_size = KVM_VGIC_V3_DIST_SIZE;
1990                 alignment = SZ_64K;
1991                 break;
1992         case KVM_VGIC_V3_ADDR_TYPE_REDIST:
1993                 type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
1994                 addr_ptr = &vgic->vgic_redist_base;
1995                 block_size = KVM_VGIC_V3_REDIST_SIZE;
1996                 alignment = SZ_64K;
1997                 break;
1998 #endif
1999         default:
2000                 r = -ENODEV;
2001                 goto out;
2002         }
2003
2004         if (vgic->vgic_model != type_needed) {
2005                 r = -ENODEV;
2006                 goto out;
2007         }
2008
2009         if (write) {
2010                 if (!IS_ALIGNED(*addr, alignment))
2011                         r = -EINVAL;
2012                 else
2013                         r = vgic_ioaddr_assign(kvm, addr_ptr, *addr,
2014                                                block_size);
2015         } else {
2016                 *addr = *addr_ptr;
2017         }
2018
2019 out:
2020         mutex_unlock(&kvm->lock);
2021         return r;
2022 }
2023
2024 int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2025 {
2026         int r;
2027
2028         switch (attr->group) {
2029         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2030                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2031                 u64 addr;
2032                 unsigned long type = (unsigned long)attr->attr;
2033
2034                 if (copy_from_user(&addr, uaddr, sizeof(addr)))
2035                         return -EFAULT;
2036
2037                 r = kvm_vgic_addr(dev->kvm, type, &addr, true);
2038                 return (r == -ENODEV) ? -ENXIO : r;
2039         }
2040         case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
2041                 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
2042                 u32 val;
2043                 int ret = 0;
2044
2045                 if (get_user(val, uaddr))
2046                         return -EFAULT;
2047
2048                 /*
2049                  * We require:
2050                  * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs
2051                  * - at most 1024 interrupts
2052                  * - a multiple of 32 interrupts
2053                  */
2054                 if (val < (VGIC_NR_PRIVATE_IRQS + 32) ||
2055                     val > VGIC_MAX_IRQS ||
2056                     (val & 31))
2057                         return -EINVAL;
2058
2059                 mutex_lock(&dev->kvm->lock);
2060
2061                 if (vgic_ready(dev->kvm) || dev->kvm->arch.vgic.nr_irqs)
2062                         ret = -EBUSY;
2063                 else
2064                         dev->kvm->arch.vgic.nr_irqs = val;
2065
2066                 mutex_unlock(&dev->kvm->lock);
2067
2068                 return ret;
2069         }
2070         case KVM_DEV_ARM_VGIC_GRP_CTRL: {
2071                 switch (attr->attr) {
2072                 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2073                         r = vgic_init(dev->kvm);
2074                         return r;
2075                 }
2076                 break;
2077         }
2078         }
2079
2080         return -ENXIO;
2081 }
2082
2083 int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2084 {
2085         int r = -ENXIO;
2086
2087         switch (attr->group) {
2088         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2089                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2090                 u64 addr;
2091                 unsigned long type = (unsigned long)attr->attr;
2092
2093                 r = kvm_vgic_addr(dev->kvm, type, &addr, false);
2094                 if (r)
2095                         return (r == -ENODEV) ? -ENXIO : r;
2096
2097                 if (copy_to_user(uaddr, &addr, sizeof(addr)))
2098                         return -EFAULT;
2099                 break;
2100         }
2101         case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
2102                 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
2103
2104                 r = put_user(dev->kvm->arch.vgic.nr_irqs, uaddr);
2105                 break;
2106         }
2107
2108         }
2109
2110         return r;
2111 }
2112
2113 int vgic_has_attr_regs(const struct vgic_io_range *ranges, phys_addr_t offset)
2114 {
2115         if (vgic_find_range(ranges, 4, offset))
2116                 return 0;
2117         else
2118                 return -ENXIO;
2119 }
2120
2121 static void vgic_init_maintenance_interrupt(void *info)
2122 {
2123         enable_percpu_irq(vgic->maint_irq, 0);
2124 }
2125
2126 static int vgic_cpu_notify(struct notifier_block *self,
2127                            unsigned long action, void *cpu)
2128 {
2129         switch (action) {
2130         case CPU_STARTING:
2131         case CPU_STARTING_FROZEN:
2132                 vgic_init_maintenance_interrupt(NULL);
2133                 break;
2134         case CPU_DYING:
2135         case CPU_DYING_FROZEN:
2136                 disable_percpu_irq(vgic->maint_irq);
2137                 break;
2138         }
2139
2140         return NOTIFY_OK;
2141 }
2142
2143 static struct notifier_block vgic_cpu_nb = {
2144         .notifier_call = vgic_cpu_notify,
2145 };
2146
2147 static const struct of_device_id vgic_ids[] = {
2148         { .compatible = "arm,cortex-a15-gic",   .data = vgic_v2_probe, },
2149         { .compatible = "arm,cortex-a7-gic",    .data = vgic_v2_probe, },
2150         { .compatible = "arm,gic-400",          .data = vgic_v2_probe, },
2151         { .compatible = "arm,gic-v3",           .data = vgic_v3_probe, },
2152         {},
2153 };
2154
2155 int kvm_vgic_hyp_init(void)
2156 {
2157         const struct of_device_id *matched_id;
2158         const int (*vgic_probe)(struct device_node *,const struct vgic_ops **,
2159                                 const struct vgic_params **);
2160         struct device_node *vgic_node;
2161         int ret;
2162
2163         vgic_node = of_find_matching_node_and_match(NULL,
2164                                                     vgic_ids, &matched_id);
2165         if (!vgic_node) {
2166                 kvm_err("error: no compatible GIC node found\n");
2167                 return -ENODEV;
2168         }
2169
2170         vgic_probe = matched_id->data;
2171         ret = vgic_probe(vgic_node, &vgic_ops, &vgic);
2172         if (ret)
2173                 return ret;
2174
2175         ret = request_percpu_irq(vgic->maint_irq, vgic_maintenance_handler,
2176                                  "vgic", kvm_get_running_vcpus());
2177         if (ret) {
2178                 kvm_err("Cannot register interrupt %d\n", vgic->maint_irq);
2179                 return ret;
2180         }
2181
2182         ret = __register_cpu_notifier(&vgic_cpu_nb);
2183         if (ret) {
2184                 kvm_err("Cannot register vgic CPU notifier\n");
2185                 goto out_free_irq;
2186         }
2187
2188         /* Callback into for arch code for setup */
2189         vgic_arch_setup(vgic);
2190
2191         on_each_cpu(vgic_init_maintenance_interrupt, NULL, 1);
2192
2193         return 0;
2194
2195 out_free_irq:
2196         free_percpu_irq(vgic->maint_irq, kvm_get_running_vcpus());
2197         return ret;
2198 }
2199
2200 int kvm_irq_map_gsi(struct kvm *kvm,
2201                     struct kvm_kernel_irq_routing_entry *entries,
2202                     int gsi)
2203 {
2204         return gsi;
2205 }
2206
2207 int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)
2208 {
2209         return pin;
2210 }
2211
2212 int kvm_set_irq(struct kvm *kvm, int irq_source_id,
2213                 u32 irq, int level, bool line_status)
2214 {
2215         unsigned int spi = irq + VGIC_NR_PRIVATE_IRQS;
2216
2217         trace_kvm_set_irq(irq, level, irq_source_id);
2218
2219         BUG_ON(!vgic_initialized(kvm));
2220
2221         if (spi > kvm->arch.vgic.nr_irqs)
2222                 return -EINVAL;
2223         return kvm_vgic_inject_irq(kvm, 0, spi, level);
2224
2225 }
2226
2227 /* MSI not implemented yet */
2228 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
2229                 struct kvm *kvm, int irq_source_id,
2230                 int level, bool line_status)
2231 {
2232         return 0;
2233 }