virt/kvm/arm/vgic/vgic-its.c

   1 /*
   2  * GICv3 ITS emulation
   3  *
   4  * Copyright (C) 2015,2016 ARM Ltd.
   5  * Author: Andre Przywara <andre.przywara@arm.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include <linux/cpu.h>
  21 #include <linux/kvm.h>
  22 #include <linux/kvm_host.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/list.h>
  25 #include <linux/uaccess.h>
  26
  27 #include <linux/irqchip/arm-gic-v3.h>
  28
  29 #include <asm/kvm_emulate.h>
  30 #include <asm/kvm_arm.h>
  31 #include <asm/kvm_mmu.h>
  32
  33 #include "vgic.h"
  34 #include "vgic-mmio.h"
  35
  36 static int vgic_its_save_tables_v0(struct vgic_its *its);
  37 static int vgic_its_restore_tables_v0(struct vgic_its *its);
  38 static int vgic_its_commit_v0(struct vgic_its *its);
  39
  40 /*
  41  * Creates a new (reference to a) struct vgic_irq for a given LPI.
  42  * If this LPI is already mapped on another ITS, we increase its refcount
  43  * and return a pointer to the existing structure.
  44  * If this is a "new" LPI, we allocate and initialize a new struct vgic_irq.
  45  * This function returns a pointer to the _unlocked_ structure.
  46  */
  47 static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid)
  48 {
  49         struct vgic_dist *dist = &kvm->arch.vgic;
  50         struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid), *oldirq;
  51
  52         /* In this case there is no put, since we keep the reference. */
  53         if (irq)
  54                 return irq;
  55
  56         irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL);
  57         if (!irq)
  58                 return ERR_PTR(-ENOMEM);
  59
  60         INIT_LIST_HEAD(&irq->lpi_list);
  61         INIT_LIST_HEAD(&irq->ap_list);
  62         spin_lock_init(&irq->irq_lock);
  63
  64         irq->config = VGIC_CONFIG_EDGE;
  65         kref_init(&irq->refcount);
  66         irq->intid = intid;
  67
  68         spin_lock(&dist->lpi_list_lock);
  69
  70         /*
  71          * There could be a race with another vgic_add_lpi(), so we need to
  72          * check that we don't add a second list entry with the same LPI.
  73          */
  74         list_for_each_entry(oldirq, &dist->lpi_list_head, lpi_list) {
  75                 if (oldirq->intid != intid)
  76                         continue;
  77
  78                 /* Someone was faster with adding this LPI, lets use that. */
  79                 kfree(irq);
  80                 irq = oldirq;
  81
  82                 /*
  83                  * This increases the refcount, the caller is expected to
  84                  * call vgic_put_irq() on the returned pointer once it's
  85                  * finished with the IRQ.
  86                  */
  87                 vgic_get_irq_kref(irq);
  88
  89                 goto out_unlock;
  90         }
  91
  92         list_add_tail(&irq->lpi_list, &dist->lpi_list_head);
  93         dist->lpi_list_count++;
  94
  95 out_unlock:
  96         spin_unlock(&dist->lpi_list_lock);
  97
  98         return irq;
  99 }
 100
 101 struct its_device {
 102         struct list_head dev_list;
 103
 104         /* the head for the list of ITTEs */
 105         struct list_head itt_head;
 106         u32 device_id;
 107 };
 108
 109 #define COLLECTION_NOT_MAPPED ((u32)~0)
 110
 111 struct its_collection {
 112         struct list_head coll_list;
 113
 114         u32 collection_id;
 115         u32 target_addr;
 116 };
 117
 118 #define its_is_collection_mapped(coll) ((coll) && \
 119                                 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
 120
 121 struct its_ite {
 122         struct list_head ite_list;
 123
 124         struct vgic_irq *irq;
 125         struct its_collection *collection;
 126         u32 lpi;
 127         u32 event_id;
 128 };
 129
 130 /**
 131  * struct vgic_its_abi - ITS abi ops and settings
 132  * @cte_esz: collection table entry size
 133  * @dte_esz: device table entry size
 134  * @ite_esz: interrupt translation table entry size
 135  * @save tables: save the ITS tables into guest RAM
 136  * @restore_tables: restore the ITS internal structs from tables
 137  *  stored in guest RAM
 138  * @commit: initialize the registers which expose the ABI settings,
 139  *  especially the entry sizes
 140  */
 141 struct vgic_its_abi {
 142         int cte_esz;
 143         int dte_esz;
 144         int ite_esz;
 145         int (*save_tables)(struct vgic_its *its);
 146         int (*restore_tables)(struct vgic_its *its);
 147         int (*commit)(struct vgic_its *its);
 148 };
 149
 150 static const struct vgic_its_abi its_table_abi_versions[] = {
 151         [0] = {.cte_esz = 8, .dte_esz = 8, .ite_esz = 8,
 152          .save_tables = vgic_its_save_tables_v0,
 153          .restore_tables = vgic_its_restore_tables_v0,
 154          .commit = vgic_its_commit_v0,
 155         },
 156 };
 157
 158 #define NR_ITS_ABIS     ARRAY_SIZE(its_table_abi_versions)
 159
 160 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
 161 {
 162         return &its_table_abi_versions[its->abi_rev];
 163 }
 164
 165 int vgic_its_set_abi(struct vgic_its *its, int rev)
 166 {
 167         const struct vgic_its_abi *abi;
 168
 169         its->abi_rev = rev;
 170         abi = vgic_its_get_abi(its);
 171         return abi->commit(its);
 172 }
 173
 174 /*
 175  * Find and returns a device in the device table for an ITS.
 176  * Must be called with the its_lock mutex held.
 177  */
 178 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
 179 {
 180         struct its_device *device;
 181
 182         list_for_each_entry(device, &its->device_list, dev_list)
 183                 if (device_id == device->device_id)
 184                         return device;
 185
 186         return NULL;
 187 }
 188
 189 /*
 190  * Find and returns an interrupt translation table entry (ITTE) for a given
 191  * Device ID/Event ID pair on an ITS.
 192  * Must be called with the its_lock mutex held.
 193  */
 194 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
 195                                   u32 event_id)
 196 {
 197         struct its_device *device;
 198         struct its_ite *ite;
 199
 200         device = find_its_device(its, device_id);
 201         if (device == NULL)
 202                 return NULL;
 203
 204         list_for_each_entry(ite, &device->itt_head, ite_list)
 205                 if (ite->event_id == event_id)
 206                         return ite;
 207
 208         return NULL;
 209 }
 210
 211 /* To be used as an iterator this macro misses the enclosing parentheses */
 212 #define for_each_lpi_its(dev, ite, its) \
 213         list_for_each_entry(dev, &(its)->device_list, dev_list) \
 214                 list_for_each_entry(ite, &(dev)->itt_head, ite_list)
 215
 216 /*
 217  * We only implement 48 bits of PA at the moment, although the ITS
 218  * supports more. Let's be restrictive here.
 219  */
 220 #define BASER_ADDRESS(x)        ((x) & GENMASK_ULL(47, 16))
 221 #define CBASER_ADDRESS(x)       ((x) & GENMASK_ULL(47, 12))
 222 #define PENDBASER_ADDRESS(x)    ((x) & GENMASK_ULL(47, 16))
 223 #define PROPBASER_ADDRESS(x)    ((x) & GENMASK_ULL(47, 12))
 224
 225 #define GIC_LPI_OFFSET 8192
 226
 227 /*
 228  * Finds and returns a collection in the ITS collection table.
 229  * Must be called with the its_lock mutex held.
 230  */
 231 static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
 232 {
 233         struct its_collection *collection;
 234
 235         list_for_each_entry(collection, &its->collection_list, coll_list) {
 236                 if (coll_id == collection->collection_id)
 237                         return collection;
 238         }
 239
 240         return NULL;
 241 }
 242
 243 #define LPI_PROP_ENABLE_BIT(p)  ((p) & LPI_PROP_ENABLED)
 244 #define LPI_PROP_PRIORITY(p)    ((p) & 0xfc)
 245
 246 /*
 247  * Reads the configuration data for a given LPI from guest memory and
 248  * updates the fields in struct vgic_irq.
 249  * If filter_vcpu is not NULL, applies only if the IRQ is targeting this
 250  * VCPU. Unconditionally applies if filter_vcpu is NULL.
 251  */
 252 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
 253                              struct kvm_vcpu *filter_vcpu)
 254 {
 255         u64 propbase = PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
 256         u8 prop;
 257         int ret;
 258
 259         ret = kvm_read_guest(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
 260                              &prop, 1);
 261
 262         if (ret)
 263                 return ret;
 264
 265         spin_lock(&irq->irq_lock);
 266
 267         if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
 268                 irq->priority = LPI_PROP_PRIORITY(prop);
 269                 irq->enabled = LPI_PROP_ENABLE_BIT(prop);
 270
 271                 vgic_queue_irq_unlock(kvm, irq);
 272         } else {
 273                 spin_unlock(&irq->irq_lock);
 274         }
 275
 276         return 0;
 277 }
 278
 279 /*
 280  * Create a snapshot of the current LPI list, so that we can enumerate all
 281  * LPIs without holding any lock.
 282  * Returns the array length and puts the kmalloc'ed array into intid_ptr.
 283  */
 284 static int vgic_copy_lpi_list(struct kvm *kvm, u32 **intid_ptr)
 285 {
 286         struct vgic_dist *dist = &kvm->arch.vgic;
 287         struct vgic_irq *irq;
 288         u32 *intids;
 289         int irq_count = dist->lpi_list_count, i = 0;
 290
 291         /*
 292          * We use the current value of the list length, which may change
 293          * after the kmalloc. We don't care, because the guest shouldn't
 294          * change anything while the command handling is still running,
 295          * and in the worst case we would miss a new IRQ, which one wouldn't
 296          * expect to be covered by this command anyway.
 297          */
 298         intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL);
 299         if (!intids)
 300                 return -ENOMEM;
 301
 302         spin_lock(&dist->lpi_list_lock);
 303         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
 304                 /* We don't need to "get" the IRQ, as we hold the list lock. */
 305                 intids[i] = irq->intid;
 306                 if (++i == irq_count)
 307                         break;
 308         }
 309         spin_unlock(&dist->lpi_list_lock);
 310
 311         *intid_ptr = intids;
 312         return irq_count;
 313 }
 314
 315 /*
 316  * Promotes the ITS view of affinity of an ITTE (which redistributor this LPI
 317  * is targeting) to the VGIC's view, which deals with target VCPUs.
 318  * Needs to be called whenever either the collection for a LPIs has
 319  * changed or the collection itself got retargeted.
 320  */
 321 static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
 322 {
 323         struct kvm_vcpu *vcpu;
 324
 325         if (!its_is_collection_mapped(ite->collection))
 326                 return;
 327
 328         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
 329
 330         spin_lock(&ite->irq->irq_lock);
 331         ite->irq->target_vcpu = vcpu;
 332         spin_unlock(&ite->irq->irq_lock);
 333 }
 334
 335 /*
 336  * Updates the target VCPU for every LPI targeting this collection.
 337  * Must be called with the its_lock mutex held.
 338  */
 339 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
 340                                        struct its_collection *coll)
 341 {
 342         struct its_device *device;
 343         struct its_ite *ite;
 344
 345         for_each_lpi_its(device, ite, its) {
 346                 if (!ite->collection || coll != ite->collection)
 347                         continue;
 348
 349                 update_affinity_ite(kvm, ite);
 350         }
 351 }
 352
 353 static u32 max_lpis_propbaser(u64 propbaser)
 354 {
 355         int nr_idbits = (propbaser & 0x1f) + 1;
 356
 357         return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS);
 358 }
 359
 360 /*
 361  * Scan the whole LPI pending table and sync the pending bit in there
 362  * with our own data structures. This relies on the LPI being
 363  * mapped before.
 364  */
 365 static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
 366 {
 367         gpa_t pendbase = PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);
 368         struct vgic_irq *irq;
 369         int last_byte_offset = -1;
 370         int ret = 0;
 371         u32 *intids;
 372         int nr_irqs, i;
 373
 374         nr_irqs = vgic_copy_lpi_list(vcpu->kvm, &intids);
 375         if (nr_irqs < 0)
 376                 return nr_irqs;
 377
 378         for (i = 0; i < nr_irqs; i++) {
 379                 int byte_offset, bit_nr;
 380                 u8 pendmask;
 381
 382                 byte_offset = intids[i] / BITS_PER_BYTE;
 383                 bit_nr = intids[i] % BITS_PER_BYTE;
 384
 385                 /*
 386                  * For contiguously allocated LPIs chances are we just read
 387                  * this very same byte in the last iteration. Reuse that.
 388                  */
 389                 if (byte_offset != last_byte_offset) {
 390                         ret = kvm_read_guest(vcpu->kvm, pendbase + byte_offset,
 391                                              &pendmask, 1);
 392                         if (ret) {
 393                                 kfree(intids);
 394                                 return ret;
 395                         }
 396                         last_byte_offset = byte_offset;
 397                 }
 398
 399                 irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]);
 400                 spin_lock(&irq->irq_lock);
 401                 irq->pending_latch = pendmask & (1U << bit_nr);
 402                 vgic_queue_irq_unlock(vcpu->kvm, irq);
 403                 vgic_put_irq(vcpu->kvm, irq);
 404         }
 405
 406         kfree(intids);
 407
 408         return ret;
 409 }
 410
 411 static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
 412                                               struct vgic_its *its,
 413                                               gpa_t addr, unsigned int len)
 414 {
 415         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
 416         u64 reg = GITS_TYPER_PLPIS;
 417
 418         /*
 419          * We use linear CPU numbers for redistributor addressing,
 420          * so GITS_TYPER.PTA is 0.
 421          * Also we force all PROPBASER registers to be the same, so
 422          * CommonLPIAff is 0 as well.
 423          * To avoid memory waste in the guest, we keep the number of IDBits and
 424          * DevBits low - as least for the time being.
 425          */
 426         reg |= 0x0f << GITS_TYPER_DEVBITS_SHIFT;
 427         reg |= 0x0f << GITS_TYPER_IDBITS_SHIFT;
 428         reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT;
 429
 430         return extract_bytes(reg, addr & 7, len);
 431 }
 432
 433 static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
 434                                              struct vgic_its *its,
 435                                              gpa_t addr, unsigned int len)
 436 {
 437         u32 val;
 438
 439         val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
 440         val |= (PRODUCT_ID_KVM << GITS_IIDR_PRODUCTID_SHIFT) | IMPLEMENTER_ARM;
 441         return val;
 442 }
 443
 444 static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm,
 445                                             struct vgic_its *its,
 446                                             gpa_t addr, unsigned int len,
 447                                             unsigned long val)
 448 {
 449         u32 rev = GITS_IIDR_REV(val);
 450
 451         if (rev >= NR_ITS_ABIS)
 452                 return -EINVAL;
 453         return vgic_its_set_abi(its, rev);
 454 }
 455
 456 static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
 457                                                struct vgic_its *its,
 458                                                gpa_t addr, unsigned int len)
 459 {
 460         switch (addr & 0xffff) {
 461         case GITS_PIDR0:
 462                 return 0x92;    /* part number, bits[7:0] */
 463         case GITS_PIDR1:
 464                 return 0xb4;    /* part number, bits[11:8] */
 465         case GITS_PIDR2:
 466                 return GIC_PIDR2_ARCH_GICv3 | 0x0b;
 467         case GITS_PIDR4:
 468                 return 0x40;    /* This is a 64K software visible page */
 469         /* The following are the ID registers for (any) GIC. */
 470         case GITS_CIDR0:
 471                 return 0x0d;
 472         case GITS_CIDR1:
 473                 return 0xf0;
 474         case GITS_CIDR2:
 475                 return 0x05;
 476         case GITS_CIDR3:
 477                 return 0xb1;
 478         }
 479
 480         return 0;
 481 }
 482
 483 /*
 484  * Find the target VCPU and the LPI number for a given devid/eventid pair
 485  * and make this IRQ pending, possibly injecting it.
 486  * Must be called with the its_lock mutex held.
 487  * Returns 0 on success, a positive error value for any ITS mapping
 488  * related errors and negative error values for generic errors.
 489  */
 490 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
 491                                 u32 devid, u32 eventid)
 492 {
 493         struct kvm_vcpu *vcpu;
 494         struct its_ite *ite;
 495
 496         if (!its->enabled)
 497                 return -EBUSY;
 498
 499         ite = find_ite(its, devid, eventid);
 500         if (!ite || !its_is_collection_mapped(ite->collection))
 501                 return E_ITS_INT_UNMAPPED_INTERRUPT;
 502
 503         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
 504         if (!vcpu)
 505                 return E_ITS_INT_UNMAPPED_INTERRUPT;
 506
 507         if (!vcpu->arch.vgic_cpu.lpis_enabled)
 508                 return -EBUSY;
 509
 510         spin_lock(&ite->irq->irq_lock);
 511         ite->irq->pending_latch = true;
 512         vgic_queue_irq_unlock(kvm, ite->irq);
 513
 514         return 0;
 515 }
 516
 517 static struct vgic_io_device *vgic_get_its_iodev(struct kvm_io_device *dev)
 518 {
 519         struct vgic_io_device *iodev;
 520
 521         if (dev->ops != &kvm_io_gic_ops)
 522                 return NULL;
 523
 524         iodev = container_of(dev, struct vgic_io_device, dev);
 525
 526         if (iodev->iodev_type != IODEV_ITS)
 527                 return NULL;
 528
 529         return iodev;
 530 }
 531
 532 /*
 533  * Queries the KVM IO bus framework to get the ITS pointer from the given
 534  * doorbell address.
 535  * We then call vgic_its_trigger_msi() with the decoded data.
 536  * According to the KVM_SIGNAL_MSI API description returns 1 on success.
 537  */
 538 int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
 539 {
 540         u64 address;
 541         struct kvm_io_device *kvm_io_dev;
 542         struct vgic_io_device *iodev;
 543         int ret;
 544
 545         if (!vgic_has_its(kvm))
 546                 return -ENODEV;
 547
 548         if (!(msi->flags & KVM_MSI_VALID_DEVID))
 549                 return -EINVAL;
 550
 551         address = (u64)msi->address_hi << 32 | msi->address_lo;
 552
 553         kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, address);
 554         if (!kvm_io_dev)
 555                 return -EINVAL;
 556
 557         iodev = vgic_get_its_iodev(kvm_io_dev);
 558         if (!iodev)
 559                 return -EINVAL;
 560
 561         mutex_lock(&iodev->its->its_lock);
 562         ret = vgic_its_trigger_msi(kvm, iodev->its, msi->devid, msi->data);
 563         mutex_unlock(&iodev->its->its_lock);
 564
 565         if (ret < 0)
 566                 return ret;
 567
 568         /*
 569          * KVM_SIGNAL_MSI demands a return value > 0 for success and 0
 570          * if the guest has blocked the MSI. So we map any LPI mapping
 571          * related error to that.
 572          */
 573         if (ret)
 574                 return 0;
 575         else
 576                 return 1;
 577 }
 578
 579 /* Requires the its_lock to be held. */
 580 static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
 581 {
 582         list_del(&ite->ite_list);
 583
 584         /* This put matches the get in vgic_add_lpi. */
 585         if (ite->irq)
 586                 vgic_put_irq(kvm, ite->irq);
 587
 588         kfree(ite);
 589 }
 590
 591 static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
 592 {
 593         return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
 594 }
 595
 596 #define its_cmd_get_command(cmd)        its_cmd_mask_field(cmd, 0,  0,  8)
 597 #define its_cmd_get_deviceid(cmd)       its_cmd_mask_field(cmd, 0, 32, 32)
 598 #define its_cmd_get_id(cmd)             its_cmd_mask_field(cmd, 1,  0, 32)
 599 #define its_cmd_get_physical_id(cmd)    its_cmd_mask_field(cmd, 1, 32, 32)
 600 #define its_cmd_get_collection(cmd)     its_cmd_mask_field(cmd, 2,  0, 16)
 601 #define its_cmd_get_target_addr(cmd)    its_cmd_mask_field(cmd, 2, 16, 32)
 602 #define its_cmd_get_validbit(cmd)       its_cmd_mask_field(cmd, 2, 63,  1)
 603
 604 /*
 605  * The DISCARD command frees an Interrupt Translation Table Entry (ITTE).
 606  * Must be called with the its_lock mutex held.
 607  */
 608 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
 609                                        u64 *its_cmd)
 610 {
 611         u32 device_id = its_cmd_get_deviceid(its_cmd);
 612         u32 event_id = its_cmd_get_id(its_cmd);
 613         struct its_ite *ite;
 614
 615
 616         ite = find_ite(its, device_id, event_id);
 617         if (ite && ite->collection) {
 618                 /*
 619                  * Though the spec talks about removing the pending state, we
 620                  * don't bother here since we clear the ITTE anyway and the
 621                  * pending state is a property of the ITTE struct.
 622                  */
 623                 its_free_ite(kvm, ite);
 624                 return 0;
 625         }
 626
 627         return E_ITS_DISCARD_UNMAPPED_INTERRUPT;
 628 }
 629
 630 /*
 631  * The MOVI command moves an ITTE to a different collection.
 632  * Must be called with the its_lock mutex held.
 633  */
 634 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
 635                                     u64 *its_cmd)
 636 {
 637         u32 device_id = its_cmd_get_deviceid(its_cmd);
 638         u32 event_id = its_cmd_get_id(its_cmd);
 639         u32 coll_id = its_cmd_get_collection(its_cmd);
 640         struct kvm_vcpu *vcpu;
 641         struct its_ite *ite;
 642         struct its_collection *collection;
 643
 644         ite = find_ite(its, device_id, event_id);
 645         if (!ite)
 646                 return E_ITS_MOVI_UNMAPPED_INTERRUPT;
 647
 648         if (!its_is_collection_mapped(ite->collection))
 649                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
 650
 651         collection = find_collection(its, coll_id);
 652         if (!its_is_collection_mapped(collection))
 653                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
 654
 655         ite->collection = collection;
 656         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
 657
 658         spin_lock(&ite->irq->irq_lock);
 659         ite->irq->target_vcpu = vcpu;
 660         spin_unlock(&ite->irq->irq_lock);
 661
 662         return 0;
 663 }
 664
 665 /*
 666  * Check whether an ID can be stored into the corresponding guest table.
 667  * For a direct table this is pretty easy, but gets a bit nasty for
 668  * indirect tables. We check whether the resulting guest physical address
 669  * is actually valid (covered by a memslot and guest accessbible).
 670  * For this we have to read the respective first level entry.
 671  */
 672 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, int id)
 673 {
 674         int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
 675         int index;
 676         u64 indirect_ptr;
 677         gfn_t gfn;
 678         int esz = GITS_BASER_ENTRY_SIZE(baser);
 679
 680         if (!(baser & GITS_BASER_INDIRECT)) {
 681                 phys_addr_t addr;
 682
 683                 if (id >= (l1_tbl_size / esz))
 684                         return false;
 685
 686                 addr = BASER_ADDRESS(baser) + id * esz;
 687                 gfn = addr >> PAGE_SHIFT;
 688
 689                 return kvm_is_visible_gfn(its->dev->kvm, gfn);
 690         }
 691
 692         /* calculate and check the index into the 1st level */
 693         index = id / (SZ_64K / esz);
 694         if (index >= (l1_tbl_size / sizeof(u64)))
 695                 return false;
 696
 697         /* Each 1st level entry is represented by a 64-bit value. */
 698         if (kvm_read_guest(its->dev->kvm,
 699                            BASER_ADDRESS(baser) + index * sizeof(indirect_ptr),
 700                            &indirect_ptr, sizeof(indirect_ptr)))
 701                 return false;
 702
 703         indirect_ptr = le64_to_cpu(indirect_ptr);
 704
 705         /* check the valid bit of the first level entry */
 706         if (!(indirect_ptr & BIT_ULL(63)))
 707                 return false;
 708
 709         /*
 710          * Mask the guest physical address and calculate the frame number.
 711          * Any address beyond our supported 48 bits of PA will be caught
 712          * by the actual check in the final step.
 713          */
 714         indirect_ptr &= GENMASK_ULL(51, 16);
 715
 716         /* Find the address of the actual entry */
 717         index = id % (SZ_64K / esz);
 718         indirect_ptr += index * esz;
 719         gfn = indirect_ptr >> PAGE_SHIFT;
 720
 721         return kvm_is_visible_gfn(its->dev->kvm, gfn);
 722 }
 723
 724 static int vgic_its_alloc_collection(struct vgic_its *its,
 725                                      struct its_collection **colp,
 726                                      u32 coll_id)
 727 {
 728         struct its_collection *collection;
 729
 730         if (!vgic_its_check_id(its, its->baser_coll_table, coll_id))
 731                 return E_ITS_MAPC_COLLECTION_OOR;
 732
 733         collection = kzalloc(sizeof(*collection), GFP_KERNEL);
 734
 735         collection->collection_id = coll_id;
 736         collection->target_addr = COLLECTION_NOT_MAPPED;
 737
 738         list_add_tail(&collection->coll_list, &its->collection_list);
 739         *colp = collection;
 740
 741         return 0;
 742 }
 743
 744 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
 745 {
 746         struct its_collection *collection;
 747         struct its_device *device;
 748         struct its_ite *ite;
 749
 750         /*
 751          * Clearing the mapping for that collection ID removes the
 752          * entry from the list. If there wasn't any before, we can
 753          * go home early.
 754          */
 755         collection = find_collection(its, coll_id);
 756         if (!collection)
 757                 return;
 758
 759         for_each_lpi_its(device, ite, its)
 760                 if (ite->collection &&
 761                     ite->collection->collection_id == coll_id)
 762                         ite->collection = NULL;
 763
 764         list_del(&collection->coll_list);
 765         kfree(collection);
 766 }
 767
 768 /*
 769  * The MAPTI and MAPI commands map LPIs to ITTEs.
 770  * Must be called with its_lock mutex held.
 771  */
 772 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
 773                                     u64 *its_cmd)
 774 {
 775         u32 device_id = its_cmd_get_deviceid(its_cmd);
 776         u32 event_id = its_cmd_get_id(its_cmd);
 777         u32 coll_id = its_cmd_get_collection(its_cmd);
 778         struct its_ite *ite;
 779         struct its_device *device;
 780         struct its_collection *collection, *new_coll = NULL;
 781         int lpi_nr;
 782         struct vgic_irq *irq;
 783
 784         device = find_its_device(its, device_id);
 785         if (!device)
 786                 return E_ITS_MAPTI_UNMAPPED_DEVICE;
 787
 788         if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI)
 789                 lpi_nr = its_cmd_get_physical_id(its_cmd);
 790         else
 791                 lpi_nr = event_id;
 792         if (lpi_nr < GIC_LPI_OFFSET ||
 793             lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser))
 794                 return E_ITS_MAPTI_PHYSICALID_OOR;
 795
 796         /* If there is an existing mapping, behavior is UNPREDICTABLE. */
 797         if (find_ite(its, device_id, event_id))
 798                 return 0;
 799
 800         collection = find_collection(its, coll_id);
 801         if (!collection) {
 802                 int ret = vgic_its_alloc_collection(its, &collection, coll_id);
 803                 if (ret)
 804                         return ret;
 805                 new_coll = collection;
 806         }
 807
 808         ite = kzalloc(sizeof(struct its_ite), GFP_KERNEL);
 809         if (!ite) {
 810                 if (new_coll)
 811                         vgic_its_free_collection(its, coll_id);
 812                 return -ENOMEM;
 813         }
 814
 815         ite->event_id   = event_id;
 816         list_add_tail(&ite->ite_list, &device->itt_head);
 817
 818         ite->collection = collection;
 819         ite->lpi = lpi_nr;
 820
 821         irq = vgic_add_lpi(kvm, lpi_nr);
 822         if (IS_ERR(irq)) {
 823                 if (new_coll)
 824                         vgic_its_free_collection(its, coll_id);
 825                 its_free_ite(kvm, ite);
 826                 return PTR_ERR(irq);
 827         }
 828         ite->irq = irq;
 829
 830         update_affinity_ite(kvm, ite);
 831
 832         /*
 833          * We "cache" the configuration table entries in out struct vgic_irq's.
 834          * However we only have those structs for mapped IRQs, so we read in
 835          * the respective config data from memory here upon mapping the LPI.
 836          */
 837         update_lpi_config(kvm, ite->irq, NULL);
 838
 839         return 0;
 840 }
 841
 842 /* Requires the its_lock to be held. */
 843 static void vgic_its_unmap_device(struct kvm *kvm, struct its_device *device)
 844 {
 845         struct its_ite *ite, *temp;
 846
 847         /*
 848          * The spec says that unmapping a device with still valid
 849          * ITTEs associated is UNPREDICTABLE. We remove all ITTEs,
 850          * since we cannot leave the memory unreferenced.
 851          */
 852         list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
 853                 its_free_ite(kvm, ite);
 854
 855         list_del(&device->dev_list);
 856         kfree(device);
 857 }
 858
 859 /*
 860  * MAPD maps or unmaps a device ID to Interrupt Translation Tables (ITTs).
 861  * Must be called with the its_lock mutex held.
 862  */
 863 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
 864                                     u64 *its_cmd)
 865 {
 866         u32 device_id = its_cmd_get_deviceid(its_cmd);
 867         bool valid = its_cmd_get_validbit(its_cmd);
 868         struct its_device *device;
 869
 870         if (!vgic_its_check_id(its, its->baser_device_table, device_id))
 871                 return E_ITS_MAPD_DEVICE_OOR;
 872
 873         device = find_its_device(its, device_id);
 874
 875         /*
 876          * The spec says that calling MAPD on an already mapped device
 877          * invalidates all cached data for this device. We implement this
 878          * by removing the mapping and re-establishing it.
 879          */
 880         if (device)
 881                 vgic_its_unmap_device(kvm, device);
 882
 883         /*
 884          * The spec does not say whether unmapping a not-mapped device
 885          * is an error, so we are done in any case.
 886          */
 887         if (!valid)
 888                 return 0;
 889
 890         device = kzalloc(sizeof(struct its_device), GFP_KERNEL);
 891         if (!device)
 892                 return -ENOMEM;
 893
 894         device->device_id = device_id;
 895         INIT_LIST_HEAD(&device->itt_head);
 896
 897         list_add_tail(&device->dev_list, &its->device_list);
 898
 899         return 0;
 900 }
 901
 902 /*
 903  * The MAPC command maps collection IDs to redistributors.
 904  * Must be called with the its_lock mutex held.
 905  */
 906 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
 907                                     u64 *its_cmd)
 908 {
 909         u16 coll_id;
 910         u32 target_addr;
 911         struct its_collection *collection;
 912         bool valid;
 913
 914         valid = its_cmd_get_validbit(its_cmd);
 915         coll_id = its_cmd_get_collection(its_cmd);
 916         target_addr = its_cmd_get_target_addr(its_cmd);
 917
 918         if (target_addr >= atomic_read(&kvm->online_vcpus))
 919                 return E_ITS_MAPC_PROCNUM_OOR;
 920
 921         if (!valid) {
 922                 vgic_its_free_collection(its, coll_id);
 923         } else {
 924                 collection = find_collection(its, coll_id);
 925
 926                 if (!collection) {
 927                         int ret;
 928
 929                         ret = vgic_its_alloc_collection(its, &collection,
 930                                                         coll_id);
 931                         if (ret)
 932                                 return ret;
 933                         collection->target_addr = target_addr;
 934                 } else {
 935                         collection->target_addr = target_addr;
 936                         update_affinity_collection(kvm, its, collection);
 937                 }
 938         }
 939
 940         return 0;
 941 }
 942
 943 /*
 944  * The CLEAR command removes the pending state for a particular LPI.
 945  * Must be called with the its_lock mutex held.
 946  */
 947 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
 948                                      u64 *its_cmd)
 949 {
 950         u32 device_id = its_cmd_get_deviceid(its_cmd);
 951         u32 event_id = its_cmd_get_id(its_cmd);
 952         struct its_ite *ite;
 953
 954
 955         ite = find_ite(its, device_id, event_id);
 956         if (!ite)
 957                 return E_ITS_CLEAR_UNMAPPED_INTERRUPT;
 958
 959         ite->irq->pending_latch = false;
 960
 961         return 0;
 962 }
 963
 964 /*
 965  * The INV command syncs the configuration bits from the memory table.
 966  * Must be called with the its_lock mutex held.
 967  */
 968 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
 969                                    u64 *its_cmd)
 970 {
 971         u32 device_id = its_cmd_get_deviceid(its_cmd);
 972         u32 event_id = its_cmd_get_id(its_cmd);
 973         struct its_ite *ite;
 974
 975
 976         ite = find_ite(its, device_id, event_id);
 977         if (!ite)
 978                 return E_ITS_INV_UNMAPPED_INTERRUPT;
 979
 980         return update_lpi_config(kvm, ite->irq, NULL);
 981 }
 982
 983 /*
 984  * The INVALL command requests flushing of all IRQ data in this collection.
 985  * Find the VCPU mapped to that collection, then iterate over the VM's list
 986  * of mapped LPIs and update the configuration for each IRQ which targets
 987  * the specified vcpu. The configuration will be read from the in-memory
 988  * configuration table.
 989  * Must be called with the its_lock mutex held.
 990  */
 991 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
 992                                       u64 *its_cmd)
 993 {
 994         u32 coll_id = its_cmd_get_collection(its_cmd);
 995         struct its_collection *collection;
 996         struct kvm_vcpu *vcpu;
 997         struct vgic_irq *irq;
 998         u32 *intids;
 999         int irq_count, i;
1000
1001         collection = find_collection(its, coll_id);
1002         if (!its_is_collection_mapped(collection))
1003                 return E_ITS_INVALL_UNMAPPED_COLLECTION;
1004
1005         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1006
1007         irq_count = vgic_copy_lpi_list(kvm, &intids);
1008         if (irq_count < 0)
1009                 return irq_count;
1010
1011         for (i = 0; i < irq_count; i++) {
1012                 irq = vgic_get_irq(kvm, NULL, intids[i]);
1013                 if (!irq)
1014                         continue;
1015                 update_lpi_config(kvm, irq, vcpu);
1016                 vgic_put_irq(kvm, irq);
1017         }
1018
1019         kfree(intids);
1020
1021         return 0;
1022 }
1023
1024 /*
1025  * The MOVALL command moves the pending state of all IRQs targeting one
1026  * redistributor to another. We don't hold the pending state in the VCPUs,
1027  * but in the IRQs instead, so there is really not much to do for us here.
1028  * However the spec says that no IRQ must target the old redistributor
1029  * afterwards, so we make sure that no LPI is using the associated target_vcpu.
1030  * This command affects all LPIs in the system that target that redistributor.
1031  */
1032 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1033                                       u64 *its_cmd)
1034 {
1035         struct vgic_dist *dist = &kvm->arch.vgic;
1036         u32 target1_addr = its_cmd_get_target_addr(its_cmd);
1037         u32 target2_addr = its_cmd_mask_field(its_cmd, 3, 16, 32);
1038         struct kvm_vcpu *vcpu1, *vcpu2;
1039         struct vgic_irq *irq;
1040
1041         if (target1_addr >= atomic_read(&kvm->online_vcpus) ||
1042             target2_addr >= atomic_read(&kvm->online_vcpus))
1043                 return E_ITS_MOVALL_PROCNUM_OOR;
1044
1045         if (target1_addr == target2_addr)
1046                 return 0;
1047
1048         vcpu1 = kvm_get_vcpu(kvm, target1_addr);
1049         vcpu2 = kvm_get_vcpu(kvm, target2_addr);
1050
1051         spin_lock(&dist->lpi_list_lock);
1052
1053         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
1054                 spin_lock(&irq->irq_lock);
1055
1056                 if (irq->target_vcpu == vcpu1)
1057                         irq->target_vcpu = vcpu2;
1058
1059                 spin_unlock(&irq->irq_lock);
1060         }
1061
1062         spin_unlock(&dist->lpi_list_lock);
1063
1064         return 0;
1065 }
1066
1067 /*
1068  * The INT command injects the LPI associated with that DevID/EvID pair.
1069  * Must be called with the its_lock mutex held.
1070  */
1071 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1072                                    u64 *its_cmd)
1073 {
1074         u32 msi_data = its_cmd_get_id(its_cmd);
1075         u64 msi_devid = its_cmd_get_deviceid(its_cmd);
1076
1077         return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1078 }
1079
1080 /*
1081  * This function is called with the its_cmd lock held, but the ITS data
1082  * structure lock dropped.
1083  */
1084 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1085                                    u64 *its_cmd)
1086 {
1087         int ret = -ENODEV;
1088
1089         mutex_lock(&its->its_lock);
1090         switch (its_cmd_get_command(its_cmd)) {
1091         case GITS_CMD_MAPD:
1092                 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1093                 break;
1094         case GITS_CMD_MAPC:
1095                 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1096                 break;
1097         case GITS_CMD_MAPI:
1098                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1099                 break;
1100         case GITS_CMD_MAPTI:
1101                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1102                 break;
1103         case GITS_CMD_MOVI:
1104                 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1105                 break;
1106         case GITS_CMD_DISCARD:
1107                 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1108                 break;
1109         case GITS_CMD_CLEAR:
1110                 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1111                 break;
1112         case GITS_CMD_MOVALL:
1113                 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1114                 break;
1115         case GITS_CMD_INT:
1116                 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1117                 break;
1118         case GITS_CMD_INV:
1119                 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1120                 break;
1121         case GITS_CMD_INVALL:
1122                 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1123                 break;
1124         case GITS_CMD_SYNC:
1125                 /* we ignore this command: we are in sync all of the time */
1126                 ret = 0;
1127                 break;
1128         }
1129         mutex_unlock(&its->its_lock);
1130
1131         return ret;
1132 }
1133
1134 static u64 vgic_sanitise_its_baser(u64 reg)
1135 {
1136         reg = vgic_sanitise_field(reg, GITS_BASER_SHAREABILITY_MASK,
1137                                   GITS_BASER_SHAREABILITY_SHIFT,
1138                                   vgic_sanitise_shareability);
1139         reg = vgic_sanitise_field(reg, GITS_BASER_INNER_CACHEABILITY_MASK,
1140                                   GITS_BASER_INNER_CACHEABILITY_SHIFT,
1141                                   vgic_sanitise_inner_cacheability);
1142         reg = vgic_sanitise_field(reg, GITS_BASER_OUTER_CACHEABILITY_MASK,
1143                                   GITS_BASER_OUTER_CACHEABILITY_SHIFT,
1144                                   vgic_sanitise_outer_cacheability);
1145
1146         /* Bits 15:12 contain bits 51:48 of the PA, which we don't support. */
1147         reg &= ~GENMASK_ULL(15, 12);
1148
1149         /* We support only one (ITS) page size: 64K */
1150         reg = (reg & ~GITS_BASER_PAGE_SIZE_MASK) | GITS_BASER_PAGE_SIZE_64K;
1151
1152         return reg;
1153 }
1154
1155 static u64 vgic_sanitise_its_cbaser(u64 reg)
1156 {
1157         reg = vgic_sanitise_field(reg, GITS_CBASER_SHAREABILITY_MASK,
1158                                   GITS_CBASER_SHAREABILITY_SHIFT,
1159                                   vgic_sanitise_shareability);
1160         reg = vgic_sanitise_field(reg, GITS_CBASER_INNER_CACHEABILITY_MASK,
1161                                   GITS_CBASER_INNER_CACHEABILITY_SHIFT,
1162                                   vgic_sanitise_inner_cacheability);
1163         reg = vgic_sanitise_field(reg, GITS_CBASER_OUTER_CACHEABILITY_MASK,
1164                                   GITS_CBASER_OUTER_CACHEABILITY_SHIFT,
1165                                   vgic_sanitise_outer_cacheability);
1166
1167         /*
1168          * Sanitise the physical address to be 64k aligned.
1169          * Also limit the physical addresses to 48 bits.
1170          */
1171         reg &= ~(GENMASK_ULL(51, 48) | GENMASK_ULL(15, 12));
1172
1173         return reg;
1174 }
1175
1176 static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
1177                                                struct vgic_its *its,
1178                                                gpa_t addr, unsigned int len)
1179 {
1180         return extract_bytes(its->cbaser, addr & 7, len);
1181 }
1182
1183 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1184                                        gpa_t addr, unsigned int len,
1185                                        unsigned long val)
1186 {
1187         /* When GITS_CTLR.Enable is 1, this register is RO. */
1188         if (its->enabled)
1189                 return;
1190
1191         mutex_lock(&its->cmd_lock);
1192         its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1193         its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1194         its->creadr = 0;
1195         /*
1196          * CWRITER is architecturally UNKNOWN on reset, but we need to reset
1197          * it to CREADR to make sure we start with an empty command buffer.
1198          */
1199         its->cwriter = its->creadr;
1200         mutex_unlock(&its->cmd_lock);
1201 }
1202
1203 #define ITS_CMD_BUFFER_SIZE(baser)      ((((baser) & 0xff) + 1) << 12)
1204 #define ITS_CMD_SIZE                    32
1205 #define ITS_CMD_OFFSET(reg)             ((reg) & GENMASK(19, 5))
1206
1207 /* Must be called with the cmd_lock held. */
1208 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1209 {
1210         gpa_t cbaser;
1211         u64 cmd_buf[4];
1212
1213         /* Commands are only processed when the ITS is enabled. */
1214         if (!its->enabled)
1215                 return;
1216
1217         cbaser = CBASER_ADDRESS(its->cbaser);
1218
1219         while (its->cwriter != its->creadr) {
1220                 int ret = kvm_read_guest(kvm, cbaser + its->creadr,
1221                                          cmd_buf, ITS_CMD_SIZE);
1222                 /*
1223                  * If kvm_read_guest() fails, this could be due to the guest
1224                  * programming a bogus value in CBASER or something else going
1225                  * wrong from which we cannot easily recover.
1226                  * According to section 6.3.2 in the GICv3 spec we can just
1227                  * ignore that command then.
1228                  */
1229                 if (!ret)
1230                         vgic_its_handle_command(kvm, its, cmd_buf);
1231
1232                 its->creadr += ITS_CMD_SIZE;
1233                 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1234                         its->creadr = 0;
1235         }
1236 }
1237
1238 /*
1239  * By writing to CWRITER the guest announces new commands to be processed.
1240  * To avoid any races in the first place, we take the its_cmd lock, which
1241  * protects our ring buffer variables, so that there is only one user
1242  * per ITS handling commands at a given time.
1243  */
1244 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1245                                         gpa_t addr, unsigned int len,
1246                                         unsigned long val)
1247 {
1248         u64 reg;
1249
1250         if (!its)
1251                 return;
1252
1253         mutex_lock(&its->cmd_lock);
1254
1255         reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1256         reg = ITS_CMD_OFFSET(reg);
1257         if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1258                 mutex_unlock(&its->cmd_lock);
1259                 return;
1260         }
1261         its->cwriter = reg;
1262
1263         vgic_its_process_commands(kvm, its);
1264
1265         mutex_unlock(&its->cmd_lock);
1266 }
1267
1268 static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
1269                                                 struct vgic_its *its,
1270                                                 gpa_t addr, unsigned int len)
1271 {
1272         return extract_bytes(its->cwriter, addr & 0x7, len);
1273 }
1274
1275 static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
1276                                                struct vgic_its *its,
1277                                                gpa_t addr, unsigned int len)
1278 {
1279         return extract_bytes(its->creadr, addr & 0x7, len);
1280 }
1281
1282 static int vgic_mmio_uaccess_write_its_creadr(struct kvm *kvm,
1283                                               struct vgic_its *its,
1284                                               gpa_t addr, unsigned int len,
1285                                               unsigned long val)
1286 {
1287         u32 cmd_offset;
1288         int ret = 0;
1289
1290         mutex_lock(&its->cmd_lock);
1291
1292         if (its->enabled) {
1293                 ret = -EBUSY;
1294                 goto out;
1295         }
1296
1297         cmd_offset = ITS_CMD_OFFSET(val);
1298         if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1299                 ret = -EINVAL;
1300                 goto out;
1301         }
1302
1303         its->creadr = cmd_offset;
1304 out:
1305         mutex_unlock(&its->cmd_lock);
1306         return ret;
1307 }
1308
1309 #define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
1310 static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
1311                                               struct vgic_its *its,
1312                                               gpa_t addr, unsigned int len)
1313 {
1314         u64 reg;
1315
1316         switch (BASER_INDEX(addr)) {
1317         case 0:
1318                 reg = its->baser_device_table;
1319                 break;
1320         case 1:
1321                 reg = its->baser_coll_table;
1322                 break;
1323         default:
1324                 reg = 0;
1325                 break;
1326         }
1327
1328         return extract_bytes(reg, addr & 7, len);
1329 }
1330
1331 #define GITS_BASER_RO_MASK      (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
1332 static void vgic_mmio_write_its_baser(struct kvm *kvm,
1333                                       struct vgic_its *its,
1334                                       gpa_t addr, unsigned int len,
1335                                       unsigned long val)
1336 {
1337         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1338         u64 entry_size, device_type;
1339         u64 reg, *regptr, clearbits = 0;
1340
1341         /* When GITS_CTLR.Enable is 1, we ignore write accesses. */
1342         if (its->enabled)
1343                 return;
1344
1345         switch (BASER_INDEX(addr)) {
1346         case 0:
1347                 regptr = &its->baser_device_table;
1348                 entry_size = abi->dte_esz;
1349                 device_type = GITS_BASER_TYPE_DEVICE;
1350                 break;
1351         case 1:
1352                 regptr = &its->baser_coll_table;
1353                 entry_size = abi->cte_esz;
1354                 device_type = GITS_BASER_TYPE_COLLECTION;
1355                 clearbits = GITS_BASER_INDIRECT;
1356                 break;
1357         default:
1358                 return;
1359         }
1360
1361         reg = update_64bit_reg(*regptr, addr & 7, len, val);
1362         reg &= ~GITS_BASER_RO_MASK;
1363         reg &= ~clearbits;
1364
1365         reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT;
1366         reg |= device_type << GITS_BASER_TYPE_SHIFT;
1367         reg = vgic_sanitise_its_baser(reg);
1368
1369         *regptr = reg;
1370 }
1371
1372 static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
1373                                              struct vgic_its *its,
1374                                              gpa_t addr, unsigned int len)
1375 {
1376         u32 reg = 0;
1377
1378         mutex_lock(&its->cmd_lock);
1379         if (its->creadr == its->cwriter)
1380                 reg |= GITS_CTLR_QUIESCENT;
1381         if (its->enabled)
1382                 reg |= GITS_CTLR_ENABLE;
1383         mutex_unlock(&its->cmd_lock);
1384
1385         return reg;
1386 }
1387
1388 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1389                                      gpa_t addr, unsigned int len,
1390                                      unsigned long val)
1391 {
1392         mutex_lock(&its->cmd_lock);
1393
1394         its->enabled = !!(val & GITS_CTLR_ENABLE);
1395
1396         /*
1397          * Try to process any pending commands. This function bails out early
1398          * if the ITS is disabled or no commands have been queued.
1399          */
1400         vgic_its_process_commands(kvm, its);
1401
1402         mutex_unlock(&its->cmd_lock);
1403 }
1404
1405 #define REGISTER_ITS_DESC(off, rd, wr, length, acc)             \
1406 {                                                               \
1407         .reg_offset = off,                                      \
1408         .len = length,                                          \
1409         .access_flags = acc,                                    \
1410         .its_read = rd,                                         \
1411         .its_write = wr,                                        \
1412 }
1413
1414 #define REGISTER_ITS_DESC_UACCESS(off, rd, wr, uwr, length, acc)\
1415 {                                                               \
1416         .reg_offset = off,                                      \
1417         .len = length,                                          \
1418         .access_flags = acc,                                    \
1419         .its_read = rd,                                         \
1420         .its_write = wr,                                        \
1421         .uaccess_its_write = uwr,                               \
1422 }
1423
1424 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1425                               gpa_t addr, unsigned int len, unsigned long val)
1426 {
1427         /* Ignore */
1428 }
1429
1430 static struct vgic_register_region its_registers[] = {
1431         REGISTER_ITS_DESC(GITS_CTLR,
1432                 vgic_mmio_read_its_ctlr, vgic_mmio_write_its_ctlr, 4,
1433                 VGIC_ACCESS_32bit),
1434         REGISTER_ITS_DESC_UACCESS(GITS_IIDR,
1435                 vgic_mmio_read_its_iidr, its_mmio_write_wi,
1436                 vgic_mmio_uaccess_write_its_iidr, 4,
1437                 VGIC_ACCESS_32bit),
1438         REGISTER_ITS_DESC(GITS_TYPER,
1439                 vgic_mmio_read_its_typer, its_mmio_write_wi, 8,
1440                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1441         REGISTER_ITS_DESC(GITS_CBASER,
1442                 vgic_mmio_read_its_cbaser, vgic_mmio_write_its_cbaser, 8,
1443                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1444         REGISTER_ITS_DESC(GITS_CWRITER,
1445                 vgic_mmio_read_its_cwriter, vgic_mmio_write_its_cwriter, 8,
1446                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1447         REGISTER_ITS_DESC_UACCESS(GITS_CREADR,
1448                 vgic_mmio_read_its_creadr, its_mmio_write_wi,
1449                 vgic_mmio_uaccess_write_its_creadr, 8,
1450                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1451         REGISTER_ITS_DESC(GITS_BASER,
1452                 vgic_mmio_read_its_baser, vgic_mmio_write_its_baser, 0x40,
1453                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1454         REGISTER_ITS_DESC(GITS_IDREGS_BASE,
1455                 vgic_mmio_read_its_idregs, its_mmio_write_wi, 0x30,
1456                 VGIC_ACCESS_32bit),
1457 };
1458
1459 /* This is called on setting the LPI enable bit in the redistributor. */
1460 void vgic_enable_lpis(struct kvm_vcpu *vcpu)
1461 {
1462         if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ))
1463                 its_sync_lpi_pending_table(vcpu);
1464 }
1465
1466 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its)
1467 {
1468         struct vgic_io_device *iodev = &its->iodev;
1469         int ret;
1470
1471         if (!its->initialized)
1472                 return -EBUSY;
1473
1474         if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base))
1475                 return -ENXIO;
1476
1477         iodev->regions = its_registers;
1478         iodev->nr_regions = ARRAY_SIZE(its_registers);
1479         kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops);
1480
1481         iodev->base_addr = its->vgic_its_base;
1482         iodev->iodev_type = IODEV_ITS;
1483         iodev->its = its;
1484         mutex_lock(&kvm->slots_lock);
1485         ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr,
1486                                       KVM_VGIC_V3_ITS_SIZE, &iodev->dev);
1487         mutex_unlock(&kvm->slots_lock);
1488
1489         return ret;
1490 }
1491
1492 #define INITIAL_BASER_VALUE                                               \
1493         (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb)                | \
1494          GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner)         | \
1495          GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable)             | \
1496          GITS_BASER_PAGE_SIZE_64K)
1497
1498 #define INITIAL_PROPBASER_VALUE                                           \
1499         (GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb)            | \
1500          GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner)     | \
1501          GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
1502
1503 static int vgic_its_create(struct kvm_device *dev, u32 type)
1504 {
1505         struct vgic_its *its;
1506
1507         if (type != KVM_DEV_TYPE_ARM_VGIC_ITS)
1508                 return -ENODEV;
1509
1510         its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL);
1511         if (!its)
1512                 return -ENOMEM;
1513
1514         mutex_init(&its->its_lock);
1515         mutex_init(&its->cmd_lock);
1516
1517         its->vgic_its_base = VGIC_ADDR_UNDEF;
1518
1519         INIT_LIST_HEAD(&its->device_list);
1520         INIT_LIST_HEAD(&its->collection_list);
1521
1522         dev->kvm->arch.vgic.has_its = true;
1523         its->initialized = false;
1524         its->enabled = false;
1525         its->dev = dev;
1526
1527         its->baser_device_table = INITIAL_BASER_VALUE                   |
1528                 ((u64)GITS_BASER_TYPE_DEVICE << GITS_BASER_TYPE_SHIFT);
1529         its->baser_coll_table = INITIAL_BASER_VALUE |
1530                 ((u64)GITS_BASER_TYPE_COLLECTION << GITS_BASER_TYPE_SHIFT);
1531         dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
1532
1533         dev->private = its;
1534
1535         return vgic_its_set_abi(its, NR_ITS_ABIS - 1);
1536 }
1537
1538 static void vgic_its_destroy(struct kvm_device *kvm_dev)
1539 {
1540         struct kvm *kvm = kvm_dev->kvm;
1541         struct vgic_its *its = kvm_dev->private;
1542         struct its_device *dev;
1543         struct its_ite *ite;
1544         struct list_head *dev_cur, *dev_temp;
1545         struct list_head *cur, *temp;
1546
1547         /*
1548          * We may end up here without the lists ever having been initialized.
1549          * Check this and bail out early to avoid dereferencing a NULL pointer.
1550          */
1551         if (!its->device_list.next)
1552                 return;
1553
1554         mutex_lock(&its->its_lock);
1555         list_for_each_safe(dev_cur, dev_temp, &its->device_list) {
1556                 dev = container_of(dev_cur, struct its_device, dev_list);
1557                 list_for_each_safe(cur, temp, &dev->itt_head) {
1558                         ite = (container_of(cur, struct its_ite, ite_list));
1559                         its_free_ite(kvm, ite);
1560                 }
1561                 list_del(dev_cur);
1562                 kfree(dev);
1563         }
1564
1565         list_for_each_safe(cur, temp, &its->collection_list) {
1566                 list_del(cur);
1567                 kfree(container_of(cur, struct its_collection, coll_list));
1568         }
1569         mutex_unlock(&its->its_lock);
1570
1571         kfree(its);
1572 }
1573
1574 int vgic_its_has_attr_regs(struct kvm_device *dev,
1575                            struct kvm_device_attr *attr)
1576 {
1577         const struct vgic_register_region *region;
1578         gpa_t offset = attr->attr;
1579         int align;
1580
1581         align = (offset < GITS_TYPER) || (offset >= GITS_PIDR4) ? 0x3 : 0x7;
1582
1583         if (offset & align)
1584                 return -EINVAL;
1585
1586         region = vgic_find_mmio_region(its_registers,
1587                                        ARRAY_SIZE(its_registers),
1588                                        offset);
1589         if (!region)
1590                 return -ENXIO;
1591
1592         return 0;
1593 }
1594
1595 int vgic_its_attr_regs_access(struct kvm_device *dev,
1596                               struct kvm_device_attr *attr,
1597                               u64 *reg, bool is_write)
1598 {
1599         const struct vgic_register_region *region;
1600         struct vgic_its *its;
1601         gpa_t addr, offset;
1602         unsigned int len;
1603         int align, ret = 0;
1604
1605         its = dev->private;
1606         offset = attr->attr;
1607
1608         /*
1609          * Although the spec supports upper/lower 32-bit accesses to
1610          * 64-bit ITS registers, the userspace ABI requires 64-bit
1611          * accesses to all 64-bit wide registers. We therefore only
1612          * support 32-bit accesses to GITS_CTLR, GITS_IIDR and GITS ID
1613          * registers
1614          */
1615         if ((offset < GITS_TYPER) || (offset >= GITS_PIDR4))
1616                 align = 0x3;
1617         else
1618                 align = 0x7;
1619
1620         if (offset & align)
1621                 return -EINVAL;
1622
1623         mutex_lock(&dev->kvm->lock);
1624
1625         if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1626                 ret = -ENXIO;
1627                 goto out;
1628         }
1629
1630         region = vgic_find_mmio_region(its_registers,
1631                                        ARRAY_SIZE(its_registers),
1632                                        offset);
1633         if (!region) {
1634                 ret = -ENXIO;
1635                 goto out;
1636         }
1637
1638         if (!lock_all_vcpus(dev->kvm)) {
1639                 ret = -EBUSY;
1640                 goto out;
1641         }
1642
1643         addr = its->vgic_its_base + offset;
1644
1645         len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4;
1646
1647         if (is_write) {
1648                 if (region->uaccess_its_write)
1649                         ret = region->uaccess_its_write(dev->kvm, its, addr,
1650                                                         len, *reg);
1651                 else
1652                         region->its_write(dev->kvm, its, addr, len, *reg);
1653         } else {
1654                 *reg = region->its_read(dev->kvm, its, addr, len);
1655         }
1656         unlock_all_vcpus(dev->kvm);
1657 out:
1658         mutex_unlock(&dev->kvm->lock);
1659         return ret;
1660 }
1661
1662 /**
1663  * vgic_its_save_tables_v0 - Save the ITS tables into guest ARM
1664  * according to v0 ABI
1665  */
1666 static int vgic_its_save_tables_v0(struct vgic_its *its)
1667 {
1668         return -ENXIO;
1669 }
1670
1671 /**
1672  * vgic_its_restore_tables_v0 - Restore the ITS tables from guest RAM
1673  * to internal data structs according to V0 ABI
1674  *
1675  */
1676 static int vgic_its_restore_tables_v0(struct vgic_its *its)
1677 {
1678         return -ENXIO;
1679 }
1680
1681 static int vgic_its_commit_v0(struct vgic_its *its)
1682 {
1683         const struct vgic_its_abi *abi;
1684
1685         abi = vgic_its_get_abi(its);
1686         its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
1687         its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
1688
1689         its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
1690                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
1691
1692         its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
1693                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
1694         return 0;
1695 }
1696
1697 static int vgic_its_has_attr(struct kvm_device *dev,
1698                              struct kvm_device_attr *attr)
1699 {
1700         switch (attr->group) {
1701         case KVM_DEV_ARM_VGIC_GRP_ADDR:
1702                 switch (attr->attr) {
1703                 case KVM_VGIC_ITS_ADDR_TYPE:
1704                         return 0;
1705                 }
1706                 break;
1707         case KVM_DEV_ARM_VGIC_GRP_CTRL:
1708                 switch (attr->attr) {
1709                 case KVM_DEV_ARM_VGIC_CTRL_INIT:
1710                         return 0;
1711                 }
1712                 break;
1713         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS:
1714                 return vgic_its_has_attr_regs(dev, attr);
1715         }
1716         return -ENXIO;
1717 }
1718
1719 static int vgic_its_set_attr(struct kvm_device *dev,
1720                              struct kvm_device_attr *attr)
1721 {
1722         struct vgic_its *its = dev->private;
1723         int ret;
1724
1725         switch (attr->group) {
1726         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
1727                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
1728                 unsigned long type = (unsigned long)attr->attr;
1729                 u64 addr;
1730
1731                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
1732                         return -ENODEV;
1733
1734                 if (copy_from_user(&addr, uaddr, sizeof(addr)))
1735                         return -EFAULT;
1736
1737                 ret = vgic_check_ioaddr(dev->kvm, &its->vgic_its_base,
1738                                         addr, SZ_64K);
1739                 if (ret)
1740                         return ret;
1741
1742                 its->vgic_its_base = addr;
1743
1744                 return 0;
1745         }
1746         case KVM_DEV_ARM_VGIC_GRP_CTRL:
1747                 switch (attr->attr) {
1748                 case KVM_DEV_ARM_VGIC_CTRL_INIT:
1749                         its->initialized = true;
1750
1751                         return 0;
1752                 }
1753                 break;
1754         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
1755                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
1756                 u64 reg;
1757
1758                 if (get_user(reg, uaddr))
1759                         return -EFAULT;
1760
1761                 return vgic_its_attr_regs_access(dev, attr, &reg, true);
1762         }
1763         }
1764         return -ENXIO;
1765 }
1766
1767 static int vgic_its_get_attr(struct kvm_device *dev,
1768                              struct kvm_device_attr *attr)
1769 {
1770         switch (attr->group) {
1771         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
1772                 struct vgic_its *its = dev->private;
1773                 u64 addr = its->vgic_its_base;
1774                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
1775                 unsigned long type = (unsigned long)attr->attr;
1776
1777                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
1778                         return -ENODEV;
1779
1780                 if (copy_to_user(uaddr, &addr, sizeof(addr)))
1781                         return -EFAULT;
1782                 break;
1783         }
1784         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
1785                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
1786                 u64 reg;
1787                 int ret;
1788
1789                 ret = vgic_its_attr_regs_access(dev, attr, &reg, false);
1790                 if (ret)
1791                         return ret;
1792                 return put_user(reg, uaddr);
1793         }
1794         default:
1795                 return -ENXIO;
1796         }
1797
1798         return 0;
1799 }
1800
1801 static struct kvm_device_ops kvm_arm_vgic_its_ops = {
1802         .name = "kvm-arm-vgic-its",
1803         .create = vgic_its_create,
1804         .destroy = vgic_its_destroy,
1805         .set_attr = vgic_its_set_attr,
1806         .get_attr = vgic_its_get_attr,
1807         .has_attr = vgic_its_has_attr,
1808 };
1809
1810 int kvm_vgic_register_its_device(void)
1811 {
1812         return kvm_register_device_ops(&kvm_arm_vgic_its_ops,
1813                                        KVM_DEV_TYPE_ARM_VGIC_ITS);
1814 }
1815
1816 /*
1817  * Registers all ITSes with the kvm_io_bus framework.
1818  * To follow the existing VGIC initialization sequence, this has to be
1819  * done as late as possible, just before the first VCPU runs.
1820  */
1821 int vgic_register_its_iodevs(struct kvm *kvm)
1822 {
1823         struct kvm_device *dev;
1824         int ret = 0;
1825
1826         list_for_each_entry(dev, &kvm->devices, vm_node) {
1827                 if (dev->ops != &kvm_arm_vgic_its_ops)
1828                         continue;
1829
1830                 ret = vgic_register_its_iodev(kvm, dev->private);
1831                 if (ret)
1832                         return ret;
1833                 /*
1834                  * We don't need to care about tearing down previously
1835                  * registered ITSes, as the kvm_io_bus framework removes
1836                  * them for us if the VM gets destroyed.
1837                  */
1838         }
1839
1840         return ret;
1841 }