arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/module.h>
  25 #include <linux/random.h>
  26 #include <linux/slab.h>
  27 #include <linux/timer.h>
  28 #include <linux/vmalloc.h>
  29 #include <asm/asm-offsets.h>
  30 #include <asm/lowcore.h>
  31 #include <asm/etr.h>
  32 #include <asm/pgtable.h>
  33 #include <asm/nmi.h>
  34 #include <asm/switch_to.h>
  35 #include <asm/isc.h>
  36 #include <asm/sclp.h>
  37 #include "kvm-s390.h"
  38 #include "gaccess.h"
  39
  40 #define KMSG_COMPONENT "kvm-s390"
  41 #undef pr_fmt
  42 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  43
  44 #define CREATE_TRACE_POINTS
  45 #include "trace.h"
  46 #include "trace-s390.h"
  47
  48 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  49 #define LOCAL_IRQS 32
  50 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  51                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  52
  53 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  54
  55 struct kvm_stats_debugfs_item debugfs_entries[] = {
  56         { "userspace_handled", VCPU_STAT(exit_userspace) },
  57         { "exit_null", VCPU_STAT(exit_null) },
  58         { "exit_validity", VCPU_STAT(exit_validity) },
  59         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  60         { "exit_external_request", VCPU_STAT(exit_external_request) },
  61         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  62         { "exit_instruction", VCPU_STAT(exit_instruction) },
  63         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  64         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  65         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  66         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  67         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  68         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  69         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  70         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  71         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  72         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  73         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  74         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  75         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  76         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  77         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  78         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  79         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  80         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  81         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  82         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  83         { "instruction_spx", VCPU_STAT(instruction_spx) },
  84         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  85         { "instruction_stap", VCPU_STAT(instruction_stap) },
  86         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  87         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  88         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  89         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  90         { "instruction_essa", VCPU_STAT(instruction_essa) },
  91         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
  92         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
  93         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
  94         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
  95         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
  96         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
  97         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
  98         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
  99         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 100         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 101         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 102         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 103         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 104         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 105         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 106         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 107         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 108         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 109         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 110         { "diagnose_10", VCPU_STAT(diagnose_10) },
 111         { "diagnose_44", VCPU_STAT(diagnose_44) },
 112         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
 113         { "diagnose_258", VCPU_STAT(diagnose_258) },
 114         { "diagnose_308", VCPU_STAT(diagnose_308) },
 115         { "diagnose_500", VCPU_STAT(diagnose_500) },
 116         { NULL }
 117 };
 118
 119 /* upper facilities limit for kvm */
 120 unsigned long kvm_s390_fac_list_mask[] = {
 121         0xffe6fffbfcfdfc40UL,
 122         0x005e800000000000UL,
 123 };
 124
 125 unsigned long kvm_s390_fac_list_mask_size(void)
 126 {
 127         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
 128         return ARRAY_SIZE(kvm_s390_fac_list_mask);
 129 }
 130
 131 static struct gmap_notifier gmap_notifier;
 132 debug_info_t *kvm_s390_dbf;
 133
 134 /* Section: not file related */
 135 int kvm_arch_hardware_enable(void)
 136 {
 137         /* every s390 is virtualization enabled ;-) */
 138         return 0;
 139 }
 140
 141 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 142
 143 /*
 144  * This callback is executed during stop_machine(). All CPUs are therefore
 145  * temporarily stopped. In order not to change guest behavior, we have to
 146  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 147  * so a CPU won't be stopped while calculating with the epoch.
 148  */
 149 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 150                           void *v)
 151 {
 152         struct kvm *kvm;
 153         struct kvm_vcpu *vcpu;
 154         int i;
 155         unsigned long long *delta = v;
 156
 157         list_for_each_entry(kvm, &vm_list, vm_list) {
 158                 kvm->arch.epoch -= *delta;
 159                 kvm_for_each_vcpu(i, vcpu, kvm) {
 160                         vcpu->arch.sie_block->epoch -= *delta;
 161                 }
 162         }
 163         return NOTIFY_OK;
 164 }
 165
 166 static struct notifier_block kvm_clock_notifier = {
 167         .notifier_call = kvm_clock_sync,
 168 };
 169
 170 int kvm_arch_hardware_setup(void)
 171 {
 172         gmap_notifier.notifier_call = kvm_gmap_notifier;
 173         gmap_register_ipte_notifier(&gmap_notifier);
 174         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 175                                        &kvm_clock_notifier);
 176         return 0;
 177 }
 178
 179 void kvm_arch_hardware_unsetup(void)
 180 {
 181         gmap_unregister_ipte_notifier(&gmap_notifier);
 182         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 183                                          &kvm_clock_notifier);
 184 }
 185
 186 int kvm_arch_init(void *opaque)
 187 {
 188         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 189         if (!kvm_s390_dbf)
 190                 return -ENOMEM;
 191
 192         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
 193                 debug_unregister(kvm_s390_dbf);
 194                 return -ENOMEM;
 195         }
 196
 197         /* Register floating interrupt controller interface. */
 198         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 199 }
 200
 201 void kvm_arch_exit(void)
 202 {
 203         debug_unregister(kvm_s390_dbf);
 204 }
 205
 206 /* Section: device related */
 207 long kvm_arch_dev_ioctl(struct file *filp,
 208                         unsigned int ioctl, unsigned long arg)
 209 {
 210         if (ioctl == KVM_S390_ENABLE_SIE)
 211                 return s390_enable_sie();
 212         return -EINVAL;
 213 }
 214
 215 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 216 {
 217         int r;
 218
 219         switch (ext) {
 220         case KVM_CAP_S390_PSW:
 221         case KVM_CAP_S390_GMAP:
 222         case KVM_CAP_SYNC_MMU:
 223 #ifdef CONFIG_KVM_S390_UCONTROL
 224         case KVM_CAP_S390_UCONTROL:
 225 #endif
 226         case KVM_CAP_ASYNC_PF:
 227         case KVM_CAP_SYNC_REGS:
 228         case KVM_CAP_ONE_REG:
 229         case KVM_CAP_ENABLE_CAP:
 230         case KVM_CAP_S390_CSS_SUPPORT:
 231         case KVM_CAP_IOEVENTFD:
 232         case KVM_CAP_DEVICE_CTRL:
 233         case KVM_CAP_ENABLE_CAP_VM:
 234         case KVM_CAP_S390_IRQCHIP:
 235         case KVM_CAP_VM_ATTRIBUTES:
 236         case KVM_CAP_MP_STATE:
 237         case KVM_CAP_S390_INJECT_IRQ:
 238         case KVM_CAP_S390_USER_SIGP:
 239         case KVM_CAP_S390_USER_STSI:
 240         case KVM_CAP_S390_SKEYS:
 241         case KVM_CAP_S390_IRQ_STATE:
 242                 r = 1;
 243                 break;
 244         case KVM_CAP_S390_MEM_OP:
 245                 r = MEM_OP_MAX_SIZE;
 246                 break;
 247         case KVM_CAP_NR_VCPUS:
 248         case KVM_CAP_MAX_VCPUS:
 249                 r = sclp.has_esca ? KVM_S390_ESCA_CPU_SLOTS
 250                                   : KVM_S390_BSCA_CPU_SLOTS;
 251                 break;
 252         case KVM_CAP_NR_MEMSLOTS:
 253                 r = KVM_USER_MEM_SLOTS;
 254                 break;
 255         case KVM_CAP_S390_COW:
 256                 r = MACHINE_HAS_ESOP;
 257                 break;
 258         case KVM_CAP_S390_VECTOR_REGISTERS:
 259                 r = MACHINE_HAS_VX;
 260                 break;
 261         case KVM_CAP_S390_RI:
 262                 r = test_facility(64);
 263                 break;
 264         default:
 265                 r = 0;
 266         }
 267         return r;
 268 }
 269
 270 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 271                                         struct kvm_memory_slot *memslot)
 272 {
 273         gfn_t cur_gfn, last_gfn;
 274         unsigned long address;
 275         struct gmap *gmap = kvm->arch.gmap;
 276
 277         /* Loop over all guest pages */
 278         last_gfn = memslot->base_gfn + memslot->npages;
 279         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 280                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 281
 282                 if (gmap_test_and_clear_dirty(address, gmap))
 283                         mark_page_dirty(kvm, cur_gfn);
 284                 cond_resched();
 285         }
 286 }
 287
 288 /* Section: vm related */
 289 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
 290
 291 /*
 292  * Get (and clear) the dirty memory log for a memory slot.
 293  */
 294 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 295                                struct kvm_dirty_log *log)
 296 {
 297         int r;
 298         unsigned long n;
 299         struct kvm_memslots *slots;
 300         struct kvm_memory_slot *memslot;
 301         int is_dirty = 0;
 302
 303         mutex_lock(&kvm->slots_lock);
 304
 305         r = -EINVAL;
 306         if (log->slot >= KVM_USER_MEM_SLOTS)
 307                 goto out;
 308
 309         slots = kvm_memslots(kvm);
 310         memslot = id_to_memslot(slots, log->slot);
 311         r = -ENOENT;
 312         if (!memslot->dirty_bitmap)
 313                 goto out;
 314
 315         kvm_s390_sync_dirty_log(kvm, memslot);
 316         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 317         if (r)
 318                 goto out;
 319
 320         /* Clear the dirty log */
 321         if (is_dirty) {
 322                 n = kvm_dirty_bitmap_bytes(memslot);
 323                 memset(memslot->dirty_bitmap, 0, n);
 324         }
 325         r = 0;
 326 out:
 327         mutex_unlock(&kvm->slots_lock);
 328         return r;
 329 }
 330
 331 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 332 {
 333         int r;
 334
 335         if (cap->flags)
 336                 return -EINVAL;
 337
 338         switch (cap->cap) {
 339         case KVM_CAP_S390_IRQCHIP:
 340                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 341                 kvm->arch.use_irqchip = 1;
 342                 r = 0;
 343                 break;
 344         case KVM_CAP_S390_USER_SIGP:
 345                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 346                 kvm->arch.user_sigp = 1;
 347                 r = 0;
 348                 break;
 349         case KVM_CAP_S390_VECTOR_REGISTERS:
 350                 mutex_lock(&kvm->lock);
 351                 if (atomic_read(&kvm->online_vcpus)) {
 352                         r = -EBUSY;
 353                 } else if (MACHINE_HAS_VX) {
 354                         set_kvm_facility(kvm->arch.model.fac->mask, 129);
 355                         set_kvm_facility(kvm->arch.model.fac->list, 129);
 356                         r = 0;
 357                 } else
 358                         r = -EINVAL;
 359                 mutex_unlock(&kvm->lock);
 360                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 361                          r ? "(not available)" : "(success)");
 362                 break;
 363         case KVM_CAP_S390_RI:
 364                 r = -EINVAL;
 365                 mutex_lock(&kvm->lock);
 366                 if (atomic_read(&kvm->online_vcpus)) {
 367                         r = -EBUSY;
 368                 } else if (test_facility(64)) {
 369                         set_kvm_facility(kvm->arch.model.fac->mask, 64);
 370                         set_kvm_facility(kvm->arch.model.fac->list, 64);
 371                         r = 0;
 372                 }
 373                 mutex_unlock(&kvm->lock);
 374                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
 375                          r ? "(not available)" : "(success)");
 376                 break;
 377         case KVM_CAP_S390_USER_STSI:
 378                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 379                 kvm->arch.user_stsi = 1;
 380                 r = 0;
 381                 break;
 382         default:
 383                 r = -EINVAL;
 384                 break;
 385         }
 386         return r;
 387 }
 388
 389 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 390 {
 391         int ret;
 392
 393         switch (attr->attr) {
 394         case KVM_S390_VM_MEM_LIMIT_SIZE:
 395                 ret = 0;
 396                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 397                          kvm->arch.mem_limit);
 398                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
 399                         ret = -EFAULT;
 400                 break;
 401         default:
 402                 ret = -ENXIO;
 403                 break;
 404         }
 405         return ret;
 406 }
 407
 408 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 409 {
 410         int ret;
 411         unsigned int idx;
 412         switch (attr->attr) {
 413         case KVM_S390_VM_MEM_ENABLE_CMMA:
 414                 /* enable CMMA only for z10 and later (EDAT_1) */
 415                 ret = -EINVAL;
 416                 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
 417                         break;
 418
 419                 ret = -EBUSY;
 420                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 421                 mutex_lock(&kvm->lock);
 422                 if (atomic_read(&kvm->online_vcpus) == 0) {
 423                         kvm->arch.use_cmma = 1;
 424                         ret = 0;
 425                 }
 426                 mutex_unlock(&kvm->lock);
 427                 break;
 428         case KVM_S390_VM_MEM_CLR_CMMA:
 429                 ret = -EINVAL;
 430                 if (!kvm->arch.use_cmma)
 431                         break;
 432
 433                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 434                 mutex_lock(&kvm->lock);
 435                 idx = srcu_read_lock(&kvm->srcu);
 436                 s390_reset_cmma(kvm->arch.gmap->mm);
 437                 srcu_read_unlock(&kvm->srcu, idx);
 438                 mutex_unlock(&kvm->lock);
 439                 ret = 0;
 440                 break;
 441         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 442                 unsigned long new_limit;
 443
 444                 if (kvm_is_ucontrol(kvm))
 445                         return -EINVAL;
 446
 447                 if (get_user(new_limit, (u64 __user *)attr->addr))
 448                         return -EFAULT;
 449
 450                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
 451                     new_limit > kvm->arch.mem_limit)
 452                         return -E2BIG;
 453
 454                 if (!new_limit)
 455                         return -EINVAL;
 456
 457                 /* gmap_alloc takes last usable address */
 458                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
 459                         new_limit -= 1;
 460
 461                 ret = -EBUSY;
 462                 mutex_lock(&kvm->lock);
 463                 if (atomic_read(&kvm->online_vcpus) == 0) {
 464                         /* gmap_alloc will round the limit up */
 465                         struct gmap *new = gmap_alloc(current->mm, new_limit);
 466
 467                         if (!new) {
 468                                 ret = -ENOMEM;
 469                         } else {
 470                                 gmap_free(kvm->arch.gmap);
 471                                 new->private = kvm;
 472                                 kvm->arch.gmap = new;
 473                                 ret = 0;
 474                         }
 475                 }
 476                 mutex_unlock(&kvm->lock);
 477                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
 478                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
 479                          (void *) kvm->arch.gmap->asce);
 480                 break;
 481         }
 482         default:
 483                 ret = -ENXIO;
 484                 break;
 485         }
 486         return ret;
 487 }
 488
 489 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 490
 491 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 492 {
 493         struct kvm_vcpu *vcpu;
 494         int i;
 495
 496         if (!test_kvm_facility(kvm, 76))
 497                 return -EINVAL;
 498
 499         mutex_lock(&kvm->lock);
 500         switch (attr->attr) {
 501         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 502                 get_random_bytes(
 503                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 504                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 505                 kvm->arch.crypto.aes_kw = 1;
 506                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 507                 break;
 508         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 509                 get_random_bytes(
 510                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 511                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 512                 kvm->arch.crypto.dea_kw = 1;
 513                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 514                 break;
 515         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 516                 kvm->arch.crypto.aes_kw = 0;
 517                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 518                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 519                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 520                 break;
 521         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 522                 kvm->arch.crypto.dea_kw = 0;
 523                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 524                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 525                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 526                 break;
 527         default:
 528                 mutex_unlock(&kvm->lock);
 529                 return -ENXIO;
 530         }
 531
 532         kvm_for_each_vcpu(i, vcpu, kvm) {
 533                 kvm_s390_vcpu_crypto_setup(vcpu);
 534                 exit_sie(vcpu);
 535         }
 536         mutex_unlock(&kvm->lock);
 537         return 0;
 538 }
 539
 540 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 541 {
 542         u8 gtod_high;
 543
 544         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 545                                            sizeof(gtod_high)))
 546                 return -EFAULT;
 547
 548         if (gtod_high != 0)
 549                 return -EINVAL;
 550         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
 551
 552         return 0;
 553 }
 554
 555 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 556 {
 557         u64 gtod;
 558
 559         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 560                 return -EFAULT;
 561
 562         kvm_s390_set_tod_clock(kvm, gtod);
 563         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
 564         return 0;
 565 }
 566
 567 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 568 {
 569         int ret;
 570
 571         if (attr->flags)
 572                 return -EINVAL;
 573
 574         switch (attr->attr) {
 575         case KVM_S390_VM_TOD_HIGH:
 576                 ret = kvm_s390_set_tod_high(kvm, attr);
 577                 break;
 578         case KVM_S390_VM_TOD_LOW:
 579                 ret = kvm_s390_set_tod_low(kvm, attr);
 580                 break;
 581         default:
 582                 ret = -ENXIO;
 583                 break;
 584         }
 585         return ret;
 586 }
 587
 588 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 589 {
 590         u8 gtod_high = 0;
 591
 592         if (copy_to_user((void __user *)attr->addr, &gtod_high,
 593                                          sizeof(gtod_high)))
 594                 return -EFAULT;
 595         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
 596
 597         return 0;
 598 }
 599
 600 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 601 {
 602         u64 gtod;
 603
 604         gtod = kvm_s390_get_tod_clock_fast(kvm);
 605         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 606                 return -EFAULT;
 607         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
 608
 609         return 0;
 610 }
 611
 612 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 613 {
 614         int ret;
 615
 616         if (attr->flags)
 617                 return -EINVAL;
 618
 619         switch (attr->attr) {
 620         case KVM_S390_VM_TOD_HIGH:
 621                 ret = kvm_s390_get_tod_high(kvm, attr);
 622                 break;
 623         case KVM_S390_VM_TOD_LOW:
 624                 ret = kvm_s390_get_tod_low(kvm, attr);
 625                 break;
 626         default:
 627                 ret = -ENXIO;
 628                 break;
 629         }
 630         return ret;
 631 }
 632
 633 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 634 {
 635         struct kvm_s390_vm_cpu_processor *proc;
 636         int ret = 0;
 637
 638         mutex_lock(&kvm->lock);
 639         if (atomic_read(&kvm->online_vcpus)) {
 640                 ret = -EBUSY;
 641                 goto out;
 642         }
 643         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 644         if (!proc) {
 645                 ret = -ENOMEM;
 646                 goto out;
 647         }
 648         if (!copy_from_user(proc, (void __user *)attr->addr,
 649                             sizeof(*proc))) {
 650                 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
 651                        sizeof(struct cpuid));
 652                 kvm->arch.model.ibc = proc->ibc;
 653                 memcpy(kvm->arch.model.fac->list, proc->fac_list,
 654                        S390_ARCH_FAC_LIST_SIZE_BYTE);
 655         } else
 656                 ret = -EFAULT;
 657         kfree(proc);
 658 out:
 659         mutex_unlock(&kvm->lock);
 660         return ret;
 661 }
 662
 663 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 664 {
 665         int ret = -ENXIO;
 666
 667         switch (attr->attr) {
 668         case KVM_S390_VM_CPU_PROCESSOR:
 669                 ret = kvm_s390_set_processor(kvm, attr);
 670                 break;
 671         }
 672         return ret;
 673 }
 674
 675 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 676 {
 677         struct kvm_s390_vm_cpu_processor *proc;
 678         int ret = 0;
 679
 680         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 681         if (!proc) {
 682                 ret = -ENOMEM;
 683                 goto out;
 684         }
 685         memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
 686         proc->ibc = kvm->arch.model.ibc;
 687         memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
 688         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
 689                 ret = -EFAULT;
 690         kfree(proc);
 691 out:
 692         return ret;
 693 }
 694
 695 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
 696 {
 697         struct kvm_s390_vm_cpu_machine *mach;
 698         int ret = 0;
 699
 700         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
 701         if (!mach) {
 702                 ret = -ENOMEM;
 703                 goto out;
 704         }
 705         get_cpu_id((struct cpuid *) &mach->cpuid);
 706         mach->ibc = sclp.ibc;
 707         memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
 708                S390_ARCH_FAC_LIST_SIZE_BYTE);
 709         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
 710                S390_ARCH_FAC_LIST_SIZE_BYTE);
 711         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
 712                 ret = -EFAULT;
 713         kfree(mach);
 714 out:
 715         return ret;
 716 }
 717
 718 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 719 {
 720         int ret = -ENXIO;
 721
 722         switch (attr->attr) {
 723         case KVM_S390_VM_CPU_PROCESSOR:
 724                 ret = kvm_s390_get_processor(kvm, attr);
 725                 break;
 726         case KVM_S390_VM_CPU_MACHINE:
 727                 ret = kvm_s390_get_machine(kvm, attr);
 728                 break;
 729         }
 730         return ret;
 731 }
 732
 733 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 734 {
 735         int ret;
 736
 737         switch (attr->group) {
 738         case KVM_S390_VM_MEM_CTRL:
 739                 ret = kvm_s390_set_mem_control(kvm, attr);
 740                 break;
 741         case KVM_S390_VM_TOD:
 742                 ret = kvm_s390_set_tod(kvm, attr);
 743                 break;
 744         case KVM_S390_VM_CPU_MODEL:
 745                 ret = kvm_s390_set_cpu_model(kvm, attr);
 746                 break;
 747         case KVM_S390_VM_CRYPTO:
 748                 ret = kvm_s390_vm_set_crypto(kvm, attr);
 749                 break;
 750         default:
 751                 ret = -ENXIO;
 752                 break;
 753         }
 754
 755         return ret;
 756 }
 757
 758 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 759 {
 760         int ret;
 761
 762         switch (attr->group) {
 763         case KVM_S390_VM_MEM_CTRL:
 764                 ret = kvm_s390_get_mem_control(kvm, attr);
 765                 break;
 766         case KVM_S390_VM_TOD:
 767                 ret = kvm_s390_get_tod(kvm, attr);
 768                 break;
 769         case KVM_S390_VM_CPU_MODEL:
 770                 ret = kvm_s390_get_cpu_model(kvm, attr);
 771                 break;
 772         default:
 773                 ret = -ENXIO;
 774                 break;
 775         }
 776
 777         return ret;
 778 }
 779
 780 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 781 {
 782         int ret;
 783
 784         switch (attr->group) {
 785         case KVM_S390_VM_MEM_CTRL:
 786                 switch (attr->attr) {
 787                 case KVM_S390_VM_MEM_ENABLE_CMMA:
 788                 case KVM_S390_VM_MEM_CLR_CMMA:
 789                 case KVM_S390_VM_MEM_LIMIT_SIZE:
 790                         ret = 0;
 791                         break;
 792                 default:
 793                         ret = -ENXIO;
 794                         break;
 795                 }
 796                 break;
 797         case KVM_S390_VM_TOD:
 798                 switch (attr->attr) {
 799                 case KVM_S390_VM_TOD_LOW:
 800                 case KVM_S390_VM_TOD_HIGH:
 801                         ret = 0;
 802                         break;
 803                 default:
 804                         ret = -ENXIO;
 805                         break;
 806                 }
 807                 break;
 808         case KVM_S390_VM_CPU_MODEL:
 809                 switch (attr->attr) {
 810                 case KVM_S390_VM_CPU_PROCESSOR:
 811                 case KVM_S390_VM_CPU_MACHINE:
 812                         ret = 0;
 813                         break;
 814                 default:
 815                         ret = -ENXIO;
 816                         break;
 817                 }
 818                 break;
 819         case KVM_S390_VM_CRYPTO:
 820                 switch (attr->attr) {
 821                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 822                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 823                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 824                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 825                         ret = 0;
 826                         break;
 827                 default:
 828                         ret = -ENXIO;
 829                         break;
 830                 }
 831                 break;
 832         default:
 833                 ret = -ENXIO;
 834                 break;
 835         }
 836
 837         return ret;
 838 }
 839
 840 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
 841 {
 842         uint8_t *keys;
 843         uint64_t hva;
 844         unsigned long curkey;
 845         int i, r = 0;
 846
 847         if (args->flags != 0)
 848                 return -EINVAL;
 849
 850         /* Is this guest using storage keys? */
 851         if (!mm_use_skey(current->mm))
 852                 return KVM_S390_GET_SKEYS_NONE;
 853
 854         /* Enforce sane limit on memory allocation */
 855         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
 856                 return -EINVAL;
 857
 858         keys = kmalloc_array(args->count, sizeof(uint8_t),
 859                              GFP_KERNEL | __GFP_NOWARN);
 860         if (!keys)
 861                 keys = vmalloc(sizeof(uint8_t) * args->count);
 862         if (!keys)
 863                 return -ENOMEM;
 864
 865         for (i = 0; i < args->count; i++) {
 866                 hva = gfn_to_hva(kvm, args->start_gfn + i);
 867                 if (kvm_is_error_hva(hva)) {
 868                         r = -EFAULT;
 869                         goto out;
 870                 }
 871
 872                 curkey = get_guest_storage_key(current->mm, hva);
 873                 if (IS_ERR_VALUE(curkey)) {
 874                         r = curkey;
 875                         goto out;
 876                 }
 877                 keys[i] = curkey;
 878         }
 879
 880         r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
 881                          sizeof(uint8_t) * args->count);
 882         if (r)
 883                 r = -EFAULT;
 884 out:
 885         kvfree(keys);
 886         return r;
 887 }
 888
 889 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
 890 {
 891         uint8_t *keys;
 892         uint64_t hva;
 893         int i, r = 0;
 894
 895         if (args->flags != 0)
 896                 return -EINVAL;
 897
 898         /* Enforce sane limit on memory allocation */
 899         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
 900                 return -EINVAL;
 901
 902         keys = kmalloc_array(args->count, sizeof(uint8_t),
 903                              GFP_KERNEL | __GFP_NOWARN);
 904         if (!keys)
 905                 keys = vmalloc(sizeof(uint8_t) * args->count);
 906         if (!keys)
 907                 return -ENOMEM;
 908
 909         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
 910                            sizeof(uint8_t) * args->count);
 911         if (r) {
 912                 r = -EFAULT;
 913                 goto out;
 914         }
 915
 916         /* Enable storage key handling for the guest */
 917         r = s390_enable_skey();
 918         if (r)
 919                 goto out;
 920
 921         for (i = 0; i < args->count; i++) {
 922                 hva = gfn_to_hva(kvm, args->start_gfn + i);
 923                 if (kvm_is_error_hva(hva)) {
 924                         r = -EFAULT;
 925                         goto out;
 926                 }
 927
 928                 /* Lowest order bit is reserved */
 929                 if (keys[i] & 0x01) {
 930                         r = -EINVAL;
 931                         goto out;
 932                 }
 933
 934                 r = set_guest_storage_key(current->mm, hva,
 935                                           (unsigned long)keys[i], 0);
 936                 if (r)
 937                         goto out;
 938         }
 939 out:
 940         kvfree(keys);
 941         return r;
 942 }
 943
 944 long kvm_arch_vm_ioctl(struct file *filp,
 945                        unsigned int ioctl, unsigned long arg)
 946 {
 947         struct kvm *kvm = filp->private_data;
 948         void __user *argp = (void __user *)arg;
 949         struct kvm_device_attr attr;
 950         int r;
 951
 952         switch (ioctl) {
 953         case KVM_S390_INTERRUPT: {
 954                 struct kvm_s390_interrupt s390int;
 955
 956                 r = -EFAULT;
 957                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
 958                         break;
 959                 r = kvm_s390_inject_vm(kvm, &s390int);
 960                 break;
 961         }
 962         case KVM_ENABLE_CAP: {
 963                 struct kvm_enable_cap cap;
 964                 r = -EFAULT;
 965                 if (copy_from_user(&cap, argp, sizeof(cap)))
 966                         break;
 967                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 968                 break;
 969         }
 970         case KVM_CREATE_IRQCHIP: {
 971                 struct kvm_irq_routing_entry routing;
 972
 973                 r = -EINVAL;
 974                 if (kvm->arch.use_irqchip) {
 975                         /* Set up dummy routing. */
 976                         memset(&routing, 0, sizeof(routing));
 977                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
 978                 }
 979                 break;
 980         }
 981         case KVM_SET_DEVICE_ATTR: {
 982                 r = -EFAULT;
 983                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 984                         break;
 985                 r = kvm_s390_vm_set_attr(kvm, &attr);
 986                 break;
 987         }
 988         case KVM_GET_DEVICE_ATTR: {
 989                 r = -EFAULT;
 990                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 991                         break;
 992                 r = kvm_s390_vm_get_attr(kvm, &attr);
 993                 break;
 994         }
 995         case KVM_HAS_DEVICE_ATTR: {
 996                 r = -EFAULT;
 997                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 998                         break;
 999                 r = kvm_s390_vm_has_attr(kvm, &attr);
1000                 break;
1001         }
1002         case KVM_S390_GET_SKEYS: {
1003                 struct kvm_s390_skeys args;
1004
1005                 r = -EFAULT;
1006                 if (copy_from_user(&args, argp,
1007                                    sizeof(struct kvm_s390_skeys)))
1008                         break;
1009                 r = kvm_s390_get_skeys(kvm, &args);
1010                 break;
1011         }
1012         case KVM_S390_SET_SKEYS: {
1013                 struct kvm_s390_skeys args;
1014
1015                 r = -EFAULT;
1016                 if (copy_from_user(&args, argp,
1017                                    sizeof(struct kvm_s390_skeys)))
1018                         break;
1019                 r = kvm_s390_set_skeys(kvm, &args);
1020                 break;
1021         }
1022         default:
1023                 r = -ENOTTY;
1024         }
1025
1026         return r;
1027 }
1028
1029 static int kvm_s390_query_ap_config(u8 *config)
1030 {
1031         u32 fcn_code = 0x04000000UL;
1032         u32 cc = 0;
1033
1034         memset(config, 0, 128);
1035         asm volatile(
1036                 "lgr 0,%1\n"
1037                 "lgr 2,%2\n"
1038                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
1039                 "0: ipm %0\n"
1040                 "srl %0,28\n"
1041                 "1:\n"
1042                 EX_TABLE(0b, 1b)
1043                 : "+r" (cc)
1044                 : "r" (fcn_code), "r" (config)
1045                 : "cc", "0", "2", "memory"
1046         );
1047
1048         return cc;
1049 }
1050
1051 static int kvm_s390_apxa_installed(void)
1052 {
1053         u8 config[128];
1054         int cc;
1055
1056         if (test_facility(12)) {
1057                 cc = kvm_s390_query_ap_config(config);
1058
1059                 if (cc)
1060                         pr_err("PQAP(QCI) failed with cc=%d", cc);
1061                 else
1062                         return config[0] & 0x40;
1063         }
1064
1065         return 0;
1066 }
1067
1068 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1069 {
1070         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1071
1072         if (kvm_s390_apxa_installed())
1073                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1074         else
1075                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1076 }
1077
1078 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1079 {
1080         get_cpu_id(cpu_id);
1081         cpu_id->version = 0xff;
1082 }
1083
1084 static int kvm_s390_crypto_init(struct kvm *kvm)
1085 {
1086         if (!test_kvm_facility(kvm, 76))
1087                 return 0;
1088
1089         kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1090                                          GFP_KERNEL | GFP_DMA);
1091         if (!kvm->arch.crypto.crycb)
1092                 return -ENOMEM;
1093
1094         kvm_s390_set_crycb_format(kvm);
1095
1096         /* Enable AES/DEA protected key functions by default */
1097         kvm->arch.crypto.aes_kw = 1;
1098         kvm->arch.crypto.dea_kw = 1;
1099         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1100                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1101         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1102                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1103
1104         return 0;
1105 }
1106
1107 static void sca_dispose(struct kvm *kvm)
1108 {
1109         if (kvm->arch.use_esca)
1110                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1111         else
1112                 free_page((unsigned long)(kvm->arch.sca));
1113         kvm->arch.sca = NULL;
1114 }
1115
1116 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1117 {
1118         int i, rc;
1119         char debug_name[16];
1120         static unsigned long sca_offset;
1121
1122         rc = -EINVAL;
1123 #ifdef CONFIG_KVM_S390_UCONTROL
1124         if (type & ~KVM_VM_S390_UCONTROL)
1125                 goto out_err;
1126         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1127                 goto out_err;
1128 #else
1129         if (type)
1130                 goto out_err;
1131 #endif
1132
1133         rc = s390_enable_sie();
1134         if (rc)
1135                 goto out_err;
1136
1137         rc = -ENOMEM;
1138
1139         kvm->arch.use_esca = 0; /* start with basic SCA */
1140         rwlock_init(&kvm->arch.sca_lock);
1141         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(GFP_KERNEL);
1142         if (!kvm->arch.sca)
1143                 goto out_err;
1144         spin_lock(&kvm_lock);
1145         sca_offset += 16;
1146         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1147                 sca_offset = 0;
1148         kvm->arch.sca = (struct bsca_block *)
1149                         ((char *) kvm->arch.sca + sca_offset);
1150         spin_unlock(&kvm_lock);
1151
1152         sprintf(debug_name, "kvm-%u", current->pid);
1153
1154         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1155         if (!kvm->arch.dbf)
1156                 goto out_err;
1157
1158         /*
1159          * The architectural maximum amount of facilities is 16 kbit. To store
1160          * this amount, 2 kbyte of memory is required. Thus we need a full
1161          * page to hold the guest facility list (arch.model.fac->list) and the
1162          * facility mask (arch.model.fac->mask). Its address size has to be
1163          * 31 bits and word aligned.
1164          */
1165         kvm->arch.model.fac =
1166                 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1167         if (!kvm->arch.model.fac)
1168                 goto out_err;
1169
1170         /* Populate the facility mask initially. */
1171         memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1172                S390_ARCH_FAC_LIST_SIZE_BYTE);
1173         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1174                 if (i < kvm_s390_fac_list_mask_size())
1175                         kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1176                 else
1177                         kvm->arch.model.fac->mask[i] = 0UL;
1178         }
1179
1180         /* Populate the facility list initially. */
1181         memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1182                S390_ARCH_FAC_LIST_SIZE_BYTE);
1183
1184         kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1185         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1186
1187         if (kvm_s390_crypto_init(kvm) < 0)
1188                 goto out_err;
1189
1190         spin_lock_init(&kvm->arch.float_int.lock);
1191         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1192                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1193         init_waitqueue_head(&kvm->arch.ipte_wq);
1194         mutex_init(&kvm->arch.ipte_mutex);
1195
1196         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1197         VM_EVENT(kvm, 3, "vm created with type %lu", type);
1198
1199         if (type & KVM_VM_S390_UCONTROL) {
1200                 kvm->arch.gmap = NULL;
1201                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1202         } else {
1203                 if (sclp.hamax == U64_MAX)
1204                         kvm->arch.mem_limit = TASK_MAX_SIZE;
1205                 else
1206                         kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1207                                                     sclp.hamax + 1);
1208                 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
1209                 if (!kvm->arch.gmap)
1210                         goto out_err;
1211                 kvm->arch.gmap->private = kvm;
1212                 kvm->arch.gmap->pfault_enabled = 0;
1213         }
1214
1215         kvm->arch.css_support = 0;
1216         kvm->arch.use_irqchip = 0;
1217         kvm->arch.epoch = 0;
1218
1219         spin_lock_init(&kvm->arch.start_stop_lock);
1220         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1221
1222         return 0;
1223 out_err:
1224         kfree(kvm->arch.crypto.crycb);
1225         free_page((unsigned long)kvm->arch.model.fac);
1226         debug_unregister(kvm->arch.dbf);
1227         sca_dispose(kvm);
1228         KVM_EVENT(3, "creation of vm failed: %d", rc);
1229         return rc;
1230 }
1231
1232 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1233 {
1234         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1235         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1236         kvm_s390_clear_local_irqs(vcpu);
1237         kvm_clear_async_pf_completion_queue(vcpu);
1238         if (!kvm_is_ucontrol(vcpu->kvm))
1239                 sca_del_vcpu(vcpu);
1240
1241         if (kvm_is_ucontrol(vcpu->kvm))
1242                 gmap_free(vcpu->arch.gmap);
1243
1244         if (vcpu->kvm->arch.use_cmma)
1245                 kvm_s390_vcpu_unsetup_cmma(vcpu);
1246         free_page((unsigned long)(vcpu->arch.sie_block));
1247
1248         kvm_vcpu_uninit(vcpu);
1249         kmem_cache_free(kvm_vcpu_cache, vcpu);
1250 }
1251
1252 static void kvm_free_vcpus(struct kvm *kvm)
1253 {
1254         unsigned int i;
1255         struct kvm_vcpu *vcpu;
1256
1257         kvm_for_each_vcpu(i, vcpu, kvm)
1258                 kvm_arch_vcpu_destroy(vcpu);
1259
1260         mutex_lock(&kvm->lock);
1261         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1262                 kvm->vcpus[i] = NULL;
1263
1264         atomic_set(&kvm->online_vcpus, 0);
1265         mutex_unlock(&kvm->lock);
1266 }
1267
1268 void kvm_arch_destroy_vm(struct kvm *kvm)
1269 {
1270         kvm_free_vcpus(kvm);
1271         free_page((unsigned long)kvm->arch.model.fac);
1272         sca_dispose(kvm);
1273         debug_unregister(kvm->arch.dbf);
1274         kfree(kvm->arch.crypto.crycb);
1275         if (!kvm_is_ucontrol(kvm))
1276                 gmap_free(kvm->arch.gmap);
1277         kvm_s390_destroy_adapters(kvm);
1278         kvm_s390_clear_float_irqs(kvm);
1279         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1280 }
1281
1282 /* Section: vcpu related */
1283 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1284 {
1285         vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1286         if (!vcpu->arch.gmap)
1287                 return -ENOMEM;
1288         vcpu->arch.gmap->private = vcpu->kvm;
1289
1290         return 0;
1291 }
1292
1293 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1294 {
1295         read_lock(&vcpu->kvm->arch.sca_lock);
1296         if (vcpu->kvm->arch.use_esca) {
1297                 struct esca_block *sca = vcpu->kvm->arch.sca;
1298
1299                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1300                 sca->cpu[vcpu->vcpu_id].sda = 0;
1301         } else {
1302                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1303
1304                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1305                 sca->cpu[vcpu->vcpu_id].sda = 0;
1306         }
1307         read_unlock(&vcpu->kvm->arch.sca_lock);
1308 }
1309
1310 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1311 {
1312         read_lock(&vcpu->kvm->arch.sca_lock);
1313         if (vcpu->kvm->arch.use_esca) {
1314                 struct esca_block *sca = vcpu->kvm->arch.sca;
1315
1316                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1317                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1318                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1319                 vcpu->arch.sie_block->ecb2 |= 0x04U;
1320                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1321         } else {
1322                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1323
1324                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1325                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1326                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1327                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1328         }
1329         read_unlock(&vcpu->kvm->arch.sca_lock);
1330 }
1331
1332 /* Basic SCA to Extended SCA data copy routines */
1333 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1334 {
1335         d->sda = s->sda;
1336         d->sigp_ctrl.c = s->sigp_ctrl.c;
1337         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1338 }
1339
1340 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1341 {
1342         int i;
1343
1344         d->ipte_control = s->ipte_control;
1345         d->mcn[0] = s->mcn;
1346         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1347                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1348 }
1349
1350 static int sca_switch_to_extended(struct kvm *kvm)
1351 {
1352         struct bsca_block *old_sca = kvm->arch.sca;
1353         struct esca_block *new_sca;
1354         struct kvm_vcpu *vcpu;
1355         unsigned int vcpu_idx;
1356         u32 scaol, scaoh;
1357
1358         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1359         if (!new_sca)
1360                 return -ENOMEM;
1361
1362         scaoh = (u32)((u64)(new_sca) >> 32);
1363         scaol = (u32)(u64)(new_sca) & ~0x3fU;
1364
1365         kvm_s390_vcpu_block_all(kvm);
1366         write_lock(&kvm->arch.sca_lock);
1367
1368         sca_copy_b_to_e(new_sca, old_sca);
1369
1370         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1371                 vcpu->arch.sie_block->scaoh = scaoh;
1372                 vcpu->arch.sie_block->scaol = scaol;
1373                 vcpu->arch.sie_block->ecb2 |= 0x04U;
1374         }
1375         kvm->arch.sca = new_sca;
1376         kvm->arch.use_esca = 1;
1377
1378         write_unlock(&kvm->arch.sca_lock);
1379         kvm_s390_vcpu_unblock_all(kvm);
1380
1381         free_page((unsigned long)old_sca);
1382
1383         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1384                  old_sca, kvm->arch.sca);
1385         return 0;
1386 }
1387
1388 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1389 {
1390         int rc;
1391
1392         if (id < KVM_S390_BSCA_CPU_SLOTS)
1393                 return true;
1394         if (!sclp.has_esca)
1395                 return false;
1396
1397         mutex_lock(&kvm->lock);
1398         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1399         mutex_unlock(&kvm->lock);
1400
1401         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1402 }
1403
1404 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1405 {
1406         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1407         kvm_clear_async_pf_completion_queue(vcpu);
1408         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1409                                     KVM_SYNC_GPRS |
1410                                     KVM_SYNC_ACRS |
1411                                     KVM_SYNC_CRS |
1412                                     KVM_SYNC_ARCH0 |
1413                                     KVM_SYNC_PFAULT;
1414         if (test_kvm_facility(vcpu->kvm, 64))
1415                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
1416         /* fprs can be synchronized via vrs, even if the guest has no vx. With
1417          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1418          */
1419         if (MACHINE_HAS_VX)
1420                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1421         else
1422                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
1423
1424         if (kvm_is_ucontrol(vcpu->kvm))
1425                 return __kvm_ucontrol_vcpu_init(vcpu);
1426
1427         return 0;
1428 }
1429
1430 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1431 {
1432         /* Save host register state */
1433         save_fpu_regs();
1434         vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1435         vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
1436
1437         if (MACHINE_HAS_VX)
1438                 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1439         else
1440                 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
1441         current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1442         if (test_fp_ctl(current->thread.fpu.fpc))
1443                 /* User space provided an invalid FPC, let's clear it */
1444                 current->thread.fpu.fpc = 0;
1445
1446         save_access_regs(vcpu->arch.host_acrs);
1447         restore_access_regs(vcpu->run->s.regs.acrs);
1448         gmap_enable(vcpu->arch.gmap);
1449         atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1450 }
1451
1452 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1453 {
1454         atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1455         gmap_disable(vcpu->arch.gmap);
1456
1457         /* Save guest register state */
1458         save_fpu_regs();
1459         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1460
1461         /* Restore host register state */
1462         current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1463         current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
1464
1465         save_access_regs(vcpu->run->s.regs.acrs);
1466         restore_access_regs(vcpu->arch.host_acrs);
1467 }
1468
1469 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1470 {
1471         /* this equals initial cpu reset in pop, but we don't switch to ESA */
1472         vcpu->arch.sie_block->gpsw.mask = 0UL;
1473         vcpu->arch.sie_block->gpsw.addr = 0UL;
1474         kvm_s390_set_prefix(vcpu, 0);
1475         vcpu->arch.sie_block->cputm     = 0UL;
1476         vcpu->arch.sie_block->ckc       = 0UL;
1477         vcpu->arch.sie_block->todpr     = 0;
1478         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1479         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
1480         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1481         /* make sure the new fpc will be lazily loaded */
1482         save_fpu_regs();
1483         current->thread.fpu.fpc = 0;
1484         vcpu->arch.sie_block->gbea = 1;
1485         vcpu->arch.sie_block->pp = 0;
1486         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1487         kvm_clear_async_pf_completion_queue(vcpu);
1488         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1489                 kvm_s390_vcpu_stop(vcpu);
1490         kvm_s390_clear_local_irqs(vcpu);
1491 }
1492
1493 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1494 {
1495         mutex_lock(&vcpu->kvm->lock);
1496         preempt_disable();
1497         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1498         preempt_enable();
1499         mutex_unlock(&vcpu->kvm->lock);
1500         if (!kvm_is_ucontrol(vcpu->kvm)) {
1501                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1502                 sca_add_vcpu(vcpu);
1503         }
1504
1505 }
1506
1507 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1508 {
1509         if (!test_kvm_facility(vcpu->kvm, 76))
1510                 return;
1511
1512         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1513
1514         if (vcpu->kvm->arch.crypto.aes_kw)
1515                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1516         if (vcpu->kvm->arch.crypto.dea_kw)
1517                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1518
1519         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1520 }
1521
1522 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1523 {
1524         free_page(vcpu->arch.sie_block->cbrlo);
1525         vcpu->arch.sie_block->cbrlo = 0;
1526 }
1527
1528 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1529 {
1530         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1531         if (!vcpu->arch.sie_block->cbrlo)
1532                 return -ENOMEM;
1533
1534         vcpu->arch.sie_block->ecb2 |= 0x80;
1535         vcpu->arch.sie_block->ecb2 &= ~0x08;
1536         return 0;
1537 }
1538
1539 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1540 {
1541         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1542
1543         vcpu->arch.cpu_id = model->cpu_id;
1544         vcpu->arch.sie_block->ibc = model->ibc;
1545         vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1546 }
1547
1548 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1549 {
1550         int rc = 0;
1551
1552         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1553                                                     CPUSTAT_SM |
1554                                                     CPUSTAT_STOPPED);
1555
1556         if (test_kvm_facility(vcpu->kvm, 78))
1557                 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1558         else if (test_kvm_facility(vcpu->kvm, 8))
1559                 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1560
1561         kvm_s390_vcpu_setup_model(vcpu);
1562
1563         vcpu->arch.sie_block->ecb   = 6;
1564         if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1565                 vcpu->arch.sie_block->ecb |= 0x10;
1566
1567         vcpu->arch.sie_block->ecb2  = 8;
1568         vcpu->arch.sie_block->eca   = 0xC1002000U;
1569         if (sclp.has_siif)
1570                 vcpu->arch.sie_block->eca |= 1;
1571         if (sclp.has_sigpif)
1572                 vcpu->arch.sie_block->eca |= 0x10000000U;
1573         if (test_kvm_facility(vcpu->kvm, 64))
1574                 vcpu->arch.sie_block->ecb3 |= 0x01;
1575         if (test_kvm_facility(vcpu->kvm, 129)) {
1576                 vcpu->arch.sie_block->eca |= 0x00020000;
1577                 vcpu->arch.sie_block->ecd |= 0x20000000;
1578         }
1579         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
1580         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1581
1582         if (vcpu->kvm->arch.use_cmma) {
1583                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1584                 if (rc)
1585                         return rc;
1586         }
1587         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1588         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1589
1590         kvm_s390_vcpu_crypto_setup(vcpu);
1591
1592         return rc;
1593 }
1594
1595 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1596                                       unsigned int id)
1597 {
1598         struct kvm_vcpu *vcpu;
1599         struct sie_page *sie_page;
1600         int rc = -EINVAL;
1601
1602         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
1603                 goto out;
1604
1605         rc = -ENOMEM;
1606
1607         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1608         if (!vcpu)
1609                 goto out;
1610
1611         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1612         if (!sie_page)
1613                 goto out_free_cpu;
1614
1615         vcpu->arch.sie_block = &sie_page->sie_block;
1616         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1617
1618         vcpu->arch.sie_block->icpua = id;
1619         spin_lock_init(&vcpu->arch.local_int.lock);
1620         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1621         vcpu->arch.local_int.wq = &vcpu->wq;
1622         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1623
1624         rc = kvm_vcpu_init(vcpu, kvm, id);
1625         if (rc)
1626                 goto out_free_sie_block;
1627         VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
1628                  vcpu->arch.sie_block);
1629         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1630
1631         return vcpu;
1632 out_free_sie_block:
1633         free_page((unsigned long)(vcpu->arch.sie_block));
1634 out_free_cpu:
1635         kmem_cache_free(kvm_vcpu_cache, vcpu);
1636 out:
1637         return ERR_PTR(rc);
1638 }
1639
1640 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1641 {
1642         return kvm_s390_vcpu_has_irq(vcpu, 0);
1643 }
1644
1645 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1646 {
1647         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1648         exit_sie(vcpu);
1649 }
1650
1651 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1652 {
1653         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1654 }
1655
1656 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1657 {
1658         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1659         exit_sie(vcpu);
1660 }
1661
1662 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1663 {
1664         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1665 }
1666
1667 /*
1668  * Kick a guest cpu out of SIE and wait until SIE is not running.
1669  * If the CPU is not running (e.g. waiting as idle) the function will
1670  * return immediately. */
1671 void exit_sie(struct kvm_vcpu *vcpu)
1672 {
1673         atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1674         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1675                 cpu_relax();
1676 }
1677
1678 /* Kick a guest cpu out of SIE to process a request synchronously */
1679 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1680 {
1681         kvm_make_request(req, vcpu);
1682         kvm_s390_vcpu_request(vcpu);
1683 }
1684
1685 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1686 {
1687         int i;
1688         struct kvm *kvm = gmap->private;
1689         struct kvm_vcpu *vcpu;
1690
1691         kvm_for_each_vcpu(i, vcpu, kvm) {
1692                 /* match against both prefix pages */
1693                 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1694                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1695                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1696                 }
1697         }
1698 }
1699
1700 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1701 {
1702         /* kvm common code refers to this, but never calls it */
1703         BUG();
1704         return 0;
1705 }
1706
1707 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1708                                            struct kvm_one_reg *reg)
1709 {
1710         int r = -EINVAL;
1711
1712         switch (reg->id) {
1713         case KVM_REG_S390_TODPR:
1714                 r = put_user(vcpu->arch.sie_block->todpr,
1715                              (u32 __user *)reg->addr);
1716                 break;
1717         case KVM_REG_S390_EPOCHDIFF:
1718                 r = put_user(vcpu->arch.sie_block->epoch,
1719                              (u64 __user *)reg->addr);
1720                 break;
1721         case KVM_REG_S390_CPU_TIMER:
1722                 r = put_user(vcpu->arch.sie_block->cputm,
1723                              (u64 __user *)reg->addr);
1724                 break;
1725         case KVM_REG_S390_CLOCK_COMP:
1726                 r = put_user(vcpu->arch.sie_block->ckc,
1727                              (u64 __user *)reg->addr);
1728                 break;
1729         case KVM_REG_S390_PFTOKEN:
1730                 r = put_user(vcpu->arch.pfault_token,
1731                              (u64 __user *)reg->addr);
1732                 break;
1733         case KVM_REG_S390_PFCOMPARE:
1734                 r = put_user(vcpu->arch.pfault_compare,
1735                              (u64 __user *)reg->addr);
1736                 break;
1737         case KVM_REG_S390_PFSELECT:
1738                 r = put_user(vcpu->arch.pfault_select,
1739                              (u64 __user *)reg->addr);
1740                 break;
1741         case KVM_REG_S390_PP:
1742                 r = put_user(vcpu->arch.sie_block->pp,
1743                              (u64 __user *)reg->addr);
1744                 break;
1745         case KVM_REG_S390_GBEA:
1746                 r = put_user(vcpu->arch.sie_block->gbea,
1747                              (u64 __user *)reg->addr);
1748                 break;
1749         default:
1750                 break;
1751         }
1752
1753         return r;
1754 }
1755
1756 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1757                                            struct kvm_one_reg *reg)
1758 {
1759         int r = -EINVAL;
1760
1761         switch (reg->id) {
1762         case KVM_REG_S390_TODPR:
1763                 r = get_user(vcpu->arch.sie_block->todpr,
1764                              (u32 __user *)reg->addr);
1765                 break;
1766         case KVM_REG_S390_EPOCHDIFF:
1767                 r = get_user(vcpu->arch.sie_block->epoch,
1768                              (u64 __user *)reg->addr);
1769                 break;
1770         case KVM_REG_S390_CPU_TIMER:
1771                 r = get_user(vcpu->arch.sie_block->cputm,
1772                              (u64 __user *)reg->addr);
1773                 break;
1774         case KVM_REG_S390_CLOCK_COMP:
1775                 r = get_user(vcpu->arch.sie_block->ckc,
1776                              (u64 __user *)reg->addr);
1777                 break;
1778         case KVM_REG_S390_PFTOKEN:
1779                 r = get_user(vcpu->arch.pfault_token,
1780                              (u64 __user *)reg->addr);
1781                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1782                         kvm_clear_async_pf_completion_queue(vcpu);
1783                 break;
1784         case KVM_REG_S390_PFCOMPARE:
1785                 r = get_user(vcpu->arch.pfault_compare,
1786                              (u64 __user *)reg->addr);
1787                 break;
1788         case KVM_REG_S390_PFSELECT:
1789                 r = get_user(vcpu->arch.pfault_select,
1790                              (u64 __user *)reg->addr);
1791                 break;
1792         case KVM_REG_S390_PP:
1793                 r = get_user(vcpu->arch.sie_block->pp,
1794                              (u64 __user *)reg->addr);
1795                 break;
1796         case KVM_REG_S390_GBEA:
1797                 r = get_user(vcpu->arch.sie_block->gbea,
1798                              (u64 __user *)reg->addr);
1799                 break;
1800         default:
1801                 break;
1802         }
1803
1804         return r;
1805 }
1806
1807 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1808 {
1809         kvm_s390_vcpu_initial_reset(vcpu);
1810         return 0;
1811 }
1812
1813 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1814 {
1815         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1816         return 0;
1817 }
1818
1819 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1820 {
1821         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1822         return 0;
1823 }
1824
1825 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1826                                   struct kvm_sregs *sregs)
1827 {
1828         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1829         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1830         restore_access_regs(vcpu->run->s.regs.acrs);
1831         return 0;
1832 }
1833
1834 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1835                                   struct kvm_sregs *sregs)
1836 {
1837         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1838         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1839         return 0;
1840 }
1841
1842 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1843 {
1844         /* make sure the new values will be lazily loaded */
1845         save_fpu_regs();
1846         if (test_fp_ctl(fpu->fpc))
1847                 return -EINVAL;
1848         current->thread.fpu.fpc = fpu->fpc;
1849         if (MACHINE_HAS_VX)
1850                 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
1851         else
1852                 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
1853         return 0;
1854 }
1855
1856 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1857 {
1858         /* make sure we have the latest values */
1859         save_fpu_regs();
1860         if (MACHINE_HAS_VX)
1861                 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
1862         else
1863                 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
1864         fpu->fpc = current->thread.fpu.fpc;
1865         return 0;
1866 }
1867
1868 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1869 {
1870         int rc = 0;
1871
1872         if (!is_vcpu_stopped(vcpu))
1873                 rc = -EBUSY;
1874         else {
1875                 vcpu->run->psw_mask = psw.mask;
1876                 vcpu->run->psw_addr = psw.addr;
1877         }
1878         return rc;
1879 }
1880
1881 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1882                                   struct kvm_translation *tr)
1883 {
1884         return -EINVAL; /* not implemented yet */
1885 }
1886
1887 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1888                               KVM_GUESTDBG_USE_HW_BP | \
1889                               KVM_GUESTDBG_ENABLE)
1890
1891 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1892                                         struct kvm_guest_debug *dbg)
1893 {
1894         int rc = 0;
1895
1896         vcpu->guest_debug = 0;
1897         kvm_s390_clear_bp_data(vcpu);
1898
1899         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1900                 return -EINVAL;
1901
1902         if (dbg->control & KVM_GUESTDBG_ENABLE) {
1903                 vcpu->guest_debug = dbg->control;
1904                 /* enforce guest PER */
1905                 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1906
1907                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1908                         rc = kvm_s390_import_bp_data(vcpu, dbg);
1909         } else {
1910                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1911                 vcpu->arch.guestdbg.last_bp = 0;
1912         }
1913
1914         if (rc) {
1915                 vcpu->guest_debug = 0;
1916                 kvm_s390_clear_bp_data(vcpu);
1917                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1918         }
1919
1920         return rc;
1921 }
1922
1923 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1924                                     struct kvm_mp_state *mp_state)
1925 {
1926         /* CHECK_STOP and LOAD are not supported yet */
1927         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1928                                        KVM_MP_STATE_OPERATING;
1929 }
1930
1931 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1932                                     struct kvm_mp_state *mp_state)
1933 {
1934         int rc = 0;
1935
1936         /* user space knows about this interface - let it control the state */
1937         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1938
1939         switch (mp_state->mp_state) {
1940         case KVM_MP_STATE_STOPPED:
1941                 kvm_s390_vcpu_stop(vcpu);
1942                 break;
1943         case KVM_MP_STATE_OPERATING:
1944                 kvm_s390_vcpu_start(vcpu);
1945                 break;
1946         case KVM_MP_STATE_LOAD:
1947         case KVM_MP_STATE_CHECK_STOP:
1948                 /* fall through - CHECK_STOP and LOAD are not supported yet */
1949         default:
1950                 rc = -ENXIO;
1951         }
1952
1953         return rc;
1954 }
1955
1956 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1957 {
1958         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1959 }
1960
1961 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1962 {
1963 retry:
1964         kvm_s390_vcpu_request_handled(vcpu);
1965         if (!vcpu->requests)
1966                 return 0;
1967         /*
1968          * We use MMU_RELOAD just to re-arm the ipte notifier for the
1969          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1970          * This ensures that the ipte instruction for this request has
1971          * already finished. We might race against a second unmapper that
1972          * wants to set the blocking bit. Lets just retry the request loop.
1973          */
1974         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1975                 int rc;
1976                 rc = gmap_ipte_notify(vcpu->arch.gmap,
1977                                       kvm_s390_get_prefix(vcpu),
1978                                       PAGE_SIZE * 2);
1979                 if (rc)
1980                         return rc;
1981                 goto retry;
1982         }
1983
1984         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1985                 vcpu->arch.sie_block->ihcpu = 0xffff;
1986                 goto retry;
1987         }
1988
1989         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1990                 if (!ibs_enabled(vcpu)) {
1991                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1992                         atomic_or(CPUSTAT_IBS,
1993                                         &vcpu->arch.sie_block->cpuflags);
1994                 }
1995                 goto retry;
1996         }
1997
1998         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1999                 if (ibs_enabled(vcpu)) {
2000                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2001                         atomic_andnot(CPUSTAT_IBS,
2002                                           &vcpu->arch.sie_block->cpuflags);
2003                 }
2004                 goto retry;
2005         }
2006
2007         /* nothing to do, just clear the request */
2008         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2009
2010         return 0;
2011 }
2012
2013 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2014 {
2015         struct kvm_vcpu *vcpu;
2016         int i;
2017
2018         mutex_lock(&kvm->lock);
2019         preempt_disable();
2020         kvm->arch.epoch = tod - get_tod_clock();
2021         kvm_s390_vcpu_block_all(kvm);
2022         kvm_for_each_vcpu(i, vcpu, kvm)
2023                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2024         kvm_s390_vcpu_unblock_all(kvm);
2025         preempt_enable();
2026         mutex_unlock(&kvm->lock);
2027 }
2028
2029 /**
2030  * kvm_arch_fault_in_page - fault-in guest page if necessary
2031  * @vcpu: The corresponding virtual cpu
2032  * @gpa: Guest physical address
2033  * @writable: Whether the page should be writable or not
2034  *
2035  * Make sure that a guest page has been faulted-in on the host.
2036  *
2037  * Return: Zero on success, negative error code otherwise.
2038  */
2039 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2040 {
2041         return gmap_fault(vcpu->arch.gmap, gpa,
2042                           writable ? FAULT_FLAG_WRITE : 0);
2043 }
2044
2045 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2046                                       unsigned long token)
2047 {
2048         struct kvm_s390_interrupt inti;
2049         struct kvm_s390_irq irq;
2050
2051         if (start_token) {
2052                 irq.u.ext.ext_params2 = token;
2053                 irq.type = KVM_S390_INT_PFAULT_INIT;
2054                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2055         } else {
2056                 inti.type = KVM_S390_INT_PFAULT_DONE;
2057                 inti.parm64 = token;
2058                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2059         }
2060 }
2061
2062 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2063                                      struct kvm_async_pf *work)
2064 {
2065         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2066         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2067 }
2068
2069 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2070                                  struct kvm_async_pf *work)
2071 {
2072         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2073         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2074 }
2075
2076 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2077                                struct kvm_async_pf *work)
2078 {
2079         /* s390 will always inject the page directly */
2080 }
2081
2082 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2083 {
2084         /*
2085          * s390 will always inject the page directly,
2086          * but we still want check_async_completion to cleanup
2087          */
2088         return true;
2089 }
2090
2091 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2092 {
2093         hva_t hva;
2094         struct kvm_arch_async_pf arch;
2095         int rc;
2096
2097         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2098                 return 0;
2099         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2100             vcpu->arch.pfault_compare)
2101                 return 0;
2102         if (psw_extint_disabled(vcpu))
2103                 return 0;
2104         if (kvm_s390_vcpu_has_irq(vcpu, 0))
2105                 return 0;
2106         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2107                 return 0;
2108         if (!vcpu->arch.gmap->pfault_enabled)
2109                 return 0;
2110
2111         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2112         hva += current->thread.gmap_addr & ~PAGE_MASK;
2113         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2114                 return 0;
2115
2116         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2117         return rc;
2118 }
2119
2120 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2121 {
2122         int rc, cpuflags;
2123
2124         /*
2125          * On s390 notifications for arriving pages will be delivered directly
2126          * to the guest but the house keeping for completed pfaults is
2127          * handled outside the worker.
2128          */
2129         kvm_check_async_pf_completion(vcpu);
2130
2131         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2132         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2133
2134         if (need_resched())
2135                 schedule();
2136
2137         if (test_cpu_flag(CIF_MCCK_PENDING))
2138                 s390_handle_mcck();
2139
2140         if (!kvm_is_ucontrol(vcpu->kvm)) {
2141                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2142                 if (rc)
2143                         return rc;
2144         }
2145
2146         rc = kvm_s390_handle_requests(vcpu);
2147         if (rc)
2148                 return rc;
2149
2150         if (guestdbg_enabled(vcpu)) {
2151                 kvm_s390_backup_guest_per_regs(vcpu);
2152                 kvm_s390_patch_guest_per_regs(vcpu);
2153         }
2154
2155         vcpu->arch.sie_block->icptcode = 0;
2156         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2157         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2158         trace_kvm_s390_sie_enter(vcpu, cpuflags);
2159
2160         return 0;
2161 }
2162
2163 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2164 {
2165         struct kvm_s390_pgm_info pgm_info = {
2166                 .code = PGM_ADDRESSING,
2167         };
2168         u8 opcode, ilen;
2169         int rc;
2170
2171         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2172         trace_kvm_s390_sie_fault(vcpu);
2173
2174         /*
2175          * We want to inject an addressing exception, which is defined as a
2176          * suppressing or terminating exception. However, since we came here
2177          * by a DAT access exception, the PSW still points to the faulting
2178          * instruction since DAT exceptions are nullifying. So we've got
2179          * to look up the current opcode to get the length of the instruction
2180          * to be able to forward the PSW.
2181          */
2182         rc = read_guest_instr(vcpu, &opcode, 1);
2183         ilen = insn_length(opcode);
2184         if (rc < 0) {
2185                 return rc;
2186         } else if (rc) {
2187                 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2188                  * Forward by arbitrary ilc, injection will take care of
2189                  * nullification if necessary.
2190                  */
2191                 pgm_info = vcpu->arch.pgm;
2192                 ilen = 4;
2193         }
2194         pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2195         kvm_s390_forward_psw(vcpu, ilen);
2196         return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
2197 }
2198
2199 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2200 {
2201         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2202                    vcpu->arch.sie_block->icptcode);
2203         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2204
2205         if (guestdbg_enabled(vcpu))
2206                 kvm_s390_restore_guest_per_regs(vcpu);
2207
2208         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2209         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
2210
2211         if (vcpu->arch.sie_block->icptcode > 0) {
2212                 int rc = kvm_handle_sie_intercept(vcpu);
2213
2214                 if (rc != -EOPNOTSUPP)
2215                         return rc;
2216                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2217                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2218                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2219                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2220                 return -EREMOTE;
2221         } else if (exit_reason != -EFAULT) {
2222                 vcpu->stat.exit_null++;
2223                 return 0;
2224         } else if (kvm_is_ucontrol(vcpu->kvm)) {
2225                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2226                 vcpu->run->s390_ucontrol.trans_exc_code =
2227                                                 current->thread.gmap_addr;
2228                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2229                 return -EREMOTE;
2230         } else if (current->thread.gmap_pfault) {
2231                 trace_kvm_s390_major_guest_pfault(vcpu);
2232                 current->thread.gmap_pfault = 0;
2233                 if (kvm_arch_setup_async_pf(vcpu))
2234                         return 0;
2235                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2236         }
2237         return vcpu_post_run_fault_in_sie(vcpu);
2238 }
2239
2240 static int __vcpu_run(struct kvm_vcpu *vcpu)
2241 {
2242         int rc, exit_reason;
2243
2244         /*
2245          * We try to hold kvm->srcu during most of vcpu_run (except when run-
2246          * ning the guest), so that memslots (and other stuff) are protected
2247          */
2248         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2249
2250         do {
2251                 rc = vcpu_pre_run(vcpu);
2252                 if (rc)
2253                         break;
2254
2255                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2256                 /*
2257                  * As PF_VCPU will be used in fault handler, between
2258                  * guest_enter and guest_exit should be no uaccess.
2259                  */
2260                 local_irq_disable();
2261                 __kvm_guest_enter();
2262                 local_irq_enable();
2263                 exit_reason = sie64a(vcpu->arch.sie_block,
2264                                      vcpu->run->s.regs.gprs);
2265                 local_irq_disable();
2266                 __kvm_guest_exit();
2267                 local_irq_enable();
2268                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2269
2270                 rc = vcpu_post_run(vcpu, exit_reason);
2271         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2272
2273         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2274         return rc;
2275 }
2276
2277 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2278 {
2279         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2280         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2281         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2282                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2283         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2284                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2285                 /* some control register changes require a tlb flush */
2286                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2287         }
2288         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2289                 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2290                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2291                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2292                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2293                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2294         }
2295         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2296                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2297                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2298                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2299                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2300                         kvm_clear_async_pf_completion_queue(vcpu);
2301         }
2302         kvm_run->kvm_dirty_regs = 0;
2303 }
2304
2305 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2306 {
2307         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2308         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2309         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2310         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2311         kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2312         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2313         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2314         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2315         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2316         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2317         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2318         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2319 }
2320
2321 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2322 {
2323         int rc;
2324         sigset_t sigsaved;
2325
2326         if (guestdbg_exit_pending(vcpu)) {
2327                 kvm_s390_prepare_debug_exit(vcpu);
2328                 return 0;
2329         }
2330
2331         if (vcpu->sigset_active)
2332                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2333
2334         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2335                 kvm_s390_vcpu_start(vcpu);
2336         } else if (is_vcpu_stopped(vcpu)) {
2337                 pr_err_ratelimited("can't run stopped vcpu %d\n",
2338                                    vcpu->vcpu_id);
2339                 return -EINVAL;
2340         }
2341
2342         sync_regs(vcpu, kvm_run);
2343
2344         might_fault();
2345         rc = __vcpu_run(vcpu);
2346
2347         if (signal_pending(current) && !rc) {
2348                 kvm_run->exit_reason = KVM_EXIT_INTR;
2349                 rc = -EINTR;
2350         }
2351
2352         if (guestdbg_exit_pending(vcpu) && !rc)  {
2353                 kvm_s390_prepare_debug_exit(vcpu);
2354                 rc = 0;
2355         }
2356
2357         if (rc == -EREMOTE) {
2358                 /* userspace support is needed, kvm_run has been prepared */
2359                 rc = 0;
2360         }
2361
2362         store_regs(vcpu, kvm_run);
2363
2364         if (vcpu->sigset_active)
2365                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2366
2367         vcpu->stat.exit_userspace++;
2368         return rc;
2369 }
2370
2371 /*
2372  * store status at address
2373  * we use have two special cases:
2374  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2375  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2376  */
2377 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2378 {
2379         unsigned char archmode = 1;
2380         freg_t fprs[NUM_FPRS];
2381         unsigned int px;
2382         u64 clkcomp;
2383         int rc;
2384
2385         px = kvm_s390_get_prefix(vcpu);
2386         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2387                 if (write_guest_abs(vcpu, 163, &archmode, 1))
2388                         return -EFAULT;
2389                 gpa = 0;
2390         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2391                 if (write_guest_real(vcpu, 163, &archmode, 1))
2392                         return -EFAULT;
2393                 gpa = px;
2394         } else
2395                 gpa -= __LC_FPREGS_SAVE_AREA;
2396
2397         /* manually convert vector registers if necessary */
2398         if (MACHINE_HAS_VX) {
2399                 convert_vx_to_fp(fprs, current->thread.fpu.vxrs);
2400                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2401                                      fprs, 128);
2402         } else {
2403                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2404                                      vcpu->run->s.regs.fprs, 128);
2405         }
2406         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2407                               vcpu->run->s.regs.gprs, 128);
2408         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2409                               &vcpu->arch.sie_block->gpsw, 16);
2410         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2411                               &px, 4);
2412         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2413                               &vcpu->run->s.regs.fpc, 4);
2414         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2415                               &vcpu->arch.sie_block->todpr, 4);
2416         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2417                               &vcpu->arch.sie_block->cputm, 8);
2418         clkcomp = vcpu->arch.sie_block->ckc >> 8;
2419         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2420                               &clkcomp, 8);
2421         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
2422                               &vcpu->run->s.regs.acrs, 64);
2423         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
2424                               &vcpu->arch.sie_block->gcr, 128);
2425         return rc ? -EFAULT : 0;
2426 }
2427
2428 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2429 {
2430         /*
2431          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2432          * copying in vcpu load/put. Lets update our copies before we save
2433          * it into the save area
2434          */
2435         save_fpu_regs();
2436         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2437         save_access_regs(vcpu->run->s.regs.acrs);
2438
2439         return kvm_s390_store_status_unloaded(vcpu, addr);
2440 }
2441
2442 /*
2443  * store additional status at address
2444  */
2445 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2446                                         unsigned long gpa)
2447 {
2448         /* Only bits 0-53 are used for address formation */
2449         if (!(gpa & ~0x3ff))
2450                 return 0;
2451
2452         return write_guest_abs(vcpu, gpa & ~0x3ff,
2453                                (void *)&vcpu->run->s.regs.vrs, 512);
2454 }
2455
2456 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2457 {
2458         if (!test_kvm_facility(vcpu->kvm, 129))
2459                 return 0;
2460
2461         /*
2462          * The guest VXRS are in the host VXRs due to the lazy
2463          * copying in vcpu load/put. We can simply call save_fpu_regs()
2464          * to save the current register state because we are in the
2465          * middle of a load/put cycle.
2466          *
2467          * Let's update our copies before we save it into the save area.
2468          */
2469         save_fpu_regs();
2470
2471         return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2472 }
2473
2474 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2475 {
2476         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2477         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2478 }
2479
2480 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2481 {
2482         unsigned int i;
2483         struct kvm_vcpu *vcpu;
2484
2485         kvm_for_each_vcpu(i, vcpu, kvm) {
2486                 __disable_ibs_on_vcpu(vcpu);
2487         }
2488 }
2489
2490 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2491 {
2492         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2493         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2494 }
2495
2496 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2497 {
2498         int i, online_vcpus, started_vcpus = 0;
2499
2500         if (!is_vcpu_stopped(vcpu))
2501                 return;
2502
2503         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2504         /* Only one cpu at a time may enter/leave the STOPPED state. */
2505         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2506         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2507
2508         for (i = 0; i < online_vcpus; i++) {
2509                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2510                         started_vcpus++;
2511         }
2512
2513         if (started_vcpus == 0) {
2514                 /* we're the only active VCPU -> speed it up */
2515                 __enable_ibs_on_vcpu(vcpu);
2516         } else if (started_vcpus == 1) {
2517                 /*
2518                  * As we are starting a second VCPU, we have to disable
2519                  * the IBS facility on all VCPUs to remove potentially
2520                  * oustanding ENABLE requests.
2521                  */
2522                 __disable_ibs_on_all_vcpus(vcpu->kvm);
2523         }
2524
2525         atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2526         /*
2527          * Another VCPU might have used IBS while we were offline.
2528          * Let's play safe and flush the VCPU at startup.
2529          */
2530         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2531         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2532         return;
2533 }
2534
2535 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2536 {
2537         int i, online_vcpus, started_vcpus = 0;
2538         struct kvm_vcpu *started_vcpu = NULL;
2539
2540         if (is_vcpu_stopped(vcpu))
2541                 return;
2542
2543         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2544         /* Only one cpu at a time may enter/leave the STOPPED state. */
2545         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2546         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2547
2548         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2549         kvm_s390_clear_stop_irq(vcpu);
2550
2551         atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2552         __disable_ibs_on_vcpu(vcpu);
2553
2554         for (i = 0; i < online_vcpus; i++) {
2555                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2556                         started_vcpus++;
2557                         started_vcpu = vcpu->kvm->vcpus[i];
2558                 }
2559         }
2560
2561         if (started_vcpus == 1) {
2562                 /*
2563                  * As we only have one VCPU left, we want to enable the
2564                  * IBS facility for that VCPU to speed it up.
2565                  */
2566                 __enable_ibs_on_vcpu(started_vcpu);
2567         }
2568
2569         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2570         return;
2571 }
2572
2573 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2574                                      struct kvm_enable_cap *cap)
2575 {
2576         int r;
2577
2578         if (cap->flags)
2579                 return -EINVAL;
2580
2581         switch (cap->cap) {
2582         case KVM_CAP_S390_CSS_SUPPORT:
2583                 if (!vcpu->kvm->arch.css_support) {
2584                         vcpu->kvm->arch.css_support = 1;
2585                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2586                         trace_kvm_s390_enable_css(vcpu->kvm);
2587                 }
2588                 r = 0;
2589                 break;
2590         default:
2591                 r = -EINVAL;
2592                 break;
2593         }
2594         return r;
2595 }
2596
2597 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2598                                   struct kvm_s390_mem_op *mop)
2599 {
2600         void __user *uaddr = (void __user *)mop->buf;
2601         void *tmpbuf = NULL;
2602         int r, srcu_idx;
2603         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2604                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
2605
2606         if (mop->flags & ~supported_flags)
2607                 return -EINVAL;
2608
2609         if (mop->size > MEM_OP_MAX_SIZE)
2610                 return -E2BIG;
2611
2612         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2613                 tmpbuf = vmalloc(mop->size);
2614                 if (!tmpbuf)
2615                         return -ENOMEM;
2616         }
2617
2618         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2619
2620         switch (mop->op) {
2621         case KVM_S390_MEMOP_LOGICAL_READ:
2622                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2623                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2624                                             mop->size, GACC_FETCH);
2625                         break;
2626                 }
2627                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2628                 if (r == 0) {
2629                         if (copy_to_user(uaddr, tmpbuf, mop->size))
2630                                 r = -EFAULT;
2631                 }
2632                 break;
2633         case KVM_S390_MEMOP_LOGICAL_WRITE:
2634                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2635                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2636                                             mop->size, GACC_STORE);
2637                         break;
2638                 }
2639                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2640                         r = -EFAULT;
2641                         break;
2642                 }
2643                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2644                 break;
2645         default:
2646                 r = -EINVAL;
2647         }
2648
2649         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2650
2651         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2652                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2653
2654         vfree(tmpbuf);
2655         return r;
2656 }
2657
2658 long kvm_arch_vcpu_ioctl(struct file *filp,
2659                          unsigned int ioctl, unsigned long arg)
2660 {
2661         struct kvm_vcpu *vcpu = filp->private_data;
2662         void __user *argp = (void __user *)arg;
2663         int idx;
2664         long r;
2665
2666         switch (ioctl) {
2667         case KVM_S390_IRQ: {
2668                 struct kvm_s390_irq s390irq;
2669
2670                 r = -EFAULT;
2671                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2672                         break;
2673                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2674                 break;
2675         }
2676         case KVM_S390_INTERRUPT: {
2677                 struct kvm_s390_interrupt s390int;
2678                 struct kvm_s390_irq s390irq;
2679
2680                 r = -EFAULT;
2681                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2682                         break;
2683                 if (s390int_to_s390irq(&s390int, &s390irq))
2684                         return -EINVAL;
2685                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2686                 break;
2687         }
2688         case KVM_S390_STORE_STATUS:
2689                 idx = srcu_read_lock(&vcpu->kvm->srcu);
2690                 r = kvm_s390_vcpu_store_status(vcpu, arg);
2691                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2692                 break;
2693         case KVM_S390_SET_INITIAL_PSW: {
2694                 psw_t psw;
2695
2696                 r = -EFAULT;
2697                 if (copy_from_user(&psw, argp, sizeof(psw)))
2698                         break;
2699                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2700                 break;
2701         }
2702         case KVM_S390_INITIAL_RESET:
2703                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2704                 break;
2705         case KVM_SET_ONE_REG:
2706         case KVM_GET_ONE_REG: {
2707                 struct kvm_one_reg reg;
2708                 r = -EFAULT;
2709                 if (copy_from_user(&reg, argp, sizeof(reg)))
2710                         break;
2711                 if (ioctl == KVM_SET_ONE_REG)
2712                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2713                 else
2714                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2715                 break;
2716         }
2717 #ifdef CONFIG_KVM_S390_UCONTROL
2718         case KVM_S390_UCAS_MAP: {
2719                 struct kvm_s390_ucas_mapping ucasmap;
2720
2721                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2722                         r = -EFAULT;
2723                         break;
2724                 }
2725
2726                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2727                         r = -EINVAL;
2728                         break;
2729                 }
2730
2731                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2732                                      ucasmap.vcpu_addr, ucasmap.length);
2733                 break;
2734         }
2735         case KVM_S390_UCAS_UNMAP: {
2736                 struct kvm_s390_ucas_mapping ucasmap;
2737
2738                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2739                         r = -EFAULT;
2740                         break;
2741                 }
2742
2743                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2744                         r = -EINVAL;
2745                         break;
2746                 }
2747
2748                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2749                         ucasmap.length);
2750                 break;
2751         }
2752 #endif
2753         case KVM_S390_VCPU_FAULT: {
2754                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2755                 break;
2756         }
2757         case KVM_ENABLE_CAP:
2758         {
2759                 struct kvm_enable_cap cap;
2760                 r = -EFAULT;
2761                 if (copy_from_user(&cap, argp, sizeof(cap)))
2762                         break;
2763                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2764                 break;
2765         }
2766         case KVM_S390_MEM_OP: {
2767                 struct kvm_s390_mem_op mem_op;
2768
2769                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2770                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2771                 else
2772                         r = -EFAULT;
2773                 break;
2774         }
2775         case KVM_S390_SET_IRQ_STATE: {
2776                 struct kvm_s390_irq_state irq_state;
2777
2778                 r = -EFAULT;
2779                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2780                         break;
2781                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2782                     irq_state.len == 0 ||
2783                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2784                         r = -EINVAL;
2785                         break;
2786                 }
2787                 r = kvm_s390_set_irq_state(vcpu,
2788                                            (void __user *) irq_state.buf,
2789                                            irq_state.len);
2790                 break;
2791         }
2792         case KVM_S390_GET_IRQ_STATE: {
2793                 struct kvm_s390_irq_state irq_state;
2794
2795                 r = -EFAULT;
2796                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2797                         break;
2798                 if (irq_state.len == 0) {
2799                         r = -EINVAL;
2800                         break;
2801                 }
2802                 r = kvm_s390_get_irq_state(vcpu,
2803                                            (__u8 __user *)  irq_state.buf,
2804                                            irq_state.len);
2805                 break;
2806         }
2807         default:
2808                 r = -ENOTTY;
2809         }
2810         return r;
2811 }
2812
2813 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2814 {
2815 #ifdef CONFIG_KVM_S390_UCONTROL
2816         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2817                  && (kvm_is_ucontrol(vcpu->kvm))) {
2818                 vmf->page = virt_to_page(vcpu->arch.sie_block);
2819                 get_page(vmf->page);
2820                 return 0;
2821         }
2822 #endif
2823         return VM_FAULT_SIGBUS;
2824 }
2825
2826 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2827                             unsigned long npages)
2828 {
2829         return 0;
2830 }
2831
2832 /* Section: memory related */
2833 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2834                                    struct kvm_memory_slot *memslot,
2835                                    const struct kvm_userspace_memory_region *mem,
2836                                    enum kvm_mr_change change)
2837 {
2838         /* A few sanity checks. We can have memory slots which have to be
2839            located/ended at a segment boundary (1MB). The memory in userland is
2840            ok to be fragmented into various different vmas. It is okay to mmap()
2841            and munmap() stuff in this slot after doing this call at any time */
2842
2843         if (mem->userspace_addr & 0xffffful)
2844                 return -EINVAL;
2845
2846         if (mem->memory_size & 0xffffful)
2847                 return -EINVAL;
2848
2849         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
2850                 return -EINVAL;
2851
2852         return 0;
2853 }
2854
2855 void kvm_arch_commit_memory_region(struct kvm *kvm,
2856                                 const struct kvm_userspace_memory_region *mem,
2857                                 const struct kvm_memory_slot *old,
2858                                 const struct kvm_memory_slot *new,
2859                                 enum kvm_mr_change change)
2860 {
2861         int rc;
2862
2863         /* If the basics of the memslot do not change, we do not want
2864          * to update the gmap. Every update causes several unnecessary
2865          * segment translation exceptions. This is usually handled just
2866          * fine by the normal fault handler + gmap, but it will also
2867          * cause faults on the prefix page of running guest CPUs.
2868          */
2869         if (old->userspace_addr == mem->userspace_addr &&
2870             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2871             old->npages * PAGE_SIZE == mem->memory_size)
2872                 return;
2873
2874         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2875                 mem->guest_phys_addr, mem->memory_size);
2876         if (rc)
2877                 pr_warn("failed to commit memory region\n");
2878         return;
2879 }
2880
2881 static int __init kvm_s390_init(void)
2882 {
2883         if (!sclp.has_sief2) {
2884                 pr_info("SIE not available\n");
2885                 return -ENODEV;
2886         }
2887
2888         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2889 }
2890
2891 static void __exit kvm_s390_exit(void)
2892 {
2893         kvm_exit();
2894 }
2895
2896 module_init(kvm_s390_init);
2897 module_exit(kvm_s390_exit);
2898
2899 /*
2900  * Enable autoloading of the kvm module.
2901  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2902  * since x86 takes a different approach.
2903  */
2904 #include <linux/miscdevice.h>
2905 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2906 MODULE_ALIAS("devname:kvm");